k

.github/actions/custom-build-and-push/action.yml

@@ -35,16 +35,6 @@ inputs:
   cache-to:
     description: 'Cache destinations'
     required: false
-  outputs:
-    description: 'Output destinations'
-    required: false
-  provenance:
-    description: 'Generate provenance attestation'
-    required: false
-    default: 'false'
-  build-args:
-    description: 'Build arguments'
-    required: false
   retry-wait-time:
     description: 'Time to wait before attempt 2 in seconds'
     required: false
@@ -72,9 +62,6 @@ runs:
         no-cache: ${{ inputs.no-cache }}
         cache-from: ${{ inputs.cache-from }}
         cache-to: ${{ inputs.cache-to }}
-        outputs: ${{ inputs.outputs }}
-        provenance: ${{ inputs.provenance }}
-        build-args: ${{ inputs.build-args }}
 
     - name: Wait before attempt 2
       if: steps.buildx1.outcome != 'success'
@@ -98,9 +85,6 @@ runs:
         no-cache: ${{ inputs.no-cache }}
         cache-from: ${{ inputs.cache-from }}
         cache-to: ${{ inputs.cache-to }}
-        outputs: ${{ inputs.outputs }}
-        provenance: ${{ inputs.provenance }}
-        build-args: ${{ inputs.build-args }}
 
     - name: Wait before attempt 3
       if: steps.buildx1.outcome != 'success' && steps.buildx2.outcome != 'success'
@@ -124,9 +108,6 @@ runs:
         no-cache: ${{ inputs.no-cache }}
         cache-from: ${{ inputs.cache-from }}
         cache-to: ${{ inputs.cache-to }}
-        outputs: ${{ inputs.outputs }}
-        provenance: ${{ inputs.provenance }}
-        build-args: ${{ inputs.build-args }}
 
     - name: Report failure
       if: steps.buildx1.outcome != 'success' && steps.buildx2.outcome != 'success' && steps.buildx3.outcome != 'success'

.github/workflows/docker-build-push-backend-container-on-tag.yml

@@ -142,25 +142,15 @@ jobs:
       # can re-enable when they figure it out
       # https://github.com/aquasecurity/trivy/discussions/7538
       # https://github.com/aquasecurity/trivy-action/issues/389
-      # Security: Using pinned digest (0.65.0@sha256:a22415a38938a56c379387a8163fcb0ce38b10ace73e593475d3658d578b2436)
-      # Security: No Docker socket mount needed for remote registry scanning
       - name: Run Trivy vulnerability scanner
-        uses: nick-fields/retry@v3
+        uses: aquasecurity/trivy-action@master
+        env:
+          TRIVY_DB_REPOSITORY: "public.ecr.aws/aquasecurity/trivy-db:2"
+          TRIVY_JAVA_DB_REPOSITORY: "public.ecr.aws/aquasecurity/trivy-java-db:1"
+          TRIVY_USERNAME: ${{ secrets.DOCKER_USERNAME }}
+          TRIVY_PASSWORD: ${{ secrets.DOCKER_TOKEN }}
         with:
-          timeout_minutes: 30
-          max_attempts: 3
-          retry_wait_seconds: 10
-          command: |
-            docker run --rm -v $HOME/.cache/trivy:/root/.cache/trivy \
-              -v ${{ github.workspace }}/backend/.trivyignore:/tmp/.trivyignore:ro \
-              -e TRIVY_DB_REPOSITORY="public.ecr.aws/aquasecurity/trivy-db:2" \
-              -e TRIVY_JAVA_DB_REPOSITORY="public.ecr.aws/aquasecurity/trivy-java-db:1" \
-              -e TRIVY_USERNAME="${{ secrets.DOCKER_USERNAME }}" \
-              -e TRIVY_PASSWORD="${{ secrets.DOCKER_TOKEN }}" \
-              aquasec/trivy@sha256:a22415a38938a56c379387a8163fcb0ce38b10ace73e593475d3658d578b2436 \
-              image \
-              --skip-version-check \
-              --timeout 20m \
-              --severity CRITICAL,HIGH \
-              --ignorefile /tmp/.trivyignore \
-              docker.io/${{ env.REGISTRY_IMAGE }}:${{ github.ref_name }}
+          # To run locally: trivy image --severity HIGH,CRITICAL onyxdotapp/onyx-backend
+          image-ref: docker.io/${{ env.REGISTRY_IMAGE }}:${{ github.ref_name }}
+          severity: "CRITICAL,HIGH"
+          trivyignores: ./backend/.trivyignore

.github/workflows/docker-build-push-cloud-web-container-on-tag.yml

@@ -139,20 +139,12 @@ jobs:
       # https://github.com/aquasecurity/trivy/discussions/7538
       # https://github.com/aquasecurity/trivy-action/issues/389
       - name: Run Trivy vulnerability scanner
-        uses: nick-fields/retry@v3
+        uses: aquasecurity/trivy-action@master
+        env:
+          TRIVY_DB_REPOSITORY: "public.ecr.aws/aquasecurity/trivy-db:2"
+          TRIVY_JAVA_DB_REPOSITORY: "public.ecr.aws/aquasecurity/trivy-java-db:1"
+          TRIVY_USERNAME: ${{ secrets.DOCKER_USERNAME }}
+          TRIVY_PASSWORD: ${{ secrets.DOCKER_TOKEN }}
         with:
-          timeout_minutes: 30
-          max_attempts: 3
-          retry_wait_seconds: 10
-          command: |
-            docker run --rm -v $HOME/.cache/trivy:/root/.cache/trivy \
-              -e TRIVY_DB_REPOSITORY="public.ecr.aws/aquasecurity/trivy-db:2" \
-              -e TRIVY_JAVA_DB_REPOSITORY="public.ecr.aws/aquasecurity/trivy-java-db:1" \
-              -e TRIVY_USERNAME="${{ secrets.DOCKER_USERNAME }}" \
-              -e TRIVY_PASSWORD="${{ secrets.DOCKER_TOKEN }}" \
-              aquasec/trivy@sha256:a22415a38938a56c379387a8163fcb0ce38b10ace73e593475d3658d578b2436 \
-              image \
-              --skip-version-check \
-              --timeout 20m \
-              --severity CRITICAL,HIGH \
-              docker.io/${{ env.REGISTRY_IMAGE }}:${{ github.ref_name }}
+          image-ref: docker.io/${{ env.REGISTRY_IMAGE }}:${{ github.ref_name }}
+          severity: "CRITICAL,HIGH"

.github/workflows/docker-build-push-model-server-container-on-tag.yml

@@ -99,7 +99,7 @@ jobs:
     needs: [check_model_server_changes]
     if: needs.check_model_server_changes.outputs.changed == 'true'
     runs-on:
-      [runs-on, runner=8cpu-linux-arm64, "run-id=${{ github.run_id }}-arm64"]
+      [runs-on, runner=8cpu-linux-x64, "run-id=${{ github.run_id }}-arm64"]
     env:
       PLATFORM_PAIR: linux-arm64
     steps:
@@ -164,20 +164,13 @@ jobs:
           fi
 
       - name: Run Trivy vulnerability scanner
-        uses: nick-fields/retry@v3
+        uses: aquasecurity/trivy-action@master
+        env:
+          TRIVY_DB_REPOSITORY: "public.ecr.aws/aquasecurity/trivy-db:2"
+          TRIVY_JAVA_DB_REPOSITORY: "public.ecr.aws/aquasecurity/trivy-java-db:1"
+          TRIVY_USERNAME: ${{ secrets.DOCKER_USERNAME }}
+          TRIVY_PASSWORD: ${{ secrets.DOCKER_TOKEN }}
         with:
-          timeout_minutes: 30
-          max_attempts: 3
-          retry_wait_seconds: 10
-          command: |
-            docker run --rm -v $HOME/.cache/trivy:/root/.cache/trivy \
-              -e TRIVY_DB_REPOSITORY="public.ecr.aws/aquasecurity/trivy-db:2" \
-              -e TRIVY_JAVA_DB_REPOSITORY="public.ecr.aws/aquasecurity/trivy-java-db:1" \
-              -e TRIVY_USERNAME="${{ secrets.DOCKER_USERNAME }}" \
-              -e TRIVY_PASSWORD="${{ secrets.DOCKER_TOKEN }}" \
-              aquasec/trivy@sha256:a22415a38938a56c379387a8163fcb0ce38b10ace73e593475d3658d578b2436 \
-              image \
-              --skip-version-check \
-              --timeout 20m \
-              --severity CRITICAL,HIGH \
-              docker.io/${{ env.REGISTRY_IMAGE }}:${{ github.ref_name }}
+          image-ref: docker.io/${{ env.REGISTRY_IMAGE }}:${{ github.ref_name }}
+          severity: "CRITICAL,HIGH"
+          timeout: "10m"

.github/workflows/docker-build-push-web-container-on-tag.yml

@@ -150,20 +150,12 @@ jobs:
       # https://github.com/aquasecurity/trivy/discussions/7538
       # https://github.com/aquasecurity/trivy-action/issues/389
       - name: Run Trivy vulnerability scanner
-        uses: nick-fields/retry@v3
+        uses: aquasecurity/trivy-action@master
+        env:
+          TRIVY_DB_REPOSITORY: "public.ecr.aws/aquasecurity/trivy-db:2"
+          TRIVY_JAVA_DB_REPOSITORY: "public.ecr.aws/aquasecurity/trivy-java-db:1"
+          TRIVY_USERNAME: ${{ secrets.DOCKER_USERNAME }}
+          TRIVY_PASSWORD: ${{ secrets.DOCKER_TOKEN }}
         with:
-          timeout_minutes: 30
-          max_attempts: 3
-          retry_wait_seconds: 10
-          command: |
-            docker run --rm -v $HOME/.cache/trivy:/root/.cache/trivy \
-              -e TRIVY_DB_REPOSITORY="public.ecr.aws/aquasecurity/trivy-db:2" \
-              -e TRIVY_JAVA_DB_REPOSITORY="public.ecr.aws/aquasecurity/trivy-java-db:1" \
-              -e TRIVY_USERNAME="${{ secrets.DOCKER_USERNAME }}" \
-              -e TRIVY_PASSWORD="${{ secrets.DOCKER_TOKEN }}" \
-              aquasec/trivy@sha256:a22415a38938a56c379387a8163fcb0ce38b10ace73e593475d3658d578b2436 \
-              image \
-              --skip-version-check \
-              --timeout 20m \
-              --severity CRITICAL,HIGH \
-              docker.io/${{ env.REGISTRY_IMAGE }}:${{ github.ref_name }}
+          image-ref: docker.io/${{ env.REGISTRY_IMAGE }}:${{ github.ref_name }}
+          severity: "CRITICAL,HIGH"

.github/workflows/helm-chart-releases.yml

@@ -27,7 +27,6 @@ jobs:
         run: |
           helm repo add bitnami https://charts.bitnami.com/bitnami
           helm repo add onyx-vespa https://onyx-dot-app.github.io/vespa-helm-charts
-          helm repo add keda https://kedacore.github.io/charts
           helm repo update
 
       - name: Build chart dependencies
@@ -47,4 +46,4 @@ jobs:
           charts_dir: deployment/helm/charts
           branch: gh-pages
           commit_username: ${{ github.actor }}
-          commit_email: ${{ github.actor }}@users.noreply.github.com
+          commit_email: ${{ github.actor }}@users.noreply.github.com

.github/workflows/pr-external-dependency-unit-tests.yml

@@ -21,10 +21,6 @@ env:
   CONFLUENCE_USER_NAME: ${{ secrets.CONFLUENCE_USER_NAME }}
   CONFLUENCE_ACCESS_TOKEN: ${{ secrets.CONFLUENCE_ACCESS_TOKEN }}
 
-  # LLMs
-  OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
-  ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
-
 jobs:
   discover-test-dirs:
     runs-on: ubuntu-latest
@@ -89,6 +85,8 @@ jobs:
         shell: script -q -e -c "bash --noprofile --norc -eo pipefail {0}"
         run: |
           py.test \
+            -n 8 \
+            --dist loadfile \
             --durations=8 \
             -o junit_family=xunit2 \
             -xv \

.github/workflows/pr-helm-chart-testing.yml

@@ -53,154 +53,27 @@ jobs:
       if: steps.list-changed.outputs.changed == 'true'
       uses: helm/kind-action@v1.12.0
 
-    - name: Pre-install cluster status check
+    - name: Run chart-testing (install)
       if: steps.list-changed.outputs.changed == 'true'
-      run: |
-        echo "=== Pre-install Cluster Status ==="
-        kubectl get nodes -o wide
-        kubectl get pods --all-namespaces
-        kubectl get storageclass
-
-    - name: Add Helm repositories and update
-      if: steps.list-changed.outputs.changed == 'true'
-      run: |
-        echo "=== Adding Helm repositories ==="
-        helm repo add bitnami https://charts.bitnami.com/bitnami
-        helm repo add vespa https://onyx-dot-app.github.io/vespa-helm-charts
-        helm repo update
-
-    - name: Pre-pull critical images
-      if: steps.list-changed.outputs.changed == 'true'
-      run: |
-        echo "=== Pre-pulling critical images to avoid timeout ==="
-        # Get kind cluster name
-        KIND_CLUSTER=$(kubectl config current-context | sed 's/kind-//')
-        echo "Kind cluster: $KIND_CLUSTER"
-        
-        # Pre-pull images that are likely to be used
-        echo "Pre-pulling PostgreSQL image..."
-        docker pull postgres:15-alpine || echo "Failed to pull postgres:15-alpine"
-        kind load docker-image postgres:15-alpine --name $KIND_CLUSTER || echo "Failed to load postgres image"
-        
-        echo "Pre-pulling Redis image..."
-        docker pull redis:7-alpine || echo "Failed to pull redis:7-alpine"
-        kind load docker-image redis:7-alpine --name $KIND_CLUSTER || echo "Failed to load redis image"
-        
-        echo "Pre-pulling Onyx images..."
-        docker pull docker.io/onyxdotapp/onyx-web-server:latest || echo "Failed to pull onyx web server"
-        docker pull docker.io/onyxdotapp/onyx-backend:latest || echo "Failed to pull onyx backend"
-        kind load docker-image docker.io/onyxdotapp/onyx-web-server:latest --name $KIND_CLUSTER || echo "Failed to load onyx web server"
-        kind load docker-image docker.io/onyxdotapp/onyx-backend:latest --name $KIND_CLUSTER || echo "Failed to load onyx backend"
-        
-        echo "=== Images loaded into Kind cluster ==="
-        docker exec $KIND_CLUSTER-control-plane crictl images | grep -E "(postgres|redis|onyx)" || echo "Some images may still be loading..."
-
-    - name: Validate chart dependencies
-      if: steps.list-changed.outputs.changed == 'true'
-      run: |
-        echo "=== Validating chart dependencies ==="
-        cd deployment/helm/charts/onyx
-        helm dependency update
-        helm lint .
-
-    - name: Run chart-testing (install) with enhanced monitoring
-      timeout-minutes: 25
-      if: steps.list-changed.outputs.changed == 'true'
-      run: |
-        echo "=== Starting chart installation with monitoring ==="
-        
-        # Function to monitor cluster state
-        monitor_cluster() {
-          while true; do
-            echo "=== Cluster Status Check at $(date) ==="
-            # Only show non-running pods to reduce noise
-            NON_RUNNING_PODS=$(kubectl get pods --all-namespaces --field-selector=status.phase!=Running,status.phase!=Succeeded --no-headers 2>/dev/null | wc -l)
-            if [ "$NON_RUNNING_PODS" -gt 0 ]; then
-              echo "Non-running pods:"
-              kubectl get pods --all-namespaces --field-selector=status.phase!=Running,status.phase!=Succeeded
-            else
-              echo "All pods running successfully"
-            fi
-            # Only show recent events if there are issues
-            RECENT_EVENTS=$(kubectl get events --sort-by=.lastTimestamp --all-namespaces --field-selector=type!=Normal 2>/dev/null | tail -5)
-            if [ -n "$RECENT_EVENTS" ]; then
-              echo "Recent warnings/errors:"
-              echo "$RECENT_EVENTS"
-            fi
-            sleep 60
-          done
-        }
-        
-        # Start monitoring in background
-        monitor_cluster &
-        MONITOR_PID=$!
-        
-        # Set up cleanup
-        cleanup() {
-          echo "=== Cleaning up monitoring process ==="
-          kill $MONITOR_PID 2>/dev/null || true
-          echo "=== Final cluster state ==="
-          kubectl get pods --all-namespaces
-          kubectl get events --all-namespaces --sort-by=.lastTimestamp | tail -20
-        }
-        
-        # Trap cleanup on exit
-        trap cleanup EXIT
-        
-        # Run the actual installation with detailed logging
-        echo "=== Starting ct install ==="
-        ct install --all \
-          --helm-extra-set-args="\
-            --set=nginx.enabled=false \
-            --set=minio.enabled=false \
-            --set=vespa.enabled=false \
-            --set=slackbot.enabled=false \
-            --set=postgresql.enabled=true \
-            --set=postgresql.primary.persistence.enabled=false \
-            --set=redis.enabled=true \
-            --set=webserver.replicaCount=1 \
-            --set=api.replicaCount=0 \
-            --set=inferenceCapability.replicaCount=0 \
-            --set=indexCapability.replicaCount=0 \
-            --set=celery_beat.replicaCount=0 \
-            --set=celery_worker_heavy.replicaCount=0 \
-            --set=celery_worker_docfetching.replicaCount=0 \
-            --set=celery_worker_docprocessing.replicaCount=0 \
-            --set=celery_worker_light.replicaCount=0 \
-            --set=celery_worker_monitoring.replicaCount=0 \
-            --set=celery_worker_primary.replicaCount=0 \
-            --set=celery_worker_user_files_indexing.replicaCount=0" \
-          --helm-extra-args="--timeout 900s --debug" \
-          --debug --config ct.yaml
-        
-        echo "=== Installation completed successfully ==="
-        kubectl get pods --all-namespaces
-
-    - name: Post-install verification
-      if: steps.list-changed.outputs.changed == 'true'
-      run: |
-        echo "=== Post-install verification ==="
-        kubectl get pods --all-namespaces
-        kubectl get services --all-namespaces
-        # Only show issues if they exist
-        kubectl describe pods --all-namespaces | grep -A 5 -B 2 "Failed\|Error\|Warning" || echo "No pod issues found"
-
-    - name: Cleanup on failure
-      if: failure() && steps.list-changed.outputs.changed == 'true'
-      run: |
-        echo "=== Cleanup on failure ==="
-        echo "=== Final cluster state ==="
-        kubectl get pods --all-namespaces
-        kubectl get events --all-namespaces --sort-by=.lastTimestamp | tail -10
-        
-        echo "=== Pod descriptions for debugging ==="
-        kubectl describe pods --all-namespaces | grep -A 10 -B 3 "Failed\|Error\|Warning\|Pending" || echo "No problematic pods found"
-        
-        echo "=== Recent logs for debugging ==="
-        kubectl logs --all-namespaces --tail=50 | grep -i "error\|timeout\|failed\|pull" || echo "No error logs found"
-        
-        echo "=== Helm releases ==="
-        helm list --all-namespaces
+      run: ct install --all \
+        --helm-extra-set-args="\
+          --set=nginx.enabled=false \
+          --set=postgresql.enabled=false \
+          --set=redis.enabled=false \
+          --set=minio.enabled=false \
+          --set=vespa.enabled=false \
+          --set=slackbot.enabled=false \
+          --set=api.replicaCount=0 \
+          --set=inferenceCapability.replicaCount=0 \
+          --set=indexCapability.replicaCount=0 \
+          --set=celery_beat.replicaCount=0 \
+          --set=celery_worker_heavy.replicaCount=0 \
+          --set=celery_worker_docprocessing.replicaCount=0 \
+          --set=celery_worker_light.replicaCount=0 \
+          --set=celery_worker_monitoring.replicaCount=0 \
+          --set=celery_worker_primary.replicaCount=0 \
+          --set=celery_worker_user_files_indexing.replicaCount=0" \
+        --debug --config ct.yaml
       # the following would install only changed charts, but we only have one chart so 
       # don't worry about that for now
       # run: ct install --target-branch ${{ github.event.repository.default_branch }}

.github/workflows/pr-integration-tests.yml

@@ -11,12 +11,6 @@ on:
       - "release/**"
 
 env:
-  # Private Registry Configuration
-  PRIVATE_REGISTRY: experimental-registry.blacksmith.sh:5000
-  PRIVATE_REGISTRY_USERNAME: ${{ secrets.PRIVATE_REGISTRY_USERNAME }}
-  PRIVATE_REGISTRY_PASSWORD: ${{ secrets.PRIVATE_REGISTRY_PASSWORD }}
-
-  # Test Environment Variables
   OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
   SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
   CONFLUENCE_TEST_SPACE_URL: ${{ secrets.CONFLUENCE_TEST_SPACE_URL }}
@@ -29,38 +23,18 @@ env:
   PERM_SYNC_SHAREPOINT_PRIVATE_KEY: ${{ secrets.PERM_SYNC_SHAREPOINT_PRIVATE_KEY }}
   PERM_SYNC_SHAREPOINT_CERTIFICATE_PASSWORD: ${{ secrets.PERM_SYNC_SHAREPOINT_CERTIFICATE_PASSWORD }}
   PERM_SYNC_SHAREPOINT_DIRECTORY_ID: ${{ secrets.PERM_SYNC_SHAREPOINT_DIRECTORY_ID }}
+  PLATFORM_PAIR: linux-amd64
 
 jobs:
-  discover-test-dirs:
-    runs-on: blacksmith-2vcpu-ubuntu-2404-arm
-    outputs:
-      test-dirs: ${{ steps.set-matrix.outputs.test-dirs }}
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-
-      - name: Discover test directories
-        id: set-matrix
-        run: |
-          # Find all leaf-level directories in both test directories
-          tests_dirs=$(find backend/tests/integration/tests -mindepth 1 -maxdepth 1 -type d ! -name "__pycache__" -exec basename {} \; | sort)
-          connector_dirs=$(find backend/tests/integration/connector_job_tests -mindepth 1 -maxdepth 1 -type d ! -name "__pycache__" -exec basename {} \; | sort)
-
-          # Create JSON array with directory info
-          all_dirs=""
-          for dir in $tests_dirs; do
-            all_dirs="$all_dirs{\"path\":\"tests/$dir\",\"name\":\"tests-$dir\"},"
-          done
-          for dir in $connector_dirs; do
-            all_dirs="$all_dirs{\"path\":\"connector_job_tests/$dir\",\"name\":\"connector-$dir\"},"
-          done
-
-          # Remove trailing comma and wrap in array
-          all_dirs="[${all_dirs%,}]"
-          echo "test-dirs=$all_dirs" >> $GITHUB_OUTPUT
-
-  prepare-build:
-    runs-on: blacksmith-2vcpu-ubuntu-2404-arm
+  integration-tests:
+    # See https://runs-on.com/runners/linux/
+    runs-on:
+      [
+        runs-on,
+        runner=32cpu-linux-x64,
+        disk=large,
+        "run-id=${{ github.run_id }}",
+      ]
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -73,12 +47,12 @@ jobs:
           cache-dependency-path: |
             backend/requirements/default.txt
             backend/requirements/dev.txt
-
-      - name: Install Python dependencies
-        run: |
+            backend/requirements/ee.txt
+      - run: |
           python -m pip install --upgrade pip
           pip install --retries 5 --timeout 30 -r backend/requirements/default.txt
           pip install --retries 5 --timeout 30 -r backend/requirements/dev.txt
+          pip install --retries 5 --timeout 30 -r backend/requirements/ee.txt
 
       - name: Generate OpenAPI schema
         working-directory: ./backend
@@ -100,151 +74,130 @@ jobs:
             --skip-validate-spec \
             --openapi-normalizer "SIMPLIFY_ONEOF_ANYOF=true,SET_OAS3_NULLABLE=true"
 
-      - name: Upload OpenAPI artifacts
-        uses: actions/upload-artifact@v4
-        with:
-          name: openapi-artifacts
-          path: backend/generated/
-
-  build-backend-image:
-    runs-on: blacksmith-16vcpu-ubuntu-2404-arm
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-
-      - name: Login to Private Registry
-        uses: docker/login-action@v3
-        with:
-          registry: ${{ env.PRIVATE_REGISTRY }}
-          username: ${{ env.PRIVATE_REGISTRY_USERNAME }}
-          password: ${{ env.PRIVATE_REGISTRY_PASSWORD }}
-
       - name: Set up Docker Buildx
-        uses: useblacksmith/setup-docker-builder@v1
+        uses: docker/setup-buildx-action@v3
 
-      - name: Build and push Backend Docker image
-        uses: useblacksmith/build-push-action@v2
-        with:
-          context: ./backend
-          file: ./backend/Dockerfile
-          platforms: linux/arm64
-          tags: ${{ env.PRIVATE_REGISTRY }}/integration-test-onyx-backend:test-${{ github.run_id }}
-          push: true
-
-  build-model-server-image:
-    runs-on: blacksmith-16vcpu-ubuntu-2404-arm
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-
-      - name: Login to Private Registry
-        uses: docker/login-action@v3
-        with:
-          registry: ${{ env.PRIVATE_REGISTRY }}
-          username: ${{ env.PRIVATE_REGISTRY_USERNAME }}
-          password: ${{ env.PRIVATE_REGISTRY_PASSWORD }}
-
-      - name: Set up Docker Buildx
-        uses: useblacksmith/setup-docker-builder@v1
-
-      - name: Build and push Model Server Docker image
-        uses: useblacksmith/build-push-action@v2
-        with:
-          context: ./backend
-          file: ./backend/Dockerfile.model_server
-          platforms: linux/arm64
-          tags: ${{ env.PRIVATE_REGISTRY }}/integration-test-onyx-model-server:test-${{ github.run_id }}
-          push: true
-          outputs: type=registry
-          provenance: false
-
-  build-integration-image:
-    needs: prepare-build
-    runs-on: blacksmith-16vcpu-ubuntu-2404-arm
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-
-      - name: Login to Private Registry
-        uses: docker/login-action@v3
-        with:
-          registry: ${{ env.PRIVATE_REGISTRY }}
-          username: ${{ env.PRIVATE_REGISTRY_USERNAME }}
-          password: ${{ env.PRIVATE_REGISTRY_PASSWORD }}
-
-      - name: Download OpenAPI artifacts
-        uses: actions/download-artifact@v4
-        with:
-          name: openapi-artifacts
-          path: backend/generated/
-
-      - name: Set up Docker Buildx
-        uses: useblacksmith/setup-docker-builder@v1
-
-      - name: Build and push integration test Docker image
-        uses: useblacksmith/build-push-action@v2
-        with:
-          context: ./backend
-          file: ./backend/tests/integration/Dockerfile
-          platforms: linux/arm64
-          tags: ${{ env.PRIVATE_REGISTRY }}/integration-test-onyx-integration:test-${{ github.run_id }}
-          push: true
-
-  integration-tests:
-    needs:
-      [
-        discover-test-dirs,
-        build-backend-image,
-        build-model-server-image,
-        build-integration-image,
-      ]
-    runs-on: blacksmith-8vcpu-ubuntu-2404-arm
-
-    strategy:
-      fail-fast: false
-      matrix:
-        test-dir: ${{ fromJson(needs.discover-test-dirs.outputs.test-dirs) }}
-
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-
-      - name: Login to Private Registry
-        uses: docker/login-action@v3
-        with:
-          registry: ${{ env.PRIVATE_REGISTRY }}
-          username: ${{ env.PRIVATE_REGISTRY_USERNAME }}
-          password: ${{ env.PRIVATE_REGISTRY_PASSWORD }}
-
-      # needed for pulling Vespa, Redis, Postgres, and Minio images
-      # otherwise, we hit the "Unauthenticated users" limit
-      # https://docs.docker.com/docker-hub/usage/
       - name: Login to Docker Hub
         uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_TOKEN }}
 
-      - name: Pull Docker images
+      # tag every docker image with "test" so that we can spin up the correct set
+      # of images during testing
+
+      # We don't need to build the Web Docker image since it's not yet used
+      # in the integration tests. We have a separate action to verify that it builds
+      # successfully.
+      - name: Pull Web Docker image
         run: |
-          # Pull all images from registry in parallel
-          echo "Pulling Docker images in parallel..."
-          # Pull images from private registry
-          (docker pull ${{ env.PRIVATE_REGISTRY }}/integration-test-onyx-backend:test-${{ github.run_id }}) &
-          (docker pull ${{ env.PRIVATE_REGISTRY }}/integration-test-onyx-model-server:test-${{ github.run_id }}) &
-          (docker pull ${{ env.PRIVATE_REGISTRY }}/integration-test-onyx-integration:test-${{ github.run_id }}) &
+          docker pull onyxdotapp/onyx-web-server:latest
+          docker tag onyxdotapp/onyx-web-server:latest onyxdotapp/onyx-web-server:test
 
-          # Wait for all background jobs to complete
-          wait
-          echo "All Docker images pulled successfully"
+      # we use the runs-on cache for docker builds
+      # in conjunction with runs-on runners, it has better speed and unlimited caching
+      # https://runs-on.com/caching/s3-cache-for-github-actions/
+      # https://runs-on.com/caching/docker/
+      # https://github.com/moby/buildkit#s3-cache-experimental
 
-          # Re-tag to remove registry prefix for docker-compose
-          docker tag ${{ env.PRIVATE_REGISTRY }}/integration-test-onyx-backend:test-${{ github.run_id }} onyxdotapp/onyx-backend:test
-          docker tag ${{ env.PRIVATE_REGISTRY }}/integration-test-onyx-model-server:test-${{ github.run_id }} onyxdotapp/onyx-model-server:test
-          docker tag ${{ env.PRIVATE_REGISTRY }}/integration-test-onyx-integration:test-${{ github.run_id }} onyxdotapp/onyx-integration:test
+      # images are built and run locally for testing purposes. Not pushed.
+      - name: Build Backend Docker image
+        uses: ./.github/actions/custom-build-and-push
+        with:
+          context: ./backend
+          file: ./backend/Dockerfile
+          platforms: linux/amd64
+          tags: onyxdotapp/onyx-backend:test
+          push: false
+          load: true
+          cache-from: type=s3,prefix=cache/${{ github.repository }}/integration-tests/backend-${{ env.PLATFORM_PAIR }}/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }}
+          cache-to: type=s3,prefix=cache/${{ github.repository }}/integration-tests/backend-${{ env.PLATFORM_PAIR }}/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }},mode=max
+
+      - name: Build Model Server Docker image
+        uses: ./.github/actions/custom-build-and-push
+        with:
+          context: ./backend
+          file: ./backend/Dockerfile.model_server
+          platforms: linux/amd64
+          tags: onyxdotapp/onyx-model-server:test
+          push: false
+          load: true
+          cache-from: type=s3,prefix=cache/${{ github.repository }}/integration-tests/model-server-${{ env.PLATFORM_PAIR }}/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }}
+          cache-to: type=s3,prefix=cache/${{ github.repository }}/integration-tests/model-server-${{ env.PLATFORM_PAIR }}/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }},mode=max
+
+      - name: Build integration test Docker image
+        uses: ./.github/actions/custom-build-and-push
+        with:
+          context: ./backend
+          file: ./backend/tests/integration/Dockerfile
+          platforms: linux/amd64
+          tags: onyxdotapp/onyx-integration:test
+          push: false
+          load: true
+          cache-from: type=s3,prefix=cache/${{ github.repository }}/integration-tests/integration-${{ env.PLATFORM_PAIR }}/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }}
+          cache-to: type=s3,prefix=cache/${{ github.repository }}/integration-tests/integration-${{ env.PLATFORM_PAIR }}/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }},mode=max
+
+      # Start containers for multi-tenant tests
+      - name: Start Docker containers for multi-tenant tests
+        run: |
+          cd deployment/docker_compose
+          ENABLE_PAID_ENTERPRISE_EDITION_FEATURES=true \
+          MULTI_TENANT=true \
+          AUTH_TYPE=cloud \
+          REQUIRE_EMAIL_VERIFICATION=false \
+          DISABLE_TELEMETRY=true \
+          IMAGE_TAG=test \
+          DEV_MODE=true \
+          docker compose -f docker-compose.multitenant-dev.yml -p onyx-stack up -d
+        id: start_docker_multi_tenant
+
+      # In practice, `cloud` Auth type would require OAUTH credentials to be set.
+      - name: Run Multi-Tenant Integration Tests
+        run: |
+          echo "Waiting for 3 minutes to ensure API server is ready..."
+          sleep 180
+          echo "Running integration tests..."
+          docker run --rm --network onyx-stack_default \
+            --name test-runner \
+            -e POSTGRES_HOST=relational_db \
+            -e POSTGRES_USER=postgres \
+            -e POSTGRES_PASSWORD=password \
+            -e DB_READONLY_USER=db_readonly_user \
+            -e DB_READONLY_PASSWORD=password \
+            -e POSTGRES_DB=postgres \
+            -e POSTGRES_USE_NULL_POOL=true \
+            -e VESPA_HOST=index \
+            -e REDIS_HOST=cache \
+            -e API_SERVER_HOST=api_server \
+            -e OPENAI_API_KEY=${OPENAI_API_KEY} \
+            -e SLACK_BOT_TOKEN=${SLACK_BOT_TOKEN} \
+            -e TEST_WEB_HOSTNAME=test-runner \
+            -e AUTH_TYPE=cloud \
+            -e MULTI_TENANT=true \
+            -e REQUIRE_EMAIL_VERIFICATION=false \
+            -e DISABLE_TELEMETRY=true \
+            -e IMAGE_TAG=test \
+            -e DEV_MODE=true \
+            onyxdotapp/onyx-integration:test \
+            /app/tests/integration/multitenant_tests
+        continue-on-error: true
+        id: run_multitenant_tests
+
+      - name: Check multi-tenant test results
+        run: |
+          if [ ${{ steps.run_multitenant_tests.outcome }} == 'failure' ]; then
+            echo "Multi-tenant integration tests failed. Exiting with error."
+            exit 1
+          else
+            echo "All multi-tenant integration tests passed successfully."
+          fi
+
+      - name: Stop multi-tenant Docker containers
+        run: |
+          cd deployment/docker_compose
+          docker compose -f docker-compose.multitenant-dev.yml -p onyx-stack down -v
 
       # NOTE: Use pre-ping/null pool to reduce flakiness due to dropped connections
-      # NOTE: don't need web server for integration tests
       - name: Start Docker containers
         run: |
           cd deployment/docker_compose
@@ -257,16 +210,7 @@ jobs:
           IMAGE_TAG=test \
           INTEGRATION_TESTS_MODE=true \
           CHECK_TTL_MANAGEMENT_TASK_FREQUENCY_IN_HOURS=0.001 \
-          docker compose -f docker-compose.dev.yml -p onyx-stack up \
-            relational_db \
-            index \
-            cache \
-            minio \
-            api_server \
-            inference_model_server \
-            indexing_model_server \
-            background \
-            -d
+          docker compose -f docker-compose.dev.yml -p onyx-stack up -d
         id: start_docker
 
       - name: Wait for service to be ready
@@ -309,44 +253,54 @@ jobs:
           docker compose -f docker-compose.mock-it-services.yml \
             -p mock-it-services-stack up -d
 
-      - name: Run Integration Tests for ${{ matrix.test-dir.name }}
-        uses: nick-fields/retry@v3
-        with:
-          timeout_minutes: 20
-          max_attempts: 3
-          retry_wait_seconds: 10
-          command: |
-            echo "Running integration tests for ${{ matrix.test-dir.path }}..."
-            docker run --rm --network onyx-stack_default \
-              --name test-runner \
-              -e POSTGRES_HOST=relational_db \
-              -e POSTGRES_USER=postgres \
-              -e POSTGRES_PASSWORD=password \
-              -e POSTGRES_DB=postgres \
-              -e DB_READONLY_USER=db_readonly_user \
-              -e DB_READONLY_PASSWORD=password \
-              -e POSTGRES_POOL_PRE_PING=true \
-              -e POSTGRES_USE_NULL_POOL=true \
-              -e VESPA_HOST=index \
-              -e REDIS_HOST=cache \
-              -e API_SERVER_HOST=api_server \
-              -e OPENAI_API_KEY=${OPENAI_API_KEY} \
-              -e SLACK_BOT_TOKEN=${SLACK_BOT_TOKEN} \
-              -e CONFLUENCE_TEST_SPACE_URL=${CONFLUENCE_TEST_SPACE_URL} \
-              -e CONFLUENCE_USER_NAME=${CONFLUENCE_USER_NAME} \
-              -e CONFLUENCE_ACCESS_TOKEN=${CONFLUENCE_ACCESS_TOKEN} \
-              -e JIRA_BASE_URL=${JIRA_BASE_URL} \
-              -e JIRA_USER_EMAIL=${JIRA_USER_EMAIL} \
-              -e JIRA_API_TOKEN=${JIRA_API_TOKEN} \
-              -e PERM_SYNC_SHAREPOINT_CLIENT_ID=${PERM_SYNC_SHAREPOINT_CLIENT_ID} \
-              -e PERM_SYNC_SHAREPOINT_PRIVATE_KEY="${PERM_SYNC_SHAREPOINT_PRIVATE_KEY}" \
-              -e PERM_SYNC_SHAREPOINT_CERTIFICATE_PASSWORD=${PERM_SYNC_SHAREPOINT_CERTIFICATE_PASSWORD} \
-              -e PERM_SYNC_SHAREPOINT_DIRECTORY_ID=${PERM_SYNC_SHAREPOINT_DIRECTORY_ID} \
-              -e TEST_WEB_HOSTNAME=test-runner \
-              -e MOCK_CONNECTOR_SERVER_HOST=mock_connector_server \
-              -e MOCK_CONNECTOR_SERVER_PORT=8001 \
-              onyxdotapp/onyx-integration:test \
-              /app/tests/integration/${{ matrix.test-dir.path }}
+      # NOTE: Use pre-ping/null to reduce flakiness due to dropped connections
+      # NOTE: `-e ENABLE_PAID_ENTERPRISE_EDITION_FEATURES=true` should be added once
+      # enterprise tests are fixed 
+      - name: Run Standard Integration Tests
+        run: |
+          echo "Running integration tests..."
+          docker run --rm --network onyx-stack_default \
+            --name test-runner \
+            -e POSTGRES_HOST=relational_db \
+            -e POSTGRES_USER=postgres \
+            -e POSTGRES_PASSWORD=password \
+            -e DB_READONLY_USER=db_readonly_user \
+            -e DB_READONLY_PASSWORD=password \
+            -e POSTGRES_DB=postgres \
+            -e POSTGRES_POOL_PRE_PING=true \
+            -e POSTGRES_USE_NULL_POOL=true \
+            -e VESPA_HOST=index \
+            -e REDIS_HOST=cache \
+            -e API_SERVER_HOST=api_server \
+            -e OPENAI_API_KEY=${OPENAI_API_KEY} \
+            -e SLACK_BOT_TOKEN=${SLACK_BOT_TOKEN} \
+            -e CONFLUENCE_TEST_SPACE_URL=${CONFLUENCE_TEST_SPACE_URL} \
+            -e CONFLUENCE_USER_NAME=${CONFLUENCE_USER_NAME} \
+            -e CONFLUENCE_ACCESS_TOKEN=${CONFLUENCE_ACCESS_TOKEN} \
+            -e JIRA_BASE_URL=${JIRA_BASE_URL} \
+            -e JIRA_USER_EMAIL=${JIRA_USER_EMAIL} \
+            -e JIRA_API_TOKEN=${JIRA_API_TOKEN} \
+            -e PERM_SYNC_SHAREPOINT_CLIENT_ID=${PERM_SYNC_SHAREPOINT_CLIENT_ID} \
+            -e PERM_SYNC_SHAREPOINT_PRIVATE_KEY="${PERM_SYNC_SHAREPOINT_PRIVATE_KEY}" \
+            -e PERM_SYNC_SHAREPOINT_CERTIFICATE_PASSWORD=${PERM_SYNC_SHAREPOINT_CERTIFICATE_PASSWORD} \
+            -e PERM_SYNC_SHAREPOINT_DIRECTORY_ID=${PERM_SYNC_SHAREPOINT_DIRECTORY_ID} \
+            -e TEST_WEB_HOSTNAME=test-runner \
+            -e MOCK_CONNECTOR_SERVER_HOST=mock_connector_server \
+            -e MOCK_CONNECTOR_SERVER_PORT=8001 \
+            onyxdotapp/onyx-integration:test \
+            /app/tests/integration/tests \
+            /app/tests/integration/connector_job_tests
+        continue-on-error: true
+        id: run_tests
+
+      - name: Check test results
+        run: |
+          if [ ${{ steps.run_tests.outcome }} == 'failure' ]; then
+            echo "Integration tests failed. Exiting with error."
+            exit 1
+          else
+            echo "All integration tests passed successfully."
+          fi
 
       # ------------------------------------------------------------
       # Always gather logs BEFORE "down":
@@ -366,7 +320,7 @@ jobs:
         if: always()
         uses: actions/upload-artifact@v4
         with:
-          name: docker-all-logs-${{ matrix.test-dir.name }}
+          name: docker-all-logs
           path: ${{ github.workspace }}/docker-compose.log
       # ------------------------------------------------------------
 
@@ -375,157 +329,3 @@ jobs:
         run: |
           cd deployment/docker_compose
           docker compose -f docker-compose.dev.yml -p onyx-stack down -v
-
-
-  multitenant-tests:
-    needs:
-      [
-        build-backend-image,
-        build-model-server-image,
-        build-integration-image,
-      ]
-    runs-on: blacksmith-8vcpu-ubuntu-2404-arm
-
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-
-      - name: Login to Private Registry
-        uses: docker/login-action@v3
-        with:
-          registry: ${{ env.PRIVATE_REGISTRY }}
-          username: ${{ env.PRIVATE_REGISTRY_USERNAME }}
-          password: ${{ env.PRIVATE_REGISTRY_PASSWORD }}
-
-      - name: Login to Docker Hub
-        uses: docker/login-action@v3
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_TOKEN }}
-
-      - name: Pull Docker images
-        run: |
-          (docker pull ${{ env.PRIVATE_REGISTRY }}/integration-test-onyx-backend:test-${{ github.run_id }}) &
-          (docker pull ${{ env.PRIVATE_REGISTRY }}/integration-test-onyx-model-server:test-${{ github.run_id }}) &
-          (docker pull ${{ env.PRIVATE_REGISTRY }}/integration-test-onyx-integration:test-${{ github.run_id }}) &
-          wait
-          docker tag ${{ env.PRIVATE_REGISTRY }}/integration-test-onyx-backend:test-${{ github.run_id }} onyxdotapp/onyx-backend:test
-          docker tag ${{ env.PRIVATE_REGISTRY }}/integration-test-onyx-model-server:test-${{ github.run_id }} onyxdotapp/onyx-model-server:test
-          docker tag ${{ env.PRIVATE_REGISTRY }}/integration-test-onyx-integration:test-${{ github.run_id }} onyxdotapp/onyx-integration:test
-
-      - name: Start Docker containers for multi-tenant tests
-        run: |
-          cd deployment/docker_compose
-          ENABLE_PAID_ENTERPRISE_EDITION_FEATURES=true \
-          MULTI_TENANT=true \
-          AUTH_TYPE=cloud \
-          REQUIRE_EMAIL_VERIFICATION=false \
-          DISABLE_TELEMETRY=true \
-          IMAGE_TAG=test \
-          DEV_MODE=true \
-          docker compose -f docker-compose.multitenant-dev.yml -p onyx-stack up \
-            relational_db \
-            index \
-            cache \
-            minio \
-            api_server \
-            inference_model_server \
-            indexing_model_server \
-            background \
-            -d
-        id: start_docker_multi_tenant
-
-      - name: Wait for service to be ready (multi-tenant)
-        run: |
-          echo "Starting wait-for-service script for multi-tenant..."
-          docker logs -f onyx-stack-api_server-1 &
-          start_time=$(date +%s)
-          timeout=300
-          while true; do
-            current_time=$(date +%s)
-            elapsed_time=$((current_time - start_time))
-            if [ $elapsed_time -ge $timeout ]; then
-              echo "Timeout reached. Service did not become ready in 5 minutes."
-              exit 1
-            fi
-            response=$(curl -s -o /dev/null -w "%{http_code}" http://localhost:8080/health || echo "curl_error")
-            if [ "$response" = "200" ]; then
-              echo "Service is ready!"
-              break
-            elif [ "$response" = "curl_error" ]; then
-              echo "Curl encountered an error; retrying..."
-            else
-              echo "Service not ready yet (HTTP $response). Retrying in 5 seconds..."
-            fi
-            sleep 5
-          done
-          echo "Finished waiting for service."
-
-      - name: Run Multi-Tenant Integration Tests
-        run: |
-          echo "Running multi-tenant integration tests..."
-          docker run --rm --network onyx-stack_default \
-            --name test-runner \
-            -e POSTGRES_HOST=relational_db \
-            -e POSTGRES_USER=postgres \
-            -e POSTGRES_PASSWORD=password \
-            -e DB_READONLY_USER=db_readonly_user \
-            -e DB_READONLY_PASSWORD=password \
-            -e POSTGRES_DB=postgres \
-            -e POSTGRES_USE_NULL_POOL=true \
-            -e VESPA_HOST=index \
-            -e REDIS_HOST=cache \
-            -e API_SERVER_HOST=api_server \
-            -e OPENAI_API_KEY=${OPENAI_API_KEY} \
-            -e SLACK_BOT_TOKEN=${SLACK_BOT_TOKEN} \
-            -e TEST_WEB_HOSTNAME=test-runner \
-            -e AUTH_TYPE=cloud \
-            -e MULTI_TENANT=true \
-            -e SKIP_RESET=true \
-            -e REQUIRE_EMAIL_VERIFICATION=false \
-            -e DISABLE_TELEMETRY=true \
-            -e IMAGE_TAG=test \
-            -e DEV_MODE=true \
-            onyxdotapp/onyx-integration:test \
-            /app/tests/integration/multitenant_tests
-
-      - name: Dump API server logs (multi-tenant)
-        if: always()
-        run: |
-          cd deployment/docker_compose
-          docker compose -f docker-compose.multitenant-dev.yml -p onyx-stack logs --no-color api_server > $GITHUB_WORKSPACE/api_server_multitenant.log || true
-
-      - name: Dump all-container logs (multi-tenant)
-        if: always()
-        run: |
-          cd deployment/docker_compose
-          docker compose -f docker-compose.multitenant-dev.yml -p onyx-stack logs --no-color > $GITHUB_WORKSPACE/docker-compose-multitenant.log || true
-
-      - name: Upload logs (multi-tenant)
-        if: always()
-        uses: actions/upload-artifact@v4
-        with:
-          name: docker-all-logs-multitenant
-          path: ${{ github.workspace }}/docker-compose-multitenant.log
-
-      - name: Stop multi-tenant Docker containers
-        if: always()
-        run: |
-          cd deployment/docker_compose
-          docker compose -f docker-compose.multitenant-dev.yml -p onyx-stack down -v
-
-  required: 
-    runs-on: blacksmith-2vcpu-ubuntu-2404-arm
-    needs: [integration-tests, multitenant-tests]
-    if: ${{ always() }}
-    steps:
-      - uses: actions/github-script@v7
-        with:
-          script: |
-            const needs = ${{ toJSON(needs) }};
-            const failed = Object.values(needs).some(n => n.result !== 'success');
-            if (failed) {
-              core.setFailed('One or more upstream jobs failed or were cancelled.');
-            } else {
-              core.notice('All required jobs succeeded.');
-            }

.github/workflows/pr-mit-integration-tests.yml

@@ -5,15 +5,12 @@ concurrency:
 
 on:
   merge_group:
-    types: [checks_requested]
+  pull_request:
+    branches:
+      - main
+      - "release/**"
 
 env:
-  # Private Registry Configuration
-  PRIVATE_REGISTRY: experimental-registry.blacksmith.sh:5000
-  PRIVATE_REGISTRY_USERNAME: ${{ secrets.PRIVATE_REGISTRY_USERNAME }}
-  PRIVATE_REGISTRY_PASSWORD: ${{ secrets.PRIVATE_REGISTRY_PASSWORD }}
-
-  # Test Environment Variables
   OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
   SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
   CONFLUENCE_TEST_SPACE_URL: ${{ secrets.CONFLUENCE_TEST_SPACE_URL }}
@@ -26,42 +23,21 @@ env:
   PERM_SYNC_SHAREPOINT_PRIVATE_KEY: ${{ secrets.PERM_SYNC_SHAREPOINT_PRIVATE_KEY }}
   PERM_SYNC_SHAREPOINT_CERTIFICATE_PASSWORD: ${{ secrets.PERM_SYNC_SHAREPOINT_CERTIFICATE_PASSWORD }}
   PERM_SYNC_SHAREPOINT_DIRECTORY_ID: ${{ secrets.PERM_SYNC_SHAREPOINT_DIRECTORY_ID }}
-
+  PLATFORM_PAIR: linux-amd64
 jobs:
-  discover-test-dirs:
-    runs-on: blacksmith-2vcpu-ubuntu-2404-arm
-    outputs:
-      test-dirs: ${{ steps.set-matrix.outputs.test-dirs }}
+  integration-tests-mit:
+    # See https://runs-on.com/runners/linux/
+    runs-on:
+      [
+        runs-on,
+        runner=32cpu-linux-x64,
+        disk=large,
+        "run-id=${{ github.run_id }}",
+      ]
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
-
-      - name: Discover test directories
-        id: set-matrix
-        run: |
-          # Find all leaf-level directories in both test directories
-          tests_dirs=$(find backend/tests/integration/tests -mindepth 1 -maxdepth 1 -type d ! -name "__pycache__" -exec basename {} \; | sort)
-          connector_dirs=$(find backend/tests/integration/connector_job_tests -mindepth 1 -maxdepth 1 -type d ! -name "__pycache__" -exec basename {} \; | sort)
-
-          # Create JSON array with directory info
-          all_dirs=""
-          for dir in $tests_dirs; do
-            all_dirs="$all_dirs{\"path\":\"tests/$dir\",\"name\":\"tests-$dir\"},"
-          done
-          for dir in $connector_dirs; do
-            all_dirs="$all_dirs{\"path\":\"connector_job_tests/$dir\",\"name\":\"connector-$dir\"},"
-          done
-
-          # Remove trailing comma and wrap in array
-          all_dirs="[${all_dirs%,}]"
-          echo "test-dirs=$all_dirs" >> $GITHUB_OUTPUT
-
-  prepare-build:
-    runs-on: blacksmith-2vcpu-ubuntu-2404-arm
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-
+        
       - name: Setup Python
         uses: actions/setup-python@v5
         with:
@@ -70,9 +46,7 @@ jobs:
           cache-dependency-path: |
             backend/requirements/default.txt
             backend/requirements/dev.txt
-
-      - name: Install Python dependencies
-        run: |
+      - run: |
           python -m pip install --upgrade pip
           pip install --retries 5 --timeout 30 -r backend/requirements/default.txt
           pip install --retries 5 --timeout 30 -r backend/requirements/dev.txt
@@ -96,153 +70,71 @@ jobs:
             --package-name onyx_openapi_client \
             --skip-validate-spec \
             --openapi-normalizer "SIMPLIFY_ONEOF_ANYOF=true,SET_OAS3_NULLABLE=true"
-
-      - name: Upload OpenAPI artifacts
-        uses: actions/upload-artifact@v4
-        with:
-          name: openapi-artifacts
-          path: backend/generated/
-
-  build-backend-image:
-    runs-on: blacksmith-16vcpu-ubuntu-2404-arm
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-
-      - name: Login to Private Registry
-        uses: docker/login-action@v3
-        with:
-          registry: ${{ env.PRIVATE_REGISTRY }}
-          username: ${{ env.PRIVATE_REGISTRY_USERNAME }}
-          password: ${{ env.PRIVATE_REGISTRY_PASSWORD }}
-
+            
       - name: Set up Docker Buildx
-        uses: useblacksmith/setup-docker-builder@v1
+        uses: docker/setup-buildx-action@v3
 
-      - name: Build and push Backend Docker image
-        uses: useblacksmith/build-push-action@v2
-        with:
-          context: ./backend
-          file: ./backend/Dockerfile
-          platforms: linux/arm64
-          tags: ${{ env.PRIVATE_REGISTRY }}/integration-test-onyx-backend:test-${{ github.run_id }}
-          push: true
-
-  build-model-server-image:
-    runs-on: blacksmith-16vcpu-ubuntu-2404-arm
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-
-      - name: Login to Private Registry
-        uses: docker/login-action@v3
-        with:
-          registry: ${{ env.PRIVATE_REGISTRY }}
-          username: ${{ env.PRIVATE_REGISTRY_USERNAME }}
-          password: ${{ env.PRIVATE_REGISTRY_PASSWORD }}
-
-      - name: Set up Docker Buildx
-        uses: useblacksmith/setup-docker-builder@v1
-
-      - name: Build and push Model Server Docker image
-        uses: useblacksmith/build-push-action@v2
-        with:
-          context: ./backend
-          file: ./backend/Dockerfile.model_server
-          platforms: linux/arm64
-          tags: ${{ env.PRIVATE_REGISTRY }}/integration-test-onyx-model-server:test-${{ github.run_id }}
-          push: true
-          outputs: type=registry
-          provenance: false
-
-  build-integration-image:
-    needs: prepare-build
-    runs-on: blacksmith-16vcpu-ubuntu-2404-arm
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-
-      - name: Login to Private Registry
-        uses: docker/login-action@v3
-        with:
-          registry: ${{ env.PRIVATE_REGISTRY }}
-          username: ${{ env.PRIVATE_REGISTRY_USERNAME }}
-          password: ${{ env.PRIVATE_REGISTRY_PASSWORD }}
-
-      - name: Download OpenAPI artifacts
-        uses: actions/download-artifact@v4
-        with:
-          name: openapi-artifacts
-          path: backend/generated/
-
-      - name: Set up Docker Buildx
-        uses: useblacksmith/setup-docker-builder@v1
-
-      - name: Build and push integration test Docker image
-        uses: useblacksmith/build-push-action@v2
-        with:
-          context: ./backend
-          file: ./backend/tests/integration/Dockerfile
-          platforms: linux/arm64
-          tags: ${{ env.PRIVATE_REGISTRY }}/integration-test-onyx-integration:test-${{ github.run_id }}
-          push: true
-
-  integration-tests-mit:
-    needs:
-      [
-        discover-test-dirs,
-        build-backend-image,
-        build-model-server-image,
-        build-integration-image,
-      ]
-    # See https://docs.blacksmith.sh/blacksmith-runners/overview
-    runs-on: blacksmith-8vcpu-ubuntu-2404-arm
-
-    strategy:
-      fail-fast: false
-      matrix:
-        test-dir: ${{ fromJson(needs.discover-test-dirs.outputs.test-dirs) }}
-
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-
-      - name: Login to Private Registry
-        uses: docker/login-action@v3
-        with:
-          registry: ${{ env.PRIVATE_REGISTRY }}
-          username: ${{ env.PRIVATE_REGISTRY_USERNAME }}
-          password: ${{ env.PRIVATE_REGISTRY_PASSWORD }}
-
-      # needed for pulling Vespa, Redis, Postgres, and Minio images
-      # otherwise, we hit the "Unauthenticated users" limit
-      # https://docs.docker.com/docker-hub/usage/
       - name: Login to Docker Hub
         uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_TOKEN }}
 
-      - name: Pull Docker images
+      # tag every docker image with "test" so that we can spin up the correct set
+      # of images during testing
+
+      # We don't need to build the Web Docker image since it's not yet used
+      # in the integration tests. We have a separate action to verify that it builds
+      # successfully.
+      - name: Pull Web Docker image
         run: |
-          # Pull all images from registry in parallel
-          echo "Pulling Docker images in parallel..."
-          # Pull images from private registry
-          (docker pull ${{ env.PRIVATE_REGISTRY }}/integration-test-onyx-backend:test-${{ github.run_id }}) &
-          (docker pull ${{ env.PRIVATE_REGISTRY }}/integration-test-onyx-model-server:test-${{ github.run_id }}) &
-          (docker pull ${{ env.PRIVATE_REGISTRY }}/integration-test-onyx-integration:test-${{ github.run_id }}) &
+          docker pull onyxdotapp/onyx-web-server:latest
+          docker tag onyxdotapp/onyx-web-server:latest onyxdotapp/onyx-web-server:test
 
-          # Wait for all background jobs to complete
-          wait
-          echo "All Docker images pulled successfully"
+      # we use the runs-on cache for docker builds
+      # in conjunction with runs-on runners, it has better speed and unlimited caching
+      # https://runs-on.com/caching/s3-cache-for-github-actions/
+      # https://runs-on.com/caching/docker/
+      # https://github.com/moby/buildkit#s3-cache-experimental
 
-          # Re-tag to remove registry prefix for docker-compose
-          docker tag ${{ env.PRIVATE_REGISTRY }}/integration-test-onyx-backend:test-${{ github.run_id }} onyxdotapp/onyx-backend:test
-          docker tag ${{ env.PRIVATE_REGISTRY }}/integration-test-onyx-model-server:test-${{ github.run_id }} onyxdotapp/onyx-model-server:test
-          docker tag ${{ env.PRIVATE_REGISTRY }}/integration-test-onyx-integration:test-${{ github.run_id }} onyxdotapp/onyx-integration:test
+      # images are built and run locally for testing purposes. Not pushed.
+      - name: Build Backend Docker image
+        uses: ./.github/actions/custom-build-and-push
+        with:
+          context: ./backend
+          file: ./backend/Dockerfile
+          platforms: linux/amd64
+          tags: onyxdotapp/onyx-backend:test
+          push: false
+          load: true
+          cache-from: type=s3,prefix=cache/${{ github.repository }}/mit-integration-tests/backend-${{ env.PLATFORM_PAIR }}/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }}
+          cache-to: type=s3,prefix=cache/${{ github.repository }}/mit-integration-tests/backend-${{ env.PLATFORM_PAIR }}/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }},mode=max
+
+      - name: Build Model Server Docker image
+        uses: ./.github/actions/custom-build-and-push
+        with:
+          context: ./backend
+          file: ./backend/Dockerfile.model_server
+          platforms: linux/amd64
+          tags: onyxdotapp/onyx-model-server:test
+          push: false
+          load: true
+          cache-from: type=s3,prefix=cache/${{ github.repository }}/mit-integration-tests/model-server-${{ env.PLATFORM_PAIR }}/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }}
+          cache-to: type=s3,prefix=cache/${{ github.repository }}/mit-integration-tests/model-server-${{ env.PLATFORM_PAIR }}/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }},mode=max
+
+      - name: Build integration test Docker image
+        uses: ./.github/actions/custom-build-and-push
+        with:
+          context: ./backend
+          file: ./backend/tests/integration/Dockerfile
+          platforms: linux/amd64
+          tags: onyxdotapp/onyx-integration:test
+          push: false
+          load: true
+          cache-from: type=s3,prefix=cache/${{ github.repository }}/mit-integration-tests/integration-${{ env.PLATFORM_PAIR }}/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }}
+          cache-to: type=s3,prefix=cache/${{ github.repository }}/mit-integration-tests/integration-${{ env.PLATFORM_PAIR }}/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }},mode=max
 
       # NOTE: Use pre-ping/null pool to reduce flakiness due to dropped connections
-      # NOTE: don't need web server for integration tests
       - name: Start Docker containers
         run: |
           cd deployment/docker_compose
@@ -253,16 +145,7 @@ jobs:
           DISABLE_TELEMETRY=true \
           IMAGE_TAG=test \
           INTEGRATION_TESTS_MODE=true \
-          docker compose -f docker-compose.dev.yml -p onyx-stack up \
-            relational_db \
-            index \
-            cache \
-            minio \
-            api_server \
-            inference_model_server \
-            indexing_model_server \
-            background \
-            -d
+          docker compose -f docker-compose.dev.yml -p onyx-stack up -d
         id: start_docker
 
       - name: Wait for service to be ready
@@ -306,44 +189,51 @@ jobs:
             -p mock-it-services-stack up -d
 
       # NOTE: Use pre-ping/null to reduce flakiness due to dropped connections
-      - name: Run Integration Tests for ${{ matrix.test-dir.name }}
-        uses: nick-fields/retry@v3
-        with:
-          timeout_minutes: 20
-          max_attempts: 3
-          retry_wait_seconds: 10
-          command: |
-            echo "Running integration tests for ${{ matrix.test-dir.path }}..."
-            docker run --rm --network onyx-stack_default \
-              --name test-runner \
-              -e POSTGRES_HOST=relational_db \
-              -e POSTGRES_USER=postgres \
-              -e POSTGRES_PASSWORD=password \
-              -e POSTGRES_DB=postgres \
-              -e DB_READONLY_USER=db_readonly_user \
-              -e DB_READONLY_PASSWORD=password \
-              -e POSTGRES_POOL_PRE_PING=true \
-              -e POSTGRES_USE_NULL_POOL=true \
-              -e VESPA_HOST=index \
-              -e REDIS_HOST=cache \
-              -e API_SERVER_HOST=api_server \
-              -e OPENAI_API_KEY=${OPENAI_API_KEY} \
-              -e SLACK_BOT_TOKEN=${SLACK_BOT_TOKEN} \
-              -e CONFLUENCE_TEST_SPACE_URL=${CONFLUENCE_TEST_SPACE_URL} \
-              -e CONFLUENCE_USER_NAME=${CONFLUENCE_USER_NAME} \
-              -e CONFLUENCE_ACCESS_TOKEN=${CONFLUENCE_ACCESS_TOKEN} \
-              -e JIRA_BASE_URL=${JIRA_BASE_URL} \
-              -e JIRA_USER_EMAIL=${JIRA_USER_EMAIL} \
-              -e JIRA_API_TOKEN=${JIRA_API_TOKEN} \
-              -e PERM_SYNC_SHAREPOINT_CLIENT_ID=${PERM_SYNC_SHAREPOINT_CLIENT_ID} \
-              -e PERM_SYNC_SHAREPOINT_PRIVATE_KEY="${PERM_SYNC_SHAREPOINT_PRIVATE_KEY}" \
-              -e PERM_SYNC_SHAREPOINT_CERTIFICATE_PASSWORD=${PERM_SYNC_SHAREPOINT_CERTIFICATE_PASSWORD} \
-              -e PERM_SYNC_SHAREPOINT_DIRECTORY_ID=${PERM_SYNC_SHAREPOINT_DIRECTORY_ID} \
-              -e TEST_WEB_HOSTNAME=test-runner \
-              -e MOCK_CONNECTOR_SERVER_HOST=mock_connector_server \
-              -e MOCK_CONNECTOR_SERVER_PORT=8001 \
-              onyxdotapp/onyx-integration:test \
-              /app/tests/integration/${{ matrix.test-dir.path }}
+      - name: Run Standard Integration Tests
+        run: |
+          echo "Running integration tests..."
+          docker run --rm --network onyx-stack_default \
+            --name test-runner \
+            -e POSTGRES_HOST=relational_db \
+            -e POSTGRES_USER=postgres \
+            -e POSTGRES_PASSWORD=password \
+            -e POSTGRES_DB=postgres \
+            -e DB_READONLY_USER=db_readonly_user \
+            -e DB_READONLY_PASSWORD=password \
+            -e POSTGRES_POOL_PRE_PING=true \
+            -e POSTGRES_USE_NULL_POOL=true \
+            -e VESPA_HOST=index \
+            -e REDIS_HOST=cache \
+            -e API_SERVER_HOST=api_server \
+            -e OPENAI_API_KEY=${OPENAI_API_KEY} \
+            -e SLACK_BOT_TOKEN=${SLACK_BOT_TOKEN} \
+            -e CONFLUENCE_TEST_SPACE_URL=${CONFLUENCE_TEST_SPACE_URL} \
+            -e CONFLUENCE_USER_NAME=${CONFLUENCE_USER_NAME} \
+            -e CONFLUENCE_ACCESS_TOKEN=${CONFLUENCE_ACCESS_TOKEN} \
+            -e JIRA_BASE_URL=${JIRA_BASE_URL} \
+            -e JIRA_USER_EMAIL=${JIRA_USER_EMAIL} \
+            -e JIRA_API_TOKEN=${JIRA_API_TOKEN} \
+            -e PERM_SYNC_SHAREPOINT_CLIENT_ID=${PERM_SYNC_SHAREPOINT_CLIENT_ID} \
+            -e PERM_SYNC_SHAREPOINT_PRIVATE_KEY="${PERM_SYNC_SHAREPOINT_PRIVATE_KEY}" \
+            -e PERM_SYNC_SHAREPOINT_CERTIFICATE_PASSWORD=${PERM_SYNC_SHAREPOINT_CERTIFICATE_PASSWORD} \
+            -e PERM_SYNC_SHAREPOINT_DIRECTORY_ID=${PERM_SYNC_SHAREPOINT_DIRECTORY_ID} \
+            -e TEST_WEB_HOSTNAME=test-runner \
+            -e MOCK_CONNECTOR_SERVER_HOST=mock_connector_server \
+            -e MOCK_CONNECTOR_SERVER_PORT=8001 \
+            onyxdotapp/onyx-integration:test \
+            /app/tests/integration/tests \
+            /app/tests/integration/connector_job_tests
+        continue-on-error: true
+        id: run_tests
+
+      - name: Check test results
+        run: |
+          if [ ${{ steps.run_tests.outcome }} == 'failure' ]; then
+            echo "Integration tests failed. Exiting with error."
+            exit 1
+          else
+            echo "All integration tests passed successfully."
+          fi
 
       # ------------------------------------------------------------
       # Always gather logs BEFORE "down":
@@ -363,7 +253,7 @@ jobs:
         if: always()
         uses: actions/upload-artifact@v4
         with:
-          name: docker-all-logs-${{ matrix.test-dir.name }}
+          name: docker-all-logs
           path: ${{ github.workspace }}/docker-compose.log
       # ------------------------------------------------------------
 
@@ -372,20 +262,3 @@ jobs:
         run: |
           cd deployment/docker_compose
           docker compose -f docker-compose.dev.yml -p onyx-stack down -v
-
-  
-  required: 
-    runs-on: blacksmith-2vcpu-ubuntu-2404-arm
-    needs: [integration-tests-mit]
-    if: ${{ always() }}
-    steps:
-      - uses: actions/github-script@v7
-        with:
-          script: |
-            const needs = ${{ toJSON(needs) }};
-            const failed = Object.values(needs).some(n => n.result !== 'success');
-            if (failed) {
-              core.setFailed('One or more upstream jobs failed or were cancelled.');
-            } else {
-              core.notice('All required jobs succeeded.');
-            }

.github/workflows/pr-playwright-tests.yml

@@ -6,165 +6,43 @@ concurrency:
 on: push
 
 env:
-  # AWS ECR Configuration
-  AWS_REGION: ${{ secrets.AWS_REGION || 'us-west-2' }}
-  ECR_REGISTRY: ${{ secrets.ECR_REGISTRY }}
-  AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID_ECR }}
-  AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY_ECR }}
-  BUILDX_NO_DEFAULT_ATTESTATIONS: 1
-  
-  # Test Environment Variables
   OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
   SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
   GEN_AI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
-  EXA_API_KEY: ${{ secrets.EXA_API_KEY }}
-
-  # for federated slack tests
-  SLACK_CLIENT_ID: ${{ secrets.SLACK_CLIENT_ID }}
-  SLACK_CLIENT_SECRET: ${{ secrets.SLACK_CLIENT_SECRET }}
-
   MOCK_LLM_RESPONSE: true
 
 jobs:
-  build-web-image:
-    runs-on: blacksmith-8vcpu-ubuntu-2404-arm
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-
-      - name: Configure AWS credentials
-        uses: aws-actions/configure-aws-credentials@v4
-        with:
-          aws-access-key-id: ${{ env.AWS_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ env.AWS_SECRET_ACCESS_KEY }}
-          aws-region: ${{ env.AWS_REGION }}
-
-      - name: Login to Amazon ECR
-        id: login-ecr
-        uses: aws-actions/amazon-ecr-login@v2
-
-      - name: Set up Docker Buildx
-        uses: useblacksmith/setup-docker-builder@v1
-
-      - name: Build and push Web Docker image
-        uses: useblacksmith/build-push-action@v2
-        with:
-          context: ./web
-          file: ./web/Dockerfile
-          platforms: linux/arm64
-          tags: ${{ env.ECR_REGISTRY }}/integration-test-onyx-web-server:playwright-test-${{ github.run_id }}
-          provenance: false
-          sbom: false
-          push: true
-
-  build-backend-image:
-    runs-on: blacksmith-8vcpu-ubuntu-2404-arm
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-
-      - name: Configure AWS credentials
-        uses: aws-actions/configure-aws-credentials@v4
-        with:
-          aws-access-key-id: ${{ env.AWS_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ env.AWS_SECRET_ACCESS_KEY }}
-          aws-region: ${{ env.AWS_REGION }}
-
-      - name: Login to Amazon ECR
-        id: login-ecr
-        uses: aws-actions/amazon-ecr-login@v2
-
-      - name: Set up Docker Buildx
-        uses: useblacksmith/setup-docker-builder@v1
-
-      - name: Build and push Backend Docker image
-        uses: useblacksmith/build-push-action@v2
-        with:
-          context: ./backend
-          file: ./backend/Dockerfile
-          platforms: linux/arm64
-          tags: ${{ env.ECR_REGISTRY }}/integration-test-onyx-backend:playwright-test-${{ github.run_id }}
-          provenance: false
-          sbom: false
-          push: true
-
-  build-model-server-image:
-    runs-on: blacksmith-8vcpu-ubuntu-2404-arm
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-
-      - name: Configure AWS credentials
-        uses: aws-actions/configure-aws-credentials@v4
-        with:
-          aws-access-key-id: ${{ env.AWS_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ env.AWS_SECRET_ACCESS_KEY }}
-          aws-region: ${{ env.AWS_REGION }}
-
-      - name: Login to Amazon ECR
-        id: login-ecr
-        uses: aws-actions/amazon-ecr-login@v2
-
-      - name: Set up Docker Buildx
-        uses: useblacksmith/setup-docker-builder@v1
-
-      - name: Build and push Model Server Docker image
-        uses: useblacksmith/build-push-action@v2
-        with:
-          context: ./backend
-          file: ./backend/Dockerfile.model_server
-          platforms: linux/arm64
-          tags: ${{ env.ECR_REGISTRY }}/integration-test-onyx-model-server:playwright-test-${{ github.run_id }}
-          provenance: false
-          sbom: false
-          push: true
-
   playwright-tests:
-    needs: [build-web-image, build-backend-image, build-model-server-image]
     name: Playwright Tests
-    runs-on: blacksmith-8vcpu-ubuntu-2404-arm
+
+    # See https://runs-on.com/runners/linux/
+    runs-on:
+      [
+        runs-on,
+        runner=32cpu-linux-x64,
+        disk=large,
+        "run-id=${{ github.run_id }}",
+      ]
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
-      - name: Configure AWS credentials
-        uses: aws-actions/configure-aws-credentials@v4
+      - name: Set up Python
+        uses: actions/setup-python@v5
         with:
-          aws-access-key-id: ${{ env.AWS_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ env.AWS_SECRET_ACCESS_KEY }}
-          aws-region: ${{ env.AWS_REGION }}
-
-      - name: Login to Amazon ECR
-        id: login-ecr
-        uses: aws-actions/amazon-ecr-login@v2
-
-      # needed for pulling Vespa, Redis, Postgres, and Minio images
-      # otherwise, we hit the "Unauthenticated users" limit
-      # https://docs.docker.com/docker-hub/usage/
-      - name: Login to Docker Hub
-        uses: docker/login-action@v3
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_TOKEN }}
-
-      - name: Pull Docker images
-        run: |
-          # Pull all images from ECR in parallel
-          echo "Pulling Docker images in parallel..."
-          (docker pull ${{ env.ECR_REGISTRY }}/integration-test-onyx-web-server:playwright-test-${{ github.run_id }}) &
-          (docker pull ${{ env.ECR_REGISTRY }}/integration-test-onyx-backend:playwright-test-${{ github.run_id }}) &
-          (docker pull ${{ env.ECR_REGISTRY }}/integration-test-onyx-model-server:playwright-test-${{ github.run_id }}) &
-
-          # Wait for all background jobs to complete
-          wait
-          echo "All Docker images pulled successfully"
-
-          # Re-tag with expected names for docker-compose
-          docker tag ${{ env.ECR_REGISTRY }}/integration-test-onyx-web-server:playwright-test-${{ github.run_id }} onyxdotapp/onyx-web-server:test
-          docker tag ${{ env.ECR_REGISTRY }}/integration-test-onyx-backend:playwright-test-${{ github.run_id }} onyxdotapp/onyx-backend:test
-          docker tag ${{ env.ECR_REGISTRY }}/integration-test-onyx-model-server:playwright-test-${{ github.run_id }} onyxdotapp/onyx-model-server:test
+          python-version: "3.11"
+          cache: "pip"
+          cache-dependency-path: |
+            backend/requirements/default.txt
+            backend/requirements/dev.txt
+            backend/requirements/model_server.txt
+      - run: |
+          python -m pip install --upgrade pip
+          pip install --retries 5 --timeout 30 -r backend/requirements/default.txt
+          pip install --retries 5 --timeout 30 -r backend/requirements/dev.txt
+          pip install --retries 5 --timeout 30 -r backend/requirements/model_server.txt
 
       - name: Setup node
         uses: actions/setup-node@v4
@@ -179,13 +57,68 @@ jobs:
         working-directory: ./web
         run: npx playwright install --with-deps
 
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_TOKEN }}
+
+      # tag every docker image with "test" so that we can spin up the correct set
+      # of images during testing
+
+      # we use the runs-on cache for docker builds
+      # in conjunction with runs-on runners, it has better speed and unlimited caching
+      # https://runs-on.com/caching/s3-cache-for-github-actions/
+      # https://runs-on.com/caching/docker/
+      # https://github.com/moby/buildkit#s3-cache-experimental
+
+      # images are built and run locally for testing purposes. Not pushed.
+
+      - name: Build Web Docker image
+        uses: ./.github/actions/custom-build-and-push
+        with:
+          context: ./web
+          file: ./web/Dockerfile
+          platforms: linux/amd64
+          tags: onyxdotapp/onyx-web-server:test
+          push: false
+          load: true
+          cache-from: type=s3,prefix=cache/${{ github.repository }}/integration-tests/web-server/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }}
+          cache-to: type=s3,prefix=cache/${{ github.repository }}/integration-tests/web-server/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }},mode=max
+
+      - name: Build Backend Docker image
+        uses: ./.github/actions/custom-build-and-push
+        with:
+          context: ./backend
+          file: ./backend/Dockerfile
+          platforms: linux/amd64
+          tags: onyxdotapp/onyx-backend:test
+          push: false
+          load: true
+          cache-from: type=s3,prefix=cache/${{ github.repository }}/integration-tests/backend/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }}
+          cache-to: type=s3,prefix=cache/${{ github.repository }}/integration-tests/backend/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }},mode=max
+
+      - name: Build Model Server Docker image
+        uses: ./.github/actions/custom-build-and-push
+        with:
+          context: ./backend
+          file: ./backend/Dockerfile.model_server
+          platforms: linux/amd64
+          tags: onyxdotapp/onyx-model-server:test
+          push: false
+          load: true
+          cache-from: type=s3,prefix=cache/${{ github.repository }}/integration-tests/model-server/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }}
+          cache-to: type=s3,prefix=cache/${{ github.repository }}/integration-tests/model-server/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }},mode=max
+
       - name: Start Docker containers
         run: |
           cd deployment/docker_compose
           ENABLE_PAID_ENTERPRISE_EDITION_FEATURES=true \
           AUTH_TYPE=basic \
-          GEN_AI_API_KEY=${{ env.OPENAI_API_KEY }} \
-          EXA_API_KEY=${{ env.EXA_API_KEY }} \
+          GEN_AI_API_KEY=${{ secrets.OPENAI_API_KEY }} \
           REQUIRE_EMAIL_VERIFICATION=false \
           DISABLE_TELEMETRY=true \
           IMAGE_TAG=test \

.github/workflows/pr-python-tests.yml

@@ -31,14 +31,12 @@ jobs:
         cache-dependency-path: |
           backend/requirements/default.txt
           backend/requirements/dev.txt
-          backend/requirements/model_server.txt
 
     - name: Install Dependencies
       run: |
         python -m pip install --upgrade pip
         pip install --retries 5 --timeout 30 -r backend/requirements/default.txt
         pip install --retries 5 --timeout 30 -r backend/requirements/dev.txt
-        pip install --retries 5 --timeout 30 -r backend/requirements/model_server.txt
 
     - name: Run Tests
       shell: script -q -e -c "bash --noprofile --norc -eo pipefail {0}"

.gitignore

@@ -17,7 +17,6 @@ backend/tests/regression/answer_quality/test_data.json
 backend/tests/regression/search_quality/eval-*
 backend/tests/regression/search_quality/search_eval_config.yaml
 backend/tests/regression/search_quality/*.json
-backend/onyx/evals/data/
 *.log
 
 # secret files
@@ -29,7 +28,6 @@ settings.json
 /deployment/data/nginx/app.conf
 *.sw?
 /backend/tests/regression/answer_quality/search_test_config.yaml
-*.egg-info
 
 # Local .terraform directories
 **/.terraform/*

.mcp.json.template

@@ -1,8 +0,0 @@
-{
-  "mcpServers": {
-    "onyx-mcp": {
-      "type": "http",
-      "url": "http://localhost:8000/mcp"
-    }
-  }
-}

.vscode/launch.template.jsonc

@@ -1,491 +1,496 @@
 /* Copy this file into '.vscode/launch.json' or merge its contents into your existing configurations. */
 
 {
-    // Use IntelliSense to learn about possible attributes.
-    // Hover to view descriptions of existing attributes.
-    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
-    "version": "0.2.0",
-    "compounds": [
-      {
-        // Dummy entry used to label the group
-        "name": "--- Compound ---",
-        "configurations": ["--- Individual ---"],
-        "presentation": {
-          "group": "1"
-        }
-      },
-      {
-        "name": "Run All Onyx Services",
-        "configurations": [
-          "Web Server",
-          "Model Server",
-          "API Server",
-          "Slack Bot",
-          "Celery primary",
-          "Celery light",
-          "Celery heavy",
-          "Celery docfetching",
-          "Celery docprocessing",
-          "Celery beat",
-          "Celery monitoring"
-        ],
-        "presentation": {
-          "group": "1"
-        },
-        "stopAll": true
-      },
-      {
-        "name": "Web / Model / API",
-        "configurations": ["Web Server", "Model Server", "API Server"],
-        "presentation": {
-          "group": "1"
-        },
-        "stopAll": true
-      },
-      {
-        "name": "Celery (all)",
-        "configurations": [
-          "Celery primary",
-          "Celery light",
-          "Celery heavy",
-          "Celery docfetching",
-          "Celery docprocessing",
-          "Celery beat",
-          "Celery monitoring"
-        ],
-        "presentation": {
-          "group": "1"
-        },
-        "stopAll": true
+  // Use IntelliSense to learn about possible attributes.
+  // Hover to view descriptions of existing attributes.
+  // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+  "version": "0.2.0",
+  "compounds": [
+    {
+      // Dummy entry used to label the group
+      "name": "--- Compound ---",
+      "configurations": ["--- Individual ---"],
+      "presentation": {
+        "group": "1"
       }
-    ],
-    "configurations": [
-      {
-        // Dummy entry used to label the group
-        "name": "--- Individual ---",
-        "type": "node",
-        "request": "launch",
-        "presentation": {
-          "group": "2",
-          "order": 0
-        }
-      },
-      {
-        "name": "Web Server",
-        "type": "node",
-        "request": "launch",
-        "cwd": "${workspaceRoot}/web",
-        "runtimeExecutable": "npm",
-        "envFile": "${workspaceFolder}/.vscode/.env",
-        "runtimeArgs": ["run", "dev"],
-        "presentation": {
-          "group": "2"
-        },
-        "console": "integratedTerminal",
-        "consoleTitle": "Web Server Console"
-      },
-      {
-        "name": "Model Server",
-        "consoleName": "Model Server",
-        "type": "debugpy",
-        "request": "launch",
-        "module": "uvicorn",
-        "cwd": "${workspaceFolder}/backend",
-        "envFile": "${workspaceFolder}/.vscode/.env",
-        "env": {
-          "LOG_LEVEL": "DEBUG",
-          "PYTHONUNBUFFERED": "1"
-        },
-        "args": ["model_server.main:app", "--reload", "--port", "9000"],
-        "presentation": {
-          "group": "2"
-        },
-        "consoleTitle": "Model Server Console"
-      },
-      {
-        "name": "API Server",
-        "consoleName": "API Server",
-        "type": "debugpy",
-        "request": "launch",
-        "module": "uvicorn",
-        "cwd": "${workspaceFolder}/backend",
-        "envFile": "${workspaceFolder}/.vscode/.env",
-        "env": {
-          "LOG_DANSWER_MODEL_INTERACTIONS": "True",
-          "LOG_LEVEL": "DEBUG",
-          "PYTHONUNBUFFERED": "1"
-        },
-        "args": ["onyx.main:app", "--reload", "--port", "8080"],
-        "presentation": {
-          "group": "2"
-        },
-        "consoleTitle": "API Server Console"
-      },
-      // For the listener to access the Slack API,
-      // DANSWER_BOT_SLACK_APP_TOKEN & DANSWER_BOT_SLACK_BOT_TOKEN need to be set in .env file located in the root of the project
-      {
-        "name": "Slack Bot",
-        "consoleName": "Slack Bot",
-        "type": "debugpy",
-        "request": "launch",
-        "program": "onyx/onyxbot/slack/listener.py",
-        "cwd": "${workspaceFolder}/backend",
-        "envFile": "${workspaceFolder}/.vscode/.env",
-        "env": {
-          "LOG_LEVEL": "DEBUG",
-          "PYTHONUNBUFFERED": "1",
-          "PYTHONPATH": "."
-        },
-        "presentation": {
-          "group": "2"
-        },
-        "consoleTitle": "Slack Bot Console"
-      },
-      {
-        "name": "Celery primary",
-        "type": "debugpy",
-        "request": "launch",
-        "module": "celery",
-        "cwd": "${workspaceFolder}/backend",
-        "envFile": "${workspaceFolder}/.vscode/.env",
-        "env": {
-          "LOG_LEVEL": "INFO",
-          "PYTHONUNBUFFERED": "1",
-          "PYTHONPATH": "."
-        },
-        "args": [
-          "-A",
-          "onyx.background.celery.versioned_apps.primary",
-          "worker",
-          "--pool=threads",
-          "--concurrency=4",
-          "--prefetch-multiplier=1",
-          "--loglevel=INFO",
-          "--hostname=primary@%n",
-          "-Q",
-          "celery"
-        ],
-        "presentation": {
-          "group": "2"
-        },
-        "consoleTitle": "Celery primary Console"
-      },
-      {
-        "name": "Celery light",
-        "type": "debugpy",
-        "request": "launch",
-        "module": "celery",
-        "cwd": "${workspaceFolder}/backend",
-        "envFile": "${workspaceFolder}/.vscode/.env",
-        "env": {
-          "LOG_LEVEL": "INFO",
-          "PYTHONUNBUFFERED": "1",
-          "PYTHONPATH": "."
-        },
-        "args": [
-          "-A",
-          "onyx.background.celery.versioned_apps.light",
-          "worker",
-          "--pool=threads",
-          "--concurrency=64",
-          "--prefetch-multiplier=8",
-          "--loglevel=INFO",
-          "--hostname=light@%n",
-          "-Q",
-          "vespa_metadata_sync,connector_deletion,doc_permissions_upsert,index_attempt_cleanup"
-        ],
-        "presentation": {
-          "group": "2"
-        },
-        "consoleTitle": "Celery light Console"
-      },
-      {
-        "name": "Celery heavy",
-        "type": "debugpy",
-        "request": "launch",
-        "module": "celery",
-        "cwd": "${workspaceFolder}/backend",
-        "envFile": "${workspaceFolder}/.vscode/.env",
-        "env": {
-          "LOG_LEVEL": "INFO",
-          "PYTHONUNBUFFERED": "1",
-          "PYTHONPATH": "."
-        },
-        "args": [
-          "-A",
-          "onyx.background.celery.versioned_apps.heavy",
-          "worker",
-          "--pool=threads",
-          "--concurrency=4",
-          "--prefetch-multiplier=1",
-          "--loglevel=INFO",
-          "--hostname=heavy@%n",
-          "-Q",
-          "connector_pruning,connector_doc_permissions_sync,connector_external_group_sync"
-        ],
-        "presentation": {
-          "group": "2"
-        },
-        "consoleTitle": "Celery heavy Console"
-      },
-      {
-        "name": "Celery docfetching",
-        "type": "debugpy",
-        "request": "launch",
-        "module": "celery",
-        "cwd": "${workspaceFolder}/backend",
-        "envFile": "${workspaceFolder}/.vscode/.env",
-        "env": {
-            "LOG_LEVEL": "DEBUG",
-            "PYTHONUNBUFFERED": "1",
-            "PYTHONPATH": "."
-        },
-        "args": [
-            "-A",
-            "onyx.background.celery.versioned_apps.docfetching",
-            "worker",
-            "--pool=threads",
-            "--concurrency=1",
-            "--prefetch-multiplier=1",
-            "--loglevel=INFO",
-            "--hostname=docfetching@%n",
-            "-Q",
-            "connector_doc_fetching,user_files_indexing"
-        ],
-        "presentation": {
-            "group": "2"
-        },
-        "consoleTitle": "Celery docfetching Console",
-        "justMyCode": false
     },
     {
-        "name": "Celery docprocessing",
-        "type": "debugpy",
-        "request": "launch",
-        "module": "celery",
-        "cwd": "${workspaceFolder}/backend",
-        "envFile": "${workspaceFolder}/.vscode/.env",
-        "env": {
-            "ENABLE_MULTIPASS_INDEXING": "false",
-            "LOG_LEVEL": "DEBUG",
-            "PYTHONUNBUFFERED": "1",
-            "PYTHONPATH": "."
-        },
-        "args": [
-            "-A",
-            "onyx.background.celery.versioned_apps.docprocessing",
-            "worker",
-            "--pool=threads",
-            "--concurrency=6",
-            "--prefetch-multiplier=1",
-            "--loglevel=INFO",
-            "--hostname=docprocessing@%n",
-            "-Q",
-            "docprocessing"
-        ],
-        "presentation": {
-            "group": "2"
-        },
-        "consoleTitle": "Celery docprocessing Console",
-        "justMyCode": false
+      "name": "Run All Onyx Services",
+      "configurations": [
+        "Web Server",
+        "Model Server",
+        "API Server",
+        "Slack Bot",
+        "Celery primary",
+        "Celery light",
+        "Celery heavy",
+        "Celery docfetching",
+        "Celery docprocessing",
+        "Celery beat",
+        "Celery monitoring",
+        "Celery user file processing"
+      ],
+      "presentation": {
+        "group": "1"
+      }
     },
-      {
-        "name": "Celery monitoring",
-        "type": "debugpy",
-        "request": "launch",
-        "module": "celery",
-        "cwd": "${workspaceFolder}/backend",
-        "envFile": "${workspaceFolder}/.vscode/.env",
-        "env": {},
-        "args": [
+    {
+      "name": "Web / Model / API",
+      "configurations": ["Web Server", "Model Server", "API Server"],
+      "presentation": {
+        "group": "1"
+      }
+    },
+    {
+      "name": "Celery (all)",
+      "configurations": [
+        "Celery primary",
+        "Celery light",
+        "Celery heavy",
+        "Celery docfetching",
+        "Celery docprocessing",
+        "Celery beat",
+        "Celery monitoring",
+        "Celery user file processing"
+      ],
+      "presentation": {
+        "group": "1"
+        },
+        "stopAll": true
+    }
+  ],
+  "configurations": [
+    
+    {
+      // Dummy entry used to label the group
+      "name": "--- Individual ---",
+      "type": "node",
+      "request": "launch",
+      "presentation": {
+        "group": "2",
+        "order": 0
+      }
+    },
+    {
+      "name": "Web Server",
+      "type": "node",
+      "request": "launch",
+      "cwd": "${workspaceRoot}/web",
+      "runtimeExecutable": "npm",
+      "envFile": "${workspaceFolder}/.vscode/.env",
+      "runtimeArgs": ["run", "dev"],
+      "presentation": {
+        "group": "2"
+      },
+      "console": "integratedTerminal",
+      "consoleTitle": "Web Server Console"
+    },
+    {
+      "name": "Model Server",
+      "consoleName": "Model Server",
+      "type": "debugpy",
+      "request": "launch",
+      "module": "uvicorn",
+      "cwd": "${workspaceFolder}/backend",
+      "envFile": "${workspaceFolder}/.vscode/.env",
+      "env": {
+        "LOG_LEVEL": "DEBUG",
+        "PYTHONUNBUFFERED": "1"
+      },
+      "args": ["model_server.main:app", "--reload", "--port", "9000"],
+      "presentation": {
+        "group": "2"
+      },
+      "consoleTitle": "Model Server Console"
+    },
+    {
+      "name": "API Server",
+      "consoleName": "API Server",
+      "type": "debugpy",
+      "request": "launch",
+      "module": "uvicorn",
+      "cwd": "${workspaceFolder}/backend",
+      "envFile": "${workspaceFolder}/.vscode/.env",
+      "env": {
+        "LOG_DANSWER_MODEL_INTERACTIONS": "True",
+        "LOG_LEVEL": "DEBUG",
+        "PYTHONUNBUFFERED": "1"
+      },
+      "args": ["onyx.main:app", "--reload", "--port", "8080"],
+      "presentation": {
+        "group": "2"
+      },
+      "consoleTitle": "API Server Console"
+    },
+    // For the listener to access the Slack API,
+    // DANSWER_BOT_SLACK_APP_TOKEN & DANSWER_BOT_SLACK_BOT_TOKEN need to be set in .env file located in the root of the project
+    {
+      "name": "Slack Bot",
+      "consoleName": "Slack Bot",
+      "type": "debugpy",
+      "request": "launch",
+      "program": "onyx/onyxbot/slack/listener.py",
+      "cwd": "${workspaceFolder}/backend",
+      "envFile": "${workspaceFolder}/.vscode/.env",
+      "env": {
+        "LOG_LEVEL": "DEBUG",
+        "PYTHONUNBUFFERED": "1",
+        "PYTHONPATH": "."
+      },
+      "presentation": {
+        "group": "2"
+      },
+      "consoleTitle": "Slack Bot Console"
+    },
+    {
+      "name": "Celery primary",
+      "type": "debugpy",
+      "request": "launch",
+      "module": "celery",
+      "cwd": "${workspaceFolder}/backend",
+      "envFile": "${workspaceFolder}/.vscode/.env",
+      "env": {
+        "LOG_LEVEL": "INFO",
+        "PYTHONUNBUFFERED": "1",
+        "PYTHONPATH": "."
+      },
+      "args": [
+        "-A",
+        "onyx.background.celery.versioned_apps.primary",
+        "worker",
+        "--pool=threads",
+        "--concurrency=4",
+        "--prefetch-multiplier=1",
+        "--loglevel=INFO",
+        "--hostname=primary@%n",
+        "-Q",
+        "celery"
+      ],
+      "presentation": {
+        "group": "2"
+      },
+      "consoleTitle": "Celery primary Console"
+    },
+    {
+      "name": "Celery light",
+      "type": "debugpy",
+      "request": "launch",
+      "module": "celery",
+      "cwd": "${workspaceFolder}/backend",
+      "envFile": "${workspaceFolder}/.vscode/.env",
+      "env": {
+        "LOG_LEVEL": "INFO",
+        "PYTHONUNBUFFERED": "1",
+        "PYTHONPATH": "."
+      },
+      "args": [
+        "-A",
+        "onyx.background.celery.versioned_apps.light",
+        "worker",
+        "--pool=threads",
+        "--concurrency=64",
+        "--prefetch-multiplier=8",
+        "--loglevel=INFO",
+        "--hostname=light@%n",
+        "-Q",
+          "vespa_metadata_sync,connector_deletion,doc_permissions_upsert,index_attempt_cleanup"
+      ],
+      "presentation": {
+        "group": "2"
+      },
+      "consoleTitle": "Celery light Console"
+    },
+    {
+      "name": "Celery heavy",
+      "type": "debugpy",
+      "request": "launch",
+      "module": "celery",
+      "cwd": "${workspaceFolder}/backend",
+      "envFile": "${workspaceFolder}/.vscode/.env",
+      "env": {
+        "LOG_LEVEL": "INFO",
+        "PYTHONUNBUFFERED": "1",
+        "PYTHONPATH": "."
+      },
+      "args": [
+        "-A",
+        "onyx.background.celery.versioned_apps.heavy",
+        "worker",
+        "--pool=threads",
+        "--concurrency=4",
+        "--prefetch-multiplier=1",
+        "--loglevel=INFO",
+        "--hostname=heavy@%n",
+        "-Q",
+        "connector_pruning,connector_doc_permissions_sync,connector_external_group_sync"
+      ],
+      "presentation": {
+        "group": "2"
+      },
+      "consoleTitle": "Celery heavy Console"
+    },
+    {
+      "name": "Celery docfetching",
+      "type": "debugpy",
+      "request": "launch",
+      "module": "celery",
+      "cwd": "${workspaceFolder}/backend",
+      "envFile": "${workspaceFolder}/.vscode/.env",
+      "env": {
+          "LOG_LEVEL": "DEBUG",
+          "PYTHONUNBUFFERED": "1",
+          "PYTHONPATH": "."
+      },
+      "args": [
           "-A",
-          "onyx.background.celery.versioned_apps.monitoring",
+          "onyx.background.celery.versioned_apps.docfetching",
           "worker",
-          "--pool=solo",
+          "--pool=threads",
           "--concurrency=1",
           "--prefetch-multiplier=1",
           "--loglevel=INFO",
-          "--hostname=monitoring@%n",
+          "--hostname=docfetching@%n",
           "-Q",
-          "monitoring"
-        ],
-        "presentation": {
+          "connector_doc_fetching,user_files_indexing"
+      ],
+      "presentation": {
           "group": "2"
-        },
-        "consoleTitle": "Celery monitoring Console"
       },
-      {
-        "name": "Celery beat",
-        "type": "debugpy",
-        "request": "launch",
-        "module": "celery",
-        "cwd": "${workspaceFolder}/backend",
-        "envFile": "${workspaceFolder}/.vscode/.env",
-        "env": {
+      "consoleTitle": "Celery docfetching Console",
+      "justMyCode": false
+  },
+  {
+      "name": "Celery docprocessing",
+      "type": "debugpy",
+      "request": "launch",
+      "module": "celery",
+      "cwd": "${workspaceFolder}/backend",
+      "envFile": "${workspaceFolder}/.vscode/.env",
+      "env": {
+          "ENABLE_MULTIPASS_INDEXING": "false",
           "LOG_LEVEL": "DEBUG",
           "PYTHONUNBUFFERED": "1",
           "PYTHONPATH": "."
-        },
-        "args": [
+      },
+      "args": [
           "-A",
-          "onyx.background.celery.versioned_apps.beat",
-          "beat",
-          "--loglevel=INFO"
-        ],
-        "presentation": {
+          "onyx.background.celery.versioned_apps.docprocessing",
+          "worker",
+          "--pool=threads",
+          "--concurrency=6",
+          "--prefetch-multiplier=1",
+          "--loglevel=INFO",
+          "--hostname=docprocessing@%n",
+          "-Q",
+          "docprocessing"
+      ],
+      "presentation": {
           "group": "2"
-        },
-        "consoleTitle": "Celery beat Console"
-      },
-      {
-        "name": "Pytest",
-        "consoleName": "Pytest",
-        "type": "debugpy",
-        "request": "launch",
-        "module": "pytest",
-        "cwd": "${workspaceFolder}/backend",
-        "envFile": "${workspaceFolder}/.vscode/.env",
-        "env": {
-          "LOG_LEVEL": "DEBUG",
-          "PYTHONUNBUFFERED": "1",
-          "PYTHONPATH": "."
-        },
-        "args": [
-          "-v"
-          // Specify a sepcific module/test to run or provide nothing to run all tests
-          //"tests/unit/onyx/llm/answering/test_prune_and_merge.py"
-        ],
-        "presentation": {
-          "group": "2"
-        },
-        "consoleTitle": "Pytest Console"
-      },
-      {
-        // Dummy entry used to label the group
-        "name": "--- Tasks ---",
-        "type": "node",
-        "request": "launch",
-        "presentation": {
-          "group": "3",
-          "order": 0
-        }
-      },
-      {
-        "name": "Clear and Restart External Volumes and Containers",
-        "type": "node",
-        "request": "launch",
-        "runtimeExecutable": "bash",
-        "runtimeArgs": [
-          "${workspaceFolder}/backend/scripts/restart_containers.sh"
-        ],
-        "cwd": "${workspaceFolder}",
-        "console": "integratedTerminal",
-        "stopOnEntry": true,
-        "presentation": {
-          "group": "3"
-        }
-      },
-      {
-        "name": "Eval CLI",
-        "type": "debugpy",
-        "request": "launch",
-        "program": "${workspaceFolder}/backend/onyx/evals/eval_cli.py",
-        "cwd": "${workspaceFolder}/backend",
-        "console": "integratedTerminal",
-        "justMyCode": false,
-        "envFile": "${workspaceFolder}/.vscode/.env",
-        "presentation": {
-          "group": "3"
-        },
-        "env": {
-          "LOG_LEVEL": "INFO",
-          "PYTHONUNBUFFERED": "1",
-          "PYTHONPATH": "."
-        },
-        "args": [
-            "--verbose"
-        ],
-        "consoleTitle": "Eval CLI Console"
-    },
-      {
-        // Celery jobs launched through a single background script (legacy)
-        // Recommend using the "Celery (all)" compound launch instead.
-        "name": "Background Jobs",
-        "consoleName": "Background Jobs",
-        "type": "debugpy",
-        "request": "launch",
-        "program": "scripts/dev_run_background_jobs.py",
-        "cwd": "${workspaceFolder}/backend",
-        "envFile": "${workspaceFolder}/.vscode/.env",
-        "env": {
-          "LOG_DANSWER_MODEL_INTERACTIONS": "True",
-          "LOG_LEVEL": "DEBUG",
-          "PYTHONUNBUFFERED": "1",
-          "PYTHONPATH": "."
-        }
-      },
-      {
-        "name": "Install Python Requirements",
-        "type": "node",
-        "request": "launch",
-        "runtimeExecutable": "bash",
-        "runtimeArgs": [
-          "-c",
-          "pip install -r backend/requirements/default.txt && pip install -r backend/requirements/dev.txt && pip install -r backend/requirements/ee.txt && pip install -r backend/requirements/model_server.txt"
-        ],
-        "cwd": "${workspaceFolder}",
-        "console": "integratedTerminal",
-        "presentation": {
-          "group": "3"
-        }
       },
+      "consoleTitle": "Celery docprocessing Console",
+      "justMyCode": false
+  },
     {
-      // script to generate the openapi schema
-      "name": "Onyx OpenAPI Schema Generator",
+      "name": "Celery monitoring",
       "type": "debugpy",
       "request": "launch",
-      "program": "scripts/onyx_openapi_schema.py",
+      "module": "celery",
       "cwd": "${workspaceFolder}/backend",
-      "envFile": "${workspaceFolder}/.env",
+      "envFile": "${workspaceFolder}/.vscode/.env",
+      "env": {},
+      "args": [
+        "-A",
+        "onyx.background.celery.versioned_apps.monitoring",
+        "worker",
+        "--pool=solo",
+        "--concurrency=1",
+        "--prefetch-multiplier=1",
+        "--loglevel=INFO",
+        "--hostname=monitoring@%n",
+        "-Q",
+        "monitoring"
+      ],
+      "presentation": {
+        "group": "2"
+      },
+      "consoleTitle": "Celery monitoring Console"
+    },
+    {
+      "name": "Celery beat",
+      "type": "debugpy",
+      "request": "launch",
+      "module": "celery",
+      "cwd": "${workspaceFolder}/backend",
+      "envFile": "${workspaceFolder}/.vscode/.env",
       "env": {
+        "LOG_LEVEL": "DEBUG",
         "PYTHONUNBUFFERED": "1",
         "PYTHONPATH": "."
       },
       "args": [
-        "--filename",
-        "generated/openapi.json"
-      ]
+        "-A",
+        "onyx.background.celery.versioned_apps.beat",
+        "beat",
+        "--loglevel=INFO"
+      ],
+      "presentation": {
+        "group": "2"
+      },
+      "consoleTitle": "Celery beat Console"
     },
     {
-      // script to debug multi tenant db issues
-      "name": "Onyx DB Manager (Top Chunks)",
+      "name": "Celery user file processing",
       "type": "debugpy",
       "request": "launch",
-      "program": "scripts/debugging/onyx_db.py",
+      "module": "celery",
+      "args": [
+        "-A",
+        "onyx.background.celery.versioned_apps.user_file_processing",
+        "worker",
+        "--loglevel=INFO",
+        "--hostname=user_file_processing@%n",
+        "--pool=threads",
+        "-Q",
+        "user_file_processing"
+      ],
       "cwd": "${workspaceFolder}/backend",
-      "envFile": "${workspaceFolder}/.env",
+      "envFile": "${workspaceFolder}/.vscode/.env",
       "env": {
+        "LOG_LEVEL": "DEBUG",
+        "PYTHONUNBUFFERED": "1",
+        "PYTHONPATH": "."
+      },
+      "presentation": {
+        "group": "2"
+      },
+      "consoleTitle": "Celery user file processing Console"
+    },
+    {
+      "name": "Pytest",
+      "consoleName": "Pytest",
+      "type": "debugpy",
+      "request": "launch",
+      "module": "pytest",
+      "cwd": "${workspaceFolder}/backend",
+      "envFile": "${workspaceFolder}/.vscode/.env",
+      "env": {
+        "LOG_LEVEL": "DEBUG",
         "PYTHONUNBUFFERED": "1",
         "PYTHONPATH": "."
       },
       "args": [
-        "--password",
-        "your_password_here",
-        "--port",
-        "5433",
-        "--report",
-        "top-chunks",
-        "--filename",
-        "generated/tenants_by_num_docs.csv"
-      ]
+        "-v"
+        // Specify a sepcific module/test to run or provide nothing to run all tests
+        //"tests/unit/onyx/llm/answering/test_prune_and_merge.py"
+      ],
+      "presentation": {
+        "group": "2"
+      },
+      "consoleTitle": "Pytest Console"
     },
-      {
-        "name": "Debug React Web App in Chrome",
-        "type": "chrome",
-        "request": "launch",
-        "url": "http://localhost:3000",
-        "webRoot": "${workspaceFolder}/web"
+    {
+      // Dummy entry used to label the group
+      "name": "--- Tasks ---",
+      "type": "node",
+      "request": "launch",
+      "presentation": {
+        "group": "3",
+        "order": 0
       }
+    },
+    {
+      "name": "Clear and Restart External Volumes and Containers",
+      "type": "node",
+      "request": "launch",
+      "runtimeExecutable": "bash",
+      "runtimeArgs": [
+        "${workspaceFolder}/backend/scripts/restart_containers.sh"
+      ],
+      "cwd": "${workspaceFolder}",
+      "console": "integratedTerminal",
+      "stopOnEntry": true,
+      "presentation": {
+        "group": "3"
+      }
+    },
+    {
+      // Celery jobs launched through a single background script (legacy)
+      // Recommend using the "Celery (all)" compound launch instead.
+      "name": "Background Jobs",
+      "consoleName": "Background Jobs",
+      "type": "debugpy",
+      "request": "launch",
+      "program": "scripts/dev_run_background_jobs.py",
+      "cwd": "${workspaceFolder}/backend",
+      "envFile": "${workspaceFolder}/.vscode/.env",
+      "env": {
+        "LOG_DANSWER_MODEL_INTERACTIONS": "True",
+        "LOG_LEVEL": "DEBUG",
+        "PYTHONUNBUFFERED": "1",
+        "PYTHONPATH": "."
+      }
+    },
+    {
+      "name": "Install Python Requirements",
+      "type": "node",
+      "request": "launch",
+      "runtimeExecutable": "bash",
+      "runtimeArgs": [
+        "-c",
+        "pip install -r backend/requirements/default.txt && pip install -r backend/requirements/dev.txt && pip install -r backend/requirements/ee.txt && pip install -r backend/requirements/model_server.txt"
+      ],
+      "cwd": "${workspaceFolder}",
+      "console": "integratedTerminal",
+      "presentation": {
+        "group": "3"
+      }
+    },
+  {
+    // script to generate the openapi schema
+    "name": "Onyx OpenAPI Schema Generator",
+    "type": "debugpy",
+    "request": "launch",
+    "program": "scripts/onyx_openapi_schema.py",
+    "cwd": "${workspaceFolder}/backend",
+    "envFile": "${workspaceFolder}/.env",
+    "env": {
+      "PYTHONUNBUFFERED": "1",
+      "PYTHONPATH": "."
+    },
+    "args": [
+      "--filename",
+      "generated/openapi.json"
     ]
-  }
- 
+  },
+  {
+    // script to debug multi tenant db issues
+    "name": "Onyx DB Manager (Top Chunks)",
+    "type": "debugpy",
+    "request": "launch",
+    "program": "scripts/debugging/onyx_db.py",
+    "cwd": "${workspaceFolder}/backend",
+    "envFile": "${workspaceFolder}/.env",
+    "env": {
+      "PYTHONUNBUFFERED": "1",
+      "PYTHONPATH": "."
+    },
+    "args": [
+      "--password",
+      "your_password_here",
+      "--port",
+      "5433",
+      "--report",
+      "top-chunks",
+      "--filename",
+      "generated/tenants_by_num_docs.csv"
+    ]
+  },
+    {
+      "name": "Debug React Web App in Chrome",
+      "type": "chrome",
+      "request": "launch",
+      "url": "http://localhost:3000",
+      "webRoot": "${workspaceFolder}/web"
+    }
+  ]
+}

AGENTS.md

@@ -1,295 +0,0 @@
-# AGENTS.md
-
-This file provides guidance to Codex when working with code in this repository.
-
-## KEY NOTES
-
-- If you run into any missing python dependency errors, try running your command with `workon onyx &&` in front
-to assume the python venv.
-- To make tests work, check the `.env` file at the root of the project to find an OpenAI key.
-- If using `playwright` to explore the frontend, you can usually log in with username `a@test.com` and password
-`a`. The app can be accessed at `http://localhost:3000`.
-- You should assume that all Onyx services are running. To verify, you can check the `backend/log` directory to
-make sure we see logs coming out from the relevant service.
-- To connect to the Postgres database, use: `docker exec -it onyx-stack-relational_db-1 psql -U postgres -c "<SQL>"`
-- When making calls to the backend, always go through the frontend. E.g. make a call to `http://localhost:3000/api/persona` not `http://localhost:8080/api/persona`
-- Put ALL db operations under the `backend/onyx/db` / `backend/ee/onyx/db` directories. Don't run queries
-outside of those directories.
-
-## Project Overview
-
-**Onyx** (formerly Danswer) is an open-source Gen-AI and Enterprise Search platform that connects to company documents, apps, and people. It features a modular architecture with both Community Edition (MIT licensed) and Enterprise Edition offerings.
-
-
-### Background Workers (Celery)
-
-Onyx uses Celery for asynchronous task processing with multiple specialized workers:
-
-#### Worker Types
-
-1. **Primary Worker** (`celery_app.py`)
-   - Coordinates core background tasks and system-wide operations
-   - Handles connector management, document sync, pruning, and periodic checks
-   - Runs with 4 threads concurrency
-   - Tasks: connector deletion, vespa sync, pruning, LLM model updates, user file sync
-
-2. **Docfetching Worker** (`docfetching`)
-   - Fetches documents from external data sources (connectors)
-   - Spawns docprocessing tasks for each document batch
-   - Implements watchdog monitoring for stuck connectors
-   - Configurable concurrency (default from env)
-
-3. **Docprocessing Worker** (`docprocessing`)
-   - Processes fetched documents through the indexing pipeline:
-     - Upserts documents to PostgreSQL
-     - Chunks documents and adds contextual information
-     - Embeds chunks via model server
-     - Writes chunks to Vespa vector database
-     - Updates document metadata
-   - Configurable concurrency (default from env)
-
-4. **Light Worker** (`light`)
-   - Handles lightweight, fast operations
-   - Tasks: vespa operations, document permissions sync, external group sync
-   - Higher concurrency for quick tasks
-
-5. **Heavy Worker** (`heavy`)
-   - Handles resource-intensive operations
-   - Primary task: document pruning operations
-   - Runs with 4 threads concurrency
-
-6. **KG Processing Worker** (`kg_processing`)
-   - Handles Knowledge Graph processing and clustering
-   - Builds relationships between documents
-   - Runs clustering algorithms
-   - Configurable concurrency
-
-7. **Monitoring Worker** (`monitoring`)
-   - System health monitoring and metrics collection
-   - Monitors Celery queues, process memory, and system status
-   - Single thread (monitoring doesn't need parallelism)
-   - Cloud-specific monitoring tasks
-
-8. **Beat Worker** (`beat`)
-   - Celery's scheduler for periodic tasks
-   - Uses DynamicTenantScheduler for multi-tenant support
-   - Schedules tasks like:
-     - Indexing checks (every 15 seconds)
-     - Connector deletion checks (every 20 seconds)
-     - Vespa sync checks (every 20 seconds)
-     - Pruning checks (every 20 seconds)
-     - KG processing (every 60 seconds)
-     - Monitoring tasks (every 5 minutes)
-     - Cleanup tasks (hourly)
-
-#### Key Features
-
-- **Thread-based Workers**: All workers use thread pools (not processes) for stability
-- **Tenant Awareness**: Multi-tenant support with per-tenant task isolation. There is a 
-middleware layer that automatically finds the appropriate tenant ID when sending tasks 
-via Celery Beat.
-- **Task Prioritization**: High, Medium, Low priority queues
-- **Monitoring**: Built-in heartbeat and liveness checking
-- **Failure Handling**: Automatic retry and failure recovery mechanisms
-- **Redis Coordination**: Inter-process communication via Redis
-- **PostgreSQL State**: Task state and metadata stored in PostgreSQL
-
-
-#### Important Notes
-
-**Defining Tasks**: 
-- Always use `@shared_task` rather than `@celery_app`
-- Put tasks under `background/celery/tasks/` or `ee/background/celery/tasks`
-
-**Defining APIs**:
-When creating new FastAPI APIs, do NOT use the `response_model` field. Instead, just type the
-function.
-
-**Testing Updates**:
-If you make any updates to a celery worker and you want to test these changes, you will need
-to ask me to restart the celery worker. There is no auto-restart on code-change mechanism.
-
-### Code Quality
-```bash
-# Install and run pre-commit hooks
-pre-commit install
-pre-commit run --all-files
-```
-
-NOTE: Always make sure everything is strictly typed (both in Python and Typescript).
-
-## Architecture Overview
-
-### Technology Stack
-- **Backend**: Python 3.11, FastAPI, SQLAlchemy, Alembic, Celery
-- **Frontend**: Next.js 15+, React 18, TypeScript, Tailwind CSS
-- **Database**: PostgreSQL with Redis caching
-- **Search**: Vespa vector database
-- **Auth**: OAuth2, SAML, multi-provider support
-- **AI/ML**: LangChain, LiteLLM, multiple embedding models
-
-### Directory Structure
-
-```
-backend/
-├── onyx/
-│   ├── auth/                    # Authentication & authorization
-│   ├── chat/                    # Chat functionality & LLM interactions
-│   ├── connectors/              # Data source connectors
-│   ├── db/                      # Database models & operations
-│   ├── document_index/          # Vespa integration
-│   ├── federated_connectors/    # External search connectors
-│   ├── llm/                     # LLM provider integrations
-│   └── server/                  # API endpoints & routers
-├── ee/                          # Enterprise Edition features
-├── alembic/                     # Database migrations
-└── tests/                       # Test suites
-
-web/
-├── src/app/                     # Next.js app router pages
-├── src/components/              # Reusable React components
-└── src/lib/                     # Utilities & business logic
-```
-
-## Database & Migrations
-
-### Running Migrations
-```bash
-# Standard migrations
-alembic upgrade head
-
-# Multi-tenant (Enterprise)
-alembic -n schema_private upgrade head
-```
-
-### Creating Migrations
-```bash
-# Auto-generate migration
-alembic revision --autogenerate -m "description"
-
-# Multi-tenant migration
-alembic -n schema_private revision --autogenerate -m "description"
-```
-
-## Testing Strategy
-
-There are 4 main types of tests within Onyx:
-
-### Unit Tests
-These should not assume any Onyx/external services are available to be called.
-Interactions with the outside world should be mocked using `unittest.mock`. Generally, only 
-write these for complex, isolated modules e.g. `citation_processing.py`.
-
-To run them:
-
-```bash
-python -m dotenv -f .vscode/.env run -- pytest -xv backend/tests/unit
-```
-
-### External Dependency Unit Tests
-These tests assume that all external dependencies of Onyx are available and callable (e.g. Postgres, Redis, 
-MinIO/S3, Vespa are running + OpenAI can be called + any request to the internet is fine + etc.).
-
-However, the actual Onyx containers are not running and with these tests we call the function to test directly.
-We can also mock components/calls at will. 
-
-The goal with these tests are to minimize mocking while giving some flexibility to mock things that are flakey, 
-need strictly controlled behavior, or need to have their internal behavior validated (e.g. verify a function is called
-with certain args, something that would be impossible with proper integration tests).
-
-A great example of this type of test is `backend/tests/external_dependency_unit/connectors/confluence/test_confluence_group_sync.py`.
-
-To run them:
-
-```bash
-python -m dotenv -f .vscode/.env run -- pytest backend/tests/external_dependency_unit
-```
-
-### Integration Tests
-Standard integration tests. Every test in `backend/tests/integration` runs against a real Onyx deployment. We cannot 
-mock anything in these tests. Prefer writing integration tests (or External Dependency Unit Tests if mocking/internal 
-verification is necessary) over any other type of test.
-
-Tests are parallelized at a directory level.
-
-When writing integration tests, make sure to check the root `conftest.py` for useful fixtures + the `backend/tests/integration/common_utils` directory for utilities. Prefer (if one exists), calling the appropriate Manager 
-class in the utils over directly calling the APIs with a library like `requests`. Prefer using fixtures rather than
-calling the utilities directly (e.g. do NOT create admin users with 
-`admin_user = UserManager.create(name="admin_user")`, instead use the `admin_user` fixture).
-
-A great example of this type of test is `backend/tests/integration/dev_apis/test_simple_chat_api.py`.
-
-To run them:
-
-```bash
-python -m dotenv -f .vscode/.env run -- pytest backend/tests/integration
-```
-
-### Playwright (E2E) Tests
-These tests are an even more complete version of the Integration Tests mentioned above. Has all services of Onyx 
-running, *including* the Web Server.
-
-Use these tests for anything that requires significant frontend <-> backend coordination.
-
-Tests are located at `web/tests/e2e`. Tests are written in TypeScript.
-
-To run them:
-
-```bash
-npx playwright test <TEST_NAME>
-```
-
-
-## Logs
-
-When (1) writing integration tests or (2) doing live tests (e.g. curl / playwright) you can get access
-to logs via the `backend/log/<service_name>_debug.log` file. All Onyx services (api_server, web_server, celery_X)
-will be tailing their logs to this file. 
-
-
-## Security Considerations
-
-- Never commit API keys or secrets to repository
-- Use encrypted credential storage for connector credentials
-- Follow RBAC patterns for new features
-- Implement proper input validation with Pydantic models
-- Use parameterized queries to prevent SQL injection
-
-## AI/LLM Integration
-
-- Multiple LLM providers supported via LiteLLM
-- Configurable models per feature (chat, search, embeddings)
-- Streaming support for real-time responses
-- Token management and rate limiting
-- Custom prompts and agent actions
-
-## UI/UX Patterns
-
-- Tailwind CSS with design system in `web/src/components/ui/`
-- Radix UI and Headless UI for accessible components
-- SWR for data fetching and caching
-- Form validation with react-hook-form
-- Error handling with popup notifications
-
-## Creating a Plan
-When creating a plan in the `plans` directory, make sure to include at least these elements:
-
-**Issues to Address**
-What the change is meant to do.
-
-**Important Notes**
-Things you come across in your research that are important to the implementation.
-
-**Implementation strategy**
-How you are going to make the changes happen. High level approach.
-
-**Tests**
-What unit (use rarely), external dependency unit, integration, and playwright tests you plan to write to 
-verify the correct behavior. Don't overtest. Usually, a given change only needs one type of test.
-
-Do NOT include these: *Timeline*, *Rollback plan*
-
-This is a minimal list - feel free to include more. Do NOT write code as part of your plan.
-Keep it high level. You can reference certain files or functions though.
-
-Before writing your plan, make sure to do research. Explore the relevant sections in the codebase.

CLAUDE.md

@@ -1,295 +0,0 @@
-# CLAUDE.md
-
-This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
-
-## KEY NOTES
-
-- If you run into any missing python dependency errors, try running your command with `workon onyx &&` in front
-to assume the python venv.
-- To make tests work, check the `.env` file at the root of the project to find an OpenAI key.
-- If using `playwright` to explore the frontend, you can usually log in with username `a@test.com` and password
-`a`. The app can be accessed at `http://localhost:3000`.
-- You should assume that all Onyx services are running. To verify, you can check the `backend/log` directory to
-make sure we see logs coming out from the relevant service.
-- To connect to the Postgres database, use: `docker exec -it onyx-stack-relational_db-1 psql -U postgres -c "<SQL>"`
-- When making calls to the backend, always go through the frontend. E.g. make a call to `http://localhost:3000/api/persona` not `http://localhost:8080/api/persona`
-- Put ALL db operations under the `backend/onyx/db` / `backend/ee/onyx/db` directories. Don't run queries
-outside of those directories.
-
-## Project Overview
-
-**Onyx** (formerly Danswer) is an open-source Gen-AI and Enterprise Search platform that connects to company documents, apps, and people. It features a modular architecture with both Community Edition (MIT licensed) and Enterprise Edition offerings.
-
-
-### Background Workers (Celery)
-
-Onyx uses Celery for asynchronous task processing with multiple specialized workers:
-
-#### Worker Types
-
-1. **Primary Worker** (`celery_app.py`)
-   - Coordinates core background tasks and system-wide operations
-   - Handles connector management, document sync, pruning, and periodic checks
-   - Runs with 4 threads concurrency
-   - Tasks: connector deletion, vespa sync, pruning, LLM model updates, user file sync
-
-2. **Docfetching Worker** (`docfetching`)
-   - Fetches documents from external data sources (connectors)
-   - Spawns docprocessing tasks for each document batch
-   - Implements watchdog monitoring for stuck connectors
-   - Configurable concurrency (default from env)
-
-3. **Docprocessing Worker** (`docprocessing`)
-   - Processes fetched documents through the indexing pipeline:
-     - Upserts documents to PostgreSQL
-     - Chunks documents and adds contextual information
-     - Embeds chunks via model server
-     - Writes chunks to Vespa vector database
-     - Updates document metadata
-   - Configurable concurrency (default from env)
-
-4. **Light Worker** (`light`)
-   - Handles lightweight, fast operations
-   - Tasks: vespa operations, document permissions sync, external group sync
-   - Higher concurrency for quick tasks
-
-5. **Heavy Worker** (`heavy`)
-   - Handles resource-intensive operations
-   - Primary task: document pruning operations
-   - Runs with 4 threads concurrency
-
-6. **KG Processing Worker** (`kg_processing`)
-   - Handles Knowledge Graph processing and clustering
-   - Builds relationships between documents
-   - Runs clustering algorithms
-   - Configurable concurrency
-
-7. **Monitoring Worker** (`monitoring`)
-   - System health monitoring and metrics collection
-   - Monitors Celery queues, process memory, and system status
-   - Single thread (monitoring doesn't need parallelism)
-   - Cloud-specific monitoring tasks
-
-8. **Beat Worker** (`beat`)
-   - Celery's scheduler for periodic tasks
-   - Uses DynamicTenantScheduler for multi-tenant support
-   - Schedules tasks like:
-     - Indexing checks (every 15 seconds)
-     - Connector deletion checks (every 20 seconds)
-     - Vespa sync checks (every 20 seconds)
-     - Pruning checks (every 20 seconds)
-     - KG processing (every 60 seconds)
-     - Monitoring tasks (every 5 minutes)
-     - Cleanup tasks (hourly)
-
-#### Key Features
-
-- **Thread-based Workers**: All workers use thread pools (not processes) for stability
-- **Tenant Awareness**: Multi-tenant support with per-tenant task isolation. There is a 
-middleware layer that automatically finds the appropriate tenant ID when sending tasks 
-via Celery Beat.
-- **Task Prioritization**: High, Medium, Low priority queues
-- **Monitoring**: Built-in heartbeat and liveness checking
-- **Failure Handling**: Automatic retry and failure recovery mechanisms
-- **Redis Coordination**: Inter-process communication via Redis
-- **PostgreSQL State**: Task state and metadata stored in PostgreSQL
-
-
-#### Important Notes
-
-**Defining Tasks**: 
-- Always use `@shared_task` rather than `@celery_app`
-- Put tasks under `background/celery/tasks/` or `ee/background/celery/tasks`
-
-**Defining APIs**:
-When creating new FastAPI APIs, do NOT use the `response_model` field. Instead, just type the
-function.
-
-**Testing Updates**:
-If you make any updates to a celery worker and you want to test these changes, you will need
-to ask me to restart the celery worker. There is no auto-restart on code-change mechanism.
-
-### Code Quality
-```bash
-# Install and run pre-commit hooks
-pre-commit install
-pre-commit run --all-files
-```
-
-NOTE: Always make sure everything is strictly typed (both in Python and Typescript).
-
-## Architecture Overview
-
-### Technology Stack
-- **Backend**: Python 3.11, FastAPI, SQLAlchemy, Alembic, Celery
-- **Frontend**: Next.js 15+, React 18, TypeScript, Tailwind CSS
-- **Database**: PostgreSQL with Redis caching
-- **Search**: Vespa vector database
-- **Auth**: OAuth2, SAML, multi-provider support
-- **AI/ML**: LangChain, LiteLLM, multiple embedding models
-
-### Directory Structure
-
-```
-backend/
-├── onyx/
-│   ├── auth/                    # Authentication & authorization
-│   ├── chat/                    # Chat functionality & LLM interactions
-│   ├── connectors/              # Data source connectors
-│   ├── db/                      # Database models & operations
-│   ├── document_index/          # Vespa integration
-│   ├── federated_connectors/    # External search connectors
-│   ├── llm/                     # LLM provider integrations
-│   └── server/                  # API endpoints & routers
-├── ee/                          # Enterprise Edition features
-├── alembic/                     # Database migrations
-└── tests/                       # Test suites
-
-web/
-├── src/app/                     # Next.js app router pages
-├── src/components/              # Reusable React components
-└── src/lib/                     # Utilities & business logic
-```
-
-## Database & Migrations
-
-### Running Migrations
-```bash
-# Standard migrations
-alembic upgrade head
-
-# Multi-tenant (Enterprise)
-alembic -n schema_private upgrade head
-```
-
-### Creating Migrations
-```bash
-# Auto-generate migration
-alembic revision --autogenerate -m "description"
-
-# Multi-tenant migration
-alembic -n schema_private revision --autogenerate -m "description"
-```
-
-## Testing Strategy
-
-There are 4 main types of tests within Onyx:
-
-### Unit Tests
-These should not assume any Onyx/external services are available to be called.
-Interactions with the outside world should be mocked using `unittest.mock`. Generally, only 
-write these for complex, isolated modules e.g. `citation_processing.py`.
-
-To run them:
-
-```bash
-python -m dotenv -f .vscode/.env run -- pytest -xv backend/tests/unit
-```
-
-### External Dependency Unit Tests
-These tests assume that all external dependencies of Onyx are available and callable (e.g. Postgres, Redis, 
-MinIO/S3, Vespa are running + OpenAI can be called + any request to the internet is fine + etc.).
-
-However, the actual Onyx containers are not running and with these tests we call the function to test directly.
-We can also mock components/calls at will. 
-
-The goal with these tests are to minimize mocking while giving some flexibility to mock things that are flakey, 
-need strictly controlled behavior, or need to have their internal behavior validated (e.g. verify a function is called
-with certain args, something that would be impossible with proper integration tests).
-
-A great example of this type of test is `backend/tests/external_dependency_unit/connectors/confluence/test_confluence_group_sync.py`.
-
-To run them:
-
-```bash
-python -m dotenv -f .vscode/.env run -- pytest backend/tests/external_dependency_unit
-```
-
-### Integration Tests
-Standard integration tests. Every test in `backend/tests/integration` runs against a real Onyx deployment. We cannot 
-mock anything in these tests. Prefer writing integration tests (or External Dependency Unit Tests if mocking/internal 
-verification is necessary) over any other type of test.
-
-Tests are parallelized at a directory level.
-
-When writing integration tests, make sure to check the root `conftest.py` for useful fixtures + the `backend/tests/integration/common_utils` directory for utilities. Prefer (if one exists), calling the appropriate Manager 
-class in the utils over directly calling the APIs with a library like `requests`. Prefer using fixtures rather than
-calling the utilities directly (e.g. do NOT create admin users with 
-`admin_user = UserManager.create(name="admin_user")`, instead use the `admin_user` fixture).
-
-A great example of this type of test is `backend/tests/integration/dev_apis/test_simple_chat_api.py`.
-
-To run them:
-
-```bash
-python -m dotenv -f .vscode/.env run -- pytest backend/tests/integration
-```
-
-### Playwright (E2E) Tests
-These tests are an even more complete version of the Integration Tests mentioned above. Has all services of Onyx 
-running, *including* the Web Server.
-
-Use these tests for anything that requires significant frontend <-> backend coordination.
-
-Tests are located at `web/tests/e2e`. Tests are written in TypeScript.
-
-To run them:
-
-```bash
-npx playwright test <TEST_NAME>
-```
-
-
-## Logs
-
-When (1) writing integration tests or (2) doing live tests (e.g. curl / playwright) you can get access
-to logs via the `backend/log/<service_name>_debug.log` file. All Onyx services (api_server, web_server, celery_X)
-will be tailing their logs to this file. 
-
-
-## Security Considerations
-
-- Never commit API keys or secrets to repository
-- Use encrypted credential storage for connector credentials
-- Follow RBAC patterns for new features
-- Implement proper input validation with Pydantic models
-- Use parameterized queries to prevent SQL injection
-
-## AI/LLM Integration
-
-- Multiple LLM providers supported via LiteLLM
-- Configurable models per feature (chat, search, embeddings)
-- Streaming support for real-time responses
-- Token management and rate limiting
-- Custom prompts and agent actions
-
-## UI/UX Patterns
-
-- Tailwind CSS with design system in `web/src/components/ui/`
-- Radix UI and Headless UI for accessible components
-- SWR for data fetching and caching
-- Form validation with react-hook-form
-- Error handling with popup notifications
-
-## Creating a Plan
-When creating a plan in the `plans` directory, make sure to include at least these elements:
-
-**Issues to Address**
-What the change is meant to do.
-
-**Important Notes**
-Things you come across in your research that are important to the implementation.
-
-**Implementation strategy**
-How you are going to make the changes happen. High level approach.
-
-**Tests**
-What unit (use rarely), external dependency unit, integration, and playwright tests you plan to write to 
-verify the correct behavior. Don't overtest. Usually, a given change only needs one type of test.
-
-Do NOT include these: *Timeline*, *Rollback plan*
-
-This is a minimal list - feel free to include more. Do NOT write code as part of your plan.
-Keep it high level. You can reference certain files or functions though.
-
-Before writing your plan, make sure to do research. Explore the relevant sections in the codebase.

README.md

@@ -57,7 +57,7 @@ https://private-user-images.githubusercontent.com/32520769/414509312-48392e83-95
 **To try it out for free and get started in seconds, check out [Onyx Cloud](https://cloud.onyx.app/signup)**.
 
 Onyx can also be run locally (even on a laptop) or deployed on a virtual machine with a single
-`docker compose` command. Checkout our [docs](https://docs.onyx.app/deployment/getting_started/quickstart) to learn more.
+`docker compose` command. Checkout our [docs](https://docs.onyx.app/quickstart) to learn more.
 
 We also have built-in support for high-availability/scalable deployment on Kubernetes.
 References [here](https://github.com/onyx-dot-app/onyx/tree/main/deployment).
@@ -97,7 +97,7 @@ Keep knowledge and access up to sync across 40+ connectors:
 - Websites
 - And more ...
 
-See the full list [here](https://docs.onyx.app/admin/connectors/overview).
+See the full list [here](https://docs.onyx.app/connectors).
 
 
 ## 📚 Licensing
@@ -113,4 +113,5 @@ To try the Onyx Enterprise Edition:
 
 
 ## 💡 Contributing
-Looking to contribute? Please check out the [Contribution Guide](CONTRIBUTING.md) for more details.
+Looking to contribute? Please check out the [Contribution Guide](CONTRIBUTING.md) for more details.
+

backend/Dockerfile.model_server

@@ -23,21 +23,15 @@ RUN mkdir -p /app && \
     chmod 755 /var/log/onyx && \
     chown onyx:onyx /var/log/onyx
 
-# --- add toolchain needed for Rust/Python builds (fastuuid) ---
-ENV RUSTUP_HOME=/usr/local/rustup \
-    CARGO_HOME=/usr/local/cargo \
-    PATH=/usr/local/cargo/bin:$PATH
-
-RUN set -eux; \
-    apt-get update && apt-get install -y --no-install-recommends \
+# Install build tools needed for compiling Rust packages like fastuuid
+RUN apt-get update && apt-get install -y \
         build-essential \
-        pkg-config \
         curl \
-        ca-certificates \
-    && rm -rf /var/lib/apt/lists/* \
-    # Install latest stable Rust (supports Cargo.lock v4)
-    && curl -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain stable \
-    && rustc --version && cargo --version
+        && rm -rf /var/lib/apt/lists/*
+
+# Install Rust (needed for fastuuid compilation)
+RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
+ENV PATH="/root/.cargo/bin:${PATH}"
 
 COPY ./requirements/model_server.txt /tmp/requirements.txt
 RUN pip install --no-cache-dir --upgrade \
@@ -45,6 +39,11 @@ RUN pip install --no-cache-dir --upgrade \
         --timeout 30 \
         -r /tmp/requirements.txt
 
+# Clean up build tools to reduce image size
+RUN apt-get remove -y build-essential curl && \
+    apt-get autoremove -y && \
+    rm -rf /root/.cargo /root/.rustup
+
 RUN apt-get remove -y --allow-remove-essential perl-base && \ 
     apt-get autoremove -y
 

backend/alembic/versions/085d844e3953_add_project__userfile_table_and_.py

@@ -0,0 +1,380 @@
+"""add project__userfile table and userfile column changes
+
+Revision ID: 085d844e3953
+Revises: 8818cf73fa1a
+Create Date: 2025-09-05 14:24:50.026940
+
+"""
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql as psql
+
+# revision identifiers, used by Alembic.
+revision = "085d844e3953"
+down_revision = "8818cf73fa1a"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # 0) Ensure UUID generator exists
+    op.execute("CREATE EXTENSION IF NOT EXISTS pgcrypto")
+
+    # Drop persona__user_folder table
+    try:
+        op.drop_table("persona__user_folder")
+    except Exception:
+        # Table might not exist, that's okay
+        pass
+
+    # Drop folder related tables and columns
+    # First try to drop the foreign key constraint if it exists
+    try:
+        # TODO(subash): do proper deletion on constraints
+        op.drop_constraint(
+            "chat_session_folder_id_fkey", "chat_session", type_="foreignkey"
+        )
+    except Exception:
+        # Constraint might not exist, that's okay
+        pass
+
+    # Then drop the folder_id column if it exists
+    try:
+        op.drop_column("chat_session", "folder_id")
+    except Exception:
+        # Column might not exist, that's okay
+        pass
+
+    # Finally drop the chat_folder table if it exists
+    try:
+        op.drop_table("chat_folder")
+    except Exception:
+        # Table might not exist, that's okay
+        pass
+
+    # 1) Add transitional UUID column on user_file + UNIQUE so FKs can reference it
+    op.add_column(
+        "user_file",
+        sa.Column(
+            "new_id",
+            psql.UUID(as_uuid=True),
+            nullable=False,
+            server_default=sa.text("gen_random_uuid()"),
+        ),
+    )
+    op.create_unique_constraint("uq_user_file_new_id", "user_file", ["new_id"])
+
+    # 2) Move FK users to the transitional UUID
+    # ---- persona__user_file.user_file_id (INT) -> UUID ----
+    op.add_column(
+        "persona__user_file",
+        sa.Column("user_file_id_uuid", psql.UUID(as_uuid=True), nullable=True),
+    )
+    op.execute(
+        """
+        UPDATE persona__user_file p
+        SET user_file_id_uuid = uf.new_id
+        FROM user_file uf
+        WHERE p.user_file_id = uf.id
+        """
+    )
+    # swap FK to reference user_file.new_id (the transitional UNIQUE)
+    op.drop_constraint(
+        "persona__user_file_user_file_id_fkey",
+        "persona__user_file",
+        type_="foreignkey",
+    )
+    op.alter_column("persona__user_file", "user_file_id_uuid", nullable=False)
+    op.create_foreign_key(
+        "persona__user_file_user_file_id_fkey",
+        "persona__user_file",
+        "user_file",
+        local_cols=["user_file_id_uuid"],
+        remote_cols=["new_id"],
+    )
+    op.drop_column("persona__user_file", "user_file_id")
+    op.alter_column(
+        "persona__user_file",
+        "user_file_id_uuid",
+        new_column_name="user_file_id",
+        existing_type=psql.UUID(as_uuid=True),
+        nullable=False,
+    )
+    # ---- end persona__user_file ----
+
+    # (Repeat 2) for any other FK tables that point to user_file.id)
+
+    # 3) Swap PK on user_file from int -> uuid
+    op.drop_constraint("user_file_pkey", "user_file", type_="primary")
+    op.drop_column("user_file", "id")
+    op.alter_column(
+        "user_file",
+        "new_id",
+        new_column_name="id",
+        existing_type=psql.UUID(as_uuid=True),
+        nullable=False,
+    )
+    op.create_primary_key("user_file_pkey", "user_file", ["id"])
+
+    # 4) Now **force** FKs to bind to the PK:
+    #    (a) drop FK(s)
+    op.drop_constraint(
+        "persona__user_file_user_file_id_fkey",
+        "persona__user_file",
+        type_="foreignkey",
+    )
+    #    (b) drop the transitional UNIQUE so it cannot be chosen
+    op.drop_constraint("uq_user_file_new_id", "user_file", type_="unique")
+    #    (c) recreate FK(s) to user_file(id) — only PK remains, so it will bind there
+    op.create_foreign_key(
+        "persona__user_file_user_file_id_fkey",
+        "persona__user_file",
+        "user_file",
+        local_cols=["user_file_id"],
+        remote_cols=["id"],
+    )
+
+    # 5) Rename user_folder -> user_project and update dependent FKs/columns
+    try:
+        op.rename_table("user_folder", "user_project")
+    except Exception:
+        # Table might already be renamed
+        pass
+
+    # Drop user_file.folder_id if it exists (we don't keep one-to-many link)
+    try:
+        op.drop_column("user_file", "folder_id")
+    except Exception:
+        pass
+
+    # 6) Safe to create new tables referencing the UUID PK
+    op.create_table(
+        "project__user_file",
+        sa.Column("project_id", sa.Integer(), nullable=False),
+        sa.Column("user_file_id", psql.UUID(as_uuid=True), nullable=False),
+        sa.ForeignKeyConstraint(["project_id"], ["user_project.id"]),
+        sa.ForeignKeyConstraint(["user_file_id"], ["user_file.id"]),
+        sa.PrimaryKeyConstraint("project_id", "user_file_id"),
+    )
+
+    # 6) Remove CCPair relationship
+    # Drop the foreign key constraint first
+    op.drop_constraint(
+        "user_file_cc_pair_id_fkey",
+        "user_file",
+        type_="foreignkey",
+    )
+    # Drop the unique constraint
+    op.drop_constraint(
+        "user_file_cc_pair_id_key",
+        "user_file",
+        type_="unique",
+    )
+    # Drop the column
+    op.drop_column("user_file", "cc_pair_id")
+
+    # 7) Add extra columns
+    op.add_column(
+        "user_file",
+        sa.Column(
+            "status",
+            sa.Enum(
+                "processing",
+                "completed",
+                "failed",
+                "canceled",
+                name="userfilestatus",
+                native_enum=False,
+            ),
+            nullable=False,
+            server_default="processing",
+        ),
+    )
+    op.add_column("user_file", sa.Column("chunk_count", sa.Integer(), nullable=True))
+    # Drop deprecated document_id column if present
+    try:
+        op.drop_column("user_file", "document_id")
+    except Exception:
+        pass
+    op.add_column(
+        "user_file",
+        sa.Column("last_accessed_at", sa.DateTime(timezone=True), nullable=True),
+    )
+    op.add_column(
+        "user_project",
+        sa.Column("prompt_id", sa.Integer(), nullable=True),
+    )
+    op.create_foreign_key(
+        "user_project_prompt_id_fkey",
+        "user_project",
+        "prompt",
+        ["prompt_id"],
+        ["id"],
+    )
+    op.add_column(
+        "chat_session",
+        sa.Column("project_id", sa.Integer(), nullable=True),
+    )
+    op.create_foreign_key(
+        "chat_session_project_id_fkey",
+        "chat_session",
+        "user_project",
+        ["project_id"],
+        ["id"],
+    )
+    # Add index on project_id for better query performance
+    op.create_index(
+        "ix_chat_session_project_id",
+        "chat_session",
+        ["project_id"],
+    )
+
+
+def downgrade() -> None:
+    # Recreate persona__user_folder table
+    op.create_table(
+        "persona__user_folder",
+        sa.Column("persona_id", sa.Integer(), nullable=False),
+        sa.Column("user_folder_id", sa.Integer(), nullable=False),
+        sa.ForeignKeyConstraint(["persona_id"], ["persona.id"]),
+        sa.ForeignKeyConstraint(["user_folder_id"], ["user_folder.id"]),
+        sa.PrimaryKeyConstraint("persona_id", "user_folder_id"),
+    )
+
+    # Recreate folder related tables and columns
+    # First create the chat_folder table
+    op.create_table(
+        "chat_folder",
+        sa.Column("id", sa.Integer(), primary_key=True),
+        sa.Column("user_id", psql.UUID(as_uuid=True), nullable=True),
+        sa.Column("name", sa.String(), nullable=True),
+        sa.Column("display_priority", sa.Integer(), nullable=True, default=0),
+    )
+    # Add foreign key for user_id after table creation
+    op.create_foreign_key(
+        "chat_folder_user_id_fkey",
+        "chat_folder",
+        "user",
+        ["user_id"],
+        ["id"],
+    )
+
+    # Add folder_id column to chat_session
+    op.add_column(
+        "chat_session",
+        sa.Column("folder_id", sa.Integer(), nullable=True),
+    )
+    # Create foreign key constraint after both tables exist
+    op.create_foreign_key(
+        "chat_session_folder_id_fkey",
+        "chat_session",
+        "chat_folder",
+        ["folder_id"],
+        ["id"],
+    )
+
+    # Drop extra columns
+    op.drop_column("user_file", "last_accessed_at")
+    # Recreate document_id on downgrade
+    try:
+        op.add_column(
+            "user_file", sa.Column("document_id", sa.String(), nullable=False)
+        )
+    except Exception:
+        pass
+    op.drop_column("user_file", "chunk_count")
+    op.drop_column("user_file", "status")
+    op.execute("DROP TYPE IF EXISTS userfilestatus")
+
+    # Drop association table
+    op.drop_table("project__user_file")
+    # Drop index before dropping the column
+    op.drop_index("ix_chat_session_project_id", table_name="chat_session")
+    op.drop_column("chat_session", "project_id")
+    # Recreate an integer PK (best-effort; original values aren’t retained)
+    op.drop_constraint(
+        "persona__user_file_user_file_id_fkey", "persona__user_file", type_="foreignkey"
+    )
+    op.drop_constraint("user_file_pkey", "user_file", type_="primary")
+
+    op.add_column(
+        "user_file",
+        sa.Column("id_int_tmp", sa.Integer(), autoincrement=True, nullable=False),
+    )
+    op.execute(
+        "CREATE SEQUENCE IF NOT EXISTS user_file_id_seq OWNED BY user_file.id_int_tmp"
+    )
+    op.execute(
+        "ALTER TABLE user_file ALTER COLUMN id_int_tmp SET DEFAULT nextval('user_file_id_seq')"
+    )
+    op.create_primary_key("user_file_pkey", "user_file", ["id_int_tmp"])
+
+    op.add_column(
+        "persona__user_file",
+        sa.Column("user_file_id_int_tmp", sa.Integer(), nullable=True),
+    )
+    op.create_foreign_key(
+        "persona__user_file_user_file_id_fkey",
+        "persona__user_file",
+        "user_file",
+        ["user_file_id_int_tmp"],
+        ["id_int_tmp"],
+    )
+
+    # Remove UUID id and rename int back to id
+    op.drop_column("user_file", "id")
+    op.alter_column(
+        "user_file",
+        "id_int_tmp",
+        new_column_name="id",
+        existing_type=sa.Integer(),
+        nullable=False,
+    )
+
+    op.drop_column("persona__user_file", "user_file_id")
+    op.alter_column(
+        "persona__user_file",
+        "user_file_id_int_tmp",
+        new_column_name="user_file_id",
+        existing_type=sa.Integer(),
+    )
+
+    # Restore CCPair relationship
+    op.add_column(
+        "user_file",
+        sa.Column("cc_pair_id", sa.Integer(), nullable=True),
+    )
+    op.create_unique_constraint(
+        "user_file_cc_pair_id_key",
+        "user_file",
+        ["cc_pair_id"],
+    )
+    op.create_foreign_key(
+        "user_file_cc_pair_id_fkey",
+        "user_file",
+        "connector_credential_pair",
+        ["cc_pair_id"],
+        ["id"],
+    )
+
+    # Rename user_project back to user_folder and revert related changes
+    try:
+        op.drop_constraint(
+            "user_project_prompt_id_fkey", "user_project", type_="foreignkey"
+        )
+    except Exception:
+        pass
+    try:
+        op.drop_column("user_project", "prompt_id")
+    except Exception:
+        pass
+    # Recreate user_file.folder_id (nullable) since we dropped it on upgrade
+    try:
+        op.add_column("user_file", sa.Column("folder_id", sa.Integer(), nullable=True))
+    except Exception:
+        pass
+    try:
+        op.rename_table("user_project", "user_folder")
+    except Exception:
+        pass

backend/alembic/versions/1c3f8a7b5d4e_add_python_tool.py

@@ -1,73 +0,0 @@
-"""add_python_tool
-
-Revision ID: 1c3f8a7b5d4e
-Revises: 505c488f6662
-Create Date: 2025-02-14 00:00:00
-
-"""
-
-from alembic import op
-import sqlalchemy as sa
-
-
-# revision identifiers, used by Alembic.
-revision = "1c3f8a7b5d4e"
-down_revision = "505c488f6662"
-branch_labels = None
-depends_on = None
-
-
-PYTHON_TOOL = {
-    "name": "PythonTool",
-    "display_name": "Code Interpreter",
-    "description": (
-        "The Code Interpreter Action lets assistants execute Python in an isolated runtime. "
-        "It can process staged files, read and write artifacts, stream stdout and stderr, "
-        "and return generated outputs for the chat session."
-    ),
-    "in_code_tool_id": "PythonTool",
-}
-
-
-def upgrade() -> None:
-    conn = op.get_bind()
-    conn.execute(sa.text("BEGIN"))
-    try:
-        existing = conn.execute(
-            sa.text("SELECT id FROM tool WHERE in_code_tool_id = :in_code_tool_id"),
-            PYTHON_TOOL,
-        ).fetchone()
-
-        if existing:
-            conn.execute(
-                sa.text(
-                    """
-                    UPDATE tool
-                    SET name = :name,
-                        display_name = :display_name,
-                        description = :description
-                    WHERE in_code_tool_id = :in_code_tool_id
-                    """
-                ),
-                PYTHON_TOOL,
-            )
-        else:
-            conn.execute(
-                sa.text(
-                    """
-                    INSERT INTO tool (name, display_name, description, in_code_tool_id)
-                    VALUES (:name, :display_name, :description, :in_code_tool_id)
-                    """
-                ),
-                PYTHON_TOOL,
-            )
-
-        conn.execute(sa.text("COMMIT"))
-    except Exception:
-        conn.execute(sa.text("ROLLBACK"))
-        raise
-
-
-def downgrade() -> None:
-    # Do not delete the tool entry on downgrade; leaving it is safe and keeps migrations idempotent.
-    pass

backend/alembic/versions/505c488f6662_merge_default_assistants_into_unified.py

@@ -1,380 +0,0 @@
-"""merge_default_assistants_into_unified
-
-Revision ID: 505c488f6662
-Revises: d09fc20a3c66
-Create Date: 2025-09-09 19:00:56.816626
-
-"""
-
-import json
-from typing import Any
-from typing import NamedTuple
-from uuid import UUID
-
-from alembic import op
-import sqlalchemy as sa
-
-
-# revision identifiers, used by Alembic.
-revision = "505c488f6662"
-down_revision = "d09fc20a3c66"
-branch_labels = None
-depends_on = None
-
-# Constants for the unified assistant
-UNIFIED_ASSISTANT_NAME = "Assistant"
-UNIFIED_ASSISTANT_DESCRIPTION = (
-    "Your AI assistant with search, web browsing, and image generation capabilities."
-)
-UNIFIED_ASSISTANT_NUM_CHUNKS = 25
-UNIFIED_ASSISTANT_DISPLAY_PRIORITY = 0
-UNIFIED_ASSISTANT_LLM_FILTER_EXTRACTION = True
-UNIFIED_ASSISTANT_LLM_RELEVANCE_FILTER = False
-UNIFIED_ASSISTANT_RECENCY_BIAS = "AUTO"  # NOTE: needs to be capitalized
-UNIFIED_ASSISTANT_CHUNKS_ABOVE = 0
-UNIFIED_ASSISTANT_CHUNKS_BELOW = 0
-UNIFIED_ASSISTANT_DATETIME_AWARE = True
-
-# NOTE: tool specific prompts are handled on the fly and automatically injected
-# into the prompt before passing to the LLM.
-DEFAULT_SYSTEM_PROMPT = """
-You are a highly capable, thoughtful, and precise assistant. Your goal is to deeply understand the \
-user's intent, ask clarifying questions when needed, think step-by-step through complex problems, \
-provide clear and accurate answers, and proactively anticipate helpful follow-up information. Always \
-prioritize being truthful, nuanced, insightful, and efficient.
-The current date is [[CURRENT_DATETIME]]
-
-You use different text styles, bolding, emojis (sparingly), block quotes, and other formatting to make \
-your responses more readable and engaging.
-You use proper Markdown and LaTeX to format your responses for math, scientific, and chemical formulas, \
-symbols, etc.: '$$\\n[expression]\\n$$' for standalone cases and '\\( [expression] \\)' when inline.
-For code you prefer to use Markdown and specify the language.
-You can use Markdown horizontal rules (---) to separate sections of your responses.
-You can use Markdown tables to format your responses for data, lists, and other structured information.
-""".strip()
-
-
-INSERT_DICT: dict[str, Any] = {
-    "name": UNIFIED_ASSISTANT_NAME,
-    "description": UNIFIED_ASSISTANT_DESCRIPTION,
-    "system_prompt": DEFAULT_SYSTEM_PROMPT,
-    "num_chunks": UNIFIED_ASSISTANT_NUM_CHUNKS,
-    "display_priority": UNIFIED_ASSISTANT_DISPLAY_PRIORITY,
-    "llm_filter_extraction": UNIFIED_ASSISTANT_LLM_FILTER_EXTRACTION,
-    "llm_relevance_filter": UNIFIED_ASSISTANT_LLM_RELEVANCE_FILTER,
-    "recency_bias": UNIFIED_ASSISTANT_RECENCY_BIAS,
-    "chunks_above": UNIFIED_ASSISTANT_CHUNKS_ABOVE,
-    "chunks_below": UNIFIED_ASSISTANT_CHUNKS_BELOW,
-    "datetime_aware": UNIFIED_ASSISTANT_DATETIME_AWARE,
-}
-
-GENERAL_ASSISTANT_ID = -1
-ART_ASSISTANT_ID = -3
-
-
-class UserRow(NamedTuple):
-    """Typed representation of user row from database query."""
-
-    id: UUID
-    chosen_assistants: list[int] | None
-    visible_assistants: list[int] | None
-    hidden_assistants: list[int] | None
-    pinned_assistants: list[int] | None
-
-
-def upgrade() -> None:
-    conn = op.get_bind()
-
-    # Start transaction
-    conn.execute(sa.text("BEGIN"))
-
-    try:
-        # Step 1: Create or update the unified assistant (ID 0)
-        search_assistant = conn.execute(
-            sa.text("SELECT * FROM persona WHERE id = 0")
-        ).fetchone()
-
-        if search_assistant:
-            # Update existing Search assistant to be the unified assistant
-            conn.execute(
-                sa.text(
-                    """
-                    UPDATE persona
-                    SET name = :name,
-                        description = :description,
-                        system_prompt = :system_prompt,
-                        num_chunks = :num_chunks,
-                        is_default_persona = true,
-                        is_visible = true,
-                        deleted = false,
-                        display_priority = :display_priority,
-                        llm_filter_extraction = :llm_filter_extraction,
-                        llm_relevance_filter = :llm_relevance_filter,
-                        recency_bias = :recency_bias,
-                        chunks_above = :chunks_above,
-                        chunks_below = :chunks_below,
-                        datetime_aware = :datetime_aware,
-                        starter_messages = null
-                    WHERE id = 0
-                """
-                ),
-                INSERT_DICT,
-            )
-        else:
-            # Create new unified assistant with ID 0
-            conn.execute(
-                sa.text(
-                    """
-                    INSERT INTO persona (
-                        id, name, description, system_prompt, num_chunks,
-                        is_default_persona, is_visible, deleted, display_priority,
-                        llm_filter_extraction, llm_relevance_filter, recency_bias,
-                        chunks_above, chunks_below, datetime_aware, starter_messages,
-                        builtin_persona
-                    ) VALUES (
-                        0, :name, :description, :system_prompt, :num_chunks,
-                        true, true, false, :display_priority, :llm_filter_extraction,
-                        :llm_relevance_filter, :recency_bias, :chunks_above, :chunks_below,
-                        :datetime_aware, null, true
-                    )
-                """
-                ),
-                INSERT_DICT,
-            )
-
-        # Step 2: Mark ALL builtin assistants as deleted (except the unified assistant ID 0)
-        conn.execute(
-            sa.text(
-                """
-                UPDATE persona
-                SET deleted = true, is_visible = false, is_default_persona = false
-                WHERE builtin_persona = true AND id != 0
-            """
-            )
-        )
-
-        # Step 3: Add all built-in tools to the unified assistant
-        # First, get the tool IDs for SearchTool, ImageGenerationTool, and WebSearchTool
-        search_tool = conn.execute(
-            sa.text("SELECT id FROM tool WHERE in_code_tool_id = 'SearchTool'")
-        ).fetchone()
-
-        if not search_tool:
-            raise ValueError(
-                "SearchTool not found in database. Ensure tools migration has run first."
-            )
-
-        image_gen_tool = conn.execute(
-            sa.text("SELECT id FROM tool WHERE in_code_tool_id = 'ImageGenerationTool'")
-        ).fetchone()
-
-        if not image_gen_tool:
-            raise ValueError(
-                "ImageGenerationTool not found in database. Ensure tools migration has run first."
-            )
-
-        # WebSearchTool is optional - may not be configured
-        web_search_tool = conn.execute(
-            sa.text("SELECT id FROM tool WHERE in_code_tool_id = 'WebSearchTool'")
-        ).fetchone()
-
-        # Clear existing tool associations for persona 0
-        conn.execute(sa.text("DELETE FROM persona__tool WHERE persona_id = 0"))
-
-        # Add tools to the unified assistant
-        conn.execute(
-            sa.text(
-                """
-                INSERT INTO persona__tool (persona_id, tool_id)
-                VALUES (0, :tool_id)
-                ON CONFLICT DO NOTHING
-            """
-            ),
-            {"tool_id": search_tool[0]},
-        )
-
-        conn.execute(
-            sa.text(
-                """
-                INSERT INTO persona__tool (persona_id, tool_id)
-                VALUES (0, :tool_id)
-                ON CONFLICT DO NOTHING
-            """
-            ),
-            {"tool_id": image_gen_tool[0]},
-        )
-
-        if web_search_tool:
-            conn.execute(
-                sa.text(
-                    """
-                    INSERT INTO persona__tool (persona_id, tool_id)
-                    VALUES (0, :tool_id)
-                    ON CONFLICT DO NOTHING
-                """
-                ),
-                {"tool_id": web_search_tool[0]},
-            )
-
-        # Step 4: Migrate existing chat sessions from all builtin assistants to unified assistant
-        conn.execute(
-            sa.text(
-                """
-                UPDATE chat_session
-                SET persona_id = 0
-                WHERE persona_id IN (
-                    SELECT id FROM persona WHERE builtin_persona = true AND id != 0
-                )
-            """
-            )
-        )
-
-        # Step 5: Migrate user preferences - remove references to all builtin assistants
-        # First, get all builtin assistant IDs (except 0)
-        builtin_assistants_result = conn.execute(
-            sa.text(
-                """
-                SELECT id FROM persona
-                WHERE builtin_persona = true AND id != 0
-            """
-            )
-        ).fetchall()
-        builtin_assistant_ids = [row[0] for row in builtin_assistants_result]
-
-        # Get all users with preferences
-        users_result = conn.execute(
-            sa.text(
-                """
-                SELECT id, chosen_assistants, visible_assistants,
-                       hidden_assistants, pinned_assistants
-                FROM "user"
-            """
-            )
-        ).fetchall()
-
-        for user_row in users_result:
-            user = UserRow(*user_row)
-            user_id: UUID = user.id
-            updates: dict[str, Any] = {}
-
-            # Remove all builtin assistants from chosen_assistants
-            if user.chosen_assistants:
-                new_chosen: list[int] = [
-                    assistant_id
-                    for assistant_id in user.chosen_assistants
-                    if assistant_id not in builtin_assistant_ids
-                ]
-                if new_chosen != user.chosen_assistants:
-                    updates["chosen_assistants"] = json.dumps(new_chosen)
-
-            # Remove all builtin assistants from visible_assistants
-            if user.visible_assistants:
-                new_visible: list[int] = [
-                    assistant_id
-                    for assistant_id in user.visible_assistants
-                    if assistant_id not in builtin_assistant_ids
-                ]
-                if new_visible != user.visible_assistants:
-                    updates["visible_assistants"] = json.dumps(new_visible)
-
-            # Add all builtin assistants to hidden_assistants
-            if user.hidden_assistants:
-                new_hidden: list[int] = list(user.hidden_assistants)
-                for old_id in builtin_assistant_ids:
-                    if old_id not in new_hidden:
-                        new_hidden.append(old_id)
-                if new_hidden != user.hidden_assistants:
-                    updates["hidden_assistants"] = json.dumps(new_hidden)
-            else:
-                updates["hidden_assistants"] = json.dumps(builtin_assistant_ids)
-
-            # Remove all builtin assistants from pinned_assistants
-            if user.pinned_assistants:
-                new_pinned: list[int] = [
-                    assistant_id
-                    for assistant_id in user.pinned_assistants
-                    if assistant_id not in builtin_assistant_ids
-                ]
-                if new_pinned != user.pinned_assistants:
-                    updates["pinned_assistants"] = json.dumps(new_pinned)
-
-            # Apply updates if any
-            if updates:
-                set_clause = ", ".join([f"{k} = :{k}" for k in updates.keys()])
-                updates["user_id"] = str(user_id)  # Convert UUID to string for SQL
-                conn.execute(
-                    sa.text(f'UPDATE "user" SET {set_clause} WHERE id = :user_id'),
-                    updates,
-                )
-
-        # Commit transaction
-        conn.execute(sa.text("COMMIT"))
-
-    except Exception as e:
-        # Rollback on error
-        conn.execute(sa.text("ROLLBACK"))
-        raise e
-
-
-def downgrade() -> None:
-    conn = op.get_bind()
-
-    # Start transaction
-    conn.execute(sa.text("BEGIN"))
-
-    try:
-        # Only restore General (ID -1) and Art (ID -3) assistants
-        # Step 1: Keep Search assistant (ID 0) as default but restore original state
-        conn.execute(
-            sa.text(
-                """
-                UPDATE persona
-                SET is_default_persona = true,
-                    is_visible = true,
-                    deleted = false
-                WHERE id = 0
-            """
-            )
-        )
-
-        # Step 2: Restore General assistant (ID -1)
-        conn.execute(
-            sa.text(
-                """
-                UPDATE persona
-                SET deleted = false,
-                    is_visible = true,
-                    is_default_persona = true
-                WHERE id = :general_assistant_id
-            """
-            ),
-            {"general_assistant_id": GENERAL_ASSISTANT_ID},
-        )
-
-        # Step 3: Restore Art assistant (ID -3)
-        conn.execute(
-            sa.text(
-                """
-                UPDATE persona
-                SET deleted = false,
-                    is_visible = true,
-                    is_default_persona = true
-                WHERE id = :art_assistant_id
-            """
-            ),
-            {"art_assistant_id": ART_ASSISTANT_ID},
-        )
-
-        # Note: We don't restore the original tool associations, names, or descriptions
-        # as those would require more complex logic to determine original state.
-        # We also cannot restore original chat session persona_ids as we don't
-        # have the original mappings.
-        # Other builtin assistants remain deleted as per the requirement.
-
-        # Commit transaction
-        conn.execute(sa.text("COMMIT"))
-
-    except Exception as e:
-        # Rollback on error
-        conn.execute(sa.text("ROLLBACK"))
-        raise e

backend/alembic/versions/abbfec3a5ac5_merge_prompt_into_persona.py

@@ -1,225 +0,0 @@
-"""merge prompt into persona
-
-Revision ID: abbfec3a5ac5
-Revises: 8818cf73fa1a
-Create Date: 2024-12-19 12:00:00.000000
-
-"""
-
-from alembic import op
-import sqlalchemy as sa
-from sqlalchemy.dialects import postgresql
-
-# revision identifiers, used by Alembic.
-revision = "abbfec3a5ac5"
-down_revision = "8818cf73fa1a"
-branch_labels = None
-depends_on = None
-
-
-MAX_PROMPT_LENGTH = 5_000_000
-
-
-def upgrade() -> None:
-    """NOTE: Prompts without any Personas will just be lost."""
-    # Step 1: Add new columns to persona table (only if they don't exist)
-
-    # Check if columns exist before adding them
-    connection = op.get_bind()
-    inspector = sa.inspect(connection)
-    existing_columns = [col["name"] for col in inspector.get_columns("persona")]
-
-    if "system_prompt" not in existing_columns:
-        op.add_column(
-            "persona",
-            sa.Column(
-                "system_prompt", sa.String(length=MAX_PROMPT_LENGTH), nullable=True
-            ),
-        )
-
-    if "task_prompt" not in existing_columns:
-        op.add_column(
-            "persona",
-            sa.Column(
-                "task_prompt", sa.String(length=MAX_PROMPT_LENGTH), nullable=True
-            ),
-        )
-
-    if "datetime_aware" not in existing_columns:
-        op.add_column(
-            "persona",
-            sa.Column(
-                "datetime_aware", sa.Boolean(), nullable=False, server_default="true"
-            ),
-        )
-
-    # Step 2: Migrate data from prompt table to persona table (only if tables exist)
-    existing_tables = inspector.get_table_names()
-
-    if "prompt" in existing_tables and "persona__prompt" in existing_tables:
-        # For personas that have associated prompts, copy the prompt data
-        op.execute(
-            """
-            UPDATE persona
-            SET
-                system_prompt = p.system_prompt,
-                task_prompt = p.task_prompt,
-                datetime_aware = p.datetime_aware
-            FROM (
-                -- Get the first prompt for each persona (in case there are multiple)
-                SELECT DISTINCT ON (pp.persona_id)
-                    pp.persona_id,
-                    pr.system_prompt,
-                    pr.task_prompt,
-                    pr.datetime_aware
-                FROM persona__prompt pp
-                JOIN prompt pr ON pp.prompt_id = pr.id
-            ) p
-            WHERE persona.id = p.persona_id
-        """
-        )
-
-        # Step 3: Update chat_message references
-        # Since chat messages referenced prompt_id, we need to update them to use persona_id
-        # This is complex as we need to map from prompt_id to persona_id
-
-        # Check if chat_message has prompt_id column
-        chat_message_columns = [
-            col["name"] for col in inspector.get_columns("chat_message")
-        ]
-        if "prompt_id" in chat_message_columns:
-            op.execute(
-                """
-                ALTER TABLE chat_message
-                DROP CONSTRAINT IF EXISTS chat_message__prompt_fk
-            """
-            )
-            op.drop_column("chat_message", "prompt_id")
-
-    # Step 4: Handle personas without prompts - set default values if needed (always run this)
-    op.execute(
-        """
-        UPDATE persona
-        SET
-            system_prompt = COALESCE(system_prompt, ''),
-            task_prompt = COALESCE(task_prompt, '')
-        WHERE system_prompt IS NULL OR task_prompt IS NULL
-    """
-    )
-
-    # Step 5: Drop the persona__prompt association table (if it exists)
-    if "persona__prompt" in existing_tables:
-        op.drop_table("persona__prompt")
-
-    # Step 6: Drop the prompt table (if it exists)
-    if "prompt" in existing_tables:
-        op.drop_table("prompt")
-
-    # Step 7: Make system_prompt and task_prompt non-nullable after migration (only if they exist)
-    op.alter_column(
-        "persona",
-        "system_prompt",
-        existing_type=sa.String(length=MAX_PROMPT_LENGTH),
-        nullable=False,
-        server_default=None,
-    )
-
-    op.alter_column(
-        "persona",
-        "task_prompt",
-        existing_type=sa.String(length=MAX_PROMPT_LENGTH),
-        nullable=False,
-        server_default=None,
-    )
-
-
-def downgrade() -> None:
-    # Step 1: Recreate the prompt table
-    op.create_table(
-        "prompt",
-        sa.Column("id", sa.Integer(), nullable=False),
-        sa.Column("user_id", postgresql.UUID(as_uuid=True), nullable=True),
-        sa.Column("name", sa.String(), nullable=False),
-        sa.Column("description", sa.String(), nullable=False),
-        sa.Column("system_prompt", sa.String(length=MAX_PROMPT_LENGTH), nullable=False),
-        sa.Column("task_prompt", sa.String(length=MAX_PROMPT_LENGTH), nullable=False),
-        sa.Column(
-            "datetime_aware", sa.Boolean(), nullable=False, server_default="true"
-        ),
-        sa.Column(
-            "default_prompt", sa.Boolean(), nullable=False, server_default="false"
-        ),
-        sa.Column("deleted", sa.Boolean(), nullable=False, server_default="false"),
-        sa.ForeignKeyConstraint(["user_id"], ["user.id"], ondelete="CASCADE"),
-        sa.PrimaryKeyConstraint("id"),
-    )
-
-    # Step 2: Recreate the persona__prompt association table
-    op.create_table(
-        "persona__prompt",
-        sa.Column("persona_id", sa.Integer(), nullable=False),
-        sa.Column("prompt_id", sa.Integer(), nullable=False),
-        sa.ForeignKeyConstraint(
-            ["persona_id"],
-            ["persona.id"],
-        ),
-        sa.ForeignKeyConstraint(
-            ["prompt_id"],
-            ["prompt.id"],
-        ),
-        sa.PrimaryKeyConstraint("persona_id", "prompt_id"),
-    )
-
-    # Step 3: Migrate data back from persona to prompt table
-    op.execute(
-        """
-        INSERT INTO prompt (
-            name,
-            description,
-            system_prompt,
-            task_prompt,
-            datetime_aware,
-            default_prompt,
-            deleted,
-            user_id
-        )
-        SELECT
-            CONCAT('Prompt for ', name),
-            description,
-            system_prompt,
-            task_prompt,
-            datetime_aware,
-            is_default_persona,
-            deleted,
-            user_id
-        FROM persona
-        WHERE system_prompt IS NOT NULL AND system_prompt != ''
-        RETURNING id, name
-        """
-    )
-
-    # Step 4: Re-establish persona__prompt relationships
-    op.execute(
-        """
-        INSERT INTO persona__prompt (persona_id, prompt_id)
-        SELECT
-            p.id as persona_id,
-            pr.id as prompt_id
-        FROM persona p
-        JOIN prompt pr ON pr.name = CONCAT('Prompt for ', p.name)
-        WHERE p.system_prompt IS NOT NULL AND p.system_prompt != ''
-    """
-    )
-
-    # Step 5: Add prompt_id column back to chat_message
-    op.add_column("chat_message", sa.Column("prompt_id", sa.Integer(), nullable=True))
-
-    # Step 6: Re-establish foreign key constraint
-    op.create_foreign_key(
-        "chat_message__prompt_fk", "chat_message", "prompt", ["prompt_id"], ["id"]
-    )
-
-    # Step 7: Remove columns from persona table
-    op.drop_column("persona", "datetime_aware")
-    op.drop_column("persona", "task_prompt")
-    op.drop_column("persona", "system_prompt")

backend/alembic/versions/b7ec9b5b505f_adjust_prompt_length.py

@@ -1,43 +0,0 @@
-"""adjust prompt length
-
-Revision ID: b7ec9b5b505f
-Revises: abbfec3a5ac5
-Create Date: 2025-09-10 18:51:15.629197
-
-"""
-
-from alembic import op
-import sqlalchemy as sa
-
-
-# revision identifiers, used by Alembic.
-revision = "b7ec9b5b505f"
-down_revision = "abbfec3a5ac5"
-branch_labels = None
-depends_on = None
-
-
-MAX_PROMPT_LENGTH = 5_000_000
-
-
-def upgrade() -> None:
-    # NOTE: need to run this since the previous migration PREVIOUSLY set the length to 8000
-    op.alter_column(
-        "persona",
-        "system_prompt",
-        existing_type=sa.String(length=8000),
-        type_=sa.String(length=MAX_PROMPT_LENGTH),
-        existing_nullable=False,
-    )
-    op.alter_column(
-        "persona",
-        "task_prompt",
-        existing_type=sa.String(length=8000),
-        type_=sa.String(length=MAX_PROMPT_LENGTH),
-        existing_nullable=False,
-    )
-
-
-def downgrade() -> None:
-    # Downgrade not necessary
-    pass

backend/alembic/versions/d09fc20a3c66_seed_builtin_tools.py

@@ -1,152 +0,0 @@
-"""seed_builtin_tools
-
-Revision ID: d09fc20a3c66
-Revises: b7ec9b5b505f
-Create Date: 2025-09-09 19:32:16.824373
-
-"""
-
-from alembic import op
-import sqlalchemy as sa
-
-
-# revision identifiers, used by Alembic.
-revision = "d09fc20a3c66"
-down_revision = "b7ec9b5b505f"
-branch_labels = None
-depends_on = None
-
-
-# Tool definitions - core tools that should always be seeded
-# Names/in_code_tool_id are the same as the class names in the tool_implementations package
-BUILT_IN_TOOLS = [
-    {
-        "name": "SearchTool",
-        "display_name": "Internal Search",
-        "description": "The Search Action allows the Assistant to search through connected knowledge to help build an answer.",
-        "in_code_tool_id": "SearchTool",
-    },
-    {
-        "name": "ImageGenerationTool",
-        "display_name": "Image Generation",
-        "description": (
-            "The Image Generation Action allows the assistant to use DALL-E 3 or GPT-IMAGE-1 to generate images. "
-            "The action will be used when the user asks the assistant to generate an image."
-        ),
-        "in_code_tool_id": "ImageGenerationTool",
-    },
-    {
-        "name": "WebSearchTool",
-        "display_name": "Web Search",
-        "description": (
-            "The Web Search Action allows the assistant "
-            "to perform internet searches for up-to-date information."
-        ),
-        "in_code_tool_id": "WebSearchTool",
-    },
-    {
-        "name": "KnowledgeGraphTool",
-        "display_name": "Knowledge Graph Search",
-        "description": (
-            "The Knowledge Graph Search Action allows the assistant to search the "
-            "Knowledge Graph for information. This tool can (for now) only be active in the KG Beta Assistant, "
-            "and it requires the Knowledge Graph to be enabled."
-        ),
-        "in_code_tool_id": "KnowledgeGraphTool",
-    },
-    {
-        "name": "OktaProfileTool",
-        "display_name": "Okta Profile",
-        "description": (
-            "The Okta Profile Action allows the assistant to fetch the current user's information from Okta. "
-            "This may include the user's name, email, phone number, address, and other details such as their "
-            "manager and direct reports."
-        ),
-        "in_code_tool_id": "OktaProfileTool",
-    },
-]
-
-
-def upgrade() -> None:
-    conn = op.get_bind()
-
-    # Start transaction
-    conn.execute(sa.text("BEGIN"))
-
-    try:
-        # Get existing tools to check what already exists
-        existing_tools = conn.execute(
-            sa.text(
-                "SELECT in_code_tool_id FROM tool WHERE in_code_tool_id IS NOT NULL"
-            )
-        ).fetchall()
-        existing_tool_ids = {row[0] for row in existing_tools}
-
-        # Insert or update built-in tools
-        for tool in BUILT_IN_TOOLS:
-            in_code_id = tool["in_code_tool_id"]
-
-            # Handle historical rename: InternetSearchTool -> WebSearchTool
-            if (
-                in_code_id == "WebSearchTool"
-                and "WebSearchTool" not in existing_tool_ids
-                and "InternetSearchTool" in existing_tool_ids
-            ):
-                # Rename the existing InternetSearchTool row in place and update fields
-                conn.execute(
-                    sa.text(
-                        """
-                        UPDATE tool
-                        SET name = :name,
-                            display_name = :display_name,
-                            description = :description,
-                            in_code_tool_id = :in_code_tool_id
-                        WHERE in_code_tool_id = 'InternetSearchTool'
-                        """
-                    ),
-                    tool,
-                )
-                # Keep the local view of existing ids in sync to avoid duplicate insert
-                existing_tool_ids.discard("InternetSearchTool")
-                existing_tool_ids.add("WebSearchTool")
-                continue
-
-            if in_code_id in existing_tool_ids:
-                # Update existing tool
-                conn.execute(
-                    sa.text(
-                        """
-                        UPDATE tool
-                        SET name = :name,
-                            display_name = :display_name,
-                            description = :description
-                        WHERE in_code_tool_id = :in_code_tool_id
-                        """
-                    ),
-                    tool,
-                )
-            else:
-                # Insert new tool
-                conn.execute(
-                    sa.text(
-                        """
-                        INSERT INTO tool (name, display_name, description, in_code_tool_id)
-                        VALUES (:name, :display_name, :description, :in_code_tool_id)
-                        """
-                    ),
-                    tool,
-                )
-
-        # Commit transaction
-        conn.execute(sa.text("COMMIT"))
-
-    except Exception as e:
-        # Rollback on error
-        conn.execute(sa.text("ROLLBACK"))
-        raise e
-
-
-def downgrade() -> None:
-    # We don't remove the tools on downgrade since it's totally fine to just
-    # have them around. If we upgrade again, it will be a no-op.
-    pass

backend/ee/onyx/main.py

@@ -17,7 +17,6 @@ from ee.onyx.server.enterprise_settings.api import (
 from ee.onyx.server.enterprise_settings.api import (
     basic_router as enterprise_settings_router,
 )
-from ee.onyx.server.evals.api import router as evals_router
 from ee.onyx.server.manage.standard_answer import router as standard_answer_router
 from ee.onyx.server.middleware.tenant_tracking import (
     add_api_server_tenant_id_middleware,
@@ -171,7 +170,6 @@ def get_application() -> FastAPI:
     include_router_with_global_prefix_prepended(application, standard_answer_router)
     include_router_with_global_prefix_prepended(application, ee_oauth_router)
     include_router_with_global_prefix_prepended(application, ee_document_cc_pair_router)
-    include_router_with_global_prefix_prepended(application, evals_router)
 
     # Enterprise-only global settings
     include_router_with_global_prefix_prepended(

backend/ee/onyx/onyxbot/slack/handlers/handle_standard_answers.py

@@ -14,6 +14,7 @@ from onyx.db.chat import create_new_chat_message
 from onyx.db.chat import get_chat_messages_by_sessions
 from onyx.db.chat import get_chat_sessions_by_slack_thread_id
 from onyx.db.chat import get_or_create_root_message
+from onyx.db.models import Prompt
 from onyx.db.models import SlackChannelConfig
 from onyx.db.models import StandardAnswer as StandardAnswerModel
 from onyx.onyxbot.slack.blocks import get_restate_blocks
@@ -80,6 +81,7 @@ def _handle_standard_answers(
     message_info: SlackMessageInfo,
     receiver_ids: list[str] | None,
     slack_channel_config: SlackChannelConfig,
+    prompt: Prompt | None,
     logger: OnyxLoggingAdapter,
     client: WebClient,
     db_session: Session,
@@ -159,6 +161,7 @@ def _handle_standard_answers(
         new_user_message = create_new_chat_message(
             chat_session_id=chat_session.id,
             parent_message=root_message,
+            prompt_id=prompt.id if prompt else None,
             message=query_msg.message,
             token_count=0,
             message_type=MessageType.USER,
@@ -179,6 +182,7 @@ def _handle_standard_answers(
         chat_message = create_new_chat_message(
             chat_session_id=chat_session.id,
             parent_message=new_user_message,
+            prompt_id=prompt.id if prompt else None,
             message=answer_message,
             token_count=0,
             message_type=MessageType.ASSISTANT,

backend/ee/onyx/server/evals/api.py

@@ -1,32 +0,0 @@
-from fastapi import APIRouter
-from fastapi import Depends
-
-from ee.onyx.auth.users import current_cloud_superuser
-from onyx.background.celery.apps.client import celery_app as client_app
-from onyx.configs.constants import OnyxCeleryTask
-from onyx.db.models import User
-from onyx.evals.models import EvalConfigurationOptions
-from onyx.server.evals.models import EvalRunAck
-from onyx.utils.logger import setup_logger
-
-logger = setup_logger()
-
-router = APIRouter(prefix="/evals")
-
-
-@router.post("/eval_run", response_model=EvalRunAck)
-def eval_run(
-    request: EvalConfigurationOptions,
-    user: User = Depends(current_cloud_superuser),
-) -> EvalRunAck:
-    """
-    Run an evaluation with the given message and optional dataset.
-    This endpoint requires a valid API key for authentication.
-    """
-    client_app.send_task(
-        OnyxCeleryTask.EVAL_RUN_TASK,
-        kwargs={
-            "configuration_dict": request.model_dump(),
-        },
-    )
-    return EvalRunAck(success=True)

backend/ee/onyx/server/query_and_chat/chat_backend.py

@@ -93,6 +93,7 @@ def handle_simplified_chat_message(
         parent_message_id=parent_message.id,
         message=chat_message_req.message,
         file_descriptors=[],
+        prompt_id=None,
         search_doc_ids=chat_message_req.search_doc_ids,
         retrieval_options=retrieval_options,
         # Simple API does not support reranking, hide complexity from user
@@ -180,6 +181,7 @@ def handle_send_message_simple_with_history(
         chat_message = create_new_chat_message(
             chat_session_id=chat_session.id,
             parent_message=chat_message,
+            prompt_id=req.prompt_id,
             message=msg.message,
             token_count=len(llm_tokenizer.encode(msg.message)),
             message_type=msg.role,
@@ -212,6 +214,7 @@ def handle_send_message_simple_with_history(
         parent_message_id=chat_message.id,
         message=query,
         file_descriptors=[],
+        prompt_id=req.prompt_id,
         search_doc_ids=req.search_doc_ids,
         retrieval_options=retrieval_options,
         # Simple API does not support reranking, hide complexity from user

backend/ee/onyx/server/query_and_chat/models.py

@@ -73,6 +73,7 @@ class BasicCreateChatMessageRequest(ChunkContext):
 class BasicCreateChatMessageWithHistoryRequest(ChunkContext):
     # Last element is the new query. All previous elements are historical context
     messages: list[ThreadMessage]
+    prompt_id: int | None
     persona_id: int
     retrieval_options: RetrievalDetails | None = None
     query_override: str | None = None
@@ -161,6 +162,7 @@ class OneShotQARequest(ChunkContext):
     persona_id: int | None = None
 
     messages: list[ThreadMessage]
+    prompt_id: int | None = None
     retrieval_options: RetrievalDetails = Field(default_factory=RetrievalDetails)
     rerank_settings: RerankingDetails | None = None
 
@@ -179,9 +181,11 @@ class OneShotQARequest(ChunkContext):
     def check_persona_fields(self) -> "OneShotQARequest":
         if self.persona_override_config is None and self.persona_id is None:
             raise ValueError("Exactly one of persona_config or persona_id must be set")
-        elif self.persona_override_config is not None and (self.persona_id is not None):
+        elif self.persona_override_config is not None and (
+            self.persona_id is not None or self.prompt_id is not None
+        ):
             raise ValueError(
-                "If persona_override_config is set, persona_id cannot be set"
+                "If persona_override_config is set, persona_id and prompt_id cannot be set"
             )
         return self
 

backend/ee/onyx/server/query_and_chat/query_backend.py

@@ -31,6 +31,7 @@ from onyx.context.search.pipeline import SearchPipeline
 from onyx.context.search.utils import dedupe_documents
 from onyx.context.search.utils import drop_llm_indices
 from onyx.context.search.utils import relevant_sections_to_indices
+from onyx.db.chat import get_prompt_by_id
 from onyx.db.engine.sql_engine import get_session
 from onyx.db.models import Persona
 from onyx.db.models import User
@@ -151,6 +152,14 @@ def get_answer_stream(
     ):
         raise KeyError("Must provide persona ID or Persona Config")
 
+    prompt = None
+    if query_request.prompt_id is not None:
+        prompt = get_prompt_by_id(
+            prompt_id=query_request.prompt_id,
+            user=user,
+            db_session=db_session,
+        )
+
     persona_info: Persona | PersonaOverrideConfig | None = None
     if query_request.persona_override_config is not None:
         persona_info = query_request.persona_override_config
@@ -185,6 +194,7 @@ def get_answer_stream(
         user=user,
         persona_id=query_request.persona_id,
         persona_override_config=query_request.persona_override_config,
+        prompt=prompt,
         message_ts_to_respond_to=None,
         retrieval_details=query_request.retrieval_options,
         rerank_settings=query_request.rerank_settings,

backend/ee/onyx/server/query_history/api.py

@@ -182,7 +182,6 @@ def admin_get_chat_sessions(
                 time_created=chat.time_created.isoformat(),
                 time_updated=chat.time_updated.isoformat(),
                 shared_status=chat.shared_status,
-                folder_id=chat.folder_id,
                 current_alternate_model=chat.current_alternate_model,
             )
             for chat in chat_sessions

backend/ee/onyx/server/seeding.py

@@ -131,35 +131,32 @@ def _seed_llms(
 def _seed_personas(db_session: Session, personas: list[PersonaUpsertRequest]) -> None:
     if personas:
         logger.notice("Seeding Personas")
-        try:
-            for persona in personas:
-                upsert_persona(
-                    user=None,  # Seeding is done as admin
-                    name=persona.name,
-                    description=persona.description,
-                    num_chunks=(
-                        persona.num_chunks if persona.num_chunks is not None else 0.0
-                    ),
-                    llm_relevance_filter=persona.llm_relevance_filter,
-                    llm_filter_extraction=persona.llm_filter_extraction,
-                    recency_bias=RecencyBiasSetting.AUTO,
-                    document_set_ids=persona.document_set_ids,
-                    llm_model_provider_override=persona.llm_model_provider_override,
-                    llm_model_version_override=persona.llm_model_version_override,
-                    starter_messages=persona.starter_messages,
-                    is_public=persona.is_public,
-                    db_session=db_session,
-                    tool_ids=persona.tool_ids,
-                    display_priority=persona.display_priority,
-                    system_prompt=persona.system_prompt,
-                    task_prompt=persona.task_prompt,
-                    datetime_aware=persona.datetime_aware,
-                    commit=False,
+        for persona in personas:
+            if not persona.prompt_ids:
+                raise ValueError(
+                    f"Invalid Persona with name {persona.name}; no prompts exist"
                 )
-            db_session.commit()
-        except Exception:
-            logger.exception("Failed to seed personas.")
-            raise
+
+            upsert_persona(
+                user=None,  # Seeding is done as admin
+                name=persona.name,
+                description=persona.description,
+                num_chunks=(
+                    persona.num_chunks if persona.num_chunks is not None else 0.0
+                ),
+                llm_relevance_filter=persona.llm_relevance_filter,
+                llm_filter_extraction=persona.llm_filter_extraction,
+                recency_bias=RecencyBiasSetting.AUTO,
+                prompt_ids=persona.prompt_ids,
+                document_set_ids=persona.document_set_ids,
+                llm_model_provider_override=persona.llm_model_provider_override,
+                llm_model_version_override=persona.llm_model_version_override,
+                starter_messages=persona.starter_messages,
+                is_public=persona.is_public,
+                db_session=db_session,
+                tool_ids=persona.tool_ids,
+                display_priority=persona.display_priority,
+            )
 
 
 def _seed_settings(settings: Settings) -> None:

backend/model_server/encoders.py

@@ -37,51 +37,24 @@ def get_embedding_model(
     model_name: str,
     max_context_length: int,
 ) -> "SentenceTransformer":
-    """
-    Loads or returns a cached SentenceTransformer, sets max_seq_length, pins device,
-    pre-warms rotary caches once, and wraps encode() with a lock to avoid cache races.
-    """
     from sentence_transformers import SentenceTransformer  # type: ignore
 
-    def _prewarm_rope(st_model: "SentenceTransformer", target_len: int) -> None:
-        """
-        Build RoPE cos/sin caches once on the final device/dtype so later forwards only read.
-        Works by calling the underlying HF model directly with dummy IDs/attention.
-        """
-        try:
-            # ensure > max seq after tokenization
-            # Ideally we would use the saved tokenizer, but whatever it's ok
-            # we'll make an assumption about tokenization here
-            long_text = "x " * (target_len * 2)
-            _ = st_model.encode(
-                [long_text],
-                batch_size=1,
-                convert_to_tensor=True,
-                show_progress_bar=False,
-                normalize_embeddings=False,
-            )
-            logger.info("RoPE pre-warm successful")
-        except Exception as e:
-            logger.warning(f"RoPE pre-warm skipped/failed: {e}")
-
-    global _GLOBAL_MODELS_DICT
+    global _GLOBAL_MODELS_DICT  # A dictionary to store models
 
     if model_name not in _GLOBAL_MODELS_DICT:
         logger.notice(f"Loading {model_name}")
+        # Some model architectures that aren't built into the Transformers or Sentence
+        # Transformer need to be downloaded to be loaded locally. This does not mean
+        # data is sent to remote servers for inference, however the remote code can
+        # be fairly arbitrary so only use trusted models
         model = SentenceTransformer(
             model_name_or_path=model_name,
             trust_remote_code=True,
         )
         model.max_seq_length = max_context_length
-        _prewarm_rope(model, max_context_length)
         _GLOBAL_MODELS_DICT[model_name] = model
-    else:
-        model = _GLOBAL_MODELS_DICT[model_name]
-        if max_context_length != model.max_seq_length:
-            model.max_seq_length = max_context_length
-            prev = getattr(model, "_rope_prewarmed_to", 0)
-            if max_context_length > int(prev or 0):
-                _prewarm_rope(model, max_context_length)
+    elif max_context_length != _GLOBAL_MODELS_DICT[model_name].max_seq_length:
+        _GLOBAL_MODELS_DICT[model_name].max_seq_length = max_context_length
 
     return _GLOBAL_MODELS_DICT[model_name]
 

backend/onyx/access/access.py

@@ -10,6 +10,7 @@ from onyx.configs.constants import PUBLIC_DOC_PAT
 from onyx.db.document import get_access_info_for_document
 from onyx.db.document import get_access_info_for_documents
 from onyx.db.models import User
+from onyx.db.models import UserFile
 from onyx.utils.variable_functionality import fetch_ee_implementation_or_noop
 from onyx.utils.variable_functionality import fetch_versioned_implementation
 
@@ -124,3 +125,20 @@ def source_should_fetch_permissions_during_indexing(source: DocumentSource) -> b
         ),
     )
     return _source_should_fetch_permissions_during_indexing_func(source)
+
+
+def get_access_for_user_files(
+    user_file_ids: list[str],
+    db_session: Session,
+) -> dict[str, DocumentAccess]:
+    user_files = db_session.query(UserFile).filter(UserFile.id.in_(user_file_ids)).all()
+    return {
+        str(user_file.id): DocumentAccess.build(
+            user_emails=[user_file.user.email],
+            user_groups=[],
+            is_public=False,
+            external_user_emails=[],
+            external_user_group_ids=[],
+        )
+        for user_file in user_files
+    }

backend/onyx/agents/agent_search/dc_search_analysis/nodes/a1_search_objects.py

@@ -39,7 +39,7 @@ def search_objects(
         raise ValueError("Search tool and persona must be provided for DivCon search")
 
     try:
-        instructions = graph_config.inputs.persona.system_prompt or ""
+        instructions = graph_config.inputs.persona.prompts[0].system_prompt
 
         agent_1_instructions = extract_section(
             instructions, "Agent Step 1:", "Agent Step 2:"

backend/onyx/agents/agent_search/dc_search_analysis/nodes/a2_research_object_source.py

@@ -43,7 +43,7 @@ def research_object_source(
         raise ValueError("Search tool and persona must be provided for DivCon search")
 
     try:
-        instructions = graph_config.inputs.persona.system_prompt or ""
+        instructions = graph_config.inputs.persona.prompts[0].system_prompt
 
         agent_2_instructions = extract_section(
             instructions, "Agent Step 2:", "Agent Step 3:"

backend/onyx/agents/agent_search/dc_search_analysis/nodes/a4_consolidate_object_research.py

@@ -33,7 +33,7 @@ def consolidate_object_research(
     if search_tool is None or graph_config.inputs.persona is None:
         raise ValueError("Search tool and persona must be provided for DivCon search")
 
-    instructions = graph_config.inputs.persona.system_prompt or ""
+    instructions = graph_config.inputs.persona.prompts[0].system_prompt
 
     agent_4_instructions = extract_section(
         instructions, "Agent Step 4:", "Agent Step 5:"

backend/onyx/agents/agent_search/dc_search_analysis/nodes/a5_consolidate_research.py

@@ -35,7 +35,7 @@ def consolidate_research(
         raise ValueError("Search tool and persona must be provided for DivCon search")
 
     # Populate prompt
-    instructions = graph_config.inputs.persona.system_prompt or ""
+    instructions = graph_config.inputs.persona.prompts[0].system_prompt
 
     try:
         agent_5_instructions = extract_section(

backend/onyx/agents/agent_search/dr/conditional_edges.py

@@ -39,7 +39,6 @@ def decision_router(state: MainState) -> list[Send | Hashable] | DRPath | str:
             DRPath.WEB_SEARCH,
             DRPath.KNOWLEDGE_GRAPH,
             DRPath.IMAGE_GENERATION,
-            DRPath.PYTHON_TOOL,
         )
         and len(state.query_list) == 0
     ):

backend/onyx/agents/agent_search/dr/constants.py

@@ -21,12 +21,10 @@ AVERAGE_TOOL_COSTS: dict[DRPath, float] = {
     DRPath.WEB_SEARCH: 1.5,
     DRPath.IMAGE_GENERATION: 3.0,
     DRPath.GENERIC_TOOL: 1.5,  # TODO: see todo in OrchestratorTool
-    DRPath.PYTHON_TOOL: 2.0,
     DRPath.CLOSER: 0.0,
 }
 
 DR_TIME_BUDGET_BY_TYPE = {
     ResearchType.THOUGHTFUL: 3.0,
     ResearchType.DEEP: 12.0,
-    ResearchType.FAST: 0.5,
 }

backend/onyx/agents/agent_search/dr/enums.py

@@ -8,7 +8,6 @@ class ResearchType(str, Enum):
     LEGACY_AGENTIC = "LEGACY_AGENTIC"  # only used for legacy agentic search migrations
     THOUGHTFUL = "THOUGHTFUL"
     DEEP = "DEEP"
-    FAST = "FAST"
 
 
 class ResearchAnswerPurpose(str, Enum):
@@ -27,7 +26,6 @@ class DRPath(str, Enum):
     WEB_SEARCH = "Web Search"
     IMAGE_GENERATION = "Image Generation"
     GENERIC_INTERNAL_TOOL = "Generic Internal Tool"
-    PYTHON_TOOL = "Python"
     CLOSER = "Closer"
     LOGGER = "Logger"
     END = "End"

backend/onyx/agents/agent_search/dr/graph_builder.py

@@ -26,9 +26,6 @@ from onyx.agents.agent_search.dr.sub_agents.image_generation.dr_image_generation
 from onyx.agents.agent_search.dr.sub_agents.kg_search.dr_kg_search_graph_builder import (
     dr_kg_search_graph_builder,
 )
-from onyx.agents.agent_search.dr.sub_agents.python_tool.dr_python_tool_graph_builder import (
-    dr_python_tool_graph_builder,
-)
 from onyx.agents.agent_search.dr.sub_agents.web_search.dr_ws_graph_builder import (
     dr_ws_graph_builder,
 )
@@ -61,15 +58,12 @@ def dr_graph_builder() -> StateGraph:
     image_generation_graph = dr_image_generation_graph_builder().compile()
     graph.add_node(DRPath.IMAGE_GENERATION, image_generation_graph)
 
-    generic_internal_tool_graph = dr_generic_internal_tool_graph_builder().compile()
-    graph.add_node(DRPath.GENERIC_INTERNAL_TOOL, generic_internal_tool_graph)
-
-    python_tool_graph = dr_python_tool_graph_builder().compile()
-    graph.add_node(DRPath.PYTHON_TOOL, python_tool_graph)
-
     custom_tool_graph = dr_custom_tool_graph_builder().compile()
     graph.add_node(DRPath.GENERIC_TOOL, custom_tool_graph)
 
+    generic_internal_tool_graph = dr_generic_internal_tool_graph_builder().compile()
+    graph.add_node(DRPath.GENERIC_INTERNAL_TOOL, generic_internal_tool_graph)
+
     graph.add_node(DRPath.CLOSER, closer)
     graph.add_node(DRPath.LOGGER, logging)
 
@@ -87,7 +81,6 @@ def dr_graph_builder() -> StateGraph:
     graph.add_edge(start_key=DRPath.IMAGE_GENERATION, end_key=DRPath.ORCHESTRATOR)
     graph.add_edge(start_key=DRPath.GENERIC_TOOL, end_key=DRPath.ORCHESTRATOR)
     graph.add_edge(start_key=DRPath.GENERIC_INTERNAL_TOOL, end_key=DRPath.ORCHESTRATOR)
-    graph.add_edge(start_key=DRPath.PYTHON_TOOL, end_key=DRPath.ORCHESTRATOR)
 
     graph.add_conditional_edges(DRPath.CLOSER, completeness_router)
     graph.add_edge(start_key=DRPath.LOGGER, end_key=END)

backend/onyx/agents/agent_search/dr/nodes/dr_a0_clarification.py

@@ -35,8 +35,6 @@ from onyx.agents.agent_search.shared_graph_utils.utils import (
 from onyx.agents.agent_search.shared_graph_utils.utils import run_with_timeout
 from onyx.agents.agent_search.shared_graph_utils.utils import write_custom_event
 from onyx.agents.agent_search.utils import create_question_prompt
-from onyx.configs.agent_configs import TF_DR_TIMEOUT_LONG
-from onyx.configs.agent_configs import TF_DR_TIMEOUT_SHORT
 from onyx.configs.constants import DocumentSource
 from onyx.configs.constants import DocumentSourceDescription
 from onyx.configs.constants import TMP_DRALPHA_PERSONA_NAME
@@ -52,13 +50,15 @@ from onyx.kg.utils.extraction_utils import get_relationship_types_str
 from onyx.llm.utils import check_number_of_tokens
 from onyx.llm.utils import get_max_input_tokens
 from onyx.natural_language_processing.utils import get_tokenizer
+from onyx.prompts.chat_prompts import PROJECT_INSTRUCTIONS_SEPARATOR
 from onyx.prompts.dr_prompts import ANSWER_PROMPT_WO_TOOL_CALLING
 from onyx.prompts.dr_prompts import DECISION_PROMPT_W_TOOL_CALLING
 from onyx.prompts.dr_prompts import DECISION_PROMPT_WO_TOOL_CALLING
 from onyx.prompts.dr_prompts import DEFAULT_DR_SYSTEM_PROMPT
+from onyx.prompts.dr_prompts import EVAL_SYSTEM_PROMPT_W_TOOL_CALLING
+from onyx.prompts.dr_prompts import EVAL_SYSTEM_PROMPT_WO_TOOL_CALLING
 from onyx.prompts.dr_prompts import REPEAT_PROMPT
 from onyx.prompts.dr_prompts import TOOL_DESCRIPTION
-from onyx.prompts.prompt_template import PromptTemplate
 from onyx.server.query_and_chat.streaming_models import MessageStart
 from onyx.server.query_and_chat.streaming_models import OverallStop
 from onyx.server.query_and_chat.streaming_models import SectionEnd
@@ -66,14 +66,13 @@ from onyx.server.query_and_chat.streaming_models import StreamingType
 from onyx.tools.tool_implementations.images.image_generation_tool import (
     ImageGenerationTool,
 )
+from onyx.tools.tool_implementations.internet_search.internet_search_tool import (
+    InternetSearchTool,
+)
 from onyx.tools.tool_implementations.knowledge_graph.knowledge_graph_tool import (
     KnowledgeGraphTool,
 )
-from onyx.tools.tool_implementations.python.python_tool import PythonTool
 from onyx.tools.tool_implementations.search.search_tool import SearchTool
-from onyx.tools.tool_implementations.web_search.web_search_tool import (
-    WebSearchTool,
-)
 from onyx.utils.b64 import get_image_type
 from onyx.utils.b64 import get_image_type_from_bytes
 from onyx.utils.logger import setup_logger
@@ -110,24 +109,19 @@ def _get_available_tools(
 
     for tool in graph_config.tooling.tools:
 
-        if not tool.is_available(db_session):
-            logger.info(f"Tool {tool.name} is not available, skipping")
-            continue
-
         tool_db_info = tool_dict.get(tool.id)
         if tool_db_info:
             incode_tool_id = tool_db_info.in_code_tool_id
         else:
             raise ValueError(f"Tool {tool.name} is not found in the database")
 
-        if isinstance(tool, WebSearchTool):
+        if isinstance(tool, InternetSearchTool):
             llm_path = DRPath.WEB_SEARCH.value
             path = DRPath.WEB_SEARCH
         elif isinstance(tool, SearchTool):
             llm_path = DRPath.INTERNAL_SEARCH.value
             path = DRPath.INTERNAL_SEARCH
         elif isinstance(tool, KnowledgeGraphTool) and include_kg:
-            # TODO (chris): move this into the `is_available` check
             if len(active_source_types) == 0:
                 logger.error(
                     "No active source types found, skipping Knowledge Graph tool"
@@ -135,9 +129,6 @@ def _get_available_tools(
                 continue
             llm_path = DRPath.KNOWLEDGE_GRAPH.value
             path = DRPath.KNOWLEDGE_GRAPH
-        elif isinstance(tool, PythonTool):
-            llm_path = DRPath.PYTHON_TOOL.value
-            path = DRPath.PYTHON_TOOL
         elif isinstance(tool, ImageGenerationTool):
             llm_path = DRPath.IMAGE_GENERATION.value
             path = DRPath.IMAGE_GENERATION
@@ -409,22 +400,30 @@ def clarifier(
     else:
         active_source_type_descriptions_str = ""
 
-    if graph_config.inputs.persona:
-        assistant_system_prompt = PromptTemplate(
-            graph_config.inputs.persona.system_prompt or DEFAULT_DR_SYSTEM_PROMPT
-        ).build()
-        if graph_config.inputs.persona.task_prompt:
+    if graph_config.inputs.persona and len(graph_config.inputs.persona.prompts) > 0:
+        assistant_system_prompt = (
+            graph_config.inputs.persona.prompts[0].system_prompt
+            or DEFAULT_DR_SYSTEM_PROMPT
+        ) + "\n\n"
+        if graph_config.inputs.persona.prompts[0].task_prompt:
             assistant_task_prompt = (
                 "\n\nHere are more specifications from the user:\n\n"
-                + PromptTemplate(graph_config.inputs.persona.task_prompt).build()
+                + graph_config.inputs.persona.prompts[0].task_prompt
             )
         else:
             assistant_task_prompt = ""
 
     else:
-        assistant_system_prompt = PromptTemplate(DEFAULT_DR_SYSTEM_PROMPT).build()
+        assistant_system_prompt = DEFAULT_DR_SYSTEM_PROMPT + "\n\n"
         assistant_task_prompt = ""
 
+    if graph_config.inputs.project_instructions:
+        assistant_system_prompt = (
+            assistant_system_prompt
+            + PROJECT_INSTRUCTIONS_SEPARATOR
+            + graph_config.inputs.project_instructions
+        )
+
     chat_history_string = (
         get_chat_history_string(
             graph_config.inputs.prompt_builder.message_history,
@@ -468,9 +467,8 @@ def clarifier(
                 llm_decision = invoke_llm_json(
                     llm=graph_config.tooling.primary_llm,
                     prompt=create_question_prompt(
-                        assistant_system_prompt,
+                        EVAL_SYSTEM_PROMPT_WO_TOOL_CALLING,
                         decision_prompt,
-                        uploaded_image_context=uploaded_image_context,
                     ),
                     schema=DecisionResponse,
                 )
@@ -498,13 +496,12 @@ def clarifier(
                 )
 
                 answer_tokens, _, _ = run_with_timeout(
-                    TF_DR_TIMEOUT_LONG,
+                    80,
                     lambda: stream_llm_answer(
                         llm=graph_config.tooling.primary_llm,
                         prompt=create_question_prompt(
                             assistant_system_prompt,
                             answer_prompt + assistant_task_prompt,
-                            uploaded_image_context=uploaded_image_context,
                         ),
                         event_name="basic_response",
                         writer=writer,
@@ -512,7 +509,7 @@ def clarifier(
                         agent_answer_level=0,
                         agent_answer_question_num=0,
                         agent_answer_type="agent_level_answer",
-                        timeout_override=TF_DR_TIMEOUT_LONG,
+                        timeout_override=60,
                         ind=current_step_nr,
                         context_docs=None,
                         replace_citations=True,
@@ -569,7 +566,7 @@ def clarifier(
 
             stream = graph_config.tooling.primary_llm.stream(
                 prompt=create_question_prompt(
-                    assistant_system_prompt,
+                    assistant_system_prompt + EVAL_SYSTEM_PROMPT_W_TOOL_CALLING,
                     decision_prompt + assistant_task_prompt,
                     uploaded_image_context=uploaded_image_context,
                 ),
@@ -623,7 +620,7 @@ def clarifier(
 
     clarification = None
 
-    if research_type == ResearchType.DEEP:
+    if research_type != ResearchType.THOUGHTFUL:
         result = _get_existing_clarification_request(graph_config)
         if result is not None:
             clarification, original_question, chat_history_string = result
@@ -653,12 +650,10 @@ def clarifier(
                 clarification_response = invoke_llm_json(
                     llm=graph_config.tooling.primary_llm,
                     prompt=create_question_prompt(
-                        assistant_system_prompt,
-                        clarification_prompt,
-                        uploaded_image_context=uploaded_image_context,
+                        assistant_system_prompt, clarification_prompt
                     ),
                     schema=ClarificationGenerationResponse,
-                    timeout_override=TF_DR_TIMEOUT_SHORT,
+                    timeout_override=25,
                     # max_tokens=1500,
                 )
             except Exception as e:
@@ -687,7 +682,7 @@ def clarifier(
                 )
 
                 _, _, _ = run_with_timeout(
-                    TF_DR_TIMEOUT_LONG,
+                    80,
                     lambda: stream_llm_answer(
                         llm=graph_config.tooling.primary_llm,
                         prompt=repeat_prompt,
@@ -696,7 +691,7 @@ def clarifier(
                         agent_answer_level=0,
                         agent_answer_question_num=0,
                         agent_answer_type="agent_level_answer",
-                        timeout_override=TF_DR_TIMEOUT_LONG,
+                        timeout_override=60,
                         answer_piece=StreamingType.MESSAGE_DELTA.value,
                         ind=current_step_nr,
                         # max_tokens=None,
@@ -782,6 +777,6 @@ def clarifier(
         active_source_types_descriptions="\n".join(active_source_types_descriptions),
         assistant_system_prompt=assistant_system_prompt,
         assistant_task_prompt=assistant_task_prompt,
-        uploaded_text_context=uploaded_text_context,
+        uploaded_test_context=uploaded_text_context,
         uploaded_image_context=uploaded_image_context,
     )

backend/onyx/agents/agent_search/dr/nodes/dr_a1_orchestrator.py

@@ -30,8 +30,6 @@ from onyx.agents.agent_search.shared_graph_utils.utils import (
 from onyx.agents.agent_search.shared_graph_utils.utils import run_with_timeout
 from onyx.agents.agent_search.shared_graph_utils.utils import write_custom_event
 from onyx.agents.agent_search.utils import create_question_prompt
-from onyx.configs.agent_configs import TF_DR_TIMEOUT_LONG
-from onyx.configs.agent_configs import TF_DR_TIMEOUT_SHORT
 from onyx.kg.utils.extraction_utils import get_entity_types_str
 from onyx.kg.utils.extraction_utils import get_relationship_types_str
 from onyx.prompts.dr_prompts import DEFAULLT_DECISION_PROMPT
@@ -140,8 +138,7 @@ def orchestrator(
 
     available_tools = state.available_tools or {}
 
-    uploaded_context = state.uploaded_text_context or ""
-    uploaded_image_context = state.uploaded_image_context or []
+    uploaded_context = state.uploaded_test_context or ""
 
     questions = [
         f"{iteration_response.tool}: {iteration_response.question}"
@@ -173,39 +170,11 @@ def orchestrator(
     reasoning_result = "(No reasoning result provided yet.)"
     tool_calls_string = "(No tool calls provided yet.)"
 
-    if research_type not in ResearchType:
-        raise ValueError(f"Invalid research type: {research_type}")
-
-    if research_type in [ResearchType.THOUGHTFUL, ResearchType.FAST]:
+    if research_type == ResearchType.THOUGHTFUL:
         if iteration_nr == 1:
-            remaining_time_budget = DR_TIME_BUDGET_BY_TYPE[research_type]
+            remaining_time_budget = DR_TIME_BUDGET_BY_TYPE[ResearchType.THOUGHTFUL]
 
-        elif remaining_time_budget <= 0:
-            return OrchestrationUpdate(
-                tools_used=[DRPath.CLOSER.value],
-                current_step_nr=current_step_nr,
-                query_list=[],
-                iteration_nr=iteration_nr,
-                log_messages=[
-                    get_langgraph_node_log_string(
-                        graph_component="main",
-                        node_name="orchestrator",
-                        node_start_time=node_start_time,
-                    )
-                ],
-                plan_of_record=plan_of_record,
-                remaining_time_budget=remaining_time_budget,
-                iteration_instructions=[
-                    IterationInstructions(
-                        iteration_nr=iteration_nr,
-                        plan=None,
-                        reasoning="Time to wrap up.",
-                        purpose="",
-                    )
-                ],
-            )
-
-        elif iteration_nr > 1 and remaining_time_budget > 0:
+        elif iteration_nr > 1:
             # for each iteration past the first one, we need to see whether we
             # have enough information to answer the question.
             # if we do, we can stop the iteration and return the answer.
@@ -231,20 +200,18 @@ def orchestrator(
             reasoning_tokens: list[str] = [""]
 
             reasoning_tokens, _, _ = run_with_timeout(
-                TF_DR_TIMEOUT_LONG,
+                80,
                 lambda: stream_llm_answer(
                     llm=graph_config.tooling.primary_llm,
                     prompt=create_question_prompt(
-                        decision_system_prompt,
-                        reasoning_prompt,
-                        uploaded_image_context=uploaded_image_context,
+                        decision_system_prompt, reasoning_prompt
                     ),
                     event_name="basic_response",
                     writer=writer,
                     agent_answer_level=0,
                     agent_answer_question_num=0,
                     agent_answer_type="agent_level_answer",
-                    timeout_override=TF_DR_TIMEOUT_LONG,
+                    timeout_override=60,
                     answer_piece=StreamingType.REASONING_DELTA.value,
                     ind=current_step_nr,
                     # max_tokens=None,
@@ -328,10 +295,9 @@ def orchestrator(
                     prompt=create_question_prompt(
                         decision_system_prompt,
                         decision_prompt,
-                        uploaded_image_context=uploaded_image_context,
                     ),
                     schema=OrchestratorDecisonsNoPlan,
-                    timeout_override=TF_DR_TIMEOUT_SHORT,
+                    timeout_override=35,
                     # max_tokens=2500,
                 )
                 next_step = orchestrator_action.next_step
@@ -354,7 +320,7 @@ def orchestrator(
             reasoning_result = "Time to wrap up."
             next_tool_name = DRPath.CLOSER.value
 
-    elif research_type == ResearchType.DEEP:
+    else:
         if iteration_nr == 1 and not plan_of_record:
             # by default, we start a new iteration, but if there is a feedback request,
             # we start a new iteration 0 again (set a bit later)
@@ -380,10 +346,9 @@ def orchestrator(
                     prompt=create_question_prompt(
                         decision_system_prompt,
                         plan_generation_prompt,
-                        uploaded_image_context=uploaded_image_context,
                     ),
                     schema=OrchestrationPlan,
-                    timeout_override=TF_DR_TIMEOUT_SHORT,
+                    timeout_override=25,
                     # max_tokens=3000,
                 )
             except Exception as e:
@@ -403,7 +368,7 @@ def orchestrator(
             )
 
             _, _, _ = run_with_timeout(
-                TF_DR_TIMEOUT_LONG,
+                80,
                 lambda: stream_llm_answer(
                     llm=graph_config.tooling.primary_llm,
                     prompt=repeat_plan_prompt,
@@ -412,7 +377,7 @@ def orchestrator(
                     agent_answer_level=0,
                     agent_answer_question_num=0,
                     agent_answer_type="agent_level_answer",
-                    timeout_override=TF_DR_TIMEOUT_LONG,
+                    timeout_override=60,
                     answer_piece=StreamingType.REASONING_DELTA.value,
                     ind=current_step_nr,
                 ),
@@ -459,10 +424,9 @@ def orchestrator(
                     prompt=create_question_prompt(
                         decision_system_prompt,
                         decision_prompt,
-                        uploaded_image_context=uploaded_image_context,
                     ),
                     schema=OrchestratorDecisonsNoPlan,
-                    timeout_override=TF_DR_TIMEOUT_LONG,
+                    timeout_override=15,
                     # max_tokens=1500,
                 )
                 next_step = orchestrator_action.next_step
@@ -496,7 +460,7 @@ def orchestrator(
         )
 
         _, _, _ = run_with_timeout(
-            TF_DR_TIMEOUT_LONG,
+            80,
             lambda: stream_llm_answer(
                 llm=graph_config.tooling.primary_llm,
                 prompt=repeat_reasoning_prompt,
@@ -505,7 +469,7 @@ def orchestrator(
                 agent_answer_level=0,
                 agent_answer_question_num=0,
                 agent_answer_type="agent_level_answer",
-                timeout_override=TF_DR_TIMEOUT_LONG,
+                timeout_override=60,
                 answer_piece=StreamingType.REASONING_DELTA.value,
                 ind=current_step_nr,
                 # max_tokens=None,
@@ -520,9 +484,6 @@ def orchestrator(
 
         current_step_nr += 1
 
-    else:
-        raise NotImplementedError(f"Research type {research_type} is not implemented.")
-
     base_next_step_purpose_prompt = get_dr_prompt_orchestration_templates(
         DRPromptPurpose.NEXT_STEP_PURPOSE,
         ResearchType.DEEP,
@@ -537,55 +498,48 @@ def orchestrator(
     )
 
     purpose_tokens: list[str] = [""]
-    purpose = ""
 
-    if research_type in [ResearchType.THOUGHTFUL, ResearchType.DEEP]:
+    try:
 
-        try:
+        write_custom_event(
+            current_step_nr,
+            ReasoningStart(),
+            writer,
+        )
 
-            write_custom_event(
-                current_step_nr,
-                ReasoningStart(),
-                writer,
-            )
-
-            purpose_tokens, _, _ = run_with_timeout(
-                TF_DR_TIMEOUT_LONG,
-                lambda: stream_llm_answer(
-                    llm=graph_config.tooling.primary_llm,
-                    prompt=create_question_prompt(
-                        decision_system_prompt,
-                        orchestration_next_step_purpose_prompt,
-                        uploaded_image_context=uploaded_image_context,
-                    ),
-                    event_name="basic_response",
-                    writer=writer,
-                    agent_answer_level=0,
-                    agent_answer_question_num=0,
-                    agent_answer_type="agent_level_answer",
-                    timeout_override=TF_DR_TIMEOUT_LONG,
-                    answer_piece=StreamingType.REASONING_DELTA.value,
-                    ind=current_step_nr,
-                    # max_tokens=None,
+        purpose_tokens, _, _ = run_with_timeout(
+            80,
+            lambda: stream_llm_answer(
+                llm=graph_config.tooling.primary_llm,
+                prompt=create_question_prompt(
+                    decision_system_prompt,
+                    orchestration_next_step_purpose_prompt,
                 ),
-            )
+                event_name="basic_response",
+                writer=writer,
+                agent_answer_level=0,
+                agent_answer_question_num=0,
+                agent_answer_type="agent_level_answer",
+                timeout_override=60,
+                answer_piece=StreamingType.REASONING_DELTA.value,
+                ind=current_step_nr,
+                # max_tokens=None,
+            ),
+        )
 
-            write_custom_event(
-                current_step_nr,
-                SectionEnd(),
-                writer,
-            )
+        write_custom_event(
+            current_step_nr,
+            SectionEnd(),
+            writer,
+        )
 
-            current_step_nr += 1
+        current_step_nr += 1
 
-        except Exception as e:
-            logger.error("Error in orchestration next step purpose.")
-            raise e
+    except Exception as e:
+        logger.error(f"Error in orchestration next step purpose: {e}")
+        raise e
 
-        purpose = cast(str, merge_content(*purpose_tokens))
-
-    elif research_type == ResearchType.FAST:
-        purpose = f"Answering the question using the {next_tool_name}"
+    purpose = cast(str, merge_content(*purpose_tokens))
 
     if not next_tool_name:
         raise ValueError("The next step has not been defined. This should not happen.")

backend/onyx/agents/agent_search/dr/nodes/dr_a2_closer.py

@@ -33,7 +33,6 @@ from onyx.agents.agent_search.shared_graph_utils.utils import (
 from onyx.agents.agent_search.shared_graph_utils.utils import write_custom_event
 from onyx.agents.agent_search.utils import create_question_prompt
 from onyx.chat.chat_utils import llm_doc_from_inference_section
-from onyx.configs.agent_configs import TF_DR_TIMEOUT_LONG
 from onyx.context.search.models import InferenceSection
 from onyx.db.chat import create_search_doc_from_inference_section
 from onyx.db.chat import update_db_session_with_messages
@@ -42,6 +41,7 @@ from onyx.db.models import ResearchAgentIteration
 from onyx.db.models import ResearchAgentIterationSubStep
 from onyx.db.models import SearchDoc as DbSearchDoc
 from onyx.llm.utils import check_number_of_tokens
+from onyx.prompts.chat_prompts import PROJECT_INSTRUCTIONS_SEPARATOR
 from onyx.prompts.dr_prompts import FINAL_ANSWER_PROMPT_W_SUB_ANSWERS
 from onyx.prompts.dr_prompts import FINAL_ANSWER_PROMPT_WITHOUT_SUB_ANSWERS
 from onyx.prompts.dr_prompts import TEST_INFO_COMPLETE_PROMPT
@@ -228,7 +228,7 @@ def closer(
     assistant_system_prompt = state.assistant_system_prompt
     assistant_task_prompt = state.assistant_task_prompt
 
-    uploaded_context = state.uploaded_text_context or ""
+    uploaded_context = state.uploaded_test_context or ""
 
     clarification = state.clarification
     prompt_question = get_prompt_question(base_question, clarification)
@@ -277,7 +277,7 @@ def closer(
                 test_info_complete_prompt + (assistant_task_prompt or ""),
             ),
             schema=TestInfoCompleteResponse,
-            timeout_override=TF_DR_TIMEOUT_LONG,
+            timeout_override=40,
             # max_tokens=1000,
         )
 
@@ -312,12 +312,10 @@ def closer(
         writer,
     )
 
-    if research_type in [ResearchType.THOUGHTFUL, ResearchType.FAST]:
+    if research_type == ResearchType.THOUGHTFUL:
         final_answer_base_prompt = FINAL_ANSWER_PROMPT_WITHOUT_SUB_ANSWERS
-    elif research_type == ResearchType.DEEP:
-        final_answer_base_prompt = FINAL_ANSWER_PROMPT_W_SUB_ANSWERS
     else:
-        raise ValueError(f"Invalid research type: {research_type}")
+        final_answer_base_prompt = FINAL_ANSWER_PROMPT_W_SUB_ANSWERS
 
     estimated_final_answer_prompt_tokens = check_number_of_tokens(
         final_answer_base_prompt.build(
@@ -349,6 +347,13 @@ def closer(
         uploaded_context=uploaded_context,
     )
 
+    if graph_config.inputs.project_instructions:
+        assistant_system_prompt = (
+            assistant_system_prompt
+            + PROJECT_INSTRUCTIONS_SEPARATOR
+            + graph_config.inputs.project_instructions
+        )
+
     all_context_llmdocs = [
         llm_doc_from_inference_section(inference_section)
         for inference_section in all_cited_documents
@@ -356,7 +361,7 @@ def closer(
 
     try:
         streamed_output, _, citation_infos = run_with_timeout(
-            int(3 * TF_DR_TIMEOUT_LONG),
+            240,
             lambda: stream_llm_answer(
                 llm=graph_config.tooling.primary_llm,
                 prompt=create_question_prompt(
@@ -368,7 +373,7 @@ def closer(
                 agent_answer_level=0,
                 agent_answer_question_num=0,
                 agent_answer_type="agent_level_answer",
-                timeout_override=int(2 * TF_DR_TIMEOUT_LONG),
+                timeout_override=60,
                 answer_piece=StreamingType.MESSAGE_DELTA.value,
                 ind=current_step_nr,
                 context_docs=all_context_llmdocs,

backend/onyx/agents/agent_search/dr/states.py

@@ -46,7 +46,7 @@ class OrchestrationSetup(OrchestrationUpdate):
     active_source_types_descriptions: str | None = None
     assistant_system_prompt: str | None = None
     assistant_task_prompt: str | None = None
-    uploaded_text_context: str | None = None
+    uploaded_test_context: str | None = None
     uploaded_image_context: list[dict[str, Any]] | None = None
 
 

backend/onyx/agents/agent_search/dr/sub_agents/basic_search/dr_basic_search_2_act.py

@@ -1,6 +1,7 @@
 import re
 from datetime import datetime
 from typing import cast
+from uuid import UUID
 
 from langchain_core.runnables import RunnableConfig
 from langgraph.types import StreamWriter
@@ -22,8 +23,6 @@ from onyx.agents.agent_search.shared_graph_utils.utils import (
 from onyx.agents.agent_search.shared_graph_utils.utils import write_custom_event
 from onyx.agents.agent_search.utils import create_question_prompt
 from onyx.chat.models import LlmDoc
-from onyx.configs.agent_configs import TF_DR_TIMEOUT_LONG
-from onyx.configs.agent_configs import TF_DR_TIMEOUT_SHORT
 from onyx.context.search.models import InferenceSection
 from onyx.db.connector import DocumentSource
 from onyx.db.engine.sql_engine import get_session_with_current_tenant
@@ -73,6 +72,7 @@ def basic_search(
 
     search_tool_info = state.available_tools[state.tools_used[-1]]
     search_tool = cast(SearchTool, search_tool_info.tool_object)
+    force_use_tool = graph_config.tooling.force_use_tool
 
     # sanity check
     if search_tool != graph_config.tooling.search_tool:
@@ -96,7 +96,7 @@ def basic_search(
                 assistant_system_prompt, base_search_processing_prompt
             ),
             schema=BaseSearchProcessingResponse,
-            timeout_override=TF_DR_TIMEOUT_SHORT,
+            timeout_override=15,
             # max_tokens=100,
         )
     except Exception as e:
@@ -141,6 +141,15 @@ def basic_search(
     retrieved_docs: list[InferenceSection] = []
     callback_container: list[list[InferenceSection]] = []
 
+    user_file_ids: list[UUID] | None = None
+    project_id: int | None = None
+    if force_use_tool.override_kwargs and isinstance(
+        force_use_tool.override_kwargs, SearchToolOverrideKwargs
+    ):
+        override_kwargs = force_use_tool.override_kwargs
+        user_file_ids = override_kwargs.user_file_ids
+        project_id = override_kwargs.project_id
+
     # new db session to avoid concurrency issues
     with get_session_with_current_tenant() as search_db_session:
         for tool_response in search_tool.run(
@@ -152,7 +161,8 @@ def basic_search(
                 alternate_db_session=search_db_session,
                 retrieved_sections_callback=callback_container.append,
                 skip_query_analysis=True,
-                original_query=rewritten_query,
+                user_file_ids=user_file_ids,
+                project_id=project_id,
             ),
         ):
             # get retrieved docs to send to the rest of the graph
@@ -206,7 +216,7 @@ def basic_search(
                 assistant_system_prompt, search_prompt + (assistant_task_prompt or "")
             ),
             schema=SearchAnswer,
-            timeout_override=TF_DR_TIMEOUT_LONG,
+            timeout_override=40,
             # max_tokens=1500,
         )
 
@@ -227,9 +237,9 @@ def basic_search(
             claims,
         ) = extract_document_citations(answer_string, claims)
 
-        if citation_numbers and (
-            (max(citation_numbers) > len(retrieved_docs)) or min(citation_numbers) < 1
-        ):
+        if (citation_numbers and max(citation_numbers) > len(retrieved_docs)) or min(
+            citation_numbers
+        ) < 1:
             raise ValueError("Citation numbers are out of range for retrieved docs.")
 
         cited_documents = {

backend/onyx/agents/agent_search/dr/sub_agents/custom_tool/dr_custom_tool_2_act.py

@@ -13,8 +13,6 @@ from onyx.agents.agent_search.models import GraphConfig
 from onyx.agents.agent_search.shared_graph_utils.utils import (
     get_langgraph_node_log_string,
 )
-from onyx.configs.agent_configs import TF_DR_TIMEOUT_LONG
-from onyx.configs.agent_configs import TF_DR_TIMEOUT_SHORT
 from onyx.prompts.dr_prompts import CUSTOM_TOOL_PREP_PROMPT
 from onyx.prompts.dr_prompts import CUSTOM_TOOL_USE_PROMPT
 from onyx.tools.tool_implementations.custom.custom_tool import CUSTOM_TOOL_RESPONSE_ID
@@ -70,7 +68,7 @@ def custom_tool_act(
             tool_use_prompt,
             tools=[custom_tool.tool_definition()],
             tool_choice="required",
-            timeout_override=TF_DR_TIMEOUT_LONG,
+            timeout_override=40,
         )
 
         # make sure we got a tool call
@@ -126,7 +124,7 @@ def custom_tool_act(
     )
     answer_string = str(
         graph_config.tooling.primary_llm.invoke(
-            tool_summary_prompt, timeout_override=TF_DR_TIMEOUT_SHORT
+            tool_summary_prompt, timeout_override=40
         ).content
     ).strip()
 

backend/onyx/agents/agent_search/dr/sub_agents/generic_internal_tool/dr_generic_internal_tool_2_act.py

@@ -13,7 +13,6 @@ from onyx.agents.agent_search.models import GraphConfig
 from onyx.agents.agent_search.shared_graph_utils.utils import (
     get_langgraph_node_log_string,
 )
-from onyx.configs.agent_configs import TF_DR_TIMEOUT_SHORT
 from onyx.prompts.dr_prompts import CUSTOM_TOOL_PREP_PROMPT
 from onyx.prompts.dr_prompts import CUSTOM_TOOL_USE_PROMPT
 from onyx.prompts.dr_prompts import OKTA_TOOL_USE_SPECIAL_PROMPT
@@ -69,7 +68,7 @@ def generic_internal_tool_act(
             tool_use_prompt,
             tools=[generic_internal_tool.tool_definition()],
             tool_choice="required",
-            timeout_override=TF_DR_TIMEOUT_SHORT,
+            timeout_override=40,
         )
 
         # make sure we got a tool call
@@ -114,7 +113,7 @@ def generic_internal_tool_act(
     )
     answer_string = str(
         graph_config.tooling.primary_llm.invoke(
-            tool_summary_prompt, timeout_override=TF_DR_TIMEOUT_SHORT
+            tool_summary_prompt, timeout_override=40
         ).content
     ).strip()
 

backend/onyx/agents/agent_search/dr/sub_agents/python_tool/__init__.py

@@ -1 +0,0 @@
-"""Python Tool sub-agent for deep research."""

backend/onyx/agents/agent_search/dr/sub_agents/python_tool/dr_python_tool_1_branch.py

@@ -1,36 +0,0 @@
-from datetime import datetime
-
-from langchain_core.runnables import RunnableConfig
-from langgraph.types import StreamWriter
-
-from onyx.agents.agent_search.dr.states import LoggerUpdate
-from onyx.agents.agent_search.dr.sub_agents.states import SubAgentInput
-from onyx.agents.agent_search.shared_graph_utils.utils import (
-    get_langgraph_node_log_string,
-)
-from onyx.utils.logger import setup_logger
-
-logger = setup_logger()
-
-
-def python_tool_branch(
-    state: SubAgentInput, config: RunnableConfig, writer: StreamWriter = lambda _: None
-) -> LoggerUpdate:
-    """Log the beginning of a Python Tool branch."""
-
-    node_start_time = datetime.now()
-    iteration_nr = state.iteration_nr
-
-    logger.debug(
-        f"Python Tool branch start for iteration {iteration_nr} at {datetime.now()}"
-    )
-
-    return LoggerUpdate(
-        log_messages=[
-            get_langgraph_node_log_string(
-                graph_component="python_tool",
-                node_name="branching",
-                node_start_time=node_start_time,
-            )
-        ],
-    )

backend/onyx/agents/agent_search/dr/sub_agents/python_tool/dr_python_tool_2_act.py

@@ -1,257 +0,0 @@
-import base64
-import json
-from datetime import datetime
-from pathlib import Path
-from typing import Any
-from typing import cast
-
-from langchain_core.messages import AIMessage
-from langchain_core.runnables import RunnableConfig
-from langgraph.types import StreamWriter
-
-from onyx.agents.agent_search.dr.sub_agents.states import BranchInput
-from onyx.agents.agent_search.dr.sub_agents.states import BranchUpdate
-from onyx.agents.agent_search.dr.sub_agents.states import IterationAnswer
-from onyx.agents.agent_search.models import GraphConfig
-from onyx.agents.agent_search.shared_graph_utils.utils import (
-    get_langgraph_node_log_string,
-)
-from onyx.configs.agent_configs import TF_DR_TIMEOUT_SHORT
-from onyx.configs.constants import MessageType
-from onyx.file_store.models import ChatFileType
-from onyx.file_store.models import InMemoryChatFile
-from onyx.llm.utils import build_content_with_imgs
-from onyx.prompts.dr_prompts import CUSTOM_TOOL_PREP_PROMPT
-from onyx.prompts.dr_prompts import PYTHON_TOOL_USE_RESPONSE_PROMPT
-from onyx.tools.tool_implementations.python.python_tool import PythonTool
-from onyx.tools.tool_implementations.python.python_tool import PythonToolResult
-from onyx.utils.logger import setup_logger
-
-logger = setup_logger()
-
-
-def _serialize_chat_files(chat_files: list[InMemoryChatFile]) -> list[dict[str, Any]]:
-    serialized_files: list[dict[str, Any]] = []
-    for chat_file in chat_files:
-        file_payload: dict[str, Any] = {
-            "id": str(chat_file.file_id),
-            "name": chat_file.filename,
-            "type": chat_file.file_type.value,
-        }
-        if chat_file.file_type == ChatFileType.IMAGE:
-            file_payload["content"] = chat_file.to_base64()
-            file_payload["is_base64"] = True
-        elif chat_file.file_type.is_text_file():
-            file_payload["content"] = chat_file.content.decode(
-                "utf-8", errors="replace"
-            )
-            file_payload["is_base64"] = False
-        else:
-            file_payload["content"] = base64.b64encode(chat_file.content).decode(
-                "utf-8"
-            )
-            file_payload["is_base64"] = True
-        serialized_files.append(file_payload)
-
-    return serialized_files
-
-
-def python_tool_act(
-    state: BranchInput,
-    config: RunnableConfig,
-    writer: StreamWriter = lambda _: None,
-) -> BranchUpdate:
-    """Execute the Python Tool with any files supplied by the user."""
-
-    node_start_time = datetime.now()
-    iteration_nr = state.iteration_nr
-    parallelization_nr = state.parallelization_nr
-
-    if not state.available_tools:
-        raise ValueError("available_tools is not set")
-
-    tool_key = state.tools_used[-1]
-    python_tool_info = state.available_tools[tool_key]
-    python_tool = cast(PythonTool | None, python_tool_info.tool_object)
-
-    if python_tool is None:
-        raise ValueError("python_tool is not set")
-
-    branch_query = state.branch_question
-    if not branch_query:
-        raise ValueError("branch_query is not set")
-
-    graph_config = cast(GraphConfig, config["metadata"]["config"])
-    base_question = graph_config.inputs.prompt_builder.raw_user_query
-    files = graph_config.inputs.files
-
-    logger.debug(
-        "Tool call start for %s %s.%s at %s",
-        python_tool.llm_name,
-        iteration_nr,
-        parallelization_nr,
-        datetime.now(),
-    )
-
-    tool_args: dict[str, Any] | None = None
-    if graph_config.tooling.using_tool_calling_llm:
-        tool_use_prompt = CUSTOM_TOOL_PREP_PROMPT.build(
-            query=branch_query,
-            base_question=base_question,
-            tool_description=python_tool_info.description,
-        )
-
-        content_with_files = build_content_with_imgs(
-            message=tool_use_prompt,
-            files=files,
-            message_type=MessageType.USER,
-        )
-
-        tool_prompt_message: dict[str, Any] = {
-            "role": "user",
-            "content": content_with_files,
-        }
-        if files:
-            tool_prompt_message["files"] = _serialize_chat_files(files)
-
-        tool_calling_msg = graph_config.tooling.primary_llm.invoke(
-            [tool_prompt_message],
-            tools=[python_tool.tool_definition()],
-            tool_choice="required",
-            timeout_override=TF_DR_TIMEOUT_SHORT,
-        )
-
-        if isinstance(tool_calling_msg, AIMessage) and tool_calling_msg.tool_calls:
-            tool_args = tool_calling_msg.tool_calls[0].get("args")
-        else:
-            logger.warning("Tool-calling LLM did not emit a tool call for Python Tool")
-
-    if tool_args is None:
-        tool_args = python_tool.get_args_for_non_tool_calling_llm(
-            query=branch_query,
-            history=[],
-            llm=graph_config.tooling.primary_llm,
-            force_run=True,
-        )
-
-    if tool_args is None:
-        raise ValueError("Failed to obtain tool arguments from LLM")
-
-    if "files" in tool_args:
-        tool_args = {key: value for key, value in tool_args.items() if key != "files"}
-
-    override_kwargs = {"files": files or []}
-
-    tool_responses = list(python_tool.run(override_kwargs=override_kwargs, **tool_args))
-
-    python_tool_result: PythonToolResult | None = None
-    for response in tool_responses:
-        if isinstance(response.response, PythonToolResult):
-            python_tool_result = response.response
-            break
-
-    if python_tool_result is None:
-        raise ValueError("Python tool did not return a valid result")
-
-    final_result = python_tool.final_result(*tool_responses)
-    tool_result_str = json.dumps(final_result, ensure_ascii=False)
-
-    tool_summary_prompt = PYTHON_TOOL_USE_RESPONSE_PROMPT.build(
-        base_question=base_question,
-        tool_response=tool_result_str,
-    )
-
-    initial_files = list(files or [])
-    generated_files: list[InMemoryChatFile] = []
-    for artifact in python_tool_result.artifacts:
-        if not artifact.file_id:
-            continue
-
-        chat_file = python_tool._available_files.get(artifact.file_id)
-        if not chat_file:
-            logger.warning(
-                "Generated artifact with id %s not found in available files",
-                artifact.file_id,
-            )
-            continue
-
-        filename = (
-            chat_file.filename
-            or artifact.display_name
-            or artifact.path
-            or str(artifact.file_id)
-        )
-        filename = Path(filename).name or str(artifact.file_id)
-        if not filename.startswith("generated_"):
-            filename = f"generated_{filename}"
-
-        generated_files.append(
-            InMemoryChatFile(
-                file_id=chat_file.file_id,
-                content=chat_file.content,
-                file_type=chat_file.file_type,
-                filename=filename,
-            )
-        )
-
-    summary_files = initial_files + generated_files
-    summary_content = build_content_with_imgs(
-        message=tool_summary_prompt,
-        files=summary_files,
-        message_type=MessageType.USER,
-    )
-
-    summary_message: dict[str, Any] = {
-        "role": "user",
-        "content": summary_content,
-    }
-    if summary_files:
-        summary_message["files"] = _serialize_chat_files(summary_files)
-
-    answer_string = str(
-        graph_config.tooling.primary_llm.invoke(
-            [summary_message],
-            timeout_override=TF_DR_TIMEOUT_SHORT,
-        ).content
-    ).strip()
-
-    artifact_file_ids = [
-        artifact.file_id
-        for artifact in python_tool_result.artifacts
-        if artifact.file_id
-    ]
-
-    logger.debug(
-        "Tool call end for %s %s.%s at %s",
-        python_tool.llm_name,
-        iteration_nr,
-        parallelization_nr,
-        datetime.now(),
-    )
-
-    return BranchUpdate(
-        branch_iteration_responses=[
-            IterationAnswer(
-                tool=python_tool.llm_name,
-                tool_id=python_tool_info.tool_id,
-                iteration_nr=iteration_nr,
-                parallelization_nr=parallelization_nr,
-                question=branch_query,
-                answer=answer_string,
-                claims=[],
-                cited_documents={},
-                reasoning="",
-                additional_data=None,
-                response_type="json",
-                data=final_result,
-                file_ids=artifact_file_ids or None,
-            )
-        ],
-        log_messages=[
-            get_langgraph_node_log_string(
-                graph_component="python_tool",
-                node_name="tool_calling",
-                node_start_time=node_start_time,
-            )
-        ],
-    )

backend/onyx/agents/agent_search/dr/sub_agents/python_tool/dr_python_tool_3_reduce.py

@@ -1,76 +0,0 @@
-from datetime import datetime
-
-from langchain_core.runnables import RunnableConfig
-from langgraph.types import StreamWriter
-
-from onyx.agents.agent_search.dr.sub_agents.states import SubAgentMainState
-from onyx.agents.agent_search.dr.sub_agents.states import SubAgentUpdate
-from onyx.agents.agent_search.shared_graph_utils.utils import (
-    get_langgraph_node_log_string,
-)
-from onyx.agents.agent_search.shared_graph_utils.utils import write_custom_event
-from onyx.server.query_and_chat.streaming_models import CustomToolDelta
-from onyx.server.query_and_chat.streaming_models import CustomToolStart
-from onyx.server.query_and_chat.streaming_models import SectionEnd
-from onyx.utils.logger import setup_logger
-
-logger = setup_logger()
-
-
-def python_tool_reducer(
-    state: SubAgentMainState,
-    config: RunnableConfig,
-    writer: StreamWriter = lambda _: None,
-) -> SubAgentUpdate:
-    """Stream the Python Tool result back to the client."""
-
-    node_start_time = datetime.now()
-    current_step_nr = state.current_step_nr
-    branch_updates = state.branch_iteration_responses
-    current_iteration = state.iteration_nr
-
-    new_updates = [
-        update for update in branch_updates if update.iteration_nr == current_iteration
-    ]
-
-    for new_update in new_updates:
-        if not new_update.response_type:
-            raise ValueError("Response type is not returned.")
-
-        write_custom_event(
-            current_step_nr,
-            CustomToolStart(
-                tool_name=new_update.tool,
-            ),
-            writer,
-        )
-
-        write_custom_event(
-            current_step_nr,
-            CustomToolDelta(
-                tool_name=new_update.tool,
-                response_type=new_update.response_type,
-                data=new_update.data,
-                file_ids=new_update.file_ids,
-            ),
-            writer,
-        )
-
-        write_custom_event(
-            current_step_nr,
-            SectionEnd(),
-            writer,
-        )
-
-        current_step_nr += 1
-
-    return SubAgentUpdate(
-        iteration_responses=new_updates,
-        log_messages=[
-            get_langgraph_node_log_string(
-                graph_component="python_tool",
-                node_name="consolidation",
-                node_start_time=node_start_time,
-            )
-        ],
-    )

backend/onyx/agents/agent_search/dr/sub_agents/python_tool/dr_python_tool_conditional_edges.py

@@ -1,26 +0,0 @@
-from collections.abc import Hashable
-
-from langgraph.types import Send
-
-from onyx.agents.agent_search.dr.sub_agents.states import BranchInput
-from onyx.agents.agent_search.dr.sub_agents.states import SubAgentInput
-
-
-def branching_router(state: SubAgentInput) -> list[Send | Hashable]:
-    """Forward the current query to the Python Tool executor."""
-
-    return [
-        Send(
-            "act",
-            BranchInput(
-                iteration_nr=state.iteration_nr,
-                parallelization_nr=parallelization_nr,
-                branch_question=query,
-                context="",
-                active_source_types=state.active_source_types,
-                tools_used=state.tools_used,
-                available_tools=state.available_tools,
-            ),
-        )
-        for parallelization_nr, query in enumerate(state.query_list[:1])
-    ]

backend/onyx/agents/agent_search/dr/sub_agents/python_tool/dr_python_tool_graph_builder.py

@@ -1,38 +0,0 @@
-from langgraph.graph import END
-from langgraph.graph import START
-from langgraph.graph import StateGraph
-
-from onyx.agents.agent_search.dr.sub_agents.python_tool.dr_python_tool_1_branch import (
-    python_tool_branch,
-)
-from onyx.agents.agent_search.dr.sub_agents.python_tool.dr_python_tool_2_act import (
-    python_tool_act,
-)
-from onyx.agents.agent_search.dr.sub_agents.python_tool.dr_python_tool_3_reduce import (
-    python_tool_reducer,
-)
-from onyx.agents.agent_search.dr.sub_agents.python_tool.dr_python_tool_conditional_edges import (
-    branching_router,
-)
-from onyx.agents.agent_search.dr.sub_agents.states import SubAgentInput
-from onyx.agents.agent_search.dr.sub_agents.states import SubAgentMainState
-from onyx.utils.logger import setup_logger
-
-logger = setup_logger()
-
-
-def dr_python_tool_graph_builder() -> StateGraph:
-    """LangGraph graph builder for the Python Tool sub-agent."""
-
-    graph = StateGraph(state_schema=SubAgentMainState, input=SubAgentInput)
-
-    graph.add_node("branch", python_tool_branch)
-    graph.add_node("act", python_tool_act)
-    graph.add_node("reducer", python_tool_reducer)
-
-    graph.add_edge(start_key=START, end_key="branch")
-    graph.add_conditional_edges("branch", branching_router)
-    graph.add_edge(start_key="act", end_key="reducer")
-    graph.add_edge(start_key="reducer", end_key=END)
-
-    return graph

backend/onyx/agents/agent_search/dr/sub_agents/web_search/dr_ws_2_search.py

@@ -25,7 +25,6 @@ from onyx.agents.agent_search.shared_graph_utils.utils import (
 )
 from onyx.agents.agent_search.shared_graph_utils.utils import write_custom_event
 from onyx.agents.agent_search.utils import create_question_prompt
-from onyx.configs.agent_configs import TF_DR_TIMEOUT_SHORT
 from onyx.prompts.dr_prompts import WEB_SEARCH_URL_SELECTION_PROMPT
 from onyx.server.query_and_chat.streaming_models import SearchToolDelta
 from onyx.utils.logger import setup_logger
@@ -109,7 +108,7 @@ def web_search(
             agent_decision_prompt + (assistant_task_prompt or ""),
         ),
         schema=WebSearchAnswer,
-        timeout_override=TF_DR_TIMEOUT_SHORT,
+        timeout_override=30,
     )
     results_to_open = [
         (search_query, search_results[i])

backend/onyx/agents/agent_search/dr/sub_agents/web_search/dr_ws_6_summarize.py

@@ -17,7 +17,6 @@ from onyx.agents.agent_search.shared_graph_utils.utils import (
     get_langgraph_node_log_string,
 )
 from onyx.agents.agent_search.utils import create_question_prompt
-from onyx.configs.agent_configs import TF_DR_TIMEOUT_SHORT
 from onyx.context.search.models import InferenceSection
 from onyx.prompts.dr_prompts import INTERNAL_SEARCH_PROMPTS
 from onyx.utils.logger import setup_logger
@@ -67,7 +66,7 @@ def is_summarize(
                 assistant_system_prompt, search_prompt + (assistant_task_prompt or "")
             ),
             schema=SearchAnswer,
-            timeout_override=TF_DR_TIMEOUT_SHORT,
+            timeout_override=40,
         )
         answer_string = search_answer_json.answer
         claims = search_answer_json.claims or []

backend/onyx/agents/agent_search/dr/utils.py

@@ -1,4 +1,3 @@
-import copy
 import re
 
 from langchain.schema.messages import BaseMessage
@@ -14,8 +13,8 @@ from onyx.agents.agent_search.shared_graph_utils.operators import (
 from onyx.context.search.models import InferenceSection
 from onyx.context.search.models import SavedSearchDoc
 from onyx.context.search.utils import chunks_or_sections_to_search_docs
-from onyx.tools.tool_implementations.web_search.web_search_tool import (
-    WebSearchTool,
+from onyx.tools.tool_implementations.internet_search.internet_search_tool import (
+    InternetSearchTool,
 )
 
 
@@ -73,7 +72,7 @@ def aggregate_context(
     ):
 
         iteration_tool = iteration_response.tool
-        is_internet = iteration_tool == WebSearchTool._NAME
+        is_internet = iteration_tool == InternetSearchTool._NAME
 
         for cited_doc in iteration_response.cited_documents.values():
             unrolled_inference_sections.append(cited_doc)
@@ -180,40 +179,11 @@ def get_chat_history_string(chat_history: list[BaseMessage], max_messages: int)
     Get the chat history (up to max_messages) as a string.
     """
     # get past max_messages USER, ASSISTANT message pairs
-
     past_messages = chat_history[-max_messages * 2 :]
-    filtered_past_messages = copy.deepcopy(past_messages)
-
-    for past_message_number, past_message in enumerate(past_messages):
-
-        if isinstance(past_message.content, list):
-            removal_indices = []
-            for content_piece_number, content_piece in enumerate(past_message.content):
-                if (
-                    isinstance(content_piece, dict)
-                    and content_piece.get("type") != "text"
-                ):
-                    removal_indices.append(content_piece_number)
-
-            # Only rebuild the content list if there are items to remove
-            if removal_indices:
-                filtered_past_messages[past_message_number].content = [
-                    content_piece
-                    for content_piece_number, content_piece in enumerate(
-                        past_message.content
-                    )
-                    if content_piece_number not in removal_indices
-                ]
-
-        else:
-            continue
-
-    return (
-        "...\n" if len(chat_history) > len(filtered_past_messages) else ""
-    ) + "\n".join(
+    return ("...\n" if len(chat_history) > len(past_messages) else "") + "\n".join(
         ("user" if isinstance(msg, HumanMessage) else "you")
         + f": {str(msg.content).strip()}"
-        for msg in filtered_past_messages
+        for msg in past_messages
     )
 
 

backend/onyx/agents/agent_search/models.py

@@ -24,6 +24,7 @@ class GraphInputs(BaseModel):
     prompt_builder: AnswerPromptBuilder
     files: list[InMemoryChatFile] | None = None
     structured_response_format: dict | None = None
+    project_instructions: str | None = None
 
     class Config:
         arbitrary_types_allowed = True

backend/onyx/agents/agent_search/shared_graph_utils/utils.py

@@ -1,3 +1,4 @@
+import os
 import re
 from collections.abc import Callable
 from collections.abc import Iterator
@@ -7,10 +8,18 @@ from typing import Any
 from typing import cast
 from typing import Literal
 from typing import TypedDict
+from uuid import UUID
 
 from langchain_core.messages import BaseMessage
+from langchain_core.messages import HumanMessage
 from langgraph.types import StreamWriter
+from sqlalchemy.orm import Session
 
+from onyx.agents.agent_search.models import GraphConfig
+from onyx.agents.agent_search.models import GraphInputs
+from onyx.agents.agent_search.models import GraphPersistence
+from onyx.agents.agent_search.models import GraphSearchConfig
+from onyx.agents.agent_search.models import GraphTooling
 from onyx.agents.agent_search.shared_graph_utils.models import BaseMessage_Content
 from onyx.agents.agent_search.shared_graph_utils.models import (
     EntityRelationshipTermExtraction,
@@ -23,6 +32,9 @@ from onyx.agents.agent_search.shared_graph_utils.models import SubQuestionAnswer
 from onyx.agents.agent_search.shared_graph_utils.operators import (
     dedup_inference_section_list,
 )
+from onyx.chat.models import AnswerStyleConfig
+from onyx.chat.models import CitationConfig
+from onyx.chat.models import DocumentPruningConfig
 from onyx.chat.models import MessageResponseIDInfo
 from onyx.chat.models import PromptConfig
 from onyx.chat.models import SectionRelevancePiece
@@ -30,16 +42,25 @@ from onyx.chat.models import StreamingError
 from onyx.chat.models import StreamStopInfo
 from onyx.chat.models import StreamStopReason
 from onyx.chat.models import StreamType
+from onyx.chat.prompt_builder.answer_prompt_builder import AnswerPromptBuilder
 from onyx.configs.agent_configs import AGENT_MAX_TOKENS_HISTORY_SUMMARY
 from onyx.configs.agent_configs import (
     AGENT_TIMEOUT_CONNECT_LLM_HISTORY_SUMMARY_GENERATION,
 )
 from onyx.configs.agent_configs import AGENT_TIMEOUT_LLM_HISTORY_SUMMARY_GENERATION
+from onyx.configs.chat_configs import CHAT_TARGET_CHUNK_PERCENTAGE
+from onyx.configs.chat_configs import MAX_CHUNKS_FED_TO_CHAT
+from onyx.configs.constants import DEFAULT_PERSONA_ID
 from onyx.configs.constants import DISPATCH_SEP_CHAR
 from onyx.configs.constants import FORMAT_DOCS_SEPARATOR
+from onyx.context.search.enums import LLMEvaluationType
 from onyx.context.search.models import InferenceSection
+from onyx.context.search.models import RetrievalDetails
+from onyx.context.search.models import SearchRequest
 from onyx.db.engine.sql_engine import get_session_with_current_tenant
+from onyx.db.persona import get_persona_by_id
 from onyx.db.persona import Persona
+from onyx.db.tools import get_tool_by_name
 from onyx.llm.chat_llm import LLMRateLimitError
 from onyx.llm.chat_llm import LLMTimeoutError
 from onyx.llm.interfaces import LLM
@@ -56,12 +77,15 @@ from onyx.prompts.agent_search import (
 from onyx.prompts.prompt_utils import handle_onyx_date_awareness
 from onyx.server.query_and_chat.streaming_models import Packet
 from onyx.server.query_and_chat.streaming_models import PacketObj
+from onyx.tools.force import ForceUseTool
 from onyx.tools.models import SearchToolOverrideKwargs
+from onyx.tools.tool_constructor import SearchToolConfig
 from onyx.tools.tool_implementations.search.search_tool import (
     SEARCH_RESPONSE_SUMMARY_ID,
 )
 from onyx.tools.tool_implementations.search.search_tool import SearchResponseSummary
 from onyx.tools.tool_implementations.search.search_tool import SearchTool
+from onyx.tools.utils import explicit_tool_calling_supported
 from onyx.utils.logger import setup_logger
 from onyx.utils.threadpool_concurrency import run_with_timeout
 
@@ -132,20 +156,136 @@ def format_entity_term_extraction(
     return "\n".join(entity_strs + relationship_strs + term_strs)
 
 
+def get_test_config(
+    db_session: Session,
+    primary_llm: LLM,
+    fast_llm: LLM,
+    search_request: SearchRequest,
+    use_agentic_search: bool = True,
+) -> GraphConfig:
+    persona = get_persona_by_id(DEFAULT_PERSONA_ID, None, db_session)
+    document_pruning_config = DocumentPruningConfig(
+        max_chunks=int(
+            persona.num_chunks
+            if persona.num_chunks is not None
+            else MAX_CHUNKS_FED_TO_CHAT
+        ),
+        max_window_percentage=CHAT_TARGET_CHUNK_PERCENTAGE,
+    )
+
+    answer_style_config = AnswerStyleConfig(
+        citation_config=CitationConfig(
+            # The docs retrieved by this flow are already relevance-filtered
+            all_docs_useful=True
+        ),
+        structured_response_format=None,
+    )
+
+    search_tool_config = SearchToolConfig(
+        answer_style_config=answer_style_config,
+        document_pruning_config=document_pruning_config,
+        retrieval_options=RetrievalDetails(),  # may want to set dedupe_docs=True
+        rerank_settings=None,  # Can use this to change reranking model
+        selected_sections=None,
+        latest_query_files=None,
+        bypass_acl=False,
+    )
+
+    prompt_config = PromptConfig.from_model(persona.prompts[0])
+
+    search_tool = SearchTool(
+        tool_id=get_tool_by_name(SearchTool._NAME, db_session).id,
+        db_session=db_session,
+        user=None,
+        persona=persona,
+        retrieval_options=search_tool_config.retrieval_options,
+        prompt_config=prompt_config,
+        llm=primary_llm,
+        fast_llm=fast_llm,
+        document_pruning_config=search_tool_config.document_pruning_config,
+        answer_style_config=search_tool_config.answer_style_config,
+        selected_sections=search_tool_config.selected_sections,
+        chunks_above=search_tool_config.chunks_above,
+        chunks_below=search_tool_config.chunks_below,
+        full_doc=search_tool_config.full_doc,
+        evaluation_type=(
+            LLMEvaluationType.BASIC
+            if persona.llm_relevance_filter
+            else LLMEvaluationType.SKIP
+        ),
+        rerank_settings=search_tool_config.rerank_settings,
+        bypass_acl=search_tool_config.bypass_acl,
+    )
+
+    graph_inputs = GraphInputs(
+        persona=search_request.persona,
+        rerank_settings=search_tool_config.rerank_settings,
+        prompt_builder=AnswerPromptBuilder(
+            user_message=HumanMessage(content=search_request.query),
+            message_history=[],
+            llm_config=primary_llm.config,
+            raw_user_query=search_request.query,
+            raw_user_uploaded_files=[],
+        ),
+        structured_response_format=answer_style_config.structured_response_format,
+    )
+
+    using_tool_calling_llm = explicit_tool_calling_supported(
+        primary_llm.config.model_provider, primary_llm.config.model_name
+    )
+    graph_tooling = GraphTooling(
+        primary_llm=primary_llm,
+        fast_llm=fast_llm,
+        search_tool=search_tool,
+        tools=[search_tool],
+        force_use_tool=ForceUseTool(force_use=False, tool_name=""),
+        using_tool_calling_llm=using_tool_calling_llm,
+    )
+
+    chat_session_id = (
+        os.environ.get("ONYX_AS_CHAT_SESSION_ID")
+        or "00000000-0000-0000-0000-000000000000"
+    )
+    assert (
+        chat_session_id is not None
+    ), "ONYX_AS_CHAT_SESSION_ID must be set for backend tests"
+    graph_persistence = GraphPersistence(
+        db_session=db_session,
+        chat_session_id=UUID(chat_session_id),
+        message_id=1,
+    )
+
+    search_behavior_config = GraphSearchConfig(
+        use_agentic_search=use_agentic_search,
+        skip_gen_ai_answer_generation=False,
+        allow_refinement=True,
+    )
+    graph_config = GraphConfig(
+        inputs=graph_inputs,
+        tooling=graph_tooling,
+        persistence=graph_persistence,
+        behavior=search_behavior_config,
+    )
+
+    return graph_config
+
+
 def get_persona_agent_prompt_expressions(
     persona: Persona | None,
 ) -> PersonaPromptExpressions:
-    if persona is None:
+    if persona is None or len(persona.prompts) == 0:
+        # TODO base_prompt should be None, but no time to properly fix
         return PersonaPromptExpressions(
             contextualized_prompt=ASSISTANT_SYSTEM_PROMPT_DEFAULT, base_prompt=""
         )
 
-    # Prompts are now embedded directly on the Persona model
-    prompt_config = PromptConfig.from_model(persona)
+    # Only a 1:1 mapping between personas and prompts currently
+    prompt = persona.prompts[0]
+    prompt_config = PromptConfig.from_model(prompt)
     datetime_aware_system_prompt = handle_onyx_date_awareness(
         prompt_str=prompt_config.system_prompt,
         prompt_config=prompt_config,
-        add_additional_info_if_no_tag=persona.datetime_aware,
+        add_additional_info_if_no_tag=prompt.datetime_aware,
     )
 
     return PersonaPromptExpressions(

backend/onyx/background/celery/apps/light.py

@@ -115,7 +115,6 @@ celery_app.autodiscover_tasks(
         "onyx.background.celery.tasks.vespa",
         "onyx.background.celery.tasks.connector_deletion",
         "onyx.background.celery.tasks.doc_permission_syncing",
-        "onyx.background.celery.tasks.user_file_folder_sync",
         "onyx.background.celery.tasks.docprocessing",
     ]
 )

backend/onyx/background/celery/apps/primary.py

@@ -20,7 +20,6 @@ import onyx.background.celery.apps.app_base as app_base
 from onyx.background.celery.apps.app_base import task_logger
 from onyx.background.celery.celery_utils import celery_is_worker_primary
 from onyx.background.celery.tasks.vespa.document_sync import reset_document_sync
-from onyx.configs.app_configs import CELERY_WORKER_PRIMARY_POOL_OVERFLOW
 from onyx.configs.constants import CELERY_PRIMARY_WORKER_LOCK_TIMEOUT
 from onyx.configs.constants import OnyxRedisConstants
 from onyx.configs.constants import OnyxRedisLocks
@@ -84,11 +83,11 @@ def on_celeryd_init(sender: str, conf: Any = None, **kwargs: Any) -> None:
 def on_worker_init(sender: Worker, **kwargs: Any) -> None:
     logger.info("worker_init signal received.")
 
+    EXTRA_CONCURRENCY = 4  # small extra fudge factor for connection limits
+
     SqlEngine.set_app_name(POSTGRES_CELERY_WORKER_PRIMARY_APP_NAME)
     pool_size = cast(int, sender.concurrency)  # type: ignore
-    SqlEngine.init_engine(
-        pool_size=pool_size, max_overflow=CELERY_WORKER_PRIMARY_POOL_OVERFLOW
-    )
+    SqlEngine.init_engine(pool_size=pool_size, max_overflow=EXTRA_CONCURRENCY)
 
     app_base.wait_for_redis(sender, **kwargs)
     app_base.wait_for_db(sender, **kwargs)
@@ -317,13 +316,11 @@ celery_app.autodiscover_tasks(
     [
         "onyx.background.celery.tasks.connector_deletion",
         "onyx.background.celery.tasks.docprocessing",
-        "onyx.background.celery.tasks.evals",
         "onyx.background.celery.tasks.periodic",
         "onyx.background.celery.tasks.pruning",
         "onyx.background.celery.tasks.shared",
         "onyx.background.celery.tasks.vespa",
         "onyx.background.celery.tasks.llm_model_update",
-        "onyx.background.celery.tasks.user_file_folder_sync",
         "onyx.background.celery.tasks.kg_processing",
     ]
 )

backend/onyx/background/celery/apps/user_file_processing.py

@@ -0,0 +1,113 @@
+from typing import Any
+from typing import cast
+
+from celery import Celery
+from celery import signals
+from celery import Task
+from celery.apps.worker import Worker
+from celery.signals import celeryd_init
+from celery.signals import worker_init
+from celery.signals import worker_process_init
+from celery.signals import worker_ready
+from celery.signals import worker_shutdown
+
+import onyx.background.celery.apps.app_base as app_base
+from onyx.configs.constants import POSTGRES_CELERY_WORKER_USER_FILE_PROCESSING_APP_NAME
+from onyx.db.engine.sql_engine import SqlEngine
+from onyx.utils.logger import setup_logger
+from shared_configs.configs import MULTI_TENANT
+
+
+logger = setup_logger()
+
+celery_app = Celery(__name__)
+celery_app.config_from_object("onyx.background.celery.configs.user_file_processing")
+celery_app.Task = app_base.TenantAwareTask  # type: ignore [misc]
+
+
+@signals.task_prerun.connect
+def on_task_prerun(
+    sender: Any | None = None,
+    task_id: str | None = None,
+    task: Task | None = None,
+    args: tuple | None = None,
+    kwargs: dict | None = None,
+    **kwds: Any,
+) -> None:
+    app_base.on_task_prerun(sender, task_id, task, args, kwargs, **kwds)
+
+
+@signals.task_postrun.connect
+def on_task_postrun(
+    sender: Any | None = None,
+    task_id: str | None = None,
+    task: Task | None = None,
+    args: tuple | None = None,
+    kwargs: dict | None = None,
+    retval: Any | None = None,
+    state: str | None = None,
+    **kwds: Any,
+) -> None:
+    app_base.on_task_postrun(sender, task_id, task, args, kwargs, retval, state, **kwds)
+
+
+@celeryd_init.connect
+def on_celeryd_init(sender: str, conf: Any = None, **kwargs: Any) -> None:
+    app_base.on_celeryd_init(sender, conf, **kwargs)
+
+
+@worker_init.connect
+def on_worker_init(sender: Worker, **kwargs: Any) -> None:
+    logger.info("worker_init signal received.")
+
+    SqlEngine.set_app_name(POSTGRES_CELERY_WORKER_USER_FILE_PROCESSING_APP_NAME)
+
+    # rkuo: Transient errors keep happening in the indexing watchdog threads.
+    # "SSL connection has been closed unexpectedly"
+    # actually setting the spawn method in the cloud fixes 95% of these.
+    # setting pre ping might help even more, but not worrying about that yet
+    pool_size = cast(int, sender.concurrency)  # type: ignore
+    SqlEngine.init_engine(pool_size=pool_size, max_overflow=8)
+
+    app_base.wait_for_redis(sender, **kwargs)
+    app_base.wait_for_db(sender, **kwargs)
+    app_base.wait_for_vespa_or_shutdown(sender, **kwargs)
+
+    # Less startup checks in multi-tenant case
+    if MULTI_TENANT:
+        return
+
+    app_base.on_secondary_worker_init(sender, **kwargs)
+
+
+@worker_ready.connect
+def on_worker_ready(sender: Any, **kwargs: Any) -> None:
+    app_base.on_worker_ready(sender, **kwargs)
+
+
+@worker_shutdown.connect
+def on_worker_shutdown(sender: Any, **kwargs: Any) -> None:
+    app_base.on_worker_shutdown(sender, **kwargs)
+
+
+@worker_process_init.connect
+def init_worker(**kwargs: Any) -> None:
+    SqlEngine.reset_engine()
+
+
+@signals.setup_logging.connect
+def on_setup_logging(
+    loglevel: Any, logfile: Any, format: Any, colorize: Any, **kwargs: Any
+) -> None:
+    app_base.on_setup_logging(loglevel, logfile, format, colorize, **kwargs)
+
+
+base_bootsteps = app_base.get_bootsteps()
+for bootstep in base_bootsteps:
+    celery_app.steps["worker"].add(bootstep)
+
+celery_app.autodiscover_tasks(
+    [
+        "onyx.background.celery.tasks.user_file_processing",
+    ]
+)

backend/onyx/background/celery/configs/primary.py

@@ -1,5 +1,4 @@
 import onyx.background.celery.configs.base as shared_config
-from onyx.configs.app_configs import CELERY_WORKER_PRIMARY_CONCURRENCY
 
 broker_url = shared_config.broker_url
 broker_connection_retry_on_startup = shared_config.broker_connection_retry_on_startup
@@ -16,6 +15,6 @@ result_expires = shared_config.result_expires  # 86400 seconds is the default
 task_default_priority = shared_config.task_default_priority
 task_acks_late = shared_config.task_acks_late
 
-worker_concurrency = CELERY_WORKER_PRIMARY_CONCURRENCY
+worker_concurrency = 4
 worker_pool = "threads"
 worker_prefetch_multiplier = 1

backend/onyx/background/celery/configs/user_file_processing.py

@@ -0,0 +1,22 @@
+import onyx.background.celery.configs.base as shared_config
+from onyx.configs.app_configs import CELERY_WORKER_USER_FILE_PROCESSING_CONCURRENCY
+
+broker_url = shared_config.broker_url
+broker_connection_retry_on_startup = shared_config.broker_connection_retry_on_startup
+broker_pool_limit = shared_config.broker_pool_limit
+broker_transport_options = shared_config.broker_transport_options
+
+redis_socket_keepalive = shared_config.redis_socket_keepalive
+redis_retry_on_timeout = shared_config.redis_retry_on_timeout
+redis_backend_health_check_interval = shared_config.redis_backend_health_check_interval
+
+result_backend = shared_config.result_backend
+result_expires = shared_config.result_expires  # 86400 seconds is the default
+
+task_default_priority = shared_config.task_default_priority
+task_acks_late = shared_config.task_acks_late
+
+# User file processing worker configuration
+worker_concurrency = CELERY_WORKER_USER_FILE_PROCESSING_CONCURRENCY
+worker_pool = "threads"
+worker_prefetch_multiplier = 1

backend/onyx/background/celery/tasks/beat_schedule.py

@@ -89,17 +89,6 @@ beat_task_templates: list[dict] = [
             "expires": BEAT_EXPIRES_DEFAULT,
         },
     },
-    {
-        "name": "check-for-user-file-folder-sync",
-        "task": OnyxCeleryTask.CHECK_FOR_USER_FILE_FOLDER_SYNC,
-        "schedule": timedelta(
-            days=1
-        ),  # This should essentially always be triggered manually for user folder updates.
-        "options": {
-            "priority": OnyxCeleryPriority.MEDIUM,
-            "expires": BEAT_EXPIRES_DEFAULT,
-        },
-    },
     {
         "name": "check-for-pruning",
         "task": OnyxCeleryTask.CHECK_FOR_PRUNING,

backend/onyx/background/celery/tasks/connector_deletion/tasks.py

@@ -28,9 +28,6 @@ from onyx.db.connector_credential_pair import add_deletion_failure_message
 from onyx.db.connector_credential_pair import (
     delete_connector_credential_pair__no_commit,
 )
-from onyx.db.connector_credential_pair import (
-    delete_userfiles_for_cc_pair__no_commit,
-)
 from onyx.db.connector_credential_pair import get_connector_credential_pair_from_id
 from onyx.db.connector_credential_pair import get_connector_credential_pairs
 from onyx.db.document import (
@@ -484,12 +481,6 @@ def monitor_connector_deletion_taskset(
             # related to the deleted DocumentByConnectorCredentialPair during commit
             db_session.expire(cc_pair)
 
-            # delete all userfiles for the cc_pair
-            delete_userfiles_for_cc_pair__no_commit(
-                db_session=db_session,
-                cc_pair_id=cc_pair_id,
-            )
-
             # finally, delete the cc-pair
             delete_connector_credential_pair__no_commit(
                 db_session=db_session,

backend/onyx/background/celery/tasks/docprocessing/tasks.py

@@ -85,6 +85,9 @@ from onyx.document_index.factory import get_default_document_index
 from onyx.file_store.document_batch_storage import DocumentBatchStorage
 from onyx.file_store.document_batch_storage import get_document_batch_storage
 from onyx.httpx.httpx_pool import HttpxPool
+from onyx.indexing.adapters.document_indexing_adapter import (
+    DocumentIndexingBatchAdapter,
+)
 from onyx.indexing.embedder import DefaultIndexingEmbedder
 from onyx.indexing.indexing_pipeline import run_indexing_pipeline
 from onyx.natural_language_processing.search_nlp_models import EmbeddingModel
@@ -1369,6 +1372,14 @@ def _docprocessing_task(
                 f"Processing {len(documents)} documents through indexing pipeline"
             )
 
+            adapter = DocumentIndexingBatchAdapter(
+                db_session=db_session,
+                connector_id=index_attempt.connector_credential_pair.connector.id,
+                credential_id=index_attempt.connector_credential_pair.credential.id,
+                tenant_id=tenant_id,
+                index_attempt_metadata=index_attempt_metadata,
+            )
+
             # real work happens here!
             index_pipeline_result = run_indexing_pipeline(
                 embedder=embedding_model,
@@ -1378,7 +1389,8 @@ def _docprocessing_task(
                 db_session=db_session,
                 tenant_id=tenant_id,
                 document_batch=documents,
-                index_attempt_metadata=index_attempt_metadata,
+                request_id=index_attempt_metadata.request_id,
+                adapter=adapter,
             )
 
         # Update batch completion and document counts atomically using database coordination

backend/onyx/background/celery/tasks/evals/tasks.py

@@ -1,35 +0,0 @@
-from typing import Any
-
-from celery import shared_task
-from celery import Task
-
-from onyx.configs.app_configs import JOB_TIMEOUT
-from onyx.configs.constants import OnyxCeleryTask
-from onyx.evals.eval import run_eval
-from onyx.evals.models import EvalConfigurationOptions
-from onyx.utils.logger import setup_logger
-
-logger = setup_logger()
-
-
-@shared_task(
-    name=OnyxCeleryTask.EVAL_RUN_TASK,
-    ignore_result=True,
-    soft_time_limit=JOB_TIMEOUT,
-    bind=True,
-    trail=False,
-)
-def eval_run_task(
-    self: Task,
-    *,
-    configuration_dict: dict[str, Any],
-) -> None:
-    """Background task to run an evaluation with the given configuration"""
-    try:
-        configuration = EvalConfigurationOptions.model_validate(configuration_dict)
-        run_eval(configuration, remote_dataset_name=configuration.dataset_name)
-        logger.info("Successfully completed eval run task")
-
-    except Exception:
-        logger.error("Failed to run eval task")
-        raise

backend/onyx/background/celery/tasks/user_file_folder_sync/tasks.py

@@ -1,266 +0,0 @@
-import time
-from typing import List
-
-from celery import shared_task
-from celery import Task
-from celery.exceptions import SoftTimeLimitExceeded
-from redis.lock import Lock as RedisLock
-from sqlalchemy.orm import Session
-from tenacity import RetryError
-
-from onyx.background.celery.apps.app_base import task_logger
-from onyx.background.celery.tasks.shared.RetryDocumentIndex import RetryDocumentIndex
-from onyx.background.celery.tasks.shared.tasks import LIGHT_SOFT_TIME_LIMIT
-from onyx.background.celery.tasks.shared.tasks import LIGHT_TIME_LIMIT
-from onyx.background.celery.tasks.shared.tasks import OnyxCeleryTaskCompletionStatus
-from onyx.configs.app_configs import JOB_TIMEOUT
-from onyx.configs.constants import CELERY_USER_FILE_FOLDER_SYNC_BEAT_LOCK_TIMEOUT
-from onyx.configs.constants import OnyxCeleryTask
-from onyx.configs.constants import OnyxRedisLocks
-from onyx.db.connector_credential_pair import (
-    get_connector_credential_pairs_with_user_files,
-)
-from onyx.db.document import get_document
-from onyx.db.engine.sql_engine import get_session_with_current_tenant
-from onyx.db.models import ConnectorCredentialPair
-from onyx.db.models import Document
-from onyx.db.models import DocumentByConnectorCredentialPair
-from onyx.db.search_settings import get_active_search_settings
-from onyx.db.user_documents import fetch_user_files_for_documents
-from onyx.db.user_documents import fetch_user_folders_for_documents
-from onyx.document_index.factory import get_default_document_index
-from onyx.document_index.interfaces import VespaDocumentUserFields
-from onyx.httpx.httpx_pool import HttpxPool
-from onyx.redis.redis_pool import get_redis_client
-from onyx.utils.logger import setup_logger
-
-logger = setup_logger()
-
-
-@shared_task(
-    name=OnyxCeleryTask.CHECK_FOR_USER_FILE_FOLDER_SYNC,
-    ignore_result=True,
-    soft_time_limit=JOB_TIMEOUT,
-    trail=False,
-    bind=True,
-)
-def check_for_user_file_folder_sync(self: Task, *, tenant_id: str) -> bool | None:
-    """Runs periodically to check for documents that need user file folder metadata updates.
-    This task fetches all connector credential pairs with user files, gets the documents
-    associated with them, and updates the user file and folder metadata in Vespa.
-    """
-
-    time_start = time.monotonic()
-
-    r = get_redis_client()
-
-    lock_beat: RedisLock = r.lock(
-        OnyxRedisLocks.CHECK_USER_FILE_FOLDER_SYNC_BEAT_LOCK,
-        timeout=CELERY_USER_FILE_FOLDER_SYNC_BEAT_LOCK_TIMEOUT,
-    )
-
-    # these tasks should never overlap
-    if not lock_beat.acquire(blocking=False):
-        return None
-
-    try:
-        with get_session_with_current_tenant() as db_session:
-            # Get all connector credential pairs that have user files
-            cc_pairs = get_connector_credential_pairs_with_user_files(db_session)
-
-            if not cc_pairs:
-                task_logger.info("No connector credential pairs with user files found")
-                return True
-
-            # Get all documents associated with these cc_pairs
-            document_ids = get_documents_for_cc_pairs(cc_pairs, db_session)
-
-            if not document_ids:
-                task_logger.info(
-                    "No documents found for connector credential pairs with user files"
-                )
-                return True
-
-            # Fetch current user file and folder IDs for these documents
-            doc_id_to_user_file_id = fetch_user_files_for_documents(
-                document_ids=document_ids, db_session=db_session
-            )
-            doc_id_to_user_folder_id = fetch_user_folders_for_documents(
-                document_ids=document_ids, db_session=db_session
-            )
-
-            # Update Vespa metadata for each document
-            for doc_id in document_ids:
-                user_file_id = doc_id_to_user_file_id.get(doc_id)
-                user_folder_id = doc_id_to_user_folder_id.get(doc_id)
-
-                if user_file_id is not None or user_folder_id is not None:
-                    # Schedule a task to update the document metadata
-                    update_user_file_folder_metadata.apply_async(
-                        args=(doc_id,),  # Use tuple instead of list for args
-                        kwargs={
-                            "tenant_id": tenant_id,
-                            "user_file_id": user_file_id,
-                            "user_folder_id": user_folder_id,
-                        },
-                        queue="vespa_metadata_sync",
-                    )
-
-            task_logger.info(
-                f"Scheduled metadata updates for {len(document_ids)} documents. "
-                f"Elapsed time: {time.monotonic() - time_start:.2f}s"
-            )
-
-            return True
-    except Exception as e:
-        task_logger.exception(f"Error in check_for_user_file_folder_sync: {e}")
-        return False
-    finally:
-        lock_beat.release()
-
-
-def get_documents_for_cc_pairs(
-    cc_pairs: List[ConnectorCredentialPair], db_session: Session
-) -> List[str]:
-    """Get all document IDs associated with the given connector credential pairs."""
-    if not cc_pairs:
-        return []
-
-    cc_pair_ids = [cc_pair.id for cc_pair in cc_pairs]
-
-    # Query to get document IDs from DocumentByConnectorCredentialPair
-    # Note: DocumentByConnectorCredentialPair uses connector_id and credential_id, not cc_pair_id
-    doc_cc_pairs = (
-        db_session.query(Document.id)
-        .join(
-            DocumentByConnectorCredentialPair,
-            Document.id == DocumentByConnectorCredentialPair.id,
-        )
-        .filter(
-            db_session.query(ConnectorCredentialPair)
-            .filter(
-                ConnectorCredentialPair.id.in_(cc_pair_ids),
-                ConnectorCredentialPair.connector_id
-                == DocumentByConnectorCredentialPair.connector_id,
-                ConnectorCredentialPair.credential_id
-                == DocumentByConnectorCredentialPair.credential_id,
-            )
-            .exists()
-        )
-        .all()
-    )
-
-    return [doc_id for (doc_id,) in doc_cc_pairs]
-
-
-@shared_task(
-    name=OnyxCeleryTask.UPDATE_USER_FILE_FOLDER_METADATA,
-    bind=True,
-    soft_time_limit=LIGHT_SOFT_TIME_LIMIT,
-    time_limit=LIGHT_TIME_LIMIT,
-    max_retries=3,
-)
-def update_user_file_folder_metadata(
-    self: Task,
-    document_id: str,
-    *,
-    tenant_id: str,
-    user_file_id: int | None,
-    user_folder_id: int | None,
-) -> bool:
-    """Updates the user file and folder metadata for a document in Vespa."""
-    start = time.monotonic()
-    completion_status = OnyxCeleryTaskCompletionStatus.UNDEFINED
-
-    try:
-        with get_session_with_current_tenant() as db_session:
-            active_search_settings = get_active_search_settings(db_session)
-            doc_index = get_default_document_index(
-                search_settings=active_search_settings.primary,
-                secondary_search_settings=active_search_settings.secondary,
-                httpx_client=HttpxPool.get("vespa"),
-            )
-
-            retry_index = RetryDocumentIndex(doc_index)
-
-            doc = get_document(document_id, db_session)
-            if not doc:
-                elapsed = time.monotonic() - start
-                task_logger.info(
-                    f"doc={document_id} "
-                    f"action=no_operation "
-                    f"elapsed={elapsed:.2f}"
-                )
-                completion_status = OnyxCeleryTaskCompletionStatus.SKIPPED
-                return False
-
-            # Create user fields object with file and folder IDs
-            user_fields = VespaDocumentUserFields(
-                user_file_id=str(user_file_id) if user_file_id is not None else None,
-                user_folder_id=(
-                    str(user_folder_id) if user_folder_id is not None else None
-                ),
-            )
-
-            # Update Vespa. OK if doc doesn't exist. Raises exception otherwise.
-            chunks_affected = retry_index.update_single(
-                document_id,
-                tenant_id=tenant_id,
-                chunk_count=doc.chunk_count,
-                fields=None,  # We're only updating user fields
-                user_fields=user_fields,
-            )
-
-            elapsed = time.monotonic() - start
-            task_logger.info(
-                f"doc={document_id} "
-                f"action=user_file_folder_sync "
-                f"user_file_id={user_file_id} "
-                f"user_folder_id={user_folder_id} "
-                f"chunks={chunks_affected} "
-                f"elapsed={elapsed:.2f}"
-            )
-            completion_status = OnyxCeleryTaskCompletionStatus.SUCCEEDED
-            return True
-
-    except SoftTimeLimitExceeded:
-        task_logger.info(f"SoftTimeLimitExceeded exception. doc={document_id}")
-        completion_status = OnyxCeleryTaskCompletionStatus.SOFT_TIME_LIMIT
-    except Exception as ex:
-        e: Exception | None = None
-        while True:
-            if isinstance(ex, RetryError):
-                task_logger.warning(
-                    f"Tenacity retry failed: num_attempts={ex.last_attempt.attempt_number}"
-                )
-
-                # only set the inner exception if it is of type Exception
-                e_temp = ex.last_attempt.exception()
-                if isinstance(e_temp, Exception):
-                    e = e_temp
-            else:
-                e = ex
-
-            task_logger.exception(
-                f"update_user_file_folder_metadata exceptioned: doc={document_id}"
-            )
-
-            completion_status = OnyxCeleryTaskCompletionStatus.RETRYABLE_EXCEPTION
-            if (
-                self.max_retries is not None
-                and self.request.retries >= self.max_retries
-            ):
-                completion_status = (
-                    OnyxCeleryTaskCompletionStatus.NON_RETRYABLE_EXCEPTION
-                )
-
-            # Exponential backoff from 2^4 to 2^6 ... i.e. 16, 32, 64
-            countdown = 2 ** (self.request.retries + 4)
-            self.retry(exc=e, countdown=countdown)  # this will raise a celery exception
-            break  # we won't hit this, but it looks weird not to have it
-    finally:
-        task_logger.info(
-            f"update_user_file_folder_metadata completed: status={completion_status.value} doc={document_id}"
-        )
-
-    return False

backend/onyx/background/celery/tasks/user_file_processing/tasks.py

@@ -0,0 +1,248 @@
+import time
+
+from celery import shared_task
+from celery import Task
+from redis.lock import Lock as RedisLock
+from sqlalchemy import select
+
+from onyx.background.celery.apps.app_base import task_logger
+from onyx.background.celery.celery_utils import httpx_init_vespa_pool
+from onyx.configs.app_configs import MANAGED_VESPA
+from onyx.configs.app_configs import VESPA_CLOUD_CERT_PATH
+from onyx.configs.app_configs import VESPA_CLOUD_KEY_PATH
+from onyx.configs.constants import CELERY_GENERIC_BEAT_LOCK_TIMEOUT
+from onyx.configs.constants import OnyxCeleryPriority
+from onyx.configs.constants import OnyxCeleryQueues
+from onyx.configs.constants import OnyxCeleryTask
+from onyx.configs.constants import OnyxRedisLocks
+from onyx.connectors.file.connector import LocalFileConnector
+from onyx.connectors.models import Document
+from onyx.db.engine.sql_engine import get_session_with_current_tenant
+from onyx.db.enums import UserFileStatus
+from onyx.db.models import UserFile
+from onyx.db.search_settings import get_active_search_settings_list
+from onyx.document_index.factory import get_default_document_index
+from onyx.httpx.httpx_pool import HttpxPool
+from onyx.indexing.adapters.user_file_indexing_adapter import UserFileIndexingAdapter
+from onyx.indexing.embedder import DefaultIndexingEmbedder
+from onyx.indexing.indexing_pipeline import run_indexing_pipeline
+from onyx.natural_language_processing.search_nlp_models import (
+    InformationContentClassificationModel,
+)
+from onyx.redis.redis_pool import get_redis_client
+from shared_configs.contextvars import CURRENT_TENANT_ID_CONTEXTVAR
+
+
+def _user_file_lock_key(user_file_id: int) -> str:
+    return f"{OnyxRedisLocks.USER_FILE_PROCESSING_LOCK_PREFIX}:{user_file_id}"
+
+
+@shared_task(
+    name=OnyxCeleryTask.CHECK_FOR_USER_FILE_PROCESSING,
+    soft_time_limit=300,
+    bind=True,
+    ignore_result=True,
+)
+def check_user_file_processing(self: Task, *, tenant_id: str) -> None:
+    """Scan for user files with PROCESSING status and enqueue per-file tasks.
+
+    Uses direct Redis locks to avoid overlapping runs.
+    """
+    task_logger.info("check_user_file_processing - Starting")
+    CURRENT_TENANT_ID_CONTEXTVAR.set(tenant_id)
+
+    redis_client = get_redis_client(tenant_id=tenant_id)
+    lock: RedisLock = redis_client.lock(
+        OnyxRedisLocks.USER_FILE_PROCESSING_BEAT_LOCK,
+        timeout=CELERY_GENERIC_BEAT_LOCK_TIMEOUT,
+    )
+
+    # Do not overlap generator runs
+    if not lock.acquire(blocking=False):
+        return None
+
+    enqueued = 0
+    try:
+        with get_session_with_current_tenant() as db_session:
+            user_file_ids = (
+                db_session.execute(
+                    select(UserFile.id).where(
+                        UserFile.status == UserFileStatus.PROCESSING
+                    )
+                )
+                .scalars()
+                .all()
+            )
+
+            for user_file_id in user_file_ids:
+                self.app.send_task(
+                    OnyxCeleryTask.PROCESS_SINGLE_USER_FILE,
+                    kwargs={"user_file_id": user_file_id, "tenant_id": tenant_id},
+                    queue=OnyxCeleryQueues.USER_FILE_PROCESSING,
+                    priority=OnyxCeleryPriority.HIGH,
+                )
+                enqueued += 1
+
+    finally:
+        if lock.owned():
+            lock.release()
+
+    task_logger.info(
+        f"check_user_file_processing - Enqueued {enqueued} tasks for tenant={tenant_id}"
+    )
+    return None
+
+
+@shared_task(
+    name=OnyxCeleryTask.PROCESS_SINGLE_USER_FILE,
+    bind=True,
+    ignore_result=True,
+)
+def process_single_user_file(self: Task, *, user_file_id: str, tenant_id: str) -> None:
+    task_logger.info(f"process_single_user_file - Starting id={user_file_id}")
+    start = time.monotonic()
+    CURRENT_TENANT_ID_CONTEXTVAR.set(tenant_id)
+
+    redis_client = get_redis_client(tenant_id=tenant_id)
+    file_lock: RedisLock = redis_client.lock(
+        _user_file_lock_key(user_file_id), timeout=CELERY_GENERIC_BEAT_LOCK_TIMEOUT
+    )
+
+    if not file_lock.acquire(blocking=False):
+        task_logger.info(
+            f"process_single_user_file - Lock held, skipping user_file_id={user_file_id}"
+        )
+        return None
+
+    documents: list[Document] = []
+    try:
+        with get_session_with_current_tenant() as db_session:
+            uf = db_session.get(UserFile, user_file_id)
+            if not uf:
+                task_logger.warning(
+                    f"process_single_user_file - UserFile not found id={user_file_id}"
+                )
+                return None
+
+            if uf.status != UserFileStatus.PROCESSING:
+                task_logger.info(
+                    f"process_single_user_file - Skipping id={user_file_id} status={uf.status}"
+                )
+                return None
+
+            connector = LocalFileConnector(
+                file_locations=[uf.file_id],
+                file_names=[uf.name] if uf.name else None,
+                zip_metadata={},
+            )
+            connector.load_credentials({})
+
+            # 20 is the documented default for httpx max_keepalive_connections
+            if MANAGED_VESPA:
+                httpx_init_vespa_pool(
+                    20, ssl_cert=VESPA_CLOUD_CERT_PATH, ssl_key=VESPA_CLOUD_KEY_PATH
+                )
+            else:
+                httpx_init_vespa_pool(20)
+
+            search_settings_list = get_active_search_settings_list(db_session)
+
+            current_search_settings = next(
+                search_settings_instance
+                for search_settings_instance in search_settings_list
+                if search_settings_instance.status.is_current()
+            )
+
+            if not current_search_settings:
+                raise RuntimeError(
+                    f"process_single_user_file - No current search settings found for tenant={tenant_id}"
+                )
+
+            try:
+                for batch in connector.load_from_state():
+                    documents.extend(batch)
+
+                adapter = UserFileIndexingAdapter(
+                    tenant_id=tenant_id,
+                    db_session=db_session,
+                )
+
+                # Set up indexing pipeline components
+                embedding_model = DefaultIndexingEmbedder.from_db_search_settings(
+                    search_settings=current_search_settings,
+                )
+
+                information_content_classification_model = (
+                    InformationContentClassificationModel()
+                )
+
+                document_index = get_default_document_index(
+                    current_search_settings,
+                    None,
+                    httpx_client=HttpxPool.get("vespa"),
+                )
+
+                task_logger.info(
+                    f"process_single_user_file - Documents before indexing: {documents}"
+                )
+
+                # update the doument id to userfile id in the documents
+                for document in documents:
+                    document.id = str(user_file_id)
+
+                # real work happens here!
+                index_pipeline_result = run_indexing_pipeline(
+                    embedder=embedding_model,
+                    information_content_classification_model=information_content_classification_model,
+                    document_index=document_index,
+                    ignore_time_skip=True,
+                    db_session=db_session,
+                    tenant_id=tenant_id,
+                    document_batch=documents,
+                    request_id=None,
+                    adapter=adapter,
+                )
+
+                task_logger.info(
+                    f"process_single_user_file - Indexing pipeline completed ={index_pipeline_result}"
+                )
+
+                if index_pipeline_result.failures:
+                    task_logger.error(
+                        f"process_single_user_file - Indexing pipeline failed id={user_file_id}"
+                    )
+                    uf.status = UserFileStatus.FAILED
+                    db_session.add(uf)
+                    db_session.commit()
+                    return None
+
+            except Exception as e:
+                task_logger.exception(
+                    f"process_single_user_file - Error id={user_file_id}: {e}"
+                )
+                uf.status = UserFileStatus.FAILED
+                db_session.add(uf)
+                db_session.commit()
+                return None
+
+        elapsed = time.monotonic() - start
+        task_logger.info(
+            f"process_single_user_file - Finished id={user_file_id} docs={len(documents)} elapsed={elapsed:.2f}s"
+        )
+        return None
+    except Exception as e:
+        # Attempt to mark the file as failed
+        with get_session_with_current_tenant() as db_session:
+            uf = db_session.get(UserFile, user_file_id)
+            if uf:
+                uf.status = UserFileStatus.FAILED
+                db_session.add(uf)
+                db_session.commit()
+
+        task_logger.exception(
+            f"process_single_user_file - Error id={user_file_id}: {e}"
+        )
+        return None
+    finally:
+        if file_lock.owned():
+            file_lock.release()

backend/onyx/background/celery/tasks/vespa/tasks.py

@@ -414,14 +414,8 @@ def monitor_document_set_taskset(
         get_document_set_by_id(db_session=db_session, document_set_id=document_set_id),
     )  # casting since we "know" a document set with this ID exists
     if document_set:
-        has_connector_pairs = bool(document_set.connector_credential_pairs)
-        # Federated connectors should keep a document set alive even without cc pairs.
-        has_federated_connectors = bool(
-            getattr(document_set, "federated_connectors", [])
-        )
-
-        if not has_connector_pairs and not has_federated_connectors:
-            # If there are no connectors of any kind, delete the document set.
+        if not document_set.connector_credential_pairs:
+            # if there are no connectors, then delete the document set.
             delete_document_set(document_set_row=document_set, db_session=db_session)
             task_logger.info(
                 f"Successfully deleted document set: document_set={document_set_id}"

backend/onyx/background/celery/versioned_apps/user_file_processing.py

@@ -0,0 +1,16 @@
+"""Factory stub for running the user file processing Celery worker."""
+
+from celery import Celery
+
+from onyx.utils.variable_functionality import set_is_ee_based_on_env_variable
+
+set_is_ee_based_on_env_variable()
+
+
+def get_app() -> Celery:
+    from onyx.background.celery.apps.user_file_processing import celery_app
+
+    return celery_app
+
+
+app = get_app()

backend/onyx/background/indexing/run_docfetching.py

@@ -578,7 +578,7 @@ def _run_indexing(
                     db_session=db_session,
                     tenant_id=tenant_id,
                     document_batch=doc_batch_cleaned,
-                    index_attempt_metadata=index_attempt_md,
+                    request_id=index_attempt_md.request_id,
                 )
 
                 batch_num += 1

backend/onyx/chat/answer.py

@@ -19,7 +19,6 @@ from onyx.chat.models import StreamStopReason
 from onyx.chat.prompt_builder.answer_prompt_builder import AnswerPromptBuilder
 from onyx.configs.agent_configs import AGENT_ALLOW_REFINEMENT
 from onyx.configs.agent_configs import INITIAL_SEARCH_DECOMPOSITION_ENABLED
-from onyx.configs.agent_configs import TF_DR_DEFAULT_FAST
 from onyx.context.search.models import RerankingDetails
 from onyx.db.kg_config import get_kg_config_settings
 from onyx.db.models import Persona
@@ -62,6 +61,7 @@ class Answer:
         use_agentic_search: bool = False,
         research_type: ResearchType | None = None,
         research_plan: dict[str, Any] | None = None,
+        project_instructions: str | None = None,
     ) -> None:
         self.is_connected: Callable[[], bool] | None = is_connected
         self._processed_stream: list[AnswerStreamPart] | None = None
@@ -97,6 +97,7 @@ class Answer:
             prompt_builder=prompt_builder,
             files=latest_query_files,
             structured_response_format=answer_style_config.structured_response_format,
+            project_instructions=project_instructions,
         )
         self.graph_tooling = GraphTooling(
             primary_llm=llm,
@@ -111,14 +112,6 @@ class Answer:
             chat_session_id=chat_session_id,
             message_id=current_agent_message_id,
         )
-
-        if use_agentic_search:
-            research_type = ResearchType.DEEP
-        elif TF_DR_DEFAULT_FAST:
-            research_type = ResearchType.FAST
-        else:
-            research_type = ResearchType.THOUGHTFUL
-
         self.search_behavior_config = GraphSearchConfig(
             use_agentic_search=use_agentic_search,
             skip_gen_ai_answer_generation=skip_gen_ai_answer_generation,
@@ -126,7 +119,9 @@ class Answer:
             allow_agent_reranking=allow_agent_reranking,
             perform_initial_search_decomposition=INITIAL_SEARCH_DECOMPOSITION_ENABLED,
             kg_config_settings=get_kg_config_settings(),
-            research_type=research_type,
+            research_type=(
+                ResearchType.DEEP if use_agentic_search else ResearchType.THOUGHTFUL
+            ),
         )
         self.graph_config = GraphConfig(
             inputs=self.graph_inputs,

backend/onyx/chat/chat_utils.py

@@ -32,15 +32,16 @@ from onyx.db.llm import fetch_existing_doc_sets
 from onyx.db.llm import fetch_existing_tools
 from onyx.db.models import ChatMessage
 from onyx.db.models import Persona
+from onyx.db.models import Prompt
 from onyx.db.models import Tool
 from onyx.db.models import User
+from onyx.db.prompts import get_prompts_by_ids
 from onyx.db.search_settings import get_current_search_settings
 from onyx.kg.models import KGException
 from onyx.kg.setup.kg_default_entity_definitions import (
     populate_missing_default_entity_types__commit,
 )
 from onyx.llm.models import PreviousMessage
-from onyx.llm.override_models import LLMOverride
 from onyx.natural_language_processing.utils import BaseTokenizer
 from onyx.server.query_and_chat.models import CreateChatMessageRequest
 from onyx.server.query_and_chat.streaming_models import CitationInfo
@@ -58,14 +59,13 @@ def prepare_chat_message_request(
     persona_id: int | None,
     # Does the question need to have a persona override
     persona_override_config: PersonaOverrideConfig | None,
+    prompt: Prompt | None,
     message_ts_to_respond_to: str | None,
     retrieval_details: RetrievalDetails | None,
     rerank_settings: RerankingDetails | None,
     db_session: Session,
     use_agentic_search: bool = False,
     skip_gen_ai_answer_generation: bool = False,
-    llm_override: LLMOverride | None = None,
-    allowed_tool_ids: list[int] | None = None,
 ) -> CreateChatMessageRequest:
     # Typically used for one shot flows like SlackBot or non-chat API endpoint use cases
     new_chat_session = create_chat_session(
@@ -83,6 +83,7 @@ def prepare_chat_message_request(
         parent_message_id=None,  # It's a standalone chat session each time
         message=message_text,
         file_descriptors=[],  # Currently SlackBot/answer api do not support files in the context
+        prompt_id=prompt.id if prompt else None,
         # Can always override the persona for the single query, if it's a normal persona
         # then it will be treated the same
         persona_override_config=persona_override_config,
@@ -91,8 +92,6 @@ def prepare_chat_message_request(
         rerank_settings=rerank_settings,
         use_agentic_search=use_agentic_search,
         skip_gen_ai_answer_generation=skip_gen_ai_answer_generation,
-        llm_override=llm_override,
-        allowed_tool_ids=allowed_tool_ids,
     )
 
 
@@ -390,11 +389,20 @@ def create_temporary_persona(
     )
 
     if persona_config.prompts:
-        # Use the first prompt from the override config for embedded prompt fields
-        first_prompt = persona_config.prompts[0]
-        persona.system_prompt = first_prompt.system_prompt
-        persona.task_prompt = first_prompt.task_prompt
-        persona.datetime_aware = first_prompt.datetime_aware
+        persona.prompts = [
+            Prompt(
+                name=p.name,
+                description=p.description,
+                system_prompt=p.system_prompt,
+                task_prompt=p.task_prompt,
+                datetime_aware=p.datetime_aware,
+            )
+            for p in persona_config.prompts
+        ]
+    elif persona_config.prompt_ids:
+        persona.prompts = get_prompts_by_ids(
+            db_session=db_session, prompt_ids=persona_config.prompt_ids
+        )
 
     persona.tools = []
     if persona_config.custom_tools_openapi:

backend/onyx/chat/models.py

@@ -30,7 +30,7 @@ from onyx.tools.models import ToolResponse
 from onyx.tools.tool_implementations.custom.base_tool_types import ToolResultType
 
 if TYPE_CHECKING:
-    from onyx.db.models import Persona
+    from onyx.db.models import Prompt
 
 
 class LlmDoc(BaseModel):
@@ -171,7 +171,6 @@ class PromptOverrideConfig(BaseModel):
     system_prompt: str
     task_prompt: str = ""
     datetime_aware: bool = True
-    include_citations: bool = True
 
 
 class PersonaOverrideConfig(BaseModel):
@@ -186,7 +185,7 @@ class PersonaOverrideConfig(BaseModel):
     llm_model_version_override: str | None = None
 
     prompts: list[PromptOverrideConfig] = Field(default_factory=list)
-    # Note: prompt_ids removed - prompts are now embedded in personas
+    prompt_ids: list[int] = Field(default_factory=list)
 
     document_set_ids: list[int] = Field(default_factory=list)
     tools: list[ToolConfig] = Field(default_factory=list)
@@ -271,7 +270,7 @@ class PromptConfig(BaseModel):
 
     @classmethod
     def from_model(
-        cls, model: "Persona", prompt_override: PromptOverride | None = None
+        cls, model: "Prompt", prompt_override: PromptOverride | None = None
     ) -> "PromptConfig":
         override_system_prompt = (
             prompt_override.system_prompt if prompt_override else None
@@ -279,8 +278,8 @@ class PromptConfig(BaseModel):
         override_task_prompt = prompt_override.task_prompt if prompt_override else None
 
         return cls(
-            system_prompt=override_system_prompt or model.system_prompt or "",
-            task_prompt=override_task_prompt or model.task_prompt or "",
+            system_prompt=override_system_prompt or model.system_prompt,
+            task_prompt=override_task_prompt or model.task_prompt,
             datetime_aware=model.datetime_aware,
         )
 

backend/onyx/chat/process_message.py

@@ -63,6 +63,7 @@ from onyx.db.models import SearchDoc as DbSearchDoc
 from onyx.db.models import ToolCall
 from onyx.db.models import User
 from onyx.db.persona import get_persona_by_id
+from onyx.db.projects import get_project_instructions
 from onyx.db.search_settings import get_current_search_settings
 from onyx.document_index.factory import get_default_document_index
 from onyx.file_store.models import FileDescriptor
@@ -88,12 +89,12 @@ from onyx.tools.tool import Tool
 from onyx.tools.tool_constructor import construct_tools
 from onyx.tools.tool_constructor import CustomToolConfig
 from onyx.tools.tool_constructor import ImageGenerationToolConfig
+from onyx.tools.tool_constructor import InternetSearchToolConfig
 from onyx.tools.tool_constructor import SearchToolConfig
-from onyx.tools.tool_constructor import WebSearchToolConfig
-from onyx.tools.tool_implementations.search.search_tool import SearchTool
-from onyx.tools.tool_implementations.web_search.web_search_tool import (
-    WebSearchTool,
+from onyx.tools.tool_implementations.internet_search.internet_search_tool import (
+    InternetSearchTool,
 )
+from onyx.tools.tool_implementations.search.search_tool import SearchTool
 from onyx.utils.logger import setup_logger
 from onyx.utils.long_term_log import LongTermLogger
 from onyx.utils.telemetry import mt_cloud_telemetry
@@ -159,10 +160,12 @@ def _get_force_search_settings(
             override_kwargs=search_tool_override_kwargs,
         )
 
-    web_search_available = any(isinstance(tool, WebSearchTool) for tool in tools)
+    internet_search_available = any(
+        isinstance(tool, InternetSearchTool) for tool in tools
+    )
     search_tool_available = any(isinstance(tool, SearchTool) for tool in tools)
 
-    if not web_search_available and not search_tool_available:
+    if not internet_search_available and not search_tool_available:
         # Does not matter much which tool is set here as force is false and neither tool is available
         return ForceUseTool(force_use=False, tool_name=SearchTool._NAME)
     # Currently, the internet search tool does not support query override
@@ -197,7 +200,9 @@ def _get_force_search_settings(
 
     return ForceUseTool(
         force_use=False,
-        tool_name=(SearchTool._NAME if search_tool_available else WebSearchTool._NAME),
+        tool_name=(
+            SearchTool._NAME if search_tool_available else InternetSearchTool._NAME
+        ),
         args=args,
         override_kwargs=None,
     )
@@ -331,8 +336,11 @@ def stream_chat_message_objects(
                 properties=None,
             )
 
-        # Note: prompt configuration is now embedded in the persona
-        # No need for separate prompt_id handling
+        # If a prompt override is specified via the API, use that with highest priority
+        # but for saving it, we are just mapping it to an existing prompt
+        prompt_id = new_msg_req.prompt_id
+        if prompt_id is None and persona.prompts:
+            prompt_id = sorted(persona.prompts, key=lambda x: x.id)[-1].id
 
         if reference_doc_ids is None and retrieval_options is None:
             raise RuntimeError(
@@ -392,6 +400,7 @@ def stream_chat_message_objects(
             user_message = create_new_chat_message(
                 chat_session_id=chat_session_id,
                 parent_message=parent_message,
+                prompt_id=prompt_id,
                 message=message_text,
                 token_count=len(llm_tokenizer_encode_func(message_text)),
                 message_type=MessageType.USER,
@@ -436,26 +445,27 @@ def stream_chat_message_objects(
         files = load_all_chat_files(history_msgs, new_msg_req.file_descriptors)
         req_file_ids = [f["id"] for f in new_msg_req.file_descriptors]
         latest_query_files = [file for file in files if file.file_id in req_file_ids]
-        user_file_ids = new_msg_req.user_file_ids or []
-        user_folder_ids = new_msg_req.user_folder_ids or []
+        user_file_ids = []
 
         if persona.user_files:
             for file in persona.user_files:
                 user_file_ids.append(file.id)
-        if persona.user_folders:
-            for folder in persona.user_folders:
-                user_folder_ids.append(folder.id)
+
+        if new_msg_req.current_message_files:
+            for file in new_msg_req.current_message_files:
+                if file["user_file_id"]:
+                    user_file_ids.append(file["user_file_id"])
 
         # Load in user files into memory and create search tool override kwargs if needed
-        # if we have enough tokens and no folders, we don't need to use search
+        # if we have enough tokens, we don't need to use search
         # we can just pass them into the prompt directly
         (
             in_memory_user_files,
             user_file_models,
             search_tool_override_kwargs_for_user_files,
         ) = parse_user_files(
-            user_file_ids=user_file_ids,
-            user_folder_ids=user_folder_ids,
+            user_file_ids=user_file_ids or [],
+            project_id=chat_session.project_id,
             db_session=db_session,
             persona=persona,
             actual_user_input=message_text,
@@ -551,13 +561,27 @@ def stream_chat_message_objects(
                 ].datetime_aware,
             )
         elif prompt_override:
-            # Apply prompt override on top of persona-embedded prompt
+            if not final_msg.prompt:
+                raise ValueError(
+                    "Prompt override cannot be applied, no base prompt found."
+                )
             prompt_config = PromptConfig.from_model(
-                persona,
+                final_msg.prompt,
                 prompt_override=prompt_override,
             )
         else:
-            prompt_config = PromptConfig.from_model(persona)
+            prompt_config = PromptConfig.from_model(
+                final_msg.prompt or persona.prompts[0]
+            )
+
+        # Retrieve project-specific instructions if this chat session is associated with a project.
+        project_instructions: str | None = (
+            get_project_instructions(
+                db_session=db_session, project_id=chat_session.project_id
+            )
+            if persona.is_default_persona
+            else None
+        )  # if the persona is not default, we don't want to use the project instructions
 
         answer_style_config = AnswerStyleConfig(
             citation_config=CitationConfig(
@@ -590,7 +614,7 @@ def stream_chat_message_objects(
                 latest_query_files=latest_query_files,
                 bypass_acl=bypass_acl,
             ),
-            internet_search_tool_config=WebSearchToolConfig(
+            internet_search_tool_config=InternetSearchToolConfig(
                 answer_style_config=answer_style_config,
                 document_pruning_config=document_pruning_config,
             ),
@@ -670,6 +694,7 @@ def stream_chat_message_objects(
             db_session=db_session,
             use_agentic_search=new_msg_req.use_agentic_search,
             skip_gen_ai_answer_generation=new_msg_req.skip_gen_ai_answer_generation,
+            project_instructions=project_instructions,
         )
 
         # Process streamed packets using the new packet processing module

backend/onyx/chat/prompt_builder/answer_prompt_builder.py

@@ -76,6 +76,7 @@ def default_build_user_message(
         if prompt_config.task_prompt
         else user_query
     )
+
     user_prompt = user_prompt.strip()
     tag_handled_prompt = handle_onyx_date_awareness(user_prompt, prompt_config)
     user_msg = HumanMessage(

backend/onyx/chat/prompt_builder/citations_prompt.py

@@ -1,11 +1,13 @@
 from langchain.schema.messages import HumanMessage
 from langchain.schema.messages import SystemMessage
+from sqlalchemy.orm import Session
 
 from onyx.chat.models import LlmDoc
 from onyx.chat.models import PromptConfig
 from onyx.configs.model_configs import GEN_AI_SINGLE_USER_MESSAGE_EXPECTED_MAX_TOKENS
 from onyx.context.search.models import InferenceChunk
 from onyx.db.models import Persona
+from onyx.db.prompts import get_default_prompt
 from onyx.db.search_settings import get_multilingual_expansion
 from onyx.file_store.models import InMemoryChatFile
 from onyx.llm.factory import get_llms_for_persona
@@ -87,12 +89,13 @@ def compute_max_document_tokens(
 
 
 def compute_max_document_tokens_for_persona(
+    db_session: Session,
     persona: Persona,
     actual_user_input: str | None = None,
 ) -> int:
-    # Use the persona directly since prompts are now embedded
+    prompt = persona.prompts[0] if persona.prompts else get_default_prompt(db_session)
     return compute_max_document_tokens(
-        prompt_config=PromptConfig.from_model(persona),
+        prompt_config=PromptConfig.from_model(prompt),
         llm_config=get_main_llm_from_tuple(get_llms_for_persona(persona)).config,
         actual_user_input=actual_user_input,
     )

backend/onyx/chat/user_files/parse_user_files.py

@@ -4,6 +4,8 @@ from sqlalchemy.orm import Session
 
 from onyx.db.models import Persona
 from onyx.db.models import UserFile
+from onyx.db.projects import get_user_files_from_project
+from onyx.db.user_file import update_last_accessed_at_for_user_files
 from onyx.file_store.models import InMemoryChatFile
 from onyx.file_store.utils import get_user_files_as_user
 from onyx.file_store.utils import load_in_memory_chat_files
@@ -15,24 +17,24 @@ logger = setup_logger()
 
 
 def parse_user_files(
-    user_file_ids: list[int],
-    user_folder_ids: list[int],
+    user_file_ids: list[UUID],
     db_session: Session,
     persona: Persona,
     actual_user_input: str,
+    project_id: int,
     # should only be None if auth is disabled
     user_id: UUID | None,
 ) -> tuple[list[InMemoryChatFile], list[UserFile], SearchToolOverrideKwargs | None]:
     """
-    Parse user files and folders into in-memory chat files and create search tool override kwargs.
-    Only creates SearchToolOverrideKwargs if token overflow occurs or folders are present.
+    Parse user files and project into in-memory chat files and create search tool override kwargs.
+    Only creates SearchToolOverrideKwargs if token overflow occurs.
 
     Args:
         user_file_ids: List of user file IDs to load
-        user_folder_ids: List of user folder IDs to load
         db_session: Database session
         persona: Persona to calculate available tokens
         actual_user_input: User's input message for token calculation
+        project_id: Project ID to validate file ownership
         user_id: User ID to validate file ownership
 
     Returns:
@@ -40,42 +42,57 @@ def parse_user_files(
             loaded user files,
             user file models,
             search tool override kwargs if token
-                overflow or folders present
+                overflow
         )
     """
-    # Return empty results if no files or folders specified
-    if not user_file_ids and not user_folder_ids:
+    # Return empty results if no files or project specified
+    if not user_file_ids and not project_id:
         return [], [], None
 
+    project_user_file_ids = []
+
+    if project_id and user_id:
+        project_user_file_ids.extend(
+            [
+                file.id
+                for file in get_user_files_from_project(project_id, user_id, db_session)
+            ]
+        )
+
     # Load user files from the database into memory
     user_files = load_in_memory_chat_files(
-        user_file_ids or [],
-        user_folder_ids or [],
+        user_file_ids + project_user_file_ids or [],
         db_session,
     )
 
     user_file_models = get_user_files_as_user(
-        user_file_ids or [],
-        user_folder_ids or [],
+        user_file_ids + project_user_file_ids or [],
         user_id,
         db_session,
     )
 
+    # Update last accessed at for the user files which are used in the chat
+    if user_file_ids or project_user_file_ids:
+        update_last_accessed_at_for_user_files(
+            user_file_ids + project_user_file_ids or [],
+            db_session,
+        )
+
     # Calculate token count for the files, need to import here to avoid circular import
     # TODO: fix this
-    from onyx.db.user_documents import calculate_user_files_token_count
+    from onyx.db.user_file import calculate_user_files_token_count
     from onyx.chat.prompt_builder.citations_prompt import (
         compute_max_document_tokens_for_persona,
     )
 
     total_tokens = calculate_user_files_token_count(
-        user_file_ids or [],
-        user_folder_ids or [],
+        user_file_ids + project_user_file_ids or [],
         db_session,
     )
 
     # Calculate available tokens for documents based on prompt, user input, etc.
     available_tokens = compute_max_document_tokens_for_persona(
+        db_session=db_session,
         persona=persona,
         actual_user_input=actual_user_input,
     )
@@ -86,20 +103,22 @@ def parse_user_files(
 
     have_enough_tokens = total_tokens <= available_tokens
 
-    # If we have enough tokens and no folders, we don't need search
+    # If we have enough tokens, we don't need search
     # we can just pass them into the prompt directly
-    if have_enough_tokens and not user_folder_ids:
+    if have_enough_tokens:
         # No search tool override needed - files can be passed directly
         return user_files, user_file_models, None
 
-    # Token overflow or folders present - need to use search tool
+    # Token overflow - need to use search tool
     override_kwargs = SearchToolOverrideKwargs(
         force_no_rerank=have_enough_tokens,
         alternate_db_session=None,
         retrieved_sections_callback=None,
         skip_query_analysis=have_enough_tokens,
-        user_file_ids=user_file_ids,
-        user_folder_ids=user_folder_ids,
+        user_file_ids=user_file_ids or [],
+        project_id=(
+            project_id if persona.is_default_persona else None
+        ),  # if the persona is not default, we don't want to use the project files
     )
 
     return user_files, user_file_models, override_kwargs

backend/onyx/configs/agent_configs.py

@@ -379,11 +379,4 @@ AGENT_MAX_TOKENS_HISTORY_SUMMARY = int(
     or AGENT_DEFAULT_MAX_TOKENS_HISTORY_SUMMARY
 )
 
-# Parameters for the Thoughtful/Deep Research flows
-TF_DR_TIMEOUT_LONG = int(os.environ.get("TF_DR_TIMEOUT_LONG") or 120)
-TF_DR_TIMEOUT_SHORT = int(os.environ.get("TF_DR_TIMEOUT_SHORT") or 60)
-
-
-TF_DR_DEFAULT_FAST = (os.environ.get("TF_DR_DEFAULT_FAST") or "False").lower() == "true"
-
 GRAPH_VERSION_NAME: str = "a"

backend/onyx/configs/app_configs.py

@@ -355,13 +355,18 @@ CELERY_WORKER_KG_PROCESSING_CONCURRENCY = int(
     os.environ.get("CELERY_WORKER_KG_PROCESSING_CONCURRENCY") or 4
 )
 
-CELERY_WORKER_PRIMARY_CONCURRENCY = int(
-    os.environ.get("CELERY_WORKER_PRIMARY_CONCURRENCY") or 4
-)
-
-CELERY_WORKER_PRIMARY_POOL_OVERFLOW = int(
-    os.environ.get("CELERY_WORKER_PRIMARY_POOL_OVERFLOW") or 4
-)
+CELERY_WORKER_USER_FILE_PROCESSING_CONCURRENCY_DEFAULT = 4
+try:
+    CELERY_WORKER_USER_FILE_PROCESSING_CONCURRENCY = int(
+        os.environ.get(
+            "CELERY_WORKER_USER_FILE_PROCESSING_CONCURRENCY",
+            CELERY_WORKER_USER_FILE_PROCESSING_CONCURRENCY_DEFAULT,
+        )
+    )
+except ValueError:
+    CELERY_WORKER_USER_FILE_PROCESSING_CONCURRENCY = (
+        CELERY_WORKER_USER_FILE_PROCESSING_CONCURRENCY_DEFAULT
+    )
 
 # The maximum number of tasks that can be queued up to sync to Vespa in a single pass
 VESPA_SYNC_MAX_TASKS = 8192
@@ -685,17 +690,6 @@ LOG_POSTGRES_CONN_COUNTS = (
 # Anonymous usage telemetry
 DISABLE_TELEMETRY = os.environ.get("DISABLE_TELEMETRY", "").lower() == "true"
 
-#####
-# Braintrust Configuration
-#####
-# Enable Braintrust tracing for LangGraph/LangChain applications
-BRAINTRUST_ENABLED = os.environ.get("BRAINTRUST_ENABLED", "").lower() == "true"
-# Braintrust project name
-BRAINTRUST_PROJECT = os.environ.get("BRAINTRUST_PROJECT", "Onyx")
-BRAINTRUST_API_KEY = os.environ.get("BRAINTRUST_API_KEY") or ""
-# Maximum concurrency for Braintrust evaluations
-BRAINTRUST_MAX_CONCURRENCY = int(os.environ.get("BRAINTRUST_MAX_CONCURRENCY") or 5)
-
 TOKEN_BUDGET_GLOBALLY_ENABLED = (
     os.environ.get("TOKEN_BUDGET_GLOBALLY_ENABLED", "").lower() == "true"
 )
@@ -758,24 +752,6 @@ AZURE_DALLE_DEPLOYMENT_NAME = os.environ.get("AZURE_DALLE_DEPLOYMENT_NAME")
 # configurable image model
 IMAGE_MODEL_NAME = os.environ.get("IMAGE_MODEL_NAME", "gpt-image-1")
 
-CODE_INTERPRETER_BASE_URL = os.environ.get("CODE_INTERPRETER_BASE_URL")
-_CODE_INTERPRETER_DEFAULT_TIMEOUT_MS_RAW = os.environ.get(
-    "CODE_INTERPRETER_DEFAULT_TIMEOUT_MS"
-)
-CODE_INTERPRETER_DEFAULT_TIMEOUT_MS = (
-    int(_CODE_INTERPRETER_DEFAULT_TIMEOUT_MS_RAW)
-    if _CODE_INTERPRETER_DEFAULT_TIMEOUT_MS_RAW
-    else 30_000
-)
-_CODE_INTERPRETER_REQUEST_TIMEOUT_SECONDS_RAW = os.environ.get(
-    "CODE_INTERPRETER_REQUEST_TIMEOUT_SECONDS"
-)
-CODE_INTERPRETER_REQUEST_TIMEOUT_SECONDS = (
-    int(_CODE_INTERPRETER_REQUEST_TIMEOUT_SECONDS_RAW)
-    if _CODE_INTERPRETER_REQUEST_TIMEOUT_SECONDS_RAW
-    else 30
-)
-
 # Use managed Vespa (Vespa Cloud). If set, must also set VESPA_CLOUD_URL, VESPA_CLOUD_CERT_PATH and VESPA_CLOUD_KEY_PATH
 MANAGED_VESPA = os.environ.get("MANAGED_VESPA", "").lower() == "true"
 
@@ -879,12 +855,12 @@ S3_FILE_STORE_BUCKET_NAME = (
 )
 S3_FILE_STORE_PREFIX = os.environ.get("S3_FILE_STORE_PREFIX") or "onyx-files"
 # S3_ENDPOINT_URL is for MinIO and other S3-compatible storage. Leave blank for AWS S3.
-S3_ENDPOINT_URL = os.environ.get("S3_ENDPOINT_URL")
+S3_ENDPOINT_URL = os.environ.get("S3_ENDPOINT_URL") or "http://localhost:9004"
 S3_VERIFY_SSL = os.environ.get("S3_VERIFY_SSL", "").lower() == "true"
 
 # S3/MinIO Access Keys
-S3_AWS_ACCESS_KEY_ID = os.environ.get("S3_AWS_ACCESS_KEY_ID")
-S3_AWS_SECRET_ACCESS_KEY = os.environ.get("S3_AWS_SECRET_ACCESS_KEY")
+S3_AWS_ACCESS_KEY_ID = os.environ.get("S3_AWS_ACCESS_KEY_ID") or "minioadmin"
+S3_AWS_SECRET_ACCESS_KEY = os.environ.get("S3_AWS_SECRET_ACCESS_KEY") or "minioadmin"
 
 # Forcing Vespa Language
 # English: en, German:de, etc. See: https://docs.vespa.ai/en/linguistics.html

backend/onyx/configs/chat_configs.py

@@ -3,7 +3,6 @@ import os
 INPUT_PROMPT_YAML = "./onyx/seeding/input_prompts.yaml"
 PROMPTS_YAML = "./onyx/seeding/prompts.yaml"
 PERSONAS_YAML = "./onyx/seeding/personas.yaml"
-USER_FOLDERS_YAML = "./onyx/seeding/user_folders.yaml"
 NUM_RETURNED_HITS = 50
 # Used for LLM filtering and reranking
 # We want this to be approximately the number of results we want to show on the first page

backend/onyx/configs/constants.py

@@ -76,6 +76,9 @@ POSTGRES_CELERY_WORKER_DOCFETCHING_APP_NAME = "celery_worker_docfetching"
 POSTGRES_CELERY_WORKER_MONITORING_APP_NAME = "celery_worker_monitoring"
 POSTGRES_CELERY_WORKER_INDEXING_CHILD_APP_NAME = "celery_worker_indexing_child"
 POSTGRES_CELERY_WORKER_KG_PROCESSING_APP_NAME = "celery_worker_kg_processing"
+POSTGRES_CELERY_WORKER_USER_FILE_PROCESSING_APP_NAME = (
+    "celery_worker_user_file_processing"
+)
 POSTGRES_PERMISSIONS_APP_NAME = "permissions"
 POSTGRES_UNKNOWN_APP_NAME = "unknown"
 
@@ -112,7 +115,6 @@ CELERY_GENERIC_BEAT_LOCK_TIMEOUT = 120
 
 CELERY_VESPA_SYNC_BEAT_LOCK_TIMEOUT = 120
 
-CELERY_USER_FILE_FOLDER_SYNC_BEAT_LOCK_TIMEOUT = 120
 
 CELERY_PRIMARY_WORKER_LOCK_TIMEOUT = 120
 
@@ -199,7 +201,6 @@ class DocumentSource(str, Enum):
     HIGHSPOT = "highspot"
 
     IMAP = "imap"
-    BITBUCKET = "bitbucket"
 
     # Special case just for integration tests
     MOCK_CONNECTOR = "mock_connector"
@@ -343,6 +344,9 @@ class OnyxCeleryQueues:
     # Indexing queue
     USER_FILES_INDEXING = "user_files_indexing"
 
+    # User file processing queue
+    USER_FILE_PROCESSING = "user_file_processing"
+
     # Document processing pipeline queue
     DOCPROCESSING = "docprocessing"
     CONNECTOR_DOC_FETCHING = "connector_doc_fetching"
@@ -368,7 +372,7 @@ class OnyxRedisLocks:
     CHECK_CONNECTOR_EXTERNAL_GROUP_SYNC_BEAT_LOCK = (
         "da_lock:check_connector_external_group_sync_beat"
     )
-    CHECK_USER_FILE_FOLDER_SYNC_BEAT_LOCK = "da_lock:check_user_file_folder_sync_beat"
+
     MONITOR_BACKGROUND_PROCESSES_LOCK = "da_lock:monitor_background_processes"
     CHECK_AVAILABLE_TENANTS_LOCK = "da_lock:check_available_tenants"
     CLOUD_PRE_PROVISION_TENANT_LOCK = "da_lock:pre_provision_tenant"
@@ -390,6 +394,10 @@ class OnyxRedisLocks:
     # KG processing
     KG_PROCESSING_LOCK = "da_lock:kg_processing"
 
+    # User file processing
+    USER_FILE_PROCESSING_BEAT_LOCK = "da_lock:check_user_file_processing_beat"
+    USER_FILE_PROCESSING_LOCK_PREFIX = "da_lock:user_file_processing"
+
 
 class OnyxRedisSignals:
     BLOCK_VALIDATE_INDEXING_FENCES = "signal:block_validate_indexing_fences"
@@ -448,8 +456,6 @@ class OnyxCeleryTask:
         f"{ONYX_CLOUD_CELERY_TASK_PREFIX}_monitor_celery_pidbox"
     )
 
-    UPDATE_USER_FILE_FOLDER_METADATA = "update_user_file_folder_metadata"
-
     CHECK_FOR_CONNECTOR_DELETION = "check_for_connector_deletion_task"
     CHECK_FOR_VESPA_SYNC_TASK = "check_for_vespa_sync_task"
     CHECK_FOR_INDEXING = "check_for_indexing"
@@ -457,8 +463,10 @@ class OnyxCeleryTask:
     CHECK_FOR_DOC_PERMISSIONS_SYNC = "check_for_doc_permissions_sync"
     CHECK_FOR_EXTERNAL_GROUP_SYNC = "check_for_external_group_sync"
     CHECK_FOR_LLM_MODEL_UPDATE = "check_for_llm_model_update"
-    CHECK_FOR_USER_FILE_FOLDER_SYNC = "check_for_user_file_folder_sync"
 
+    # User file processing
+    CHECK_FOR_USER_FILE_PROCESSING = "check_for_user_file_processing"
+    PROCESS_SINGLE_USER_FILE = "process_single_user_file"
     # Connector checkpoint cleanup
     CHECK_FOR_CHECKPOINT_CLEANUP = "check_for_checkpoint_cleanup"
     CLEANUP_CHECKPOINT = "cleanup_checkpoint"
@@ -497,8 +505,6 @@ class OnyxCeleryTask:
 
     GENERATE_USAGE_REPORT_TASK = "generate_usage_report_task"
 
-    EVAL_RUN_TASK = "eval_run_task"
-
     EXPORT_QUERY_HISTORY_TASK = "export_query_history_task"
     EXPORT_QUERY_HISTORY_CLEANUP_TASK = "export_query_history_cleanup_task"
 
@@ -544,7 +550,6 @@ DocumentSourceDescription: dict[DocumentSource, str] = {
     DocumentSource.GITHUB: "github data (issues, PRs)",
     DocumentSource.GITBOOK: "gitbook data",
     DocumentSource.GITLAB: "gitlab data",
-    DocumentSource.BITBUCKET: "bitbucket data",
     DocumentSource.GURU: "guru data",
     DocumentSource.BOOKSTACK: "bookstack data",
     DocumentSource.OUTLINE: "outline data",

backend/onyx/connectors/bitbucket/connector.py

@@ -1,345 +0,0 @@
-from __future__ import annotations
-
-import copy
-from collections.abc import Callable
-from collections.abc import Iterator
-from datetime import datetime
-from datetime import timezone
-from typing import Any
-from typing import TYPE_CHECKING
-
-from typing_extensions import override
-
-from onyx.configs.app_configs import INDEX_BATCH_SIZE
-from onyx.configs.app_configs import REQUEST_TIMEOUT_SECONDS
-from onyx.configs.constants import DocumentSource
-from onyx.connectors.bitbucket.utils import build_auth_client
-from onyx.connectors.bitbucket.utils import list_repositories
-from onyx.connectors.bitbucket.utils import map_pr_to_document
-from onyx.connectors.bitbucket.utils import paginate
-from onyx.connectors.bitbucket.utils import PR_LIST_RESPONSE_FIELDS
-from onyx.connectors.bitbucket.utils import SLIM_PR_LIST_RESPONSE_FIELDS
-from onyx.connectors.exceptions import CredentialExpiredError
-from onyx.connectors.exceptions import InsufficientPermissionsError
-from onyx.connectors.exceptions import UnexpectedValidationError
-from onyx.connectors.interfaces import CheckpointedConnector
-from onyx.connectors.interfaces import CheckpointOutput
-from onyx.connectors.interfaces import SecondsSinceUnixEpoch
-from onyx.connectors.interfaces import SlimConnector
-from onyx.connectors.models import ConnectorCheckpoint
-from onyx.connectors.models import ConnectorFailure
-from onyx.connectors.models import ConnectorMissingCredentialError
-from onyx.connectors.models import DocumentFailure
-from onyx.connectors.models import SlimDocument
-from onyx.indexing.indexing_heartbeat import IndexingHeartbeatInterface
-from onyx.utils.logger import setup_logger
-
-if TYPE_CHECKING:
-    import httpx
-
-logger = setup_logger()
-
-
-class BitbucketConnectorCheckpoint(ConnectorCheckpoint):
-    """Checkpoint state for resumable Bitbucket PR indexing.
-
-    Fields:
-        repos_queue: Materialized list of repository slugs to process.
-        current_repo_index: Index of the repository currently being processed.
-        next_url: Bitbucket "next" URL for continuing pagination within the current repo.
-    """
-
-    repos_queue: list[str] = []
-    current_repo_index: int = 0
-    next_url: str | None = None
-
-
-class BitbucketConnector(
-    CheckpointedConnector[BitbucketConnectorCheckpoint],
-    SlimConnector,
-):
-    """Connector for indexing Bitbucket Cloud pull requests.
-
-    Args:
-        workspace: Bitbucket workspace ID.
-        repositories: Comma-separated list of repository slugs to index.
-        projects: Comma-separated list of project keys to index all repositories within.
-        batch_size: Max number of documents to yield per batch.
-    """
-
-    def __init__(
-        self,
-        workspace: str,
-        repositories: str | None = None,
-        projects: str | None = None,
-        batch_size: int = INDEX_BATCH_SIZE,
-    ) -> None:
-        self.workspace = workspace
-        self._repositories = (
-            [s.strip() for s in repositories.split(",") if s.strip()]
-            if repositories
-            else None
-        )
-        self._projects: list[str] | None = (
-            [s.strip() for s in projects.split(",") if s.strip()] if projects else None
-        )
-        self.batch_size = batch_size
-        self.email: str | None = None
-        self.api_token: str | None = None
-
-    def load_credentials(self, credentials: dict[str, Any]) -> dict[str, Any] | None:
-        """Load API token-based credentials.
-
-        Expects a dict with keys: `bitbucket_email`, `bitbucket_api_token`.
-        """
-        self.email = credentials.get("bitbucket_email")
-        self.api_token = credentials.get("bitbucket_api_token")
-        if not self.email or not self.api_token:
-            raise ConnectorMissingCredentialError("Bitbucket")
-        return None
-
-    def _client(self) -> httpx.Client:
-        """Build an authenticated HTTP client or raise if credentials missing."""
-        if not self.email or not self.api_token:
-            raise ConnectorMissingCredentialError("Bitbucket")
-        return build_auth_client(self.email, self.api_token)
-
-    def _iter_pull_requests_for_repo(
-        self,
-        client: httpx.Client,
-        repo_slug: str,
-        params: dict[str, Any] | None = None,
-        start_url: str | None = None,
-        on_page: Callable[[str | None], None] | None = None,
-    ) -> Iterator[dict[str, Any]]:
-        base = f"https://api.bitbucket.org/2.0/repositories/{self.workspace}/{repo_slug}/pullrequests"
-        yield from paginate(
-            client,
-            base,
-            params,
-            start_url=start_url,
-            on_page=on_page,
-        )
-
-    def _build_params(
-        self,
-        fields: str = PR_LIST_RESPONSE_FIELDS,
-        start: SecondsSinceUnixEpoch | None = None,
-        end: SecondsSinceUnixEpoch | None = None,
-    ) -> dict[str, Any]:
-        """Build Bitbucket fetch params.
-
-        Always include OPEN, MERGED, and DECLINED PRs. If both ``start`` and
-        ``end`` are provided, apply a single updated_on time window.
-        """
-
-        def _iso(ts: SecondsSinceUnixEpoch) -> str:
-            return datetime.fromtimestamp(ts, tz=timezone.utc).isoformat()
-
-        def _tc_epoch(
-            lower_epoch: SecondsSinceUnixEpoch | None,
-            upper_epoch: SecondsSinceUnixEpoch | None,
-        ) -> str | None:
-            if lower_epoch is not None and upper_epoch is not None:
-                lower_iso = _iso(lower_epoch)
-                upper_iso = _iso(upper_epoch)
-                return f'(updated_on >= "{lower_iso}" AND updated_on <= "{upper_iso}")'
-            return None
-
-        params: dict[str, Any] = {"fields": fields, "pagelen": 50}
-        time_clause = _tc_epoch(start, end)
-        q = '(state = "OPEN" OR state = "MERGED" OR state = "DECLINED")'
-        if time_clause:
-            q = f"{q} AND {time_clause}"
-        params["q"] = q
-        return params
-
-    def _iter_target_repositories(self, client: httpx.Client) -> Iterator[str]:
-        """Yield repository slugs based on configuration.
-
-        Priority:
-        - repositories list
-        - projects list (list repos by project key)
-        - workspace (all repos)
-        """
-        if self._repositories:
-            for slug in self._repositories:
-                yield slug
-            return
-        if self._projects:
-            for project_key in self._projects:
-                for repo in list_repositories(client, self.workspace, project_key):
-                    slug_val = repo.get("slug")
-                    if isinstance(slug_val, str) and slug_val:
-                        yield slug_val
-            return
-        for repo in list_repositories(client, self.workspace, None):
-            slug_val = repo.get("slug")
-            if isinstance(slug_val, str) and slug_val:
-                yield slug_val
-
-    @override
-    def load_from_checkpoint(
-        self,
-        start: SecondsSinceUnixEpoch,
-        end: SecondsSinceUnixEpoch,
-        checkpoint: BitbucketConnectorCheckpoint,
-    ) -> CheckpointOutput[BitbucketConnectorCheckpoint]:
-        """Resumable PR ingestion across repos and pages within a time window.
-
-        Yields Documents (or ConnectorFailure for per-PR mapping failures) and returns
-        an updated checkpoint that records repo position and next page URL.
-        """
-        new_checkpoint = copy.deepcopy(checkpoint)
-
-        with self._client() as client:
-            # Materialize target repositories once
-            if not new_checkpoint.repos_queue:
-                # Preserve explicit order; otherwise ensure deterministic ordering
-                repos_list = list(self._iter_target_repositories(client))
-                new_checkpoint.repos_queue = sorted(set(repos_list))
-                new_checkpoint.current_repo_index = 0
-                new_checkpoint.next_url = None
-
-            repos = new_checkpoint.repos_queue
-            if not repos or new_checkpoint.current_repo_index >= len(repos):
-                new_checkpoint.has_more = False
-                return new_checkpoint
-
-            repo_slug = repos[new_checkpoint.current_repo_index]
-
-            first_page_params = self._build_params(
-                fields=PR_LIST_RESPONSE_FIELDS,
-                start=start,
-                end=end,
-            )
-
-            def _on_page(next_url: str | None) -> None:
-                new_checkpoint.next_url = next_url
-
-            for pr in self._iter_pull_requests_for_repo(
-                client,
-                repo_slug,
-                params=first_page_params,
-                start_url=new_checkpoint.next_url,
-                on_page=_on_page,
-            ):
-                try:
-                    document = map_pr_to_document(pr, self.workspace, repo_slug)
-                    yield document
-                except Exception as e:
-                    pr_id = pr.get("id")
-                    pr_link = (
-                        f"https://bitbucket.org/{self.workspace}/{repo_slug}/pull-requests/{pr_id}"
-                        if pr_id is not None
-                        else None
-                    )
-                    yield ConnectorFailure(
-                        failed_document=DocumentFailure(
-                            document_id=(
-                                f"{DocumentSource.BITBUCKET.value}:{self.workspace}:{repo_slug}:pr:{pr_id}"
-                                if pr_id is not None
-                                else f"{DocumentSource.BITBUCKET.value}:{self.workspace}:{repo_slug}:pr:unknown"
-                            ),
-                            document_link=pr_link,
-                        ),
-                        failure_message=f"Failed to process Bitbucket PR: {e}",
-                        exception=e,
-                    )
-
-            # Advance to next repository (if any) and set has_more accordingly
-            new_checkpoint.current_repo_index += 1
-            new_checkpoint.next_url = None
-            new_checkpoint.has_more = new_checkpoint.current_repo_index < len(repos)
-
-        return new_checkpoint
-
-    @override
-    def build_dummy_checkpoint(self) -> BitbucketConnectorCheckpoint:
-        """Create an initial checkpoint with work remaining."""
-        return BitbucketConnectorCheckpoint(has_more=True)
-
-    @override
-    def validate_checkpoint_json(
-        self, checkpoint_json: str
-    ) -> BitbucketConnectorCheckpoint:
-        """Validate and deserialize a checkpoint instance from JSON."""
-        return BitbucketConnectorCheckpoint.model_validate_json(checkpoint_json)
-
-    def retrieve_all_slim_documents(
-        self,
-        start: SecondsSinceUnixEpoch | None = None,
-        end: SecondsSinceUnixEpoch | None = None,
-        callback: IndexingHeartbeatInterface | None = None,
-    ) -> Iterator[list[SlimDocument]]:
-        """Return only document IDs for all existing pull requests."""
-        batch: list[SlimDocument] = []
-        params = self._build_params(
-            fields=SLIM_PR_LIST_RESPONSE_FIELDS,
-            start=start,
-            end=end,
-        )
-        with self._client() as client:
-            for slug in self._iter_target_repositories(client):
-                for pr in self._iter_pull_requests_for_repo(
-                    client, slug, params=params
-                ):
-                    pr_id = pr["id"]
-                    doc_id = f"{DocumentSource.BITBUCKET.value}:{self.workspace}:{slug}:pr:{pr_id}"
-                    batch.append(SlimDocument(id=doc_id))
-                    if len(batch) >= self.batch_size:
-                        yield batch
-                        batch = []
-                        if callback:
-                            if callback.should_stop():
-                                # Note: this is not actually used for permission sync yet, just pruning
-                                raise RuntimeError(
-                                    "bitbucket_pr_sync: Stop signal detected"
-                                )
-                            callback.progress("bitbucket_pr_sync", len(batch))
-        if batch:
-            yield batch
-
-    def validate_connector_settings(self) -> None:
-        """Validate Bitbucket credentials and workspace access by probing a lightweight endpoint.
-
-        Raises:
-            CredentialExpiredError: on HTTP 401
-            InsufficientPermissionsError: on HTTP 403
-            UnexpectedValidationError: on any other failure
-        """
-        try:
-            with self._client() as client:
-                url = f"https://api.bitbucket.org/2.0/repositories/{self.workspace}"
-                resp = client.get(
-                    url,
-                    params={"pagelen": 1, "fields": "pagelen"},
-                    timeout=REQUEST_TIMEOUT_SECONDS,
-                )
-                if resp.status_code == 401:
-                    raise CredentialExpiredError(
-                        "Invalid or expired Bitbucket credentials (HTTP 401)."
-                    )
-                if resp.status_code == 403:
-                    raise InsufficientPermissionsError(
-                        "Insufficient permissions to access Bitbucket workspace (HTTP 403)."
-                    )
-                if resp.status_code < 200 or resp.status_code >= 300:
-                    raise UnexpectedValidationError(
-                        f"Unexpected Bitbucket error (status={resp.status_code})."
-                    )
-        except Exception as e:
-            # Network or other unexpected errors
-            if isinstance(
-                e,
-                (
-                    CredentialExpiredError,
-                    InsufficientPermissionsError,
-                    UnexpectedValidationError,
-                    ConnectorMissingCredentialError,
-                ),
-            ):
-                raise
-            raise UnexpectedValidationError(
-                f"Unexpected error while validating Bitbucket settings: {e}"
-            )

backend/onyx/connectors/bitbucket/utils.py

@@ -1,294 +0,0 @@
-from __future__ import annotations
-
-import time
-from collections.abc import Callable
-from collections.abc import Iterator
-from datetime import datetime
-from datetime import timezone
-from typing import Any
-
-import httpx
-
-from onyx.configs.app_configs import REQUEST_TIMEOUT_SECONDS
-from onyx.configs.constants import DocumentSource
-from onyx.connectors.cross_connector_utils.rate_limit_wrapper import (
-    rate_limit_builder,
-)
-from onyx.connectors.models import BasicExpertInfo
-from onyx.connectors.models import Document
-from onyx.connectors.models import ImageSection
-from onyx.connectors.models import TextSection
-from onyx.utils.logger import setup_logger
-from onyx.utils.retry_wrapper import retry_builder
-
-logger = setup_logger()
-
-# Fields requested from Bitbucket PR list endpoint to ensure rich PR data
-PR_LIST_RESPONSE_FIELDS: str = ",".join(
-    [
-        "next",
-        "page",
-        "pagelen",
-        "values.author",
-        "values.close_source_branch",
-        "values.closed_by",
-        "values.comment_count",
-        "values.created_on",
-        "values.description",
-        "values.destination",
-        "values.draft",
-        "values.id",
-        "values.links",
-        "values.merge_commit",
-        "values.participants",
-        "values.reason",
-        "values.rendered",
-        "values.reviewers",
-        "values.source",
-        "values.state",
-        "values.summary",
-        "values.task_count",
-        "values.title",
-        "values.type",
-        "values.updated_on",
-    ]
-)
-
-# Minimal fields for slim retrieval (IDs only)
-SLIM_PR_LIST_RESPONSE_FIELDS: str = ",".join(
-    [
-        "next",
-        "page",
-        "pagelen",
-        "values.id",
-    ]
-)
-
-
-# Minimal fields for repository list calls
-REPO_LIST_RESPONSE_FIELDS: str = ",".join(
-    [
-        "next",
-        "page",
-        "pagelen",
-        "values.slug",
-        "values.full_name",
-        "values.project.key",
-    ]
-)
-
-
-class BitbucketRetriableError(Exception):
-    """Raised for retriable Bitbucket conditions (429, 5xx)."""
-
-
-class BitbucketNonRetriableError(Exception):
-    """Raised for non-retriable Bitbucket client errors (4xx except 429)."""
-
-
-@retry_builder(
-    tries=6,
-    delay=1,
-    backoff=2,
-    max_delay=30,
-    exceptions=(BitbucketRetriableError, httpx.RequestError),
-)
-@rate_limit_builder(max_calls=60, period=60)
-def bitbucket_get(
-    client: httpx.Client, url: str, params: dict[str, Any] | None = None
-) -> httpx.Response:
-    """Perform a GET against Bitbucket with retry and rate limiting.
-
-    Retries on 429 and 5xx responses, and on transport errors. Honors
-    `Retry-After` header for 429 when present by sleeping before retrying.
-    """
-    try:
-        response = client.get(url, params=params, timeout=REQUEST_TIMEOUT_SECONDS)
-    except httpx.RequestError:
-        # Allow retry_builder to handle retries of transport errors
-        raise
-
-    try:
-        response.raise_for_status()
-    except httpx.HTTPStatusError as e:
-        status = e.response.status_code if e.response is not None else None
-        if status == 429:
-            retry_after = e.response.headers.get("Retry-After") if e.response else None
-            if retry_after is not None:
-                try:
-                    time.sleep(int(retry_after))
-                except (TypeError, ValueError):
-                    pass
-            raise BitbucketRetriableError("Bitbucket rate limit exceeded (429)") from e
-        if status is not None and 500 <= status < 600:
-            raise BitbucketRetriableError(f"Bitbucket server error: {status}") from e
-        if status is not None and 400 <= status < 500:
-            raise BitbucketNonRetriableError(f"Bitbucket client error: {status}") from e
-        # Unknown status, propagate
-        raise
-
-    return response
-
-
-def build_auth_client(email: str, api_token: str) -> httpx.Client:
-    """Create an authenticated httpx client for Bitbucket Cloud API."""
-    return httpx.Client(auth=(email, api_token), http2=True)
-
-
-def paginate(
-    client: httpx.Client,
-    url: str,
-    params: dict[str, Any] | None = None,
-    start_url: str | None = None,
-    on_page: Callable[[str | None], None] | None = None,
-) -> Iterator[dict[str, Any]]:
-    """Iterate over paginated Bitbucket API responses yielding individual values.
-
-    Args:
-        client: Authenticated HTTP client.
-        url: Base collection URL (first page when start_url is None).
-        params: Query params for the first page.
-        start_url: If provided, start from this absolute URL (ignores params).
-        on_page: Optional callback invoked after each page with the next page URL.
-    """
-    next_url = start_url or url
-    # If resuming from a next URL, do not pass params again
-    query = params.copy() if params else None
-    query = None if start_url else query
-    while next_url:
-        resp = bitbucket_get(client, next_url, params=query)
-        data = resp.json()
-        values = data.get("values", [])
-        for item in values:
-            yield item
-        next_url = data.get("next")
-        if on_page is not None:
-            on_page(next_url)
-        # only include params on first call, next_url will contain all necessary params
-        query = None
-
-
-def list_repositories(
-    client: httpx.Client, workspace: str, project_key: str | None = None
-) -> Iterator[dict[str, Any]]:
-    """List repositories in a workspace, optionally filtered by project key."""
-    base_url = f"https://api.bitbucket.org/2.0/repositories/{workspace}"
-    params: dict[str, Any] = {
-        "fields": REPO_LIST_RESPONSE_FIELDS,
-        "pagelen": 100,
-        # Ensure deterministic ordering
-        "sort": "full_name",
-    }
-    if project_key:
-        params["q"] = f'project.key="{project_key}"'
-    yield from paginate(client, base_url, params)
-
-
-def map_pr_to_document(pr: dict[str, Any], workspace: str, repo_slug: str) -> Document:
-    """Map a Bitbucket pull request JSON to Onyx Document."""
-    pr_id = pr["id"]
-    title = pr.get("title") or f"PR {pr_id}"
-    description = pr.get("description") or ""
-    state = pr.get("state")
-    draft = pr.get("draft", False)
-    author = pr.get("author", {})
-    reviewers = pr.get("reviewers", [])
-    participants = pr.get("participants", [])
-
-    link = pr.get("links", {}).get("html", {}).get("href") or (
-        f"https://bitbucket.org/{workspace}/{repo_slug}/pull-requests/{pr_id}"
-    )
-
-    created_on = pr.get("created_on")
-    updated_on = pr.get("updated_on")
-    updated_dt = (
-        datetime.fromisoformat(updated_on.replace("Z", "+00:00")).astimezone(
-            timezone.utc
-        )
-        if isinstance(updated_on, str)
-        else None
-    )
-
-    source_branch = pr.get("source", {}).get("branch", {}).get("name", "")
-    destination_branch = pr.get("destination", {}).get("branch", {}).get("name", "")
-
-    approved_by = [
-        _get_user_name(p.get("user", {})) for p in participants if p.get("approved")
-    ]
-
-    primary_owner = None
-    if author:
-        primary_owner = BasicExpertInfo(
-            display_name=_get_user_name(author),
-        )
-
-    secondary_owners = [
-        BasicExpertInfo(display_name=_get_user_name(r)) for r in reviewers
-    ] or None
-
-    reviewer_names = [_get_user_name(r) for r in reviewers]
-
-    # Create a concise summary of key PR info
-    created_date = created_on.split("T")[0] if created_on else "N/A"
-    updated_date = updated_on.split("T")[0] if updated_on else "N/A"
-    content_text = (
-        "Pull Request Information:\n"
-        f"- Pull Request ID: {pr_id}\n"
-        f"- Title: {title}\n"
-        f"- State: {state or 'N/A'} {'(Draft)' if draft else ''}\n"
-    )
-    if state == "DECLINED":
-        content_text += f"- Reason: {pr.get('reason', 'N/A')}\n"
-    content_text += (
-        f"- Author: {_get_user_name(author) if author else 'N/A'}\n"
-        f"- Reviewers: {', '.join(reviewer_names) if reviewer_names else 'N/A'}\n"
-        f"- Branch: {source_branch} -> {destination_branch}\n"
-        f"- Created: {created_date}\n"
-        f"- Updated: {updated_date}"
-    )
-    if description:
-        content_text += f"\n\nDescription:\n{description}"
-    sections: list[TextSection | ImageSection] = [
-        TextSection(link=link, text=content_text)
-    ]
-
-    metadata: dict[str, str | list[str]] = {
-        "object_type": "PullRequest",
-        "workspace": workspace,
-        "repository": repo_slug,
-        "pr_key": f"{workspace}/{repo_slug}#{pr_id}",
-        "id": str(pr_id),
-        "title": title,
-        "state": state or "",
-        "draft": str(bool(draft)),
-        "link": link,
-        "author": _get_user_name(author) if author else "",
-        "reviewers": reviewer_names,
-        "approved_by": approved_by,
-        "comment_count": str(pr.get("comment_count", "")),
-        "task_count": str(pr.get("task_count", "")),
-        "created_on": created_on or "",
-        "updated_on": updated_on or "",
-        "source_branch": source_branch,
-        "destination_branch": destination_branch,
-        "closed_by": (
-            _get_user_name(pr.get("closed_by", {})) if pr.get("closed_by") else ""
-        ),
-        "close_source_branch": str(bool(pr.get("close_source_branch", False))),
-    }
-
-    return Document(
-        id=f"{DocumentSource.BITBUCKET.value}:{workspace}:{repo_slug}:pr:{pr_id}",
-        sections=sections,
-        source=DocumentSource.BITBUCKET,
-        semantic_identifier=f"#{pr_id}: {title}",
-        title=title,
-        doc_updated_at=updated_dt,
-        primary_owners=[primary_owner] if primary_owner else None,
-        secondary_owners=secondary_owners,
-        metadata=metadata,
-    )
-
-
-def _get_user_name(user: dict[str, Any]) -> str:
-    return user.get("display_name") or user.get("nickname") or "unknown"

backend/onyx/connectors/cross_connector_utils/miscellaneous_utils.py

@@ -87,7 +87,7 @@ def process_onyx_metadata(
     metadata: dict[str, Any],
 ) -> tuple[OnyxMetadata, dict[str, Any]]:
     """
-    Users may set Onyx metadata and custom tags in text files. https://docs.onyx.app/admin/connectors/official/file
+    Users may set Onyx metadata and custom tags in text files. https://docs.onyx.app/connectors/file
     Any unrecognized fields are treated as custom tags.
     """
     p_owner_names = metadata.get("primary_owners")

backend/onyx/connectors/factory.py

@@ -9,7 +9,6 @@ from onyx.configs.llm_configs import get_image_extraction_and_analysis_enabled
 from onyx.connectors.airtable.airtable_connector import AirtableConnector
 from onyx.connectors.asana.connector import AsanaConnector
 from onyx.connectors.axero.connector import AxeroConnector
-from onyx.connectors.bitbucket.connector import BitbucketConnector
 from onyx.connectors.blob.connector import BlobStorageConnector
 from onyx.connectors.bookstack.connector import BookstackConnector
 from onyx.connectors.clickup.connector import ClickupConnector
@@ -126,7 +125,6 @@ def identify_connector_class(
         DocumentSource.AIRTABLE: AirtableConnector,
         DocumentSource.HIGHSPOT: HighspotConnector,
         DocumentSource.IMAP: ImapConnector,
-        DocumentSource.BITBUCKET: BitbucketConnector,
         # just for integration tests
         DocumentSource.MOCK_CONNECTOR: MockConnector,
     }

backend/onyx/connectors/file/connector.py

@@ -153,7 +153,7 @@ def _process_file(
         content_type=file_type,
     )
 
-    # Each file may have file-specific ONYX_METADATA https://docs.onyx.app/admin/connectors/official/file
+    # Each file may have file-specific ONYX_METADATA https://docs.onyx.app/connectors/file
     # If so, we should add it to any metadata processed so far
     if extraction_result.metadata:
         logger.debug(

backend/onyx/connectors/google_utils/shared_constants.py

@@ -44,7 +44,7 @@ USER_FIELDS = "nextPageToken, users(primaryEmail)"
 MISSING_SCOPES_ERROR_STR = "client not authorized for any of the scopes requested"
 
 # Documentation and error messages
-SCOPE_DOC_URL = "https://docs.onyx.app/admin/connectors/official/google_drive/overview"
+SCOPE_DOC_URL = "https://docs.onyx.app/connectors/google_drive/overview"
 ONYX_SCOPE_INSTRUCTIONS = (
     "You have upgraded Onyx without updating the Google Auth scopes. "
     f"Please refer to the documentation to learn how to update the scopes: {SCOPE_DOC_URL}"

backend/onyx/connectors/imap/connector.py

@@ -423,7 +423,7 @@ def _sanitize_mailbox_names(mailboxes: list[str]) -> list[str]:
 
 def _parse_addrs(raw_header: str) -> list[tuple[str, str]]:
     addrs = raw_header.split(",")
-    name_addr_pairs = [parseaddr(addr=addr) for addr in addrs if addr]
+    name_addr_pairs = [parseaddr(addr=addr, strict=True) for addr in addrs if addr]
     return [(name, addr) for name, addr in name_addr_pairs if addr]
 
 

backend/onyx/connectors/imap/models.py

@@ -35,10 +35,10 @@ class EmailHeaders(BaseModel):
             if not value:
                 return None
 
-            decoded_value, encoding = email.header.decode_header(value)[0]
+            decoded_value, _encoding = email.header.decode_header(value)[0]
+
             if isinstance(decoded_value, bytes):
-                encoding = encoding or "utf-8"
-                return decoded_value.decode(encoding, errors="replace")
+                return decoded_value.decode()
             elif isinstance(decoded_value, str):
                 return decoded_value
             else:

backend/onyx/connectors/sharepoint/connector.py

@@ -57,7 +57,6 @@ from onyx.connectors.sharepoint.connector_utils import get_sharepoint_external_a
 from onyx.file_processing.extract_file_text import ACCEPTED_IMAGE_FILE_EXTENSIONS
 from onyx.file_processing.extract_file_text import extract_text_and_images
 from onyx.file_processing.extract_file_text import get_file_ext
-from onyx.file_processing.file_validation import EXCLUDED_IMAGE_TYPES
 from onyx.file_processing.image_utils import store_image_and_create_section
 from onyx.utils.logger import setup_logger
 
@@ -142,10 +141,6 @@ class SharepointAuthMethod(Enum):
     CERTIFICATE = "certificate"
 
 
-class SizeCapExceeded(Exception):
-    """Exception raised when the size cap is exceeded."""
-
-
 def load_certificate_from_pfx(pfx_data: bytes, password: str) -> CertificateData | None:
     """Load certificate from .pfx file for MSAL authentication"""
     try:
@@ -244,7 +239,7 @@ def _download_with_cap(url: str, timeout: int, cap: int) -> bytes:
     Behavior:
     - Checks `Content-Length` first and aborts early if it exceeds `cap`.
     - Otherwise streams the body in chunks and stops once `cap` is surpassed.
-    - Raises `SizeCapExceeded` when the cap would be exceeded.
+    - Raises `RuntimeError('size_cap_exceeded')` when the cap would be exceeded.
     - Returns the full bytes if the content fits within `cap`.
     """
     with requests.get(url, stream=True, timeout=timeout) as resp:
@@ -258,7 +253,7 @@ def _download_with_cap(url: str, timeout: int, cap: int) -> bytes:
                 logger.warning(
                     f"Content-Length {content_len} exceeds cap {cap}; skipping download."
                 )
-                raise SizeCapExceeded("pre_download")
+                raise RuntimeError("size_cap_exceeded")
 
         buf = io.BytesIO()
         # Stream in 64KB chunks; adjust if needed for slower networks.
@@ -271,32 +266,11 @@ def _download_with_cap(url: str, timeout: int, cap: int) -> bytes:
                 logger.warning(
                     f"Streaming download exceeded cap {cap} bytes; aborting early."
                 )
-                raise SizeCapExceeded("during_download")
+                raise RuntimeError("size_cap_exceeded")
 
         return buf.getvalue()
 
 
-def _download_via_sdk_with_cap(
-    driveitem: DriveItem, bytes_allowed: int, chunk_size: int = 64 * 1024
-) -> bytes:
-    """Use the Office365 SDK streaming download with a hard byte cap.
-
-    Raises SizeCapExceeded("during_sdk_download") if the cap would be exceeded.
-    """
-    buf = io.BytesIO()
-
-    def on_chunk(bytes_read: int) -> None:
-        # bytes_read is total bytes seen so far per SDK contract
-        if bytes_read > bytes_allowed:
-            raise SizeCapExceeded("during_sdk_download")
-
-    # modifies the driveitem to change its download behavior
-    driveitem.download_session(buf, chunk_downloaded=on_chunk, chunk_size=chunk_size)
-    # Execute the configured request with retries using existing helper
-    sleep_and_retry(driveitem.context, "download_session")
-    return buf.getvalue()
-
-
 def _convert_driveitem_to_document_with_permissions(
     driveitem: DriveItem,
     drive_name: str,
@@ -315,16 +289,6 @@ def _convert_driveitem_to_document_with_permissions(
     file_size: int | None = None
     try:
         item_json = driveitem.to_json()
-        mime_type = item_json.get("file", {}).get("mimeType")
-        if not mime_type or mime_type in EXCLUDED_IMAGE_TYPES:
-            # NOTE: this function should be refactored to look like Drive doc_conversion.py pattern
-            # for now, this skip must happen before we download the file
-            # Similar to Google Drive, we'll just semi-silently skip excluded image types
-            logger.debug(
-                f"Skipping malformed or excluded mime type {mime_type} for {driveitem.name}"
-            )
-            return None
-
         size_value = item_json.get("size")
         if size_value is not None:
             file_size = int(size_value)
@@ -347,16 +311,19 @@ def _convert_driveitem_to_document_with_permissions(
     content_bytes: bytes | None = None
     if download_url:
         try:
-            # Use this to test the sdk size cap
-            # raise requests.RequestException("test")
             content_bytes = _download_with_cap(
                 download_url,
                 REQUEST_TIMEOUT_SECONDS,
                 SHAREPOINT_CONNECTOR_SIZE_THRESHOLD,
             )
-        except SizeCapExceeded as e:
-            logger.warning(f"Skipping '{driveitem.name}' exceeded size cap: {str(e)}")
-            return None
+        except RuntimeError as e:
+            if "size_cap_exceeded" in str(e):
+                logger.warning(
+                    f"Skipping '{driveitem.name}' exceeded size cap during streaming."
+                )
+                return None
+            else:
+                raise
         except requests.RequestException as e:
             status = e.response.status_code if e.response is not None else -1
             logger.warning(
@@ -365,15 +332,13 @@ def _convert_driveitem_to_document_with_permissions(
 
     # Fallback to SDK content if needed
     if content_bytes is None:
-        try:
-            content_bytes = _download_via_sdk_with_cap(
-                driveitem, SHAREPOINT_CONNECTOR_SIZE_THRESHOLD
-            )
-        except SizeCapExceeded:
-            logger.warning(
-                f"Skipping '{driveitem.name}' exceeded size cap during SDK streaming."
-            )
-            return None
+        content = sleep_and_retry(driveitem.get_content(), "get_content")
+        if content is None or not isinstance(
+            getattr(content, "value", None), (bytes, bytearray)
+        ):
+            logger.warning(f"Could not access content for '{driveitem.name}'")
+            raise ValueError(f"Could not access content for '{driveitem.name}'")
+        content_bytes = bytes(content.value)
 
     sections: list[TextSection | ImageSection] = []
     file_ext = driveitem.name.split(".")[-1]
@@ -383,7 +348,6 @@ def _convert_driveitem_to_document_with_permissions(
             f"Zero-length content for '{driveitem.name}'. Skipping text/image extraction."
         )
     elif "." + file_ext in ACCEPTED_IMAGE_FILE_EXTENSIONS:
-        # NOTE: this if should use is_valid_image_type instead with mime_type
         image_section, _ = store_image_and_create_section(
             image_data=content_bytes,
             file_id=driveitem.id,
@@ -394,26 +358,23 @@ def _convert_driveitem_to_document_with_permissions(
         sections.append(image_section)
     else:
         # Note: we don't process Onyx metadata for connectors like Drive & Sharepoint, but could
-        def _store_embedded_image(img_data: bytes, img_name: str) -> None:
-            image_section, _ = store_image_and_create_section(
-                image_data=img_data,
-                file_id=f"{driveitem.id}_img_{len(sections)}",
-                display_name=img_name or f"{driveitem.name} - image {len(sections)}",
-                file_origin=FileOrigin.CONNECTOR,
-            )
-            image_section.link = driveitem.web_url
-            sections.append(image_section)
-
         extraction_result = extract_text_and_images(
-            file=io.BytesIO(content_bytes),
-            file_name=driveitem.name,
-            image_callback=_store_embedded_image,
+            file=io.BytesIO(content_bytes), file_name=driveitem.name
         )
         if extraction_result.text_content:
             sections.append(
                 TextSection(link=driveitem.web_url, text=extraction_result.text_content)
             )
-        # Any embedded images were stored via the callback; the returned list may be empty.
+
+        for idx, (img_data, img_name) in enumerate(extraction_result.embedded_images):
+            image_section, _ = store_image_and_create_section(
+                image_data=img_data,
+                file_id=f"{driveitem.id}_img_{idx}",
+                display_name=img_name or f"{driveitem.name} - image {idx}",
+                file_origin=FileOrigin.CONNECTOR,
+            )
+            image_section.link = driveitem.web_url
+            sections.append(image_section)
 
     if include_permissions and ctx is not None:
         logger.info(f"Getting external access for {driveitem.name}")
@@ -756,7 +717,6 @@ class SharepointConnector(
                     for folder_part in site_descriptor.folder_path.split("/"):
                         root_folder = root_folder.get_by_path(folder_part)
 
-                # TODO: consider ways to avoid materializing the entire list of files in memory
                 query = root_folder.get_files(
                     recursive=True,
                     page_size=1000,
@@ -772,13 +732,12 @@ class SharepointConnector(
                         item
                         for item in driveitems
                         if item.parent_reference.path
-                        and "root:/" in item.parent_reference.path
-                        and (
-                            item.parent_reference.path.split("root:/")[1]
-                            == site_descriptor.folder_path
-                            or item.parent_reference.path.split("root:/")[1].startswith(
-                                site_descriptor.folder_path + "/"
-                            )
+                        and any(
+                            path_part == site_descriptor.folder_path
+                            or path_part.startswith(site_descriptor.folder_path + "/")
+                            for path_part in item.parent_reference.path.split("root:/")[
+                                1
+                            ].split("/")
                         )
                     ]
                     if len(driveitems) == 0:
@@ -865,7 +824,6 @@ class SharepointConnector(
                             root_folder = root_folder.get_by_path(folder_part)
 
                     # Get all items recursively
-                    # TODO: consider ways to avoid materializing the entire list of files in memory
                     query = root_folder.get_files(
                         recursive=True,
                         page_size=1000,
@@ -890,13 +848,14 @@ class SharepointConnector(
                             item
                             for item in driveitems
                             if item.parent_reference.path
-                            and "root:/" in item.parent_reference.path
-                            and (
-                                item.parent_reference.path.split("root:/")[1]
-                                == site_descriptor.folder_path
-                                or item.parent_reference.path.split("root:/")[
-                                    1
-                                ].startswith(site_descriptor.folder_path + "/")
+                            and any(
+                                path_part == site_descriptor.folder_path
+                                or path_part.startswith(
+                                    site_descriptor.folder_path + "/"
+                                )
+                                for path_part in item.parent_reference.path.split(
+                                    "root:/"
+                                )[1].split("/")
                             )
                         ]
                         if len(driveitems) == 0:
@@ -1014,8 +973,6 @@ class SharepointConnector(
         all_pages = pages_data.get("value", [])
 
         # Handle pagination if there are more pages
-        # TODO: This accumulates all pages in memory and can be heavy on large tenants.
-        #       We should process each page incrementally to avoid unbounded growth.
         while "@odata.nextLink" in pages_data:
             next_url = pages_data["@odata.nextLink"]
             response = requests.get(
@@ -1029,7 +986,7 @@ class SharepointConnector(
 
         # Filter pages based on time window if specified
         if start is not None or end is not None:
-            filtered_pages: list[dict[str, Any]] = []
+            filtered_pages = []
             for page in all_pages:
                 page_modified = page.get("lastModifiedDateTime")
                 if page_modified:

backend/onyx/connectors/slack/connector.py

@@ -761,7 +761,7 @@ class SlackConnector(
         Step 2: Loop through each channel. For each channel:
             Step 2.1: Get messages within the time range.
             Step 2.2: Process messages in parallel, yield back docs.
-            Step 2.3: Update checkpoint with new_oldest, seen_thread_ts, and current_channel.
+            Step 2.3: Update checkpoint with new_latest, seen_thread_ts, and current_channel.
                       Slack returns messages from newest to oldest, so we need to keep track of
                       the latest message we've seen in each channel.
             Step 2.4: If there are no more messages in the channel, switch the current
@@ -837,8 +837,7 @@ class SlackConnector(
 
             channel_message_ts = checkpoint.channel_completion_map.get(channel_id)
             if channel_message_ts:
-                # Set oldest to the checkpoint timestamp to resume from where we left off
-                oldest = channel_message_ts
+                latest = channel_message_ts
 
             logger.debug(
                 f"Getting messages for channel {channel} within range {oldest} - {latest}"
@@ -856,8 +855,7 @@ class SlackConnector(
                 f"{latest=}"
             )
 
-            # message_batch[0] is the newest message (Slack returns newest to oldest)
-            new_oldest = message_batch[0]["ts"] if message_batch else latest
+            new_latest = message_batch[-1]["ts"] if message_batch else latest
 
             num_threads_start = len(seen_thread_ts)
 
@@ -908,14 +906,15 @@ class SlackConnector(
             num_threads_processed = len(seen_thread_ts) - num_threads_start
 
             # calculate a percentage progress for the current channel by determining
-            # how much of the time range we've processed so far
-            new_oldest_seconds_epoch = SecondsSinceUnixEpoch(new_oldest)
-            range_start = start if start else max(0, channel_created)
-            if new_oldest_seconds_epoch < range_start:
+            # our viable range start and end, and the latest timestamp we are querying
+            # up to
+            new_latest_seconds_epoch = SecondsSinceUnixEpoch(new_latest)
+            if new_latest_seconds_epoch > end:
                 range_complete = 0.0
             else:
-                range_complete = new_oldest_seconds_epoch - range_start
+                range_complete = end - new_latest_seconds_epoch
 
+            range_start = max(0, channel_created)
             range_total = end - range_start
             if range_total <= 0:
                 range_total = 1
@@ -936,7 +935,7 @@ class SlackConnector(
             )
 
             checkpoint.seen_thread_ts = list(seen_thread_ts)
-            checkpoint.channel_completion_map[channel["id"]] = new_oldest
+            checkpoint.channel_completion_map[channel["id"]] = new_latest
 
             # bypass channels where the first set of messages seen are all bots
             # check at least MIN_BOT_MESSAGE_THRESHOLD messages are in the batch

backend/onyx/connectors/zendesk/connector.py

@@ -1,6 +1,5 @@
 import copy
 import time
-from collections.abc import Callable
 from collections.abc import Iterator
 from typing import Any
 from typing import cast
@@ -15,9 +14,6 @@ from onyx.configs.constants import DocumentSource
 from onyx.connectors.cross_connector_utils.miscellaneous_utils import (
     time_str_to_utc,
 )
-from onyx.connectors.cross_connector_utils.rate_limit_wrapper import (
-    rate_limit_builder,
-)
 from onyx.connectors.exceptions import ConnectorValidationError
 from onyx.connectors.exceptions import CredentialExpiredError
 from onyx.connectors.exceptions import InsufficientPermissionsError
@@ -51,30 +47,14 @@ class ZendeskCredentialsNotSetUpError(PermissionError):
 
 
 class ZendeskClient:
-    def __init__(
-        self,
-        subdomain: str,
-        email: str,
-        token: str,
-        calls_per_minute: int | None = None,
-    ):
+    def __init__(self, subdomain: str, email: str, token: str):
         self.base_url = f"https://{subdomain}.zendesk.com/api/v2"
         self.auth = (f"{email}/token", token)
-        self.make_request = request_with_rate_limit(self, calls_per_minute)
 
-
-def request_with_rate_limit(
-    client: ZendeskClient, max_calls_per_minute: int | None = None
-) -> Callable[[str, dict[str, Any]], dict[str, Any]]:
     @retry_builder()
-    @(
-        rate_limit_builder(max_calls=max_calls_per_minute, period=60)
-        if max_calls_per_minute
-        else lambda x: x
-    )
-    def make_request(endpoint: str, params: dict[str, Any]) -> dict[str, Any]:
+    def make_request(self, endpoint: str, params: dict[str, Any]) -> dict[str, Any]:
         response = requests.get(
-            f"{client.base_url}/{endpoint}", auth=client.auth, params=params
+            f"{self.base_url}/{endpoint}", auth=self.auth, params=params
         )
 
         if response.status_code == 429:
@@ -92,8 +72,6 @@ def request_with_rate_limit(
         response.raise_for_status()
         return response.json()
 
-    return make_request
-
 
 class ZendeskPageResponse(BaseModel):
     data: list[dict[str, Any]]
@@ -381,13 +359,11 @@ class ZendeskConnector(
     def __init__(
         self,
         content_type: str = "articles",
-        calls_per_minute: int | None = None,
     ) -> None:
         self.content_type = content_type
         self.subdomain = ""
         # Fetch all tags ahead of time
         self.content_tags: dict[str, str] = {}
-        self.calls_per_minute = calls_per_minute
 
     def load_credentials(self, credentials: dict[str, Any]) -> dict[str, Any] | None:
         # Subdomain is actually the whole URL
@@ -399,10 +375,7 @@ class ZendeskConnector(
         self.subdomain = subdomain
 
         self.client = ZendeskClient(
-            subdomain,
-            credentials["zendesk_email"],
-            credentials["zendesk_token"],
-            calls_per_minute=self.calls_per_minute,
+            subdomain, credentials["zendesk_email"], credentials["zendesk_token"]
         )
         return None
 

backend/onyx/context/search/models.py

@@ -1,5 +1,6 @@
 from datetime import datetime
 from typing import Any
+from uuid import UUID
 
 from pydantic import BaseModel
 from pydantic import ConfigDict
@@ -119,8 +120,8 @@ class BaseFilters(BaseModel):
 
 
 class UserFileFilters(BaseModel):
-    user_file_ids: list[int] | None = None
-    user_folder_ids: list[int] | None = None
+    user_file_ids: list[UUID] | None = None
+    project_id: int | None = None
 
 
 class IndexFilters(BaseFilters, UserFileFilters):