try backporting workflows - may not work

Enable Vercel Preview

.github/workflows/docker-build-push-backend-container-on-tag.yml

@@ -6,7 +6,7 @@ on:
       - "*"
 
 env:
-  REGISTRY_IMAGE: ${{ contains(github.ref_name, 'cloud') && 'danswer/danswer-backend-cloud' || 'danswer/danswer-backend' }}
+  REGISTRY_IMAGE: ${{ contains(github.ref_name, 'cloud') && 'onyxdotapp/onyx-backend-cloud' || 'onyxdotapp/onyx-backend' }}
   LATEST_TAG: ${{ contains(github.ref_name, 'latest') }}
 
 jobs:
@@ -44,7 +44,7 @@ jobs:
             ${{ env.REGISTRY_IMAGE }}:${{ github.ref_name }}
             ${{ env.LATEST_TAG == 'true' && format('{0}:latest', env.REGISTRY_IMAGE) || '' }}
           build-args: |
-            DANSWER_VERSION=${{ github.ref_name }}
+            ONYX_VERSION=${{ github.ref_name }}
 
       # trivy has their own rate limiting issues causing this action to flake
       # we worked around it by hardcoding to different db repos in env
@@ -57,7 +57,7 @@ jobs:
           TRIVY_DB_REPOSITORY: "public.ecr.aws/aquasecurity/trivy-db:2"
           TRIVY_JAVA_DB_REPOSITORY: "public.ecr.aws/aquasecurity/trivy-java-db:1"
         with:
-          # To run locally: trivy image --severity HIGH,CRITICAL danswer/danswer-backend
+          # To run locally: trivy image --severity HIGH,CRITICAL onyxdotapp/onyx-backend
           image-ref: docker.io/${{ env.REGISTRY_IMAGE }}:${{ github.ref_name }}
           severity: "CRITICAL,HIGH"
           trivyignores: ./backend/.trivyignore

.github/workflows/docker-build-push-cloud-web-container-on-tag.yml

@@ -7,7 +7,7 @@ on:
       - "*"
 
 env:
-  REGISTRY_IMAGE: danswer/danswer-web-server-cloud
+  REGISTRY_IMAGE: onyxdotapp/onyx-web-server-cloud
   LATEST_TAG: ${{ contains(github.ref_name, 'latest') }}
 
 jobs:
@@ -60,7 +60,7 @@ jobs:
           platforms: ${{ matrix.platform }}
           push: true
           build-args: |
-            DANSWER_VERSION=${{ github.ref_name }}
+            ONYX_VERSION=${{ github.ref_name }}
             NEXT_PUBLIC_CLOUD_ENABLED=true
             NEXT_PUBLIC_POSTHOG_KEY=${{ secrets.POSTHOG_KEY }}
             NEXT_PUBLIC_POSTHOG_HOST=${{ secrets.POSTHOG_HOST }}

.github/workflows/docker-build-push-model-server-container-on-tag.yml

@@ -6,20 +6,31 @@ on:
       - "*"
 
 env:
-  REGISTRY_IMAGE: ${{ contains(github.ref_name, 'cloud') && 'danswer/danswer-model-server-cloud' || 'danswer/danswer-model-server' }}
+  REGISTRY_IMAGE: ${{ contains(github.ref_name, 'cloud') && 'onyxdotapp/onyx-model-server-cloud' || 'onyxdotapp/onyx-model-server' }}
   LATEST_TAG: ${{ contains(github.ref_name, 'latest') }}
+  DOCKER_BUILDKIT: 1
+  BUILDKIT_PROGRESS: plain
 
 jobs:
-  build-and-push:
-    # See https://runs-on.com/runners/linux/
-    runs-on: [runs-on, runner=8cpu-linux-x64, "run-id=${{ github.run_id }}"]
-
+  build-amd64:
+    runs-on:
+      [runs-on, runner=8cpu-linux-x64, "run-id=${{ github.run_id }}-amd64"]
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
 
+      - name: System Info
+        run: |
+          df -h
+          free -h
+          docker system prune -af --volumes
+
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
+        with:
+          driver-opts: |
+            image=moby/buildkit:latest
+            network=host
 
       - name: Login to Docker Hub
         uses: docker/login-action@v3
@@ -27,29 +38,86 @@ jobs:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_TOKEN }}
 
-      - name: Model Server Image Docker Build and Push
+      - name: Build and Push AMD64
         uses: docker/build-push-action@v5
         with:
           context: ./backend
           file: ./backend/Dockerfile.model_server
-          platforms: linux/amd64,linux/arm64
+          platforms: linux/amd64
           push: true
-          tags: |
-            ${{ env.REGISTRY_IMAGE }}:${{ github.ref_name }}
-            ${{ env.LATEST_TAG == 'true' && format('{0}:latest', env.REGISTRY_IMAGE) || '' }}
+          tags: ${{ env.REGISTRY_IMAGE }}:${{ github.ref_name }}-amd64
           build-args: |
             DANSWER_VERSION=${{ github.ref_name }}
+          outputs: type=registry
+          provenance: false
+
+  build-arm64:
+    runs-on:
+      [runs-on, runner=8cpu-linux-x64, "run-id=${{ github.run_id }}-arm64"]
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: System Info
+        run: |
+          df -h
+          free -h
+          docker system prune -af --volumes
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          driver-opts: |
+            image=moby/buildkit:latest
+            network=host
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_TOKEN }}
+
+      - name: Build and Push ARM64
+        uses: docker/build-push-action@v5
+        with:
+          context: ./backend
+          file: ./backend/Dockerfile.model_server
+          platforms: linux/arm64
+          push: true
+          tags: ${{ env.REGISTRY_IMAGE }}:${{ github.ref_name }}-arm64
+          build-args: |
+            DANSWER_VERSION=${{ github.ref_name }}
+          outputs: type=registry
+          provenance: false
+
+  merge-and-scan:
+    needs: [build-amd64, build-arm64]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_TOKEN }}
+
+      - name: Create and Push Multi-arch Manifest
+        run: |
+          docker buildx create --use
+          docker buildx imagetools create -t ${{ env.REGISTRY_IMAGE }}:${{ github.ref_name }} \
+            ${{ env.REGISTRY_IMAGE }}:${{ github.ref_name }}-amd64 \
+            ${{ env.REGISTRY_IMAGE }}:${{ github.ref_name }}-arm64
+          if [[ "${{ env.LATEST_TAG }}" == "true" ]]; then
+            docker buildx imagetools create -t ${{ env.REGISTRY_IMAGE }}:latest \
+              ${{ env.REGISTRY_IMAGE }}:${{ github.ref_name }}-amd64 \
+              ${{ env.REGISTRY_IMAGE }}:${{ github.ref_name }}-arm64
+          fi
 
-      # trivy has their own rate limiting issues causing this action to flake
-      # we worked around it by hardcoding to different db repos in env
-      # can re-enable when they figure it out
-      # https://github.com/aquasecurity/trivy/discussions/7538
-      # https://github.com/aquasecurity/trivy-action/issues/389
       - name: Run Trivy vulnerability scanner
         uses: aquasecurity/trivy-action@master
         env:
           TRIVY_DB_REPOSITORY: "public.ecr.aws/aquasecurity/trivy-db:2"
           TRIVY_JAVA_DB_REPOSITORY: "public.ecr.aws/aquasecurity/trivy-java-db:1"
         with:
-          image-ref: docker.io/danswer/danswer-model-server:${{ github.ref_name }}
+          image-ref: docker.io/onyxdotapp/onyx-model-server:${{ github.ref_name }}
           severity: "CRITICAL,HIGH"
+          timeout: "10m"

.github/workflows/docker-build-push-web-container-on-tag.yml

@@ -3,12 +3,12 @@ name: Build and Push Web Image on Tag
 on:
   push:
     tags:
-      - '*'
+      - "*"
 
 env:
-  REGISTRY_IMAGE: danswer/danswer-web-server
+  REGISTRY_IMAGE: onyxdotapp/onyx-web-server
   LATEST_TAG: ${{ contains(github.ref_name, 'latest') }}
-  
+
 jobs:
   build:
     runs-on:
@@ -27,11 +27,11 @@ jobs:
       - name: Prepare
         run: |
           platform=${{ matrix.platform }}
-          echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV          
-      
+          echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV
+
       - name: Checkout
         uses: actions/checkout@v4
-      
+
       - name: Docker meta
         id: meta
         uses: docker/metadata-action@v5
@@ -40,16 +40,16 @@ jobs:
           tags: |
             type=raw,value=${{ env.REGISTRY_IMAGE }}:${{ github.ref_name }}
             type=raw,value=${{ env.LATEST_TAG == 'true' && format('{0}:latest', env.REGISTRY_IMAGE) || '' }}
-      
+
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
-      
+
       - name: Login to Docker Hub
         uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_TOKEN }}
-    
+
       - name: Build and push by digest
         id: build
         uses: docker/build-push-action@v5
@@ -59,18 +59,18 @@ jobs:
           platforms: ${{ matrix.platform }}
           push: true
           build-args: |
-            DANSWER_VERSION=${{ github.ref_name }}
-          # needed due to weird interactions with the builds for different platforms  
+            ONYX_VERSION=${{ github.ref_name }}
+          # needed due to weird interactions with the builds for different platforms
           no-cache: true
           labels: ${{ steps.meta.outputs.labels }}
           outputs: type=image,name=${{ env.REGISTRY_IMAGE }},push-by-digest=true,name-canonical=true,push=true
-      
+
       - name: Export digest
         run: |
           mkdir -p /tmp/digests
           digest="${{ steps.build.outputs.digest }}"
-          touch "/tmp/digests/${digest#sha256:}"          
-      
+          touch "/tmp/digests/${digest#sha256:}"
+
       - name: Upload digest
         uses: actions/upload-artifact@v4
         with:
@@ -90,42 +90,42 @@ jobs:
           path: /tmp/digests
           pattern: digests-*
           merge-multiple: true
-      
+
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
-      
+
       - name: Docker meta
         id: meta
         uses: docker/metadata-action@v5
         with:
           images: ${{ env.REGISTRY_IMAGE }}
-      
+
       - name: Login to Docker Hub
         uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_TOKEN }}
-      
+
       - name: Create manifest list and push
         working-directory: /tmp/digests
         run: |
           docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
-            $(printf '${{ env.REGISTRY_IMAGE }}@sha256:%s ' *)          
-      
+            $(printf '${{ env.REGISTRY_IMAGE }}@sha256:%s ' *)
+
       - name: Inspect image
         run: |
           docker buildx imagetools inspect ${{ env.REGISTRY_IMAGE }}:${{ steps.meta.outputs.version }}
 
-    # trivy has their own rate limiting issues causing this action to flake
-    # we worked around it by hardcoding to different db repos in env
-    # can re-enable when they figure it out
-    # https://github.com/aquasecurity/trivy/discussions/7538
-    # https://github.com/aquasecurity/trivy-action/issues/389
+      # trivy has their own rate limiting issues causing this action to flake
+      # we worked around it by hardcoding to different db repos in env
+      # can re-enable when they figure it out
+      # https://github.com/aquasecurity/trivy/discussions/7538
+      # https://github.com/aquasecurity/trivy-action/issues/389
       - name: Run Trivy vulnerability scanner
         uses: aquasecurity/trivy-action@master
         env:
-          TRIVY_DB_REPOSITORY: 'public.ecr.aws/aquasecurity/trivy-db:2'
-          TRIVY_JAVA_DB_REPOSITORY: 'public.ecr.aws/aquasecurity/trivy-java-db:1'
+          TRIVY_DB_REPOSITORY: "public.ecr.aws/aquasecurity/trivy-db:2"
+          TRIVY_JAVA_DB_REPOSITORY: "public.ecr.aws/aquasecurity/trivy-java-db:1"
         with:
           image-ref: docker.io/${{ env.REGISTRY_IMAGE }}:${{ github.ref_name }}
-          severity: 'CRITICAL,HIGH'
+          severity: "CRITICAL,HIGH"

.github/workflows/docker-tag-latest.yml

@@ -7,31 +7,31 @@ on:
   workflow_dispatch:
     inputs:
       version:
-        description: 'The version (ie v0.0.1) to tag as latest'
+        description: "The version (ie v0.0.1) to tag as latest"
         required: true
 
 jobs:
   tag:
     # See https://runs-on.com/runners/linux/
     # use a lower powered instance since this just does i/o to docker hub
-    runs-on: [runs-on,runner=2cpu-linux-x64,"run-id=${{ github.run_id }}"]
+    runs-on: [runs-on, runner=2cpu-linux-x64, "run-id=${{ github.run_id }}"]
     steps:
-    - name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@v1
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
 
-    - name: Login to Docker Hub
-      uses: docker/login-action@v1
-      with:
-        username: ${{ secrets.DOCKER_USERNAME }}
-        password: ${{ secrets.DOCKER_TOKEN }}
+      - name: Login to Docker Hub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_TOKEN }}
 
-    - name: Enable Docker CLI experimental features
-      run: echo "DOCKER_CLI_EXPERIMENTAL=enabled" >> $GITHUB_ENV
+      - name: Enable Docker CLI experimental features
+        run: echo "DOCKER_CLI_EXPERIMENTAL=enabled" >> $GITHUB_ENV
 
-    - name: Pull, Tag and Push Web Server Image
-      run: |
-        docker buildx imagetools create -t danswer/danswer-web-server:latest danswer/danswer-web-server:${{ github.event.inputs.version }}
+      - name: Pull, Tag and Push Web Server Image
+        run: |
+          docker buildx imagetools create -t onyxdotapp/onyx-web-server:latest onyxdotapp/onyx-web-server:${{ github.event.inputs.version }}
 
-    - name: Pull, Tag and Push API Server Image
-      run: |
-        docker buildx imagetools create -t danswer/danswer-backend:latest danswer/danswer-backend:${{ github.event.inputs.version }}
+      - name: Pull, Tag and Push API Server Image
+        run: |
+          docker buildx imagetools create -t onyxdotapp/onyx-backend:latest onyxdotapp/onyx-backend:${{ github.event.inputs.version }}

.github/workflows/hotfix-release-branches.yml

@@ -8,43 +8,42 @@ on:
   workflow_dispatch:
     inputs:
       hotfix_commit:
-        description: 'Hotfix commit hash'
+        description: "Hotfix commit hash"
         required: true
       hotfix_suffix:
-        description: 'Hotfix branch suffix (e.g. hotfix/v0.8-{suffix})'
+        description: "Hotfix branch suffix (e.g. hotfix/v0.8-{suffix})"
         required: true
       release_branch_pattern:
-        description: 'Release branch pattern (regex)'
+        description: "Release branch pattern (regex)"
         required: true
-        default: 'release/.*'
+        default: "release/.*"
       auto_merge:
-        description: 'Automatically merge the hotfix PRs'
+        description: "Automatically merge the hotfix PRs"
         required: true
         type: choice
-        default: 'true'
+        default: "true"
         options:
           - true
           - false
-          
+
 jobs:
   hotfix_release_branches:
     permissions: write-all
     # See https://runs-on.com/runners/linux/
     # use a lower powered instance since this just does i/o to docker hub
-    runs-on: [runs-on,runner=2cpu-linux-x64,"run-id=${{ github.run_id }}"]
+    runs-on: [runs-on, runner=2cpu-linux-x64, "run-id=${{ github.run_id }}"]
     steps:
-    
       # needs RKUO_DEPLOY_KEY for write access to merge PR's
       - name: Checkout Repository
         uses: actions/checkout@v4
         with:
           ssh-key: "${{ secrets.RKUO_DEPLOY_KEY }}"
           fetch-depth: 0
-          
+
       - name: Set up Git user
         run: |
           git config user.name "Richard Kuo [bot]"
-          git config user.email "rkuo[bot]@danswer.ai"
+          git config user.email "rkuo[bot]@onyx.app"
 
       - name: Fetch All Branches
         run: |
@@ -62,10 +61,10 @@ jobs:
             echo "No release branches found matching pattern '${{ github.event.inputs.release_branch_pattern }}'."
             exit 1
           fi
-          
+
           echo "Found release branches:"
           echo "$BRANCHES"
-          
+
           # Join the branches into a single line separated by commas
           BRANCHES_JOINED=$(echo "$BRANCHES" | tr '\n' ',' | sed 's/,$//')
 
@@ -169,4 +168,4 @@ jobs:
                 echo "Failed to merge pull request #$PR_NUMBER."
               fi
             fi
-          done
+          done

.github/workflows/pr-backport-autotrigger.yml

@@ -4,7 +4,7 @@ name: Backport on Merge
 
 on:
   pull_request:
-    types: [closed]  # Later we check for merge so only PRs that go in can get backported
+    types: [closed] # Later we check for merge so only PRs that go in can get backported
 
 permissions:
   contents: write
@@ -26,9 +26,9 @@ jobs:
       - name: Set up Git user
         run: |
           git config user.name "Richard Kuo [bot]"
-          git config user.email "rkuo[bot]@danswer.ai"
+          git config user.email "rkuo[bot]@onyx.app"
           git fetch --prune
-      
+
       - name: Check for Backport Checkbox
         id: checkbox-check
         run: |
@@ -51,14 +51,14 @@ jobs:
           # Fetch latest tags for beta and stable
           LATEST_BETA_TAG=$(git tag -l "v[0-9]*.[0-9]*.[0-9]*-beta.[0-9]*" | grep -E "^v[0-9]+\.[0-9]+\.[0-9]+-beta\.[0-9]+$" | grep -v -- "-cloud" | sort -Vr | head -n 1)
           LATEST_STABLE_TAG=$(git tag -l "v[0-9]*.[0-9]*.[0-9]*" | grep -E "^v[0-9]+\.[0-9]+\.[0-9]+$" | sort -Vr | head -n 1)
-          
+
           # Handle case where no beta tags exist
           if [[ -z "$LATEST_BETA_TAG" ]]; then
             NEW_BETA_TAG="v1.0.0-beta.1"
           else
             NEW_BETA_TAG=$(echo $LATEST_BETA_TAG | awk -F '[.-]' '{print $1 "." $2 "." $3 "-beta." ($NF+1)}')
           fi
-          
+
           # Increment latest stable tag
           NEW_STABLE_TAG=$(echo $LATEST_STABLE_TAG | awk -F '.' '{print $1 "." $2 "." ($3+1)}')
           echo "latest_beta_tag=$LATEST_BETA_TAG" >> $GITHUB_OUTPUT
@@ -80,10 +80,10 @@ jobs:
         run: |
           set -e
           echo "Backporting to beta ${{ steps.list-branches.outputs.beta }} and stable ${{ steps.list-branches.outputs.stable }}"
-          
+
           # Echo the merge commit SHA
           echo "Merge commit SHA: ${{ github.event.pull_request.merge_commit_sha }}"
-          
+
           # Fetch all history for all branches and tags
           git fetch --prune
 
@@ -98,7 +98,7 @@ jobs:
             echo "Cherry-pick to beta failed due to conflicts."
             exit 1
           }
-          
+
           # Create new beta branch/tag
           git tag ${{ steps.list-branches.outputs.new_beta_tag }}
           # Push the changes and tag to the beta branch using PAT
@@ -110,13 +110,13 @@ jobs:
           echo "Last 5 commits on stable branch:"
           git log -n 5 --pretty=format:"%H"
           echo ""  # Newline for formatting
-          
+
           # Cherry-pick the merge commit from the merged PR
           git cherry-pick -m 1 ${{ github.event.pull_request.merge_commit_sha }} || {
             echo "Cherry-pick to stable failed due to conflicts."
             exit 1
           }
-          
+
           # Create new stable branch/tag
           git tag ${{ steps.list-branches.outputs.new_stable_tag }}
           # Push the changes and tag to the stable branch using PAT

.github/workflows/pr-chromatic-tests.yml

@@ -14,18 +14,24 @@ jobs:
     name: Playwright Tests
 
     # See https://runs-on.com/runners/linux/
-    runs-on: [runs-on,runner=8cpu-linux-x64,ram=16,"run-id=${{ github.run_id }}"]
+    runs-on:
+      [
+        runs-on,
+        runner=32cpu-linux-x64,
+        disk=large,
+        "run-id=${{ github.run_id }}",
+      ]
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
         with:
           fetch-depth: 0
-          
+
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: '3.11'
-          cache: 'pip'
+          python-version: "3.11"
+          cache: "pip"
           cache-dependency-path: |
             backend/requirements/default.txt
             backend/requirements/dev.txt
@@ -35,7 +41,7 @@ jobs:
           pip install --retries 5 --timeout 30 -r backend/requirements/default.txt
           pip install --retries 5 --timeout 30 -r backend/requirements/dev.txt
           pip install --retries 5 --timeout 30 -r backend/requirements/model_server.txt
-        
+
       - name: Setup node
         uses: actions/setup-node@v4
         with:
@@ -48,7 +54,7 @@ jobs:
       - name: Install playwright browsers
         working-directory: ./web
         run: npx playwright install --with-deps
-        
+
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
 
@@ -60,13 +66,13 @@ jobs:
 
       # tag every docker image with "test" so that we can spin up the correct set
       # of images during testing
-      
+
       # we use the runs-on cache for docker builds
       # in conjunction with runs-on runners, it has better speed and unlimited caching
       # https://runs-on.com/caching/s3-cache-for-github-actions/
       # https://runs-on.com/caching/docker/
       # https://github.com/moby/buildkit#s3-cache-experimental
-      
+
       # images are built and run locally for testing purposes. Not pushed.
 
       - name: Build Web Docker image
@@ -75,7 +81,7 @@ jobs:
           context: ./web
           file: ./web/Dockerfile
           platforms: linux/amd64
-          tags: danswer/danswer-web-server:test
+          tags: onyxdotapp/onyx-web-server:test
           push: false
           load: true
           cache-from: type=s3,prefix=cache/${{ github.repository }}/integration-tests/web-server/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }}
@@ -87,7 +93,7 @@ jobs:
           context: ./backend
           file: ./backend/Dockerfile
           platforms: linux/amd64
-          tags: danswer/danswer-backend:test
+          tags: onyxdotapp/onyx-backend:test
           push: false
           load: true
           cache-from: type=s3,prefix=cache/${{ github.repository }}/integration-tests/backend/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }}
@@ -99,7 +105,7 @@ jobs:
           context: ./backend
           file: ./backend/Dockerfile.model_server
           platforms: linux/amd64
-          tags: danswer/danswer-model-server:test
+          tags: onyxdotapp/onyx-model-server:test
           push: false
           load: true
           cache-from: type=s3,prefix=cache/${{ github.repository }}/integration-tests/model-server/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }}
@@ -110,6 +116,7 @@ jobs:
           cd deployment/docker_compose
           ENABLE_PAID_ENTERPRISE_EDITION_FEATURES=true \
           AUTH_TYPE=basic \
+          GEN_AI_API_KEY=${{ secrets.OPENAI_API_KEY }} \
           REQUIRE_EMAIL_VERIFICATION=false \
           DISABLE_TELEMETRY=true \
           IMAGE_TAG=test \
@@ -119,12 +126,12 @@ jobs:
       - name: Wait for service to be ready
         run: |
           echo "Starting wait-for-service script..."
-          
+
           docker logs -f danswer-stack-api_server-1 &
 
           start_time=$(date +%s)
           timeout=300  # 5 minutes in seconds
-          
+
           while true; do
             current_time=$(date +%s)
             elapsed_time=$((current_time - start_time))
@@ -152,7 +159,7 @@ jobs:
 
       - name: Run pytest playwright test init
         working-directory: ./backend
-        env: 
+        env:
           PYTEST_IGNORE_SKIP: true
         run: pytest -s tests/integration/tests/playwright/test_playwright.py
 
@@ -168,7 +175,7 @@ jobs:
           name: test-results
           path: ./web/test-results
           retention-days: 30
-                    
+
       # save before stopping the containers so the logs can be captured
       - name: Save Docker logs
         if: success() || failure()
@@ -176,7 +183,7 @@ jobs:
           cd deployment/docker_compose
           docker compose -f docker-compose.dev.yml -p danswer-stack logs > docker-compose.log
           mv docker-compose.log ${{ github.workspace }}/docker-compose.log
-      
+
       - name: Upload logs
         if: success() || failure()
         uses: actions/upload-artifact@v4
@@ -191,35 +198,41 @@ jobs:
 
   chromatic-tests:
     name: Chromatic Tests
-    
+
     needs: playwright-tests
-    runs-on: [runs-on,runner=8cpu-linux-x64,ram=16,"run-id=${{ github.run_id }}"]
+    runs-on:
+      [
+        runs-on,
+        runner=32cpu-linux-x64,
+        disk=large,
+        "run-id=${{ github.run_id }}",
+      ]
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
         with:
           fetch-depth: 0
-          
+
       - name: Setup node
         uses: actions/setup-node@v4
         with:
           node-version: 22
-          
+
       - name: Install node dependencies
         working-directory: ./web
         run: npm ci
-        
+
       - name: Download Playwright test results
         uses: actions/download-artifact@v4
         with:
           name: test-results
           path: ./web/test-results
-          
+
       - name: Run Chromatic
         uses: chromaui/action@latest
         with:
           playwright: true
           projectToken: ${{ secrets.CHROMATIC_PROJECT_TOKEN }}
           workingDir: ./web
-        env: 
+        env:
           CHROMATIC_ARCHIVE_LOCATION: ./test-results

.github/workflows/pr-integration-tests.yml

@@ -8,7 +8,7 @@ on:
   pull_request:
     branches:
       - main
-      - 'release/**'
+      - "release/**"
 
 env:
   OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
@@ -16,11 +16,11 @@ env:
   CONFLUENCE_TEST_SPACE_URL: ${{ secrets.CONFLUENCE_TEST_SPACE_URL }}
   CONFLUENCE_USER_NAME: ${{ secrets.CONFLUENCE_USER_NAME }}
   CONFLUENCE_ACCESS_TOKEN: ${{ secrets.CONFLUENCE_ACCESS_TOKEN }}
-  
+
 jobs:
   integration-tests:
     # See https://runs-on.com/runners/linux/
-    runs-on: [runs-on,runner=8cpu-linux-x64,ram=16,"run-id=${{ github.run_id }}"]
+    runs-on: [runs-on, runner=32cpu-linux-x64, "run-id=${{ github.run_id }}"]
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -36,21 +36,21 @@ jobs:
 
       # tag every docker image with "test" so that we can spin up the correct set
       # of images during testing
-      
+
       # We don't need to build the Web Docker image since it's not yet used
-      # in the integration tests. We have a separate action to verify that it builds 
+      # in the integration tests. We have a separate action to verify that it builds
       # successfully.
       - name: Pull Web Docker image
         run: |
-          docker pull danswer/danswer-web-server:latest
-          docker tag danswer/danswer-web-server:latest danswer/danswer-web-server:test
+          docker pull onyxdotapp/onyx-web-server:latest
+          docker tag onyxdotapp/onyx-web-server:latest onyxdotapp/onyx-web-server:test
 
       # we use the runs-on cache for docker builds
       # in conjunction with runs-on runners, it has better speed and unlimited caching
       # https://runs-on.com/caching/s3-cache-for-github-actions/
       # https://runs-on.com/caching/docker/
       # https://github.com/moby/buildkit#s3-cache-experimental
-      
+
       # images are built and run locally for testing purposes. Not pushed.
       - name: Build Backend Docker image
         uses: ./.github/actions/custom-build-and-push
@@ -58,7 +58,7 @@ jobs:
           context: ./backend
           file: ./backend/Dockerfile
           platforms: linux/amd64
-          tags: danswer/danswer-backend:test
+          tags: onyxdotapp/onyx-backend:test
           push: false
           load: true
           cache-from: type=s3,prefix=cache/${{ github.repository }}/integration-tests/backend/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }}
@@ -70,19 +70,19 @@ jobs:
           context: ./backend
           file: ./backend/Dockerfile.model_server
           platforms: linux/amd64
-          tags: danswer/danswer-model-server:test
+          tags: onyxdotapp/onyx-model-server:test
           push: false
           load: true
           cache-from: type=s3,prefix=cache/${{ github.repository }}/integration-tests/model-server/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }}
           cache-to: type=s3,prefix=cache/${{ github.repository }}/integration-tests/model-server/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }},mode=max
-      
+
       - name: Build integration test Docker image
         uses: ./.github/actions/custom-build-and-push
         with:
           context: ./backend
           file: ./backend/tests/integration/Dockerfile
           platforms: linux/amd64
-          tags: danswer/danswer-integration:test
+          tags: onyxdotapp/onyx-integration:test
           push: false
           load: true
           cache-from: type=s3,prefix=cache/${{ github.repository }}/integration-tests/integration/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }}
@@ -119,7 +119,7 @@ jobs:
             -e TEST_WEB_HOSTNAME=test-runner \
             -e AUTH_TYPE=cloud \
             -e MULTI_TENANT=true \
-            danswer/danswer-integration:test \
+            onyxdotapp/onyx-integration:test \
             /app/tests/integration/multitenant_tests
         continue-on-error: true
         id: run_multitenant_tests
@@ -131,15 +131,14 @@ jobs:
             exit 1
           else
             echo "All integration tests passed successfully."
-          fi 
+          fi
 
       - name: Stop multi-tenant Docker containers
         run: |
           cd deployment/docker_compose
           docker compose -f docker-compose.dev.yml -p danswer-stack down -v
 
-
-      - name: Start Docker containers 
+      - name: Start Docker containers
         run: |
           cd deployment/docker_compose
           ENABLE_PAID_ENTERPRISE_EDITION_FEATURES=true \
@@ -153,12 +152,12 @@ jobs:
       - name: Wait for service to be ready
         run: |
           echo "Starting wait-for-service script..."
-          
+
           docker logs -f danswer-stack-api_server-1 &
 
           start_time=$(date +%s)
           timeout=300  # 5 minutes in seconds
-          
+
           while true; do
             current_time=$(date +%s)
             elapsed_time=$((current_time - start_time))
@@ -202,7 +201,7 @@ jobs:
             -e CONFLUENCE_USER_NAME=${CONFLUENCE_USER_NAME} \
             -e CONFLUENCE_ACCESS_TOKEN=${CONFLUENCE_ACCESS_TOKEN} \
             -e TEST_WEB_HOSTNAME=test-runner \
-            danswer/danswer-integration:test \
+            onyxdotapp/onyx-integration:test \
             /app/tests/integration/tests \
             /app/tests/integration/connector_job_tests
         continue-on-error: true
@@ -229,7 +228,7 @@ jobs:
         run: |
           cd deployment/docker_compose
           docker compose -f docker-compose.dev.yml -p danswer-stack down -v
-      
+
       - name: Upload logs
         if: success() || failure()
         uses: actions/upload-artifact@v4

.github/workflows/tag-nightly.yml

@@ -2,53 +2,52 @@ name: Nightly Tag Push
 
 on:
   schedule:
-    - cron: '0 10 * * *' # Runs every day at 2 AM PST / 3 AM PDT / 10 AM UTC
+    - cron: "0 10 * * *" # Runs every day at 2 AM PST / 3 AM PDT / 10 AM UTC
 
 permissions:
-  contents: write  # Allows pushing tags to the repository
+  contents: write # Allows pushing tags to the repository
 
 jobs:
   create-and-push-tag:
-    runs-on: [runs-on,runner=2cpu-linux-x64,"run-id=${{ github.run_id }}"]
+    runs-on: [runs-on, runner=2cpu-linux-x64, "run-id=${{ github.run_id }}"]
 
     steps:
-    # actions using GITHUB_TOKEN cannot trigger another workflow, but we do want this to trigger docker pushes
-    # see https://github.com/orgs/community/discussions/27028#discussioncomment-3254367 for the workaround we
-    # implement here which needs an actual user's deploy key
-    - name: Checkout code
-      uses: actions/checkout@v4
-      with:
-        ssh-key: "${{ secrets.RKUO_DEPLOY_KEY }}"
+      # actions using GITHUB_TOKEN cannot trigger another workflow, but we do want this to trigger docker pushes
+      # see https://github.com/orgs/community/discussions/27028#discussioncomment-3254367 for the workaround we
+      # implement here which needs an actual user's deploy key
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ssh-key: "${{ secrets.RKUO_DEPLOY_KEY }}"
 
-    - name: Set up Git user
-      run: |
-        git config user.name "Richard Kuo [bot]"
-        git config user.email "rkuo[bot]@danswer.ai"
+      - name: Set up Git user
+        run: |
+          git config user.name "Richard Kuo [bot]"
+          git config user.email "rkuo[bot]@onyx.app"
 
-    - name: Check for existing nightly tag
-      id: check_tag
-      run: |
-        if git tag --points-at HEAD --list "nightly-latest*" | grep -q .; then
-          echo "A tag starting with 'nightly-latest' already exists on HEAD."
-          echo "tag_exists=true" >> $GITHUB_OUTPUT
-        else
-          echo "No tag starting with 'nightly-latest' exists on HEAD."
-          echo "tag_exists=false" >> $GITHUB_OUTPUT
-        fi
-        
-    # don't tag again if HEAD already has a nightly-latest tag on it
-    - name: Create Nightly Tag
-      if: steps.check_tag.outputs.tag_exists == 'false'
-      env:
-        DATE: ${{ github.run_id }}
-      run: |
-        TAG_NAME="nightly-latest-$(date +'%Y%m%d')"
-        echo "Creating tag: $TAG_NAME"
-        git tag $TAG_NAME
+      - name: Check for existing nightly tag
+        id: check_tag
+        run: |
+          if git tag --points-at HEAD --list "nightly-latest*" | grep -q .; then
+            echo "A tag starting with 'nightly-latest' already exists on HEAD."
+            echo "tag_exists=true" >> $GITHUB_OUTPUT
+          else
+            echo "No tag starting with 'nightly-latest' exists on HEAD."
+            echo "tag_exists=false" >> $GITHUB_OUTPUT
+          fi
 
-    - name: Push Tag
-      if: steps.check_tag.outputs.tag_exists == 'false'
-      run: |
-        TAG_NAME="nightly-latest-$(date +'%Y%m%d')"
-        git push origin $TAG_NAME
-        
+      # don't tag again if HEAD already has a nightly-latest tag on it
+      - name: Create Nightly Tag
+        if: steps.check_tag.outputs.tag_exists == 'false'
+        env:
+          DATE: ${{ github.run_id }}
+        run: |
+          TAG_NAME="nightly-latest-$(date +'%Y%m%d')"
+          echo "Creating tag: $TAG_NAME"
+          git tag $TAG_NAME
+
+      - name: Push Tag
+        if: steps.check_tag.outputs.tag_exists == 'false'
+        run: |
+          TAG_NAME="nightly-latest-$(date +'%Y%m%d')"
+          git push origin $TAG_NAME

README.md

@@ -1,48 +1,48 @@
-<!-- DANSWER_METADATA={"link": "https://github.com/danswer-ai/danswer/blob/main/README.md"} -->
+<!-- DANSWER_METADATA={"link": "https://github.com/onyx-dot-app/onyx/blob/main/README.md"} -->
 <a name="readme-top"></a>
 
 <h2 align="center">
-<a href="https://www.danswer.ai/"> <img width="50%" src="https://github.com/danswer-owners/danswer/blob/1fabd9372d66cd54238847197c33f091a724803b/DanswerWithName.png?raw=true)" /></a>
+<a href="https://www.onyx.app/"> <img width="50%" src="https://github.com/onyx-dot-app/onyx/blob/logo/LogoOnyx.png?raw=true)" /></a>
 </h2>
 
 <p align="center">
-<p align="center">Open Source Gen-AI Chat + Unified Search.</p>
+<p align="center">Open Source Gen-AI + Enterprise Search.</p>
 
 <p align="center">
-<a href="https://docs.danswer.dev/" target="_blank">
+<a href="https://docs.onyx.app/" target="_blank">
     <img src="https://img.shields.io/badge/docs-view-blue" alt="Documentation">
 </a>
-<a href="https://join.slack.com/t/danswer/shared_invite/zt-2twesxdr6-5iQitKZQpgq~hYIZ~dv3KA" target="_blank">
+<a href="https://join.slack.com/t/onyx-dot-app/shared_invite/zt-2sslpdbyq-iIbTaNIVPBw_i_4vrujLYQ" target="_blank">
     <img src="https://img.shields.io/badge/slack-join-blue.svg?logo=slack" alt="Slack">
 </a>
 <a href="https://discord.gg/TDJ59cGV2X" target="_blank">
     <img src="https://img.shields.io/badge/discord-join-blue.svg?logo=discord&logoColor=white" alt="Discord">
 </a>
-<a href="https://github.com/danswer-ai/danswer/blob/main/README.md" target="_blank">
+<a href="https://github.com/onyx-dot-app/onyx/blob/main/README.md" target="_blank">
     <img src="https://img.shields.io/static/v1?label=license&message=MIT&color=blue" alt="License">
 </a>
 </p>
 
-<strong>[Danswer](https://www.danswer.ai/)</strong> is the AI Assistant connected to your company's docs, apps, and people. 
-Danswer provides a Chat interface and plugs into any LLM of your choice. Danswer can be deployed anywhere and for any 
+<strong>[Onyx](https://www.onyx.app/)</strong> (Formerly Danswer) is the AI Assistant connected to your company's docs, apps, and people. 
+Onyx provides a Chat interface and plugs into any LLM of your choice. Onyx can be deployed anywhere and for any 
 scale - on a laptop, on-premise, or to cloud. Since you own the deployment, your user data and chats are fully in your 
-own control. Danswer is MIT licensed and designed to be modular and easily extensible. The system also comes fully ready 
+own control. Onyx is dual Licensed with most of it under MIT license and designed to be modular and easily extensible. The system also comes fully ready 
 for production usage with user authentication, role management (admin/basic users), chat persistence, and a UI for 
-configuring Personas (AI Assistants) and their Prompts.
+configuring AI Assistants.
 
-Danswer also serves as a Unified Search across all common workplace tools such as Slack, Google Drive, Confluence, etc.
-By combining LLMs and team specific knowledge, Danswer becomes a subject matter expert for the team. Imagine ChatGPT if
+Onyx also serves as a Enterprise Search across all common workplace tools such as Slack, Google Drive, Confluence, etc.
+By combining LLMs and team specific knowledge, Onyx becomes a subject matter expert for the team. Imagine ChatGPT if
 it had access to your team's unique knowledge! It enables questions such as "A customer wants feature X, is this already
 supported?" or "Where's the pull request for feature Y?"
 
 <h3>Usage</h3>
 
-Danswer Web App:
+Onyx Web App:
 
 https://github.com/danswer-ai/danswer/assets/32520769/563be14c-9304-47b5-bf0a-9049c2b6f410
 
 
-Or, plug Danswer into your existing Slack workflows (more integrations to come 😁):
+Or, plug Onyx into your existing Slack workflows (more integrations to come 😁):
 
 https://github.com/danswer-ai/danswer/assets/25087905/3e19739b-d178-4371-9a38-011430bdec1b
 
@@ -52,16 +52,16 @@ For more details on the Admin UI to manage connectors and users, check out our
 
 ## Deployment
 
-Danswer can easily be run locally (even on a laptop) or deployed on a virtual machine with a single
-`docker compose` command. Checkout our [docs](https://docs.danswer.dev/quickstart) to learn more.
+Onyx can easily be run locally (even on a laptop) or deployed on a virtual machine with a single
+`docker compose` command. Checkout our [docs](https://docs.onyx.app/quickstart) to learn more.
 
-We also have built-in support for deployment on Kubernetes. Files for that can be found [here](https://github.com/danswer-ai/danswer/tree/main/deployment/kubernetes).
+We also have built-in support for deployment on Kubernetes. Files for that can be found [here](https://github.com/onyx-dot-app/onyx/tree/main/deployment/kubernetes).
 
 
 ## 💃 Main Features 
 * Chat UI with the ability to select documents to chat with.
 * Create custom AI Assistants with different prompts and backing knowledge sets.
-* Connect Danswer with LLM of your choice (self-host for a fully airgapped solution).
+* Connect Onyx with LLM of your choice (self-host for a fully airgapped solution).
 * Document Search + AI Answers for natural language queries.
 * Connectors to all common workplace tools like Google Drive, Confluence, Slack, etc.
 * Slack integration to get answers and search results directly in Slack.
@@ -75,12 +75,12 @@ We also have built-in support for deployment on Kubernetes. Files for that can b
 * Organizational understanding and ability to locate and suggest experts from your team.
 
 
-## Other Notable Benefits of Danswer
+## Other Notable Benefits of Onyx
 * User Authentication with document level access management.
 * Best in class Hybrid Search across all sources (BM-25 + prefix aware embedding models).
 * Admin Dashboard to configure connectors, document-sets, access, etc.
 * Custom deep learning models + learn from user feedback.
-* Easy deployment and ability to host Danswer anywhere of your choosing.
+* Easy deployment and ability to host Onyx anywhere of your choosing.
 
 
 ## 🔌 Connectors
@@ -108,10 +108,10 @@ Efficiently pulls the latest changes from:
 
 ## 📚 Editions
 
-There are two editions of Danswer:
+There are two editions of Onyx:
 
-  * Danswer Community Edition (CE) is available freely under the MIT Expat license. This version has ALL the core features discussed above. This is the version of Danswer you will get if you follow the Deployment guide above.
-  * Danswer Enterprise Edition (EE) includes extra features that are primarily useful for larger organizations. Specifically, this includes:
+  * Onyx Community Edition (CE) is available freely under the MIT Expat license. This version has ALL the core features discussed above. This is the version of Onyx you will get if you follow the Deployment guide above.
+  * Onyx Enterprise Edition (EE) includes extra features that are primarily useful for larger organizations. Specifically, this includes:
     * Single Sign-On (SSO), with support for both SAML and OIDC
     * Role-based access control
     * Document permission inheritance from connected sources
@@ -119,24 +119,24 @@ There are two editions of Danswer:
     * Whitelabeling
     * API key authentication
     * Encryption of secrets
-    * Any many more! Checkout [our website](https://www.danswer.ai/) for the latest.
+    * Any many more! Checkout [our website](https://www.onyx.app/) for the latest.
 
-To try the Danswer Enterprise Edition: 
+To try the Onyx Enterprise Edition: 
 
-  1. Checkout our [Cloud product](https://app.danswer.ai/signup).
-  2. For self-hosting, contact us at [founders@danswer.ai](mailto:founders@danswer.ai) or book a call with us on our [Cal](https://cal.com/team/danswer/founders).
+  1. Checkout our [Cloud product](https://cloud.onyx.app/signup).
+  2. For self-hosting, contact us at [founders@onyx.app](mailto:founders@onyx.app) or book a call with us on our [Cal](https://cal.com/team/danswer/founders).
 
 ## 💡 Contributing
 Looking to contribute? Please check out the [Contribution Guide](CONTRIBUTING.md) for more details.
 
 ## ⭐Star History
 
-[![Star History Chart](https://api.star-history.com/svg?repos=danswer-ai/danswer&type=Date)](https://star-history.com/#danswer-ai/danswer&Date)
+[![Star History Chart](https://api.star-history.com/svg?repos=onyx-dot-app/onyx&type=Date)](https://star-history.com/#onyx-dot-app/onyx&Date)
 
 ## ✨Contributors
 
-<a href="https://github.com/danswer-ai/danswer/graphs/contributors">
-  <img alt="contributors" src="https://contrib.rocks/image?repo=danswer-ai/danswer"/>
+<a href="https://github.com/onyx-dot-app/onyx/graphs/contributors">
+  <img alt="contributors" src="https://contrib.rocks/image?repo=onyx-dot-app/onyx"/>
 </a>
 
 <p align="right" style="font-size: 14px; color: #555; margin-top: 20px;">

backend/alembic/versions/bf7a81109301_delete_input_prompts.py

@@ -0,0 +1,57 @@
+"""delete_input_prompts
+
+Revision ID: bf7a81109301
+Revises: f7a894b06d02
+Create Date: 2024-12-09 12:00:49.884228
+
+"""
+from alembic import op
+import sqlalchemy as sa
+import fastapi_users_db_sqlalchemy
+
+
+# revision identifiers, used by Alembic.
+revision = "bf7a81109301"
+down_revision = "f7a894b06d02"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.drop_table("inputprompt__user")
+    op.drop_table("inputprompt")
+
+
+def downgrade() -> None:
+    op.create_table(
+        "inputprompt",
+        sa.Column("id", sa.Integer(), autoincrement=True, nullable=False),
+        sa.Column("prompt", sa.String(), nullable=False),
+        sa.Column("content", sa.String(), nullable=False),
+        sa.Column("active", sa.Boolean(), nullable=False),
+        sa.Column("is_public", sa.Boolean(), nullable=False),
+        sa.Column(
+            "user_id",
+            fastapi_users_db_sqlalchemy.generics.GUID(),
+            nullable=True,
+        ),
+        sa.ForeignKeyConstraint(
+            ["user_id"],
+            ["user.id"],
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_table(
+        "inputprompt__user",
+        sa.Column("input_prompt_id", sa.Integer(), nullable=False),
+        sa.Column("user_id", sa.Integer(), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["input_prompt_id"],
+            ["inputprompt.id"],
+        ),
+        sa.ForeignKeyConstraint(
+            ["user_id"],
+            ["inputprompt.id"],
+        ),
+        sa.PrimaryKeyConstraint("input_prompt_id", "user_id"),
+    )

backend/alembic/versions/f7a894b06d02_non_nullbale_slack_bot_id_in_channel_.py

@@ -0,0 +1,40 @@
+"""non-nullbale slack bot id in channel config
+
+Revision ID: f7a894b06d02
+Revises: 9f696734098f
+Create Date: 2024-12-06 12:55:42.845723
+
+"""
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = "f7a894b06d02"
+down_revision = "9f696734098f"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # Delete all rows with null slack_bot_id
+    op.execute("DELETE FROM slack_channel_config WHERE slack_bot_id IS NULL")
+
+    # Make slack_bot_id non-nullable
+    op.alter_column(
+        "slack_channel_config",
+        "slack_bot_id",
+        existing_type=sa.Integer(),
+        nullable=False,
+    )
+
+
+def downgrade() -> None:
+    # Make slack_bot_id nullable again
+    op.alter_column(
+        "slack_channel_config",
+        "slack_bot_id",
+        existing_type=sa.Integer(),
+        nullable=True,
+    )

backend/danswer/auth/api_key.py

@@ -1,3 +1,4 @@
+import hashlib
 import secrets
 import uuid
 from urllib.parse import quote
@@ -18,7 +19,8 @@ _API_KEY_HEADER_NAME = "Authorization"
 # organizations like the Internet Engineering Task Force (IETF).
 _API_KEY_HEADER_ALTERNATIVE_NAME = "X-Danswer-Authorization"
 _BEARER_PREFIX = "Bearer "
-_API_KEY_PREFIX = "dn_"
+_API_KEY_PREFIX = "on_"
+_DEPRECATED_API_KEY_PREFIX = "dn_"
 _API_KEY_LEN = 192
 
 
@@ -52,7 +54,9 @@ def extract_tenant_from_api_key_header(request: Request) -> str | None:
 
     api_key = raw_api_key_header[len(_BEARER_PREFIX) :].strip()
 
-    if not api_key.startswith(_API_KEY_PREFIX):
+    if not api_key.startswith(_API_KEY_PREFIX) and not api_key.startswith(
+        _DEPRECATED_API_KEY_PREFIX
+    ):
         return None
 
     parts = api_key[len(_API_KEY_PREFIX) :].split(".", 1)
@@ -63,10 +67,19 @@ def extract_tenant_from_api_key_header(request: Request) -> str | None:
     return unquote(tenant_id) if tenant_id else None
 
 
+def _deprecated_hash_api_key(api_key: str) -> str:
+    return sha256_crypt.hash(api_key, salt="", rounds=API_KEY_HASH_ROUNDS)
+
+
 def hash_api_key(api_key: str) -> str:
     # NOTE: no salt is needed, as the API key is randomly generated
     # and overlaps are impossible
-    return sha256_crypt.hash(api_key, salt="", rounds=API_KEY_HASH_ROUNDS)
+    if api_key.startswith(_API_KEY_PREFIX):
+        return hashlib.sha256(api_key.encode("utf-8")).hexdigest()
+    elif api_key.startswith(_DEPRECATED_API_KEY_PREFIX):
+        return _deprecated_hash_api_key(api_key)
+    else:
+        raise ValueError(f"Invalid API key prefix: {api_key[:3]}")
 
 
 def build_displayable_api_key(api_key: str) -> str:

backend/danswer/auth/invited_users.py

@@ -9,7 +9,6 @@ from danswer.utils.special_types import JSON_ro
 def get_invited_users() -> list[str]:
     try:
         store = get_kv_store()
-
         return cast(list, store.load(KV_USER_STORE_KEY))
     except KvKeyNotFoundError:
         return list()

backend/danswer/auth/users.py

@@ -58,7 +58,6 @@ from danswer.auth.schemas import UserRole
 from danswer.auth.schemas import UserUpdate
 from danswer.configs.app_configs import AUTH_TYPE
 from danswer.configs.app_configs import DISABLE_AUTH
-from danswer.configs.app_configs import DISABLE_VERIFICATION
 from danswer.configs.app_configs import EMAIL_FROM
 from danswer.configs.app_configs import REQUIRE_EMAIL_VERIFICATION
 from danswer.configs.app_configs import SESSION_EXPIRE_TIME_SECONDS
@@ -132,11 +131,12 @@ def get_display_email(email: str | None, space_less: bool = False) -> str:
 
 
 def user_needs_to_be_verified() -> bool:
-    # all other auth types besides basic should require users to be
-    # verified
-    return not DISABLE_VERIFICATION and (
-        AUTH_TYPE != AuthType.BASIC or REQUIRE_EMAIL_VERIFICATION
-    )
+    if AUTH_TYPE == AuthType.BASIC or AUTH_TYPE == AuthType.CLOUD:
+        return REQUIRE_EMAIL_VERIFICATION
+
+    # For other auth types, if the user is authenticated it's assumed that
+    # the user is already verified via the external IDP
+    return False
 
 
 def verify_email_is_invited(email: str) -> None:

backend/danswer/background/celery/tasks/doc_permission_syncing/tasks.py

@@ -219,7 +219,7 @@ def connector_permission_sync_generator_task(
 
     r = get_redis_client(tenant_id=tenant_id)
 
-    lock = r.lock(
+    lock: RedisLock = r.lock(
         DanswerRedisLocks.CONNECTOR_DOC_PERMISSIONS_SYNC_LOCK_PREFIX
         + f"_{redis_connector.id}",
         timeout=CELERY_PERMISSIONS_SYNC_LOCK_TIMEOUT,

backend/danswer/background/celery/tasks/indexing/tasks.py

@@ -598,7 +598,7 @@ def connector_indexing_proxy_task(
                         db_session,
                         "Connector termination signal detected",
                     )
-            finally:
+            except Exception:
                 # if the DB exceptions, we'll just get an unfriendly failure message
                 # in the UI instead of the cancellation message
                 logger.exception(
@@ -640,12 +640,16 @@ def connector_indexing_proxy_task(
                 continue
 
         if job.status == "error":
+            exit_code: int | None = None
+            if job.process:
+                exit_code = job.process.exitcode
             task_logger.error(
                 "Indexing watchdog - spawned task exceptioned: "
                 f"attempt={index_attempt_id} "
                 f"tenant={tenant_id} "
                 f"cc_pair={cc_pair_id} "
                 f"search_settings={search_settings_id} "
+                f"exit_code={exit_code} "
                 f"error={job.exception()}"
             )
 

backend/danswer/background/celery/tasks/vespa/tasks.py

@@ -680,17 +680,28 @@ def monitor_ccpair_indexing_taskset(
                 )
                 task_logger.warning(msg)
 
-                index_attempt = get_index_attempt(db_session, payload.index_attempt_id)
-                if index_attempt:
-                    if (
-                        index_attempt.status != IndexingStatus.CANCELED
-                        and index_attempt.status != IndexingStatus.FAILED
-                    ):
-                        mark_attempt_failed(
-                            index_attempt_id=payload.index_attempt_id,
-                            db_session=db_session,
-                            failure_reason=msg,
-                        )
+                try:
+                    index_attempt = get_index_attempt(
+                        db_session, payload.index_attempt_id
+                    )
+                    if index_attempt:
+                        if (
+                            index_attempt.status != IndexingStatus.CANCELED
+                            and index_attempt.status != IndexingStatus.FAILED
+                        ):
+                            mark_attempt_failed(
+                                index_attempt_id=payload.index_attempt_id,
+                                db_session=db_session,
+                                failure_reason=msg,
+                            )
+                except Exception:
+                    task_logger.exception(
+                        "monitor_ccpair_indexing_taskset - transient exception marking index attempt as failed: "
+                        f"attempt={payload.index_attempt_id} "
+                        f"tenant={tenant_id} "
+                        f"cc_pair={cc_pair_id} "
+                        f"search_settings={search_settings_id}"
+                    )
 
                 redis_connector_index.reset()
         return

backend/danswer/background/indexing/job_client.py

@@ -82,7 +82,7 @@ class SimpleJob:
             return "running"
         elif self.process.exitcode is None:
             return "cancelled"
-        elif self.process.exitcode > 0:
+        elif self.process.exitcode != 0:
             return "error"
         else:
             return "finished"
@@ -123,7 +123,8 @@ class SimpleJobClient:
         self._cleanup_completed_jobs()
         if len(self.jobs) >= self.n_workers:
             logger.debug(
-                f"No available workers to run job. Currently running '{len(self.jobs)}' jobs, with a limit of '{self.n_workers}'."
+                f"No available workers to run job. "
+                f"Currently running '{len(self.jobs)}' jobs, with a limit of '{self.n_workers}'."
             )
             return None
 

backend/danswer/chat/answer.py

@@ -6,27 +6,27 @@ from langchain.schema.messages import BaseMessage
 from langchain_core.messages import AIMessageChunk
 from langchain_core.messages import ToolCall
 
+from danswer.chat.llm_response_handler import LLMResponseHandlerManager
 from danswer.chat.models import AnswerQuestionPossibleReturn
+from danswer.chat.models import AnswerStyleConfig
 from danswer.chat.models import CitationInfo
 from danswer.chat.models import DanswerAnswerPiece
-from danswer.file_store.utils import InMemoryChatFile
-from danswer.llm.answering.llm_response_handler import LLMCall
-from danswer.llm.answering.llm_response_handler import LLMResponseHandlerManager
-from danswer.llm.answering.models import AnswerStyleConfig
-from danswer.llm.answering.models import PreviousMessage
-from danswer.llm.answering.models import PromptConfig
-from danswer.llm.answering.prompts.build import AnswerPromptBuilder
-from danswer.llm.answering.prompts.build import default_build_system_message
-from danswer.llm.answering.prompts.build import default_build_user_message
-from danswer.llm.answering.stream_processing.answer_response_handler import (
+from danswer.chat.models import PromptConfig
+from danswer.chat.prompt_builder.build import AnswerPromptBuilder
+from danswer.chat.prompt_builder.build import default_build_system_message
+from danswer.chat.prompt_builder.build import default_build_user_message
+from danswer.chat.prompt_builder.build import LLMCall
+from danswer.chat.stream_processing.answer_response_handler import (
     CitationResponseHandler,
 )
-from danswer.llm.answering.stream_processing.answer_response_handler import (
+from danswer.chat.stream_processing.answer_response_handler import (
     DummyAnswerResponseHandler,
 )
-from danswer.llm.answering.stream_processing.utils import map_document_id_order
-from danswer.llm.answering.tool.tool_response_handler import ToolResponseHandler
+from danswer.chat.stream_processing.utils import map_document_id_order
+from danswer.chat.tool_handling.tool_response_handler import ToolResponseHandler
+from danswer.file_store.utils import InMemoryChatFile
 from danswer.llm.interfaces import LLM
+from danswer.llm.models import PreviousMessage
 from danswer.natural_language_processing.utils import get_tokenizer
 from danswer.tools.force import ForceUseTool
 from danswer.tools.models import ToolResponse
@@ -206,7 +206,9 @@ class Answer:
         # + figure out what the next LLM call should be
         tool_call_handler = ToolResponseHandler(current_llm_call.tools)
 
-        search_result = SearchTool.get_search_result(current_llm_call) or []
+        search_result, displayed_search_results_map = SearchTool.get_search_result(
+            current_llm_call
+        ) or ([], {})
 
         # Quotes are no longer supported
         # answer_handler: AnswerResponseHandler
@@ -224,6 +226,7 @@ class Answer:
         answer_handler = CitationResponseHandler(
             context_docs=search_result,
             doc_id_to_rank_map=map_document_id_order(search_result),
+            display_doc_order_dict=displayed_search_results_map,
         )
 
         response_handler_manager = LLMResponseHandlerManager(

backend/danswer/chat/chat_utils.py

@@ -26,7 +26,7 @@ from danswer.db.models import Prompt
 from danswer.db.models import Tool
 from danswer.db.models import User
 from danswer.db.persona import get_prompts_by_ids
-from danswer.llm.answering.models import PreviousMessage
+from danswer.llm.models import PreviousMessage
 from danswer.natural_language_processing.utils import BaseTokenizer
 from danswer.server.query_and_chat.models import CreateChatMessageRequest
 from danswer.tools.tool_implementations.custom.custom_tool import (

backend/danswer/chat/llm_response_handler.py

@@ -1,58 +1,22 @@
 from collections.abc import Callable
 from collections.abc import Generator
 from collections.abc import Iterator
-from typing import TYPE_CHECKING
 
 from langchain_core.messages import BaseMessage
-from pydantic.v1 import BaseModel as BaseModel__v1
 
-from danswer.chat.models import CitationInfo
-from danswer.chat.models import DanswerAnswerPiece
+from danswer.chat.models import ResponsePart
 from danswer.chat.models import StreamStopInfo
 from danswer.chat.models import StreamStopReason
-from danswer.file_store.models import InMemoryChatFile
-from danswer.llm.answering.prompts.build import AnswerPromptBuilder
-from danswer.tools.force import ForceUseTool
-from danswer.tools.models import ToolCallFinalResult
-from danswer.tools.models import ToolCallKickoff
-from danswer.tools.models import ToolResponse
-from danswer.tools.tool import Tool
-
-
-if TYPE_CHECKING:
-    from danswer.llm.answering.stream_processing.answer_response_handler import (
-        AnswerResponseHandler,
-    )
-    from danswer.llm.answering.tool.tool_response_handler import ToolResponseHandler
-
-
-ResponsePart = (
-    DanswerAnswerPiece
-    | CitationInfo
-    | ToolCallKickoff
-    | ToolResponse
-    | ToolCallFinalResult
-    | StreamStopInfo
-)
-
-
-class LLMCall(BaseModel__v1):
-    prompt_builder: AnswerPromptBuilder
-    tools: list[Tool]
-    force_use_tool: ForceUseTool
-    files: list[InMemoryChatFile]
-    tool_call_info: list[ToolCallKickoff | ToolResponse | ToolCallFinalResult]
-    using_tool_calling_llm: bool
-
-    class Config:
-        arbitrary_types_allowed = True
+from danswer.chat.prompt_builder.build import LLMCall
+from danswer.chat.stream_processing.answer_response_handler import AnswerResponseHandler
+from danswer.chat.tool_handling.tool_response_handler import ToolResponseHandler
 
 
 class LLMResponseHandlerManager:
     def __init__(
         self,
-        tool_handler: "ToolResponseHandler",
-        answer_handler: "AnswerResponseHandler",
+        tool_handler: ToolResponseHandler,
+        answer_handler: AnswerResponseHandler,
         is_cancelled: Callable[[], bool],
     ):
         self.tool_handler = tool_handler

backend/danswer/chat/models.py

@@ -1,10 +1,14 @@
+from collections.abc import Callable
 from collections.abc import Iterator
 from datetime import datetime
 from enum import Enum
 from typing import Any
+from typing import TYPE_CHECKING
 
 from pydantic import BaseModel
+from pydantic import ConfigDict
 from pydantic import Field
+from pydantic import model_validator
 
 from danswer.configs.constants import DocumentSource
 from danswer.configs.constants import MessageType
@@ -12,8 +16,15 @@ from danswer.context.search.enums import QueryFlow
 from danswer.context.search.enums import RecencyBiasSetting
 from danswer.context.search.enums import SearchType
 from danswer.context.search.models import RetrievalDocs
+from danswer.llm.override_models import PromptOverride
+from danswer.tools.models import ToolCallFinalResult
+from danswer.tools.models import ToolCallKickoff
+from danswer.tools.models import ToolResponse
 from danswer.tools.tool_implementations.custom.base_tool_types import ToolResultType
 
+if TYPE_CHECKING:
+    from danswer.db.models import Prompt
+
 
 class LlmDoc(BaseModel):
     """This contains the minimal set information for the LLM portion including citations"""
@@ -210,3 +221,109 @@ AnswerQuestionStreamReturn = Iterator[AnswerQuestionPossibleReturn]
 class LLMMetricsContainer(BaseModel):
     prompt_tokens: int
     response_tokens: int
+
+
+StreamProcessor = Callable[[Iterator[str]], AnswerQuestionStreamReturn]
+
+
+class DocumentPruningConfig(BaseModel):
+    max_chunks: int | None = None
+    max_window_percentage: float | None = None
+    max_tokens: int | None = None
+    # different pruning behavior is expected when the
+    # user manually selects documents they want to chat with
+    # e.g. we don't want to truncate each document to be no more
+    # than one chunk long
+    is_manually_selected_docs: bool = False
+    # If user specifies to include additional context Chunks for each match, then different pruning
+    # is used. As many Sections as possible are included, and the last Section is truncated
+    # If this is false, all of the Sections are truncated if they are longer than the expected Chunk size.
+    # Sections are often expected to be longer than the maximum Chunk size but Chunks should not be.
+    use_sections: bool = True
+    # If using tools, then we need to consider the tool length
+    tool_num_tokens: int = 0
+    # If using a tool message to represent the docs, then we have to JSON serialize
+    # the document content, which adds to the token count.
+    using_tool_message: bool = False
+
+
+class ContextualPruningConfig(DocumentPruningConfig):
+    num_chunk_multiple: int
+
+    @classmethod
+    def from_doc_pruning_config(
+        cls, num_chunk_multiple: int, doc_pruning_config: DocumentPruningConfig
+    ) -> "ContextualPruningConfig":
+        return cls(num_chunk_multiple=num_chunk_multiple, **doc_pruning_config.dict())
+
+
+class CitationConfig(BaseModel):
+    all_docs_useful: bool = False
+
+
+class QuotesConfig(BaseModel):
+    pass
+
+
+class AnswerStyleConfig(BaseModel):
+    citation_config: CitationConfig | None = None
+    quotes_config: QuotesConfig | None = None
+    document_pruning_config: DocumentPruningConfig = Field(
+        default_factory=DocumentPruningConfig
+    )
+    # forces the LLM to return a structured response, see
+    # https://platform.openai.com/docs/guides/structured-outputs/introduction
+    # right now, only used by the simple chat API
+    structured_response_format: dict | None = None
+
+    @model_validator(mode="after")
+    def check_quotes_and_citation(self) -> "AnswerStyleConfig":
+        if self.citation_config is None and self.quotes_config is None:
+            raise ValueError(
+                "One of `citation_config` or `quotes_config` must be provided"
+            )
+
+        if self.citation_config is not None and self.quotes_config is not None:
+            raise ValueError(
+                "Only one of `citation_config` or `quotes_config` must be provided"
+            )
+
+        return self
+
+
+class PromptConfig(BaseModel):
+    """Final representation of the Prompt configuration passed
+    into the `Answer` object."""
+
+    system_prompt: str
+    task_prompt: str
+    datetime_aware: bool
+    include_citations: bool
+
+    @classmethod
+    def from_model(
+        cls, model: "Prompt", prompt_override: PromptOverride | None = None
+    ) -> "PromptConfig":
+        override_system_prompt = (
+            prompt_override.system_prompt if prompt_override else None
+        )
+        override_task_prompt = prompt_override.task_prompt if prompt_override else None
+
+        return cls(
+            system_prompt=override_system_prompt or model.system_prompt,
+            task_prompt=override_task_prompt or model.task_prompt,
+            datetime_aware=model.datetime_aware,
+            include_citations=model.include_citations,
+        )
+
+    model_config = ConfigDict(frozen=True)
+
+
+ResponsePart = (
+    DanswerAnswerPiece
+    | CitationInfo
+    | ToolCallKickoff
+    | ToolResponse
+    | ToolCallFinalResult
+    | StreamStopInfo
+)

backend/danswer/chat/process_message.py

@@ -6,19 +6,24 @@ from typing import cast
 
 from sqlalchemy.orm import Session
 
+from danswer.chat.answer import Answer
 from danswer.chat.chat_utils import create_chat_chain
 from danswer.chat.chat_utils import create_temporary_persona
 from danswer.chat.models import AllCitations
+from danswer.chat.models import AnswerStyleConfig
 from danswer.chat.models import ChatDanswerBotResponse
+from danswer.chat.models import CitationConfig
 from danswer.chat.models import CitationInfo
 from danswer.chat.models import CustomToolResponse
 from danswer.chat.models import DanswerAnswerPiece
 from danswer.chat.models import DanswerContexts
+from danswer.chat.models import DocumentPruningConfig
 from danswer.chat.models import FileChatDisplay
 from danswer.chat.models import FinalUsedContextDocsResponse
 from danswer.chat.models import LLMRelevanceFilterResponse
 from danswer.chat.models import MessageResponseIDInfo
 from danswer.chat.models import MessageSpecificCitations
+from danswer.chat.models import PromptConfig
 from danswer.chat.models import QADocsResponse
 from danswer.chat.models import StreamingError
 from danswer.chat.models import StreamStopInfo
@@ -57,16 +62,11 @@ from danswer.document_index.factory import get_default_document_index
 from danswer.file_store.models import ChatFileType
 from danswer.file_store.models import FileDescriptor
 from danswer.file_store.utils import load_all_chat_files
-from danswer.file_store.utils import save_files_from_urls
-from danswer.llm.answering.answer import Answer
-from danswer.llm.answering.models import AnswerStyleConfig
-from danswer.llm.answering.models import CitationConfig
-from danswer.llm.answering.models import DocumentPruningConfig
-from danswer.llm.answering.models import PreviousMessage
-from danswer.llm.answering.models import PromptConfig
+from danswer.file_store.utils import save_files
 from danswer.llm.exceptions import GenAIDisabledException
 from danswer.llm.factory import get_llms_for_persona
 from danswer.llm.factory import get_main_llm_from_tuple
+from danswer.llm.models import PreviousMessage
 from danswer.llm.utils import litellm_exception_to_error_msg
 from danswer.natural_language_processing.utils import get_tokenizer
 from danswer.server.query_and_chat.models import ChatMessageDetail
@@ -119,6 +119,7 @@ from danswer.utils.logger import setup_logger
 from danswer.utils.long_term_log import LongTermLogger
 from danswer.utils.timing import log_function_time
 from danswer.utils.timing import log_generator_function_time
+from shared_configs.contextvars import CURRENT_TENANT_ID_CONTEXTVAR
 
 
 logger = setup_logger()
@@ -302,6 +303,7 @@ def stream_chat_message_objects(
     3. [always] A set of streamed LLM tokens or an error anywhere along the line if something fails
     4. [always] Details on the final AI response message that is created
     """
+    tenant_id = CURRENT_TENANT_ID_CONTEXTVAR.get()
     use_existing_user_message = new_msg_req.use_existing_user_message
     existing_assistant_message_id = new_msg_req.existing_assistant_message_id
 
@@ -678,7 +680,8 @@ def stream_chat_message_objects(
 
         reference_db_search_docs = None
         qa_docs_response = None
-        ai_message_files = None  # any files to associate with the AI message e.g. dall-e generated images
+        # any files to associate with the AI message e.g. dall-e generated images
+        ai_message_files = []
         dropped_indices = None
         tool_result = None
 
@@ -733,8 +736,14 @@ def stream_chat_message_objects(
                         list[ImageGenerationResponse], packet.response
                     )
 
-                    file_ids = save_files_from_urls(
-                        [img.url for img in img_generation_response]
+                    file_ids = save_files(
+                        urls=[img.url for img in img_generation_response if img.url],
+                        base64_files=[
+                            img.image_data
+                            for img in img_generation_response
+                            if img.image_data
+                        ],
+                        tenant_id=tenant_id,
                     )
                     ai_message_files = [
                         FileDescriptor(id=str(file_id), type=ChatFileType.IMAGE)
@@ -760,15 +769,19 @@ def stream_chat_message_objects(
                         or custom_tool_response.response_type == "csv"
                     ):
                         file_ids = custom_tool_response.tool_result.file_ids
-                        ai_message_files = [
-                            FileDescriptor(
-                                id=str(file_id),
-                                type=ChatFileType.IMAGE
-                                if custom_tool_response.response_type == "image"
-                                else ChatFileType.CSV,
-                            )
-                            for file_id in file_ids
-                        ]
+                        ai_message_files.extend(
+                            [
+                                FileDescriptor(
+                                    id=str(file_id),
+                                    type=(
+                                        ChatFileType.IMAGE
+                                        if custom_tool_response.response_type == "image"
+                                        else ChatFileType.CSV
+                                    ),
+                                )
+                                for file_id in file_ids
+                            ]
+                        )
                         yield FileChatDisplay(
                             file_ids=[str(file_id) for file_id in file_ids]
                         )
@@ -818,7 +831,8 @@ def stream_chat_message_objects(
                 citations_list=answer.citations,
                 db_docs=reference_db_search_docs,
             )
-            yield AllCitations(citations=answer.citations)
+            if not answer.is_cancelled():
+                yield AllCitations(citations=answer.citations)
 
         # Saving Gen AI answer and responding with message info
         tool_name_to_tool_id: dict[str, int] = {}

backend/danswer/chat/prompt_builder/build.py

@@ -4,20 +4,26 @@ from typing import cast
 from langchain_core.messages import BaseMessage
 from langchain_core.messages import HumanMessage
 from langchain_core.messages import SystemMessage
+from pydantic.v1 import BaseModel as BaseModel__v1
 
+from danswer.chat.models import PromptConfig
+from danswer.chat.prompt_builder.citations_prompt import compute_max_llm_input_tokens
+from danswer.chat.prompt_builder.utils import translate_history_to_basemessages
 from danswer.file_store.models import InMemoryChatFile
-from danswer.llm.answering.models import PreviousMessage
-from danswer.llm.answering.models import PromptConfig
-from danswer.llm.answering.prompts.citations_prompt import compute_max_llm_input_tokens
 from danswer.llm.interfaces import LLMConfig
+from danswer.llm.models import PreviousMessage
 from danswer.llm.utils import build_content_with_imgs
 from danswer.llm.utils import check_message_tokens
 from danswer.llm.utils import message_to_prompt_and_imgs
-from danswer.llm.utils import translate_history_to_basemessages
 from danswer.natural_language_processing.utils import get_tokenizer
 from danswer.prompts.chat_prompts import CHAT_USER_CONTEXT_FREE_PROMPT
 from danswer.prompts.prompt_utils import add_date_time_to_prompt
 from danswer.prompts.prompt_utils import drop_messages_history_overflow
+from danswer.tools.force import ForceUseTool
+from danswer.tools.models import ToolCallFinalResult
+from danswer.tools.models import ToolCallKickoff
+from danswer.tools.models import ToolResponse
+from danswer.tools.tool import Tool
 
 
 def default_build_system_message(
@@ -139,3 +145,15 @@ class AnswerPromptBuilder:
         return drop_messages_history_overflow(
             final_messages_with_tokens, self.max_tokens
         )
+
+
+class LLMCall(BaseModel__v1):
+    prompt_builder: AnswerPromptBuilder
+    tools: list[Tool]
+    force_use_tool: ForceUseTool
+    files: list[InMemoryChatFile]
+    tool_call_info: list[ToolCallKickoff | ToolResponse | ToolCallFinalResult]
+    using_tool_calling_llm: bool
+
+    class Config:
+        arbitrary_types_allowed = True

backend/danswer/chat/prompt_builder/citations_prompt.py

@@ -2,12 +2,12 @@ from langchain.schema.messages import HumanMessage
 from langchain.schema.messages import SystemMessage
 
 from danswer.chat.models import LlmDoc
+from danswer.chat.models import PromptConfig
 from danswer.configs.model_configs import GEN_AI_SINGLE_USER_MESSAGE_EXPECTED_MAX_TOKENS
 from danswer.context.search.models import InferenceChunk
 from danswer.db.models import Persona
 from danswer.db.persona import get_default_prompt__read_only
 from danswer.db.search_settings import get_multilingual_expansion
-from danswer.llm.answering.models import PromptConfig
 from danswer.llm.factory import get_llms_for_persona
 from danswer.llm.factory import get_main_llm_from_tuple
 from danswer.llm.interfaces import LLMConfig

backend/danswer/chat/prompt_builder/quotes_prompt.py

@@ -1,10 +1,10 @@
 from langchain.schema.messages import HumanMessage
 
 from danswer.chat.models import LlmDoc
+from danswer.chat.models import PromptConfig
 from danswer.configs.chat_configs import LANGUAGE_HINT
 from danswer.context.search.models import InferenceChunk
 from danswer.db.search_settings import get_multilingual_expansion
-from danswer.llm.answering.models import PromptConfig
 from danswer.llm.utils import message_to_prompt_and_imgs
 from danswer.prompts.direct_qa_prompts import CONTEXT_BLOCK
 from danswer.prompts.direct_qa_prompts import HISTORY_BLOCK

backend/danswer/chat/prompt_builder/utils.py

@@ -0,0 +1,62 @@
+from langchain.schema.messages import AIMessage
+from langchain.schema.messages import BaseMessage
+from langchain.schema.messages import HumanMessage
+
+from danswer.configs.constants import MessageType
+from danswer.db.models import ChatMessage
+from danswer.file_store.models import InMemoryChatFile
+from danswer.llm.models import PreviousMessage
+from danswer.llm.utils import build_content_with_imgs
+from danswer.prompts.direct_qa_prompts import PARAMATERIZED_PROMPT
+from danswer.prompts.direct_qa_prompts import PARAMATERIZED_PROMPT_WITHOUT_CONTEXT
+
+
+def build_dummy_prompt(
+    system_prompt: str, task_prompt: str, retrieval_disabled: bool
+) -> str:
+    if retrieval_disabled:
+        return PARAMATERIZED_PROMPT_WITHOUT_CONTEXT.format(
+            user_query="<USER_QUERY>",
+            system_prompt=system_prompt,
+            task_prompt=task_prompt,
+        ).strip()
+
+    return PARAMATERIZED_PROMPT.format(
+        context_docs_str="<CONTEXT_DOCS>",
+        user_query="<USER_QUERY>",
+        system_prompt=system_prompt,
+        task_prompt=task_prompt,
+    ).strip()
+
+
+def translate_danswer_msg_to_langchain(
+    msg: ChatMessage | PreviousMessage,
+) -> BaseMessage:
+    files: list[InMemoryChatFile] = []
+
+    # If the message is a `ChatMessage`, it doesn't have the downloaded files
+    # attached. Just ignore them for now.
+    if not isinstance(msg, ChatMessage):
+        files = msg.files
+    content = build_content_with_imgs(msg.message, files, message_type=msg.message_type)
+
+    if msg.message_type == MessageType.SYSTEM:
+        raise ValueError("System messages are not currently part of history")
+    if msg.message_type == MessageType.ASSISTANT:
+        return AIMessage(content=content)
+    if msg.message_type == MessageType.USER:
+        return HumanMessage(content=content)
+
+    raise ValueError(f"New message type {msg.message_type} not handled")
+
+
+def translate_history_to_basemessages(
+    history: list[ChatMessage] | list["PreviousMessage"],
+) -> tuple[list[BaseMessage], list[int]]:
+    history_basemessages = [
+        translate_danswer_msg_to_langchain(msg)
+        for msg in history
+        if msg.token_count != 0
+    ]
+    history_token_counts = [msg.token_count for msg in history if msg.token_count != 0]
+    return history_basemessages, history_token_counts

backend/danswer/chat/prune_and_merge.py

@@ -5,16 +5,16 @@ from typing import TypeVar
 
 from pydantic import BaseModel
 
+from danswer.chat.models import ContextualPruningConfig
 from danswer.chat.models import (
     LlmDoc,
 )
+from danswer.chat.models import PromptConfig
+from danswer.chat.prompt_builder.citations_prompt import compute_max_document_tokens
 from danswer.configs.constants import IGNORE_FOR_QA
 from danswer.configs.model_configs import DOC_EMBEDDING_CONTEXT_SIZE
 from danswer.context.search.models import InferenceChunk
 from danswer.context.search.models import InferenceSection
-from danswer.llm.answering.models import ContextualPruningConfig
-from danswer.llm.answering.models import PromptConfig
-from danswer.llm.answering.prompts.citations_prompt import compute_max_document_tokens
 from danswer.llm.interfaces import LLMConfig
 from danswer.natural_language_processing.utils import get_tokenizer
 from danswer.natural_language_processing.utils import tokenizer_trim_content

backend/danswer/chat/stream_processing/answer_response_handler.py

@@ -3,13 +3,11 @@ from collections.abc import Generator
 
 from langchain_core.messages import BaseMessage
 
+from danswer.chat.llm_response_handler import ResponsePart
 from danswer.chat.models import CitationInfo
 from danswer.chat.models import LlmDoc
-from danswer.llm.answering.llm_response_handler import ResponsePart
-from danswer.llm.answering.stream_processing.citation_processing import (
-    CitationProcessor,
-)
-from danswer.llm.answering.stream_processing.utils import DocumentIdOrderMapping
+from danswer.chat.stream_processing.citation_processing import CitationProcessor
+from danswer.chat.stream_processing.utils import DocumentIdOrderMapping
 from danswer.utils.logger import setup_logger
 
 logger = setup_logger()
@@ -37,13 +35,18 @@ class DummyAnswerResponseHandler(AnswerResponseHandler):
 
 class CitationResponseHandler(AnswerResponseHandler):
     def __init__(
-        self, context_docs: list[LlmDoc], doc_id_to_rank_map: DocumentIdOrderMapping
+        self,
+        context_docs: list[LlmDoc],
+        doc_id_to_rank_map: DocumentIdOrderMapping,
+        display_doc_order_dict: dict[str, int],
     ):
         self.context_docs = context_docs
         self.doc_id_to_rank_map = doc_id_to_rank_map
+        self.display_doc_order_dict = display_doc_order_dict
         self.citation_processor = CitationProcessor(
             context_docs=self.context_docs,
             doc_id_to_rank_map=self.doc_id_to_rank_map,
+            display_doc_order_dict=self.display_doc_order_dict,
         )
         self.processed_text = ""
         self.citations: list[CitationInfo] = []

backend/danswer/chat/stream_processing/citation_processing.py

@@ -4,8 +4,8 @@ from collections.abc import Generator
 from danswer.chat.models import CitationInfo
 from danswer.chat.models import DanswerAnswerPiece
 from danswer.chat.models import LlmDoc
+from danswer.chat.stream_processing.utils import DocumentIdOrderMapping
 from danswer.configs.chat_configs import STOP_STREAM_PAT
-from danswer.llm.answering.stream_processing.utils import DocumentIdOrderMapping
 from danswer.prompts.constants import TRIPLE_BACKTICK
 from danswer.utils.logger import setup_logger
 
@@ -22,12 +22,16 @@ class CitationProcessor:
         self,
         context_docs: list[LlmDoc],
         doc_id_to_rank_map: DocumentIdOrderMapping,
+        display_doc_order_dict: dict[str, int],
         stop_stream: str | None = STOP_STREAM_PAT,
     ):
         self.context_docs = context_docs
         self.doc_id_to_rank_map = doc_id_to_rank_map
         self.stop_stream = stop_stream
         self.order_mapping = doc_id_to_rank_map.order_mapping
+        self.display_doc_order_dict = (
+            display_doc_order_dict  # original order of docs to displayed to user
+        )
         self.llm_out = ""
         self.max_citation_num = len(context_docs)
         self.citation_order: list[int] = []
@@ -98,6 +102,18 @@ class CitationProcessor:
                         self.citation_order.index(real_citation_num) + 1
                     )
 
+                    # get the value that was displayed to user, should always
+                    # be in the display_doc_order_dict. But check anyways
+                    if context_llm_doc.document_id in self.display_doc_order_dict:
+                        displayed_citation_num = self.display_doc_order_dict[
+                            context_llm_doc.document_id
+                        ]
+                    else:
+                        displayed_citation_num = real_citation_num
+                        logger.warning(
+                            f"Doc {context_llm_doc.document_id} not in display_doc_order_dict. Used LLM citation number instead."
+                        )
+
                     # Skip consecutive citations of the same work
                     if target_citation_num in self.current_citations:
                         start, end = citation.span()
@@ -118,6 +134,7 @@ class CitationProcessor:
                                 doc_id = int(match.group(1))
                                 context_llm_doc = self.context_docs[doc_id - 1]
                                 yield CitationInfo(
+                                    # stay with the original for now (order of LLM cites)
                                     citation_num=target_citation_num,
                                     document_id=context_llm_doc.document_id,
                                 )
@@ -139,6 +156,7 @@ class CitationProcessor:
                     if target_citation_num not in self.cited_inds:
                         self.cited_inds.add(target_citation_num)
                         yield CitationInfo(
+                            # stay with the original for now (order of LLM cites)
                             citation_num=target_citation_num,
                             document_id=context_llm_doc.document_id,
                         )
@@ -148,7 +166,8 @@ class CitationProcessor:
                         prev_length = len(self.curr_segment)
                         self.curr_segment = (
                             self.curr_segment[: start + length_to_add]
-                            + f"[[{target_citation_num}]]({link})"
+                            + f"[[{displayed_citation_num}]]({link})"  # use the value that was displayed to user
+                            # + f"[[{target_citation_num}]]({link})"
                             + self.curr_segment[end + length_to_add :]
                         )
                         length_to_add += len(self.curr_segment) - prev_length
@@ -156,7 +175,8 @@ class CitationProcessor:
                         prev_length = len(self.curr_segment)
                         self.curr_segment = (
                             self.curr_segment[: start + length_to_add]
-                            + f"[[{target_citation_num}]]()"
+                            + f"[[{displayed_citation_num}]]()"  # use the value that was displayed to user
+                            # + f"[[{target_citation_num}]]()"
                             + self.curr_segment[end + length_to_add :]
                         )
                         length_to_add += len(self.curr_segment) - prev_length

backend/danswer/chat/tool_handling/tool_response_handler.py

@@ -4,8 +4,8 @@ from langchain_core.messages import AIMessageChunk
 from langchain_core.messages import BaseMessage
 from langchain_core.messages import ToolCall
 
-from danswer.llm.answering.llm_response_handler import LLMCall
-from danswer.llm.answering.llm_response_handler import ResponsePart
+from danswer.chat.models import ResponsePart
+from danswer.chat.prompt_builder.build import LLMCall
 from danswer.llm.interfaces import LLM
 from danswer.tools.force import ForceUseTool
 from danswer.tools.message import build_tool_message

backend/danswer/configs/app_configs.py

@@ -43,9 +43,6 @@ WEB_DOMAIN = os.environ.get("WEB_DOMAIN") or "http://localhost:3000"
 AUTH_TYPE = AuthType((os.environ.get("AUTH_TYPE") or AuthType.DISABLED.value).lower())
 DISABLE_AUTH = AUTH_TYPE == AuthType.DISABLED
 
-# Necessary for cloud integration tests
-DISABLE_VERIFICATION = os.environ.get("DISABLE_VERIFICATION", "").lower() == "true"
-
 # Encryption key secret is used to encrypt connector credentials, api keys, and other sensitive
 # information. This provides an extra layer of security on top of Postgres access controls
 # and is available in Danswer EE
@@ -84,7 +81,14 @@ OAUTH_CLIENT_SECRET = (
     or ""
 )
 
+# for future OAuth connector support
+# OAUTH_CONFLUENCE_CLIENT_ID = os.environ.get("OAUTH_CONFLUENCE_CLIENT_ID", "")
+# OAUTH_CONFLUENCE_CLIENT_SECRET = os.environ.get("OAUTH_CONFLUENCE_CLIENT_SECRET", "")
+# OAUTH_JIRA_CLIENT_ID = os.environ.get("OAUTH_JIRA_CLIENT_ID", "")
+# OAUTH_JIRA_CLIENT_SECRET = os.environ.get("OAUTH_JIRA_CLIENT_SECRET", "")
+
 USER_AUTH_SECRET = os.environ.get("USER_AUTH_SECRET", "")
+
 # for basic auth
 REQUIRE_EMAIL_VERIFICATION = (
     os.environ.get("REQUIRE_EMAIL_VERIFICATION", "").lower() == "true"
@@ -118,6 +122,8 @@ VESPA_HOST = os.environ.get("VESPA_HOST") or "localhost"
 VESPA_CONFIG_SERVER_HOST = os.environ.get("VESPA_CONFIG_SERVER_HOST") or VESPA_HOST
 VESPA_PORT = os.environ.get("VESPA_PORT") or "8081"
 VESPA_TENANT_PORT = os.environ.get("VESPA_TENANT_PORT") or "19071"
+# the number of times to try and connect to vespa on startup before giving up
+VESPA_NUM_ATTEMPTS_ON_STARTUP = int(os.environ.get("NUM_RETRIES_ON_STARTUP") or 10)
 
 VESPA_CLOUD_URL = os.environ.get("VESPA_CLOUD_URL", "")
 
@@ -342,6 +348,12 @@ GITLAB_CONNECTOR_INCLUDE_CODE_FILES = (
     os.environ.get("GITLAB_CONNECTOR_INCLUDE_CODE_FILES", "").lower() == "true"
 )
 
+# Egnyte specific configs
+EGNYTE_LOCALHOST_OVERRIDE = os.getenv("EGNYTE_LOCALHOST_OVERRIDE")
+EGNYTE_BASE_DOMAIN = os.getenv("EGNYTE_DOMAIN")
+EGNYTE_CLIENT_ID = os.getenv("EGNYTE_CLIENT_ID")
+EGNYTE_CLIENT_SECRET = os.getenv("EGNYTE_CLIENT_SECRET")
+
 DASK_JOB_CLIENT_ENABLED = (
     os.environ.get("DASK_JOB_CLIENT_ENABLED", "").lower() == "true"
 )
@@ -405,21 +417,28 @@ LARGE_CHUNK_RATIO = 4
 # We don't want the metadata to overwhelm the actual contents of the chunk
 SKIP_METADATA_IN_CHUNK = os.environ.get("SKIP_METADATA_IN_CHUNK", "").lower() == "true"
 # Timeout to wait for job's last update before killing it, in hours
-CLEANUP_INDEXING_JOBS_TIMEOUT = int(os.environ.get("CLEANUP_INDEXING_JOBS_TIMEOUT", 3))
+CLEANUP_INDEXING_JOBS_TIMEOUT = int(
+    os.environ.get("CLEANUP_INDEXING_JOBS_TIMEOUT") or 3
+)
 
 # The indexer will warn in the logs whenver a document exceeds this threshold (in bytes)
 INDEXING_SIZE_WARNING_THRESHOLD = int(
-    os.environ.get("INDEXING_SIZE_WARNING_THRESHOLD", 100 * 1024 * 1024)
+    os.environ.get("INDEXING_SIZE_WARNING_THRESHOLD") or 100 * 1024 * 1024
 )
 
 # during indexing, will log verbose memory diff stats every x batches and at the end.
 # 0 disables this behavior and is the default.
-INDEXING_TRACER_INTERVAL = int(os.environ.get("INDEXING_TRACER_INTERVAL", 0))
+INDEXING_TRACER_INTERVAL = int(os.environ.get("INDEXING_TRACER_INTERVAL") or 0)
 
 # During an indexing attempt, specifies the number of batches which are allowed to
 # exception without aborting the attempt.
-INDEXING_EXCEPTION_LIMIT = int(os.environ.get("INDEXING_EXCEPTION_LIMIT", 0))
+INDEXING_EXCEPTION_LIMIT = int(os.environ.get("INDEXING_EXCEPTION_LIMIT") or 0)
 
+# Maximum file size in a document to be indexed
+MAX_DOCUMENT_CHARS = int(os.environ.get("MAX_DOCUMENT_CHARS") or 5_000_000)
+MAX_FILE_SIZE_BYTES = int(
+    os.environ.get("MAX_FILE_SIZE_BYTES") or 2 * 1024 * 1024 * 1024
+)  # 2GB in bytes
 
 #####
 # Miscellaneous

backend/danswer/configs/chat_configs.py

@@ -3,7 +3,6 @@ import os
 
 PROMPTS_YAML = "./danswer/seeding/prompts.yaml"
 PERSONAS_YAML = "./danswer/seeding/personas.yaml"
-INPUT_PROMPT_YAML = "./danswer/seeding/input_prompts.yaml"
 
 NUM_RETURNED_HITS = 50
 # Used for LLM filtering and reranking

backend/danswer/configs/constants.py

@@ -132,6 +132,7 @@ class DocumentSource(str, Enum):
     NOT_APPLICABLE = "not_applicable"
     FRESHDESK = "freshdesk"
     FIREFLIES = "fireflies"
+    EGNYTE = "egnyte"
 
 
 DocumentSourceRequiringTenantContext: list[DocumentSource] = [DocumentSource.FILE]

backend/danswer/configs/tool_configs.py

@@ -2,6 +2,8 @@ import json
 import os
 
 
+IMAGE_GENERATION_OUTPUT_FORMAT = os.environ.get("IMAGE_GENERATION_OUTPUT_FORMAT", "url")
+
 # if specified, will pass through request headers to the call to API calls made by custom tools
 CUSTOM_TOOL_PASS_THROUGH_HEADERS: list[str] | None = None
 _CUSTOM_TOOL_PASS_THROUGH_HEADERS_RAW = os.environ.get(

backend/danswer/connectors/confluence/connector.py

@@ -15,6 +15,7 @@ from danswer.connectors.confluence.utils import attachment_to_content
 from danswer.connectors.confluence.utils import build_confluence_document_id
 from danswer.connectors.confluence.utils import datetime_from_string
 from danswer.connectors.confluence.utils import extract_text_from_confluence_html
+from danswer.connectors.confluence.utils import validate_attachment_filetype
 from danswer.connectors.interfaces import GenerateDocumentsOutput
 from danswer.connectors.interfaces import GenerateSlimDocumentOutput
 from danswer.connectors.interfaces import LoadConnector
@@ -276,9 +277,11 @@ class ConfluenceConnector(LoadConnector, PollConnector, SlimConnector):
         ):
             # If the page has restrictions, add them to the perm_sync_data
             # These will be used by doc_sync.py to sync permissions
-            perm_sync_data = {
-                "restrictions": page.get("restrictions", {}),
-                "space_key": page.get("space", {}).get("key"),
+            page_restrictions = page.get("restrictions")
+            page_space_key = page.get("space", {}).get("key")
+            page_perm_sync_data = {
+                "restrictions": page_restrictions or {},
+                "space_key": page_space_key,
             }
 
             doc_metadata_list.append(
@@ -288,7 +291,7 @@ class ConfluenceConnector(LoadConnector, PollConnector, SlimConnector):
                         page["_links"]["webui"],
                         self.is_cloud,
                     ),
-                    perm_sync_data=perm_sync_data,
+                    perm_sync_data=page_perm_sync_data,
                 )
             )
             attachment_cql = f"type=attachment and container='{page['id']}'"
@@ -298,6 +301,21 @@ class ConfluenceConnector(LoadConnector, PollConnector, SlimConnector):
                 expand=restrictions_expand,
                 limit=_SLIM_DOC_BATCH_SIZE,
             ):
+                if not validate_attachment_filetype(attachment):
+                    continue
+                attachment_restrictions = attachment.get("restrictions")
+                if not attachment_restrictions:
+                    attachment_restrictions = page_restrictions
+
+                attachment_space_key = attachment.get("space", {}).get("key")
+                if not attachment_space_key:
+                    attachment_space_key = page_space_key
+
+                attachment_perm_sync_data = {
+                    "restrictions": attachment_restrictions or {},
+                    "space_key": attachment_space_key,
+                }
+
                 doc_metadata_list.append(
                     SlimDocument(
                         id=build_confluence_document_id(
@@ -305,7 +323,7 @@ class ConfluenceConnector(LoadConnector, PollConnector, SlimConnector):
                             attachment["_links"]["webui"],
                             self.is_cloud,
                         ),
-                        perm_sync_data=perm_sync_data,
+                        perm_sync_data=attachment_perm_sync_data,
                     )
                 )
             if len(doc_metadata_list) > _SLIM_DOC_BATCH_SIZE:

backend/danswer/connectors/confluence/onyx_confluence.py

@@ -368,4 +368,5 @@ def build_confluence_client(
         backoff_and_retry=True,
         max_backoff_retries=10,
         max_backoff_seconds=60,
+        cloud=is_cloud,
     )

backend/danswer/connectors/confluence/utils.py

@@ -177,19 +177,23 @@ def extract_text_from_confluence_html(
     return format_document_soup(soup)
 
 
-def attachment_to_content(
-    confluence_client: OnyxConfluence,
-    attachment: dict[str, Any],
-) -> str | None:
-    """If it returns None, assume that we should skip this attachment."""
-    if attachment["metadata"]["mediaType"] in [
+def validate_attachment_filetype(attachment: dict[str, Any]) -> bool:
+    return attachment["metadata"]["mediaType"] not in [
         "image/jpeg",
         "image/png",
         "image/gif",
         "image/svg+xml",
         "video/mp4",
         "video/quicktime",
-    ]:
+    ]
+
+
+def attachment_to_content(
+    confluence_client: OnyxConfluence,
+    attachment: dict[str, Any],
+) -> str | None:
+    """If it returns None, assume that we should skip this attachment."""
+    if not validate_attachment_filetype(attachment):
         return None
 
     download_link = confluence_client.url + attachment["_links"]["download"]
@@ -245,7 +249,7 @@ def build_confluence_document_id(
     return f"{base_url}{content_url}"
 
 
-def extract_referenced_attachment_names(page_text: str) -> list[str]:
+def _extract_referenced_attachment_names(page_text: str) -> list[str]:
     """Parse a Confluence html page to generate a list of current
         attachments in use
 

backend/danswer/connectors/egnyte/connector.py

@@ -0,0 +1,384 @@
+import io
+import os
+from collections.abc import Generator
+from datetime import datetime
+from datetime import timezone
+from logging import Logger
+from typing import Any
+from typing import cast
+from typing import IO
+
+import requests
+from retry import retry
+
+from danswer.configs.app_configs import EGNYTE_BASE_DOMAIN
+from danswer.configs.app_configs import EGNYTE_CLIENT_ID
+from danswer.configs.app_configs import EGNYTE_CLIENT_SECRET
+from danswer.configs.app_configs import EGNYTE_LOCALHOST_OVERRIDE
+from danswer.configs.app_configs import INDEX_BATCH_SIZE
+from danswer.configs.constants import DocumentSource
+from danswer.connectors.interfaces import GenerateDocumentsOutput
+from danswer.connectors.interfaces import LoadConnector
+from danswer.connectors.interfaces import OAuthConnector
+from danswer.connectors.interfaces import PollConnector
+from danswer.connectors.interfaces import SecondsSinceUnixEpoch
+from danswer.connectors.models import BasicExpertInfo
+from danswer.connectors.models import ConnectorMissingCredentialError
+from danswer.connectors.models import Document
+from danswer.connectors.models import Section
+from danswer.file_processing.extract_file_text import detect_encoding
+from danswer.file_processing.extract_file_text import extract_file_text
+from danswer.file_processing.extract_file_text import get_file_ext
+from danswer.file_processing.extract_file_text import is_text_file_extension
+from danswer.file_processing.extract_file_text import is_valid_file_ext
+from danswer.file_processing.extract_file_text import read_text_file
+from danswer.utils.logger import setup_logger
+
+
+logger = setup_logger()
+
+_EGNYTE_API_BASE = "https://{domain}.egnyte.com/pubapi/v1"
+_EGNYTE_APP_BASE = "https://{domain}.egnyte.com"
+_TIMEOUT = 60
+
+
+def _request_with_retries(
+    method: str,
+    url: str,
+    data: dict[str, Any] | None = None,
+    headers: dict[str, Any] | None = None,
+    params: dict[str, Any] | None = None,
+    timeout: int = _TIMEOUT,
+    stream: bool = False,
+    tries: int = 8,
+    delay: float = 1,
+    backoff: float = 2,
+) -> requests.Response:
+    @retry(tries=tries, delay=delay, backoff=backoff, logger=cast(Logger, logger))
+    def _make_request() -> requests.Response:
+        response = requests.request(
+            method,
+            url,
+            data=data,
+            headers=headers,
+            params=params,
+            timeout=timeout,
+            stream=stream,
+        )
+        try:
+            response.raise_for_status()
+        except requests.exceptions.HTTPError as e:
+            if e.response.status_code != 403:
+                logger.exception(
+                    f"Failed to call Egnyte API.\n"
+                    f"URL: {url}\n"
+                    f"Headers: {headers}\n"
+                    f"Data: {data}\n"
+                    f"Params: {params}"
+                )
+            raise e
+        return response
+
+    return _make_request()
+
+
+def _parse_last_modified(last_modified: str) -> datetime:
+    return datetime.strptime(last_modified, "%a, %d %b %Y %H:%M:%S %Z").replace(
+        tzinfo=timezone.utc
+    )
+
+
+def _process_egnyte_file(
+    file_metadata: dict[str, Any],
+    file_content: IO,
+    base_url: str,
+    folder_path: str | None = None,
+) -> Document | None:
+    """Process an Egnyte file into a Document object
+
+    Args:
+        file_data: The file data from Egnyte API
+        file_content: The raw content of the file in bytes
+        base_url: The base URL for the Egnyte instance
+        folder_path: Optional folder path to filter results
+    """
+    # Skip if file path doesn't match folder path filter
+    if folder_path and not file_metadata["path"].startswith(folder_path):
+        raise ValueError(
+            f"File path {file_metadata['path']} does not match folder path {folder_path}"
+        )
+
+    file_name = file_metadata["name"]
+    extension = get_file_ext(file_name)
+    if not is_valid_file_ext(extension):
+        logger.warning(f"Skipping file '{file_name}' with extension '{extension}'")
+        return None
+
+    # Extract text content based on file type
+    if is_text_file_extension(file_name):
+        encoding = detect_encoding(file_content)
+        file_content_raw, file_metadata = read_text_file(
+            file_content, encoding=encoding, ignore_danswer_metadata=False
+        )
+    else:
+        file_content_raw = extract_file_text(
+            file=file_content,
+            file_name=file_name,
+            break_on_unprocessable=True,
+        )
+
+    # Build the web URL for the file
+    web_url = f"{base_url}/navigate/file/{file_metadata['group_id']}"
+
+    # Create document metadata
+    metadata: dict[str, str | list[str]] = {
+        "file_path": file_metadata["path"],
+        "last_modified": file_metadata.get("last_modified", ""),
+    }
+
+    # Add lock info if present
+    if lock_info := file_metadata.get("lock_info"):
+        metadata[
+            "lock_owner"
+        ] = f"{lock_info.get('first_name', '')} {lock_info.get('last_name', '')}"
+
+    # Create the document owners
+    primary_owner = None
+    if uploaded_by := file_metadata.get("uploaded_by"):
+        primary_owner = BasicExpertInfo(
+            email=uploaded_by,  # Using username as email since that's what we have
+        )
+
+    # Create the document
+    return Document(
+        id=f"egnyte-{file_metadata['entry_id']}",
+        sections=[Section(text=file_content_raw.strip(), link=web_url)],
+        source=DocumentSource.EGNYTE,
+        semantic_identifier=file_name,
+        metadata=metadata,
+        doc_updated_at=(
+            _parse_last_modified(file_metadata["last_modified"])
+            if "last_modified" in file_metadata
+            else None
+        ),
+        primary_owners=[primary_owner] if primary_owner else None,
+    )
+
+
+class EgnyteConnector(LoadConnector, PollConnector, OAuthConnector):
+    def __init__(
+        self,
+        folder_path: str | None = None,
+        batch_size: int = INDEX_BATCH_SIZE,
+    ) -> None:
+        self.domain = ""  # will always be set in `load_credentials`
+        self.folder_path = folder_path or ""  # Root folder if not specified
+        self.batch_size = batch_size
+        self.access_token: str | None = None
+
+    @classmethod
+    def oauth_id(cls) -> DocumentSource:
+        return DocumentSource.EGNYTE
+
+    @classmethod
+    def oauth_authorization_url(cls, base_domain: str, state: str) -> str:
+        if not EGNYTE_CLIENT_ID:
+            raise ValueError("EGNYTE_CLIENT_ID environment variable must be set")
+        if not EGNYTE_BASE_DOMAIN:
+            raise ValueError("EGNYTE_DOMAIN environment variable must be set")
+
+        if EGNYTE_LOCALHOST_OVERRIDE:
+            base_domain = EGNYTE_LOCALHOST_OVERRIDE
+
+        callback_uri = f"{base_domain.strip('/')}/connector/oauth/callback/egnyte"
+        return (
+            f"https://{EGNYTE_BASE_DOMAIN}.egnyte.com/puboauth/token"
+            f"?client_id={EGNYTE_CLIENT_ID}"
+            f"&redirect_uri={callback_uri}"
+            f"&scope=Egnyte.filesystem"
+            f"&state={state}"
+            f"&response_type=code"
+        )
+
+    @classmethod
+    def oauth_code_to_token(cls, code: str) -> dict[str, Any]:
+        if not EGNYTE_CLIENT_ID:
+            raise ValueError("EGNYTE_CLIENT_ID environment variable must be set")
+        if not EGNYTE_CLIENT_SECRET:
+            raise ValueError("EGNYTE_CLIENT_SECRET environment variable must be set")
+        if not EGNYTE_BASE_DOMAIN:
+            raise ValueError("EGNYTE_DOMAIN environment variable must be set")
+
+        # Exchange code for token
+        url = f"https://{EGNYTE_BASE_DOMAIN}.egnyte.com/puboauth/token"
+        data = {
+            "client_id": EGNYTE_CLIENT_ID,
+            "client_secret": EGNYTE_CLIENT_SECRET,
+            "code": code,
+            "grant_type": "authorization_code",
+            "redirect_uri": f"{EGNYTE_LOCALHOST_OVERRIDE or ''}/connector/oauth/callback/egnyte",
+            "scope": "Egnyte.filesystem",
+        }
+        headers = {"Content-Type": "application/x-www-form-urlencoded"}
+
+        response = _request_with_retries(
+            method="POST",
+            url=url,
+            data=data,
+            headers=headers,
+            # try a lot faster since this is a realtime flow
+            backoff=0,
+            delay=0.1,
+        )
+        if not response.ok:
+            raise RuntimeError(f"Failed to exchange code for token: {response.text}")
+
+        token_data = response.json()
+        return {
+            "domain": EGNYTE_BASE_DOMAIN,
+            "access_token": token_data["access_token"],
+        }
+
+    def load_credentials(self, credentials: dict[str, Any]) -> dict[str, Any] | None:
+        self.domain = credentials["domain"]
+        self.access_token = credentials["access_token"]
+        return None
+
+    def _get_files_list(
+        self,
+        path: str,
+    ) -> list[dict[str, Any]]:
+        if not self.access_token or not self.domain:
+            raise ConnectorMissingCredentialError("Egnyte")
+
+        headers = {
+            "Authorization": f"Bearer {self.access_token}",
+        }
+
+        params: dict[str, Any] = {
+            "list_content": True,
+        }
+
+        url = f"{_EGNYTE_API_BASE.format(domain=self.domain)}/fs/{path or ''}"
+        response = _request_with_retries(
+            method="GET", url=url, headers=headers, params=params, timeout=_TIMEOUT
+        )
+        if not response.ok:
+            raise RuntimeError(f"Failed to fetch files from Egnyte: {response.text}")
+
+        data = response.json()
+        all_files: list[dict[str, Any]] = []
+
+        # Add files from current directory
+        all_files.extend(data.get("files", []))
+
+        # Recursively traverse folders
+        for item in data.get("folders", []):
+            all_files.extend(self._get_files_list(item["path"]))
+
+        return all_files
+
+    def _filter_files(
+        self,
+        files: list[dict[str, Any]],
+        start_time: datetime | None = None,
+        end_time: datetime | None = None,
+    ) -> list[dict[str, Any]]:
+        filtered_files = []
+        for file in files:
+            if file["is_folder"]:
+                continue
+
+            file_modified = _parse_last_modified(file["last_modified"])
+            if start_time and file_modified < start_time:
+                continue
+            if end_time and file_modified > end_time:
+                continue
+
+            filtered_files.append(file)
+
+        return filtered_files
+
+    def _process_files(
+        self,
+        start_time: datetime | None = None,
+        end_time: datetime | None = None,
+    ) -> Generator[list[Document], None, None]:
+        files = self._get_files_list(self.folder_path)
+        files = self._filter_files(files, start_time, end_time)
+
+        current_batch: list[Document] = []
+        for file in files:
+            try:
+                # Set up request with streaming enabled
+                headers = {
+                    "Authorization": f"Bearer {self.access_token}",
+                }
+                url = f"{_EGNYTE_API_BASE.format(domain=self.domain)}/fs-content/{file['path']}"
+                response = _request_with_retries(
+                    method="GET",
+                    url=url,
+                    headers=headers,
+                    timeout=_TIMEOUT,
+                    stream=True,
+                )
+
+                if not response.ok:
+                    logger.error(
+                        f"Failed to fetch file content: {file['path']} (status code: {response.status_code})"
+                    )
+                    continue
+
+                # Stream the response content into a BytesIO buffer
+                buffer = io.BytesIO()
+                for chunk in response.iter_content(chunk_size=8192):
+                    if chunk:
+                        buffer.write(chunk)
+
+                # Reset buffer's position to the start
+                buffer.seek(0)
+
+                # Process the streamed file content
+                doc = _process_egnyte_file(
+                    file_metadata=file,
+                    file_content=buffer,
+                    base_url=_EGNYTE_APP_BASE.format(domain=self.domain),
+                    folder_path=self.folder_path,
+                )
+
+                if doc is not None:
+                    current_batch.append(doc)
+
+                    if len(current_batch) >= self.batch_size:
+                        yield current_batch
+                        current_batch = []
+
+            except Exception:
+                logger.exception(f"Failed to process file {file['path']}")
+                continue
+
+        if current_batch:
+            yield current_batch
+
+    def load_from_state(self) -> GenerateDocumentsOutput:
+        yield from self._process_files()
+
+    def poll_source(
+        self, start: SecondsSinceUnixEpoch, end: SecondsSinceUnixEpoch
+    ) -> GenerateDocumentsOutput:
+        start_time = datetime.fromtimestamp(start, tz=timezone.utc)
+        end_time = datetime.fromtimestamp(end, tz=timezone.utc)
+
+        yield from self._process_files(start_time=start_time, end_time=end_time)
+
+
+if __name__ == "__main__":
+    connector = EgnyteConnector()
+    connector.load_credentials(
+        {
+            "domain": os.environ["EGNYTE_DOMAIN"],
+            "access_token": os.environ["EGNYTE_ACCESS_TOKEN"],
+        }
+    )
+    document_batches = connector.load_from_state()
+    print(next(document_batches))

backend/danswer/connectors/factory.py

@@ -15,6 +15,7 @@ from danswer.connectors.danswer_jira.connector import JiraConnector
 from danswer.connectors.discourse.connector import DiscourseConnector
 from danswer.connectors.document360.connector import Document360Connector
 from danswer.connectors.dropbox.connector import DropboxConnector
+from danswer.connectors.egnyte.connector import EgnyteConnector
 from danswer.connectors.file.connector import LocalFileConnector
 from danswer.connectors.fireflies.connector import FirefliesConnector
 from danswer.connectors.freshdesk.connector import FreshdeskConnector
@@ -40,7 +41,6 @@ from danswer.connectors.salesforce.connector import SalesforceConnector
 from danswer.connectors.sharepoint.connector import SharepointConnector
 from danswer.connectors.slab.connector import SlabConnector
 from danswer.connectors.slack.connector import SlackPollConnector
-from danswer.connectors.slack.load_connector import SlackLoadConnector
 from danswer.connectors.teams.connector import TeamsConnector
 from danswer.connectors.web.connector import WebConnector
 from danswer.connectors.wikipedia.connector import WikipediaConnector
@@ -63,7 +63,6 @@ def identify_connector_class(
         DocumentSource.WEB: WebConnector,
         DocumentSource.FILE: LocalFileConnector,
         DocumentSource.SLACK: {
-            InputType.LOAD_STATE: SlackLoadConnector,
             InputType.POLL: SlackPollConnector,
             InputType.SLIM_RETRIEVAL: SlackPollConnector,
         },
@@ -103,6 +102,7 @@ def identify_connector_class(
         DocumentSource.XENFORO: XenforoConnector,
         DocumentSource.FRESHDESK: FreshdeskConnector,
         DocumentSource.FIREFLIES: FirefliesConnector,
+        DocumentSource.EGNYTE: EgnyteConnector,
     }
     connector_by_source = connector_map.get(source, {})
 

backend/danswer/connectors/file/connector.py

@@ -17,11 +17,11 @@ from danswer.connectors.models import BasicExpertInfo
 from danswer.connectors.models import Document
 from danswer.connectors.models import Section
 from danswer.db.engine import get_session_with_tenant
-from danswer.file_processing.extract_file_text import check_file_ext_is_valid
 from danswer.file_processing.extract_file_text import detect_encoding
 from danswer.file_processing.extract_file_text import extract_file_text
 from danswer.file_processing.extract_file_text import get_file_ext
 from danswer.file_processing.extract_file_text import is_text_file_extension
+from danswer.file_processing.extract_file_text import is_valid_file_ext
 from danswer.file_processing.extract_file_text import load_files_from_zip
 from danswer.file_processing.extract_file_text import read_pdf_file
 from danswer.file_processing.extract_file_text import read_text_file
@@ -50,7 +50,7 @@ def _read_files_and_metadata(
             file_content, ignore_dirs=True
         ):
             yield os.path.join(directory_path, file_info.filename), file, metadata
-    elif check_file_ext_is_valid(extension):
+    elif is_valid_file_ext(extension):
         yield file_name, file_content, metadata
     else:
         logger.warning(f"Skipping file '{file_name}' with extension '{extension}'")
@@ -63,7 +63,7 @@ def _process_file(
     pdf_pass: str | None = None,
 ) -> list[Document]:
     extension = get_file_ext(file_name)
-    if not check_file_ext_is_valid(extension):
+    if not is_valid_file_ext(extension):
         logger.warning(f"Skipping file '{file_name}' with extension '{extension}'")
         return []
 

backend/danswer/connectors/google_drive/connector.py

@@ -4,11 +4,13 @@ from concurrent.futures import as_completed
 from concurrent.futures import ThreadPoolExecutor
 from functools import partial
 from typing import Any
+from typing import cast
 
 from google.oauth2.credentials import Credentials as OAuthCredentials  # type: ignore
 from google.oauth2.service_account import Credentials as ServiceAccountCredentials  # type: ignore
 
 from danswer.configs.app_configs import INDEX_BATCH_SIZE
+from danswer.configs.app_configs import MAX_FILE_SIZE_BYTES
 from danswer.configs.constants import DocumentSource
 from danswer.connectors.google_drive.doc_conversion import build_slim_document
 from danswer.connectors.google_drive.doc_conversion import (
@@ -452,12 +454,14 @@ class GoogleDriveConnector(LoadConnector, PollConnector, SlimConnector):
             if isinstance(self.creds, ServiceAccountCredentials)
             else self._manage_oauth_retrieval
         )
-        return retrieval_method(
+        drive_files = retrieval_method(
             is_slim=is_slim,
             start=start,
             end=end,
         )
 
+        return drive_files
+
     def _extract_docs_from_google_drive(
         self,
         start: SecondsSinceUnixEpoch | None = None,
@@ -473,6 +477,15 @@ class GoogleDriveConnector(LoadConnector, PollConnector, SlimConnector):
         files_to_process = []
         # Gather the files into batches to be processed in parallel
         for file in self._fetch_drive_items(is_slim=False, start=start, end=end):
+            if (
+                file.get("size")
+                and int(cast(str, file.get("size"))) > MAX_FILE_SIZE_BYTES
+            ):
+                logger.warning(
+                    f"Skipping file {file.get('name', 'Unknown')} as it is too large: {file.get('size')} bytes"
+                )
+                continue
+
             files_to_process.append(file)
             if len(files_to_process) >= LARGE_BATCH_SIZE:
                 yield from _process_files_batch(

backend/danswer/connectors/google_drive/file_retrieval.py

@@ -16,7 +16,7 @@ logger = setup_logger()
 
 FILE_FIELDS = (
     "nextPageToken, files(mimeType, id, name, permissions, modifiedTime, webViewLink, "
-    "shortcutDetails, owners(emailAddress))"
+    "shortcutDetails, owners(emailAddress), size)"
 )
 SLIM_FILE_FIELDS = (
     "nextPageToken, files(mimeType, id, name, permissions(emailAddress, type), "

backend/danswer/connectors/interfaces.py

@@ -2,6 +2,7 @@ import abc
 from collections.abc import Iterator
 from typing import Any
 
+from danswer.configs.constants import DocumentSource
 from danswer.connectors.models import Document
 from danswer.connectors.models import SlimDocument
 
@@ -64,6 +65,23 @@ class SlimConnector(BaseConnector):
         raise NotImplementedError
 
 
+class OAuthConnector(BaseConnector):
+    @classmethod
+    @abc.abstractmethod
+    def oauth_id(cls) -> DocumentSource:
+        raise NotImplementedError
+
+    @classmethod
+    @abc.abstractmethod
+    def oauth_authorization_url(cls, base_domain: str, state: str) -> str:
+        raise NotImplementedError
+
+    @classmethod
+    @abc.abstractmethod
+    def oauth_code_to_token(cls, code: str) -> dict[str, Any]:
+        raise NotImplementedError
+
+
 # Event driven
 class EventConnector(BaseConnector):
     @abc.abstractmethod

backend/danswer/connectors/linear/connector.py

@@ -132,7 +132,6 @@ class LinearConnector(LoadConnector, PollConnector):
                             branchName
                             customerTicketCount
                             description
-                            descriptionData
                             comments {
                                 nodes {
                                     url
@@ -215,5 +214,6 @@ class LinearConnector(LoadConnector, PollConnector):
 if __name__ == "__main__":
     connector = LinearConnector()
     connector.load_credentials({"linear_api_key": os.environ["LINEAR_API_KEY"]})
+
     document_batches = connector.load_from_state()
     print(next(document_batches))

backend/danswer/connectors/slack/connector.py

@@ -134,7 +134,6 @@ def get_latest_message_time(thread: ThreadType) -> datetime:
 
 
 def thread_to_doc(
-    workspace: str,
     channel: ChannelType,
     thread: ThreadType,
     slack_cleaner: SlackTextCleaner,
@@ -171,15 +170,15 @@ def thread_to_doc(
         else first_message
     )
 
-    doc_sem_id = f"{initial_sender_name} in #{channel['name']}: {snippet}"
+    doc_sem_id = f"{initial_sender_name} in #{channel['name']}: {snippet}".replace(
+        "\n", " "
+    )
 
     return Document(
         id=f"{channel_id}__{thread[0]['ts']}",
         sections=[
             Section(
-                link=get_message_link(
-                    event=m, workspace=workspace, channel_id=channel_id
-                ),
+                link=get_message_link(event=m, client=client, channel_id=channel_id),
                 text=slack_cleaner.index_clean(cast(str, m["text"])),
             )
             for m in thread
@@ -263,7 +262,6 @@ def filter_channels(
 
 def _get_all_docs(
     client: WebClient,
-    workspace: str,
     channels: list[str] | None = None,
     channel_name_regex_enabled: bool = False,
     oldest: str | None = None,
@@ -310,7 +308,6 @@ def _get_all_docs(
                 if filtered_thread:
                     channel_docs += 1
                     yield thread_to_doc(
-                        workspace=workspace,
                         channel=channel,
                         thread=filtered_thread,
                         slack_cleaner=slack_cleaner,
@@ -373,14 +370,12 @@ def _get_all_doc_ids(
 class SlackPollConnector(PollConnector, SlimConnector):
     def __init__(
         self,
-        workspace: str,
         channels: list[str] | None = None,
         # if specified, will treat the specified channel strings as
         # regexes, and will only index channels that fully match the regexes
         channel_regex_enabled: bool = False,
         batch_size: int = INDEX_BATCH_SIZE,
     ) -> None:
-        self.workspace = workspace
         self.channels = channels
         self.channel_regex_enabled = channel_regex_enabled
         self.batch_size = batch_size
@@ -414,7 +409,6 @@ class SlackPollConnector(PollConnector, SlimConnector):
         documents: list[Document] = []
         for document in _get_all_docs(
             client=self.client,
-            workspace=self.workspace,
             channels=self.channels,
             channel_name_regex_enabled=self.channel_regex_enabled,
             # NOTE: need to impute to `None` instead of using 0.0, since Slack will
@@ -438,7 +432,6 @@ if __name__ == "__main__":
 
     slack_channel = os.environ.get("SLACK_CHANNEL")
     connector = SlackPollConnector(
-        workspace=os.environ["SLACK_WORKSPACE"],
         channels=[slack_channel] if slack_channel else None,
     )
     connector.load_credentials({"slack_bot_token": os.environ["SLACK_BOT_TOKEN"]})

backend/danswer/connectors/slack/load_connector.py

@@ -1,140 +0,0 @@
-import json
-import os
-from datetime import datetime
-from datetime import timezone
-from pathlib import Path
-from typing import Any
-from typing import cast
-
-from danswer.configs.app_configs import INDEX_BATCH_SIZE
-from danswer.configs.constants import DocumentSource
-from danswer.connectors.interfaces import GenerateDocumentsOutput
-from danswer.connectors.interfaces import LoadConnector
-from danswer.connectors.models import Document
-from danswer.connectors.models import Section
-from danswer.connectors.slack.connector import filter_channels
-from danswer.connectors.slack.utils import get_message_link
-from danswer.utils.logger import setup_logger
-
-
-logger = setup_logger()
-
-
-def get_event_time(event: dict[str, Any]) -> datetime | None:
-    ts = event.get("ts")
-    if not ts:
-        return None
-    return datetime.fromtimestamp(float(ts), tz=timezone.utc)
-
-
-class SlackLoadConnector(LoadConnector):
-    # WARNING: DEPRECATED, DO NOT USE
-    def __init__(
-        self,
-        workspace: str,
-        export_path_str: str,
-        channels: list[str] | None = None,
-        # if specified, will treat the specified channel strings as
-        # regexes, and will only index channels that fully match the regexes
-        channel_regex_enabled: bool = False,
-        batch_size: int = INDEX_BATCH_SIZE,
-    ) -> None:
-        self.workspace = workspace
-        self.channels = channels
-        self.channel_regex_enabled = channel_regex_enabled
-        self.export_path_str = export_path_str
-        self.batch_size = batch_size
-
-    def load_credentials(self, credentials: dict[str, Any]) -> dict[str, Any] | None:
-        if credentials:
-            logger.warning("Unexpected credentials provided for Slack Load Connector")
-        return None
-
-    @staticmethod
-    def _process_batch_event(
-        slack_event: dict[str, Any],
-        channel: dict[str, Any],
-        matching_doc: Document | None,
-        workspace: str,
-    ) -> Document | None:
-        if (
-            slack_event["type"] == "message"
-            and slack_event.get("subtype") != "channel_join"
-        ):
-            if matching_doc:
-                return Document(
-                    id=matching_doc.id,
-                    sections=matching_doc.sections
-                    + [
-                        Section(
-                            link=get_message_link(
-                                event=slack_event,
-                                workspace=workspace,
-                                channel_id=channel["id"],
-                            ),
-                            text=slack_event["text"],
-                        )
-                    ],
-                    source=matching_doc.source,
-                    semantic_identifier=matching_doc.semantic_identifier,
-                    title="",  # slack docs don't really have a "title"
-                    doc_updated_at=get_event_time(slack_event),
-                    metadata=matching_doc.metadata,
-                )
-
-            return Document(
-                id=slack_event["ts"],
-                sections=[
-                    Section(
-                        link=get_message_link(
-                            event=slack_event,
-                            workspace=workspace,
-                            channel_id=channel["id"],
-                        ),
-                        text=slack_event["text"],
-                    )
-                ],
-                source=DocumentSource.SLACK,
-                semantic_identifier=channel["name"],
-                title="",  # slack docs don't really have a "title"
-                doc_updated_at=get_event_time(slack_event),
-                metadata={},
-            )
-
-        return None
-
-    def load_from_state(self) -> GenerateDocumentsOutput:
-        export_path = Path(self.export_path_str)
-
-        with open(export_path / "channels.json") as f:
-            all_channels = json.load(f)
-
-        filtered_channels = filter_channels(
-            all_channels, self.channels, self.channel_regex_enabled
-        )
-
-        document_batch: dict[str, Document] = {}
-        for channel_info in filtered_channels:
-            channel_dir_path = export_path / cast(str, channel_info["name"])
-            channel_file_paths = [
-                channel_dir_path / file_name
-                for file_name in os.listdir(channel_dir_path)
-            ]
-            for path in channel_file_paths:
-                with open(path) as f:
-                    events = cast(list[dict[str, Any]], json.load(f))
-                for slack_event in events:
-                    doc = self._process_batch_event(
-                        slack_event=slack_event,
-                        channel=channel_info,
-                        matching_doc=document_batch.get(
-                            slack_event.get("thread_ts", "")
-                        ),
-                        workspace=self.workspace,
-                    )
-                    if doc:
-                        document_batch[doc.id] = doc
-                        if len(document_batch) >= self.batch_size:
-                            yield list(document_batch.values())
-
-        yield list(document_batch.values())

backend/danswer/connectors/slack/utils.py

@@ -2,6 +2,7 @@ import re
 import time
 from collections.abc import Callable
 from collections.abc import Generator
+from functools import lru_cache
 from functools import wraps
 from typing import Any
 from typing import cast
@@ -21,19 +22,21 @@ basic_retry_wrapper = retry_builder()
 _SLACK_LIMIT = 900
 
 
+@lru_cache()
+def get_base_url(token: str) -> str:
+    """Retrieve and cache the base URL of the Slack workspace based on the client token."""
+    client = WebClient(token=token)
+    return client.auth_test()["url"]
+
+
 def get_message_link(
-    event: dict[str, Any], workspace: str, channel_id: str | None = None
+    event: dict[str, Any], client: WebClient, channel_id: str | None = None
 ) -> str:
-    channel_id = channel_id or cast(
-        str, event["channel"]
-    )  # channel must either be present in the event or passed in
-    message_ts = cast(str, event["ts"])
-    message_ts_without_dot = message_ts.replace(".", "")
-    thread_ts = cast(str | None, event.get("thread_ts"))
-    return (
-        f"https://{workspace}.slack.com/archives/{channel_id}/p{message_ts_without_dot}"
-        + (f"?thread_ts={thread_ts}" if thread_ts else "")
-    )
+    channel_id = channel_id or event["channel"]
+    message_ts = event["ts"]
+    response = client.chat_getPermalink(channel=channel_id, message_ts=message_ts)
+    permalink = response["permalink"]
+    return permalink
 
 
 def _make_slack_api_call_logged(

backend/danswer/connectors/teams/connector.py

@@ -33,7 +33,7 @@ def get_created_datetime(chat_message: ChatMessage) -> datetime:
 
 def _extract_channel_members(channel: Channel) -> list[BasicExpertInfo]:
     channel_members_list: list[BasicExpertInfo] = []
-    members = channel.members.get().execute_query()
+    members = channel.members.get().execute_query_retry()
     for member in members:
         channel_members_list.append(BasicExpertInfo(display_name=member.display_name))
     return channel_members_list
@@ -51,7 +51,7 @@ def _get_threads_from_channel(
         end = end.replace(tzinfo=timezone.utc)
 
     query = channel.messages.get()
-    base_messages: list[ChatMessage] = query.execute_query()
+    base_messages: list[ChatMessage] = query.execute_query_retry()
 
     threads: list[list[ChatMessage]] = []
     for base_message in base_messages:
@@ -65,7 +65,7 @@ def _get_threads_from_channel(
             continue
 
         reply_query = base_message.replies.get_all()
-        replies = reply_query.execute_query()
+        replies = reply_query.execute_query_retry()
 
         # start a list containing the base message and its replies
         thread: list[ChatMessage] = [base_message]
@@ -82,7 +82,7 @@ def _get_channels_from_teams(
     channels_list: list[Channel] = []
     for team in teams:
         query = team.channels.get()
-        channels = query.execute_query()
+        channels = query.execute_query_retry()
         channels_list.extend(channels)
 
     return channels_list
@@ -210,7 +210,7 @@ class TeamsConnector(LoadConnector, PollConnector):
 
         teams_list: list[Team] = []
 
-        teams = self.graph_client.teams.get().execute_query()
+        teams = self.graph_client.teams.get().execute_query_retry()
 
         if len(self.requested_team_list) > 0:
             adjusted_request_strings = [
@@ -234,14 +234,25 @@ class TeamsConnector(LoadConnector, PollConnector):
             raise ConnectorMissingCredentialError("Teams")
 
         teams = self._get_all_teams()
+        logger.debug(f"Found available teams: {[str(t) for t in teams]}")
+        if not teams:
+            msg = "No teams found."
+            logger.error(msg)
+            raise ValueError(msg)
 
         channels = _get_channels_from_teams(
             teams=teams,
         )
+        logger.debug(f"Found available channels: {[c.id for c in channels]}")
+        if not channels:
+            msg = "No channels found."
+            logger.error(msg)
+            raise ValueError(msg)
 
         # goes over channels, converts them into Document objects and then yields them in batches
         doc_batch: list[Document] = []
         for channel in channels:
+            logger.debug(f"Fetching threads from channel: {channel.id}")
             thread_list = _get_threads_from_channel(channel, start=start, end=end)
             for thread in thread_list:
                 converted_doc = _convert_thread_to_document(channel, thread)
@@ -259,8 +270,8 @@ class TeamsConnector(LoadConnector, PollConnector):
     def poll_source(
         self, start: SecondsSinceUnixEpoch, end: SecondsSinceUnixEpoch
     ) -> GenerateDocumentsOutput:
-        start_datetime = datetime.utcfromtimestamp(start)
-        end_datetime = datetime.utcfromtimestamp(end)
+        start_datetime = datetime.fromtimestamp(start, timezone.utc)
+        end_datetime = datetime.fromtimestamp(end, timezone.utc)
         return self._fetch_from_teams(start=start_datetime, end=end_datetime)
 
 

backend/danswer/context/search/pipeline.py

@@ -5,7 +5,11 @@ from typing import cast
 
 from sqlalchemy.orm import Session
 
+from danswer.chat.models import PromptConfig
 from danswer.chat.models import SectionRelevancePiece
+from danswer.chat.prune_and_merge import _merge_sections
+from danswer.chat.prune_and_merge import ChunkRange
+from danswer.chat.prune_and_merge import merge_chunk_intervals
 from danswer.configs.chat_configs import DISABLE_LLM_DOC_RELEVANCE
 from danswer.context.search.enums import LLMEvaluationType
 from danswer.context.search.enums import QueryFlow
@@ -27,10 +31,6 @@ from danswer.db.models import User
 from danswer.db.search_settings import get_current_search_settings
 from danswer.document_index.factory import get_default_document_index
 from danswer.document_index.interfaces import VespaChunkRequest
-from danswer.llm.answering.models import PromptConfig
-from danswer.llm.answering.prune_and_merge import _merge_sections
-from danswer.llm.answering.prune_and_merge import ChunkRange
-from danswer.llm.answering.prune_and_merge import merge_chunk_intervals
 from danswer.llm.interfaces import LLM
 from danswer.secondary_llm_flows.agentic_evaluation import evaluate_inference_section
 from danswer.utils.logger import setup_logger

backend/danswer/danswerbot/slack/blocks.py

@@ -204,7 +204,8 @@ def _build_documents_blocks(
             continue
         seen_docs_identifiers.add(d.document_id)
 
-        doc_sem_id = d.semantic_identifier
+        # Strip newlines from the semantic identifier for Slackbot formatting
+        doc_sem_id = d.semantic_identifier.replace("\n", " ")
         if d.source_type == DocumentSource.SLACK.value:
             doc_sem_id = "#" + doc_sem_id
 

backend/danswer/danswerbot/slack/handlers/handle_regular_answer.py

@@ -373,7 +373,9 @@ def handle_regular_answer(
         respond_in_thread(
             client=client,
             channel=channel,
-            receiver_ids=receiver_ids,
+            receiver_ids=[message_info.sender]
+            if message_info.is_bot_msg and message_info.sender
+            else receiver_ids,
             text="Hello! Danswer has some results for you!",
             blocks=all_blocks,
             thread_ts=message_ts_to_respond_to,

backend/danswer/danswerbot/slack/utils.py

@@ -11,6 +11,7 @@ from retry import retry
 from slack_sdk import WebClient
 from slack_sdk.errors import SlackApiError
 from slack_sdk.models.blocks import Block
+from slack_sdk.models.blocks import SectionBlock
 from slack_sdk.models.metadata import Metadata
 from slack_sdk.socket_mode import SocketModeClient
 
@@ -140,6 +141,40 @@ def remove_danswer_bot_tag(message_str: str, client: WebClient) -> str:
     return re.sub(rf"<@{bot_tag_id}>\s", "", message_str)
 
 
+def _check_for_url_in_block(block: Block) -> bool:
+    """
+    Check if the block has a key that contains "url" in it
+    """
+    block_dict = block.to_dict()
+
+    def check_dict_for_url(d: dict) -> bool:
+        for key, value in d.items():
+            if "url" in key.lower():
+                return True
+            if isinstance(value, dict):
+                if check_dict_for_url(value):
+                    return True
+            elif isinstance(value, list):
+                for item in value:
+                    if isinstance(item, dict) and check_dict_for_url(item):
+                        return True
+        return False
+
+    return check_dict_for_url(block_dict)
+
+
+def _build_error_block(error_message: str) -> Block:
+    """
+    Build an error block to display in slack so that the user can see
+    the error without completely breaking
+    """
+    display_text = (
+        "There was an error displaying all of the Onyx answers."
+        f" Please let an admin or an onyx developer know. Error: {error_message}"
+    )
+    return SectionBlock(text=display_text)
+
+
 @retry(
     tries=DANSWER_BOT_NUM_RETRIES,
     delay=0.25,
@@ -162,24 +197,9 @@ def respond_in_thread(
     message_ids: list[str] = []
     if not receiver_ids:
         slack_call = make_slack_api_rate_limited(client.chat_postMessage)
-        response = slack_call(
-            channel=channel,
-            text=text,
-            blocks=blocks,
-            thread_ts=thread_ts,
-            metadata=metadata,
-            unfurl_links=unfurl,
-            unfurl_media=unfurl,
-        )
-        if not response.get("ok"):
-            raise RuntimeError(f"Failed to post message: {response}")
-        message_ids.append(response["message_ts"])
-    else:
-        slack_call = make_slack_api_rate_limited(client.chat_postEphemeral)
-        for receiver in receiver_ids:
+        try:
             response = slack_call(
                 channel=channel,
-                user=receiver,
                 text=text,
                 blocks=blocks,
                 thread_ts=thread_ts,
@@ -187,8 +207,68 @@ def respond_in_thread(
                 unfurl_links=unfurl,
                 unfurl_media=unfurl,
             )
-            if not response.get("ok"):
-                raise RuntimeError(f"Failed to post message: {response}")
+        except Exception as e:
+            logger.warning(f"Failed to post message: {e} \n blocks: {blocks}")
+            logger.warning("Trying again without blocks that have urls")
+
+            if not blocks:
+                raise e
+
+            blocks_without_urls = [
+                block for block in blocks if not _check_for_url_in_block(block)
+            ]
+            blocks_without_urls.append(_build_error_block(str(e)))
+
+            # Try again wtihout blocks containing url
+            response = slack_call(
+                channel=channel,
+                text=text,
+                blocks=blocks_without_urls,
+                thread_ts=thread_ts,
+                metadata=metadata,
+                unfurl_links=unfurl,
+                unfurl_media=unfurl,
+            )
+
+        message_ids.append(response["message_ts"])
+    else:
+        slack_call = make_slack_api_rate_limited(client.chat_postEphemeral)
+        for receiver in receiver_ids:
+            try:
+                response = slack_call(
+                    channel=channel,
+                    user=receiver,
+                    text=text,
+                    blocks=blocks,
+                    thread_ts=thread_ts,
+                    metadata=metadata,
+                    unfurl_links=unfurl,
+                    unfurl_media=unfurl,
+                )
+            except Exception as e:
+                logger.warning(f"Failed to post message: {e} \n blocks: {blocks}")
+                logger.warning("Trying again without blocks that have urls")
+
+                if not blocks:
+                    raise e
+
+                blocks_without_urls = [
+                    block for block in blocks if not _check_for_url_in_block(block)
+                ]
+                blocks_without_urls.append(_build_error_block(str(e)))
+
+                # Try again wtihout blocks containing url
+                response = slack_call(
+                    channel=channel,
+                    user=receiver,
+                    text=text,
+                    blocks=blocks_without_urls,
+                    thread_ts=thread_ts,
+                    metadata=metadata,
+                    unfurl_links=unfurl,
+                    unfurl_media=unfurl,
+                )
+
             message_ids.append(response["message_ts"])
 
     return message_ids

backend/danswer/db/credentials.py

@@ -20,7 +20,6 @@ from danswer.db.models import DocumentByConnectorCredentialPair
 from danswer.db.models import User
 from danswer.db.models import User__UserGroup
 from danswer.server.documents.models import CredentialBase
-from danswer.server.documents.models import CredentialDataUpdateRequest
 from danswer.utils.logger import setup_logger
 
 
@@ -248,7 +247,6 @@ def create_credential(
     )
 
     db_session.commit()
-
     return credential
 
 
@@ -263,7 +261,8 @@ def _cleanup_credential__user_group_relationships__no_commit(
 
 def alter_credential(
     credential_id: int,
-    credential_data: CredentialDataUpdateRequest,
+    name: str,
+    credential_json: dict[str, Any],
     user: User,
     db_session: Session,
 ) -> Credential | None:
@@ -273,11 +272,13 @@ def alter_credential(
     if credential is None:
         return None
 
-    credential.name = credential_data.name
+    credential.name = name
 
-    # Update only the keys present in credential_data.credential_json
-    for key, value in credential_data.credential_json.items():
-        credential.credential_json[key] = value
+    # Assign a new dictionary to credential.credential_json
+    credential.credential_json = {
+        **credential.credential_json,
+        **credential_json,
+    }
 
     credential.user_id = user.id if user is not None else None
     db_session.commit()
@@ -310,8 +311,8 @@ def update_credential_json(
     credential = fetch_credential_by_id(credential_id, user, db_session)
     if credential is None:
         return None
-    credential.credential_json = credential_json
 
+    credential.credential_json = credential_json
     db_session.commit()
     return credential
 

backend/danswer/db/index_attempt.py

@@ -522,12 +522,16 @@ def expire_index_attempts(
     search_settings_id: int,
     db_session: Session,
 ) -> None:
-    delete_query = (
-        delete(IndexAttempt)
+    not_started_query = (
+        update(IndexAttempt)
         .where(IndexAttempt.search_settings_id == search_settings_id)
         .where(IndexAttempt.status == IndexingStatus.NOT_STARTED)
+        .values(
+            status=IndexingStatus.CANCELED,
+            error_msg="Canceled, likely due to model swap",
+        )
     )
-    db_session.execute(delete_query)
+    db_session.execute(not_started_query)
 
     update_query = (
         update(IndexAttempt)
@@ -549,9 +553,14 @@ def cancel_indexing_attempts_for_ccpair(
     include_secondary_index: bool = False,
 ) -> None:
     stmt = (
-        delete(IndexAttempt)
+        update(IndexAttempt)
         .where(IndexAttempt.connector_credential_pair_id == cc_pair_id)
         .where(IndexAttempt.status == IndexingStatus.NOT_STARTED)
+        .values(
+            status=IndexingStatus.CANCELED,
+            error_msg="Canceled by user",
+            time_started=datetime.now(timezone.utc),
+        )
     )
 
     if not include_secondary_index:

backend/danswer/db/input_prompt.py

@@ -1,202 +0,0 @@
-from uuid import UUID
-
-from fastapi import HTTPException
-from sqlalchemy import select
-from sqlalchemy.orm import Session
-
-from danswer.db.models import InputPrompt
-from danswer.db.models import User
-from danswer.server.features.input_prompt.models import InputPromptSnapshot
-from danswer.server.manage.models import UserInfo
-from danswer.utils.logger import setup_logger
-
-
-logger = setup_logger()
-
-
-def insert_input_prompt_if_not_exists(
-    user: User | None,
-    input_prompt_id: int | None,
-    prompt: str,
-    content: str,
-    active: bool,
-    is_public: bool,
-    db_session: Session,
-    commit: bool = True,
-) -> InputPrompt:
-    if input_prompt_id is not None:
-        input_prompt = (
-            db_session.query(InputPrompt).filter_by(id=input_prompt_id).first()
-        )
-    else:
-        query = db_session.query(InputPrompt).filter(InputPrompt.prompt == prompt)
-        if user:
-            query = query.filter(InputPrompt.user_id == user.id)
-        else:
-            query = query.filter(InputPrompt.user_id.is_(None))
-        input_prompt = query.first()
-
-    if input_prompt is None:
-        input_prompt = InputPrompt(
-            id=input_prompt_id,
-            prompt=prompt,
-            content=content,
-            active=active,
-            is_public=is_public or user is None,
-            user_id=user.id if user else None,
-        )
-        db_session.add(input_prompt)
-
-    if commit:
-        db_session.commit()
-
-    return input_prompt
-
-
-def insert_input_prompt(
-    prompt: str,
-    content: str,
-    is_public: bool,
-    user: User | None,
-    db_session: Session,
-) -> InputPrompt:
-    input_prompt = InputPrompt(
-        prompt=prompt,
-        content=content,
-        active=True,
-        is_public=is_public or user is None,
-        user_id=user.id if user is not None else None,
-    )
-    db_session.add(input_prompt)
-    db_session.commit()
-
-    return input_prompt
-
-
-def update_input_prompt(
-    user: User | None,
-    input_prompt_id: int,
-    prompt: str,
-    content: str,
-    active: bool,
-    db_session: Session,
-) -> InputPrompt:
-    input_prompt = db_session.scalar(
-        select(InputPrompt).where(InputPrompt.id == input_prompt_id)
-    )
-    if input_prompt is None:
-        raise ValueError(f"No input prompt with id {input_prompt_id}")
-
-    if not validate_user_prompt_authorization(user, input_prompt):
-        raise HTTPException(status_code=401, detail="You don't own this prompt")
-
-    input_prompt.prompt = prompt
-    input_prompt.content = content
-    input_prompt.active = active
-
-    db_session.commit()
-    return input_prompt
-
-
-def validate_user_prompt_authorization(
-    user: User | None, input_prompt: InputPrompt
-) -> bool:
-    prompt = InputPromptSnapshot.from_model(input_prompt=input_prompt)
-
-    if prompt.user_id is not None:
-        if user is None:
-            return False
-
-        user_details = UserInfo.from_model(user)
-        if str(user_details.id) != str(prompt.user_id):
-            return False
-    return True
-
-
-def remove_public_input_prompt(input_prompt_id: int, db_session: Session) -> None:
-    input_prompt = db_session.scalar(
-        select(InputPrompt).where(InputPrompt.id == input_prompt_id)
-    )
-
-    if input_prompt is None:
-        raise ValueError(f"No input prompt with id {input_prompt_id}")
-
-    if not input_prompt.is_public:
-        raise HTTPException(status_code=400, detail="This prompt is not public")
-
-    db_session.delete(input_prompt)
-    db_session.commit()
-
-
-def remove_input_prompt(
-    user: User | None, input_prompt_id: int, db_session: Session
-) -> None:
-    input_prompt = db_session.scalar(
-        select(InputPrompt).where(InputPrompt.id == input_prompt_id)
-    )
-    if input_prompt is None:
-        raise ValueError(f"No input prompt with id {input_prompt_id}")
-
-    if input_prompt.is_public:
-        raise HTTPException(
-            status_code=400, detail="Cannot delete public prompts with this method"
-        )
-
-    if not validate_user_prompt_authorization(user, input_prompt):
-        raise HTTPException(status_code=401, detail="You do not own this prompt")
-
-    db_session.delete(input_prompt)
-    db_session.commit()
-
-
-def fetch_input_prompt_by_id(
-    id: int, user_id: UUID | None, db_session: Session
-) -> InputPrompt:
-    query = select(InputPrompt).where(InputPrompt.id == id)
-
-    if user_id:
-        query = query.where(
-            (InputPrompt.user_id == user_id) | (InputPrompt.user_id is None)
-        )
-    else:
-        # If no user_id is provided, only fetch prompts without a user_id (aka public)
-        query = query.where(InputPrompt.user_id == None)  # noqa
-
-    result = db_session.scalar(query)
-
-    if result is None:
-        raise HTTPException(422, "No input prompt found")
-
-    return result
-
-
-def fetch_public_input_prompts(
-    db_session: Session,
-) -> list[InputPrompt]:
-    query = select(InputPrompt).where(InputPrompt.is_public)
-    return list(db_session.scalars(query).all())
-
-
-def fetch_input_prompts_by_user(
-    db_session: Session,
-    user_id: UUID | None,
-    active: bool | None = None,
-    include_public: bool = False,
-) -> list[InputPrompt]:
-    query = select(InputPrompt)
-
-    if user_id is not None:
-        if include_public:
-            query = query.where(
-                (InputPrompt.user_id == user_id) | InputPrompt.is_public
-            )
-        else:
-            query = query.where(InputPrompt.user_id == user_id)
-
-    elif include_public:
-        query = query.where(InputPrompt.is_public)
-
-    if active is not None:
-        query = query.where(InputPrompt.active == active)
-
-    return list(db_session.scalars(query).all())

backend/danswer/db/models.py

@@ -159,9 +159,6 @@ class User(SQLAlchemyBaseUserTableUUID, Base):
     )
 
     prompts: Mapped[list["Prompt"]] = relationship("Prompt", back_populates="user")
-    input_prompts: Mapped[list["InputPrompt"]] = relationship(
-        "InputPrompt", back_populates="user"
-    )
 
     # Personas owned by this user
     personas: Mapped[list["Persona"]] = relationship("Persona", back_populates="user")
@@ -178,31 +175,6 @@ class User(SQLAlchemyBaseUserTableUUID, Base):
     )
 
 
-class InputPrompt(Base):
-    __tablename__ = "inputprompt"
-
-    id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
-    prompt: Mapped[str] = mapped_column(String)
-    content: Mapped[str] = mapped_column(String)
-    active: Mapped[bool] = mapped_column(Boolean)
-    user: Mapped[User | None] = relationship("User", back_populates="input_prompts")
-    is_public: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
-    user_id: Mapped[UUID | None] = mapped_column(
-        ForeignKey("user.id", ondelete="CASCADE"), nullable=True
-    )
-
-
-class InputPrompt__User(Base):
-    __tablename__ = "inputprompt__user"
-
-    input_prompt_id: Mapped[int] = mapped_column(
-        ForeignKey("inputprompt.id"), primary_key=True
-    )
-    user_id: Mapped[UUID | None] = mapped_column(
-        ForeignKey("inputprompt.id"), primary_key=True
-    )
-
-
 class AccessToken(SQLAlchemyBaseAccessTokenTableUUID, Base):
     pass
 
@@ -596,6 +568,25 @@ class Connector(Base):
         list["DocumentByConnectorCredentialPair"]
     ] = relationship("DocumentByConnectorCredentialPair", back_populates="connector")
 
+    # synchronize this validation logic with RefreshFrequencySchema etc on front end
+    # until we have a centralized validation schema
+
+    # TODO(rkuo): experiment with SQLAlchemy validators rather than manual checks
+    # https://docs.sqlalchemy.org/en/20/orm/mapped_attributes.html
+    def validate_refresh_freq(self) -> None:
+        if self.refresh_freq is not None:
+            if self.refresh_freq < 60:
+                raise ValueError(
+                    "refresh_freq must be greater than or equal to 60 seconds."
+                )
+
+    def validate_prune_freq(self) -> None:
+        if self.prune_freq is not None:
+            if self.prune_freq < 86400:
+                raise ValueError(
+                    "prune_freq must be greater than or equal to 86400 seconds."
+                )
+
 
 class Credential(Base):
     __tablename__ = "credential"
@@ -1490,7 +1481,9 @@ class SlackChannelConfig(Base):
     __tablename__ = "slack_channel_config"
 
     id: Mapped[int] = mapped_column(primary_key=True)
-    slack_bot_id: Mapped[int] = mapped_column(ForeignKey("slack_bot.id"), nullable=True)
+    slack_bot_id: Mapped[int] = mapped_column(
+        ForeignKey("slack_bot.id"), nullable=False
+    )
     persona_id: Mapped[int | None] = mapped_column(
         ForeignKey("persona.id"), nullable=True
     )
@@ -1528,6 +1521,7 @@ class SlackBot(Base):
     slack_channel_configs: Mapped[list[SlackChannelConfig]] = relationship(
         "SlackChannelConfig",
         back_populates="slack_bot",
+        cascade="all, delete-orphan",
     )
 
 

backend/danswer/db/persona.py

@@ -453,9 +453,9 @@ def upsert_persona(
     """
 
     if persona_id is not None:
-        persona = db_session.query(Persona).filter_by(id=persona_id).first()
+        existing_persona = db_session.query(Persona).filter_by(id=persona_id).first()
     else:
-        persona = _get_persona_by_name(
+        existing_persona = _get_persona_by_name(
             persona_name=name, user=user, db_session=db_session
         )
 
@@ -481,62 +481,78 @@ def upsert_persona(
     prompts = None
     if prompt_ids is not None:
         prompts = db_session.query(Prompt).filter(Prompt.id.in_(prompt_ids)).all()
-        if not prompts and prompt_ids:
-            raise ValueError("prompts not found")
+
+    if prompts is not None and len(prompts) == 0:
+        raise ValueError(
+            f"Invalid Persona config, no valid prompts "
+            f"specified. Specified IDs were: '{prompt_ids}'"
+        )
 
     # ensure all specified tools are valid
     if tools:
         validate_persona_tools(tools)
 
-    if persona:
+    if existing_persona:
         # Built-in personas can only be updated through YAML configuration.
         # This ensures that core system personas are not modified unintentionally.
-        if persona.builtin_persona and not builtin_persona:
+        if existing_persona.builtin_persona and not builtin_persona:
             raise ValueError("Cannot update builtin persona with non-builtin.")
 
         # this checks if the user has permission to edit the persona
-        persona = fetch_persona_by_id(
-            db_session=db_session, persona_id=persona.id, user=user, get_editable=True
+        # will raise an Exception if the user does not have permission
+        existing_persona = fetch_persona_by_id(
+            db_session=db_session,
+            persona_id=existing_persona.id,
+            user=user,
+            get_editable=True,
         )
 
         # The following update excludes `default`, `built-in`, and display priority.
         # Display priority is handled separately in the `display-priority` endpoint.
         # `default` and `built-in` properties can only be set when creating a persona.
-        persona.name = name
-        persona.description = description
-        persona.num_chunks = num_chunks
-        persona.chunks_above = chunks_above
-        persona.chunks_below = chunks_below
-        persona.llm_relevance_filter = llm_relevance_filter
-        persona.llm_filter_extraction = llm_filter_extraction
-        persona.recency_bias = recency_bias
-        persona.llm_model_provider_override = llm_model_provider_override
-        persona.llm_model_version_override = llm_model_version_override
-        persona.starter_messages = starter_messages
-        persona.deleted = False  # Un-delete if previously deleted
-        persona.is_public = is_public
-        persona.icon_color = icon_color
-        persona.icon_shape = icon_shape
+        existing_persona.name = name
+        existing_persona.description = description
+        existing_persona.num_chunks = num_chunks
+        existing_persona.chunks_above = chunks_above
+        existing_persona.chunks_below = chunks_below
+        existing_persona.llm_relevance_filter = llm_relevance_filter
+        existing_persona.llm_filter_extraction = llm_filter_extraction
+        existing_persona.recency_bias = recency_bias
+        existing_persona.llm_model_provider_override = llm_model_provider_override
+        existing_persona.llm_model_version_override = llm_model_version_override
+        existing_persona.starter_messages = starter_messages
+        existing_persona.deleted = False  # Un-delete if previously deleted
+        existing_persona.is_public = is_public
+        existing_persona.icon_color = icon_color
+        existing_persona.icon_shape = icon_shape
         if remove_image or uploaded_image_id:
-            persona.uploaded_image_id = uploaded_image_id
-        persona.is_visible = is_visible
-        persona.search_start_date = search_start_date
-        persona.category_id = category_id
+            existing_persona.uploaded_image_id = uploaded_image_id
+        existing_persona.is_visible = is_visible
+        existing_persona.search_start_date = search_start_date
+        existing_persona.category_id = category_id
         # Do not delete any associations manually added unless
         # a new updated list is provided
         if document_sets is not None:
-            persona.document_sets.clear()
-            persona.document_sets = document_sets or []
+            existing_persona.document_sets.clear()
+            existing_persona.document_sets = document_sets or []
 
         if prompts is not None:
-            persona.prompts.clear()
-            persona.prompts = prompts or []
+            existing_persona.prompts.clear()
+            existing_persona.prompts = prompts
 
         if tools is not None:
-            persona.tools = tools or []
+            existing_persona.tools = tools or []
+
+        persona = existing_persona
 
     else:
-        persona = Persona(
+        if not prompts:
+            raise ValueError(
+                "Invalid Persona config. "
+                "Must specify at least one prompt for a new persona."
+            )
+
+        new_persona = Persona(
             id=persona_id,
             user_id=user.id if user else None,
             is_public=is_public,
@@ -549,7 +565,7 @@ def upsert_persona(
             llm_filter_extraction=llm_filter_extraction,
             recency_bias=recency_bias,
             builtin_persona=builtin_persona,
-            prompts=prompts or [],
+            prompts=prompts,
             document_sets=document_sets or [],
             llm_model_provider_override=llm_model_provider_override,
             llm_model_version_override=llm_model_version_override,
@@ -564,8 +580,8 @@ def upsert_persona(
             is_default_persona=is_default_persona,
             category_id=category_id,
         )
-        db_session.add(persona)
-
+        db_session.add(new_persona)
+        persona = new_persona
     if commit:
         db_session.commit()
     else:

backend/danswer/document_index/vespa/app_config/schemas/danswer_chunk.sd

@@ -4,6 +4,8 @@ schema DANSWER_CHUNK_NAME {
         # Not to be confused with the UUID generated for this chunk which is called documentid by default
         field document_id type string {
             indexing: summary | attribute
+            attribute: fast-search
+            rank: filter
         }
         field chunk_id type int {
             indexing: summary | attribute

backend/danswer/file_processing/extract_file_text.py

@@ -6,6 +6,7 @@ import zipfile
 from collections.abc import Callable
 from collections.abc import Iterator
 from email.parser import Parser as EmailParser
+from io import BytesIO
 from pathlib import Path
 from typing import Any
 from typing import Dict
@@ -15,13 +16,17 @@ import chardet
 import docx  # type: ignore
 import openpyxl  # type: ignore
 import pptx  # type: ignore
+from docx import Document
+from fastapi import UploadFile
 from pypdf import PdfReader
 from pypdf.errors import PdfStreamError
 
 from danswer.configs.constants import DANSWER_METADATA_FILENAME
+from danswer.configs.constants import FileOrigin
 from danswer.file_processing.html_utils import parse_html_page_basic
 from danswer.file_processing.unstructured import get_unstructured_api_key
 from danswer.file_processing.unstructured import unstructured_to_text
+from danswer.file_store.file_store import FileStore
 from danswer.utils.logger import setup_logger
 
 logger = setup_logger()
@@ -65,7 +70,7 @@ def get_file_ext(file_path_or_name: str | Path) -> str:
     return extension
 
 
-def check_file_ext_is_valid(ext: str) -> bool:
+def is_valid_file_ext(ext: str) -> bool:
     return ext in VALID_FILE_EXTENSIONS
 
 
@@ -359,7 +364,7 @@ def extract_file_text(
             elif file_name is not None:
                 final_extension = get_file_ext(file_name)
 
-            if check_file_ext_is_valid(final_extension):
+            if is_valid_file_ext(final_extension):
                 return extension_to_function.get(final_extension, file_io_to_text)(file)
 
         # Either the file somehow has no name or the extension is not one that we recognize
@@ -375,3 +380,35 @@ def extract_file_text(
             ) from e
         logger.warning(f"Failed to process file {file_name or 'Unknown'}: {str(e)}")
         return ""
+
+
+def convert_docx_to_txt(
+    file: UploadFile, file_store: FileStore, file_path: str
+) -> None:
+    file.file.seek(0)
+    docx_content = file.file.read()
+    doc = Document(BytesIO(docx_content))
+
+    # Extract text from the document
+    full_text = []
+    for para in doc.paragraphs:
+        full_text.append(para.text)
+
+    # Join the extracted text
+    text_content = "\n".join(full_text)
+
+    txt_file_path = docx_to_txt_filename(file_path)
+    file_store.save_file(
+        file_name=txt_file_path,
+        content=BytesIO(text_content.encode("utf-8")),
+        display_name=file.filename,
+        file_origin=FileOrigin.CONNECTOR,
+        file_type="text/plain",
+    )
+
+
+def docx_to_txt_filename(file_path: str) -> str:
+    """
+    Convert a .docx file path to its corresponding .txt file path.
+    """
+    return file_path.rsplit(".", 1)[0] + ".txt"

backend/danswer/file_store/utils.py

@@ -1,6 +1,6 @@
+import base64
 from collections.abc import Callable
 from io import BytesIO
-from typing import Any
 from typing import cast
 from uuid import uuid4
 
@@ -13,8 +13,8 @@ from danswer.db.models import ChatMessage
 from danswer.file_store.file_store import get_default_file_store
 from danswer.file_store.models import FileDescriptor
 from danswer.file_store.models import InMemoryChatFile
+from danswer.utils.b64 import get_image_type
 from danswer.utils.threadpool_concurrency import run_functions_tuples_in_parallel
-from shared_configs.contextvars import CURRENT_TENANT_ID_CONTEXTVAR
 
 
 def load_chat_file(
@@ -75,11 +75,58 @@ def save_file_from_url(url: str, tenant_id: str) -> str:
         return unique_id
 
 
-def save_files_from_urls(urls: list[str]) -> list[str]:
-    tenant_id = CURRENT_TENANT_ID_CONTEXTVAR.get()
+def save_file_from_base64(base64_string: str, tenant_id: str) -> str:
+    with get_session_with_tenant(tenant_id) as db_session:
+        unique_id = str(uuid4())
+        file_store = get_default_file_store(db_session)
+        file_store.save_file(
+            file_name=unique_id,
+            content=BytesIO(base64.b64decode(base64_string)),
+            display_name="GeneratedImage",
+            file_origin=FileOrigin.CHAT_IMAGE_GEN,
+            file_type=get_image_type(base64_string),
+        )
+        return unique_id
 
-    funcs: list[tuple[Callable[..., Any], tuple[Any, ...]]] = [
-        (save_file_from_url, (url, tenant_id)) for url in urls
+
+def save_file(
+    tenant_id: str,
+    url: str | None = None,
+    base64_data: str | None = None,
+) -> str:
+    """Save a file from either a URL or base64 encoded string.
+
+    Args:
+        tenant_id: The tenant ID to save the file under
+        url: URL to download file from
+        base64_data: Base64 encoded file data
+
+    Returns:
+        The unique ID of the saved file
+
+    Raises:
+        ValueError: If neither url nor base64_data is provided, or if both are provided
+    """
+    if url is not None and base64_data is not None:
+        raise ValueError("Cannot specify both url and base64_data")
+
+    if url is not None:
+        return save_file_from_url(url, tenant_id)
+    elif base64_data is not None:
+        return save_file_from_base64(base64_data, tenant_id)
+    else:
+        raise ValueError("Must specify either url or base64_data")
+
+
+def save_files(urls: list[str], base64_files: list[str], tenant_id: str) -> list[str]:
+    # NOTE: be explicit about typing so that if we change things, we get notified
+    funcs: list[
+        tuple[
+            Callable[[str, str | None, str | None], str],
+            tuple[str, str | None, str | None],
+        ]
+    ] = [(save_file, (tenant_id, url, None)) for url in urls] + [
+        (save_file, (tenant_id, None, base64_file)) for base64_file in base64_files
     ]
-    # Must pass in tenant_id here, since this is called by multithreading
+
     return run_functions_tuples_in_parallel(funcs)

backend/danswer/indexing/indexing_pipeline.py

@@ -1,4 +1,5 @@
 import traceback
+from collections.abc import Callable
 from functools import partial
 from http import HTTPStatus
 from typing import Protocol
@@ -12,6 +13,7 @@ from danswer.access.access import get_access_for_documents
 from danswer.access.models import DocumentAccess
 from danswer.configs.app_configs import ENABLE_MULTIPASS_INDEXING
 from danswer.configs.app_configs import INDEXING_EXCEPTION_LIMIT
+from danswer.configs.app_configs import MAX_DOCUMENT_CHARS
 from danswer.configs.constants import DEFAULT_BOOST
 from danswer.connectors.cross_connector_utils.miscellaneous_utils import (
     get_experts_stores_representations,
@@ -202,40 +204,13 @@ def index_doc_batch_with_handler(
 
 
 def index_doc_batch_prepare(
-    document_batch: list[Document],
+    documents: list[Document],
     index_attempt_metadata: IndexAttemptMetadata,
     db_session: Session,
     ignore_time_skip: bool = False,
 ) -> DocumentBatchPrepareContext | None:
     """Sets up the documents in the relational DB (source of truth) for permissions, metadata, etc.
     This preceeds indexing it into the actual document index."""
-    documents: list[Document] = []
-    for document in document_batch:
-        empty_contents = not any(section.text.strip() for section in document.sections)
-        if (
-            (not document.title or not document.title.strip())
-            and not document.semantic_identifier.strip()
-            and empty_contents
-        ):
-            # Skip documents that have neither title nor content
-            # If the document doesn't have either, then there is no useful information in it
-            # This is again verified later in the pipeline after chunking but at that point there should
-            # already be no documents that are empty.
-            logger.warning(
-                f"Skipping document with ID {document.id} as it has neither title nor content."
-            )
-            continue
-
-        if document.title is not None and not document.title.strip() and empty_contents:
-            # The title is explicitly empty ("" and not None) and the document is empty
-            # so when building the chunk text representation, it will be empty and unuseable
-            logger.warning(
-                f"Skipping document with ID {document.id} as the chunks will be empty."
-            )
-            continue
-
-        documents.append(document)
-
     # Create a trimmed list of docs that don't have a newer updated at
     # Shortcuts the time-consuming flow on connector index retries
     document_ids: list[str] = [document.id for document in documents]
@@ -282,17 +257,64 @@ def index_doc_batch_prepare(
     )
 
 
+def filter_documents(document_batch: list[Document]) -> list[Document]:
+    documents: list[Document] = []
+    for document in document_batch:
+        empty_contents = not any(section.text.strip() for section in document.sections)
+        if (
+            (not document.title or not document.title.strip())
+            and not document.semantic_identifier.strip()
+            and empty_contents
+        ):
+            # Skip documents that have neither title nor content
+            # If the document doesn't have either, then there is no useful information in it
+            # This is again verified later in the pipeline after chunking but at that point there should
+            # already be no documents that are empty.
+            logger.warning(
+                f"Skipping document with ID {document.id} as it has neither title nor content."
+            )
+            continue
+
+        if document.title is not None and not document.title.strip() and empty_contents:
+            # The title is explicitly empty ("" and not None) and the document is empty
+            # so when building the chunk text representation, it will be empty and unuseable
+            logger.warning(
+                f"Skipping document with ID {document.id} as the chunks will be empty."
+            )
+            continue
+
+        section_chars = sum(len(section.text) for section in document.sections)
+        if (
+            MAX_DOCUMENT_CHARS
+            and len(document.title or document.semantic_identifier) + section_chars
+            > MAX_DOCUMENT_CHARS
+        ):
+            # Skip documents that are too long, later on there are more memory intensive steps done on the text
+            # and the container will run out of memory and crash. Several other checks are included upstream but
+            # those are at the connector level so a catchall is still needed.
+            # Assumption here is that files that are that long, are generated files and not the type users
+            # generally care for.
+            logger.warning(
+                f"Skipping document with ID {document.id} as it is too long."
+            )
+            continue
+
+        documents.append(document)
+    return documents
+
+
 @log_function_time(debug_only=True)
 def index_doc_batch(
     *,
+    document_batch: list[Document],
     chunker: Chunker,
     embedder: IndexingEmbedder,
     document_index: DocumentIndex,
-    document_batch: list[Document],
     index_attempt_metadata: IndexAttemptMetadata,
     db_session: Session,
     ignore_time_skip: bool = False,
     tenant_id: str | None = None,
+    filter_fnc: Callable[[list[Document]], list[Document]] = filter_documents,
 ) -> tuple[int, int]:
     """Takes different pieces of the indexing pipeline and applies it to a batch of documents
     Note that the documents should already be batched at this point so that it does not inflate the
@@ -309,8 +331,11 @@ def index_doc_batch(
         is_public=False,
     )
 
+    logger.debug("Filtering Documents")
+    filtered_documents = filter_fnc(document_batch)
+
     ctx = index_doc_batch_prepare(
-        document_batch=document_batch,
+        documents=filtered_documents,
         index_attempt_metadata=index_attempt_metadata,
         ignore_time_skip=ignore_time_skip,
         db_session=db_session,

backend/danswer/llm/answering/models.py

@@ -1,163 +0,0 @@
-from collections.abc import Callable
-from collections.abc import Iterator
-from typing import TYPE_CHECKING
-
-from langchain.schema.messages import AIMessage
-from langchain.schema.messages import BaseMessage
-from langchain.schema.messages import HumanMessage
-from langchain.schema.messages import SystemMessage
-from pydantic import BaseModel
-from pydantic import ConfigDict
-from pydantic import Field
-from pydantic import model_validator
-
-from danswer.chat.models import AnswerQuestionStreamReturn
-from danswer.configs.constants import MessageType
-from danswer.file_store.models import InMemoryChatFile
-from danswer.llm.override_models import PromptOverride
-from danswer.llm.utils import build_content_with_imgs
-from danswer.tools.models import ToolCallFinalResult
-
-if TYPE_CHECKING:
-    from danswer.db.models import ChatMessage
-    from danswer.db.models import Prompt
-
-
-StreamProcessor = Callable[[Iterator[str]], AnswerQuestionStreamReturn]
-
-
-class PreviousMessage(BaseModel):
-    """Simplified version of `ChatMessage`"""
-
-    message: str
-    token_count: int
-    message_type: MessageType
-    files: list[InMemoryChatFile]
-    tool_call: ToolCallFinalResult | None
-
-    @classmethod
-    def from_chat_message(
-        cls, chat_message: "ChatMessage", available_files: list[InMemoryChatFile]
-    ) -> "PreviousMessage":
-        message_file_ids = (
-            [file["id"] for file in chat_message.files] if chat_message.files else []
-        )
-        return cls(
-            message=chat_message.message,
-            token_count=chat_message.token_count,
-            message_type=chat_message.message_type,
-            files=[
-                file
-                for file in available_files
-                if str(file.file_id) in message_file_ids
-            ],
-            tool_call=ToolCallFinalResult(
-                tool_name=chat_message.tool_call.tool_name,
-                tool_args=chat_message.tool_call.tool_arguments,
-                tool_result=chat_message.tool_call.tool_result,
-            )
-            if chat_message.tool_call
-            else None,
-        )
-
-    def to_langchain_msg(self) -> BaseMessage:
-        content = build_content_with_imgs(self.message, self.files)
-        if self.message_type == MessageType.USER:
-            return HumanMessage(content=content)
-        elif self.message_type == MessageType.ASSISTANT:
-            return AIMessage(content=content)
-        else:
-            return SystemMessage(content=content)
-
-
-class DocumentPruningConfig(BaseModel):
-    max_chunks: int | None = None
-    max_window_percentage: float | None = None
-    max_tokens: int | None = None
-    # different pruning behavior is expected when the
-    # user manually selects documents they want to chat with
-    # e.g. we don't want to truncate each document to be no more
-    # than one chunk long
-    is_manually_selected_docs: bool = False
-    # If user specifies to include additional context Chunks for each match, then different pruning
-    # is used. As many Sections as possible are included, and the last Section is truncated
-    # If this is false, all of the Sections are truncated if they are longer than the expected Chunk size.
-    # Sections are often expected to be longer than the maximum Chunk size but Chunks should not be.
-    use_sections: bool = True
-    # If using tools, then we need to consider the tool length
-    tool_num_tokens: int = 0
-    # If using a tool message to represent the docs, then we have to JSON serialize
-    # the document content, which adds to the token count.
-    using_tool_message: bool = False
-
-
-class ContextualPruningConfig(DocumentPruningConfig):
-    num_chunk_multiple: int
-
-    @classmethod
-    def from_doc_pruning_config(
-        cls, num_chunk_multiple: int, doc_pruning_config: DocumentPruningConfig
-    ) -> "ContextualPruningConfig":
-        return cls(num_chunk_multiple=num_chunk_multiple, **doc_pruning_config.dict())
-
-
-class CitationConfig(BaseModel):
-    all_docs_useful: bool = False
-
-
-class QuotesConfig(BaseModel):
-    pass
-
-
-class AnswerStyleConfig(BaseModel):
-    citation_config: CitationConfig | None = None
-    quotes_config: QuotesConfig | None = None
-    document_pruning_config: DocumentPruningConfig = Field(
-        default_factory=DocumentPruningConfig
-    )
-    # forces the LLM to return a structured response, see
-    # https://platform.openai.com/docs/guides/structured-outputs/introduction
-    # right now, only used by the simple chat API
-    structured_response_format: dict | None = None
-
-    @model_validator(mode="after")
-    def check_quotes_and_citation(self) -> "AnswerStyleConfig":
-        if self.citation_config is None and self.quotes_config is None:
-            raise ValueError(
-                "One of `citation_config` or `quotes_config` must be provided"
-            )
-
-        if self.citation_config is not None and self.quotes_config is not None:
-            raise ValueError(
-                "Only one of `citation_config` or `quotes_config` must be provided"
-            )
-
-        return self
-
-
-class PromptConfig(BaseModel):
-    """Final representation of the Prompt configuration passed
-    into the `Answer` object."""
-
-    system_prompt: str
-    task_prompt: str
-    datetime_aware: bool
-    include_citations: bool
-
-    @classmethod
-    def from_model(
-        cls, model: "Prompt", prompt_override: PromptOverride | None = None
-    ) -> "PromptConfig":
-        override_system_prompt = (
-            prompt_override.system_prompt if prompt_override else None
-        )
-        override_task_prompt = prompt_override.task_prompt if prompt_override else None
-
-        return cls(
-            system_prompt=override_system_prompt or model.system_prompt,
-            task_prompt=override_task_prompt or model.task_prompt,
-            datetime_aware=model.datetime_aware,
-            include_citations=model.include_citations,
-        )
-
-    model_config = ConfigDict(frozen=True)

backend/danswer/llm/answering/prompts/utils.py

@@ -1,20 +0,0 @@
-from danswer.prompts.direct_qa_prompts import PARAMATERIZED_PROMPT
-from danswer.prompts.direct_qa_prompts import PARAMATERIZED_PROMPT_WITHOUT_CONTEXT
-
-
-def build_dummy_prompt(
-    system_prompt: str, task_prompt: str, retrieval_disabled: bool
-) -> str:
-    if retrieval_disabled:
-        return PARAMATERIZED_PROMPT_WITHOUT_CONTEXT.format(
-            user_query="<USER_QUERY>",
-            system_prompt=system_prompt,
-            task_prompt=task_prompt,
-        ).strip()
-
-    return PARAMATERIZED_PROMPT.format(
-        context_docs_str="<CONTEXT_DOCS>",
-        user_query="<USER_QUERY>",
-        system_prompt=system_prompt,
-        task_prompt=task_prompt,
-    ).strip()

backend/danswer/llm/chat_llm.py

@@ -268,12 +268,16 @@ class DefaultMultiLLM(LLM):
 
         # NOTE: have to set these as environment variables for Litellm since
         # not all are able to passed in but they always support them set as env
-        # variables
+        # variables. We'll also try passing them in, since litellm just ignores
+        # addtional kwargs (and some kwargs MUST be passed in rather than set as
+        # env variables)
         if custom_config:
             for k, v in custom_config.items():
                 os.environ[k] = v
 
         model_kwargs = model_kwargs or {}
+        if custom_config:
+            model_kwargs.update(custom_config)
         if extra_headers:
             model_kwargs.update({"extra_headers": extra_headers})
         if extra_body:

backend/danswer/llm/models.py

@@ -0,0 +1,59 @@
+from typing import TYPE_CHECKING
+
+from langchain.schema.messages import AIMessage
+from langchain.schema.messages import BaseMessage
+from langchain.schema.messages import HumanMessage
+from langchain.schema.messages import SystemMessage
+from pydantic import BaseModel
+
+from danswer.configs.constants import MessageType
+from danswer.file_store.models import InMemoryChatFile
+from danswer.llm.utils import build_content_with_imgs
+from danswer.tools.models import ToolCallFinalResult
+
+if TYPE_CHECKING:
+    from danswer.db.models import ChatMessage
+
+
+class PreviousMessage(BaseModel):
+    """Simplified version of `ChatMessage`"""
+
+    message: str
+    token_count: int
+    message_type: MessageType
+    files: list[InMemoryChatFile]
+    tool_call: ToolCallFinalResult | None
+
+    @classmethod
+    def from_chat_message(
+        cls, chat_message: "ChatMessage", available_files: list[InMemoryChatFile]
+    ) -> "PreviousMessage":
+        message_file_ids = (
+            [file["id"] for file in chat_message.files] if chat_message.files else []
+        )
+        return cls(
+            message=chat_message.message,
+            token_count=chat_message.token_count,
+            message_type=chat_message.message_type,
+            files=[
+                file
+                for file in available_files
+                if str(file.file_id) in message_file_ids
+            ],
+            tool_call=ToolCallFinalResult(
+                tool_name=chat_message.tool_call.tool_name,
+                tool_args=chat_message.tool_call.tool_arguments,
+                tool_result=chat_message.tool_call.tool_result,
+            )
+            if chat_message.tool_call
+            else None,
+        )
+
+    def to_langchain_msg(self) -> BaseMessage:
+        content = build_content_with_imgs(self.message, self.files)
+        if self.message_type == MessageType.USER:
+            return HumanMessage(content=content)
+        elif self.message_type == MessageType.ASSISTANT:
+            return AIMessage(content=content)
+        else:
+            return SystemMessage(content=content)

backend/danswer/llm/utils.py

@@ -1,15 +1,11 @@
 import copy
-import io
 import json
 from collections.abc import Callable
 from collections.abc import Iterator
 from typing import Any
 from typing import cast
-from typing import TYPE_CHECKING
-from typing import Union
 
 import litellm  # type: ignore
-import pandas as pd
 import tiktoken
 from langchain.prompts.base import StringPromptValue
 from langchain.prompts.chat import ChatPromptValue
@@ -36,17 +32,15 @@ from danswer.configs.constants import MessageType
 from danswer.configs.model_configs import GEN_AI_MAX_TOKENS
 from danswer.configs.model_configs import GEN_AI_MODEL_FALLBACK_MAX_TOKENS
 from danswer.configs.model_configs import GEN_AI_NUM_RESERVED_OUTPUT_TOKENS
-from danswer.db.models import ChatMessage
 from danswer.file_store.models import ChatFileType
 from danswer.file_store.models import InMemoryChatFile
 from danswer.llm.interfaces import LLM
 from danswer.prompts.constants import CODE_BLOCK_PAT
+from danswer.utils.b64 import get_image_type
+from danswer.utils.b64 import get_image_type_from_bytes
 from danswer.utils.logger import setup_logger
 from shared_configs.configs import LOG_LEVEL
 
-if TYPE_CHECKING:
-    from danswer.llm.answering.models import PreviousMessage
-
 logger = setup_logger()
 
 
@@ -104,92 +98,39 @@ def litellm_exception_to_error_msg(
     return error_msg
 
 
-def translate_danswer_msg_to_langchain(
-    msg: Union[ChatMessage, "PreviousMessage"],
-) -> BaseMessage:
-    files: list[InMemoryChatFile] = []
-
-    # If the message is a `ChatMessage`, it doesn't have the downloaded files
-    # attached. Just ignore them for now.
-    if not isinstance(msg, ChatMessage):
-        files = msg.files
-    content = build_content_with_imgs(msg.message, files, message_type=msg.message_type)
-
-    if msg.message_type == MessageType.SYSTEM:
-        raise ValueError("System messages are not currently part of history")
-    if msg.message_type == MessageType.ASSISTANT:
-        return AIMessage(content=content)
-    if msg.message_type == MessageType.USER:
-        return HumanMessage(content=content)
-
-    raise ValueError(f"New message type {msg.message_type} not handled")
-
-
-def translate_history_to_basemessages(
-    history: list[ChatMessage] | list["PreviousMessage"],
-) -> tuple[list[BaseMessage], list[int]]:
-    history_basemessages = [
-        translate_danswer_msg_to_langchain(msg)
-        for msg in history
-        if msg.token_count != 0
-    ]
-    history_token_counts = [msg.token_count for msg in history if msg.token_count != 0]
-    return history_basemessages, history_token_counts
-
-
-#  Processes CSV files to show the first 5 rows and max_columns (default 40) columns
-def _process_csv_file(file: InMemoryChatFile, max_columns: int = 40) -> str:
-    df = pd.read_csv(io.StringIO(file.content.decode("utf-8")))
-
-    csv_preview = df.head().to_string(max_cols=max_columns)
-
-    file_name_section = (
-        f"CSV FILE NAME: {file.filename}\n"
-        if file.filename
-        else "CSV FILE (NO NAME PROVIDED):\n"
-    )
-    return f"{file_name_section}{CODE_BLOCK_PAT.format(csv_preview)}\n\n\n"
-
-
 def _build_content(
     message: str,
     files: list[InMemoryChatFile] | None = None,
 ) -> str:
     """Applies all non-image files."""
-    text_files = (
-        [file for file in files if file.file_type == ChatFileType.PLAIN_TEXT]
-        if files
-        else None
-    )
+    if not files:
+        return message
 
-    csv_files = (
-        [file for file in files if file.file_type == ChatFileType.CSV]
-        if files
-        else None
-    )
+    text_files = [
+        file
+        for file in files
+        if file.file_type in (ChatFileType.PLAIN_TEXT, ChatFileType.CSV)
+    ]
 
-    if not text_files and not csv_files:
+    if not text_files:
         return message
 
     final_message_with_files = "FILES:\n\n"
-    for file in text_files or []:
+    for file in text_files:
         file_content = file.content.decode("utf-8")
         file_name_section = f"DOCUMENT: {file.filename}\n" if file.filename else ""
         final_message_with_files += (
             f"{file_name_section}{CODE_BLOCK_PAT.format(file_content.strip())}\n\n\n"
         )
-    for file in csv_files or []:
-        final_message_with_files += _process_csv_file(file)
 
-    final_message_with_files += message
-
-    return final_message_with_files
+    return final_message_with_files + message
 
 
 def build_content_with_imgs(
     message: str,
     files: list[InMemoryChatFile] | None = None,
     img_urls: list[str] | None = None,
+    b64_imgs: list[str] | None = None,
     message_type: MessageType = MessageType.USER,
 ) -> str | list[str | dict[str, Any]]:  # matching Langchain's BaseMessage content type
     files = files or []
@@ -202,6 +143,7 @@ def build_content_with_imgs(
     )
 
     img_urls = img_urls or []
+    b64_imgs = b64_imgs or []
 
     message_main_content = _build_content(message, files)
 
@@ -220,11 +162,22 @@ def build_content_with_imgs(
             {
                 "type": "image_url",
                 "image_url": {
-                    "url": f"data:image/jpeg;base64,{file.to_base64()}",
+                    "url": (
+                        f"data:{get_image_type_from_bytes(file.content)};"
+                        f"base64,{file.to_base64()}"
+                    ),
                 },
             }
-            for file in files
-            if file.file_type == "image"
+            for file in img_files
+        ]
+        + [
+            {
+                "type": "image_url",
+                "image_url": {
+                    "url": f"data:{get_image_type(b64_img)};base64,{b64_img}",
+                },
+            }
+            for b64_img in b64_imgs
         ]
         + [
             {

backend/danswer/main.py

@@ -52,12 +52,9 @@ from danswer.server.documents.connector import router as connector_router
 from danswer.server.documents.credential import router as credential_router
 from danswer.server.documents.document import router as document_router
 from danswer.server.documents.indexing import router as indexing_router
+from danswer.server.documents.standard_oauth import router as oauth_router
 from danswer.server.features.document_set.api import router as document_set_router
 from danswer.server.features.folder.api import router as folder_router
-from danswer.server.features.input_prompt.api import (
-    admin_router as admin_input_prompt_router,
-)
-from danswer.server.features.input_prompt.api import basic_router as input_prompt_router
 from danswer.server.features.notifications.api import router as notification_router
 from danswer.server.features.persona.api import admin_router as admin_persona_router
 from danswer.server.features.persona.api import basic_router as persona_router
@@ -105,7 +102,6 @@ from shared_configs.configs import CORS_ALLOWED_ORIGIN
 from shared_configs.configs import MULTI_TENANT
 from shared_configs.configs import SENTRY_DSN
 
-
 logger = setup_logger()
 
 
@@ -259,8 +255,6 @@ def get_application() -> FastAPI:
     )
     include_router_with_global_prefix_prepended(application, persona_router)
     include_router_with_global_prefix_prepended(application, admin_persona_router)
-    include_router_with_global_prefix_prepended(application, input_prompt_router)
-    include_router_with_global_prefix_prepended(application, admin_input_prompt_router)
     include_router_with_global_prefix_prepended(application, notification_router)
     include_router_with_global_prefix_prepended(application, prompt_router)
     include_router_with_global_prefix_prepended(application, tool_router)
@@ -283,6 +277,7 @@ def get_application() -> FastAPI:
     )
     include_router_with_global_prefix_prepended(application, long_term_logs_router)
     include_router_with_global_prefix_prepended(application, api_key_router)
+    include_router_with_global_prefix_prepended(application, oauth_router)
 
     if AUTH_TYPE == AuthType.DISABLED:
         # Server logs this during auth setup verification step

backend/danswer/prompts/prompt_utils.py

@@ -5,11 +5,11 @@ from typing import cast
 from langchain_core.messages import BaseMessage
 
 from danswer.chat.models import LlmDoc
+from danswer.chat.models import PromptConfig
 from danswer.configs.chat_configs import LANGUAGE_HINT
 from danswer.configs.constants import DocumentSource
 from danswer.context.search.models import InferenceChunk
 from danswer.db.models import Prompt
-from danswer.llm.answering.models import PromptConfig
 from danswer.prompts.chat_prompts import ADDITIONAL_INFO
 from danswer.prompts.chat_prompts import CITATION_REMINDER
 from danswer.prompts.constants import CODE_BLOCK_PAT

backend/danswer/secondary_llm_flows/choose_search.py

@@ -3,14 +3,14 @@ from langchain.schema import HumanMessage
 from langchain.schema import SystemMessage
 
 from danswer.chat.chat_utils import combine_message_chain
+from danswer.chat.prompt_builder.utils import translate_danswer_msg_to_langchain
 from danswer.configs.chat_configs import DISABLE_LLM_CHOOSE_SEARCH
 from danswer.configs.model_configs import GEN_AI_HISTORY_CUTOFF
 from danswer.db.models import ChatMessage
-from danswer.llm.answering.models import PreviousMessage
 from danswer.llm.interfaces import LLM
+from danswer.llm.models import PreviousMessage
 from danswer.llm.utils import dict_based_prompt_to_langchain_prompt
 from danswer.llm.utils import message_to_string
-from danswer.llm.utils import translate_danswer_msg_to_langchain
 from danswer.prompts.chat_prompts import AGGRESSIVE_SEARCH_TEMPLATE
 from danswer.prompts.chat_prompts import NO_SEARCH
 from danswer.prompts.chat_prompts import REQUIRE_SEARCH_HINT

backend/danswer/secondary_llm_flows/query_expansion.py

@@ -4,10 +4,10 @@ from danswer.chat.chat_utils import combine_message_chain
 from danswer.configs.chat_configs import DISABLE_LLM_QUERY_REPHRASE
 from danswer.configs.model_configs import GEN_AI_HISTORY_CUTOFF
 from danswer.db.models import ChatMessage
-from danswer.llm.answering.models import PreviousMessage
 from danswer.llm.exceptions import GenAIDisabledException
 from danswer.llm.factory import get_default_llms
 from danswer.llm.interfaces import LLM
+from danswer.llm.models import PreviousMessage
 from danswer.llm.utils import dict_based_prompt_to_langchain_prompt
 from danswer.llm.utils import message_to_string
 from danswer.prompts.chat_prompts import HISTORY_QUERY_REPHRASE

backend/danswer/seeding/input_prompts.yaml

@@ -1,24 +0,0 @@
-input_prompts:
-  - id: -5
-    prompt: "Elaborate"
-    content: "Elaborate on the above, give me a more in depth explanation."
-    active: true
-    is_public: true
-
-  - id: -4
-    prompt: "Reword"
-    content: "Help me rewrite the following politely and concisely for professional communication:\n"
-    active: true
-    is_public: true
-
-  - id: -3
-    prompt: "Email"
-    content: "Write a professional email for me including a subject line, signature, etc. Template the parts that need editing with [ ]. The email should cover the following points:\n"
-    active: true
-    is_public: true
-
-  - id: -2
-    prompt: "Debug"
-    content: "Provide step-by-step troubleshooting instructions for the following issue:\n"
-    active: true
-    is_public: true

backend/danswer/seeding/load_docs.py

@@ -196,7 +196,7 @@ def seed_initial_documents(
     docs, chunks = _create_indexable_chunks(processed_docs, tenant_id)
 
     index_doc_batch_prepare(
-        document_batch=docs,
+        documents=docs,
         index_attempt_metadata=IndexAttemptMetadata(
             connector_id=connector_id,
             credential_id=PUBLIC_CREDENTIAL_ID,

backend/danswer/seeding/load_yamls.py

@@ -1,13 +1,11 @@
 import yaml
 from sqlalchemy.orm import Session
 
-from danswer.configs.chat_configs import INPUT_PROMPT_YAML
 from danswer.configs.chat_configs import MAX_CHUNKS_FED_TO_CHAT
 from danswer.configs.chat_configs import PERSONAS_YAML
 from danswer.configs.chat_configs import PROMPTS_YAML
 from danswer.context.search.enums import RecencyBiasSetting
 from danswer.db.document_set import get_or_create_document_set_by_name
-from danswer.db.input_prompt import insert_input_prompt_if_not_exists
 from danswer.db.models import DocumentSet as DocumentSetDBModel
 from danswer.db.models import Persona
 from danswer.db.models import Prompt as PromptDBModel
@@ -79,6 +77,9 @@ def load_personas_from_yaml(
             if prompts:
                 prompt_ids = [prompt.id for prompt in prompts if prompt is not None]
 
+        if not prompt_ids:
+            raise ValueError("Invalid Persona config, no prompts exist")
+
         p_id = persona.get("id")
         tool_ids = []
 
@@ -123,45 +124,24 @@ def load_personas_from_yaml(
             tool_ids=tool_ids,
             builtin_persona=True,
             is_public=True,
-            display_priority=existing_persona.display_priority
-            if existing_persona is not None
-            else persona.get("display_priority"),
-            is_visible=existing_persona.is_visible
-            if existing_persona is not None
-            else persona.get("is_visible"),
+            display_priority=(
+                existing_persona.display_priority
+                if existing_persona is not None
+                else persona.get("display_priority")
+            ),
+            is_visible=(
+                existing_persona.is_visible
+                if existing_persona is not None
+                else persona.get("is_visible")
+            ),
             db_session=db_session,
         )
 
 
-def load_input_prompts_from_yaml(
-    db_session: Session, input_prompts_yaml: str = INPUT_PROMPT_YAML
-) -> None:
-    with open(input_prompts_yaml, "r") as file:
-        data = yaml.safe_load(file)
-
-    all_input_prompts = data.get("input_prompts", [])
-    for input_prompt in all_input_prompts:
-        # If these prompts are deleted (which is a hard delete in the DB), on server startup
-        # they will be recreated, but the user can always just deactivate them, just a light inconvenience
-
-        insert_input_prompt_if_not_exists(
-            user=None,
-            input_prompt_id=input_prompt.get("id"),
-            prompt=input_prompt["prompt"],
-            content=input_prompt["content"],
-            is_public=input_prompt["is_public"],
-            active=input_prompt.get("active", True),
-            db_session=db_session,
-            commit=True,
-        )
-
-
 def load_chat_yamls(
     db_session: Session,
     prompt_yaml: str = PROMPTS_YAML,
     personas_yaml: str = PERSONAS_YAML,
-    input_prompts_yaml: str = INPUT_PROMPT_YAML,
 ) -> None:
     load_prompts_from_yaml(db_session, prompt_yaml)
     load_personas_from_yaml(db_session, personas_yaml)
-    load_input_prompts_from_yaml(db_session, input_prompts_yaml)

backend/danswer/server/documents/cc_pair.py

@@ -33,8 +33,6 @@ from danswer.db.engine import get_current_tenant_id
 from danswer.db.engine import get_session
 from danswer.db.enums import AccessType
 from danswer.db.enums import ConnectorCredentialPairStatus
-from danswer.db.index_attempt import cancel_indexing_attempts_for_ccpair
-from danswer.db.index_attempt import cancel_indexing_attempts_past_model
 from danswer.db.index_attempt import count_index_attempts_for_connector
 from danswer.db.index_attempt import get_latest_index_attempt_for_cc_pair_id
 from danswer.db.index_attempt import get_paginated_index_attempts_for_cc_pair_id
@@ -45,6 +43,7 @@ from danswer.db.search_settings import get_current_search_settings
 from danswer.redis.redis_connector import RedisConnector
 from danswer.redis.redis_pool import get_redis_client
 from danswer.server.documents.models import CCPairFullInfo
+from danswer.server.documents.models import CCPropertyUpdateRequest
 from danswer.server.documents.models import CCStatusUpdateRequest
 from danswer.server.documents.models import ConnectorCredentialPairIdentifier
 from danswer.server.documents.models import ConnectorCredentialPairMetadata
@@ -192,9 +191,6 @@ def update_cc_pair_status(
             db_session
         )
 
-        cancel_indexing_attempts_for_ccpair(cc_pair_id, db_session)
-        cancel_indexing_attempts_past_model(db_session)
-
         redis_connector = RedisConnector(tenant_id, cc_pair_id)
 
         try:
@@ -308,6 +304,46 @@ def update_cc_pair_name(
         raise HTTPException(status_code=400, detail="Name must be unique")
 
 
+@router.put("/admin/cc-pair/{cc_pair_id}/property")
+def update_cc_pair_property(
+    cc_pair_id: int,
+    update_request: CCPropertyUpdateRequest,  # in seconds
+    user: User | None = Depends(current_curator_or_admin_user),
+    db_session: Session = Depends(get_session),
+) -> StatusResponse[int]:
+    cc_pair = get_connector_credential_pair_from_id(
+        cc_pair_id=cc_pair_id,
+        db_session=db_session,
+        user=user,
+        get_editable=True,
+    )
+    if not cc_pair:
+        raise HTTPException(
+            status_code=400, detail="CC Pair not found for current user's permissions"
+        )
+
+    # Can we centralize logic for updating connector properties
+    # so that we don't need to manually validate everywhere?
+    if update_request.name == "refresh_frequency":
+        cc_pair.connector.refresh_freq = int(update_request.value)
+        cc_pair.connector.validate_refresh_freq()
+        db_session.commit()
+
+        msg = "Refresh frequency updated successfully"
+    elif update_request.name == "pruning_frequency":
+        cc_pair.connector.prune_freq = int(update_request.value)
+        cc_pair.connector.validate_prune_freq()
+        db_session.commit()
+
+        msg = "Pruning frequency updated successfully"
+    else:
+        raise HTTPException(
+            status_code=400, detail=f"Property name {update_request.name} is not valid."
+        )
+
+    return StatusResponse(success=True, message=msg, data=cc_pair_id)
+
+
 @router.get("/admin/cc-pair/{cc_pair_id}/last_pruned")
 def get_cc_pair_last_pruned(
     cc_pair_id: int,

backend/danswer/server/documents/connector.py

@@ -86,6 +86,7 @@ from danswer.db.models import SearchSettings
 from danswer.db.models import User
 from danswer.db.search_settings import get_current_search_settings
 from danswer.db.search_settings import get_secondary_search_settings
+from danswer.file_processing.extract_file_text import convert_docx_to_txt
 from danswer.file_store.file_store import get_default_file_store
 from danswer.key_value_store.interface import KvKeyNotFoundError
 from danswer.redis.redis_connector import RedisConnector
@@ -393,6 +394,12 @@ def upload_files(
                 file_origin=FileOrigin.CONNECTOR,
                 file_type=file.content_type or "text/plain",
             )
+
+            if file.content_type and file.content_type.startswith(
+                "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
+            ):
+                convert_docx_to_txt(file, file_store, file_path)
+
     except ValueError as e:
         raise HTTPException(status_code=400, detail=str(e))
     return FileUploadResponse(file_paths=deduped_file_paths)
@@ -1010,37 +1017,18 @@ def get_connector_by_id(
 
 
 class BasicCCPairInfo(BaseModel):
-    docs_indexed: int
     has_successful_run: bool
     source: DocumentSource
 
 
-@router.get("/indexing-status")
+@router.get("/connector-status")
 def get_basic_connector_indexing_status(
     _: User = Depends(current_user),
     db_session: Session = Depends(get_session),
 ) -> list[BasicCCPairInfo]:
     cc_pairs = get_connector_credential_pairs(db_session)
-    cc_pair_identifiers = [
-        ConnectorCredentialPairIdentifier(
-            connector_id=cc_pair.connector_id, credential_id=cc_pair.credential_id
-        )
-        for cc_pair in cc_pairs
-    ]
-    document_count_info = get_document_counts_for_cc_pairs(
-        db_session=db_session,
-        cc_pair_identifiers=cc_pair_identifiers,
-    )
-    cc_pair_to_document_cnt = {
-        (connector_id, credential_id): cnt
-        for connector_id, credential_id, cnt in document_count_info
-    }
     return [
         BasicCCPairInfo(
-            docs_indexed=cc_pair_to_document_cnt.get(
-                (cc_pair.connector_id, cc_pair.credential_id)
-            )
-            or 0,
             has_successful_run=cc_pair.last_successful_index_time is not None,
             source=cc_pair.connector.source,
         )

backend/danswer/server/documents/credential.py

@@ -181,7 +181,13 @@ def update_credential_data(
     user: User = Depends(current_user),
     db_session: Session = Depends(get_session),
 ) -> CredentialBase:
-    credential = alter_credential(credential_id, credential_update, user, db_session)
+    credential = alter_credential(
+        credential_id,
+        credential_update.name,
+        credential_update.credential_json,
+        user,
+        db_session,
+    )
 
     if credential is None:
         raise HTTPException(

backend/danswer/server/documents/models.py

@@ -364,6 +364,11 @@ class RunConnectorRequest(BaseModel):
     from_beginning: bool = False
 
 
+class CCPropertyUpdateRequest(BaseModel):
+    name: str
+    value: str
+
+
 """Connectors Models"""
 
 

backend/danswer/server/documents/standard_oauth.py

@@ -0,0 +1,142 @@
+import uuid
+from typing import Annotated
+from typing import cast
+
+from fastapi import APIRouter
+from fastapi import Depends
+from fastapi import HTTPException
+from fastapi import Query
+from pydantic import BaseModel
+from sqlalchemy.orm import Session
+
+from danswer.auth.users import current_user
+from danswer.configs.app_configs import WEB_DOMAIN
+from danswer.configs.constants import DocumentSource
+from danswer.connectors.interfaces import OAuthConnector
+from danswer.db.credentials import create_credential
+from danswer.db.engine import get_current_tenant_id
+from danswer.db.engine import get_session
+from danswer.db.models import User
+from danswer.redis.redis_pool import get_redis_client
+from danswer.server.documents.models import CredentialBase
+from danswer.utils.logger import setup_logger
+from danswer.utils.subclasses import find_all_subclasses_in_dir
+
+logger = setup_logger()
+
+router = APIRouter(prefix="/connector/oauth")
+
+_OAUTH_STATE_KEY_FMT = "oauth_state:{state}"
+_OAUTH_STATE_EXPIRATION_SECONDS = 10 * 60  # 10 minutes
+
+# Cache for OAuth connectors, populated at module load time
+_OAUTH_CONNECTORS: dict[DocumentSource, type[OAuthConnector]] = {}
+
+
+def _discover_oauth_connectors() -> dict[DocumentSource, type[OAuthConnector]]:
+    """Walk through the connectors package to find all OAuthConnector implementations"""
+    global _OAUTH_CONNECTORS
+    if _OAUTH_CONNECTORS:  # Return cached connectors if already discovered
+        return _OAUTH_CONNECTORS
+
+    oauth_connectors = find_all_subclasses_in_dir(
+        cast(type[OAuthConnector], OAuthConnector), "danswer.connectors"
+    )
+
+    _OAUTH_CONNECTORS = {cls.oauth_id(): cls for cls in oauth_connectors}
+    return _OAUTH_CONNECTORS
+
+
+# Discover OAuth connectors at module load time
+_discover_oauth_connectors()
+
+
+class AuthorizeResponse(BaseModel):
+    redirect_url: str
+
+
+@router.get("/authorize/{source}")
+def oauth_authorize(
+    source: DocumentSource,
+    desired_return_url: Annotated[str | None, Query()] = None,
+    _: User = Depends(current_user),
+    tenant_id: str | None = Depends(get_current_tenant_id),
+) -> AuthorizeResponse:
+    """Initiates the OAuth flow by redirecting to the provider's auth page"""
+    oauth_connectors = _discover_oauth_connectors()
+
+    if source not in oauth_connectors:
+        raise HTTPException(status_code=400, detail=f"Unknown OAuth source: {source}")
+
+    connector_cls = oauth_connectors[source]
+    base_url = WEB_DOMAIN
+
+    # store state in redis
+    if not desired_return_url:
+        desired_return_url = f"{base_url}/admin/connectors/{source}?step=0"
+    redis_client = get_redis_client(tenant_id=tenant_id)
+    state = str(uuid.uuid4())
+    redis_client.set(
+        _OAUTH_STATE_KEY_FMT.format(state=state),
+        desired_return_url,
+        ex=_OAUTH_STATE_EXPIRATION_SECONDS,
+    )
+
+    return AuthorizeResponse(
+        redirect_url=connector_cls.oauth_authorization_url(base_url, state)
+    )
+
+
+class CallbackResponse(BaseModel):
+    redirect_url: str
+
+
+@router.get("/callback/{source}")
+def oauth_callback(
+    source: DocumentSource,
+    code: Annotated[str, Query()],
+    state: Annotated[str, Query()],
+    db_session: Session = Depends(get_session),
+    user: User = Depends(current_user),
+    tenant_id: str | None = Depends(get_current_tenant_id),
+) -> CallbackResponse:
+    """Handles the OAuth callback and exchanges the code for tokens"""
+    oauth_connectors = _discover_oauth_connectors()
+
+    if source not in oauth_connectors:
+        raise HTTPException(status_code=400, detail=f"Unknown OAuth source: {source}")
+
+    connector_cls = oauth_connectors[source]
+
+    # get state from redis
+    redis_client = get_redis_client(tenant_id=tenant_id)
+    original_url_bytes = cast(
+        bytes, redis_client.get(_OAUTH_STATE_KEY_FMT.format(state=state))
+    )
+    if not original_url_bytes:
+        raise HTTPException(status_code=400, detail="Invalid OAuth state")
+    original_url = original_url_bytes.decode("utf-8")
+
+    token_info = connector_cls.oauth_code_to_token(code)
+
+    # Create a new credential with the token info
+    credential_data = CredentialBase(
+        credential_json=token_info,
+        admin_public=True,  # Or based on some logic/parameter
+        source=source,
+        name=f"{source.title()} OAuth Credential",
+    )
+
+    credential = create_credential(
+        credential_data=credential_data,
+        user=user,
+        db_session=db_session,
+    )
+
+    return CallbackResponse(
+        redirect_url=(
+            f"{original_url}?credentialId={credential.id}"
+            if "?" not in original_url
+            else f"{original_url}&credentialId={credential.id}"
+        )
+    )

backend/danswer/server/features/input_prompt/api.py

@@ -1,134 +0,0 @@
-from fastapi import APIRouter
-from fastapi import Depends
-from fastapi import HTTPException
-from sqlalchemy.orm import Session
-
-from danswer.auth.users import current_admin_user
-from danswer.auth.users import current_user
-from danswer.db.engine import get_session
-from danswer.db.input_prompt import fetch_input_prompt_by_id
-from danswer.db.input_prompt import fetch_input_prompts_by_user
-from danswer.db.input_prompt import fetch_public_input_prompts
-from danswer.db.input_prompt import insert_input_prompt
-from danswer.db.input_prompt import remove_input_prompt
-from danswer.db.input_prompt import remove_public_input_prompt
-from danswer.db.input_prompt import update_input_prompt
-from danswer.db.models import User
-from danswer.server.features.input_prompt.models import CreateInputPromptRequest
-from danswer.server.features.input_prompt.models import InputPromptSnapshot
-from danswer.server.features.input_prompt.models import UpdateInputPromptRequest
-from danswer.utils.logger import setup_logger
-
-logger = setup_logger()
-
-basic_router = APIRouter(prefix="/input_prompt")
-admin_router = APIRouter(prefix="/admin/input_prompt")
-
-
-@basic_router.get("")
-def list_input_prompts(
-    user: User | None = Depends(current_user),
-    include_public: bool = False,
-    db_session: Session = Depends(get_session),
-) -> list[InputPromptSnapshot]:
-    user_prompts = fetch_input_prompts_by_user(
-        user_id=user.id if user is not None else None,
-        db_session=db_session,
-        include_public=include_public,
-    )
-    return [InputPromptSnapshot.from_model(prompt) for prompt in user_prompts]
-
-
-@basic_router.get("/{input_prompt_id}")
-def get_input_prompt(
-    input_prompt_id: int,
-    user: User | None = Depends(current_user),
-    db_session: Session = Depends(get_session),
-) -> InputPromptSnapshot:
-    input_prompt = fetch_input_prompt_by_id(
-        id=input_prompt_id,
-        user_id=user.id if user is not None else None,
-        db_session=db_session,
-    )
-    return InputPromptSnapshot.from_model(input_prompt=input_prompt)
-
-
-@basic_router.post("")
-def create_input_prompt(
-    create_input_prompt_request: CreateInputPromptRequest,
-    user: User | None = Depends(current_user),
-    db_session: Session = Depends(get_session),
-) -> InputPromptSnapshot:
-    input_prompt = insert_input_prompt(
-        prompt=create_input_prompt_request.prompt,
-        content=create_input_prompt_request.content,
-        is_public=create_input_prompt_request.is_public,
-        user=user,
-        db_session=db_session,
-    )
-    return InputPromptSnapshot.from_model(input_prompt)
-
-
-@basic_router.patch("/{input_prompt_id}")
-def patch_input_prompt(
-    input_prompt_id: int,
-    update_input_prompt_request: UpdateInputPromptRequest,
-    user: User | None = Depends(current_user),
-    db_session: Session = Depends(get_session),
-) -> InputPromptSnapshot:
-    try:
-        updated_input_prompt = update_input_prompt(
-            user=user,
-            input_prompt_id=input_prompt_id,
-            prompt=update_input_prompt_request.prompt,
-            content=update_input_prompt_request.content,
-            active=update_input_prompt_request.active,
-            db_session=db_session,
-        )
-    except ValueError as e:
-        error_msg = "Error occurred while updated input prompt"
-        logger.warn(f"{error_msg}. Stack trace: {e}")
-        raise HTTPException(status_code=404, detail=error_msg)
-
-    return InputPromptSnapshot.from_model(updated_input_prompt)
-
-
-@basic_router.delete("/{input_prompt_id}")
-def delete_input_prompt(
-    input_prompt_id: int,
-    user: User | None = Depends(current_user),
-    db_session: Session = Depends(get_session),
-) -> None:
-    try:
-        remove_input_prompt(user, input_prompt_id, db_session)
-
-    except ValueError as e:
-        error_msg = "Error occurred while deleting input prompt"
-        logger.warn(f"{error_msg}. Stack trace: {e}")
-        raise HTTPException(status_code=404, detail=error_msg)
-
-
-@admin_router.delete("/{input_prompt_id}")
-def delete_public_input_prompt(
-    input_prompt_id: int,
-    _: User | None = Depends(current_admin_user),
-    db_session: Session = Depends(get_session),
-) -> None:
-    try:
-        remove_public_input_prompt(input_prompt_id, db_session)
-
-    except ValueError as e:
-        error_msg = "Error occurred while deleting input prompt"
-        logger.warn(f"{error_msg}. Stack trace: {e}")
-        raise HTTPException(status_code=404, detail=error_msg)
-
-
-@admin_router.get("")
-def list_public_input_prompts(
-    _: User | None = Depends(current_admin_user),
-    db_session: Session = Depends(get_session),
-) -> list[InputPromptSnapshot]:
-    user_prompts = fetch_public_input_prompts(
-        db_session=db_session,
-    )
-    return [InputPromptSnapshot.from_model(prompt) for prompt in user_prompts]

backend/danswer/server/features/input_prompt/models.py

@@ -1,47 +0,0 @@
-from uuid import UUID
-
-from pydantic import BaseModel
-
-from danswer.db.models import InputPrompt
-from danswer.utils.logger import setup_logger
-
-logger = setup_logger()
-
-
-class CreateInputPromptRequest(BaseModel):
-    prompt: str
-    content: str
-    is_public: bool
-
-
-class UpdateInputPromptRequest(BaseModel):
-    prompt: str
-    content: str
-    active: bool
-
-
-class InputPromptResponse(BaseModel):
-    id: int
-    prompt: str
-    content: str
-    active: bool
-
-
-class InputPromptSnapshot(BaseModel):
-    id: int
-    prompt: str
-    content: str
-    active: bool
-    user_id: UUID | None
-    is_public: bool
-
-    @classmethod
-    def from_model(cls, input_prompt: InputPrompt) -> "InputPromptSnapshot":
-        return InputPromptSnapshot(
-            id=input_prompt.id,
-            prompt=input_prompt.prompt,
-            content=input_prompt.content,
-            active=input_prompt.active,
-            user_id=input_prompt.user_id,
-            is_public=input_prompt.is_public,
-        )

backend/danswer/server/features/persona/api.py

@@ -13,6 +13,7 @@ from danswer.auth.users import current_admin_user
 from danswer.auth.users import current_curator_or_admin_user
 from danswer.auth.users import current_limited_user
 from danswer.auth.users import current_user
+from danswer.chat.prompt_builder.utils import build_dummy_prompt
 from danswer.configs.constants import FileOrigin
 from danswer.configs.constants import NotificationType
 from danswer.db.engine import get_session
@@ -33,7 +34,6 @@ from danswer.db.persona import update_persona_shared_users
 from danswer.db.persona import update_persona_visibility
 from danswer.file_store.file_store import get_default_file_store
 from danswer.file_store.models import ChatFileType
-from danswer.llm.answering.prompts.utils import build_dummy_prompt
 from danswer.server.features.persona.models import CreatePersonaRequest
 from danswer.server.features.persona.models import ImageGenerationToolStatus
 from danswer.server.features.persona.models import PersonaCategoryCreate

backend/danswer/server/manage/models.py

@@ -266,5 +266,7 @@ class FullModelVersionResponse(BaseModel):
 class AllUsersResponse(BaseModel):
     accepted: list[FullUserSnapshot]
     invited: list[InvitedUserSnapshot]
+    slack_users: list[FullUserSnapshot]
     accepted_pages: int
     invited_pages: int
+    slack_users_pages: int

backend/danswer/server/manage/users.py

@@ -119,6 +119,7 @@ def set_user_role(
 def list_all_users(
     q: str | None = None,
     accepted_page: int | None = None,
+    slack_users_page: int | None = None,
     invited_page: int | None = None,
     user: User | None = Depends(current_curator_or_admin_user),
     db_session: Session = Depends(get_session),
@@ -131,7 +132,12 @@ def list_all_users(
         for user in list_users(db_session, email_filter_string=q)
         if not is_api_key_email_address(user.email)
     ]
-    accepted_emails = {user.email for user in users}
+
+    slack_users = [user for user in users if user.role == UserRole.SLACK_USER]
+    accepted_users = [user for user in users if user.role != UserRole.SLACK_USER]
+
+    accepted_emails = {user.email for user in accepted_users}
+    slack_users_emails = {user.email for user in slack_users}
     invited_emails = get_invited_users()
     if q:
         invited_emails = [
@@ -139,10 +145,11 @@ def list_all_users(
         ]
 
     accepted_count = len(accepted_emails)
+    slack_users_count = len(slack_users_emails)
     invited_count = len(invited_emails)
 
     # If any of q, accepted_page, or invited_page is None, return all users
-    if accepted_page is None or invited_page is None:
+    if accepted_page is None or invited_page is None or slack_users_page is None:
         return AllUsersResponse(
             accepted=[
                 FullUserSnapshot(
@@ -153,11 +160,23 @@ def list_all_users(
                         UserStatus.LIVE if user.is_active else UserStatus.DEACTIVATED
                     ),
                 )
-                for user in users
+                for user in accepted_users
+            ],
+            slack_users=[
+                FullUserSnapshot(
+                    id=user.id,
+                    email=user.email,
+                    role=user.role,
+                    status=(
+                        UserStatus.LIVE if user.is_active else UserStatus.DEACTIVATED
+                    ),
+                )
+                for user in slack_users
             ],
             invited=[InvitedUserSnapshot(email=email) for email in invited_emails],
             accepted_pages=1,
             invited_pages=1,
+            slack_users_pages=1,
         )
 
     # Otherwise, return paginated results
@@ -169,13 +188,27 @@ def list_all_users(
                 role=user.role,
                 status=UserStatus.LIVE if user.is_active else UserStatus.DEACTIVATED,
             )
-            for user in users
+            for user in accepted_users
         ][accepted_page * USERS_PAGE_SIZE : (accepted_page + 1) * USERS_PAGE_SIZE],
+        slack_users=[
+            FullUserSnapshot(
+                id=user.id,
+                email=user.email,
+                role=user.role,
+                status=UserStatus.LIVE if user.is_active else UserStatus.DEACTIVATED,
+            )
+            for user in slack_users
+        ][
+            slack_users_page
+            * USERS_PAGE_SIZE : (slack_users_page + 1)
+            * USERS_PAGE_SIZE
+        ],
         invited=[InvitedUserSnapshot(email=email) for email in invited_emails][
             invited_page * USERS_PAGE_SIZE : (invited_page + 1) * USERS_PAGE_SIZE
         ],
         accepted_pages=accepted_count // USERS_PAGE_SIZE + 1,
         invited_pages=invited_count // USERS_PAGE_SIZE + 1,
+        slack_users_pages=slack_users_count // USERS_PAGE_SIZE + 1,
     )
 
 
@@ -194,11 +227,11 @@ def bulk_invite_users(
         )
 
     tenant_id = CURRENT_TENANT_ID_CONTEXTVAR.get()
-    normalized_emails = []
+    new_invited_emails = []
     try:
         for email in emails:
             email_info = validate_email(email)
-            normalized_emails.append(email_info.normalized)  # type: ignore
+            new_invited_emails.append(email_info.normalized)
 
     except (EmailUndeliverableError, EmailNotValidError) as e:
         raise HTTPException(
@@ -210,7 +243,7 @@ def bulk_invite_users(
         try:
             fetch_ee_implementation_or_noop(
                 "danswer.server.tenants.provisioning", "add_users_to_tenant", None
-            )(normalized_emails, tenant_id)
+            )(new_invited_emails, tenant_id)
 
         except IntegrityError as e:
             if isinstance(e.orig, UniqueViolation):
@@ -224,7 +257,7 @@ def bulk_invite_users(
 
     initial_invited_users = get_invited_users()
 
-    all_emails = list(set(normalized_emails) | set(initial_invited_users))
+    all_emails = list(set(new_invited_emails) | set(initial_invited_users))
     number_of_invited_users = write_invited_users(all_emails)
 
     if not MULTI_TENANT:
@@ -236,7 +269,7 @@ def bulk_invite_users(
         )(CURRENT_TENANT_ID_CONTEXTVAR.get(), get_total_users_count(db_session))
         if ENABLE_EMAIL_INVITES:
             try:
-                for email in all_emails:
+                for email in new_invited_emails:
                     send_user_email_invite(email, current_user)
             except Exception as e:
                 logger.error(f"Error sending email invite to invited users: {e}")
@@ -250,7 +283,7 @@ def bulk_invite_users(
         write_invited_users(initial_invited_users)  # Reset to original state
         fetch_ee_implementation_or_noop(
             "danswer.server.tenants.user_mapping", "remove_users_from_tenant", None
-        )(normalized_emails, tenant_id)
+        )(new_invited_emails, tenant_id)
         raise e
 
 

backend/danswer/server/query_and_chat/chat_backend.py

@@ -1,6 +1,7 @@
 import asyncio
 import io
 import json
+import os
 import uuid
 from collections.abc import Callable
 from collections.abc import Generator
@@ -23,6 +24,9 @@ from danswer.auth.users import current_user
 from danswer.chat.chat_utils import create_chat_chain
 from danswer.chat.chat_utils import extract_headers
 from danswer.chat.process_message import stream_chat_message
+from danswer.chat.prompt_builder.citations_prompt import (
+    compute_max_document_tokens_for_persona,
+)
 from danswer.configs.app_configs import WEB_DOMAIN
 from danswer.configs.constants import FileOrigin
 from danswer.configs.constants import MessageType
@@ -47,13 +51,11 @@ from danswer.db.models import User
 from danswer.db.persona import get_persona_by_id
 from danswer.document_index.document_index_utils import get_both_index_names
 from danswer.document_index.factory import get_default_document_index
+from danswer.file_processing.extract_file_text import docx_to_txt_filename
 from danswer.file_processing.extract_file_text import extract_file_text
 from danswer.file_store.file_store import get_default_file_store
 from danswer.file_store.models import ChatFileType
 from danswer.file_store.models import FileDescriptor
-from danswer.llm.answering.prompts.citations_prompt import (
-    compute_max_document_tokens_for_persona,
-)
 from danswer.llm.exceptions import GenAIDisabledException
 from danswer.llm.factory import get_default_llms
 from danswer.llm.factory import get_llms_for_persona
@@ -718,6 +720,18 @@ def fetch_chat_file(
     if not file_record:
         raise HTTPException(status_code=404, detail="File not found")
 
+    original_file_name = file_record.display_name
+    if file_record.file_type.startswith(
+        "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
+    ):
+        # Check if a converted text file exists for .docx files
+        txt_file_name = docx_to_txt_filename(original_file_name)
+        txt_file_id = os.path.join(os.path.dirname(file_id), txt_file_name)
+        txt_file_record = file_store.read_file_record(txt_file_id)
+        if txt_file_record:
+            file_record = txt_file_record
+            file_id = txt_file_id
+
     media_type = file_record.file_type
     file_io = file_store.read_file(file_id, mode="b")
 

backend/danswer/server/query_and_chat/models.py

@@ -1,5 +1,6 @@
 from datetime import datetime
 from typing import Any
+from typing import TYPE_CHECKING
 from uuid import UUID
 
 from pydantic import BaseModel
@@ -22,6 +23,9 @@ from danswer.llm.override_models import LLMOverride
 from danswer.llm.override_models import PromptOverride
 from danswer.tools.models import ToolCallFinalResult
 
+if TYPE_CHECKING:
+    pass
+
 
 class SourceTag(Tag):
     source: DocumentSource

backend/danswer/setup.py

@@ -4,6 +4,7 @@ from sqlalchemy.orm import Session
 
 from danswer.configs.app_configs import DISABLE_INDEX_UPDATE_ON_SWAP
 from danswer.configs.app_configs import MANAGED_VESPA
+from danswer.configs.app_configs import VESPA_NUM_ATTEMPTS_ON_STARTUP
 from danswer.configs.constants import KV_REINDEX_KEY
 from danswer.configs.constants import KV_SEARCH_SETTINGS
 from danswer.configs.model_configs import FAST_GEN_AI_MODEL_VERSION
@@ -38,7 +39,6 @@ from danswer.key_value_store.interface import KvKeyNotFoundError
 from danswer.natural_language_processing.search_nlp_models import EmbeddingModel
 from danswer.natural_language_processing.search_nlp_models import warm_up_bi_encoder
 from danswer.natural_language_processing.search_nlp_models import warm_up_cross_encoder
-from danswer.seeding.load_docs import seed_initial_documents
 from danswer.seeding.load_yamls import load_chat_yamls
 from danswer.server.manage.llm.models import LLMProviderUpsertRequest
 from danswer.server.settings.store import load_settings
@@ -150,7 +150,7 @@ def setup_danswer(
     # update multipass indexing setting based on GPU availability
     update_default_multipass_indexing(db_session)
 
-    seed_initial_documents(db_session, tenant_id, cohere_enabled)
+    # seed_initial_documents(db_session, tenant_id, cohere_enabled)
 
 
 def translate_saved_search_settings(db_session: Session) -> None:
@@ -221,13 +221,13 @@ def setup_vespa(
     document_index: DocumentIndex,
     index_setting: IndexingSetting,
     secondary_index_setting: IndexingSetting | None,
+    num_attempts: int = VESPA_NUM_ATTEMPTS_ON_STARTUP,
 ) -> bool:
     # Vespa startup is a bit slow, so give it a few seconds
     WAIT_SECONDS = 5
-    VESPA_ATTEMPTS = 5
-    for x in range(VESPA_ATTEMPTS):
+    for x in range(num_attempts):
         try:
-            logger.notice(f"Setting up Vespa (attempt {x+1}/{VESPA_ATTEMPTS})...")
+            logger.notice(f"Setting up Vespa (attempt {x+1}/{num_attempts})...")
             document_index.ensure_indices_exist(
                 index_embedding_dim=index_setting.model_dim,
                 secondary_index_embedding_dim=secondary_index_setting.model_dim
@@ -244,7 +244,7 @@ def setup_vespa(
             time.sleep(WAIT_SECONDS)
 
     logger.error(
-        f"Vespa setup did not succeed. Attempt limit reached. ({VESPA_ATTEMPTS})"
+        f"Vespa setup did not succeed. Attempt limit reached. ({num_attempts})"
     )
     return False
 

backend/danswer/tools/base_tool.py

@@ -7,7 +7,7 @@ from danswer.llm.utils import message_to_prompt_and_imgs
 from danswer.tools.tool import Tool
 
 if TYPE_CHECKING:
-    from danswer.llm.answering.prompts.build import AnswerPromptBuilder
+    from danswer.chat.prompt_builder.build import AnswerPromptBuilder
     from danswer.tools.tool_implementations.custom.custom_tool import (
         CustomToolCallSummary,
     )

backend/danswer/tools/tool.py

@@ -3,13 +3,13 @@ from collections.abc import Generator
 from typing import Any
 from typing import TYPE_CHECKING
 
-from danswer.llm.answering.models import PreviousMessage
 from danswer.llm.interfaces import LLM
+from danswer.llm.models import PreviousMessage
 from danswer.utils.special_types import JSON_ro
 
 
 if TYPE_CHECKING:
-    from danswer.llm.answering.prompts.build import AnswerPromptBuilder
+    from danswer.chat.prompt_builder.build import AnswerPromptBuilder
     from danswer.tools.message import ToolCallSummary
     from danswer.tools.models import ToolResponse
 

backend/danswer/tools/tool_constructor.py

@@ -5,6 +5,10 @@ from pydantic import BaseModel
 from pydantic import Field
 from sqlalchemy.orm import Session
 
+from danswer.chat.models import AnswerStyleConfig
+from danswer.chat.models import CitationConfig
+from danswer.chat.models import DocumentPruningConfig
+from danswer.chat.models import PromptConfig
 from danswer.configs.app_configs import AZURE_DALLE_API_BASE
 from danswer.configs.app_configs import AZURE_DALLE_API_KEY
 from danswer.configs.app_configs import AZURE_DALLE_API_VERSION
@@ -19,10 +23,6 @@ from danswer.db.llm import fetch_existing_llm_providers
 from danswer.db.models import Persona
 from danswer.db.models import User
 from danswer.file_store.models import InMemoryChatFile
-from danswer.llm.answering.models import AnswerStyleConfig
-from danswer.llm.answering.models import CitationConfig
-from danswer.llm.answering.models import DocumentPruningConfig
-from danswer.llm.answering.models import PromptConfig
 from danswer.llm.interfaces import LLM
 from danswer.llm.interfaces import LLMConfig
 from danswer.natural_language_processing.utils import get_tokenizer

backend/danswer/tools/tool_implementations/custom/custom_tool.py

@@ -15,14 +15,14 @@ from langchain_core.messages import SystemMessage
 from pydantic import BaseModel
 from requests import JSONDecodeError
 
+from danswer.chat.prompt_builder.build import AnswerPromptBuilder
 from danswer.configs.constants import FileOrigin
 from danswer.db.engine import get_session_with_default_tenant
 from danswer.file_store.file_store import get_default_file_store
 from danswer.file_store.models import ChatFileType
 from danswer.file_store.models import InMemoryChatFile
-from danswer.llm.answering.models import PreviousMessage
-from danswer.llm.answering.prompts.build import AnswerPromptBuilder
 from danswer.llm.interfaces import LLM
+from danswer.llm.models import PreviousMessage
 from danswer.tools.base_tool import BaseTool
 from danswer.tools.message import ToolCallSummary
 from danswer.tools.models import CHAT_SESSION_ID_PLACEHOLDER

backend/danswer/tools/tool_implementations/images/image_generation_tool.py

@@ -4,14 +4,16 @@ from enum import Enum
 from typing import Any
 from typing import cast
 
+import requests
 from litellm import image_generation  # type: ignore
 from pydantic import BaseModel
 
 from danswer.chat.chat_utils import combine_message_chain
+from danswer.chat.prompt_builder.build import AnswerPromptBuilder
 from danswer.configs.model_configs import GEN_AI_HISTORY_CUTOFF
-from danswer.llm.answering.models import PreviousMessage
-from danswer.llm.answering.prompts.build import AnswerPromptBuilder
+from danswer.configs.tool_configs import IMAGE_GENERATION_OUTPUT_FORMAT
 from danswer.llm.interfaces import LLM
+from danswer.llm.models import PreviousMessage
 from danswer.llm.utils import build_content_with_imgs
 from danswer.llm.utils import message_to_string
 from danswer.prompts.constants import GENERAL_SEP_PAT
@@ -56,9 +58,18 @@ Follow Up Input:
 """.strip()
 
 
+class ImageFormat(str, Enum):
+    URL = "url"
+    BASE64 = "b64_json"
+
+
+_DEFAULT_OUTPUT_FORMAT = ImageFormat(IMAGE_GENERATION_OUTPUT_FORMAT)
+
+
 class ImageGenerationResponse(BaseModel):
     revised_prompt: str
-    url: str
+    url: str | None
+    image_data: str | None
 
 
 class ImageShape(str, Enum):
@@ -80,6 +91,7 @@ class ImageGenerationTool(Tool):
         model: str = "dall-e-3",
         num_imgs: int = 2,
         additional_headers: dict[str, str] | None = None,
+        output_format: ImageFormat = _DEFAULT_OUTPUT_FORMAT,
     ) -> None:
         self.api_key = api_key
         self.api_base = api_base
@@ -89,6 +101,7 @@ class ImageGenerationTool(Tool):
         self.num_imgs = num_imgs
 
         self.additional_headers = additional_headers
+        self.output_format = output_format
 
     @property
     def name(self) -> str:
@@ -168,7 +181,7 @@ class ImageGenerationTool(Tool):
         )
 
         return build_content_with_imgs(
-            json.dumps(
+            message=json.dumps(
                 [
                     {
                         "revised_prompt": image_generation.revised_prompt,
@@ -177,13 +190,10 @@ class ImageGenerationTool(Tool):
                     for image_generation in image_generations
                 ]
             ),
-            # NOTE: we can't pass in the image URLs here, since OpenAI doesn't allow
-            # Tool messages to contain images
-            # img_urls=[image_generation.url for image_generation in image_generations],
         )
 
     def _generate_image(
-        self, prompt: str, shape: ImageShape
+        self, prompt: str, shape: ImageShape, format: ImageFormat
     ) -> ImageGenerationResponse:
         if shape == ImageShape.LANDSCAPE:
             size = "1792x1024"
@@ -197,20 +207,32 @@ class ImageGenerationTool(Tool):
                 prompt=prompt,
                 model=self.model,
                 api_key=self.api_key,
-                # need to pass in None rather than empty str
                 api_base=self.api_base or None,
                 api_version=self.api_version or None,
                 size=size,
                 n=1,
+                response_format=format,
                 extra_headers=build_llm_extra_headers(self.additional_headers),
             )
+
+            if format == ImageFormat.URL:
+                url = response.data[0]["url"]
+                image_data = None
+            else:
+                url = None
+                image_data = response.data[0]["b64_json"]
+
             return ImageGenerationResponse(
                 revised_prompt=response.data[0]["revised_prompt"],
-                url=response.data[0]["url"],
+                url=url,
+                image_data=image_data,
             )
 
+        except requests.RequestException as e:
+            logger.error(f"Error fetching or converting image: {e}")
+            raise ValueError("Failed to fetch or convert the generated image")
         except Exception as e:
-            logger.debug(f"Error occured during image generation: {e}")
+            logger.debug(f"Error occurred during image generation: {e}")
 
             error_message = str(e)
             if "OpenAIException" in str(type(e)):
@@ -235,9 +257,8 @@ class ImageGenerationTool(Tool):
     def run(self, **kwargs: str) -> Generator[ToolResponse, None, None]:
         prompt = cast(str, kwargs["prompt"])
         shape = ImageShape(kwargs.get("shape", ImageShape.SQUARE))
+        format = self.output_format
 
-        # dalle3 only supports 1 image at a time, which is why we have to
-        # parallelize this via threading
         results = cast(
             list[ImageGenerationResponse],
             run_functions_tuples_in_parallel(
@@ -247,6 +268,7 @@ class ImageGenerationTool(Tool):
                         (
                             prompt,
                             shape,
+                            format,
                         ),
                     )
                     for _ in range(self.num_imgs)
@@ -288,11 +310,17 @@ class ImageGenerationTool(Tool):
         if img_generation_response is None:
             raise ValueError("No image generation response found")
 
-        img_urls = [img.url for img in img_generation_response]
+        img_urls = [img.url for img in img_generation_response if img.url is not None]
+        b64_imgs = [
+            img.image_data
+            for img in img_generation_response
+            if img.image_data is not None
+        ]
         prompt_builder.update_user_prompt(
             build_image_generation_user_prompt(
                 query=prompt_builder.get_user_message_content(),
                 img_urls=img_urls,
+                b64_imgs=b64_imgs,
             )
         )
 

backend/danswer/tools/tool_implementations/images/prompt.py

@@ -11,11 +11,14 @@ Can you please summarize them in a sentence or two? Do NOT include image urls or
 
 
 def build_image_generation_user_prompt(
-    query: str, img_urls: list[str] | None = None
+    query: str,
+    img_urls: list[str] | None = None,
+    b64_imgs: list[str] | None = None,
 ) -> HumanMessage:
     return HumanMessage(
         content=build_content_with_imgs(
             message=IMG_GENERATION_SUMMARY_PROMPT.format(query=query).strip(),
+            b64_imgs=b64_imgs,
             img_urls=img_urls,
         )
     )

backend/danswer/tools/tool_implementations/internet_search/internet_search_tool.py

@@ -7,15 +7,15 @@ from typing import cast
 import httpx
 
 from danswer.chat.chat_utils import combine_message_chain
+from danswer.chat.models import AnswerStyleConfig
 from danswer.chat.models import LlmDoc
+from danswer.chat.models import PromptConfig
+from danswer.chat.prompt_builder.build import AnswerPromptBuilder
 from danswer.configs.constants import DocumentSource
 from danswer.configs.model_configs import GEN_AI_HISTORY_CUTOFF
 from danswer.context.search.models import SearchDoc
-from danswer.llm.answering.models import AnswerStyleConfig
-from danswer.llm.answering.models import PreviousMessage
-from danswer.llm.answering.models import PromptConfig
-from danswer.llm.answering.prompts.build import AnswerPromptBuilder
 from danswer.llm.interfaces import LLM
+from danswer.llm.models import PreviousMessage
 from danswer.llm.utils import message_to_string
 from danswer.prompts.chat_prompts import INTERNET_SEARCH_QUERY_REPHRASE
 from danswer.prompts.constants import GENERAL_SEP_PAT

backend/danswer/tools/tool_implementations/search/search_tool.py

@@ -7,10 +7,19 @@ from pydantic import BaseModel
 from sqlalchemy.orm import Session
 
 from danswer.chat.chat_utils import llm_doc_from_inference_section
+from danswer.chat.llm_response_handler import LLMCall
+from danswer.chat.models import AnswerStyleConfig
+from danswer.chat.models import ContextualPruningConfig
 from danswer.chat.models import DanswerContext
 from danswer.chat.models import DanswerContexts
+from danswer.chat.models import DocumentPruningConfig
 from danswer.chat.models import LlmDoc
+from danswer.chat.models import PromptConfig
 from danswer.chat.models import SectionRelevancePiece
+from danswer.chat.prompt_builder.build import AnswerPromptBuilder
+from danswer.chat.prompt_builder.citations_prompt import compute_max_llm_input_tokens
+from danswer.chat.prune_and_merge import prune_and_merge_sections
+from danswer.chat.prune_and_merge import prune_sections
 from danswer.configs.chat_configs import CONTEXT_CHUNKS_ABOVE
 from danswer.configs.chat_configs import CONTEXT_CHUNKS_BELOW
 from danswer.configs.model_configs import GEN_AI_MODEL_FALLBACK_MAX_TOKENS
@@ -25,17 +34,8 @@ from danswer.context.search.models import SearchRequest
 from danswer.context.search.pipeline import SearchPipeline
 from danswer.db.models import Persona
 from danswer.db.models import User
-from danswer.llm.answering.llm_response_handler import LLMCall
-from danswer.llm.answering.models import AnswerStyleConfig
-from danswer.llm.answering.models import ContextualPruningConfig
-from danswer.llm.answering.models import DocumentPruningConfig
-from danswer.llm.answering.models import PreviousMessage
-from danswer.llm.answering.models import PromptConfig
-from danswer.llm.answering.prompts.build import AnswerPromptBuilder
-from danswer.llm.answering.prompts.citations_prompt import compute_max_llm_input_tokens
-from danswer.llm.answering.prune_and_merge import prune_and_merge_sections
-from danswer.llm.answering.prune_and_merge import prune_sections
 from danswer.llm.interfaces import LLM
+from danswer.llm.models import PreviousMessage
 from danswer.secondary_llm_flows.choose_search import check_if_need_search
 from danswer.secondary_llm_flows.query_expansion import history_based_query_rephrase
 from danswer.tools.message import ToolCallSummary
@@ -48,6 +48,9 @@ from danswer.tools.tool_implementations.search_like_tool_utils import (
 from danswer.tools.tool_implementations.search_like_tool_utils import (
     FINAL_CONTEXT_DOCUMENTS_ID,
 )
+from danswer.tools.tool_implementations.search_like_tool_utils import (
+    ORIGINAL_CONTEXT_DOCUMENTS_ID,
+)
 from danswer.utils.logger import setup_logger
 from danswer.utils.special_types import JSON_ro
 
@@ -391,15 +394,35 @@ class SearchTool(Tool):
     """Other utility functions"""
 
     @classmethod
-    def get_search_result(cls, llm_call: LLMCall) -> list[LlmDoc] | None:
+    def get_search_result(
+        cls, llm_call: LLMCall
+    ) -> tuple[list[LlmDoc], dict[str, int]] | None:
+        """
+        Returns the final search results and a map of docs to their original search rank (which is what is displayed to user)
+        """
         if not llm_call.tool_call_info:
             return None
 
+        final_search_results = []
+        doc_id_to_original_search_rank_map = {}
+
         for yield_item in llm_call.tool_call_info:
             if (
                 isinstance(yield_item, ToolResponse)
                 and yield_item.id == FINAL_CONTEXT_DOCUMENTS_ID
             ):
-                return cast(list[LlmDoc], yield_item.response)
+                final_search_results = cast(list[LlmDoc], yield_item.response)
+            elif (
+                isinstance(yield_item, ToolResponse)
+                and yield_item.id == ORIGINAL_CONTEXT_DOCUMENTS_ID
+            ):
+                search_contexts = yield_item.response.contexts
+                original_doc_search_rank = 1
+                for idx, doc in enumerate(search_contexts):
+                    if doc.document_id not in doc_id_to_original_search_rank_map:
+                        doc_id_to_original_search_rank_map[
+                            doc.document_id
+                        ] = original_doc_search_rank
+                        original_doc_search_rank += 1
 
-        return None
+        return final_search_results, doc_id_to_original_search_rank_map