don't wait in multi tenant case

.github/workflows/docker-build-push-backend-container-on-tag.yml

@@ -3,61 +3,61 @@ name: Build and Push Backend Image on Tag
 on:
   push:
     tags:
-      - '*'
+      - "*"
 
 env:
-  REGISTRY_IMAGE: danswer/danswer-backend
+  REGISTRY_IMAGE: ${{ contains(github.ref_name, 'cloud') && 'danswer/danswer-backend-cloud' || 'danswer/danswer-backend' }}
   LATEST_TAG: ${{ contains(github.ref_name, 'latest') }}
-  
+
 jobs:
   build-and-push:
-    # TODO: investigate a matrix build like the web container 
+    # TODO: investigate a matrix build like the web container
     # See https://runs-on.com/runners/linux/
-    runs-on: [runs-on,runner=8cpu-linux-x64,"run-id=${{ github.run_id }}"]
+    runs-on: [runs-on, runner=8cpu-linux-x64, "run-id=${{ github.run_id }}"]
 
     steps:
-    - name: Checkout code
-      uses: actions/checkout@v4
+      - name: Checkout code
+        uses: actions/checkout@v4
 
-    - name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@v3
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
 
-    - name: Login to Docker Hub
-      uses: docker/login-action@v3
-      with:
-        username: ${{ secrets.DOCKER_USERNAME }}
-        password: ${{ secrets.DOCKER_TOKEN }}
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_TOKEN }}
 
-    - name: Install build-essential
-      run: |
-        sudo apt-get update
-        sudo apt-get install -y build-essential
-          
-    - name: Backend Image Docker Build and Push
-      uses: docker/build-push-action@v5
-      with:
-        context: ./backend
-        file: ./backend/Dockerfile
-        platforms: linux/amd64,linux/arm64
-        push: true
-        tags: |
-          ${{ env.REGISTRY_IMAGE }}:${{ github.ref_name }}
-          ${{ env.LATEST_TAG == 'true' && format('{0}:latest', env.REGISTRY_IMAGE) || '' }}
-        build-args: |
-          DANSWER_VERSION=${{ github.ref_name }}
+      - name: Install build-essential
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y build-essential
 
-    # trivy has their own rate limiting issues causing this action to flake
-    # we worked around it by hardcoding to different db repos in env
-    # can re-enable when they figure it out
-    # https://github.com/aquasecurity/trivy/discussions/7538
-    # https://github.com/aquasecurity/trivy-action/issues/389
-    - name: Run Trivy vulnerability scanner
-      uses: aquasecurity/trivy-action@master
-      env:
-        TRIVY_DB_REPOSITORY: 'public.ecr.aws/aquasecurity/trivy-db:2'
-        TRIVY_JAVA_DB_REPOSITORY: 'public.ecr.aws/aquasecurity/trivy-java-db:1'
-      with:
-        # To run locally: trivy image --severity HIGH,CRITICAL danswer/danswer-backend
-        image-ref: docker.io/${{ env.REGISTRY_IMAGE }}:${{ github.ref_name }}
-        severity: 'CRITICAL,HIGH'
-        trivyignores: ./backend/.trivyignore
+      - name: Backend Image Docker Build and Push
+        uses: docker/build-push-action@v5
+        with:
+          context: ./backend
+          file: ./backend/Dockerfile
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: |
+            ${{ env.REGISTRY_IMAGE }}:${{ github.ref_name }}
+            ${{ env.LATEST_TAG == 'true' && format('{0}:latest', env.REGISTRY_IMAGE) || '' }}
+          build-args: |
+            DANSWER_VERSION=${{ github.ref_name }}
+
+      # trivy has their own rate limiting issues causing this action to flake
+      # we worked around it by hardcoding to different db repos in env
+      # can re-enable when they figure it out
+      # https://github.com/aquasecurity/trivy/discussions/7538
+      # https://github.com/aquasecurity/trivy-action/issues/389
+      - name: Run Trivy vulnerability scanner
+        uses: aquasecurity/trivy-action@master
+        env:
+          TRIVY_DB_REPOSITORY: "public.ecr.aws/aquasecurity/trivy-db:2"
+          TRIVY_JAVA_DB_REPOSITORY: "public.ecr.aws/aquasecurity/trivy-java-db:1"
+        with:
+          # To run locally: trivy image --severity HIGH,CRITICAL danswer/danswer-backend
+          image-ref: docker.io/${{ env.REGISTRY_IMAGE }}:${{ github.ref_name }}
+          severity: "CRITICAL,HIGH"
+          trivyignores: ./backend/.trivyignore

.github/workflows/docker-build-push-cloud-web-container-on-tag.yml

@@ -4,12 +4,12 @@ name: Build and Push Cloud Web Image on Tag
 on:
   push:
     tags:
-      - '*'
+      - "*"
 
 env:
-  REGISTRY_IMAGE: danswer/danswer-cloud-web-server
+  REGISTRY_IMAGE: danswer/danswer-web-server-cloud
   LATEST_TAG: ${{ contains(github.ref_name, 'latest') }}
-  
+
 jobs:
   build:
     runs-on:
@@ -28,11 +28,11 @@ jobs:
       - name: Prepare
         run: |
           platform=${{ matrix.platform }}
-          echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV          
-      
+          echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV
+
       - name: Checkout
         uses: actions/checkout@v4
-      
+
       - name: Docker meta
         id: meta
         uses: docker/metadata-action@v5
@@ -41,16 +41,16 @@ jobs:
           tags: |
             type=raw,value=${{ env.REGISTRY_IMAGE }}:${{ github.ref_name }}
             type=raw,value=${{ env.LATEST_TAG == 'true' && format('{0}:latest', env.REGISTRY_IMAGE) || '' }}
-      
+
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
-      
+
       - name: Login to Docker Hub
         uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_TOKEN }}
-    
+
       - name: Build and push by digest
         id: build
         uses: docker/build-push-action@v5
@@ -65,17 +65,17 @@ jobs:
             NEXT_PUBLIC_POSTHOG_KEY=${{ secrets.POSTHOG_KEY }}
             NEXT_PUBLIC_POSTHOG_HOST=${{ secrets.POSTHOG_HOST }}
             NEXT_PUBLIC_SENTRY_DSN=${{ secrets.SENTRY_DSN }}
-          # needed due to weird interactions with the builds for different platforms  
+          # needed due to weird interactions with the builds for different platforms
           no-cache: true
           labels: ${{ steps.meta.outputs.labels }}
           outputs: type=image,name=${{ env.REGISTRY_IMAGE }},push-by-digest=true,name-canonical=true,push=true
-      
+
       - name: Export digest
         run: |
           mkdir -p /tmp/digests
           digest="${{ steps.build.outputs.digest }}"
-          touch "/tmp/digests/${digest#sha256:}"          
-      
+          touch "/tmp/digests/${digest#sha256:}"
+
       - name: Upload digest
         uses: actions/upload-artifact@v4
         with:
@@ -95,42 +95,42 @@ jobs:
           path: /tmp/digests
           pattern: digests-*
           merge-multiple: true
-      
+
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
-      
+
       - name: Docker meta
         id: meta
         uses: docker/metadata-action@v5
         with:
           images: ${{ env.REGISTRY_IMAGE }}
-      
+
       - name: Login to Docker Hub
         uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_TOKEN }}
-      
+
       - name: Create manifest list and push
         working-directory: /tmp/digests
         run: |
           docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
-            $(printf '${{ env.REGISTRY_IMAGE }}@sha256:%s ' *)          
-      
+            $(printf '${{ env.REGISTRY_IMAGE }}@sha256:%s ' *)
+
       - name: Inspect image
         run: |
           docker buildx imagetools inspect ${{ env.REGISTRY_IMAGE }}:${{ steps.meta.outputs.version }}
 
-    # trivy has their own rate limiting issues causing this action to flake
-    # we worked around it by hardcoding to different db repos in env
-    # can re-enable when they figure it out
-    # https://github.com/aquasecurity/trivy/discussions/7538
-    # https://github.com/aquasecurity/trivy-action/issues/389
+      # trivy has their own rate limiting issues causing this action to flake
+      # we worked around it by hardcoding to different db repos in env
+      # can re-enable when they figure it out
+      # https://github.com/aquasecurity/trivy/discussions/7538
+      # https://github.com/aquasecurity/trivy-action/issues/389
       - name: Run Trivy vulnerability scanner
         uses: aquasecurity/trivy-action@master
         env:
-          TRIVY_DB_REPOSITORY: 'public.ecr.aws/aquasecurity/trivy-db:2'
-          TRIVY_JAVA_DB_REPOSITORY: 'public.ecr.aws/aquasecurity/trivy-java-db:1'
+          TRIVY_DB_REPOSITORY: "public.ecr.aws/aquasecurity/trivy-db:2"
+          TRIVY_JAVA_DB_REPOSITORY: "public.ecr.aws/aquasecurity/trivy-java-db:1"
         with:
           image-ref: docker.io/${{ env.REGISTRY_IMAGE }}:${{ github.ref_name }}
-          severity: 'CRITICAL,HIGH'
+          severity: "CRITICAL,HIGH"

.github/workflows/docker-build-push-model-server-container-on-tag.yml

@@ -6,7 +6,13 @@ on:
       - '*'
 
 env:
-  REGISTRY_IMAGE: danswer/danswer-model-server
+  REGISTRY_IMAGE: ${{ contains(github.ref_name, 'cloud') && 'danswer/danswer-model-server-cloud' || 'danswer/danswer-model-server' }}
+  LATEST_TAG: ${{ contains(github.ref_name, 'latest') }}
+
+jobs:
+  build-and-push:
+    # See https://runs-on.com/runners/linux/
+    runs-on: [runs-on,runner=8cpu-linux-x64,"run-id=${{ github.run_id }}"]
   LATEST_TAG: ${{ contains(github.ref_name, 'latest') }}
   
 jobs:

backend/danswer/background/celery/apps/app_base.py

@@ -146,6 +146,8 @@ def on_celeryd_init(sender: Any = None, conf: Any = None, **kwargs: Any) -> None
 def wait_for_redis(sender: Any, **kwargs: Any) -> None:
     """Waits for redis to become ready subject to a hardcoded timeout.
     Will raise WorkerShutdown to kill the celery worker if the timeout is reached."""
+    if MULTI_TENANT:
+        return
 
     r = get_redis_client(tenant_id=None)
 
@@ -188,7 +190,8 @@ def wait_for_redis(sender: Any, **kwargs: Any) -> None:
 def wait_for_db(sender: Any, **kwargs: Any) -> None:
     """Waits for the db to become ready subject to a hardcoded timeout.
     Will raise WorkerShutdown to kill the celery worker if the timeout is reached."""
-
+    if MULTI_TENANT:
+        return
     WAIT_INTERVAL = 5
     WAIT_LIMIT = 60
 
@@ -230,6 +233,8 @@ def wait_for_db(sender: Any, **kwargs: Any) -> None:
 def wait_for_vespa(sender: Any, **kwargs: Any) -> None:
     """Waits for Vespa to become ready subject to a hardcoded timeout.
     Will raise WorkerShutdown to kill the celery worker if the timeout is reached."""
+    if MULTI_TENANT:
+        return
 
     WAIT_INTERVAL = 5
     WAIT_LIMIT = 60

deployment/cloud_kubernetes/keda/celery-worker-indexing-scaledobject.yaml

@@ -0,0 +1,43 @@
+apiVersion: keda.sh/v1alpha1
+kind: ScaledObject
+metadata:
+  name: celery-worker-indexing-scaledobject
+  namespace: danswer
+  labels:
+    app: celery-worker-indexing
+spec:
+  scaleTargetRef:
+    name: celery-worker-indexing
+  minReplicaCount: 1
+  maxReplicaCount: 10 # Adjust based on expected workload
+  triggers:
+    - type: redis
+      metadata:
+        sslEnabled: "true" # Set to "true" if using SSL
+        host: "master.redisencrypted.4h4xor.use2.cache.amazonaws.com"
+        port: "6379"
+        enableTLS: "true"
+        listName: connector_indexing
+        listLength: "1" # Scale up as soon as there's a task
+        password: "50lbsAuthTokenDanswer"
+        databaseIndex: "15"
+    - type: redis
+      metadata:
+        sslEnabled: "true" # Set to "true" if using SSL
+        host: "master.redisencrypted.4h4xor.use2.cache.amazonaws.com"
+        port: "6379"
+        enableTLS: "true"
+        listName: connector_indexing:2
+        listLength: "1" # Scale up as soon as there's a task
+        password: "50lbsAuthTokenDanswer"
+        databaseIndex: "15"
+    - type: redis
+      metadata:
+        sslEnabled: "true" # Set to "true" if using SSL
+        host: "master.redisencrypted.4h4xor.use2.cache.amazonaws.com"
+        port: "6379"
+        enableTLS: "true"
+        listName: connector_indexing:3
+        listLength: "1" # Scale up as soon as there's a task
+        password: "50lbsAuthTokenDanswer"
+        databaseIndex: "15"

deployment/cloud_kubernetes/keda/celery-worker-light-scaledobject.yaml

@@ -0,0 +1,58 @@
+apiVersion: keda.sh/v1alpha1
+kind: ScaledObject
+metadata:
+  name: celery-worker-light-scaledobject
+  namespace: danswer
+  labels:
+    app: celery-worker-light
+spec:
+  scaleTargetRef:
+    name: celery-worker-light
+  minReplicaCount: 1
+  maxReplicaCount: 15
+  triggers:
+    - type: redis
+      metadata:
+        host: "master.redisencrypted.4h4xor.use2.cache.amazonaws.com"
+        port: "6379"
+        enableTLS: "true"
+        listName: vespa_metadata_sync
+        listLength: "1" # Scale up as soon as there's a task
+        password: "50lbsAuthTokenDanswer"
+        databaseIndex: "15"
+    - type: redis
+      metadata:
+        host: "master.redisencrypted.4h4xor.use2.cache.amazonaws.com"
+        port: "6379"
+        enableTLS: "true"
+        listName: vespa_metadata_sync:2
+        listLength: "1" # Scale up as soon as there's a task
+        password: "50lbsAuthTokenDanswer"
+        databaseIndex: "15"
+    - type: redis
+      metadata:
+        host: "master.redisencrypted.4h4xor.use2.cache.amazonaws.com"
+        port: "6379"
+        enableTLS: "true"
+        listName: vespa_metadata_sync:3
+        listLength: "1" # Scale up as soon as there's a task
+        password: "50lbsAuthTokenDanswer"
+        databaseIndex: "15"
+    - type: redis
+      metadata:
+        host: "master.redisencrypted.4h4xor.use2.cache.amazonaws.com"
+        port: "6379"
+        enableTLS: "true"
+        listName: connector_deletion
+        listLength: "1" # Scale up as soon as there's a task
+        password: "50lbsAuthTokenDanswer"
+        databaseIndex: "15"
+    - type: redis
+      metadata:
+        host: "master.redisencrypted.4h4xor.use2.cache.amazonaws.com"
+        port: "6379"
+        enableTLS: "true"
+        listName: connector_deletion:2
+        listLength: "1" # Scale up as soon as there's a task
+        password: "50lbsAuthTokenDanswer"
+        databaseIndex: "15"

deployment/cloud_kubernetes/keda/celery-worker-primary-scaledobject.yaml

@@ -0,0 +1,70 @@
+apiVersion: keda.sh/v1alpha1
+kind: ScaledObject
+metadata:
+  name: celery-worker-primary-scaledobject
+  namespace: danswer
+  labels:
+    app: celery-worker-primary
+spec:
+  scaleTargetRef:
+    name: celery-worker-primary
+  pollingInterval: 15 # Check every 15 seconds
+  cooldownPeriod: 30 # Wait 30 seconds before scaling down
+  minReplicaCount: 1
+  maxReplicaCount: 20
+  triggers:
+    - type: redis
+      metadata:
+        host: "master.redisencrypted.4h4xor.use2.cache.amazonaws.com"
+        port: "6379"
+        enableTLS: "true"
+        listName: celery
+        listLength: "1" # Scale up as soon as there's a task
+        password: "50lbsAuthTokenDanswer"
+        databaseIndex: "15"
+
+    - type: redis
+      metadata:
+        host: "master.redisencrypted.4h4xor.use2.cache.amazonaws.com"
+        port: "6379"
+        enableTLS: "true"
+        listName: celery:1
+        listLength: "1" # Scale up as soon as there's a task
+        password: "50lbsAuthTokenDanswer"
+        databaseIndex: "15"
+    - type: redis
+      metadata:
+        host: "master.redisencrypted.4h4xor.use2.cache.amazonaws.com"
+        port: "6379"
+        enableTLS: "true"
+        listName: celery:2
+        listLength: "1" # Scale up as soon as there's a task
+        password: "50lbsAuthTokenDanswer"
+        databaseIndex: "15"
+    - type: redis
+      metadata:
+        host: "master.redisencrypted.4h4xor.use2.cache.amazonaws.com"
+        port: "6379"
+        enableTLS: "true"
+        listName: celery:3
+        listLength: "1" # Scale up as soon as there's a task
+        password: "50lbsAuthTokenDanswer"
+        databaseIndex: "15"
+    - type: redis
+      metadata:
+        host: "master.redisencrypted.4h4xor.use2.cache.amazonaws.com"
+        port: "6379"
+        enableTLS: "true"
+        listName: periodic_tasks
+        listLength: "1"
+        password: "50lbsAuthTokenDanswer"
+        databaseIndex: "15"
+    - type: redis
+      metadata:
+        host: "master.redisencrypted.4h4xor.use2.cache.amazonaws.com"
+        port: "6379"
+        enableTLS: "true"
+        listName: periodic_tasks:2
+        listLength: "1"
+        password: "50lbsAuthTokenDanswer"
+        databaseIndex: "15"

deployment/cloud_kubernetes/workers/indexing_worker.yaml

@@ -26,6 +26,8 @@ spec:
               "--hostname=indexing@%n",
               "-Q",
               "connector_indexing",
+              "--prefetch-multiplier=1",
+              "--concurrency=10",
             ]
           env:
             - name: REDIS_PASSWORD

deployment/cloud_kubernetes/workers/light_worker.yaml

@@ -26,6 +26,8 @@ spec:
               "--hostname=light@%n",
               "-Q",
               "vespa_metadata_sync,connector_deletion",
+              "--prefetch-multiplier=1",
+              "--concurrency=10",
             ]
           env:
             - name: REDIS_PASSWORD

deployment/cloud_kubernetes/workers/primary.yaml

@@ -25,7 +25,9 @@ spec:
               "--loglevel=INFO",
               "--hostname=primary@%n",
               "-Q",
-              "celery,periodic_tasks,vespa_metadata_sync",
+              "celery,periodic_tasks",
+              "--prefetch-multiplier=1",
+              "--concurrency=10",
             ]
           env:
             - name: REDIS_PASSWORD