more reasonable limits

backend/danswer/background/celery/tasks/beat_schedule.py

@@ -8,25 +8,25 @@ tasks_to_schedule = [
     {
         "name": "check-for-vespa-sync",
         "task": "check_for_vespa_sync_task",
-        "schedule": timedelta(seconds=5),
+        "schedule": timedelta(seconds=60),
         "options": {"priority": DanswerCeleryPriority.HIGH},
     },
     {
         "name": "check-for-connector-deletion",
         "task": "check_for_connector_deletion_task",
-        "schedule": timedelta(seconds=20),
+        "schedule": timedelta(seconds=60),
         "options": {"priority": DanswerCeleryPriority.HIGH},
     },
     {
         "name": "check-for-indexing",
         "task": "check_for_indexing",
-        "schedule": timedelta(seconds=10),
+        "schedule": timedelta(seconds=60),
         "options": {"priority": DanswerCeleryPriority.HIGH},
     },
     {
         "name": "check-for-prune",
         "task": "check_for_pruning",
-        "schedule": timedelta(seconds=10),
+        "schedule": timedelta(seconds=60),
         "options": {"priority": DanswerCeleryPriority.HIGH},
     },
     {
@@ -38,7 +38,7 @@ tasks_to_schedule = [
     {
         "name": "monitor-vespa-sync",
         "task": "monitor_vespa_sync",
-        "schedule": timedelta(seconds=5),
+        "schedule": timedelta(seconds=60),
         "options": {"priority": DanswerCeleryPriority.HIGH},
     },
 ]

deployment/cloud_kubernetes/keda/celery-worker-indexing-scaledobject.yaml

@@ -8,8 +8,8 @@ metadata:
 spec:
   scaleTargetRef:
     name: celery-worker-indexing
-  minReplicaCount: 1
-  maxReplicaCount: 10
+  minReplicaCount: 0
+  maxReplicaCount: 0
   triggers:
     - type: redis
       metadata:

deployment/cloud_kubernetes/keda/celery-worker-light-scaledobject.yaml

@@ -8,12 +8,12 @@ metadata:
 spec:
   scaleTargetRef:
     name: celery-worker-light
-  minReplicaCount: 1
+  minReplicaCount: 5
   maxReplicaCount: 20
   triggers:
     - type: redis
       metadata:
-        host: "{host}"
+        host: "master.redisencrypted.4h4xor.use2.cache.amazonaws.com"
         port: "6379"
         enableTLS: "true"
         listName: vespa_metadata_sync
@@ -23,7 +23,7 @@ spec:
         name: celery-worker-auth
     - type: redis
       metadata:
-        host: "{host}"
+        host: "master.redisencrypted.4h4xor.use2.cache.amazonaws.com"
         port: "6379"
         enableTLS: "true"
         listName: vespa_metadata_sync:2
@@ -33,7 +33,7 @@ spec:
         name: celery-worker-auth
     - type: redis
       metadata:
-        host: "{host}"
+        host: "master.redisencrypted.4h4xor.use2.cache.amazonaws.com"
         port: "6379"
         enableTLS: "true"
         listName: vespa_metadata_sync:3
@@ -43,7 +43,7 @@ spec:
         name: celery-worker-auth
     - type: redis
       metadata:
-        host: "{host}"
+        host: "master.redisencrypted.4h4xor.use2.cache.amazonaws.com"
         port: "6379"
         enableTLS: "true"
         listName: connector_deletion
@@ -53,7 +53,7 @@ spec:
         name: celery-worker-auth
     - type: redis
       metadata:
-        host: "{host}"
+        host: "master.redisencrypted.4h4xor.use2.cache.amazonaws.com"
         port: "6379"
         enableTLS: "true"
         listName: connector_deletion:2

deployment/cloud_kubernetes/keda/celery-worker-primary-scaledobject.yaml

@@ -10,8 +10,8 @@ spec:
     name: celery-worker-primary
   pollingInterval: 15 # Check every 15 seconds
   cooldownPeriod: 30 # Wait 30 seconds before scaling down
-  minReplicaCount: 1
-  maxReplicaCount: 1
+  minReplicaCount: 0
+  maxReplicaCount: 0
   triggers:
     - type: redis
       metadata:

deployment/cloud_kubernetes/keda/keda-redis-secret.yaml

@@ -5,5 +5,3 @@ metadata:
   namespace: danswer
 type: Opaque
 data:
-  host: { { base64-encoded-hostname } }
-  password: { { base64-encoded-password } }