k

* Properly find context window for ollama llama

* Better ollama support + upgrade litellm

* Ugprade OpenAI as well

* Fix mypy

.github/workflows/multi-tenant-tests.yml

@@ -0,0 +1,111 @@
+name: Run Multi-Tenant Integration Tests
+
+on:
+  workflow_dispatch:
+  pull_request:
+    branches:
+      - main
+      - "release/**"
+
+env:
+  OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+  SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
+  CONFLUENCE_TEST_SPACE_URL: ${{ secrets.CONFLUENCE_TEST_SPACE_URL }}
+  CONFLUENCE_USER_NAME: ${{ secrets.CONFLUENCE_USER_NAME }}
+  CONFLUENCE_ACCESS_TOKEN: ${{ secrets.CONFLUENCE_ACCESS_TOKEN }}
+
+jobs:
+  multi-tenant-integration-tests:
+    runs-on:
+      [runs-on, runner=8cpu-linux-x64, ram=16, "run-id=${{ github.run_id }}"]
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_TOKEN }}
+
+      - name: Pull Required Docker Images
+        run: |
+          docker pull danswer/danswer-backend:latest
+          docker tag danswer/danswer-backend:latest danswer/danswer-backend:test
+
+          docker pull danswer/danswer-model-server:latest
+          docker tag danswer/danswer-model-server:latest danswer/danswer-model-server:test
+
+          docker pull danswer/danswer-web-server:latest
+          docker tag danswer/danswer-web-server:latest danswer/danswer-web-server:test
+
+          docker pull danswer/control-tenants-service:latest
+          docker tag danswer/control-tenants-service:latest danswer/control-tenants-service:test
+
+      - name: Build Integration Test Docker Image
+        uses: ./.github/actions/custom-build-and-push
+        with:
+          context: ./backend
+          file: ./backend/tests/integration/Dockerfile
+          platforms: linux/amd64
+          tags: danswer/danswer-integration:test
+          push: false
+          load: true
+
+      - name: Start Docker Containers for Multi-Tenant Tests
+        run: |
+          cd deployment/docker_compose
+          ENABLE_PAID_ENTERPRISE_EDITION_FEATURES=true \
+          MULTI_TENANT=true \
+          INTEGRATION_TEST_MODE=true \  
+          AUTH_TYPE=basic \
+          REQUIRE_EMAIL_VERIFICATION=false \
+          DISABLE_TELEMETRY=true \
+          IMAGE_TAG=test \
+          CONTROL_TENANTS_SERVICE_IMAGE=danswer/control-tenants-service:test \
+          docker compose -f docker-compose.dev.yml -f docker-compose.multi-tenant.yml -p danswer-stack up -d
+
+      - name: Run Multi-Tenant Integration Tests
+        run: |
+          echo "Running multi-tenant integration tests..."
+          docker run --rm --network danswer-stack_default \
+            --name test-runner \
+            -e POSTGRES_HOST=relational_db \
+            -e POSTGRES_USER=postgres \
+            -e POSTGRES_PASSWORD=password \
+            -e POSTGRES_DB=postgres \
+            -e VESPA_HOST=index \
+            -e REDIS_HOST=cache \
+            -e API_SERVER_HOST=api_server \
+            -e OPENAI_API_KEY=${OPENAI_API_KEY} \
+            -e SLACK_BOT_TOKEN=${SLACK_BOT_TOKEN} \
+            -e TEST_WEB_HOSTNAME=test-runner \
+            -e AUTH_TYPE=cloud \
+            -e MULTI_TENANT=true \
+            danswer/danswer-integration:test \
+            /app/tests/integration/multitenant_tests
+        continue-on-error: true
+        id: run_multitenant_tests
+
+      - name: Check Multi-Tenant Test Results
+        run: |
+          if [ ${{ steps.run_multitenant_tests.outcome }} == 'failure' ]; then
+            echo "Integration tests failed. Exiting with error."
+            exit 1
+          else
+            echo "All integration tests passed successfully."
+          fi
+
+      - name: Stop Docker Containers
+        run: |
+          cd deployment/docker_compose
+          docker compose -f docker-compose.dev.yml -p danswer-stack down -v
+
+      - name: Upload Logs
+        if: success() || failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: docker-logs
+          path: ${{ github.workspace }}/docker-compose.log

.github/workflows/pr-integration-tests.yml

@@ -8,7 +8,7 @@ on:
   pull_request:
     branches:
       - main
-      - 'release/**'
+      - "release/**"
 
 env:
   OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
@@ -16,11 +16,12 @@ env:
   CONFLUENCE_TEST_SPACE_URL: ${{ secrets.CONFLUENCE_TEST_SPACE_URL }}
   CONFLUENCE_USER_NAME: ${{ secrets.CONFLUENCE_USER_NAME }}
   CONFLUENCE_ACCESS_TOKEN: ${{ secrets.CONFLUENCE_ACCESS_TOKEN }}
-  
+
 jobs:
   integration-tests:
     # See https://runs-on.com/runners/linux/
-    runs-on: [runs-on,runner=8cpu-linux-x64,ram=16,"run-id=${{ github.run_id }}"]
+    runs-on:
+      [runs-on, runner=8cpu-linux-x64, ram=16, "run-id=${{ github.run_id }}"]
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -36,9 +37,9 @@ jobs:
 
       # tag every docker image with "test" so that we can spin up the correct set
       # of images during testing
-      
+
       # We don't need to build the Web Docker image since it's not yet used
-      # in the integration tests. We have a separate action to verify that it builds 
+      # in the integration tests. We have a separate action to verify that it builds
       # successfully.
       - name: Pull Web Docker image
         run: |
@@ -50,7 +51,7 @@ jobs:
       # https://runs-on.com/caching/s3-cache-for-github-actions/
       # https://runs-on.com/caching/docker/
       # https://github.com/moby/buildkit#s3-cache-experimental
-      
+
       # images are built and run locally for testing purposes. Not pushed.
       - name: Build Backend Docker image
         uses: ./.github/actions/custom-build-and-push
@@ -75,7 +76,7 @@ jobs:
           load: true
           cache-from: type=s3,prefix=cache/${{ github.repository }}/integration-tests/model-server/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }}
           cache-to: type=s3,prefix=cache/${{ github.repository }}/integration-tests/model-server/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }},mode=max
-      
+
       - name: Build integration test Docker image
         uses: ./.github/actions/custom-build-and-push
         with:
@@ -88,58 +89,7 @@ jobs:
           cache-from: type=s3,prefix=cache/${{ github.repository }}/integration-tests/integration/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }}
           cache-to: type=s3,prefix=cache/${{ github.repository }}/integration-tests/integration/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }},mode=max
 
-      # Start containers for multi-tenant tests
-      - name: Start Docker containers for multi-tenant tests
-        run: |
-          cd deployment/docker_compose
-          ENABLE_PAID_ENTERPRISE_EDITION_FEATURES=true \
-          MULTI_TENANT=true \
-          AUTH_TYPE=basic \
-          REQUIRE_EMAIL_VERIFICATION=false \
-          DISABLE_TELEMETRY=true \
-          IMAGE_TAG=test \
-          docker compose -f docker-compose.dev.yml -p danswer-stack up -d
-        id: start_docker_multi_tenant
-
-      # In practice, `cloud` Auth type would require OAUTH credentials to be set.
-      - name: Run Multi-Tenant Integration Tests
-        run: |
-          echo "Running integration tests..."
-          docker run --rm --network danswer-stack_default \
-            --name test-runner \
-            -e POSTGRES_HOST=relational_db \
-            -e POSTGRES_USER=postgres \
-            -e POSTGRES_PASSWORD=password \
-            -e POSTGRES_DB=postgres \
-            -e VESPA_HOST=index \
-            -e REDIS_HOST=cache \
-            -e API_SERVER_HOST=api_server \
-            -e OPENAI_API_KEY=${OPENAI_API_KEY} \
-            -e SLACK_BOT_TOKEN=${SLACK_BOT_TOKEN} \
-            -e TEST_WEB_HOSTNAME=test-runner \
-            -e AUTH_TYPE=cloud \
-            -e MULTI_TENANT=true \
-            danswer/danswer-integration:test \
-            /app/tests/integration/multitenant_tests
-        continue-on-error: true
-        id: run_multitenant_tests
-
-      - name: Check multi-tenant test results
-        run: |
-          if [ ${{ steps.run_tests.outcome }} == 'failure' ]; then
-            echo "Integration tests failed. Exiting with error."
-            exit 1
-          else
-            echo "All integration tests passed successfully."
-          fi 
-
-      - name: Stop multi-tenant Docker containers
-        run: |
-          cd deployment/docker_compose
-          docker compose -f docker-compose.dev.yml -p danswer-stack down -v
-
-
-      - name: Start Docker containers 
+      - name: Start Docker containers
         run: |
           cd deployment/docker_compose
           ENABLE_PAID_ENTERPRISE_EDITION_FEATURES=true \
@@ -153,12 +103,12 @@ jobs:
       - name: Wait for service to be ready
         run: |
           echo "Starting wait-for-service script..."
-          
+
           docker logs -f danswer-stack-api_server-1 &
 
           start_time=$(date +%s)
           timeout=300  # 5 minutes in seconds
-          
+
           while true; do
             current_time=$(date +%s)
             elapsed_time=$((current_time - start_time))
@@ -229,7 +179,7 @@ jobs:
         run: |
           cd deployment/docker_compose
           docker compose -f docker-compose.dev.yml -p danswer-stack down -v
-      
+
       - name: Upload logs
         if: success() || failure()
         uses: actions/upload-artifact@v4

backend/alembic/versions/93560ba1b118_add_web_ui_option_to_slack_config.py

@@ -0,0 +1,35 @@
+"""add web ui option to slack config
+
+Revision ID: 93560ba1b118
+Revises: 6d562f86c78b
+Create Date: 2024-11-24 06:36:17.490612
+
+"""
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "93560ba1b118"
+down_revision = "6d562f86c78b"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # Add show_continue_in_web_ui with default False to all existing channel_configs
+    op.execute(
+        """
+        UPDATE slack_channel_config
+        SET channel_config = channel_config || '{"show_continue_in_web_ui": false}'::jsonb
+        WHERE NOT channel_config ? 'show_continue_in_web_ui'
+        """
+    )
+
+
+def downgrade() -> None:
+    # Remove show_continue_in_web_ui from all channel_configs
+    op.execute(
+        """
+        UPDATE slack_channel_config
+        SET channel_config = channel_config - 'show_continue_in_web_ui'
+        """
+    )

backend/alembic/versions/abe7378b8217_add_indexing_trigger_to_cc_pair.py

@@ -0,0 +1,30 @@
+"""add indexing trigger to cc_pair
+
+Revision ID: abe7378b8217
+Revises: 6d562f86c78b
+Create Date: 2024-11-26 19:09:53.481171
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = "abe7378b8217"
+down_revision = "93560ba1b118"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.add_column(
+        "connector_credential_pair",
+        sa.Column(
+            "indexing_trigger",
+            sa.Enum("UPDATE", "REINDEX", name="indexingmode", native_enum=False),
+            nullable=True,
+        ),
+    )
+
+
+def downgrade() -> None:
+    op.drop_column("connector_credential_pair", "indexing_trigger")

backend/danswer/background/celery/tasks/connector_deletion/tasks.py

@@ -5,7 +5,6 @@ from celery import Celery
 from celery import shared_task
 from celery import Task
 from celery.exceptions import SoftTimeLimitExceeded
-from redis import Redis
 from redis.lock import Lock as RedisLock
 from sqlalchemy.orm import Session
 
@@ -37,7 +36,7 @@ class TaskDependencyError(RuntimeError):
 def check_for_connector_deletion_task(self: Task, *, tenant_id: str | None) -> None:
     r = get_redis_client(tenant_id=tenant_id)
 
-    lock_beat = r.lock(
+    lock_beat: RedisLock = r.lock(
         DanswerRedisLocks.CHECK_CONNECTOR_DELETION_BEAT_LOCK,
         timeout=CELERY_VESPA_SYNC_BEAT_LOCK_TIMEOUT,
     )
@@ -60,7 +59,7 @@ def check_for_connector_deletion_task(self: Task, *, tenant_id: str | None) -> N
                 redis_connector = RedisConnector(tenant_id, cc_pair_id)
                 try:
                     try_generate_document_cc_pair_cleanup_tasks(
-                        self.app, cc_pair_id, db_session, r, lock_beat, tenant_id
+                        self.app, cc_pair_id, db_session, lock_beat, tenant_id
                     )
                 except TaskDependencyError as e:
                     # this means we wanted to start deleting but dependent tasks were running
@@ -86,7 +85,6 @@ def try_generate_document_cc_pair_cleanup_tasks(
     app: Celery,
     cc_pair_id: int,
     db_session: Session,
-    r: Redis,
     lock_beat: RedisLock,
     tenant_id: str | None,
 ) -> int | None:

backend/danswer/background/celery/tasks/doc_permission_syncing/tasks.py

@@ -8,6 +8,7 @@ from celery import shared_task
 from celery import Task
 from celery.exceptions import SoftTimeLimitExceeded
 from redis import Redis
+from redis.lock import Lock as RedisLock
 
 from danswer.access.models import DocExternalAccess
 from danswer.background.celery.apps.app_base import task_logger
@@ -27,7 +28,7 @@ from danswer.db.models import ConnectorCredentialPair
 from danswer.db.users import batch_add_ext_perm_user_if_not_exists
 from danswer.redis.redis_connector import RedisConnector
 from danswer.redis.redis_connector_doc_perm_sync import (
-    RedisConnectorPermissionSyncData,
+    RedisConnectorPermissionSyncPayload,
 )
 from danswer.redis.redis_pool import get_redis_client
 from danswer.utils.logger import doc_permission_sync_ctx
@@ -138,7 +139,7 @@ def try_creating_permissions_sync_task(
 
     LOCK_TIMEOUT = 30
 
-    lock = r.lock(
+    lock: RedisLock = r.lock(
         DANSWER_REDIS_FUNCTION_LOCK_PREFIX + "try_generate_permissions_sync_tasks",
         timeout=LOCK_TIMEOUT,
     )
@@ -162,7 +163,7 @@ def try_creating_permissions_sync_task(
 
         custom_task_id = f"{redis_connector.permissions.generator_task_key}_{uuid4()}"
 
-        app.send_task(
+        result = app.send_task(
             "connector_permission_sync_generator_task",
             kwargs=dict(
                 cc_pair_id=cc_pair_id,
@@ -174,8 +175,8 @@ def try_creating_permissions_sync_task(
         )
 
         # set a basic fence to start
-        payload = RedisConnectorPermissionSyncData(
-            started=None,
+        payload = RedisConnectorPermissionSyncPayload(
+            started=None, celery_task_id=result.id
         )
 
         redis_connector.permissions.set_fence(payload)
@@ -241,13 +242,17 @@ def connector_permission_sync_generator_task(
 
             doc_sync_func = DOC_PERMISSIONS_FUNC_MAP.get(source_type)
             if doc_sync_func is None:
-                raise ValueError(f"No doc sync func found for {source_type}")
+                raise ValueError(
+                    f"No doc sync func found for {source_type} with cc_pair={cc_pair_id}"
+                )
 
-            logger.info(f"Syncing docs for {source_type}")
+            logger.info(f"Syncing docs for {source_type} with cc_pair={cc_pair_id}")
 
-            payload = RedisConnectorPermissionSyncData(
-                started=datetime.now(timezone.utc),
-            )
+            payload = redis_connector.permissions.payload
+            if not payload:
+                raise ValueError(f"No fence payload found: cc_pair={cc_pair_id}")
+
+            payload.started = datetime.now(timezone.utc)
             redis_connector.permissions.set_fence(payload)
 
             document_external_accesses: list[DocExternalAccess] = doc_sync_func(cc_pair)

backend/danswer/background/celery/tasks/external_group_syncing/tasks.py

@@ -8,6 +8,7 @@ from celery import shared_task
 from celery import Task
 from celery.exceptions import SoftTimeLimitExceeded
 from redis import Redis
+from redis.lock import Lock as RedisLock
 
 from danswer.background.celery.apps.app_base import task_logger
 from danswer.configs.app_configs import JOB_TIMEOUT
@@ -24,6 +25,9 @@ from danswer.db.enums import AccessType
 from danswer.db.enums import ConnectorCredentialPairStatus
 from danswer.db.models import ConnectorCredentialPair
 from danswer.redis.redis_connector import RedisConnector
+from danswer.redis.redis_connector_ext_group_sync import (
+    RedisConnectorExternalGroupSyncPayload,
+)
 from danswer.redis.redis_pool import get_redis_client
 from danswer.utils.logger import setup_logger
 from ee.danswer.db.connector_credential_pair import get_all_auto_sync_cc_pairs
@@ -49,7 +53,7 @@ def _is_external_group_sync_due(cc_pair: ConnectorCredentialPair) -> bool:
     if cc_pair.access_type != AccessType.SYNC:
         return False
 
-    # skip pruning if not active
+    # skip external group sync if not active
     if cc_pair.status != ConnectorCredentialPairStatus.ACTIVE:
         return False
 
@@ -107,7 +111,7 @@ def check_for_external_group_sync(self: Task, *, tenant_id: str | None) -> None:
                     cc_pair_ids_to_sync.append(cc_pair.id)
 
         for cc_pair_id in cc_pair_ids_to_sync:
-            tasks_created = try_creating_permissions_sync_task(
+            tasks_created = try_creating_external_group_sync_task(
                 self.app, cc_pair_id, r, tenant_id
             )
             if not tasks_created:
@@ -125,7 +129,7 @@ def check_for_external_group_sync(self: Task, *, tenant_id: str | None) -> None:
             lock_beat.release()
 
 
-def try_creating_permissions_sync_task(
+def try_creating_external_group_sync_task(
     app: Celery,
     cc_pair_id: int,
     r: Redis,
@@ -156,7 +160,7 @@ def try_creating_permissions_sync_task(
 
         custom_task_id = f"{redis_connector.external_group_sync.taskset_key}_{uuid4()}"
 
-        _ = app.send_task(
+        result = app.send_task(
             "connector_external_group_sync_generator_task",
             kwargs=dict(
                 cc_pair_id=cc_pair_id,
@@ -166,8 +170,13 @@ def try_creating_permissions_sync_task(
             task_id=custom_task_id,
             priority=DanswerCeleryPriority.HIGH,
         )
-        # set a basic fence to start
-        redis_connector.external_group_sync.set_fence(True)
+
+        payload = RedisConnectorExternalGroupSyncPayload(
+            started=datetime.now(timezone.utc),
+            celery_task_id=result.id,
+        )
+
+        redis_connector.external_group_sync.set_fence(payload)
 
     except Exception:
         task_logger.exception(
@@ -195,7 +204,7 @@ def connector_external_group_sync_generator_task(
     tenant_id: str | None,
 ) -> None:
     """
-    Permission sync task that handles document permission syncing for a given connector credential pair
+    Permission sync task that handles external group syncing for a given connector credential pair
     This task assumes that the task has already been properly fenced
     """
 
@@ -203,7 +212,7 @@ def connector_external_group_sync_generator_task(
 
     r = get_redis_client(tenant_id=tenant_id)
 
-    lock = r.lock(
+    lock: RedisLock = r.lock(
         DanswerRedisLocks.CONNECTOR_EXTERNAL_GROUP_SYNC_LOCK_PREFIX
         + f"_{redis_connector.id}",
         timeout=CELERY_EXTERNAL_GROUP_SYNC_LOCK_TIMEOUT,
@@ -228,9 +237,13 @@ def connector_external_group_sync_generator_task(
 
             ext_group_sync_func = GROUP_PERMISSIONS_FUNC_MAP.get(source_type)
             if ext_group_sync_func is None:
-                raise ValueError(f"No external group sync func found for {source_type}")
+                raise ValueError(
+                    f"No external group sync func found for {source_type} for cc_pair: {cc_pair_id}"
+                )
 
-            logger.info(f"Syncing docs for {source_type}")
+            logger.info(
+                f"Syncing external groups for {source_type} for cc_pair: {cc_pair_id}"
+            )
 
             external_user_groups: list[ExternalUserGroup] = ext_group_sync_func(cc_pair)
 
@@ -249,7 +262,6 @@ def connector_external_group_sync_generator_task(
             )
 
             mark_cc_pair_as_external_group_synced(db_session, cc_pair.id)
-
     except Exception as e:
         task_logger.exception(
             f"Failed to run external group sync: cc_pair={cc_pair_id}"
@@ -260,6 +272,6 @@ def connector_external_group_sync_generator_task(
         raise e
     finally:
         # we always want to clear the fence after the task is done or failed so it doesn't get stuck
-        redis_connector.external_group_sync.set_fence(False)
+        redis_connector.external_group_sync.set_fence(None)
         if lock.owned():
             lock.release()

backend/danswer/background/celery/tasks/indexing/tasks.py

@@ -25,11 +25,13 @@ from danswer.configs.constants import DanswerCeleryPriority
 from danswer.configs.constants import DanswerCeleryQueues
 from danswer.configs.constants import DanswerRedisLocks
 from danswer.configs.constants import DocumentSource
+from danswer.db.connector import mark_ccpair_with_indexing_trigger
 from danswer.db.connector_credential_pair import fetch_connector_credential_pairs
 from danswer.db.connector_credential_pair import get_connector_credential_pair_from_id
 from danswer.db.engine import get_db_current_time
 from danswer.db.engine import get_session_with_tenant
 from danswer.db.enums import ConnectorCredentialPairStatus
+from danswer.db.enums import IndexingMode
 from danswer.db.enums import IndexingStatus
 from danswer.db.enums import IndexModelStatus
 from danswer.db.index_attempt import create_index_attempt
@@ -37,12 +39,13 @@ from danswer.db.index_attempt import delete_index_attempt
 from danswer.db.index_attempt import get_all_index_attempts_by_status
 from danswer.db.index_attempt import get_index_attempt
 from danswer.db.index_attempt import get_last_attempt_for_cc_pair
+from danswer.db.index_attempt import mark_attempt_canceled
 from danswer.db.index_attempt import mark_attempt_failed
 from danswer.db.models import ConnectorCredentialPair
 from danswer.db.models import IndexAttempt
 from danswer.db.models import SearchSettings
+from danswer.db.search_settings import get_active_search_settings
 from danswer.db.search_settings import get_current_search_settings
-from danswer.db.search_settings import get_secondary_search_settings
 from danswer.db.swap_index import check_index_swap
 from danswer.indexing.indexing_heartbeat import IndexingHeartbeatInterface
 from danswer.natural_language_processing.search_nlp_models import EmbeddingModel
@@ -159,7 +162,7 @@ def get_unfenced_index_attempt_ids(db_session: Session, r: redis.Redis) -> list[
 )
 def check_for_indexing(self: Task, *, tenant_id: str | None) -> int | None:
     tasks_created = 0
-
+    locked = False
     r = get_redis_client(tenant_id=tenant_id)
 
     lock_beat: RedisLock = r.lock(
@@ -172,6 +175,8 @@ def check_for_indexing(self: Task, *, tenant_id: str | None) -> int | None:
         if not lock_beat.acquire(blocking=False):
             return None
 
+        locked = True
+
         # check for search settings swap
         with get_session_with_tenant(tenant_id=tenant_id) as db_session:
             old_search_settings = check_index_swap(db_session=db_session)
@@ -205,17 +210,10 @@ def check_for_indexing(self: Task, *, tenant_id: str | None) -> int | None:
 
             redis_connector = RedisConnector(tenant_id, cc_pair_id)
             with get_session_with_tenant(tenant_id) as db_session:
-                # Get the primary search settings
-                primary_search_settings = get_current_search_settings(db_session)
-                search_settings = [primary_search_settings]
-
-                # Check for secondary search settings
-                secondary_search_settings = get_secondary_search_settings(db_session)
-                if secondary_search_settings is not None:
-                    # If secondary settings exist, add them to the list
-                    search_settings.append(secondary_search_settings)
-
-                for search_settings_instance in search_settings:
+                search_settings_list: list[SearchSettings] = get_active_search_settings(
+                    db_session
+                )
+                for search_settings_instance in search_settings_list:
                     redis_connector_index = redis_connector.new_index(
                         search_settings_instance.id
                     )
@@ -231,22 +229,46 @@ def check_for_indexing(self: Task, *, tenant_id: str | None) -> int | None:
                     last_attempt = get_last_attempt_for_cc_pair(
                         cc_pair.id, search_settings_instance.id, db_session
                     )
+
+                    search_settings_primary = False
+                    if search_settings_instance.id == search_settings_list[0].id:
+                        search_settings_primary = True
+
                     if not _should_index(
                         cc_pair=cc_pair,
                         last_index=last_attempt,
                         search_settings_instance=search_settings_instance,
-                        secondary_index_building=len(search_settings) > 1,
+                        search_settings_primary=search_settings_primary,
+                        secondary_index_building=len(search_settings_list) > 1,
                         db_session=db_session,
                     ):
                         continue
 
+                    reindex = False
+                    if search_settings_instance.id == search_settings_list[0].id:
+                        # the indexing trigger is only checked and cleared with the primary search settings
+                        if cc_pair.indexing_trigger is not None:
+                            if cc_pair.indexing_trigger == IndexingMode.REINDEX:
+                                reindex = True
+
+                            task_logger.info(
+                                f"Connector indexing manual trigger detected: "
+                                f"cc_pair={cc_pair.id} "
+                                f"search_settings={search_settings_instance.id} "
+                                f"indexing_mode={cc_pair.indexing_trigger}"
+                            )
+
+                            mark_ccpair_with_indexing_trigger(
+                                cc_pair.id, None, db_session
+                            )
+
                     # using a task queue and only allowing one task per cc_pair/search_setting
                     # prevents us from starving out certain attempts
                     attempt_id = try_creating_indexing_task(
                         self.app,
                         cc_pair,
                         search_settings_instance,
-                        False,
+                        reindex,
                         db_session,
                         r,
                         tenant_id,
@@ -256,7 +278,7 @@ def check_for_indexing(self: Task, *, tenant_id: str | None) -> int | None:
                             f"Connector indexing queued: "
                             f"index_attempt={attempt_id} "
                             f"cc_pair={cc_pair.id} "
-                            f"search_settings={search_settings_instance.id} "
+                            f"search_settings={search_settings_instance.id}"
                         )
                         tasks_created += 1
 
@@ -281,7 +303,6 @@ def check_for_indexing(self: Task, *, tenant_id: str | None) -> int | None:
                 mark_attempt_failed(
                     attempt.id, db_session, failure_reason=failure_reason
                 )
-
     except SoftTimeLimitExceeded:
         task_logger.info(
             "Soft time limit exceeded, task is being terminated gracefully."
@@ -289,13 +310,14 @@ def check_for_indexing(self: Task, *, tenant_id: str | None) -> int | None:
     except Exception:
         task_logger.exception(f"Unexpected exception: tenant={tenant_id}")
     finally:
-        if lock_beat.owned():
-            lock_beat.release()
-        else:
-            task_logger.error(
-                "check_for_indexing - Lock not owned on completion: "
-                f"tenant={tenant_id}"
-            )
+        if locked:
+            if lock_beat.owned():
+                lock_beat.release()
+            else:
+                task_logger.error(
+                    "check_for_indexing - Lock not owned on completion: "
+                    f"tenant={tenant_id}"
+                )
 
     return tasks_created
 
@@ -304,6 +326,7 @@ def _should_index(
     cc_pair: ConnectorCredentialPair,
     last_index: IndexAttempt | None,
     search_settings_instance: SearchSettings,
+    search_settings_primary: bool,
     secondary_index_building: bool,
     db_session: Session,
 ) -> bool:
@@ -368,6 +391,11 @@ def _should_index(
     ):
         return False
 
+    if search_settings_primary:
+        if cc_pair.indexing_trigger is not None:
+            # if a manual indexing trigger is on the cc pair, honor it for primary search settings
+            return True
+
     # if no attempt has ever occurred, we should index regardless of refresh_freq
     if not last_index:
         return True
@@ -495,8 +523,11 @@ def try_creating_indexing_task(
     return index_attempt_id
 
 
-@shared_task(name="connector_indexing_proxy_task", acks_late=False, track_started=True)
+@shared_task(
+    name="connector_indexing_proxy_task", bind=True, acks_late=False, track_started=True
+)
 def connector_indexing_proxy_task(
+    self: Task,
     index_attempt_id: int,
     cc_pair_id: int,
     search_settings_id: int,
@@ -509,6 +540,10 @@ def connector_indexing_proxy_task(
         f"cc_pair={cc_pair_id} "
         f"search_settings={search_settings_id}"
     )
+
+    if not self.request.id:
+        task_logger.error("self.request.id is None!")
+
     client = SimpleJobClient()
 
     job = client.submit(
@@ -537,8 +572,30 @@ def connector_indexing_proxy_task(
         f"search_settings={search_settings_id}"
     )
 
+    redis_connector = RedisConnector(tenant_id, cc_pair_id)
+    redis_connector_index = redis_connector.new_index(search_settings_id)
+
     while True:
-        sleep(10)
+        sleep(5)
+
+        if self.request.id and redis_connector_index.terminating(self.request.id):
+            task_logger.warning(
+                "Indexing proxy - termination signal detected: "
+                f"attempt={index_attempt_id} "
+                f"tenant={tenant_id} "
+                f"cc_pair={cc_pair_id} "
+                f"search_settings={search_settings_id}"
+            )
+
+            with get_session_with_tenant(tenant_id) as db_session:
+                mark_attempt_canceled(
+                    index_attempt_id,
+                    db_session,
+                    "Connector termination signal detected",
+                )
+
+            job.cancel()
+            break
 
         # do nothing for ongoing jobs that haven't been stopped
         if not job.done():

backend/danswer/background/celery/tasks/vespa/tasks.py

@@ -46,6 +46,7 @@ from danswer.db.document_set import fetch_document_sets_for_document
 from danswer.db.document_set import get_document_set_by_id
 from danswer.db.document_set import mark_document_set_as_synced
 from danswer.db.engine import get_session_with_tenant
+from danswer.db.enums import IndexingStatus
 from danswer.db.index_attempt import delete_index_attempts
 from danswer.db.index_attempt import get_index_attempt
 from danswer.db.index_attempt import mark_attempt_failed
@@ -58,7 +59,7 @@ from danswer.redis.redis_connector_credential_pair import RedisConnectorCredenti
 from danswer.redis.redis_connector_delete import RedisConnectorDelete
 from danswer.redis.redis_connector_doc_perm_sync import RedisConnectorPermissionSync
 from danswer.redis.redis_connector_doc_perm_sync import (
-    RedisConnectorPermissionSyncData,
+    RedisConnectorPermissionSyncPayload,
 )
 from danswer.redis.redis_connector_index import RedisConnectorIndex
 from danswer.redis.redis_connector_prune import RedisConnectorPrune
@@ -588,7 +589,7 @@ def monitor_ccpair_permissions_taskset(
     if remaining > 0:
         return
 
-    payload: RedisConnectorPermissionSyncData | None = (
+    payload: RedisConnectorPermissionSyncPayload | None = (
         redis_connector.permissions.payload
     )
     start_time: datetime | None = payload.started if payload else None
@@ -596,9 +597,7 @@ def monitor_ccpair_permissions_taskset(
     mark_cc_pair_as_permissions_synced(db_session, int(cc_pair_id), start_time)
     task_logger.info(f"Successfully synced permissions for cc_pair={cc_pair_id}")
 
-    redis_connector.permissions.taskset_clear()
-    redis_connector.permissions.generator_clear()
-    redis_connector.permissions.set_fence(None)
+    redis_connector.permissions.reset()
 
 
 def monitor_ccpair_indexing_taskset(
@@ -678,11 +677,15 @@ def monitor_ccpair_indexing_taskset(
 
                 index_attempt = get_index_attempt(db_session, payload.index_attempt_id)
                 if index_attempt:
-                    mark_attempt_failed(
-                        index_attempt_id=payload.index_attempt_id,
-                        db_session=db_session,
-                        failure_reason=msg,
-                    )
+                    if (
+                        index_attempt.status != IndexingStatus.CANCELED
+                        and index_attempt.status != IndexingStatus.FAILED
+                    ):
+                        mark_attempt_failed(
+                            index_attempt_id=payload.index_attempt_id,
+                            db_session=db_session,
+                            failure_reason=msg,
+                        )
 
                 redis_connector_index.reset()
         return
@@ -692,6 +695,7 @@ def monitor_ccpair_indexing_taskset(
     task_logger.info(
         f"Connector indexing finished: cc_pair={cc_pair_id} "
         f"search_settings={search_settings_id} "
+        f"progress={progress} "
         f"status={status_enum.name} "
         f"elapsed_submitted={elapsed_submitted.total_seconds():.2f}"
     )
@@ -724,7 +728,7 @@ def monitor_vespa_sync(self: Task, tenant_id: str | None) -> bool:
 
         # print current queue lengths
         r_celery = self.app.broker_connection().channel().client  # type: ignore
-        n_celery = celery_get_queue_length("celery", r)
+        n_celery = celery_get_queue_length("celery", r_celery)
         n_indexing = celery_get_queue_length(
             DanswerCeleryQueues.CONNECTOR_INDEXING, r_celery
         )

backend/danswer/background/celery/versioned_apps/beat.py

@@ -1,6 +1,8 @@
 """Factory stub for running celery worker / celery beat."""
+from celery import Celery
+
 from danswer.background.celery.apps.beat import celery_app
 from danswer.utils.variable_functionality import set_is_ee_based_on_env_variable
 
 set_is_ee_based_on_env_variable()
-app = celery_app
+app: Celery = celery_app

backend/danswer/background/celery/versioned_apps/primary.py

@@ -1,8 +1,10 @@
 """Factory stub for running celery worker / celery beat."""
+from celery import Celery
+
 from danswer.utils.variable_functionality import fetch_versioned_implementation
 from danswer.utils.variable_functionality import set_is_ee_based_on_env_variable
 
 set_is_ee_based_on_env_variable()
-app = fetch_versioned_implementation(
+app: Celery = fetch_versioned_implementation(
     "danswer.background.celery.apps.primary", "celery_app"
 )

backend/danswer/background/indexing/run_indexing.py

@@ -19,6 +19,7 @@ from danswer.db.connector_credential_pair import get_last_successful_attempt_tim
 from danswer.db.connector_credential_pair import update_connector_credential_pair
 from danswer.db.engine import get_session_with_tenant
 from danswer.db.enums import ConnectorCredentialPairStatus
+from danswer.db.index_attempt import mark_attempt_canceled
 from danswer.db.index_attempt import mark_attempt_failed
 from danswer.db.index_attempt import mark_attempt_partially_succeeded
 from danswer.db.index_attempt import mark_attempt_succeeded
@@ -87,6 +88,10 @@ def _get_connector_runner(
     )
 
 
+class ConnectorStopSignal(Exception):
+    """A custom exception used to signal a stop in processing."""
+
+
 def _run_indexing(
     db_session: Session,
     index_attempt: IndexAttempt,
@@ -208,9 +213,7 @@ def _run_indexing(
                 # contents still need to be initially pulled.
                 if callback:
                     if callback.should_stop():
-                        raise RuntimeError(
-                            "_run_indexing: Connector stop signal detected"
-                        )
+                        raise ConnectorStopSignal("Connector stop signal detected")
 
                 # TODO: should we move this into the above callback instead?
                 db_session.refresh(db_cc_pair)
@@ -304,26 +307,16 @@ def _run_indexing(
                 )
         except Exception as e:
             logger.exception(
-                f"Connector run ran into exception after elapsed time: {time.time() - start_time} seconds"
+                f"Connector run exceptioned after elapsed time: {time.time() - start_time} seconds"
             )
-            # Only mark the attempt as a complete failure if this is the first indexing window.
-            # Otherwise, some progress was made - the next run will not start from the beginning.
-            # In this case, it is not accurate to mark it as a failure. When the next run begins,
-            # if that fails immediately, it will be marked as a failure.
-            #
-            # NOTE: if the connector is manually disabled, we should mark it as a failure regardless
-            # to give better clarity in the UI, as the next run will never happen.
-            if (
-                ind == 0
-                or not db_cc_pair.status.is_active()
-                or index_attempt.status != IndexingStatus.IN_PROGRESS
-            ):
-                mark_attempt_failed(
+
+            if isinstance(e, ConnectorStopSignal):
+                mark_attempt_canceled(
                     index_attempt.id,
                     db_session,
-                    failure_reason=str(e),
-                    full_exception_trace=traceback.format_exc(),
+                    reason=str(e),
                 )
+
                 if is_primary:
                     update_connector_credential_pair(
                         db_session=db_session,
@@ -335,6 +328,37 @@ def _run_indexing(
                 if INDEXING_TRACER_INTERVAL > 0:
                     tracer.stop()
                 raise e
+            else:
+                # Only mark the attempt as a complete failure if this is the first indexing window.
+                # Otherwise, some progress was made - the next run will not start from the beginning.
+                # In this case, it is not accurate to mark it as a failure. When the next run begins,
+                # if that fails immediately, it will be marked as a failure.
+                #
+                # NOTE: if the connector is manually disabled, we should mark it as a failure regardless
+                # to give better clarity in the UI, as the next run will never happen.
+                if (
+                    ind == 0
+                    or not db_cc_pair.status.is_active()
+                    or index_attempt.status != IndexingStatus.IN_PROGRESS
+                ):
+                    mark_attempt_failed(
+                        index_attempt.id,
+                        db_session,
+                        failure_reason=str(e),
+                        full_exception_trace=traceback.format_exc(),
+                    )
+
+                    if is_primary:
+                        update_connector_credential_pair(
+                            db_session=db_session,
+                            connector_id=db_connector.id,
+                            credential_id=db_credential.id,
+                            net_docs=net_doc_change,
+                        )
+
+                    if INDEXING_TRACER_INTERVAL > 0:
+                        tracer.stop()
+                    raise e
 
             # break => similar to success case. As mentioned above, if the next run fails for the same
             # reason it will then be marked as a failure

backend/danswer/configs/model_configs.py

@@ -70,7 +70,9 @@ GEN_AI_NUM_RESERVED_OUTPUT_TOKENS = int(
 )
 
 # Typically, GenAI models nowadays are at least 4K tokens
-GEN_AI_MODEL_FALLBACK_MAX_TOKENS = 4096
+GEN_AI_MODEL_FALLBACK_MAX_TOKENS = int(
+    os.environ.get("GEN_AI_MODEL_FALLBACK_MAX_TOKENS") or 4096
+)
 
 # Number of tokens from chat history to include at maximum
 # 3000 should be enough context regardless of use, no need to include as much as possible

backend/danswer/connectors/confluence/connector.py

@@ -51,7 +51,7 @@ _RESTRICTIONS_EXPANSION_FIELDS = [
     "restrictions.read.restrictions.group",
 ]
 
-_SLIM_DOC_BATCH_SIZE = 1000
+_SLIM_DOC_BATCH_SIZE = 5000
 
 
 class ConfluenceConnector(LoadConnector, PollConnector, SlimConnector):
@@ -301,5 +301,8 @@ class ConfluenceConnector(LoadConnector, PollConnector, SlimConnector):
                         perm_sync_data=perm_sync_data,
                     )
                 )
-            yield doc_metadata_list
-            doc_metadata_list = []
+            if len(doc_metadata_list) > _SLIM_DOC_BATCH_SIZE:
+                yield doc_metadata_list[:_SLIM_DOC_BATCH_SIZE]
+                doc_metadata_list = doc_metadata_list[_SLIM_DOC_BATCH_SIZE:]
+
+        yield doc_metadata_list

backend/danswer/connectors/confluence/onyx_confluence.py

@@ -120,7 +120,7 @@ def handle_confluence_rate_limit(confluence_call: F) -> F:
     return cast(F, wrapped_call)
 
 
-_DEFAULT_PAGINATION_LIMIT = 100
+_DEFAULT_PAGINATION_LIMIT = 1000
 
 
 class OnyxConfluence(Confluence):

backend/danswer/danswerbot/slack/blocks.py

@@ -18,20 +18,30 @@ from slack_sdk.models.blocks.block_elements import ImageElement
 
 from danswer.chat.models import DanswerQuote
 from danswer.configs.app_configs import DISABLE_GENERATIVE_AI
+from danswer.configs.app_configs import WEB_DOMAIN
 from danswer.configs.constants import DocumentSource
 from danswer.configs.constants import SearchFeedbackType
 from danswer.configs.danswerbot_configs import DANSWER_BOT_NUM_DOCS_TO_DISPLAY
 from danswer.context.search.models import SavedSearchDoc
+from danswer.danswerbot.slack.constants import CONTINUE_IN_WEB_UI_ACTION_ID
 from danswer.danswerbot.slack.constants import DISLIKE_BLOCK_ACTION_ID
 from danswer.danswerbot.slack.constants import FEEDBACK_DOC_BUTTON_BLOCK_ACTION_ID
 from danswer.danswerbot.slack.constants import FOLLOWUP_BUTTON_ACTION_ID
 from danswer.danswerbot.slack.constants import FOLLOWUP_BUTTON_RESOLVED_ACTION_ID
 from danswer.danswerbot.slack.constants import IMMEDIATE_RESOLVED_BUTTON_ACTION_ID
 from danswer.danswerbot.slack.constants import LIKE_BLOCK_ACTION_ID
+from danswer.danswerbot.slack.formatting import format_slack_message
 from danswer.danswerbot.slack.icons import source_to_github_img_link
+from danswer.danswerbot.slack.models import SlackMessageInfo
+from danswer.danswerbot.slack.utils import build_continue_in_web_ui_id
 from danswer.danswerbot.slack.utils import build_feedback_id
 from danswer.danswerbot.slack.utils import remove_slack_text_interactions
 from danswer.danswerbot.slack.utils import translate_vespa_highlight_to_slack
+from danswer.db.chat import get_chat_session_by_message_id
+from danswer.db.engine import get_session_with_tenant
+from danswer.db.models import ChannelConfig
+from danswer.db.models import Persona
+from danswer.one_shot_answer.models import OneShotQAResponse
 from danswer.utils.text_processing import decode_escapes
 from danswer.utils.text_processing import replace_whitespaces_w_space
 
@@ -101,12 +111,12 @@ def _split_text(text: str, limit: int = 3000) -> list[str]:
     return chunks
 
 
-def clean_markdown_link_text(text: str) -> str:
+def _clean_markdown_link_text(text: str) -> str:
     # Remove any newlines within the text
     return text.replace("\n", " ").strip()
 
 
-def build_qa_feedback_block(
+def _build_qa_feedback_block(
     message_id: int, feedback_reminder_id: str | None = None
 ) -> Block:
     return ActionsBlock(
@@ -115,7 +125,6 @@ def build_qa_feedback_block(
             ButtonElement(
                 action_id=LIKE_BLOCK_ACTION_ID,
                 text="👍 Helpful",
-                style="primary",
                 value=feedback_reminder_id,
             ),
             ButtonElement(
@@ -155,7 +164,7 @@ def get_document_feedback_blocks() -> Block:
     )
 
 
-def build_doc_feedback_block(
+def _build_doc_feedback_block(
     message_id: int,
     document_id: str,
     document_rank: int,
@@ -182,7 +191,7 @@ def get_restate_blocks(
     ]
 
 
-def build_documents_blocks(
+def _build_documents_blocks(
     documents: list[SavedSearchDoc],
     message_id: int | None,
     num_docs_to_display: int = DANSWER_BOT_NUM_DOCS_TO_DISPLAY,
@@ -223,7 +232,7 @@ def build_documents_blocks(
 
         feedback: ButtonElement | dict = {}
         if message_id is not None:
-            feedback = build_doc_feedback_block(
+            feedback = _build_doc_feedback_block(
                 message_id=message_id,
                 document_id=d.document_id,
                 document_rank=rank,
@@ -241,7 +250,7 @@ def build_documents_blocks(
     return section_blocks
 
 
-def build_sources_blocks(
+def _build_sources_blocks(
     cited_documents: list[tuple[int, SavedSearchDoc]],
     num_docs_to_display: int = DANSWER_BOT_NUM_DOCS_TO_DISPLAY,
 ) -> list[Block]:
@@ -286,7 +295,7 @@ def build_sources_blocks(
             + ([days_ago_str] if days_ago_str else [])
         )
 
-        document_title = clean_markdown_link_text(doc_sem_id)
+        document_title = _clean_markdown_link_text(doc_sem_id)
         img_link = source_to_github_img_link(d.source_type)
 
         section_blocks.append(
@@ -317,7 +326,50 @@ def build_sources_blocks(
     return section_blocks
 
 
-def build_quotes_block(
+def _priority_ordered_documents_blocks(
+    answer: OneShotQAResponse,
+) -> list[Block]:
+    docs_response = answer.docs if answer.docs else None
+    top_docs = docs_response.top_documents if docs_response else []
+    llm_doc_inds = answer.llm_selected_doc_indices or []
+    llm_docs = [top_docs[i] for i in llm_doc_inds]
+    remaining_docs = [
+        doc for idx, doc in enumerate(top_docs) if idx not in llm_doc_inds
+    ]
+    priority_ordered_docs = llm_docs + remaining_docs
+    if not priority_ordered_docs:
+        return []
+
+    document_blocks = _build_documents_blocks(
+        documents=priority_ordered_docs,
+        message_id=answer.chat_message_id,
+    )
+    if document_blocks:
+        document_blocks = [DividerBlock()] + document_blocks
+    return document_blocks
+
+
+def _build_citations_blocks(
+    answer: OneShotQAResponse,
+) -> list[Block]:
+    docs_response = answer.docs if answer.docs else None
+    top_docs = docs_response.top_documents if docs_response else []
+    citations = answer.citations or []
+    cited_docs = []
+    for citation in citations:
+        matching_doc = next(
+            (d for d in top_docs if d.document_id == citation.document_id),
+            None,
+        )
+        if matching_doc:
+            cited_docs.append((citation.citation_num, matching_doc))
+
+    cited_docs.sort()
+    citations_block = _build_sources_blocks(cited_documents=cited_docs)
+    return citations_block
+
+
+def _build_quotes_block(
     quotes: list[DanswerQuote],
 ) -> list[Block]:
     quote_lines: list[str] = []
@@ -359,58 +411,70 @@ def build_quotes_block(
     return [SectionBlock(text="*Relevant Snippets*\n" + "\n".join(quote_lines))]
 
 
-def build_qa_response_blocks(
-    message_id: int | None,
-    answer: str | None,
-    quotes: list[DanswerQuote] | None,
-    source_filters: list[DocumentSource] | None,
-    time_cutoff: datetime | None,
-    favor_recent: bool,
+def _build_qa_response_blocks(
+    answer: OneShotQAResponse,
     skip_quotes: bool = False,
     process_message_for_citations: bool = False,
-    skip_ai_feedback: bool = False,
-    feedback_reminder_id: str | None = None,
 ) -> list[Block]:
+    retrieval_info = answer.docs
+    if not retrieval_info:
+        # This should not happen, even with no docs retrieved, there is still info returned
+        raise RuntimeError("Failed to retrieve docs, cannot answer question.")
+
+    formatted_answer = format_slack_message(answer.answer) if answer.answer else None
+    quotes = answer.quotes.quotes if answer.quotes else None
+
     if DISABLE_GENERATIVE_AI:
         return []
 
     quotes_blocks: list[Block] = []
 
     filter_block: Block | None = None
-    if time_cutoff or favor_recent or source_filters:
+    if (
+        retrieval_info.applied_time_cutoff
+        or retrieval_info.recency_bias_multiplier > 1
+        or retrieval_info.applied_source_filters
+    ):
         filter_text = "Filters: "
-        if source_filters:
-            sources_str = ", ".join([s.value for s in source_filters])
+        if retrieval_info.applied_source_filters:
+            sources_str = ", ".join(
+                [s.value for s in retrieval_info.applied_source_filters]
+            )
             filter_text += f"`Sources in [{sources_str}]`"
-            if time_cutoff or favor_recent:
+            if (
+                retrieval_info.applied_time_cutoff
+                or retrieval_info.recency_bias_multiplier > 1
+            ):
                 filter_text += " and "
-        if time_cutoff is not None:
-            time_str = time_cutoff.strftime("%b %d, %Y")
+        if retrieval_info.applied_time_cutoff is not None:
+            time_str = retrieval_info.applied_time_cutoff.strftime("%b %d, %Y")
             filter_text += f"`Docs Updated >= {time_str}` "
-        if favor_recent:
-            if time_cutoff is not None:
+        if retrieval_info.recency_bias_multiplier > 1:
+            if retrieval_info.applied_time_cutoff is not None:
                 filter_text += "+ "
             filter_text += "`Prioritize Recently Updated Docs`"
 
         filter_block = SectionBlock(text=f"_{filter_text}_")
 
-    if not answer:
+    if not formatted_answer:
         answer_blocks = [
             SectionBlock(
                 text="Sorry, I was unable to find an answer, but I did find some potentially relevant docs 🤓"
             )
         ]
     else:
-        answer_processed = decode_escapes(remove_slack_text_interactions(answer))
+        answer_processed = decode_escapes(
+            remove_slack_text_interactions(formatted_answer)
+        )
         if process_message_for_citations:
             answer_processed = _process_citations_for_slack(answer_processed)
         answer_blocks = [
             SectionBlock(text=text) for text in _split_text(answer_processed)
         ]
         if quotes:
-            quotes_blocks = build_quotes_block(quotes)
+            quotes_blocks = _build_quotes_block(quotes)
 
-        # if no quotes OR `build_quotes_block()` did not give back any blocks
+        # if no quotes OR `_build_quotes_block()` did not give back any blocks
         if not quotes_blocks:
             quotes_blocks = [
                 SectionBlock(
@@ -425,20 +489,37 @@ def build_qa_response_blocks(
 
     response_blocks.extend(answer_blocks)
 
-    if message_id is not None and not skip_ai_feedback:
-        response_blocks.append(
-            build_qa_feedback_block(
-                message_id=message_id, feedback_reminder_id=feedback_reminder_id
-            )
-        )
-
     if not skip_quotes:
         response_blocks.extend(quotes_blocks)
 
     return response_blocks
 
 
-def build_follow_up_block(message_id: int | None) -> ActionsBlock:
+def _build_continue_in_web_ui_block(
+    tenant_id: str | None,
+    message_id: int | None,
+) -> Block:
+    if message_id is None:
+        raise ValueError("No message id provided to build continue in web ui block")
+    with get_session_with_tenant(tenant_id) as db_session:
+        chat_session = get_chat_session_by_message_id(
+            db_session=db_session,
+            message_id=message_id,
+        )
+        return ActionsBlock(
+            block_id=build_continue_in_web_ui_id(message_id),
+            elements=[
+                ButtonElement(
+                    action_id=CONTINUE_IN_WEB_UI_ACTION_ID,
+                    text="Continue Chat in Danswer!",
+                    style="primary",
+                    url=f"{WEB_DOMAIN}/chat?slackChatId={chat_session.id}",
+                ),
+            ],
+        )
+
+
+def _build_follow_up_block(message_id: int | None) -> ActionsBlock:
     return ActionsBlock(
         block_id=build_feedback_id(message_id) if message_id is not None else None,
         elements=[
@@ -483,3 +564,77 @@ def build_follow_up_resolved_blocks(
         ]
     )
     return [text_block, button_block]
+
+
+def build_slack_response_blocks(
+    tenant_id: str | None,
+    message_info: SlackMessageInfo,
+    answer: OneShotQAResponse,
+    persona: Persona | None,
+    channel_conf: ChannelConfig | None,
+    use_citations: bool,
+    feedback_reminder_id: str | None,
+    skip_ai_feedback: bool = False,
+) -> list[Block]:
+    """
+    This function is a top level function that builds all the blocks for the Slack response.
+    It also handles combining all the blocks together.
+    """
+    # If called with the DanswerBot slash command, the question is lost so we have to reshow it
+    restate_question_block = get_restate_blocks(
+        message_info.thread_messages[-1].message, message_info.is_bot_msg
+    )
+
+    answer_blocks = _build_qa_response_blocks(
+        answer=answer,
+        skip_quotes=persona is not None or use_citations,
+        process_message_for_citations=use_citations,
+    )
+
+    web_follow_up_block = []
+    if channel_conf and channel_conf.get("show_continue_in_web_ui"):
+        web_follow_up_block.append(
+            _build_continue_in_web_ui_block(
+                tenant_id=tenant_id,
+                message_id=answer.chat_message_id,
+            )
+        )
+
+    follow_up_block = []
+    if channel_conf and channel_conf.get("follow_up_tags") is not None:
+        follow_up_block.append(
+            _build_follow_up_block(message_id=answer.chat_message_id)
+        )
+
+    ai_feedback_block = []
+    if answer.chat_message_id is not None and not skip_ai_feedback:
+        ai_feedback_block.append(
+            _build_qa_feedback_block(
+                message_id=answer.chat_message_id,
+                feedback_reminder_id=feedback_reminder_id,
+            )
+        )
+
+    citations_blocks = []
+    document_blocks = []
+    if use_citations:
+        # if citations are enabled, only show cited documents
+        citations_blocks = _build_citations_blocks(answer)
+    else:
+        document_blocks = _priority_ordered_documents_blocks(answer)
+
+    citations_divider = [DividerBlock()] if citations_blocks else []
+    buttons_divider = [DividerBlock()] if web_follow_up_block or follow_up_block else []
+
+    all_blocks = (
+        restate_question_block
+        + answer_blocks
+        + ai_feedback_block
+        + citations_divider
+        + citations_blocks
+        + document_blocks
+        + buttons_divider
+        + web_follow_up_block
+        + follow_up_block
+    )
+    return all_blocks

backend/danswer/danswerbot/slack/constants.py

@@ -2,6 +2,7 @@ from enum import Enum
 
 LIKE_BLOCK_ACTION_ID = "feedback-like"
 DISLIKE_BLOCK_ACTION_ID = "feedback-dislike"
+CONTINUE_IN_WEB_UI_ACTION_ID = "continue-in-web-ui"
 FEEDBACK_DOC_BUTTON_BLOCK_ACTION_ID = "feedback-doc-button"
 IMMEDIATE_RESOLVED_BUTTON_ACTION_ID = "immediate-resolved-button"
 FOLLOWUP_BUTTON_ACTION_ID = "followup-button"

backend/danswer/danswerbot/slack/handlers/handle_buttons.py

@@ -28,7 +28,7 @@ from danswer.danswerbot.slack.models import SlackMessageInfo
 from danswer.danswerbot.slack.utils import build_feedback_id
 from danswer.danswerbot.slack.utils import decompose_action_id
 from danswer.danswerbot.slack.utils import fetch_group_ids_from_names
-from danswer.danswerbot.slack.utils import fetch_user_ids_from_emails
+from danswer.danswerbot.slack.utils import fetch_slack_user_ids_from_emails
 from danswer.danswerbot.slack.utils import get_channel_name_from_id
 from danswer.danswerbot.slack.utils import get_feedback_visibility
 from danswer.danswerbot.slack.utils import read_slack_thread
@@ -267,7 +267,7 @@ def handle_followup_button(
             tag_names = slack_channel_config.channel_config.get("follow_up_tags")
             remaining = None
             if tag_names:
-                tag_ids, remaining = fetch_user_ids_from_emails(
+                tag_ids, remaining = fetch_slack_user_ids_from_emails(
                     tag_names, client.web_client
                 )
             if remaining:

backend/danswer/danswerbot/slack/handlers/handle_message.py

@@ -13,7 +13,7 @@ from danswer.danswerbot.slack.handlers.handle_standard_answers import (
     handle_standard_answers,
 )
 from danswer.danswerbot.slack.models import SlackMessageInfo
-from danswer.danswerbot.slack.utils import fetch_user_ids_from_emails
+from danswer.danswerbot.slack.utils import fetch_slack_user_ids_from_emails
 from danswer.danswerbot.slack.utils import fetch_user_ids_from_groups
 from danswer.danswerbot.slack.utils import respond_in_thread
 from danswer.danswerbot.slack.utils import slack_usage_report
@@ -184,7 +184,7 @@ def handle_message(
     send_to: list[str] | None = None
     missing_users: list[str] | None = None
     if respond_member_group_list:
-        send_to, missing_ids = fetch_user_ids_from_emails(
+        send_to, missing_ids = fetch_slack_user_ids_from_emails(
             respond_member_group_list, client
         )
 

backend/danswer/danswerbot/slack/handlers/handle_regular_answer.py

@@ -7,7 +7,6 @@ from typing import TypeVar
 
 from retry import retry
 from slack_sdk import WebClient
-from slack_sdk.models.blocks import DividerBlock
 from slack_sdk.models.blocks import SectionBlock
 
 from danswer.configs.app_configs import DISABLE_GENERATIVE_AI
@@ -25,12 +24,7 @@ from danswer.context.search.enums import OptionalSearchSetting
 from danswer.context.search.models import BaseFilters
 from danswer.context.search.models import RerankingDetails
 from danswer.context.search.models import RetrievalDetails
-from danswer.danswerbot.slack.blocks import build_documents_blocks
-from danswer.danswerbot.slack.blocks import build_follow_up_block
-from danswer.danswerbot.slack.blocks import build_qa_response_blocks
-from danswer.danswerbot.slack.blocks import build_sources_blocks
-from danswer.danswerbot.slack.blocks import get_restate_blocks
-from danswer.danswerbot.slack.formatting import format_slack_message
+from danswer.danswerbot.slack.blocks import build_slack_response_blocks
 from danswer.danswerbot.slack.handlers.utils import send_team_member_message
 from danswer.danswerbot.slack.models import SlackMessageInfo
 from danswer.danswerbot.slack.utils import respond_in_thread
@@ -411,62 +405,16 @@ def handle_regular_answer(
             )
         return True
 
-    # If called with the DanswerBot slash command, the question is lost so we have to reshow it
-    restate_question_block = get_restate_blocks(messages[-1].message, is_bot_msg)
-    formatted_answer = format_slack_message(answer.answer) if answer.answer else None
-
-    answer_blocks = build_qa_response_blocks(
-        message_id=answer.chat_message_id,
-        answer=formatted_answer,
-        quotes=answer.quotes.quotes if answer.quotes else None,
-        source_filters=retrieval_info.applied_source_filters,
-        time_cutoff=retrieval_info.applied_time_cutoff,
-        favor_recent=retrieval_info.recency_bias_multiplier > 1,
-        # currently Personas don't support quotes
-        # if citations are enabled, also don't use quotes
-        skip_quotes=persona is not None or use_citations,
-        process_message_for_citations=use_citations,
+    all_blocks = build_slack_response_blocks(
+        tenant_id=tenant_id,
+        message_info=message_info,
+        answer=answer,
+        persona=persona,
+        channel_conf=channel_conf,
+        use_citations=use_citations,
         feedback_reminder_id=feedback_reminder_id,
     )
 
-    # Get the chunks fed to the LLM only, then fill with other docs
-    llm_doc_inds = answer.llm_selected_doc_indices or []
-    llm_docs = [top_docs[i] for i in llm_doc_inds]
-    remaining_docs = [
-        doc for idx, doc in enumerate(top_docs) if idx not in llm_doc_inds
-    ]
-    priority_ordered_docs = llm_docs + remaining_docs
-
-    document_blocks = []
-    citations_block = []
-    # if citations are enabled, only show cited documents
-    if use_citations:
-        citations = answer.citations or []
-        cited_docs = []
-        for citation in citations:
-            matching_doc = next(
-                (d for d in top_docs if d.document_id == citation.document_id),
-                None,
-            )
-            if matching_doc:
-                cited_docs.append((citation.citation_num, matching_doc))
-
-        cited_docs.sort()
-        citations_block = build_sources_blocks(cited_documents=cited_docs)
-    elif priority_ordered_docs:
-        document_blocks = build_documents_blocks(
-            documents=priority_ordered_docs,
-            message_id=answer.chat_message_id,
-        )
-        document_blocks = [DividerBlock()] + document_blocks
-
-    all_blocks = (
-        restate_question_block + answer_blocks + citations_block + document_blocks
-    )
-
-    if channel_conf and channel_conf.get("follow_up_tags") is not None:
-        all_blocks.append(build_follow_up_block(message_id=answer.chat_message_id))
-
     try:
         respond_in_thread(
             client=client,

backend/danswer/danswerbot/slack/utils.py

@@ -3,9 +3,9 @@ import random
 import re
 import string
 import time
+import uuid
 from typing import Any
 from typing import cast
-from typing import Optional
 
 from retry import retry
 from slack_sdk import WebClient
@@ -216,6 +216,13 @@ def build_feedback_id(
     return unique_prefix + ID_SEPARATOR + feedback_id
 
 
+def build_continue_in_web_ui_id(
+    message_id: int,
+) -> str:
+    unique_prefix = str(uuid.uuid4())[:10]
+    return unique_prefix + ID_SEPARATOR + str(message_id)
+
+
 def decompose_action_id(feedback_id: str) -> tuple[int, str | None, int | None]:
     """Decompose into query_id, document_id, document_rank, see above function"""
     try:
@@ -313,7 +320,7 @@ def get_channel_name_from_id(
         raise e
 
 
-def fetch_user_ids_from_emails(
+def fetch_slack_user_ids_from_emails(
     user_emails: list[str], client: WebClient
 ) -> tuple[list[str], list[str]]:
     user_ids: list[str] = []
@@ -522,7 +529,7 @@ class SlackRateLimiter:
             self.last_reset_time = time.time()
 
     def notify(
-        self, client: WebClient, channel: str, position: int, thread_ts: Optional[str]
+        self, client: WebClient, channel: str, position: int, thread_ts: str | None
     ) -> None:
         respond_in_thread(
             client=client,

backend/danswer/db/chat.py

@@ -3,6 +3,7 @@ from datetime import datetime
 from datetime import timedelta
 from uuid import UUID
 
+from fastapi import HTTPException
 from sqlalchemy import delete
 from sqlalchemy import desc
 from sqlalchemy import func
@@ -30,6 +31,7 @@ from danswer.db.models import SearchDoc
 from danswer.db.models import SearchDoc as DBSearchDoc
 from danswer.db.models import ToolCall
 from danswer.db.models import User
+from danswer.db.persona import get_best_persona_id_for_user
 from danswer.db.pg_file_store import delete_lobj_by_name
 from danswer.file_store.models import FileDescriptor
 from danswer.llm.override_models import LLMOverride
@@ -250,6 +252,50 @@ def create_chat_session(
     return chat_session
 
 
+def duplicate_chat_session_for_user_from_slack(
+    db_session: Session,
+    user: User | None,
+    chat_session_id: UUID,
+) -> ChatSession:
+    """
+    This takes a chat session id for a session in Slack and:
+    - Creates a new chat session in the DB
+    - Tries to copy the persona from the original chat session
+        (if it is available to the user clicking the button)
+    - Sets the user to the given user (if provided)
+    """
+    chat_session = get_chat_session_by_id(
+        chat_session_id=chat_session_id,
+        user_id=None,  # Ignore user permissions for this
+        db_session=db_session,
+    )
+    if not chat_session:
+        raise HTTPException(status_code=400, detail="Invalid Chat Session ID provided")
+
+    # This enforces permissions and sets a default
+    new_persona_id = get_best_persona_id_for_user(
+        db_session=db_session,
+        user=user,
+        persona_id=chat_session.persona_id,
+    )
+
+    return create_chat_session(
+        db_session=db_session,
+        user_id=user.id if user else None,
+        persona_id=new_persona_id,
+        # Set this to empty string so the frontend will force a rename
+        description="",
+        llm_override=chat_session.llm_override,
+        prompt_override=chat_session.prompt_override,
+        # Chat sessions from Slack should put people in the chat UI, not the search
+        one_shot=False,
+        # Chat is in UI now so this is false
+        danswerbot_flow=False,
+        # Maybe we want this in the future to track if it was created from Slack
+        slack_thread_id=None,
+    )
+
+
 def update_chat_session(
     db_session: Session,
     user_id: UUID | None,
@@ -336,6 +382,28 @@ def get_chat_message(
     return chat_message
 
 
+def get_chat_session_by_message_id(
+    db_session: Session,
+    message_id: int,
+) -> ChatSession:
+    """
+    Should only be used for Slack
+    Get the chat session associated with a specific message ID
+    Note: this ignores permission checks.
+    """
+    stmt = select(ChatMessage).where(ChatMessage.id == message_id)
+
+    result = db_session.execute(stmt)
+    chat_message = result.scalar_one_or_none()
+
+    if chat_message is None:
+        raise ValueError(
+            f"Unable to find chat session associated with message ID: {message_id}"
+        )
+
+    return chat_message.chat_session
+
+
 def get_chat_messages_by_sessions(
     chat_session_ids: list[UUID],
     user_id: UUID | None,
@@ -355,6 +423,44 @@ def get_chat_messages_by_sessions(
     return db_session.execute(stmt).scalars().all()
 
 
+def add_chats_to_session_from_slack_thread(
+    db_session: Session,
+    slack_chat_session_id: UUID,
+    new_chat_session_id: UUID,
+) -> None:
+    new_root_message = get_or_create_root_message(
+        chat_session_id=new_chat_session_id,
+        db_session=db_session,
+    )
+
+    for chat_message in get_chat_messages_by_sessions(
+        chat_session_ids=[slack_chat_session_id],
+        user_id=None,  # Ignore user permissions for this
+        db_session=db_session,
+        skip_permission_check=True,
+    ):
+        if chat_message.message_type == MessageType.SYSTEM:
+            continue
+        # Duplicate the message
+        new_root_message = create_new_chat_message(
+            db_session=db_session,
+            chat_session_id=new_chat_session_id,
+            parent_message=new_root_message,
+            message=chat_message.message,
+            files=chat_message.files,
+            rephrased_query=chat_message.rephrased_query,
+            error=chat_message.error,
+            citations=chat_message.citations,
+            reference_docs=chat_message.search_docs,
+            tool_call=chat_message.tool_call,
+            prompt_id=chat_message.prompt_id,
+            token_count=chat_message.token_count,
+            message_type=chat_message.message_type,
+            alternate_assistant_id=chat_message.alternate_assistant_id,
+            overridden_model=chat_message.overridden_model,
+        )
+
+
 def get_search_docs_for_chat_message(
     chat_message_id: int, db_session: Session
 ) -> list[SearchDoc]:

backend/danswer/db/connector.py

@@ -12,6 +12,7 @@ from sqlalchemy.orm import Session
 from danswer.configs.app_configs import DEFAULT_PRUNING_FREQ
 from danswer.configs.constants import DocumentSource
 from danswer.connectors.models import InputType
+from danswer.db.enums import IndexingMode
 from danswer.db.models import Connector
 from danswer.db.models import ConnectorCredentialPair
 from danswer.db.models import IndexAttempt
@@ -311,3 +312,25 @@ def mark_cc_pair_as_external_group_synced(db_session: Session, cc_pair_id: int)
     # If this changes, we need to update this function.
     cc_pair.last_time_external_group_sync = datetime.now(timezone.utc)
     db_session.commit()
+
+
+def mark_ccpair_with_indexing_trigger(
+    cc_pair_id: int, indexing_mode: IndexingMode | None, db_session: Session
+) -> None:
+    """indexing_mode sets a field which will be picked up by a background task
+    to trigger indexing. Set to None to disable the trigger."""
+    try:
+        cc_pair = db_session.execute(
+            select(ConnectorCredentialPair)
+            .where(ConnectorCredentialPair.id == cc_pair_id)
+            .with_for_update()
+        ).scalar_one()
+
+        if cc_pair is None:
+            raise ValueError(f"No cc_pair with ID: {cc_pair_id}")
+
+        cc_pair.indexing_trigger = indexing_mode
+        db_session.commit()
+    except Exception:
+        db_session.rollback()
+        raise

backend/danswer/db/connector_credential_pair.py

@@ -324,8 +324,11 @@ def associate_default_cc_pair(db_session: Session) -> None:
 def _relate_groups_to_cc_pair__no_commit(
     db_session: Session,
     cc_pair_id: int,
-    user_group_ids: list[int],
+    user_group_ids: list[int] | None = None,
 ) -> None:
+    if not user_group_ids:
+        return
+
     for group_id in user_group_ids:
         db_session.add(
             UserGroup__ConnectorCredentialPair(
@@ -402,12 +405,11 @@ def add_credential_to_connector(
     db_session.flush()  # make sure the association has an id
     db_session.refresh(association)
 
-    if groups and access_type != AccessType.SYNC:
-        _relate_groups_to_cc_pair__no_commit(
-            db_session=db_session,
-            cc_pair_id=association.id,
-            user_group_ids=groups,
-        )
+    _relate_groups_to_cc_pair__no_commit(
+        db_session=db_session,
+        cc_pair_id=association.id,
+        user_group_ids=groups,
+    )
 
     db_session.commit()
 

backend/danswer/db/enums.py

@@ -19,6 +19,11 @@ class IndexingStatus(str, PyEnum):
         return self in terminal_states
 
 
+class IndexingMode(str, PyEnum):
+    UPDATE = "update"
+    REINDEX = "reindex"
+
+
 # these may differ in the future, which is why we're okay with this duplication
 class DeletionStatus(str, PyEnum):
     NOT_STARTED = "not_started"

backend/danswer/db/models.py

@@ -42,7 +42,7 @@ from danswer.configs.constants import DEFAULT_BOOST
 from danswer.configs.constants import DocumentSource
 from danswer.configs.constants import FileOrigin
 from danswer.configs.constants import MessageType
-from danswer.db.enums import AccessType
+from danswer.db.enums import AccessType, IndexingMode
 from danswer.configs.constants import NotificationType
 from danswer.configs.constants import SearchFeedbackType
 from danswer.configs.constants import TokenRateLimitScope
@@ -438,6 +438,10 @@ class ConnectorCredentialPair(Base):
 
     total_docs_indexed: Mapped[int] = mapped_column(Integer, default=0)
 
+    indexing_trigger: Mapped[IndexingMode | None] = mapped_column(
+        Enum(IndexingMode, native_enum=False), nullable=True
+    )
+
     connector: Mapped["Connector"] = relationship(
         "Connector", back_populates="credentials"
     )
@@ -1480,6 +1484,7 @@ class ChannelConfig(TypedDict):
     # If None then no follow up
     # If empty list, follow up with no tags
     follow_up_tags: NotRequired[list[str]]
+    show_continue_in_web_ui: NotRequired[bool]  # defaults to False
 
 
 class SlackBotResponseType(str, PyEnum):

backend/danswer/db/persona.py

@@ -113,6 +113,31 @@ def fetch_persona_by_id(
     return persona
 
 
+def get_best_persona_id_for_user(
+    db_session: Session, user: User | None, persona_id: int | None = None
+) -> int | None:
+    if persona_id is not None:
+        stmt = select(Persona).where(Persona.id == persona_id).distinct()
+        stmt = _add_user_filters(
+            stmt=stmt,
+            user=user,
+            # We don't want to filter by editable here, we just want to see if the
+            # persona is usable by the user
+            get_editable=False,
+        )
+        persona = db_session.scalars(stmt).one_or_none()
+        if persona:
+            return persona.id
+
+    # If the persona is not found, or the slack bot is using doc sets instead of personas,
+    # we need to find the best persona for the user
+    # This is the persona with the highest display priority that the user has access to
+    stmt = select(Persona).order_by(Persona.display_priority.desc()).distinct()
+    stmt = _add_user_filters(stmt=stmt, user=user, get_editable=True)
+    persona = db_session.scalars(stmt).one_or_none()
+    return persona.id if persona else None
+
+
 def _get_persona_by_name(
     persona_name: str, user: User | None, db_session: Session
 ) -> Persona | None:
@@ -160,7 +185,7 @@ def create_update_persona(
             "persona_id": persona_id,
             "user": user,
             "db_session": db_session,
-            **create_persona_request.dict(exclude={"users", "groups"}),
+            **create_persona_request.model_dump(exclude={"users", "groups"}),
         }
 
         persona = upsert_persona(**persona_data)
@@ -733,6 +758,8 @@ def get_prompt_by_name(
     if user and user.role != UserRole.ADMIN:
         stmt = stmt.where(Prompt.user_id == user.id)
 
+    # Order by ID to ensure consistent result when multiple prompts exist
+    stmt = stmt.order_by(Prompt.id).limit(1)
     result = db_session.execute(stmt).scalar_one_or_none()
     return result
 

backend/danswer/db/search_settings.py

@@ -143,6 +143,25 @@ def get_secondary_search_settings(db_session: Session) -> SearchSettings | None:
     return latest_settings
 
 
+def get_active_search_settings(db_session: Session) -> list[SearchSettings]:
+    """Returns active search settings. The first entry will always be the current search
+    settings. If there are new search settings that are being migrated to, those will be
+    the second entry."""
+    search_settings_list: list[SearchSettings] = []
+
+    # Get the primary search settings
+    primary_search_settings = get_current_search_settings(db_session)
+    search_settings_list.append(primary_search_settings)
+
+    # Check for secondary search settings
+    secondary_search_settings = get_secondary_search_settings(db_session)
+    if secondary_search_settings is not None:
+        # If secondary settings exist, add them to the list
+        search_settings_list.append(secondary_search_settings)
+
+    return search_settings_list
+
+
 def get_all_search_settings(db_session: Session) -> list[SearchSettings]:
     query = select(SearchSettings).order_by(SearchSettings.id.desc())
     result = db_session.execute(query)

backend/danswer/file_processing/extract_file_text.py

@@ -295,7 +295,7 @@ def pptx_to_text(file: IO[Any]) -> str:
 
 
 def xlsx_to_text(file: IO[Any]) -> str:
-    workbook = openpyxl.load_workbook(file)
+    workbook = openpyxl.load_workbook(file, read_only=True)
     text_content = []
     for sheet in workbook.worksheets:
         sheet_string = "\n".join(

backend/danswer/llm/answering/stream_processing/citation_processing.py

@@ -67,9 +67,9 @@ class CitationProcessor:
                 if piece_that_comes_after == "\n" and in_code_block(self.llm_out):
                     self.curr_segment = self.curr_segment.replace("```", "```plaintext")
 
-        citation_pattern = r"\[(\d+)\]"
+        citation_pattern = r"\[(\d+)\]|\[\[(\d+)\]\]"  # [1], [[1]], etc.
         citations_found = list(re.finditer(citation_pattern, self.curr_segment))
-        possible_citation_pattern = r"(\[\d*$)"  # [1, [, etc
+        possible_citation_pattern = r"(\[+\d*$)"  # [1, [, [[, [[2, etc.
         possible_citation_found = re.search(
             possible_citation_pattern, self.curr_segment
         )
@@ -77,13 +77,15 @@ class CitationProcessor:
         if len(citations_found) == 0 and len(self.llm_out) - self.past_cite_count > 5:
             self.current_citations = []
 
-        result = ""  # Initialize result here
+        result = ""
         if citations_found and not in_code_block(self.llm_out):
             last_citation_end = 0
             length_to_add = 0
             while len(citations_found) > 0:
                 citation = citations_found.pop(0)
-                numerical_value = int(citation.group(1))
+                numerical_value = int(
+                    next(group for group in citation.groups() if group is not None)
+                )
 
                 if 1 <= numerical_value <= self.max_citation_num:
                     context_llm_doc = self.context_docs[numerical_value - 1]
@@ -131,14 +133,6 @@ class CitationProcessor:
 
                     link = context_llm_doc.link
 
-                    # Replace the citation in the current segment
-                    start, end = citation.span()
-                    self.curr_segment = (
-                        self.curr_segment[: start + length_to_add]
-                        + f"[{target_citation_num}]"
-                        + self.curr_segment[end + length_to_add :]
-                    )
-
                     self.past_cite_count = len(self.llm_out)
                     self.current_citations.append(target_citation_num)
 
@@ -149,6 +143,7 @@ class CitationProcessor:
                             document_id=context_llm_doc.document_id,
                         )
 
+                    start, end = citation.span()
                     if link:
                         prev_length = len(self.curr_segment)
                         self.curr_segment = (

backend/danswer/llm/chat_llm.py

@@ -26,7 +26,9 @@ from langchain_core.messages.tool import ToolMessage
 from langchain_core.prompt_values import PromptValue
 
 from danswer.configs.app_configs import LOG_DANSWER_MODEL_INTERACTIONS
-from danswer.configs.model_configs import DISABLE_LITELLM_STREAMING
+from danswer.configs.model_configs import (
+    DISABLE_LITELLM_STREAMING,
+)
 from danswer.configs.model_configs import GEN_AI_TEMPERATURE
 from danswer.configs.model_configs import LITELLM_EXTRA_BODY
 from danswer.llm.interfaces import LLM
@@ -161,7 +163,9 @@ def _convert_delta_to_message_chunk(
 
     if role == "user":
         return HumanMessageChunk(content=content)
-    elif role == "assistant":
+    # NOTE: if tool calls are present, then it's an assistant.
+    # In Ollama, the role will be None for tool-calls
+    elif role == "assistant" or tool_calls:
         if tool_calls:
             tool_call = tool_calls[0]
             tool_name = tool_call.function.name or (curr_msg and curr_msg.name) or ""
@@ -236,6 +240,7 @@ class DefaultMultiLLM(LLM):
         custom_config: dict[str, str] | None = None,
         extra_headers: dict[str, str] | None = None,
         extra_body: dict | None = LITELLM_EXTRA_BODY,
+        model_kwargs: dict[str, Any] | None = None,
         long_term_logger: LongTermLogger | None = None,
     ):
         self._timeout = timeout
@@ -268,7 +273,7 @@ class DefaultMultiLLM(LLM):
             for k, v in custom_config.items():
                 os.environ[k] = v
 
-        model_kwargs: dict[str, Any] = {}
+        model_kwargs = model_kwargs or {}
         if extra_headers:
             model_kwargs.update({"extra_headers": extra_headers})
         if extra_body:

backend/danswer/llm/factory.py

@@ -1,5 +1,8 @@
+from typing import Any
+
 from danswer.configs.app_configs import DISABLE_GENERATIVE_AI
 from danswer.configs.chat_configs import QA_TIMEOUT
+from danswer.configs.model_configs import GEN_AI_MODEL_FALLBACK_MAX_TOKENS
 from danswer.configs.model_configs import GEN_AI_TEMPERATURE
 from danswer.db.engine import get_session_context_manager
 from danswer.db.llm import fetch_default_provider
@@ -13,6 +16,15 @@ from danswer.utils.headers import build_llm_extra_headers
 from danswer.utils.long_term_log import LongTermLogger
 
 
+def _build_extra_model_kwargs(provider: str) -> dict[str, Any]:
+    """Ollama requires us to specify the max context window.
+
+    For now, just using the GEN_AI_MODEL_FALLBACK_MAX_TOKENS value.
+    TODO: allow model-specific values to be configured via the UI.
+    """
+    return {"num_ctx": GEN_AI_MODEL_FALLBACK_MAX_TOKENS} if provider == "ollama" else {}
+
+
 def get_main_llm_from_tuple(
     llms: tuple[LLM, LLM],
 ) -> LLM:
@@ -132,5 +144,6 @@ def get_llm(
         temperature=temperature,
         custom_config=custom_config,
         extra_headers=build_llm_extra_headers(additional_headers),
+        model_kwargs=_build_extra_model_kwargs(provider),
         long_term_logger=long_term_logger,
     )

backend/danswer/llm/utils.py

@@ -1,3 +1,4 @@
+import copy
 import io
 import json
 from collections.abc import Callable
@@ -385,6 +386,62 @@ def test_llm(llm: LLM) -> str | None:
     return error_msg
 
 
+def get_model_map() -> dict:
+    starting_map = copy.deepcopy(cast(dict, litellm.model_cost))
+
+    # NOTE: we could add additional models here in the future,
+    # but for now there is no point. Ollama allows the user to
+    # to specify their desired max context window, and it's
+    # unlikely to be standard across users even for the same model
+    # (it heavily depends on their hardware). For now, we'll just
+    # rely on GEN_AI_MODEL_FALLBACK_MAX_TOKENS to cover this.
+    # for model_name in [
+    #     "llama3.2",
+    #     "llama3.2:1b",
+    #     "llama3.2:3b",
+    #     "llama3.2:11b",
+    #     "llama3.2:90b",
+    # ]:
+    #     starting_map[f"ollama/{model_name}"] = {
+    #         "max_tokens": 128000,
+    #         "max_input_tokens": 128000,
+    #         "max_output_tokens": 128000,
+    #     }
+
+    return starting_map
+
+
+def _strip_extra_provider_from_model_name(model_name: str) -> str:
+    return model_name.split("/")[1] if "/" in model_name else model_name
+
+
+def _strip_colon_from_model_name(model_name: str) -> str:
+    return ":".join(model_name.split(":")[:-1]) if ":" in model_name else model_name
+
+
+def _find_model_obj(
+    model_map: dict, provider: str, model_names: list[str | None]
+) -> dict | None:
+    # Filter out None values and deduplicate model names
+    filtered_model_names = [name for name in model_names if name]
+
+    # First try all model names with provider prefix
+    for model_name in filtered_model_names:
+        model_obj = model_map.get(f"{provider}/{model_name}")
+        if model_obj:
+            logger.debug(f"Using model object for {provider}/{model_name}")
+            return model_obj
+
+    # Then try all model names without provider prefix
+    for model_name in filtered_model_names:
+        model_obj = model_map.get(model_name)
+        if model_obj:
+            logger.debug(f"Using model object for {model_name}")
+            return model_obj
+
+    return None
+
+
 def get_llm_max_tokens(
     model_map: dict,
     model_name: str,
@@ -397,22 +454,22 @@ def get_llm_max_tokens(
         return GEN_AI_MAX_TOKENS
 
     try:
-        model_obj = model_map.get(f"{model_provider}/{model_name}")
-        if model_obj:
-            logger.debug(f"Using model object for {model_provider}/{model_name}")
-
-        if not model_obj:
-            model_obj = model_map.get(model_name)
-            if model_obj:
-                logger.debug(f"Using model object for {model_name}")
-
-        if not model_obj:
-            model_name_split = model_name.split("/")
-            if len(model_name_split) > 1:
-                model_obj = model_map.get(model_name_split[1])
-            if model_obj:
-                logger.debug(f"Using model object for {model_name_split[1]}")
-
+        extra_provider_stripped_model_name = _strip_extra_provider_from_model_name(
+            model_name
+        )
+        model_obj = _find_model_obj(
+            model_map,
+            model_provider,
+            [
+                model_name,
+                # Remove leading extra provider. Usually for cases where user has a
+                # customer model proxy which appends another prefix
+                extra_provider_stripped_model_name,
+                # remove :XXXX from the end, if present. Needed for ollama.
+                _strip_colon_from_model_name(model_name),
+                _strip_colon_from_model_name(extra_provider_stripped_model_name),
+            ],
+        )
         if not model_obj:
             raise RuntimeError(
                 f"No litellm entry found for {model_provider}/{model_name}"
@@ -488,7 +545,7 @@ def get_max_input_tokens(
     # `model_cost` dict is a named public interface:
     # https://litellm.vercel.app/docs/completion/token_usage#7-model_cost
     # model_map is  litellm.model_cost
-    litellm_model_map = litellm.model_cost
+    litellm_model_map = get_model_map()
 
     input_toks = (
         get_llm_max_tokens(

backend/danswer/main.py

@@ -26,6 +26,7 @@ from danswer.auth.schemas import UserRead
 from danswer.auth.schemas import UserUpdate
 from danswer.auth.users import auth_backend
 from danswer.auth.users import BasicAuthenticationError
+from danswer.auth.users import create_danswer_oauth_router
 from danswer.auth.users import fastapi_users
 from danswer.configs.app_configs import APP_API_PREFIX
 from danswer.configs.app_configs import APP_HOST
@@ -44,6 +45,7 @@ from danswer.configs.constants import AuthType
 from danswer.configs.constants import POSTGRES_WEB_APP_NAME
 from danswer.db.engine import SqlEngine
 from danswer.db.engine import warm_up_connections
+from danswer.server.api_key.api import router as api_key_router
 from danswer.server.auth_check import check_router_auth
 from danswer.server.danswer_api.ingestion import router as danswer_api_router
 from danswer.server.documents.cc_pair import router as cc_pair_router
@@ -280,6 +282,7 @@ def get_application() -> FastAPI:
         application, get_full_openai_assistants_api_router()
     )
     include_router_with_global_prefix_prepended(application, long_term_logs_router)
+    include_router_with_global_prefix_prepended(application, api_key_router)
 
     if AUTH_TYPE == AuthType.DISABLED:
         # Server logs this during auth setup verification step
@@ -323,7 +326,7 @@ def get_application() -> FastAPI:
         oauth_client = GoogleOAuth2(OAUTH_CLIENT_ID, OAUTH_CLIENT_SECRET)
         include_router_with_global_prefix_prepended(
             application,
-            fastapi_users.get_oauth_router(
+            create_danswer_oauth_router(
                 oauth_client,
                 auth_backend,
                 USER_AUTH_SECRET,

backend/danswer/natural_language_processing/exceptions.py

@@ -0,0 +1,4 @@
+class ModelServerRateLimitError(Exception):
+    """
+    Exception raised for rate limiting errors from the model server.
+    """

backend/danswer/natural_language_processing/search_nlp_models.py

@@ -6,6 +6,9 @@ from typing import Any
 
 import requests
 from httpx import HTTPError
+from requests import JSONDecodeError
+from requests import RequestException
+from requests import Response
 from retry import retry
 
 from danswer.configs.app_configs import LARGE_CHUNK_RATIO
@@ -16,6 +19,9 @@ from danswer.configs.model_configs import (
 from danswer.configs.model_configs import DOC_EMBEDDING_CONTEXT_SIZE
 from danswer.db.models import SearchSettings
 from danswer.indexing.indexing_heartbeat import IndexingHeartbeatInterface
+from danswer.natural_language_processing.exceptions import (
+    ModelServerRateLimitError,
+)
 from danswer.natural_language_processing.utils import get_tokenizer
 from danswer.natural_language_processing.utils import tokenizer_trim_content
 from danswer.utils.logger import setup_logger
@@ -99,28 +105,43 @@ class EmbeddingModel:
         self.embed_server_endpoint = f"{model_server_url}/encoder/bi-encoder-embed"
 
     def _make_model_server_request(self, embed_request: EmbedRequest) -> EmbedResponse:
-        def _make_request() -> EmbedResponse:
+        def _make_request() -> Response:
             response = requests.post(
                 self.embed_server_endpoint, json=embed_request.model_dump()
             )
-            try:
-                response.raise_for_status()
-            except requests.HTTPError as e:
-                try:
-                    error_detail = response.json().get("detail", str(e))
-                except Exception:
-                    error_detail = response.text
-                raise HTTPError(f"HTTP error occurred: {error_detail}") from e
-            except requests.RequestException as e:
-                raise HTTPError(f"Request failed: {str(e)}") from e
+            # signify that this is a rate limit error
+            if response.status_code == 429:
+                raise ModelServerRateLimitError(response.text)
 
-            return EmbedResponse(**response.json())
+            response.raise_for_status()
+            return response
 
-        # only perform retries for the non-realtime embedding of passages (e.g. for indexing)
+        final_make_request_func = _make_request
+
+        # if the text type is a passage, add some default
+        # retries + handling for rate limiting
         if embed_request.text_type == EmbedTextType.PASSAGE:
-            return retry(tries=3, delay=5)(_make_request)()
-        else:
-            return _make_request()
+            final_make_request_func = retry(
+                tries=3,
+                delay=5,
+                exceptions=(RequestException, ValueError, JSONDecodeError),
+            )(final_make_request_func)
+            # use 10 second delay as per Azure suggestion
+            final_make_request_func = retry(
+                tries=10, delay=10, exceptions=ModelServerRateLimitError
+            )(final_make_request_func)
+
+        try:
+            response = final_make_request_func()
+            return EmbedResponse(**response.json())
+        except requests.HTTPError as e:
+            try:
+                error_detail = response.json().get("detail", str(e))
+            except Exception:
+                error_detail = response.text
+            raise HTTPError(f"HTTP error occurred: {error_detail}") from e
+        except requests.RequestException as e:
+            raise HTTPError(f"Request failed: {str(e)}") from e
 
     def _batch_encode_texts(
         self,

backend/danswer/natural_language_processing/utils.py

@@ -131,7 +131,7 @@ def _try_initialize_tokenizer(
             return tokenizer
         except Exception as hf_error:
             logger.warning(
-                f"Error initializing HuggingFaceTokenizer for {model_name}: {hf_error}"
+                f"Failed to initialize HuggingFaceTokenizer for {model_name}: {hf_error}"
             )
 
     # If both initializations fail, return None

backend/danswer/one_shot_answer/answer_question.py

@@ -47,6 +47,7 @@ from danswer.one_shot_answer.models import DirectQARequest
 from danswer.one_shot_answer.models import OneShotQAResponse
 from danswer.one_shot_answer.models import QueryRephrase
 from danswer.one_shot_answer.qa_utils import combine_message_thread
+from danswer.one_shot_answer.qa_utils import slackify_message_thread
 from danswer.secondary_llm_flows.answer_validation import get_answer_validity
 from danswer.secondary_llm_flows.query_expansion import thread_based_query_rephrase
 from danswer.server.query_and_chat.models import ChatMessageDetail
@@ -194,13 +195,22 @@ def stream_answer_objects(
             )
         prompt = persona.prompts[0]
 
+    user_message_str = query_msg.message
+    # For this endpoint, we only save one user message to the chat session
+    # However, for slackbot, we want to include the history of the entire thread
+    if danswerbot_flow:
+        # Right now, we only support bringing over citations and search docs
+        # from the last message in the thread, not the entire thread
+        # in the future, we may want to retrieve the entire thread
+        user_message_str = slackify_message_thread(query_req.messages)
+
     # Create the first User query message
     new_user_message = create_new_chat_message(
         chat_session_id=chat_session.id,
         parent_message=root_message,
         prompt_id=query_req.prompt_id,
-        message=query_msg.message,
-        token_count=len(llm_tokenizer.encode(query_msg.message)),
+        message=user_message_str,
+        token_count=len(llm_tokenizer.encode(user_message_str)),
         message_type=MessageType.USER,
         db_session=db_session,
         commit=True,

backend/danswer/one_shot_answer/qa_utils.py

@@ -51,3 +51,31 @@ def combine_message_thread(
         total_token_count += message_token_count
 
     return "\n\n".join(message_strs)
+
+
+def slackify_message(message: ThreadMessage) -> str:
+    if message.role != MessageType.USER:
+        return message.message
+
+    return f"{message.sender or 'Unknown User'} said in Slack:\n{message.message}"
+
+
+def slackify_message_thread(messages: list[ThreadMessage]) -> str:
+    if not messages:
+        return ""
+
+    message_strs: list[str] = []
+    for message in messages:
+        if message.role == MessageType.USER:
+            message_text = (
+                f"{message.sender or 'Unknown User'} said in Slack:\n{message.message}"
+            )
+        elif message.role == MessageType.ASSISTANT:
+            message_text = f"DanswerBot said in Slack:\n{message.message}"
+        else:
+            message_text = (
+                f"{message.role.value.upper()} said in Slack:\n{message.message}"
+            )
+        message_strs.append(message_text)
+
+    return "\n\n".join(message_strs)

backend/danswer/redis/redis_connector.py

@@ -1,5 +1,8 @@
+import time
+
 import redis
 
+from danswer.db.models import SearchSettings
 from danswer.redis.redis_connector_delete import RedisConnectorDelete
 from danswer.redis.redis_connector_doc_perm_sync import RedisConnectorPermissionSync
 from danswer.redis.redis_connector_ext_group_sync import RedisConnectorExternalGroupSync
@@ -31,6 +34,44 @@ class RedisConnector:
             self.tenant_id, self.id, search_settings_id, self.redis
         )
 
+    def wait_for_indexing_termination(
+        self,
+        search_settings_list: list[SearchSettings],
+        timeout: float = 15.0,
+    ) -> bool:
+        """
+        Returns True if all indexing for the given redis connector is finished within the given timeout.
+        Returns False if the timeout is exceeded
+
+        This check does not guarantee that current indexings being terminated
+        won't get restarted midflight
+        """
+
+        finished = False
+
+        start = time.monotonic()
+
+        while True:
+            still_indexing = False
+            for search_settings in search_settings_list:
+                redis_connector_index = self.new_index(search_settings.id)
+                if redis_connector_index.fenced:
+                    still_indexing = True
+                    break
+
+            if not still_indexing:
+                finished = True
+                break
+
+            now = time.monotonic()
+            if now - start > timeout:
+                break
+
+            time.sleep(1)
+            continue
+
+        return finished
+
     @staticmethod
     def get_id_from_fence_key(key: str) -> str | None:
         """

backend/danswer/redis/redis_connector_doc_perm_sync.py

@@ -14,8 +14,9 @@ from danswer.configs.constants import DanswerCeleryPriority
 from danswer.configs.constants import DanswerCeleryQueues
 
 
-class RedisConnectorPermissionSyncData(BaseModel):
+class RedisConnectorPermissionSyncPayload(BaseModel):
     started: datetime | None
+    celery_task_id: str | None
 
 
 class RedisConnectorPermissionSync:
@@ -78,14 +79,14 @@ class RedisConnectorPermissionSync:
         return False
 
     @property
-    def payload(self) -> RedisConnectorPermissionSyncData | None:
+    def payload(self) -> RedisConnectorPermissionSyncPayload | None:
         # read related data and evaluate/print task progress
         fence_bytes = cast(bytes, self.redis.get(self.fence_key))
         if fence_bytes is None:
             return None
 
         fence_str = fence_bytes.decode("utf-8")
-        payload = RedisConnectorPermissionSyncData.model_validate_json(
+        payload = RedisConnectorPermissionSyncPayload.model_validate_json(
             cast(str, fence_str)
         )
 
@@ -93,7 +94,7 @@ class RedisConnectorPermissionSync:
 
     def set_fence(
         self,
-        payload: RedisConnectorPermissionSyncData | None,
+        payload: RedisConnectorPermissionSyncPayload | None,
     ) -> None:
         if not payload:
             self.redis.delete(self.fence_key)
@@ -162,6 +163,12 @@ class RedisConnectorPermissionSync:
 
         return len(async_results)
 
+    def reset(self) -> None:
+        self.redis.delete(self.generator_progress_key)
+        self.redis.delete(self.generator_complete_key)
+        self.redis.delete(self.taskset_key)
+        self.redis.delete(self.fence_key)
+
     @staticmethod
     def remove_from_taskset(id: int, task_id: str, r: redis.Redis) -> None:
         taskset_key = f"{RedisConnectorPermissionSync.TASKSET_PREFIX}_{id}"

backend/danswer/redis/redis_connector_ext_group_sync.py

@@ -1,11 +1,18 @@
+from datetime import datetime
 from typing import cast
 
 import redis
 from celery import Celery
+from pydantic import BaseModel
 from redis.lock import Lock as RedisLock
 from sqlalchemy.orm import Session
 
 
+class RedisConnectorExternalGroupSyncPayload(BaseModel):
+    started: datetime | None
+    celery_task_id: str | None
+
+
 class RedisConnectorExternalGroupSync:
     """Manages interactions with redis for external group syncing tasks. Should only be accessed
     through RedisConnector."""
@@ -68,12 +75,29 @@ class RedisConnectorExternalGroupSync:
 
         return False
 
-    def set_fence(self, value: bool) -> None:
-        if not value:
+    @property
+    def payload(self) -> RedisConnectorExternalGroupSyncPayload | None:
+        # read related data and evaluate/print task progress
+        fence_bytes = cast(bytes, self.redis.get(self.fence_key))
+        if fence_bytes is None:
+            return None
+
+        fence_str = fence_bytes.decode("utf-8")
+        payload = RedisConnectorExternalGroupSyncPayload.model_validate_json(
+            cast(str, fence_str)
+        )
+
+        return payload
+
+    def set_fence(
+        self,
+        payload: RedisConnectorExternalGroupSyncPayload | None,
+    ) -> None:
+        if not payload:
             self.redis.delete(self.fence_key)
             return
 
-        self.redis.set(self.fence_key, 0)
+        self.redis.set(self.fence_key, payload.model_dump_json())
 
     @property
     def generator_complete(self) -> int | None:

backend/danswer/redis/redis_connector_index.py

@@ -29,6 +29,8 @@ class RedisConnectorIndex:
 
     GENERATOR_LOCK_PREFIX = "da_lock:indexing"
 
+    TERMINATE_PREFIX = PREFIX + "_terminate"  # connectorindexing_terminate
+
     def __init__(
         self,
         tenant_id: str | None,
@@ -51,6 +53,7 @@ class RedisConnectorIndex:
         self.generator_lock_key = (
             f"{self.GENERATOR_LOCK_PREFIX}_{id}/{search_settings_id}"
         )
+        self.terminate_key = f"{self.TERMINATE_PREFIX}_{id}/{search_settings_id}"
 
     @classmethod
     def fence_key_with_ids(cls, cc_pair_id: int, search_settings_id: int) -> str:
@@ -92,6 +95,18 @@ class RedisConnectorIndex:
 
         self.redis.set(self.fence_key, payload.model_dump_json())
 
+    def terminating(self, celery_task_id: str) -> bool:
+        if self.redis.exists(f"{self.terminate_key}_{celery_task_id}"):
+            return True
+
+        return False
+
+    def set_terminate(self, celery_task_id: str) -> None:
+        """This sets a signal. It does not block!"""
+        # We shouldn't need very long to terminate the spawned task.
+        # 10 minute TTL is good.
+        self.redis.set(f"{self.terminate_key}_{celery_task_id}", 0, ex=600)
+
     def set_generator_complete(self, payload: int | None) -> None:
         if not payload:
             self.redis.delete(self.generator_complete_key)

backend/danswer/seeding/load_docs.py

@@ -33,6 +33,7 @@ from danswer.server.documents.models import ConnectorBase
 from danswer.utils.logger import setup_logger
 from danswer.utils.retry_wrapper import retry_builder
 from danswer.utils.variable_functionality import fetch_versioned_implementation
+from ee.danswer.configs.app_configs import INTEGRATION_TEST_MODE
 
 logger = setup_logger()
 
@@ -127,6 +128,9 @@ def seed_initial_documents(
     - Indexing the documents into Vespa
     - Create a fake index attempt with fake times
     """
+    if INTEGRATION_TEST_MODE:
+        return
+
     logger.info("Seeding initial documents")
 
     kv_store = get_kv_store()

backend/danswer/server/documents/cc_pair.py

@@ -6,6 +6,7 @@ from fastapi import APIRouter
 from fastapi import Depends
 from fastapi import HTTPException
 from fastapi import Query
+from fastapi.responses import JSONResponse
 from sqlalchemy.exc import IntegrityError
 from sqlalchemy.orm import Session
 
@@ -37,7 +38,9 @@ from danswer.db.index_attempt import cancel_indexing_attempts_past_model
 from danswer.db.index_attempt import count_index_attempts_for_connector
 from danswer.db.index_attempt import get_latest_index_attempt_for_cc_pair_id
 from danswer.db.index_attempt import get_paginated_index_attempts_for_cc_pair_id
+from danswer.db.models import SearchSettings
 from danswer.db.models import User
+from danswer.db.search_settings import get_active_search_settings
 from danswer.db.search_settings import get_current_search_settings
 from danswer.redis.redis_connector import RedisConnector
 from danswer.redis.redis_pool import get_redis_client
@@ -158,7 +161,19 @@ def update_cc_pair_status(
     status_update_request: CCStatusUpdateRequest,
     user: User | None = Depends(current_curator_or_admin_user),
     db_session: Session = Depends(get_session),
-) -> None:
+    tenant_id: str | None = Depends(get_current_tenant_id),
+) -> JSONResponse:
+    """This method may wait up to 30 seconds if pausing the connector due to the need to
+    terminate tasks in progress. Tasks are not guaranteed to terminate within the
+    timeout.
+
+    Returns HTTPStatus.OK if everything finished.
+    Returns HTTPStatus.ACCEPTED if the connector is being paused, but background tasks
+    did not finish within the timeout.
+    """
+    WAIT_TIMEOUT = 15.0
+    still_terminating = False
+
     cc_pair = get_connector_credential_pair_from_id(
         cc_pair_id=cc_pair_id,
         db_session=db_session,
@@ -173,10 +188,76 @@ def update_cc_pair_status(
         )
 
     if status_update_request.status == ConnectorCredentialPairStatus.PAUSED:
-        cancel_indexing_attempts_for_ccpair(cc_pair_id, db_session)
+        search_settings_list: list[SearchSettings] = get_active_search_settings(
+            db_session
+        )
 
+        cancel_indexing_attempts_for_ccpair(cc_pair_id, db_session)
         cancel_indexing_attempts_past_model(db_session)
 
+        redis_connector = RedisConnector(tenant_id, cc_pair_id)
+
+        try:
+            redis_connector.stop.set_fence(True)
+            while True:
+                logger.debug(
+                    f"Wait for indexing soft termination starting: cc_pair={cc_pair_id}"
+                )
+                wait_succeeded = redis_connector.wait_for_indexing_termination(
+                    search_settings_list, WAIT_TIMEOUT
+                )
+                if wait_succeeded:
+                    logger.debug(
+                        f"Wait for indexing soft termination succeeded: cc_pair={cc_pair_id}"
+                    )
+                    break
+
+                logger.debug(
+                    "Wait for indexing soft termination timed out. "
+                    f"Moving to hard termination: cc_pair={cc_pair_id} timeout={WAIT_TIMEOUT:.2f}"
+                )
+
+                for search_settings in search_settings_list:
+                    redis_connector_index = redis_connector.new_index(
+                        search_settings.id
+                    )
+                    if not redis_connector_index.fenced:
+                        continue
+
+                    index_payload = redis_connector_index.payload
+                    if not index_payload:
+                        continue
+
+                    if not index_payload.celery_task_id:
+                        continue
+
+                    # Revoke the task to prevent it from running
+                    primary_app.control.revoke(index_payload.celery_task_id)
+
+                    # If it is running, then signaling for termination will get the
+                    # watchdog thread to kill the spawned task
+                    redis_connector_index.set_terminate(index_payload.celery_task_id)
+
+                logger.debug(
+                    f"Wait for indexing hard termination starting: cc_pair={cc_pair_id}"
+                )
+                wait_succeeded = redis_connector.wait_for_indexing_termination(
+                    search_settings_list, WAIT_TIMEOUT
+                )
+                if wait_succeeded:
+                    logger.debug(
+                        f"Wait for indexing hard termination succeeded: cc_pair={cc_pair_id}"
+                    )
+                    break
+
+                logger.debug(
+                    f"Wait for indexing hard termination timed out: cc_pair={cc_pair_id}"
+                )
+                still_terminating = True
+                break
+        finally:
+            redis_connector.stop.set_fence(False)
+
     update_connector_credential_pair_from_id(
         db_session=db_session,
         cc_pair_id=cc_pair_id,
@@ -185,6 +266,18 @@ def update_cc_pair_status(
 
     db_session.commit()
 
+    if still_terminating:
+        return JSONResponse(
+            status_code=HTTPStatus.ACCEPTED,
+            content={
+                "message": "Request accepted, background task termination still in progress"
+            },
+        )
+
+    return JSONResponse(
+        status_code=HTTPStatus.OK, content={"message": str(HTTPStatus.OK)}
+    )
+
 
 @router.put("/admin/cc-pair/{cc_pair_id}/name")
 def update_cc_pair_name(
@@ -267,9 +360,9 @@ def prune_cc_pair(
         )
 
     logger.info(
-        f"Pruning cc_pair: cc_pair_id={cc_pair_id} "
-        f"connector_id={cc_pair.connector_id} "
-        f"credential_id={cc_pair.credential_id} "
+        f"Pruning cc_pair: cc_pair={cc_pair_id} "
+        f"connector={cc_pair.connector_id} "
+        f"credential={cc_pair.credential_id} "
         f"{cc_pair.connector.name} connector."
     )
     tasks_created = try_creating_prune_generator_task(

backend/danswer/server/documents/connector.py

@@ -17,9 +17,9 @@ from danswer.auth.users import current_admin_user
 from danswer.auth.users import current_curator_or_admin_user
 from danswer.auth.users import current_user
 from danswer.background.celery.celery_utils import get_deletion_attempt_snapshot
-from danswer.background.celery.tasks.indexing.tasks import try_creating_indexing_task
 from danswer.background.celery.versioned_apps.primary import app as primary_app
 from danswer.configs.app_configs import ENABLED_CONNECTOR_TYPES
+from danswer.configs.constants import DanswerCeleryPriority
 from danswer.configs.constants import DocumentSource
 from danswer.configs.constants import FileOrigin
 from danswer.connectors.google_utils.google_auth import (
@@ -59,6 +59,7 @@ from danswer.db.connector import delete_connector
 from danswer.db.connector import fetch_connector_by_id
 from danswer.db.connector import fetch_connectors
 from danswer.db.connector import get_connector_credential_ids
+from danswer.db.connector import mark_ccpair_with_indexing_trigger
 from danswer.db.connector import update_connector
 from danswer.db.connector_credential_pair import add_credential_to_connector
 from danswer.db.connector_credential_pair import get_cc_pair_groups_for_ids
@@ -74,6 +75,7 @@ from danswer.db.document import get_document_counts_for_cc_pairs
 from danswer.db.engine import get_current_tenant_id
 from danswer.db.engine import get_session
 from danswer.db.enums import AccessType
+from danswer.db.enums import IndexingMode
 from danswer.db.index_attempt import get_index_attempts_for_cc_pair
 from danswer.db.index_attempt import get_latest_index_attempt_for_cc_pair_id
 from danswer.db.index_attempt import get_latest_index_attempts
@@ -86,7 +88,6 @@ from danswer.db.search_settings import get_secondary_search_settings
 from danswer.file_store.file_store import get_default_file_store
 from danswer.key_value_store.interface import KvKeyNotFoundError
 from danswer.redis.redis_connector import RedisConnector
-from danswer.redis.redis_pool import get_redis_client
 from danswer.server.documents.models import AuthStatus
 from danswer.server.documents.models import AuthUrl
 from danswer.server.documents.models import ConnectorCredentialPairIdentifier
@@ -792,12 +793,10 @@ def connector_run_once(
     _: User = Depends(current_curator_or_admin_user),
     db_session: Session = Depends(get_session),
     tenant_id: str = Depends(get_current_tenant_id),
-) -> StatusResponse[list[int]]:
+) -> StatusResponse[int]:
     """Used to trigger indexing on a set of cc_pairs associated with a
     single connector."""
 
-    r = get_redis_client(tenant_id=tenant_id)
-
     connector_id = run_info.connector_id
     specified_credential_ids = run_info.credential_ids
 
@@ -843,54 +842,41 @@ def connector_run_once(
         )
     ]
 
-    search_settings = get_current_search_settings(db_session)
-
     connector_credential_pairs = [
         get_connector_credential_pair(connector_id, credential_id, db_session)
         for credential_id in credential_ids
         if credential_id not in skipped_credentials
     ]
 
-    index_attempt_ids = []
+    num_triggers = 0
     for cc_pair in connector_credential_pairs:
         if cc_pair is not None:
-            attempt_id = try_creating_indexing_task(
-                primary_app,
-                cc_pair,
-                search_settings,
-                run_info.from_beginning,
-                db_session,
-                r,
-                tenant_id,
+            indexing_mode = IndexingMode.UPDATE
+            if run_info.from_beginning:
+                indexing_mode = IndexingMode.REINDEX
+
+            mark_ccpair_with_indexing_trigger(cc_pair.id, indexing_mode, db_session)
+            num_triggers += 1
+
+            logger.info(
+                f"connector_run_once - marking cc_pair with indexing trigger: "
+                f"connector={run_info.connector_id} "
+                f"cc_pair={cc_pair.id} "
+                f"indexing_trigger={indexing_mode}"
             )
-            if attempt_id:
-                logger.info(
-                    f"connector_run_once - try_creating_indexing_task succeeded: "
-                    f"connector={run_info.connector_id} "
-                    f"cc_pair={cc_pair.id} "
-                    f"attempt={attempt_id} "
-                )
-                index_attempt_ids.append(attempt_id)
-            else:
-                logger.info(
-                    f"connector_run_once - try_creating_indexing_task failed: "
-                    f"connector={run_info.connector_id} "
-                    f"cc_pair={cc_pair.id}"
-                )
 
-    if not index_attempt_ids:
-        msg = "No new indexing attempts created, indexing jobs are queued or running."
-        logger.info(msg)
-        raise HTTPException(
-            status_code=400,
-            detail=msg,
-        )
+    # run the beat task to pick up the triggers immediately
+    primary_app.send_task(
+        "check_for_indexing",
+        priority=DanswerCeleryPriority.HIGH,
+        kwargs={"tenant_id": tenant_id},
+    )
 
-    msg = f"Successfully created {len(index_attempt_ids)} index attempts. {index_attempt_ids}"
+    msg = f"Marked {num_triggers} index attempts with indexing triggers."
     return StatusResponse(
         success=True,
         message=msg,
-        data=index_attempt_ids,
+        data=num_triggers,
     )
 
 

backend/danswer/server/manage/models.py

@@ -156,6 +156,7 @@ class SlackChannelConfigCreationRequest(BaseModel):
     channel_name: str
     respond_tag_only: bool = False
     respond_to_bots: bool = False
+    show_continue_in_web_ui: bool = False
     enable_auto_filters: bool = False
     # If no team members, assume respond in the channel to everyone
     respond_member_group_list: list[str] = Field(default_factory=list)

backend/danswer/server/manage/slack_bot.py

@@ -80,6 +80,10 @@ def _form_channel_config(
     if follow_up_tags is not None:
         channel_config["follow_up_tags"] = follow_up_tags
 
+    channel_config[
+        "show_continue_in_web_ui"
+    ] = slack_channel_config_creation_request.show_continue_in_web_ui
+
     channel_config[
         "respond_to_bots"
     ] = slack_channel_config_creation_request.respond_to_bots

backend/danswer/server/query_and_chat/chat_backend.py

@@ -27,9 +27,11 @@ from danswer.configs.app_configs import WEB_DOMAIN
 from danswer.configs.constants import FileOrigin
 from danswer.configs.constants import MessageType
 from danswer.configs.model_configs import LITELLM_PASS_THROUGH_HEADERS
+from danswer.db.chat import add_chats_to_session_from_slack_thread
 from danswer.db.chat import create_chat_session
 from danswer.db.chat import create_new_chat_message
 from danswer.db.chat import delete_chat_session
+from danswer.db.chat import duplicate_chat_session_for_user_from_slack
 from danswer.db.chat import get_chat_message
 from danswer.db.chat import get_chat_messages_by_session
 from danswer.db.chat import get_chat_session_by_id
@@ -532,6 +534,38 @@ def seed_chat(
     )
 
 
+class SeedChatFromSlackRequest(BaseModel):
+    chat_session_id: UUID
+
+
+class SeedChatFromSlackResponse(BaseModel):
+    redirect_url: str
+
+
+@router.post("/seed-chat-session-from-slack")
+def seed_chat_from_slack(
+    chat_seed_request: SeedChatFromSlackRequest,
+    user: User | None = Depends(current_user),
+    db_session: Session = Depends(get_session),
+) -> SeedChatFromSlackResponse:
+    slack_chat_session_id = chat_seed_request.chat_session_id
+    new_chat_session = duplicate_chat_session_for_user_from_slack(
+        db_session=db_session,
+        user=user,
+        chat_session_id=slack_chat_session_id,
+    )
+
+    add_chats_to_session_from_slack_thread(
+        db_session=db_session,
+        slack_chat_session_id=slack_chat_session_id,
+        new_chat_session_id=new_chat_session.id,
+    )
+
+    return SeedChatFromSlackResponse(
+        redirect_url=f"{WEB_DOMAIN}/chat?chatId={new_chat_session.id}"
+    )
+
+
 """File upload"""
 
 

backend/ee/danswer/configs/app_configs.py

@@ -19,3 +19,5 @@ STRIPE_PRICE_ID = os.environ.get("STRIPE_PRICE")
 OPENAI_DEFAULT_API_KEY = os.environ.get("OPENAI_DEFAULT_API_KEY")
 ANTHROPIC_DEFAULT_API_KEY = os.environ.get("ANTHROPIC_DEFAULT_API_KEY")
 COHERE_DEFAULT_API_KEY = os.environ.get("COHERE_DEFAULT_API_KEY")
+
+INTEGRATION_TEST_MODE = os.environ.get("INTEGRATION_TEST_MODE")

backend/ee/danswer/db/user_group.py

@@ -11,6 +11,7 @@ from sqlalchemy import update
 from sqlalchemy.orm import Session
 
 from danswer.db.connector_credential_pair import get_connector_credential_pair_from_id
+from danswer.db.enums import AccessType
 from danswer.db.enums import ConnectorCredentialPairStatus
 from danswer.db.models import ConnectorCredentialPair
 from danswer.db.models import Credential__UserGroup
@@ -298,6 +299,11 @@ def fetch_user_groups_for_documents(
     db_session: Session,
     document_ids: list[str],
 ) -> Sequence[tuple[str, list[str]]]:
+    """
+    Fetches all user groups that have access to the given documents.
+
+    NOTE: this doesn't include groups if the cc_pair is access type SYNC
+    """
     stmt = (
         select(Document.id, func.array_agg(UserGroup.name))
         .join(
@@ -306,7 +312,11 @@ def fetch_user_groups_for_documents(
         )
         .join(
             ConnectorCredentialPair,
-            ConnectorCredentialPair.id == UserGroup__ConnectorCredentialPair.cc_pair_id,
+            and_(
+                ConnectorCredentialPair.id
+                == UserGroup__ConnectorCredentialPair.cc_pair_id,
+                ConnectorCredentialPair.access_type != AccessType.SYNC,
+            ),
         )
         .join(
             DocumentByConnectorCredentialPair,

backend/ee/danswer/external_permissions/confluence/doc_sync.py

@@ -16,7 +16,7 @@ from danswer.utils.logger import setup_logger
 logger = setup_logger()
 
 _VIEWSPACE_PERMISSION_TYPE = "VIEWSPACE"
-_REQUEST_PAGINATION_LIMIT = 100
+_REQUEST_PAGINATION_LIMIT = 5000
 
 
 def _get_server_space_permissions(
@@ -97,6 +97,7 @@ def _get_space_permissions(
     confluence_client: OnyxConfluence,
     is_cloud: bool,
 ) -> dict[str, ExternalAccess]:
+    logger.debug("Getting space permissions")
     # Gets all the spaces in the Confluence instance
     all_space_keys = []
     start = 0
@@ -113,6 +114,7 @@ def _get_space_permissions(
         start += len(spaces_batch.get("results", []))
 
     # Gets the permissions for each space
+    logger.debug(f"Got {len(all_space_keys)} spaces from confluence")
     space_permissions_by_space_key: dict[str, ExternalAccess] = {}
     for space_key in all_space_keys:
         if is_cloud:
@@ -242,6 +244,7 @@ def _fetch_all_page_restrictions_for_space(
 
         logger.warning(f"No permissions found for document {slim_doc.id}")
 
+    logger.debug("Finished fetching all page restrictions for space")
     return document_restrictions
 
 
@@ -254,27 +257,28 @@ def confluence_doc_sync(
     it in postgres so that when it gets created later, the permissions are
     already populated
     """
+    logger.debug("Starting confluence doc sync")
     confluence_connector = ConfluenceConnector(
         **cc_pair.connector.connector_specific_config
     )
     confluence_connector.load_credentials(cc_pair.credential.credential_json)
-    if confluence_connector.confluence_client is None:
-        raise ValueError("Failed to load credentials")
-    confluence_client = confluence_connector.confluence_client
 
     is_cloud = cc_pair.connector.connector_specific_config.get("is_cloud", False)
 
     space_permissions_by_space_key = _get_space_permissions(
-        confluence_client=confluence_client,
+        confluence_client=confluence_connector.confluence_client,
         is_cloud=is_cloud,
     )
 
     slim_docs = []
+    logger.debug("Fetching all slim documents from confluence")
     for doc_batch in confluence_connector.retrieve_all_slim_documents():
+        logger.debug(f"Got {len(doc_batch)} slim documents from confluence")
         slim_docs.extend(doc_batch)
 
+    logger.debug("Fetching all page restrictions for space")
     return _fetch_all_page_restrictions_for_space(
-        confluence_client=confluence_client,
+        confluence_client=confluence_connector.confluence_client,
         slim_docs=slim_docs,
         space_permissions_by_space_key=space_permissions_by_space_key,
     )

backend/ee/danswer/external_permissions/confluence/group_sync.py

@@ -14,7 +14,10 @@ def _build_group_member_email_map(
 ) -> dict[str, set[str]]:
     group_member_emails: dict[str, set[str]] = {}
     for user_result in confluence_client.paginated_cql_user_retrieval():
-        user = user_result["user"]
+        user = user_result.get("user", {})
+        if not user:
+            logger.warning(f"user result missing user field: {user_result}")
+            continue
         email = user.get("email")
         if not email:
             # This field is only present in Confluence Server

backend/ee/danswer/external_permissions/sync_params.py

@@ -57,9 +57,9 @@ DOC_PERMISSION_SYNC_PERIODS: dict[DocumentSource, int] = {
 
 # If nothing is specified here, we run the doc_sync every time the celery beat runs
 EXTERNAL_GROUP_SYNC_PERIODS: dict[DocumentSource, int] = {
-    # Polling is not supported so we fetch all group permissions every 60 seconds
-    DocumentSource.GOOGLE_DRIVE: 60,
-    DocumentSource.CONFLUENCE: 60,
+    # Polling is not supported so we fetch all group permissions every 5 minutes
+    DocumentSource.GOOGLE_DRIVE: 5 * 60,
+    DocumentSource.CONFLUENCE: 5 * 60,
 }
 
 

backend/ee/danswer/main.py

@@ -13,7 +13,6 @@ from danswer.configs.app_configs import WEB_DOMAIN
 from danswer.configs.constants import AuthType
 from danswer.main import get_application as get_application_base
 from danswer.main import include_router_with_global_prefix_prepended
-from danswer.server.api_key.api import router as api_key_router
 from danswer.utils.logger import setup_logger
 from danswer.utils.variable_functionality import global_version
 from ee.danswer.configs.app_configs import OPENID_CONFIG_URL
@@ -116,8 +115,6 @@ def get_application() -> FastAPI:
     # Analytics endpoints
     include_router_with_global_prefix_prepended(application, analytics_router)
     include_router_with_global_prefix_prepended(application, query_history_router)
-    # Api key management
-    include_router_with_global_prefix_prepended(application, api_key_router)
     # EE only backend APIs
     include_router_with_global_prefix_prepended(application, query_router)
     include_router_with_global_prefix_prepended(application, chat_router)

backend/model_server/encoders.py

@@ -11,6 +11,7 @@ from fastapi import APIRouter
 from fastapi import HTTPException
 from google.oauth2 import service_account  # type: ignore
 from litellm import embedding
+from litellm.exceptions import RateLimitError
 from retry import retry
 from sentence_transformers import CrossEncoder  # type: ignore
 from sentence_transformers import SentenceTransformer  # type: ignore
@@ -205,28 +206,22 @@ class CloudEmbedding:
         model_name: str | None = None,
         deployment_name: str | None = None,
     ) -> list[Embedding]:
-        try:
-            if self.provider == EmbeddingProvider.OPENAI:
-                return self._embed_openai(texts, model_name)
-            elif self.provider == EmbeddingProvider.AZURE:
-                return self._embed_azure(texts, f"azure/{deployment_name}")
-            elif self.provider == EmbeddingProvider.LITELLM:
-                return self._embed_litellm_proxy(texts, model_name)
+        if self.provider == EmbeddingProvider.OPENAI:
+            return self._embed_openai(texts, model_name)
+        elif self.provider == EmbeddingProvider.AZURE:
+            return self._embed_azure(texts, f"azure/{deployment_name}")
+        elif self.provider == EmbeddingProvider.LITELLM:
+            return self._embed_litellm_proxy(texts, model_name)
 
-            embedding_type = EmbeddingModelTextType.get_type(self.provider, text_type)
-            if self.provider == EmbeddingProvider.COHERE:
-                return self._embed_cohere(texts, model_name, embedding_type)
-            elif self.provider == EmbeddingProvider.VOYAGE:
-                return self._embed_voyage(texts, model_name, embedding_type)
-            elif self.provider == EmbeddingProvider.GOOGLE:
-                return self._embed_vertex(texts, model_name, embedding_type)
-            else:
-                raise ValueError(f"Unsupported provider: {self.provider}")
-        except Exception as e:
-            raise HTTPException(
-                status_code=500,
-                detail=f"Error embedding text with {self.provider}: {str(e)}",
-            )
+        embedding_type = EmbeddingModelTextType.get_type(self.provider, text_type)
+        if self.provider == EmbeddingProvider.COHERE:
+            return self._embed_cohere(texts, model_name, embedding_type)
+        elif self.provider == EmbeddingProvider.VOYAGE:
+            return self._embed_voyage(texts, model_name, embedding_type)
+        elif self.provider == EmbeddingProvider.GOOGLE:
+            return self._embed_vertex(texts, model_name, embedding_type)
+        else:
+            raise ValueError(f"Unsupported provider: {self.provider}")
 
     @staticmethod
     def create(
@@ -430,6 +425,11 @@ async def process_embed_request(
             prefix=prefix,
         )
         return EmbedResponse(embeddings=embeddings)
+    except RateLimitError as e:
+        raise HTTPException(
+            status_code=429,
+            detail=str(e),
+        )
     except Exception as e:
         exception_detail = f"Error during embedding process:\n{str(e)}"
         logger.exception(exception_detail)

backend/requirements/default.txt

@@ -29,7 +29,7 @@ trafilatura==1.12.2
 langchain==0.1.17
 langchain-core==0.1.50
 langchain-text-splitters==0.0.1
-litellm==1.50.2
+litellm==1.53.1
 lxml==5.3.0
 lxml_html_clean==0.2.2
 llama-index==0.9.45
@@ -38,7 +38,7 @@ msal==1.28.0
 nltk==3.8.1
 Office365-REST-Python-Client==2.5.9
 oauthlib==3.2.2
-openai==1.52.2
+openai==1.55.3
 openpyxl==3.1.2
 playwright==1.41.2
 psutil==5.9.5

backend/tests/daily/embedding/test_embeddings.py

@@ -7,6 +7,7 @@ from shared_configs.enums import EmbedTextType
 from shared_configs.model_server_models import EmbeddingProvider
 
 VALID_SAMPLE = ["hi", "hello my name is bob", "woah there!!!. 😃"]
+VALID_LONG_SAMPLE = ["hi " * 999]
 # openai limit is 2048, cohere is supposed to be 96 but in practice that doesn't
 # seem to be true
 TOO_LONG_SAMPLE = ["a"] * 2500
@@ -99,3 +100,42 @@ def local_nomic_embedding_model() -> EmbeddingModel:
 def test_local_nomic_embedding(local_nomic_embedding_model: EmbeddingModel) -> None:
     _run_embeddings(VALID_SAMPLE, local_nomic_embedding_model, 768)
     _run_embeddings(TOO_LONG_SAMPLE, local_nomic_embedding_model, 768)
+
+
+@pytest.fixture
+def azure_embedding_model() -> EmbeddingModel:
+    return EmbeddingModel(
+        server_host="localhost",
+        server_port=9000,
+        model_name="text-embedding-3-large",
+        normalize=True,
+        query_prefix=None,
+        passage_prefix=None,
+        api_key=os.getenv("AZURE_API_KEY"),
+        provider_type=EmbeddingProvider.AZURE,
+        api_url=os.getenv("AZURE_API_URL"),
+    )
+
+
+# NOTE (chris): this test doesn't work, and I do not know why
+# def test_azure_embedding_model_rate_limit(azure_embedding_model: EmbeddingModel):
+#     """NOTE: this test relies on a very low rate limit for the Azure API +
+#     this test only being run once in a 1 minute window"""
+#     # VALID_LONG_SAMPLE is 999 tokens, so the second call should run into rate
+#     # limits assuming the limit is 1000 tokens per minute
+#     result = azure_embedding_model.encode(VALID_LONG_SAMPLE, EmbedTextType.QUERY)
+#     assert len(result) == 1
+#     assert len(result[0]) == 1536
+
+#     # this should fail
+#     with pytest.raises(ModelServerRateLimitError):
+#         azure_embedding_model.encode(VALID_LONG_SAMPLE, EmbedTextType.QUERY)
+#         azure_embedding_model.encode(VALID_LONG_SAMPLE, EmbedTextType.QUERY)
+#         azure_embedding_model.encode(VALID_LONG_SAMPLE, EmbedTextType.QUERY)
+
+#     # this should succeed, since passage requests retry up to 10 times
+#     start = time.time()
+#     result = azure_embedding_model.encode(VALID_LONG_SAMPLE, EmbedTextType.PASSAGE)
+#     assert len(result) == 1
+#     assert len(result[0]) == 1536
+#     assert time.time() - start > 30  # make sure we waited, even though we hit rate limits

backend/tests/integration/common_utils/managers/cc_pair.py

@@ -240,7 +240,85 @@ class CCPairManager:
         result.raise_for_status()
 
     @staticmethod
-    def wait_for_indexing(
+    def wait_for_indexing_inactive(
+        cc_pair: DATestCCPair,
+        timeout: float = MAX_DELAY,
+        user_performing_action: DATestUser | None = None,
+    ) -> None:
+        """wait for the number of docs to be indexed on the connector.
+        This is used to test pausing a connector in the middle of indexing and
+        terminating that indexing."""
+        print(f"Indexing wait for inactive starting: cc_pair={cc_pair.id}")
+        start = time.monotonic()
+        while True:
+            fetched_cc_pairs = CCPairManager.get_indexing_statuses(
+                user_performing_action
+            )
+            for fetched_cc_pair in fetched_cc_pairs:
+                if fetched_cc_pair.cc_pair_id != cc_pair.id:
+                    continue
+
+                if fetched_cc_pair.in_progress:
+                    continue
+
+                print(f"Indexing is inactive: cc_pair={cc_pair.id}")
+                return
+
+            elapsed = time.monotonic() - start
+            if elapsed > timeout:
+                raise TimeoutError(
+                    f"Indexing wait for inactive timed out: cc_pair={cc_pair.id} timeout={timeout}s"
+                )
+
+            print(
+                f"Indexing wait for inactive still waiting: cc_pair={cc_pair.id} elapsed={elapsed:.2f} timeout={timeout}s"
+            )
+            time.sleep(5)
+
+    @staticmethod
+    def wait_for_indexing_in_progress(
+        cc_pair: DATestCCPair,
+        timeout: float = MAX_DELAY,
+        num_docs: int = 16,
+        user_performing_action: DATestUser | None = None,
+    ) -> None:
+        """wait for the number of docs to be indexed on the connector.
+        This is used to test pausing a connector in the middle of indexing and
+        terminating that indexing."""
+        start = time.monotonic()
+        while True:
+            fetched_cc_pairs = CCPairManager.get_indexing_statuses(
+                user_performing_action
+            )
+            for fetched_cc_pair in fetched_cc_pairs:
+                if fetched_cc_pair.cc_pair_id != cc_pair.id:
+                    continue
+
+                if not fetched_cc_pair.in_progress:
+                    continue
+
+                if fetched_cc_pair.docs_indexed >= num_docs:
+                    print(
+                        "Indexed at least the requested number of docs: "
+                        f"cc_pair={cc_pair.id} "
+                        f"docs_indexed={fetched_cc_pair.docs_indexed} "
+                        f"num_docs={num_docs}"
+                    )
+                    return
+
+            elapsed = time.monotonic() - start
+            if elapsed > timeout:
+                raise TimeoutError(
+                    f"Indexing in progress wait timed out: cc_pair={cc_pair.id} timeout={timeout}s"
+                )
+
+            print(
+                f"Indexing in progress waiting: cc_pair={cc_pair.id} elapsed={elapsed:.2f} timeout={timeout}s"
+            )
+            time.sleep(5)
+
+    @staticmethod
+    def wait_for_indexing_completion(
         cc_pair: DATestCCPair,
         after: datetime,
         timeout: float = MAX_DELAY,

backend/tests/integration/common_utils/managers/tenant.py

@@ -1,84 +0,0 @@
-from datetime import datetime
-from datetime import timedelta
-
-import jwt
-import requests
-
-from danswer.server.manage.models import AllUsersResponse
-from danswer.server.models import FullUserSnapshot
-from danswer.server.models import InvitedUserSnapshot
-from tests.integration.common_utils.constants import API_SERVER_URL
-from tests.integration.common_utils.constants import GENERAL_HEADERS
-from tests.integration.common_utils.test_models import DATestUser
-
-
-def generate_auth_token() -> str:
-    payload = {
-        "iss": "control_plane",
-        "exp": datetime.utcnow() + timedelta(minutes=5),
-        "iat": datetime.utcnow(),
-        "scope": "tenant:create",
-    }
-    token = jwt.encode(payload, "", algorithm="HS256")
-    return token
-
-
-class TenantManager:
-    @staticmethod
-    def create(
-        tenant_id: str | None = None,
-        initial_admin_email: str | None = None,
-        referral_source: str | None = None,
-    ) -> dict[str, str]:
-        body = {
-            "tenant_id": tenant_id,
-            "initial_admin_email": initial_admin_email,
-            "referral_source": referral_source,
-        }
-
-        token = generate_auth_token()
-        headers = {
-            "Authorization": f"Bearer {token}",
-            "X-API-KEY": "",
-            "Content-Type": "application/json",
-        }
-
-        response = requests.post(
-            url=f"{API_SERVER_URL}/tenants/create",
-            json=body,
-            headers=headers,
-        )
-
-        response.raise_for_status()
-
-        return response.json()
-
-    @staticmethod
-    def get_all_users(
-        user_performing_action: DATestUser | None = None,
-    ) -> AllUsersResponse:
-        response = requests.get(
-            url=f"{API_SERVER_URL}/manage/users",
-            headers=user_performing_action.headers
-            if user_performing_action
-            else GENERAL_HEADERS,
-        )
-        response.raise_for_status()
-
-        data = response.json()
-        return AllUsersResponse(
-            accepted=[FullUserSnapshot(**user) for user in data["accepted"]],
-            invited=[InvitedUserSnapshot(**user) for user in data["invited"]],
-            accepted_pages=data["accepted_pages"],
-            invited_pages=data["invited_pages"],
-        )
-
-    @staticmethod
-    def verify_user_in_tenant(
-        user: DATestUser, user_performing_action: DATestUser | None = None
-    ) -> None:
-        all_users = TenantManager.get_all_users(user_performing_action)
-        for accepted_user in all_users.accepted:
-            if accepted_user.email == user.email and accepted_user.id == user.id:
-                return
-        raise ValueError(f"User {user.email} not found in tenant")

backend/tests/integration/common_utils/reset.py

@@ -211,7 +211,6 @@ def reset_postgres_multitenant() -> None:
     for schema in tenant_schemas:
         schema_name = schema[0]
         cur.execute(f'DROP SCHEMA "{schema_name}" CASCADE')
-
     cur.close()
     conn.close()
 

backend/tests/integration/connector_job_tests/slack/test_permission_sync.py

@@ -14,6 +14,7 @@ from tests.integration.common_utils.managers.document_search import (
 )
 from tests.integration.common_utils.managers.llm_provider import LLMProviderManager
 from tests.integration.common_utils.managers.user import UserManager
+from tests.integration.common_utils.managers.user_group import UserGroupManager
 from tests.integration.common_utils.test_models import DATestCCPair
 from tests.integration.common_utils.test_models import DATestConnector
 from tests.integration.common_utils.test_models import DATestCredential
@@ -77,7 +78,7 @@ def test_slack_permission_sync(
         access_type=AccessType.SYNC,
         user_performing_action=admin_user,
     )
-    CCPairManager.wait_for_indexing(
+    CCPairManager.wait_for_indexing_completion(
         cc_pair=cc_pair,
         after=before,
         user_performing_action=admin_user,
@@ -112,7 +113,7 @@ def test_slack_permission_sync(
     # Run indexing
     before = datetime.now(timezone.utc)
     CCPairManager.run_once(cc_pair, admin_user)
-    CCPairManager.wait_for_indexing(
+    CCPairManager.wait_for_indexing_completion(
         cc_pair=cc_pair,
         after=before,
         user_performing_action=admin_user,
@@ -215,3 +216,124 @@ def test_slack_permission_sync(
     # Ensure test_user_1 can only see messages from the public channel
     assert public_message in danswer_doc_message_strings
     assert private_message not in danswer_doc_message_strings
+
+
+def test_slack_group_permission_sync(
+    reset: None,
+    vespa_client: vespa_fixture,
+    slack_test_setup: tuple[dict[str, Any], dict[str, Any]],
+) -> None:
+    """
+    This test ensures that permission sync overrides danswer group access.
+    """
+    public_channel, private_channel = slack_test_setup
+
+    # Creating an admin user (first user created is automatically an admin)
+    admin_user: DATestUser = UserManager.create(
+        email="admin@onyx-test.com",
+    )
+
+    # Creating a non-admin user
+    test_user_1: DATestUser = UserManager.create(
+        email="test_user_1@onyx-test.com",
+    )
+
+    # Create a user group and adding the non-admin user to it
+    user_group = UserGroupManager.create(
+        name="test_group",
+        user_ids=[test_user_1.id],
+        cc_pair_ids=[],
+        user_performing_action=admin_user,
+    )
+    UserGroupManager.wait_for_sync(
+        user_groups_to_check=[user_group],
+        user_performing_action=admin_user,
+    )
+
+    slack_client = SlackManager.get_slack_client(os.environ["SLACK_BOT_TOKEN"])
+    email_id_map = SlackManager.build_slack_user_email_id_map(slack_client)
+    admin_user_id = email_id_map[admin_user.email]
+
+    LLMProviderManager.create(user_performing_action=admin_user)
+
+    # Add only admin to the private channel
+    SlackManager.set_channel_members(
+        slack_client=slack_client,
+        admin_user_id=admin_user_id,
+        channel=private_channel,
+        user_ids=[admin_user_id],
+    )
+
+    before = datetime.now(timezone.utc)
+    credential = CredentialManager.create(
+        source=DocumentSource.SLACK,
+        credential_json={
+            "slack_bot_token": os.environ["SLACK_BOT_TOKEN"],
+        },
+        user_performing_action=admin_user,
+    )
+
+    # Create connector with sync access and assign it to the user group
+    connector = ConnectorManager.create(
+        name="Slack",
+        input_type=InputType.POLL,
+        source=DocumentSource.SLACK,
+        connector_specific_config={
+            "workspace": "onyx-test-workspace",
+            "channels": [private_channel["name"]],
+        },
+        access_type=AccessType.SYNC,
+        groups=[user_group.id],
+        user_performing_action=admin_user,
+    )
+
+    cc_pair = CCPairManager.create(
+        credential_id=credential.id,
+        connector_id=connector.id,
+        access_type=AccessType.SYNC,
+        user_performing_action=admin_user,
+        groups=[user_group.id],
+    )
+
+    # Add a test message to the private channel
+    private_message = "This is a secret message: 987654"
+    SlackManager.add_message_to_channel(
+        slack_client=slack_client,
+        channel=private_channel,
+        message=private_message,
+    )
+
+    # Run indexing
+    CCPairManager.run_once(cc_pair, admin_user)
+    CCPairManager.wait_for_indexing_completion(
+        cc_pair=cc_pair,
+        after=before,
+        user_performing_action=admin_user,
+    )
+
+    # Run permission sync
+    CCPairManager.sync(
+        cc_pair=cc_pair,
+        user_performing_action=admin_user,
+    )
+    CCPairManager.wait_for_sync(
+        cc_pair=cc_pair,
+        after=before,
+        number_of_updated_docs=1,
+        user_performing_action=admin_user,
+    )
+
+    # Verify admin can see the message
+    admin_docs = DocumentSearchManager.search_documents(
+        query="secret message",
+        user_performing_action=admin_user,
+    )
+    assert private_message in admin_docs
+
+    # Verify test_user_1 cannot see the message despite being in the group
+    # (Slack permissions should take precedence)
+    user_1_docs = DocumentSearchManager.search_documents(
+        query="secret message",
+        user_performing_action=test_user_1,
+    )
+    assert private_message not in user_1_docs

backend/tests/integration/connector_job_tests/slack/test_prune.py

@@ -74,7 +74,7 @@ def test_slack_prune(
         access_type=AccessType.SYNC,
         user_performing_action=admin_user,
     )
-    CCPairManager.wait_for_indexing(
+    CCPairManager.wait_for_indexing_completion(
         cc_pair=cc_pair,
         after=before,
         user_performing_action=admin_user,
@@ -113,7 +113,7 @@ def test_slack_prune(
     # Run indexing
     before = datetime.now(timezone.utc)
     CCPairManager.run_once(cc_pair, admin_user)
-    CCPairManager.wait_for_indexing(
+    CCPairManager.wait_for_indexing_completion(
         cc_pair=cc_pair,
         after=before,
         user_performing_action=admin_user,

backend/tests/integration/multitenant_tests/syncing/test_search_permissions.py

@@ -4,7 +4,6 @@ from tests.integration.common_utils.managers.cc_pair import CCPairManager
 from tests.integration.common_utils.managers.chat import ChatSessionManager
 from tests.integration.common_utils.managers.document import DocumentManager
 from tests.integration.common_utils.managers.llm_provider import LLMProviderManager
-from tests.integration.common_utils.managers.tenant import TenantManager
 from tests.integration.common_utils.managers.user import UserManager
 from tests.integration.common_utils.test_models import DATestAPIKey
 from tests.integration.common_utils.test_models import DATestCCPair
@@ -13,66 +12,49 @@ from tests.integration.common_utils.test_models import DATestUser
 
 
 def test_multi_tenant_access_control(reset_multitenant: None) -> None:
-    # Create Tenant 1 and its Admin User
-    TenantManager.create("tenant_dev1", "test1@test.com", "Data Plane Registration")
-    test_user1: DATestUser = UserManager.create(name="test1", email="test1@test.com")
-    assert UserManager.verify_role(test_user1, UserRole.ADMIN)
+    # Create Tenants and Admin Users
+    test_user1: DATestUser = UserManager.create(name="test1", email="test_1@test.com")
+    test_user2: DATestUser = UserManager.create(name="test2", email="test_2@test.com")
 
-    # Create Tenant 2 and its Admin User
-    TenantManager.create("tenant_dev2", "test2@test.com", "Data Plane Registration")
-    test_user2: DATestUser = UserManager.create(name="test2", email="test2@test.com")
+    assert UserManager.verify_role(test_user1, UserRole.ADMIN)
     assert UserManager.verify_role(test_user2, UserRole.ADMIN)
 
-    # Create connectors for Tenant 1
+    # Create connectors and seed documents for Tenant 1
     cc_pair_1: DATestCCPair = CCPairManager.create_from_scratch(
-        user_performing_action=test_user1,
-    )
-    api_key_1: DATestAPIKey = APIKeyManager.create(
-        user_performing_action=test_user1,
+        user_performing_action=test_user1
     )
+    api_key_1: DATestAPIKey = APIKeyManager.create(user_performing_action=test_user1)
     api_key_1.headers.update(test_user1.headers)
     LLMProviderManager.create(user_performing_action=test_user1)
 
-    # Seed documents for Tenant 1
     cc_pair_1.documents = []
-    doc1_tenant1 = DocumentManager.seed_doc_with_content(
-        cc_pair=cc_pair_1,
-        content="Tenant 1 Document Content",
-        api_key=api_key_1,
-    )
-    doc2_tenant1 = DocumentManager.seed_doc_with_content(
-        cc_pair=cc_pair_1,
-        content="Tenant 1 Document Content",
-        api_key=api_key_1,
-    )
-    cc_pair_1.documents.extend([doc1_tenant1, doc2_tenant1])
+    docs_tenant1 = [
+        DocumentManager.seed_doc_with_content(
+            cc_pair=cc_pair_1, content="Tenant 1 Document Content", api_key=api_key_1
+        )
+        for _ in range(2)
+    ]
+    cc_pair_1.documents.extend(docs_tenant1)
 
-    # Create connectors for Tenant 2
+    # Create connectors and seed documents for Tenant 2
     cc_pair_2: DATestCCPair = CCPairManager.create_from_scratch(
-        user_performing_action=test_user2,
-    )
-    api_key_2: DATestAPIKey = APIKeyManager.create(
-        user_performing_action=test_user2,
+        user_performing_action=test_user2
     )
+    api_key_2: DATestAPIKey = APIKeyManager.create(user_performing_action=test_user2)
     api_key_2.headers.update(test_user2.headers)
     LLMProviderManager.create(user_performing_action=test_user2)
 
-    # Seed documents for Tenant 2
     cc_pair_2.documents = []
-    doc1_tenant2 = DocumentManager.seed_doc_with_content(
-        cc_pair=cc_pair_2,
-        content="Tenant 2 Document Content",
-        api_key=api_key_2,
-    )
-    doc2_tenant2 = DocumentManager.seed_doc_with_content(
-        cc_pair=cc_pair_2,
-        content="Tenant 2 Document Content",
-        api_key=api_key_2,
-    )
-    cc_pair_2.documents.extend([doc1_tenant2, doc2_tenant2])
+    docs_tenant2 = [
+        DocumentManager.seed_doc_with_content(
+            cc_pair=cc_pair_2, content="Tenant 2 Document Content", api_key=api_key_2
+        )
+        for _ in range(2)
+    ]
+    cc_pair_2.documents.extend(docs_tenant2)
 
-    tenant1_doc_ids = {doc1_tenant1.id, doc2_tenant1.id}
-    tenant2_doc_ids = {doc1_tenant2.id, doc2_tenant2.id}
+    tenant1_doc_ids = {doc.id for doc in docs_tenant1}
+    tenant2_doc_ids = {doc.id for doc in docs_tenant2}
 
     # Create chat sessions for each user
     chat_session1: DATestChatSession = ChatSessionManager.create(
@@ -82,69 +64,63 @@ def test_multi_tenant_access_control(reset_multitenant: None) -> None:
         user_performing_action=test_user2
     )
 
-    # User 1 sends a message and gets a response
+    # Test access for Tenant 1
     response1 = ChatSessionManager.send_message(
         chat_session_id=chat_session1.id,
         message="What is in Tenant 1's documents?",
         user_performing_action=test_user1,
     )
-    # Assert that the search tool was used
     assert response1.tool_name == "run_search"
-
-    response_doc_ids = {doc["document_id"] for doc in response1.tool_result or []}
+    response1_doc_ids = {doc["document_id"] for doc in response1.tool_result or []}
     assert tenant1_doc_ids.issubset(
-        response_doc_ids
+        response1_doc_ids
     ), "Not all Tenant 1 document IDs are in the response"
-    assert not response_doc_ids.intersection(
+    assert not response1_doc_ids.intersection(
         tenant2_doc_ids
-    ), "Tenant 2 document IDs should not be in the response"
-
-    # Assert that the contents are correct
+    ), "Tenant 2's document IDs should not be in the response"
     for doc in response1.tool_result or []:
         assert doc["content"] == "Tenant 1 Document Content"
 
-    # User 2 sends a message and gets a response
+    # Test access for Tenant 2
     response2 = ChatSessionManager.send_message(
         chat_session_id=chat_session2.id,
         message="What is in Tenant 2's documents?",
         user_performing_action=test_user2,
     )
-    # Assert that the search tool was used
     assert response2.tool_name == "run_search"
-    # Assert that the tool_result contains Tenant 2's documents
-    response_doc_ids = {doc["document_id"] for doc in response2.tool_result or []}
+    response2_doc_ids = {doc["document_id"] for doc in response2.tool_result or []}
     assert tenant2_doc_ids.issubset(
-        response_doc_ids
+        response2_doc_ids
     ), "Not all Tenant 2 document IDs are in the response"
-    assert not response_doc_ids.intersection(
+    assert not response2_doc_ids.intersection(
         tenant1_doc_ids
-    ), "Tenant 1 document IDs should not be in the response"
-    # Assert that the contents are correct
+    ), "Tenant 1's document IDs should not be in the response"
     for doc in response2.tool_result or []:
         assert doc["content"] == "Tenant 2 Document Content"
 
-    # User 1 tries to access Tenant 2's documents
-    response_cross = ChatSessionManager.send_message(
+    # Test cross-tenant access attempts
+    response_cross1 = ChatSessionManager.send_message(
         chat_session_id=chat_session1.id,
         message="What is in Tenant 2's documents?",
         user_performing_action=test_user1,
     )
-    # Assert that the search tool was used
-    assert response_cross.tool_name == "run_search"
-    # Assert that the tool_result is empty or does not contain Tenant 2's documents
-    response_doc_ids = {doc["document_id"] for doc in response_cross.tool_result or []}
-    # Ensure none of Tenant 2's document IDs are in the response
-    assert not response_doc_ids.intersection(tenant2_doc_ids)
+    assert response_cross1.tool_name == "run_search"
+    response_cross1_doc_ids = {
+        doc["document_id"] for doc in response_cross1.tool_result or []
+    }
+    assert not response_cross1_doc_ids.intersection(
+        tenant2_doc_ids
+    ), "Tenant 2's document IDs should not be in the response"
 
-    # User 2 tries to access Tenant 1's documents
     response_cross2 = ChatSessionManager.send_message(
         chat_session_id=chat_session2.id,
         message="What is in Tenant 1's documents?",
         user_performing_action=test_user2,
     )
-    # Assert that the search tool was used
     assert response_cross2.tool_name == "run_search"
-    # Assert that the tool_result is empty or does not contain Tenant 1's documents
-    response_doc_ids = {doc["document_id"] for doc in response_cross2.tool_result or []}
-    # Ensure none of Tenant 1's document IDs are in the response
-    assert not response_doc_ids.intersection(tenant1_doc_ids)
+    response_cross2_doc_ids = {
+        doc["document_id"] for doc in response_cross2.tool_result or []
+    }
+    assert not response_cross2_doc_ids.intersection(
+        tenant1_doc_ids
+    ), "Tenant 1's document IDs should not be in the response"

backend/tests/integration/multitenant_tests/tenants/test_tenant_creation.py

@@ -1,41 +0,0 @@
-from danswer.configs.constants import DocumentSource
-from danswer.db.enums import AccessType
-from danswer.db.models import UserRole
-from tests.integration.common_utils.managers.cc_pair import CCPairManager
-from tests.integration.common_utils.managers.connector import ConnectorManager
-from tests.integration.common_utils.managers.credential import CredentialManager
-from tests.integration.common_utils.managers.tenant import TenantManager
-from tests.integration.common_utils.managers.user import UserManager
-from tests.integration.common_utils.test_models import DATestUser
-
-
-# Test flow from creating tenant to registering as a user
-def test_tenant_creation(reset_multitenant: None) -> None:
-    TenantManager.create("tenant_dev", "test@test.com", "Data Plane Registration")
-    test_user: DATestUser = UserManager.create(name="test", email="test@test.com")
-
-    assert UserManager.verify_role(test_user, UserRole.ADMIN)
-
-    test_credential = CredentialManager.create(
-        name="admin_test_credential",
-        source=DocumentSource.FILE,
-        curator_public=False,
-        user_performing_action=test_user,
-    )
-
-    test_connector = ConnectorManager.create(
-        name="admin_test_connector",
-        source=DocumentSource.FILE,
-        access_type=AccessType.PRIVATE,
-        user_performing_action=test_user,
-    )
-
-    test_cc_pair = CCPairManager.create(
-        connector_id=test_connector.id,
-        credential_id=test_credential.id,
-        name="admin_test_cc_pair",
-        access_type=AccessType.PRIVATE,
-        user_performing_action=test_user,
-    )
-
-    CCPairManager.verify(cc_pair=test_cc_pair, user_performing_action=test_user)

backend/tests/integration/tests/connector/test_connector_creation.py

@@ -58,7 +58,7 @@ def test_overlapping_connector_creation(reset: None) -> None:
         user_performing_action=admin_user,
     )
 
-    CCPairManager.wait_for_indexing(
+    CCPairManager.wait_for_indexing_completion(
         cc_pair_1, now, timeout=120, user_performing_action=admin_user
     )
 
@@ -71,7 +71,7 @@ def test_overlapping_connector_creation(reset: None) -> None:
         user_performing_action=admin_user,
     )
 
-    CCPairManager.wait_for_indexing(
+    CCPairManager.wait_for_indexing_completion(
         cc_pair_2, now, timeout=120, user_performing_action=admin_user
     )
 
@@ -82,3 +82,48 @@ def test_overlapping_connector_creation(reset: None) -> None:
     assert info_2
 
     assert info_1.num_docs_indexed == info_2.num_docs_indexed
+
+
+def test_connector_pause_while_indexing(reset: None) -> None:
+    """Tests that we can pause a connector while indexing is in progress and that
+    tasks end early or abort as a result.
+
+    TODO: This does not specifically test for soft or hard termination code paths.
+    Design specific tests for those use cases.
+    """
+    admin_user: DATestUser = UserManager.create(name="admin_user")
+
+    config = {
+        "wiki_base": os.environ["CONFLUENCE_TEST_SPACE_URL"],
+        "space": "",
+        "is_cloud": True,
+        "page_id": "",
+    }
+
+    credential = {
+        "confluence_username": os.environ["CONFLUENCE_USER_NAME"],
+        "confluence_access_token": os.environ["CONFLUENCE_ACCESS_TOKEN"],
+    }
+
+    # store the time before we create the connector so that we know after
+    # when the indexing should have started
+    datetime.now(timezone.utc)
+
+    # create connector
+    cc_pair_1 = CCPairManager.create_from_scratch(
+        source=DocumentSource.CONFLUENCE,
+        connector_specific_config=config,
+        credential_json=credential,
+        user_performing_action=admin_user,
+    )
+
+    CCPairManager.wait_for_indexing_in_progress(
+        cc_pair_1, timeout=60, num_docs=16, user_performing_action=admin_user
+    )
+
+    CCPairManager.pause_cc_pair(cc_pair_1, user_performing_action=admin_user)
+
+    CCPairManager.wait_for_indexing_inactive(
+        cc_pair_1, timeout=60, user_performing_action=admin_user
+    )
+    return

backend/tests/integration/tests/pruning/test_pruning.py

@@ -135,7 +135,7 @@ def test_web_pruning(reset: None, vespa_client: vespa_fixture) -> None:
                 user_performing_action=admin_user,
             )
 
-            CCPairManager.wait_for_indexing(
+            CCPairManager.wait_for_indexing_completion(
                 cc_pair_1, now, timeout=60, user_performing_action=admin_user
             )
 

backend/tests/unit/danswer/llm/answering/stream_processing/test_citation_processing.py

@@ -385,6 +385,16 @@ def process_text(
             "Here is some text[[1]](https://0.com). Some other text",
             ["doc_0"],
         ),
+        # ['To', ' set', ' up', ' D', 'answer', ',', ' if', ' you', ' are', ' running', ' it', ' yourself', ' and',
+        # ' need', ' access', ' to', ' certain', ' features', ' like', ' auto', '-sync', 'ing', ' document',
+        # '-level', ' access', ' permissions', ',', ' you', ' should', ' reach', ' out', ' to', ' the', ' D',
+        # 'answer', ' team', ' to', ' receive', ' access', ' [[', '4', ']].', '']
+        (
+            "Unique tokens with double brackets and a single token that ends the citation and has characters after it.",
+            ["... to receive access", " [[", "1", "]].", ""],
+            "... to receive access [[1]](https://0.com).",
+            ["doc_0"],
+        ),
     ],
 )
 def test_citation_extraction(

deployment/docker_compose/docker-compose.multi-tenant.yml

@@ -0,0 +1,30 @@
+version: "3"
+services:
+  control-tenants-service:
+    image: danswer/control-tenants-service:test
+    environment:
+      - POSTGRES_HOST=relational_db
+      - POSTGRES_PORT=5432
+    ports:
+      - "8082:8082"
+    depends_on:
+      relational_db:
+        condition: service_healthy
+
+  relational_db:
+    image: postgres:15.2-alpine
+    restart: always
+    ports:
+      - "5434:5432"
+    volumes:
+      - db_volume:/var/lib/postgresql/data
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U postgres -d postgres"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+      start_period: 10s
+    command: ["postgres", "-c", "log_statement=all"]
+
+volumes:
+  db_volume:

web/src/app/admin/assistants/AssistantEditor.tsx

@@ -405,7 +405,7 @@ export function AssistantEditor({
                   message: `"${assistant.name}" has been added to your list.`,
                   type: "success",
                 });
-                router.refresh();
+                await refreshAssistants();
               } else {
                 setPopup({
                   message: `"${assistant.name}" could not be added to your list.`,

web/src/app/admin/assistants/PersonaTable.tsx

@@ -90,7 +90,7 @@ export function PersonasTable() {
         message: `Failed to update persona order - ${await response.text()}`,
       });
       setFinalPersonas(assistants);
-      router.refresh();
+      await refreshAssistants();
       return;
     }
 
@@ -151,7 +151,7 @@ export function PersonasTable() {
                       persona.is_visible
                     );
                     if (response.ok) {
-                      router.refresh();
+                      await refreshAssistants();
                     } else {
                       setPopup({
                         type: "error",
@@ -183,7 +183,7 @@ export function PersonasTable() {
                       onClick={async () => {
                         const response = await deletePersona(persona.id);
                         if (response.ok) {
-                          router.refresh();
+                          await refreshAssistants();
                         } else {
                           alert(
                             `Failed to delete persona - ${await response.text()}`

web/src/app/admin/assistants/lib.ts

@@ -259,29 +259,8 @@ export async function updatePersona(
 ): Promise<[Response, Response | null]> {
   const { id, existingPromptId } = personaUpdateRequest;
 
-  let fileId = null;
-  if (personaUpdateRequest.uploaded_image) {
-    fileId = await uploadFile(personaUpdateRequest.uploaded_image);
-    if (!fileId) {
-      return [new Response(null, { status: 400 }), null];
-    }
-  }
-
-  const updatePersonaResponse = await fetch(`/api/persona/${id}`, {
-    method: "PATCH",
-    headers: {
-      "Content-Type": "application/json",
-    },
-    body: JSON.stringify(
-      buildPersonaAPIBody(personaUpdateRequest, existingPromptId ?? 0, fileId)
-    ),
-  });
-
-  if (!updatePersonaResponse.ok) {
-    return [updatePersonaResponse, null];
-  }
-
   let promptResponse;
+  let promptId: number | null = null;
   if (existingPromptId !== undefined) {
     promptResponse = await updatePrompt({
       promptId: existingPromptId,
@@ -290,6 +269,7 @@ export async function updatePersona(
       taskPrompt: personaUpdateRequest.task_prompt,
       includeCitations: personaUpdateRequest.include_citations,
     });
+    promptId = existingPromptId;
   } else {
     promptResponse = await createPrompt({
       personaName: personaUpdateRequest.name,
@@ -297,7 +277,30 @@ export async function updatePersona(
       taskPrompt: personaUpdateRequest.task_prompt,
       includeCitations: personaUpdateRequest.include_citations,
     });
+    promptId = promptResponse.ok
+      ? ((await promptResponse.json()).id as number)
+      : null;
   }
+  let fileId = null;
+  if (personaUpdateRequest.uploaded_image) {
+    fileId = await uploadFile(personaUpdateRequest.uploaded_image);
+    if (!fileId) {
+      return [promptResponse, null];
+    }
+  }
+
+  const updatePersonaResponse =
+    promptResponse.ok && promptId !== null
+      ? await fetch(`/api/persona/${id}`, {
+          method: "PATCH",
+          headers: {
+            "Content-Type": "application/json",
+          },
+          body: JSON.stringify(
+            buildPersonaAPIBody(personaUpdateRequest, promptId, fileId)
+          ),
+        })
+      : null;
 
   return [promptResponse, updatePersonaResponse];
 }

web/src/app/admin/bots/[bot-id]/SlackChannelConfigsTable.tsx

@@ -60,21 +60,24 @@ export function SlackChannelConfigsTable({
               .slice(numToDisplay * (page - 1), numToDisplay * page)
               .map((slackChannelConfig) => {
                 return (
-                  <TableRow key={slackChannelConfig.id}>
+                  <TableRow
+                    key={slackChannelConfig.id}
+                    className="cursor-pointer hover:bg-gray-100 transition-colors"
+                    onClick={() => {
+                      window.location.href = `/admin/bots/${slackBotId}/channels/${slackChannelConfig.id}`;
+                    }}
+                  >
                     <TableCell>
                       <div className="flex gap-x-2">
-                        <Link
-                          className="cursor-pointer my-auto"
-                          href={`/admin/bots/${slackBotId}/channels/${slackChannelConfig.id}`}
-                        >
+                        <div className="my-auto">
                           <EditIcon />
-                        </Link>
+                        </div>
                         <div className="my-auto">
                           {"#" + slackChannelConfig.channel_config.channel_name}
                         </div>
                       </div>
                     </TableCell>
-                    <TableCell>
+                    <TableCell onClick={(e) => e.stopPropagation()}>
                       {slackChannelConfig.persona &&
                       !isPersonaASlackBotPersona(slackChannelConfig.persona) ? (
                         <Link
@@ -98,10 +101,11 @@ export function SlackChannelConfigsTable({
                           : "-"}
                       </div>
                     </TableCell>
-                    <TableCell>
+                    <TableCell onClick={(e) => e.stopPropagation()}>
                       <div
                         className="cursor-pointer hover:text-destructive"
-                        onClick={async () => {
+                        onClick={async (e) => {
+                          e.stopPropagation();
                           const response = await deleteSlackChannelConfig(
                             slackChannelConfig.id
                           );

web/src/app/admin/bots/[bot-id]/channels/SlackChannelConfigCreationForm.tsx

@@ -81,6 +81,11 @@ export const SlackChannelConfigCreationForm = ({
             respond_to_bots:
               existingSlackChannelConfig?.channel_config?.respond_to_bots ||
               false,
+            show_continue_in_web_ui:
+              // If we're updating, we want to keep the existing value
+              // Otherwise, we want to default to true
+              existingSlackChannelConfig?.channel_config
+                ?.show_continue_in_web_ui ?? !isUpdate,
             enable_auto_filters:
               existingSlackChannelConfig?.enable_auto_filters || false,
             respond_member_group_list:
@@ -119,6 +124,7 @@ export const SlackChannelConfigCreationForm = ({
             questionmark_prefilter_enabled: Yup.boolean().required(),
             respond_tag_only: Yup.boolean().required(),
             respond_to_bots: Yup.boolean().required(),
+            show_continue_in_web_ui: Yup.boolean().required(),
             enable_auto_filters: Yup.boolean().required(),
             respond_member_group_list: Yup.array().of(Yup.string()).required(),
             still_need_help_enabled: Yup.boolean().required(),
@@ -270,7 +276,13 @@ export const SlackChannelConfigCreationForm = ({
 
                 {showAdvancedOptions && (
                   <div className="mt-4">
-                    <div className="w-64 mb-4">
+                    <BooleanFormField
+                      name="show_continue_in_web_ui"
+                      removeIndent
+                      label="Show Continue in Web UI button"
+                      tooltip="If set, will show a button at the bottom of the response that allows the user to continue the conversation in the Danswer Web UI"
+                    />
+                    <div className="w-64 mb-4 mt-4">
                       <SelectorFormField
                         name="response_type"
                         label="Answer Type"

web/src/app/admin/bots/[bot-id]/lib.ts

@@ -15,6 +15,7 @@ interface SlackChannelConfigCreationRequest {
   questionmark_prefilter_enabled: boolean;
   respond_tag_only: boolean;
   respond_to_bots: boolean;
+  show_continue_in_web_ui: boolean;
   respond_member_group_list: string[];
   follow_up_tags?: string[];
   usePersona: boolean;
@@ -43,6 +44,7 @@ const buildRequestBodyFromCreationRequest = (
     channel_name: creationRequest.channel_name,
     respond_tag_only: creationRequest.respond_tag_only,
     respond_to_bots: creationRequest.respond_to_bots,
+    show_continue_in_web_ui: creationRequest.show_continue_in_web_ui,
     enable_auto_filters: creationRequest.enable_auto_filters,
     respond_member_group_list: creationRequest.respond_member_group_list,
     answer_filters: buildFiltersFromCreationRequest(creationRequest),

web/src/app/admin/bots/[bot-id]/page.tsx

@@ -22,7 +22,6 @@ function SlackBotEditPage({
   const unwrappedParams = use(params);
   const { popup, setPopup } = usePopup();
 
-  console.log("unwrappedParams", unwrappedParams);
   const {
     data: slackBot,
     isLoading: isSlackBotLoading,

web/src/app/admin/configuration/search/UpgradingPage.tsx

@@ -161,7 +161,7 @@ export default function UpgradingPage({
                     reindexingProgress={sortedReindexingProgress}
                   />
                 ) : (
-                  <ErrorCallout errorTitle="Failed to fetch re-indexing progress" />
+                  <ErrorCallout errorTitle="Failed to fetch reindexing progress" />
                 )}
               </>
             ) : (
@@ -171,7 +171,7 @@ export default function UpgradingPage({
                 </h3>
                 <p className="mb-4 text-text-800">
                   You&apos;re currently switching embedding models, but there
-                  are no connectors to re-index. This means the transition will
+                  are no connectors to reindex. This means the transition will
                   be quick and seamless!
                 </p>
                 <p className="text-text-600">

web/src/app/admin/connector/[ccPairId]/ModifyStatusButtonCluster.tsx

@@ -6,6 +6,8 @@ import { usePopup } from "@/components/admin/connectors/Popup";
 import { mutate } from "swr";
 import { buildCCPairInfoUrl } from "./lib";
 import { setCCPairStatus } from "@/lib/ccPair";
+import { useState } from "react";
+import { LoadingAnimation } from "@/components/Loading";
 
 export function ModifyStatusButtonCluster({
   ccPair,
@@ -13,44 +15,72 @@ export function ModifyStatusButtonCluster({
   ccPair: CCPairFullInfo;
 }) {
   const { popup, setPopup } = usePopup();
+  const [isUpdating, setIsUpdating] = useState(false);
+
+  const handleStatusChange = async (
+    newStatus: ConnectorCredentialPairStatus
+  ) => {
+    if (isUpdating) return; // Prevent double-clicks or multiple requests
+    setIsUpdating(true);
+
+    try {
+      // Call the backend to update the status
+      await setCCPairStatus(ccPair.id, newStatus, setPopup);
+
+      // Use mutate to revalidate the status on the backend
+      await mutate(buildCCPairInfoUrl(ccPair.id));
+    } catch (error) {
+      console.error("Failed to update status", error);
+    } finally {
+      // Reset local updating state and button text after mutation
+      setIsUpdating(false);
+    }
+  };
+
+  // Compute the button text based on current state and backend status
+  const buttonText =
+    ccPair.status === ConnectorCredentialPairStatus.PAUSED
+      ? "Re-Enable"
+      : "Pause";
+
+  const tooltip =
+    ccPair.status === ConnectorCredentialPairStatus.PAUSED
+      ? "Click to start indexing again!"
+      : "When paused, the connector's documents will still be visible. However, no new documents will be indexed.";
 
   return (
     <>
       {popup}
-      {ccPair.status === ConnectorCredentialPairStatus.PAUSED ? (
-        <Button
-          variant="success-reverse"
-          onClick={() =>
-            setCCPairStatus(
-              ccPair.id,
-              ConnectorCredentialPairStatus.ACTIVE,
-              setPopup,
-              () => mutate(buildCCPairInfoUrl(ccPair.id))
-            )
-          }
-          tooltip="Click to start indexing again!"
-        >
-          Re-Enable
-        </Button>
-      ) : (
-        <Button
-          variant="default"
-          onClick={() =>
-            setCCPairStatus(
-              ccPair.id,
-              ConnectorCredentialPairStatus.PAUSED,
-              setPopup,
-              () => mutate(buildCCPairInfoUrl(ccPair.id))
-            )
-          }
-          tooltip={
-            "When paused, the connectors documents will still" +
-            " be visible. However, no new documents will be indexed."
-          }
-        >
-          Pause
-        </Button>
-      )}
+      <Button
+        className="flex items-center justify-center w-auto min-w-[100px] px-4 py-2"
+        variant={
+          ccPair.status === ConnectorCredentialPairStatus.PAUSED
+            ? "success-reverse"
+            : "default"
+        }
+        disabled={isUpdating}
+        onClick={() =>
+          handleStatusChange(
+            ccPair.status === ConnectorCredentialPairStatus.PAUSED
+              ? ConnectorCredentialPairStatus.ACTIVE
+              : ConnectorCredentialPairStatus.PAUSED
+          )
+        }
+        tooltip={tooltip}
+      >
+        {isUpdating ? (
+          <LoadingAnimation
+            text={
+              ccPair.status === ConnectorCredentialPairStatus.PAUSED
+                ? "Resuming"
+                : "Pausing"
+            }
+            size="text-md"
+          />
+        ) : (
+          buttonText
+        )}
+      </Button>
     </>
   );
 }

web/src/app/admin/connector/[ccPairId]/ReIndexButton.tsx

@@ -121,7 +121,7 @@ export function ReIndexButton({
       {popup}
       <Button
         variant="success-reverse"
-        className="ml-auto"
+        className="ml-auto min-w-[100px]"
         onClick={() => {
           setReIndexPopupVisible(true);
         }}

web/src/app/admin/connector/[ccPairId]/page.tsx

@@ -25,6 +25,7 @@ import { ReIndexButton } from "./ReIndexButton";
 import { buildCCPairInfoUrl } from "./lib";
 import { CCPairFullInfo, ConnectorCredentialPairStatus } from "./types";
 import { EditableStringFieldDisplay } from "@/components/EditableStringFieldDisplay";
+import { Button } from "@/components/ui/button";
 
 // since the uploaded files are cleaned up after some period of time
 // re-indexing will not work for the file connector. Also, it would not

web/src/app/auth/login/EmailPasswordForm.tsx

@@ -15,10 +15,12 @@ export function EmailPasswordForm({
   isSignup = false,
   shouldVerify,
   referralSource,
+  nextUrl,
 }: {
   isSignup?: boolean;
   shouldVerify?: boolean;
   referralSource?: string;
+  nextUrl?: string | null;
 }) {
   const router = useRouter();
   const { popup, setPopup } = usePopup();
@@ -69,7 +71,7 @@ export function EmailPasswordForm({
               await requestEmailVerification(values.email);
               router.push("/auth/waiting-on-verification");
             } else {
-              router.push("/");
+              router.push(nextUrl ? encodeURI(nextUrl) : "/");
             }
           } else {
             setIsWorking(false);

web/src/app/auth/login/page.tsx

@@ -22,6 +22,9 @@ const Page = async (props: {
 }) => {
   const searchParams = await props.searchParams;
   const autoRedirectDisabled = searchParams?.disableAutoRedirect === "true";
+  const nextUrl = Array.isArray(searchParams?.next)
+    ? searchParams?.next[0]
+    : searchParams?.next || null;
 
   // catch cases where the backend is completely unreachable here
   // without try / catch, will just raise an exception and the page
@@ -37,10 +40,6 @@ const Page = async (props: {
     console.log(`Some fetch failed for the login page - ${e}`);
   }
 
-  const nextUrl = Array.isArray(searchParams?.next)
-    ? searchParams?.next[0]
-    : searchParams?.next || null;
-
   // simply take the user to the home page if Auth is disabled
   if (authTypeMetadata?.authType === "disabled") {
     return redirect("/");
@@ -100,12 +99,15 @@ const Page = async (props: {
               <span className="px-4 text-gray-500">or</span>
               <div className="flex-grow border-t border-gray-300"></div>
             </div>
-            <EmailPasswordForm shouldVerify={true} />
+            <EmailPasswordForm shouldVerify={true} nextUrl={nextUrl} />
 
             <div className="flex">
               <Text className="mt-4 mx-auto">
                 Don&apos;t have an account?{" "}
-                <Link href="/auth/signup" className="text-link font-medium">
+                <Link
+                  href={`/auth/signup${searchParams?.next ? `?next=${searchParams.next}` : ""}`}
+                  className="text-link font-medium"
+                >
                   Create an account
                 </Link>
               </Text>
@@ -120,11 +122,14 @@ const Page = async (props: {
                 <LoginText />
               </Title>
             </div>
-            <EmailPasswordForm />
+            <EmailPasswordForm nextUrl={nextUrl} />
             <div className="flex">
               <Text className="mt-4 mx-auto">
                 Don&apos;t have an account?{" "}
-                <Link href="/auth/signup" className="text-link font-medium">
+                <Link
+                  href={`/auth/signup${searchParams?.next ? `?next=${searchParams.next}` : ""}`}
+                  className="text-link font-medium"
+                >
                   Create an account
                 </Link>
               </Text>

web/src/app/auth/signup/page.tsx

@@ -15,7 +15,14 @@ import AuthFlowContainer from "@/components/auth/AuthFlowContainer";
 import ReferralSourceSelector from "./ReferralSourceSelector";
 import { Separator } from "@/components/ui/separator";
 
-const Page = async () => {
+const Page = async (props: {
+  searchParams?: Promise<{ [key: string]: string | string[] | undefined }>;
+}) => {
+  const searchParams = await props.searchParams;
+  const nextUrl = Array.isArray(searchParams?.next)
+    ? searchParams?.next[0]
+    : searchParams?.next || null;
+
   // catch cases where the backend is completely unreachable here
   // without try / catch, will just raise an exception and the page
   // will not render
@@ -86,12 +93,19 @@ const Page = async () => {
           <EmailPasswordForm
             isSignup
             shouldVerify={authTypeMetadata?.requiresVerification}
+            nextUrl={nextUrl}
           />
 
           <div className="flex">
             <Text className="mt-4 mx-auto">
               Already have an account?{" "}
-              <Link href="/auth/login" className="text-link font-medium">
+              <Link
+                href={{
+                  pathname: "/auth/login",
+                  query: { ...searchParams },
+                }}
+                className="text-link font-medium"
+              >
                 Log In
               </Link>
             </Text>

web/src/app/chat/ChatPage.tsx

@@ -161,6 +161,8 @@ export function ChatPage({
 
   const { user, isAdmin, isLoadingUser, refreshUser } = useUser();
 
+  const slackChatId = searchParams.get("slackChatId");
+
   const existingChatIdRaw = searchParams.get("chatId");
   const [sendOnLoad, setSendOnLoad] = useState<string | null>(
     searchParams.get(SEARCH_PARAM_NAMES.SEND_ON_LOAD)
@@ -259,7 +261,7 @@ export function ChatPage({
     refreshRecentAssistants,
   } = useAssistants();
 
-  const liveAssistant: Persona | undefined =
+  const liveAssistant =
     alternativeAssistant ||
     selectedAssistant ||
     recentAssistants[0] ||
@@ -269,7 +271,6 @@ export function ChatPage({
   const noAssistants = liveAssistant == null || liveAssistant == undefined;
   // always set the model override for the chat session, when an assistant, llm provider, or user preference exists
   useEffect(() => {
-    if (noAssistants) return;
     const personaDefault = getLLMProviderOverrideForPersona(
       liveAssistant,
       llmProviders
@@ -404,6 +405,7 @@ export function ChatPage({
         }
         return;
       }
+      setIsReady(true);
       const shouldScrollToBottom =
         visibleRange.get(existingChatSessionId) === undefined ||
         visibleRange.get(existingChatSessionId)?.end == 0;
@@ -469,9 +471,12 @@ export function ChatPage({
         });
         // force re-name if the chat session doesn't have one
         if (!chatSession.description) {
-          await nameChatSession(existingChatSessionId, seededMessage);
+          await nameChatSession(existingChatSessionId);
           refreshChatSessions();
         }
+      } else if (newMessageHistory.length === 2 && !chatSession.description) {
+        await nameChatSession(existingChatSessionId);
+        refreshChatSessions();
       }
     }
 
@@ -754,7 +759,7 @@ export function ChatPage({
   useEffect(() => {
     async function fetchMaxTokens() {
       const response = await fetch(
-        `/api/chat/max-selected-document-tokens?persona_id=${liveAssistant?.id}`
+        `/api/chat/max-selected-document-tokens?persona_id=${liveAssistant.id}`
       );
       if (response.ok) {
         const maxTokens = (await response.json()).max_tokens as number;
@@ -1429,7 +1434,7 @@ export function ChatPage({
 
       if (!searchParamBasedChatSessionName) {
         await new Promise((resolve) => setTimeout(resolve, 200));
-        await nameChatSession(currChatSessionId, currMessage);
+        await nameChatSession(currChatSessionId);
         refreshChatSessions();
       }
 
@@ -1810,23 +1815,54 @@ export function ChatPage({
       });
     };
   }
-  if (noAssistants)
-    return (
-      <>
-        <HealthCheckBanner />
-        <NoAssistantModal isAdmin={isAdmin} />
-      </>
-    );
+
+  useEffect(() => {
+    const handleSlackChatRedirect = async () => {
+      if (!slackChatId) return;
+
+      // Set isReady to false before starting retrieval to display loading text
+      setIsReady(false);
+
+      try {
+        const response = await fetch("/api/chat/seed-chat-session-from-slack", {
+          method: "POST",
+          headers: {
+            "Content-Type": "application/json",
+          },
+          body: JSON.stringify({
+            chat_session_id: slackChatId,
+          }),
+        });
+
+        if (!response.ok) {
+          throw new Error("Failed to seed chat from Slack");
+        }
+
+        const data = await response.json();
+        router.push(data.redirect_url);
+      } catch (error) {
+        console.error("Error seeding chat from Slack:", error);
+        setPopup({
+          message: "Failed to load chat from Slack",
+          type: "error",
+        });
+      }
+    };
+
+    handleSlackChatRedirect();
+  }, [searchParams, router]);
 
   return (
     <>
       <HealthCheckBanner />
 
-      {showApiKeyModal && !shouldShowWelcomeModal && (
+      {showApiKeyModal && !shouldShowWelcomeModal ? (
         <ApiKeyModal
           hide={() => setShowApiKeyModal(false)}
           setPopup={setPopup}
         />
+      ) : (
+        noAssistants && <NoAssistantModal isAdmin={isAdmin} />
       )}
 
       {/* ChatPopup is a custom popup that displays a admin-specified message on initial user visit. 

web/src/app/chat/lib.tsx

@@ -203,7 +203,7 @@ export async function* sendMessage({
   yield* handleSSEStream<PacketType>(response);
 }
 
-export async function nameChatSession(chatSessionId: string, message: string) {
+export async function nameChatSession(chatSessionId: string) {
   const response = await fetch("/api/chat/rename-chat-session", {
     method: "PUT",
     headers: {
@@ -212,7 +212,6 @@ export async function nameChatSession(chatSessionId: string, message: string) {
     body: JSON.stringify({
       chat_session_id: chatSessionId,
       name: null,
-      first_message: message,
     }),
   });
   return response;
@@ -263,7 +262,6 @@ export async function renameChatSession(
     body: JSON.stringify({
       chat_session_id: chatSessionId,
       name: newName,
-      first_message: null,
     }),
   });
   return response;

web/src/app/chat/modal/FeedbackModal.tsx

@@ -5,15 +5,19 @@ import { FeedbackType } from "../types";
 import { Modal } from "@/components/Modal";
 import { FilledLikeIcon } from "@/components/icons/icons";
 
-const predefinedPositiveFeedbackOptions =
-  process.env.NEXT_PUBLIC_POSITIVE_PREDEFINED_FEEDBACK_OPTIONS?.split(",") ||
-  [];
-const predefinedNegativeFeedbackOptions =
-  process.env.NEXT_PUBLIC_NEGATIVE_PREDEFINED_FEEDBACK_OPTIONS?.split(",") || [
-    "Retrieved documents were not relevant",
-    "AI misread the documents",
-    "Cited source had incorrect information",
-  ];
+const predefinedPositiveFeedbackOptions = process.env
+  .NEXT_PUBLIC_POSITIVE_PREDEFINED_FEEDBACK_OPTIONS
+  ? process.env.NEXT_PUBLIC_POSITIVE_PREDEFINED_FEEDBACK_OPTIONS.split(",")
+  : [];
+
+const predefinedNegativeFeedbackOptions = process.env
+  .NEXT_PUBLIC_NEGATIVE_PREDEFINED_FEEDBACK_OPTIONS
+  ? process.env.NEXT_PUBLIC_NEGATIVE_PREDEFINED_FEEDBACK_OPTIONS.split(",")
+  : [
+      "Retrieved documents were not relevant",
+      "AI misread the documents",
+      "Cited source had incorrect information",
+    ];
 
 interface FeedbackModalProps {
   feedbackType: FeedbackType;

web/src/app/ee/admin/performance/usage/QueryPerformanceChart.tsx

@@ -62,6 +62,7 @@ export function QueryPerformanceChart({
     chart = (
       <AreaChartDisplay
         className="mt-4"
+        stacked={false}
         data={dateRange.map((dateStr) => {
           const queryAnalyticsForDate = dateToQueryAnalytics.get(dateStr);
           const userAnalyticsForDate = dateToUserAnalytics.get(dateStr);

web/src/components/ui/areaChart.tsx

@@ -24,18 +24,12 @@ interface AreaChartProps {
   categories?: string[];
   index?: string;
   colors?: string[];
-  startEndOnly?: boolean;
   showXAxis?: boolean;
   showYAxis?: boolean;
   yAxisWidth?: number;
   showAnimation?: boolean;
   showTooltip?: boolean;
-  showLegend?: boolean;
   showGridLines?: boolean;
-  showGradient?: boolean;
-  autoMinValue?: boolean;
-  minValue?: number;
-  maxValue?: number;
   connectNulls?: boolean;
   allowDecimals?: boolean;
   className?: string;
@@ -43,6 +37,7 @@ interface AreaChartProps {
   description?: string;
   xAxisFormatter?: (value: any) => string;
   yAxisFormatter?: (value: any) => string;
+  stacked?: boolean;
 }
 
 export function AreaChartDisplay({
@@ -50,18 +45,12 @@ export function AreaChartDisplay({
   categories = [],
   index,
   colors = ["indigo", "fuchsia"],
-  startEndOnly = false,
   showXAxis = true,
   showYAxis = true,
   yAxisWidth = 56,
   showAnimation = true,
   showTooltip = true,
-  showLegend = false,
   showGridLines = true,
-  showGradient = true,
-  autoMinValue = false,
-  minValue,
-  maxValue,
   connectNulls = false,
   allowDecimals = true,
   className,
@@ -69,6 +58,7 @@ export function AreaChartDisplay({
   description,
   xAxisFormatter = (dateStr: string) => dateStr,
   yAxisFormatter = (number: number) => number.toString(),
+  stacked = false,
 }: AreaChartProps) {
   return (
     <Card className={className}>
@@ -113,7 +103,7 @@ export function AreaChartDisplay({
                   key={category}
                   type="monotone"
                   dataKey={category}
-                  stackId="1"
+                  stackId={stacked ? "1" : category}
                   stroke={colors[ind % colors.length]}
                   fill={colors[ind % colors.length]}
                   fillOpacity={0.3}

web/src/lib/types.ts

@@ -208,6 +208,7 @@ export interface ChannelConfig {
   channel_name: string;
   respond_tag_only?: boolean;
   respond_to_bots?: boolean;
+  show_continue_in_web_ui?: boolean;
   respond_member_group_list?: string[];
   answer_filters?: AnswerFilterOption[];
   follow_up_tags?: string[];

web/src/lib/userSS.ts

@@ -62,12 +62,17 @@ const getOIDCAuthUrlSS = async (nextUrl: string | null): Promise<string> => {
   return data.authorization_url;
 };
 
-const getGoogleOAuthUrlSS = async (): Promise<string> => {
-  const res = await fetch(buildUrl(`/auth/oauth/authorize`), {
-    headers: {
-      cookie: processCookies(await cookies()),
-    },
-  });
+const getGoogleOAuthUrlSS = async (nextUrl: string | null): Promise<string> => {
+  const res = await fetch(
+    buildUrl(
+      `/auth/oauth/authorize${nextUrl ? `?next=${encodeURIComponent(nextUrl)}` : ""}`
+    ),
+    {
+      headers: {
+        cookie: processCookies(await cookies()),
+      },
+    }
+  );
   if (!res.ok) {
     throw new Error("Failed to fetch data");
   }
@@ -76,8 +81,12 @@ const getGoogleOAuthUrlSS = async (): Promise<string> => {
   return data.authorization_url;
 };
 
-const getSAMLAuthUrlSS = async (): Promise<string> => {
-  const res = await fetch(buildUrl("/auth/saml/authorize"));
+const getSAMLAuthUrlSS = async (nextUrl: string | null): Promise<string> => {
+  const res = await fetch(
+    buildUrl(
+      `/auth/saml/authorize${nextUrl ? `?next=${encodeURIComponent(nextUrl)}` : ""}`
+    )
+  );
   if (!res.ok) {
     throw new Error("Failed to fetch data");
   }
@@ -97,13 +106,13 @@ export const getAuthUrlSS = async (
     case "basic":
       return "";
     case "google_oauth": {
-      return await getGoogleOAuthUrlSS();
+      return await getGoogleOAuthUrlSS(nextUrl);
     }
     case "cloud": {
-      return await getGoogleOAuthUrlSS();
+      return await getGoogleOAuthUrlSS(nextUrl);
     }
     case "saml": {
-      return await getSAMLAuthUrlSS();
+      return await getSAMLAuthUrlSS(nextUrl);
     }
     case "oidc": {
       return await getOIDCAuthUrlSS(nextUrl);