quick nit

.github/workflows/pr-integration-tests.yml

@@ -145,7 +145,7 @@ jobs:
         run: |
           cd deployment/docker_compose
           docker compose -f docker-compose.multitenant-dev.yml -p onyx-stack down -v
-
+      
       # NOTE: Use pre-ping/null pool to reduce flakiness due to dropped connections
       - name: Start Docker containers
         run: |
@@ -157,7 +157,6 @@ jobs:
           REQUIRE_EMAIL_VERIFICATION=false \
           DISABLE_TELEMETRY=true \
           IMAGE_TAG=test \
-          INTEGRATION_TESTS_MODE=true \
           docker compose -f docker-compose.dev.yml -p onyx-stack up -d
         id: start_docker
 
@@ -200,7 +199,7 @@ jobs:
           cd backend/tests/integration/mock_services
           docker compose -f docker-compose.mock-it-services.yml \
             -p mock-it-services-stack up -d
-
+      
       # NOTE: Use pre-ping/null to reduce flakiness due to dropped connections
       - name: Run Standard Integration Tests
         run: |

.github/workflows/pr-python-model-tests.yml

@@ -1,16 +1,10 @@
-name: Model Server Tests
+name: Connector Tests
 
 on:
   schedule:
     # This cron expression runs the job daily at 16:00 UTC (9am PT)
     - cron: "0 16 * * *"
-  workflow_dispatch:
-    inputs:
-      branch:
-        description: 'Branch to run the workflow on'
-        required: false
-        default: 'main'
-        
+
 env:
   # Bedrock
   AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
@@ -32,23 +26,6 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4
 
-      - name: Login to Docker Hub
-        uses: docker/login-action@v3
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_TOKEN }}
-
-      # tag every docker image with "test" so that we can spin up the correct set
-      # of images during testing
-
-      # We don't need to build the Web Docker image since it's not yet used
-      # in the integration tests. We have a separate action to verify that it builds
-      # successfully.
-      - name: Pull Model Server Docker image
-        run: |
-          docker pull onyxdotapp/onyx-model-server:latest
-          docker tag onyxdotapp/onyx-model-server:latest onyxdotapp/onyx-model-server:test
-          
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
@@ -64,49 +41,6 @@ jobs:
           pip install --retries 5 --timeout 30 -r backend/requirements/default.txt
           pip install --retries 5 --timeout 30 -r backend/requirements/dev.txt
 
-      - name: Start Docker containers
-        run: |
-          cd deployment/docker_compose
-          ENABLE_PAID_ENTERPRISE_EDITION_FEATURES=true \
-          AUTH_TYPE=basic \
-          REQUIRE_EMAIL_VERIFICATION=false \
-          DISABLE_TELEMETRY=true \
-          IMAGE_TAG=test \
-          docker compose -f docker-compose.dev.yml -p onyx-stack up -d indexing_model_server
-        id: start_docker
-
-      - name: Wait for service to be ready
-        run: |
-          echo "Starting wait-for-service script..."
-
-          start_time=$(date +%s)
-          timeout=300  # 5 minutes in seconds
-
-          while true; do
-            current_time=$(date +%s)
-            elapsed_time=$((current_time - start_time))
-            
-            if [ $elapsed_time -ge $timeout ]; then
-              echo "Timeout reached. Service did not become ready in 5 minutes."
-              exit 1
-            fi
-            
-            # Use curl with error handling to ignore specific exit code 56
-            response=$(curl -s -o /dev/null -w "%{http_code}" http://localhost:9000/api/health || echo "curl_error")
-            
-            if [ "$response" = "200" ]; then
-              echo "Service is ready!"
-              break
-            elif [ "$response" = "curl_error" ]; then
-              echo "Curl encountered an error, possibly exit code 56. Continuing to retry..."
-            else
-              echo "Service not ready yet (HTTP status $response). Retrying in 5 seconds..."
-            fi
-            
-            sleep 5
-          done
-          echo "Finished waiting for service."
-          
       - name: Run Tests
         shell: script -q -e -c "bash --noprofile --norc -eo pipefail {0}"
         run: |
@@ -122,10 +56,3 @@ jobs:
             -H 'Content-type: application/json' \
             --data '{"text":"Scheduled Model Tests failed! Check the run at: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"}' \
             $SLACK_WEBHOOK
-            
-      - name: Stop Docker containers
-        if: always()
-        run: |
-          cd deployment/docker_compose
-          docker compose -f docker-compose.dev.yml -p onyx-stack down -v
-          

README.md

@@ -26,12 +26,12 @@
 
 <strong>[Onyx](https://www.onyx.app/)</strong> (formerly Danswer) is the AI platform connected to your company's docs, apps, and people.
 Onyx provides a feature rich Chat interface and plugs into any LLM of your choice.
-Keep knowledge and access controls sync-ed across over 40 connectors like Google Drive, Slack, Confluence, Salesforce, etc.
-Create custom AI agents with unique prompts, knowledge, and actions that the agents can take.
+There are over 40 supported connectors such as Google Drive, Slack, Confluence, Salesforce, etc. which keep knowledge and permissions up to date.
+Create custom AI agents with unique prompts, knowledge, and actions the agents can take.
 Onyx can be deployed securely anywhere and for any scale - on a laptop, on-premise, or to cloud.
 
 
-<h3>Feature Highlights</h3>
+<h3>Feature Showcase</h3>
 
 **Deep research over your team's knowledge:**
 
@@ -63,21 +63,22 @@ We also have built-in support for high-availability/scalable deployment on Kuber
 References [here](https://github.com/onyx-dot-app/onyx/tree/main/deployment).
 
 
-## 🔍 Other Notable Benefits of Onyx
-- Custom deep learning models for indexing and inference time, only through Onyx + learning from user feedback.
-- Flexible security features like SSO (OIDC/SAML/OAuth2), RBAC, encryption of credentials, etc.
-- Knowledge curation features like document-sets, query history, usage analytics, etc.
-- Scalable deployment options tested up to many tens of thousands users and hundreds of millions of documents.
-
-
 ## 🚧 Roadmap
-- New methods in information retrieval (StructRAG, LightGraphRAG, etc.)
+- Extensions to the Chrome Plugin
+- Latest methods in information retrieval (StructRAG, LightGraphRAG, etc.)
 - Personalized Search
 - Organizational understanding and ability to locate and suggest experts from your team.
 - Code Search
 - SQL and Structured Query Language
 
 
+## 🔍 Other Notable Benefits of Onyx
+- Custom deep learning models only through Onyx + learn from user feedback.
+- Flexible security features like SSO (OIDC/SAML/OAuth2), RBAC, encryption of credentials, etc.
+- Knowledge curation features like document-sets, query history, usage analytics, etc.
+- Scalable deployment options tested up to many tens of thousands users and hundreds of millions of documents.
+
+
 ## 🔌 Connectors
 Keep knowledge and access up to sync across 40+ connectors:
 

backend/alembic/versions/f13db29f3101_add_composite_index_for_last_modified_.py

@@ -1,27 +0,0 @@
-"""Add composite index for last_modified and last_synced to document
-
-Revision ID: f13db29f3101
-Revises: b388730a2899
-Create Date: 2025-02-18 22:48:11.511389
-
-"""
-from alembic import op
-
-# revision identifiers, used by Alembic.
-revision = "f13db29f3101"
-down_revision = "acaab4ef4507"
-branch_labels: str | None = None
-depends_on: str | None = None
-
-
-def upgrade() -> None:
-    op.create_index(
-        "ix_document_sync_status",
-        "document",
-        ["last_modified", "last_synced"],
-        unique=False,
-    )
-
-
-def downgrade() -> None:
-    op.drop_index("ix_document_sync_status", table_name="document")

backend/ee/onyx/db/connector_credential_pair.py

@@ -4,7 +4,6 @@ from sqlalchemy.orm import Session
 from onyx.configs.constants import DocumentSource
 from onyx.db.connector_credential_pair import get_connector_credential_pair
 from onyx.db.enums import AccessType
-from onyx.db.enums import ConnectorCredentialPairStatus
 from onyx.db.models import Connector
 from onyx.db.models import ConnectorCredentialPair
 from onyx.db.models import UserGroup__ConnectorCredentialPair
@@ -36,11 +35,10 @@ def _delete_connector_credential_pair_user_groups_relationship__no_commit(
 def get_cc_pairs_by_source(
     db_session: Session,
     source_type: DocumentSource,
-    access_type: AccessType | None = None,
-    status: ConnectorCredentialPairStatus | None = None,
+    only_sync: bool,
 ) -> list[ConnectorCredentialPair]:
     """
-    Get all cc_pairs for a given source type with optional filtering by access_type and status
+    Get all cc_pairs for a given source type (and optionally only sync)
     result is sorted by cc_pair id
     """
     query = (
@@ -50,11 +48,8 @@ def get_cc_pairs_by_source(
         .order_by(ConnectorCredentialPair.id)
     )
 
-    if access_type is not None:
-        query = query.filter(ConnectorCredentialPair.access_type == access_type)
-
-    if status is not None:
-        query = query.filter(ConnectorCredentialPair.status == status)
+    if only_sync:
+        query = query.filter(ConnectorCredentialPair.access_type == AccessType.SYNC)
 
     cc_pairs = query.all()
     return cc_pairs

backend/ee/onyx/external_permissions/google_drive/doc_sync.py

@@ -62,14 +62,12 @@ def _fetch_permissions_for_permission_ids(
         user_email=(owner_email or google_drive_connector.primary_admin_email),
     )
 
-    # We continue on 404 or 403 because the document may not exist or the user may not have access to it
     fetched_permissions = execute_paginated_retrieval(
         retrieval_function=drive_service.permissions().list,
         list_key="permissions",
         fileId=doc_id,
         fields="permissions(id, emailAddress, type, domain)",
         supportsAllDrives=True,
-        continue_on_404_or_403=True,
     )
 
     permissions_for_doc_id = []
@@ -106,13 +104,7 @@ def _get_permissions_from_slim_doc(
     user_emails: set[str] = set()
     group_emails: set[str] = set()
     public = False
-    skipped_permissions = 0
-
     for permission in permissions_list:
-        if not permission:
-            skipped_permissions += 1
-            continue
-
         permission_type = permission["type"]
         if permission_type == "user":
             user_emails.add(permission["emailAddress"])
@@ -129,11 +121,6 @@ def _get_permissions_from_slim_doc(
         elif permission_type == "anyone":
             public = True
 
-    if skipped_permissions > 0:
-        logger.warning(
-            f"Skipped {skipped_permissions} permissions of {len(permissions_list)} for document {slim_doc.id}"
-        )
-
     drive_id = permission_info.get("drive_id")
     group_ids = group_emails | ({drive_id} if drive_id is not None else set())
 

backend/ee/onyx/server/query_and_chat/token_limit.py

@@ -13,7 +13,7 @@ from sqlalchemy import select
 from sqlalchemy.orm import Session
 
 from onyx.db.api_key import is_api_key_email_address
-from onyx.db.engine import get_session_with_current_tenant
+from onyx.db.engine import get_session_with_tenant
 from onyx.db.models import ChatMessage
 from onyx.db.models import ChatSession
 from onyx.db.models import TokenRateLimit
@@ -28,21 +28,21 @@ from onyx.server.query_and_chat.token_limit import _user_is_rate_limited_by_glob
 from onyx.utils.threadpool_concurrency import run_functions_tuples_in_parallel
 
 
-def _check_token_rate_limits(user: User | None) -> None:
+def _check_token_rate_limits(user: User | None, tenant_id: str) -> None:
     if user is None:
         # Unauthenticated users are only rate limited by global settings
-        _user_is_rate_limited_by_global()
+        _user_is_rate_limited_by_global(tenant_id)
 
     elif is_api_key_email_address(user.email):
         # API keys are only rate limited by global settings
-        _user_is_rate_limited_by_global()
+        _user_is_rate_limited_by_global(tenant_id)
 
     else:
         run_functions_tuples_in_parallel(
             [
-                (_user_is_rate_limited, (user.id,)),
-                (_user_is_rate_limited_by_group, (user.id,)),
-                (_user_is_rate_limited_by_global, ()),
+                (_user_is_rate_limited, (user.id, tenant_id)),
+                (_user_is_rate_limited_by_group, (user.id, tenant_id)),
+                (_user_is_rate_limited_by_global, (tenant_id,)),
             ]
         )
 
@@ -52,8 +52,8 @@ User rate limits
 """
 
 
-def _user_is_rate_limited(user_id: UUID) -> None:
-    with get_session_with_current_tenant() as db_session:
+def _user_is_rate_limited(user_id: UUID, tenant_id: str) -> None:
+    with get_session_with_tenant(tenant_id=tenant_id) as db_session:
         user_rate_limits = fetch_all_user_token_rate_limits(
             db_session=db_session, enabled_only=True, ordered=False
         )
@@ -93,8 +93,8 @@ User Group rate limits
 """
 
 
-def _user_is_rate_limited_by_group(user_id: UUID) -> None:
-    with get_session_with_current_tenant() as db_session:
+def _user_is_rate_limited_by_group(user_id: UUID, tenant_id: str | None) -> None:
+    with get_session_with_tenant(tenant_id=tenant_id) as db_session:
         group_rate_limits = _fetch_all_user_group_rate_limits(user_id, db_session)
 
         if group_rate_limits:

backend/model_server/encoders.py

@@ -98,17 +98,12 @@ class CloudEmbedding:
             return final_embeddings
         except Exception as e:
             error_string = (
-                f"Exception embedding text with OpenAI - {type(e)}: "
-                f"Model: {model} "
-                f"Provider: {self.provider} "
-                f"Exception: {e}"
+                f"Error embedding text with OpenAI: {str(e)} \n"
+                f"Model: {model} \n"
+                f"Provider: {self.provider} \n"
+                f"Texts: {texts}"
             )
             logger.error(error_string)
-
-            # only log text when it's not an authentication error.
-            if not isinstance(e, openai.AuthenticationError):
-                logger.debug(f"Exception texts: {texts}")
-
             raise RuntimeError(error_string)
 
     async def _embed_cohere(

backend/onyx/auth/email_utils.py

@@ -10,7 +10,6 @@ from onyx.configs.app_configs import SMTP_PORT
 from onyx.configs.app_configs import SMTP_SERVER
 from onyx.configs.app_configs import SMTP_USER
 from onyx.configs.app_configs import WEB_DOMAIN
-from onyx.configs.constants import AuthType
 from onyx.configs.constants import TENANT_ID_COOKIE_NAME
 from onyx.db.models import User
 
@@ -188,51 +187,23 @@ def send_subscription_cancellation_email(user_email: str) -> None:
     send_email(user_email, subject, html_content, text_content)
 
 
-def send_user_email_invite(
-    user_email: str, current_user: User, auth_type: AuthType
-) -> None:
+def send_user_email_invite(user_email: str, current_user: User) -> None:
     subject = "Invitation to Join Onyx Organization"
     heading = "You've Been Invited!"
-
-    # the exact action taken by the user, and thus the message, depends on the auth type
-    message = f"<p>You have been invited by {current_user.email} to join an organization on Onyx.</p>"
-    if auth_type == AuthType.CLOUD:
-        message += (
-            "<p>To join the organization, please click the button below to set a password "
-            "or login with Google and complete your registration.</p>"
-        )
-    elif auth_type == AuthType.BASIC:
-        message += (
-            "<p>To join the organization, please click the button below to set a password "
-            "and complete your registration.</p>"
-        )
-    elif auth_type == AuthType.GOOGLE_OAUTH:
-        message += (
-            "<p>To join the organization, please click the button below to login with Google "
-            "and complete your registration.</p>"
-        )
-    elif auth_type == AuthType.OIDC or auth_type == AuthType.SAML:
-        message += (
-            "<p>To join the organization, please click the button below to"
-            " complete your registration.</p>"
-        )
-    else:
-        raise ValueError(f"Invalid auth type: {auth_type}")
-
+    message = (
+        f"<p>You have been invited by {current_user.email} to join an organization on Onyx.</p>"
+        "<p>To join the organization, please click the button below to set a password "
+        "or login with Google and complete your registration.</p>"
+    )
     cta_text = "Join Organization"
     cta_link = f"{WEB_DOMAIN}/auth/signup?email={user_email}"
     html_content = build_html_email(heading, message, cta_text, cta_link)
-
-    # text content is the fallback for clients that don't support HTML
-    # not as critical, so not having special cases for each auth type
     text_content = (
         f"You have been invited by {current_user.email} to join an organization on Onyx.\n"
         "To join the organization, please visit the following link:\n"
         f"{WEB_DOMAIN}/auth/signup?email={user_email}\n"
+        "You'll be asked to set a password or login with Google to complete your registration."
     )
-    if auth_type == AuthType.CLOUD:
-        text_content += "You'll be asked to set a password or login with Google to complete your registration."
-
     send_email(user_email, subject, html_content, text_content)
 
 

backend/onyx/auth/users.py

@@ -420,8 +420,7 @@ class UserManager(UUIDIDMixin, BaseUserManager[User, uuid.UUID]):
             except exceptions.UserNotExists:
                 try:
                     # Attempt to get user by email
-                    user = cast(User, await self.user_db.get_by_email(account_email))
-
+                    user = await self.get_by_email(account_email)
                     if not associate_by_email:
                         raise exceptions.UserAlreadyExists()
 

backend/onyx/background/celery/apps/app_base.py

@@ -140,7 +140,7 @@ def on_task_postrun(
         f"{f'for tenant_id={tenant_id}' if tenant_id else ''}"
     )
 
-    r = get_redis_client(tenant_id=tenant_id)
+    r = get_redis_client()
 
     if task_id.startswith(RedisConnectorCredentialPair.PREFIX):
         r.srem(RedisConnectorCredentialPair.get_taskset_key(), task_id)

backend/onyx/background/celery/celery_redis.py

@@ -92,8 +92,7 @@ def celery_find_task(task_id: str, queue: str, r: Redis) -> int:
 
 
 def celery_get_queued_task_ids(queue: str, r: Redis) -> set[str]:
-    """This is a redis specific way to build a list of tasks in a queue and return them
-    as a set.
+    """This is a redis specific way to build a list of tasks in a queue.
 
     This helps us read the queue once and then efficiently look for missing tasks
     in the queue.

backend/onyx/background/celery/tasks/connector_deletion/tasks.py

@@ -8,21 +8,16 @@ from celery import Celery
 from celery import shared_task
 from celery import Task
 from celery.exceptions import SoftTimeLimitExceeded
-from pydantic import ValidationError
 from redis import Redis
 from redis.lock import Lock as RedisLock
 from sqlalchemy.orm import Session
 
 from onyx.background.celery.apps.app_base import task_logger
-from onyx.background.celery.celery_redis import celery_get_queue_length
-from onyx.background.celery.celery_redis import celery_get_queued_task_ids
 from onyx.configs.app_configs import JOB_TIMEOUT
 from onyx.configs.constants import CELERY_GENERIC_BEAT_LOCK_TIMEOUT
-from onyx.configs.constants import OnyxCeleryQueues
 from onyx.configs.constants import OnyxCeleryTask
 from onyx.configs.constants import OnyxRedisConstants
 from onyx.configs.constants import OnyxRedisLocks
-from onyx.configs.constants import OnyxRedisSignals
 from onyx.db.connector import fetch_connector_by_id
 from onyx.db.connector_credential_pair import add_deletion_failure_message
 from onyx.db.connector_credential_pair import (
@@ -69,7 +64,6 @@ def check_for_connector_deletion_task(
 ) -> bool | None:
     r = get_redis_client()
     r_replica = get_redis_replica_client()
-    r_celery: Redis = self.app.broker_connection().channel().client  # type: ignore
 
     lock_beat: RedisLock = r.lock(
         OnyxRedisLocks.CHECK_CONNECTOR_DELETION_BEAT_LOCK,
@@ -81,21 +75,6 @@ def check_for_connector_deletion_task(
         return None
 
     try:
-        # we want to run this less frequently than the overall task
-        lock_beat.reacquire()
-        if not r.exists(OnyxRedisSignals.BLOCK_VALIDATE_CONNECTOR_DELETION_FENCES):
-            # clear fences that don't have associated celery tasks in progress
-            try:
-                validate_connector_deletion_fences(
-                    tenant_id, r, r_replica, r_celery, lock_beat
-                )
-            except Exception:
-                task_logger.exception(
-                    "Exception while validating connector deletion fences"
-                )
-
-            r.set(OnyxRedisSignals.BLOCK_VALIDATE_CONNECTOR_DELETION_FENCES, 1, ex=300)
-
         # collect cc_pair_ids
         cc_pair_ids: list[int] = []
         with get_session_with_current_tenant() as db_session:
@@ -190,7 +169,6 @@ def try_generate_document_cc_pair_cleanup_tasks(
         return None
 
     # set a basic fence to start
-    redis_connector.delete.set_active()
     fence_payload = RedisConnectorDeletePayload(
         num_tasks=None,
         submitted=datetime.now(timezone.utc),
@@ -423,171 +401,3 @@ def monitor_connector_deletion_taskset(
     )
 
     redis_connector.delete.reset()
-
-
-def validate_connector_deletion_fences(
-    tenant_id: str | None,
-    r: Redis,
-    r_replica: Redis,
-    r_celery: Redis,
-    lock_beat: RedisLock,
-) -> None:
-    # building lookup table can be expensive, so we won't bother
-    # validating until the queue is small
-    CONNECTION_DELETION_VALIDATION_MAX_QUEUE_LEN = 1024
-
-    queue_len = celery_get_queue_length(OnyxCeleryQueues.CONNECTOR_DELETION, r_celery)
-    if queue_len > CONNECTION_DELETION_VALIDATION_MAX_QUEUE_LEN:
-        return
-
-    queued_upsert_tasks = celery_get_queued_task_ids(
-        OnyxCeleryQueues.CONNECTOR_DELETION, r_celery
-    )
-
-    # validate all existing connector deletion jobs
-    lock_beat.reacquire()
-    keys = cast(set[Any], r_replica.smembers(OnyxRedisConstants.ACTIVE_FENCES))
-    for key in keys:
-        key_bytes = cast(bytes, key)
-        key_str = key_bytes.decode("utf-8")
-        if not key_str.startswith(RedisConnectorDelete.FENCE_PREFIX):
-            continue
-
-        validate_connector_deletion_fence(
-            tenant_id,
-            key_bytes,
-            queued_upsert_tasks,
-            r,
-        )
-
-        lock_beat.reacquire()
-
-    return
-
-
-def validate_connector_deletion_fence(
-    tenant_id: str | None,
-    key_bytes: bytes,
-    queued_tasks: set[str],
-    r: Redis,
-) -> None:
-    """Checks for the error condition where an indexing fence is set but the associated celery tasks don't exist.
-    This can happen if the indexing worker hard crashes or is terminated.
-    Being in this bad state means the fence will never clear without help, so this function
-    gives the help.
-
-    How this works:
-    1. This function renews the active signal with a 5 minute TTL under the following conditions
-    1.2. When the task is seen in the redis queue
-    1.3. When the task is seen in the reserved / prefetched list
-
-    2. Externally, the active signal is renewed when:
-    2.1. The fence is created
-    2.2. The indexing watchdog checks the spawned task.
-
-    3. The TTL allows us to get through the transitions on fence startup
-    and when the task starts executing.
-
-    More TTL clarification: it is seemingly impossible to exactly query Celery for
-    whether a task is in the queue or currently executing.
-    1. An unknown task id is always returned as state PENDING.
-    2. Redis can be inspected for the task id, but the task id is gone between the time a worker receives the task
-    and the time it actually starts on the worker.
-
-    queued_tasks: the celery queue of lightweight permission sync tasks
-    reserved_tasks: prefetched tasks for sync task generator
-    """
-    # if the fence doesn't exist, there's nothing to do
-    fence_key = key_bytes.decode("utf-8")
-    cc_pair_id_str = RedisConnector.get_id_from_fence_key(fence_key)
-    if cc_pair_id_str is None:
-        task_logger.warning(
-            f"validate_connector_deletion_fence - could not parse id from {fence_key}"
-        )
-        return
-
-    cc_pair_id = int(cc_pair_id_str)
-    # parse out metadata and initialize the helper class with it
-    redis_connector = RedisConnector(tenant_id, int(cc_pair_id))
-
-    # check to see if the fence/payload exists
-    if not redis_connector.delete.fenced:
-        return
-
-    # in the cloud, the payload format may have changed ...
-    # it's a little sloppy, but just reset the fence for now if that happens
-    # TODO: add intentional cleanup/abort logic
-    try:
-        payload = redis_connector.delete.payload
-    except ValidationError:
-        task_logger.exception(
-            "validate_connector_deletion_fence - "
-            "Resetting fence because fence schema is out of date: "
-            f"cc_pair={cc_pair_id} "
-            f"fence={fence_key}"
-        )
-
-        redis_connector.delete.reset()
-        return
-
-    if not payload:
-        return
-
-    # OK, there's actually something for us to validate
-
-    # look up every task in the current taskset in the celery queue
-    # every entry in the taskset should have an associated entry in the celery task queue
-    # because we get the celery tasks first, the entries in our own permissions taskset
-    # should be roughly a subset of the tasks in celery
-
-    # this check isn't very exact, but should be sufficient over a period of time
-    # A single successful check over some number of attempts is sufficient.
-
-    # TODO: if the number of tasks in celery is much lower than than the taskset length
-    # we might be able to shortcut the lookup since by definition some of the tasks
-    # must not exist in celery.
-
-    tasks_scanned = 0
-    tasks_not_in_celery = 0  # a non-zero number after completing our check is bad
-
-    for member in r.sscan_iter(redis_connector.delete.taskset_key):
-        tasks_scanned += 1
-
-        member_bytes = cast(bytes, member)
-        member_str = member_bytes.decode("utf-8")
-        if member_str in queued_tasks:
-            continue
-
-        tasks_not_in_celery += 1
-
-    task_logger.info(
-        "validate_connector_deletion_fence task check: "
-        f"tasks_scanned={tasks_scanned} tasks_not_in_celery={tasks_not_in_celery}"
-    )
-
-    # we're active if there are still tasks to run and those tasks all exist in celery
-    if tasks_scanned > 0 and tasks_not_in_celery == 0:
-        redis_connector.delete.set_active()
-        return
-
-    # we may want to enable this check if using the active task list somehow isn't good enough
-    # if redis_connector_index.generator_locked():
-    #     logger.info(f"{payload.celery_task_id} is currently executing.")
-
-    # if we get here, we didn't find any direct indication that the associated celery tasks exist,
-    # but they still might be there due to gaps in our ability to check states during transitions
-    # Checking the active signal safeguards us against these transition periods
-    # (which has a duration that allows us to bridge those gaps)
-    if redis_connector.delete.active():
-        return
-
-    # celery tasks don't exist and the active signal has expired, possibly due to a crash. Clean it up.
-    task_logger.warning(
-        "validate_connector_deletion_fence - "
-        "Resetting fence because no associated celery tasks were found: "
-        f"cc_pair={cc_pair_id} "
-        f"fence={fence_key}"
-    )
-
-    redis_connector.delete.reset()
-    return

backend/onyx/background/celery/tasks/doc_permission_syncing/tasks.py

@@ -30,7 +30,6 @@ from onyx.background.celery.celery_redis import celery_find_task
 from onyx.background.celery.celery_redis import celery_get_queue_length
 from onyx.background.celery.celery_redis import celery_get_queued_task_ids
 from onyx.background.celery.celery_redis import celery_get_unacked_task_ids
-from onyx.background.celery.tasks.shared.tasks import OnyxCeleryTaskCompletionStatus
 from onyx.configs.app_configs import JOB_TIMEOUT
 from onyx.configs.constants import CELERY_GENERIC_BEAT_LOCK_TIMEOUT
 from onyx.configs.constants import CELERY_PERMISSIONS_SYNC_LOCK_TIMEOUT
@@ -43,10 +42,8 @@ from onyx.configs.constants import OnyxCeleryTask
 from onyx.configs.constants import OnyxRedisConstants
 from onyx.configs.constants import OnyxRedisLocks
 from onyx.configs.constants import OnyxRedisSignals
-from onyx.connectors.factory import validate_ccpair_for_user
 from onyx.db.connector import mark_cc_pair_as_permissions_synced
 from onyx.db.connector_credential_pair import get_connector_credential_pair_from_id
-from onyx.db.connector_credential_pair import update_connector_credential_pair
 from onyx.db.document import upsert_document_by_connector_credential_pair
 from onyx.db.engine import get_session_with_current_tenant
 from onyx.db.enums import AccessType
@@ -66,7 +63,6 @@ from onyx.redis.redis_pool import get_redis_replica_client
 from onyx.redis.redis_pool import redis_lock_dump
 from onyx.server.utils import make_short_id
 from onyx.utils.logger import doc_permission_sync_ctx
-from onyx.utils.logger import format_error_for_logging
 from onyx.utils.logger import LoggerContextVars
 from onyx.utils.logger import setup_logger
 
@@ -197,19 +193,12 @@ def check_for_doc_permissions_sync(self: Task, *, tenant_id: str) -> bool | None
                     monitor_ccpair_permissions_taskset(
                         tenant_id, key_bytes, r, db_session
                     )
-        task_logger.info(f"check_for_doc_permissions_sync finished: tenant={tenant_id}")
     except SoftTimeLimitExceeded:
         task_logger.info(
             "Soft time limit exceeded, task is being terminated gracefully."
         )
-    except Exception as e:
-        error_msg = format_error_for_logging(e)
-        task_logger.warning(
-            f"Unexpected check_for_doc_permissions_sync exception: tenant={tenant_id} {error_msg}"
-        )
-        task_logger.exception(
-            f"Unexpected check_for_doc_permissions_sync exception: tenant={tenant_id}"
-        )
+    except Exception:
+        task_logger.exception(f"Unexpected exception: tenant={tenant_id}")
     finally:
         if lock_beat.owned():
             lock_beat.release()
@@ -293,19 +282,13 @@ def try_creating_permissions_sync_task(
         redis_connector.permissions.set_fence(payload)
 
         payload_id = payload.id
-    except Exception as e:
-        error_msg = format_error_for_logging(e)
-        task_logger.warning(
-            f"Unexpected try_creating_permissions_sync_task exception: cc_pair={cc_pair_id} {error_msg}"
-        )
+    except Exception:
+        task_logger.exception(f"Unexpected exception: cc_pair={cc_pair_id}")
         return None
     finally:
         if lock.owned():
             lock.release()
 
-    task_logger.info(
-        f"try_creating_permissions_sync_task finished: cc_pair={cc_pair_id} payload_id={payload_id}"
-    )
     return payload_id
 
 
@@ -405,30 +388,6 @@ def connector_permission_sync_generator_task(
                     f"No connector credential pair found for id: {cc_pair_id}"
                 )
 
-            try:
-                created = validate_ccpair_for_user(
-                    cc_pair.connector.id,
-                    cc_pair.credential.id,
-                    db_session,
-                    tenant_id,
-                    enforce_creation=False,
-                )
-                if not created:
-                    task_logger.warning(
-                        f"Unable to create connector credential pair for id: {cc_pair_id}"
-                    )
-            except Exception:
-                task_logger.exception(
-                    f"validate_ccpair_permissions_sync exceptioned: cc_pair={cc_pair_id}"
-                )
-                update_connector_credential_pair(
-                    db_session=db_session,
-                    connector_id=cc_pair.connector.id,
-                    credential_id=cc_pair.credential.id,
-                    status=ConnectorCredentialPairStatus.INVALID,
-                )
-                raise
-
             source_type = cc_pair.connector.source
 
             doc_sync_func = DOC_PERMISSIONS_FUNC_MAP.get(source_type)
@@ -480,10 +439,6 @@ def connector_permission_sync_generator_task(
             redis_connector.permissions.generator_complete = tasks_generated
 
     except Exception as e:
-        error_msg = format_error_for_logging(e)
-        task_logger.warning(
-            f"Permission sync exceptioned: cc_pair={cc_pair_id} payload_id={payload_id} {error_msg}"
-        )
         task_logger.exception(
             f"Permission sync exceptioned: cc_pair={cc_pair_id} payload_id={payload_id}"
         )
@@ -518,8 +473,6 @@ def update_external_document_permissions_task(
 ) -> bool:
     start = time.monotonic()
 
-    completion_status = OnyxCeleryTaskCompletionStatus.UNDEFINED
-
     document_external_access = DocExternalAccess.from_dict(
         serialized_doc_external_access
     )
@@ -559,28 +512,13 @@ def update_external_document_permissions_task(
                 f"elapsed={elapsed:.2f}"
             )
 
-        completion_status = OnyxCeleryTaskCompletionStatus.SUCCEEDED
-    except Exception as e:
-        error_msg = format_error_for_logging(e)
-        task_logger.warning(
-            f"Exception in update_external_document_permissions_task: connector_id={connector_id} doc_id={doc_id} {error_msg}"
-        )
+    except Exception:
         task_logger.exception(
-            f"update_external_document_permissions_task exceptioned: "
+            f"Exception in update_external_document_permissions_task: "
             f"connector_id={connector_id} doc_id={doc_id}"
         )
-        completion_status = OnyxCeleryTaskCompletionStatus.NON_RETRYABLE_EXCEPTION
-    finally:
-        task_logger.info(
-            f"update_external_document_permissions_task completed: status={completion_status.value} doc={doc_id}"
-        )
-
-    if completion_status != OnyxCeleryTaskCompletionStatus.SUCCEEDED:
         return False
 
-    task_logger.info(
-        f"update_external_document_permissions_task finished: connector_id={connector_id} doc_id={doc_id}"
-    )
     return True
 
 

backend/onyx/background/celery/tasks/external_group_syncing/tasks.py

@@ -37,11 +37,8 @@ from onyx.configs.constants import OnyxCeleryTask
 from onyx.configs.constants import OnyxRedisConstants
 from onyx.configs.constants import OnyxRedisLocks
 from onyx.configs.constants import OnyxRedisSignals
-from onyx.connectors.exceptions import ConnectorValidationError
-from onyx.connectors.factory import validate_ccpair_for_user
 from onyx.db.connector import mark_cc_pair_as_external_group_synced
 from onyx.db.connector_credential_pair import get_connector_credential_pair_from_id
-from onyx.db.connector_credential_pair import update_connector_credential_pair
 from onyx.db.engine import get_session_with_current_tenant
 from onyx.db.enums import AccessType
 from onyx.db.enums import ConnectorCredentialPairStatus
@@ -58,7 +55,6 @@ from onyx.redis.redis_connector_ext_group_sync import (
 from onyx.redis.redis_pool import get_redis_client
 from onyx.redis.redis_pool import get_redis_replica_client
 from onyx.server.utils import make_short_id
-from onyx.utils.logger import format_error_for_logging
 from onyx.utils.logger import setup_logger
 
 logger = setup_logger()
@@ -152,10 +148,7 @@ def check_for_external_group_sync(self: Task, *, tenant_id: str | None) -> bool
             for source in GROUP_PERMISSIONS_IS_CC_PAIR_AGNOSTIC:
                 # These are ordered by cc_pair id so the first one is the one we want
                 cc_pairs_to_dedupe = get_cc_pairs_by_source(
-                    db_session,
-                    source,
-                    access_type=AccessType.SYNC,
-                    status=ConnectorCredentialPairStatus.ACTIVE,
+                    db_session, source, only_sync=True
                 )
                 # We only want to sync one cc_pair per source type
                 # in GROUP_PERMISSIONS_IS_CC_PAIR_AGNOSTIC so we dedupe here
@@ -202,17 +195,12 @@ def check_for_external_group_sync(self: Task, *, tenant_id: str | None) -> bool
         task_logger.info(
             "Soft time limit exceeded, task is being terminated gracefully."
         )
-    except Exception as e:
-        error_msg = format_error_for_logging(e)
-        task_logger.warning(
-            f"Unexpected check_for_external_group_sync exception: tenant={tenant_id} {error_msg}"
-        )
+    except Exception:
         task_logger.exception(f"Unexpected exception: tenant={tenant_id}")
     finally:
         if lock_beat.owned():
             lock_beat.release()
 
-    task_logger.info(f"check_for_external_group_sync finished: tenant={tenant_id}")
     return True
 
 
@@ -279,19 +267,12 @@ def try_creating_external_group_sync_task(
         redis_connector.external_group_sync.set_fence(payload)
 
         payload_id = payload.id
-    except Exception as e:
-        error_msg = format_error_for_logging(e)
-        task_logger.warning(
-            f"Unexpected try_creating_external_group_sync_task exception: cc_pair={cc_pair_id} {error_msg}"
-        )
+    except Exception:
         task_logger.exception(
             f"Unexpected exception while trying to create external group sync task: cc_pair={cc_pair_id}"
         )
         return None
 
-    task_logger.info(
-        f"try_creating_external_group_sync_task finished: cc_pair={cc_pair_id} payload_id={payload_id}"
-    )
     return payload_id
 
 
@@ -380,37 +361,12 @@ def connector_external_group_sync_generator_task(
             cc_pair = get_connector_credential_pair_from_id(
                 db_session=db_session,
                 cc_pair_id=cc_pair_id,
-                eager_load_credential=True,
             )
             if cc_pair is None:
                 raise ValueError(
                     f"No connector credential pair found for id: {cc_pair_id}"
                 )
 
-            try:
-                created = validate_ccpair_for_user(
-                    cc_pair.connector.id,
-                    cc_pair.credential.id,
-                    db_session,
-                    tenant_id,
-                    enforce_creation=False,
-                )
-                if not created:
-                    task_logger.warning(
-                        f"Unable to create connector credential pair for id: {cc_pair_id}"
-                    )
-            except Exception:
-                task_logger.exception(
-                    f"validate_ccpair_permissions_sync exceptioned: cc_pair={cc_pair_id}"
-                )
-                update_connector_credential_pair(
-                    db_session=db_session,
-                    connector_id=cc_pair.connector.id,
-                    credential_id=cc_pair.credential.id,
-                    status=ConnectorCredentialPairStatus.INVALID,
-                )
-                raise
-
             source_type = cc_pair.connector.source
 
             ext_group_sync_func = GROUP_PERMISSIONS_FUNC_MAP.get(source_type)
@@ -422,18 +378,8 @@ def connector_external_group_sync_generator_task(
             logger.info(
                 f"Syncing external groups for {source_type} for cc_pair: {cc_pair_id}"
             )
-            external_user_groups: list[ExternalUserGroup] = []
-            try:
-                external_user_groups = ext_group_sync_func(cc_pair)
-            except ConnectorValidationError as e:
-                msg = f"Error syncing external groups for {source_type} for cc_pair: {cc_pair_id} {e}"
-                update_connector_credential_pair(
-                    db_session=db_session,
-                    connector_id=cc_pair.connector.id,
-                    credential_id=cc_pair.credential.id,
-                    status=ConnectorCredentialPairStatus.INVALID,
-                )
-                raise e
+
+            external_user_groups: list[ExternalUserGroup] = ext_group_sync_func(cc_pair)
 
             logger.info(
                 f"Syncing {len(external_user_groups)} external user groups for {source_type}"
@@ -459,14 +405,6 @@ def connector_external_group_sync_generator_task(
                 sync_status=SyncStatus.SUCCESS,
             )
     except Exception as e:
-        error_msg = format_error_for_logging(e)
-        task_logger.warning(
-            f"External group sync exceptioned: cc_pair={cc_pair_id} payload_id={payload.id} {error_msg}"
-        )
-        task_logger.exception(
-            f"External group sync exceptioned: cc_pair={cc_pair_id} payload_id={payload.id}"
-        )
-
         msg = f"External group sync exceptioned: cc_pair={cc_pair_id} payload_id={payload.id}"
         task_logger.exception(msg)
         emit_background_error(msg + f"\n\n{e}", cc_pair_id=cc_pair_id)

backend/onyx/background/celery/tasks/indexing/tasks.py

@@ -48,7 +48,7 @@ from onyx.configs.constants import OnyxCeleryTask
 from onyx.configs.constants import OnyxRedisConstants
 from onyx.configs.constants import OnyxRedisLocks
 from onyx.configs.constants import OnyxRedisSignals
-from onyx.connectors.exceptions import ConnectorValidationError
+from onyx.connectors.interfaces import ConnectorValidationError
 from onyx.db.connector import mark_ccpair_with_indexing_trigger
 from onyx.db.connector_credential_pair import fetch_connector_credential_pairs
 from onyx.db.connector_credential_pair import get_connector_credential_pair_from_id
@@ -899,9 +899,6 @@ def connector_indexing_proxy_task(
 
     TODO(rkuo): refactor this so that there is a single return path where we canonically
     log the result of running this function.
-
-    NOTE: we try/except all db access in this function because as a watchdog, this function
-    needs to be extremely stable.
     """
     start = time.monotonic()
 
@@ -927,7 +924,6 @@ def connector_indexing_proxy_task(
         task_logger.error("self.request.id is None!")
 
     client = SimpleJobClient()
-    task_logger.info(f"submitting connector_indexing_task with tenant_id={tenant_id}")
 
     job = client.submit(
         connector_indexing_task,
@@ -1020,7 +1016,7 @@ def connector_indexing_proxy_task(
                     job.release()
                     break
 
-            # if a termination signal is detected, break (exit point will clean up)
+            # if a termination signal is detected, clean up and break
             if self.request.id and redis_connector_index.terminating(self.request.id):
                 task_logger.warning(
                     log_builder.build("Indexing watchdog - termination signal detected")
@@ -1029,7 +1025,6 @@ def connector_indexing_proxy_task(
                 result.status = IndexingWatchdogTerminalStatus.TERMINATED_BY_SIGNAL
                 break
 
-            # if activity timeout is detected, break (exit point will clean up)
             if not redis_connector_index.connector_active():
                 task_logger.warning(
                     log_builder.build(
@@ -1038,6 +1033,25 @@ def connector_indexing_proxy_task(
                     )
                 )
 
+                try:
+                    with get_session_with_current_tenant() as db_session:
+                        mark_attempt_failed(
+                            index_attempt_id,
+                            db_session,
+                            "Indexing watchdog - activity timeout exceeded: "
+                            f"attempt={index_attempt_id} "
+                            f"timeout={CELERY_INDEXING_WATCHDOG_CONNECTOR_TIMEOUT}s",
+                        )
+                except Exception:
+                    # if the DB exceptions, we'll just get an unfriendly failure message
+                    # in the UI instead of the cancellation message
+                    logger.exception(
+                        log_builder.build(
+                            "Indexing watchdog - transient exception marking index attempt as failed"
+                        )
+                    )
+
+                job.cancel()
                 result.status = (
                     IndexingWatchdogTerminalStatus.TERMINATED_BY_ACTIVITY_TIMEOUT
                 )
@@ -1056,15 +1070,15 @@ def connector_indexing_proxy_task(
 
                     if not index_attempt.is_finished():
                         continue
-
             except Exception:
+                # if the DB exceptioned, just restart the check.
+                # polling the index attempt status doesn't need to be strongly consistent
                 task_logger.exception(
                     log_builder.build(
                         "Indexing watchdog - transient exception looking up index attempt"
                     )
                 )
                 continue
-
     except Exception as e:
         result.status = IndexingWatchdogTerminalStatus.WATCHDOG_EXCEPTIONED
         if isinstance(e, ConnectorValidationError):
@@ -1125,6 +1139,8 @@ def connector_indexing_proxy_task(
                     "Connector termination signal detected",
                 )
         except Exception:
+            # if the DB exceptions, we'll just get an unfriendly failure message
+            # in the UI instead of the cancellation message
             task_logger.exception(
                 log_builder.build(
                     "Indexing watchdog - transient exception marking index attempt as canceled"
@@ -1132,25 +1148,6 @@ def connector_indexing_proxy_task(
             )
 
         job.cancel()
-    elif result.status == IndexingWatchdogTerminalStatus.TERMINATED_BY_ACTIVITY_TIMEOUT:
-        try:
-            with get_session_with_current_tenant() as db_session:
-                mark_attempt_failed(
-                    index_attempt_id,
-                    db_session,
-                    "Indexing watchdog - activity timeout exceeded: "
-                    f"attempt={index_attempt_id} "
-                    f"timeout={CELERY_INDEXING_WATCHDOG_CONNECTOR_TIMEOUT}s",
-                )
-        except Exception:
-            logger.exception(
-                log_builder.build(
-                    "Indexing watchdog - transient exception marking index attempt as failed"
-                )
-            )
-        job.cancel()
-    else:
-        pass
 
     task_logger.info(
         log_builder.build(

backend/onyx/background/celery/tasks/pruning/tasks.py

@@ -55,7 +55,6 @@ from onyx.redis.redis_connector_prune import RedisConnectorPrunePayload
 from onyx.redis.redis_pool import get_redis_client
 from onyx.redis.redis_pool import get_redis_replica_client
 from onyx.server.utils import make_short_id
-from onyx.utils.logger import format_error_for_logging
 from onyx.utils.logger import LoggerContextVars
 from onyx.utils.logger import pruning_ctx
 from onyx.utils.logger import setup_logger
@@ -195,14 +194,12 @@ def check_for_pruning(self: Task, *, tenant_id: str | None) -> bool | None:
         task_logger.info(
             "Soft time limit exceeded, task is being terminated gracefully."
         )
-    except Exception as e:
-        error_msg = format_error_for_logging(e)
-        task_logger.warning(f"Unexpected pruning check exception: {error_msg}")
+    except Exception:
         task_logger.exception("Unexpected exception during pruning check")
     finally:
         if lock_beat.owned():
             lock_beat.release()
-    task_logger.info(f"check_for_pruning finished: tenant={tenant_id}")
+
     return True
 
 
@@ -304,19 +301,13 @@ def try_creating_prune_generator_task(
         redis_connector.prune.set_fence(payload)
 
         payload_id = payload.id
-    except Exception as e:
-        error_msg = format_error_for_logging(e)
-        task_logger.warning(
-            f"Unexpected try_creating_prune_generator_task exception: cc_pair={cc_pair.id} {error_msg}"
-        )
+    except Exception:
         task_logger.exception(f"Unexpected exception: cc_pair={cc_pair.id}")
         return None
     finally:
         if lock.owned():
             lock.release()
-    task_logger.info(
-        f"try_creating_prune_generator_task finished: cc_pair={cc_pair.id} payload_id={payload_id}"
-    )
+
     return payload_id
 
 

backend/onyx/background/celery/tasks/shared/tasks.py

@@ -1,5 +1,4 @@
 import time
-from enum import Enum
 from http import HTTPStatus
 
 import httpx
@@ -46,24 +45,6 @@ LIGHT_SOFT_TIME_LIMIT = 105
 LIGHT_TIME_LIMIT = LIGHT_SOFT_TIME_LIMIT + 15
 
 
-class OnyxCeleryTaskCompletionStatus(str, Enum):
-    """The different statuses the watchdog can finish with.
-
-    TODO: create broader success/failure/abort categories
-    """
-
-    UNDEFINED = "undefined"
-
-    SUCCEEDED = "succeeded"
-
-    SKIPPED = "skipped"
-
-    SOFT_TIME_LIMIT = "soft_time_limit"
-
-    NON_RETRYABLE_EXCEPTION = "non_retryable_exception"
-    RETRYABLE_EXCEPTION = "retryable_exception"
-
-
 @shared_task(
     name=OnyxCeleryTask.DOCUMENT_BY_CC_PAIR_CLEANUP_TASK,
     soft_time_limit=LIGHT_SOFT_TIME_LIMIT,
@@ -97,8 +78,6 @@ def document_by_cc_pair_cleanup_task(
 
     start = time.monotonic()
 
-    completion_status = OnyxCeleryTaskCompletionStatus.UNDEFINED
-
     try:
         with get_session_with_current_tenant() as db_session:
             action = "skip"
@@ -131,9 +110,6 @@ def document_by_cc_pair_cleanup_task(
                     db_session=db_session,
                     document_ids=[document_id],
                 )
-                db_session.commit()
-
-                completion_status = OnyxCeleryTaskCompletionStatus.SUCCEEDED
             elif count > 1:
                 action = "update"
 
@@ -177,11 +153,10 @@ def document_by_cc_pair_cleanup_task(
                 )
 
                 mark_document_as_synced(document_id, db_session)
-                db_session.commit()
-
-                completion_status = OnyxCeleryTaskCompletionStatus.SUCCEEDED
             else:
-                completion_status = OnyxCeleryTaskCompletionStatus.SKIPPED
+                pass
+
+            db_session.commit()
 
             elapsed = time.monotonic() - start
             task_logger.info(
@@ -193,79 +168,57 @@ def document_by_cc_pair_cleanup_task(
             )
     except SoftTimeLimitExceeded:
         task_logger.info(f"SoftTimeLimitExceeded exception. doc={document_id}")
-        completion_status = OnyxCeleryTaskCompletionStatus.SOFT_TIME_LIMIT
+        return False
     except Exception as ex:
         e: Exception | None = None
-        while True:
-            if isinstance(ex, RetryError):
-                task_logger.warning(
-                    f"Tenacity retry failed: num_attempts={ex.last_attempt.attempt_number}"
-                )
-
-                # only set the inner exception if it is of type Exception
-                e_temp = ex.last_attempt.exception()
-                if isinstance(e_temp, Exception):
-                    e = e_temp
-            else:
-                e = ex
-
-            if isinstance(e, httpx.HTTPStatusError):
-                if e.response.status_code == HTTPStatus.BAD_REQUEST:
-                    task_logger.exception(
-                        f"Non-retryable HTTPStatusError: "
-                        f"doc={document_id} "
-                        f"status={e.response.status_code}"
-                    )
-                completion_status = (
-                    OnyxCeleryTaskCompletionStatus.NON_RETRYABLE_EXCEPTION
-                )
-                break
-
-            task_logger.exception(
-                f"document_by_cc_pair_cleanup_task exceptioned: doc={document_id}"
+        if isinstance(ex, RetryError):
+            task_logger.warning(
+                f"Tenacity retry failed: num_attempts={ex.last_attempt.attempt_number}"
             )
 
-            completion_status = OnyxCeleryTaskCompletionStatus.RETRYABLE_EXCEPTION
-            if (
-                self.max_retries is not None
-                and self.request.retries >= self.max_retries
-            ):
-                # This is the last attempt! mark the document as dirty in the db so that it
-                # eventually gets fixed out of band via stale document reconciliation
-                task_logger.warning(
-                    f"Max celery task retries reached. Marking doc as dirty for reconciliation: "
-                    f"doc={document_id}"
-                )
-                with get_session_with_current_tenant() as db_session:
-                    # delete the cc pair relationship now and let reconciliation clean it up
-                    # in vespa
-                    delete_document_by_connector_credential_pair__no_commit(
-                        db_session=db_session,
-                        document_id=document_id,
-                        connector_credential_pair_identifier=ConnectorCredentialPairIdentifier(
-                            connector_id=connector_id,
-                            credential_id=credential_id,
-                        ),
-                    )
-                    mark_document_as_modified(document_id, db_session)
-                completion_status = (
-                    OnyxCeleryTaskCompletionStatus.NON_RETRYABLE_EXCEPTION
-                )
-                break
+            # only set the inner exception if it is of type Exception
+            e_temp = ex.last_attempt.exception()
+            if isinstance(e_temp, Exception):
+                e = e_temp
+        else:
+            e = ex
 
-            # Exponential backoff from 2^4 to 2^6 ... i.e. 16, 32, 64
+        if isinstance(e, httpx.HTTPStatusError):
+            if e.response.status_code == HTTPStatus.BAD_REQUEST:
+                task_logger.exception(
+                    f"Non-retryable HTTPStatusError: "
+                    f"doc={document_id} "
+                    f"status={e.response.status_code}"
+                )
+            return False
+
+        task_logger.exception(f"Unexpected exception: doc={document_id}")
+
+        if self.request.retries < DOCUMENT_BY_CC_PAIR_CLEANUP_MAX_RETRIES:
+            # Still retrying. Exponential backoff from 2^4 to 2^6 ... i.e. 16, 32, 64
             countdown = 2 ** (self.request.retries + 4)
-            self.retry(exc=e, countdown=countdown)  # this will raise a celery exception
-            break  # we won't hit this, but it looks weird not to have it
-    finally:
-        task_logger.info(
-            f"document_by_cc_pair_cleanup_task completed: status={completion_status.value} doc={document_id}"
-        )
-
-    if completion_status != OnyxCeleryTaskCompletionStatus.SUCCEEDED:
+            self.retry(exc=e, countdown=countdown)
+        else:
+            # This is the last attempt! mark the document as dirty in the db so that it
+            # eventually gets fixed out of band via stale document reconciliation
+            task_logger.warning(
+                f"Max celery task retries reached. Marking doc as dirty for reconciliation: "
+                f"doc={document_id}"
+            )
+            with get_session_with_current_tenant() as db_session:
+                # delete the cc pair relationship now and let reconciliation clean it up
+                # in vespa
+                delete_document_by_connector_credential_pair__no_commit(
+                    db_session=db_session,
+                    document_id=document_id,
+                    connector_credential_pair_identifier=ConnectorCredentialPairIdentifier(
+                        connector_id=connector_id,
+                        credential_id=credential_id,
+                    ),
+                )
+                mark_document_as_modified(document_id, db_session)
         return False
 
-    task_logger.info(f"document_by_cc_pair_cleanup_task finished: doc={document_id}")
     return True
 
 

backend/onyx/background/celery/tasks/vespa/tasks.py

@@ -19,7 +19,6 @@ from onyx.background.celery.apps.app_base import task_logger
 from onyx.background.celery.tasks.shared.RetryDocumentIndex import RetryDocumentIndex
 from onyx.background.celery.tasks.shared.tasks import LIGHT_SOFT_TIME_LIMIT
 from onyx.background.celery.tasks.shared.tasks import LIGHT_TIME_LIMIT
-from onyx.background.celery.tasks.shared.tasks import OnyxCeleryTaskCompletionStatus
 from onyx.configs.app_configs import JOB_TIMEOUT
 from onyx.configs.app_configs import VESPA_SYNC_MAX_TASKS
 from onyx.configs.constants import CELERY_VESPA_SYNC_BEAT_LOCK_TIMEOUT
@@ -528,8 +527,6 @@ def vespa_metadata_sync_task(
 ) -> bool:
     start = time.monotonic()
 
-    completion_status = OnyxCeleryTaskCompletionStatus.UNDEFINED
-
     try:
         with get_session_with_current_tenant() as db_session:
             active_search_settings = get_active_search_settings(db_session)
@@ -543,103 +540,75 @@ def vespa_metadata_sync_task(
 
             doc = get_document(document_id, db_session)
             if not doc:
-                elapsed = time.monotonic() - start
-                task_logger.info(
-                    f"doc={document_id} "
-                    f"action=no_operation "
-                    f"elapsed={elapsed:.2f}"
-                )
-                completion_status = OnyxCeleryTaskCompletionStatus.SKIPPED
-            else:
-                # document set sync
-                doc_sets = fetch_document_sets_for_document(document_id, db_session)
-                update_doc_sets: set[str] = set(doc_sets)
+                return False
 
-                # User group sync
-                doc_access = get_access_for_document(
-                    document_id=document_id, db_session=db_session
-                )
+            # document set sync
+            doc_sets = fetch_document_sets_for_document(document_id, db_session)
+            update_doc_sets: set[str] = set(doc_sets)
 
-                fields = VespaDocumentFields(
-                    document_sets=update_doc_sets,
-                    access=doc_access,
-                    boost=doc.boost,
-                    hidden=doc.hidden,
-                )
-
-                # update Vespa. OK if doc doesn't exist. Raises exception otherwise.
-                chunks_affected = retry_index.update_single(
-                    document_id,
-                    tenant_id=tenant_id,
-                    chunk_count=doc.chunk_count,
-                    fields=fields,
-                )
-
-                # update db last. Worst case = we crash right before this and
-                # the sync might repeat again later
-                mark_document_as_synced(document_id, db_session)
-
-                elapsed = time.monotonic() - start
-                task_logger.info(
-                    f"doc={document_id} "
-                    f"action=sync "
-                    f"chunks={chunks_affected} "
-                    f"elapsed={elapsed:.2f}"
-                )
-                completion_status = OnyxCeleryTaskCompletionStatus.SUCCEEDED
-    except SoftTimeLimitExceeded:
-        task_logger.info(f"SoftTimeLimitExceeded exception. doc={document_id}")
-        completion_status = OnyxCeleryTaskCompletionStatus.SOFT_TIME_LIMIT
-    except Exception as ex:
-        e: Exception | None = None
-        while True:
-            if isinstance(ex, RetryError):
-                task_logger.warning(
-                    f"Tenacity retry failed: num_attempts={ex.last_attempt.attempt_number}"
-                )
-
-                # only set the inner exception if it is of type Exception
-                e_temp = ex.last_attempt.exception()
-                if isinstance(e_temp, Exception):
-                    e = e_temp
-            else:
-                e = ex
-
-            if isinstance(e, httpx.HTTPStatusError):
-                if e.response.status_code == HTTPStatus.BAD_REQUEST:
-                    task_logger.exception(
-                        f"Non-retryable HTTPStatusError: "
-                        f"doc={document_id} "
-                        f"status={e.response.status_code}"
-                    )
-                completion_status = (
-                    OnyxCeleryTaskCompletionStatus.NON_RETRYABLE_EXCEPTION
-                )
-                break
-
-            task_logger.exception(
-                f"vespa_metadata_sync_task exceptioned: doc={document_id}"
+            # User group sync
+            doc_access = get_access_for_document(
+                document_id=document_id, db_session=db_session
             )
 
-            completion_status = OnyxCeleryTaskCompletionStatus.RETRYABLE_EXCEPTION
-            if (
-                self.max_retries is not None
-                and self.request.retries >= self.max_retries
-            ):
-                completion_status = (
-                    OnyxCeleryTaskCompletionStatus.NON_RETRYABLE_EXCEPTION
-                )
+            fields = VespaDocumentFields(
+                document_sets=update_doc_sets,
+                access=doc_access,
+                boost=doc.boost,
+                hidden=doc.hidden,
+            )
 
-            # Exponential backoff from 2^4 to 2^6 ... i.e. 16, 32, 64
-            countdown = 2 ** (self.request.retries + 4)
-            self.retry(exc=e, countdown=countdown)  # this will raise a celery exception
-            break  # we won't hit this, but it looks weird not to have it
-    finally:
-        task_logger.info(
-            f"vespa_metadata_sync_task completed: status={completion_status.value} doc={document_id}"
+            # update Vespa. OK if doc doesn't exist. Raises exception otherwise.
+            chunks_affected = retry_index.update_single(
+                document_id,
+                tenant_id=tenant_id,
+                chunk_count=doc.chunk_count,
+                fields=fields,
+            )
+
+            # update db last. Worst case = we crash right before this and
+            # the sync might repeat again later
+            mark_document_as_synced(document_id, db_session)
+
+            elapsed = time.monotonic() - start
+            task_logger.info(
+                f"doc={document_id} "
+                f"action=sync "
+                f"chunks={chunks_affected} "
+                f"elapsed={elapsed:.2f}"
+            )
+    except SoftTimeLimitExceeded:
+        task_logger.info(f"SoftTimeLimitExceeded exception. doc={document_id}")
+        return False
+    except Exception as ex:
+        e: Exception | None = None
+        if isinstance(ex, RetryError):
+            task_logger.warning(
+                f"Tenacity retry failed: num_attempts={ex.last_attempt.attempt_number}"
+            )
+
+            # only set the inner exception if it is of type Exception
+            e_temp = ex.last_attempt.exception()
+            if isinstance(e_temp, Exception):
+                e = e_temp
+        else:
+            e = ex
+
+        if isinstance(e, httpx.HTTPStatusError):
+            if e.response.status_code == HTTPStatus.BAD_REQUEST:
+                task_logger.exception(
+                    f"Non-retryable HTTPStatusError: "
+                    f"doc={document_id} "
+                    f"status={e.response.status_code}"
+                )
+            return False
+
+        task_logger.exception(
+            f"Unexpected exception during vespa metadata sync: doc={document_id}"
         )
 
-    if completion_status != OnyxCeleryTaskCompletionStatus.SUCCEEDED:
-        return False
+        # Exponential backoff from 2^4 to 2^6 ... i.e. 16, 32, 64
+        countdown = 2 ** (self.request.retries + 4)
+        self.retry(exc=e, countdown=countdown)
 
     return True

backend/onyx/background/error_logging.py

@@ -1,5 +1,3 @@
-from sqlalchemy.exc import IntegrityError
-
 from onyx.db.background_error import create_background_error
 from onyx.db.engine import get_session_with_current_tenant
 
@@ -12,9 +10,4 @@ def emit_background_error(
 
     In the future, could create notifications based on the severity."""
     with get_session_with_current_tenant() as db_session:
-        try:
-            create_background_error(db_session, message, cc_pair_id)
-        except IntegrityError as e:
-            # Log an error if the cc_pair_id was deleted or any other exception occurs
-            error_message = f"Failed to create background error: {str(e)}. Original message: {message}"
-            create_background_error(db_session, error_message, None)
+        create_background_error(db_session, message, cc_pair_id)

backend/onyx/background/indexing/job_client.py

@@ -17,9 +17,6 @@ from typing import Optional
 from onyx.configs.constants import POSTGRES_CELERY_WORKER_INDEXING_CHILD_APP_NAME
 from onyx.db.engine import SqlEngine
 from onyx.utils.logger import setup_logger
-from shared_configs.configs import POSTGRES_DEFAULT_SCHEMA
-from shared_configs.configs import TENANT_ID_PREFIX
-from shared_configs.contextvars import CURRENT_TENANT_ID_CONTEXTVAR
 
 logger = setup_logger()
 
@@ -57,15 +54,6 @@ def _initializer(
         kwargs = {}
 
     logger.info("Initializing spawned worker child process.")
-    # 1. Get tenant_id from args or fallback to default
-    tenant_id = POSTGRES_DEFAULT_SCHEMA
-    for arg in reversed(args):
-        if isinstance(arg, str) and arg.startswith(TENANT_ID_PREFIX):
-            tenant_id = arg
-            break
-
-    # 2. Set the tenant context before running anything
-    token = CURRENT_TENANT_ID_CONTEXTVAR.set(tenant_id)
 
     # Reset the engine in the child process
     SqlEngine.reset_engine()
@@ -93,8 +81,6 @@ def _initializer(
         queue.put(error_msg)  # Send the exception to the parent process
 
         sys.exit(255)  # use 255 to indicate a generic exception
-    finally:
-        CURRENT_TENANT_ID_CONTEXTVAR.reset(token)
 
 
 def _run_in_process(

backend/onyx/background/indexing/run_indexing.py

@@ -15,14 +15,13 @@ from onyx.background.indexing.memory_tracer import MemoryTracer
 from onyx.configs.app_configs import INDEX_BATCH_SIZE
 from onyx.configs.app_configs import INDEXING_SIZE_WARNING_THRESHOLD
 from onyx.configs.app_configs import INDEXING_TRACER_INTERVAL
-from onyx.configs.app_configs import INTEGRATION_TESTS_MODE
 from onyx.configs.app_configs import LEAVE_CONNECTOR_ACTIVE_ON_INITIALIZATION_FAILURE
 from onyx.configs.app_configs import POLL_CONNECTOR_OFFSET
 from onyx.configs.constants import DocumentSource
 from onyx.configs.constants import MilestoneRecordType
 from onyx.connectors.connector_runner import ConnectorRunner
-from onyx.connectors.exceptions import ConnectorValidationError
 from onyx.connectors.factory import instantiate_connector
+from onyx.connectors.interfaces import ConnectorValidationError
 from onyx.connectors.models import ConnectorCheckpoint
 from onyx.connectors.models import ConnectorFailure
 from onyx.connectors.models import Document
@@ -90,8 +89,8 @@ def _get_connector_runner(
         )
 
         # validate the connector settings
-        if not INTEGRATION_TESTS_MODE:
-            runnable_connector.validate_connector_settings()
+
+        runnable_connector.validate_connector_settings()
 
     except Exception as e:
         logger.exception(f"Unable to instantiate connector due to {e}")

backend/onyx/chat/chat_utils.py

@@ -190,8 +190,7 @@ def create_chat_chain(
             and previous_message.message_type == MessageType.ASSISTANT
             and mainline_messages
         ):
-            if current_message.refined_answer_improvement:
-                mainline_messages[-1] = current_message
+            mainline_messages[-1] = current_message
         else:
             mainline_messages.append(current_message)
 

backend/onyx/chat/models.py

@@ -142,15 +142,6 @@ class MessageResponseIDInfo(BaseModel):
     reserved_assistant_message_id: int
 
 
-class AgentMessageIDInfo(BaseModel):
-    level: int
-    message_id: int
-
-
-class AgenticMessageResponseIDInfo(BaseModel):
-    agentic_message_ids: list[AgentMessageIDInfo]
-
-
 class StreamingError(BaseModel):
     error: str
     stack_trace: str | None = None

backend/onyx/chat/process_message.py

@@ -11,8 +11,6 @@ from onyx.agents.agent_search.orchestration.nodes.call_tool import ToolCallExcep
 from onyx.chat.answer import Answer
 from onyx.chat.chat_utils import create_chat_chain
 from onyx.chat.chat_utils import create_temporary_persona
-from onyx.chat.models import AgenticMessageResponseIDInfo
-from onyx.chat.models import AgentMessageIDInfo
 from onyx.chat.models import AgentSearchPacket
 from onyx.chat.models import AllCitations
 from onyx.chat.models import AnswerPostInfo
@@ -310,7 +308,6 @@ ChatPacket = (
     | CustomToolResponse
     | MessageSpecificCitations
     | MessageResponseIDInfo
-    | AgenticMessageResponseIDInfo
     | StreamStopInfo
     | AgentSearchPacket
 )
@@ -747,13 +744,14 @@ def stream_chat_message_objects(
                 files=latest_query_files,
                 single_message_history=single_message_history,
             ),
-            system_message=default_build_system_message(prompt_config, llm.config),
+            system_message=default_build_system_message(prompt_config),
             message_history=message_history,
             llm_config=llm.config,
             raw_user_query=final_msg.message,
             raw_user_uploaded_files=latest_query_files or [],
             single_message_history=single_message_history,
         )
+        prompt_builder.update_system_prompt(default_build_system_message(prompt_config))
 
         # LLM prompt building, response capturing, etc.
         answer = Answer(
@@ -869,6 +867,7 @@ def stream_chat_message_objects(
                             for img in img_generation_response
                             if img.image_data
                         ],
+                        tenant_id=tenant_id,
                     )
                     info.ai_message_files.extend(
                         [
@@ -1036,7 +1035,6 @@ def stream_chat_message_objects(
         next_level = 1
         prev_message = gen_ai_response_message
         agent_answers = answer.llm_answer_by_level()
-        agentic_message_ids = []
         while next_level in agent_answers:
             next_answer = agent_answers[next_level]
             info = info_by_subq[
@@ -1061,18 +1059,17 @@ def stream_chat_message_objects(
                 refined_answer_improvement=refined_answer_improvement,
                 is_agentic=True,
             )
-            agentic_message_ids.append(
-                AgentMessageIDInfo(level=next_level, message_id=next_answer_message.id)
-            )
             next_level += 1
             prev_message = next_answer_message
 
         logger.debug("Committing messages")
         db_session.commit()  # actually save user / assistant message
 
-        yield AgenticMessageResponseIDInfo(agentic_message_ids=agentic_message_ids)
+        msg_detail_response = translate_db_message_to_chat_message_detail(
+            gen_ai_response_message
+        )
 
-        yield translate_db_message_to_chat_message_detail(gen_ai_response_message)
+        yield msg_detail_response
     except Exception as e:
         error_msg = str(e)
         logger.exception(error_msg)

backend/onyx/chat/prompt_builder/answer_prompt_builder.py

@@ -12,7 +12,6 @@ from onyx.chat.prompt_builder.citations_prompt import compute_max_llm_input_toke
 from onyx.chat.prompt_builder.utils import translate_history_to_basemessages
 from onyx.file_store.models import InMemoryChatFile
 from onyx.llm.interfaces import LLMConfig
-from onyx.llm.llm_provider_options import OPENAI_PROVIDER_NAME
 from onyx.llm.models import PreviousMessage
 from onyx.llm.utils import build_content_with_imgs
 from onyx.llm.utils import check_message_tokens
@@ -20,7 +19,6 @@ from onyx.llm.utils import message_to_prompt_and_imgs
 from onyx.llm.utils import model_supports_image_input
 from onyx.natural_language_processing.utils import get_tokenizer
 from onyx.prompts.chat_prompts import CHAT_USER_CONTEXT_FREE_PROMPT
-from onyx.prompts.chat_prompts import CODE_BLOCK_MARKDOWN
 from onyx.prompts.direct_qa_prompts import HISTORY_BLOCK
 from onyx.prompts.prompt_utils import drop_messages_history_overflow
 from onyx.prompts.prompt_utils import handle_onyx_date_awareness
@@ -33,16 +31,8 @@ from onyx.tools.tool import Tool
 
 def default_build_system_message(
     prompt_config: PromptConfig,
-    llm_config: LLMConfig,
 ) -> SystemMessage | None:
     system_prompt = prompt_config.system_prompt.strip()
-    # See https://simonwillison.net/tags/markdown/ for context on this temporary fix
-    # for o-series markdown generation
-    if (
-        llm_config.model_provider == OPENAI_PROVIDER_NAME
-        and llm_config.model_name.startswith("o")
-    ):
-        system_prompt = CODE_BLOCK_MARKDOWN + system_prompt
     tag_handled_prompt = handle_onyx_date_awareness(
         system_prompt,
         prompt_config,
@@ -120,8 +110,21 @@ class AnswerPromptBuilder:
             ),
         )
 
-        self.update_system_prompt(system_message)
-        self.update_user_prompt(user_message)
+        self.system_message_and_token_cnt: tuple[SystemMessage, int] | None = (
+            (
+                system_message,
+                check_message_tokens(system_message, self.llm_tokenizer_encode_func),
+            )
+            if system_message
+            else None
+        )
+        self.user_message_and_token_cnt = (
+            user_message,
+            check_message_tokens(
+                user_message,
+                self.llm_tokenizer_encode_func,
+            ),
+        )
 
         self.new_messages_and_token_cnts: list[tuple[BaseMessage, int]] = []
 

backend/onyx/configs/app_configs.py

@@ -626,8 +626,6 @@ POD_NAMESPACE = os.environ.get("POD_NAMESPACE")
 
 DEV_MODE = os.environ.get("DEV_MODE", "").lower() == "true"
 
-INTEGRATION_TESTS_MODE = os.environ.get("INTEGRATION_TESTS_MODE", "").lower() == "true"
-
 MOCK_CONNECTOR_FILE_PATH = os.environ.get("MOCK_CONNECTOR_FILE_PATH")
 
 TEST_ENV = os.environ.get("TEST_ENV", "").lower() == "true"

backend/onyx/configs/constants.py

@@ -342,9 +342,6 @@ class OnyxRedisSignals:
     BLOCK_PRUNING = "signal:block_pruning"
     BLOCK_VALIDATE_PRUNING_FENCES = "signal:block_validate_pruning_fences"
     BLOCK_BUILD_FENCE_LOOKUP_TABLE = "signal:block_build_fence_lookup_table"
-    BLOCK_VALIDATE_CONNECTOR_DELETION_FENCES = (
-        "signal:block_validate_connector_deletion_fences"
-    )
 
 
 class OnyxRedisConstants:

backend/onyx/connectors/blob/connector.py

@@ -7,18 +7,11 @@ from typing import Optional
 
 import boto3  # type: ignore
 from botocore.client import Config  # type: ignore
-from botocore.exceptions import ClientError
-from botocore.exceptions import NoCredentialsError
-from botocore.exceptions import PartialCredentialsError
 from mypy_boto3_s3 import S3Client  # type: ignore
 
 from onyx.configs.app_configs import INDEX_BATCH_SIZE
 from onyx.configs.constants import BlobType
 from onyx.configs.constants import DocumentSource
-from onyx.connectors.exceptions import ConnectorValidationError
-from onyx.connectors.exceptions import CredentialExpiredError
-from onyx.connectors.exceptions import InsufficientPermissionsError
-from onyx.connectors.exceptions import UnexpectedError
 from onyx.connectors.interfaces import GenerateDocumentsOutput
 from onyx.connectors.interfaces import LoadConnector
 from onyx.connectors.interfaces import PollConnector
@@ -247,73 +240,6 @@ class BlobStorageConnector(LoadConnector, PollConnector):
 
         return None
 
-    def validate_connector_settings(self) -> None:
-        if self.s3_client is None:
-            raise ConnectorMissingCredentialError(
-                "Blob storage credentials not loaded."
-            )
-
-        if not self.bucket_name:
-            raise ConnectorValidationError(
-                "No bucket name was provided in connector settings."
-            )
-
-        try:
-            # We only fetch one object/page as a light-weight validation step.
-            # This ensures we trigger typical S3 permission checks (ListObjectsV2, etc.).
-            self.s3_client.list_objects_v2(
-                Bucket=self.bucket_name, Prefix=self.prefix, MaxKeys=1
-            )
-
-        except NoCredentialsError:
-            raise ConnectorMissingCredentialError(
-                "No valid blob storage credentials found or provided to boto3."
-            )
-        except PartialCredentialsError:
-            raise ConnectorMissingCredentialError(
-                "Partial or incomplete blob storage credentials provided to boto3."
-            )
-        except ClientError as e:
-            error_code = e.response["Error"].get("Code", "")
-            status_code = e.response["ResponseMetadata"].get("HTTPStatusCode")
-
-            # Most common S3 error cases
-            if error_code in [
-                "AccessDenied",
-                "InvalidAccessKeyId",
-                "SignatureDoesNotMatch",
-            ]:
-                if status_code == 403 or error_code == "AccessDenied":
-                    raise InsufficientPermissionsError(
-                        f"Insufficient permissions to list objects in bucket '{self.bucket_name}'. "
-                        "Please check your bucket policy and/or IAM policy."
-                    )
-                if status_code == 401 or error_code == "SignatureDoesNotMatch":
-                    raise CredentialExpiredError(
-                        "Provided blob storage credentials appear invalid or expired."
-                    )
-
-                raise CredentialExpiredError(
-                    f"Credential issue encountered ({error_code})."
-                )
-
-            if error_code == "NoSuchBucket" or status_code == 404:
-                raise ConnectorValidationError(
-                    f"Bucket '{self.bucket_name}' does not exist or cannot be found."
-                )
-
-            raise ConnectorValidationError(
-                f"Unexpected S3 client error (code={error_code}, status={status_code}): {e}"
-            )
-
-        except Exception as e:
-            # Catch-all for anything not captured by the above
-            # Since we are unsure of the error and it may not disable the connector,
-            #  raise an unexpected error (does not disable connector)
-            raise UnexpectedError(
-                f"Unexpected error during blob storage settings validation: {e}"
-            )
-
 
 if __name__ == "__main__":
     credentials_dict = {

backend/onyx/connectors/bookstack/connector.py

@@ -9,10 +9,10 @@ from onyx.configs.constants import DocumentSource
 from onyx.connectors.bookstack.client import BookStackApiClient
 from onyx.connectors.bookstack.client import BookStackClientRequestFailedError
 from onyx.connectors.cross_connector_utils.miscellaneous_utils import time_str_to_utc
-from onyx.connectors.exceptions import ConnectorValidationError
-from onyx.connectors.exceptions import CredentialExpiredError
-from onyx.connectors.exceptions import InsufficientPermissionsError
+from onyx.connectors.interfaces import ConnectorValidationError
+from onyx.connectors.interfaces import CredentialExpiredError
 from onyx.connectors.interfaces import GenerateDocumentsOutput
+from onyx.connectors.interfaces import InsufficientPermissionsError
 from onyx.connectors.interfaces import LoadConnector
 from onyx.connectors.interfaces import PollConnector
 from onyx.connectors.interfaces import SecondsSinceUnixEpoch

backend/onyx/connectors/confluence/connector.py

@@ -4,8 +4,6 @@ from datetime import timezone
 from typing import Any
 from urllib.parse import quote
 
-from requests.exceptions import HTTPError
-
 from onyx.configs.app_configs import CONFLUENCE_CONNECTOR_LABELS_TO_SKIP
 from onyx.configs.app_configs import CONFLUENCE_TIMEZONE_OFFSET
 from onyx.configs.app_configs import CONTINUE_ON_CONNECTOR_FAILURE
@@ -18,10 +16,6 @@ from onyx.connectors.confluence.utils import build_confluence_document_id
 from onyx.connectors.confluence.utils import datetime_from_string
 from onyx.connectors.confluence.utils import extract_text_from_confluence_html
 from onyx.connectors.confluence.utils import validate_attachment_filetype
-from onyx.connectors.exceptions import ConnectorValidationError
-from onyx.connectors.exceptions import CredentialExpiredError
-from onyx.connectors.exceptions import InsufficientPermissionsError
-from onyx.connectors.exceptions import UnexpectedError
 from onyx.connectors.interfaces import GenerateDocumentsOutput
 from onyx.connectors.interfaces import GenerateSlimDocumentOutput
 from onyx.connectors.interfaces import LoadConnector
@@ -403,33 +397,3 @@ class ConfluenceConnector(LoadConnector, PollConnector, SlimConnector):
                     callback.progress("retrieve_all_slim_documents", 1)
 
         yield doc_metadata_list
-
-    def validate_connector_settings(self) -> None:
-        if self._confluence_client is None:
-            raise ConnectorMissingCredentialError("Confluence credentials not loaded.")
-
-        try:
-            spaces = self._confluence_client.get_all_spaces(limit=1)
-        except HTTPError as e:
-            status_code = e.response.status_code if e.response else None
-            if status_code == 401:
-                raise CredentialExpiredError(
-                    "Invalid or expired Confluence credentials (HTTP 401)."
-                )
-            elif status_code == 403:
-                raise InsufficientPermissionsError(
-                    "Insufficient permissions to access Confluence resources (HTTP 403)."
-                )
-            raise UnexpectedError(
-                f"Unexpected Confluence error (status={status_code}): {e}"
-            )
-        except Exception as e:
-            raise UnexpectedError(
-                f"Unexpected error while validating Confluence settings: {e}"
-            )
-
-        if not spaces or not spaces.get("results"):
-            raise ConnectorValidationError(
-                "No Confluence spaces found. Either your credentials lack permissions, or "
-                "there truly are no spaces in this Confluence instance."
-            )

backend/onyx/connectors/confluence/onyx_confluence.py

@@ -11,7 +11,6 @@ from atlassian import Confluence  # type:ignore
 from pydantic import BaseModel
 from requests import HTTPError
 
-from onyx.connectors.exceptions import ConnectorValidationError
 from onyx.utils.logger import setup_logger
 
 logger = setup_logger()
@@ -509,15 +508,11 @@ def build_confluence_client(
     is_cloud: bool,
     wiki_base: str,
 ) -> OnyxConfluence:
-    try:
-        _validate_connector_configuration(
-            credentials=credentials,
-            is_cloud=is_cloud,
-            wiki_base=wiki_base,
-        )
-    except Exception as e:
-        raise ConnectorValidationError(str(e))
-
+    _validate_connector_configuration(
+        credentials=credentials,
+        is_cloud=is_cloud,
+        wiki_base=wiki_base,
+    )
     return OnyxConfluence(
         api_version="cloud" if is_cloud else "latest",
         # Remove trailing slash from wiki_base if present

backend/onyx/connectors/dropbox/connector.py

@@ -10,10 +10,10 @@ from dropbox.files import FolderMetadata  # type:ignore
 
 from onyx.configs.app_configs import INDEX_BATCH_SIZE
 from onyx.configs.constants import DocumentSource
-from onyx.connectors.exceptions import ConnectorValidationError
-from onyx.connectors.exceptions import CredentialInvalidError
-from onyx.connectors.exceptions import InsufficientPermissionsError
+from onyx.connectors.interfaces import ConnectorValidationError
+from onyx.connectors.interfaces import CredentialInvalidError
 from onyx.connectors.interfaces import GenerateDocumentsOutput
+from onyx.connectors.interfaces import InsufficientPermissionsError
 from onyx.connectors.interfaces import LoadConnector
 from onyx.connectors.interfaces import PollConnector
 from onyx.connectors.interfaces import SecondsSinceUnixEpoch

backend/onyx/connectors/exceptions.py

@@ -1,49 +0,0 @@
-class ValidationError(Exception):
-    """General exception for validation errors."""
-
-    def __init__(self, message: str):
-        self.message = message
-        super().__init__(self.message)
-
-
-class ConnectorValidationError(ValidationError):
-    """General exception for connector validation errors."""
-
-    def __init__(self, message: str):
-        self.message = message
-        super().__init__(self.message)
-
-
-class UnexpectedError(ValidationError):
-    """Raised when an unexpected error occurs during connector validation.
-
-    Unexpected errors don't necessarily mean the credential is invalid,
-    but rather that there was an error during the validation process
-    or we encountered a currently unhandled error case.
-    """
-
-    def __init__(self, message: str = "Unexpected error during connector validation"):
-        super().__init__(message)
-
-
-class CredentialInvalidError(ConnectorValidationError):
-    """Raised when a connector's credential is invalid."""
-
-    def __init__(self, message: str = "Credential is invalid"):
-        super().__init__(message)
-
-
-class CredentialExpiredError(ConnectorValidationError):
-    """Raised when a connector's credential is expired."""
-
-    def __init__(self, message: str = "Credential has expired"):
-        super().__init__(message)
-
-
-class InsufficientPermissionsError(ConnectorValidationError):
-    """Raised when the credential does not have sufficient API permissions."""
-
-    def __init__(
-        self, message: str = "Insufficient permissions for the requested operation"
-    ):
-        super().__init__(message)

backend/onyx/connectors/factory.py

@@ -3,7 +3,6 @@ from typing import Type
 
 from sqlalchemy.orm import Session
 
-from onyx.configs.app_configs import INTEGRATION_TESTS_MODE
 from onyx.configs.constants import DocumentSource
 from onyx.configs.constants import DocumentSourceRequiringTenantContext
 from onyx.connectors.airtable.airtable_connector import AirtableConnector
@@ -18,7 +17,6 @@ from onyx.connectors.discourse.connector import DiscourseConnector
 from onyx.connectors.document360.connector import Document360Connector
 from onyx.connectors.dropbox.connector import DropboxConnector
 from onyx.connectors.egnyte.connector import EgnyteConnector
-from onyx.connectors.exceptions import ConnectorValidationError
 from onyx.connectors.file.connector import LocalFileConnector
 from onyx.connectors.fireflies.connector import FirefliesConnector
 from onyx.connectors.freshdesk.connector import FreshdeskConnector
@@ -33,6 +31,7 @@ from onyx.connectors.guru.connector import GuruConnector
 from onyx.connectors.hubspot.connector import HubSpotConnector
 from onyx.connectors.interfaces import BaseConnector
 from onyx.connectors.interfaces import CheckpointConnector
+from onyx.connectors.interfaces import ConnectorValidationError
 from onyx.connectors.interfaces import EventConnector
 from onyx.connectors.interfaces import LoadConnector
 from onyx.connectors.interfaces import PollConnector
@@ -56,8 +55,9 @@ from onyx.connectors.zendesk.connector import ZendeskConnector
 from onyx.connectors.zulip.connector import ZulipConnector
 from onyx.db.connector import fetch_connector_by_id
 from onyx.db.credentials import backend_update_credential_json
-from onyx.db.credentials import fetch_credential_by_id
+from onyx.db.credentials import fetch_credential_by_id_for_user
 from onyx.db.models import Credential
+from onyx.db.models import User
 
 
 class ConnectorMissingException(Exception):
@@ -184,30 +184,21 @@ def validate_ccpair_for_user(
     connector_id: int,
     credential_id: int,
     db_session: Session,
+    user: User | None,
     tenant_id: str | None,
-    enforce_creation: bool = True,
-) -> bool:
-    if INTEGRATION_TESTS_MODE:
-        return True
-
+) -> None:
     # Validate the connector settings
     connector = fetch_connector_by_id(connector_id, db_session)
-    credential = fetch_credential_by_id(
+    credential = fetch_credential_by_id_for_user(
         credential_id,
+        user,
         db_session,
+        get_editable=False,
     )
-
-    if not connector:
-        raise ValueError("Connector not found")
-
-    if (
-        connector.source == DocumentSource.INGESTION_API
-        or connector.source == DocumentSource.MOCK_CONNECTOR
-    ):
-        return True
-
     if not credential:
         raise ValueError("Credential not found")
+    if not connector:
+        raise ValueError("Connector not found")
 
     try:
         runnable_connector = instantiate_connector(
@@ -218,13 +209,7 @@ def validate_ccpair_for_user(
             credential=credential,
             tenant_id=tenant_id,
         )
-    except ConnectorValidationError as e:
-        raise e
     except Exception as e:
-        if enforce_creation:
-            raise ConnectorValidationError(str(e))
-        else:
-            return False
+        raise ConnectorValidationError(str(e))
 
     runnable_connector.validate_connector_settings()
-    return True

backend/onyx/connectors/gitbook/connector.py

@@ -229,20 +229,16 @@ class GitbookConnector(LoadConnector, PollConnector):
 
         try:
             content = self.client.get(f"/spaces/{self.space_id}/content")
-            pages: list[dict[str, Any]] = content.get("pages", [])
+            pages = content.get("pages", [])
+
             current_batch: list[Document] = []
+            for page in pages:
+                updated_at = datetime.fromisoformat(page["updatedAt"])
 
-            while pages:
-                page = pages.pop(0)
-
-                updated_at_raw = page.get("updatedAt")
-                if updated_at_raw is None:
-                    # if updatedAt is not present, that means the page has never been edited
-                    continue
-
-                updated_at = datetime.fromisoformat(updated_at_raw)
                 if start and updated_at < start:
-                    continue
+                    if current_batch:
+                        yield current_batch
+                    return
                 if end and updated_at > end:
                     continue
 
@@ -254,8 +250,6 @@ class GitbookConnector(LoadConnector, PollConnector):
                     yield current_batch
                     current_batch = []
 
-                pages.extend(page.get("pages", []))
-
             if current_batch:
                 yield current_batch
 

backend/onyx/connectors/github/connector.py

@@ -17,14 +17,14 @@ from github.PullRequest import PullRequest
 from onyx.configs.app_configs import GITHUB_CONNECTOR_BASE_URL
 from onyx.configs.app_configs import INDEX_BATCH_SIZE
 from onyx.configs.constants import DocumentSource
-from onyx.connectors.exceptions import ConnectorValidationError
-from onyx.connectors.exceptions import CredentialExpiredError
-from onyx.connectors.exceptions import InsufficientPermissionsError
-from onyx.connectors.exceptions import UnexpectedError
+from onyx.connectors.interfaces import ConnectorValidationError
+from onyx.connectors.interfaces import CredentialExpiredError
 from onyx.connectors.interfaces import GenerateDocumentsOutput
+from onyx.connectors.interfaces import InsufficientPermissionsError
 from onyx.connectors.interfaces import LoadConnector
 from onyx.connectors.interfaces import PollConnector
 from onyx.connectors.interfaces import SecondsSinceUnixEpoch
+from onyx.connectors.interfaces import UnexpectedError
 from onyx.connectors.models import ConnectorMissingCredentialError
 from onyx.connectors.models import Document
 from onyx.connectors.models import Section

backend/onyx/connectors/gmail/connector.py

@@ -305,7 +305,6 @@ class GmailConnector(LoadConnector, PollConnector, SlimConnector):
                     userId=user_email,
                     fields=THREAD_FIELDS,
                     id=thread["id"],
-                    continue_on_404_or_403=True,
                 )
                 # full_threads is an iterator containing a single thread
                 # so we need to convert it to a list and grab the first element
@@ -337,7 +336,6 @@ class GmailConnector(LoadConnector, PollConnector, SlimConnector):
                 userId=user_email,
                 fields=THREAD_LIST_FIELDS,
                 q=query,
-                continue_on_404_or_403=True,
             ):
                 doc_batch.append(
                     SlimDocument(

backend/onyx/connectors/google_drive/connector.py

@@ -13,9 +13,6 @@ from googleapiclient.errors import HttpError  # type: ignore
 from onyx.configs.app_configs import INDEX_BATCH_SIZE
 from onyx.configs.app_configs import MAX_FILE_SIZE_BYTES
 from onyx.configs.constants import DocumentSource
-from onyx.connectors.exceptions import ConnectorValidationError
-from onyx.connectors.exceptions import CredentialExpiredError
-from onyx.connectors.exceptions import InsufficientPermissionsError
 from onyx.connectors.google_drive.doc_conversion import build_slim_document
 from onyx.connectors.google_drive.doc_conversion import (
     convert_drive_item_to_document,
@@ -45,7 +42,6 @@ from onyx.connectors.interfaces import LoadConnector
 from onyx.connectors.interfaces import PollConnector
 from onyx.connectors.interfaces import SecondsSinceUnixEpoch
 from onyx.connectors.interfaces import SlimConnector
-from onyx.connectors.models import ConnectorMissingCredentialError
 from onyx.indexing.indexing_heartbeat import IndexingHeartbeatInterface
 from onyx.utils.logger import setup_logger
 from onyx.utils.retry_wrapper import retry_builder
@@ -141,7 +137,7 @@ class GoogleDriveConnector(LoadConnector, PollConnector, SlimConnector):
                 "Please visit the docs for help with the new setup: "
                 f"{SCOPE_DOC_URL}"
             )
-            raise ConnectorValidationError(
+            raise ValueError(
                 "Google Drive connector received old input parameters. "
                 "Please visit the docs for help with the new setup: "
                 f"{SCOPE_DOC_URL}"
@@ -155,7 +151,7 @@ class GoogleDriveConnector(LoadConnector, PollConnector, SlimConnector):
             and not my_drive_emails
             and not shared_drive_urls
         ):
-            raise ConnectorValidationError(
+            raise ValueError(
                 "Nothing to index. Please specify at least one of the following: "
                 "include_shared_drives, include_my_drives, include_files_shared_with_me, "
                 "shared_folder_urls, or my_drive_emails"
@@ -224,14 +220,7 @@ class GoogleDriveConnector(LoadConnector, PollConnector, SlimConnector):
         return self._creds
 
     def load_credentials(self, credentials: dict[str, Any]) -> dict[str, str] | None:
-        try:
-            self._primary_admin_email = credentials[DB_CREDENTIALS_PRIMARY_ADMIN_KEY]
-        except KeyError:
-            raise ValueError(
-                "Primary admin email missing, "
-                "should not call this property "
-                "before calling load_credentials"
-            )
+        self._primary_admin_email = credentials[DB_CREDENTIALS_PRIMARY_ADMIN_KEY]
 
         self._creds, new_creds_dict = get_google_creds(
             credentials=credentials,
@@ -613,50 +602,3 @@ class GoogleDriveConnector(LoadConnector, PollConnector, SlimConnector):
             if MISSING_SCOPES_ERROR_STR in str(e):
                 raise PermissionError(ONYX_SCOPE_INSTRUCTIONS) from e
             raise e
-
-    def validate_connector_settings(self) -> None:
-        if self._creds is None:
-            raise ConnectorMissingCredentialError(
-                "Google Drive credentials not loaded."
-            )
-
-        if self._primary_admin_email is None:
-            raise ConnectorValidationError(
-                "Primary admin email not found in credentials. "
-                "Ensure DB_CREDENTIALS_PRIMARY_ADMIN_KEY is set."
-            )
-
-        try:
-            drive_service = get_drive_service(self._creds, self._primary_admin_email)
-            drive_service.files().list(pageSize=1, fields="files(id)").execute()
-
-            if isinstance(self._creds, ServiceAccountCredentials):
-                retry_builder()(get_root_folder_id)(drive_service)
-
-        except HttpError as e:
-            status_code = e.resp.status if e.resp else None
-            if status_code == 401:
-                raise CredentialExpiredError(
-                    "Invalid or expired Google Drive credentials (401)."
-                )
-            elif status_code == 403:
-                raise InsufficientPermissionsError(
-                    "Google Drive app lacks required permissions (403). "
-                    "Please ensure the necessary scopes are granted and Drive "
-                    "apps are enabled."
-                )
-            else:
-                raise ConnectorValidationError(
-                    f"Unexpected Google Drive error (status={status_code}): {e}"
-                )
-
-        except Exception as e:
-            # Check for scope-related hints from the error message
-            if MISSING_SCOPES_ERROR_STR in str(e):
-                raise InsufficientPermissionsError(
-                    "Google Drive credentials are missing required scopes. "
-                    f"{ONYX_SCOPE_INSTRUCTIONS}"
-                )
-            raise ConnectorValidationError(
-                f"Unexpected error during Google Drive validation: {e}"
-            )

backend/onyx/connectors/hubspot/connector.py

@@ -87,18 +87,16 @@ class HubSpotConnector(LoadConnector, PollConnector):
                         contact = api_client.crm.contacts.basic_api.get_by_id(
                             contact_id=contact.id
                         )
-                        email = contact.properties.get("email")
-                        if email is not None:
-                            associated_emails.append(email)
+                        associated_emails.append(contact.properties["email"])
 
                 if notes:
                     for note in notes.results:
                         note = api_client.crm.objects.notes.basic_api.get_by_id(
                             note_id=note.id, properties=["content", "hs_body_preview"]
                         )
-                        preview = note.properties.get("hs_body_preview")
-                        if preview is not None:
-                            associated_notes.append(preview)
+                        if note.properties["hs_body_preview"] is None:
+                            continue
+                        associated_notes.append(note.properties["hs_body_preview"])
 
             associated_emails_str = " ,".join(associated_emails)
             associated_notes_str = " ".join(associated_notes)

backend/onyx/connectors/interfaces.py

@@ -146,3 +146,46 @@ class CheckpointConnector(BaseConnector):
         ```
         """
         raise NotImplementedError
+
+
+class ConnectorValidationError(Exception):
+    """General exception for connector validation errors."""
+
+    def __init__(self, message: str):
+        self.message = message
+        super().__init__(self.message)
+
+
+class UnexpectedError(Exception):
+    """Raised when an unexpected error occurs during connector validation.
+
+    Unexpected errors don't necessarily mean the credential is invalid,
+    but rather that there was an error during the validation process
+    or we encountered a currently unhandled error case.
+    """
+
+    def __init__(self, message: str = "Unexpected error during connector validation"):
+        super().__init__(message)
+
+
+class CredentialInvalidError(ConnectorValidationError):
+    """Raised when a connector's credential is invalid."""
+
+    def __init__(self, message: str = "Credential is invalid"):
+        super().__init__(message)
+
+
+class CredentialExpiredError(ConnectorValidationError):
+    """Raised when a connector's credential is expired."""
+
+    def __init__(self, message: str = "Credential has expired"):
+        super().__init__(message)
+
+
+class InsufficientPermissionsError(ConnectorValidationError):
+    """Raised when the credential does not have sufficient API permissions."""
+
+    def __init__(
+        self, message: str = "Insufficient permissions for the requested operation"
+    ):
+        super().__init__(message)

backend/onyx/connectors/notion/connector.py

@@ -16,11 +16,10 @@ from onyx.configs.constants import DocumentSource
 from onyx.connectors.cross_connector_utils.rate_limit_wrapper import (
     rl_requests,
 )
-from onyx.connectors.exceptions import ConnectorValidationError
-from onyx.connectors.exceptions import CredentialExpiredError
-from onyx.connectors.exceptions import InsufficientPermissionsError
-from onyx.connectors.exceptions import UnexpectedError
+from onyx.connectors.interfaces import ConnectorValidationError
+from onyx.connectors.interfaces import CredentialExpiredError
 from onyx.connectors.interfaces import GenerateDocumentsOutput
+from onyx.connectors.interfaces import InsufficientPermissionsError
 from onyx.connectors.interfaces import LoadConnector
 from onyx.connectors.interfaces import PollConnector
 from onyx.connectors.interfaces import SecondsSinceUnixEpoch
@@ -671,12 +670,12 @@ class NotionConnector(LoadConnector, PollConnector):
                     "Please try again later."
                 )
             else:
-                raise UnexpectedError(
+                raise Exception(
                     f"Unexpected Notion HTTP error (status={status_code}): {http_err}"
                 ) from http_err
 
         except Exception as exc:
-            raise UnexpectedError(
+            raise Exception(
                 f"Unexpected error during Notion settings validation: {exc}"
             )
 

backend/onyx/connectors/onyx_jira/connector.py

@@ -12,11 +12,11 @@ from onyx.configs.app_configs import JIRA_CONNECTOR_LABELS_TO_SKIP
 from onyx.configs.app_configs import JIRA_CONNECTOR_MAX_TICKET_SIZE
 from onyx.configs.constants import DocumentSource
 from onyx.connectors.cross_connector_utils.miscellaneous_utils import time_str_to_utc
-from onyx.connectors.exceptions import ConnectorValidationError
-from onyx.connectors.exceptions import CredentialExpiredError
-from onyx.connectors.exceptions import InsufficientPermissionsError
+from onyx.connectors.interfaces import ConnectorValidationError
+from onyx.connectors.interfaces import CredentialExpiredError
 from onyx.connectors.interfaces import GenerateDocumentsOutput
 from onyx.connectors.interfaces import GenerateSlimDocumentOutput
+from onyx.connectors.interfaces import InsufficientPermissionsError
 from onyx.connectors.interfaces import LoadConnector
 from onyx.connectors.interfaces import PollConnector
 from onyx.connectors.interfaces import SecondsSinceUnixEpoch

backend/onyx/connectors/slack/connector.py

@@ -18,10 +18,6 @@ from slack_sdk.errors import SlackApiError
 from onyx.configs.app_configs import ENABLE_EXPENSIVE_EXPERT_CALLS
 from onyx.configs.app_configs import INDEX_BATCH_SIZE
 from onyx.configs.constants import DocumentSource
-from onyx.connectors.exceptions import ConnectorValidationError
-from onyx.connectors.exceptions import CredentialExpiredError
-from onyx.connectors.exceptions import InsufficientPermissionsError
-from onyx.connectors.exceptions import UnexpectedError
 from onyx.connectors.interfaces import CheckpointConnector
 from onyx.connectors.interfaces import CheckpointOutput
 from onyx.connectors.interfaces import GenerateSlimDocumentOutput
@@ -86,14 +82,14 @@ def get_channels(
     get_public: bool = True,
     get_private: bool = True,
 ) -> list[ChannelType]:
-    """Get all channels in the workspace."""
+    """Get all channels in the workspace"""
     channels: list[dict[str, Any]] = []
     channel_types = []
     if get_public:
         channel_types.append("public_channel")
     if get_private:
         channel_types.append("private_channel")
-    # Try fetching both public and private channels first:
+    # try getting private channels as well at first
     try:
         channels = _collect_paginated_channels(
             client=client,
@@ -101,19 +97,19 @@ def get_channels(
             channel_types=channel_types,
         )
     except SlackApiError as e:
-        logger.info(
-            f"Unable to fetch private channels due to: {e}. Trying again without private channels."
-        )
+        logger.info(f"Unable to fetch private channels due to - {e}")
+        logger.info("trying again without private channels")
         if get_public:
             channel_types = ["public_channel"]
         else:
-            logger.warning("No channels to fetch.")
+            logger.warning("No channels to fetch")
             return []
         channels = _collect_paginated_channels(
             client=client,
             exclude_archived=exclude_archived,
             channel_types=channel_types,
         )
+
     return channels
 
 
@@ -670,86 +666,6 @@ class SlackConnector(SlimConnector, CheckpointConnector):
             )
             return checkpoint
 
-    def validate_connector_settings(self) -> None:
-        """
-        1. Verify the bot token is valid for the workspace (via auth_test).
-        2. Ensure the bot has enough scope to list channels.
-        3. Check that every channel specified in self.channels exists.
-        """
-        if self.client is None:
-            raise ConnectorMissingCredentialError("Slack credentials not loaded.")
-
-        try:
-            # 1) Validate connection to workspace
-            auth_response = self.client.auth_test()
-            if not auth_response.get("ok", False):
-                error_msg = auth_response.get(
-                    "error", "Unknown error from Slack auth_test"
-                )
-                raise ConnectorValidationError(f"Failed Slack auth_test: {error_msg}")
-
-            # 2) Minimal test to confirm listing channels works
-            test_resp = self.client.conversations_list(
-                limit=1, types=["public_channel"]
-            )
-            if not test_resp.get("ok", False):
-                error_msg = test_resp.get("error", "Unknown error from Slack")
-                if error_msg == "invalid_auth":
-                    raise ConnectorValidationError(
-                        f"Invalid Slack bot token ({error_msg})."
-                    )
-                elif error_msg == "not_authed":
-                    raise CredentialExpiredError(
-                        f"Invalid or expired Slack bot token ({error_msg})."
-                    )
-                raise UnexpectedError(f"Slack API returned a failure: {error_msg}")
-
-            # 3) If channels are specified, verify each is accessible
-            if self.channels:
-                accessible_channels = get_channels(
-                    client=self.client,
-                    exclude_archived=True,
-                    get_public=True,
-                    get_private=True,
-                )
-                # For quick lookups by name or ID, build a map:
-                accessible_channel_names = {ch["name"] for ch in accessible_channels}
-                accessible_channel_ids = {ch["id"] for ch in accessible_channels}
-
-                for user_channel in self.channels:
-                    if (
-                        user_channel not in accessible_channel_names
-                        and user_channel not in accessible_channel_ids
-                    ):
-                        raise ConnectorValidationError(
-                            f"Channel '{user_channel}' not found or inaccessible in this workspace."
-                        )
-
-        except SlackApiError as e:
-            slack_error = e.response.get("error", "")
-            if slack_error == "missing_scope":
-                raise InsufficientPermissionsError(
-                    "Slack bot token lacks the necessary scope to list/access channels. "
-                    "Please ensure your Slack app has 'channels:read' (and/or 'groups:read' for private channels)."
-                )
-            elif slack_error == "invalid_auth":
-                raise CredentialExpiredError(
-                    f"Invalid Slack bot token ({slack_error})."
-                )
-            elif slack_error == "not_authed":
-                raise CredentialExpiredError(
-                    f"Invalid or expired Slack bot token ({slack_error})."
-                )
-            raise UnexpectedError(
-                f"Unexpected Slack error '{slack_error}' during settings validation."
-            )
-        except ConnectorValidationError as e:
-            raise e
-        except Exception as e:
-            raise UnexpectedError(
-                f"Unexpected error during Slack settings validation: {e}"
-            )
-
 
 if __name__ == "__main__":
     import os

backend/onyx/connectors/teams/connector.py

@@ -5,7 +5,6 @@ from typing import Any
 
 import msal  # type: ignore
 from office365.graph_client import GraphClient  # type: ignore
-from office365.runtime.client_request_exception import ClientRequestException  # type: ignore
 from office365.teams.channels.channel import Channel  # type: ignore
 from office365.teams.chats.messages.message import ChatMessage  # type: ignore
 from office365.teams.team import Team  # type: ignore
@@ -13,10 +12,6 @@ from office365.teams.team import Team  # type: ignore
 from onyx.configs.app_configs import INDEX_BATCH_SIZE
 from onyx.configs.constants import DocumentSource
 from onyx.connectors.cross_connector_utils.miscellaneous_utils import time_str_to_utc
-from onyx.connectors.exceptions import ConnectorValidationError
-from onyx.connectors.exceptions import CredentialExpiredError
-from onyx.connectors.exceptions import InsufficientPermissionsError
-from onyx.connectors.exceptions import UnexpectedError
 from onyx.connectors.interfaces import GenerateDocumentsOutput
 from onyx.connectors.interfaces import LoadConnector
 from onyx.connectors.interfaces import PollConnector
@@ -284,50 +279,6 @@ class TeamsConnector(LoadConnector, PollConnector):
         end_datetime = datetime.fromtimestamp(end, timezone.utc)
         return self._fetch_from_teams(start=start_datetime, end=end_datetime)
 
-    def validate_connector_settings(self) -> None:
-        if self.graph_client is None:
-            raise ConnectorMissingCredentialError("Teams credentials not loaded.")
-
-        try:
-            # Minimal call to confirm we can retrieve Teams
-            found_teams = self._get_all_teams()
-
-        except ClientRequestException as e:
-            status_code = e.response.status_code
-            if status_code == 401:
-                raise CredentialExpiredError(
-                    "Invalid or expired Microsoft Teams credentials (401 Unauthorized)."
-                )
-            elif status_code == 403:
-                raise InsufficientPermissionsError(
-                    "Your app lacks sufficient permissions to read Teams (403 Forbidden)."
-                )
-            raise UnexpectedError(f"Unexpected error retrieving teams: {e}")
-
-        except Exception as e:
-            error_str = str(e).lower()
-            if (
-                "unauthorized" in error_str
-                or "401" in error_str
-                or "invalid_grant" in error_str
-            ):
-                raise CredentialExpiredError(
-                    "Invalid or expired Microsoft Teams credentials."
-                )
-            elif "forbidden" in error_str or "403" in error_str:
-                raise InsufficientPermissionsError(
-                    "App lacks required permissions to read from Microsoft Teams."
-                )
-            raise ConnectorValidationError(
-                f"Unexpected error during Teams validation: {e}"
-            )
-
-        if not found_teams:
-            raise ConnectorValidationError(
-                "No Teams found for the given credentials. "
-                "Either there are no Teams in this tenant, or your app does not have permission to view them."
-            )
-
 
 if __name__ == "__main__":
     connector = TeamsConnector(teams=os.environ["TEAMS"].split(","))

backend/onyx/connectors/web/connector.py

@@ -25,12 +25,12 @@ from onyx.configs.app_configs import WEB_CONNECTOR_OAUTH_CLIENT_SECRET
 from onyx.configs.app_configs import WEB_CONNECTOR_OAUTH_TOKEN_URL
 from onyx.configs.app_configs import WEB_CONNECTOR_VALIDATE_URLS
 from onyx.configs.constants import DocumentSource
-from onyx.connectors.exceptions import ConnectorValidationError
-from onyx.connectors.exceptions import CredentialExpiredError
-from onyx.connectors.exceptions import InsufficientPermissionsError
-from onyx.connectors.exceptions import UnexpectedError
+from onyx.connectors.interfaces import ConnectorValidationError
+from onyx.connectors.interfaces import CredentialExpiredError
 from onyx.connectors.interfaces import GenerateDocumentsOutput
+from onyx.connectors.interfaces import InsufficientPermissionsError
 from onyx.connectors.interfaces import LoadConnector
+from onyx.connectors.interfaces import UnexpectedError
 from onyx.connectors.models import Document
 from onyx.connectors.models import Section
 from onyx.file_processing.extract_file_text import read_pdf_file
@@ -440,10 +440,7 @@ class WebConnector(LoadConnector):
                 "No URL configured. Please provide at least one valid URL."
             )
 
-        if (
-            self.web_connector_type == WEB_CONNECTOR_VALID_SETTINGS.SITEMAP.value
-            or self.web_connector_type == WEB_CONNECTOR_VALID_SETTINGS.RECURSIVE.value
-        ):
+        if self.web_connector_type == WEB_CONNECTOR_VALID_SETTINGS.SITEMAP.value:
             return None
 
         # We'll just test the first URL for connectivity and correctness

backend/onyx/db/connector_credential_pair.py

@@ -194,14 +194,9 @@ def get_connector_credential_pair_from_id_for_user(
 def get_connector_credential_pair_from_id(
     db_session: Session,
     cc_pair_id: int,
-    eager_load_credential: bool = False,
 ) -> ConnectorCredentialPair | None:
     stmt = select(ConnectorCredentialPair).distinct()
     stmt = stmt.where(ConnectorCredentialPair.id == cc_pair_id)
-
-    if eager_load_credential:
-        stmt = stmt.options(joinedload(ConnectorCredentialPair.credential))
-
     result = db_session.execute(stmt)
     return result.scalar_one_or_none()
 
@@ -401,8 +396,8 @@ def add_credential_to_connector(
     # If we are in the seeding flow, we shouldn't need to check if the credential belongs to the user
     if seeding_flow:
         credential = fetch_credential_by_id(
-            credential_id=credential_id,
             db_session=db_session,
+            credential_id=credential_id,
         )
     else:
         credential = fetch_credential_by_id_for_user(

backend/onyx/db/credentials.py

@@ -169,8 +169,8 @@ def fetch_credential_by_id_for_user(
 
 
 def fetch_credential_by_id(
-    credential_id: int,
     db_session: Session,
+    credential_id: int,
 ) -> Credential | None:
     stmt = select(Credential).distinct()
     stmt = stmt.where(Credential.id == credential_id)
@@ -422,8 +422,8 @@ def create_initial_public_credential(db_session: Session) -> None:
         "There must exist an empty public credential for data connectors that do not require additional Auth."
     )
     first_credential = fetch_credential_by_id(
-        credential_id=PUBLIC_CREDENTIAL_ID,
         db_session=db_session,
+        credential_id=PUBLIC_CREDENTIAL_ID,
     )
 
     if first_credential is not None:

backend/onyx/db/document.py

@@ -60,8 +60,9 @@ def count_documents_by_needs_sync(session: Session) -> int:
     This function executes the query and returns the count of
     documents matching the criteria."""
 
-    return (
-        session.query(DbDocument.id)
+    count = (
+        session.query(func.count(DbDocument.id.distinct()))
+        .select_from(DbDocument)
         .join(
             DocumentByConnectorCredentialPair,
             DbDocument.id == DocumentByConnectorCredentialPair.id,
@@ -72,53 +73,63 @@ def count_documents_by_needs_sync(session: Session) -> int:
                 DbDocument.last_synced.is_(None),
             )
         )
-        .count()
+        .scalar()
     )
 
+    return count
+
 
 def construct_document_select_for_connector_credential_pair_by_needs_sync(
     connector_id: int, credential_id: int
 ) -> Select:
-    return (
-        select(DbDocument)
-        .join(
-            DocumentByConnectorCredentialPair,
-            DbDocument.id == DocumentByConnectorCredentialPair.id,
-        )
-        .where(
-            and_(
-                DocumentByConnectorCredentialPair.connector_id == connector_id,
-                DocumentByConnectorCredentialPair.credential_id == credential_id,
-                or_(
-                    DbDocument.last_modified > DbDocument.last_synced,
-                    DbDocument.last_synced.is_(None),
-                ),
-            )
+    initial_doc_ids_stmt = select(DocumentByConnectorCredentialPair.id).where(
+        and_(
+            DocumentByConnectorCredentialPair.connector_id == connector_id,
+            DocumentByConnectorCredentialPair.credential_id == credential_id,
         )
     )
 
+    stmt = (
+        select(DbDocument)
+        .where(
+            DbDocument.id.in_(initial_doc_ids_stmt),
+            or_(
+                DbDocument.last_modified
+                > DbDocument.last_synced,  # last_modified is newer than last_synced
+                DbDocument.last_synced.is_(None),  # never synced
+            ),
+        )
+        .distinct()
+    )
+
+    return stmt
+
 
 def construct_document_id_select_for_connector_credential_pair_by_needs_sync(
     connector_id: int, credential_id: int
 ) -> Select:
-    return (
-        select(DbDocument.id)
-        .join(
-            DocumentByConnectorCredentialPair,
-            DbDocument.id == DocumentByConnectorCredentialPair.id,
-        )
-        .where(
-            and_(
-                DocumentByConnectorCredentialPair.connector_id == connector_id,
-                DocumentByConnectorCredentialPair.credential_id == credential_id,
-                or_(
-                    DbDocument.last_modified > DbDocument.last_synced,
-                    DbDocument.last_synced.is_(None),
-                ),
-            )
+    initial_doc_ids_stmt = select(DocumentByConnectorCredentialPair.id).where(
+        and_(
+            DocumentByConnectorCredentialPair.connector_id == connector_id,
+            DocumentByConnectorCredentialPair.credential_id == credential_id,
         )
     )
 
+    stmt = (
+        select(DbDocument.id)
+        .where(
+            DbDocument.id.in_(initial_doc_ids_stmt),
+            or_(
+                DbDocument.last_modified
+                > DbDocument.last_synced,  # last_modified is newer than last_synced
+                DbDocument.last_synced.is_(None),  # never synced
+            ),
+        )
+        .distinct()
+    )
+
+    return stmt
+
 
 def get_all_documents_needing_vespa_sync_for_cc_pair(
     db_session: Session, cc_pair_id: int

backend/onyx/db/models.py

@@ -570,14 +570,6 @@ class Document(Base):
         back_populates="documents",
     )
 
-    __table_args__ = (
-        Index(
-            "ix_document_sync_status",
-            last_modified,
-            last_synced,
-        ),
-    )
-
 
 class Tag(Base):
     __tablename__ = "tag"

backend/onyx/file_store/utils.py

@@ -8,7 +8,7 @@ import requests
 from sqlalchemy.orm import Session
 
 from onyx.configs.constants import FileOrigin
-from onyx.db.engine import get_session_with_current_tenant
+from onyx.db.engine import get_session_with_tenant
 from onyx.db.models import ChatMessage
 from onyx.file_store.file_store import get_default_file_store
 from onyx.file_store.models import FileDescriptor
@@ -53,11 +53,11 @@ def load_all_chat_files(
     return files
 
 
-def save_file_from_url(url: str) -> str:
+def save_file_from_url(url: str, tenant_id: str) -> str:
     """NOTE: using multiple sessions here, since this is often called
     using multithreading. In practice, sharing a session has resulted in
     weird errors."""
-    with get_session_with_current_tenant() as db_session:
+    with get_session_with_tenant(tenant_id=tenant_id) as db_session:
         response = requests.get(url)
         response.raise_for_status()
 
@@ -75,8 +75,8 @@ def save_file_from_url(url: str) -> str:
         return unique_id
 
 
-def save_file_from_base64(base64_string: str) -> str:
-    with get_session_with_current_tenant() as db_session:
+def save_file_from_base64(base64_string: str, tenant_id: str) -> str:
+    with get_session_with_tenant(tenant_id=tenant_id) as db_session:
         unique_id = str(uuid4())
         file_store = get_default_file_store(db_session)
         file_store.save_file(
@@ -90,12 +90,14 @@ def save_file_from_base64(base64_string: str) -> str:
 
 
 def save_file(
+    tenant_id: str,
     url: str | None = None,
     base64_data: str | None = None,
 ) -> str:
     """Save a file from either a URL or base64 encoded string.
 
     Args:
+        tenant_id: The tenant ID to save the file under
         url: URL to download file from
         base64_data: Base64 encoded file data
 
@@ -109,22 +111,22 @@ def save_file(
         raise ValueError("Cannot specify both url and base64_data")
 
     if url is not None:
-        return save_file_from_url(url)
+        return save_file_from_url(url, tenant_id)
     elif base64_data is not None:
-        return save_file_from_base64(base64_data)
+        return save_file_from_base64(base64_data, tenant_id)
     else:
         raise ValueError("Must specify either url or base64_data")
 
 
-def save_files(urls: list[str], base64_files: list[str]) -> list[str]:
+def save_files(urls: list[str], base64_files: list[str], tenant_id: str) -> list[str]:
     # NOTE: be explicit about typing so that if we change things, we get notified
     funcs: list[
         tuple[
-            Callable[[str | None, str | None], str],
-            tuple[str | None, str | None],
+            Callable[[str, str | None, str | None], str],
+            tuple[str, str | None, str | None],
         ]
-    ] = [(save_file, (url, None)) for url in urls] + [
-        (save_file, (None, base64_file)) for base64_file in base64_files
+    ] = [(save_file, (tenant_id, url, None)) for url in urls] + [
+        (save_file, (tenant_id, None, base64_file)) for base64_file in base64_files
     ]
 
     return run_functions_tuples_in_parallel(funcs)

backend/onyx/indexing/indexing_pipeline.py

@@ -173,10 +173,7 @@ def index_doc_batch_with_handler(
             tenant_id=tenant_id,
         )
     except Exception as e:
-        # don't log the batch directly, it's too much text
-        document_ids = [doc.id for doc in document_batch]
-        logger.exception(f"Failed to index document batch: {document_ids}")
-
+        logger.exception(f"Failed to index document batch: {document_batch}")
         index_pipeline_result = IndexingPipelineResult(
             new_docs=0,
             total_docs=len(document_batch),

backend/onyx/llm/models.py

@@ -23,7 +23,6 @@ class PreviousMessage(BaseModel):
     message_type: MessageType
     files: list[InMemoryChatFile]
     tool_call: ToolCallFinalResult | None
-    refined_answer_improvement: bool | None
 
     @classmethod
     def from_chat_message(
@@ -48,7 +47,6 @@ class PreviousMessage(BaseModel):
             )
             if chat_message.tool_call
             else None,
-            refined_answer_improvement=chat_message.refined_answer_improvement,
         )
 
     def to_langchain_msg(self) -> BaseMessage:

backend/onyx/prompts/chat_prompts.py

@@ -18,7 +18,6 @@ Remember to provide inline citations in the format [1], [2], [3], etc.
 
 ADDITIONAL_INFO = "\n\nAdditional Information:\n\t- {datetime_info}."
 
-CODE_BLOCK_MARKDOWN = "Formatting re-enabled. "
 
 CHAT_USER_PROMPT = f"""
 Refer to the following context documents when responding to me.{{optional_ignore_statement}}

backend/onyx/redis/redis_connector_delete.py

@@ -33,12 +33,6 @@ class RedisConnectorDelete:
     FENCE_PREFIX = f"{PREFIX}_fence"  # "connectordeletion_fence"
     TASKSET_PREFIX = f"{PREFIX}_taskset"  # "connectordeletion_taskset"
 
-    # used to signal the overall workflow is still active
-    # it's impossible to get the exact state of the system at a single point in time
-    # so we need a signal with a TTL to bridge gaps in our checks
-    ACTIVE_PREFIX = PREFIX + "_active"
-    ACTIVE_TTL = 3600
-
     def __init__(self, tenant_id: str | None, id: int, redis: redis.Redis) -> None:
         self.tenant_id: str | None = tenant_id
         self.id = id
@@ -47,8 +41,6 @@ class RedisConnectorDelete:
         self.fence_key: str = f"{self.FENCE_PREFIX}_{id}"
         self.taskset_key = f"{self.TASKSET_PREFIX}_{id}"
 
-        self.active_key = f"{self.ACTIVE_PREFIX}_{id}"
-
     def taskset_clear(self) -> None:
         self.redis.delete(self.taskset_key)
 
@@ -85,20 +77,6 @@ class RedisConnectorDelete:
         self.redis.set(self.fence_key, payload.model_dump_json())
         self.redis.sadd(OnyxRedisConstants.ACTIVE_FENCES, self.fence_key)
 
-    def set_active(self) -> None:
-        """This sets a signal to keep the permissioning flow from getting cleaned up within
-        the expiration time.
-
-        The slack in timing is needed to avoid race conditions where simply checking
-        the celery queue and task status could result in race conditions."""
-        self.redis.set(self.active_key, 0, ex=self.ACTIVE_TTL)
-
-    def active(self) -> bool:
-        if self.redis.exists(self.active_key):
-            return True
-
-        return False
-
     def _generate_task_id(self) -> str:
         # celery's default task id format is "dd32ded3-00aa-4884-8b21-42f8332e7fac"
         # we prefix the task id so it's easier to keep track of who created the task
@@ -163,7 +141,6 @@ class RedisConnectorDelete:
 
     def reset(self) -> None:
         self.redis.srem(OnyxRedisConstants.ACTIVE_FENCES, self.fence_key)
-        self.redis.delete(self.active_key)
         self.redis.delete(self.taskset_key)
         self.redis.delete(self.fence_key)
 
@@ -176,9 +153,6 @@ class RedisConnectorDelete:
     @staticmethod
     def reset_all(r: redis.Redis) -> None:
         """Deletes all redis values for all connectors"""
-        for key in r.scan_iter(RedisConnectorDelete.ACTIVE_PREFIX + "*"):
-            r.delete(key)
-
         for key in r.scan_iter(RedisConnectorDelete.TASKSET_PREFIX + "*"):
             r.delete(key)
 

backend/onyx/server/documents/cc_pair.py

@@ -25,8 +25,8 @@ from onyx.background.celery.versioned_apps.primary import app as primary_app
 from onyx.background.indexing.models import IndexAttemptErrorPydantic
 from onyx.configs.constants import OnyxCeleryPriority
 from onyx.configs.constants import OnyxCeleryTask
-from onyx.connectors.exceptions import ValidationError
 from onyx.connectors.factory import validate_ccpair_for_user
+from onyx.connectors.interfaces import ConnectorValidationError
 from onyx.db.connector import delete_connector
 from onyx.db.connector_credential_pair import add_credential_to_connector
 from onyx.db.connector_credential_pair import (
@@ -620,7 +620,9 @@ def associate_credential_to_connector(
     )
 
     try:
-        validate_ccpair_for_user(connector_id, credential_id, db_session, tenant_id)
+        validate_ccpair_for_user(
+            connector_id, credential_id, db_session, user, tenant_id
+        )
 
         response = add_credential_to_connector(
             db_session=db_session,
@@ -647,7 +649,7 @@ def associate_credential_to_connector(
 
         return response
 
-    except ValidationError as e:
+    except ConnectorValidationError as e:
         # If validation fails, delete the connector and commit the changes
         # Ensures we don't leave invalid connectors in the database
         # NOTE: consensus is that it makes sense to unify connector and ccpair creation flows
@@ -658,6 +660,7 @@ def associate_credential_to_connector(
         raise HTTPException(
             status_code=400, detail="Connector validation error: " + str(e)
         )
+
     except IntegrityError as e:
         logger.error(f"IntegrityError: {e}")
         raise HTTPException(status_code=400, detail="Name must be unique")

backend/onyx/server/documents/connector.py

@@ -28,7 +28,6 @@ from onyx.configs.constants import FileOrigin
 from onyx.configs.constants import MilestoneRecordType
 from onyx.configs.constants import OnyxCeleryPriority
 from onyx.configs.constants import OnyxCeleryTask
-from onyx.connectors.exceptions import ConnectorValidationError
 from onyx.connectors.factory import validate_ccpair_for_user
 from onyx.connectors.google_utils.google_auth import (
     get_google_oauth_creds,
@@ -63,6 +62,7 @@ from onyx.connectors.google_utils.shared_constants import DB_CREDENTIALS_DICT_TO
 from onyx.connectors.google_utils.shared_constants import (
     GoogleOAuthAuthenticationMethod,
 )
+from onyx.connectors.interfaces import ConnectorValidationError
 from onyx.db.connector import create_connector
 from onyx.db.connector import delete_connector
 from onyx.db.connector import fetch_connector_by_id
@@ -854,6 +854,7 @@ def create_connector_with_mock_credential(
             connector_id=connector_id,
             credential_id=credential_id,
             db_session=db_session,
+            user=user,
             tenant_id=tenant_id,
         )
         response = add_credential_to_connector(

backend/onyx/server/documents/credential.py

@@ -106,6 +106,7 @@ def swap_credentials_for_connector(
         credential_swap_req.connector_id,
         credential_swap_req.new_credential_id,
         db_session,
+        user,
         tenant_id,
     )
 

backend/onyx/server/manage/slack_bot.py

@@ -1,3 +1,5 @@
+from typing import Any
+
 from fastapi import APIRouter
 from fastapi import Depends
 from fastapi import HTTPException
@@ -343,9 +345,6 @@ def list_bot_configs(
     ]
 
 
-MAX_CHANNELS = 200
-
-
 @router.get(
     "/admin/slack-app/bots/{bot_id}/channels",
 )
@@ -354,40 +353,38 @@ def get_all_channels_from_slack_api(
     db_session: Session = Depends(get_session),
     _: User | None = Depends(current_admin_user),
 ) -> list[SlackChannel]:
-    """
-    Fetches all channels from the Slack API.
-    If the workspace has 200 or more channels, we raise an error.
-    """
     tokens = fetch_slack_bot_tokens(db_session, bot_id)
     if not tokens or "bot_token" not in tokens:
         raise HTTPException(
             status_code=404, detail="Bot token not found for the given bot ID"
         )
 
-    client = WebClient(token=tokens["bot_token"])
+    bot_token = tokens["bot_token"]
+    client = WebClient(token=bot_token)
 
     try:
-        response = client.conversations_list(
-            types="public_channel,private_channel",
-            exclude_archived=True,
-            limit=MAX_CHANNELS,
-        )
-
-        channels = [
-            SlackChannel(id=channel["id"], name=channel["name"])
-            for channel in response["channels"]
-        ]
-
-        if len(channels) == MAX_CHANNELS:
-            raise HTTPException(
-                status_code=400,
-                detail=f"Workspace has {MAX_CHANNELS} or more channels.",
+        channels = []
+        cursor = None
+        while True:
+            response = client.conversations_list(
+                types="public_channel,private_channel",
+                exclude_archived=True,
+                limit=1000,
+                cursor=cursor,
             )
+            for channel in response["channels"]:
+                channels.append(SlackChannel(id=channel["id"], name=channel["name"]))
+
+            response_metadata: dict[str, Any] = response.get("response_metadata", {})
+            if isinstance(response_metadata, dict):
+                cursor = response_metadata.get("next_cursor")
+                if not cursor:
+                    break
+            else:
+                break
 
         return channels
-
     except SlackApiError as e:
         raise HTTPException(
-            status_code=500,
-            detail=f"Error fetching channels from Slack API: {str(e)}",
+            status_code=500, detail=f"Error fetching channels from Slack API: {str(e)}"
         )

backend/onyx/server/manage/users.py

@@ -311,23 +311,19 @@ def bulk_invite_users(
     all_emails = list(set(new_invited_emails) | set(initial_invited_users))
     number_of_invited_users = write_invited_users(all_emails)
 
-    # send out email invitations if enabled
-    if ENABLE_EMAIL_INVITES:
-        try:
-            for email in new_invited_emails:
-                send_user_email_invite(email, current_user, AUTH_TYPE)
-        except Exception as e:
-            logger.error(f"Error sending email invite to invited users: {e}")
-
     if not MULTI_TENANT:
         return number_of_invited_users
-
-    # for billing purposes, write to the control plane about the number of new users
     try:
         logger.info("Registering tenant users")
         fetch_ee_implementation_or_noop(
             "onyx.server.tenants.billing", "register_tenant_users", None
         )(tenant_id, get_total_users_count(db_session))
+        if ENABLE_EMAIL_INVITES:
+            try:
+                for email in new_invited_emails:
+                    send_user_email_invite(email, current_user)
+            except Exception as e:
+                logger.error(f"Error sending email invite to invited users: {e}")
 
         return number_of_invited_users
     except Exception as e:

backend/onyx/server/query_and_chat/token_limit.py

@@ -13,6 +13,7 @@ from sqlalchemy.orm import Session
 
 from onyx.auth.users import current_chat_accesssible_user
 from onyx.db.engine import get_session_context_manager
+from onyx.db.engine import get_session_with_tenant
 from onyx.db.models import ChatMessage
 from onyx.db.models import ChatSession
 from onyx.db.models import TokenRateLimit
@@ -20,6 +21,7 @@ from onyx.db.models import User
 from onyx.db.token_limit import fetch_all_global_token_rate_limits
 from onyx.utils.logger import setup_logger
 from onyx.utils.variable_functionality import fetch_versioned_implementation
+from shared_configs.contextvars import get_current_tenant_id
 
 
 logger = setup_logger()
@@ -37,13 +39,13 @@ def check_token_rate_limits(
         return
 
     versioned_rate_limit_strategy = fetch_versioned_implementation(
-        "onyx.server.query_and_chat.token_limit", _check_token_rate_limits.__name__
+        "onyx.server.query_and_chat.token_limit", "_check_token_rate_limits"
     )
-    return versioned_rate_limit_strategy(user)
+    return versioned_rate_limit_strategy(user, get_current_tenant_id())
 
 
-def _check_token_rate_limits(_: User | None) -> None:
-    _user_is_rate_limited_by_global()
+def _check_token_rate_limits(_: User | None, tenant_id: str | None) -> None:
+    _user_is_rate_limited_by_global(tenant_id)
 
 
 """
@@ -51,8 +53,8 @@ Global rate limits
 """
 
 
-def _user_is_rate_limited_by_global() -> None:
-    with get_session_context_manager() as db_session:
+def _user_is_rate_limited_by_global(tenant_id: str | None) -> None:
+    with get_session_with_tenant(tenant_id=tenant_id) as db_session:
         global_rate_limits = fetch_all_global_token_rate_limits(
             db_session=db_session, enabled_only=True, ordered=False
         )

backend/onyx/server/settings/models.py

@@ -45,10 +45,10 @@ class Settings(BaseModel):
     gpu_enabled: bool | None = None
     application_status: ApplicationStatus = ApplicationStatus.ACTIVE
     anonymous_user_enabled: bool | None = None
-    pro_search_enabled: bool | None = None
+    pro_search_disabled: bool | None = None
 
-    temperature_override_enabled: bool | None = False
-    auto_scroll: bool | None = False
+    temperature_override_enabled: bool = False
+    auto_scroll: bool = False
 
 
 class UserSettings(Settings):

backend/onyx/utils/logger.py

@@ -253,8 +253,3 @@ def print_loggers() -> None:
 
         print(f"  Propagate: {logger.propagate}")
         print()
-
-
-def format_error_for_logging(e: Exception) -> str:
-    """Clean error message by removing newlines for better logging."""
-    return str(e).replace("\n", " ")

backend/onyx/utils/threadpool_concurrency.py

@@ -1,4 +1,3 @@
-import contextvars
 import threading
 import uuid
 from collections.abc import Callable
@@ -15,6 +14,10 @@ logger = setup_logger()
 R = TypeVar("R")
 
 
+# WARNING: it is not currently well understood whether we lose access to contextvars when functions are
+# executed through this wrapper Do NOT try to acquire a db session in a function run through this unless
+# you have heavily tested that multi-tenancy is respected. If/when we know for sure that it is or
+# is not safe, update this comment.
 def run_functions_tuples_in_parallel(
     functions_with_args: list[tuple[Callable, tuple]],
     allow_failures: bool = False,
@@ -42,11 +45,8 @@ def run_functions_tuples_in_parallel(
 
     results = []
     with ThreadPoolExecutor(max_workers=workers) as executor:
-        # The primary reason for propagating contextvars is to allow acquiring a db session
-        # that respects tenant id. Context.run is expected to be low-overhead, but if we later
-        # find that it is increasing latency we can make using it optional.
         future_to_index = {
-            executor.submit(contextvars.copy_context().run, func, *args): i
+            executor.submit(func, *args): i
             for i, (func, args) in enumerate(functions_with_args)
         }
 
@@ -83,6 +83,10 @@ class FunctionCall(Generic[R]):
         return self.func(*self.args, **self.kwargs)
 
 
+# WARNING: it is not currently well understood whether we lose access to contextvars when functions are
+# executed through this wrapper Do NOT try to acquire a db session in a function run through this unless
+# you have heavily tested that multi-tenancy is respected. If/when we know for sure that it is or
+# is not safe, update this comment.
 def run_functions_in_parallel(
     function_calls: list[FunctionCall],
     allow_failures: bool = False,
@@ -98,9 +102,7 @@ def run_functions_in_parallel(
 
     with ThreadPoolExecutor(max_workers=len(function_calls)) as executor:
         future_to_id = {
-            executor.submit(
-                contextvars.copy_context().run, func_call.execute
-            ): func_call.result_id
+            executor.submit(func_call.execute): func_call.result_id
             for func_call in function_calls
         }
 
@@ -141,6 +143,10 @@ class TimeoutThread(threading.Thread):
         )
 
 
+# WARNING: it is not currently well understood whether we lose access to contextvars when functions are
+# executed through this wrapper Do NOT try to acquire a db session in a function run through this unless
+# you have heavily tested that multi-tenancy is respected. If/when we know for sure that it is or
+# is not safe, update this comment.
 def run_with_timeout(
     timeout: float, func: Callable[..., R], *args: Any, **kwargs: Any
 ) -> R:
@@ -148,8 +154,7 @@ def run_with_timeout(
     Executes a function with a timeout. If the function doesn't complete within the specified
     timeout, raises TimeoutError.
     """
-    context = contextvars.copy_context()
-    task = TimeoutThread(timeout, context.run, func, *args, **kwargs)
+    task = TimeoutThread(timeout, func, *args, **kwargs)
     task.start()
     task.join(timeout)
 

backend/requirements/ee.txt

@@ -1,4 +1,3 @@
 cohere==5.6.1
 posthog==3.7.4
 python3-saml==1.15.0
-xmlsec==1.3.14

backend/scripts/debugging/onyx_redis.py

@@ -3,7 +3,6 @@ import json
 import logging
 import sys
 import time
-from enum import Enum
 from logging import getLogger
 from typing import cast
 from uuid import UUID
@@ -21,13 +20,10 @@ from onyx.configs.app_configs import REDIS_PORT
 from onyx.configs.app_configs import REDIS_SSL
 from onyx.db.engine import get_session_with_tenant
 from onyx.db.users import get_user_by_email
-from onyx.redis.redis_connector import RedisConnector
-from onyx.redis.redis_connector_index import RedisConnectorIndex
 from onyx.redis.redis_pool import RedisPool
 from shared_configs.configs import MULTI_TENANT
 from shared_configs.configs import POSTGRES_DEFAULT_SCHEMA
 from shared_configs.contextvars import CURRENT_TENANT_ID_CONTEXTVAR
-from shared_configs.contextvars import get_current_tenant_id
 
 # Tool to run helpful operations on Redis in production
 # This is targeted for internal usage and may not have all the necessary parameters
@@ -46,19 +42,6 @@ SCAN_ITER_COUNT = 10000
 BATCH_DEFAULT = 1000
 
 
-class OnyxRedisCommand(Enum):
-    purge_connectorsync_taskset = "purge_connectorsync_taskset"
-    purge_documentset_taskset = "purge_documentset_taskset"
-    purge_usergroup_taskset = "purge_usergroup_taskset"
-    purge_locks_blocking_deletion = "purge_locks_blocking_deletion"
-    purge_vespa_syncing = "purge_vespa_syncing"
-    get_user_token = "get_user_token"
-    delete_user_token = "delete_user_token"
-
-    def __str__(self) -> str:
-        return self.value
-
-
 def get_user_id(user_email: str) -> tuple[UUID, str]:
     tenant_id = (
         get_tenant_id_for_email(user_email) if MULTI_TENANT else POSTGRES_DEFAULT_SCHEMA
@@ -72,79 +55,50 @@ def get_user_id(user_email: str) -> tuple[UUID, str]:
 
 
 def onyx_redis(
-    command: OnyxRedisCommand,
+    command: str,
     batch: int,
     dry_run: bool,
-    ssl: bool,
     host: str,
     port: int,
     db: int,
     password: str | None,
     user_email: str | None = None,
-    cc_pair_id: int | None = None,
 ) -> int:
-    # this is global and not tenant aware
     pool = RedisPool.create_pool(
         host=host,
         port=port,
         db=db,
         password=password if password else "",
-        ssl=ssl,
+        ssl=REDIS_SSL,
         ssl_cert_reqs="optional",
         ssl_ca_certs=None,
     )
 
     r = Redis(connection_pool=pool)
 
-    logger.info("Redis ping starting. This may hang if your settings are incorrect.")
-
     try:
         r.ping()
     except:
         logger.exception("Redis ping exceptioned")
         raise
 
-    logger.info("Redis ping succeeded.")
-
-    if command == OnyxRedisCommand.purge_connectorsync_taskset:
+    if command == "purge_connectorsync_taskset":
         """Purge connector tasksets. Used when the tasks represented in the tasksets
         have been purged."""
         return purge_by_match_and_type(
             "*connectorsync_taskset*", "set", batch, dry_run, r
         )
-    elif command == OnyxRedisCommand.purge_documentset_taskset:
+    elif command == "purge_documentset_taskset":
         return purge_by_match_and_type(
             "*documentset_taskset*", "set", batch, dry_run, r
         )
-    elif command == OnyxRedisCommand.purge_usergroup_taskset:
+    elif command == "purge_usergroup_taskset":
         return purge_by_match_and_type("*usergroup_taskset*", "set", batch, dry_run, r)
-    elif command == OnyxRedisCommand.purge_locks_blocking_deletion:
-        if cc_pair_id is None:
-            logger.error("You must specify --cc-pair with purge_deletion_locks")
-            return 1
-
-        tenant_id = get_current_tenant_id()
-        logger.info(f"Purging locks associated with deleting cc_pair={cc_pair_id}.")
-        redis_connector = RedisConnector(tenant_id, cc_pair_id)
-
-        match_pattern = f"{tenant_id}:{RedisConnectorIndex.FENCE_PREFIX}_{cc_pair_id}/*"
-        purge_by_match_and_type(match_pattern, "string", batch, dry_run, r)
-
-        redis_delete_if_exists_helper(
-            f"{tenant_id}:{redis_connector.prune.fence_key}", dry_run, r
-        )
-        redis_delete_if_exists_helper(
-            f"{tenant_id}:{redis_connector.permissions.fence_key}", dry_run, r
-        )
-        redis_delete_if_exists_helper(
-            f"{tenant_id}:{redis_connector.external_group_sync.fence_key}", dry_run, r
-        )
-        return 0
-    elif command == OnyxRedisCommand.purge_vespa_syncing:
+    elif command == "purge_vespa_syncing":
         return purge_by_match_and_type(
             "*connectorsync:vespa_syncing*", "string", batch, dry_run, r
         )
-    elif command == OnyxRedisCommand.get_user_token:
+    elif command == "get_user_token":
         if not user_email:
             logger.error("You must specify --user-email with get_user_token")
             return 1
@@ -155,7 +109,7 @@ def onyx_redis(
         else:
             print(f"No token found for user {user_email}")
             return 2
-    elif command == OnyxRedisCommand.delete_user_token:
+    elif command == "delete_user_token":
         if not user_email:
             logger.error("You must specify --user-email with delete_user_token")
             return 1
@@ -177,25 +131,6 @@ def flush_batch_delete(batch_keys: list[bytes], r: Redis) -> None:
         pipe.execute()
 
 
-def redis_delete_if_exists_helper(key: str, dry_run: bool, r: Redis) -> bool:
-    """Returns True if the key was found, False if not.
-    This function exists for logging purposes as the delete operation itself
-    doesn't really need to check the existence of the key.
-    """
-
-    if not r.exists(key):
-        logger.info(f"Did not find {key}.")
-        return False
-
-    if dry_run:
-        logger.info(f"(DRY-RUN) Deleting {key}.")
-    else:
-        logger.info(f"Deleting {key}.")
-        r.delete(key)
-
-    return True
-
-
 def purge_by_match_and_type(
     match_pattern: str, match_type: str, batch_size: int, dry_run: bool, r: Redis
 ) -> int:
@@ -203,12 +138,6 @@ def purge_by_match_and_type(
     match_type: https://redis.io/docs/latest/commands/type/
     """
 
-    logger.info(
-        f"purge_by_match_and_type start: "
-        f"match_pattern={match_pattern} "
-        f"match_type={match_type}"
-    )
-
     # cursor = "0"
     # while cursor != 0:
     #     cursor, data = self.scan(
@@ -235,15 +164,13 @@ def purge_by_match_and_type(
         logger.info(f"Deleting item {count}: {key_str}")
 
         batch_keys.append(key)
-
-        # flush if batch size has been reached
         if len(batch_keys) >= batch_size:
             flush_batch_delete(batch_keys, r)
             batch_keys.clear()
 
-    # final flush
-    flush_batch_delete(batch_keys, r)
-    batch_keys.clear()
+    if len(batch_keys) >= batch_size:
+        flush_batch_delete(batch_keys, r)
+        batch_keys.clear()
 
     logger.info(f"Deleted {count} matches.")
 
@@ -352,21 +279,7 @@ def delete_user_token_from_redis(
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description="Onyx Redis Manager")
-    parser.add_argument(
-        "--command",
-        type=OnyxRedisCommand,
-        help="The command to run",
-        choices=list(OnyxRedisCommand),
-        required=True,
-    )
-
-    parser.add_argument(
-        "--ssl",
-        type=bool,
-        default=REDIS_SSL,
-        help="Use SSL when connecting to Redis. Usually True for prod and False for local testing",
-        required=False,
-    )
+    parser.add_argument("--command", type=str, help="Operation to run", required=True)
 
     parser.add_argument(
         "--host",
@@ -429,13 +342,6 @@ if __name__ == "__main__":
         required=False,
     )
 
-    parser.add_argument(
-        "--cc-pair",
-        type=int,
-        help="A connector credential pair id. Used with the purge_deletion_locks command.",
-        required=False,
-    )
-
     args = parser.parse_args()
 
     if args.tenant_id:
@@ -462,12 +368,10 @@ if __name__ == "__main__":
         command=args.command,
         batch=args.batch,
         dry_run=args.dry_run,
-        ssl=args.ssl,
         host=args.host,
         port=args.port,
         db=args.db,
         password=args.password,
         user_email=args.user_email,
-        cc_pair_id=args.cc_pair,
     )
     sys.exit(exitcode)

backend/scripts/debugging/onyx_vespa.py

@@ -508,7 +508,6 @@ def get_number_of_chunks_we_think_exist(
 class VespaDebugging:
     # Class for managing Vespa debugging actions.
     def __init__(self, tenant_id: str | None = None):
-        CURRENT_TENANT_ID_CONTEXTVAR.set(tenant_id)
         self.tenant_id = POSTGRES_DEFAULT_SCHEMA if not tenant_id else tenant_id
         self.index_name = get_index_name(self.tenant_id)
 

backend/tests/integration/common_utils/managers/connector.py

@@ -30,10 +30,8 @@ class ConnectorManager:
             name=name,
             source=source,
             input_type=input_type,
-            connector_specific_config=(
-                connector_specific_config
-                or ({"file_locations": []} if source == DocumentSource.FILE else {})
-            ),
+            connector_specific_config=connector_specific_config
+            or {"file_locations": []},
             access_type=access_type,
             groups=groups or [],
         )

backend/tests/integration/common_utils/managers/user.py

@@ -88,6 +88,8 @@ class UserManager:
         if not session_cookie:
             raise Exception("Failed to login")
 
+        print(f"Logged in as {test_user.email}")
+
         # Set cookies in the headers
         test_user.headers["Cookie"] = f"fastapiusersauth={session_cookie}; "
         test_user.cookies = {"fastapiusersauth": session_cookie}

backend/tests/unit/onyx/chat/test_answer.py

@@ -70,7 +70,7 @@ def _answer_fixture_impl(
                 files=[],
                 single_message_history=None,
             ),
-            system_message=default_build_system_message(prompt_config, mock_llm.config),
+            system_message=default_build_system_message(prompt_config),
             message_history=[],
             llm_config=mock_llm.config,
             raw_user_query=QUERY,

backend/tests/unit/onyx/utils/test_threadpool_contextvars.py

@@ -1,131 +0,0 @@
-import contextvars
-import time
-
-from onyx.utils.threadpool_concurrency import FunctionCall
-from onyx.utils.threadpool_concurrency import run_functions_in_parallel
-from onyx.utils.threadpool_concurrency import run_functions_tuples_in_parallel
-from onyx.utils.threadpool_concurrency import run_with_timeout
-
-# Create a test contextvar
-test_var = contextvars.ContextVar("test_var", default="default")
-
-
-def get_contextvar_value() -> str:
-    """Helper function that runs in a thread and returns the contextvar value"""
-    # Add a small sleep to ensure we're actually running in a different thread
-    time.sleep(0.1)
-    return test_var.get()
-
-
-def test_run_with_timeout_preserves_contextvar() -> None:
-    """Test that run_with_timeout preserves contextvar values"""
-    # Set a value in the main thread
-    test_var.set("test_value")
-
-    # Run function with timeout and verify the value is preserved
-    result = run_with_timeout(1.0, get_contextvar_value)
-    assert result == "test_value"
-
-
-def test_run_functions_in_parallel_preserves_contextvar() -> None:
-    """Test that run_functions_in_parallel preserves contextvar values"""
-    # Set a value in the main thread
-    test_var.set("parallel_test")
-
-    # Create multiple function calls
-    function_calls = [
-        FunctionCall(get_contextvar_value),
-        FunctionCall(get_contextvar_value),
-    ]
-
-    # Run in parallel and verify all results have the correct value
-    results = run_functions_in_parallel(function_calls)
-
-    for result_id, value in results.items():
-        assert value == "parallel_test"
-
-
-def test_run_functions_tuples_preserves_contextvar() -> None:
-    """Test that run_functions_tuples_in_parallel preserves contextvar values"""
-    # Set a value in the main thread
-    test_var.set("tuple_test")
-
-    # Create list of function tuples
-    functions_with_args = [
-        (get_contextvar_value, ()),
-        (get_contextvar_value, ()),
-    ]
-
-    # Run in parallel and verify all results have the correct value
-    results = run_functions_tuples_in_parallel(functions_with_args)
-
-    for result in results:
-        assert result == "tuple_test"
-
-
-def test_nested_contextvar_modifications() -> None:
-    """Test that modifications to contextvars in threads don't affect other threads"""
-
-    def modify_and_return_contextvar(new_value: str) -> tuple[str, str]:
-        """Helper that modifies the contextvar and returns both values"""
-        original = test_var.get()
-        test_var.set(new_value)
-        time.sleep(0.1)  # Ensure threads overlap
-        return original, test_var.get()
-
-    # Set initial value
-    test_var.set("initial")
-
-    # Run multiple functions that modify the contextvar
-    functions_with_args = [
-        (modify_and_return_contextvar, ("thread1",)),
-        (modify_and_return_contextvar, ("thread2",)),
-    ]
-
-    results = run_functions_tuples_in_parallel(functions_with_args)
-
-    # Verify each thread saw the initial value and its own modification
-    for original, modified in results:
-        assert original == "initial"  # Each thread should see the initial value
-        assert modified in [
-            "thread1",
-            "thread2",
-        ]  # Each thread should see its own modification
-
-    # Verify the main thread's value wasn't affected
-    assert test_var.get() == "initial"
-
-
-def test_contextvar_isolation_between_runs() -> None:
-    """Test that contextvar changes don't leak between separate parallel runs"""
-
-    def set_and_return_contextvar(value: str) -> str:
-        test_var.set(value)
-        return test_var.get()
-
-    # First run
-    test_var.set("first_run")
-    first_results = run_functions_tuples_in_parallel(
-        [
-            (set_and_return_contextvar, ("thread1",)),
-            (set_and_return_contextvar, ("thread2",)),
-        ]
-    )
-
-    # Verify first run results
-    assert all(result in ["thread1", "thread2"] for result in first_results)
-
-    # Second run should still see the main thread's value
-    assert test_var.get() == "first_run"
-
-    # Second run with different value
-    test_var.set("second_run")
-    second_results = run_functions_tuples_in_parallel(
-        [
-            (set_and_return_contextvar, ("thread3",)),
-            (set_and_return_contextvar, ("thread4",)),
-        ]
-    )
-
-    # Verify second run results
-    assert all(result in ["thread3", "thread4"] for result in second_results)

deployment/data/nginx/app.conf.template

@@ -4,24 +4,6 @@ log_format custom_main '$remote_addr - $remote_user [$time_local] "$request" '
                 '"$http_user_agent" "$http_x_forwarded_for" '
                 'rt=$request_time';
 
-# Map X-Forwarded-Proto or fallback to $scheme
-map $http_x_forwarded_proto $forwarded_proto {
-    default $http_x_forwarded_proto;
-    ""      $scheme;
-}
-
-# Map X-Forwarded-Host or fallback to $host
-map $http_x_forwarded_host $forwarded_host {
-    default $http_x_forwarded_host;
-    ""      $host;
-}
-
-# Map X-Forwarded-Port or fallback to server port
-map $http_x_forwarded_port $forwarded_port {
-    default $http_x_forwarded_port;
-    ""      $server_port;
-}
-
 upstream api_server {
     # fail_timeout=0 means we always retry an upstream even if it failed
     # to return a good HTTP response
@@ -39,7 +21,8 @@ upstream web_server {
 }
 
 server {
-    listen 80 default_server;
+    listen 80;
+    server_name ${DOMAIN};
 
     client_max_body_size 5G;    # Maximum upload size
 
@@ -53,9 +36,8 @@ server {
         # misc headers
         proxy_set_header X-Real-IP $remote_addr;
         proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-        proxy_set_header X-Forwarded-Proto $forwarded_proto;
-        proxy_set_header X-Forwarded-Host $forwarded_host; 
-        proxy_set_header X-Forwarded-Port $forwarded_port;
+        proxy_set_header X-Forwarded-Proto $scheme;
+        proxy_set_header X-Forwarded-Host $host; 
         proxy_set_header Host $host;
 
         # need to use 1.1 to support chunked transfers
@@ -72,9 +54,8 @@ server {
         # misc headers
         proxy_set_header X-Real-IP $remote_addr;
         proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-        proxy_set_header X-Forwarded-Proto $forwarded_proto;
-        proxy_set_header X-Forwarded-Host $forwarded_host; 
-        proxy_set_header X-Forwarded-Port $forwarded_port;
+        proxy_set_header X-Forwarded-Proto $scheme;
+        proxy_set_header X-Forwarded-Host $host; 
         proxy_set_header Host $host;
 
         proxy_http_version 1.1;
@@ -91,25 +72,14 @@ server {
 }
 
 server {
-    listen 443 ssl default_server;
+    listen 443 ssl;
+    server_name ${DOMAIN};
 
     client_max_body_size 5G;    # Maximum upload size
     
     location / {
-        # misc headers
-        proxy_set_header X-Real-IP $remote_addr;
-        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-        # don't use forwarded schema, host, or port here - this is the entry point
-        proxy_set_header X-Forwarded-Proto $scheme;
-        proxy_set_header X-Forwarded-Host $host; 
-        proxy_set_header X-Forwarded-Port $server_port;
-        proxy_set_header Host $host;
-
         proxy_http_version 1.1;
         proxy_buffering off;
-        # we don't want nginx trying to do something clever with
-        # redirects, we set the Host: header above already.
-        proxy_redirect off;
         proxy_pass http://localhost:80;
     }
 

deployment/data/nginx/app.conf.template.dev

@@ -21,7 +21,8 @@ upstream web_server {
 }
 
 server {
-    listen 80 default_server;
+    listen 80;
+    server_name ${DOMAIN};
 
     client_max_body_size 5G;    # Maximum upload size    
 
@@ -36,8 +37,7 @@ server {
         proxy_set_header X-Real-IP $remote_addr;
         proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
         proxy_set_header X-Forwarded-Proto $scheme;
-        proxy_set_header X-Forwarded-Host $host;
-        proxy_set_header X-Forwarded-Port $server_port;
+        proxy_set_header X-Forwarded-Host $host; 
         proxy_set_header Host $host;
 
         # need to use 1.1 to support chunked transfers
@@ -55,8 +55,7 @@ server {
         proxy_set_header X-Real-IP $remote_addr;
         proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
         proxy_set_header X-Forwarded-Proto $scheme;
-        proxy_set_header X-Forwarded-Host $host;
-        proxy_set_header X-Forwarded-Port $server_port;
+        proxy_set_header X-Forwarded-Host $host; 
         proxy_set_header Host $host;
 
         proxy_http_version 1.1;

deployment/data/nginx/app.conf.template.no-letsencrypt

@@ -4,24 +4,6 @@ log_format custom_main '$remote_addr - $remote_user [$time_local] "$request" '
                 '"$http_user_agent" "$http_x_forwarded_for" '
                 'rt=$request_time';
 
-# Map X-Forwarded-Proto or fallback to $scheme
-map $http_x_forwarded_proto $forwarded_proto {
-    default $http_x_forwarded_proto;
-    ""      $scheme;
-}
-
-# Map X-Forwarded-Host or fallback to $host
-map $http_x_forwarded_host $forwarded_host {
-    default $http_x_forwarded_host;
-    ""      $host;
-}
-
-# Map X-Forwarded-Port or fallback to server port
-map $http_x_forwarded_port $forwarded_port {
-    default $http_x_forwarded_port;
-    ""      $server_port;
-}
-
 upstream api_server {
     # fail_timeout=0 means we always retry an upstream even if it failed
     # to return a good HTTP response
@@ -39,7 +21,8 @@ upstream web_server {
 }
 
 server {
-    listen 80 default_server;
+    listen 80;
+    server_name ${DOMAIN};
 
     client_max_body_size 5G;    # Maximum upload size
 
@@ -53,9 +36,8 @@ server {
         # misc headers
         proxy_set_header X-Real-IP $remote_addr;
         proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-        proxy_set_header X-Forwarded-Proto $forwarded_proto;
-        proxy_set_header X-Forwarded-Host $forwarded_host; 
-        proxy_set_header X-Forwarded-Port $forwarded_port;
+        proxy_set_header X-Forwarded-Proto $scheme;
+        proxy_set_header X-Forwarded-Host $host; 
         proxy_set_header Host $host;
 
         # need to use 1.1 to support chunked transfers
@@ -72,9 +54,8 @@ server {
         # misc headers
         proxy_set_header X-Real-IP $remote_addr;
         proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-        proxy_set_header X-Forwarded-Proto $forwarded_proto;
-        proxy_set_header X-Forwarded-Host $forwarded_host; 
-        proxy_set_header X-Forwarded-Port $forwarded_port;
+        proxy_set_header X-Forwarded-Proto $scheme;
+        proxy_set_header X-Forwarded-Host $host; 
         proxy_set_header Host $host;
 
         proxy_http_version 1.1;
@@ -87,25 +68,14 @@ server {
 }
 
 server {
-    listen 443 ssl default_server;
+    listen 443 ssl;
+    server_name ${DOMAIN};
 
     client_max_body_size 5G;    # Maximum upload size
     
     location / {
-        # misc headers
-        proxy_set_header X-Real-IP $remote_addr;
-        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-        # don't use forwarded schema, host, or port here - this is the entry point
-        proxy_set_header X-Forwarded-Proto $scheme;
-        proxy_set_header X-Forwarded-Host $host; 
-        proxy_set_header X-Forwarded-Port $server_port;
-        proxy_set_header Host $host;
-
         proxy_http_version 1.1;
         proxy_buffering off;
-        # we don't want nginx trying to do something clever with
-        # redirects, we set the Host: header above already.
-        proxy_redirect off;
         proxy_pass http://localhost:80;
     }
 

deployment/docker_compose/docker-compose.dev.yml

@@ -36,7 +36,6 @@ services:
       - OPENID_CONFIG_URL=${OPENID_CONFIG_URL:-}
       - TRACK_EXTERNAL_IDP_EXPIRY=${TRACK_EXTERNAL_IDP_EXPIRY:-}
       - CORS_ALLOWED_ORIGIN=${CORS_ALLOWED_ORIGIN:-}
-      - INTEGRATION_TESTS_MODE=${INTEGRATION_TESTS_MODE:-}
       # Gen AI Settings
       - GEN_AI_MAX_TOKENS=${GEN_AI_MAX_TOKENS:-}
       - QA_TIMEOUT=${QA_TIMEOUT:-}

web/src/app/admin/assistants/AssistantEditor.tsx

@@ -113,6 +113,7 @@ export function AssistantEditor({
   documentSets,
   user,
   defaultPublic,
+  redirectType,
   llmProviders,
   tools,
   shouldAddAssistantToUserPreferences,
@@ -123,6 +124,7 @@ export function AssistantEditor({
   documentSets: DocumentSet[];
   user: User | null;
   defaultPublic: boolean;
+  redirectType: SuccessfulPersonaUpdateRedirectType;
   llmProviders: FullLLMProvider[];
   tools: ToolSnapshot[];
   shouldAddAssistantToUserPreferences?: boolean;
@@ -500,7 +502,7 @@ export function AssistantEditor({
             )
             .map((message: { message: string; name?: string }) => ({
               message: message.message,
-              name: message.message,
+              name: message.name || message.message,
             }));
 
           // don't set groups if marked as public

web/src/app/admin/bots/[bot-id]/channels/SlackChannelConfigCreationForm.tsx

@@ -1,7 +1,7 @@
 "use client";
 
-import React, { useMemo } from "react";
-import { Formik, Form } from "formik";
+import React, { useMemo, useState, useEffect } from "react";
+import { Formik, Form, Field } from "formik";
 import * as Yup from "yup";
 import { usePopup } from "@/components/admin/connectors/Popup";
 import {
@@ -13,13 +13,17 @@ import {
   createSlackChannelConfig,
   isPersonaASlackBotPersona,
   updateSlackChannelConfig,
+  fetchSlackChannels,
 } from "../lib";
 import CardSection from "@/components/admin/CardSection";
 import { useRouter } from "next/navigation";
 import { Persona } from "@/app/admin/assistants/interfaces";
 import { StandardAnswerCategoryResponse } from "@/components/standardAnswers/getStandardAnswerCategoriesIfEE";
-import { SEARCH_TOOL_ID } from "@/app/chat/tools/constants";
-import { SlackChannelConfigFormFields } from "./SlackChannelConfigFormFields";
+import { SEARCH_TOOL_ID, SEARCH_TOOL_NAME } from "@/app/chat/tools/constants";
+import {
+  SlackChannelConfigFormFields,
+  SlackChannelConfigFormFieldsProps,
+} from "./SlackChannelConfigFormFields";
 
 export const SlackChannelConfigCreationForm = ({
   slack_bot_id,

web/src/app/admin/bots/[bot-id]/channels/SlackChannelConfigFormFields.tsx

@@ -1,7 +1,13 @@
 "use client";
 
 import React, { useState, useEffect, useMemo } from "react";
-import { FieldArray, useFormikContext, ErrorMessage, Field } from "formik";
+import {
+  FieldArray,
+  Form,
+  useFormikContext,
+  ErrorMessage,
+  Field,
+} from "formik";
 import { CCPairDescriptor, DocumentSet } from "@/lib/types";
 import {
   Label,
@@ -12,13 +18,14 @@ import {
 } from "@/components/admin/connectors/Field";
 import { Button } from "@/components/ui/button";
 import { Persona } from "@/app/admin/assistants/interfaces";
+import { AdvancedOptionsToggle } from "@/components/AdvancedOptionsToggle";
 import { DocumentSetSelectable } from "@/components/documentSet/DocumentSetSelectable";
 import CollapsibleSection from "@/app/admin/assistants/CollapsibleSection";
 import { StandardAnswerCategoryResponse } from "@/components/standardAnswers/getStandardAnswerCategoriesIfEE";
 import { StandardAnswerCategoryDropdownField } from "@/components/standardAnswers/StandardAnswerCategoryDropdown";
 import { RadioGroup } from "@/components/ui/radio-group";
 import { RadioGroupItemField } from "@/components/ui/RadioGroupItemField";
-import { AlertCircle } from "lucide-react";
+import { AlertCircle, View } from "lucide-react";
 import { useRouter } from "next/navigation";
 import {
   Tooltip,
@@ -43,7 +50,6 @@ import {
 import { Separator } from "@/components/ui/separator";
 
 import { CheckFormField } from "@/components/ui/CheckField";
-import { Input } from "@/components/ui/input";
 
 export interface SlackChannelConfigFormFieldsProps {
   isUpdate: boolean;
@@ -172,13 +178,9 @@ export function SlackChannelConfigFormFields({
     );
   }, [documentSets]);
 
-  const {
-    data: channelOptions,
-    error,
-    isLoading,
-  } = useSWR(
+  const { data: channelOptions, isLoading } = useSWR(
     `/api/manage/admin/slack-app/bots/${slack_bot_id}/channels`,
-    async () => {
+    async (url: string) => {
       const channels = await fetchSlackChannels(slack_bot_id);
       return channels.map((channel: any) => ({
         name: channel.name,
@@ -225,34 +227,20 @@ export function SlackChannelConfigFormFields({
             >
               Select A Slack Channel:
             </label>{" "}
-            {error ? (
-              <div>
-                <div className="text-red-600 text-sm mb-4">
-                  {error.message || "Unable to fetch Slack channels."}
-                  {" Please enter the channel name manually."}
-                </div>
-                <TextFormField
-                  name="channel_name"
-                  label="Channel Name"
-                  placeholder="Enter channel name"
+            <Field name="channel_name">
+              {({ field, form }: { field: any; form: any }) => (
+                <SearchMultiSelectDropdown
+                  options={channelOptions || []}
+                  onSelect={(selected) => {
+                    form.setFieldValue("channel_name", selected.name);
+                  }}
+                  initialSearchTerm={field.value}
+                  onSearchTermChange={(term) => {
+                    form.setFieldValue("channel_name", term);
+                  }}
                 />
-              </div>
-            ) : (
-              <Field name="channel_name">
-                {({ field, form }: { field: any; form: any }) => (
-                  <SearchMultiSelectDropdown
-                    options={channelOptions || []}
-                    onSelect={(selected) => {
-                      form.setFieldValue("channel_name", selected.name);
-                    }}
-                    initialSearchTerm={field.value}
-                    onSearchTermChange={(term) => {
-                      form.setFieldValue("channel_name", term);
-                    }}
-                  />
-                )}
-              </Field>
-            )}
+              )}
+            </Field>
           </>
         )}
         <div className="space-y-2 mt-4">

web/src/app/admin/connector/[ccPairId]/ModifyStatusButtonCluster.tsx

@@ -1,18 +1,13 @@
 "use client";
 
 import { Button } from "@/components/ui/button";
-import {
-  CCPairFullInfo,
-  ConnectorCredentialPairStatus,
-  statusIsNotCurrentlyActive,
-} from "./types";
+import { CCPairFullInfo, ConnectorCredentialPairStatus } from "./types";
 import { usePopup } from "@/components/admin/connectors/Popup";
 import { mutate } from "swr";
 import { buildCCPairInfoUrl } from "./lib";
 import { setCCPairStatus } from "@/lib/ccPair";
 import { useState } from "react";
 import { LoadingAnimation } from "@/components/Loading";
-import { ConfirmEntityModal } from "@/components/modals/ConfirmEntityModal";
 
 export function ModifyStatusButtonCluster({
   ccPair,
@@ -21,24 +16,11 @@ export function ModifyStatusButtonCluster({
 }) {
   const { popup, setPopup } = usePopup();
   const [isUpdating, setIsUpdating] = useState(false);
-  const [showConfirmModal, setShowConfirmModal] = useState(false);
 
   const handleStatusChange = async (
     newStatus: ConnectorCredentialPairStatus
   ) => {
     if (isUpdating) return; // Prevent double-clicks or multiple requests
-
-    if (
-      ccPair.status === ConnectorCredentialPairStatus.INVALID &&
-      newStatus === ConnectorCredentialPairStatus.ACTIVE
-    ) {
-      setShowConfirmModal(true);
-    } else {
-      await updateStatus(newStatus);
-    }
-  };
-
-  const updateStatus = async (newStatus: ConnectorCredentialPairStatus) => {
     setIsUpdating(true);
 
     try {
@@ -56,23 +38,30 @@ export function ModifyStatusButtonCluster({
   };
 
   // Compute the button text based on current state and backend status
-  const isNotActive = statusIsNotCurrentlyActive(ccPair.status);
-  const buttonText = isNotActive ? "Re-Enable" : "Pause";
+  const buttonText =
+    ccPair.status === ConnectorCredentialPairStatus.PAUSED
+      ? "Re-Enable"
+      : "Pause";
 
-  const tooltip = isNotActive
-    ? "Click to start indexing again!"
-    : "When paused, the connector's documents will still be visible. However, no new documents will be indexed.";
+  const tooltip =
+    ccPair.status === ConnectorCredentialPairStatus.PAUSED
+      ? "Click to start indexing again!"
+      : "When paused, the connector's documents will still be visible. However, no new documents will be indexed.";
 
   return (
     <>
       {popup}
       <Button
         className="flex items-center justify-center w-auto min-w-[100px] px-4 py-2"
-        variant={isNotActive ? "success-reverse" : "default"}
+        variant={
+          ccPair.status === ConnectorCredentialPairStatus.PAUSED
+            ? "success-reverse"
+            : "default"
+        }
         disabled={isUpdating}
         onClick={() =>
           handleStatusChange(
-            isNotActive
+            ccPair.status === ConnectorCredentialPairStatus.PAUSED
               ? ConnectorCredentialPairStatus.ACTIVE
               : ConnectorCredentialPairStatus.PAUSED
           )
@@ -81,27 +70,17 @@ export function ModifyStatusButtonCluster({
       >
         {isUpdating ? (
           <LoadingAnimation
-            text={isNotActive ? "Resuming" : "Pausing"}
+            text={
+              ccPair.status === ConnectorCredentialPairStatus.PAUSED
+                ? "Resuming"
+                : "Pausing"
+            }
             size="text-md"
           />
         ) : (
           buttonText
         )}
       </Button>
-      {showConfirmModal && (
-        <ConfirmEntityModal
-          entityType="Invalid Connector"
-          entityName={ccPair.name}
-          onClose={() => setShowConfirmModal(false)}
-          onSubmit={() => {
-            setShowConfirmModal(false);
-            updateStatus(ConnectorCredentialPairStatus.ACTIVE);
-          }}
-          additionalDetails="This connector was previously marked as invalid. Please verify that your configuration is correct before re-enabling. Are you sure you want to proceed?"
-          actionButtonText="Re-Enable"
-          variant="action"
-        />
-      )}
     </>
   );
 }

web/src/app/admin/connector/[ccPairId]/ReIndexButton.tsx

@@ -123,8 +123,7 @@ export function ReIndexButton({
         disabled={
           isDisabled ||
           ccPairStatus == ConnectorCredentialPairStatus.DELETING ||
-          ccPairStatus == ConnectorCredentialPairStatus.PAUSED ||
-          ccPairStatus == ConnectorCredentialPairStatus.INVALID
+          ccPairStatus == ConnectorCredentialPairStatus.PAUSED
         }
         tooltip={getCCPairStatusMessage(isDisabled, isIndexing, ccPairStatus)}
       >

web/src/app/admin/connector/[ccPairId]/types.ts

@@ -15,18 +15,6 @@ export enum ConnectorCredentialPairStatus {
   INVALID = "INVALID",
 }
 
-/**
- * Returns true if the status is not currently active (i.e. paused or invalid), but not deleting
- */
-export function statusIsNotCurrentlyActive(
-  status: ConnectorCredentialPairStatus
-): boolean {
-  return (
-    status === ConnectorCredentialPairStatus.PAUSED ||
-    status === ConnectorCredentialPairStatus.INVALID
-  );
-}
-
 export interface CCPairFullInfo {
   id: number;
   name: string;

web/src/app/admin/settings/SettingsForm.tsx

@@ -240,11 +240,11 @@ export function SettingsForm() {
       />
 
       <Checkbox
-        label="Agent Search"
-        sublabel="If set, users will be able to use Agent Search."
-        checked={settings.pro_search_enabled ?? true}
+        label="Pro Search Disabled"
+        sublabel="If set, users will not be able to use Pro Search."
+        checked={settings.pro_search_disabled ?? false}
         onChange={(e) =>
-          handleToggleSettingsField("pro_search_enabled", e.target.checked)
+          handleToggleSettingsField("pro_search_disabled", e.target.checked)
         }
       />
 

web/src/app/admin/settings/interfaces.ts

@@ -10,7 +10,7 @@ export interface Settings {
   notifications: Notification[];
   needs_reindexing: boolean;
   gpu_enabled: boolean;
-  pro_search_enabled: boolean | null;
+  pro_search_disabled: boolean | null;
   application_status: ApplicationStatus;
   auto_scroll: boolean;
   temperature_override_enabled: boolean;

web/src/app/assistants/edit/[id]/page.tsx

@@ -21,7 +21,11 @@ export default async function Page(props: { params: Promise<{ id: string }> }) {
         <div className="px-32">
           <div className="mx-auto container">
             <CardSection className="!border-none !bg-transparent !ring-none">
-              <AssistantEditor {...values} defaultPublic={false} />
+              <AssistantEditor
+                {...values}
+                defaultPublic={false}
+                redirectType={SuccessfulPersonaUpdateRedirectType.CHAT}
+              />
             </CardSection>
           </div>
         </div>

web/src/app/assistants/new/page.tsx

@@ -26,6 +26,7 @@ export default async function Page() {
               <AssistantEditor
                 {...values}
                 defaultPublic={false}
+                redirectType={SuccessfulPersonaUpdateRedirectType.CHAT}
                 shouldAddAssistantToUserPreferences={true}
               />
             </CardSection>

web/src/app/chat/ChatPage.tsx

@@ -23,7 +23,6 @@ import {
   SubQuestionDetail,
   constructSubQuestions,
   DocumentsResponse,
-  AgenticMessageResponseIDInfo,
 } from "./interfaces";
 
 import Prism from "prismjs";
@@ -47,7 +46,6 @@ import {
   removeMessage,
   sendMessage,
   setMessageAsLatest,
-  updateLlmOverrideForChatSession,
   updateParentChildren,
   uploadFilesForChat,
   useScrollonStream,
@@ -66,7 +64,7 @@ import {
 import { usePopup } from "@/components/admin/connectors/Popup";
 import { SEARCH_PARAM_NAMES, shouldSubmitOnLoad } from "./searchParams";
 import { useDocumentSelection } from "./useDocumentSelection";
-import { LlmDescriptor, useFilters, useLlmManager } from "@/lib/hooks";
+import { LlmOverride, useFilters, useLlmOverride } from "@/lib/hooks";
 import { ChatState, FeedbackType, RegenerationState } from "./types";
 import { DocumentResults } from "./documentSidebar/DocumentResults";
 import { OnyxInitializingLoader } from "@/components/OnyxInitializingLoader";
@@ -90,11 +88,7 @@ import {
 import { buildFilters } from "@/lib/search/utils";
 import { SettingsContext } from "@/components/settings/SettingsProvider";
 import Dropzone from "react-dropzone";
-import {
-  checkLLMSupportsImageInput,
-  getFinalLLM,
-  structureValue,
-} from "@/lib/llm/utils";
+import { checkLLMSupportsImageInput, getFinalLLM } from "@/lib/llm/utils";
 import { ChatInputBar } from "./input/ChatInputBar";
 import { useChatContext } from "@/components/context/ChatContext";
 import { v4 as uuidv4 } from "uuid";
@@ -199,6 +193,16 @@ export function ChatPage({
     return screenSize;
   }
 
+  const { height: screenHeight } = useScreenSize();
+
+  const getContainerHeight = () => {
+    if (autoScrollEnabled) return undefined;
+
+    if (screenHeight < 600) return "20vh";
+    if (screenHeight < 1200) return "30vh";
+    return "40vh";
+  };
+
   // handle redirect if chat page is disabled
   // NOTE: this must be done here, in a client component since
   // settings are passed in via Context and therefore aren't
@@ -217,7 +221,6 @@ export function ChatPage({
     setProSearchEnabled(!proSearchEnabled);
   };
 
-  const isInitialLoad = useRef(true);
   const [userSettingsToggled, setUserSettingsToggled] = useState(false);
 
   const {
@@ -352,7 +355,7 @@ export function ChatPage({
     ]
   );
 
-  const llmManager = useLlmManager(
+  const llmOverrideManager = useLlmOverride(
     llmProviders,
     selectedChatSession,
     liveAssistant
@@ -516,17 +519,8 @@ export function ChatPage({
       scrollInitialized.current = false;
 
       if (!hasPerformedInitialScroll) {
-        if (isInitialLoad.current) {
-          setHasPerformedInitialScroll(true);
-          isInitialLoad.current = false;
-        }
         clientScrollToBottom();
-
-        setTimeout(() => {
-          setHasPerformedInitialScroll(true);
-        }, 100);
       } else if (isChatSessionSwitch) {
-        setHasPerformedInitialScroll(true);
         clientScrollToBottom(true);
       }
 
@@ -1135,56 +1129,6 @@ export function ChatPage({
     });
   };
   const [uncaughtError, setUncaughtError] = useState<string | null>(null);
-  const [agenticGenerating, setAgenticGenerating] = useState(false);
-
-  const autoScrollEnabled =
-    (user?.preferences?.auto_scroll && !agenticGenerating) ?? false;
-
-  useScrollonStream({
-    chatState: currentSessionChatState,
-    scrollableDivRef,
-    scrollDist,
-    endDivRef,
-    debounceNumber,
-    mobile: settings?.isMobile,
-    enableAutoScroll: autoScrollEnabled,
-  });
-
-  // Track whether a message has been sent during this page load, keyed by chat session id
-  const [sessionHasSentLocalUserMessage, setSessionHasSentLocalUserMessage] =
-    useState<Map<string | null, boolean>>(new Map());
-
-  // Update the local state for a session once the user sends a message
-  const markSessionMessageSent = (sessionId: string | null) => {
-    setSessionHasSentLocalUserMessage((prev) => {
-      const newMap = new Map(prev);
-      newMap.set(sessionId, true);
-      return newMap;
-    });
-  };
-  const currentSessionHasSentLocalUserMessage = useMemo(
-    () => (sessionId: string | null) => {
-      return sessionHasSentLocalUserMessage.size === 0
-        ? undefined
-        : sessionHasSentLocalUserMessage.get(sessionId) || false;
-    },
-    [sessionHasSentLocalUserMessage]
-  );
-
-  const { height: screenHeight } = useScreenSize();
-
-  const getContainerHeight = useMemo(() => {
-    return () => {
-      if (!currentSessionHasSentLocalUserMessage(chatSessionIdRef.current)) {
-        return undefined;
-      }
-      if (autoScrollEnabled) return undefined;
-
-      if (screenHeight < 600) return "40vh";
-      if (screenHeight < 1200) return "50vh";
-      return "60vh";
-    };
-  }, [autoScrollEnabled, screenHeight, currentSessionHasSentLocalUserMessage]);
 
   const onSubmit = async ({
     messageIdToResend,
@@ -1193,7 +1137,7 @@ export function ChatPage({
     forceSearch,
     isSeededChat,
     alternativeAssistantOverride = null,
-    modelOverride,
+    modelOverRide,
     regenerationRequest,
     overrideFileDescriptors,
   }: {
@@ -1203,7 +1147,7 @@ export function ChatPage({
     forceSearch?: boolean;
     isSeededChat?: boolean;
     alternativeAssistantOverride?: Persona | null;
-    modelOverride?: LlmDescriptor;
+    modelOverRide?: LlmOverride;
     regenerationRequest?: RegenerationRequest | null;
     overrideFileDescriptors?: FileDescriptor[];
   } = {}) => {
@@ -1211,9 +1155,6 @@ export function ChatPage({
     let frozenSessionId = currentSessionId();
     updateCanContinue(false, frozenSessionId);
 
-    // Mark that we've sent a message for this session in the current page load
-    markSessionMessageSent(frozenSessionId);
-
     if (currentChatState() != "input") {
       if (currentChatState() == "uploading") {
         setPopup({
@@ -1249,22 +1190,6 @@ export function ChatPage({
       currChatSessionId = chatSessionIdRef.current as string;
     }
     frozenSessionId = currChatSessionId;
-    // update the selected model for the chat session if one is specified so that
-    // it persists across page reloads. Do not `await` here so that the message
-    // request can continue and this will just happen in the background.
-    // NOTE: only set the model override for the chat session once we send a
-    // message with it. If the user switches models and then starts a new
-    // chat session, it is unexpected for that model to be used when they
-    // return to this session the next day.
-    let finalLLM = modelOverride || llmManager.currentLlm;
-    updateLlmOverrideForChatSession(
-      currChatSessionId,
-      structureValue(
-        finalLLM.name || "",
-        finalLLM.provider || "",
-        finalLLM.modelName || ""
-      )
-    );
 
     updateStatesWithNewSessionId(currChatSessionId);
 
@@ -1324,14 +1249,11 @@ export function ChatPage({
         : null) ||
       (messageMap.size === 1 ? Array.from(messageMap.values())[0] : null);
 
-    let currentAssistantId;
-    if (alternativeAssistantOverride) {
-      currentAssistantId = alternativeAssistantOverride.id;
-    } else if (alternativeAssistant) {
-      currentAssistantId = alternativeAssistant.id;
-    } else {
-      currentAssistantId = liveAssistant.id;
-    }
+    const currentAssistantId = alternativeAssistantOverride
+      ? alternativeAssistantOverride.id
+      : alternativeAssistant
+        ? alternativeAssistant.id
+        : liveAssistant.id;
 
     resetInputBar();
     let messageUpdates: Message[] | null = null;
@@ -1358,8 +1280,6 @@ export function ChatPage({
     let toolCall: ToolCallMetadata | null = null;
     let isImprovement: boolean | undefined = undefined;
     let isStreamingQuestions = true;
-    let includeAgentic = false;
-    let secondLevelMessageId: number | null = null;
 
     let initialFetchDetails: null | {
       user_message_id: number;
@@ -1403,18 +1323,20 @@ export function ChatPage({
         forceSearch,
         regenerate: regenerationRequest !== undefined,
         modelProvider:
-          modelOverride?.name || llmManager.currentLlm.name || undefined,
+          modelOverRide?.name ||
+          llmOverrideManager.llmOverride.name ||
+          undefined,
         modelVersion:
-          modelOverride?.modelName ||
-          llmManager.currentLlm.modelName ||
+          modelOverRide?.modelName ||
+          llmOverrideManager.llmOverride.modelName ||
           searchParams.get(SEARCH_PARAM_NAMES.MODEL_VERSION) ||
           undefined,
-        temperature: llmManager.temperature || undefined,
+        temperature: llmOverrideManager.temperature || undefined,
         systemPromptOverride:
           searchParams.get(SEARCH_PARAM_NAMES.SYSTEM_PROMPT) || undefined,
         useExistingUserMessage: isSeededChat,
         useLanggraph:
-          settings?.settings.pro_search_enabled &&
+          !settings?.settings.pro_search_disabled &&
           proSearchEnabled &&
           retrievalEnabled,
       });
@@ -1495,17 +1417,6 @@ export function ChatPage({
             resetRegenerationState();
           } else {
             const { user_message_id, frozenMessageMap } = initialFetchDetails;
-            if (Object.hasOwn(packet, "agentic_message_ids")) {
-              const agenticMessageIds = (packet as AgenticMessageResponseIDInfo)
-                .agentic_message_ids;
-              const level1MessageId = agenticMessageIds.find(
-                (item) => item.level === 1
-              )?.message_id;
-              if (level1MessageId) {
-                secondLevelMessageId = level1MessageId;
-                includeAgentic = true;
-              }
-            }
 
             setChatState((prevState) => {
               if (prevState.get(chatSessionIdRef.current!) === "loading") {
@@ -1657,10 +1568,7 @@ export function ChatPage({
                   };
                 }
               );
-            } else if (
-              Object.hasOwn(packet, "error") &&
-              (packet as any).error != null
-            ) {
+            } else if (Object.hasOwn(packet, "error")) {
               if (
                 sub_questions.length > 0 &&
                 sub_questions
@@ -1672,8 +1580,8 @@ export function ChatPage({
                 setAgenticGenerating(false);
                 setAlternativeGeneratingAssistant(null);
                 setSubmittedMessage("");
-
-                throw new Error((packet as StreamingError).error);
+                return;
+                // throw new Error((packet as StreamingError).error);
               } else {
                 error = (packet as StreamingError).error;
                 stackTrace = (packet as StreamingError).stack_trace;
@@ -1756,19 +1664,6 @@ export function ChatPage({
                 second_level_generating: second_level_generating,
                 agentic_docs: agenticDocs,
               },
-              ...(includeAgentic
-                ? [
-                    {
-                      messageId: secondLevelMessageId!,
-                      message: second_level_answer,
-                      type: "assistant" as const,
-                      files: [],
-                      toolCall: null,
-                      parentMessageId:
-                        initialFetchDetails.assistant_message_id!,
-                    },
-                  ]
-                : []),
             ]);
           }
         }
@@ -1877,7 +1772,7 @@ export function ChatPage({
     const [_, llmModel] = getFinalLLM(
       llmProviders,
       liveAssistant,
-      llmManager.currentLlm
+      llmOverrideManager.llmOverride
     );
     const llmAcceptsImages = checkLLMSupportsImageInput(llmModel);
 
@@ -1932,6 +1827,7 @@ export function ChatPage({
   // Used to maintain a "time out" for history sidebar so our existing refs can have time to process change
   const [untoggled, setUntoggled] = useState(false);
   const [loadingError, setLoadingError] = useState<string | null>(null);
+  const [agenticGenerating, setAgenticGenerating] = useState(false);
 
   const explicitlyUntoggle = () => {
     setShowHistorySidebar(false);
@@ -1973,6 +1869,19 @@ export function ChatPage({
     isAnonymousUser: user?.is_anonymous_user,
   });
 
+  const autoScrollEnabled =
+    (user?.preferences?.auto_scroll && !agenticGenerating) ?? false;
+
+  useScrollonStream({
+    chatState: currentSessionChatState,
+    scrollableDivRef,
+    scrollDist,
+    endDivRef,
+    debounceNumber,
+    mobile: settings?.isMobile,
+    enableAutoScroll: autoScrollEnabled,
+  });
+
   // Virtualization + Scrolling related effects and functions
   const scrollInitialized = useRef(false);
   interface VisibleRange {
@@ -2182,7 +2091,7 @@ export function ChatPage({
   }, [searchParams, router]);
 
   useEffect(() => {
-    llmManager.updateImageFilesPresent(imageFileInMessageHistory);
+    llmOverrideManager.updateImageFilesPresent(imageFileInMessageHistory);
   }, [imageFileInMessageHistory]);
 
   const pathname = usePathname();
@@ -2236,9 +2145,9 @@ export function ChatPage({
 
   function createRegenerator(regenerationRequest: RegenerationRequest) {
     // Returns new function that only needs `modelOverRide` to be specified when called
-    return async function (modelOverride: LlmDescriptor) {
+    return async function (modelOverRide: LlmOverride) {
       return await onSubmit({
-        modelOverride,
+        modelOverRide,
         messageIdToResend: regenerationRequest.parentMessage.messageId,
         regenerationRequest,
         forceSearch: regenerationRequest.forceSearch,
@@ -2319,7 +2228,9 @@ export function ChatPage({
       {(settingsToggled || userSettingsToggled) && (
         <UserSettingsModal
           setPopup={setPopup}
-          setCurrentLlm={(newLlm) => llmManager.updateCurrentLlm(newLlm)}
+          setLlmOverride={(newOverride) =>
+            llmOverrideManager.updateLLMOverride(newOverride)
+          }
           defaultModel={user?.preferences.default_model!}
           llmProviders={llmProviders}
           onClose={() => {
@@ -2383,7 +2294,7 @@ export function ChatPage({
         <ShareChatSessionModal
           assistantId={liveAssistant?.id}
           message={message}
-          modelOverride={llmManager.currentLlm}
+          modelOverride={llmOverrideManager.llmOverride}
           chatSessionId={sharedChatSession.id}
           existingSharedStatus={sharedChatSession.shared_status}
           onClose={() => setSharedChatSession(null)}
@@ -2401,7 +2312,7 @@ export function ChatPage({
         <ShareChatSessionModal
           message={message}
           assistantId={liveAssistant?.id}
-          modelOverride={llmManager.currentLlm}
+          modelOverride={llmOverrideManager.llmOverride}
           chatSessionId={chatSessionIdRef.current}
           existingSharedStatus={chatSessionSharedStatus}
           onClose={() => setSharingModalVisible(false)}
@@ -2631,7 +2542,6 @@ export function ChatPage({
                             style={{ overflowAnchor: "none" }}
                             key={currentSessionId()}
                             className={
-                              (hasPerformedInitialScroll ? "" : " hidden ") +
                               "desktop:-ml-4 w-full mx-auto " +
                               "absolute mobile:top-0 desktop:top-0 left-0 " +
                               (settings?.enterpriseSettings
@@ -2782,11 +2692,6 @@ export function ChatPage({
                                     ? messageHistory[i + 1]?.documents
                                     : undefined;
 
-                                const nextMessage =
-                                  messageHistory[i + 1]?.type === "assistant"
-                                    ? messageHistory[i + 1]
-                                    : undefined;
-
                                 return (
                                   <div
                                     className="text-text"
@@ -2815,10 +2720,7 @@ export function ChatPage({
                                             selectedMessageForDocDisplay ==
                                               secondLevelMessage?.messageId)
                                         }
-                                        isImprovement={
-                                          message.isImprovement ||
-                                          nextMessage?.isImprovement
-                                        }
+                                        isImprovement={message.isImprovement}
                                         secondLevelGenerating={
                                           (message.second_level_generating &&
                                             currentSessionChatState !==
@@ -3118,7 +3020,7 @@ export function ChatPage({
                                               messageId: message.messageId,
                                               parentMessage: parentMessage!,
                                               forceSearch: true,
-                                            })(llmManager.currentLlm);
+                                            })(llmOverrideManager.llmOverride);
                                           } else {
                                             setPopup({
                                               type: "error",
@@ -3263,7 +3165,7 @@ export function ChatPage({
                               availableDocumentSets={documentSets}
                               availableTags={tags}
                               filterManager={filterManager}
-                              llmManager={llmManager}
+                              llmOverrideManager={llmOverrideManager}
                               removeDocs={() => {
                                 clearSelectedDocuments();
                               }}

web/src/app/chat/RegenerateOption.tsx

@@ -1,8 +1,8 @@
 import { useChatContext } from "@/components/context/ChatContext";
 import {
   getDisplayNameForModel,
-  LlmDescriptor,
-  useLlmManager,
+  LlmOverride,
+  useLlmOverride,
 } from "@/lib/hooks";
 import { StringOrNumberOption } from "@/components/Dropdown";
 
@@ -106,13 +106,13 @@ export default function RegenerateOption({
   onDropdownVisibleChange,
 }: {
   selectedAssistant: Persona;
-  regenerate: (modelOverRide: LlmDescriptor) => Promise<void>;
+  regenerate: (modelOverRide: LlmOverride) => Promise<void>;
   overriddenModel?: string;
   onHoverChange: (isHovered: boolean) => void;
   onDropdownVisibleChange: (isVisible: boolean) => void;
 }) {
   const { llmProviders } = useChatContext();
-  const llmManager = useLlmManager(llmProviders);
+  const llmOverrideManager = useLlmOverride(llmProviders);
 
   const [_, llmName] = getFinalLLM(llmProviders, selectedAssistant, null);
 
@@ -148,7 +148,7 @@ export default function RegenerateOption({
   );
 
   const currentModelName =
-    llmManager?.currentLlm.modelName ||
+    llmOverrideManager?.llmOverride.modelName ||
     (selectedAssistant
       ? selectedAssistant.llm_model_version_override || llmName
       : llmName);

web/src/app/chat/input/ChatInputBar.tsx

@@ -6,7 +6,7 @@ import { Persona } from "@/app/admin/assistants/interfaces";
 import LLMPopover from "./LLMPopover";
 import { InputPrompt } from "@/app/chat/interfaces";
 
-import { FilterManager, LlmManager } from "@/lib/hooks";
+import { FilterManager, LlmOverrideManager } from "@/lib/hooks";
 import { useChatContext } from "@/components/context/ChatContext";
 import { ChatFileType, FileDescriptor } from "../interfaces";
 import {
@@ -180,7 +180,7 @@ interface ChatInputBarProps {
   setMessage: (message: string) => void;
   stopGenerating: () => void;
   onSubmit: () => void;
-  llmManager: LlmManager;
+  llmOverrideManager: LlmOverrideManager;
   chatState: ChatState;
   alternativeAssistant: Persona | null;
   // assistants
@@ -225,7 +225,7 @@ export function ChatInputBar({
   availableSources,
   availableDocumentSets,
   availableTags,
-  llmManager,
+  llmOverrideManager,
   proSearchEnabled,
   setProSearchEnabled,
 }: ChatInputBarProps) {
@@ -781,7 +781,7 @@ export function ChatInputBar({
 
                 <LLMPopover
                   llmProviders={llmProviders}
-                  llmManager={llmManager}
+                  llmOverrideManager={llmOverrideManager}
                   requiresImageGeneration={false}
                   currentAssistant={selectedAssistant}
                 />
@@ -805,12 +805,13 @@ export function ChatInputBar({
                 )}
               </div>
               <div className="flex items-center my-auto">
-                {retrievalEnabled && settings?.settings.pro_search_enabled && (
-                  <AgenticToggle
-                    proSearchEnabled={proSearchEnabled}
-                    setProSearchEnabled={setProSearchEnabled}
-                  />
-                )}
+                {retrievalEnabled &&
+                  !settings?.settings.pro_search_disabled && (
+                    <AgenticToggle
+                      proSearchEnabled={proSearchEnabled}
+                      setProSearchEnabled={setProSearchEnabled}
+                    />
+                  )}
                 <button
                   id="onyx-chat-input-send-button"
                   className={`cursor-pointer ${

web/src/app/chat/input/LLMPopover.tsx

@@ -16,7 +16,7 @@ import {
   LLMProviderDescriptor,
 } from "@/app/admin/configuration/llm/interfaces";
 import { Persona } from "@/app/admin/assistants/interfaces";
-import { LlmManager } from "@/lib/hooks";
+import { LlmOverrideManager } from "@/lib/hooks";
 
 import {
   Tooltip,
@@ -31,19 +31,21 @@ import { useUser } from "@/components/user/UserProvider";
 
 interface LLMPopoverProps {
   llmProviders: LLMProviderDescriptor[];
-  llmManager: LlmManager;
+  llmOverrideManager: LlmOverrideManager;
   requiresImageGeneration?: boolean;
   currentAssistant?: Persona;
 }
 
 export default function LLMPopover({
   llmProviders,
-  llmManager,
+  llmOverrideManager,
   requiresImageGeneration,
   currentAssistant,
 }: LLMPopoverProps) {
   const [isOpen, setIsOpen] = useState(false);
   const { user } = useUser();
+  const { llmOverride, updateLLMOverride } = llmOverrideManager;
+  const currentLlm = llmOverride.modelName;
 
   const llmOptionsByProvider: {
     [provider: string]: {
@@ -91,19 +93,19 @@ export default function LLMPopover({
     : null;
 
   const [localTemperature, setLocalTemperature] = useState(
-    llmManager.temperature ?? 0.5
+    llmOverrideManager.temperature ?? 0.5
   );
 
   useEffect(() => {
-    setLocalTemperature(llmManager.temperature ?? 0.5);
-  }, [llmManager.temperature]);
+    setLocalTemperature(llmOverrideManager.temperature ?? 0.5);
+  }, [llmOverrideManager.temperature]);
 
   const handleTemperatureChange = (value: number[]) => {
     setLocalTemperature(value[0]);
   };
 
   const handleTemperatureChangeComplete = (value: number[]) => {
-    llmManager.updateTemperature(value[0]);
+    llmOverrideManager.updateTemperature(value[0]);
   };
 
   return (
@@ -118,15 +120,15 @@ export default function LLMPopover({
             toggle
             flexPriority="stiff"
             name={getDisplayNameForModel(
-              llmManager?.currentLlm.modelName ||
+              llmOverrideManager?.llmOverride.modelName ||
                 defaultModelDisplayName ||
                 "Models"
             )}
             Icon={getProviderIcon(
-              llmManager?.currentLlm.provider ||
+              llmOverrideManager?.llmOverride.provider ||
                 defaultProvider?.provider ||
                 "anthropic",
-              llmManager?.currentLlm.modelName ||
+              llmOverrideManager?.llmOverride.modelName ||
                 defaultProvider?.default_model_name ||
                 "claude-3-5-sonnet-20240620"
             )}
@@ -145,12 +147,12 @@ export default function LLMPopover({
                 <button
                   key={index}
                   className={`w-full flex items-center gap-x-2 px-3 py-2 text-sm text-left hover:bg-background-100 dark:hover:bg-neutral-800 transition-colors duration-150 ${
-                    llmManager.currentLlm.modelName === name
+                    currentLlm === name
                       ? "bg-background-100 dark:bg-neutral-900 text-text"
                       : "text-text-darker"
                   }`}
                   onClick={() => {
-                    llmManager.updateCurrentLlm(destructureValue(value));
+                    updateLLMOverride(destructureValue(value));
                     setIsOpen(false);
                   }}
                 >
@@ -170,7 +172,7 @@ export default function LLMPopover({
                       );
                     }
                   })()}
-                  {llmManager.imageFilesPresent &&
+                  {llmOverrideManager.imageFilesPresent &&
                     !checkLLMSupportsImageInput(name) && (
                       <TooltipProvider>
                         <Tooltip delayDuration={0}>
@@ -197,7 +199,7 @@ export default function LLMPopover({
             <div className="w-full px-3 py-2">
               <Slider
                 value={[localTemperature]}
-                max={llmManager.maxTemperature}
+                max={llmOverrideManager.maxTemperature}
                 min={0}
                 step={0.01}
                 onValueChange={handleTemperatureChange}

web/src/app/chat/interfaces.ts

@@ -155,15 +155,6 @@ export interface MessageResponseIDInfo {
   reserved_assistant_message_id: number;
 }
 
-export interface AgentMessageIDInfo {
-  level: number;
-  message_id: number;
-}
-
-export interface AgenticMessageResponseIDInfo {
-  agentic_message_ids: AgentMessageIDInfo[];
-}
-
 export interface DocumentsResponse {
   top_documents: OnyxDocument[];
   rephrased_query: string | null;

web/src/app/chat/lib.tsx

@@ -25,7 +25,6 @@ import {
   RetrievalType,
   StreamingError,
   ToolCallMetadata,
-  AgenticMessageResponseIDInfo,
 } from "./interfaces";
 import { Persona } from "../admin/assistants/interfaces";
 import { ReadonlyURLSearchParams } from "next/navigation";
@@ -65,7 +64,7 @@ export function getChatRetentionInfo(
   };
 }
 
-export async function updateLlmOverrideForChatSession(
+export async function updateModelOverrideForChatSession(
   chatSessionId: string,
   newAlternateModel: string
 ) {
@@ -155,8 +154,7 @@ export type PacketType =
   | AgentAnswerPiece
   | SubQuestionPiece
   | ExtendedToolResponse
-  | RefinedAnswerImprovement
-  | AgenticMessageResponseIDInfo;
+  | RefinedAnswerImprovement;
 
 export async function* sendMessage({
   regenerate,
@@ -236,7 +234,7 @@ export async function* sendMessage({
           }
         : null,
     use_existing_user_message: useExistingUserMessage,
-    use_agentic_search: useLanggraph ?? false,
+    use_agentic_search: useLanggraph,
   });
 
   const response = await fetch(`/api/chat/send-message`, {

web/src/app/chat/message/AgenticMessage.tsx

@@ -44,7 +44,7 @@ import { ValidSources } from "@/lib/types";
 import { useMouseTracking } from "./hooks";
 import { SettingsContext } from "@/components/settings/SettingsProvider";
 import RegenerateOption from "../RegenerateOption";
-import { LlmDescriptor } from "@/lib/hooks";
+import { LlmOverride } from "@/lib/hooks";
 import { ContinueGenerating } from "./ContinueMessage";
 import { MemoizedAnchor, MemoizedParagraph } from "./MemoizedTextComponents";
 import { extractCodeText, preprocessLaTeX } from "./codeUtils";
@@ -117,7 +117,7 @@ export const AgenticMessage = ({
   isComplete?: boolean;
   handleFeedback?: (feedbackType: FeedbackType) => void;
   overriddenModel?: string;
-  regenerate?: (modelOverRide: LlmDescriptor) => Promise<void>;
+  regenerate?: (modelOverRide: LlmOverride) => Promise<void>;
   setPresentingDocument?: (document: OnyxDocument) => void;
   toggleDocDisplay?: (agentic: boolean) => void;
   error?: string | null;

web/src/app/chat/message/Messages.tsx

@@ -58,7 +58,7 @@ import { useMouseTracking } from "./hooks";
 import { SettingsContext } from "@/components/settings/SettingsProvider";
 import GeneratingImageDisplay from "../tools/GeneratingImageDisplay";
 import RegenerateOption from "../RegenerateOption";
-import { LlmDescriptor } from "@/lib/hooks";
+import { LlmOverride } from "@/lib/hooks";
 import { ContinueGenerating } from "./ContinueMessage";
 import { MemoizedAnchor, MemoizedParagraph } from "./MemoizedTextComponents";
 import { extractCodeText, preprocessLaTeX } from "./codeUtils";
@@ -213,7 +213,7 @@ export const AIMessage = ({
   handleForceSearch?: () => void;
   retrievalDisabled?: boolean;
   overriddenModel?: string;
-  regenerate?: (modelOverRide: LlmDescriptor) => Promise<void>;
+  regenerate?: (modelOverRide: LlmOverride) => Promise<void>;
   setPresentingDocument: (document: OnyxDocument) => void;
   removePadding?: boolean;
 }) => {

web/src/app/chat/modal/ShareChatSessionModal.tsx

@@ -11,7 +11,7 @@ import { CopyButton } from "@/components/CopyButton";
 import { SEARCH_PARAM_NAMES } from "../searchParams";
 import { usePopup } from "@/components/admin/connectors/Popup";
 import { structureValue } from "@/lib/llm/utils";
-import { LlmDescriptor } from "@/lib/hooks";
+import { LlmOverride } from "@/lib/hooks";
 import { Separator } from "@/components/ui/separator";
 import { AdvancedOptionsToggle } from "@/components/AdvancedOptionsToggle";
 
@@ -38,7 +38,7 @@ async function generateShareLink(chatSessionId: string) {
 async function generateSeedLink(
   message?: string,
   assistantId?: number,
-  modelOverride?: LlmDescriptor
+  modelOverride?: LlmOverride
 ) {
   const baseUrl = `${window.location.protocol}//${window.location.host}`;
   const model = modelOverride
@@ -92,7 +92,7 @@ export function ShareChatSessionModal({
   onClose: () => void;
   message?: string;
   assistantId?: number;
-  modelOverride?: LlmDescriptor;
+  modelOverride?: LlmOverride;
 }) {
   const [shareLink, setShareLink] = useState<string>(
     existingSharedStatus === ChatSessionSharedStatus.Public

web/src/app/chat/modal/UserSettingsModal.tsx

@@ -1,6 +1,6 @@
 import { useContext, useEffect, useRef, useState } from "react";
 import { Modal } from "@/components/Modal";
-import { getDisplayNameForModel, LlmDescriptor } from "@/lib/hooks";
+import { getDisplayNameForModel, LlmOverride } from "@/lib/hooks";
 import { LLMProviderDescriptor } from "@/app/admin/configuration/llm/interfaces";
 
 import { destructureValue, structureValue } from "@/lib/llm/utils";
@@ -31,12 +31,12 @@ export function UserSettingsModal({
   setPopup,
   llmProviders,
   onClose,
-  setCurrentLlm,
+  setLlmOverride,
   defaultModel,
 }: {
   setPopup: (popupSpec: PopupSpec | null) => void;
   llmProviders: LLMProviderDescriptor[];
-  setCurrentLlm?: (newLlm: LlmDescriptor) => void;
+  setLlmOverride?: (newOverride: LlmOverride) => void;
   onClose: () => void;
   defaultModel: string | null;
 }) {
@@ -127,14 +127,18 @@ export function UserSettingsModal({
     );
   });
 
+  const llmOptions = Object.entries(llmOptionsByProvider).flatMap(
+    ([provider, options]) => [...options]
+  );
+
   const router = useRouter();
   const handleChangedefaultModel = async (defaultModel: string | null) => {
     try {
       const response = await setUserDefaultModel(defaultModel);
 
       if (response.ok) {
-        if (defaultModel && setCurrentLlm) {
-          setCurrentLlm(destructureValue(defaultModel));
+        if (defaultModel && setLlmOverride) {
+          setLlmOverride(destructureValue(defaultModel));
         }
         setPopup({
           message: "Default model updated successfully",

web/src/app/layout.tsx

@@ -21,9 +21,11 @@ import { fetchAssistantData } from "@/lib/chat/fetchAssistantdata";
 import { AppProvider } from "@/components/context/AppProvider";
 import { PHProvider } from "./providers";
 import { getCurrentUserSS } from "@/lib/userSS";
+import CardSection from "@/components/admin/CardSection";
 import { Suspense } from "react";
 import PostHogPageView from "./PostHogPageView";
 import Script from "next/script";
+import { LogoType } from "@/components/logo/Logo";
 import { Hanken_Grotesk } from "next/font/google";
 import { WebVitals } from "./web-vitals";
 import { ThemeProvider } from "next-themes";

web/src/components/credentials/CredentialSection.tsx

@@ -168,7 +168,7 @@ export default function CredentialSection({
           onClick={() => {
             setShowModifyCredential(true);
           }}
-          className="flex items-center gap-x-2 cursor-pointer bg-neutral-800 border-neutral-600 border-2 hover:bg-neutral-700 p-1.5 rounded-lg text-neutral-300"
+          className="flex items-center gap-x-2 cursor-pointer bg-background-100 border-border border-2 hover:bg-border p-1.5 rounded-lg text-text-700"
         >
           <FaSwatchbook />
           Update Credentials

web/src/components/modals/ConfirmEntityModal.tsx

@@ -37,7 +37,7 @@ export const ConfirmEntityModal = ({
   };
 
   return (
-    <Modal width="rounded max-w-md w-full" onOutsideClick={onClose}>
+    <Modal width="rounded max-w-sm w-full" onOutsideClick={onClose}>
       <>
         <div className="flex mb-4">
           <h2 className="my-auto text-2xl font-bold">

web/src/components/settings/lib.ts

@@ -51,7 +51,7 @@ export async function fetchSettingsSS(): Promise<CombinedSettings | null> {
           notifications: [],
           needs_reindexing: false,
           anonymous_user_enabled: false,
-          pro_search_enabled: true,
+          pro_search_disabled: false,
           temperature_override_enabled: true,
         };
       } else {
@@ -95,8 +95,8 @@ export async function fetchSettingsSS(): Promise<CombinedSettings | null> {
       }
     }
 
-    if (settings.pro_search_enabled == null) {
-      settings.pro_search_enabled = true;
+    if (enterpriseSettings && settings.pro_search_disabled == null) {
+      settings.pro_search_disabled = true;
     }
 
     const webVersion = getWebVersion();

web/src/lib/hooks.ts

@@ -360,18 +360,18 @@ export const useUsers = ({ includeApiKeys }: UseUsersParams) => {
   };
 };
 
-export interface LlmDescriptor {
+export interface LlmOverride {
   name: string;
   provider: string;
   modelName: string;
 }
 
-export interface LlmManager {
-  currentLlm: LlmDescriptor;
-  updateCurrentLlm: (newOverride: LlmDescriptor) => void;
+export interface LlmOverrideManager {
+  llmOverride: LlmOverride;
+  updateLLMOverride: (newOverride: LlmOverride) => void;
   temperature: number;
   updateTemperature: (temperature: number) => void;
-  updateModelOverrideBasedOnChatSession: (chatSession?: ChatSession) => void;
+  updateModelOverrideForChatSession: (chatSession?: ChatSession) => void;
   imageFilesPresent: boolean;
   updateImageFilesPresent: (present: boolean) => void;
   liveAssistant: Persona | null;
@@ -400,7 +400,7 @@ Thus, the input should be
 
 Changes take place as
 - liveAssistant or currentChatSession changes (and the associated model override is set)
-- (updateCurrentLlm) User explicitly setting a model override (and we explicitly override and set the userSpecifiedOverride which we'll use in place of the user preferences unless overridden by an assistant)
+- (uploadLLMOverride) User explicitly setting a model override (and we explicitly override and set the userSpecifiedOverride which we'll use in place of the user preferences unless overridden by an assistant)
 
 If we have a live assistant, we should use that model override
 
@@ -419,78 +419,55 @@ This approach ensures that user preferences are maintained for existing chats wh
 providing appropriate defaults for new conversations based on the available tools.
 */
 
-export function useLlmManager(
+export function useLlmOverride(
   llmProviders: LLMProviderDescriptor[],
   currentChatSession?: ChatSession,
   liveAssistant?: Persona
-): LlmManager {
+): LlmOverrideManager {
   const { user } = useUser();
 
-  const [userHasManuallyOverriddenLLM, setUserHasManuallyOverriddenLLM] =
-    useState(false);
   const [chatSession, setChatSession] = useState<ChatSession | null>(null);
-  const [currentLlm, setCurrentLlm] = useState<LlmDescriptor>({
-    name: "",
-    provider: "",
-    modelName: "",
-  });
 
-  const llmUpdate = () => {
-    /* Should be called when the live assistant or current chat session changes */
+  const llmOverrideUpdate = () => {
+    if (liveAssistant?.llm_model_version_override) {
+      setLlmOverride(
+        getValidLlmOverride(liveAssistant.llm_model_version_override)
+      );
+    } else if (currentChatSession?.current_alternate_model) {
+      setLlmOverride(
+        getValidLlmOverride(currentChatSession.current_alternate_model)
+      );
+    } else if (user?.preferences?.default_model) {
+      setLlmOverride(getValidLlmOverride(user.preferences.default_model));
+      return;
+    } else {
+      const defaultProvider = llmProviders.find(
+        (provider) => provider.is_default_provider
+      );
 
-    // separate function so we can `return` to break out
-    const _llmUpdate = () => {
-      // if the user has overridden in this session and just switched to a brand
-      // new session, use their manually specified model
-      if (userHasManuallyOverriddenLLM && !currentChatSession) {
-        return;
+      if (defaultProvider) {
+        setLlmOverride({
+          name: defaultProvider.name,
+          provider: defaultProvider.provider,
+          modelName: defaultProvider.default_model_name,
+        });
       }
-
-      if (currentChatSession?.current_alternate_model) {
-        setCurrentLlm(
-          getValidLlmDescriptor(currentChatSession.current_alternate_model)
-        );
-      } else if (liveAssistant?.llm_model_version_override) {
-        setCurrentLlm(
-          getValidLlmDescriptor(liveAssistant.llm_model_version_override)
-        );
-      } else if (userHasManuallyOverriddenLLM) {
-        // if the user has an override and there's nothing special about the
-        // current chat session, use the override
-        return;
-      } else if (user?.preferences?.default_model) {
-        setCurrentLlm(getValidLlmDescriptor(user.preferences.default_model));
-      } else {
-        const defaultProvider = llmProviders.find(
-          (provider) => provider.is_default_provider
-        );
-
-        if (defaultProvider) {
-          setCurrentLlm({
-            name: defaultProvider.name,
-            provider: defaultProvider.provider,
-            modelName: defaultProvider.default_model_name,
-          });
-        }
-      }
-    };
-
-    _llmUpdate();
+    }
     setChatSession(currentChatSession || null);
   };
 
-  const getValidLlmDescriptor = (
-    modelName: string | null | undefined
-  ): LlmDescriptor => {
-    if (modelName) {
-      const model = destructureValue(modelName);
+  const getValidLlmOverride = (
+    overrideModel: string | null | undefined
+  ): LlmOverride => {
+    if (overrideModel) {
+      const model = destructureValue(overrideModel);
       if (!(model.modelName && model.modelName.length > 0)) {
         const provider = llmProviders.find((p) =>
-          p.model_names.includes(modelName)
+          p.model_names.includes(overrideModel)
         );
         if (provider) {
           return {
-            modelName: modelName,
+            modelName: overrideModel,
             name: provider.name,
             provider: provider.provider,
           };
@@ -514,32 +491,38 @@ export function useLlmManager(
     setImageFilesPresent(present);
   };
 
-  // Manually set the LLM
-  const updateCurrentLlm = (newLlm: LlmDescriptor) => {
+  const [llmOverride, setLlmOverride] = useState<LlmOverride>({
+    name: "",
+    provider: "",
+    modelName: "",
+  });
+
+  // Manually set the override
+  const updateLLMOverride = (newOverride: LlmOverride) => {
     const provider =
-      newLlm.provider || findProviderForModel(llmProviders, newLlm.modelName);
+      newOverride.provider ||
+      findProviderForModel(llmProviders, newOverride.modelName);
     const structuredValue = structureValue(
-      newLlm.name,
+      newOverride.name,
       provider,
-      newLlm.modelName
+      newOverride.modelName
     );
-    setCurrentLlm(getValidLlmDescriptor(structuredValue));
-    setUserHasManuallyOverriddenLLM(true);
+    setLlmOverride(getValidLlmOverride(structuredValue));
   };
 
-  const updateModelOverrideBasedOnChatSession = (chatSession?: ChatSession) => {
+  const updateModelOverrideForChatSession = (chatSession?: ChatSession) => {
     if (chatSession && chatSession.current_alternate_model?.length > 0) {
-      setCurrentLlm(getValidLlmDescriptor(chatSession.current_alternate_model));
+      setLlmOverride(getValidLlmOverride(chatSession.current_alternate_model));
     }
   };
 
   const [temperature, setTemperature] = useState<number>(() => {
-    llmUpdate();
+    llmOverrideUpdate();
 
     if (currentChatSession?.current_temperature_override != null) {
       return Math.min(
         currentChatSession.current_temperature_override,
-        isAnthropic(currentLlm.provider, currentLlm.modelName) ? 1.0 : 2.0
+        isAnthropic(llmOverride.provider, llmOverride.modelName) ? 1.0 : 2.0
       );
     } else if (
       liveAssistant?.tools.some((tool) => tool.name === SEARCH_TOOL_ID)
@@ -550,23 +533,22 @@ export function useLlmManager(
   });
 
   const maxTemperature = useMemo(() => {
-    return isAnthropic(currentLlm.provider, currentLlm.modelName) ? 1.0 : 2.0;
-  }, [currentLlm]);
+    return isAnthropic(llmOverride.provider, llmOverride.modelName) ? 1.0 : 2.0;
+  }, [llmOverride]);
 
   useEffect(() => {
-    if (isAnthropic(currentLlm.provider, currentLlm.modelName)) {
+    if (isAnthropic(llmOverride.provider, llmOverride.modelName)) {
       const newTemperature = Math.min(temperature, 1.0);
       setTemperature(newTemperature);
       if (chatSession?.id) {
         updateTemperatureOverrideForChatSession(chatSession.id, newTemperature);
       }
     }
-  }, [currentLlm]);
+  }, [llmOverride]);
 
   useEffect(() => {
-    llmUpdate();
-
     if (!chatSession && currentChatSession) {
+      setChatSession(currentChatSession || null);
       if (temperature) {
         updateTemperatureOverrideForChatSession(
           currentChatSession.id,
@@ -588,7 +570,7 @@ export function useLlmManager(
   }, [liveAssistant, currentChatSession]);
 
   const updateTemperature = (temperature: number) => {
-    if (isAnthropic(currentLlm.provider, currentLlm.modelName)) {
+    if (isAnthropic(llmOverride.provider, llmOverride.modelName)) {
       setTemperature((prevTemp) => Math.min(temperature, 1.0));
     } else {
       setTemperature(temperature);
@@ -599,9 +581,9 @@ export function useLlmManager(
   };
 
   return {
-    updateModelOverrideBasedOnChatSession,
-    currentLlm,
-    updateCurrentLlm,
+    updateModelOverrideForChatSession,
+    llmOverride,
+    updateLLMOverride,
     temperature,
     updateTemperature,
     imageFilesPresent,