improve scroll

.github/workflows/pr-python-model-tests.yml

@@ -17,13 +17,8 @@ env:
   AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
   AWS_REGION_NAME: ${{ secrets.AWS_REGION_NAME }}
 
-  # API keys for testing
-  COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }}
-  LITELLM_API_KEY: ${{ secrets.LITELLM_API_KEY }}
-  LITELLM_API_URL: ${{ secrets.LITELLM_API_URL }}
+  # OpenAI
   OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
-  AZURE_API_KEY: ${{ secrets.AZURE_API_KEY }}
-  AZURE_API_URL: ${{ secrets.AZURE_API_URL }}
 
 jobs:
   model-check:
@@ -77,7 +72,7 @@ jobs:
           REQUIRE_EMAIL_VERIFICATION=false \
           DISABLE_TELEMETRY=true \
           IMAGE_TAG=test \
-          docker compose -f docker-compose.model-server-test.yml -p onyx-stack up -d indexing_model_server
+          docker compose -f docker-compose.dev.yml -p onyx-stack up -d indexing_model_server
         id: start_docker
 
       - name: Wait for service to be ready
@@ -128,22 +123,9 @@ jobs:
             --data '{"text":"Scheduled Model Tests failed! Check the run at: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"}' \
             $SLACK_WEBHOOK
             
-      - name: Dump all-container logs (optional)
-        if: always()
-        run: |
-          cd deployment/docker_compose
-          docker compose -f docker-compose.model-server-test.yml -p onyx-stack logs --no-color > $GITHUB_WORKSPACE/docker-compose.log || true
-
-      - name: Upload logs
-        if: always()
-        uses: actions/upload-artifact@v4
-        with:
-          name: docker-all-logs
-          path: ${{ github.workspace }}/docker-compose.log
-          
       - name: Stop Docker containers
         if: always()
         run: |
           cd deployment/docker_compose
-          docker compose -f docker-compose.model-server-test.yml -p onyx-stack down -v
+          docker compose -f docker-compose.dev.yml -p onyx-stack down -v
           

backend/alembic/versions/8f43500ee275_add_index.py

@@ -1,32 +0,0 @@
-"""add index
-
-Revision ID: 8f43500ee275
-Revises: da42808081e3
-Create Date: 2025-02-24 17:35:33.072714
-
-"""
-from alembic import op
-
-
-# revision identifiers, used by Alembic.
-revision = "8f43500ee275"
-down_revision = "da42808081e3"
-branch_labels = None
-depends_on = None
-
-
-def upgrade() -> None:
-    # Create a basic index on the lowercase message column for direct text matching
-    # Limit to 1500 characters to stay well under the 2856 byte limit of btree version 4
-    # op.execute(
-    #     """
-    #     CREATE INDEX idx_chat_message_message_lower
-    #     ON chat_message (LOWER(substring(message, 1, 1500)))
-    #     """
-    # )
-    pass
-
-
-def downgrade() -> None:
-    # Drop the index
-    op.execute("DROP INDEX IF EXISTS idx_chat_message_message_lower;")

backend/alembic/versions/da42808081e3_migrate_jira_connectors_to_new_format.py

@@ -1,120 +0,0 @@
-"""migrate jira connectors to new format
-
-Revision ID: da42808081e3
-Revises: f13db29f3101
-Create Date: 2025-02-24 11:24:54.396040
-
-"""
-from alembic import op
-import sqlalchemy as sa
-import json
-
-from onyx.configs.constants import DocumentSource
-from onyx.connectors.onyx_jira.utils import extract_jira_project
-
-
-# revision identifiers, used by Alembic.
-revision = "da42808081e3"
-down_revision = "f13db29f3101"
-branch_labels = None
-depends_on = None
-
-
-def upgrade() -> None:
-    # Get all Jira connectors
-    conn = op.get_bind()
-
-    # First get all Jira connectors
-    jira_connectors = conn.execute(
-        sa.text(
-            """
-            SELECT id, connector_specific_config
-            FROM connector
-            WHERE source = :source
-            """
-        ),
-        {"source": DocumentSource.JIRA.value.upper()},
-    ).fetchall()
-
-    # Update each connector's config
-    for connector_id, old_config in jira_connectors:
-        if not old_config:
-            continue
-
-        # Extract project key from URL if it exists
-        new_config: dict[str, str | None] = {}
-        if project_url := old_config.get("jira_project_url"):
-            # Parse the URL to get base and project
-            try:
-                jira_base, project_key = extract_jira_project(project_url)
-                new_config = {"jira_base_url": jira_base, "project_key": project_key}
-            except ValueError:
-                # If URL parsing fails, just use the URL as the base
-                new_config = {
-                    "jira_base_url": project_url.split("/projects/")[0],
-                    "project_key": None,
-                }
-        else:
-            # For connectors without a project URL, we need admin intervention
-            # Mark these for review
-            print(
-                f"WARNING: Jira connector {connector_id} has no project URL configured"
-            )
-            continue
-
-        # Update the connector config
-        conn.execute(
-            sa.text(
-                """
-                UPDATE connector
-                SET connector_specific_config = :new_config
-                WHERE id = :id
-                """
-            ),
-            {"id": connector_id, "new_config": json.dumps(new_config)},
-        )
-
-
-def downgrade() -> None:
-    # Get all Jira connectors
-    conn = op.get_bind()
-
-    # First get all Jira connectors
-    jira_connectors = conn.execute(
-        sa.text(
-            """
-            SELECT id, connector_specific_config
-            FROM connector
-            WHERE source = :source
-            """
-        ),
-        {"source": DocumentSource.JIRA.value.upper()},
-    ).fetchall()
-
-    # Update each connector's config back to the old format
-    for connector_id, new_config in jira_connectors:
-        if not new_config:
-            continue
-
-        old_config = {}
-        base_url = new_config.get("jira_base_url")
-        project_key = new_config.get("project_key")
-
-        if base_url and project_key:
-            old_config = {"jira_project_url": f"{base_url}/projects/{project_key}"}
-        elif base_url:
-            old_config = {"jira_project_url": base_url}
-        else:
-            continue
-
-        # Update the connector config
-        conn.execute(
-            sa.text(
-                """
-                UPDATE connector
-                SET connector_specific_config = :old_config
-                WHERE id = :id
-                """
-            ),
-            {"id": connector_id, "old_config": old_config},
-        )

backend/ee/onyx/background/celery/apps/primary.py

@@ -5,9 +5,11 @@ from onyx.background.celery.apps.primary import celery_app
 from onyx.background.task_utils import build_celery_task_wrapper
 from onyx.configs.app_configs import JOB_TIMEOUT
 from onyx.db.chat import delete_chat_sessions_older_than
-from onyx.db.engine import get_session_with_current_tenant
+from onyx.db.engine import get_session_with_tenant
 from onyx.server.settings.store import load_settings
 from onyx.utils.logger import setup_logger
+from shared_configs.configs import MULTI_TENANT
+from shared_configs.contextvars import CURRENT_TENANT_ID_CONTEXTVAR
 
 logger = setup_logger()
 
@@ -16,8 +18,10 @@ logger = setup_logger()
 
 @build_celery_task_wrapper(name_chat_ttl_task)
 @celery_app.task(soft_time_limit=JOB_TIMEOUT)
-def perform_ttl_management_task(retention_limit_days: int, *, tenant_id: str) -> None:
-    with get_session_with_current_tenant() as db_session:
+def perform_ttl_management_task(
+    retention_limit_days: int, *, tenant_id: str | None
+) -> None:
+    with get_session_with_tenant(tenant_id=tenant_id) as db_session:
         delete_chat_sessions_older_than(retention_limit_days, db_session)
 
 
@@ -31,19 +35,24 @@ def perform_ttl_management_task(retention_limit_days: int, *, tenant_id: str) ->
     ignore_result=True,
     soft_time_limit=JOB_TIMEOUT,
 )
-def check_ttl_management_task(*, tenant_id: str) -> None:
+def check_ttl_management_task(*, tenant_id: str | None) -> None:
     """Runs periodically to check if any ttl tasks should be run and adds them
     to the queue"""
+    token = None
+    if MULTI_TENANT and tenant_id is not None:
+        token = CURRENT_TENANT_ID_CONTEXTVAR.set(tenant_id)
 
     settings = load_settings()
     retention_limit_days = settings.maximum_chat_retention_days
-    with get_session_with_current_tenant() as db_session:
+    with get_session_with_tenant(tenant_id=tenant_id) as db_session:
         if should_perform_chat_ttl_check(retention_limit_days, db_session):
             perform_ttl_management_task.apply_async(
                 kwargs=dict(
                     retention_limit_days=retention_limit_days, tenant_id=tenant_id
                 ),
             )
+    if token is not None:
+        CURRENT_TENANT_ID_CONTEXTVAR.reset(token)
 
 
 @celery_app.task(
@@ -51,9 +60,9 @@ def check_ttl_management_task(*, tenant_id: str) -> None:
     ignore_result=True,
     soft_time_limit=JOB_TIMEOUT,
 )
-def autogenerate_usage_report_task(*, tenant_id: str) -> None:
+def autogenerate_usage_report_task(*, tenant_id: str | None) -> None:
     """This generates usage report under the /admin/generate-usage/report endpoint"""
-    with get_session_with_current_tenant() as db_session:
+    with get_session_with_tenant(tenant_id=tenant_id) as db_session:
         create_new_usage_report(
             db_session=db_session,
             user_id=None,

backend/ee/onyx/background/celery/tasks/vespa/tasks.py

@@ -18,7 +18,7 @@ logger = setup_logger()
 
 
 def monitor_usergroup_taskset(
-    tenant_id: str, key_bytes: bytes, r: Redis, db_session: Session
+    tenant_id: str | None, key_bytes: bytes, r: Redis, db_session: Session
 ) -> None:
     """This function is likely to move in the worker refactor happening next."""
     fence_key = key_bytes.decode("utf-8")

backend/ee/onyx/db/connector_credential_pair.py

@@ -4,7 +4,6 @@ from sqlalchemy.orm import Session
 from onyx.configs.constants import DocumentSource
 from onyx.db.connector_credential_pair import get_connector_credential_pair
 from onyx.db.enums import AccessType
-from onyx.db.enums import ConnectorCredentialPairStatus
 from onyx.db.models import Connector
 from onyx.db.models import ConnectorCredentialPair
 from onyx.db.models import UserGroup__ConnectorCredentialPair
@@ -36,11 +35,10 @@ def _delete_connector_credential_pair_user_groups_relationship__no_commit(
 def get_cc_pairs_by_source(
     db_session: Session,
     source_type: DocumentSource,
-    access_type: AccessType | None = None,
-    status: ConnectorCredentialPairStatus | None = None,
+    only_sync: bool,
 ) -> list[ConnectorCredentialPair]:
     """
-    Get all cc_pairs for a given source type with optional filtering by access_type and status
+    Get all cc_pairs for a given source type (and optionally only sync)
     result is sorted by cc_pair id
     """
     query = (
@@ -50,11 +48,8 @@ def get_cc_pairs_by_source(
         .order_by(ConnectorCredentialPair.id)
     )
 
-    if access_type is not None:
-        query = query.filter(ConnectorCredentialPair.access_type == access_type)
-
-    if status is not None:
-        query = query.filter(ConnectorCredentialPair.status == status)
+    if only_sync:
+        query = query.filter(ConnectorCredentialPair.access_type == AccessType.SYNC)
 
     cc_pairs = query.all()
     return cc_pairs

backend/ee/onyx/external_permissions/google_drive/doc_sync.py

@@ -62,14 +62,12 @@ def _fetch_permissions_for_permission_ids(
         user_email=(owner_email or google_drive_connector.primary_admin_email),
     )
 
-    # We continue on 404 or 403 because the document may not exist or the user may not have access to it
     fetched_permissions = execute_paginated_retrieval(
         retrieval_function=drive_service.permissions().list,
         list_key="permissions",
         fileId=doc_id,
         fields="permissions(id, emailAddress, type, domain)",
         supportsAllDrives=True,
-        continue_on_404_or_403=True,
     )
 
     permissions_for_doc_id = []
@@ -106,13 +104,7 @@ def _get_permissions_from_slim_doc(
     user_emails: set[str] = set()
     group_emails: set[str] = set()
     public = False
-    skipped_permissions = 0
-
     for permission in permissions_list:
-        if not permission:
-            skipped_permissions += 1
-            continue
-
         permission_type = permission["type"]
         if permission_type == "user":
             user_emails.add(permission["emailAddress"])
@@ -129,11 +121,6 @@ def _get_permissions_from_slim_doc(
         elif permission_type == "anyone":
             public = True
 
-    if skipped_permissions > 0:
-        logger.warning(
-            f"Skipped {skipped_permissions} permissions of {len(permissions_list)} for document {slim_doc.id}"
-        )
-
     drive_id = permission_info.get("drive_id")
     group_ids = group_emails | ({drive_id} if drive_id is not None else set())
 

backend/ee/onyx/server/query_and_chat/token_limit.py

@@ -13,7 +13,7 @@ from sqlalchemy import select
 from sqlalchemy.orm import Session
 
 from onyx.db.api_key import is_api_key_email_address
-from onyx.db.engine import get_session_with_current_tenant
+from onyx.db.engine import get_session_with_tenant
 from onyx.db.models import ChatMessage
 from onyx.db.models import ChatSession
 from onyx.db.models import TokenRateLimit
@@ -28,21 +28,21 @@ from onyx.server.query_and_chat.token_limit import _user_is_rate_limited_by_glob
 from onyx.utils.threadpool_concurrency import run_functions_tuples_in_parallel
 
 
-def _check_token_rate_limits(user: User | None) -> None:
+def _check_token_rate_limits(user: User | None, tenant_id: str) -> None:
     if user is None:
         # Unauthenticated users are only rate limited by global settings
-        _user_is_rate_limited_by_global()
+        _user_is_rate_limited_by_global(tenant_id)
 
     elif is_api_key_email_address(user.email):
         # API keys are only rate limited by global settings
-        _user_is_rate_limited_by_global()
+        _user_is_rate_limited_by_global(tenant_id)
 
     else:
         run_functions_tuples_in_parallel(
             [
-                (_user_is_rate_limited, (user.id,)),
-                (_user_is_rate_limited_by_group, (user.id,)),
-                (_user_is_rate_limited_by_global, ()),
+                (_user_is_rate_limited, (user.id, tenant_id)),
+                (_user_is_rate_limited_by_group, (user.id, tenant_id)),
+                (_user_is_rate_limited_by_global, (tenant_id,)),
             ]
         )
 
@@ -52,8 +52,8 @@ User rate limits
 """
 
 
-def _user_is_rate_limited(user_id: UUID) -> None:
-    with get_session_with_current_tenant() as db_session:
+def _user_is_rate_limited(user_id: UUID, tenant_id: str) -> None:
+    with get_session_with_tenant(tenant_id=tenant_id) as db_session:
         user_rate_limits = fetch_all_user_token_rate_limits(
             db_session=db_session, enabled_only=True, ordered=False
         )
@@ -93,8 +93,8 @@ User Group rate limits
 """
 
 
-def _user_is_rate_limited_by_group(user_id: UUID) -> None:
-    with get_session_with_current_tenant() as db_session:
+def _user_is_rate_limited_by_group(user_id: UUID, tenant_id: str | None) -> None:
+    with get_session_with_tenant(tenant_id=tenant_id) as db_session:
         group_rate_limits = _fetch_all_user_group_rate_limits(user_id, db_session)
 
         if group_rate_limits:

backend/ee/onyx/server/query_history/api.py

@@ -2,7 +2,6 @@ import csv
 import io
 from datetime import datetime
 from datetime import timezone
-from http import HTTPStatus
 from uuid import UUID
 
 from fastapi import APIRouter
@@ -22,10 +21,8 @@ from ee.onyx.server.query_history.models import QuestionAnswerPairSnapshot
 from onyx.auth.users import current_admin_user
 from onyx.auth.users import get_display_email
 from onyx.chat.chat_utils import create_chat_chain
-from onyx.configs.app_configs import ONYX_QUERY_HISTORY_TYPE
 from onyx.configs.constants import MessageType
 from onyx.configs.constants import QAFeedbackType
-from onyx.configs.constants import QueryHistoryType
 from onyx.configs.constants import SessionType
 from onyx.db.chat import get_chat_session_by_id
 from onyx.db.chat import get_chat_sessions_by_user
@@ -38,8 +35,6 @@ from onyx.server.query_and_chat.models import ChatSessionsResponse
 
 router = APIRouter()
 
-ONYX_ANONYMIZED_EMAIL = "anonymous@anonymous.invalid"
-
 
 def fetch_and_process_chat_session_history(
     db_session: Session,
@@ -112,17 +107,6 @@ def get_user_chat_sessions(
     _: User | None = Depends(current_admin_user),
     db_session: Session = Depends(get_session),
 ) -> ChatSessionsResponse:
-    # we specifically don't allow this endpoint if "anonymized" since
-    # this is a direct query on the user id
-    if ONYX_QUERY_HISTORY_TYPE in [
-        QueryHistoryType.DISABLED,
-        QueryHistoryType.ANONYMIZED,
-    ]:
-        raise HTTPException(
-            status_code=HTTPStatus.FORBIDDEN,
-            detail="Per user query history has been disabled by the administrator.",
-        )
-
     try:
         chat_sessions = get_chat_sessions_by_user(
             user_id=user_id, deleted=False, db_session=db_session, limit=0
@@ -138,7 +122,6 @@ def get_user_chat_sessions(
                 name=chat.description,
                 persona_id=chat.persona_id,
                 time_created=chat.time_created.isoformat(),
-                time_updated=chat.time_updated.isoformat(),
                 shared_status=chat.shared_status,
                 folder_id=chat.folder_id,
                 current_alternate_model=chat.current_alternate_model,
@@ -158,12 +141,6 @@ def get_chat_session_history(
     _: User | None = Depends(current_admin_user),
     db_session: Session = Depends(get_session),
 ) -> PaginatedReturn[ChatSessionMinimal]:
-    if ONYX_QUERY_HISTORY_TYPE == QueryHistoryType.DISABLED:
-        raise HTTPException(
-            status_code=HTTPStatus.FORBIDDEN,
-            detail="Query history has been disabled by the administrator.",
-        )
-
     page_of_chat_sessions = get_page_of_chat_sessions(
         page_num=page_num,
         page_size=page_size,
@@ -180,16 +157,11 @@ def get_chat_session_history(
         feedback_filter=feedback_type,
     )
 
-    minimal_chat_sessions: list[ChatSessionMinimal] = []
-
-    for chat_session in page_of_chat_sessions:
-        minimal_chat_session = ChatSessionMinimal.from_chat_session(chat_session)
-        if ONYX_QUERY_HISTORY_TYPE == QueryHistoryType.ANONYMIZED:
-            minimal_chat_session.user_email = ONYX_ANONYMIZED_EMAIL
-        minimal_chat_sessions.append(minimal_chat_session)
-
     return PaginatedReturn(
-        items=minimal_chat_sessions,
+        items=[
+            ChatSessionMinimal.from_chat_session(chat_session)
+            for chat_session in page_of_chat_sessions
+        ],
         total_items=total_filtered_chat_sessions_count,
     )
 
@@ -200,12 +172,6 @@ def get_chat_session_admin(
     _: User | None = Depends(current_admin_user),
     db_session: Session = Depends(get_session),
 ) -> ChatSessionSnapshot:
-    if ONYX_QUERY_HISTORY_TYPE == QueryHistoryType.DISABLED:
-        raise HTTPException(
-            status_code=HTTPStatus.FORBIDDEN,
-            detail="Query history has been disabled by the administrator.",
-        )
-
     try:
         chat_session = get_chat_session_by_id(
             chat_session_id=chat_session_id,
@@ -227,9 +193,6 @@ def get_chat_session_admin(
             f"Could not create snapshot for chat session with id '{chat_session_id}'",
         )
 
-    if ONYX_QUERY_HISTORY_TYPE == QueryHistoryType.ANONYMIZED:
-        snapshot.user_email = ONYX_ANONYMIZED_EMAIL
-
     return snapshot
 
 
@@ -240,12 +203,6 @@ def get_query_history_as_csv(
     end: datetime | None = None,
     db_session: Session = Depends(get_session),
 ) -> StreamingResponse:
-    if ONYX_QUERY_HISTORY_TYPE == QueryHistoryType.DISABLED:
-        raise HTTPException(
-            status_code=HTTPStatus.FORBIDDEN,
-            detail="Query history has been disabled by the administrator.",
-        )
-
     complete_chat_session_history = fetch_and_process_chat_session_history(
         db_session=db_session,
         start=start or datetime.fromtimestamp(0, tz=timezone.utc),
@@ -256,9 +213,6 @@ def get_query_history_as_csv(
 
     question_answer_pairs: list[QuestionAnswerPairSnapshot] = []
     for chat_session_snapshot in complete_chat_session_history:
-        if ONYX_QUERY_HISTORY_TYPE == QueryHistoryType.ANONYMIZED:
-            chat_session_snapshot.user_email = ONYX_ANONYMIZED_EMAIL
-
         question_answer_pairs.extend(
             QuestionAnswerPairSnapshot.from_chat_session_snapshot(chat_session_snapshot)
         )

backend/ee/onyx/server/tenants/provisioning.py

@@ -224,7 +224,7 @@ def configure_default_api_keys(db_session: Session) -> None:
             name="Anthropic",
             provider=ANTHROPIC_PROVIDER_NAME,
             api_key=ANTHROPIC_DEFAULT_API_KEY,
-            default_model_name="claude-3-7-sonnet-20250219",
+            default_model_name="claude-3-5-sonnet-20241022",
             fast_default_model_name="claude-3-5-sonnet-20241022",
             model_names=ANTHROPIC_MODEL_NAMES,
         )

backend/ee/onyx/server/tenants/user_mapping.py

@@ -28,7 +28,7 @@ def get_tenant_id_for_email(email: str) -> str:
 
 
 def user_owns_a_tenant(email: str) -> bool:
-    with get_session_with_tenant(tenant_id=POSTGRES_DEFAULT_SCHEMA) as db_session:
+    with get_session_with_tenant(tenant_id=None) as db_session:
         result = (
             db_session.query(UserTenantMapping)
             .filter(UserTenantMapping.email == email)
@@ -38,7 +38,7 @@ def user_owns_a_tenant(email: str) -> bool:
 
 
 def add_users_to_tenant(emails: list[str], tenant_id: str) -> None:
-    with get_session_with_tenant(tenant_id=POSTGRES_DEFAULT_SCHEMA) as db_session:
+    with get_session_with_tenant(tenant_id=None) as db_session:
         try:
             for email in emails:
                 db_session.add(UserTenantMapping(email=email, tenant_id=tenant_id))
@@ -48,7 +48,7 @@ def add_users_to_tenant(emails: list[str], tenant_id: str) -> None:
 
 
 def remove_users_from_tenant(emails: list[str], tenant_id: str) -> None:
-    with get_session_with_tenant(tenant_id=POSTGRES_DEFAULT_SCHEMA) as db_session:
+    with get_session_with_tenant(tenant_id=None) as db_session:
         try:
             mappings_to_delete = (
                 db_session.query(UserTenantMapping)
@@ -71,7 +71,7 @@ def remove_users_from_tenant(emails: list[str], tenant_id: str) -> None:
 
 
 def remove_all_users_from_tenant(tenant_id: str) -> None:
-    with get_session_with_tenant(tenant_id=POSTGRES_DEFAULT_SCHEMA) as db_session:
+    with get_session_with_tenant(tenant_id=None) as db_session:
         db_session.query(UserTenantMapping).filter(
             UserTenantMapping.tenant_id == tenant_id
         ).delete()

backend/onyx/auth/api_key.py

@@ -10,7 +10,6 @@ from pydantic import BaseModel
 
 from onyx.auth.schemas import UserRole
 from onyx.configs.app_configs import API_KEY_HASH_ROUNDS
-from shared_configs.configs import MULTI_TENANT
 
 
 _API_KEY_HEADER_NAME = "Authorization"
@@ -36,7 +35,8 @@ class ApiKeyDescriptor(BaseModel):
 
 
 def generate_api_key(tenant_id: str | None = None) -> str:
-    if not MULTI_TENANT or not tenant_id:
+    # For backwards compatibility, if no tenant_id, generate old style key
+    if not tenant_id:
         return _API_KEY_PREFIX + secrets.token_urlsafe(_API_KEY_LEN)
 
     encoded_tenant = quote(tenant_id)  # URL encode the tenant ID

backend/onyx/auth/email_utils.py

@@ -2,8 +2,6 @@ import smtplib
 from datetime import datetime
 from email.mime.multipart import MIMEMultipart
 from email.mime.text import MIMEText
-from email.utils import formatdate
-from email.utils import make_msgid
 
 from onyx.configs.app_configs import EMAIL_CONFIGURED
 from onyx.configs.app_configs import EMAIL_FROM
@@ -15,7 +13,6 @@ from onyx.configs.app_configs import WEB_DOMAIN
 from onyx.configs.constants import AuthType
 from onyx.configs.constants import TENANT_ID_COOKIE_NAME
 from onyx.db.models import User
-from shared_configs.configs import MULTI_TENANT
 
 HTML_EMAIL_TEMPLATE = """\
 <!DOCTYPE html>
@@ -153,9 +150,8 @@ def send_email(
     msg = MIMEMultipart("alternative")
     msg["Subject"] = subject
     msg["To"] = user_email
-    msg["From"] = mail_from
-    msg["Date"] = formatdate(localtime=True)
-    msg["Message-ID"] = make_msgid(domain="onyx.app")
+    if mail_from:
+        msg["From"] = mail_from
 
     part_text = MIMEText(text_body, "plain")
     part_html = MIMEText(html_body, "html")
@@ -177,7 +173,7 @@ def send_subscription_cancellation_email(user_email: str) -> None:
     subject = "Your Onyx Subscription Has Been Canceled"
     heading = "Subscription Canceled"
     message = (
-        "<p>We're sorry to see you go.</p>"
+        "<p>We’re sorry to see you go.</p>"
         "<p>Your subscription has been canceled and will end on your next billing date.</p>"
         "<p>If you change your mind, you can always come back!</p>"
     )
@@ -243,13 +239,13 @@ def send_user_email_invite(
 def send_forgot_password_email(
     user_email: str,
     token: str,
-    tenant_id: str,
     mail_from: str = EMAIL_FROM,
+    tenant_id: str | None = None,
 ) -> None:
     # Builds a forgot password email with or without fancy HTML
     subject = "Onyx Forgot Password"
     link = f"{WEB_DOMAIN}/auth/reset-password?token={token}"
-    if MULTI_TENANT:
+    if tenant_id:
         link += f"&{TENANT_ID_COOKIE_NAME}={tenant_id}"
     message = f"<p>Click the following link to reset your password:</p><p>{link}</p>"
     html_content = build_html_email("Reset Your Password", message)

backend/onyx/auth/users.py

@@ -214,7 +214,7 @@ def verify_email_is_invited(email: str) -> None:
     raise PermissionError("User not on allowed user whitelist")
 
 
-def verify_email_in_whitelist(email: str, tenant_id: str) -> None:
+def verify_email_in_whitelist(email: str, tenant_id: str | None = None) -> None:
     with get_session_with_tenant(tenant_id=tenant_id) as db_session:
         if not get_user_by_email(email, db_session):
             verify_email_is_invited(email)
@@ -420,7 +420,7 @@ class UserManager(UUIDIDMixin, BaseUserManager[User, uuid.UUID]):
             except exceptions.UserNotExists:
                 try:
                     # Attempt to get user by email
-                    user = cast(User, await self.user_db.get_by_email(account_email))
+                    user = await self.get_by_email(account_email)
                     if not associate_by_email:
                         raise exceptions.UserAlreadyExists()
 
@@ -553,7 +553,7 @@ class UserManager(UUIDIDMixin, BaseUserManager[User, uuid.UUID]):
             async_return_default_schema,
         )(email=user.email)
 
-        send_forgot_password_email(user.email, tenant_id=tenant_id, token=token)
+        send_forgot_password_email(user.email, token, tenant_id=tenant_id)
 
     async def on_after_request_verify(
         self, user: User, token: str, request: Optional[Request] = None

backend/onyx/background/celery/apps/app_base.py

@@ -2,7 +2,6 @@ import logging
 import multiprocessing
 import time
 from typing import Any
-from typing import cast
 
 import sentry_sdk
 from celery import Task
@@ -132,9 +131,9 @@ def on_task_postrun(
     # Get tenant_id directly from kwargs- each celery task has a tenant_id kwarg
     if not kwargs:
         logger.error(f"Task {task.name} (ID: {task_id}) is missing kwargs")
-        tenant_id = POSTGRES_DEFAULT_SCHEMA
+        tenant_id = None
     else:
-        tenant_id = cast(str, kwargs.get("tenant_id", POSTGRES_DEFAULT_SCHEMA))
+        tenant_id = kwargs.get("tenant_id")
 
     task_logger.debug(
         f"Task {task.name} (ID: {task_id}) completed with state: {state} "

backend/onyx/background/celery/celery_redis.py

@@ -92,8 +92,7 @@ def celery_find_task(task_id: str, queue: str, r: Redis) -> int:
 
 
 def celery_get_queued_task_ids(queue: str, r: Redis) -> set[str]:
-    """This is a redis specific way to build a list of tasks in a queue and return them
-    as a set.
+    """This is a redis specific way to build a list of tasks in a queue.
 
     This helps us read the queue once and then efficiently look for missing tasks
     in the queue.

backend/onyx/background/celery/celery_utils.py

@@ -34,7 +34,7 @@ def _get_deletion_status(
     connector_id: int,
     credential_id: int,
     db_session: Session,
-    tenant_id: str,
+    tenant_id: str | None = None,
 ) -> TaskQueueState | None:
     """We no longer store TaskQueueState in the DB for a deletion attempt.
     This function populates TaskQueueState by just checking redis.
@@ -67,7 +67,7 @@ def get_deletion_attempt_snapshot(
     connector_id: int,
     credential_id: int,
     db_session: Session,
-    tenant_id: str,
+    tenant_id: str | None = None,
 ) -> DeletionAttemptSnapshot | None:
     deletion_task = _get_deletion_status(
         connector_id, credential_id, db_session, tenant_id

backend/onyx/background/celery/tasks/connector_deletion/tasks.py

@@ -8,21 +8,16 @@ from celery import Celery
 from celery import shared_task
 from celery import Task
 from celery.exceptions import SoftTimeLimitExceeded
-from pydantic import ValidationError
 from redis import Redis
 from redis.lock import Lock as RedisLock
 from sqlalchemy.orm import Session
 
 from onyx.background.celery.apps.app_base import task_logger
-from onyx.background.celery.celery_redis import celery_get_queue_length
-from onyx.background.celery.celery_redis import celery_get_queued_task_ids
 from onyx.configs.app_configs import JOB_TIMEOUT
 from onyx.configs.constants import CELERY_GENERIC_BEAT_LOCK_TIMEOUT
-from onyx.configs.constants import OnyxCeleryQueues
 from onyx.configs.constants import OnyxCeleryTask
 from onyx.configs.constants import OnyxRedisConstants
 from onyx.configs.constants import OnyxRedisLocks
-from onyx.configs.constants import OnyxRedisSignals
 from onyx.db.connector import fetch_connector_by_id
 from onyx.db.connector_credential_pair import add_deletion_failure_message
 from onyx.db.connector_credential_pair import (
@@ -57,51 +52,6 @@ class TaskDependencyError(RuntimeError):
     with connector deletion."""
 
 
-def revoke_tasks_blocking_deletion(
-    redis_connector: RedisConnector, db_session: Session, app: Celery
-) -> None:
-    search_settings_list = get_all_search_settings(db_session)
-    for search_settings in search_settings_list:
-        redis_connector_index = redis_connector.new_index(search_settings.id)
-        try:
-            index_payload = redis_connector_index.payload
-            if index_payload and index_payload.celery_task_id:
-                app.control.revoke(index_payload.celery_task_id)
-                task_logger.info(
-                    f"Revoked indexing task {index_payload.celery_task_id}."
-                )
-        except Exception:
-            task_logger.exception("Exception while revoking indexing task")
-
-    try:
-        permissions_sync_payload = redis_connector.permissions.payload
-        if permissions_sync_payload and permissions_sync_payload.celery_task_id:
-            app.control.revoke(permissions_sync_payload.celery_task_id)
-            task_logger.info(
-                f"Revoked permissions sync task {permissions_sync_payload.celery_task_id}."
-            )
-    except Exception:
-        task_logger.exception("Exception while revoking pruning task")
-
-    try:
-        prune_payload = redis_connector.prune.payload
-        if prune_payload and prune_payload.celery_task_id:
-            app.control.revoke(prune_payload.celery_task_id)
-            task_logger.info(f"Revoked pruning task {prune_payload.celery_task_id}.")
-    except Exception:
-        task_logger.exception("Exception while revoking permissions sync task")
-
-    try:
-        external_group_sync_payload = redis_connector.external_group_sync.payload
-        if external_group_sync_payload and external_group_sync_payload.celery_task_id:
-            app.control.revoke(external_group_sync_payload.celery_task_id)
-            task_logger.info(
-                f"Revoked external group sync task {external_group_sync_payload.celery_task_id}."
-            )
-    except Exception:
-        task_logger.exception("Exception while revoking external group sync task")
-
-
 @shared_task(
     name=OnyxCeleryTask.CHECK_FOR_CONNECTOR_DELETION,
     ignore_result=True,
@@ -109,36 +59,22 @@ def revoke_tasks_blocking_deletion(
     trail=False,
     bind=True,
 )
-def check_for_connector_deletion_task(self: Task, *, tenant_id: str) -> bool | None:
+def check_for_connector_deletion_task(
+    self: Task, *, tenant_id: str | None
+) -> bool | None:
     r = get_redis_client()
     r_replica = get_redis_replica_client()
-    r_celery: Redis = self.app.broker_connection().channel().client  # type: ignore
 
     lock_beat: RedisLock = r.lock(
         OnyxRedisLocks.CHECK_CONNECTOR_DELETION_BEAT_LOCK,
         timeout=CELERY_GENERIC_BEAT_LOCK_TIMEOUT,
     )
 
-    # Prevent this task from overlapping with itself
+    # these tasks should never overlap
     if not lock_beat.acquire(blocking=False):
         return None
 
     try:
-        # we want to run this less frequently than the overall task
-        lock_beat.reacquire()
-        if not r.exists(OnyxRedisSignals.BLOCK_VALIDATE_CONNECTOR_DELETION_FENCES):
-            # clear fences that don't have associated celery tasks in progress
-            try:
-                validate_connector_deletion_fences(
-                    tenant_id, r, r_replica, r_celery, lock_beat
-                )
-            except Exception:
-                task_logger.exception(
-                    "Exception while validating connector deletion fences"
-                )
-
-            r.set(OnyxRedisSignals.BLOCK_VALIDATE_CONNECTOR_DELETION_FENCES, 1, ex=300)
-
         # collect cc_pair_ids
         cc_pair_ids: list[int] = []
         with get_session_with_current_tenant() as db_session:
@@ -156,38 +92,9 @@ def check_for_connector_deletion_task(self: Task, *, tenant_id: str) -> bool | N
                     )
                 except TaskDependencyError as e:
                     # this means we wanted to start deleting but dependent tasks were running
-                    # on the first error, we set a stop signal and revoke the dependent tasks
-                    # on subsequent errors, we hard reset blocking fences after our specified timeout
-                    # is exceeded
+                    # Leave a stop signal to clear indexing and pruning tasks more quickly
                     task_logger.info(str(e))
-
-                    if not redis_connector.stop.fenced:
-                        # one time revoke of celery tasks
-                        task_logger.info("Revoking any tasks blocking deletion.")
-                        revoke_tasks_blocking_deletion(
-                            redis_connector, db_session, self.app
-                        )
-                        redis_connector.stop.set_fence(True)
-                        redis_connector.stop.set_timeout()
-                    else:
-                        # stop signal already set
-                        if redis_connector.stop.timed_out:
-                            # waiting too long, just reset blocking fences
-                            task_logger.info(
-                                "Timed out waiting for tasks blocking deletion. Resetting blocking fences."
-                            )
-                            search_settings_list = get_all_search_settings(db_session)
-                            for search_settings in search_settings_list:
-                                redis_connector_index = redis_connector.new_index(
-                                    search_settings.id
-                                )
-                                redis_connector_index.reset()
-                            redis_connector.prune.reset()
-                            redis_connector.permissions.reset()
-                            redis_connector.external_group_sync.reset()
-                        else:
-                            # just wait
-                            pass
+                    redis_connector.stop.set_fence(True)
                 else:
                     # clear the stop signal if it exists ... no longer needed
                     redis_connector.stop.set_fence(False)
@@ -222,7 +129,7 @@ def try_generate_document_cc_pair_cleanup_tasks(
     cc_pair_id: int,
     db_session: Session,
     lock_beat: RedisLock,
-    tenant_id: str,
+    tenant_id: str | None,
 ) -> int | None:
     """Returns an int if syncing is needed. The int represents the number of sync tasks generated.
     Note that syncing can still be required even if the number of sync tasks generated is zero.
@@ -262,7 +169,6 @@ def try_generate_document_cc_pair_cleanup_tasks(
         return None
 
     # set a basic fence to start
-    redis_connector.delete.set_active()
     fence_payload = RedisConnectorDeletePayload(
         num_tasks=None,
         submitted=datetime.now(timezone.utc),
@@ -343,7 +249,7 @@ def try_generate_document_cc_pair_cleanup_tasks(
 
 
 def monitor_connector_deletion_taskset(
-    tenant_id: str, key_bytes: bytes, r: Redis
+    tenant_id: str | None, key_bytes: bytes, r: Redis
 ) -> None:
     fence_key = key_bytes.decode("utf-8")
     cc_pair_id_str = RedisConnector.get_id_from_fence_key(fence_key)
@@ -495,171 +401,3 @@ def monitor_connector_deletion_taskset(
     )
 
     redis_connector.delete.reset()
-
-
-def validate_connector_deletion_fences(
-    tenant_id: str,
-    r: Redis,
-    r_replica: Redis,
-    r_celery: Redis,
-    lock_beat: RedisLock,
-) -> None:
-    # building lookup table can be expensive, so we won't bother
-    # validating until the queue is small
-    CONNECTION_DELETION_VALIDATION_MAX_QUEUE_LEN = 1024
-
-    queue_len = celery_get_queue_length(OnyxCeleryQueues.CONNECTOR_DELETION, r_celery)
-    if queue_len > CONNECTION_DELETION_VALIDATION_MAX_QUEUE_LEN:
-        return
-
-    queued_upsert_tasks = celery_get_queued_task_ids(
-        OnyxCeleryQueues.CONNECTOR_DELETION, r_celery
-    )
-
-    # validate all existing connector deletion jobs
-    lock_beat.reacquire()
-    keys = cast(set[Any], r_replica.smembers(OnyxRedisConstants.ACTIVE_FENCES))
-    for key in keys:
-        key_bytes = cast(bytes, key)
-        key_str = key_bytes.decode("utf-8")
-        if not key_str.startswith(RedisConnectorDelete.FENCE_PREFIX):
-            continue
-
-        validate_connector_deletion_fence(
-            tenant_id,
-            key_bytes,
-            queued_upsert_tasks,
-            r,
-        )
-
-        lock_beat.reacquire()
-
-    return
-
-
-def validate_connector_deletion_fence(
-    tenant_id: str,
-    key_bytes: bytes,
-    queued_tasks: set[str],
-    r: Redis,
-) -> None:
-    """Checks for the error condition where an indexing fence is set but the associated celery tasks don't exist.
-    This can happen if the indexing worker hard crashes or is terminated.
-    Being in this bad state means the fence will never clear without help, so this function
-    gives the help.
-
-    How this works:
-    1. This function renews the active signal with a 5 minute TTL under the following conditions
-    1.2. When the task is seen in the redis queue
-    1.3. When the task is seen in the reserved / prefetched list
-
-    2. Externally, the active signal is renewed when:
-    2.1. The fence is created
-    2.2. The indexing watchdog checks the spawned task.
-
-    3. The TTL allows us to get through the transitions on fence startup
-    and when the task starts executing.
-
-    More TTL clarification: it is seemingly impossible to exactly query Celery for
-    whether a task is in the queue or currently executing.
-    1. An unknown task id is always returned as state PENDING.
-    2. Redis can be inspected for the task id, but the task id is gone between the time a worker receives the task
-    and the time it actually starts on the worker.
-
-    queued_tasks: the celery queue of lightweight permission sync tasks
-    reserved_tasks: prefetched tasks for sync task generator
-    """
-    # if the fence doesn't exist, there's nothing to do
-    fence_key = key_bytes.decode("utf-8")
-    cc_pair_id_str = RedisConnector.get_id_from_fence_key(fence_key)
-    if cc_pair_id_str is None:
-        task_logger.warning(
-            f"validate_connector_deletion_fence - could not parse id from {fence_key}"
-        )
-        return
-
-    cc_pair_id = int(cc_pair_id_str)
-    # parse out metadata and initialize the helper class with it
-    redis_connector = RedisConnector(tenant_id, int(cc_pair_id))
-
-    # check to see if the fence/payload exists
-    if not redis_connector.delete.fenced:
-        return
-
-    # in the cloud, the payload format may have changed ...
-    # it's a little sloppy, but just reset the fence for now if that happens
-    # TODO: add intentional cleanup/abort logic
-    try:
-        payload = redis_connector.delete.payload
-    except ValidationError:
-        task_logger.exception(
-            "validate_connector_deletion_fence - "
-            "Resetting fence because fence schema is out of date: "
-            f"cc_pair={cc_pair_id} "
-            f"fence={fence_key}"
-        )
-
-        redis_connector.delete.reset()
-        return
-
-    if not payload:
-        return
-
-    # OK, there's actually something for us to validate
-
-    # look up every task in the current taskset in the celery queue
-    # every entry in the taskset should have an associated entry in the celery task queue
-    # because we get the celery tasks first, the entries in our own permissions taskset
-    # should be roughly a subset of the tasks in celery
-
-    # this check isn't very exact, but should be sufficient over a period of time
-    # A single successful check over some number of attempts is sufficient.
-
-    # TODO: if the number of tasks in celery is much lower than than the taskset length
-    # we might be able to shortcut the lookup since by definition some of the tasks
-    # must not exist in celery.
-
-    tasks_scanned = 0
-    tasks_not_in_celery = 0  # a non-zero number after completing our check is bad
-
-    for member in r.sscan_iter(redis_connector.delete.taskset_key):
-        tasks_scanned += 1
-
-        member_bytes = cast(bytes, member)
-        member_str = member_bytes.decode("utf-8")
-        if member_str in queued_tasks:
-            continue
-
-        tasks_not_in_celery += 1
-
-    task_logger.info(
-        "validate_connector_deletion_fence task check: "
-        f"tasks_scanned={tasks_scanned} tasks_not_in_celery={tasks_not_in_celery}"
-    )
-
-    # we're active if there are still tasks to run and those tasks all exist in celery
-    if tasks_scanned > 0 and tasks_not_in_celery == 0:
-        redis_connector.delete.set_active()
-        return
-
-    # we may want to enable this check if using the active task list somehow isn't good enough
-    # if redis_connector_index.generator_locked():
-    #     logger.info(f"{payload.celery_task_id} is currently executing.")
-
-    # if we get here, we didn't find any direct indication that the associated celery tasks exist,
-    # but they still might be there due to gaps in our ability to check states during transitions
-    # Checking the active signal safeguards us against these transition periods
-    # (which has a duration that allows us to bridge those gaps)
-    if redis_connector.delete.active():
-        return
-
-    # celery tasks don't exist and the active signal has expired, possibly due to a crash. Clean it up.
-    task_logger.warning(
-        "validate_connector_deletion_fence - "
-        "Resetting fence because no associated celery tasks were found: "
-        f"cc_pair={cc_pair_id} "
-        f"fence={fence_key}"
-    )
-
-    redis_connector.delete.reset()
-    return

backend/onyx/background/celery/tasks/doc_permission_syncing/tasks.py

@@ -30,7 +30,6 @@ from onyx.background.celery.celery_redis import celery_find_task
 from onyx.background.celery.celery_redis import celery_get_queue_length
 from onyx.background.celery.celery_redis import celery_get_queued_task_ids
 from onyx.background.celery.celery_redis import celery_get_unacked_task_ids
-from onyx.background.celery.tasks.shared.tasks import OnyxCeleryTaskCompletionStatus
 from onyx.configs.app_configs import JOB_TIMEOUT
 from onyx.configs.constants import CELERY_GENERIC_BEAT_LOCK_TIMEOUT
 from onyx.configs.constants import CELERY_PERMISSIONS_SYNC_LOCK_TIMEOUT
@@ -43,10 +42,8 @@ from onyx.configs.constants import OnyxCeleryTask
 from onyx.configs.constants import OnyxRedisConstants
 from onyx.configs.constants import OnyxRedisLocks
 from onyx.configs.constants import OnyxRedisSignals
-from onyx.connectors.factory import validate_ccpair_for_user
 from onyx.db.connector import mark_cc_pair_as_permissions_synced
 from onyx.db.connector_credential_pair import get_connector_credential_pair_from_id
-from onyx.db.connector_credential_pair import update_connector_credential_pair
 from onyx.db.document import upsert_document_by_connector_credential_pair
 from onyx.db.engine import get_session_with_current_tenant
 from onyx.db.enums import AccessType
@@ -66,7 +63,6 @@ from onyx.redis.redis_pool import get_redis_replica_client
 from onyx.redis.redis_pool import redis_lock_dump
 from onyx.server.utils import make_short_id
 from onyx.utils.logger import doc_permission_sync_ctx
-from onyx.utils.logger import format_error_for_logging
 from onyx.utils.logger import LoggerContextVars
 from onyx.utils.logger import setup_logger
 
@@ -197,19 +193,12 @@ def check_for_doc_permissions_sync(self: Task, *, tenant_id: str) -> bool | None
                     monitor_ccpair_permissions_taskset(
                         tenant_id, key_bytes, r, db_session
                     )
-        task_logger.info(f"check_for_doc_permissions_sync finished: tenant={tenant_id}")
     except SoftTimeLimitExceeded:
         task_logger.info(
             "Soft time limit exceeded, task is being terminated gracefully."
         )
-    except Exception as e:
-        error_msg = format_error_for_logging(e)
-        task_logger.warning(
-            f"Unexpected check_for_doc_permissions_sync exception: tenant={tenant_id} {error_msg}"
-        )
-        task_logger.exception(
-            f"Unexpected check_for_doc_permissions_sync exception: tenant={tenant_id}"
-        )
+    except Exception:
+        task_logger.exception(f"Unexpected exception: tenant={tenant_id}")
     finally:
         if lock_beat.owned():
             lock_beat.release()
@@ -221,7 +210,7 @@ def try_creating_permissions_sync_task(
     app: Celery,
     cc_pair_id: int,
     r: Redis,
-    tenant_id: str,
+    tenant_id: str | None,
 ) -> str | None:
     """Returns a randomized payload id on success.
     Returns None if no syncing is required."""
@@ -293,19 +282,13 @@ def try_creating_permissions_sync_task(
         redis_connector.permissions.set_fence(payload)
 
         payload_id = payload.id
-    except Exception as e:
-        error_msg = format_error_for_logging(e)
-        task_logger.warning(
-            f"Unexpected try_creating_permissions_sync_task exception: cc_pair={cc_pair_id} {error_msg}"
-        )
+    except Exception:
+        task_logger.exception(f"Unexpected exception: cc_pair={cc_pair_id}")
         return None
     finally:
         if lock.owned():
             lock.release()
 
-    task_logger.info(
-        f"try_creating_permissions_sync_task finished: cc_pair={cc_pair_id} payload_id={payload_id}"
-    )
     return payload_id
 
 
@@ -320,7 +303,7 @@ def try_creating_permissions_sync_task(
 def connector_permission_sync_generator_task(
     self: Task,
     cc_pair_id: int,
-    tenant_id: str,
+    tenant_id: str | None,
 ) -> None:
     """
     Permission sync task that handles document permission syncing for a given connector credential pair
@@ -405,29 +388,6 @@ def connector_permission_sync_generator_task(
                     f"No connector credential pair found for id: {cc_pair_id}"
                 )
 
-            try:
-                created = validate_ccpair_for_user(
-                    cc_pair.connector.id,
-                    cc_pair.credential.id,
-                    db_session,
-                    enforce_creation=False,
-                )
-                if not created:
-                    task_logger.warning(
-                        f"Unable to create connector credential pair for id: {cc_pair_id}"
-                    )
-            except Exception:
-                task_logger.exception(
-                    f"validate_ccpair_permissions_sync exceptioned: cc_pair={cc_pair_id}"
-                )
-                update_connector_credential_pair(
-                    db_session=db_session,
-                    connector_id=cc_pair.connector.id,
-                    credential_id=cc_pair.credential.id,
-                    status=ConnectorCredentialPairStatus.INVALID,
-                )
-                raise
-
             source_type = cc_pair.connector.source
 
             doc_sync_func = DOC_PERMISSIONS_FUNC_MAP.get(source_type)
@@ -479,10 +439,6 @@ def connector_permission_sync_generator_task(
             redis_connector.permissions.generator_complete = tasks_generated
 
     except Exception as e:
-        error_msg = format_error_for_logging(e)
-        task_logger.warning(
-            f"Permission sync exceptioned: cc_pair={cc_pair_id} payload_id={payload_id} {error_msg}"
-        )
         task_logger.exception(
             f"Permission sync exceptioned: cc_pair={cc_pair_id} payload_id={payload_id}"
         )
@@ -509,7 +465,7 @@ def connector_permission_sync_generator_task(
 )
 def update_external_document_permissions_task(
     self: Task,
-    tenant_id: str,
+    tenant_id: str | None,
     serialized_doc_external_access: dict,
     source_string: str,
     connector_id: int,
@@ -517,8 +473,6 @@ def update_external_document_permissions_task(
 ) -> bool:
     start = time.monotonic()
 
-    completion_status = OnyxCeleryTaskCompletionStatus.UNDEFINED
-
     document_external_access = DocExternalAccess.from_dict(
         serialized_doc_external_access
     )
@@ -558,33 +512,18 @@ def update_external_document_permissions_task(
                 f"elapsed={elapsed:.2f}"
             )
 
-        completion_status = OnyxCeleryTaskCompletionStatus.SUCCEEDED
-    except Exception as e:
-        error_msg = format_error_for_logging(e)
-        task_logger.warning(
-            f"Exception in update_external_document_permissions_task: connector_id={connector_id} doc_id={doc_id} {error_msg}"
-        )
+    except Exception:
         task_logger.exception(
-            f"update_external_document_permissions_task exceptioned: "
+            f"Exception in update_external_document_permissions_task: "
             f"connector_id={connector_id} doc_id={doc_id}"
         )
-        completion_status = OnyxCeleryTaskCompletionStatus.NON_RETRYABLE_EXCEPTION
-    finally:
-        task_logger.info(
-            f"update_external_document_permissions_task completed: status={completion_status.value} doc={doc_id}"
-        )
-
-    if completion_status != OnyxCeleryTaskCompletionStatus.SUCCEEDED:
         return False
 
-    task_logger.info(
-        f"update_external_document_permissions_task finished: connector_id={connector_id} doc_id={doc_id}"
-    )
     return True
 
 
 def validate_permission_sync_fences(
-    tenant_id: str,
+    tenant_id: str | None,
     r: Redis,
     r_replica: Redis,
     r_celery: Redis,
@@ -631,7 +570,7 @@ def validate_permission_sync_fences(
 
 
 def validate_permission_sync_fence(
-    tenant_id: str,
+    tenant_id: str | None,
     key_bytes: bytes,
     queued_tasks: set[str],
     reserved_tasks: set[str],
@@ -841,7 +780,7 @@ class PermissionSyncCallback(IndexingHeartbeatInterface):
 
 
 def monitor_ccpair_permissions_taskset(
-    tenant_id: str, key_bytes: bytes, r: Redis, db_session: Session
+    tenant_id: str | None, key_bytes: bytes, r: Redis, db_session: Session
 ) -> None:
     fence_key = key_bytes.decode("utf-8")
     cc_pair_id_str = RedisConnector.get_id_from_fence_key(fence_key)

backend/onyx/background/celery/tasks/external_group_syncing/tasks.py

@@ -37,11 +37,8 @@ from onyx.configs.constants import OnyxCeleryTask
 from onyx.configs.constants import OnyxRedisConstants
 from onyx.configs.constants import OnyxRedisLocks
 from onyx.configs.constants import OnyxRedisSignals
-from onyx.connectors.exceptions import ConnectorValidationError
-from onyx.connectors.factory import validate_ccpair_for_user
 from onyx.db.connector import mark_cc_pair_as_external_group_synced
 from onyx.db.connector_credential_pair import get_connector_credential_pair_from_id
-from onyx.db.connector_credential_pair import update_connector_credential_pair
 from onyx.db.engine import get_session_with_current_tenant
 from onyx.db.enums import AccessType
 from onyx.db.enums import ConnectorCredentialPairStatus
@@ -58,7 +55,6 @@ from onyx.redis.redis_connector_ext_group_sync import (
 from onyx.redis.redis_pool import get_redis_client
 from onyx.redis.redis_pool import get_redis_replica_client
 from onyx.server.utils import make_short_id
-from onyx.utils.logger import format_error_for_logging
 from onyx.utils.logger import setup_logger
 
 logger = setup_logger()
@@ -123,7 +119,7 @@ def _is_external_group_sync_due(cc_pair: ConnectorCredentialPair) -> bool:
     soft_time_limit=JOB_TIMEOUT,
     bind=True,
 )
-def check_for_external_group_sync(self: Task, *, tenant_id: str) -> bool | None:
+def check_for_external_group_sync(self: Task, *, tenant_id: str | None) -> bool | None:
     # we need to use celery's redis client to access its redis data
     # (which lives on a different db number)
     r = get_redis_client()
@@ -152,10 +148,7 @@ def check_for_external_group_sync(self: Task, *, tenant_id: str) -> bool | None:
             for source in GROUP_PERMISSIONS_IS_CC_PAIR_AGNOSTIC:
                 # These are ordered by cc_pair id so the first one is the one we want
                 cc_pairs_to_dedupe = get_cc_pairs_by_source(
-                    db_session,
-                    source,
-                    access_type=AccessType.SYNC,
-                    status=ConnectorCredentialPairStatus.ACTIVE,
+                    db_session, source, only_sync=True
                 )
                 # We only want to sync one cc_pair per source type
                 # in GROUP_PERMISSIONS_IS_CC_PAIR_AGNOSTIC so we dedupe here
@@ -202,17 +195,12 @@ def check_for_external_group_sync(self: Task, *, tenant_id: str) -> bool | None:
         task_logger.info(
             "Soft time limit exceeded, task is being terminated gracefully."
         )
-    except Exception as e:
-        error_msg = format_error_for_logging(e)
-        task_logger.warning(
-            f"Unexpected check_for_external_group_sync exception: tenant={tenant_id} {error_msg}"
-        )
+    except Exception:
         task_logger.exception(f"Unexpected exception: tenant={tenant_id}")
     finally:
         if lock_beat.owned():
             lock_beat.release()
 
-    task_logger.info(f"check_for_external_group_sync finished: tenant={tenant_id}")
     return True
 
 
@@ -220,7 +208,7 @@ def try_creating_external_group_sync_task(
     app: Celery,
     cc_pair_id: int,
     r: Redis,
-    tenant_id: str,
+    tenant_id: str | None,
 ) -> str | None:
     """Returns an int if syncing is needed. The int represents the number of sync tasks generated.
     Returns None if no syncing is required."""
@@ -279,19 +267,12 @@ def try_creating_external_group_sync_task(
         redis_connector.external_group_sync.set_fence(payload)
 
         payload_id = payload.id
-    except Exception as e:
-        error_msg = format_error_for_logging(e)
-        task_logger.warning(
-            f"Unexpected try_creating_external_group_sync_task exception: cc_pair={cc_pair_id} {error_msg}"
-        )
+    except Exception:
         task_logger.exception(
             f"Unexpected exception while trying to create external group sync task: cc_pair={cc_pair_id}"
         )
         return None
 
-    task_logger.info(
-        f"try_creating_external_group_sync_task finished: cc_pair={cc_pair_id} payload_id={payload_id}"
-    )
     return payload_id
 
 
@@ -306,7 +287,7 @@ def try_creating_external_group_sync_task(
 def connector_external_group_sync_generator_task(
     self: Task,
     cc_pair_id: int,
-    tenant_id: str,
+    tenant_id: str | None,
 ) -> None:
     """
     External group sync task for a given connector credential pair
@@ -387,29 +368,6 @@ def connector_external_group_sync_generator_task(
                     f"No connector credential pair found for id: {cc_pair_id}"
                 )
 
-            try:
-                created = validate_ccpair_for_user(
-                    cc_pair.connector.id,
-                    cc_pair.credential.id,
-                    db_session,
-                    enforce_creation=False,
-                )
-                if not created:
-                    task_logger.warning(
-                        f"Unable to create connector credential pair for id: {cc_pair_id}"
-                    )
-            except Exception:
-                task_logger.exception(
-                    f"validate_ccpair_permissions_sync exceptioned: cc_pair={cc_pair_id}"
-                )
-                update_connector_credential_pair(
-                    db_session=db_session,
-                    connector_id=cc_pair.connector.id,
-                    credential_id=cc_pair.credential.id,
-                    status=ConnectorCredentialPairStatus.INVALID,
-                )
-                raise
-
             source_type = cc_pair.connector.source
 
             ext_group_sync_func = GROUP_PERMISSIONS_FUNC_MAP.get(source_type)
@@ -421,18 +379,8 @@ def connector_external_group_sync_generator_task(
             logger.info(
                 f"Syncing external groups for {source_type} for cc_pair: {cc_pair_id}"
             )
-            external_user_groups: list[ExternalUserGroup] = []
-            try:
-                external_user_groups = ext_group_sync_func(cc_pair)
-            except ConnectorValidationError as e:
-                msg = f"Error syncing external groups for {source_type} for cc_pair: {cc_pair_id} {e}"
-                update_connector_credential_pair(
-                    db_session=db_session,
-                    connector_id=cc_pair.connector.id,
-                    credential_id=cc_pair.credential.id,
-                    status=ConnectorCredentialPairStatus.INVALID,
-                )
-                raise e
+
+            external_user_groups: list[ExternalUserGroup] = ext_group_sync_func(cc_pair)
 
             logger.info(
                 f"Syncing {len(external_user_groups)} external user groups for {source_type}"
@@ -458,14 +406,6 @@ def connector_external_group_sync_generator_task(
                 sync_status=SyncStatus.SUCCESS,
             )
     except Exception as e:
-        error_msg = format_error_for_logging(e)
-        task_logger.warning(
-            f"External group sync exceptioned: cc_pair={cc_pair_id} payload_id={payload.id} {error_msg}"
-        )
-        task_logger.exception(
-            f"External group sync exceptioned: cc_pair={cc_pair_id} payload_id={payload.id}"
-        )
-
         msg = f"External group sync exceptioned: cc_pair={cc_pair_id} payload_id={payload.id}"
         task_logger.exception(msg)
         emit_background_error(msg + f"\n\n{e}", cc_pair_id=cc_pair_id)
@@ -493,7 +433,7 @@ def connector_external_group_sync_generator_task(
 
 
 def validate_external_group_sync_fences(
-    tenant_id: str,
+    tenant_id: str | None,
     celery_app: Celery,
     r: Redis,
     r_replica: Redis,
@@ -525,7 +465,7 @@ def validate_external_group_sync_fences(
 
 
 def validate_external_group_sync_fence(
-    tenant_id: str,
+    tenant_id: str | None,
     key_bytes: bytes,
     reserved_tasks: set[str],
     r_celery: Redis,

backend/onyx/background/celery/tasks/indexing/tasks.py

@@ -48,7 +48,7 @@ from onyx.configs.constants import OnyxCeleryTask
 from onyx.configs.constants import OnyxRedisConstants
 from onyx.configs.constants import OnyxRedisLocks
 from onyx.configs.constants import OnyxRedisSignals
-from onyx.connectors.exceptions import ConnectorValidationError
+from onyx.connectors.interfaces import ConnectorValidationError
 from onyx.db.connector import mark_ccpair_with_indexing_trigger
 from onyx.db.connector_credential_pair import fetch_connector_credential_pairs
 from onyx.db.connector_credential_pair import get_connector_credential_pair_from_id
@@ -182,7 +182,7 @@ class SimpleJobResult:
 
 
 class ConnectorIndexingContext(BaseModel):
-    tenant_id: str
+    tenant_id: str | None
     cc_pair_id: int
     search_settings_id: int
     index_attempt_id: int
@@ -210,7 +210,7 @@ class ConnectorIndexingLogBuilder:
 
 
 def monitor_ccpair_indexing_taskset(
-    tenant_id: str, key_bytes: bytes, r: Redis, db_session: Session
+    tenant_id: str | None, key_bytes: bytes, r: Redis, db_session: Session
 ) -> None:
     # if the fence doesn't exist, there's nothing to do
     fence_key = key_bytes.decode("utf-8")
@@ -358,7 +358,7 @@ def monitor_ccpair_indexing_taskset(
     soft_time_limit=300,
     bind=True,
 )
-def check_for_indexing(self: Task, *, tenant_id: str) -> int | None:
+def check_for_indexing(self: Task, *, tenant_id: str | None) -> int | None:
     """a lightweight task used to kick off indexing tasks.
     Occcasionally does some validation of existing state to clear up error conditions"""
 
@@ -598,7 +598,7 @@ def connector_indexing_task(
     cc_pair_id: int,
     search_settings_id: int,
     is_ee: bool,
-    tenant_id: str,
+    tenant_id: str | None,
 ) -> int | None:
     """Indexing task. For a cc pair, this task pulls all document IDs from the source
     and compares those IDs to locally stored documents and deletes all locally stored IDs missing
@@ -890,7 +890,7 @@ def connector_indexing_proxy_task(
     index_attempt_id: int,
     cc_pair_id: int,
     search_settings_id: int,
-    tenant_id: str,
+    tenant_id: str | None,
 ) -> None:
     """celery out of process task execution strategy is pool=prefork, but it uses fork,
     and forking is inherently unstable.
@@ -899,9 +899,6 @@ def connector_indexing_proxy_task(
 
     TODO(rkuo): refactor this so that there is a single return path where we canonically
     log the result of running this function.
-
-    NOTE: we try/except all db access in this function because as a watchdog, this function
-    needs to be extremely stable.
     """
     start = time.monotonic()
 
@@ -927,7 +924,6 @@ def connector_indexing_proxy_task(
         task_logger.error("self.request.id is None!")
 
     client = SimpleJobClient()
-    task_logger.info(f"submitting connector_indexing_task with tenant_id={tenant_id}")
 
     job = client.submit(
         connector_indexing_task,
@@ -1020,7 +1016,7 @@ def connector_indexing_proxy_task(
                     job.release()
                     break
 
-            # if a termination signal is detected, break (exit point will clean up)
+            # if a termination signal is detected, clean up and break
             if self.request.id and redis_connector_index.terminating(self.request.id):
                 task_logger.warning(
                     log_builder.build("Indexing watchdog - termination signal detected")
@@ -1029,7 +1025,6 @@ def connector_indexing_proxy_task(
                 result.status = IndexingWatchdogTerminalStatus.TERMINATED_BY_SIGNAL
                 break
 
-            # if activity timeout is detected, break (exit point will clean up)
             if not redis_connector_index.connector_active():
                 task_logger.warning(
                     log_builder.build(
@@ -1038,6 +1033,25 @@ def connector_indexing_proxy_task(
                     )
                 )
 
+                try:
+                    with get_session_with_current_tenant() as db_session:
+                        mark_attempt_failed(
+                            index_attempt_id,
+                            db_session,
+                            "Indexing watchdog - activity timeout exceeded: "
+                            f"attempt={index_attempt_id} "
+                            f"timeout={CELERY_INDEXING_WATCHDOG_CONNECTOR_TIMEOUT}s",
+                        )
+                except Exception:
+                    # if the DB exceptions, we'll just get an unfriendly failure message
+                    # in the UI instead of the cancellation message
+                    logger.exception(
+                        log_builder.build(
+                            "Indexing watchdog - transient exception marking index attempt as failed"
+                        )
+                    )
+
+                job.cancel()
                 result.status = (
                     IndexingWatchdogTerminalStatus.TERMINATED_BY_ACTIVITY_TIMEOUT
                 )
@@ -1056,15 +1070,15 @@ def connector_indexing_proxy_task(
 
                     if not index_attempt.is_finished():
                         continue
-
             except Exception:
+                # if the DB exceptioned, just restart the check.
+                # polling the index attempt status doesn't need to be strongly consistent
                 task_logger.exception(
                     log_builder.build(
                         "Indexing watchdog - transient exception looking up index attempt"
                     )
                 )
                 continue
-
     except Exception as e:
         result.status = IndexingWatchdogTerminalStatus.WATCHDOG_EXCEPTIONED
         if isinstance(e, ConnectorValidationError):
@@ -1125,6 +1139,8 @@ def connector_indexing_proxy_task(
                     "Connector termination signal detected",
                 )
         except Exception:
+            # if the DB exceptions, we'll just get an unfriendly failure message
+            # in the UI instead of the cancellation message
             task_logger.exception(
                 log_builder.build(
                     "Indexing watchdog - transient exception marking index attempt as canceled"
@@ -1132,25 +1148,6 @@ def connector_indexing_proxy_task(
             )
 
         job.cancel()
-    elif result.status == IndexingWatchdogTerminalStatus.TERMINATED_BY_ACTIVITY_TIMEOUT:
-        try:
-            with get_session_with_current_tenant() as db_session:
-                mark_attempt_failed(
-                    index_attempt_id,
-                    db_session,
-                    "Indexing watchdog - activity timeout exceeded: "
-                    f"attempt={index_attempt_id} "
-                    f"timeout={CELERY_INDEXING_WATCHDOG_CONNECTOR_TIMEOUT}s",
-                )
-        except Exception:
-            logger.exception(
-                log_builder.build(
-                    "Indexing watchdog - transient exception marking index attempt as failed"
-                )
-            )
-        job.cancel()
-    else:
-        pass
 
     task_logger.info(
         log_builder.build(
@@ -1170,7 +1167,7 @@ def connector_indexing_proxy_task(
     name=OnyxCeleryTask.CHECK_FOR_CHECKPOINT_CLEANUP,
     soft_time_limit=300,
 )
-def check_for_checkpoint_cleanup(*, tenant_id: str) -> None:
+def check_for_checkpoint_cleanup(*, tenant_id: str | None) -> None:
     """Clean up old checkpoints that are older than 7 days."""
     locked = False
     redis_client = get_redis_client(tenant_id=tenant_id)

backend/onyx/background/celery/tasks/indexing/utils.py

@@ -187,7 +187,7 @@ class IndexingCallback(IndexingCallbackBase):
 
 
 def validate_indexing_fence(
-    tenant_id: str,
+    tenant_id: str | None,
     key_bytes: bytes,
     reserved_tasks: set[str],
     r_celery: Redis,
@@ -311,7 +311,7 @@ def validate_indexing_fence(
 
 
 def validate_indexing_fences(
-    tenant_id: str,
+    tenant_id: str | None,
     r_replica: Redis,
     r_celery: Redis,
     lock_beat: RedisLock,
@@ -442,7 +442,7 @@ def try_creating_indexing_task(
     reindex: bool,
     db_session: Session,
     r: Redis,
-    tenant_id: str,
+    tenant_id: str | None,
 ) -> int | None:
     """Checks for any conditions that should block the indexing task from being
     created, then creates the task.

backend/onyx/background/celery/tasks/llm_model_update/tasks.py

@@ -59,7 +59,7 @@ def _process_model_list_response(model_list_json: Any) -> list[str]:
     trail=False,
     bind=True,
 )
-def check_for_llm_model_update(self: Task, *, tenant_id: str) -> bool | None:
+def check_for_llm_model_update(self: Task, *, tenant_id: str | None) -> bool | None:
     if not LLM_MODEL_UPDATE_API_URL:
         raise ValueError("LLM model update API URL not configured")
 

backend/onyx/background/celery/tasks/monitoring/tasks.py

@@ -91,7 +91,7 @@ class Metric(BaseModel):
         }
         task_logger.info(json.dumps(data))
 
-    def emit(self, tenant_id: str) -> None:
+    def emit(self, tenant_id: str | None) -> None:
         # Convert value to appropriate type based on the input value
         bool_value = None
         float_value = None
@@ -656,7 +656,7 @@ def build_job_id(
     queue=OnyxCeleryQueues.MONITORING,
     bind=True,
 )
-def monitor_background_processes(self: Task, *, tenant_id: str) -> None:
+def monitor_background_processes(self: Task, *, tenant_id: str | None) -> None:
     """Collect and emit metrics about background processes.
     This task runs periodically to gather metrics about:
     - Queue lengths for different Celery queues
@@ -864,7 +864,7 @@ def cloud_monitor_celery_queues(
 
 
 @shared_task(name=OnyxCeleryTask.MONITOR_CELERY_QUEUES, ignore_result=True, bind=True)
-def monitor_celery_queues(self: Task, *, tenant_id: str) -> None:
+def monitor_celery_queues(self: Task, *, tenant_id: str | None) -> None:
     return monitor_celery_queues_helper(self)
 
 

backend/onyx/background/celery/tasks/periodic/tasks.py

@@ -24,7 +24,7 @@ from onyx.db.engine import get_session_with_current_tenant
     bind=True,
     base=AbortableTask,
 )
-def kombu_message_cleanup_task(self: Any, tenant_id: str) -> int:
+def kombu_message_cleanup_task(self: Any, tenant_id: str | None) -> int:
     """Runs periodically to clean up the kombu_message table"""
 
     # we will select messages older than this amount to clean up

backend/onyx/background/celery/tasks/pruning/tasks.py

@@ -55,7 +55,6 @@ from onyx.redis.redis_connector_prune import RedisConnectorPrunePayload
 from onyx.redis.redis_pool import get_redis_client
 from onyx.redis.redis_pool import get_redis_replica_client
 from onyx.server.utils import make_short_id
-from onyx.utils.logger import format_error_for_logging
 from onyx.utils.logger import LoggerContextVars
 from onyx.utils.logger import pruning_ctx
 from onyx.utils.logger import setup_logger
@@ -114,7 +113,7 @@ def _is_pruning_due(cc_pair: ConnectorCredentialPair) -> bool:
     soft_time_limit=JOB_TIMEOUT,
     bind=True,
 )
-def check_for_pruning(self: Task, *, tenant_id: str) -> bool | None:
+def check_for_pruning(self: Task, *, tenant_id: str | None) -> bool | None:
     r = get_redis_client()
     r_replica = get_redis_replica_client()
     r_celery: Redis = self.app.broker_connection().channel().client  # type: ignore
@@ -195,14 +194,12 @@ def check_for_pruning(self: Task, *, tenant_id: str) -> bool | None:
         task_logger.info(
             "Soft time limit exceeded, task is being terminated gracefully."
         )
-    except Exception as e:
-        error_msg = format_error_for_logging(e)
-        task_logger.warning(f"Unexpected pruning check exception: {error_msg}")
+    except Exception:
         task_logger.exception("Unexpected exception during pruning check")
     finally:
         if lock_beat.owned():
             lock_beat.release()
-    task_logger.info(f"check_for_pruning finished: tenant={tenant_id}")
+
     return True
 
 
@@ -211,7 +208,7 @@ def try_creating_prune_generator_task(
     cc_pair: ConnectorCredentialPair,
     db_session: Session,
     r: Redis,
-    tenant_id: str,
+    tenant_id: str | None,
 ) -> str | None:
     """Checks for any conditions that should block the pruning generator task from being
     created, then creates the task.
@@ -304,19 +301,13 @@ def try_creating_prune_generator_task(
         redis_connector.prune.set_fence(payload)
 
         payload_id = payload.id
-    except Exception as e:
-        error_msg = format_error_for_logging(e)
-        task_logger.warning(
-            f"Unexpected try_creating_prune_generator_task exception: cc_pair={cc_pair.id} {error_msg}"
-        )
+    except Exception:
         task_logger.exception(f"Unexpected exception: cc_pair={cc_pair.id}")
         return None
     finally:
         if lock.owned():
             lock.release()
-    task_logger.info(
-        f"try_creating_prune_generator_task finished: cc_pair={cc_pair.id} payload_id={payload_id}"
-    )
+
     return payload_id
 
 
@@ -333,7 +324,7 @@ def connector_pruning_generator_task(
     cc_pair_id: int,
     connector_id: int,
     credential_id: int,
-    tenant_id: str,
+    tenant_id: str | None,
 ) -> None:
     """connector pruning task. For a cc pair, this task pulls all document IDs from the source
     and compares those IDs to locally stored documents and deletes all locally stored IDs missing
@@ -521,7 +512,7 @@ def connector_pruning_generator_task(
 
 
 def monitor_ccpair_pruning_taskset(
-    tenant_id: str, key_bytes: bytes, r: Redis, db_session: Session
+    tenant_id: str | None, key_bytes: bytes, r: Redis, db_session: Session
 ) -> None:
     fence_key = key_bytes.decode("utf-8")
     cc_pair_id_str = RedisConnector.get_id_from_fence_key(fence_key)
@@ -567,7 +558,7 @@ def monitor_ccpair_pruning_taskset(
 
 
 def validate_pruning_fences(
-    tenant_id: str,
+    tenant_id: str | None,
     r: Redis,
     r_replica: Redis,
     r_celery: Redis,
@@ -615,7 +606,7 @@ def validate_pruning_fences(
 
 
 def validate_pruning_fence(
-    tenant_id: str,
+    tenant_id: str | None,
     key_bytes: bytes,
     reserved_tasks: set[str],
     queued_tasks: set[str],

backend/onyx/background/celery/tasks/shared/RetryDocumentIndex.py

@@ -32,7 +32,7 @@ class RetryDocumentIndex:
         self,
         doc_id: str,
         *,
-        tenant_id: str,
+        tenant_id: str | None,
         chunk_count: int | None,
     ) -> int:
         return self.index.delete_single(
@@ -50,7 +50,7 @@ class RetryDocumentIndex:
         self,
         doc_id: str,
         *,
-        tenant_id: str,
+        tenant_id: str | None,
         chunk_count: int | None,
         fields: VespaDocumentFields,
     ) -> int:

backend/onyx/background/celery/tasks/shared/tasks.py

@@ -1,5 +1,4 @@
 import time
-from enum import Enum
 from http import HTTPStatus
 
 import httpx
@@ -46,24 +45,6 @@ LIGHT_SOFT_TIME_LIMIT = 105
 LIGHT_TIME_LIMIT = LIGHT_SOFT_TIME_LIMIT + 15
 
 
-class OnyxCeleryTaskCompletionStatus(str, Enum):
-    """The different statuses the watchdog can finish with.
-
-    TODO: create broader success/failure/abort categories
-    """
-
-    UNDEFINED = "undefined"
-
-    SUCCEEDED = "succeeded"
-
-    SKIPPED = "skipped"
-
-    SOFT_TIME_LIMIT = "soft_time_limit"
-
-    NON_RETRYABLE_EXCEPTION = "non_retryable_exception"
-    RETRYABLE_EXCEPTION = "retryable_exception"
-
-
 @shared_task(
     name=OnyxCeleryTask.DOCUMENT_BY_CC_PAIR_CLEANUP_TASK,
     soft_time_limit=LIGHT_SOFT_TIME_LIMIT,
@@ -76,7 +57,7 @@ def document_by_cc_pair_cleanup_task(
     document_id: str,
     connector_id: int,
     credential_id: int,
-    tenant_id: str,
+    tenant_id: str | None,
 ) -> bool:
     """A lightweight subtask used to clean up document to cc pair relationships.
     Created by connection deletion and connector pruning parent tasks."""
@@ -97,8 +78,6 @@ def document_by_cc_pair_cleanup_task(
 
     start = time.monotonic()
 
-    completion_status = OnyxCeleryTaskCompletionStatus.UNDEFINED
-
     try:
         with get_session_with_current_tenant() as db_session:
             action = "skip"
@@ -131,9 +110,6 @@ def document_by_cc_pair_cleanup_task(
                     db_session=db_session,
                     document_ids=[document_id],
                 )
-                db_session.commit()
-
-                completion_status = OnyxCeleryTaskCompletionStatus.SUCCEEDED
             elif count > 1:
                 action = "update"
 
@@ -177,11 +153,10 @@ def document_by_cc_pair_cleanup_task(
                 )
 
                 mark_document_as_synced(document_id, db_session)
-                db_session.commit()
-
-                completion_status = OnyxCeleryTaskCompletionStatus.SUCCEEDED
             else:
-                completion_status = OnyxCeleryTaskCompletionStatus.SKIPPED
+                pass
+
+            db_session.commit()
 
             elapsed = time.monotonic() - start
             task_logger.info(
@@ -193,79 +168,57 @@ def document_by_cc_pair_cleanup_task(
             )
     except SoftTimeLimitExceeded:
         task_logger.info(f"SoftTimeLimitExceeded exception. doc={document_id}")
-        completion_status = OnyxCeleryTaskCompletionStatus.SOFT_TIME_LIMIT
+        return False
     except Exception as ex:
         e: Exception | None = None
-        while True:
-            if isinstance(ex, RetryError):
-                task_logger.warning(
-                    f"Tenacity retry failed: num_attempts={ex.last_attempt.attempt_number}"
-                )
-
-                # only set the inner exception if it is of type Exception
-                e_temp = ex.last_attempt.exception()
-                if isinstance(e_temp, Exception):
-                    e = e_temp
-            else:
-                e = ex
-
-            if isinstance(e, httpx.HTTPStatusError):
-                if e.response.status_code == HTTPStatus.BAD_REQUEST:
-                    task_logger.exception(
-                        f"Non-retryable HTTPStatusError: "
-                        f"doc={document_id} "
-                        f"status={e.response.status_code}"
-                    )
-                completion_status = (
-                    OnyxCeleryTaskCompletionStatus.NON_RETRYABLE_EXCEPTION
-                )
-                break
-
-            task_logger.exception(
-                f"document_by_cc_pair_cleanup_task exceptioned: doc={document_id}"
+        if isinstance(ex, RetryError):
+            task_logger.warning(
+                f"Tenacity retry failed: num_attempts={ex.last_attempt.attempt_number}"
             )
 
-            completion_status = OnyxCeleryTaskCompletionStatus.RETRYABLE_EXCEPTION
-            if (
-                self.max_retries is not None
-                and self.request.retries >= self.max_retries
-            ):
-                # This is the last attempt! mark the document as dirty in the db so that it
-                # eventually gets fixed out of band via stale document reconciliation
-                task_logger.warning(
-                    f"Max celery task retries reached. Marking doc as dirty for reconciliation: "
-                    f"doc={document_id}"
-                )
-                with get_session_with_current_tenant() as db_session:
-                    # delete the cc pair relationship now and let reconciliation clean it up
-                    # in vespa
-                    delete_document_by_connector_credential_pair__no_commit(
-                        db_session=db_session,
-                        document_id=document_id,
-                        connector_credential_pair_identifier=ConnectorCredentialPairIdentifier(
-                            connector_id=connector_id,
-                            credential_id=credential_id,
-                        ),
-                    )
-                    mark_document_as_modified(document_id, db_session)
-                completion_status = (
-                    OnyxCeleryTaskCompletionStatus.NON_RETRYABLE_EXCEPTION
-                )
-                break
+            # only set the inner exception if it is of type Exception
+            e_temp = ex.last_attempt.exception()
+            if isinstance(e_temp, Exception):
+                e = e_temp
+        else:
+            e = ex
 
-            # Exponential backoff from 2^4 to 2^6 ... i.e. 16, 32, 64
+        if isinstance(e, httpx.HTTPStatusError):
+            if e.response.status_code == HTTPStatus.BAD_REQUEST:
+                task_logger.exception(
+                    f"Non-retryable HTTPStatusError: "
+                    f"doc={document_id} "
+                    f"status={e.response.status_code}"
+                )
+            return False
+
+        task_logger.exception(f"Unexpected exception: doc={document_id}")
+
+        if self.request.retries < DOCUMENT_BY_CC_PAIR_CLEANUP_MAX_RETRIES:
+            # Still retrying. Exponential backoff from 2^4 to 2^6 ... i.e. 16, 32, 64
             countdown = 2 ** (self.request.retries + 4)
-            self.retry(exc=e, countdown=countdown)  # this will raise a celery exception
-            break  # we won't hit this, but it looks weird not to have it
-    finally:
-        task_logger.info(
-            f"document_by_cc_pair_cleanup_task completed: status={completion_status.value} doc={document_id}"
-        )
-
-    if completion_status != OnyxCeleryTaskCompletionStatus.SUCCEEDED:
+            self.retry(exc=e, countdown=countdown)
+        else:
+            # This is the last attempt! mark the document as dirty in the db so that it
+            # eventually gets fixed out of band via stale document reconciliation
+            task_logger.warning(
+                f"Max celery task retries reached. Marking doc as dirty for reconciliation: "
+                f"doc={document_id}"
+            )
+            with get_session_with_current_tenant() as db_session:
+                # delete the cc pair relationship now and let reconciliation clean it up
+                # in vespa
+                delete_document_by_connector_credential_pair__no_commit(
+                    db_session=db_session,
+                    document_id=document_id,
+                    connector_credential_pair_identifier=ConnectorCredentialPairIdentifier(
+                        connector_id=connector_id,
+                        credential_id=credential_id,
+                    ),
+                )
+                mark_document_as_modified(document_id, db_session)
         return False
 
-    task_logger.info(f"document_by_cc_pair_cleanup_task finished: doc={document_id}")
     return True
 
 
@@ -297,8 +250,7 @@ def cloud_beat_task_generator(
         return None
 
     last_lock_time = time.monotonic()
-    tenant_ids: list[str] = []
-    num_processed_tenants = 0
+    tenant_ids: list[str] | list[None] = []
 
     try:
         tenant_ids = get_all_tenant_ids()
@@ -326,8 +278,6 @@ def cloud_beat_task_generator(
                 expires=expires,
                 ignore_result=True,
             )
-
-            num_processed_tenants += 1
     except SoftTimeLimitExceeded:
         task_logger.info(
             "Soft time limit exceeded, task is being terminated gracefully."
@@ -347,7 +297,6 @@ def cloud_beat_task_generator(
     task_logger.info(
         f"cloud_beat_task_generator finished: "
         f"task={task_name} "
-        f"num_processed_tenants={num_processed_tenants} "
         f"num_tenants={len(tenant_ids)} "
         f"elapsed={time_elapsed:.2f}"
     )

backend/onyx/background/celery/tasks/vespa/tasks.py

@@ -19,7 +19,6 @@ from onyx.background.celery.apps.app_base import task_logger
 from onyx.background.celery.tasks.shared.RetryDocumentIndex import RetryDocumentIndex
 from onyx.background.celery.tasks.shared.tasks import LIGHT_SOFT_TIME_LIMIT
 from onyx.background.celery.tasks.shared.tasks import LIGHT_TIME_LIMIT
-from onyx.background.celery.tasks.shared.tasks import OnyxCeleryTaskCompletionStatus
 from onyx.configs.app_configs import JOB_TIMEOUT
 from onyx.configs.app_configs import VESPA_SYNC_MAX_TASKS
 from onyx.configs.constants import CELERY_VESPA_SYNC_BEAT_LOCK_TIMEOUT
@@ -76,7 +75,7 @@ logger = setup_logger()
     trail=False,
     bind=True,
 )
-def check_for_vespa_sync_task(self: Task, *, tenant_id: str) -> bool | None:
+def check_for_vespa_sync_task(self: Task, *, tenant_id: str | None) -> bool | None:
     """Runs periodically to check if any document needs syncing.
     Generates sets of tasks for Celery if syncing is needed."""
 
@@ -208,7 +207,7 @@ def try_generate_stale_document_sync_tasks(
     db_session: Session,
     r: Redis,
     lock_beat: RedisLock,
-    tenant_id: str,
+    tenant_id: str | None,
 ) -> int | None:
     # the fence is up, do nothing
 
@@ -284,7 +283,7 @@ def try_generate_document_set_sync_tasks(
     db_session: Session,
     r: Redis,
     lock_beat: RedisLock,
-    tenant_id: str,
+    tenant_id: str | None,
 ) -> int | None:
     lock_beat.reacquire()
 
@@ -361,7 +360,7 @@ def try_generate_user_group_sync_tasks(
     db_session: Session,
     r: Redis,
     lock_beat: RedisLock,
-    tenant_id: str,
+    tenant_id: str | None,
 ) -> int | None:
     lock_beat.reacquire()
 
@@ -448,7 +447,7 @@ def monitor_connector_taskset(r: Redis) -> None:
 
 
 def monitor_document_set_taskset(
-    tenant_id: str, key_bytes: bytes, r: Redis, db_session: Session
+    tenant_id: str | None, key_bytes: bytes, r: Redis, db_session: Session
 ) -> None:
     fence_key = key_bytes.decode("utf-8")
     document_set_id_str = RedisDocumentSet.get_id_from_fence_key(fence_key)
@@ -523,11 +522,11 @@ def monitor_document_set_taskset(
     time_limit=LIGHT_TIME_LIMIT,
     max_retries=3,
 )
-def vespa_metadata_sync_task(self: Task, document_id: str, *, tenant_id: str) -> bool:
+def vespa_metadata_sync_task(
+    self: Task, document_id: str, *, tenant_id: str | None
+) -> bool:
     start = time.monotonic()
 
-    completion_status = OnyxCeleryTaskCompletionStatus.UNDEFINED
-
     try:
         with get_session_with_current_tenant() as db_session:
             active_search_settings = get_active_search_settings(db_session)
@@ -541,103 +540,75 @@ def vespa_metadata_sync_task(self: Task, document_id: str, *, tenant_id: str) ->
 
             doc = get_document(document_id, db_session)
             if not doc:
-                elapsed = time.monotonic() - start
-                task_logger.info(
-                    f"doc={document_id} "
-                    f"action=no_operation "
-                    f"elapsed={elapsed:.2f}"
-                )
-                completion_status = OnyxCeleryTaskCompletionStatus.SKIPPED
-            else:
-                # document set sync
-                doc_sets = fetch_document_sets_for_document(document_id, db_session)
-                update_doc_sets: set[str] = set(doc_sets)
+                return False
 
-                # User group sync
-                doc_access = get_access_for_document(
-                    document_id=document_id, db_session=db_session
-                )
+            # document set sync
+            doc_sets = fetch_document_sets_for_document(document_id, db_session)
+            update_doc_sets: set[str] = set(doc_sets)
 
-                fields = VespaDocumentFields(
-                    document_sets=update_doc_sets,
-                    access=doc_access,
-                    boost=doc.boost,
-                    hidden=doc.hidden,
-                )
-
-                # update Vespa. OK if doc doesn't exist. Raises exception otherwise.
-                chunks_affected = retry_index.update_single(
-                    document_id,
-                    tenant_id=tenant_id,
-                    chunk_count=doc.chunk_count,
-                    fields=fields,
-                )
-
-                # update db last. Worst case = we crash right before this and
-                # the sync might repeat again later
-                mark_document_as_synced(document_id, db_session)
-
-                elapsed = time.monotonic() - start
-                task_logger.info(
-                    f"doc={document_id} "
-                    f"action=sync "
-                    f"chunks={chunks_affected} "
-                    f"elapsed={elapsed:.2f}"
-                )
-                completion_status = OnyxCeleryTaskCompletionStatus.SUCCEEDED
-    except SoftTimeLimitExceeded:
-        task_logger.info(f"SoftTimeLimitExceeded exception. doc={document_id}")
-        completion_status = OnyxCeleryTaskCompletionStatus.SOFT_TIME_LIMIT
-    except Exception as ex:
-        e: Exception | None = None
-        while True:
-            if isinstance(ex, RetryError):
-                task_logger.warning(
-                    f"Tenacity retry failed: num_attempts={ex.last_attempt.attempt_number}"
-                )
-
-                # only set the inner exception if it is of type Exception
-                e_temp = ex.last_attempt.exception()
-                if isinstance(e_temp, Exception):
-                    e = e_temp
-            else:
-                e = ex
-
-            if isinstance(e, httpx.HTTPStatusError):
-                if e.response.status_code == HTTPStatus.BAD_REQUEST:
-                    task_logger.exception(
-                        f"Non-retryable HTTPStatusError: "
-                        f"doc={document_id} "
-                        f"status={e.response.status_code}"
-                    )
-                completion_status = (
-                    OnyxCeleryTaskCompletionStatus.NON_RETRYABLE_EXCEPTION
-                )
-                break
-
-            task_logger.exception(
-                f"vespa_metadata_sync_task exceptioned: doc={document_id}"
+            # User group sync
+            doc_access = get_access_for_document(
+                document_id=document_id, db_session=db_session
             )
 
-            completion_status = OnyxCeleryTaskCompletionStatus.RETRYABLE_EXCEPTION
-            if (
-                self.max_retries is not None
-                and self.request.retries >= self.max_retries
-            ):
-                completion_status = (
-                    OnyxCeleryTaskCompletionStatus.NON_RETRYABLE_EXCEPTION
-                )
+            fields = VespaDocumentFields(
+                document_sets=update_doc_sets,
+                access=doc_access,
+                boost=doc.boost,
+                hidden=doc.hidden,
+            )
 
-            # Exponential backoff from 2^4 to 2^6 ... i.e. 16, 32, 64
-            countdown = 2 ** (self.request.retries + 4)
-            self.retry(exc=e, countdown=countdown)  # this will raise a celery exception
-            break  # we won't hit this, but it looks weird not to have it
-    finally:
-        task_logger.info(
-            f"vespa_metadata_sync_task completed: status={completion_status.value} doc={document_id}"
+            # update Vespa. OK if doc doesn't exist. Raises exception otherwise.
+            chunks_affected = retry_index.update_single(
+                document_id,
+                tenant_id=tenant_id,
+                chunk_count=doc.chunk_count,
+                fields=fields,
+            )
+
+            # update db last. Worst case = we crash right before this and
+            # the sync might repeat again later
+            mark_document_as_synced(document_id, db_session)
+
+            elapsed = time.monotonic() - start
+            task_logger.info(
+                f"doc={document_id} "
+                f"action=sync "
+                f"chunks={chunks_affected} "
+                f"elapsed={elapsed:.2f}"
+            )
+    except SoftTimeLimitExceeded:
+        task_logger.info(f"SoftTimeLimitExceeded exception. doc={document_id}")
+        return False
+    except Exception as ex:
+        e: Exception | None = None
+        if isinstance(ex, RetryError):
+            task_logger.warning(
+                f"Tenacity retry failed: num_attempts={ex.last_attempt.attempt_number}"
+            )
+
+            # only set the inner exception if it is of type Exception
+            e_temp = ex.last_attempt.exception()
+            if isinstance(e_temp, Exception):
+                e = e_temp
+        else:
+            e = ex
+
+        if isinstance(e, httpx.HTTPStatusError):
+            if e.response.status_code == HTTPStatus.BAD_REQUEST:
+                task_logger.exception(
+                    f"Non-retryable HTTPStatusError: "
+                    f"doc={document_id} "
+                    f"status={e.response.status_code}"
+                )
+            return False
+
+        task_logger.exception(
+            f"Unexpected exception during vespa metadata sync: doc={document_id}"
         )
 
-    if completion_status != OnyxCeleryTaskCompletionStatus.SUCCEEDED:
-        return False
+        # Exponential backoff from 2^4 to 2^6 ... i.e. 16, 32, 64
+        countdown = 2 ** (self.request.retries + 4)
+        self.retry(exc=e, countdown=countdown)
 
     return True

backend/onyx/background/error_logging.py

@@ -1,5 +1,3 @@
-from sqlalchemy.exc import IntegrityError
-
 from onyx.db.background_error import create_background_error
 from onyx.db.engine import get_session_with_current_tenant
 
@@ -12,9 +10,4 @@ def emit_background_error(
 
     In the future, could create notifications based on the severity."""
     with get_session_with_current_tenant() as db_session:
-        try:
-            create_background_error(db_session, message, cc_pair_id)
-        except IntegrityError as e:
-            # Log an error if the cc_pair_id was deleted or any other exception occurs
-            error_message = f"Failed to create background error: {str(e)}. Original message: {message}"
-            create_background_error(db_session, error_message, None)
+        create_background_error(db_session, message, cc_pair_id)

backend/onyx/background/indexing/job_client.py

@@ -17,9 +17,6 @@ from typing import Optional
 from onyx.configs.constants import POSTGRES_CELERY_WORKER_INDEXING_CHILD_APP_NAME
 from onyx.db.engine import SqlEngine
 from onyx.utils.logger import setup_logger
-from shared_configs.configs import POSTGRES_DEFAULT_SCHEMA
-from shared_configs.configs import TENANT_ID_PREFIX
-from shared_configs.contextvars import CURRENT_TENANT_ID_CONTEXTVAR
 
 logger = setup_logger()
 
@@ -57,15 +54,6 @@ def _initializer(
         kwargs = {}
 
     logger.info("Initializing spawned worker child process.")
-    # 1. Get tenant_id from args or fallback to default
-    tenant_id = POSTGRES_DEFAULT_SCHEMA
-    for arg in reversed(args):
-        if isinstance(arg, str) and arg.startswith(TENANT_ID_PREFIX):
-            tenant_id = arg
-            break
-
-    # 2. Set the tenant context before running anything
-    token = CURRENT_TENANT_ID_CONTEXTVAR.set(tenant_id)
 
     # Reset the engine in the child process
     SqlEngine.reset_engine()
@@ -93,8 +81,6 @@ def _initializer(
         queue.put(error_msg)  # Send the exception to the parent process
 
         sys.exit(255)  # use 255 to indicate a generic exception
-    finally:
-        CURRENT_TENANT_ID_CONTEXTVAR.reset(token)
 
 
 def _run_in_process(

backend/onyx/background/indexing/run_indexing.py

@@ -21,8 +21,8 @@ from onyx.configs.app_configs import POLL_CONNECTOR_OFFSET
 from onyx.configs.constants import DocumentSource
 from onyx.configs.constants import MilestoneRecordType
 from onyx.connectors.connector_runner import ConnectorRunner
-from onyx.connectors.exceptions import ConnectorValidationError
 from onyx.connectors.factory import instantiate_connector
+from onyx.connectors.interfaces import ConnectorValidationError
 from onyx.connectors.models import ConnectorCheckpoint
 from onyx.connectors.models import ConnectorFailure
 from onyx.connectors.models import Document
@@ -55,7 +55,6 @@ from onyx.utils.logger import setup_logger
 from onyx.utils.logger import TaskAttemptSingleton
 from onyx.utils.telemetry import create_milestone_and_report
 from onyx.utils.variable_functionality import global_version
-from shared_configs.configs import MULTI_TENANT
 
 logger = setup_logger()
 
@@ -68,6 +67,7 @@ def _get_connector_runner(
     batch_size: int,
     start_time: datetime,
     end_time: datetime,
+    tenant_id: str | None,
     leave_connector_active: bool = LEAVE_CONNECTOR_ACTIVE_ON_INITIALIZATION_FAILURE,
 ) -> ConnectorRunner:
     """
@@ -86,6 +86,7 @@ def _get_connector_runner(
             input_type=task,
             connector_specific_config=attempt.connector_credential_pair.connector.connector_specific_config,
             credential=attempt.connector_credential_pair.credential,
+            tenant_id=tenant_id,
         )
 
         # validate the connector settings
@@ -240,7 +241,7 @@ def _check_failure_threshold(
 def _run_indexing(
     db_session: Session,
     index_attempt_id: int,
-    tenant_id: str,
+    tenant_id: str | None,
     callback: IndexingHeartbeatInterface | None = None,
 ) -> None:
     """
@@ -387,6 +388,7 @@ def _run_indexing(
                 batch_size=INDEX_BATCH_SIZE,
                 start_time=window_start,
                 end_time=window_end,
+                tenant_id=tenant_id,
             )
 
             # don't use a checkpoint if we're explicitly indexing from
@@ -679,7 +681,7 @@ def _run_indexing(
 
 def run_indexing_entrypoint(
     index_attempt_id: int,
-    tenant_id: str,
+    tenant_id: str | None,
     connector_credential_pair_id: int,
     is_ee: bool = False,
     callback: IndexingHeartbeatInterface | None = None,
@@ -699,7 +701,7 @@ def run_indexing_entrypoint(
         attempt = transition_attempt_to_in_progress(index_attempt_id, db_session)
 
         tenant_str = ""
-        if MULTI_TENANT:
+        if tenant_id is not None:
             tenant_str = f" for tenant {tenant_id}"
 
         connector_name = attempt.connector_credential_pair.connector.name

backend/onyx/chat/process_message.py

@@ -747,13 +747,14 @@ def stream_chat_message_objects(
                 files=latest_query_files,
                 single_message_history=single_message_history,
             ),
-            system_message=default_build_system_message(prompt_config, llm.config),
+            system_message=default_build_system_message(prompt_config),
             message_history=message_history,
             llm_config=llm.config,
             raw_user_query=final_msg.message,
             raw_user_uploaded_files=latest_query_files or [],
             single_message_history=single_message_history,
         )
+        prompt_builder.update_system_prompt(default_build_system_message(prompt_config))
 
         # LLM prompt building, response capturing, etc.
         answer = Answer(
@@ -869,6 +870,7 @@ def stream_chat_message_objects(
                             for img in img_generation_response
                             if img.image_data
                         ],
+                        tenant_id=tenant_id,
                     )
                     info.ai_message_files.extend(
                         [

backend/onyx/chat/prompt_builder/answer_prompt_builder.py

@@ -12,7 +12,6 @@ from onyx.chat.prompt_builder.citations_prompt import compute_max_llm_input_toke
 from onyx.chat.prompt_builder.utils import translate_history_to_basemessages
 from onyx.file_store.models import InMemoryChatFile
 from onyx.llm.interfaces import LLMConfig
-from onyx.llm.llm_provider_options import OPENAI_PROVIDER_NAME
 from onyx.llm.models import PreviousMessage
 from onyx.llm.utils import build_content_with_imgs
 from onyx.llm.utils import check_message_tokens
@@ -20,7 +19,6 @@ from onyx.llm.utils import message_to_prompt_and_imgs
 from onyx.llm.utils import model_supports_image_input
 from onyx.natural_language_processing.utils import get_tokenizer
 from onyx.prompts.chat_prompts import CHAT_USER_CONTEXT_FREE_PROMPT
-from onyx.prompts.chat_prompts import CODE_BLOCK_MARKDOWN
 from onyx.prompts.direct_qa_prompts import HISTORY_BLOCK
 from onyx.prompts.prompt_utils import drop_messages_history_overflow
 from onyx.prompts.prompt_utils import handle_onyx_date_awareness
@@ -33,16 +31,8 @@ from onyx.tools.tool import Tool
 
 def default_build_system_message(
     prompt_config: PromptConfig,
-    llm_config: LLMConfig,
 ) -> SystemMessage | None:
     system_prompt = prompt_config.system_prompt.strip()
-    # See https://simonwillison.net/tags/markdown/ for context on this temporary fix
-    # for o-series markdown generation
-    if (
-        llm_config.model_provider == OPENAI_PROVIDER_NAME
-        and llm_config.model_name.startswith("o")
-    ):
-        system_prompt = CODE_BLOCK_MARKDOWN + system_prompt
     tag_handled_prompt = handle_onyx_date_awareness(
         system_prompt,
         prompt_config,
@@ -120,8 +110,21 @@ class AnswerPromptBuilder:
             ),
         )
 
-        self.update_system_prompt(system_message)
-        self.update_user_prompt(user_message)
+        self.system_message_and_token_cnt: tuple[SystemMessage, int] | None = (
+            (
+                system_message,
+                check_message_tokens(system_message, self.llm_tokenizer_encode_func),
+            )
+            if system_message
+            else None
+        )
+        self.user_message_and_token_cnt = (
+            user_message,
+            check_message_tokens(
+                user_message,
+                self.llm_tokenizer_encode_func,
+            ),
+        )
 
         self.new_messages_and_token_cnts: list[tuple[BaseMessage, int]] = []
 

backend/onyx/configs/app_configs.py

@@ -6,7 +6,6 @@ from typing import cast
 from onyx.auth.schemas import AuthBackend
 from onyx.configs.constants import AuthType
 from onyx.configs.constants import DocumentIndexType
-from onyx.configs.constants import QueryHistoryType
 from onyx.file_processing.enums import HtmlBasedConnectorTransformLinksStrategy
 
 #####
@@ -30,9 +29,6 @@ GENERATIVE_MODEL_ACCESS_CHECK_FREQ = int(
 )  # 1 day
 DISABLE_GENERATIVE_AI = os.environ.get("DISABLE_GENERATIVE_AI", "").lower() == "true"
 
-ONYX_QUERY_HISTORY_TYPE = QueryHistoryType(
-    (os.environ.get("ONYX_QUERY_HISTORY_TYPE") or QueryHistoryType.NORMAL.value).lower()
-)
 
 #####
 # Web Configs
@@ -162,7 +158,7 @@ POSTGRES_USER = os.environ.get("POSTGRES_USER") or "postgres"
 POSTGRES_PASSWORD = urllib.parse.quote_plus(
     os.environ.get("POSTGRES_PASSWORD") or "password"
 )
-POSTGRES_HOST = os.environ.get("POSTGRES_HOST") or "localhost"
+POSTGRES_HOST = os.environ.get("POSTGRES_HOST") or "127.0.0.1"
 POSTGRES_PORT = os.environ.get("POSTGRES_PORT") or "5432"
 POSTGRES_DB = os.environ.get("POSTGRES_DB") or "postgres"
 AWS_REGION_NAME = os.environ.get("AWS_REGION_NAME") or "us-east-2"

backend/onyx/configs/constants.py

@@ -213,12 +213,6 @@ class AuthType(str, Enum):
     CLOUD = "cloud"
 
 
-class QueryHistoryType(str, Enum):
-    DISABLED = "disabled"
-    ANONYMIZED = "anonymized"
-    NORMAL = "normal"
-
-
 # Special characters for password validation
 PASSWORD_SPECIAL_CHARS = "!@#$%^&*()_+-=[]{}|;:,.<>?"
 
@@ -348,9 +342,6 @@ class OnyxRedisSignals:
     BLOCK_PRUNING = "signal:block_pruning"
     BLOCK_VALIDATE_PRUNING_FENCES = "signal:block_validate_pruning_fences"
     BLOCK_BUILD_FENCE_LOOKUP_TABLE = "signal:block_build_fence_lookup_table"
-    BLOCK_VALIDATE_CONNECTOR_DELETION_FENCES = (
-        "signal:block_validate_connector_deletion_fences"
-    )
 
 
 class OnyxRedisConstants:

backend/onyx/connectors/blob/connector.py

@@ -7,18 +7,11 @@ from typing import Optional
 
 import boto3  # type: ignore
 from botocore.client import Config  # type: ignore
-from botocore.exceptions import ClientError
-from botocore.exceptions import NoCredentialsError
-from botocore.exceptions import PartialCredentialsError
 from mypy_boto3_s3 import S3Client  # type: ignore
 
 from onyx.configs.app_configs import INDEX_BATCH_SIZE
 from onyx.configs.constants import BlobType
 from onyx.configs.constants import DocumentSource
-from onyx.connectors.exceptions import ConnectorValidationError
-from onyx.connectors.exceptions import CredentialExpiredError
-from onyx.connectors.exceptions import InsufficientPermissionsError
-from onyx.connectors.exceptions import UnexpectedError
 from onyx.connectors.interfaces import GenerateDocumentsOutput
 from onyx.connectors.interfaces import LoadConnector
 from onyx.connectors.interfaces import PollConnector
@@ -247,73 +240,6 @@ class BlobStorageConnector(LoadConnector, PollConnector):
 
         return None
 
-    def validate_connector_settings(self) -> None:
-        if self.s3_client is None:
-            raise ConnectorMissingCredentialError(
-                "Blob storage credentials not loaded."
-            )
-
-        if not self.bucket_name:
-            raise ConnectorValidationError(
-                "No bucket name was provided in connector settings."
-            )
-
-        try:
-            # We only fetch one object/page as a light-weight validation step.
-            # This ensures we trigger typical S3 permission checks (ListObjectsV2, etc.).
-            self.s3_client.list_objects_v2(
-                Bucket=self.bucket_name, Prefix=self.prefix, MaxKeys=1
-            )
-
-        except NoCredentialsError:
-            raise ConnectorMissingCredentialError(
-                "No valid blob storage credentials found or provided to boto3."
-            )
-        except PartialCredentialsError:
-            raise ConnectorMissingCredentialError(
-                "Partial or incomplete blob storage credentials provided to boto3."
-            )
-        except ClientError as e:
-            error_code = e.response["Error"].get("Code", "")
-            status_code = e.response["ResponseMetadata"].get("HTTPStatusCode")
-
-            # Most common S3 error cases
-            if error_code in [
-                "AccessDenied",
-                "InvalidAccessKeyId",
-                "SignatureDoesNotMatch",
-            ]:
-                if status_code == 403 or error_code == "AccessDenied":
-                    raise InsufficientPermissionsError(
-                        f"Insufficient permissions to list objects in bucket '{self.bucket_name}'. "
-                        "Please check your bucket policy and/or IAM policy."
-                    )
-                if status_code == 401 or error_code == "SignatureDoesNotMatch":
-                    raise CredentialExpiredError(
-                        "Provided blob storage credentials appear invalid or expired."
-                    )
-
-                raise CredentialExpiredError(
-                    f"Credential issue encountered ({error_code})."
-                )
-
-            if error_code == "NoSuchBucket" or status_code == 404:
-                raise ConnectorValidationError(
-                    f"Bucket '{self.bucket_name}' does not exist or cannot be found."
-                )
-
-            raise ConnectorValidationError(
-                f"Unexpected S3 client error (code={error_code}, status={status_code}): {e}"
-            )
-
-        except Exception as e:
-            # Catch-all for anything not captured by the above
-            # Since we are unsure of the error and it may not disable the connector,
-            #  raise an unexpected error (does not disable connector)
-            raise UnexpectedError(
-                f"Unexpected error during blob storage settings validation: {e}"
-            )
-
 
 if __name__ == "__main__":
     credentials_dict = {

backend/onyx/connectors/bookstack/connector.py

@@ -9,10 +9,10 @@ from onyx.configs.constants import DocumentSource
 from onyx.connectors.bookstack.client import BookStackApiClient
 from onyx.connectors.bookstack.client import BookStackClientRequestFailedError
 from onyx.connectors.cross_connector_utils.miscellaneous_utils import time_str_to_utc
-from onyx.connectors.exceptions import ConnectorValidationError
-from onyx.connectors.exceptions import CredentialExpiredError
-from onyx.connectors.exceptions import InsufficientPermissionsError
+from onyx.connectors.interfaces import ConnectorValidationError
+from onyx.connectors.interfaces import CredentialExpiredError
 from onyx.connectors.interfaces import GenerateDocumentsOutput
+from onyx.connectors.interfaces import InsufficientPermissionsError
 from onyx.connectors.interfaces import LoadConnector
 from onyx.connectors.interfaces import PollConnector
 from onyx.connectors.interfaces import SecondsSinceUnixEpoch

backend/onyx/connectors/confluence/connector.py

@@ -4,8 +4,6 @@ from datetime import timezone
 from typing import Any
 from urllib.parse import quote
 
-from requests.exceptions import HTTPError
-
 from onyx.configs.app_configs import CONFLUENCE_CONNECTOR_LABELS_TO_SKIP
 from onyx.configs.app_configs import CONFLUENCE_TIMEZONE_OFFSET
 from onyx.configs.app_configs import CONTINUE_ON_CONNECTOR_FAILURE
@@ -18,10 +16,6 @@ from onyx.connectors.confluence.utils import build_confluence_document_id
 from onyx.connectors.confluence.utils import datetime_from_string
 from onyx.connectors.confluence.utils import extract_text_from_confluence_html
 from onyx.connectors.confluence.utils import validate_attachment_filetype
-from onyx.connectors.exceptions import ConnectorValidationError
-from onyx.connectors.exceptions import CredentialExpiredError
-from onyx.connectors.exceptions import InsufficientPermissionsError
-from onyx.connectors.exceptions import UnexpectedError
 from onyx.connectors.interfaces import GenerateDocumentsOutput
 from onyx.connectors.interfaces import GenerateSlimDocumentOutput
 from onyx.connectors.interfaces import LoadConnector
@@ -403,33 +397,3 @@ class ConfluenceConnector(LoadConnector, PollConnector, SlimConnector):
                     callback.progress("retrieve_all_slim_documents", 1)
 
         yield doc_metadata_list
-
-    def validate_connector_settings(self) -> None:
-        if self._confluence_client is None:
-            raise ConnectorMissingCredentialError("Confluence credentials not loaded.")
-
-        try:
-            spaces = self._confluence_client.get_all_spaces(limit=1)
-        except HTTPError as e:
-            status_code = e.response.status_code if e.response else None
-            if status_code == 401:
-                raise CredentialExpiredError(
-                    "Invalid or expired Confluence credentials (HTTP 401)."
-                )
-            elif status_code == 403:
-                raise InsufficientPermissionsError(
-                    "Insufficient permissions to access Confluence resources (HTTP 403)."
-                )
-            raise UnexpectedError(
-                f"Unexpected Confluence error (status={status_code}): {e}"
-            )
-        except Exception as e:
-            raise UnexpectedError(
-                f"Unexpected error while validating Confluence settings: {e}"
-            )
-
-        if not spaces or not spaces.get("results"):
-            raise ConnectorValidationError(
-                "No Confluence spaces found. Either your credentials lack permissions, or "
-                "there truly are no spaces in this Confluence instance."
-            )

backend/onyx/connectors/confluence/onyx_confluence.py

@@ -11,9 +11,6 @@ from atlassian import Confluence  # type:ignore
 from pydantic import BaseModel
 from requests import HTTPError
 
-from onyx.connectors.confluence.utils import get_start_param_from_url
-from onyx.connectors.confluence.utils import update_param_in_path
-from onyx.connectors.exceptions import ConnectorValidationError
 from onyx.utils.logger import setup_logger
 
 logger = setup_logger()
@@ -163,7 +160,7 @@ class OnyxConfluence(Confluence):
                 )
 
     def _paginate_url(
-        self, url_suffix: str, limit: int | None = None, auto_paginate: bool = False
+        self, url_suffix: str, limit: int | None = None
     ) -> Iterator[dict[str, Any]]:
         """
         This will paginate through the top level query.
@@ -238,41 +235,9 @@ class OnyxConfluence(Confluence):
                 raise e
 
             # yield the results individually
-            results = cast(list[dict[str, Any]], next_response.get("results", []))
-            yield from results
+            yield from next_response.get("results", [])
 
-            old_url_suffix = url_suffix
-            url_suffix = cast(str, next_response.get("_links", {}).get("next", ""))
-
-            # make sure we don't update the start by more than the amount
-            # of results we were able to retrieve. The Confluence API has a
-            # weird behavior where if you pass in a limit that is too large for
-            # the configured server, it will artificially limit the amount of
-            # results returned BUT will not apply this to the start parameter.
-            # This will cause us to miss results.
-            if url_suffix and "start" in url_suffix:
-                new_start = get_start_param_from_url(url_suffix)
-                previous_start = get_start_param_from_url(old_url_suffix)
-                if new_start - previous_start > len(results):
-                    logger.warning(
-                        f"Start was updated by more than the amount of results "
-                        f"retrieved. This is a bug with Confluence. Start: {new_start}, "
-                        f"Previous Start: {previous_start}, Len Results: {len(results)}."
-                    )
-
-                    # Update the url_suffix to use the adjusted start
-                    adjusted_start = previous_start + len(results)
-                    url_suffix = update_param_in_path(
-                        url_suffix, "start", str(adjusted_start)
-                    )
-
-            # some APIs don't properly paginate, so we need to manually update the `start` param
-            if auto_paginate and len(results) > 0:
-                previous_start = get_start_param_from_url(old_url_suffix)
-                updated_start = previous_start + len(results)
-                url_suffix = update_param_in_path(
-                    old_url_suffix, "start", str(updated_start)
-                )
+            url_suffix = next_response.get("_links", {}).get("next")
 
     def paginated_cql_retrieval(
         self,
@@ -332,9 +297,7 @@ class OnyxConfluence(Confluence):
             url = "rest/api/search/user"
             expand_string = f"&expand={expand}" if expand else ""
             url += f"?cql={cql}{expand_string}"
-            # endpoint doesn't properly paginate, so we need to manually update the `start` param
-            # thus the auto_paginate flag
-            for user_result in self._paginate_url(url, limit, auto_paginate=True):
+            for user_result in self._paginate_url(url, limit):
                 # Example response:
                 # {
                 #     'user': {
@@ -545,15 +508,11 @@ def build_confluence_client(
     is_cloud: bool,
     wiki_base: str,
 ) -> OnyxConfluence:
-    try:
-        _validate_connector_configuration(
-            credentials=credentials,
-            is_cloud=is_cloud,
-            wiki_base=wiki_base,
-        )
-    except Exception as e:
-        raise ConnectorValidationError(str(e))
-
+    _validate_connector_configuration(
+        credentials=credentials,
+        is_cloud=is_cloud,
+        wiki_base=wiki_base,
+    )
     return OnyxConfluence(
         api_version="cloud" if is_cloud else "latest",
         # Remove trailing slash from wiki_base if present

backend/onyx/connectors/confluence/utils.py

@@ -2,10 +2,7 @@ import io
 from datetime import datetime
 from datetime import timezone
 from typing import Any
-from typing import TYPE_CHECKING
-from urllib.parse import parse_qs
 from urllib.parse import quote
-from urllib.parse import urlparse
 
 import bs4
 
@@ -13,13 +10,13 @@ from onyx.configs.app_configs import (
     CONFLUENCE_CONNECTOR_ATTACHMENT_CHAR_COUNT_THRESHOLD,
 )
 from onyx.configs.app_configs import CONFLUENCE_CONNECTOR_ATTACHMENT_SIZE_THRESHOLD
+from onyx.connectors.confluence.onyx_confluence import (
+    OnyxConfluence,
+)
 from onyx.file_processing.extract_file_text import extract_file_text
 from onyx.file_processing.html_utils import format_document_soup
 from onyx.utils.logger import setup_logger
 
-if TYPE_CHECKING:
-    from onyx.connectors.confluence.onyx_confluence import OnyxConfluence
-
 logger = setup_logger()
 
 
@@ -27,7 +24,7 @@ _USER_EMAIL_CACHE: dict[str, str | None] = {}
 
 
 def get_user_email_from_username__server(
-    confluence_client: "OnyxConfluence", user_name: str
+    confluence_client: OnyxConfluence, user_name: str
 ) -> str | None:
     global _USER_EMAIL_CACHE
     if _USER_EMAIL_CACHE.get(user_name) is None:
@@ -50,7 +47,7 @@ _USER_NOT_FOUND = "Unknown Confluence User"
 _USER_ID_TO_DISPLAY_NAME_CACHE: dict[str, str | None] = {}
 
 
-def _get_user(confluence_client: "OnyxConfluence", user_id: str) -> str:
+def _get_user(confluence_client: OnyxConfluence, user_id: str) -> str:
     """Get Confluence Display Name based on the account-id or userkey value
 
     Args:
@@ -81,7 +78,7 @@ def _get_user(confluence_client: "OnyxConfluence", user_id: str) -> str:
 
 
 def extract_text_from_confluence_html(
-    confluence_client: "OnyxConfluence",
+    confluence_client: OnyxConfluence,
     confluence_object: dict[str, Any],
     fetched_titles: set[str],
 ) -> str:
@@ -194,7 +191,7 @@ def validate_attachment_filetype(attachment: dict[str, Any]) -> bool:
 
 
 def attachment_to_content(
-    confluence_client: "OnyxConfluence",
+    confluence_client: OnyxConfluence,
     attachment: dict[str, Any],
 ) -> str | None:
     """If it returns None, assume that we should skip this attachment."""
@@ -282,32 +279,3 @@ def datetime_from_string(datetime_string: str) -> datetime:
         datetime_object = datetime_object.astimezone(timezone.utc)
 
     return datetime_object
-
-
-def get_single_param_from_url(url: str, param: str) -> str | None:
-    """Get a parameter from a url"""
-    parsed_url = urlparse(url)
-    return parse_qs(parsed_url.query).get(param, [None])[0]
-
-
-def get_start_param_from_url(url: str) -> int:
-    """Get the start parameter from a url"""
-    start_str = get_single_param_from_url(url, "start")
-    if start_str is None:
-        return 0
-    return int(start_str)
-
-
-def update_param_in_path(path: str, param: str, value: str) -> str:
-    """Update a parameter in a path. Path should look something like:
-
-    /api/rest/users?start=0&limit=10
-    """
-    parsed_url = urlparse(path)
-    query_params = parse_qs(parsed_url.query)
-    query_params[param] = [value]
-    return (
-        path.split("?")[0]
-        + "?"
-        + "&".join(f"{k}={quote(v[0])}" for k, v in query_params.items())
-    )

backend/onyx/connectors/dropbox/connector.py

@@ -10,10 +10,10 @@ from dropbox.files import FolderMetadata  # type:ignore
 
 from onyx.configs.app_configs import INDEX_BATCH_SIZE
 from onyx.configs.constants import DocumentSource
-from onyx.connectors.exceptions import ConnectorValidationError
-from onyx.connectors.exceptions import CredentialInvalidError
-from onyx.connectors.exceptions import InsufficientPermissionsError
+from onyx.connectors.interfaces import ConnectorValidationError
+from onyx.connectors.interfaces import CredentialInvalidError
 from onyx.connectors.interfaces import GenerateDocumentsOutput
+from onyx.connectors.interfaces import InsufficientPermissionsError
 from onyx.connectors.interfaces import LoadConnector
 from onyx.connectors.interfaces import PollConnector
 from onyx.connectors.interfaces import SecondsSinceUnixEpoch

backend/onyx/connectors/exceptions.py

@@ -1,49 +0,0 @@
-class ValidationError(Exception):
-    """General exception for validation errors."""
-
-    def __init__(self, message: str):
-        self.message = message
-        super().__init__(self.message)
-
-
-class ConnectorValidationError(ValidationError):
-    """General exception for connector validation errors."""
-
-    def __init__(self, message: str):
-        self.message = message
-        super().__init__(self.message)
-
-
-class UnexpectedError(ValidationError):
-    """Raised when an unexpected error occurs during connector validation.
-
-    Unexpected errors don't necessarily mean the credential is invalid,
-    but rather that there was an error during the validation process
-    or we encountered a currently unhandled error case.
-    """
-
-    def __init__(self, message: str = "Unexpected error during connector validation"):
-        super().__init__(message)
-
-
-class CredentialInvalidError(ConnectorValidationError):
-    """Raised when a connector's credential is invalid."""
-
-    def __init__(self, message: str = "Credential is invalid"):
-        super().__init__(message)
-
-
-class CredentialExpiredError(ConnectorValidationError):
-    """Raised when a connector's credential is expired."""
-
-    def __init__(self, message: str = "Credential has expired"):
-        super().__init__(message)
-
-
-class InsufficientPermissionsError(ConnectorValidationError):
-    """Raised when the credential does not have sufficient API permissions."""
-
-    def __init__(
-        self, message: str = "Insufficient permissions for the requested operation"
-    ):
-        super().__init__(message)

backend/onyx/connectors/factory.py

@@ -5,6 +5,7 @@ from sqlalchemy.orm import Session
 
 from onyx.configs.app_configs import INTEGRATION_TESTS_MODE
 from onyx.configs.constants import DocumentSource
+from onyx.configs.constants import DocumentSourceRequiringTenantContext
 from onyx.connectors.airtable.airtable_connector import AirtableConnector
 from onyx.connectors.asana.connector import AsanaConnector
 from onyx.connectors.axero.connector import AxeroConnector
@@ -17,7 +18,6 @@ from onyx.connectors.discourse.connector import DiscourseConnector
 from onyx.connectors.document360.connector import Document360Connector
 from onyx.connectors.dropbox.connector import DropboxConnector
 from onyx.connectors.egnyte.connector import EgnyteConnector
-from onyx.connectors.exceptions import ConnectorValidationError
 from onyx.connectors.file.connector import LocalFileConnector
 from onyx.connectors.fireflies.connector import FirefliesConnector
 from onyx.connectors.freshdesk.connector import FreshdeskConnector
@@ -32,6 +32,7 @@ from onyx.connectors.guru.connector import GuruConnector
 from onyx.connectors.hubspot.connector import HubSpotConnector
 from onyx.connectors.interfaces import BaseConnector
 from onyx.connectors.interfaces import CheckpointConnector
+from onyx.connectors.interfaces import ConnectorValidationError
 from onyx.connectors.interfaces import EventConnector
 from onyx.connectors.interfaces import LoadConnector
 from onyx.connectors.interfaces import PollConnector
@@ -55,8 +56,9 @@ from onyx.connectors.zendesk.connector import ZendeskConnector
 from onyx.connectors.zulip.connector import ZulipConnector
 from onyx.db.connector import fetch_connector_by_id
 from onyx.db.credentials import backend_update_credential_json
-from onyx.db.credentials import fetch_credential_by_id
+from onyx.db.credentials import fetch_credential_by_id_for_user
 from onyx.db.models import Credential
+from onyx.db.models import User
 
 
 class ConnectorMissingException(Exception):
@@ -163,9 +165,13 @@ def instantiate_connector(
     input_type: InputType,
     connector_specific_config: dict[str, Any],
     credential: Credential,
+    tenant_id: str | None = None,
 ) -> BaseConnector:
     connector_class = identify_connector_class(source, input_type)
 
+    if source in DocumentSourceRequiringTenantContext:
+        connector_specific_config["tenant_id"] = tenant_id
+
     connector = connector_class(**connector_specific_config)
     new_credentials = connector.load_credentials(credential.credential_json)
 
@@ -179,16 +185,19 @@ def validate_ccpair_for_user(
     connector_id: int,
     credential_id: int,
     db_session: Session,
-    enforce_creation: bool = True,
-) -> bool:
+    user: User | None,
+    tenant_id: str | None,
+) -> None:
     if INTEGRATION_TESTS_MODE:
-        return True
+        return
 
     # Validate the connector settings
     connector = fetch_connector_by_id(connector_id, db_session)
-    credential = fetch_credential_by_id(
+    credential = fetch_credential_by_id_for_user(
         credential_id,
+        user,
         db_session,
+        get_editable=False,
     )
 
     if not connector:
@@ -198,7 +207,7 @@ def validate_ccpair_for_user(
         connector.source == DocumentSource.INGESTION_API
         or connector.source == DocumentSource.MOCK_CONNECTOR
     ):
-        return True
+        return
 
     if not credential:
         raise ValueError("Credential not found")
@@ -210,14 +219,9 @@ def validate_ccpair_for_user(
             input_type=connector.input_type,
             connector_specific_config=connector.connector_specific_config,
             credential=credential,
+            tenant_id=tenant_id,
         )
-    except ConnectorValidationError as e:
-        raise e
     except Exception as e:
-        if enforce_creation:
-            raise ConnectorValidationError(str(e))
-        else:
-            return False
+        raise ConnectorValidationError(str(e))
 
     runnable_connector.validate_connector_settings()
-    return True

backend/onyx/connectors/file/connector.py

@@ -16,7 +16,7 @@ from onyx.connectors.interfaces import LoadConnector
 from onyx.connectors.models import BasicExpertInfo
 from onyx.connectors.models import Document
 from onyx.connectors.models import Section
-from onyx.db.engine import get_session_with_current_tenant
+from onyx.db.engine import get_session_with_tenant
 from onyx.file_processing.extract_file_text import detect_encoding
 from onyx.file_processing.extract_file_text import extract_file_text
 from onyx.file_processing.extract_file_text import get_file_ext
@@ -27,6 +27,8 @@ from onyx.file_processing.extract_file_text import read_pdf_file
 from onyx.file_processing.extract_file_text import read_text_file
 from onyx.file_store.file_store import get_default_file_store
 from onyx.utils.logger import setup_logger
+from shared_configs.configs import POSTGRES_DEFAULT_SCHEMA
+from shared_configs.contextvars import CURRENT_TENANT_ID_CONTEXTVAR
 
 logger = setup_logger()
 
@@ -163,10 +165,12 @@ class LocalFileConnector(LoadConnector):
     def __init__(
         self,
         file_locations: list[Path | str],
+        tenant_id: str = POSTGRES_DEFAULT_SCHEMA,
         batch_size: int = INDEX_BATCH_SIZE,
     ) -> None:
         self.file_locations = [Path(file_location) for file_location in file_locations]
         self.batch_size = batch_size
+        self.tenant_id = tenant_id
         self.pdf_pass: str | None = None
 
     def load_credentials(self, credentials: dict[str, Any]) -> dict[str, Any] | None:
@@ -175,8 +179,9 @@ class LocalFileConnector(LoadConnector):
 
     def load_from_state(self) -> GenerateDocumentsOutput:
         documents: list[Document] = []
+        token = CURRENT_TENANT_ID_CONTEXTVAR.set(self.tenant_id)
 
-        with get_session_with_current_tenant() as db_session:
+        with get_session_with_tenant(tenant_id=self.tenant_id) as db_session:
             for file_path in self.file_locations:
                 current_datetime = datetime.now(timezone.utc)
                 files = _read_files_and_metadata(
@@ -198,6 +203,8 @@ class LocalFileConnector(LoadConnector):
             if documents:
                 yield documents
 
+        CURRENT_TENANT_ID_CONTEXTVAR.reset(token)
+
 
 if __name__ == "__main__":
     connector = LocalFileConnector(file_locations=[os.environ["TEST_FILE"]])

backend/onyx/connectors/github/connector.py

@@ -17,14 +17,14 @@ from github.PullRequest import PullRequest
 from onyx.configs.app_configs import GITHUB_CONNECTOR_BASE_URL
 from onyx.configs.app_configs import INDEX_BATCH_SIZE
 from onyx.configs.constants import DocumentSource
-from onyx.connectors.exceptions import ConnectorValidationError
-from onyx.connectors.exceptions import CredentialExpiredError
-from onyx.connectors.exceptions import InsufficientPermissionsError
-from onyx.connectors.exceptions import UnexpectedError
+from onyx.connectors.interfaces import ConnectorValidationError
+from onyx.connectors.interfaces import CredentialExpiredError
 from onyx.connectors.interfaces import GenerateDocumentsOutput
+from onyx.connectors.interfaces import InsufficientPermissionsError
 from onyx.connectors.interfaces import LoadConnector
 from onyx.connectors.interfaces import PollConnector
 from onyx.connectors.interfaces import SecondsSinceUnixEpoch
+from onyx.connectors.interfaces import UnexpectedError
 from onyx.connectors.models import ConnectorMissingCredentialError
 from onyx.connectors.models import Document
 from onyx.connectors.models import Section
@@ -124,7 +124,7 @@ class GithubConnector(LoadConnector, PollConnector):
     def __init__(
         self,
         repo_owner: str,
-        repo_name: str | None = None,
+        repo_name: str,
         batch_size: int = INDEX_BATCH_SIZE,
         state_filter: str = "all",
         include_prs: bool = True,
@@ -162,81 +162,53 @@ class GithubConnector(LoadConnector, PollConnector):
             _sleep_after_rate_limit_exception(github_client)
             return self._get_github_repo(github_client, attempt_num + 1)
 
-    def _get_all_repos(
-        self, github_client: Github, attempt_num: int = 0
-    ) -> list[Repository.Repository]:
-        if attempt_num > _MAX_NUM_RATE_LIMIT_RETRIES:
-            raise RuntimeError(
-                "Re-tried fetching repos too many times. Something is going wrong with fetching objects from Github"
-            )
-
-        try:
-            # Try to get organization first
-            try:
-                org = github_client.get_organization(self.repo_owner)
-                return list(org.get_repos())
-            except GithubException:
-                # If not an org, try as a user
-                user = github_client.get_user(self.repo_owner)
-                return list(user.get_repos())
-        except RateLimitExceededException:
-            _sleep_after_rate_limit_exception(github_client)
-            return self._get_all_repos(github_client, attempt_num + 1)
-
     def _fetch_from_github(
         self, start: datetime | None = None, end: datetime | None = None
     ) -> GenerateDocumentsOutput:
         if self.github_client is None:
             raise ConnectorMissingCredentialError("GitHub")
 
-        repos = (
-            [self._get_github_repo(self.github_client)]
-            if self.repo_name
-            else self._get_all_repos(self.github_client)
-        )
+        repo = self._get_github_repo(self.github_client)
 
-        for repo in repos:
-            if self.include_prs:
-                logger.info(f"Fetching PRs for repo: {repo.name}")
-                pull_requests = repo.get_pulls(
-                    state=self.state_filter, sort="updated", direction="desc"
-                )
+        if self.include_prs:
+            pull_requests = repo.get_pulls(
+                state=self.state_filter, sort="updated", direction="desc"
+            )
 
-                for pr_batch in _batch_github_objects(
-                    pull_requests, self.github_client, self.batch_size
-                ):
-                    doc_batch: list[Document] = []
-                    for pr in pr_batch:
-                        if start is not None and pr.updated_at < start:
-                            yield doc_batch
-                            break
-                        if end is not None and pr.updated_at > end:
-                            continue
-                        doc_batch.append(_convert_pr_to_document(cast(PullRequest, pr)))
-                    yield doc_batch
+            for pr_batch in _batch_github_objects(
+                pull_requests, self.github_client, self.batch_size
+            ):
+                doc_batch: list[Document] = []
+                for pr in pr_batch:
+                    if start is not None and pr.updated_at < start:
+                        yield doc_batch
+                        return
+                    if end is not None and pr.updated_at > end:
+                        continue
+                    doc_batch.append(_convert_pr_to_document(cast(PullRequest, pr)))
+                yield doc_batch
 
-            if self.include_issues:
-                logger.info(f"Fetching issues for repo: {repo.name}")
-                issues = repo.get_issues(
-                    state=self.state_filter, sort="updated", direction="desc"
-                )
+        if self.include_issues:
+            issues = repo.get_issues(
+                state=self.state_filter, sort="updated", direction="desc"
+            )
 
-                for issue_batch in _batch_github_objects(
-                    issues, self.github_client, self.batch_size
-                ):
-                    doc_batch = []
-                    for issue in issue_batch:
-                        issue = cast(Issue, issue)
-                        if start is not None and issue.updated_at < start:
-                            yield doc_batch
-                            break
-                        if end is not None and issue.updated_at > end:
-                            continue
-                        if issue.pull_request is not None:
-                            # PRs are handled separately
-                            continue
-                        doc_batch.append(_convert_issue_to_document(issue))
-                    yield doc_batch
+            for issue_batch in _batch_github_objects(
+                issues, self.github_client, self.batch_size
+            ):
+                doc_batch = []
+                for issue in issue_batch:
+                    issue = cast(Issue, issue)
+                    if start is not None and issue.updated_at < start:
+                        yield doc_batch
+                        return
+                    if end is not None and issue.updated_at > end:
+                        continue
+                    if issue.pull_request is not None:
+                        # PRs are handled separately
+                        continue
+                    doc_batch.append(_convert_issue_to_document(issue))
+                yield doc_batch
 
     def load_from_state(self) -> GenerateDocumentsOutput:
         return self._fetch_from_github()
@@ -262,26 +234,16 @@ class GithubConnector(LoadConnector, PollConnector):
         if self.github_client is None:
             raise ConnectorMissingCredentialError("GitHub credentials not loaded.")
 
-        if not self.repo_owner:
+        if not self.repo_owner or not self.repo_name:
             raise ConnectorValidationError(
-                "Invalid connector settings: 'repo_owner' must be provided."
+                "Invalid connector settings: 'repo_owner' and 'repo_name' must be provided."
             )
 
         try:
-            if self.repo_name:
-                test_repo = self.github_client.get_repo(
-                    f"{self.repo_owner}/{self.repo_name}"
-                )
-                test_repo.get_contents("")
-            else:
-                # Try to get organization first
-                try:
-                    org = self.github_client.get_organization(self.repo_owner)
-                    org.get_repos().totalCount  # Just check if we can access repos
-                except GithubException:
-                    # If not an org, try as a user
-                    user = self.github_client.get_user(self.repo_owner)
-                    user.get_repos().totalCount  # Just check if we can access repos
+            test_repo = self.github_client.get_repo(
+                f"{self.repo_owner}/{self.repo_name}"
+            )
+            test_repo.get_contents("")
 
         except RateLimitExceededException:
             raise UnexpectedError(
@@ -298,14 +260,9 @@ class GithubConnector(LoadConnector, PollConnector):
                     "Your GitHub token does not have sufficient permissions for this repository (HTTP 403)."
                 )
             elif e.status == 404:
-                if self.repo_name:
-                    raise ConnectorValidationError(
-                        f"GitHub repository not found with name: {self.repo_owner}/{self.repo_name}"
-                    )
-                else:
-                    raise ConnectorValidationError(
-                        f"GitHub user or organization not found: {self.repo_owner}"
-                    )
+                raise ConnectorValidationError(
+                    f"GitHub repository not found with name: {self.repo_owner}/{self.repo_name}"
+                )
             else:
                 raise ConnectorValidationError(
                     f"Unexpected GitHub error (status={e.status}): {e.data}"

backend/onyx/connectors/gmail/connector.py

@@ -305,7 +305,6 @@ class GmailConnector(LoadConnector, PollConnector, SlimConnector):
                     userId=user_email,
                     fields=THREAD_FIELDS,
                     id=thread["id"],
-                    continue_on_404_or_403=True,
                 )
                 # full_threads is an iterator containing a single thread
                 # so we need to convert it to a list and grab the first element
@@ -337,7 +336,6 @@ class GmailConnector(LoadConnector, PollConnector, SlimConnector):
                 userId=user_email,
                 fields=THREAD_LIST_FIELDS,
                 q=query,
-                continue_on_404_or_403=True,
             ):
                 doc_batch.append(
                     SlimDocument(

backend/onyx/connectors/google_drive/connector.py

@@ -13,9 +13,6 @@ from googleapiclient.errors import HttpError  # type: ignore
 from onyx.configs.app_configs import INDEX_BATCH_SIZE
 from onyx.configs.app_configs import MAX_FILE_SIZE_BYTES
 from onyx.configs.constants import DocumentSource
-from onyx.connectors.exceptions import ConnectorValidationError
-from onyx.connectors.exceptions import CredentialExpiredError
-from onyx.connectors.exceptions import InsufficientPermissionsError
 from onyx.connectors.google_drive.doc_conversion import build_slim_document
 from onyx.connectors.google_drive.doc_conversion import (
     convert_drive_item_to_document,
@@ -45,7 +42,6 @@ from onyx.connectors.interfaces import LoadConnector
 from onyx.connectors.interfaces import PollConnector
 from onyx.connectors.interfaces import SecondsSinceUnixEpoch
 from onyx.connectors.interfaces import SlimConnector
-from onyx.connectors.models import ConnectorMissingCredentialError
 from onyx.indexing.indexing_heartbeat import IndexingHeartbeatInterface
 from onyx.utils.logger import setup_logger
 from onyx.utils.retry_wrapper import retry_builder
@@ -141,7 +137,7 @@ class GoogleDriveConnector(LoadConnector, PollConnector, SlimConnector):
                 "Please visit the docs for help with the new setup: "
                 f"{SCOPE_DOC_URL}"
             )
-            raise ConnectorValidationError(
+            raise ValueError(
                 "Google Drive connector received old input parameters. "
                 "Please visit the docs for help with the new setup: "
                 f"{SCOPE_DOC_URL}"
@@ -155,7 +151,7 @@ class GoogleDriveConnector(LoadConnector, PollConnector, SlimConnector):
             and not my_drive_emails
             and not shared_drive_urls
         ):
-            raise ConnectorValidationError(
+            raise ValueError(
                 "Nothing to index. Please specify at least one of the following: "
                 "include_shared_drives, include_my_drives, include_files_shared_with_me, "
                 "shared_folder_urls, or my_drive_emails"
@@ -613,50 +609,3 @@ class GoogleDriveConnector(LoadConnector, PollConnector, SlimConnector):
             if MISSING_SCOPES_ERROR_STR in str(e):
                 raise PermissionError(ONYX_SCOPE_INSTRUCTIONS) from e
             raise e
-
-    def validate_connector_settings(self) -> None:
-        if self._creds is None:
-            raise ConnectorMissingCredentialError(
-                "Google Drive credentials not loaded."
-            )
-
-        if self._primary_admin_email is None:
-            raise ConnectorValidationError(
-                "Primary admin email not found in credentials. "
-                "Ensure DB_CREDENTIALS_PRIMARY_ADMIN_KEY is set."
-            )
-
-        try:
-            drive_service = get_drive_service(self._creds, self._primary_admin_email)
-            drive_service.files().list(pageSize=1, fields="files(id)").execute()
-
-            if isinstance(self._creds, ServiceAccountCredentials):
-                retry_builder()(get_root_folder_id)(drive_service)
-
-        except HttpError as e:
-            status_code = e.resp.status if e.resp else None
-            if status_code == 401:
-                raise CredentialExpiredError(
-                    "Invalid or expired Google Drive credentials (401)."
-                )
-            elif status_code == 403:
-                raise InsufficientPermissionsError(
-                    "Google Drive app lacks required permissions (403). "
-                    "Please ensure the necessary scopes are granted and Drive "
-                    "apps are enabled."
-                )
-            else:
-                raise ConnectorValidationError(
-                    f"Unexpected Google Drive error (status={status_code}): {e}"
-                )
-
-        except Exception as e:
-            # Check for scope-related hints from the error message
-            if MISSING_SCOPES_ERROR_STR in str(e):
-                raise InsufficientPermissionsError(
-                    "Google Drive credentials are missing required scopes. "
-                    f"{ONYX_SCOPE_INSTRUCTIONS}"
-                )
-            raise ConnectorValidationError(
-                f"Unexpected error during Google Drive validation: {e}"
-            )

backend/onyx/connectors/google_drive/doc_conversion.py

@@ -1,9 +1,7 @@
 import io
 from datetime import datetime
 from datetime import timezone
-from tempfile import NamedTemporaryFile
 
-import openpyxl  # type: ignore
 from googleapiclient.discovery import build  # type: ignore
 from googleapiclient.errors import HttpError  # type: ignore
 
@@ -45,15 +43,12 @@ def _extract_sections_basic(
 ) -> list[Section]:
     mime_type = file["mimeType"]
     link = file["webViewLink"]
-    supported_file_types = set(item.value for item in GDriveMimeType)
 
-    if mime_type not in supported_file_types:
+    if mime_type not in set(item.value for item in GDriveMimeType):
         # Unsupported file types can still have a title, finding this way is still useful
         return [Section(link=link, text=UNSUPPORTED_FILE_TYPE_CONTENT)]
 
     try:
-        # ---------------------------
-        # Google Sheets extraction
         if mime_type == GDriveMimeType.SPREADSHEET.value:
             try:
                 sheets_service = build(
@@ -114,53 +109,7 @@ def _extract_sections_basic(
                     f"Ran into exception '{e}' when pulling data from Google Sheet '{file['name']}'."
                     " Falling back to basic extraction."
                 )
-        # ---------------------------
-        # Microsoft Excel (.xlsx or .xls) extraction branch
-        elif mime_type in [
-            GDriveMimeType.SPREADSHEET_OPEN_FORMAT.value,
-            GDriveMimeType.SPREADSHEET_MS_EXCEL.value,
-        ]:
-            try:
-                response = service.files().get_media(fileId=file["id"]).execute()
 
-                with NamedTemporaryFile(suffix=".xlsx", delete=True) as tmp:
-                    tmp.write(response)
-                    tmp_path = tmp.name
-
-                    section_separator = "\n\n"
-                    workbook = openpyxl.load_workbook(tmp_path, read_only=True)
-
-                    # Work similarly to the xlsx_to_text function used for file connector
-                    # but returns Sections instead of a string
-                    sections = [
-                        Section(
-                            link=link,
-                            text=(
-                                f"Sheet: {sheet.title}\n\n"
-                                + section_separator.join(
-                                    ",".join(map(str, row))
-                                    for row in sheet.iter_rows(
-                                        min_row=1, values_only=True
-                                    )
-                                    if row
-                                )
-                            ),
-                        )
-                        for sheet in workbook.worksheets
-                    ]
-
-                return sections
-
-            except Exception as e:
-                logger.warning(
-                    f"Error extracting data from Excel file '{file['name']}': {e}"
-                )
-                return [
-                    Section(link=link, text="Error extracting data from Excel file")
-                ]
-
-        # ---------------------------
-        # Export for Google Docs, PPT, and fallback for spreadsheets
         if mime_type in [
             GDriveMimeType.DOC.value,
             GDriveMimeType.PPT.value,
@@ -179,8 +128,6 @@ def _extract_sections_basic(
             )
             return [Section(link=link, text=text)]
 
-        # ---------------------------
-        # Plain text and Markdown files
         elif mime_type in [
             GDriveMimeType.PLAIN_TEXT.value,
             GDriveMimeType.MARKDOWN.value,
@@ -194,8 +141,6 @@ def _extract_sections_basic(
                     .decode("utf-8"),
                 )
             ]
-        # ---------------------------
-        # Word, PowerPoint, PDF files
         if mime_type in [
             GDriveMimeType.WORD_DOC.value,
             GDriveMimeType.POWERPOINT.value,
@@ -225,11 +170,7 @@ def _extract_sections_basic(
                     Section(link=link, text=pptx_to_text(file=io.BytesIO(response)))
                 ]
 
-        # Catch-all case, should not happen since there should be specific handling
-        # for each of the supported file types
-        error_message = f"Unsupported file type: {mime_type}"
-        logger.error(error_message)
-        raise ValueError(error_message)
+        return [Section(link=link, text=UNSUPPORTED_FILE_TYPE_CONTENT)]
 
     except Exception:
         return [Section(link=link, text=UNSUPPORTED_FILE_TYPE_CONTENT)]

backend/onyx/connectors/google_drive/models.py

@@ -5,10 +5,6 @@ from typing import Any
 class GDriveMimeType(str, Enum):
     DOC = "application/vnd.google-apps.document"
     SPREADSHEET = "application/vnd.google-apps.spreadsheet"
-    SPREADSHEET_OPEN_FORMAT = (
-        "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
-    )
-    SPREADSHEET_MS_EXCEL = "application/vnd.ms-excel"
     PDF = "application/pdf"
     WORD_DOC = "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
     PPT = "application/vnd.google-apps.presentation"

backend/onyx/connectors/hubspot/connector.py

@@ -87,18 +87,16 @@ class HubSpotConnector(LoadConnector, PollConnector):
                         contact = api_client.crm.contacts.basic_api.get_by_id(
                             contact_id=contact.id
                         )
-                        email = contact.properties.get("email")
-                        if email is not None:
-                            associated_emails.append(email)
+                        associated_emails.append(contact.properties["email"])
 
                 if notes:
                     for note in notes.results:
                         note = api_client.crm.objects.notes.basic_api.get_by_id(
                             note_id=note.id, properties=["content", "hs_body_preview"]
                         )
-                        preview = note.properties.get("hs_body_preview")
-                        if preview is not None:
-                            associated_notes.append(preview)
+                        if note.properties["hs_body_preview"] is None:
+                            continue
+                        associated_notes.append(note.properties["hs_body_preview"])
 
             associated_emails_str = " ,".join(associated_emails)
             associated_notes_str = " ".join(associated_notes)

backend/onyx/connectors/interfaces.py

@@ -146,3 +146,46 @@ class CheckpointConnector(BaseConnector):
         ```
         """
         raise NotImplementedError
+
+
+class ConnectorValidationError(Exception):
+    """General exception for connector validation errors."""
+
+    def __init__(self, message: str):
+        self.message = message
+        super().__init__(self.message)
+
+
+class UnexpectedError(Exception):
+    """Raised when an unexpected error occurs during connector validation.
+
+    Unexpected errors don't necessarily mean the credential is invalid,
+    but rather that there was an error during the validation process
+    or we encountered a currently unhandled error case.
+    """
+
+    def __init__(self, message: str = "Unexpected error during connector validation"):
+        super().__init__(message)
+
+
+class CredentialInvalidError(ConnectorValidationError):
+    """Raised when a connector's credential is invalid."""
+
+    def __init__(self, message: str = "Credential is invalid"):
+        super().__init__(message)
+
+
+class CredentialExpiredError(ConnectorValidationError):
+    """Raised when a connector's credential is expired."""
+
+    def __init__(self, message: str = "Credential has expired"):
+        super().__init__(message)
+
+
+class InsufficientPermissionsError(ConnectorValidationError):
+    """Raised when the credential does not have sufficient API permissions."""
+
+    def __init__(
+        self, message: str = "Insufficient permissions for the requested operation"
+    ):
+        super().__init__(message)

backend/onyx/connectors/notion/connector.py

@@ -16,11 +16,10 @@ from onyx.configs.constants import DocumentSource
 from onyx.connectors.cross_connector_utils.rate_limit_wrapper import (
     rl_requests,
 )
-from onyx.connectors.exceptions import ConnectorValidationError
-from onyx.connectors.exceptions import CredentialExpiredError
-from onyx.connectors.exceptions import InsufficientPermissionsError
-from onyx.connectors.exceptions import UnexpectedError
+from onyx.connectors.interfaces import ConnectorValidationError
+from onyx.connectors.interfaces import CredentialExpiredError
 from onyx.connectors.interfaces import GenerateDocumentsOutput
+from onyx.connectors.interfaces import InsufficientPermissionsError
 from onyx.connectors.interfaces import LoadConnector
 from onyx.connectors.interfaces import PollConnector
 from onyx.connectors.interfaces import SecondsSinceUnixEpoch
@@ -671,12 +670,12 @@ class NotionConnector(LoadConnector, PollConnector):
                     "Please try again later."
                 )
             else:
-                raise UnexpectedError(
+                raise Exception(
                     f"Unexpected Notion HTTP error (status={status_code}): {http_err}"
                 ) from http_err
 
         except Exception as exc:
-            raise UnexpectedError(
+            raise Exception(
                 f"Unexpected error during Notion settings validation: {exc}"
             )
 

backend/onyx/connectors/onyx_jira/connector.py

@@ -12,11 +12,11 @@ from onyx.configs.app_configs import JIRA_CONNECTOR_LABELS_TO_SKIP
 from onyx.configs.app_configs import JIRA_CONNECTOR_MAX_TICKET_SIZE
 from onyx.configs.constants import DocumentSource
 from onyx.connectors.cross_connector_utils.miscellaneous_utils import time_str_to_utc
-from onyx.connectors.exceptions import ConnectorValidationError
-from onyx.connectors.exceptions import CredentialExpiredError
-from onyx.connectors.exceptions import InsufficientPermissionsError
+from onyx.connectors.interfaces import ConnectorValidationError
+from onyx.connectors.interfaces import CredentialExpiredError
 from onyx.connectors.interfaces import GenerateDocumentsOutput
 from onyx.connectors.interfaces import GenerateSlimDocumentOutput
+from onyx.connectors.interfaces import InsufficientPermissionsError
 from onyx.connectors.interfaces import LoadConnector
 from onyx.connectors.interfaces import PollConnector
 from onyx.connectors.interfaces import SecondsSinceUnixEpoch
@@ -29,6 +29,7 @@ from onyx.connectors.onyx_jira.utils import best_effort_basic_expert_info
 from onyx.connectors.onyx_jira.utils import best_effort_get_field_from_issue
 from onyx.connectors.onyx_jira.utils import build_jira_client
 from onyx.connectors.onyx_jira.utils import build_jira_url
+from onyx.connectors.onyx_jira.utils import extract_jira_project
 from onyx.connectors.onyx_jira.utils import extract_text_from_adf
 from onyx.connectors.onyx_jira.utils import get_comment_strs
 from onyx.indexing.indexing_heartbeat import IndexingHeartbeatInterface
@@ -159,8 +160,7 @@ def fetch_jira_issues_batch(
 class JiraConnector(LoadConnector, PollConnector, SlimConnector):
     def __init__(
         self,
-        jira_base_url: str,
-        project_key: str | None = None,
+        jira_project_url: str,
         comment_email_blacklist: list[str] | None = None,
         batch_size: int = INDEX_BATCH_SIZE,
         # if a ticket has one of the labels specified in this list, we will just
@@ -169,12 +169,11 @@ class JiraConnector(LoadConnector, PollConnector, SlimConnector):
         labels_to_skip: list[str] = JIRA_CONNECTOR_LABELS_TO_SKIP,
     ) -> None:
         self.batch_size = batch_size
-        self.jira_base = jira_base_url.rstrip("/")  # Remove trailing slash if present
-        self.jira_project = project_key
-        self._comment_email_blacklist = comment_email_blacklist or []
-        self.labels_to_skip = set(labels_to_skip)
-
+        self.jira_base, self._jira_project = extract_jira_project(jira_project_url)
         self._jira_client: JIRA | None = None
+        self._comment_email_blacklist = comment_email_blacklist or []
+
+        self.labels_to_skip = set(labels_to_skip)
 
     @property
     def comment_email_blacklist(self) -> tuple:
@@ -189,9 +188,7 @@ class JiraConnector(LoadConnector, PollConnector, SlimConnector):
     @property
     def quoted_jira_project(self) -> str:
         # Quote the project name to handle reserved words
-        if not self.jira_project:
-            return ""
-        return f'"{self.jira_project}"'
+        return f'"{self._jira_project}"'
 
     def load_credentials(self, credentials: dict[str, Any]) -> dict[str, Any] | None:
         self._jira_client = build_jira_client(
@@ -200,14 +197,8 @@ class JiraConnector(LoadConnector, PollConnector, SlimConnector):
         )
         return None
 
-    def _get_jql_query(self) -> str:
-        """Get the JQL query based on whether a specific project is set"""
-        if self.jira_project:
-            return f"project = {self.quoted_jira_project}"
-        return ""  # Empty string means all accessible projects
-
     def load_from_state(self) -> GenerateDocumentsOutput:
-        jql = self._get_jql_query()
+        jql = f"project = {self.quoted_jira_project}"
 
         document_batch = []
         for doc in fetch_jira_issues_batch(
@@ -234,10 +225,11 @@ class JiraConnector(LoadConnector, PollConnector, SlimConnector):
             "%Y-%m-%d %H:%M"
         )
 
-        base_jql = self._get_jql_query()
         jql = (
-            f"{base_jql} AND " if base_jql else ""
-        ) + f"updated >= '{start_date_str}' AND updated <= '{end_date_str}'"
+            f"project = {self.quoted_jira_project} AND "
+            f"updated >= '{start_date_str}' AND "
+            f"updated <= '{end_date_str}'"
+        )
 
         document_batch = []
         for doc in fetch_jira_issues_batch(
@@ -260,7 +252,7 @@ class JiraConnector(LoadConnector, PollConnector, SlimConnector):
         end: SecondsSinceUnixEpoch | None = None,
         callback: IndexingHeartbeatInterface | None = None,
     ) -> GenerateSlimDocumentOutput:
-        jql = self._get_jql_query()
+        jql = f"project = {self.quoted_jira_project}"
 
         slim_doc_batch = []
         for issue in _paginate_jql_search(
@@ -287,63 +279,43 @@ class JiraConnector(LoadConnector, PollConnector, SlimConnector):
         if self._jira_client is None:
             raise ConnectorMissingCredentialError("Jira")
 
-        # If a specific project is set, validate it exists
-        if self.jira_project:
-            try:
-                self.jira_client.project(self.jira_project)
-            except Exception as e:
-                status_code = getattr(e, "status_code", None)
+        if not self._jira_project:
+            raise ConnectorValidationError(
+                "Invalid connector settings: 'jira_project' must be provided."
+            )
 
-                if status_code == 401:
-                    raise CredentialExpiredError(
-                        "Jira credential appears to be expired or invalid (HTTP 401)."
-                    )
-                elif status_code == 403:
-                    raise InsufficientPermissionsError(
-                        "Your Jira token does not have sufficient permissions for this project (HTTP 403)."
-                    )
-                elif status_code == 404:
-                    raise ConnectorValidationError(
-                        f"Jira project not found with key: {self.jira_project}"
-                    )
-                elif status_code == 429:
-                    raise ConnectorValidationError(
-                        "Validation failed due to Jira rate-limits being exceeded. Please try again later."
-                    )
+        try:
+            self.jira_client.project(self._jira_project)
 
-                raise RuntimeError(f"Unexpected Jira error during validation: {e}")
-        else:
-            # If no project specified, validate we can access the Jira API
-            try:
-                # Try to list projects to validate access
-                self.jira_client.projects()
-            except Exception as e:
-                status_code = getattr(e, "status_code", None)
-                if status_code == 401:
-                    raise CredentialExpiredError(
-                        "Jira credential appears to be expired or invalid (HTTP 401)."
-                    )
-                elif status_code == 403:
-                    raise InsufficientPermissionsError(
-                        "Your Jira token does not have sufficient permissions to list projects (HTTP 403)."
-                    )
-                elif status_code == 429:
-                    raise ConnectorValidationError(
-                        "Validation failed due to Jira rate-limits being exceeded. Please try again later."
-                    )
+        except Exception as e:
+            status_code = getattr(e, "status_code", None)
 
-                raise RuntimeError(f"Unexpected Jira error during validation: {e}")
+            if status_code == 401:
+                raise CredentialExpiredError(
+                    "Jira credential appears to be expired or invalid (HTTP 401)."
+                )
+            elif status_code == 403:
+                raise InsufficientPermissionsError(
+                    "Your Jira token does not have sufficient permissions for this project (HTTP 403)."
+                )
+            elif status_code == 404:
+                raise ConnectorValidationError(
+                    f"Jira project not found with key: {self._jira_project}"
+                )
+            elif status_code == 429:
+                raise ConnectorValidationError(
+                    "Validation failed due to Jira rate-limits being exceeded. Please try again later."
+                )
+            else:
+                raise Exception(f"Unexpected Jira error during validation: {e}")
 
 
 if __name__ == "__main__":
     import os
 
     connector = JiraConnector(
-        jira_base_url=os.environ["JIRA_BASE_URL"],
-        project_key=os.environ.get("JIRA_PROJECT_KEY"),
-        comment_email_blacklist=[],
+        os.environ["JIRA_PROJECT_URL"], comment_email_blacklist=[]
     )
-
     connector.load_credentials(
         {
             "jira_user_email": os.environ["JIRA_USER_EMAIL"],

backend/onyx/connectors/slack/connector.py

@@ -18,10 +18,6 @@ from slack_sdk.errors import SlackApiError
 from onyx.configs.app_configs import ENABLE_EXPENSIVE_EXPERT_CALLS
 from onyx.configs.app_configs import INDEX_BATCH_SIZE
 from onyx.configs.constants import DocumentSource
-from onyx.connectors.exceptions import ConnectorValidationError
-from onyx.connectors.exceptions import CredentialExpiredError
-from onyx.connectors.exceptions import InsufficientPermissionsError
-from onyx.connectors.exceptions import UnexpectedError
 from onyx.connectors.interfaces import CheckpointConnector
 from onyx.connectors.interfaces import CheckpointOutput
 from onyx.connectors.interfaces import GenerateSlimDocumentOutput
@@ -86,14 +82,14 @@ def get_channels(
     get_public: bool = True,
     get_private: bool = True,
 ) -> list[ChannelType]:
-    """Get all channels in the workspace."""
+    """Get all channels in the workspace"""
     channels: list[dict[str, Any]] = []
     channel_types = []
     if get_public:
         channel_types.append("public_channel")
     if get_private:
         channel_types.append("private_channel")
-    # Try fetching both public and private channels first:
+    # try getting private channels as well at first
     try:
         channels = _collect_paginated_channels(
             client=client,
@@ -101,19 +97,19 @@ def get_channels(
             channel_types=channel_types,
         )
     except SlackApiError as e:
-        logger.info(
-            f"Unable to fetch private channels due to: {e}. Trying again without private channels."
-        )
+        logger.info(f"Unable to fetch private channels due to - {e}")
+        logger.info("trying again without private channels")
         if get_public:
             channel_types = ["public_channel"]
         else:
-            logger.warning("No channels to fetch.")
+            logger.warning("No channels to fetch")
             return []
         channels = _collect_paginated_channels(
             client=client,
             exclude_archived=exclude_archived,
             channel_types=channel_types,
         )
+
     return channels
 
 
@@ -670,86 +666,6 @@ class SlackConnector(SlimConnector, CheckpointConnector):
             )
             return checkpoint
 
-    def validate_connector_settings(self) -> None:
-        """
-        1. Verify the bot token is valid for the workspace (via auth_test).
-        2. Ensure the bot has enough scope to list channels.
-        3. Check that every channel specified in self.channels exists.
-        """
-        if self.client is None:
-            raise ConnectorMissingCredentialError("Slack credentials not loaded.")
-
-        try:
-            # 1) Validate connection to workspace
-            auth_response = self.client.auth_test()
-            if not auth_response.get("ok", False):
-                error_msg = auth_response.get(
-                    "error", "Unknown error from Slack auth_test"
-                )
-                raise ConnectorValidationError(f"Failed Slack auth_test: {error_msg}")
-
-            # 2) Minimal test to confirm listing channels works
-            test_resp = self.client.conversations_list(
-                limit=1, types=["public_channel"]
-            )
-            if not test_resp.get("ok", False):
-                error_msg = test_resp.get("error", "Unknown error from Slack")
-                if error_msg == "invalid_auth":
-                    raise ConnectorValidationError(
-                        f"Invalid Slack bot token ({error_msg})."
-                    )
-                elif error_msg == "not_authed":
-                    raise CredentialExpiredError(
-                        f"Invalid or expired Slack bot token ({error_msg})."
-                    )
-                raise UnexpectedError(f"Slack API returned a failure: {error_msg}")
-
-            # 3) If channels are specified, verify each is accessible
-            if self.channels:
-                accessible_channels = get_channels(
-                    client=self.client,
-                    exclude_archived=True,
-                    get_public=True,
-                    get_private=True,
-                )
-                # For quick lookups by name or ID, build a map:
-                accessible_channel_names = {ch["name"] for ch in accessible_channels}
-                accessible_channel_ids = {ch["id"] for ch in accessible_channels}
-
-                for user_channel in self.channels:
-                    if (
-                        user_channel not in accessible_channel_names
-                        and user_channel not in accessible_channel_ids
-                    ):
-                        raise ConnectorValidationError(
-                            f"Channel '{user_channel}' not found or inaccessible in this workspace."
-                        )
-
-        except SlackApiError as e:
-            slack_error = e.response.get("error", "")
-            if slack_error == "missing_scope":
-                raise InsufficientPermissionsError(
-                    "Slack bot token lacks the necessary scope to list/access channels. "
-                    "Please ensure your Slack app has 'channels:read' (and/or 'groups:read' for private channels)."
-                )
-            elif slack_error == "invalid_auth":
-                raise CredentialExpiredError(
-                    f"Invalid Slack bot token ({slack_error})."
-                )
-            elif slack_error == "not_authed":
-                raise CredentialExpiredError(
-                    f"Invalid or expired Slack bot token ({slack_error})."
-                )
-            raise UnexpectedError(
-                f"Unexpected Slack error '{slack_error}' during settings validation."
-            )
-        except ConnectorValidationError as e:
-            raise e
-        except Exception as e:
-            raise UnexpectedError(
-                f"Unexpected error during Slack settings validation: {e}"
-            )
-
 
 if __name__ == "__main__":
     import os

backend/onyx/connectors/teams/connector.py

@@ -5,7 +5,6 @@ from typing import Any
 
 import msal  # type: ignore
 from office365.graph_client import GraphClient  # type: ignore
-from office365.runtime.client_request_exception import ClientRequestException  # type: ignore
 from office365.teams.channels.channel import Channel  # type: ignore
 from office365.teams.chats.messages.message import ChatMessage  # type: ignore
 from office365.teams.team import Team  # type: ignore
@@ -13,10 +12,6 @@ from office365.teams.team import Team  # type: ignore
 from onyx.configs.app_configs import INDEX_BATCH_SIZE
 from onyx.configs.constants import DocumentSource
 from onyx.connectors.cross_connector_utils.miscellaneous_utils import time_str_to_utc
-from onyx.connectors.exceptions import ConnectorValidationError
-from onyx.connectors.exceptions import CredentialExpiredError
-from onyx.connectors.exceptions import InsufficientPermissionsError
-from onyx.connectors.exceptions import UnexpectedError
 from onyx.connectors.interfaces import GenerateDocumentsOutput
 from onyx.connectors.interfaces import LoadConnector
 from onyx.connectors.interfaces import PollConnector
@@ -284,50 +279,6 @@ class TeamsConnector(LoadConnector, PollConnector):
         end_datetime = datetime.fromtimestamp(end, timezone.utc)
         return self._fetch_from_teams(start=start_datetime, end=end_datetime)
 
-    def validate_connector_settings(self) -> None:
-        if self.graph_client is None:
-            raise ConnectorMissingCredentialError("Teams credentials not loaded.")
-
-        try:
-            # Minimal call to confirm we can retrieve Teams
-            found_teams = self._get_all_teams()
-
-        except ClientRequestException as e:
-            status_code = e.response.status_code
-            if status_code == 401:
-                raise CredentialExpiredError(
-                    "Invalid or expired Microsoft Teams credentials (401 Unauthorized)."
-                )
-            elif status_code == 403:
-                raise InsufficientPermissionsError(
-                    "Your app lacks sufficient permissions to read Teams (403 Forbidden)."
-                )
-            raise UnexpectedError(f"Unexpected error retrieving teams: {e}")
-
-        except Exception as e:
-            error_str = str(e).lower()
-            if (
-                "unauthorized" in error_str
-                or "401" in error_str
-                or "invalid_grant" in error_str
-            ):
-                raise CredentialExpiredError(
-                    "Invalid or expired Microsoft Teams credentials."
-                )
-            elif "forbidden" in error_str or "403" in error_str:
-                raise InsufficientPermissionsError(
-                    "App lacks required permissions to read from Microsoft Teams."
-                )
-            raise ConnectorValidationError(
-                f"Unexpected error during Teams validation: {e}"
-            )
-
-        if not found_teams:
-            raise ConnectorValidationError(
-                "No Teams found for the given credentials. "
-                "Either there are no Teams in this tenant, or your app does not have permission to view them."
-            )
-
 
 if __name__ == "__main__":
     connector = TeamsConnector(teams=os.environ["TEAMS"].split(","))

backend/onyx/connectors/web/connector.py

@@ -25,12 +25,12 @@ from onyx.configs.app_configs import WEB_CONNECTOR_OAUTH_CLIENT_SECRET
 from onyx.configs.app_configs import WEB_CONNECTOR_OAUTH_TOKEN_URL
 from onyx.configs.app_configs import WEB_CONNECTOR_VALIDATE_URLS
 from onyx.configs.constants import DocumentSource
-from onyx.connectors.exceptions import ConnectorValidationError
-from onyx.connectors.exceptions import CredentialExpiredError
-from onyx.connectors.exceptions import InsufficientPermissionsError
-from onyx.connectors.exceptions import UnexpectedError
+from onyx.connectors.interfaces import ConnectorValidationError
+from onyx.connectors.interfaces import CredentialExpiredError
 from onyx.connectors.interfaces import GenerateDocumentsOutput
+from onyx.connectors.interfaces import InsufficientPermissionsError
 from onyx.connectors.interfaces import LoadConnector
+from onyx.connectors.interfaces import UnexpectedError
 from onyx.connectors.models import Document
 from onyx.connectors.models import Section
 from onyx.file_processing.extract_file_text import read_pdf_file
@@ -440,10 +440,7 @@ class WebConnector(LoadConnector):
                 "No URL configured. Please provide at least one valid URL."
             )
 
-        if (
-            self.web_connector_type == WEB_CONNECTOR_VALID_SETTINGS.SITEMAP.value
-            or self.web_connector_type == WEB_CONNECTOR_VALID_SETTINGS.RECURSIVE.value
-        ):
+        if self.web_connector_type == WEB_CONNECTOR_VALID_SETTINGS.SITEMAP.value:
             return None
 
         # We'll just test the first URL for connectivity and correctness

backend/onyx/db/api_key.py

@@ -16,6 +16,7 @@ from onyx.configs.constants import UNNAMED_KEY_PLACEHOLDER
 from onyx.db.models import ApiKey
 from onyx.db.models import User
 from onyx.server.api_key.models import APIKeyArgs
+from shared_configs.configs import MULTI_TENANT
 from shared_configs.contextvars import get_current_tenant_id
 
 
@@ -72,7 +73,7 @@ def insert_api_key(
     # Get tenant_id from context var (will be default schema for single tenant)
     tenant_id = get_current_tenant_id()
 
-    api_key = generate_api_key(tenant_id)
+    api_key = generate_api_key(tenant_id if MULTI_TENANT else None)
     api_key_user_id = uuid.uuid4()
 
     display_name = api_key_args.name or UNNAMED_KEY_PLACEHOLDER

backend/onyx/db/chat.py

@@ -168,7 +168,7 @@ def get_chat_sessions_by_user(
     if not include_onyxbot_flows:
         stmt = stmt.where(ChatSession.onyxbot_flow.is_(False))
 
-    stmt = stmt.order_by(desc(ChatSession.time_updated))
+    stmt = stmt.order_by(desc(ChatSession.time_created))
 
     if deleted is not None:
         stmt = stmt.where(ChatSession.deleted == deleted)

backend/onyx/db/chat_search.py

@@ -1,152 +0,0 @@
-from typing import List
-from typing import Optional
-from typing import Tuple
-from uuid import UUID
-
-from sqlalchemy import desc
-from sqlalchemy import func
-from sqlalchemy import literal
-from sqlalchemy import Select
-from sqlalchemy import select
-from sqlalchemy import union_all
-from sqlalchemy.orm import joinedload
-from sqlalchemy.orm import Session
-
-from onyx.db.models import ChatMessage
-from onyx.db.models import ChatSession
-
-
-def search_chat_sessions(
-    user_id: UUID | None,
-    db_session: Session,
-    query: Optional[str] = None,
-    page: int = 1,
-    page_size: int = 10,
-    include_deleted: bool = False,
-    include_onyxbot_flows: bool = False,
-) -> Tuple[List[ChatSession], bool]:
-    """
-    Search for chat sessions based on the provided query.
-    If no query is provided, returns recent chat sessions.
-
-    Returns a tuple of (chat_sessions, has_more)
-    """
-    offset = (page - 1) * page_size
-
-    # If no search query, we use standard SQLAlchemy pagination
-    if not query or not query.strip():
-        stmt = select(ChatSession)
-        if user_id:
-            stmt = stmt.where(ChatSession.user_id == user_id)
-        if not include_onyxbot_flows:
-            stmt = stmt.where(ChatSession.onyxbot_flow.is_(False))
-        if not include_deleted:
-            stmt = stmt.where(ChatSession.deleted.is_(False))
-
-        stmt = stmt.order_by(desc(ChatSession.time_created))
-
-        # Apply pagination
-        stmt = stmt.offset(offset).limit(page_size + 1)
-        result = db_session.execute(stmt.options(joinedload(ChatSession.persona)))
-        chat_sessions = result.scalars().all()
-
-        has_more = len(chat_sessions) > page_size
-        if has_more:
-            chat_sessions = chat_sessions[:page_size]
-
-        return list(chat_sessions), has_more
-
-    words = query.lower().strip().split()
-
-    # Message mach subquery
-    message_matches = []
-    for word in words:
-        word_like = f"%{word}%"
-        message_match: Select = (
-            select(ChatMessage.chat_session_id, literal(1.0).label("search_rank"))
-            .join(ChatSession, ChatSession.id == ChatMessage.chat_session_id)
-            .where(func.lower(ChatMessage.message).like(word_like))
-        )
-
-        if user_id:
-            message_match = message_match.where(ChatSession.user_id == user_id)
-
-        message_matches.append(message_match)
-
-    if message_matches:
-        message_matches_query = union_all(*message_matches).alias("message_matches")
-    else:
-        return [], False
-
-    # Description matches
-    description_match: Select = select(
-        ChatSession.id.label("chat_session_id"), literal(0.5).label("search_rank")
-    ).where(func.lower(ChatSession.description).like(f"%{query.lower()}%"))
-
-    if user_id:
-        description_match = description_match.where(ChatSession.user_id == user_id)
-    if not include_onyxbot_flows:
-        description_match = description_match.where(ChatSession.onyxbot_flow.is_(False))
-    if not include_deleted:
-        description_match = description_match.where(ChatSession.deleted.is_(False))
-
-    # Combine all match sources
-    combined_matches = union_all(
-        message_matches_query.select(), description_match
-    ).alias("combined_matches")
-
-    # Use CTE to group and get max rank
-    session_ranks = (
-        select(
-            combined_matches.c.chat_session_id,
-            func.max(combined_matches.c.search_rank).label("rank"),
-        )
-        .group_by(combined_matches.c.chat_session_id)
-        .alias("session_ranks")
-    )
-
-    # Get ranked sessions with pagination
-    ranked_query = (
-        db_session.query(session_ranks.c.chat_session_id, session_ranks.c.rank)
-        .order_by(desc(session_ranks.c.rank), session_ranks.c.chat_session_id)
-        .offset(offset)
-        .limit(page_size + 1)
-    )
-
-    result = ranked_query.all()
-
-    # Extract session IDs and ranks
-    session_ids_with_ranks = {row.chat_session_id: row.rank for row in result}
-    session_ids = list(session_ids_with_ranks.keys())
-
-    if not session_ids:
-        return [], False
-
-    # Now, let's query the actual ChatSession objects using the IDs
-    stmt = select(ChatSession).where(ChatSession.id.in_(session_ids))
-
-    if user_id:
-        stmt = stmt.where(ChatSession.user_id == user_id)
-    if not include_onyxbot_flows:
-        stmt = stmt.where(ChatSession.onyxbot_flow.is_(False))
-    if not include_deleted:
-        stmt = stmt.where(ChatSession.deleted.is_(False))
-
-    # Full objects with eager loading
-    result = db_session.execute(stmt.options(joinedload(ChatSession.persona)))
-    chat_sessions = result.scalars().all()
-
-    # Sort based on above ranking
-    chat_sessions = sorted(
-        chat_sessions,
-        key=lambda session: (
-            -session_ids_with_ranks.get(session.id, 0),  # Rank (higher first)
-            session.time_created.timestamp() * -1,  # Then by time (newest first)
-        ),
-    )
-
-    has_more = len(chat_sessions) > page_size
-    if has_more:
-        chat_sessions = chat_sessions[:page_size]
-
-    return chat_sessions, has_more

backend/onyx/db/connector_credential_pair.py

@@ -1,5 +1,4 @@
 from datetime import datetime
-from typing import TypeVarTuple
 
 from fastapi import HTTPException
 from sqlalchemy import delete
@@ -9,18 +8,15 @@ from sqlalchemy import Select
 from sqlalchemy import select
 from sqlalchemy.orm import aliased
 from sqlalchemy.orm import joinedload
-from sqlalchemy.orm import selectinload
 from sqlalchemy.orm import Session
 
 from onyx.configs.app_configs import DISABLE_AUTH
 from onyx.db.connector import fetch_connector_by_id
 from onyx.db.credentials import fetch_credential_by_id
 from onyx.db.credentials import fetch_credential_by_id_for_user
-from onyx.db.engine import get_session_context_manager
 from onyx.db.enums import AccessType
 from onyx.db.enums import ConnectorCredentialPairStatus
 from onyx.db.models import ConnectorCredentialPair
-from onyx.db.models import Credential
 from onyx.db.models import IndexAttempt
 from onyx.db.models import IndexingStatus
 from onyx.db.models import IndexModelStatus
@@ -35,12 +31,10 @@ from onyx.utils.variable_functionality import fetch_ee_implementation_or_noop
 
 logger = setup_logger()
 
-R = TypeVarTuple("R")
-
 
 def _add_user_filters(
-    stmt: Select[tuple[*R]], user: User | None, get_editable: bool = True
-) -> Select[tuple[*R]]:
+    stmt: Select, user: User | None, get_editable: bool = True
+) -> Select:
     # If user is None and auth is disabled, assume the user is an admin
     if (user is None and DISABLE_AUTH) or (user and user.role == UserRole.ADMIN):
         return stmt
@@ -104,52 +98,17 @@ def get_connector_credential_pairs_for_user(
     get_editable: bool = True,
     ids: list[int] | None = None,
     eager_load_connector: bool = False,
-    eager_load_credential: bool = False,
-    eager_load_user: bool = False,
 ) -> list[ConnectorCredentialPair]:
-    if eager_load_user:
-        assert (
-            eager_load_credential
-        ), "eager_load_credential must be True if eager_load_user is True"
     stmt = select(ConnectorCredentialPair).distinct()
 
     if eager_load_connector:
-        stmt = stmt.options(selectinload(ConnectorCredentialPair.connector))
-
-    if eager_load_credential:
-        load_opts = selectinload(ConnectorCredentialPair.credential)
-        if eager_load_user:
-            load_opts = load_opts.joinedload(Credential.user)
-        stmt = stmt.options(load_opts)
+        stmt = stmt.options(joinedload(ConnectorCredentialPair.connector))
 
     stmt = _add_user_filters(stmt, user, get_editable)
     if ids:
         stmt = stmt.where(ConnectorCredentialPair.id.in_(ids))
 
-    return list(db_session.scalars(stmt).unique().all())
-
-
-# For use with our thread-level parallelism utils. Note that any relationships
-# you wish to use MUST be eagerly loaded, as the session will not be available
-# after this function to allow lazy loading.
-def get_connector_credential_pairs_for_user_parallel(
-    user: User | None,
-    get_editable: bool = True,
-    ids: list[int] | None = None,
-    eager_load_connector: bool = False,
-    eager_load_credential: bool = False,
-    eager_load_user: bool = False,
-) -> list[ConnectorCredentialPair]:
-    with get_session_context_manager() as db_session:
-        return get_connector_credential_pairs_for_user(
-            db_session,
-            user,
-            get_editable,
-            ids,
-            eager_load_connector,
-            eager_load_credential,
-            eager_load_user,
-        )
+    return list(db_session.scalars(stmt).all())
 
 
 def get_connector_credential_pairs(
@@ -192,16 +151,6 @@ def get_cc_pair_groups_for_ids(
     return list(db_session.scalars(stmt).all())
 
 
-# For use with our thread-level parallelism utils. Note that any relationships
-# you wish to use MUST be eagerly loaded, as the session will not be available
-# after this function to allow lazy loading.
-def get_cc_pair_groups_for_ids_parallel(
-    cc_pair_ids: list[int],
-) -> list[UserGroup__ConnectorCredentialPair]:
-    with get_session_context_manager() as db_session:
-        return get_cc_pair_groups_for_ids(db_session, cc_pair_ids)
-
-
 def get_connector_credential_pair_for_user(
     db_session: Session,
     connector_id: int,
@@ -452,8 +401,8 @@ def add_credential_to_connector(
     # If we are in the seeding flow, we shouldn't need to check if the credential belongs to the user
     if seeding_flow:
         credential = fetch_credential_by_id(
-            credential_id=credential_id,
             db_session=db_session,
+            credential_id=credential_id,
         )
     else:
         credential = fetch_credential_by_id_for_user(

backend/onyx/db/credentials.py

@@ -169,8 +169,8 @@ def fetch_credential_by_id_for_user(
 
 
 def fetch_credential_by_id(
-    credential_id: int,
     db_session: Session,
+    credential_id: int,
 ) -> Credential | None:
     stmt = select(Credential).distinct()
     stmt = stmt.where(Credential.id == credential_id)
@@ -422,8 +422,8 @@ def create_initial_public_credential(db_session: Session) -> None:
         "There must exist an empty public credential for data connectors that do not require additional Auth."
     )
     first_credential = fetch_credential_by_id(
-        credential_id=PUBLIC_CREDENTIAL_ID,
         db_session=db_session,
+        credential_id=PUBLIC_CREDENTIAL_ID,
     )
 
     if first_credential is not None:

backend/onyx/db/document.py

@@ -24,7 +24,6 @@ from sqlalchemy.sql.expression import null
 from onyx.configs.constants import DEFAULT_BOOST
 from onyx.configs.constants import DocumentSource
 from onyx.db.connector_credential_pair import get_connector_credential_pair_from_id
-from onyx.db.engine import get_session_context_manager
 from onyx.db.enums import AccessType
 from onyx.db.enums import ConnectorCredentialPairStatus
 from onyx.db.feedback import delete_document_feedback_for_documents__no_commit
@@ -230,12 +229,12 @@ def get_document_connector_counts(
 
 
 def get_document_counts_for_cc_pairs(
-    db_session: Session, cc_pairs: list[ConnectorCredentialPairIdentifier]
+    db_session: Session, cc_pair_identifiers: list[ConnectorCredentialPairIdentifier]
 ) -> Sequence[tuple[int, int, int]]:
     """Returns a sequence of tuples of (connector_id, credential_id, document count)"""
 
     # Prepare a list of (connector_id, credential_id) tuples
-    cc_ids = [(x.connector_id, x.credential_id) for x in cc_pairs]
+    cc_ids = [(x.connector_id, x.credential_id) for x in cc_pair_identifiers]
 
     stmt = (
         select(
@@ -261,16 +260,6 @@ def get_document_counts_for_cc_pairs(
     return db_session.execute(stmt).all()  # type: ignore
 
 
-# For use with our thread-level parallelism utils. Note that any relationships
-# you wish to use MUST be eagerly loaded, as the session will not be available
-# after this function to allow lazy loading.
-def get_document_counts_for_cc_pairs_parallel(
-    cc_pairs: list[ConnectorCredentialPairIdentifier],
-) -> Sequence[tuple[int, int, int]]:
-    with get_session_context_manager() as db_session:
-        return get_document_counts_for_cc_pairs(db_session, cc_pairs)
-
-
 def get_access_info_for_document(
     db_session: Session,
     document_id: str,

backend/onyx/db/engine.py

@@ -218,7 +218,6 @@ class SqlEngine:
             final_engine_kwargs.update(engine_kwargs)
 
         logger.info(f"Creating engine with kwargs: {final_engine_kwargs}")
-        # echo=True here for inspecting all emitted db queries
         engine = create_engine(connection_string, **final_engine_kwargs)
 
         if USE_IAM_AUTH:
@@ -258,11 +257,11 @@ class SqlEngine:
                 cls._engine = None
 
 
-def get_all_tenant_ids() -> list[str]:
+def get_all_tenant_ids() -> list[str] | list[None]:
     """Returning [None] means the only tenant is the 'public' or self hosted tenant."""
 
     if not MULTI_TENANT:
-        return [POSTGRES_DEFAULT_SCHEMA]
+        return [None]
 
     with get_session_with_shared_schema() as session:
         result = session.execute(
@@ -417,7 +416,7 @@ def get_session_with_shared_schema() -> Generator[Session, None, None]:
 
 
 @contextmanager
-def get_session_with_tenant(*, tenant_id: str) -> Generator[Session, None, None]:
+def get_session_with_tenant(*, tenant_id: str | None) -> Generator[Session, None, None]:
     """
     Generate a database session for a specific tenant.
     """

backend/onyx/db/index_attempt.py

@@ -2,7 +2,6 @@ from collections.abc import Sequence
 from datetime import datetime
 from datetime import timedelta
 from datetime import timezone
-from typing import TypeVarTuple
 
 from sqlalchemy import and_
 from sqlalchemy import delete
@@ -10,13 +9,9 @@ from sqlalchemy import desc
 from sqlalchemy import func
 from sqlalchemy import select
 from sqlalchemy import update
-from sqlalchemy.orm import contains_eager
-from sqlalchemy.orm import joinedload
 from sqlalchemy.orm import Session
-from sqlalchemy.sql import Select
 
 from onyx.connectors.models import ConnectorFailure
-from onyx.db.engine import get_session_context_manager
 from onyx.db.models import IndexAttempt
 from onyx.db.models import IndexAttemptError
 from onyx.db.models import IndexingStatus
@@ -373,33 +368,19 @@ def get_latest_index_attempts_by_status(
     return db_session.execute(stmt).scalars().all()
 
 
-T = TypeVarTuple("T")
-
-
-def _add_only_finished_clause(stmt: Select[tuple[*T]]) -> Select[tuple[*T]]:
-    return stmt.where(
-        IndexAttempt.status.not_in(
-            [IndexingStatus.NOT_STARTED, IndexingStatus.IN_PROGRESS]
-        ),
-    )
-
-
 def get_latest_index_attempts(
     secondary_index: bool,
     db_session: Session,
-    eager_load_cc_pair: bool = False,
-    only_finished: bool = False,
 ) -> Sequence[IndexAttempt]:
     ids_stmt = select(
         IndexAttempt.connector_credential_pair_id,
         func.max(IndexAttempt.id).label("max_id"),
     ).join(SearchSettings, IndexAttempt.search_settings_id == SearchSettings.id)
 
-    status = IndexModelStatus.FUTURE if secondary_index else IndexModelStatus.PRESENT
-    ids_stmt = ids_stmt.where(SearchSettings.status == status)
-
-    if only_finished:
-        ids_stmt = _add_only_finished_clause(ids_stmt)
+    if secondary_index:
+        ids_stmt = ids_stmt.where(SearchSettings.status == IndexModelStatus.FUTURE)
+    else:
+        ids_stmt = ids_stmt.where(SearchSettings.status == IndexModelStatus.PRESENT)
 
     ids_stmt = ids_stmt.group_by(IndexAttempt.connector_credential_pair_id)
     ids_subquery = ids_stmt.subquery()
@@ -414,53 +395,7 @@ def get_latest_index_attempts(
         .where(IndexAttempt.id == ids_subquery.c.max_id)
     )
 
-    if only_finished:
-        stmt = _add_only_finished_clause(stmt)
-
-    if eager_load_cc_pair:
-        stmt = stmt.options(
-            joinedload(IndexAttempt.connector_credential_pair),
-            joinedload(IndexAttempt.error_rows),
-        )
-
-    return db_session.execute(stmt).scalars().unique().all()
-
-
-# For use with our thread-level parallelism utils. Note that any relationships
-# you wish to use MUST be eagerly loaded, as the session will not be available
-# after this function to allow lazy loading.
-def get_latest_index_attempts_parallel(
-    secondary_index: bool,
-    eager_load_cc_pair: bool = False,
-    only_finished: bool = False,
-) -> Sequence[IndexAttempt]:
-    with get_session_context_manager() as db_session:
-        return get_latest_index_attempts(
-            secondary_index,
-            db_session,
-            eager_load_cc_pair,
-            only_finished,
-        )
-
-
-def get_latest_index_attempt_for_cc_pair_id(
-    db_session: Session,
-    connector_credential_pair_id: int,
-    secondary_index: bool,
-    only_finished: bool = True,
-) -> IndexAttempt | None:
-    stmt = select(IndexAttempt)
-    stmt = stmt.where(
-        IndexAttempt.connector_credential_pair_id == connector_credential_pair_id,
-    )
-    if only_finished:
-        stmt = _add_only_finished_clause(stmt)
-
-    status = IndexModelStatus.FUTURE if secondary_index else IndexModelStatus.PRESENT
-    stmt = stmt.join(SearchSettings).where(SearchSettings.status == status)
-    stmt = stmt.order_by(desc(IndexAttempt.time_created))
-    stmt = stmt.limit(1)
-    return db_session.execute(stmt).scalar_one_or_none()
+    return db_session.execute(stmt).scalars().all()
 
 
 def count_index_attempts_for_connector(
@@ -518,12 +453,37 @@ def get_paginated_index_attempts_for_cc_pair_id(
 
     # Apply pagination
     stmt = stmt.offset(page * page_size).limit(page_size)
-    stmt = stmt.options(
-        contains_eager(IndexAttempt.connector_credential_pair),
-        joinedload(IndexAttempt.error_rows),
-    )
 
-    return list(db_session.execute(stmt).scalars().unique().all())
+    return list(db_session.execute(stmt).scalars().all())
+
+
+def get_latest_index_attempt_for_cc_pair_id(
+    db_session: Session,
+    connector_credential_pair_id: int,
+    secondary_index: bool,
+    only_finished: bool = True,
+) -> IndexAttempt | None:
+    stmt = select(IndexAttempt)
+    stmt = stmt.where(
+        IndexAttempt.connector_credential_pair_id == connector_credential_pair_id,
+    )
+    if only_finished:
+        stmt = stmt.where(
+            IndexAttempt.status.not_in(
+                [IndexingStatus.NOT_STARTED, IndexingStatus.IN_PROGRESS]
+            ),
+        )
+    if secondary_index:
+        stmt = stmt.join(SearchSettings).where(
+            SearchSettings.status == IndexModelStatus.FUTURE
+        )
+    else:
+        stmt = stmt.join(SearchSettings).where(
+            SearchSettings.status == IndexModelStatus.PRESENT
+        )
+    stmt = stmt.order_by(desc(IndexAttempt.time_created))
+    stmt = stmt.limit(1)
+    return db_session.execute(stmt).scalar_one_or_none()
 
 
 def get_index_attempts_for_cc_pair(

backend/onyx/db/persona.py

@@ -100,14 +100,9 @@ def _add_user_filters(
             .correlate(Persona)
         )
     else:
-        # Group the public persona conditions
-        public_condition = (Persona.is_public == True) & (  # noqa: E712
-            Persona.is_visible == True  # noqa: E712
-        )
-
-        where_clause |= public_condition
+        where_clause |= Persona.is_public == True  # noqa: E712
+        where_clause &= Persona.is_visible == True  # noqa: E712
         where_clause |= Persona__User.user_id == user.id
-
     where_clause |= Persona.user_id == user.id
 
     return stmt.where(where_clause)

backend/onyx/document_index/document_index_utils.py

@@ -81,7 +81,7 @@ def translate_boost_count_to_multiplier(boost: int) -> float:
 # Vespa's Document API.
 def get_document_chunk_ids(
     enriched_document_info_list: list[EnrichedDocumentIndexingInfo],
-    tenant_id: str,
+    tenant_id: str | None,
     large_chunks_enabled: bool,
 ) -> list[UUID]:
     doc_chunk_ids = []
@@ -139,7 +139,7 @@ def get_uuid_from_chunk_info(
     *,
     document_id: str,
     chunk_id: int,
-    tenant_id: str,
+    tenant_id: str | None,
     large_chunk_id: int | None = None,
 ) -> UUID:
     """NOTE: be VERY carefuly about changing this function. If changed without a migration,
@@ -154,7 +154,7 @@ def get_uuid_from_chunk_info(
         "large_" + str(large_chunk_id) if large_chunk_id is not None else str(chunk_id)
     )
     unique_identifier_string = "_".join([doc_str, chunk_index])
-    if MULTI_TENANT:
+    if tenant_id and MULTI_TENANT:
         unique_identifier_string += "_" + tenant_id
 
     uuid_value = uuid.uuid5(uuid.NAMESPACE_X500, unique_identifier_string)

backend/onyx/document_index/interfaces.py

@@ -43,7 +43,7 @@ class IndexBatchParams:
 
     doc_id_to_previous_chunk_cnt: dict[str, int | None]
     doc_id_to_new_chunk_cnt: dict[str, int]
-    tenant_id: str
+    tenant_id: str | None
     large_chunks_enabled: bool
 
 
@@ -222,7 +222,7 @@ class Deletable(abc.ABC):
         self,
         doc_id: str,
         *,
-        tenant_id: str,
+        tenant_id: str | None,
         chunk_count: int | None,
     ) -> int:
         """
@@ -249,7 +249,7 @@ class Updatable(abc.ABC):
         self,
         doc_id: str,
         *,
-        tenant_id: str,
+        tenant_id: str | None,
         chunk_count: int | None,
         fields: VespaDocumentFields,
     ) -> int:
@@ -270,7 +270,9 @@ class Updatable(abc.ABC):
         raise NotImplementedError
 
     @abc.abstractmethod
-    def update(self, update_requests: list[UpdateRequest], *, tenant_id: str) -> None:
+    def update(
+        self, update_requests: list[UpdateRequest], *, tenant_id: str | None
+    ) -> None:
         """
         Updates some set of chunks. The document and fields to update are specified in the update
         requests. Each update request in the list applies its changes to a list of document ids.

backend/onyx/document_index/vespa/index.py

@@ -468,7 +468,9 @@ class VespaIndex(DocumentIndex):
                         failure_msg = f"Failed to update document: {future_to_document_id[future]}"
                         raise requests.HTTPError(failure_msg) from e
 
-    def update(self, update_requests: list[UpdateRequest], *, tenant_id: str) -> None:
+    def update(
+        self, update_requests: list[UpdateRequest], *, tenant_id: str | None
+    ) -> None:
         logger.debug(f"Updating {len(update_requests)} documents in Vespa")
 
         # Handle Vespa character limitations
@@ -616,7 +618,7 @@ class VespaIndex(DocumentIndex):
         doc_id: str,
         *,
         chunk_count: int | None,
-        tenant_id: str,
+        tenant_id: str | None,
         fields: VespaDocumentFields,
     ) -> int:
         """Note: if the document id does not exist, the update will be a no-op and the
@@ -659,7 +661,7 @@ class VespaIndex(DocumentIndex):
         self,
         doc_id: str,
         *,
-        tenant_id: str,
+        tenant_id: str | None,
         chunk_count: int | None,
     ) -> int:
         total_chunks_deleted = 0

backend/onyx/file_store/utils.py

@@ -8,7 +8,7 @@ import requests
 from sqlalchemy.orm import Session
 
 from onyx.configs.constants import FileOrigin
-from onyx.db.engine import get_session_with_current_tenant
+from onyx.db.engine import get_session_with_tenant
 from onyx.db.models import ChatMessage
 from onyx.file_store.file_store import get_default_file_store
 from onyx.file_store.models import FileDescriptor
@@ -53,11 +53,11 @@ def load_all_chat_files(
     return files
 
 
-def save_file_from_url(url: str) -> str:
+def save_file_from_url(url: str, tenant_id: str) -> str:
     """NOTE: using multiple sessions here, since this is often called
     using multithreading. In practice, sharing a session has resulted in
     weird errors."""
-    with get_session_with_current_tenant() as db_session:
+    with get_session_with_tenant(tenant_id=tenant_id) as db_session:
         response = requests.get(url)
         response.raise_for_status()
 
@@ -75,8 +75,8 @@ def save_file_from_url(url: str) -> str:
         return unique_id
 
 
-def save_file_from_base64(base64_string: str) -> str:
-    with get_session_with_current_tenant() as db_session:
+def save_file_from_base64(base64_string: str, tenant_id: str) -> str:
+    with get_session_with_tenant(tenant_id=tenant_id) as db_session:
         unique_id = str(uuid4())
         file_store = get_default_file_store(db_session)
         file_store.save_file(
@@ -90,12 +90,14 @@ def save_file_from_base64(base64_string: str) -> str:
 
 
 def save_file(
+    tenant_id: str,
     url: str | None = None,
     base64_data: str | None = None,
 ) -> str:
     """Save a file from either a URL or base64 encoded string.
 
     Args:
+        tenant_id: The tenant ID to save the file under
         url: URL to download file from
         base64_data: Base64 encoded file data
 
@@ -109,22 +111,22 @@ def save_file(
         raise ValueError("Cannot specify both url and base64_data")
 
     if url is not None:
-        return save_file_from_url(url)
+        return save_file_from_url(url, tenant_id)
     elif base64_data is not None:
-        return save_file_from_base64(base64_data)
+        return save_file_from_base64(base64_data, tenant_id)
     else:
         raise ValueError("Must specify either url or base64_data")
 
 
-def save_files(urls: list[str], base64_files: list[str]) -> list[str]:
+def save_files(urls: list[str], base64_files: list[str], tenant_id: str) -> list[str]:
     # NOTE: be explicit about typing so that if we change things, we get notified
     funcs: list[
         tuple[
-            Callable[[str | None, str | None], str],
-            tuple[str | None, str | None],
+            Callable[[str, str | None, str | None], str],
+            tuple[str, str | None, str | None],
         ]
-    ] = [(save_file, (url, None)) for url in urls] + [
-        (save_file, (None, base64_file)) for base64_file in base64_files
+    ] = [(save_file, (tenant_id, url, None)) for url in urls] + [
+        (save_file, (tenant_id, None, base64_file)) for base64_file in base64_files
     ]
 
     return run_functions_tuples_in_parallel(funcs)

backend/onyx/indexing/indexing_pipeline.py

@@ -158,8 +158,8 @@ def index_doc_batch_with_handler(
     document_batch: list[Document],
     index_attempt_metadata: IndexAttemptMetadata,
     db_session: Session,
-    tenant_id: str,
     ignore_time_skip: bool = False,
+    tenant_id: str | None = None,
 ) -> IndexingPipelineResult:
     try:
         index_pipeline_result = index_doc_batch(
@@ -173,10 +173,7 @@ def index_doc_batch_with_handler(
             tenant_id=tenant_id,
         )
     except Exception as e:
-        # don't log the batch directly, it's too much text
-        document_ids = [doc.id for doc in document_batch]
-        logger.exception(f"Failed to index document batch: {document_ids}")
-
+        logger.exception(f"Failed to index document batch: {document_batch}")
         index_pipeline_result = IndexingPipelineResult(
             new_docs=0,
             total_docs=len(document_batch),
@@ -317,8 +314,8 @@ def index_doc_batch(
     document_index: DocumentIndex,
     index_attempt_metadata: IndexAttemptMetadata,
     db_session: Session,
-    tenant_id: str,
     ignore_time_skip: bool = False,
+    tenant_id: str | None = None,
     filter_fnc: Callable[[list[Document]], list[Document]] = filter_documents,
 ) -> IndexingPipelineResult:
     """Takes different pieces of the indexing pipeline and applies it to a batch of documents
@@ -525,9 +522,9 @@ def build_indexing_pipeline(
     embedder: IndexingEmbedder,
     document_index: DocumentIndex,
     db_session: Session,
-    tenant_id: str,
     chunker: Chunker | None = None,
     ignore_time_skip: bool = False,
+    tenant_id: str | None = None,
     callback: IndexingHeartbeatInterface | None = None,
 ) -> IndexingPipelineProtocol:
     """Builds a pipeline which takes in a list (batch) of docs and indexes them."""

backend/onyx/indexing/models.py

@@ -84,7 +84,7 @@ class DocMetadataAwareIndexChunk(IndexChunk):
            negative -> ranked lower.
     """
 
-    tenant_id: str
+    tenant_id: str | None = None
     access: "DocumentAccess"
     document_sets: set[str]
     boost: int
@@ -96,7 +96,7 @@ class DocMetadataAwareIndexChunk(IndexChunk):
         access: "DocumentAccess",
         document_sets: set[str],
         boost: int,
-        tenant_id: str,
+        tenant_id: str | None,
     ) -> "DocMetadataAwareIndexChunk":
         index_chunk_data = index_chunk.model_dump()
         return cls(

backend/onyx/llm/llm_provider_options.py

@@ -103,7 +103,7 @@ def fetch_available_well_known_llms() -> list[WellKnownLLMProviderDescriptor]:
             api_version_required=False,
             custom_config_keys=[],
             llm_names=fetch_models_for_provider(ANTHROPIC_PROVIDER_NAME),
-            default_model="claude-3-7-sonnet-20250219",
+            default_model="claude-3-5-sonnet-20241022",
             default_fast_model="claude-3-5-sonnet-20241022",
         ),
         WellKnownLLMProviderDescriptor(

backend/onyx/main.py

@@ -219,7 +219,7 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
 
         # If we are multi-tenant, we need to only set up initial public tables
         with Session(engine) as db_session:
-            setup_onyx(db_session, POSTGRES_DEFAULT_SCHEMA)
+            setup_onyx(db_session, None)
     else:
         setup_multitenant_onyx()
 

backend/onyx/onyxbot/slack/blocks.py

@@ -410,7 +410,7 @@ def _build_qa_response_blocks(
 
 
 def _build_continue_in_web_ui_block(
-    tenant_id: str,
+    tenant_id: str | None,
     message_id: int | None,
 ) -> Block:
     if message_id is None:
@@ -482,7 +482,7 @@ def build_follow_up_resolved_blocks(
 
 def build_slack_response_blocks(
     answer: ChatOnyxBotResponse,
-    tenant_id: str,
+    tenant_id: str | None,
     message_info: SlackMessageInfo,
     channel_conf: ChannelConfig | None,
     use_citations: bool,

backend/onyx/onyxbot/slack/handlers/handle_buttons.py

@@ -151,7 +151,7 @@ def handle_slack_feedback(
     user_id_to_post_confirmation: str,
     channel_id_to_post_confirmation: str,
     thread_ts_to_post_confirmation: str,
-    tenant_id: str,
+    tenant_id: str | None,
 ) -> None:
     message_id, doc_id, doc_rank = decompose_action_id(feedback_id)
 

backend/onyx/onyxbot/slack/handlers/handle_message.py

@@ -109,7 +109,7 @@ def handle_message(
     slack_channel_config: SlackChannelConfig,
     client: WebClient,
     feedback_reminder_id: str | None,
-    tenant_id: str,
+    tenant_id: str | None,
 ) -> bool:
     """Potentially respond to the user message depending on filters and if an answer was generated
 

backend/onyx/onyxbot/slack/handlers/handle_regular_answer.py

@@ -72,7 +72,7 @@ def handle_regular_answer(
     channel: str,
     logger: OnyxLoggingAdapter,
     feedback_reminder_id: str | None,
-    tenant_id: str,
+    tenant_id: str | None,
     num_retries: int = DANSWER_BOT_NUM_RETRIES,
     thread_context_percent: float = MAX_THREAD_CONTEXT_PERCENTAGE,
     should_respond_with_error_msgs: bool = DANSWER_BOT_DISPLAY_ERROR_MSGS,

backend/onyx/onyxbot/slack/listener.py

@@ -17,12 +17,10 @@ from prometheus_client import Gauge
 from prometheus_client import start_http_server
 from redis.lock import Lock
 from slack_sdk import WebClient
-from slack_sdk.errors import SlackApiError
 from slack_sdk.socket_mode.request import SocketModeRequest
 from slack_sdk.socket_mode.response import SocketModeResponse
 from sqlalchemy.orm import Session
 
-from ee.onyx.server.tenants.product_gating import get_gated_tenants
 from onyx.chat.models import ThreadMessage
 from onyx.configs.app_configs import DEV_MODE
 from onyx.configs.app_configs import POD_NAME
@@ -123,13 +121,13 @@ _OFFICIAL_SLACKBOT_USER_ID = "USLACKBOT"
 class SlackbotHandler:
     def __init__(self) -> None:
         logger.info("Initializing SlackbotHandler")
-        self.tenant_ids: Set[str] = set()
+        self.tenant_ids: Set[str | None] = set()
         # The keys for these dictionaries are tuples of (tenant_id, slack_bot_id)
-        self.socket_clients: Dict[tuple[str, int], TenantSocketModeClient] = {}
-        self.slack_bot_tokens: Dict[tuple[str, int], SlackBotTokens] = {}
+        self.socket_clients: Dict[tuple[str | None, int], TenantSocketModeClient] = {}
+        self.slack_bot_tokens: Dict[tuple[str | None, int], SlackBotTokens] = {}
 
         # Store Redis lock objects here so we can release them properly
-        self.redis_locks: Dict[str, Lock] = {}
+        self.redis_locks: Dict[str | None, Lock] = {}
 
         self.running = True
         self.pod_id = self.get_pod_id()
@@ -193,7 +191,7 @@ class SlackbotHandler:
             self._shutdown_event.wait(timeout=TENANT_HEARTBEAT_INTERVAL)
 
     def _manage_clients_per_tenant(
-        self, db_session: Session, tenant_id: str, bot: SlackBot
+        self, db_session: Session, tenant_id: str | None, bot: SlackBot
     ) -> None:
         """
         - If the tokens are missing or empty, close the socket client and remove them.
@@ -251,12 +249,7 @@ class SlackbotHandler:
         - If yes, store them in self.tenant_ids and manage the socket connections.
         - If a tenant in self.tenant_ids no longer has Slack bots, remove it (and release the lock in this scope).
         """
-
-        all_tenants = [
-            tenant_id
-            for tenant_id in get_all_tenant_ids()
-            if tenant_id not in get_gated_tenants()
-        ]
+        all_tenants = get_all_tenant_ids()
 
         token: Token[str | None]
 
@@ -385,7 +378,7 @@ class SlackbotHandler:
             finally:
                 CURRENT_TENANT_ID_CONTEXTVAR.reset(token)
 
-    def _remove_tenant(self, tenant_id: str) -> None:
+    def _remove_tenant(self, tenant_id: str | None) -> None:
         """
         Helper to remove a tenant from `self.tenant_ids` and close any socket clients.
         (Lock release now happens in `acquire_tenants()`, not here.)
@@ -415,7 +408,7 @@ class SlackbotHandler:
             )
 
     def start_socket_client(
-        self, slack_bot_id: int, tenant_id: str, slack_bot_tokens: SlackBotTokens
+        self, slack_bot_id: int, tenant_id: str | None, slack_bot_tokens: SlackBotTokens
     ) -> None:
         socket_client: TenantSocketModeClient = _get_socket_client(
             slack_bot_tokens, tenant_id, slack_bot_id
@@ -423,7 +416,6 @@ class SlackbotHandler:
 
         try:
             bot_info = socket_client.web_client.auth_test()
-
             if bot_info["ok"]:
                 bot_user_id = bot_info["user_id"]
                 user_info = socket_client.web_client.users_info(user=bot_user_id)
@@ -434,23 +426,9 @@ class SlackbotHandler:
                     logger.info(
                         f"Started socket client for Slackbot with name '{bot_name}' (tenant: {tenant_id}, app: {slack_bot_id})"
                     )
-        except SlackApiError as e:
-            # Only error out if we get a not_authed error
-            if "not_authed" in str(e):
-                self.tenant_ids.add(tenant_id)
-                logger.error(
-                    f"Authentication error: Invalid or expired credentials for tenant: {tenant_id}, app: {slack_bot_id}. "
-                    "Error: {e}"
-                )
-                return
-            # Log other Slack API errors but continue
-            logger.error(
-                f"Slack API error fetching bot info: {e} for tenant: {tenant_id}, app: {slack_bot_id}"
-            )
         except Exception as e:
-            # Log other exceptions but continue
-            logger.error(
-                f"Error fetching bot info: {e} for tenant: {tenant_id}, app: {slack_bot_id}"
+            logger.warning(
+                f"Could not fetch bot name: {e} for tenant: {tenant_id}, app: {slack_bot_id}"
             )
 
         # Append the event handler
@@ -912,7 +890,7 @@ def create_process_slack_event() -> (
 
 
 def _get_socket_client(
-    slack_bot_tokens: SlackBotTokens, tenant_id: str, slack_bot_id: int
+    slack_bot_tokens: SlackBotTokens, tenant_id: str | None, slack_bot_id: int
 ) -> TenantSocketModeClient:
     # For more info on how to set this up, checkout the docs:
     # https://docs.onyx.app/slack_bot_setup

backend/onyx/onyxbot/slack/utils.py

@@ -570,7 +570,7 @@ def read_slack_thread(
 
 
 def slack_usage_report(
-    action: str, sender_id: str | None, client: WebClient, tenant_id: str
+    action: str, sender_id: str | None, client: WebClient, tenant_id: str | None
 ) -> None:
     if DISABLE_TELEMETRY:
         return
@@ -663,7 +663,9 @@ def get_feedback_visibility() -> FeedbackVisibility:
 
 
 class TenantSocketModeClient(SocketModeClient):
-    def __init__(self, tenant_id: str, slack_bot_id: int, *args: Any, **kwargs: Any):
+    def __init__(
+        self, tenant_id: str | None, slack_bot_id: int, *args: Any, **kwargs: Any
+    ):
         super().__init__(*args, **kwargs)
         self.tenant_id = tenant_id
         self.slack_bot_id = slack_bot_id

backend/onyx/prompts/chat_prompts.py

@@ -18,7 +18,6 @@ Remember to provide inline citations in the format [1], [2], [3], etc.
 
 ADDITIONAL_INFO = "\n\nAdditional Information:\n\t- {datetime_info}."
 
-CODE_BLOCK_MARKDOWN = "Formatting re-enabled. "
 
 CHAT_USER_PROMPT = f"""
 Refer to the following context documents when responding to me.{{optional_ignore_statement}}

backend/onyx/redis/redis_connector.py

@@ -16,10 +16,10 @@ class RedisConnector:
     """Composes several classes to simplify interacting with a connector and its
     associated background tasks / associated redis interactions."""
 
-    def __init__(self, tenant_id: str, id: int) -> None:
+    def __init__(self, tenant_id: str | None, id: int) -> None:
         """id: a connector credential pair id"""
 
-        self.tenant_id: str = tenant_id
+        self.tenant_id: str | None = tenant_id
         self.id: int = id
         self.redis: redis.Redis = get_redis_client(tenant_id=tenant_id)
 

backend/onyx/redis/redis_connector_credential_pair.py

@@ -31,7 +31,7 @@ class RedisConnectorCredentialPair(RedisObjectHelper):
     PREFIX = "connectorsync"
     TASKSET_PREFIX = PREFIX + "_taskset"
 
-    def __init__(self, tenant_id: str, id: int) -> None:
+    def __init__(self, tenant_id: str | None, id: int) -> None:
         super().__init__(tenant_id, str(id))
 
         # documents that should be skipped
@@ -60,7 +60,7 @@ class RedisConnectorCredentialPair(RedisObjectHelper):
         db_session: Session,
         redis_client: Redis,
         lock: RedisLock,
-        tenant_id: str,
+        tenant_id: str | None,
     ) -> tuple[int, int] | None:
         """We can limit the number of tasks generated here, which is useful to prevent
         one tenant from overwhelming the sync queue.

backend/onyx/redis/redis_connector_delete.py

@@ -33,22 +33,14 @@ class RedisConnectorDelete:
     FENCE_PREFIX = f"{PREFIX}_fence"  # "connectordeletion_fence"
     TASKSET_PREFIX = f"{PREFIX}_taskset"  # "connectordeletion_taskset"
 
-    # used to signal the overall workflow is still active
-    # it's impossible to get the exact state of the system at a single point in time
-    # so we need a signal with a TTL to bridge gaps in our checks
-    ACTIVE_PREFIX = PREFIX + "_active"
-    ACTIVE_TTL = 3600
-
-    def __init__(self, tenant_id: str, id: int, redis: redis.Redis) -> None:
-        self.tenant_id: str = tenant_id
+    def __init__(self, tenant_id: str | None, id: int, redis: redis.Redis) -> None:
+        self.tenant_id: str | None = tenant_id
         self.id = id
         self.redis = redis
 
         self.fence_key: str = f"{self.FENCE_PREFIX}_{id}"
         self.taskset_key = f"{self.TASKSET_PREFIX}_{id}"
 
-        self.active_key = f"{self.ACTIVE_PREFIX}_{id}"
-
     def taskset_clear(self) -> None:
         self.redis.delete(self.taskset_key)
 
@@ -85,20 +77,6 @@ class RedisConnectorDelete:
         self.redis.set(self.fence_key, payload.model_dump_json())
         self.redis.sadd(OnyxRedisConstants.ACTIVE_FENCES, self.fence_key)
 
-    def set_active(self) -> None:
-        """This sets a signal to keep the permissioning flow from getting cleaned up within
-        the expiration time.
-
-        The slack in timing is needed to avoid race conditions where simply checking
-        the celery queue and task status could result in race conditions."""
-        self.redis.set(self.active_key, 0, ex=self.ACTIVE_TTL)
-
-    def active(self) -> bool:
-        if self.redis.exists(self.active_key):
-            return True
-
-        return False
-
     def _generate_task_id(self) -> str:
         # celery's default task id format is "dd32ded3-00aa-4884-8b21-42f8332e7fac"
         # we prefix the task id so it's easier to keep track of who created the task
@@ -163,7 +141,6 @@ class RedisConnectorDelete:
 
     def reset(self) -> None:
         self.redis.srem(OnyxRedisConstants.ACTIVE_FENCES, self.fence_key)
-        self.redis.delete(self.active_key)
         self.redis.delete(self.taskset_key)
         self.redis.delete(self.fence_key)
 
@@ -176,9 +153,6 @@ class RedisConnectorDelete:
     @staticmethod
     def reset_all(r: redis.Redis) -> None:
         """Deletes all redis values for all connectors"""
-        for key in r.scan_iter(RedisConnectorDelete.ACTIVE_PREFIX + "*"):
-            r.delete(key)
-
         for key in r.scan_iter(RedisConnectorDelete.TASKSET_PREFIX + "*"):
             r.delete(key)
 

backend/onyx/redis/redis_connector_doc_perm_sync.py

@@ -52,8 +52,8 @@ class RedisConnectorPermissionSync:
     ACTIVE_PREFIX = PREFIX + "_active"
     ACTIVE_TTL = CELERY_PERMISSIONS_SYNC_LOCK_TIMEOUT * 2
 
-    def __init__(self, tenant_id: str, id: int, redis: redis.Redis) -> None:
-        self.tenant_id: str = tenant_id
+    def __init__(self, tenant_id: str | None, id: int, redis: redis.Redis) -> None:
+        self.tenant_id: str | None = tenant_id
         self.id = id
         self.redis = redis
 

backend/onyx/redis/redis_connector_ext_group_sync.py

@@ -44,8 +44,8 @@ class RedisConnectorExternalGroupSync:
     ACTIVE_PREFIX = PREFIX + "_active"
     ACTIVE_TTL = 3600
 
-    def __init__(self, tenant_id: str, id: int, redis: redis.Redis) -> None:
-        self.tenant_id: str = tenant_id
+    def __init__(self, tenant_id: str | None, id: int, redis: redis.Redis) -> None:
+        self.tenant_id: str | None = tenant_id
         self.id = id
         self.redis = redis
 

backend/onyx/redis/redis_connector_index.py

@@ -52,12 +52,12 @@ class RedisConnectorIndex:
 
     def __init__(
         self,
-        tenant_id: str,
+        tenant_id: str | None,
         id: int,
         search_settings_id: int,
         redis: redis.Redis,
     ) -> None:
-        self.tenant_id: str = tenant_id
+        self.tenant_id: str | None = tenant_id
         self.id = id
         self.search_settings_id = search_settings_id
         self.redis = redis
@@ -93,7 +93,10 @@ class RedisConnectorIndex:
 
     @property
     def fenced(self) -> bool:
-        return bool(self.redis.exists(self.fence_key))
+        if self.redis.exists(self.fence_key):
+            return True
+
+        return False
 
     @property
     def payload(self) -> RedisConnectorIndexPayload | None:
@@ -103,7 +106,9 @@ class RedisConnectorIndex:
             return None
 
         fence_str = fence_bytes.decode("utf-8")
-        return RedisConnectorIndexPayload.model_validate_json(cast(str, fence_str))
+        payload = RedisConnectorIndexPayload.model_validate_json(cast(str, fence_str))
+
+        return payload
 
     def set_fence(
         self,
@@ -118,7 +123,10 @@ class RedisConnectorIndex:
         self.redis.sadd(OnyxRedisConstants.ACTIVE_FENCES, self.fence_key)
 
     def terminating(self, celery_task_id: str) -> bool:
-        return bool(self.redis.exists(f"{self.terminate_key}_{celery_task_id}"))
+        if self.redis.exists(f"{self.terminate_key}_{celery_task_id}"):
+            return True
+
+        return False
 
     def set_terminate(self, celery_task_id: str) -> None:
         """This sets a signal. It does not block!"""
@@ -138,7 +146,10 @@ class RedisConnectorIndex:
 
     def watchdog_signaled(self) -> bool:
         """Check the state of the watchdog."""
-        return bool(self.redis.exists(self.watchdog_key))
+        if self.redis.exists(self.watchdog_key):
+            return True
+
+        return False
 
     def set_active(self) -> None:
         """This sets a signal to keep the indexing flow from getting cleaned up within
@@ -149,7 +160,10 @@ class RedisConnectorIndex:
         self.redis.set(self.active_key, 0, ex=self.ACTIVE_TTL)
 
     def active(self) -> bool:
-        return bool(self.redis.exists(self.active_key))
+        if self.redis.exists(self.active_key):
+            return True
+
+        return False
 
     def set_connector_active(self) -> None:
         """This sets a signal to keep the indexing flow from getting cleaned up within
@@ -166,7 +180,10 @@ class RedisConnectorIndex:
         return False
 
     def generator_locked(self) -> bool:
-        return bool(self.redis.exists(self.generator_lock_key))
+        if self.redis.exists(self.generator_lock_key):
+            return True
+
+        return False
 
     def set_generator_complete(self, payload: int | None) -> None:
         if not payload:

backend/onyx/redis/redis_connector_prune.py

@@ -52,8 +52,8 @@ class RedisConnectorPrune:
     ACTIVE_PREFIX = PREFIX + "_active"
     ACTIVE_TTL = CELERY_PRUNING_LOCK_TIMEOUT * 2
 
-    def __init__(self, tenant_id: str, id: int, redis: redis.Redis) -> None:
-        self.tenant_id: str = tenant_id
+    def __init__(self, tenant_id: str | None, id: int, redis: redis.Redis) -> None:
+        self.tenant_id: str | None = tenant_id
         self.id = id
         self.redis = redis
 

backend/onyx/redis/redis_connector_stop.py

@@ -5,21 +5,14 @@ class RedisConnectorStop:
     """Manages interactions with redis for stop signaling. Should only be accessed
     through RedisConnector."""
 
-    PREFIX = "connectorstop"
-    FENCE_PREFIX = f"{PREFIX}_fence"
+    FENCE_PREFIX = "connectorstop_fence"
 
-    # if this timeout is exceeded, the caller may decide to take more
-    # drastic measures
-    TIMEOUT_PREFIX = f"{PREFIX}_timeout"
-    TIMEOUT_TTL = 300
-
-    def __init__(self, tenant_id: str, id: int, redis: redis.Redis) -> None:
-        self.tenant_id: str = tenant_id
+    def __init__(self, tenant_id: str | None, id: int, redis: redis.Redis) -> None:
+        self.tenant_id: str | None = tenant_id
         self.id: int = id
         self.redis = redis
 
         self.fence_key: str = f"{self.FENCE_PREFIX}_{id}"
-        self.timeout_key: str = f"{self.TIMEOUT_PREFIX}_{id}"
 
     @property
     def fenced(self) -> bool:
@@ -35,22 +28,7 @@ class RedisConnectorStop:
 
         self.redis.set(self.fence_key, 0)
 
-    @property
-    def timed_out(self) -> bool:
-        if self.redis.exists(self.timeout_key):
-            return False
-
-        return True
-
-    def set_timeout(self) -> None:
-        """After calling this, call timed_out to determine if the timeout has been
-        exceeded."""
-        self.redis.set(f"{self.timeout_key}", 0, ex=self.TIMEOUT_TTL)
-
     @staticmethod
     def reset_all(r: redis.Redis) -> None:
         for key in r.scan_iter(RedisConnectorStop.FENCE_PREFIX + "*"):
             r.delete(key)
-
-        for key in r.scan_iter(RedisConnectorStop.TIMEOUT_PREFIX + "*"):
-            r.delete(key)

backend/onyx/redis/redis_document_set.py

@@ -23,7 +23,7 @@ class RedisDocumentSet(RedisObjectHelper):
     FENCE_PREFIX = PREFIX + "_fence"
     TASKSET_PREFIX = PREFIX + "_taskset"
 
-    def __init__(self, tenant_id: str, id: int) -> None:
+    def __init__(self, tenant_id: str | None, id: int) -> None:
         super().__init__(tenant_id, str(id))
 
     @property
@@ -58,7 +58,7 @@ class RedisDocumentSet(RedisObjectHelper):
         db_session: Session,
         redis_client: Redis,
         lock: RedisLock,
-        tenant_id: str,
+        tenant_id: str | None,
     ) -> tuple[int, int] | None:
         """Max tasks is ignored for now until we can build the logic to mark the
         document set up to date over multiple batches.

backend/onyx/redis/redis_object_helper.py

@@ -14,8 +14,8 @@ class RedisObjectHelper(ABC):
     FENCE_PREFIX = PREFIX + "_fence"
     TASKSET_PREFIX = PREFIX + "_taskset"
 
-    def __init__(self, tenant_id: str, id: str):
-        self._tenant_id: str = tenant_id
+    def __init__(self, tenant_id: str | None, id: str):
+        self._tenant_id: str | None = tenant_id
         self._id: str = id
         self.redis = get_redis_client(tenant_id=tenant_id)
 
@@ -87,7 +87,7 @@ class RedisObjectHelper(ABC):
         db_session: Session,
         redis_client: Redis,
         lock: RedisLock,
-        tenant_id: str,
+        tenant_id: str | None,
     ) -> tuple[int, int] | None:
         """First element should be the number of actual tasks generated, second should
         be the number of docs that were candidates to be synced for the cc pair.

backend/onyx/redis/redis_usergroup.py

@@ -24,7 +24,7 @@ class RedisUserGroup(RedisObjectHelper):
     FENCE_PREFIX = PREFIX + "_fence"
     TASKSET_PREFIX = PREFIX + "_taskset"
 
-    def __init__(self, tenant_id: str, id: int) -> None:
+    def __init__(self, tenant_id: str | None, id: int) -> None:
         super().__init__(tenant_id, str(id))
 
     @property
@@ -59,7 +59,7 @@ class RedisUserGroup(RedisObjectHelper):
         db_session: Session,
         redis_client: Redis,
         lock: RedisLock,
-        tenant_id: str,
+        tenant_id: str | None,
     ) -> tuple[int, int] | None:
         """Max tasks is ignored for now until we can build the logic to mark the
         user group up to date over multiple batches.

backend/onyx/seeding/load_docs.py

@@ -37,15 +37,13 @@ from onyx.key_value_store.interface import KvKeyNotFoundError
 from onyx.server.documents.models import ConnectorBase
 from onyx.utils.logger import setup_logger
 from onyx.utils.variable_functionality import fetch_versioned_implementation
-from shared_configs.configs import MULTI_TENANT
-from shared_configs.configs import POSTGRES_DEFAULT_SCHEMA
 
 logger = setup_logger()
 
 
 def _create_indexable_chunks(
     preprocessed_docs: list[dict],
-    tenant_id: str,
+    tenant_id: str | None,
 ) -> tuple[list[Document], list[DocMetadataAwareIndexChunk]]:
     ids_to_documents = {}
     chunks = []
@@ -88,7 +86,7 @@ def _create_indexable_chunks(
                 mini_chunk_embeddings=[],
             ),
             title_embedding=preprocessed_doc["title_embedding"],
-            tenant_id=tenant_id if MULTI_TENANT else POSTGRES_DEFAULT_SCHEMA,
+            tenant_id=tenant_id,
             access=default_public_access,
             document_sets=set(),
             boost=DEFAULT_BOOST,
@@ -113,7 +111,7 @@ def load_processed_docs(cohere_enabled: bool) -> list[dict]:
 
 
 def seed_initial_documents(
-    db_session: Session, tenant_id: str, cohere_enabled: bool = False
+    db_session: Session, tenant_id: str | None, cohere_enabled: bool = False
 ) -> None:
     """
     Seed initial documents so users don't have an empty index to start

backend/onyx/server/documents/cc_pair.py

@@ -25,8 +25,8 @@ from onyx.background.celery.versioned_apps.primary import app as primary_app
 from onyx.background.indexing.models import IndexAttemptErrorPydantic
 from onyx.configs.constants import OnyxCeleryPriority
 from onyx.configs.constants import OnyxCeleryTask
-from onyx.connectors.exceptions import ValidationError
 from onyx.connectors.factory import validate_ccpair_for_user
+from onyx.connectors.interfaces import ConnectorValidationError
 from onyx.db.connector import delete_connector
 from onyx.db.connector_credential_pair import add_credential_to_connector
 from onyx.db.connector_credential_pair import (
@@ -123,15 +123,15 @@ def get_cc_pair_full_info(
     )
     is_editable_for_current_user = editable_cc_pair is not None
 
+    cc_pair_identifier = ConnectorCredentialPairIdentifier(
+        connector_id=cc_pair.connector_id,
+        credential_id=cc_pair.credential_id,
+    )
+
     document_count_info_list = list(
         get_document_counts_for_cc_pairs(
             db_session=db_session,
-            cc_pairs=[
-                ConnectorCredentialPairIdentifier(
-                    connector_id=cc_pair.connector_id,
-                    credential_id=cc_pair.credential_id,
-                )
-            ],
+            cc_pair_identifiers=[cc_pair_identifier],
         )
     )
     documents_indexed = (
@@ -620,7 +620,9 @@ def associate_credential_to_connector(
     )
 
     try:
-        validate_ccpair_for_user(connector_id, credential_id, db_session)
+        validate_ccpair_for_user(
+            connector_id, credential_id, db_session, user, tenant_id
+        )
 
         response = add_credential_to_connector(
             db_session=db_session,
@@ -647,7 +649,7 @@ def associate_credential_to_connector(
 
         return response
 
-    except ValidationError as e:
+    except ConnectorValidationError as e:
         # If validation fails, delete the connector and commit the changes
         # Ensures we don't leave invalid connectors in the database
         # NOTE: consensus is that it makes sense to unify connector and ccpair creation flows
@@ -658,6 +660,7 @@ def associate_credential_to_connector(
         raise HTTPException(
             status_code=400, detail="Connector validation error: " + str(e)
         )
+
     except IntegrityError as e:
         logger.error(f"IntegrityError: {e}")
         raise HTTPException(status_code=400, detail="Name must be unique")

backend/onyx/server/documents/connector.py

@@ -28,7 +28,6 @@ from onyx.configs.constants import FileOrigin
 from onyx.configs.constants import MilestoneRecordType
 from onyx.configs.constants import OnyxCeleryPriority
 from onyx.configs.constants import OnyxCeleryTask
-from onyx.connectors.exceptions import ConnectorValidationError
 from onyx.connectors.factory import validate_ccpair_for_user
 from onyx.connectors.google_utils.google_auth import (
     get_google_oauth_creds,
@@ -63,6 +62,7 @@ from onyx.connectors.google_utils.shared_constants import DB_CREDENTIALS_DICT_TO
 from onyx.connectors.google_utils.shared_constants import (
     GoogleOAuthAuthenticationMethod,
 )
+from onyx.connectors.interfaces import ConnectorValidationError
 from onyx.db.connector import create_connector
 from onyx.db.connector import delete_connector
 from onyx.db.connector import fetch_connector_by_id
@@ -72,31 +72,25 @@ from onyx.db.connector import mark_ccpair_with_indexing_trigger
 from onyx.db.connector import update_connector
 from onyx.db.connector_credential_pair import add_credential_to_connector
 from onyx.db.connector_credential_pair import get_cc_pair_groups_for_ids
-from onyx.db.connector_credential_pair import get_cc_pair_groups_for_ids_parallel
 from onyx.db.connector_credential_pair import get_connector_credential_pair
 from onyx.db.connector_credential_pair import get_connector_credential_pairs_for_user
-from onyx.db.connector_credential_pair import (
-    get_connector_credential_pairs_for_user_parallel,
-)
 from onyx.db.credentials import cleanup_gmail_credentials
 from onyx.db.credentials import cleanup_google_drive_credentials
 from onyx.db.credentials import create_credential
 from onyx.db.credentials import delete_service_account_credentials
 from onyx.db.credentials import fetch_credential_by_id_for_user
 from onyx.db.deletion_attempt import check_deletion_attempt_is_allowed
-from onyx.db.document import get_document_counts_for_cc_pairs_parallel
-from onyx.db.engine import get_current_tenant_id
+from onyx.db.document import get_document_counts_for_cc_pairs
 from onyx.db.engine import get_session
 from onyx.db.enums import AccessType
 from onyx.db.enums import IndexingMode
 from onyx.db.index_attempt import get_index_attempts_for_cc_pair
+from onyx.db.index_attempt import get_latest_index_attempt_for_cc_pair_id
+from onyx.db.index_attempt import get_latest_index_attempts
 from onyx.db.index_attempt import get_latest_index_attempts_by_status
-from onyx.db.index_attempt import get_latest_index_attempts_parallel
-from onyx.db.models import ConnectorCredentialPair
-from onyx.db.models import IndexAttempt
 from onyx.db.models import IndexingStatus
+from onyx.db.models import SearchSettings
 from onyx.db.models import User
-from onyx.db.models import UserGroup__ConnectorCredentialPair
 from onyx.db.search_settings import get_current_search_settings
 from onyx.db.search_settings import get_secondary_search_settings
 from onyx.file_processing.extract_file_text import convert_docx_to_txt
@@ -125,8 +119,8 @@ from onyx.server.documents.models import RunConnectorRequest
 from onyx.server.models import StatusResponse
 from onyx.utils.logger import setup_logger
 from onyx.utils.telemetry import create_milestone_and_report
-from onyx.utils.threadpool_concurrency import run_functions_tuples_in_parallel
 from onyx.utils.variable_functionality import fetch_ee_implementation_or_noop
+from shared_configs.contextvars import get_current_tenant_id
 
 logger = setup_logger()
 
@@ -584,8 +578,6 @@ def get_connector_status(
     cc_pairs = get_connector_credential_pairs_for_user(
         db_session=db_session,
         user=user,
-        eager_load_connector=True,
-        eager_load_credential=True,
     )
 
     group_cc_pair_relationships = get_cc_pair_groups_for_ids(
@@ -640,35 +632,23 @@ def get_connector_indexing_status(
     # Additional checks are done to make sure the connector and credential still exist.
     # TODO: make this one query ... possibly eager load or wrap in a read transaction
     # to avoid the complexity of trying to error check throughout the function
-
-    # see https://stackoverflow.com/questions/75758327/
-    # sqlalchemy-method-connection-for-bind-is-already-in-progress
-    # for why we can't pass in the current db_session to these functions
-    (
-        cc_pairs,
-        latest_index_attempts,
-        latest_finished_index_attempts,
-    ) = run_functions_tuples_in_parallel(
-        [
-            (
-                # Gets the connector/credential pairs for the user
-                get_connector_credential_pairs_for_user_parallel,
-                (user, get_editable, None, True, True, True),
-            ),
-            (
-                # Gets the most recent index attempt for each connector/credential pair
-                get_latest_index_attempts_parallel,
-                (secondary_index, True, False),
-            ),
-            (
-                # Gets the most recent FINISHED index attempt for each connector/credential pair
-                get_latest_index_attempts_parallel,
-                (secondary_index, True, True),
-            ),
-        ]
+    cc_pairs = get_connector_credential_pairs_for_user(
+        db_session=db_session,
+        user=user,
+        get_editable=get_editable,
+    )
+
+    cc_pair_identifiers = [
+        ConnectorCredentialPairIdentifier(
+            connector_id=cc_pair.connector_id, credential_id=cc_pair.credential_id
+        )
+        for cc_pair in cc_pairs
+    ]
+
+    latest_index_attempts = get_latest_index_attempts(
+        secondary_index=secondary_index,
+        db_session=db_session,
     )
-    cc_pairs = cast(list[ConnectorCredentialPair], cc_pairs)
-    latest_index_attempts = cast(list[IndexAttempt], latest_index_attempts)
 
     cc_pair_to_latest_index_attempt = {
         (
@@ -678,60 +658,31 @@ def get_connector_indexing_status(
         for index_attempt in latest_index_attempts
     }
 
-    cc_pair_to_latest_finished_index_attempt = {
-        (
-            index_attempt.connector_credential_pair.connector_id,
-            index_attempt.connector_credential_pair.credential_id,
-        ): index_attempt
-        for index_attempt in latest_finished_index_attempts
-    }
-
-    document_count_info, group_cc_pair_relationships = run_functions_tuples_in_parallel(
-        [
-            (
-                get_document_counts_for_cc_pairs_parallel,
-                (
-                    [
-                        ConnectorCredentialPairIdentifier(
-                            connector_id=cc_pair.connector_id,
-                            credential_id=cc_pair.credential_id,
-                        )
-                        for cc_pair in cc_pairs
-                    ],
-                ),
-            ),
-            (
-                get_cc_pair_groups_for_ids_parallel,
-                ([cc_pair.id for cc_pair in cc_pairs],),
-            ),
-        ]
+    document_count_info = get_document_counts_for_cc_pairs(
+        db_session=db_session,
+        cc_pair_identifiers=cc_pair_identifiers,
     )
-    document_count_info = cast(list[tuple[int, int, int]], document_count_info)
-    group_cc_pair_relationships = cast(
-        list[UserGroup__ConnectorCredentialPair], group_cc_pair_relationships
-    )
-
     cc_pair_to_document_cnt = {
         (connector_id, credential_id): cnt
         for connector_id, credential_id, cnt in document_count_info
     }
 
+    group_cc_pair_relationships = get_cc_pair_groups_for_ids(
+        db_session=db_session,
+        cc_pair_ids=[cc_pair.id for cc_pair in cc_pairs],
+    )
     group_cc_pair_relationships_dict: dict[int, list[int]] = {}
     for relationship in group_cc_pair_relationships:
         group_cc_pair_relationships_dict.setdefault(relationship.cc_pair_id, []).append(
             relationship.user_group_id
         )
 
-    connector_to_cc_pair_ids: dict[int, list[int]] = {}
-    for cc_pair in cc_pairs:
-        connector_to_cc_pair_ids.setdefault(cc_pair.connector_id, []).append(cc_pair.id)
+    search_settings: SearchSettings | None = None
+    if not secondary_index:
+        search_settings = get_current_search_settings(db_session)
+    else:
+        search_settings = get_secondary_search_settings(db_session)
 
-    get_search_settings = (
-        get_secondary_search_settings
-        if secondary_index
-        else get_current_search_settings
-    )
-    search_settings = get_search_settings(db_session)
     for cc_pair in cc_pairs:
         # TODO remove this to enable ingestion API
         if cc_pair.name == "DefaultCCPair":
@@ -754,8 +705,11 @@ def get_connector_indexing_status(
             (connector.id, credential.id)
         )
 
-        latest_finished_attempt = cc_pair_to_latest_finished_index_attempt.get(
-            (connector.id, credential.id)
+        latest_finished_attempt = get_latest_index_attempt_for_cc_pair_id(
+            db_session=db_session,
+            connector_credential_pair_id=cc_pair.id,
+            secondary_index=secondary_index,
+            only_finished=True,
         )
 
         indexing_statuses.append(
@@ -764,9 +718,7 @@ def get_connector_indexing_status(
                 name=cc_pair.name,
                 in_progress=in_progress,
                 cc_pair_status=cc_pair.status,
-                connector=ConnectorSnapshot.from_connector_db_model(
-                    connector, connector_to_cc_pair_ids.get(connector.id, [])
-                ),
+                connector=ConnectorSnapshot.from_connector_db_model(connector),
                 credential=CredentialSnapshot.from_credential_db_model(credential),
                 access_type=cc_pair.access_type,
                 owner=credential.user.email if credential.user else "",
@@ -902,6 +854,8 @@ def create_connector_with_mock_credential(
             connector_id=connector_id,
             credential_id=credential_id,
             db_session=db_session,
+            user=user,
+            tenant_id=tenant_id,
         )
         response = add_credential_to_connector(
             db_session=db_session,

backend/onyx/server/documents/credential.py

@@ -18,6 +18,7 @@ from onyx.db.credentials import fetch_credentials_by_source_for_user
 from onyx.db.credentials import fetch_credentials_for_user
 from onyx.db.credentials import swap_credentials_connector
 from onyx.db.credentials import update_credential
+from onyx.db.engine import get_current_tenant_id
 from onyx.db.engine import get_session
 from onyx.db.models import DocumentSource
 from onyx.db.models import User
@@ -99,11 +100,14 @@ def swap_credentials_for_connector(
     credential_swap_req: CredentialSwapRequest,
     user: User | None = Depends(current_user),
     db_session: Session = Depends(get_session),
+    tenant_id: str | None = Depends(get_current_tenant_id),
 ) -> StatusResponse:
     validate_ccpair_for_user(
         credential_swap_req.connector_id,
         credential_swap_req.new_credential_id,
         db_session,
+        user,
+        tenant_id,
     )
 
     connector_credential_pair = swap_credentials_connector(

backend/onyx/server/documents/models.py

@@ -83,9 +83,7 @@ class ConnectorSnapshot(ConnectorBase):
     source: DocumentSource
 
     @classmethod
-    def from_connector_db_model(
-        cls, connector: Connector, credential_ids: list[int] | None = None
-    ) -> "ConnectorSnapshot":
+    def from_connector_db_model(cls, connector: Connector) -> "ConnectorSnapshot":
         return ConnectorSnapshot(
             id=connector.id,
             name=connector.name,
@@ -94,10 +92,9 @@ class ConnectorSnapshot(ConnectorBase):
             connector_specific_config=connector.connector_specific_config,
             refresh_freq=connector.refresh_freq,
             prune_freq=connector.prune_freq,
-            credential_ids=(
-                credential_ids
-                or [association.credential.id for association in connector.credentials]
-            ),
+            credential_ids=[
+                association.credential.id for association in connector.credentials
+            ],
             indexing_start=connector.indexing_start,
             time_created=connector.time_created,
             time_updated=connector.time_updated,

backend/onyx/server/features/folder/api.py

@@ -49,7 +49,6 @@ def get_folders(
                         name=chat_session.description,
                         persona_id=chat_session.persona_id,
                         time_created=chat_session.time_created.isoformat(),
-                        time_updated=chat_session.time_updated.isoformat(),
                         shared_status=chat_session.shared_status,
                         folder_id=folder.id,
                     )

backend/onyx/server/manage/slack_bot.py

@@ -1,3 +1,5 @@
+from typing import Any
+
 from fastapi import APIRouter
 from fastapi import Depends
 from fastapi import HTTPException
@@ -343,9 +345,6 @@ def list_bot_configs(
     ]
 
 
-MAX_CHANNELS = 200
-
-
 @router.get(
     "/admin/slack-app/bots/{bot_id}/channels",
 )
@@ -354,40 +353,38 @@ def get_all_channels_from_slack_api(
     db_session: Session = Depends(get_session),
     _: User | None = Depends(current_admin_user),
 ) -> list[SlackChannel]:
-    """
-    Fetches all channels from the Slack API.
-    If the workspace has 200 or more channels, we raise an error.
-    """
     tokens = fetch_slack_bot_tokens(db_session, bot_id)
     if not tokens or "bot_token" not in tokens:
         raise HTTPException(
             status_code=404, detail="Bot token not found for the given bot ID"
         )
 
-    client = WebClient(token=tokens["bot_token"])
+    bot_token = tokens["bot_token"]
+    client = WebClient(token=bot_token)
 
     try:
-        response = client.conversations_list(
-            types="public_channel,private_channel",
-            exclude_archived=True,
-            limit=MAX_CHANNELS,
-        )
-
-        channels = [
-            SlackChannel(id=channel["id"], name=channel["name"])
-            for channel in response["channels"]
-        ]
-
-        if len(channels) == MAX_CHANNELS:
-            raise HTTPException(
-                status_code=400,
-                detail=f"Workspace has {MAX_CHANNELS} or more channels.",
+        channels = []
+        cursor = None
+        while True:
+            response = client.conversations_list(
+                types="public_channel,private_channel",
+                exclude_archived=True,
+                limit=1000,
+                cursor=cursor,
             )
+            for channel in response["channels"]:
+                channels.append(SlackChannel(id=channel["id"], name=channel["name"]))
+
+            response_metadata: dict[str, Any] = response.get("response_metadata", {})
+            if isinstance(response_metadata, dict):
+                cursor = response_metadata.get("next_cursor")
+                if not cursor:
+                    break
+            else:
+                break
 
         return channels
-
     except SlackApiError as e:
         raise HTTPException(
-            status_code=500,
-            detail=f"Error fetching channels from Slack API: {str(e)}",
+            status_code=500, detail=f"Error fetching channels from Slack API: {str(e)}"
         )

backend/onyx/server/openai_assistants_api/threads_api.py

@@ -147,11 +147,9 @@ def list_threads(
                 name=chat.description,
                 persona_id=chat.persona_id,
                 time_created=chat.time_created.isoformat(),
-                time_updated=chat.time_updated.isoformat(),
                 shared_status=chat.shared_status,
                 folder_id=chat.folder_id,
                 current_alternate_model=chat.current_alternate_model,
-                current_temperature_override=chat.temperature_override,
             )
             for chat in chat_sessions
         ]