k

.github/workflows/pr-python-connector-tests.yml

@@ -74,9 +74,7 @@ jobs:
           python -m pip install --upgrade pip
           pip install --retries 5 --timeout 30 -r backend/requirements/default.txt
           pip install --retries 5 --timeout 30 -r backend/requirements/dev.txt
-          playwright install chromium
-          playwright install-deps chromium
-          
+
       - name: Run Tests
         shell: script -q -e -c "bash --noprofile --norc -eo pipefail {0}"
         run: py.test -o junit_family=xunit2 -xv --ff backend/tests/daily/connectors

backend/alembic/versions/acaab4ef4507_remove_inactive_ccpair_status_on_.py

@@ -1,29 +0,0 @@
-"""remove inactive ccpair status on downgrade
-
-Revision ID: acaab4ef4507
-Revises: b388730a2899
-Create Date: 2025-02-16 18:21:41.330212
-
-"""
-from alembic import op
-from onyx.db.models import ConnectorCredentialPair
-from onyx.db.enums import ConnectorCredentialPairStatus
-from sqlalchemy import update
-
-# revision identifiers, used by Alembic.
-revision = "acaab4ef4507"
-down_revision = "b388730a2899"
-branch_labels = None
-depends_on = None
-
-
-def upgrade() -> None:
-    pass
-
-
-def downgrade() -> None:
-    op.execute(
-        update(ConnectorCredentialPair)
-        .where(ConnectorCredentialPair.status == ConnectorCredentialPairStatus.INVALID)
-        .values(status=ConnectorCredentialPairStatus.ACTIVE)
-    )

backend/alembic/versions/f13db29f3101_add_composite_index_for_last_modified_.py

@@ -1,27 +0,0 @@
-"""Add composite index for last_modified and last_synced to document
-
-Revision ID: f13db29f3101
-Revises: b388730a2899
-Create Date: 2025-02-18 22:48:11.511389
-
-"""
-from alembic import op
-
-# revision identifiers, used by Alembic.
-revision = "f13db29f3101"
-down_revision = "acaab4ef4507"
-branch_labels: str | None = None
-depends_on: str | None = None
-
-
-def upgrade() -> None:
-    op.create_index(
-        "ix_document_sync_status",
-        "document",
-        ["last_modified", "last_synced"],
-        unique=False,
-    )
-
-
-def downgrade() -> None:
-    op.drop_index("ix_document_sync_status", table_name="document")

backend/onyx/auth/email_utils.py

@@ -10,7 +10,6 @@ from onyx.configs.app_configs import SMTP_PORT
 from onyx.configs.app_configs import SMTP_SERVER
 from onyx.configs.app_configs import SMTP_USER
 from onyx.configs.app_configs import WEB_DOMAIN
-from onyx.configs.constants import AuthType
 from onyx.configs.constants import TENANT_ID_COOKIE_NAME
 from onyx.db.models import User
 
@@ -188,51 +187,23 @@ def send_subscription_cancellation_email(user_email: str) -> None:
     send_email(user_email, subject, html_content, text_content)
 
 
-def send_user_email_invite(
-    user_email: str, current_user: User, auth_type: AuthType
-) -> None:
+def send_user_email_invite(user_email: str, current_user: User) -> None:
     subject = "Invitation to Join Onyx Organization"
     heading = "You've Been Invited!"
-
-    # the exact action taken by the user, and thus the message, depends on the auth type
-    message = f"<p>You have been invited by {current_user.email} to join an organization on Onyx.</p>"
-    if auth_type == AuthType.CLOUD:
-        message += (
-            "<p>To join the organization, please click the button below to set a password "
-            "or login with Google and complete your registration.</p>"
-        )
-    elif auth_type == AuthType.BASIC:
-        message += (
-            "<p>To join the organization, please click the button below to set a password "
-            "and complete your registration.</p>"
-        )
-    elif auth_type == AuthType.GOOGLE_OAUTH:
-        message += (
-            "<p>To join the organization, please click the button below to login with Google "
-            "and complete your registration.</p>"
-        )
-    elif auth_type == AuthType.OIDC or auth_type == AuthType.SAML:
-        message += (
-            "<p>To join the organization, please click the button below to"
-            " complete your registration.</p>"
-        )
-    else:
-        raise ValueError(f"Invalid auth type: {auth_type}")
-
+    message = (
+        f"<p>You have been invited by {current_user.email} to join an organization on Onyx.</p>"
+        "<p>To join the organization, please click the button below to set a password "
+        "or login with Google and complete your registration.</p>"
+    )
     cta_text = "Join Organization"
     cta_link = f"{WEB_DOMAIN}/auth/signup?email={user_email}"
     html_content = build_html_email(heading, message, cta_text, cta_link)
-
-    # text content is the fallback for clients that don't support HTML
-    # not as critical, so not having special cases for each auth type
     text_content = (
         f"You have been invited by {current_user.email} to join an organization on Onyx.\n"
         "To join the organization, please visit the following link:\n"
         f"{WEB_DOMAIN}/auth/signup?email={user_email}\n"
+        "You'll be asked to set a password or login with Google to complete your registration."
     )
-    if auth_type == AuthType.CLOUD:
-        text_content += "You'll be asked to set a password or login with Google to complete your registration."
-
     send_email(user_email, subject, html_content, text_content)
 
 

backend/onyx/auth/users.py

@@ -95,7 +95,6 @@ from onyx.db.models import User
 from onyx.db.users import get_user_by_email
 from onyx.redis.redis_pool import get_async_redis_connection
 from onyx.redis.redis_pool import get_redis_client
-from onyx.server.utils import BasicAuthenticationError
 from onyx.utils.logger import setup_logger
 from onyx.utils.telemetry import create_milestone_and_report
 from onyx.utils.telemetry import optional_telemetry
@@ -110,6 +109,11 @@ from shared_configs.contextvars import get_current_tenant_id
 logger = setup_logger()
 
 
+class BasicAuthenticationError(HTTPException):
+    def __init__(self, detail: str):
+        super().__init__(status_code=status.HTTP_403_FORBIDDEN, detail=detail)
+
+
 def is_user_admin(user: User | None) -> bool:
     if AUTH_TYPE == AuthType.DISABLED:
         return True

backend/onyx/background/celery/apps/app_base.py

@@ -140,7 +140,7 @@ def on_task_postrun(
         f"{f'for tenant_id={tenant_id}' if tenant_id else ''}"
     )
 
-    r = get_redis_client(tenant_id=tenant_id)
+    r = get_redis_client()
 
     if task_id.startswith(RedisConnectorCredentialPair.PREFIX):
         r.srem(RedisConnectorCredentialPair.get_taskset_key(), task_id)

backend/onyx/background/celery/tasks/indexing/tasks.py

@@ -48,7 +48,6 @@ from onyx.configs.constants import OnyxCeleryTask
 from onyx.configs.constants import OnyxRedisConstants
 from onyx.configs.constants import OnyxRedisLocks
 from onyx.configs.constants import OnyxRedisSignals
-from onyx.connectors.interfaces import ConnectorValidationError
 from onyx.db.connector import mark_ccpair_with_indexing_trigger
 from onyx.db.connector_credential_pair import fetch_connector_credential_pairs
 from onyx.db.connector_credential_pair import get_connector_credential_pair_from_id
@@ -108,9 +107,6 @@ class IndexingWatchdogTerminalStatus(str, Enum):
         "index_attempt_mismatch"  # expected index attempt metadata not found in db
     )
 
-    CONNECTOR_VALIDATION_ERROR = (
-        "connector_validation_error"  # the connector validation failed
-    )
     CONNECTOR_EXCEPTIONED = "connector_exceptioned"  # the connector itself exceptioned
     WATCHDOG_EXCEPTIONED = "watchdog_exceptioned"  # the watchdog exceptioned
 
@@ -131,7 +127,6 @@ class IndexingWatchdogTerminalStatus(str, Enum):
         _ENUM_TO_CODE: dict[IndexingWatchdogTerminalStatus, int] = {
             IndexingWatchdogTerminalStatus.PROCESS_SIGNAL_SIGKILL: -9,
             IndexingWatchdogTerminalStatus.OUT_OF_MEMORY: 137,
-            IndexingWatchdogTerminalStatus.CONNECTOR_VALIDATION_ERROR: 247,
             IndexingWatchdogTerminalStatus.BLOCKED_BY_DELETION: 248,
             IndexingWatchdogTerminalStatus.BLOCKED_BY_STOP_SIGNAL: 249,
             IndexingWatchdogTerminalStatus.FENCE_NOT_FOUND: 250,
@@ -149,7 +144,6 @@ class IndexingWatchdogTerminalStatus(str, Enum):
         _CODE_TO_ENUM: dict[int, IndexingWatchdogTerminalStatus] = {
             -9: IndexingWatchdogTerminalStatus.PROCESS_SIGNAL_SIGKILL,
             137: IndexingWatchdogTerminalStatus.OUT_OF_MEMORY,
-            247: IndexingWatchdogTerminalStatus.CONNECTOR_VALIDATION_ERROR,
             248: IndexingWatchdogTerminalStatus.BLOCKED_BY_DELETION,
             249: IndexingWatchdogTerminalStatus.BLOCKED_BY_STOP_SIGNAL,
             250: IndexingWatchdogTerminalStatus.FENCE_NOT_FOUND,
@@ -802,15 +796,6 @@ def connector_indexing_task(
         # get back the total number of indexed docs and return it
         n_final_progress = redis_connector_index.get_progress()
         redis_connector_index.set_generator_complete(HTTPStatus.OK.value)
-    except ConnectorValidationError:
-        raise SimpleJobException(
-            f"Indexing task failed: attempt={index_attempt_id} "
-            f"tenant={tenant_id} "
-            f"cc_pair={cc_pair_id} "
-            f"search_settings={search_settings_id}",
-            code=IndexingWatchdogTerminalStatus.CONNECTOR_VALIDATION_ERROR.code,
-        )
-
     except Exception as e:
         logger.exception(
             f"Indexing spawned task failed: attempt={index_attempt_id} "
@@ -818,8 +803,8 @@ def connector_indexing_task(
             f"cc_pair={cc_pair_id} "
             f"search_settings={search_settings_id}"
         )
-        raise e
 
+        raise e
     finally:
         if lock.owned():
             lock.release()
@@ -1079,13 +1064,9 @@ def connector_indexing_proxy_task(
                     )
                 )
                 continue
-    except Exception as e:
+    except Exception:
         result.status = IndexingWatchdogTerminalStatus.WATCHDOG_EXCEPTIONED
-        if isinstance(e, ConnectorValidationError):
-            # No need to expose full stack trace for validation errors
-            result.exception_str = str(e)
-        else:
-            result.exception_str = traceback.format_exc()
+        result.exception_str = traceback.format_exc()
 
     # handle exit and reporting
     elapsed = time.monotonic() - start

backend/onyx/background/celery/tasks/pruning/tasks.py

@@ -431,7 +431,6 @@ def connector_pruning_generator_task(
                 f"cc_pair={cc_pair_id} "
                 f"connector_source={cc_pair.connector.source}"
             )
-
             runnable_connector = instantiate_connector(
                 db_session,
                 cc_pair.connector.source,

backend/onyx/background/indexing/run_indexing.py

@@ -21,7 +21,6 @@ from onyx.configs.constants import DocumentSource
 from onyx.configs.constants import MilestoneRecordType
 from onyx.connectors.connector_runner import ConnectorRunner
 from onyx.connectors.factory import instantiate_connector
-from onyx.connectors.interfaces import ConnectorValidationError
 from onyx.connectors.models import ConnectorCheckpoint
 from onyx.connectors.models import ConnectorFailure
 from onyx.connectors.models import Document
@@ -87,11 +86,6 @@ def _get_connector_runner(
             credential=attempt.connector_credential_pair.credential,
             tenant_id=tenant_id,
         )
-
-        # validate the connector settings
-
-        runnable_connector.validate_connector_settings()
-
     except Exception as e:
         logger.exception(f"Unable to instantiate connector due to {e}")
 
@@ -573,28 +567,8 @@ def _run_indexing(
             "Connector run exceptioned after elapsed time: "
             f"{time.monotonic() - start_time} seconds"
         )
-        if isinstance(e, ConnectorValidationError):
-            # On validation errors during indexing, we want to cancel the indexing attempt
-            # and mark the CCPair as invalid. This prevents the connector from being
-            # used in the future until the credentials are updated.
-            with get_session_with_current_tenant() as db_session_temp:
-                mark_attempt_canceled(
-                    index_attempt_id,
-                    db_session_temp,
-                    reason=str(e),
-                )
 
-                if ctx.is_primary:
-                    update_connector_credential_pair(
-                        db_session=db_session_temp,
-                        connector_id=ctx.connector_id,
-                        credential_id=ctx.credential_id,
-                        status=ConnectorCredentialPairStatus.INVALID,
-                    )
-            memory_tracer.stop()
-            raise e
-
-        elif isinstance(e, ConnectorStopSignal):
+        if isinstance(e, ConnectorStopSignal):
             with get_session_with_current_tenant() as db_session_temp:
                 mark_attempt_canceled(
                     index_attempt_id,

backend/onyx/chat/chat_utils.py

@@ -190,8 +190,7 @@ def create_chat_chain(
             and previous_message.message_type == MessageType.ASSISTANT
             and mainline_messages
         ):
-            if current_message.refined_answer_improvement:
-                mainline_messages[-1] = current_message
+            mainline_messages[-1] = current_message
         else:
             mainline_messages.append(current_message)
 

backend/onyx/chat/models.py

@@ -142,15 +142,6 @@ class MessageResponseIDInfo(BaseModel):
     reserved_assistant_message_id: int
 
 
-class AgentMessageIDInfo(BaseModel):
-    level: int
-    message_id: int
-
-
-class AgenticMessageResponseIDInfo(BaseModel):
-    agentic_message_ids: list[AgentMessageIDInfo]
-
-
 class StreamingError(BaseModel):
     error: str
     stack_trace: str | None = None

backend/onyx/chat/process_message.py

@@ -11,8 +11,6 @@ from onyx.agents.agent_search.orchestration.nodes.call_tool import ToolCallExcep
 from onyx.chat.answer import Answer
 from onyx.chat.chat_utils import create_chat_chain
 from onyx.chat.chat_utils import create_temporary_persona
-from onyx.chat.models import AgenticMessageResponseIDInfo
-from onyx.chat.models import AgentMessageIDInfo
 from onyx.chat.models import AgentSearchPacket
 from onyx.chat.models import AllCitations
 from onyx.chat.models import AnswerPostInfo
@@ -310,7 +308,6 @@ ChatPacket = (
     | CustomToolResponse
     | MessageSpecificCitations
     | MessageResponseIDInfo
-    | AgenticMessageResponseIDInfo
     | StreamStopInfo
     | AgentSearchPacket
 )
@@ -1038,7 +1035,6 @@ def stream_chat_message_objects(
         next_level = 1
         prev_message = gen_ai_response_message
         agent_answers = answer.llm_answer_by_level()
-        agentic_message_ids = []
         while next_level in agent_answers:
             next_answer = agent_answers[next_level]
             info = info_by_subq[
@@ -1063,18 +1059,17 @@ def stream_chat_message_objects(
                 refined_answer_improvement=refined_answer_improvement,
                 is_agentic=True,
             )
-            agentic_message_ids.append(
-                AgentMessageIDInfo(level=next_level, message_id=next_answer_message.id)
-            )
             next_level += 1
             prev_message = next_answer_message
 
         logger.debug("Committing messages")
         db_session.commit()  # actually save user / assistant message
 
-        yield AgenticMessageResponseIDInfo(agentic_message_ids=agentic_message_ids)
+        msg_detail_response = translate_db_message_to_chat_message_detail(
+            gen_ai_response_message
+        )
 
-        yield translate_db_message_to_chat_message_detail(gen_ai_response_message)
+        yield msg_detail_response
     except Exception as e:
         error_msg = str(e)
         logger.exception(error_msg)

backend/onyx/connectors/bookstack/client.py

@@ -5,8 +5,6 @@ import requests
 
 class BookStackClientRequestFailedError(ConnectionError):
     def __init__(self, status: int, error: str) -> None:
-        self.status_code = status
-        self.error = error
         super().__init__(
             "BookStack Client request failed with status {status}: {error}".format(
                 status=status, error=error

backend/onyx/connectors/bookstack/connector.py

@@ -7,12 +7,8 @@ from typing import Any
 from onyx.configs.app_configs import INDEX_BATCH_SIZE
 from onyx.configs.constants import DocumentSource
 from onyx.connectors.bookstack.client import BookStackApiClient
-from onyx.connectors.bookstack.client import BookStackClientRequestFailedError
 from onyx.connectors.cross_connector_utils.miscellaneous_utils import time_str_to_utc
-from onyx.connectors.interfaces import ConnectorValidationError
-from onyx.connectors.interfaces import CredentialExpiredError
 from onyx.connectors.interfaces import GenerateDocumentsOutput
-from onyx.connectors.interfaces import InsufficientPermissionsError
 from onyx.connectors.interfaces import LoadConnector
 from onyx.connectors.interfaces import PollConnector
 from onyx.connectors.interfaces import SecondsSinceUnixEpoch
@@ -218,39 +214,3 @@ class BookstackConnector(LoadConnector, PollConnector):
                     break
                 else:
                     time.sleep(0.2)
-
-    def validate_connector_settings(self) -> None:
-        """
-        Validate that the BookStack credentials and connector settings are correct.
-        Specifically checks that we can make an authenticated request to BookStack.
-        """
-        if not self.bookstack_client:
-            raise ConnectorMissingCredentialError(
-                "BookStack credentials have not been loaded."
-            )
-
-        try:
-            # Attempt to fetch a small batch of books (arbitrary endpoint) to verify credentials
-            _ = self.bookstack_client.get(
-                "/books", params={"count": "1", "offset": "0"}
-            )
-
-        except BookStackClientRequestFailedError as e:
-            # Check for HTTP status codes
-            if e.status_code == 401:
-                raise CredentialExpiredError(
-                    "Your BookStack credentials appear to be invalid or expired (HTTP 401)."
-                ) from e
-            elif e.status_code == 403:
-                raise InsufficientPermissionsError(
-                    "The configured BookStack token does not have sufficient permissions (HTTP 403)."
-                ) from e
-            else:
-                raise ConnectorValidationError(
-                    f"Unexpected BookStack error (status={e.status_code}): {e}"
-                ) from e
-
-        except Exception as exc:
-            raise ConnectorValidationError(
-                f"Unexpected error while validating BookStack connector settings: {exc}"
-            ) from exc

backend/onyx/connectors/cross_connector_utils/miscellaneous_utils.py

@@ -1,4 +1,3 @@
-import re
 from collections.abc import Callable
 from collections.abc import Iterator
 from datetime import datetime
@@ -25,22 +24,16 @@ def datetime_to_utc(dt: datetime) -> datetime:
 
 
 def time_str_to_utc(datetime_str: str) -> datetime:
-    # Remove all timezone abbreviations in parentheses
-    datetime_str = re.sub(r"\([A-Z]+\)", "", datetime_str).strip()
-
-    # Remove any remaining parentheses and their contents
-    datetime_str = re.sub(r"\(.*?\)", "", datetime_str).strip()
-
     try:
         dt = parse(datetime_str)
     except ValueError:
-        # Fix common format issues (e.g. "0000" => "+0000")
+        # Handle malformed timezone by attempting to fix common format issues
         if "0000" in datetime_str:
-            datetime_str = datetime_str.replace(" 0000", " +0000")
-            dt = parse(datetime_str)
+            # Convert "0000" to "+0000" for proper timezone parsing
+            fixed_dt_str = datetime_str.replace(" 0000", " +0000")
+            dt = parse(fixed_dt_str)
         else:
             raise
-
     return datetime_to_utc(dt)
 
 

backend/onyx/connectors/dropbox/connector.py

@@ -4,16 +4,12 @@ from typing import Any
 
 from dropbox import Dropbox  # type: ignore
 from dropbox.exceptions import ApiError  # type:ignore
-from dropbox.exceptions import AuthError  # type:ignore
 from dropbox.files import FileMetadata  # type:ignore
 from dropbox.files import FolderMetadata  # type:ignore
 
 from onyx.configs.app_configs import INDEX_BATCH_SIZE
 from onyx.configs.constants import DocumentSource
-from onyx.connectors.interfaces import ConnectorValidationError
-from onyx.connectors.interfaces import CredentialInvalidError
 from onyx.connectors.interfaces import GenerateDocumentsOutput
-from onyx.connectors.interfaces import InsufficientPermissionsError
 from onyx.connectors.interfaces import LoadConnector
 from onyx.connectors.interfaces import PollConnector
 from onyx.connectors.interfaces import SecondsSinceUnixEpoch
@@ -145,29 +141,6 @@ class DropboxConnector(LoadConnector, PollConnector):
 
         return None
 
-    def validate_connector_settings(self) -> None:
-        if self.dropbox_client is None:
-            raise ConnectorMissingCredentialError("Dropbox credentials not loaded.")
-
-        try:
-            self.dropbox_client.files_list_folder(path="", limit=1)
-        except AuthError as e:
-            logger.exception("Failed to validate Dropbox credentials")
-            raise CredentialInvalidError(f"Dropbox credential is invalid: {e.error}")
-        except ApiError as e:
-            if (
-                e.error is not None
-                and "insufficient_permissions" in str(e.error).lower()
-            ):
-                raise InsufficientPermissionsError(
-                    "Your Dropbox token does not have sufficient permissions."
-                )
-            raise ConnectorValidationError(
-                f"Unexpected Dropbox error during validation: {e.user_message_text or e}"
-            )
-        except Exception as e:
-            raise Exception(f"Unexpected error during Dropbox settings validation: {e}")
-
 
 if __name__ == "__main__":
     import os

backend/onyx/connectors/factory.py

@@ -31,7 +31,6 @@ from onyx.connectors.guru.connector import GuruConnector
 from onyx.connectors.hubspot.connector import HubSpotConnector
 from onyx.connectors.interfaces import BaseConnector
 from onyx.connectors.interfaces import CheckpointConnector
-from onyx.connectors.interfaces import ConnectorValidationError
 from onyx.connectors.interfaces import EventConnector
 from onyx.connectors.interfaces import LoadConnector
 from onyx.connectors.interfaces import PollConnector
@@ -53,11 +52,8 @@ from onyx.connectors.wikipedia.connector import WikipediaConnector
 from onyx.connectors.xenforo.connector import XenforoConnector
 from onyx.connectors.zendesk.connector import ZendeskConnector
 from onyx.connectors.zulip.connector import ZulipConnector
-from onyx.db.connector import fetch_connector_by_id
 from onyx.db.credentials import backend_update_credential_json
-from onyx.db.credentials import fetch_credential_by_id_for_user
 from onyx.db.models import Credential
-from onyx.db.models import User
 
 
 class ConnectorMissingException(Exception):
@@ -178,43 +174,3 @@ def instantiate_connector(
         backend_update_credential_json(credential, new_credentials, db_session)
 
     return connector
-
-
-def validate_ccpair_for_user(
-    connector_id: int,
-    credential_id: int,
-    db_session: Session,
-    user: User | None,
-    tenant_id: str | None,
-) -> None:
-    # Validate the connector settings
-    connector = fetch_connector_by_id(connector_id, db_session)
-    credential = fetch_credential_by_id_for_user(
-        credential_id,
-        user,
-        db_session,
-        get_editable=False,
-    )
-
-    if not connector:
-        raise ValueError("Connector not found")
-
-    if connector.source == DocumentSource.INGESTION_API:
-        return
-
-    if not credential:
-        raise ValueError("Credential not found")
-
-    try:
-        runnable_connector = instantiate_connector(
-            db_session=db_session,
-            source=connector.source,
-            input_type=connector.input_type,
-            connector_specific_config=connector.connector_specific_config,
-            credential=credential,
-            tenant_id=tenant_id,
-        )
-    except Exception as e:
-        raise ConnectorValidationError(str(e))
-
-    runnable_connector.validate_connector_settings()

backend/onyx/connectors/fireflies/connector.py

@@ -187,12 +187,12 @@ class FirefliesConnector(PollConnector, LoadConnector):
         return self._process_transcripts()
 
     def poll_source(
-        self, start: SecondsSinceUnixEpoch, end: SecondsSinceUnixEpoch
+        self, start_unixtime: SecondsSinceUnixEpoch, end_unixtime: SecondsSinceUnixEpoch
     ) -> GenerateDocumentsOutput:
-        start_datetime = datetime.fromtimestamp(start, tz=timezone.utc).strftime(
-            "%Y-%m-%dT%H:%M:%S.000Z"
-        )
-        end_datetime = datetime.fromtimestamp(end, tz=timezone.utc).strftime(
+        start_datetime = datetime.fromtimestamp(
+            start_unixtime, tz=timezone.utc
+        ).strftime("%Y-%m-%dT%H:%M:%S.000Z")
+        end_datetime = datetime.fromtimestamp(end_unixtime, tz=timezone.utc).strftime(
             "%Y-%m-%dT%H:%M:%S.000Z"
         )
 

backend/onyx/connectors/gitbook/connector.py

@@ -229,20 +229,16 @@ class GitbookConnector(LoadConnector, PollConnector):
 
         try:
             content = self.client.get(f"/spaces/{self.space_id}/content")
-            pages: list[dict[str, Any]] = content.get("pages", [])
+            pages = content.get("pages", [])
+
             current_batch: list[Document] = []
+            for page in pages:
+                updated_at = datetime.fromisoformat(page["updatedAt"])
 
-            while pages:
-                page = pages.pop(0)
-
-                updated_at_raw = page.get("updatedAt")
-                if updated_at_raw is None:
-                    # if updatedAt is not present, that means the page has never been edited
-                    continue
-
-                updated_at = datetime.fromisoformat(updated_at_raw)
                 if start and updated_at < start:
-                    continue
+                    if current_batch:
+                        yield current_batch
+                    return
                 if end and updated_at > end:
                     continue
 
@@ -254,8 +250,6 @@ class GitbookConnector(LoadConnector, PollConnector):
                     yield current_batch
                     current_batch = []
 
-                pages.extend(page.get("pages", []))
-
             if current_batch:
                 yield current_batch
 

backend/onyx/connectors/github/connector.py

@@ -9,7 +9,6 @@ from typing import cast
 from github import Github
 from github import RateLimitExceededException
 from github import Repository
-from github.GithubException import GithubException
 from github.Issue import Issue
 from github.PaginatedList import PaginatedList
 from github.PullRequest import PullRequest
@@ -17,20 +16,17 @@ from github.PullRequest import PullRequest
 from onyx.configs.app_configs import GITHUB_CONNECTOR_BASE_URL
 from onyx.configs.app_configs import INDEX_BATCH_SIZE
 from onyx.configs.constants import DocumentSource
-from onyx.connectors.interfaces import ConnectorValidationError
-from onyx.connectors.interfaces import CredentialExpiredError
 from onyx.connectors.interfaces import GenerateDocumentsOutput
-from onyx.connectors.interfaces import InsufficientPermissionsError
 from onyx.connectors.interfaces import LoadConnector
 from onyx.connectors.interfaces import PollConnector
 from onyx.connectors.interfaces import SecondsSinceUnixEpoch
-from onyx.connectors.interfaces import UnexpectedError
 from onyx.connectors.models import ConnectorMissingCredentialError
 from onyx.connectors.models import Document
 from onyx.connectors.models import Section
 from onyx.utils.batching import batch_generator
 from onyx.utils.logger import setup_logger
 
+
 logger = setup_logger()
 
 
@@ -230,48 +226,6 @@ class GithubConnector(LoadConnector, PollConnector):
 
         return self._fetch_from_github(adjusted_start_datetime, end_datetime)
 
-    def validate_connector_settings(self) -> None:
-        if self.github_client is None:
-            raise ConnectorMissingCredentialError("GitHub credentials not loaded.")
-
-        if not self.repo_owner or not self.repo_name:
-            raise ConnectorValidationError(
-                "Invalid connector settings: 'repo_owner' and 'repo_name' must be provided."
-            )
-
-        try:
-            test_repo = self.github_client.get_repo(
-                f"{self.repo_owner}/{self.repo_name}"
-            )
-            test_repo.get_contents("")
-
-        except RateLimitExceededException:
-            raise UnexpectedError(
-                "Validation failed due to GitHub rate-limits being exceeded. Please try again later."
-            )
-
-        except GithubException as e:
-            if e.status == 401:
-                raise CredentialExpiredError(
-                    "GitHub credential appears to be invalid or expired (HTTP 401)."
-                )
-            elif e.status == 403:
-                raise InsufficientPermissionsError(
-                    "Your GitHub token does not have sufficient permissions for this repository (HTTP 403)."
-                )
-            elif e.status == 404:
-                raise ConnectorValidationError(
-                    f"GitHub repository not found with name: {self.repo_owner}/{self.repo_name}"
-                )
-            else:
-                raise ConnectorValidationError(
-                    f"Unexpected GitHub error (status={e.status}): {e.data}"
-                )
-        except Exception as exc:
-            raise Exception(
-                f"Unexpected error during GitHub settings validation: {exc}"
-            )
-
 
 if __name__ == "__main__":
     import os

backend/onyx/connectors/gmail/connector.py

@@ -297,7 +297,6 @@ class GmailConnector(LoadConnector, PollConnector, SlimConnector):
                 userId=user_email,
                 fields=THREAD_LIST_FIELDS,
                 q=query,
-                continue_on_404_or_403=True,
             ):
                 full_threads = execute_paginated_retrieval(
                     retrieval_function=gmail_service.users().threads().get,

backend/onyx/connectors/interfaces.py

@@ -12,6 +12,7 @@ from onyx.connectors.models import Document
 from onyx.connectors.models import SlimDocument
 from onyx.indexing.indexing_heartbeat import IndexingHeartbeatInterface
 
+
 SecondsSinceUnixEpoch = float
 
 GenerateDocumentsOutput = Iterator[list[Document]]
@@ -44,14 +45,6 @@ class BaseConnector(abc.ABC):
                 raise RuntimeError(custom_parser_req_msg)
         return metadata_lines
 
-    def validate_connector_settings(self) -> None:
-        """
-        Override this if your connector needs to validate credentials or settings.
-        Raise an exception if invalid, otherwise do nothing.
-
-        Default is a no-op (always successful).
-        """
-
 
 # Large set update or reindex, generally pulling a complete state or from a savestate file
 class LoadConnector(BaseConnector):
@@ -146,46 +139,3 @@ class CheckpointConnector(BaseConnector):
         ```
         """
         raise NotImplementedError
-
-
-class ConnectorValidationError(Exception):
-    """General exception for connector validation errors."""
-
-    def __init__(self, message: str):
-        self.message = message
-        super().__init__(self.message)
-
-
-class UnexpectedError(Exception):
-    """Raised when an unexpected error occurs during connector validation.
-
-    Unexpected errors don't necessarily mean the credential is invalid,
-    but rather that there was an error during the validation process
-    or we encountered a currently unhandled error case.
-    """
-
-    def __init__(self, message: str = "Unexpected error during connector validation"):
-        super().__init__(message)
-
-
-class CredentialInvalidError(ConnectorValidationError):
-    """Raised when a connector's credential is invalid."""
-
-    def __init__(self, message: str = "Credential is invalid"):
-        super().__init__(message)
-
-
-class CredentialExpiredError(ConnectorValidationError):
-    """Raised when a connector's credential is expired."""
-
-    def __init__(self, message: str = "Credential has expired"):
-        super().__init__(message)
-
-
-class InsufficientPermissionsError(ConnectorValidationError):
-    """Raised when the credential does not have sufficient API permissions."""
-
-    def __init__(
-        self, message: str = "Insufficient permissions for the requested operation"
-    ):
-        super().__init__(message)

backend/onyx/connectors/notion/connector.py

@@ -7,7 +7,6 @@ from datetime import timezone
 from typing import Any
 from typing import Optional
 
-import requests
 from retry import retry
 
 from onyx.configs.app_configs import INDEX_BATCH_SIZE
@@ -16,14 +15,10 @@ from onyx.configs.constants import DocumentSource
 from onyx.connectors.cross_connector_utils.rate_limit_wrapper import (
     rl_requests,
 )
-from onyx.connectors.interfaces import ConnectorValidationError
-from onyx.connectors.interfaces import CredentialExpiredError
 from onyx.connectors.interfaces import GenerateDocumentsOutput
-from onyx.connectors.interfaces import InsufficientPermissionsError
 from onyx.connectors.interfaces import LoadConnector
 from onyx.connectors.interfaces import PollConnector
 from onyx.connectors.interfaces import SecondsSinceUnixEpoch
-from onyx.connectors.models import ConnectorMissingCredentialError
 from onyx.connectors.models import Document
 from onyx.connectors.models import Section
 from onyx.utils.batching import batch_generator
@@ -621,64 +616,6 @@ class NotionConnector(LoadConnector, PollConnector):
             else:
                 break
 
-    def validate_connector_settings(self) -> None:
-        if not self.headers.get("Authorization"):
-            raise ConnectorMissingCredentialError("Notion credentials not loaded.")
-
-        try:
-            # We'll do a minimal search call (page_size=1) to confirm accessibility
-            if self.root_page_id:
-                # If root_page_id is set, fetch the specific page
-                res = rl_requests.get(
-                    f"https://api.notion.com/v1/pages/{self.root_page_id}",
-                    headers=self.headers,
-                    timeout=_NOTION_CALL_TIMEOUT,
-                )
-            else:
-                # If root_page_id is not set, perform a minimal search
-                test_query = {
-                    "filter": {"property": "object", "value": "page"},
-                    "page_size": 1,
-                }
-                res = rl_requests.post(
-                    "https://api.notion.com/v1/search",
-                    headers=self.headers,
-                    json=test_query,
-                    timeout=_NOTION_CALL_TIMEOUT,
-                )
-            res.raise_for_status()
-
-        except requests.exceptions.HTTPError as http_err:
-            status_code = http_err.response.status_code if http_err.response else None
-
-            if status_code == 401:
-                raise CredentialExpiredError(
-                    "Notion credential appears to be invalid or expired (HTTP 401)."
-                )
-            elif status_code == 403:
-                raise InsufficientPermissionsError(
-                    "Your Notion token does not have sufficient permissions (HTTP 403)."
-                )
-            elif status_code == 404:
-                # Typically means resource not found or not shared. Could be root_page_id is invalid.
-                raise ConnectorValidationError(
-                    "Notion resource not found or not shared with the integration (HTTP 404)."
-                )
-            elif status_code == 429:
-                raise ConnectorValidationError(
-                    "Validation failed due to Notion rate-limits being exceeded (HTTP 429). "
-                    "Please try again later."
-                )
-            else:
-                raise Exception(
-                    f"Unexpected Notion HTTP error (status={status_code}): {http_err}"
-                ) from http_err
-
-        except Exception as exc:
-            raise Exception(
-                f"Unexpected error during Notion settings validation: {exc}"
-            )
-
 
 if __name__ == "__main__":
     import os

backend/onyx/connectors/onyx_jira/connector.py

@@ -12,11 +12,8 @@ from onyx.configs.app_configs import JIRA_CONNECTOR_LABELS_TO_SKIP
 from onyx.configs.app_configs import JIRA_CONNECTOR_MAX_TICKET_SIZE
 from onyx.configs.constants import DocumentSource
 from onyx.connectors.cross_connector_utils.miscellaneous_utils import time_str_to_utc
-from onyx.connectors.interfaces import ConnectorValidationError
-from onyx.connectors.interfaces import CredentialExpiredError
 from onyx.connectors.interfaces import GenerateDocumentsOutput
 from onyx.connectors.interfaces import GenerateSlimDocumentOutput
-from onyx.connectors.interfaces import InsufficientPermissionsError
 from onyx.connectors.interfaces import LoadConnector
 from onyx.connectors.interfaces import PollConnector
 from onyx.connectors.interfaces import SecondsSinceUnixEpoch
@@ -275,40 +272,6 @@ class JiraConnector(LoadConnector, PollConnector, SlimConnector):
 
         yield slim_doc_batch
 
-    def validate_connector_settings(self) -> None:
-        if self._jira_client is None:
-            raise ConnectorMissingCredentialError("Jira")
-
-        if not self._jira_project:
-            raise ConnectorValidationError(
-                "Invalid connector settings: 'jira_project' must be provided."
-            )
-
-        try:
-            self.jira_client.project(self._jira_project)
-
-        except Exception as e:
-            status_code = getattr(e, "status_code", None)
-
-            if status_code == 401:
-                raise CredentialExpiredError(
-                    "Jira credential appears to be expired or invalid (HTTP 401)."
-                )
-            elif status_code == 403:
-                raise InsufficientPermissionsError(
-                    "Your Jira token does not have sufficient permissions for this project (HTTP 403)."
-                )
-            elif status_code == 404:
-                raise ConnectorValidationError(
-                    f"Jira project not found with key: {self._jira_project}"
-                )
-            elif status_code == 429:
-                raise ConnectorValidationError(
-                    "Validation failed due to Jira rate-limits being exceeded. Please try again later."
-                )
-            else:
-                raise Exception(f"Unexpected Jira error during validation: {e}")
-
 
 if __name__ == "__main__":
     import os

backend/onyx/connectors/web/connector.py

@@ -25,12 +25,8 @@ from onyx.configs.app_configs import WEB_CONNECTOR_OAUTH_CLIENT_SECRET
 from onyx.configs.app_configs import WEB_CONNECTOR_OAUTH_TOKEN_URL
 from onyx.configs.app_configs import WEB_CONNECTOR_VALIDATE_URLS
 from onyx.configs.constants import DocumentSource
-from onyx.connectors.interfaces import ConnectorValidationError
-from onyx.connectors.interfaces import CredentialExpiredError
 from onyx.connectors.interfaces import GenerateDocumentsOutput
-from onyx.connectors.interfaces import InsufficientPermissionsError
 from onyx.connectors.interfaces import LoadConnector
-from onyx.connectors.interfaces import UnexpectedError
 from onyx.connectors.models import Document
 from onyx.connectors.models import Section
 from onyx.file_processing.extract_file_text import read_pdf_file
@@ -41,8 +37,6 @@ from shared_configs.configs import MULTI_TENANT
 
 logger = setup_logger()
 
-WEB_CONNECTOR_MAX_SCROLL_ATTEMPTS = 20
-
 
 class WEB_CONNECTOR_VALID_SETTINGS(str, Enum):
     # Given a base site, index everything under that path
@@ -176,35 +170,26 @@ def start_playwright() -> Tuple[Playwright, BrowserContext]:
 
 
 def extract_urls_from_sitemap(sitemap_url: str) -> list[str]:
-    try:
-        response = requests.get(sitemap_url)
-        response.raise_for_status()
+    response = requests.get(sitemap_url)
+    response.raise_for_status()
 
-        soup = BeautifulSoup(response.content, "html.parser")
-        urls = [
-            _ensure_absolute_url(sitemap_url, loc_tag.text)
-            for loc_tag in soup.find_all("loc")
-        ]
+    soup = BeautifulSoup(response.content, "html.parser")
+    urls = [
+        _ensure_absolute_url(sitemap_url, loc_tag.text)
+        for loc_tag in soup.find_all("loc")
+    ]
 
-        if len(urls) == 0 and len(soup.find_all("urlset")) == 0:
-            # the given url doesn't look like a sitemap, let's try to find one
-            urls = list_pages_for_site(sitemap_url)
+    if len(urls) == 0 and len(soup.find_all("urlset")) == 0:
+        # the given url doesn't look like a sitemap, let's try to find one
+        urls = list_pages_for_site(sitemap_url)
 
-        if len(urls) == 0:
-            raise ValueError(
-                f"No URLs found in sitemap {sitemap_url}. Try using the 'single' or 'recursive' scraping options instead."
-            )
-
-        return urls
-    except requests.RequestException as e:
-        raise RuntimeError(f"Failed to fetch sitemap from {sitemap_url}: {e}")
-    except ValueError as e:
-        raise RuntimeError(f"Error processing sitemap {sitemap_url}: {e}")
-    except Exception as e:
-        raise RuntimeError(
-            f"Unexpected error while processing sitemap {sitemap_url}: {e}"
+    if len(urls) == 0:
+        raise ValueError(
+            f"No URLs found in sitemap {sitemap_url}. Try using the 'single' or 'recursive' scraping options instead."
         )
 
+    return urls
+
 
 def _ensure_absolute_url(source_url: str, maybe_relative_url: str) -> str:
     if not urlparse(maybe_relative_url).netloc:
@@ -240,14 +225,10 @@ class WebConnector(LoadConnector):
         web_connector_type: str = WEB_CONNECTOR_VALID_SETTINGS.RECURSIVE.value,
         mintlify_cleanup: bool = True,  # Mostly ok to apply to other websites as well
         batch_size: int = INDEX_BATCH_SIZE,
-        scroll_before_scraping: bool = False,
-        **kwargs: Any,
     ) -> None:
         self.mintlify_cleanup = mintlify_cleanup
         self.batch_size = batch_size
         self.recursive = False
-        self.scroll_before_scraping = scroll_before_scraping
-        self.web_connector_type = web_connector_type
 
         if web_connector_type == WEB_CONNECTOR_VALID_SETTINGS.RECURSIVE.value:
             self.recursive = True
@@ -363,18 +344,6 @@ class WebConnector(LoadConnector):
                         continue
                     visited_links.add(current_url)
 
-                if self.scroll_before_scraping:
-                    scroll_attempts = 0
-                    previous_height = page.evaluate("document.body.scrollHeight")
-                    while scroll_attempts < WEB_CONNECTOR_MAX_SCROLL_ATTEMPTS:
-                        page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
-                        page.wait_for_load_state("networkidle", timeout=30000)
-                        new_height = page.evaluate("document.body.scrollHeight")
-                        if new_height == previous_height:
-                            break  # Stop scrolling when no more content is loaded
-                        previous_height = new_height
-                        scroll_attempts += 1
-
                 content = page.content()
                 soup = BeautifulSoup(content, "html.parser")
 
@@ -433,53 +402,6 @@ class WebConnector(LoadConnector):
                 raise RuntimeError(last_error)
             raise RuntimeError("No valid pages found.")
 
-    def validate_connector_settings(self) -> None:
-        # Make sure we have at least one valid URL to check
-        if not self.to_visit_list:
-            raise ConnectorValidationError(
-                "No URL configured. Please provide at least one valid URL."
-            )
-
-        if self.web_connector_type == WEB_CONNECTOR_VALID_SETTINGS.SITEMAP.value:
-            return None
-
-        # We'll just test the first URL for connectivity and correctness
-        test_url = self.to_visit_list[0]
-
-        # Check that the URL is allowed and well-formed
-        try:
-            protected_url_check(test_url)
-        except ValueError as e:
-            raise ConnectorValidationError(
-                f"Protected URL check failed for '{test_url}': {e}"
-            )
-        except ConnectionError as e:
-            # Typically DNS or other network issues
-            raise ConnectorValidationError(str(e))
-
-        # Make a quick request to see if we get a valid response
-        try:
-            check_internet_connection(test_url)
-        except Exception as e:
-            err_str = str(e)
-            if "401" in err_str:
-                raise CredentialExpiredError(
-                    f"Unauthorized access to '{test_url}': {e}"
-                )
-            elif "403" in err_str:
-                raise InsufficientPermissionsError(
-                    f"Forbidden access to '{test_url}': {e}"
-                )
-            elif "404" in err_str:
-                raise ConnectorValidationError(f"Page not found for '{test_url}': {e}")
-            elif "Max retries exceeded" in err_str and "NameResolutionError" in err_str:
-                raise ConnectorValidationError(
-                    f"Unable to resolve hostname for '{test_url}'. Please check the URL and your internet connection."
-                )
-            else:
-                # Could be a 5xx or another error, treat as unexpected
-                raise UnexpectedError(f"Unexpected error validating '{test_url}': {e}")
-
 
 if __name__ == "__main__":
     connector = WebConnector("https://docs.onyx.app/")

backend/onyx/db/credentials.py

@@ -14,7 +14,6 @@ from onyx.configs.constants import DocumentSource
 from onyx.connectors.google_utils.shared_constants import (
     DB_CREDENTIALS_DICT_SERVICE_ACCOUNT_KEY,
 )
-from onyx.db.enums import ConnectorCredentialPairStatus
 from onyx.db.models import ConnectorCredentialPair
 from onyx.db.models import Credential
 from onyx.db.models import Credential__UserGroup
@@ -246,10 +245,6 @@ def swap_credentials_connector(
     existing_pair.credential_id = new_credential_id
     existing_pair.credential = new_credential
 
-    # Update ccpair status if it's in INVALID state
-    if existing_pair.status == ConnectorCredentialPairStatus.INVALID:
-        existing_pair.status = ConnectorCredentialPairStatus.ACTIVE
-
     # Commit the changes
     db_session.commit()
 

backend/onyx/db/document.py

@@ -60,8 +60,9 @@ def count_documents_by_needs_sync(session: Session) -> int:
     This function executes the query and returns the count of
     documents matching the criteria."""
 
-    return (
-        session.query(DbDocument.id)
+    count = (
+        session.query(func.count(DbDocument.id.distinct()))
+        .select_from(DbDocument)
         .join(
             DocumentByConnectorCredentialPair,
             DbDocument.id == DocumentByConnectorCredentialPair.id,
@@ -72,53 +73,63 @@ def count_documents_by_needs_sync(session: Session) -> int:
                 DbDocument.last_synced.is_(None),
             )
         )
-        .count()
+        .scalar()
     )
 
+    return count
+
 
 def construct_document_select_for_connector_credential_pair_by_needs_sync(
     connector_id: int, credential_id: int
 ) -> Select:
-    return (
-        select(DbDocument)
-        .join(
-            DocumentByConnectorCredentialPair,
-            DbDocument.id == DocumentByConnectorCredentialPair.id,
-        )
-        .where(
-            and_(
-                DocumentByConnectorCredentialPair.connector_id == connector_id,
-                DocumentByConnectorCredentialPair.credential_id == credential_id,
-                or_(
-                    DbDocument.last_modified > DbDocument.last_synced,
-                    DbDocument.last_synced.is_(None),
-                ),
-            )
+    initial_doc_ids_stmt = select(DocumentByConnectorCredentialPair.id).where(
+        and_(
+            DocumentByConnectorCredentialPair.connector_id == connector_id,
+            DocumentByConnectorCredentialPair.credential_id == credential_id,
         )
     )
 
+    stmt = (
+        select(DbDocument)
+        .where(
+            DbDocument.id.in_(initial_doc_ids_stmt),
+            or_(
+                DbDocument.last_modified
+                > DbDocument.last_synced,  # last_modified is newer than last_synced
+                DbDocument.last_synced.is_(None),  # never synced
+            ),
+        )
+        .distinct()
+    )
+
+    return stmt
+
 
 def construct_document_id_select_for_connector_credential_pair_by_needs_sync(
     connector_id: int, credential_id: int
 ) -> Select:
-    return (
-        select(DbDocument.id)
-        .join(
-            DocumentByConnectorCredentialPair,
-            DbDocument.id == DocumentByConnectorCredentialPair.id,
-        )
-        .where(
-            and_(
-                DocumentByConnectorCredentialPair.connector_id == connector_id,
-                DocumentByConnectorCredentialPair.credential_id == credential_id,
-                or_(
-                    DbDocument.last_modified > DbDocument.last_synced,
-                    DbDocument.last_synced.is_(None),
-                ),
-            )
+    initial_doc_ids_stmt = select(DocumentByConnectorCredentialPair.id).where(
+        and_(
+            DocumentByConnectorCredentialPair.connector_id == connector_id,
+            DocumentByConnectorCredentialPair.credential_id == credential_id,
         )
     )
 
+    stmt = (
+        select(DbDocument.id)
+        .where(
+            DbDocument.id.in_(initial_doc_ids_stmt),
+            or_(
+                DbDocument.last_modified
+                > DbDocument.last_synced,  # last_modified is newer than last_synced
+                DbDocument.last_synced.is_(None),  # never synced
+            ),
+        )
+        .distinct()
+    )
+
+    return stmt
+
 
 def get_all_documents_needing_vespa_sync_for_cc_pair(
     db_session: Session, cc_pair_id: int

backend/onyx/db/enums.py

@@ -73,7 +73,6 @@ class ConnectorCredentialPairStatus(str, PyEnum):
     ACTIVE = "ACTIVE"
     PAUSED = "PAUSED"
     DELETING = "DELETING"
-    INVALID = "INVALID"
 
     def is_active(self) -> bool:
         return self == ConnectorCredentialPairStatus.ACTIVE

backend/onyx/db/models.py

@@ -570,14 +570,6 @@ class Document(Base):
         back_populates="documents",
     )
 
-    __table_args__ = (
-        Index(
-            "ix_document_sync_status",
-            last_modified,
-            last_synced,
-        ),
-    )
-
 
 class Tag(Base):
     __tablename__ = "tag"

backend/onyx/llm/models.py

@@ -23,7 +23,6 @@ class PreviousMessage(BaseModel):
     message_type: MessageType
     files: list[InMemoryChatFile]
     tool_call: ToolCallFinalResult | None
-    refined_answer_improvement: bool | None
 
     @classmethod
     def from_chat_message(
@@ -48,7 +47,6 @@ class PreviousMessage(BaseModel):
             )
             if chat_message.tool_call
             else None,
-            refined_answer_improvement=chat_message.refined_answer_improvement,
         )
 
     def to_langchain_msg(self) -> BaseMessage:

backend/onyx/server/documents/cc_pair.py

@@ -25,9 +25,6 @@ from onyx.background.celery.versioned_apps.primary import app as primary_app
 from onyx.background.indexing.models import IndexAttemptErrorPydantic
 from onyx.configs.constants import OnyxCeleryPriority
 from onyx.configs.constants import OnyxCeleryTask
-from onyx.connectors.factory import validate_ccpair_for_user
-from onyx.connectors.interfaces import ConnectorValidationError
-from onyx.db.connector import delete_connector
 from onyx.db.connector_credential_pair import add_credential_to_connector
 from onyx.db.connector_credential_pair import (
     get_connector_credential_pair_from_id_for_user,
@@ -620,10 +617,6 @@ def associate_credential_to_connector(
     )
 
     try:
-        validate_ccpair_for_user(
-            connector_id, credential_id, db_session, user, tenant_id
-        )
-
         response = add_credential_to_connector(
             db_session=db_session,
             user=user,
@@ -648,27 +641,10 @@ def associate_credential_to_connector(
         )
 
         return response
-
-    except ConnectorValidationError as e:
-        # If validation fails, delete the connector and commit the changes
-        # Ensures we don't leave invalid connectors in the database
-        # NOTE: consensus is that it makes sense to unify connector and ccpair creation flows
-        # which would rid us of needing to handle cases like these
-        delete_connector(db_session, connector_id)
-        db_session.commit()
-
-        raise HTTPException(
-            status_code=400, detail="Connector validation error: " + str(e)
-        )
-
     except IntegrityError as e:
         logger.error(f"IntegrityError: {e}")
         raise HTTPException(status_code=400, detail="Name must be unique")
 
-    except Exception as e:
-        logger.exception(f"Unexpected error: {e}")
-        raise HTTPException(status_code=500, detail="Unexpected error")
-
 
 @router.delete("/connector/{connector_id}/credential/{credential_id}")
 def dissociate_credential_from_connector(

backend/onyx/server/documents/connector.py

@@ -28,7 +28,6 @@ from onyx.configs.constants import FileOrigin
 from onyx.configs.constants import MilestoneRecordType
 from onyx.configs.constants import OnyxCeleryPriority
 from onyx.configs.constants import OnyxCeleryTask
-from onyx.connectors.factory import validate_ccpair_for_user
 from onyx.connectors.google_utils.google_auth import (
     get_google_oauth_creds,
 )
@@ -62,7 +61,6 @@ from onyx.connectors.google_utils.shared_constants import DB_CREDENTIALS_DICT_TO
 from onyx.connectors.google_utils.shared_constants import (
     GoogleOAuthAuthenticationMethod,
 )
-from onyx.connectors.interfaces import ConnectorValidationError
 from onyx.db.connector import create_connector
 from onyx.db.connector import delete_connector
 from onyx.db.connector import fetch_connector_by_id
@@ -846,22 +844,11 @@ def create_connector_with_mock_credential(
             db_session=db_session,
         )
 
-        # Store the created connector and credential IDs
-        connector_id = cast(int, connector_response.id)
-        credential_id = credential.id
-
-        validate_ccpair_for_user(
-            connector_id=connector_id,
-            credential_id=credential_id,
-            db_session=db_session,
-            user=user,
-            tenant_id=tenant_id,
-        )
         response = add_credential_to_connector(
             db_session=db_session,
             user=user,
-            connector_id=connector_id,
-            credential_id=credential_id,
+            connector_id=cast(int, connector_response.id),  # will aways be an int
+            credential_id=credential.id,
             access_type=connector_data.access_type,
             cc_pair_name=connector_data.name,
             groups=connector_data.groups,
@@ -886,12 +873,9 @@ def create_connector_with_mock_credential(
             properties=None,
             db_session=db_session,
         )
+
         return response
 
-    except ConnectorValidationError as e:
-        raise HTTPException(
-            status_code=400, detail="Connector validation error: " + str(e)
-        )
     except ValueError as e:
         raise HTTPException(status_code=400, detail=str(e))
 

backend/onyx/server/documents/credential.py

@@ -7,7 +7,6 @@ from sqlalchemy.orm import Session
 from onyx.auth.users import current_admin_user
 from onyx.auth.users import current_curator_or_admin_user
 from onyx.auth.users import current_user
-from onyx.connectors.factory import validate_ccpair_for_user
 from onyx.db.credentials import alter_credential
 from onyx.db.credentials import cleanup_gmail_credentials
 from onyx.db.credentials import create_credential
@@ -18,7 +17,6 @@ from onyx.db.credentials import fetch_credentials_by_source_for_user
 from onyx.db.credentials import fetch_credentials_for_user
 from onyx.db.credentials import swap_credentials_connector
 from onyx.db.credentials import update_credential
-from onyx.db.engine import get_current_tenant_id
 from onyx.db.engine import get_session
 from onyx.db.models import DocumentSource
 from onyx.db.models import User
@@ -100,16 +98,7 @@ def swap_credentials_for_connector(
     credential_swap_req: CredentialSwapRequest,
     user: User | None = Depends(current_user),
     db_session: Session = Depends(get_session),
-    tenant_id: str | None = Depends(get_current_tenant_id),
 ) -> StatusResponse:
-    validate_ccpair_for_user(
-        credential_swap_req.connector_id,
-        credential_swap_req.new_credential_id,
-        db_session,
-        user,
-        tenant_id,
-    )
-
     connector_credential_pair = swap_credentials_connector(
         new_credential_id=credential_swap_req.new_credential_id,
         connector_id=credential_swap_req.connector_id,

backend/onyx/server/manage/users.py

@@ -311,23 +311,19 @@ def bulk_invite_users(
     all_emails = list(set(new_invited_emails) | set(initial_invited_users))
     number_of_invited_users = write_invited_users(all_emails)
 
-    # send out email invitations if enabled
-    if ENABLE_EMAIL_INVITES:
-        try:
-            for email in new_invited_emails:
-                send_user_email_invite(email, current_user, AUTH_TYPE)
-        except Exception as e:
-            logger.error(f"Error sending email invite to invited users: {e}")
-
     if not MULTI_TENANT:
         return number_of_invited_users
-
-    # for billing purposes, write to the control plane about the number of new users
     try:
         logger.info("Registering tenant users")
         fetch_ee_implementation_or_noop(
             "onyx.server.tenants.billing", "register_tenant_users", None
         )(tenant_id, get_total_users_count(db_session))
+        if ENABLE_EMAIL_INVITES:
+            try:
+                for email in new_invited_emails:
+                    send_user_email_invite(email, current_user)
+            except Exception as e:
+                logger.error(f"Error sending email invite to invited users: {e}")
 
         return number_of_invited_users
     except Exception as e:

backend/onyx/server/settings/models.py

@@ -45,7 +45,7 @@ class Settings(BaseModel):
     gpu_enabled: bool | None = None
     application_status: ApplicationStatus = ApplicationStatus.ACTIVE
     anonymous_user_enabled: bool | None = None
-    pro_search_enabled: bool | None = None
+    pro_search_disabled: bool | None = None
 
     temperature_override_enabled: bool = False
     auto_scroll: bool = False

backend/requirements/ee.txt

@@ -1,4 +1,3 @@
 cohere==5.6.1
 posthog==3.7.4
 python3-saml==1.15.0
-xmlsec==1.3.14

backend/tests/daily/connectors/web/test_web_connector.py

@@ -1,44 +0,0 @@
-import pytest
-
-from onyx.connectors.models import Document
-from onyx.connectors.web.connector import WEB_CONNECTOR_VALID_SETTINGS
-from onyx.connectors.web.connector import WebConnector
-
-
-# NOTE(rkuo): we will probably need to adjust this test to point at our own test site
-# to avoid depending on a third party site
-@pytest.fixture
-def web_connector(request: pytest.FixtureRequest) -> WebConnector:
-    scroll_before_scraping = request.param
-    connector = WebConnector(
-        base_url="https://developer.onewelcome.com",
-        web_connector_type=WEB_CONNECTOR_VALID_SETTINGS.SINGLE.value,
-        scroll_before_scraping=scroll_before_scraping,
-    )
-    return connector
-
-
-@pytest.mark.parametrize("web_connector", [True], indirect=True)
-def test_web_connector_scroll(web_connector: WebConnector) -> None:
-    all_docs: list[Document] = []
-    document_batches = web_connector.load_from_state()
-    for doc_batch in document_batches:
-        for doc in doc_batch:
-            all_docs.append(doc)
-
-    assert len(all_docs) == 1
-    doc = all_docs[0]
-    assert "Onegini Identity Cloud" in doc.sections[0].text
-
-
-@pytest.mark.parametrize("web_connector", [False], indirect=True)
-def test_web_connector_no_scroll(web_connector: WebConnector) -> None:
-    all_docs: list[Document] = []
-    document_batches = web_connector.load_from_state()
-    for doc_batch in document_batches:
-        for doc in doc_batch:
-            all_docs.append(doc)
-
-    assert len(all_docs) == 1
-    doc = all_docs[0]
-    assert "Onegini Identity Cloud" not in doc.sections[0].text

backend/tests/integration/common_utils/managers/connector.py

@@ -30,8 +30,7 @@ class ConnectorManager:
             name=name,
             source=source,
             input_type=input_type,
-            connector_specific_config=connector_specific_config
-            or {"file_locations": []},
+            connector_specific_config=connector_specific_config or {},
             access_type=access_type,
             groups=groups or [],
         )

deployment/data/nginx/app.conf.template

@@ -4,24 +4,6 @@ log_format custom_main '$remote_addr - $remote_user [$time_local] "$request" '
                 '"$http_user_agent" "$http_x_forwarded_for" '
                 'rt=$request_time';
 
-# Map X-Forwarded-Proto or fallback to $scheme
-map $http_x_forwarded_proto $forwarded_proto {
-    default $http_x_forwarded_proto;
-    ""      $scheme;
-}
-
-# Map X-Forwarded-Host or fallback to $host
-map $http_x_forwarded_host $forwarded_host {
-    default $http_x_forwarded_host;
-    ""      $host;
-}
-
-# Map X-Forwarded-Port or fallback to server port
-map $http_x_forwarded_port $forwarded_port {
-    default $http_x_forwarded_port;
-    ""      $server_port;
-}
-
 upstream api_server {
     # fail_timeout=0 means we always retry an upstream even if it failed
     # to return a good HTTP response
@@ -39,7 +21,8 @@ upstream web_server {
 }
 
 server {
-    listen 80 default_server;
+    listen 80;
+    server_name ${DOMAIN};
 
     client_max_body_size 5G;    # Maximum upload size
 
@@ -53,9 +36,8 @@ server {
         # misc headers
         proxy_set_header X-Real-IP $remote_addr;
         proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-        proxy_set_header X-Forwarded-Proto $forwarded_proto;
-        proxy_set_header X-Forwarded-Host $forwarded_host; 
-        proxy_set_header X-Forwarded-Port $forwarded_port;
+        proxy_set_header X-Forwarded-Proto $scheme;
+        proxy_set_header X-Forwarded-Host $host; 
         proxy_set_header Host $host;
 
         # need to use 1.1 to support chunked transfers
@@ -72,9 +54,8 @@ server {
         # misc headers
         proxy_set_header X-Real-IP $remote_addr;
         proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-        proxy_set_header X-Forwarded-Proto $forwarded_proto;
-        proxy_set_header X-Forwarded-Host $forwarded_host; 
-        proxy_set_header X-Forwarded-Port $forwarded_port;
+        proxy_set_header X-Forwarded-Proto $scheme;
+        proxy_set_header X-Forwarded-Host $host; 
         proxy_set_header Host $host;
 
         proxy_http_version 1.1;
@@ -91,25 +72,14 @@ server {
 }
 
 server {
-    listen 443 ssl default_server;
+    listen 443 ssl;
+    server_name ${DOMAIN};
 
     client_max_body_size 5G;    # Maximum upload size
     
     location / {
-        # misc headers
-        proxy_set_header X-Real-IP $remote_addr;
-        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-        # don't use forwarded schema, host, or port here - this is the entry point
-        proxy_set_header X-Forwarded-Proto $scheme;
-        proxy_set_header X-Forwarded-Host $host; 
-        proxy_set_header X-Forwarded-Port $server_port;
-        proxy_set_header Host $host;
-
         proxy_http_version 1.1;
         proxy_buffering off;
-        # we don't want nginx trying to do something clever with
-        # redirects, we set the Host: header above already.
-        proxy_redirect off;
         proxy_pass http://localhost:80;
     }
 

deployment/data/nginx/app.conf.template.dev

@@ -21,7 +21,8 @@ upstream web_server {
 }
 
 server {
-    listen 80 default_server;
+    listen 80;
+    server_name ${DOMAIN};
 
     client_max_body_size 5G;    # Maximum upload size    
 
@@ -36,8 +37,7 @@ server {
         proxy_set_header X-Real-IP $remote_addr;
         proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
         proxy_set_header X-Forwarded-Proto $scheme;
-        proxy_set_header X-Forwarded-Host $host;
-        proxy_set_header X-Forwarded-Port $server_port;
+        proxy_set_header X-Forwarded-Host $host; 
         proxy_set_header Host $host;
 
         # need to use 1.1 to support chunked transfers
@@ -55,8 +55,7 @@ server {
         proxy_set_header X-Real-IP $remote_addr;
         proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
         proxy_set_header X-Forwarded-Proto $scheme;
-        proxy_set_header X-Forwarded-Host $host;
-        proxy_set_header X-Forwarded-Port $server_port;
+        proxy_set_header X-Forwarded-Host $host; 
         proxy_set_header Host $host;
 
         proxy_http_version 1.1;

deployment/data/nginx/app.conf.template.no-letsencrypt

@@ -4,24 +4,6 @@ log_format custom_main '$remote_addr - $remote_user [$time_local] "$request" '
                 '"$http_user_agent" "$http_x_forwarded_for" '
                 'rt=$request_time';
 
-# Map X-Forwarded-Proto or fallback to $scheme
-map $http_x_forwarded_proto $forwarded_proto {
-    default $http_x_forwarded_proto;
-    ""      $scheme;
-}
-
-# Map X-Forwarded-Host or fallback to $host
-map $http_x_forwarded_host $forwarded_host {
-    default $http_x_forwarded_host;
-    ""      $host;
-}
-
-# Map X-Forwarded-Port or fallback to server port
-map $http_x_forwarded_port $forwarded_port {
-    default $http_x_forwarded_port;
-    ""      $server_port;
-}
-
 upstream api_server {
     # fail_timeout=0 means we always retry an upstream even if it failed
     # to return a good HTTP response
@@ -39,7 +21,8 @@ upstream web_server {
 }
 
 server {
-    listen 80 default_server;
+    listen 80;
+    server_name ${DOMAIN};
 
     client_max_body_size 5G;    # Maximum upload size
 
@@ -53,9 +36,8 @@ server {
         # misc headers
         proxy_set_header X-Real-IP $remote_addr;
         proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-        proxy_set_header X-Forwarded-Proto $forwarded_proto;
-        proxy_set_header X-Forwarded-Host $forwarded_host; 
-        proxy_set_header X-Forwarded-Port $forwarded_port;
+        proxy_set_header X-Forwarded-Proto $scheme;
+        proxy_set_header X-Forwarded-Host $host; 
         proxy_set_header Host $host;
 
         # need to use 1.1 to support chunked transfers
@@ -72,9 +54,8 @@ server {
         # misc headers
         proxy_set_header X-Real-IP $remote_addr;
         proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-        proxy_set_header X-Forwarded-Proto $forwarded_proto;
-        proxy_set_header X-Forwarded-Host $forwarded_host; 
-        proxy_set_header X-Forwarded-Port $forwarded_port;
+        proxy_set_header X-Forwarded-Proto $scheme;
+        proxy_set_header X-Forwarded-Host $host; 
         proxy_set_header Host $host;
 
         proxy_http_version 1.1;
@@ -87,25 +68,14 @@ server {
 }
 
 server {
-    listen 443 ssl default_server;
+    listen 443 ssl;
+    server_name ${DOMAIN};
 
     client_max_body_size 5G;    # Maximum upload size
     
     location / {
-        # misc headers
-        proxy_set_header X-Real-IP $remote_addr;
-        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-        # don't use forwarded schema, host, or port here - this is the entry point
-        proxy_set_header X-Forwarded-Proto $scheme;
-        proxy_set_header X-Forwarded-Host $host; 
-        proxy_set_header X-Forwarded-Port $server_port;
-        proxy_set_header Host $host;
-
         proxy_http_version 1.1;
         proxy_buffering off;
-        # we don't want nginx trying to do something clever with
-        # redirects, we set the Host: header above already.
-        proxy_redirect off;
         proxy_pass http://localhost:80;
     }
 

deployment/data/nginx/run-nginx.sh

@@ -1,5 +1,5 @@
 # fill in the template
-envsubst '$SSL_CERT_FILE_NAME $SSL_CERT_KEY_FILE_NAME' < "/etc/nginx/conf.d/$1" > /etc/nginx/conf.d/app.conf
+envsubst '$DOMAIN $SSL_CERT_FILE_NAME $SSL_CERT_KEY_FILE_NAME' < "/etc/nginx/conf.d/$1" > /etc/nginx/conf.d/app.conf
 
 # wait for the api_server to be ready
 echo "Waiting for API server to boot up; this may take a minute or two..."

web/src/app/admin/configuration/search/UpgradingPage.tsx

@@ -67,13 +67,12 @@ export default function UpgradingPage({
   };
   const statusOrder: Record<ValidStatuses, number> = useMemo(
     () => ({
-      invalid: 0,
-      failed: 1,
-      canceled: 2,
-      completed_with_errors: 3,
-      not_started: 4,
-      in_progress: 5,
-      success: 6,
+      failed: 0,
+      canceled: 1,
+      completed_with_errors: 2,
+      not_started: 3,
+      in_progress: 4,
+      success: 5,
     }),
     []
   );

web/src/app/admin/connector/[ccPairId]/ReIndexButton.tsx

@@ -4,14 +4,9 @@ import { PopupSpec, usePopup } from "@/components/admin/connectors/Popup";
 import { Button } from "@/components/ui/button";
 import Text from "@/components/ui/text";
 import { triggerIndexing } from "./lib";
-import { mutate } from "swr";
-import { buildCCPairInfoUrl, getTooltipMessage } from "./lib";
 import { useState } from "react";
 import { Modal } from "@/components/Modal";
 import { Separator } from "@/components/ui/separator";
-import { ConnectorCredentialPairStatus } from "./types";
-import { CCPairStatus } from "@/components/Status";
-import { getCCPairStatusMessage } from "@/lib/ccPair";
 
 function ReIndexPopup({
   connectorId,
@@ -88,16 +83,16 @@ export function ReIndexButton({
   ccPairId,
   connectorId,
   credentialId,
-  isIndexing,
   isDisabled,
-  ccPairStatus,
+  isIndexing,
+  isDeleting,
 }: {
   ccPairId: number;
   connectorId: number;
   credentialId: number;
-  isIndexing: boolean;
   isDisabled: boolean;
-  ccPairStatus: ConnectorCredentialPairStatus;
+  isIndexing: boolean;
+  isDeleting: boolean;
 }) {
   const { popup, setPopup } = usePopup();
   const [reIndexPopupVisible, setReIndexPopupVisible] = useState(false);
@@ -120,14 +115,18 @@ export function ReIndexButton({
         onClick={() => {
           setReIndexPopupVisible(true);
         }}
-        disabled={
-          isDisabled ||
-          ccPairStatus == ConnectorCredentialPairStatus.DELETING ||
-          ccPairStatus == ConnectorCredentialPairStatus.PAUSED
+        disabled={isDisabled || isDeleting}
+        tooltip={
+          isDeleting
+            ? "Cannot index while connector is deleting"
+            : isIndexing
+              ? "Indexing is already in progress"
+              : isDisabled
+                ? "Connector must be re-enabled before indexing"
+                : undefined
         }
-        tooltip={getCCPairStatusMessage(isDisabled, isIndexing, ccPairStatus)}
       >
-        Re-Index
+        Index
       </Button>
     </>
   );

web/src/app/admin/connector/[ccPairId]/lib.ts

@@ -40,24 +40,3 @@ export async function triggerIndexing(
   }
   mutate(buildCCPairInfoUrl(ccPairId));
 }
-
-export function getTooltipMessage(
-  isInvalid: boolean,
-  isDeleting: boolean,
-  isIndexing: boolean,
-  isDisabled: boolean
-): string | undefined {
-  if (isInvalid) {
-    return "Connector is in an invalid state. Please update the credentials or configuration before re-indexing.";
-  }
-  if (isDeleting) {
-    return "Cannot index while connector is deleting";
-  }
-  if (isIndexing) {
-    return "Indexing is already in progress";
-  }
-  if (isDisabled) {
-    return "Connector must be re-enabled before indexing";
-  }
-  return undefined;
-}

web/src/app/admin/connector/[ccPairId]/page.tsx

@@ -43,7 +43,6 @@ import IndexAttemptErrorsModal from "./IndexAttemptErrorsModal";
 import usePaginatedFetch from "@/hooks/usePaginatedFetch";
 import { IndexAttemptSnapshot } from "@/lib/types";
 import { Spinner } from "@/components/Spinner";
-import { Callout } from "@/components/ui/callout";
 
 // synchronize these validations with the SQLAlchemy connector class until we have a
 // centralized schema for both frontend and backend
@@ -364,7 +363,6 @@ function Main({ ccPairId }: { ccPairId: number }) {
           <div className="ml-auto flex gap-x-2">
             <ReIndexButton
               ccPairId={ccPair.id}
-              ccPairStatus={ccPair.status}
               connectorId={ccPair.connector.id}
               credentialId={ccPair.credential.id}
               isDisabled={
@@ -372,6 +370,7 @@ function Main({ ccPairId }: { ccPairId: number }) {
                 ccPair.status === ConnectorCredentialPairStatus.PAUSED
               }
               isIndexing={ccPair.indexing}
+              isDeleting={isDeleting}
             />
 
             {!isDeleting && <ModifyStatusButtonCluster ccPair={ccPair} />}
@@ -380,7 +379,8 @@ function Main({ ccPairId }: { ccPairId: number }) {
       </div>
       <CCPairStatus
         status={ccPair.last_index_attempt_status || "not_started"}
-        ccPairStatus={ccPair.status}
+        disabled={ccPair.status === ConnectorCredentialPairStatus.PAUSED}
+        isDeleting={isDeleting}
       />
       <div className="text-sm mt-1">
         Creator:{" "}
@@ -424,16 +424,6 @@ function Main({ ccPairId }: { ccPairId: number }) {
             />
           </>
         )}
-
-      {ccPair.status === ConnectorCredentialPairStatus.INVALID && (
-        <div className="mt-2">
-          <Callout type="warning" title="Invalid Connector State">
-            This connector is in an invalid state. Please update your
-            credentials or create a new connector before re-indexing.
-          </Callout>
-        </div>
-      )}
-
       <Separator />
       <ConfigDisplay
         connectorSpecificConfig={ccPair.connector.connector_specific_config}

web/src/app/admin/connector/[ccPairId]/types.ts

@@ -12,7 +12,6 @@ export enum ConnectorCredentialPairStatus {
   ACTIVE = "ACTIVE",
   PAUSED = "PAUSED",
   DELETING = "DELETING",
-  INVALID = "INVALID",
 }
 
 export interface CCPairFullInfo {

web/src/app/admin/connectors/[connector]/AddConnectorPage.tsx

@@ -418,7 +418,7 @@ export default function AddConnector({
           } else {
             const errorData = await linkCredentialResponse.json();
             setPopup({
-              message: errorData.message || errorData.detail,
+              message: errorData.message,
               type: "error",
             });
           }

web/src/app/admin/indexing/status/CCPairIndexingStatusTable.tsx

@@ -159,19 +159,6 @@ function ConnectorRow({
           Paused
         </Badge>
       );
-    } else if (
-      ccPairsIndexingStatus.cc_pair_status ===
-      ConnectorCredentialPairStatus.INVALID
-    ) {
-      return (
-        <Badge
-          tooltip="Connector is in an invalid state. Please update the credentials or create a new connector."
-          circle
-          variant="invalid"
-        >
-          Invalid
-        </Badge>
-      );
     }
 
     // ACTIVE case

web/src/app/admin/settings/SettingsForm.tsx

@@ -240,11 +240,11 @@ export function SettingsForm() {
       />
 
       <Checkbox
-        label="Agent Search"
-        sublabel="If set, users will be able to use Agent Search."
-        checked={settings.pro_search_enabled ?? true}
+        label="Pro Search Disabled"
+        sublabel="If set, users will not be able to use Pro Search."
+        checked={settings.pro_search_disabled ?? false}
         onChange={(e) =>
-          handleToggleSettingsField("pro_search_enabled", e.target.checked)
+          handleToggleSettingsField("pro_search_disabled", e.target.checked)
         }
       />
 

web/src/app/admin/settings/interfaces.ts

@@ -10,7 +10,7 @@ export interface Settings {
   notifications: Notification[];
   needs_reindexing: boolean;
   gpu_enabled: boolean;
-  pro_search_enabled: boolean | null;
+  pro_search_disabled: boolean | null;
   application_status: ApplicationStatus;
   auto_scroll: boolean;
   temperature_override_enabled: boolean;

web/src/app/chat/ChatPage.tsx

@@ -23,7 +23,6 @@ import {
   SubQuestionDetail,
   constructSubQuestions,
   DocumentsResponse,
-  AgenticMessageResponseIDInfo,
 } from "./interfaces";
 
 import Prism from "prismjs";
@@ -1281,8 +1280,6 @@ export function ChatPage({
     let toolCall: ToolCallMetadata | null = null;
     let isImprovement: boolean | undefined = undefined;
     let isStreamingQuestions = true;
-    let includeAgentic = false;
-    let secondLevelMessageId: number | null = null;
 
     let initialFetchDetails: null | {
       user_message_id: number;
@@ -1339,7 +1336,7 @@ export function ChatPage({
           searchParams.get(SEARCH_PARAM_NAMES.SYSTEM_PROMPT) || undefined,
         useExistingUserMessage: isSeededChat,
         useLanggraph:
-          settings?.settings.pro_search_enabled &&
+          !settings?.settings.pro_search_disabled &&
           proSearchEnabled &&
           retrievalEnabled,
       });
@@ -1420,17 +1417,6 @@ export function ChatPage({
             resetRegenerationState();
           } else {
             const { user_message_id, frozenMessageMap } = initialFetchDetails;
-            if (Object.hasOwn(packet, "agentic_message_ids")) {
-              const agenticMessageIds = (packet as AgenticMessageResponseIDInfo)
-                .agentic_message_ids;
-              const level1MessageId = agenticMessageIds.find(
-                (item) => item.level === 1
-              )?.message_id;
-              if (level1MessageId) {
-                secondLevelMessageId = level1MessageId;
-                includeAgentic = true;
-              }
-            }
 
             setChatState((prevState) => {
               if (prevState.get(chatSessionIdRef.current!) === "loading") {
@@ -1582,10 +1568,7 @@ export function ChatPage({
                   };
                 }
               );
-            } else if (
-              Object.hasOwn(packet, "error") &&
-              (packet as any).error != null
-            ) {
+            } else if (Object.hasOwn(packet, "error")) {
               if (
                 sub_questions.length > 0 &&
                 sub_questions
@@ -1597,8 +1580,8 @@ export function ChatPage({
                 setAgenticGenerating(false);
                 setAlternativeGeneratingAssistant(null);
                 setSubmittedMessage("");
-
-                throw new Error((packet as StreamingError).error);
+                return;
+                // throw new Error((packet as StreamingError).error);
               } else {
                 error = (packet as StreamingError).error;
                 stackTrace = (packet as StreamingError).stack_trace;
@@ -1681,19 +1664,6 @@ export function ChatPage({
                 second_level_generating: second_level_generating,
                 agentic_docs: agenticDocs,
               },
-              ...(includeAgentic
-                ? [
-                    {
-                      messageId: secondLevelMessageId!,
-                      message: second_level_answer,
-                      type: "assistant" as const,
-                      files: [],
-                      toolCall: null,
-                      parentMessageId:
-                        initialFetchDetails.assistant_message_id!,
-                    },
-                  ]
-                : []),
             ]);
           }
         }
@@ -2722,11 +2692,6 @@ export function ChatPage({
                                     ? messageHistory[i + 1]?.documents
                                     : undefined;
 
-                                const nextMessage =
-                                  messageHistory[i + 1]?.type === "assistant"
-                                    ? messageHistory[i + 1]
-                                    : undefined;
-
                                 return (
                                   <div
                                     className="text-text"
@@ -2755,10 +2720,7 @@ export function ChatPage({
                                             selectedMessageForDocDisplay ==
                                               secondLevelMessage?.messageId)
                                         }
-                                        isImprovement={
-                                          message.isImprovement ||
-                                          nextMessage?.isImprovement
-                                        }
+                                        isImprovement={message.isImprovement}
                                         secondLevelGenerating={
                                           (message.second_level_generating &&
                                             currentSessionChatState !==

web/src/app/chat/input/ChatInputBar.tsx

@@ -805,12 +805,13 @@ export function ChatInputBar({
                 )}
               </div>
               <div className="flex items-center my-auto">
-                {retrievalEnabled && settings?.settings.pro_search_enabled && (
-                  <AgenticToggle
-                    proSearchEnabled={proSearchEnabled}
-                    setProSearchEnabled={setProSearchEnabled}
-                  />
-                )}
+                {retrievalEnabled &&
+                  !settings?.settings.pro_search_disabled && (
+                    <AgenticToggle
+                      proSearchEnabled={proSearchEnabled}
+                      setProSearchEnabled={setProSearchEnabled}
+                    />
+                  )}
                 <button
                   id="onyx-chat-input-send-button"
                   className={`cursor-pointer ${

web/src/app/chat/input/LLMPopover.tsx

@@ -1,4 +1,9 @@
-import React, { useState, useEffect } from "react";
+import React, {
+  useState,
+  useEffect,
+  useCallback,
+  useLayoutEffect,
+} from "react";
 import {
   Popover,
   PopoverContent,
@@ -28,6 +33,7 @@ import { FiAlertTriangle } from "react-icons/fi";
 
 import { Slider } from "@/components/ui/slider";
 import { useUser } from "@/components/user/UserProvider";
+import { TruncatedText } from "@/components/ui/truncatedText";
 
 interface LLMPopoverProps {
   llmProviders: LLMProviderDescriptor[];
@@ -160,9 +166,7 @@ export default function LLMPopover({
                     size: 16,
                     className: "flex-none my-auto text-black",
                   })}
-                  <span className="line-clamp-1 ">
-                    {getDisplayNameForModel(name)}
-                  </span>
+                  <TruncatedText text={getDisplayNameForModel(name)} />
                   {(() => {
                     if (currentAssistant?.llm_model_version_override === name) {
                       return (

web/src/app/chat/interfaces.ts

@@ -155,15 +155,6 @@ export interface MessageResponseIDInfo {
   reserved_assistant_message_id: number;
 }
 
-export interface AgentMessageIDInfo {
-  level: number;
-  message_id: number;
-}
-
-export interface AgenticMessageResponseIDInfo {
-  agentic_message_ids: AgentMessageIDInfo[];
-}
-
 export interface DocumentsResponse {
   top_documents: OnyxDocument[];
   rephrased_query: string | null;

web/src/app/chat/lib.tsx

@@ -25,7 +25,6 @@ import {
   RetrievalType,
   StreamingError,
   ToolCallMetadata,
-  AgenticMessageResponseIDInfo,
 } from "./interfaces";
 import { Persona } from "../admin/assistants/interfaces";
 import { ReadonlyURLSearchParams } from "next/navigation";
@@ -155,8 +154,7 @@ export type PacketType =
   | AgentAnswerPiece
   | SubQuestionPiece
   | ExtendedToolResponse
-  | RefinedAnswerImprovement
-  | AgenticMessageResponseIDInfo;
+  | RefinedAnswerImprovement;
 
 export async function* sendMessage({
   regenerate,

web/src/app/chat/sessionSidebar/HistorySidebar.tsx

@@ -4,10 +4,7 @@ import React, {
   ForwardedRef,
   forwardRef,
   useContext,
-  useState,
   useCallback,
-  useLayoutEffect,
-  useRef,
 } from "react";
 import Link from "next/link";
 import {
@@ -50,9 +47,9 @@ import {
 } from "@dnd-kit/sortable";
 import { useSortable } from "@dnd-kit/sortable";
 import { CSS } from "@dnd-kit/utilities";
-import { CirclePlus, CircleX, PinIcon } from "lucide-react";
+import { CircleX, PinIcon } from "lucide-react";
 import { restrictToVerticalAxis } from "@dnd-kit/modifiers";
-import { turborepoTraceAccess } from "next/dist/build/turborepo-access-trace";
+import { TruncatedText } from "@/components/ui/truncatedText";
 
 interface HistorySidebarProps {
   liveAssistant?: Persona | null;
@@ -101,24 +98,6 @@ const SortableAssistant: React.FC<SortableAssistantProps> = ({
     ...(isDragging ? { zIndex: 1000, position: "relative" as const } : {}),
   };
 
-  const nameRef = useRef<HTMLParagraphElement>(null);
-  const hiddenNameRef = useRef<HTMLSpanElement>(null);
-  const [isNameTruncated, setIsNameTruncated] = useState(false);
-
-  useLayoutEffect(() => {
-    const checkTruncation = () => {
-      if (nameRef.current && hiddenNameRef.current) {
-        const visibleWidth = nameRef.current.offsetWidth;
-        const fullTextWidth = hiddenNameRef.current.offsetWidth;
-        setIsNameTruncated(fullTextWidth > visibleWidth);
-      }
-    };
-
-    checkTruncation();
-    window.addEventListener("resize", checkTruncation);
-    return () => window.removeEventListener("resize", checkTruncation);
-  }, [assistant.name]);
-
   return (
     <div
       ref={setNodeRef}
@@ -146,27 +125,11 @@ const SortableAssistant: React.FC<SortableAssistantProps> = ({
         } relative flex items-center gap-x-2 py-1 px-2 rounded-md`}
       >
         <AssistantIcon assistant={assistant} size={16} className="flex-none" />
-        <TooltipProvider>
-          <Tooltip>
-            <TooltipTrigger asChild>
-              <p
-                ref={nameRef}
-                className="text-base text-left w-fit line-clamp-1 text-ellipsis text-black dark:text-[#D4D4D4]"
-              >
-                {assistant.name}
-              </p>
-            </TooltipTrigger>
-            {isNameTruncated && (
-              <TooltipContent>{assistant.name}</TooltipContent>
-            )}
-          </Tooltip>
-        </TooltipProvider>
-        <span
-          ref={hiddenNameRef}
-          className="absolute left-[-9999px] whitespace-nowrap"
-        >
-          {assistant.name}
-        </span>
+        <TruncatedText
+          className="text-base mr-4 text-left w-fit line-clamp-1 text-ellipsis text-black dark:text-[#D4D4D4]"
+          text={assistant.name}
+        />
+
         <TooltipProvider>
           <Tooltip>
             <TooltipTrigger asChild>

web/src/app/layout.tsx

@@ -21,9 +21,11 @@ import { fetchAssistantData } from "@/lib/chat/fetchAssistantdata";
 import { AppProvider } from "@/components/context/AppProvider";
 import { PHProvider } from "./providers";
 import { getCurrentUserSS } from "@/lib/userSS";
+import CardSection from "@/components/admin/CardSection";
 import { Suspense } from "react";
 import PostHogPageView from "./PostHogPageView";
 import Script from "next/script";
+import { LogoType } from "@/components/logo/Logo";
 import { Hanken_Grotesk } from "next/font/google";
 import { WebVitals } from "./web-vitals";
 import { ThemeProvider } from "next-themes";

web/src/components/HoverPopup.tsx

@@ -8,32 +8,58 @@ interface HoverPopupProps {
   style?: "basic" | "dark";
 }
 
-import {
-  Tooltip,
-  TooltipContent,
-  TooltipProvider,
-  TooltipTrigger,
-} from "@/components/ui/tooltip";
-
 export const HoverPopup = ({
   mainContent,
   popupContent,
   classNameModifications,
   direction = "bottom",
+  style = "basic",
 }: HoverPopupProps) => {
+  const [hovered, setHovered] = useState(false);
+
+  let popupDirectionClass;
+  let popupStyle = {};
+  switch (direction) {
+    case "left":
+      popupDirectionClass = "top-0 left-0 transform";
+      popupStyle = { transform: "translateX(calc(-100% - 5px))" };
+      break;
+    case "left-top":
+      popupDirectionClass = "bottom-0 left-0";
+      popupStyle = { transform: "translate(calc(-100% - 5px), 0)" };
+      break;
+    case "bottom":
+      popupDirectionClass = "top-0 left-0 mt-6 pt-2";
+      break;
+    case "top":
+      popupDirectionClass = "top-0 left-0 translate-y-[-100%] pb-2";
+      break;
+  }
+
   return (
-    <TooltipProvider>
-      <Tooltip>
-        <TooltipTrigger asChild>
-          <div>{mainContent}</div>
-        </TooltipTrigger>
-        <TooltipContent
-          side={direction === "left-top" ? "left" : direction}
-          className={classNameModifications}
+    <div
+      className="relative flex"
+      onMouseEnter={() => {
+        setHovered(true);
+      }}
+      onMouseLeave={() => setHovered(false)}
+    >
+      {hovered && (
+        <div
+          className={`absolute ${popupDirectionClass} z-30`}
+          style={popupStyle}
         >
-          {popupContent}
-        </TooltipContent>
-      </Tooltip>
-    </TooltipProvider>
+          <div
+            className={
+              `px-3 py-2 rounded bg-background border border-border` +
+              (classNameModifications || "")
+            }
+          >
+            {popupContent}
+          </div>
+        </div>
+      )}
+      <div>{mainContent}</div>
+    </div>
   );
 };

web/src/components/Status.tsx

@@ -10,7 +10,6 @@ import {
   FiPauseCircle,
 } from "react-icons/fi";
 import { HoverPopup } from "./HoverPopup";
-import { ConnectorCredentialPairStatus } from "@/app/admin/connector/[ccPairId]/types";
 
 export function IndexAttemptStatus({
   status,
@@ -71,12 +70,6 @@ export function IndexAttemptStatus({
         Canceled
       </Badge>
     );
-  } else if (status === "invalid") {
-    badge = (
-      <Badge variant="invalid" icon={FiAlertTriangle}>
-        Invalid
-      </Badge>
-    );
   } else {
     badge = (
       <Badge variant="outline" icon={FiMinus}>
@@ -90,33 +83,29 @@ export function IndexAttemptStatus({
 
 export function CCPairStatus({
   status,
-  ccPairStatus,
+  disabled,
+  isDeleting,
   size = "md",
 }: {
   status: ValidStatuses;
-  ccPairStatus: ConnectorCredentialPairStatus;
+  disabled: boolean;
+  isDeleting: boolean;
   size?: "xs" | "sm" | "md" | "lg";
 }) {
   let badge;
 
-  if (ccPairStatus == ConnectorCredentialPairStatus.DELETING) {
+  if (isDeleting) {
     badge = (
       <Badge variant="destructive" icon={FiAlertTriangle}>
         Deleting
       </Badge>
     );
-  } else if (ccPairStatus == ConnectorCredentialPairStatus.PAUSED) {
+  } else if (disabled) {
     badge = (
       <Badge variant="paused" icon={FiPauseCircle}>
         Paused
       </Badge>
     );
-  } else if (ccPairStatus == ConnectorCredentialPairStatus.INVALID) {
-    badge = (
-      <Badge variant="invalid" icon={FiAlertTriangle}>
-        Invalid
-      </Badge>
-    );
   } else if (status === "failed") {
     badge = (
       <Badge variant="destructive" icon={FiAlertTriangle}>

web/src/components/credentials/CredentialSection.tsx

@@ -79,24 +79,14 @@ export default function CredentialSection({
     selectedCredential: Credential<any>,
     connectorId: number
   ) => {
-    const response = await swapCredential(selectedCredential.id, connectorId);
-    if (response.ok) {
-      mutate(buildSimilarCredentialInfoURL(sourceType));
-      refresh();
+    await swapCredential(selectedCredential.id, connectorId);
+    mutate(buildSimilarCredentialInfoURL(sourceType));
+    refresh();
 
-      setPopup({
-        message: "Swapped credential successfully!",
-        type: "success",
-      });
-    } else {
-      const errorData = await response.json();
-      setPopup({
-        message: `Issue swapping credential: ${
-          errorData.detail || errorData.message || "Unknown error"
-        }`,
-        type: "error",
-      });
-    }
+    setPopup({
+      message: "Swapped credential succesfully!",
+      type: "success",
+    });
   };
 
   const onUpdateCredential = async (

web/src/components/settings/lib.ts

@@ -51,7 +51,7 @@ export async function fetchSettingsSS(): Promise<CombinedSettings | null> {
           notifications: [],
           needs_reindexing: false,
           anonymous_user_enabled: false,
-          pro_search_enabled: true,
+          pro_search_disabled: false,
           temperature_override_enabled: true,
         };
       } else {
@@ -95,8 +95,8 @@ export async function fetchSettingsSS(): Promise<CombinedSettings | null> {
       }
     }
 
-    if (settings.pro_search_enabled == null) {
-      settings.pro_search_enabled = true;
+    if (enterpriseSettings && settings.pro_search_disabled == null) {
+      settings.pro_search_disabled = true;
     }
 
     const webVersion = getWebVersion();

web/src/components/ui/badge.tsx

@@ -1,11 +1,6 @@
 import * as React from "react";
 import { cva, type VariantProps } from "class-variance-authority";
-import {
-  Tooltip,
-  TooltipContent,
-  TooltipProvider,
-  TooltipTrigger,
-} from "@/components/ui/tooltip";
+
 import { cn } from "@/lib/utils";
 
 const badgeVariants = cva(
@@ -13,8 +8,6 @@ const badgeVariants = cva(
   {
     variants: {
       variant: {
-        invalid:
-          "border-orange-200 bg-orange-50 text-orange-600 dark:border-orange-700 dark:bg-orange-900 dark:text-orange-50",
         outline:
           "border-neutral-200 bg-neutral-50 text-neutral-600 dark:border-neutral-700 dark:bg-neutral-900 dark:text-neutral-50",
         purple:
@@ -64,13 +57,11 @@ function Badge({
   icon: Icon,
   size = "sm",
   circle,
-  tooltip,
   ...props
 }: BadgeProps & {
   icon?: React.ElementType;
   size?: "sm" | "md" | "xs";
   circle?: boolean;
-  tooltip?: string;
 }) {
   const sizeClasses = {
     sm: "px-2.5 py-0.5 text-xs",
@@ -78,7 +69,7 @@ function Badge({
     xs: "px-1.5 py-0.25 text-[.5rem]",
   };
 
-  const BadgeContent = (
+  return (
     <div
       className={cn(
         "flex-none inline-flex items-center whitespace-nowrap overflow-hidden",
@@ -107,21 +98,6 @@ function Badge({
       <span className="truncate">{props.children}</span>
     </div>
   );
-
-  if (tooltip) {
-    return (
-      <TooltipProvider>
-        <Tooltip>
-          <TooltipTrigger asChild>{BadgeContent}</TooltipTrigger>
-          <TooltipContent>
-            <p>{tooltip}</p>
-          </TooltipContent>
-        </Tooltip>
-      </TooltipProvider>
-    );
-  }
-
-  return BadgeContent;
 }
 
 export { Badge, badgeVariants };

web/src/components/ui/button.tsx

@@ -88,6 +88,7 @@ export interface ButtonProps
   tooltip?: string;
   reverse?: boolean;
 }
+
 const Button = React.forwardRef<HTMLButtonElement, ButtonProps>(
   (
     {
@@ -123,9 +124,7 @@ const Button = React.forwardRef<HTMLButtonElement, ButtonProps>(
       return (
         <TooltipProvider>
           <Tooltip>
-            <TooltipTrigger>
-              <div>{button}</div>
-            </TooltipTrigger>
+            <TooltipTrigger asChild>{button}</TooltipTrigger>
             <TooltipContent showTick={true}>
               <p>{tooltip}</p>
             </TooltipContent>

web/src/components/ui/truncatedText.tsx

@@ -0,0 +1,86 @@
+import React, {
+  useState,
+  useRef,
+  useLayoutEffect,
+  HTMLAttributes,
+} from "react";
+import {
+  Tooltip,
+  TooltipContent,
+  TooltipProvider,
+  TooltipTrigger,
+} from "@/components/ui/tooltip";
+
+interface TruncatedTextProps extends HTMLAttributes<HTMLSpanElement> {
+  text: string;
+  tooltipClassName?: string;
+  tooltipSide?: "top" | "right" | "bottom" | "left";
+  tooltipSideOffset?: number;
+}
+
+/**
+ * Renders passed in text on a single line. If text is truncated,
+ * shows a tooltip on hover with the full text.
+ */
+export function TruncatedText({
+  text,
+  tooltipClassName,
+  tooltipSide = "right",
+  tooltipSideOffset = 5,
+  className = "",
+  ...rest
+}: TruncatedTextProps) {
+  const [isTruncated, setIsTruncated] = useState(false);
+  const visibleRef = useRef<HTMLSpanElement>(null);
+  const hiddenRef = useRef<HTMLSpanElement>(null);
+
+  useLayoutEffect(() => {
+    function checkTruncation() {
+      if (visibleRef.current && hiddenRef.current) {
+        const visibleWidth = visibleRef.current.offsetWidth;
+        const fullTextWidth = hiddenRef.current.offsetWidth;
+        setIsTruncated(fullTextWidth > visibleWidth);
+      }
+    }
+
+    checkTruncation();
+    window.addEventListener("resize", checkTruncation);
+    return () => window.removeEventListener("resize", checkTruncation);
+  }, [text]);
+
+  return (
+    <TooltipProvider>
+      <Tooltip>
+        <TooltipTrigger asChild>
+          <span
+            ref={visibleRef}
+            // Ensure the text can actually truncate via line-clamp or overflow
+            className={`line-clamp-1 break-all flex-grow ${className}`}
+            {...rest}
+          >
+            {text}
+          </span>
+        </TooltipTrigger>
+        {/* Hide offscreen to measure full text width */}
+        <span
+          ref={hiddenRef}
+          className="absolute left-[-9999px] whitespace-nowrap pointer-events-none"
+          aria-hidden="true"
+        >
+          {text}
+        </span>
+        {isTruncated && (
+          <TooltipContent
+            side={tooltipSide}
+            sideOffset={tooltipSideOffset}
+            className={tooltipClassName}
+          >
+            <p className="text-xs max-w-[200px] whitespace-normal break-words">
+              {text}
+            </p>
+          </TooltipContent>
+        )}
+      </Tooltip>
+    </TooltipProvider>
+  );
+}

web/src/lib/ccPair.ts

@@ -46,23 +46,3 @@ export async function setCCPairStatus(
       });
   }
 }
-
-export const getCCPairStatusMessage = (
-  isDisabled: boolean,
-  isIndexing: boolean,
-  ccPairStatus: ConnectorCredentialPairStatus
-) => {
-  if (ccPairStatus === ConnectorCredentialPairStatus.INVALID) {
-    return "Connector is in an invalid state. Please update the credentials or configuration before re-indexing.";
-  }
-  if (ccPairStatus === ConnectorCredentialPairStatus.DELETING) {
-    return "Cannot index while connector is deleting";
-  }
-  if (isIndexing) {
-    return "Indexing is already in progress";
-  }
-  if (isDisabled) {
-    return "Connector must be re-enabled before indexing";
-  }
-  return undefined;
-};

web/src/lib/connectors/connectors.tsx

@@ -152,17 +152,7 @@ export const connectorConfigs: Record<
         ],
       },
     ],
-    advanced_values: [
-      {
-        type: "checkbox",
-        query: "Scroll before scraping:",
-        label: "Scroll before scraping",
-        description:
-          "Enable if the website requires scrolling for the desired content to load",
-        name: "scroll_before_scraping",
-        optional: true,
-      },
-    ],
+    advanced_values: [],
     overrideDefaultFreq: 60 * 60 * 24,
   },
   github: {

web/src/lib/types.ts

@@ -98,7 +98,6 @@ export type ValidInputTypes =
   | "event"
   | "slim_retrieval";
 export type ValidStatuses =
-  | "invalid"
   | "success"
   | "completed_with_errors"
   | "canceled"