k

.github/workflows/pr-python-connector-tests.yml

@@ -48,11 +48,6 @@ env:
   # Gitbook
   GITBOOK_SPACE_ID: ${{ secrets.GITBOOK_SPACE_ID }}
   GITBOOK_API_KEY: ${{ secrets.GITBOOK_API_KEY }}
-  # Notion
-  NOTION_INTEGRATION_TOKEN: ${{ secrets.NOTION_INTEGRATION_TOKEN }}
-  # Highspot
-  HIGHSPOT_KEY: ${{ secrets.HIGHSPOT_KEY }}
-  HIGHSPOT_SECRET: ${{ secrets.HIGHSPOT_SECRET }}
 
 jobs:
   connectors-check:

README.md

@@ -114,4 +114,3 @@ To try the Onyx Enterprise Edition:
 
 ## 💡 Contributing
 Looking to contribute? Please check out the [Contribution Guide](CONTRIBUTING.md) for more details.
-

backend/Dockerfile

@@ -8,7 +8,7 @@ Edition features outside of personal development or testing purposes. Please rea
 founders@onyx.app for more information. Please visit https://github.com/onyx-dot-app/onyx"
 
 # Default ONYX_VERSION, typically overriden during builds by GitHub Actions.
-ARG ONYX_VERSION=0.0.0-dev
+ARG ONYX_VERSION=0.8-dev
 # DO_NOT_TRACK is used to disable telemetry for Unstructured
 ENV ONYX_VERSION=${ONYX_VERSION} \
     DANSWER_RUNNING_IN_DOCKER="true" \

backend/Dockerfile.model_server

@@ -7,7 +7,7 @@ You can find it at https://hub.docker.com/r/onyx/onyx-model-server. For more det
 visit https://github.com/onyx-dot-app/onyx."
 
 # Default ONYX_VERSION, typically overriden during builds by GitHub Actions.
-ARG ONYX_VERSION=0.0.0-dev
+ARG ONYX_VERSION=0.8-dev
 ENV ONYX_VERSION=${ONYX_VERSION} \
     DANSWER_RUNNING_IN_DOCKER="true"
 
@@ -31,8 +31,7 @@ RUN python -c "from transformers import AutoTokenizer; \
 AutoTokenizer.from_pretrained('distilbert-base-uncased'); \
 AutoTokenizer.from_pretrained('mixedbread-ai/mxbai-rerank-xsmall-v1'); \
 from huggingface_hub import snapshot_download; \
-snapshot_download(repo_id='onyx-dot-app/hybrid-intent-token-classifier'); \
-snapshot_download(repo_id='onyx-dot-app/information-content-model'); \
+snapshot_download(repo_id='danswer/hybrid-intent-token-classifier', revision='v1.0.3'); \
 snapshot_download('nomic-ai/nomic-embed-text-v1'); \
 snapshot_download('mixedbread-ai/mxbai-rerank-xsmall-v1'); \
 from sentence_transformers import SentenceTransformer; \

backend/alembic/versions/3781a5eb12cb_add_chunk_stats_table.py

@@ -1,51 +0,0 @@
-"""add chunk stats table
-
-Revision ID: 3781a5eb12cb
-Revises: df46c75b714e
-Create Date: 2025-03-10 10:02:30.586666
-
-"""
-from alembic import op
-import sqlalchemy as sa
-
-# revision identifiers, used by Alembic.
-revision = "3781a5eb12cb"
-down_revision = "df46c75b714e"
-branch_labels = None
-depends_on = None
-
-
-def upgrade() -> None:
-    op.create_table(
-        "chunk_stats",
-        sa.Column("id", sa.String(), primary_key=True, index=True),
-        sa.Column(
-            "document_id",
-            sa.String(),
-            sa.ForeignKey("document.id"),
-            nullable=False,
-            index=True,
-        ),
-        sa.Column("chunk_in_doc_id", sa.Integer(), nullable=False),
-        sa.Column("information_content_boost", sa.Float(), nullable=True),
-        sa.Column(
-            "last_modified",
-            sa.DateTime(timezone=True),
-            nullable=False,
-            index=True,
-            server_default=sa.func.now(),
-        ),
-        sa.Column("last_synced", sa.DateTime(timezone=True), nullable=True, index=True),
-        sa.UniqueConstraint(
-            "document_id", "chunk_in_doc_id", name="uq_chunk_stats_doc_chunk"
-        ),
-    )
-
-    op.create_index(
-        "ix_chunk_sync_status", "chunk_stats", ["last_modified", "last_synced"]
-    )
-
-
-def downgrade() -> None:
-    op.drop_index("ix_chunk_sync_status", table_name="chunk_stats")
-    op.drop_table("chunk_stats")

backend/alembic/versions/4d58345da04a_lowercase_user_emails.py

@@ -5,10 +5,7 @@ Revises: f1ca58b2f2ec
 Create Date: 2025-01-29 07:48:46.784041
 
 """
-import logging
-from typing import cast
 from alembic import op
-from sqlalchemy.exc import IntegrityError
 from sqlalchemy.sql import text
 
 
@@ -18,45 +15,21 @@ down_revision = "f1ca58b2f2ec"
 branch_labels = None
 depends_on = None
 
-logger = logging.getLogger("alembic.runtime.migration")
-
 
 def upgrade() -> None:
-    """Conflicts on lowercasing will result in the uppercased email getting a
-    unique integer suffix when converted to lowercase."""
-
+    # Get database connection
     connection = op.get_bind()
 
-    # Fetch all user emails that are not already lowercase
-    user_emails = connection.execute(
-        text('SELECT id, email FROM "user" WHERE email != LOWER(email)')
-    ).fetchall()
-
-    for user_id, email in user_emails:
-        email = cast(str, email)
-        username, domain = email.rsplit("@", 1)
-        new_email = f"{username.lower()}@{domain.lower()}"
-        attempt = 1
-
-        while True:
-            try:
-                # Try updating the email
-                connection.execute(
-                    text('UPDATE "user" SET email = :new_email WHERE id = :user_id'),
-                    {"new_email": new_email, "user_id": user_id},
-                )
-                break  # Success, exit loop
-            except IntegrityError:
-                next_email = f"{username.lower()}_{attempt}@{domain.lower()}"
-                # Email conflict occurred, append `_1`, `_2`, etc., to the username
-                logger.warning(
-                    f"Conflict while lowercasing email: "
-                    f"old_email={email} "
-                    f"conflicting_email={new_email} "
-                    f"next_email={next_email}"
-                )
-                new_email = next_email
-                attempt += 1
+    # Update all user emails to lowercase
+    connection.execute(
+        text(
+            """
+            UPDATE "user"
+            SET email = LOWER(email)
+            WHERE email != LOWER(email)
+            """
+        )
+    )
 
 
 def downgrade() -> None:

backend/alembic/versions/df46c75b714e_add_default_vision_provider_to_llm_.py

@@ -1,36 +0,0 @@
-"""add_default_vision_provider_to_llm_provider
-
-Revision ID: df46c75b714e
-Revises: 3934b1bc7b62
-Create Date: 2025-03-11 16:20:19.038945
-
-"""
-from alembic import op
-import sqlalchemy as sa
-
-
-# revision identifiers, used by Alembic.
-revision = "df46c75b714e"
-down_revision = "3934b1bc7b62"
-branch_labels = None
-depends_on = None
-
-
-def upgrade() -> None:
-    op.add_column(
-        "llm_provider",
-        sa.Column(
-            "is_default_vision_provider",
-            sa.Boolean(),
-            nullable=True,
-            server_default=sa.false(),
-        ),
-    )
-    op.add_column(
-        "llm_provider", sa.Column("default_vision_model", sa.String(), nullable=True)
-    )
-
-
-def downgrade() -> None:
-    op.drop_column("llm_provider", "default_vision_model")
-    op.drop_column("llm_provider", "is_default_vision_provider")

backend/alembic_tenants/versions/3b45e0018bf1_add_new_available_tenant_table.py

@@ -1,33 +0,0 @@
-"""add new available tenant table
-
-Revision ID: 3b45e0018bf1
-Revises: ac842f85f932
-Create Date: 2025-03-06 09:55:18.229910
-
-"""
-import sqlalchemy as sa
-
-from alembic import op
-
-
-# revision identifiers, used by Alembic.
-revision = "3b45e0018bf1"
-down_revision = "ac842f85f932"
-branch_labels = None
-depends_on = None
-
-
-def upgrade() -> None:
-    # Create new_available_tenant table
-    op.create_table(
-        "available_tenant",
-        sa.Column("tenant_id", sa.String(), nullable=False),
-        sa.Column("alembic_version", sa.String(), nullable=False),
-        sa.Column("date_created", sa.DateTime(), nullable=False),
-        sa.PrimaryKeyConstraint("tenant_id"),
-    )
-
-
-def downgrade() -> None:
-    # Drop new_available_tenant table
-    op.drop_table("available_tenant")

backend/alembic_tenants/versions/ac842f85f932_new_column_user_tenant_mapping.py

@@ -1,51 +0,0 @@
-"""new column user tenant mapping
-
-Revision ID: ac842f85f932
-Revises: 34e3630c7f32
-Create Date: 2025-03-03 13:30:14.802874
-
-"""
-import sqlalchemy as sa
-
-from alembic import op
-
-
-# revision identifiers, used by Alembic.
-revision = "ac842f85f932"
-down_revision = "34e3630c7f32"
-branch_labels = None
-depends_on = None
-
-
-def upgrade() -> None:
-    # Add active column with default value of True
-    op.add_column(
-        "user_tenant_mapping",
-        sa.Column(
-            "active",
-            sa.Boolean(),
-            nullable=False,
-            server_default="true",
-        ),
-        schema="public",
-    )
-
-    op.drop_constraint("uq_email", "user_tenant_mapping", schema="public")
-
-    # Create a unique index for active=true records
-    # This ensures a user can only be active in one tenant at a time
-    op.execute(
-        "CREATE UNIQUE INDEX uq_user_active_email_idx ON public.user_tenant_mapping (email) WHERE active = true"
-    )
-
-
-def downgrade() -> None:
-    # Drop the unique index for active=true records
-    op.execute("DROP INDEX IF EXISTS uq_user_active_email_idx")
-
-    op.create_unique_constraint(
-        "uq_email", "user_tenant_mapping", ["email"], schema="public"
-    )
-
-    # Remove the active column
-    op.drop_column("user_tenant_mapping", "active", schema="public")

backend/ee/onyx/external_permissions/confluence/doc_sync.py

@@ -2,7 +2,6 @@
 Rules defined here:
 https://confluence.atlassian.com/conf85/check-who-can-view-a-page-1283360557.html
 """
-from collections.abc import Generator
 from typing import Any
 
 from ee.onyx.configs.app_configs import CONFLUENCE_ANONYMOUS_ACCESS_IS_PUBLIC
@@ -264,11 +263,13 @@ def _fetch_all_page_restrictions(
     space_permissions_by_space_key: dict[str, ExternalAccess],
     is_cloud: bool,
     callback: IndexingHeartbeatInterface | None,
-) -> Generator[DocExternalAccess, None, None]:
+) -> list[DocExternalAccess]:
     """
     For all pages, if a page has restrictions, then use those restrictions.
     Otherwise, use the space's restrictions.
     """
+    document_restrictions: list[DocExternalAccess] = []
+
     for slim_doc in slim_docs:
         if callback:
             if callback.should_stop():
@@ -285,9 +286,11 @@ def _fetch_all_page_restrictions(
             confluence_client=confluence_client,
             perm_sync_data=slim_doc.perm_sync_data,
         ):
-            yield DocExternalAccess(
-                doc_id=slim_doc.id,
-                external_access=restrictions,
+            document_restrictions.append(
+                DocExternalAccess(
+                    doc_id=slim_doc.id,
+                    external_access=restrictions,
+                )
             )
             # If there are restrictions, then we don't need to use the space's restrictions
             continue
@@ -321,9 +324,11 @@ def _fetch_all_page_restrictions(
             continue
 
         # If there are no restrictions, then use the space's restrictions
-        yield DocExternalAccess(
-            doc_id=slim_doc.id,
-            external_access=space_permissions,
+        document_restrictions.append(
+            DocExternalAccess(
+                doc_id=slim_doc.id,
+                external_access=space_permissions,
+            )
         )
         if (
             not space_permissions.is_public
@@ -337,12 +342,13 @@ def _fetch_all_page_restrictions(
             )
 
     logger.debug("Finished fetching all page restrictions for space")
+    return document_restrictions
 
 
 def confluence_doc_sync(
     cc_pair: ConnectorCredentialPair,
     callback: IndexingHeartbeatInterface | None,
-) -> Generator[DocExternalAccess, None, None]:
+) -> list[DocExternalAccess]:
     """
     Adds the external permissions to the documents in postgres
     if the document doesn't already exists in postgres, we create
@@ -381,7 +387,7 @@ def confluence_doc_sync(
         slim_docs.extend(doc_batch)
 
     logger.debug("Fetching all page restrictions for space")
-    yield from _fetch_all_page_restrictions(
+    return _fetch_all_page_restrictions(
         confluence_client=confluence_connector.confluence_client,
         slim_docs=slim_docs,
         space_permissions_by_space_key=space_permissions_by_space_key,

backend/ee/onyx/external_permissions/gmail/doc_sync.py

@@ -1,4 +1,3 @@
-from collections.abc import Generator
 from datetime import datetime
 from datetime import timezone
 
@@ -35,7 +34,7 @@ def _get_slim_doc_generator(
 def gmail_doc_sync(
     cc_pair: ConnectorCredentialPair,
     callback: IndexingHeartbeatInterface | None,
-) -> Generator[DocExternalAccess, None, None]:
+) -> list[DocExternalAccess]:
     """
     Adds the external permissions to the documents in postgres
     if the document doesn't already exists in postgres, we create
@@ -49,6 +48,7 @@ def gmail_doc_sync(
         cc_pair, gmail_connector, callback=callback
     )
 
+    document_external_access: list[DocExternalAccess] = []
     for slim_doc_batch in slim_doc_generator:
         for slim_doc in slim_doc_batch:
             if callback:
@@ -60,14 +60,17 @@ def gmail_doc_sync(
             if slim_doc.perm_sync_data is None:
                 logger.warning(f"No permissions found for document {slim_doc.id}")
                 continue
-
             if user_email := slim_doc.perm_sync_data.get("user_email"):
                 ext_access = ExternalAccess(
                     external_user_emails=set([user_email]),
                     external_user_group_ids=set(),
                     is_public=False,
                 )
-                yield DocExternalAccess(
-                    doc_id=slim_doc.id,
-                    external_access=ext_access,
+                document_external_access.append(
+                    DocExternalAccess(
+                        doc_id=slim_doc.id,
+                        external_access=ext_access,
+                    )
                 )
+
+    return document_external_access

backend/ee/onyx/external_permissions/google_drive/doc_sync.py

@@ -1,4 +1,3 @@
-from collections.abc import Generator
 from datetime import datetime
 from datetime import timezone
 from typing import Any
@@ -148,7 +147,7 @@ def _get_permissions_from_slim_doc(
 def gdrive_doc_sync(
     cc_pair: ConnectorCredentialPair,
     callback: IndexingHeartbeatInterface | None,
-) -> Generator[DocExternalAccess, None, None]:
+) -> list[DocExternalAccess]:
     """
     Adds the external permissions to the documents in postgres
     if the document doesn't already exists in postgres, we create
@@ -162,6 +161,7 @@ def gdrive_doc_sync(
 
     slim_doc_generator = _get_slim_doc_generator(cc_pair, google_drive_connector)
 
+    document_external_accesses = []
     for slim_doc_batch in slim_doc_generator:
         for slim_doc in slim_doc_batch:
             if callback:
@@ -174,7 +174,10 @@ def gdrive_doc_sync(
                 google_drive_connector=google_drive_connector,
                 slim_doc=slim_doc,
             )
-            yield DocExternalAccess(
-                external_access=ext_access,
-                doc_id=slim_doc.id,
+            document_external_accesses.append(
+                DocExternalAccess(
+                    external_access=ext_access,
+                    doc_id=slim_doc.id,
+                )
             )
+    return document_external_accesses

backend/ee/onyx/external_permissions/slack/doc_sync.py

@@ -1,5 +1,3 @@
-from collections.abc import Generator
-
 from slack_sdk import WebClient
 
 from ee.onyx.external_permissions.slack.utils import fetch_user_id_to_email_map
@@ -16,6 +14,35 @@ from onyx.utils.logger import setup_logger
 logger = setup_logger()
 
 
+def _get_slack_document_ids_and_channels(
+    cc_pair: ConnectorCredentialPair, callback: IndexingHeartbeatInterface | None
+) -> dict[str, list[str]]:
+    slack_connector = SlackConnector(**cc_pair.connector.connector_specific_config)
+    slack_connector.load_credentials(cc_pair.credential.credential_json)
+
+    slim_doc_generator = slack_connector.retrieve_all_slim_documents(callback=callback)
+
+    channel_doc_map: dict[str, list[str]] = {}
+    for doc_metadata_batch in slim_doc_generator:
+        for doc_metadata in doc_metadata_batch:
+            if doc_metadata.perm_sync_data is None:
+                continue
+            channel_id = doc_metadata.perm_sync_data["channel_id"]
+            if channel_id not in channel_doc_map:
+                channel_doc_map[channel_id] = []
+            channel_doc_map[channel_id].append(doc_metadata.id)
+
+        if callback:
+            if callback.should_stop():
+                raise RuntimeError(
+                    "_get_slack_document_ids_and_channels: Stop signal detected"
+                )
+
+            callback.progress("_get_slack_document_ids_and_channels", 1)
+
+    return channel_doc_map
+
+
 def _fetch_workspace_permissions(
     user_id_to_email_map: dict[str, str],
 ) -> ExternalAccess:
@@ -95,37 +122,10 @@ def _fetch_channel_permissions(
     return channel_permissions
 
 
-def _get_slack_document_access(
-    cc_pair: ConnectorCredentialPair,
-    channel_permissions: dict[str, ExternalAccess],
-    callback: IndexingHeartbeatInterface | None,
-) -> Generator[DocExternalAccess, None, None]:
-    slack_connector = SlackConnector(**cc_pair.connector.connector_specific_config)
-    slack_connector.load_credentials(cc_pair.credential.credential_json)
-
-    slim_doc_generator = slack_connector.retrieve_all_slim_documents(callback=callback)
-
-    for doc_metadata_batch in slim_doc_generator:
-        for doc_metadata in doc_metadata_batch:
-            if doc_metadata.perm_sync_data is None:
-                continue
-            channel_id = doc_metadata.perm_sync_data["channel_id"]
-            yield DocExternalAccess(
-                external_access=channel_permissions[channel_id],
-                doc_id=doc_metadata.id,
-            )
-
-        if callback:
-            if callback.should_stop():
-                raise RuntimeError("_get_slack_document_access: Stop signal detected")
-
-            callback.progress("_get_slack_document_access", 1)
-
-
 def slack_doc_sync(
     cc_pair: ConnectorCredentialPair,
     callback: IndexingHeartbeatInterface | None,
-) -> Generator[DocExternalAccess, None, None]:
+) -> list[DocExternalAccess]:
     """
     Adds the external permissions to the documents in postgres
     if the document doesn't already exists in postgres, we create
@@ -136,12 +136,9 @@ def slack_doc_sync(
         token=cc_pair.credential.credential_json["slack_bot_token"]
     )
     user_id_to_email_map = fetch_user_id_to_email_map(slack_client)
-    if not user_id_to_email_map:
-        raise ValueError(
-            "No user id to email map found. Please check to make sure that "
-            "your Slack bot token has the `users:read.email` scope"
-        )
-
+    channel_doc_map = _get_slack_document_ids_and_channels(
+        cc_pair=cc_pair, callback=callback
+    )
     workspace_permissions = _fetch_workspace_permissions(
         user_id_to_email_map=user_id_to_email_map,
     )
@@ -151,8 +148,18 @@ def slack_doc_sync(
         user_id_to_email_map=user_id_to_email_map,
     )
 
-    yield from _get_slack_document_access(
-        cc_pair=cc_pair,
-        channel_permissions=channel_permissions,
-        callback=callback,
-    )
+    document_external_accesses = []
+    for channel_id, ext_access in channel_permissions.items():
+        doc_ids = channel_doc_map.get(channel_id)
+        if not doc_ids:
+            # No documents found for channel the channel_id
+            continue
+
+        for doc_id in doc_ids:
+            document_external_accesses.append(
+                DocExternalAccess(
+                    external_access=ext_access,
+                    doc_id=doc_id,
+                )
+            )
+    return document_external_accesses

backend/ee/onyx/external_permissions/sync_params.py

@@ -1,5 +1,4 @@
 from collections.abc import Callable
-from collections.abc import Generator
 
 from ee.onyx.configs.app_configs import CONFLUENCE_PERMISSION_DOC_SYNC_FREQUENCY
 from ee.onyx.configs.app_configs import CONFLUENCE_PERMISSION_GROUP_SYNC_FREQUENCY
@@ -24,7 +23,7 @@ DocSyncFuncType = Callable[
         ConnectorCredentialPair,
         IndexingHeartbeatInterface | None,
     ],
-    Generator[DocExternalAccess, None, None],
+    list[DocExternalAccess],
 ]
 
 GroupSyncFuncType = Callable[

backend/ee/onyx/server/enterprise_settings/api.py

@@ -15,8 +15,8 @@ from sqlalchemy.orm import Session
 
 from ee.onyx.server.enterprise_settings.models import AnalyticsScriptUpload
 from ee.onyx.server.enterprise_settings.models import EnterpriseSettings
-from ee.onyx.server.enterprise_settings.store import get_logo_filename
-from ee.onyx.server.enterprise_settings.store import get_logotype_filename
+from ee.onyx.server.enterprise_settings.store import _LOGO_FILENAME
+from ee.onyx.server.enterprise_settings.store import _LOGOTYPE_FILENAME
 from ee.onyx.server.enterprise_settings.store import load_analytics_script
 from ee.onyx.server.enterprise_settings.store import load_settings
 from ee.onyx.server.enterprise_settings.store import store_analytics_script
@@ -28,7 +28,7 @@ from onyx.auth.users import get_user_manager
 from onyx.auth.users import UserManager
 from onyx.db.engine import get_session
 from onyx.db.models import User
-from onyx.file_store.file_store import PostgresBackedFileStore
+from onyx.file_store.file_store import get_default_file_store
 from onyx.utils.logger import setup_logger
 
 admin_router = APIRouter(prefix="/admin/enterprise-settings")
@@ -131,49 +131,31 @@ def put_logo(
     upload_logo(file=file, db_session=db_session, is_logotype=is_logotype)
 
 
-def fetch_logo_helper(db_session: Session) -> Response:
+def fetch_logo_or_logotype(is_logotype: bool, db_session: Session) -> Response:
     try:
-        file_store = PostgresBackedFileStore(db_session)
-        onyx_file = file_store.get_file_with_mime_type(get_logo_filename())
-        if not onyx_file:
-            raise ValueError("get_onyx_file returned None!")
+        file_store = get_default_file_store(db_session)
+        filename = _LOGOTYPE_FILENAME if is_logotype else _LOGO_FILENAME
+        file_io = file_store.read_file(filename, mode="b")
+        # NOTE: specifying "image/jpeg" here, but it still works for pngs
+        # TODO: do this properly
+        return Response(content=file_io.read(), media_type="image/jpeg")
     except Exception:
         raise HTTPException(
             status_code=404,
-            detail="No logo file found",
+            detail=f"No {'logotype' if is_logotype else 'logo'} file found",
         )
-    else:
-        return Response(content=onyx_file.data, media_type=onyx_file.mime_type)
-
-
-def fetch_logotype_helper(db_session: Session) -> Response:
-    try:
-        file_store = PostgresBackedFileStore(db_session)
-        onyx_file = file_store.get_file_with_mime_type(get_logotype_filename())
-        if not onyx_file:
-            raise ValueError("get_onyx_file returned None!")
-    except Exception:
-        raise HTTPException(
-            status_code=404,
-            detail="No logotype file found",
-        )
-    else:
-        return Response(content=onyx_file.data, media_type=onyx_file.mime_type)
 
 
 @basic_router.get("/logotype")
 def fetch_logotype(db_session: Session = Depends(get_session)) -> Response:
-    return fetch_logotype_helper(db_session)
+    return fetch_logo_or_logotype(is_logotype=True, db_session=db_session)
 
 
 @basic_router.get("/logo")
 def fetch_logo(
     is_logotype: bool = False, db_session: Session = Depends(get_session)
 ) -> Response:
-    if is_logotype:
-        return fetch_logotype_helper(db_session)
-
-    return fetch_logo_helper(db_session)
+    return fetch_logo_or_logotype(is_logotype=is_logotype, db_session=db_session)
 
 
 @admin_router.put("/custom-analytics-script")

backend/ee/onyx/server/enterprise_settings/store.py

@@ -13,7 +13,6 @@ from ee.onyx.server.enterprise_settings.models import EnterpriseSettings
 from onyx.configs.constants import FileOrigin
 from onyx.configs.constants import KV_CUSTOM_ANALYTICS_SCRIPT_KEY
 from onyx.configs.constants import KV_ENTERPRISE_SETTINGS_KEY
-from onyx.configs.constants import ONYX_DEFAULT_APPLICATION_NAME
 from onyx.file_store.file_store import get_default_file_store
 from onyx.key_value_store.factory import get_kv_store
 from onyx.key_value_store.interface import KvKeyNotFoundError
@@ -22,18 +21,8 @@ from onyx.utils.logger import setup_logger
 
 logger = setup_logger()
 
-_LOGO_FILENAME = "__logo__"
-_LOGOTYPE_FILENAME = "__logotype__"
-
 
 def load_settings() -> EnterpriseSettings:
-    """Loads settings data directly from DB. This should be used primarily
-    for checking what is actually in the DB, aka for editing and saving back settings.
-
-    Runtime settings actually used by the application should be checked with
-    load_runtime_settings as defaults may be applied at runtime.
-    """
-
     dynamic_config_store = get_kv_store()
     try:
         settings = EnterpriseSettings(
@@ -47,24 +36,9 @@ def load_settings() -> EnterpriseSettings:
 
 
 def store_settings(settings: EnterpriseSettings) -> None:
-    """Stores settings directly to the kv store / db."""
-
     get_kv_store().store(KV_ENTERPRISE_SETTINGS_KEY, settings.model_dump())
 
 
-def load_runtime_settings() -> EnterpriseSettings:
-    """Loads settings from DB and applies any defaults or transformations for use
-    at runtime.
-
-    Should not be stored back to the DB.
-    """
-    enterprise_settings = load_settings()
-    if not enterprise_settings.application_name:
-        enterprise_settings.application_name = ONYX_DEFAULT_APPLICATION_NAME
-
-    return enterprise_settings
-
-
 _CUSTOM_ANALYTICS_SECRET_KEY = os.environ.get("CUSTOM_ANALYTICS_SECRET_KEY")
 
 
@@ -86,6 +60,10 @@ def store_analytics_script(analytics_script_upload: AnalyticsScriptUpload) -> No
     get_kv_store().store(KV_CUSTOM_ANALYTICS_SCRIPT_KEY, analytics_script_upload.script)
 
 
+_LOGO_FILENAME = "__logo__"
+_LOGOTYPE_FILENAME = "__logotype__"
+
+
 def is_valid_file_type(filename: str) -> bool:
     valid_extensions = (".png", ".jpg", ".jpeg")
     return filename.endswith(valid_extensions)
@@ -138,11 +116,3 @@ def upload_logo(
         file_type=file_type,
     )
     return True
-
-
-def get_logo_filename() -> str:
-    return _LOGO_FILENAME
-
-
-def get_logotype_filename() -> str:
-    return _LOGOTYPE_FILENAME

backend/ee/onyx/server/query_and_chat/chat_backend.py

@@ -1,14 +1,10 @@
 import re
-from typing import cast
 
 from fastapi import APIRouter
 from fastapi import Depends
 from fastapi import HTTPException
 from sqlalchemy.orm import Session
 
-from ee.onyx.server.query_and_chat.models import AgentAnswer
-from ee.onyx.server.query_and_chat.models import AgentSubQuery
-from ee.onyx.server.query_and_chat.models import AgentSubQuestion
 from ee.onyx.server.query_and_chat.models import BasicCreateChatMessageRequest
 from ee.onyx.server.query_and_chat.models import (
     BasicCreateChatMessageWithHistoryRequest,
@@ -18,19 +14,13 @@ from ee.onyx.server.query_and_chat.models import SimpleDoc
 from onyx.auth.users import current_user
 from onyx.chat.chat_utils import combine_message_thread
 from onyx.chat.chat_utils import create_chat_chain
-from onyx.chat.models import AgentAnswerPiece
 from onyx.chat.models import AllCitations
-from onyx.chat.models import ExtendedToolResponse
 from onyx.chat.models import FinalUsedContextDocsResponse
 from onyx.chat.models import LlmDoc
 from onyx.chat.models import LLMRelevanceFilterResponse
 from onyx.chat.models import OnyxAnswerPiece
 from onyx.chat.models import QADocsResponse
-from onyx.chat.models import RefinedAnswerImprovement
 from onyx.chat.models import StreamingError
-from onyx.chat.models import SubQueryPiece
-from onyx.chat.models import SubQuestionIdentifier
-from onyx.chat.models import SubQuestionPiece
 from onyx.chat.process_message import ChatPacketStream
 from onyx.chat.process_message import stream_chat_message_objects
 from onyx.configs.chat_configs import CHAT_TARGET_CHUNK_PERCENTAGE
@@ -99,12 +89,6 @@ def _convert_packet_stream_to_response(
     final_context_docs: list[LlmDoc] = []
 
     answer = ""
-
-    # accumulate stream data with these dicts
-    agent_sub_questions: dict[tuple[int, int], AgentSubQuestion] = {}
-    agent_answers: dict[tuple[int, int], AgentAnswer] = {}
-    agent_sub_queries: dict[tuple[int, int, int], AgentSubQuery] = {}
-
     for packet in packets:
         if isinstance(packet, OnyxAnswerPiece) and packet.answer_piece:
             answer += packet.answer_piece
@@ -113,15 +97,6 @@ def _convert_packet_stream_to_response(
 
             # TODO: deprecate `simple_search_docs`
             response.simple_search_docs = _translate_doc_response_to_simple_doc(packet)
-
-            # This is a no-op if agent_sub_questions hasn't already been filled
-            if packet.level is not None and packet.level_question_num is not None:
-                id = (packet.level, packet.level_question_num)
-                if id in agent_sub_questions:
-                    agent_sub_questions[id].document_ids = [
-                        saved_search_doc.document_id
-                        for saved_search_doc in packet.top_documents
-                    ]
         elif isinstance(packet, StreamingError):
             response.error_msg = packet.error
         elif isinstance(packet, ChatMessageDetail):
@@ -138,104 +113,11 @@ def _convert_packet_stream_to_response(
                 citation.citation_num: citation.document_id
                 for citation in packet.citations
             }
-        # agentic packets
-        elif isinstance(packet, SubQuestionPiece):
-            if packet.level is not None and packet.level_question_num is not None:
-                id = (packet.level, packet.level_question_num)
-                if agent_sub_questions.get(id) is None:
-                    agent_sub_questions[id] = AgentSubQuestion(
-                        level=packet.level,
-                        level_question_num=packet.level_question_num,
-                        sub_question=packet.sub_question,
-                        document_ids=[],
-                    )
-                else:
-                    agent_sub_questions[id].sub_question += packet.sub_question
-
-        elif isinstance(packet, AgentAnswerPiece):
-            if packet.level is not None and packet.level_question_num is not None:
-                id = (packet.level, packet.level_question_num)
-                if agent_answers.get(id) is None:
-                    agent_answers[id] = AgentAnswer(
-                        level=packet.level,
-                        level_question_num=packet.level_question_num,
-                        answer=packet.answer_piece,
-                        answer_type=packet.answer_type,
-                    )
-                else:
-                    agent_answers[id].answer += packet.answer_piece
-        elif isinstance(packet, SubQueryPiece):
-            if packet.level is not None and packet.level_question_num is not None:
-                sub_query_id = (
-                    packet.level,
-                    packet.level_question_num,
-                    packet.query_id,
-                )
-                if agent_sub_queries.get(sub_query_id) is None:
-                    agent_sub_queries[sub_query_id] = AgentSubQuery(
-                        level=packet.level,
-                        level_question_num=packet.level_question_num,
-                        sub_query=packet.sub_query,
-                        query_id=packet.query_id,
-                    )
-                else:
-                    agent_sub_queries[sub_query_id].sub_query += packet.sub_query
-        elif isinstance(packet, ExtendedToolResponse):
-            # we shouldn't get this ... it gets intercepted and translated to QADocsResponse
-            logger.warning(
-                "_convert_packet_stream_to_response: Unexpected chat packet type ExtendedToolResponse!"
-            )
-        elif isinstance(packet, RefinedAnswerImprovement):
-            response.agent_refined_answer_improvement = (
-                packet.refined_answer_improvement
-            )
-        else:
-            logger.warning(
-                f"_convert_packet_stream_to_response - Unrecognized chat packet: type={type(packet)}"
-            )
 
     response.final_context_doc_indices = _get_final_context_doc_indices(
         final_context_docs, response.top_documents
     )
 
-    # organize / sort agent metadata for output
-    if len(agent_sub_questions) > 0:
-        response.agent_sub_questions = cast(
-            dict[int, list[AgentSubQuestion]],
-            SubQuestionIdentifier.make_dict_by_level(agent_sub_questions),
-        )
-
-    if len(agent_answers) > 0:
-        # return the agent_level_answer from the first level or the last one depending
-        # on agent_refined_answer_improvement
-        response.agent_answers = cast(
-            dict[int, list[AgentAnswer]],
-            SubQuestionIdentifier.make_dict_by_level(agent_answers),
-        )
-        if response.agent_answers:
-            selected_answer_level = (
-                0
-                if not response.agent_refined_answer_improvement
-                else len(response.agent_answers) - 1
-            )
-            level_answers = response.agent_answers[selected_answer_level]
-            for level_answer in level_answers:
-                if level_answer.answer_type != "agent_level_answer":
-                    continue
-
-                answer = level_answer.answer
-                break
-
-    if len(agent_sub_queries) > 0:
-        # subqueries are often emitted with trailing whitespace ... clean it up here
-        # perhaps fix at the source?
-        for v in agent_sub_queries.values():
-            v.sub_query = v.sub_query.strip()
-
-        response.agent_sub_queries = (
-            AgentSubQuery.make_dict_by_level_and_question_index(agent_sub_queries)
-        )
-
     response.answer = answer
     if answer:
         response.answer_citationless = remove_answer_citations(answer)

backend/ee/onyx/server/query_and_chat/models.py

@@ -1,5 +1,3 @@
-from collections import OrderedDict
-from typing import Literal
 from uuid import UUID
 
 from pydantic import BaseModel
@@ -11,7 +9,6 @@ from onyx.chat.models import CitationInfo
 from onyx.chat.models import OnyxContexts
 from onyx.chat.models import PersonaOverrideConfig
 from onyx.chat.models import QADocsResponse
-from onyx.chat.models import SubQuestionIdentifier
 from onyx.chat.models import ThreadMessage
 from onyx.configs.constants import DocumentSource
 from onyx.context.search.enums import LLMEvaluationType
@@ -91,64 +88,6 @@ class SimpleDoc(BaseModel):
     metadata: dict | None
 
 
-class AgentSubQuestion(SubQuestionIdentifier):
-    sub_question: str
-    document_ids: list[str]
-
-
-class AgentAnswer(SubQuestionIdentifier):
-    answer: str
-    answer_type: Literal["agent_sub_answer", "agent_level_answer"]
-
-
-class AgentSubQuery(SubQuestionIdentifier):
-    sub_query: str
-    query_id: int
-
-    @staticmethod
-    def make_dict_by_level_and_question_index(
-        original_dict: dict[tuple[int, int, int], "AgentSubQuery"]
-    ) -> dict[int, dict[int, list["AgentSubQuery"]]]:
-        """Takes a dict of tuple(level, question num, query_id) to sub queries.
-
-        returns a dict of level to dict[question num to list of query_id's]
-        Ordering is asc for readability.
-        """
-        # In this function, when we sort int | None, we deliberately push None to the end
-
-        # map entries to the level_question_dict
-        level_question_dict: dict[int, dict[int, list["AgentSubQuery"]]] = {}
-        for k1, obj in original_dict.items():
-            level = k1[0]
-            question = k1[1]
-
-            if level not in level_question_dict:
-                level_question_dict[level] = {}
-
-            if question not in level_question_dict[level]:
-                level_question_dict[level][question] = []
-
-            level_question_dict[level][question].append(obj)
-
-        # sort each query_id list and question_index
-        for key1, obj1 in level_question_dict.items():
-            for key2, value2 in obj1.items():
-                # sort the query_id list of each question_index
-                level_question_dict[key1][key2] = sorted(
-                    value2, key=lambda o: o.query_id
-                )
-            # sort the question_index dict of level
-            level_question_dict[key1] = OrderedDict(
-                sorted(level_question_dict[key1].items(), key=lambda x: (x is None, x))
-            )
-
-        # sort the top dict of levels
-        sorted_dict = OrderedDict(
-            sorted(level_question_dict.items(), key=lambda x: (x is None, x))
-        )
-        return sorted_dict
-
-
 class ChatBasicResponse(BaseModel):
     # This is built piece by piece, any of these can be None as the flow could break
     answer: str | None = None
@@ -168,12 +107,6 @@ class ChatBasicResponse(BaseModel):
     simple_search_docs: list[SimpleDoc] | None = None
     llm_chunks_indices: list[int] | None = None
 
-    # agentic fields
-    agent_sub_questions: dict[int, list[AgentSubQuestion]] | None = None
-    agent_answers: dict[int, list[AgentAnswer]] | None = None
-    agent_sub_queries: dict[int, dict[int, list[AgentSubQuery]]] | None = None
-    agent_refined_answer_improvement: bool | None = None
-
 
 class OneShotQARequest(ChunkContext):
     # Supports simplier APIs that don't deal with chat histories or message edits

backend/ee/onyx/server/tenants/admin_api.py

@@ -1,45 +0,0 @@
-from fastapi import APIRouter
-from fastapi import Depends
-from fastapi import HTTPException
-from fastapi import Response
-
-from ee.onyx.auth.users import current_cloud_superuser
-from ee.onyx.server.tenants.models import ImpersonateRequest
-from ee.onyx.server.tenants.user_mapping import get_tenant_id_for_email
-from onyx.auth.users import auth_backend
-from onyx.auth.users import get_redis_strategy
-from onyx.auth.users import User
-from onyx.db.engine import get_session_with_tenant
-from onyx.db.users import get_user_by_email
-from onyx.utils.logger import setup_logger
-
-logger = setup_logger()
-
-router = APIRouter(prefix="/tenants")
-
-
-@router.post("/impersonate")
-async def impersonate_user(
-    impersonate_request: ImpersonateRequest,
-    _: User = Depends(current_cloud_superuser),
-) -> Response:
-    """Allows a cloud superuser to impersonate another user by generating an impersonation JWT token"""
-    tenant_id = get_tenant_id_for_email(impersonate_request.email)
-
-    with get_session_with_tenant(tenant_id=tenant_id) as tenant_session:
-        user_to_impersonate = get_user_by_email(
-            impersonate_request.email, tenant_session
-        )
-        if user_to_impersonate is None:
-            raise HTTPException(status_code=404, detail="User not found")
-        token = await get_redis_strategy().write_token(user_to_impersonate)
-
-    response = await auth_backend.transport.get_login_response(token)
-    response.set_cookie(
-        key="fastapiusersauth",
-        value=token,
-        httponly=True,
-        secure=True,
-        samesite="lax",
-    )
-    return response

backend/ee/onyx/server/tenants/anonymous_users_api.py

@@ -1,98 +0,0 @@
-from fastapi import APIRouter
-from fastapi import Depends
-from fastapi import HTTPException
-from fastapi import Response
-from sqlalchemy.exc import IntegrityError
-
-from ee.onyx.auth.users import generate_anonymous_user_jwt_token
-from ee.onyx.configs.app_configs import ANONYMOUS_USER_COOKIE_NAME
-from ee.onyx.server.tenants.anonymous_user_path import get_anonymous_user_path
-from ee.onyx.server.tenants.anonymous_user_path import (
-    get_tenant_id_for_anonymous_user_path,
-)
-from ee.onyx.server.tenants.anonymous_user_path import modify_anonymous_user_path
-from ee.onyx.server.tenants.anonymous_user_path import validate_anonymous_user_path
-from ee.onyx.server.tenants.models import AnonymousUserPath
-from onyx.auth.users import anonymous_user_enabled
-from onyx.auth.users import current_admin_user
-from onyx.auth.users import optional_user
-from onyx.auth.users import User
-from onyx.configs.constants import FASTAPI_USERS_AUTH_COOKIE_NAME
-from onyx.db.engine import get_session_with_shared_schema
-from onyx.utils.logger import setup_logger
-from shared_configs.contextvars import get_current_tenant_id
-
-logger = setup_logger()
-
-router = APIRouter(prefix="/tenants")
-
-
-@router.get("/anonymous-user-path")
-async def get_anonymous_user_path_api(
-    _: User | None = Depends(current_admin_user),
-) -> AnonymousUserPath:
-    tenant_id = get_current_tenant_id()
-
-    if tenant_id is None:
-        raise HTTPException(status_code=404, detail="Tenant not found")
-
-    with get_session_with_shared_schema() as db_session:
-        current_path = get_anonymous_user_path(tenant_id, db_session)
-
-    return AnonymousUserPath(anonymous_user_path=current_path)
-
-
-@router.post("/anonymous-user-path")
-async def set_anonymous_user_path_api(
-    anonymous_user_path: str,
-    _: User | None = Depends(current_admin_user),
-) -> None:
-    tenant_id = get_current_tenant_id()
-    try:
-        validate_anonymous_user_path(anonymous_user_path)
-    except ValueError as e:
-        raise HTTPException(status_code=400, detail=str(e))
-
-    with get_session_with_shared_schema() as db_session:
-        try:
-            modify_anonymous_user_path(tenant_id, anonymous_user_path, db_session)
-        except IntegrityError:
-            raise HTTPException(
-                status_code=409,
-                detail="The anonymous user path is already in use. Please choose a different path.",
-            )
-        except Exception as e:
-            logger.exception(f"Failed to modify anonymous user path: {str(e)}")
-            raise HTTPException(
-                status_code=500,
-                detail="An unexpected error occurred while modifying the anonymous user path",
-            )
-
-
-@router.post("/anonymous-user")
-async def login_as_anonymous_user(
-    anonymous_user_path: str,
-    _: User | None = Depends(optional_user),
-) -> Response:
-    with get_session_with_shared_schema() as db_session:
-        tenant_id = get_tenant_id_for_anonymous_user_path(
-            anonymous_user_path, db_session
-        )
-        if not tenant_id:
-            raise HTTPException(status_code=404, detail="Tenant not found")
-
-    if not anonymous_user_enabled(tenant_id=tenant_id):
-        raise HTTPException(status_code=403, detail="Anonymous user is not enabled")
-
-    token = generate_anonymous_user_jwt_token(tenant_id)
-
-    response = Response()
-    response.delete_cookie(FASTAPI_USERS_AUTH_COOKIE_NAME)
-    response.set_cookie(
-        key=ANONYMOUS_USER_COOKIE_NAME,
-        value=token,
-        httponly=True,
-        secure=True,
-        samesite="strict",
-    )
-    return response

backend/ee/onyx/server/tenants/api.py

@@ -1,24 +1,269 @@
+import stripe
 from fastapi import APIRouter
+from fastapi import Depends
+from fastapi import HTTPException
+from fastapi import Response
+from sqlalchemy.exc import IntegrityError
+from sqlalchemy.orm import Session
 
-from ee.onyx.server.tenants.admin_api import router as admin_router
-from ee.onyx.server.tenants.anonymous_users_api import router as anonymous_users_router
-from ee.onyx.server.tenants.billing_api import router as billing_router
-from ee.onyx.server.tenants.team_membership_api import router as team_membership_router
-from ee.onyx.server.tenants.tenant_management_api import (
-    router as tenant_management_router,
-)
-from ee.onyx.server.tenants.user_invitations_api import (
-    router as user_invitations_router,
+from ee.onyx.auth.users import current_cloud_superuser
+from ee.onyx.auth.users import generate_anonymous_user_jwt_token
+from ee.onyx.configs.app_configs import ANONYMOUS_USER_COOKIE_NAME
+from ee.onyx.configs.app_configs import STRIPE_SECRET_KEY
+from ee.onyx.server.tenants.access import control_plane_dep
+from ee.onyx.server.tenants.anonymous_user_path import get_anonymous_user_path
+from ee.onyx.server.tenants.anonymous_user_path import (
+    get_tenant_id_for_anonymous_user_path,
 )
+from ee.onyx.server.tenants.anonymous_user_path import modify_anonymous_user_path
+from ee.onyx.server.tenants.anonymous_user_path import validate_anonymous_user_path
+from ee.onyx.server.tenants.billing import fetch_billing_information
+from ee.onyx.server.tenants.billing import fetch_stripe_checkout_session
+from ee.onyx.server.tenants.billing import fetch_tenant_stripe_information
+from ee.onyx.server.tenants.models import AnonymousUserPath
+from ee.onyx.server.tenants.models import BillingInformation
+from ee.onyx.server.tenants.models import ImpersonateRequest
+from ee.onyx.server.tenants.models import ProductGatingRequest
+from ee.onyx.server.tenants.models import ProductGatingResponse
+from ee.onyx.server.tenants.models import SubscriptionSessionResponse
+from ee.onyx.server.tenants.models import SubscriptionStatusResponse
+from ee.onyx.server.tenants.product_gating import store_product_gating
+from ee.onyx.server.tenants.provisioning import delete_user_from_control_plane
+from ee.onyx.server.tenants.user_mapping import get_tenant_id_for_email
+from ee.onyx.server.tenants.user_mapping import remove_all_users_from_tenant
+from ee.onyx.server.tenants.user_mapping import remove_users_from_tenant
+from onyx.auth.users import anonymous_user_enabled
+from onyx.auth.users import auth_backend
+from onyx.auth.users import current_admin_user
+from onyx.auth.users import get_redis_strategy
+from onyx.auth.users import optional_user
+from onyx.auth.users import User
+from onyx.configs.app_configs import WEB_DOMAIN
+from onyx.configs.constants import FASTAPI_USERS_AUTH_COOKIE_NAME
+from onyx.db.auth import get_user_count
+from onyx.db.engine import get_session
+from onyx.db.engine import get_session_with_shared_schema
+from onyx.db.engine import get_session_with_tenant
+from onyx.db.users import delete_user_from_db
+from onyx.db.users import get_user_by_email
+from onyx.server.manage.models import UserByEmail
+from onyx.utils.logger import setup_logger
+from shared_configs.contextvars import CURRENT_TENANT_ID_CONTEXTVAR
+from shared_configs.contextvars import get_current_tenant_id
 
-# Create a main router to include all sub-routers
-# Note: We don't add a prefix here as each router already has the /tenants prefix
-router = APIRouter()
+stripe.api_key = STRIPE_SECRET_KEY
+logger = setup_logger()
+router = APIRouter(prefix="/tenants")
 
-# Include all the individual routers
-router.include_router(admin_router)
-router.include_router(anonymous_users_router)
-router.include_router(billing_router)
-router.include_router(team_membership_router)
-router.include_router(tenant_management_router)
-router.include_router(user_invitations_router)
+
+@router.get("/anonymous-user-path")
+async def get_anonymous_user_path_api(
+    _: User | None = Depends(current_admin_user),
+) -> AnonymousUserPath:
+    tenant_id = get_current_tenant_id()
+
+    if tenant_id is None:
+        raise HTTPException(status_code=404, detail="Tenant not found")
+
+    with get_session_with_shared_schema() as db_session:
+        current_path = get_anonymous_user_path(tenant_id, db_session)
+
+    return AnonymousUserPath(anonymous_user_path=current_path)
+
+
+@router.post("/anonymous-user-path")
+async def set_anonymous_user_path_api(
+    anonymous_user_path: str,
+    _: User | None = Depends(current_admin_user),
+) -> None:
+    tenant_id = get_current_tenant_id()
+    try:
+        validate_anonymous_user_path(anonymous_user_path)
+    except ValueError as e:
+        raise HTTPException(status_code=400, detail=str(e))
+
+    with get_session_with_shared_schema() as db_session:
+        try:
+            modify_anonymous_user_path(tenant_id, anonymous_user_path, db_session)
+        except IntegrityError:
+            raise HTTPException(
+                status_code=409,
+                detail="The anonymous user path is already in use. Please choose a different path.",
+            )
+        except Exception as e:
+            logger.exception(f"Failed to modify anonymous user path: {str(e)}")
+            raise HTTPException(
+                status_code=500,
+                detail="An unexpected error occurred while modifying the anonymous user path",
+            )
+
+
+@router.post("/anonymous-user")
+async def login_as_anonymous_user(
+    anonymous_user_path: str,
+    _: User | None = Depends(optional_user),
+) -> Response:
+    with get_session_with_shared_schema() as db_session:
+        tenant_id = get_tenant_id_for_anonymous_user_path(
+            anonymous_user_path, db_session
+        )
+        if not tenant_id:
+            raise HTTPException(status_code=404, detail="Tenant not found")
+
+    if not anonymous_user_enabled(tenant_id=tenant_id):
+        raise HTTPException(status_code=403, detail="Anonymous user is not enabled")
+
+    token = generate_anonymous_user_jwt_token(tenant_id)
+
+    response = Response()
+    response.delete_cookie(FASTAPI_USERS_AUTH_COOKIE_NAME)
+    response.set_cookie(
+        key=ANONYMOUS_USER_COOKIE_NAME,
+        value=token,
+        httponly=True,
+        secure=True,
+        samesite="strict",
+    )
+    return response
+
+
+@router.post("/product-gating")
+def gate_product(
+    product_gating_request: ProductGatingRequest, _: None = Depends(control_plane_dep)
+) -> ProductGatingResponse:
+    """
+    Gating the product means that the product is not available to the tenant.
+    They will be directed to the billing page.
+    We gate the product when their subscription has ended.
+    """
+    try:
+        store_product_gating(
+            product_gating_request.tenant_id, product_gating_request.application_status
+        )
+        return ProductGatingResponse(updated=True, error=None)
+
+    except Exception as e:
+        logger.exception("Failed to gate product")
+        return ProductGatingResponse(updated=False, error=str(e))
+
+
+@router.get("/billing-information")
+async def billing_information(
+    _: User = Depends(current_admin_user),
+) -> BillingInformation | SubscriptionStatusResponse:
+    logger.info("Fetching billing information")
+    tenant_id = get_current_tenant_id()
+    return fetch_billing_information(tenant_id)
+
+
+@router.post("/create-customer-portal-session")
+async def create_customer_portal_session(
+    _: User = Depends(current_admin_user),
+) -> dict:
+    tenant_id = get_current_tenant_id()
+
+    try:
+        stripe_info = fetch_tenant_stripe_information(tenant_id)
+        stripe_customer_id = stripe_info.get("stripe_customer_id")
+        if not stripe_customer_id:
+            raise HTTPException(status_code=400, detail="Stripe customer ID not found")
+        logger.info(stripe_customer_id)
+
+        portal_session = stripe.billing_portal.Session.create(
+            customer=stripe_customer_id,
+            return_url=f"{WEB_DOMAIN}/admin/billing",
+        )
+        logger.info(portal_session)
+        return {"url": portal_session.url}
+    except Exception as e:
+        logger.exception("Failed to create customer portal session")
+        raise HTTPException(status_code=500, detail=str(e))
+
+
+@router.post("/create-subscription-session")
+async def create_subscription_session(
+    _: User = Depends(current_admin_user),
+) -> SubscriptionSessionResponse:
+    try:
+        tenant_id = CURRENT_TENANT_ID_CONTEXTVAR.get()
+        if not tenant_id:
+            raise HTTPException(status_code=400, detail="Tenant ID not found")
+        session_id = fetch_stripe_checkout_session(tenant_id)
+        return SubscriptionSessionResponse(sessionId=session_id)
+
+    except Exception as e:
+        logger.exception("Failed to create resubscription session")
+        raise HTTPException(status_code=500, detail=str(e))
+
+
+@router.post("/impersonate")
+async def impersonate_user(
+    impersonate_request: ImpersonateRequest,
+    _: User = Depends(current_cloud_superuser),
+) -> Response:
+    """Allows a cloud superuser to impersonate another user by generating an impersonation JWT token"""
+    tenant_id = get_tenant_id_for_email(impersonate_request.email)
+
+    with get_session_with_tenant(tenant_id=tenant_id) as tenant_session:
+        user_to_impersonate = get_user_by_email(
+            impersonate_request.email, tenant_session
+        )
+        if user_to_impersonate is None:
+            raise HTTPException(status_code=404, detail="User not found")
+        token = await get_redis_strategy().write_token(user_to_impersonate)
+
+    response = await auth_backend.transport.get_login_response(token)
+    response.set_cookie(
+        key="fastapiusersauth",
+        value=token,
+        httponly=True,
+        secure=True,
+        samesite="lax",
+    )
+    return response
+
+
+@router.post("/leave-organization")
+async def leave_organization(
+    user_email: UserByEmail,
+    current_user: User | None = Depends(current_admin_user),
+    db_session: Session = Depends(get_session),
+) -> None:
+    tenant_id = get_current_tenant_id()
+
+    if current_user is None or current_user.email != user_email.user_email:
+        raise HTTPException(
+            status_code=403, detail="You can only leave the organization as yourself"
+        )
+
+    user_to_delete = get_user_by_email(user_email.user_email, db_session)
+    if user_to_delete is None:
+        raise HTTPException(status_code=404, detail="User not found")
+
+    num_admin_users = await get_user_count(only_admin_users=True)
+
+    should_delete_tenant = num_admin_users == 1
+
+    if should_delete_tenant:
+        logger.info(
+            "Last admin user is leaving the organization. Deleting tenant from control plane."
+        )
+        try:
+            await delete_user_from_control_plane(tenant_id, user_to_delete.email)
+            logger.debug("User deleted from control plane")
+        except Exception as e:
+            logger.exception(
+                f"Failed to delete user from control plane for tenant {tenant_id}: {e}"
+            )
+            raise HTTPException(
+                status_code=500,
+                detail=f"Failed to remove user from control plane: {str(e)}",
+            )
+
+    db_session.expunge(user_to_delete)
+    delete_user_from_db(user_to_delete, db_session)
+
+    if should_delete_tenant:
+        remove_all_users_from_tenant(tenant_id)
+    else:
+        remove_users_from_tenant([user_to_delete.email], tenant_id)

backend/ee/onyx/server/tenants/billing_api.py

@@ -1,96 +0,0 @@
-import stripe
-from fastapi import APIRouter
-from fastapi import Depends
-from fastapi import HTTPException
-
-from ee.onyx.auth.users import current_admin_user
-from ee.onyx.configs.app_configs import STRIPE_SECRET_KEY
-from ee.onyx.server.tenants.access import control_plane_dep
-from ee.onyx.server.tenants.billing import fetch_billing_information
-from ee.onyx.server.tenants.billing import fetch_stripe_checkout_session
-from ee.onyx.server.tenants.billing import fetch_tenant_stripe_information
-from ee.onyx.server.tenants.models import BillingInformation
-from ee.onyx.server.tenants.models import ProductGatingRequest
-from ee.onyx.server.tenants.models import ProductGatingResponse
-from ee.onyx.server.tenants.models import SubscriptionSessionResponse
-from ee.onyx.server.tenants.models import SubscriptionStatusResponse
-from ee.onyx.server.tenants.product_gating import store_product_gating
-from onyx.auth.users import User
-from onyx.configs.app_configs import WEB_DOMAIN
-from onyx.utils.logger import setup_logger
-from shared_configs.contextvars import CURRENT_TENANT_ID_CONTEXTVAR
-from shared_configs.contextvars import get_current_tenant_id
-
-stripe.api_key = STRIPE_SECRET_KEY
-logger = setup_logger()
-
-router = APIRouter(prefix="/tenants")
-
-
-@router.post("/product-gating")
-def gate_product(
-    product_gating_request: ProductGatingRequest, _: None = Depends(control_plane_dep)
-) -> ProductGatingResponse:
-    """
-    Gating the product means that the product is not available to the tenant.
-    They will be directed to the billing page.
-    We gate the product when their subscription has ended.
-    """
-    try:
-        store_product_gating(
-            product_gating_request.tenant_id, product_gating_request.application_status
-        )
-        return ProductGatingResponse(updated=True, error=None)
-
-    except Exception as e:
-        logger.exception("Failed to gate product")
-        return ProductGatingResponse(updated=False, error=str(e))
-
-
-@router.get("/billing-information")
-async def billing_information(
-    _: User = Depends(current_admin_user),
-) -> BillingInformation | SubscriptionStatusResponse:
-    logger.info("Fetching billing information")
-    tenant_id = get_current_tenant_id()
-    return fetch_billing_information(tenant_id)
-
-
-@router.post("/create-customer-portal-session")
-async def create_customer_portal_session(
-    _: User = Depends(current_admin_user),
-) -> dict:
-    tenant_id = get_current_tenant_id()
-
-    try:
-        stripe_info = fetch_tenant_stripe_information(tenant_id)
-        stripe_customer_id = stripe_info.get("stripe_customer_id")
-        if not stripe_customer_id:
-            raise HTTPException(status_code=400, detail="Stripe customer ID not found")
-        logger.info(stripe_customer_id)
-
-        portal_session = stripe.billing_portal.Session.create(
-            customer=stripe_customer_id,
-            return_url=f"{WEB_DOMAIN}/admin/billing",
-        )
-        logger.info(portal_session)
-        return {"url": portal_session.url}
-    except Exception as e:
-        logger.exception("Failed to create customer portal session")
-        raise HTTPException(status_code=500, detail=str(e))
-
-
-@router.post("/create-subscription-session")
-async def create_subscription_session(
-    _: User = Depends(current_admin_user),
-) -> SubscriptionSessionResponse:
-    try:
-        tenant_id = CURRENT_TENANT_ID_CONTEXTVAR.get()
-        if not tenant_id:
-            raise HTTPException(status_code=400, detail="Tenant ID not found")
-        session_id = fetch_stripe_checkout_session(tenant_id)
-        return SubscriptionSessionResponse(sessionId=session_id)
-
-    except Exception as e:
-        logger.exception("Failed to create resubscription session")
-        raise HTTPException(status_code=500, detail=str(e))

backend/ee/onyx/server/tenants/models.py

@@ -67,30 +67,3 @@ class ProductGatingResponse(BaseModel):
 
 class SubscriptionSessionResponse(BaseModel):
     sessionId: str
-
-
-class TenantByDomainResponse(BaseModel):
-    tenant_id: str
-    number_of_users: int
-    creator_email: str
-
-
-class TenantByDomainRequest(BaseModel):
-    email: str
-
-
-class RequestInviteRequest(BaseModel):
-    tenant_id: str
-
-
-class RequestInviteResponse(BaseModel):
-    success: bool
-    message: str
-
-
-class PendingUserSnapshot(BaseModel):
-    email: str
-
-
-class ApproveUserRequest(BaseModel):
-    email: str

backend/ee/onyx/server/tenants/provisioning.py

@@ -4,7 +4,6 @@ import uuid
 
 import aiohttp  # Async HTTP client
 import httpx
-import requests
 from fastapi import HTTPException
 from fastapi import Request
 from sqlalchemy import select
@@ -15,7 +14,6 @@ from ee.onyx.configs.app_configs import COHERE_DEFAULT_API_KEY
 from ee.onyx.configs.app_configs import HUBSPOT_TRACKING_URL
 from ee.onyx.configs.app_configs import OPENAI_DEFAULT_API_KEY
 from ee.onyx.server.tenants.access import generate_data_plane_token
-from ee.onyx.server.tenants.models import TenantByDomainResponse
 from ee.onyx.server.tenants.models import TenantCreationPayload
 from ee.onyx.server.tenants.models import TenantDeletionPayload
 from ee.onyx.server.tenants.schema_management import create_schema_if_not_exists
@@ -28,12 +26,11 @@ from onyx.auth.users import exceptions
 from onyx.configs.app_configs import CONTROL_PLANE_API_BASE_URL
 from onyx.configs.app_configs import DEV_MODE
 from onyx.configs.constants import MilestoneRecordType
-from onyx.db.engine import get_session_with_shared_schema
 from onyx.db.engine import get_session_with_tenant
+from onyx.db.engine import get_sqlalchemy_engine
 from onyx.db.llm import update_default_provider
 from onyx.db.llm import upsert_cloud_embedding_provider
 from onyx.db.llm import upsert_llm_provider
-from onyx.db.models import AvailableTenant
 from onyx.db.models import IndexModelStatus
 from onyx.db.models import SearchSettings
 from onyx.db.models import UserTenantMapping
@@ -63,72 +60,42 @@ async def get_or_provision_tenant(
     This function should only be called after we have verified we want this user's tenant to exist.
     It returns the tenant ID associated with the email, creating a new tenant if necessary.
     """
-    # Early return for non-multi-tenant mode
     if not MULTI_TENANT:
         return POSTGRES_DEFAULT_SCHEMA
 
     if referral_source and request:
         await submit_to_hubspot(email, referral_source, request)
 
-    # First, check if the user already has a tenant
-    tenant_id: str | None = None
     try:
         tenant_id = get_tenant_id_for_email(email)
-        return tenant_id
     except exceptions.UserNotExists:
-        # User doesn't exist, so we need to create a new tenant or assign an existing one
-        pass
-
-    try:
-        # Try to get a pre-provisioned tenant
-        tenant_id = await get_available_tenant()
-
-        if tenant_id:
-            # If we have a pre-provisioned tenant, assign it to the user
-            await assign_tenant_to_user(tenant_id, email, referral_source)
-            logger.info(f"Assigned pre-provisioned tenant {tenant_id} to user {email}")
-            return tenant_id
-        else:
-            # If no pre-provisioned tenant is available, create a new one on-demand
+        # If tenant does not exist and in Multi tenant mode, provision a new tenant
+        try:
             tenant_id = await create_tenant(email, referral_source)
-            return tenant_id
+        except Exception as e:
+            logger.error(f"Tenant provisioning failed: {e}")
+            raise HTTPException(status_code=500, detail="Failed to provision tenant.")
 
-    except Exception as e:
-        # If we've encountered an error, log and raise an exception
-        error_msg = "Failed to provision tenant"
-        logger.error(error_msg, exc_info=e)
+    if not tenant_id:
         raise HTTPException(
-            status_code=500,
-            detail="Failed to provision tenant. Please try again later.",
+            status_code=401, detail="User does not belong to an organization"
         )
 
+    return tenant_id
+
 
 async def create_tenant(email: str, referral_source: str | None = None) -> str:
-    """
-    Create a new tenant on-demand when no pre-provisioned tenants are available.
-    This is the fallback method when we can't use a pre-provisioned tenant.
-
-    """
     tenant_id = TENANT_ID_PREFIX + str(uuid.uuid4())
-    logger.info(f"Creating new tenant {tenant_id} for user {email}")
-
     try:
         # Provision tenant on data plane
         await provision_tenant(tenant_id, email)
-
-        # Notify control plane if not already done in provision_tenant
-        if not DEV_MODE and referral_source:
+        # Notify control plane
+        if not DEV_MODE:
             await notify_control_plane(tenant_id, email, referral_source)
-
     except Exception as e:
-        logger.exception(f"Tenant provisioning failed: {str(e)}")
-        # Attempt to rollback the tenant provisioning
-        try:
-            await rollback_tenant_provisioning(tenant_id)
-        except Exception:
-            logger.exception(f"Failed to rollback tenant provisioning for {tenant_id}")
+        logger.error(f"Tenant provisioning failed: {e}")
+        await rollback_tenant_provisioning(tenant_id)
         raise HTTPException(status_code=500, detail="Failed to provision tenant.")
-
     return tenant_id
 
 
@@ -142,25 +109,54 @@ async def provision_tenant(tenant_id: str, email: str) -> None:
         )
 
     logger.debug(f"Provisioning tenant {tenant_id} for user {email}")
+    token = None
 
     try:
-        # Create the schema for the tenant
         if not create_schema_if_not_exists(tenant_id):
             logger.debug(f"Created schema for tenant {tenant_id}")
         else:
             logger.debug(f"Schema already exists for tenant {tenant_id}")
 
-        # Set up the tenant with all necessary configurations
-        await setup_tenant(tenant_id)
+        token = CURRENT_TENANT_ID_CONTEXTVAR.set(tenant_id)
 
-        # Assign the tenant to the user
-        await assign_tenant_to_user(tenant_id, email)
+        # Await the Alembic migrations
+        await asyncio.to_thread(run_alembic_migrations, tenant_id)
+
+        with get_session_with_tenant(tenant_id=tenant_id) as db_session:
+            configure_default_api_keys(db_session)
+
+            current_search_settings = (
+                db_session.query(SearchSettings)
+                .filter_by(status=IndexModelStatus.FUTURE)
+                .first()
+            )
+            cohere_enabled = (
+                current_search_settings is not None
+                and current_search_settings.provider_type == EmbeddingProvider.COHERE
+            )
+            setup_onyx(db_session, tenant_id, cohere_enabled=cohere_enabled)
+
+        add_users_to_tenant([email], tenant_id)
+
+        with get_session_with_tenant(tenant_id=tenant_id) as db_session:
+            create_milestone_and_report(
+                user=None,
+                distinct_id=tenant_id,
+                event_type=MilestoneRecordType.TENANT_CREATED,
+                properties={
+                    "email": email,
+                },
+                db_session=db_session,
+            )
 
     except Exception as e:
         logger.exception(f"Failed to create tenant {tenant_id}")
         raise HTTPException(
             status_code=500, detail=f"Failed to create tenant: {str(e)}"
         )
+    finally:
+        if token is not None:
+            CURRENT_TENANT_ID_CONTEXTVAR.reset(token)
 
 
 async def notify_control_plane(
@@ -191,74 +187,20 @@ async def notify_control_plane(
 
 
 async def rollback_tenant_provisioning(tenant_id: str) -> None:
-    """
-    Logic to rollback tenant provisioning on data plane.
-    Handles each step independently to ensure maximum cleanup even if some steps fail.
-    """
+    # Logic to rollback tenant provisioning on data plane
     logger.info(f"Rolling back tenant provisioning for tenant_id: {tenant_id}")
-
-    # Track if any part of the rollback fails
-    rollback_errors = []
-
-    # 1. Try to drop the tenant's schema
     try:
+        # Drop the tenant's schema to rollback provisioning
         drop_schema(tenant_id)
-        logger.info(f"Successfully dropped schema for tenant {tenant_id}")
+
+        # Remove tenant mapping
+        with Session(get_sqlalchemy_engine()) as db_session:
+            db_session.query(UserTenantMapping).filter(
+                UserTenantMapping.tenant_id == tenant_id
+            ).delete()
+            db_session.commit()
     except Exception as e:
-        error_msg = f"Failed to drop schema for tenant {tenant_id}: {str(e)}"
-        logger.error(error_msg)
-        rollback_errors.append(error_msg)
-
-    # 2. Try to remove tenant mapping
-    try:
-        with get_session_with_shared_schema() as db_session:
-            db_session.begin()
-            try:
-                db_session.query(UserTenantMapping).filter(
-                    UserTenantMapping.tenant_id == tenant_id
-                ).delete()
-                db_session.commit()
-                logger.info(
-                    f"Successfully removed user mappings for tenant {tenant_id}"
-                )
-            except Exception as e:
-                db_session.rollback()
-                raise e
-    except Exception as e:
-        error_msg = f"Failed to remove user mappings for tenant {tenant_id}: {str(e)}"
-        logger.error(error_msg)
-        rollback_errors.append(error_msg)
-
-    # 3. If this tenant was in the available tenants table, remove it
-    try:
-        with get_session_with_shared_schema() as db_session:
-            db_session.begin()
-            try:
-                available_tenant = (
-                    db_session.query(AvailableTenant)
-                    .filter(AvailableTenant.tenant_id == tenant_id)
-                    .first()
-                )
-
-                if available_tenant:
-                    db_session.delete(available_tenant)
-                    db_session.commit()
-                    logger.info(
-                        f"Removed tenant {tenant_id} from available tenants table"
-                    )
-            except Exception as e:
-                db_session.rollback()
-                raise e
-    except Exception as e:
-        error_msg = f"Failed to remove tenant {tenant_id} from available tenants table: {str(e)}"
-        logger.error(error_msg)
-        rollback_errors.append(error_msg)
-
-    # Log summary of rollback operation
-    if rollback_errors:
-        logger.error(f"Tenant rollback completed with {len(rollback_errors)} errors")
-    else:
-        logger.info(f"Tenant rollback completed successfully for tenant {tenant_id}")
+        logger.error(f"Failed to rollback tenant provisioning: {e}")
 
 
 def configure_default_api_keys(db_session: Session) -> None:
@@ -271,7 +213,6 @@ def configure_default_api_keys(db_session: Session) -> None:
             fast_default_model_name="claude-3-5-sonnet-20241022",
             model_names=ANTHROPIC_MODEL_NAMES,
             display_model_names=["claude-3-5-sonnet-20241022"],
-            api_key_changed=True,
         )
         try:
             full_provider = upsert_llm_provider(anthropic_provider, db_session)
@@ -284,7 +225,7 @@ def configure_default_api_keys(db_session: Session) -> None:
         )
 
     if OPENAI_DEFAULT_API_KEY:
-        openai_provider = LLMProviderUpsertRequest(
+        open_provider = LLMProviderUpsertRequest(
             name="OpenAI",
             provider=OPENAI_PROVIDER_NAME,
             api_key=OPENAI_DEFAULT_API_KEY,
@@ -292,10 +233,9 @@ def configure_default_api_keys(db_session: Session) -> None:
             fast_default_model_name="gpt-4o-mini",
             model_names=OPEN_AI_MODEL_NAMES,
             display_model_names=["o1", "o3-mini", "gpt-4o", "gpt-4o-mini"],
-            api_key_changed=True,
         )
         try:
-            full_provider = upsert_llm_provider(openai_provider, db_session)
+            full_provider = upsert_llm_provider(open_provider, db_session)
             update_default_provider(full_provider.id, db_session)
         except Exception as e:
             logger.error(f"Failed to configure OpenAI provider: {e}")
@@ -413,155 +353,3 @@ async def delete_user_from_control_plane(tenant_id: str, email: str) -> None:
                 raise Exception(
                     f"Failed to delete tenant on control plane: {error_text}"
                 )
-
-
-def get_tenant_by_domain_from_control_plane(
-    domain: str,
-    tenant_id: str,
-) -> TenantByDomainResponse | None:
-    """
-    Fetches tenant information from the control plane based on the email domain.
-
-    Args:
-        domain: The email domain to search for (e.g., "example.com")
-
-    Returns:
-        A dictionary containing tenant information if found, None otherwise
-    """
-    token = generate_data_plane_token()
-    headers = {
-        "Authorization": f"Bearer {token}",
-        "Content-Type": "application/json",
-    }
-
-    try:
-        response = requests.get(
-            f"{CONTROL_PLANE_API_BASE_URL}/tenant-by-domain",
-            headers=headers,
-            json={"domain": domain, "tenant_id": tenant_id},
-        )
-
-        if response.status_code != 200:
-            logger.error(f"Control plane tenant lookup failed: {response.text}")
-            return None
-
-        response_data = response.json()
-        if not response_data:
-            return None
-
-        return TenantByDomainResponse(
-            tenant_id=response_data.get("tenant_id"),
-            number_of_users=response_data.get("number_of_users"),
-            creator_email=response_data.get("creator_email"),
-        )
-    except Exception as e:
-        logger.error(f"Error fetching tenant by domain: {str(e)}")
-        return None
-
-
-async def get_available_tenant() -> str | None:
-    """
-    Get an available pre-provisioned tenant from the NewAvailableTenant table.
-    Returns the tenant_id if one is available, None otherwise.
-    Uses row-level locking to prevent race conditions when multiple processes
-    try to get an available tenant simultaneously.
-    """
-    if not MULTI_TENANT:
-        return None
-
-    with get_session_with_shared_schema() as db_session:
-        try:
-            db_session.begin()
-
-            # Get the oldest available tenant with FOR UPDATE lock to prevent race conditions
-            available_tenant = (
-                db_session.query(AvailableTenant)
-                .order_by(AvailableTenant.date_created)
-                .with_for_update(skip_locked=True)  # Skip locked rows to avoid blocking
-                .first()
-            )
-
-            if available_tenant:
-                tenant_id = available_tenant.tenant_id
-                # Remove the tenant from the available tenants table
-                db_session.delete(available_tenant)
-                db_session.commit()
-                logger.info(f"Using pre-provisioned tenant {tenant_id}")
-                return tenant_id
-            else:
-                db_session.rollback()
-                return None
-        except Exception:
-            logger.exception("Error getting available tenant")
-            db_session.rollback()
-            return None
-
-
-async def setup_tenant(tenant_id: str) -> None:
-    """
-    Set up a tenant with all necessary configurations.
-    This is a centralized function that handles all tenant setup logic.
-    """
-    token = None
-    try:
-        token = CURRENT_TENANT_ID_CONTEXTVAR.set(tenant_id)
-
-        # Run Alembic migrations
-        await asyncio.to_thread(run_alembic_migrations, tenant_id)
-
-        # Configure the tenant with default settings
-        with get_session_with_tenant(tenant_id=tenant_id) as db_session:
-            # Configure default API keys
-            configure_default_api_keys(db_session)
-
-            # Set up Onyx with appropriate settings
-            current_search_settings = (
-                db_session.query(SearchSettings)
-                .filter_by(status=IndexModelStatus.FUTURE)
-                .first()
-            )
-            cohere_enabled = (
-                current_search_settings is not None
-                and current_search_settings.provider_type == EmbeddingProvider.COHERE
-            )
-            setup_onyx(db_session, tenant_id, cohere_enabled=cohere_enabled)
-
-    except Exception as e:
-        logger.exception(f"Failed to set up tenant {tenant_id}")
-        raise e
-    finally:
-        if token is not None:
-            CURRENT_TENANT_ID_CONTEXTVAR.reset(token)
-
-
-async def assign_tenant_to_user(
-    tenant_id: str, email: str, referral_source: str | None = None
-) -> None:
-    """
-    Assign a tenant to a user and perform necessary operations.
-    Uses transaction handling to ensure atomicity and includes retry logic
-    for control plane notifications.
-    """
-    # First, add the user to the tenant in a transaction
-
-    try:
-        add_users_to_tenant([email], tenant_id)
-
-        # Create milestone record in the same transaction context as the tenant assignment
-        with get_session_with_tenant(tenant_id=tenant_id) as db_session:
-            create_milestone_and_report(
-                user=None,
-                distinct_id=tenant_id,
-                event_type=MilestoneRecordType.TENANT_CREATED,
-                properties={
-                    "email": email,
-                },
-                db_session=db_session,
-            )
-    except Exception:
-        logger.exception(f"Failed to assign tenant {tenant_id} to user {email}")
-        raise Exception("Failed to assign tenant to user")
-
-    # Notify control plane with retry logic
-    if not DEV_MODE:
-        await notify_control_plane(tenant_id, email, referral_source)

backend/ee/onyx/server/tenants/schema_management.py

@@ -74,21 +74,3 @@ def drop_schema(tenant_id: str) -> None:
             text("DROP SCHEMA IF EXISTS %(schema_name)s CASCADE"),
             {"schema_name": tenant_id},
         )
-
-
-def get_current_alembic_version(tenant_id: str) -> str:
-    """Get the current Alembic version for a tenant."""
-    from alembic.runtime.migration import MigrationContext
-    from sqlalchemy import text
-
-    engine = get_sqlalchemy_engine()
-
-    # Set the search path to the tenant's schema
-    with engine.connect() as connection:
-        connection.execute(text(f'SET search_path TO "{tenant_id}"'))
-
-        # Get the current version from the alembic_version table
-        context = MigrationContext.configure(connection)
-        current_rev = context.get_current_revision()
-
-    return current_rev or "head"

backend/ee/onyx/server/tenants/team_membership_api.py

@@ -1,67 +0,0 @@
-from fastapi import APIRouter
-from fastapi import Depends
-from fastapi import HTTPException
-from sqlalchemy.orm import Session
-
-from ee.onyx.server.tenants.provisioning import delete_user_from_control_plane
-from ee.onyx.server.tenants.user_mapping import remove_all_users_from_tenant
-from ee.onyx.server.tenants.user_mapping import remove_users_from_tenant
-from onyx.auth.users import current_admin_user
-from onyx.auth.users import User
-from onyx.db.auth import get_user_count
-from onyx.db.engine import get_session
-from onyx.db.users import delete_user_from_db
-from onyx.db.users import get_user_by_email
-from onyx.server.manage.models import UserByEmail
-from onyx.utils.logger import setup_logger
-from shared_configs.contextvars import get_current_tenant_id
-
-logger = setup_logger()
-
-router = APIRouter(prefix="/tenants")
-
-
-@router.post("/leave-team")
-async def leave_organization(
-    user_email: UserByEmail,
-    current_user: User | None = Depends(current_admin_user),
-    db_session: Session = Depends(get_session),
-) -> None:
-    tenant_id = get_current_tenant_id()
-
-    if current_user is None or current_user.email != user_email.user_email:
-        raise HTTPException(
-            status_code=403, detail="You can only leave the organization as yourself"
-        )
-
-    user_to_delete = get_user_by_email(user_email.user_email, db_session)
-    if user_to_delete is None:
-        raise HTTPException(status_code=404, detail="User not found")
-
-    num_admin_users = await get_user_count(only_admin_users=True)
-
-    should_delete_tenant = num_admin_users == 1
-
-    if should_delete_tenant:
-        logger.info(
-            "Last admin user is leaving the organization. Deleting tenant from control plane."
-        )
-        try:
-            await delete_user_from_control_plane(tenant_id, user_to_delete.email)
-            logger.debug("User deleted from control plane")
-        except Exception as e:
-            logger.exception(
-                f"Failed to delete user from control plane for tenant {tenant_id}: {e}"
-            )
-            raise HTTPException(
-                status_code=500,
-                detail=f"Failed to remove user from control plane: {str(e)}",
-            )
-
-    db_session.expunge(user_to_delete)
-    delete_user_from_db(user_to_delete, db_session)
-
-    if should_delete_tenant:
-        remove_all_users_from_tenant(tenant_id)
-    else:
-        remove_users_from_tenant([user_to_delete.email], tenant_id)

backend/ee/onyx/server/tenants/tenant_management_api.py

@@ -1,39 +0,0 @@
-from fastapi import APIRouter
-from fastapi import Depends
-
-from ee.onyx.server.tenants.models import TenantByDomainResponse
-from ee.onyx.server.tenants.provisioning import get_tenant_by_domain_from_control_plane
-from onyx.auth.users import current_user
-from onyx.auth.users import User
-from onyx.utils.logger import setup_logger
-from shared_configs.contextvars import get_current_tenant_id
-
-logger = setup_logger()
-
-router = APIRouter(prefix="/tenants")
-
-FORBIDDEN_COMMON_EMAIL_SUBSTRINGS = [
-    "gmail",
-    "outlook",
-    "yahoo",
-    "hotmail",
-    "icloud",
-    "msn",
-    "hotmail",
-    "hotmail.co.uk",
-]
-
-
-@router.get("/existing-team-by-domain")
-def get_existing_tenant_by_domain(
-    user: User | None = Depends(current_user),
-) -> TenantByDomainResponse | None:
-    if not user:
-        return None
-    domain = user.email.split("@")[1]
-    if any(substring in domain for substring in FORBIDDEN_COMMON_EMAIL_SUBSTRINGS):
-        return None
-
-    tenant_id = get_current_tenant_id()
-
-    return get_tenant_by_domain_from_control_plane(domain, tenant_id)

backend/ee/onyx/server/tenants/user_invitations_api.py

@@ -1,90 +0,0 @@
-from fastapi import APIRouter
-from fastapi import Depends
-from fastapi import HTTPException
-
-from ee.onyx.server.tenants.models import ApproveUserRequest
-from ee.onyx.server.tenants.models import PendingUserSnapshot
-from ee.onyx.server.tenants.models import RequestInviteRequest
-from ee.onyx.server.tenants.user_mapping import accept_user_invite
-from ee.onyx.server.tenants.user_mapping import approve_user_invite
-from ee.onyx.server.tenants.user_mapping import deny_user_invite
-from ee.onyx.server.tenants.user_mapping import invite_self_to_tenant
-from onyx.auth.invited_users import get_pending_users
-from onyx.auth.users import current_admin_user
-from onyx.auth.users import current_user
-from onyx.auth.users import User
-from onyx.utils.logger import setup_logger
-from shared_configs.contextvars import get_current_tenant_id
-
-logger = setup_logger()
-
-router = APIRouter(prefix="/tenants")
-
-
-@router.post("/users/invite/request")
-async def request_invite(
-    invite_request: RequestInviteRequest,
-    user: User | None = Depends(current_admin_user),
-) -> None:
-    if user is None:
-        raise HTTPException(status_code=401, detail="User not authenticated")
-    try:
-        invite_self_to_tenant(user.email, invite_request.tenant_id)
-    except Exception as e:
-        logger.exception(
-            f"Failed to invite self to tenant {invite_request.tenant_id}: {e}"
-        )
-        raise HTTPException(status_code=500, detail=str(e))
-
-
-@router.get("/users/pending")
-def list_pending_users(
-    _: User | None = Depends(current_admin_user),
-) -> list[PendingUserSnapshot]:
-    pending_emails = get_pending_users()
-    return [PendingUserSnapshot(email=email) for email in pending_emails]
-
-
-@router.post("/users/invite/approve")
-async def approve_user(
-    approve_user_request: ApproveUserRequest,
-    _: User | None = Depends(current_admin_user),
-) -> None:
-    tenant_id = get_current_tenant_id()
-    approve_user_invite(approve_user_request.email, tenant_id)
-
-
-@router.post("/users/invite/accept")
-async def accept_invite(
-    invite_request: RequestInviteRequest,
-    user: User | None = Depends(current_user),
-) -> None:
-    """
-    Accept an invitation to join a tenant.
-    """
-    if not user:
-        raise HTTPException(status_code=401, detail="Not authenticated")
-
-    try:
-        accept_user_invite(user.email, invite_request.tenant_id)
-    except Exception as e:
-        logger.exception(f"Failed to accept invite: {str(e)}")
-        raise HTTPException(status_code=500, detail="Failed to accept invitation")
-
-
-@router.post("/users/invite/deny")
-async def deny_invite(
-    invite_request: RequestInviteRequest,
-    user: User | None = Depends(current_user),
-) -> None:
-    """
-    Deny an invitation to join a tenant.
-    """
-    if not user:
-        raise HTTPException(status_code=401, detail="Not authenticated")
-
-    try:
-        deny_user_invite(user.email, invite_request.tenant_id)
-    except Exception as e:
-        logger.exception(f"Failed to deny invite: {str(e)}")
-        raise HTTPException(status_code=500, detail="Failed to deny invitation")

backend/ee/onyx/server/tenants/user_mapping.py

@@ -1,56 +1,27 @@
+import logging
+
 from fastapi_users import exceptions
 from sqlalchemy import select
+from sqlalchemy.orm import Session
 
-from onyx.auth.invited_users import get_invited_users
-from onyx.auth.invited_users import get_pending_users
-from onyx.auth.invited_users import write_invited_users
-from onyx.auth.invited_users import write_pending_users
-from onyx.db.engine import get_session_with_shared_schema
 from onyx.db.engine import get_session_with_tenant
+from onyx.db.engine import get_sqlalchemy_engine
 from onyx.db.models import UserTenantMapping
-from onyx.server.manage.models import TenantSnapshot
-from onyx.setup import setup_logger
 from shared_configs.configs import MULTI_TENANT
 from shared_configs.configs import POSTGRES_DEFAULT_SCHEMA
-from shared_configs.contextvars import CURRENT_TENANT_ID_CONTEXTVAR
 
-logger = setup_logger()
+logger = logging.getLogger(__name__)
 
 
 def get_tenant_id_for_email(email: str) -> str:
     if not MULTI_TENANT:
         return POSTGRES_DEFAULT_SCHEMA
     # Implement logic to get tenant_id from the mapping table
-    try:
-        with get_session_with_shared_schema() as db_session:
-            # First try to get an active tenant
-            result = db_session.execute(
-                select(UserTenantMapping).where(
-                    UserTenantMapping.email == email,
-                    UserTenantMapping.active == True,  # noqa: E712
-                )
-            )
-            mapping = result.scalar_one_or_none()
-            tenant_id = mapping.tenant_id if mapping else None
-
-            # If no active tenant found, try to get the first inactive one
-            if tenant_id is None:
-                result = db_session.execute(
-                    select(UserTenantMapping).where(
-                        UserTenantMapping.email == email,
-                        UserTenantMapping.active == False,  # noqa: E712
-                    )
-                )
-                mapping = result.scalar_one_or_none()
-                if mapping:
-                    # Mark this mapping as active
-                    mapping.active = True
-                    db_session.commit()
-                    tenant_id = mapping.tenant_id
-
-    except Exception as e:
-        logger.exception(f"Error getting tenant id for email {email}: {e}")
-        raise exceptions.UserNotExists()
+    with Session(get_sqlalchemy_engine()) as db_session:
+        result = db_session.execute(
+            select(UserTenantMapping.tenant_id).where(UserTenantMapping.email == email)
+        )
+        tenant_id = result.scalar_one_or_none()
     if tenant_id is None:
         raise exceptions.UserNotExists()
     return tenant_id
@@ -67,39 +38,13 @@ def user_owns_a_tenant(email: str) -> bool:
 
 
 def add_users_to_tenant(emails: list[str], tenant_id: str) -> None:
-    """
-    Add users to a tenant with proper transaction handling.
-    Checks if users already have a tenant mapping to avoid duplicates.
-    """
     with get_session_with_tenant(tenant_id=POSTGRES_DEFAULT_SCHEMA) as db_session:
         try:
-            # Start a transaction
-            db_session.begin()
-
             for email in emails:
-                # Check if the user already has a mapping to this tenant
-                existing_mapping = (
-                    db_session.query(UserTenantMapping)
-                    .filter(
-                        UserTenantMapping.email == email,
-                        UserTenantMapping.tenant_id == tenant_id,
-                    )
-                    .with_for_update()
-                    .first()
-                )
-
-                if not existing_mapping:
-                    # Only add if mapping doesn't exist
-                    db_session.add(UserTenantMapping(email=email, tenant_id=tenant_id))
-
-            # Commit the transaction
-            db_session.commit()
-            logger.info(f"Successfully added users {emails} to tenant {tenant_id}")
-
+                db_session.add(UserTenantMapping(email=email, tenant_id=tenant_id))
         except Exception:
             logger.exception(f"Failed to add users to tenant {tenant_id}")
-            db_session.rollback()
-            raise
+        db_session.commit()
 
 
 def remove_users_from_tenant(emails: list[str], tenant_id: str) -> None:
@@ -131,187 +76,3 @@ def remove_all_users_from_tenant(tenant_id: str) -> None:
             UserTenantMapping.tenant_id == tenant_id
         ).delete()
         db_session.commit()
-
-
-def invite_self_to_tenant(email: str, tenant_id: str) -> None:
-    token = CURRENT_TENANT_ID_CONTEXTVAR.set(tenant_id)
-    try:
-        pending_users = get_pending_users()
-        if email in pending_users:
-            return
-        write_pending_users(pending_users + [email])
-    finally:
-        CURRENT_TENANT_ID_CONTEXTVAR.reset(token)
-
-
-def approve_user_invite(email: str, tenant_id: str) -> None:
-    """
-    Approve a user invite to a tenant.
-    This will delete all existing records for this email and create a new mapping entry for the user in this tenant.
-    """
-    with get_session_with_shared_schema() as db_session:
-        # Delete all existing records for this email
-        db_session.query(UserTenantMapping).filter(
-            UserTenantMapping.email == email
-        ).delete()
-
-        # Create a new mapping entry for the user in this tenant
-        new_mapping = UserTenantMapping(email=email, tenant_id=tenant_id, active=True)
-        db_session.add(new_mapping)
-        db_session.commit()
-
-    # Also remove the user from pending users list
-    # Remove from pending users
-    pending_users = get_pending_users()
-    if email in pending_users:
-        pending_users.remove(email)
-        write_pending_users(pending_users)
-
-    # Add to invited users
-    invited_users = get_invited_users()
-    if email not in invited_users:
-        invited_users.append(email)
-        write_invited_users(invited_users)
-
-
-def accept_user_invite(email: str, tenant_id: str) -> None:
-    """
-    Accept an invitation to join a tenant.
-    This activates the user's mapping to the tenant.
-    """
-    with get_session_with_shared_schema() as db_session:
-        try:
-            # First check if there's an active mapping for this user and tenant
-            active_mapping = (
-                db_session.query(UserTenantMapping)
-                .filter(
-                    UserTenantMapping.email == email,
-                    UserTenantMapping.active == True,  # noqa: E712
-                )
-                .first()
-            )
-
-            # If an active mapping exists, delete it
-            if active_mapping:
-                db_session.delete(active_mapping)
-                logger.info(
-                    f"Deleted existing active mapping for user {email} in tenant {tenant_id}"
-                )
-
-            # Find the inactive mapping for this user and tenant
-            mapping = (
-                db_session.query(UserTenantMapping)
-                .filter(
-                    UserTenantMapping.email == email,
-                    UserTenantMapping.tenant_id == tenant_id,
-                    UserTenantMapping.active == False,  # noqa: E712
-                )
-                .first()
-            )
-
-            if mapping:
-                # Set all other mappings for this user to inactive
-                db_session.query(UserTenantMapping).filter(
-                    UserTenantMapping.email == email,
-                    UserTenantMapping.active == True,  # noqa: E712
-                ).update({"active": False})
-
-                # Activate this mapping
-                mapping.active = True
-                db_session.commit()
-                logger.info(f"User {email} accepted invitation to tenant {tenant_id}")
-            else:
-                logger.warning(
-                    f"No invitation found for user {email} in tenant {tenant_id}"
-                )
-
-        except Exception as e:
-            db_session.rollback()
-            logger.exception(
-                f"Failed to accept invitation for user {email} to tenant {tenant_id}: {str(e)}"
-            )
-            raise
-
-
-def deny_user_invite(email: str, tenant_id: str) -> None:
-    """
-    Deny an invitation to join a tenant.
-    This removes the user's mapping to the tenant.
-    """
-    with get_session_with_shared_schema() as db_session:
-        # Delete the mapping for this user and tenant
-        result = (
-            db_session.query(UserTenantMapping)
-            .filter(
-                UserTenantMapping.email == email,
-                UserTenantMapping.tenant_id == tenant_id,
-                UserTenantMapping.active == False,  # noqa: E712
-            )
-            .delete()
-        )
-
-        db_session.commit()
-        if result:
-            logger.info(f"User {email} denied invitation to tenant {tenant_id}")
-        else:
-            logger.warning(
-                f"No invitation found for user {email} in tenant {tenant_id}"
-            )
-    token = CURRENT_TENANT_ID_CONTEXTVAR.set(tenant_id)
-    try:
-        pending_users = get_invited_users()
-        if email in pending_users:
-            pending_users.remove(email)
-            write_invited_users(pending_users)
-    finally:
-        CURRENT_TENANT_ID_CONTEXTVAR.reset(token)
-
-
-def get_tenant_count(tenant_id: str) -> int:
-    """
-    Get the number of active users for this tenant
-    """
-    with get_session_with_shared_schema() as db_session:
-        # Count the number of active users for this tenant
-        user_count = (
-            db_session.query(UserTenantMapping)
-            .filter(
-                UserTenantMapping.tenant_id == tenant_id,
-                UserTenantMapping.active == True,  # noqa: E712
-            )
-            .count()
-        )
-
-        return user_count
-
-
-def get_tenant_invitation(email: str) -> TenantSnapshot | None:
-    """
-    Get the first tenant invitation for this user
-    """
-    with get_session_with_shared_schema() as db_session:
-        # Get the first tenant invitation for this user
-        invitation = (
-            db_session.query(UserTenantMapping)
-            .filter(
-                UserTenantMapping.email == email,
-                UserTenantMapping.active == False,  # noqa: E712
-            )
-            .first()
-        )
-
-        if invitation:
-            # Get the user count for this tenant
-            user_count = (
-                db_session.query(UserTenantMapping)
-                .filter(
-                    UserTenantMapping.tenant_id == invitation.tenant_id,
-                    UserTenantMapping.active == True,  # noqa: E712
-                )
-                .count()
-            )
-            return TenantSnapshot(
-                tenant_id=invitation.tenant_id, number_of_users=user_count
-            )
-
-        return None

backend/model_server/constants.py

@@ -3,7 +3,6 @@ from shared_configs.enums import EmbedTextType
 
 
 MODEL_WARM_UP_STRING = "hi " * 512
-INFORMATION_CONTENT_MODEL_WARM_UP_STRING = "hi " * 16
 DEFAULT_OPENAI_MODEL = "text-embedding-3-small"
 DEFAULT_COHERE_MODEL = "embed-english-light-v3.0"
 DEFAULT_VOYAGE_MODEL = "voyage-large-2-instruct"

backend/model_server/custom_models.py

@@ -1,14 +1,11 @@
-import numpy as np
 import torch
 import torch.nn.functional as F
 from fastapi import APIRouter
 from huggingface_hub import snapshot_download  # type: ignore
-from setfit import SetFitModel  # type: ignore[import]
 from transformers import AutoTokenizer  # type: ignore
 from transformers import BatchEncoding  # type: ignore
 from transformers import PreTrainedTokenizer  # type: ignore
 
-from model_server.constants import INFORMATION_CONTENT_MODEL_WARM_UP_STRING
 from model_server.constants import MODEL_WARM_UP_STRING
 from model_server.onyx_torch_model import ConnectorClassifier
 from model_server.onyx_torch_model import HybridClassifier
@@ -16,22 +13,11 @@ from model_server.utils import simple_log_function_time
 from onyx.utils.logger import setup_logger
 from shared_configs.configs import CONNECTOR_CLASSIFIER_MODEL_REPO
 from shared_configs.configs import CONNECTOR_CLASSIFIER_MODEL_TAG
-from shared_configs.configs import (
-    INDEXING_INFORMATION_CONTENT_CLASSIFICATION_CUTOFF_LENGTH,
-)
-from shared_configs.configs import INDEXING_INFORMATION_CONTENT_CLASSIFICATION_MAX
-from shared_configs.configs import INDEXING_INFORMATION_CONTENT_CLASSIFICATION_MIN
-from shared_configs.configs import (
-    INDEXING_INFORMATION_CONTENT_CLASSIFICATION_TEMPERATURE,
-)
 from shared_configs.configs import INDEXING_ONLY
-from shared_configs.configs import INFORMATION_CONTENT_MODEL_TAG
-from shared_configs.configs import INFORMATION_CONTENT_MODEL_VERSION
 from shared_configs.configs import INTENT_MODEL_TAG
 from shared_configs.configs import INTENT_MODEL_VERSION
 from shared_configs.model_server_models import ConnectorClassificationRequest
 from shared_configs.model_server_models import ConnectorClassificationResponse
-from shared_configs.model_server_models import ContentClassificationPrediction
 from shared_configs.model_server_models import IntentRequest
 from shared_configs.model_server_models import IntentResponse
 
@@ -45,10 +31,6 @@ _CONNECTOR_CLASSIFIER_MODEL: ConnectorClassifier | None = None
 _INTENT_TOKENIZER: AutoTokenizer | None = None
 _INTENT_MODEL: HybridClassifier | None = None
 
-_INFORMATION_CONTENT_MODEL: SetFitModel | None = None
-
-_INFORMATION_CONTENT_MODEL_PROMPT_PREFIX: str = ""  # spec to model version!
-
 
 def get_connector_classifier_tokenizer() -> AutoTokenizer:
     global _CONNECTOR_CLASSIFIER_TOKENIZER
@@ -103,7 +85,7 @@ def get_intent_model_tokenizer() -> AutoTokenizer:
 
 def get_local_intent_model(
     model_name_or_path: str = INTENT_MODEL_VERSION,
-    tag: str | None = INTENT_MODEL_TAG,
+    tag: str = INTENT_MODEL_TAG,
 ) -> HybridClassifier:
     global _INTENT_MODEL
     if _INTENT_MODEL is None:
@@ -120,9 +102,7 @@ def get_local_intent_model(
             try:
                 # Attempt to download the model snapshot
                 logger.notice(f"Downloading model snapshot for {model_name_or_path}")
-                local_path = snapshot_download(
-                    repo_id=model_name_or_path, revision=tag, local_files_only=False
-                )
+                local_path = snapshot_download(repo_id=model_name_or_path, revision=tag)
                 _INTENT_MODEL = HybridClassifier.from_pretrained(local_path)
             except Exception as e:
                 logger.error(
@@ -132,44 +112,6 @@ def get_local_intent_model(
     return _INTENT_MODEL
 
 
-def get_local_information_content_model(
-    model_name_or_path: str = INFORMATION_CONTENT_MODEL_VERSION,
-    tag: str | None = INFORMATION_CONTENT_MODEL_TAG,
-) -> SetFitModel:
-    global _INFORMATION_CONTENT_MODEL
-    if _INFORMATION_CONTENT_MODEL is None:
-        try:
-            # Calculate where the cache should be, then load from local if available
-            logger.notice(
-                f"Loading content information model from local cache: {model_name_or_path}"
-            )
-            local_path = snapshot_download(
-                repo_id=model_name_or_path, revision=tag, local_files_only=True
-            )
-            _INFORMATION_CONTENT_MODEL = SetFitModel.from_pretrained(local_path)
-            logger.notice(
-                f"Loaded content information model from local cache: {local_path}"
-            )
-        except Exception as e:
-            logger.warning(f"Failed to load content information model directly: {e}")
-            try:
-                # Attempt to download the model snapshot
-                logger.notice(
-                    f"Downloading content information model snapshot for {model_name_or_path}"
-                )
-                local_path = snapshot_download(
-                    repo_id=model_name_or_path, revision=tag, local_files_only=False
-                )
-                _INFORMATION_CONTENT_MODEL = SetFitModel.from_pretrained(local_path)
-            except Exception as e:
-                logger.error(
-                    f"Failed to load content information model even after attempted snapshot download: {e}"
-                )
-                raise
-
-    return _INFORMATION_CONTENT_MODEL
-
-
 def tokenize_connector_classification_query(
     connectors: list[str],
     query: str,
@@ -253,13 +195,6 @@ def warm_up_intent_model() -> None:
     )
 
 
-def warm_up_information_content_model() -> None:
-    logger.notice("Warming up Content Model")  # TODO: add version if needed
-
-    information_content_model = get_local_information_content_model()
-    information_content_model(INFORMATION_CONTENT_MODEL_WARM_UP_STRING)
-
-
 @simple_log_function_time()
 def run_inference(tokens: BatchEncoding) -> tuple[list[float], list[float]]:
     intent_model = get_local_intent_model()
@@ -283,117 +218,6 @@ def run_inference(tokens: BatchEncoding) -> tuple[list[float], list[float]]:
     return intent_probabilities.tolist(), token_positive_probs
 
 
-@simple_log_function_time()
-def run_content_classification_inference(
-    text_inputs: list[str],
-) -> list[ContentClassificationPrediction]:
-    """
-    Assign a score to the segments in question. The model stored in get_local_information_content_model()
-    creates the 'model score' based on its training, and the scores are then converted to a 0.0-1.0 scale.
-    In the code outside of the model/inference model servers that score will be converted into the actual
-    boost factor.
-    """
-
-    def _prob_to_score(prob: float) -> float:
-        """
-        Conversion of base score to 0.0 - 1.0 score. Note that the min/max values depend on the model!
-        """
-        _MIN_BASE_SCORE = 0.25
-        _MAX_BASE_SCORE = 0.75
-        if prob < _MIN_BASE_SCORE:
-            raw_score = 0.0
-        elif prob < _MAX_BASE_SCORE:
-            raw_score = (prob - _MIN_BASE_SCORE) / (_MAX_BASE_SCORE - _MIN_BASE_SCORE)
-        else:
-            raw_score = 1.0
-        return (
-            INDEXING_INFORMATION_CONTENT_CLASSIFICATION_MIN
-            + (
-                INDEXING_INFORMATION_CONTENT_CLASSIFICATION_MAX
-                - INDEXING_INFORMATION_CONTENT_CLASSIFICATION_MIN
-            )
-            * raw_score
-        )
-
-    _BATCH_SIZE = 32
-    content_model = get_local_information_content_model()
-
-    # Process inputs in batches
-    all_output_classes: list[int] = []
-    all_base_output_probabilities: list[float] = []
-
-    for i in range(0, len(text_inputs), _BATCH_SIZE):
-        batch = text_inputs[i : i + _BATCH_SIZE]
-        batch_with_prefix = []
-        batch_indices = []
-
-        # Pre-allocate results for this batch
-        batch_output_classes: list[np.ndarray] = [np.array(1)] * len(batch)
-        batch_probabilities: list[np.ndarray] = [np.array(1.0)] * len(batch)
-
-        # Pre-process batch to handle long input exceptions
-        for j, text in enumerate(batch):
-            if len(text) == 0:
-                # if no input, treat as non-informative from the model's perspective
-                batch_output_classes[j] = np.array(0)
-                batch_probabilities[j] = np.array(0.0)
-                logger.warning("Input for Content Information Model is empty")
-
-            elif (
-                len(text.split())
-                <= INDEXING_INFORMATION_CONTENT_CLASSIFICATION_CUTOFF_LENGTH
-            ):
-                # if input is short, use the model
-                batch_with_prefix.append(
-                    _INFORMATION_CONTENT_MODEL_PROMPT_PREFIX + text
-                )
-                batch_indices.append(j)
-            else:
-                # if longer than cutoff, treat as informative (stay with default), but issue warning
-                logger.warning("Input for Content Information Model too long")
-
-        if batch_with_prefix:  # Only run model if we have valid inputs
-            # Get predictions for the batch
-            model_output_classes = content_model(batch_with_prefix)
-            model_output_probabilities = content_model.predict_proba(batch_with_prefix)
-
-            # Place results in the correct positions
-            for idx, batch_idx in enumerate(batch_indices):
-                batch_output_classes[batch_idx] = model_output_classes[idx].numpy()
-                batch_probabilities[batch_idx] = model_output_probabilities[idx][
-                    1
-                ].numpy()  # x[1] is prob of the positive class
-
-        all_output_classes.extend([int(x) for x in batch_output_classes])
-        all_base_output_probabilities.extend([float(x) for x in batch_probabilities])
-
-    logits = [
-        np.log(p / (1 - p)) if p != 0.0 and p != 1.0 else (100 if p == 1.0 else -100)
-        for p in all_base_output_probabilities
-    ]
-    scaled_logits = [
-        logit / INDEXING_INFORMATION_CONTENT_CLASSIFICATION_TEMPERATURE
-        for logit in logits
-    ]
-    output_probabilities_with_temp = [
-        np.exp(scaled_logit) / (1 + np.exp(scaled_logit))
-        for scaled_logit in scaled_logits
-    ]
-
-    prediction_scores = [
-        _prob_to_score(p_temp) for p_temp in output_probabilities_with_temp
-    ]
-
-    content_classification_predictions = [
-        ContentClassificationPrediction(
-            predicted_label=predicted_label, content_boost_factor=output_score
-        )
-        for predicted_label, output_score in zip(all_output_classes, prediction_scores)
-    ]
-
-    return content_classification_predictions
-
-
 def map_keywords(
     input_ids: torch.Tensor, tokenizer: AutoTokenizer, is_keyword: list[bool]
 ) -> list[str]:
@@ -538,10 +362,3 @@ async def process_analysis_request(
 
     is_keyword, keywords = run_analysis(intent_request)
     return IntentResponse(is_keyword=is_keyword, keywords=keywords)
-
-
-@router.post("/content-classification")
-async def process_content_classification_request(
-    content_classification_requests: list[str],
-) -> list[ContentClassificationPrediction]:
-    return run_content_classification_inference(content_classification_requests)

backend/model_server/encoders.py

@@ -62,60 +62,6 @@ _OPENAI_MAX_INPUT_LEN = 2048
 # Cohere allows up to 96 embeddings in a single embedding calling
 _COHERE_MAX_INPUT_LEN = 96
 
-# Authentication error string constants
-_AUTH_ERROR_401 = "401"
-_AUTH_ERROR_UNAUTHORIZED = "unauthorized"
-_AUTH_ERROR_INVALID_API_KEY = "invalid api key"
-_AUTH_ERROR_PERMISSION = "permission"
-
-
-def is_authentication_error(error: Exception) -> bool:
-    """Check if an exception is related to authentication issues.
-
-    Args:
-        error: The exception to check
-
-    Returns:
-        bool: True if the error appears to be authentication-related
-    """
-    error_str = str(error).lower()
-    return (
-        _AUTH_ERROR_401 in error_str
-        or _AUTH_ERROR_UNAUTHORIZED in error_str
-        or _AUTH_ERROR_INVALID_API_KEY in error_str
-        or _AUTH_ERROR_PERMISSION in error_str
-    )
-
-
-def format_embedding_error(
-    error: Exception,
-    service_name: str,
-    model: str | None,
-    provider: EmbeddingProvider,
-    status_code: int | None = None,
-) -> str:
-    """
-    Format a standardized error string for embedding errors.
-    """
-    detail = f"Status {status_code}" if status_code else f"{type(error)}"
-
-    return (
-        f"{'HTTP error' if status_code else 'Exception'} embedding text with {service_name} - {detail}: "
-        f"Model: {model} "
-        f"Provider: {provider} "
-        f"Exception: {error}"
-    )
-
-
-# Custom exception for authentication errors
-class AuthenticationError(Exception):
-    """Raised when authentication fails with a provider."""
-
-    def __init__(self, provider: str, message: str = "API key is invalid or expired"):
-        self.provider = provider
-        self.message = message
-        super().__init__(f"{provider} authentication failed: {message}")
-
 
 class CloudEmbedding:
     def __init__(
@@ -146,17 +92,31 @@ class CloudEmbedding:
         )
 
         final_embeddings: list[Embedding] = []
+        try:
+            for text_batch in batch_list(texts, _OPENAI_MAX_INPUT_LEN):
+                response = await client.embeddings.create(
+                    input=text_batch,
+                    model=model,
+                    dimensions=reduced_dimension or openai.NOT_GIVEN,
+                )
+                final_embeddings.extend(
+                    [embedding.embedding for embedding in response.data]
+                )
+            return final_embeddings
+        except Exception as e:
+            error_string = (
+                f"Exception embedding text with OpenAI - {type(e)}: "
+                f"Model: {model} "
+                f"Provider: {self.provider} "
+                f"Exception: {e}"
+            )
+            logger.error(error_string)
 
-        for text_batch in batch_list(texts, _OPENAI_MAX_INPUT_LEN):
-            response = await client.embeddings.create(
-                input=text_batch,
-                model=model,
-                dimensions=reduced_dimension or openai.NOT_GIVEN,
-            )
-            final_embeddings.extend(
-                [embedding.embedding for embedding in response.data]
-            )
-        return final_embeddings
+            # only log text when it's not an authentication error.
+            if not isinstance(e, openai.AuthenticationError):
+                logger.debug(f"Exception texts: {texts}")
+
+            raise RuntimeError(error_string)
 
     async def _embed_cohere(
         self, texts: list[str], model: str | None, embedding_type: str
@@ -195,6 +155,7 @@ class CloudEmbedding:
             input_type=embedding_type,
             truncation=True,
         )
+
         return response.embeddings
 
     async def _embed_azure(
@@ -278,51 +239,22 @@ class CloudEmbedding:
         deployment_name: str | None = None,
         reduced_dimension: int | None = None,
     ) -> list[Embedding]:
-        try:
-            if self.provider == EmbeddingProvider.OPENAI:
-                return await self._embed_openai(texts, model_name, reduced_dimension)
-            elif self.provider == EmbeddingProvider.AZURE:
-                return await self._embed_azure(texts, f"azure/{deployment_name}")
-            elif self.provider == EmbeddingProvider.LITELLM:
-                return await self._embed_litellm_proxy(texts, model_name)
+        if self.provider == EmbeddingProvider.OPENAI:
+            return await self._embed_openai(texts, model_name, reduced_dimension)
+        elif self.provider == EmbeddingProvider.AZURE:
+            return await self._embed_azure(texts, f"azure/{deployment_name}")
+        elif self.provider == EmbeddingProvider.LITELLM:
+            return await self._embed_litellm_proxy(texts, model_name)
 
-            embedding_type = EmbeddingModelTextType.get_type(self.provider, text_type)
-            if self.provider == EmbeddingProvider.COHERE:
-                return await self._embed_cohere(texts, model_name, embedding_type)
-            elif self.provider == EmbeddingProvider.VOYAGE:
-                return await self._embed_voyage(texts, model_name, embedding_type)
-            elif self.provider == EmbeddingProvider.GOOGLE:
-                return await self._embed_vertex(texts, model_name, embedding_type)
-            else:
-                raise ValueError(f"Unsupported provider: {self.provider}")
-        except openai.AuthenticationError:
-            raise AuthenticationError(provider="OpenAI")
-        except httpx.HTTPStatusError as e:
-            if e.response.status_code == 401:
-                raise AuthenticationError(provider=str(self.provider))
-
-            error_string = format_embedding_error(
-                e,
-                str(self.provider),
-                model_name or deployment_name,
-                self.provider,
-                status_code=e.response.status_code,
-            )
-            logger.error(error_string)
-            logger.debug(f"Exception texts: {texts}")
-
-            raise RuntimeError(error_string)
-        except Exception as e:
-            if is_authentication_error(e):
-                raise AuthenticationError(provider=str(self.provider))
-
-            error_string = format_embedding_error(
-                e, str(self.provider), model_name or deployment_name, self.provider
-            )
-            logger.error(error_string)
-            logger.debug(f"Exception texts: {texts}")
-
-            raise RuntimeError(error_string)
+        embedding_type = EmbeddingModelTextType.get_type(self.provider, text_type)
+        if self.provider == EmbeddingProvider.COHERE:
+            return await self._embed_cohere(texts, model_name, embedding_type)
+        elif self.provider == EmbeddingProvider.VOYAGE:
+            return await self._embed_voyage(texts, model_name, embedding_type)
+        elif self.provider == EmbeddingProvider.GOOGLE:
+            return await self._embed_vertex(texts, model_name, embedding_type)
+        else:
+            raise ValueError(f"Unsupported provider: {self.provider}")
 
     @staticmethod
     def create(
@@ -637,13 +569,6 @@ async def process_embed_request(
             gpu_type=gpu_type,
         )
         return EmbedResponse(embeddings=embeddings)
-    except AuthenticationError as e:
-        # Handle authentication errors consistently
-        logger.error(f"Authentication error: {e.provider}")
-        raise HTTPException(
-            status_code=401,
-            detail=f"Authentication failed: {e.message}",
-        )
     except RateLimitError as e:
         raise HTTPException(
             status_code=429,

backend/model_server/main.py

@@ -13,7 +13,6 @@ from sentry_sdk.integrations.starlette import StarletteIntegration
 from transformers import logging as transformer_logging  # type:ignore
 
 from model_server.custom_models import router as custom_models_router
-from model_server.custom_models import warm_up_information_content_model
 from model_server.custom_models import warm_up_intent_model
 from model_server.encoders import router as encoders_router
 from model_server.management_endpoints import router as management_router
@@ -65,31 +64,19 @@ async def lifespan(app: FastAPI) -> AsyncGenerator:
 
     app.state.gpu_type = gpu_type
 
-    try:
-        if TEMP_HF_CACHE_PATH.is_dir():
-            logger.notice("Moving contents of temp_huggingface to huggingface cache.")
-            _move_files_recursively(TEMP_HF_CACHE_PATH, HF_CACHE_PATH)
-            shutil.rmtree(TEMP_HF_CACHE_PATH, ignore_errors=True)
-            logger.notice("Moved contents of temp_huggingface to huggingface cache.")
-    except Exception as e:
-        logger.warning(
-            f"Error moving contents of temp_huggingface to huggingface cache: {e}. "
-            "This is not a critical error and the model server will continue to run."
-        )
+    if TEMP_HF_CACHE_PATH.is_dir():
+        logger.notice("Moving contents of temp_huggingface to huggingface cache.")
+        _move_files_recursively(TEMP_HF_CACHE_PATH, HF_CACHE_PATH)
+        shutil.rmtree(TEMP_HF_CACHE_PATH, ignore_errors=True)
+        logger.notice("Moved contents of temp_huggingface to huggingface cache.")
 
     torch.set_num_threads(max(MIN_THREADS_ML_MODELS, torch.get_num_threads()))
     logger.notice(f"Torch Threads: {torch.get_num_threads()}")
 
     if not INDEXING_ONLY:
-        logger.notice(
-            "The intent model should run on the model server. The information content model should not run here."
-        )
         warm_up_intent_model()
     else:
-        logger.notice(
-            "The content information model should run on the indexing model server. The intent model should not run here."
-        )
-        warm_up_information_content_model()
+        logger.notice("This model server should only run document indexing.")
 
     yield
 

backend/onyx/access/models.py

@@ -20,7 +20,7 @@ class ExternalAccess:
 class DocExternalAccess:
     """
     This is just a class to wrap the external access and the document ID
-    together. It's used for syncing document permissions to Vespa.
+    together. It's used for syncing document permissions to Redis.
     """
 
     external_access: ExternalAccess

backend/onyx/auth/email_utils.py

@@ -1,6 +1,5 @@
 import smtplib
 from datetime import datetime
-from email.mime.image import MIMEImage
 from email.mime.multipart import MIMEMultipart
 from email.mime.text import MIMEText
 from email.utils import formatdate
@@ -14,13 +13,8 @@ from onyx.configs.app_configs import SMTP_SERVER
 from onyx.configs.app_configs import SMTP_USER
 from onyx.configs.app_configs import WEB_DOMAIN
 from onyx.configs.constants import AuthType
-from onyx.configs.constants import ONYX_DEFAULT_APPLICATION_NAME
-from onyx.configs.constants import ONYX_SLACK_URL
 from onyx.configs.constants import TENANT_ID_COOKIE_NAME
 from onyx.db.models import User
-from onyx.server.runtime.onyx_runtime import OnyxRuntime
-from onyx.utils.file import FileWithMimeType
-from onyx.utils.variable_functionality import fetch_versioned_implementation
 from shared_configs.configs import MULTI_TENANT
 
 HTML_EMAIL_TEMPLATE = """\
@@ -103,8 +97,8 @@ HTML_EMAIL_TEMPLATE = """\
       <td class="header">
         <img
           style="background-color: #ffffff; border-radius: 8px;"
-          src="cid:logo.png"
-          alt="{application_name} Logo"
+          src="https://www.onyx.app/logos/customer/onyx.png"
+          alt="Onyx Logo"
         >
       </td>
     </tr>
@@ -119,8 +113,9 @@ HTML_EMAIL_TEMPLATE = """\
     </tr>
     <tr>
       <td class="footer">
-        © {year} {application_name}. All rights reserved.
-        {slack_fragment}
+        © {year} Onyx. All rights reserved.
+        <br>
+        Have questions? Join our Slack community <a href="https://join.slack.com/t/onyx-dot-app/shared_invite/zt-2twesxdr6-5iQitKZQpgq~hYIZ~dv3KA">here</a>.
       </td>
     </tr>
   </table>
@@ -130,27 +125,17 @@ HTML_EMAIL_TEMPLATE = """\
 
 
 def build_html_email(
-    application_name: str | None,
-    heading: str,
-    message: str,
-    cta_text: str | None = None,
-    cta_link: str | None = None,
+    heading: str, message: str, cta_text: str | None = None, cta_link: str | None = None
 ) -> str:
-    slack_fragment = ""
-    if application_name == ONYX_DEFAULT_APPLICATION_NAME:
-        slack_fragment = f'<br>Have questions? Join our Slack community <a href="{ONYX_SLACK_URL}">here</a>.'
-
     if cta_text and cta_link:
         cta_block = f'<a class="cta-button" href="{cta_link}">{cta_text}</a>'
     else:
         cta_block = ""
     return HTML_EMAIL_TEMPLATE.format(
-        application_name=application_name,
         title=heading,
         heading=heading,
         message=message,
         cta_block=cta_block,
-        slack_fragment=slack_fragment,
         year=datetime.now().year,
     )
 
@@ -161,7 +146,6 @@ def send_email(
     html_body: str,
     text_body: str,
     mail_from: str = EMAIL_FROM,
-    inline_png: tuple[str, bytes] | None = None,
 ) -> None:
     if not EMAIL_CONFIGURED:
         raise ValueError("Email is not configured.")
@@ -169,8 +153,7 @@ def send_email(
     msg = MIMEMultipart("alternative")
     msg["Subject"] = subject
     msg["To"] = user_email
-    if mail_from:
-        msg["From"] = mail_from
+    msg["From"] = mail_from
     msg["Date"] = formatdate(localtime=True)
     msg["Message-ID"] = make_msgid(domain="onyx.app")
 
@@ -180,12 +163,6 @@ def send_email(
     msg.attach(part_text)
     msg.attach(part_html)
 
-    if inline_png:
-        img = MIMEImage(inline_png[1], _subtype="png")
-        img.add_header("Content-ID", inline_png[0])  # CID reference
-        img.add_header("Content-Disposition", "inline", filename=inline_png[0])
-        msg.attach(img)
-
     try:
         with smtplib.SMTP(SMTP_SERVER, SMTP_PORT) as s:
             s.starttls()
@@ -196,21 +173,8 @@ def send_email(
 
 
 def send_subscription_cancellation_email(user_email: str) -> None:
-    """This is templated but isn't meaningful for whitelabeling."""
-
     # Example usage of the reusable HTML
-    try:
-        load_runtime_settings_fn = fetch_versioned_implementation(
-            "onyx.server.enterprise_settings.store", "load_runtime_settings"
-        )
-        settings = load_runtime_settings_fn()
-        application_name = settings.application_name
-    except ModuleNotFoundError:
-        application_name = ONYX_DEFAULT_APPLICATION_NAME
-
-    onyx_file = OnyxRuntime.get_emailable_logo()
-
-    subject = f"Your {application_name} Subscription Has Been Canceled"
+    subject = "Your Onyx Subscription Has Been Canceled"
     heading = "Subscription Canceled"
     message = (
         "<p>We're sorry to see you go.</p>"
@@ -219,48 +183,23 @@ def send_subscription_cancellation_email(user_email: str) -> None:
     )
     cta_text = "Renew Subscription"
     cta_link = "https://www.onyx.app/pricing"
-    html_content = build_html_email(
-        application_name,
-        heading,
-        message,
-        cta_text,
-        cta_link,
-    )
+    html_content = build_html_email(heading, message, cta_text, cta_link)
     text_content = (
         "We're sorry to see you go.\n"
         "Your subscription has been canceled and will end on your next billing date.\n"
         "If you change your mind, visit https://www.onyx.app/pricing"
     )
-    send_email(
-        user_email,
-        subject,
-        html_content,
-        text_content,
-        inline_png=("logo.png", onyx_file.data),
-    )
+    send_email(user_email, subject, html_content, text_content)
 
 
 def send_user_email_invite(
     user_email: str, current_user: User, auth_type: AuthType
 ) -> None:
-    onyx_file: FileWithMimeType | None = None
-
-    try:
-        load_runtime_settings_fn = fetch_versioned_implementation(
-            "onyx.server.enterprise_settings.store", "load_runtime_settings"
-        )
-        settings = load_runtime_settings_fn()
-        application_name = settings.application_name
-    except ModuleNotFoundError:
-        application_name = ONYX_DEFAULT_APPLICATION_NAME
-
-    onyx_file = OnyxRuntime.get_emailable_logo()
-
-    subject = f"Invitation to Join {application_name} Organization"
+    subject = "Invitation to Join Onyx Organization"
     heading = "You've Been Invited!"
 
     # the exact action taken by the user, and thus the message, depends on the auth type
-    message = f"<p>You have been invited by {current_user.email} to join an organization on {application_name}.</p>"
+    message = f"<p>You have been invited by {current_user.email} to join an organization on Onyx.</p>"
     if auth_type == AuthType.CLOUD:
         message += (
             "<p>To join the organization, please click the button below to set a password "
@@ -286,32 +225,19 @@ def send_user_email_invite(
 
     cta_text = "Join Organization"
     cta_link = f"{WEB_DOMAIN}/auth/signup?email={user_email}"
-
-    html_content = build_html_email(
-        application_name,
-        heading,
-        message,
-        cta_text,
-        cta_link,
-    )
+    html_content = build_html_email(heading, message, cta_text, cta_link)
 
     # text content is the fallback for clients that don't support HTML
     # not as critical, so not having special cases for each auth type
     text_content = (
-        f"You have been invited by {current_user.email} to join an organization on {application_name}.\n"
+        f"You have been invited by {current_user.email} to join an organization on Onyx.\n"
         "To join the organization, please visit the following link:\n"
         f"{WEB_DOMAIN}/auth/signup?email={user_email}\n"
     )
     if auth_type == AuthType.CLOUD:
         text_content += "You'll be asked to set a password or login with Google to complete your registration."
 
-    send_email(
-        user_email,
-        subject,
-        html_content,
-        text_content,
-        inline_png=("logo.png", onyx_file.data),
-    )
+    send_email(user_email, subject, html_content, text_content)
 
 
 def send_forgot_password_email(
@@ -321,36 +247,14 @@ def send_forgot_password_email(
     mail_from: str = EMAIL_FROM,
 ) -> None:
     # Builds a forgot password email with or without fancy HTML
-    try:
-        load_runtime_settings_fn = fetch_versioned_implementation(
-            "onyx.server.enterprise_settings.store", "load_runtime_settings"
-        )
-        settings = load_runtime_settings_fn()
-        application_name = settings.application_name
-    except ModuleNotFoundError:
-        application_name = ONYX_DEFAULT_APPLICATION_NAME
-
-    onyx_file = OnyxRuntime.get_emailable_logo()
-
-    subject = f"{application_name} Forgot Password"
+    subject = "Onyx Forgot Password"
     link = f"{WEB_DOMAIN}/auth/reset-password?token={token}"
     if MULTI_TENANT:
         link += f"&{TENANT_ID_COOKIE_NAME}={tenant_id}"
     message = f"<p>Click the following link to reset your password:</p><p>{link}</p>"
-    html_content = build_html_email(
-        application_name,
-        "Reset Your Password",
-        message,
-    )
+    html_content = build_html_email("Reset Your Password", message)
     text_content = f"Click the following link to reset your password: {link}"
-    send_email(
-        user_email,
-        subject,
-        html_content,
-        text_content,
-        mail_from,
-        inline_png=("logo.png", onyx_file.data),
-    )
+    send_email(user_email, subject, html_content, text_content, mail_from)
 
 
 def send_user_verification_email(
@@ -359,33 +263,11 @@ def send_user_verification_email(
     mail_from: str = EMAIL_FROM,
 ) -> None:
     # Builds a verification email
-    try:
-        load_runtime_settings_fn = fetch_versioned_implementation(
-            "onyx.server.enterprise_settings.store", "load_runtime_settings"
-        )
-        settings = load_runtime_settings_fn()
-        application_name = settings.application_name
-    except ModuleNotFoundError:
-        application_name = ONYX_DEFAULT_APPLICATION_NAME
-
-    onyx_file = OnyxRuntime.get_emailable_logo()
-
-    subject = f"{application_name} Email Verification"
+    subject = "Onyx Email Verification"
     link = f"{WEB_DOMAIN}/auth/verify-email?token={token}"
     message = (
         f"<p>Click the following link to verify your email address:</p><p>{link}</p>"
     )
-    html_content = build_html_email(
-        application_name,
-        "Verify Your Email",
-        message,
-    )
+    html_content = build_html_email("Verify Your Email", message)
     text_content = f"Click the following link to verify your email address: {link}"
-    send_email(
-        user_email,
-        subject,
-        html_content,
-        text_content,
-        mail_from,
-        inline_png=("logo.png", onyx_file.data),
-    )
+    send_email(user_email, subject, html_content, text_content, mail_from)

backend/onyx/auth/invited_users.py

@@ -1,6 +1,5 @@
 from typing import cast
 
-from onyx.configs.constants import KV_PENDING_USERS_KEY
 from onyx.configs.constants import KV_USER_STORE_KEY
 from onyx.key_value_store.factory import get_kv_store
 from onyx.key_value_store.interface import KvKeyNotFoundError
@@ -19,17 +18,3 @@ def write_invited_users(emails: list[str]) -> int:
     store = get_kv_store()
     store.store(KV_USER_STORE_KEY, cast(JSON_ro, emails))
     return len(emails)
-
-
-def get_pending_users() -> list[str]:
-    try:
-        store = get_kv_store()
-        return cast(list, store.load(KV_PENDING_USERS_KEY))
-    except KvKeyNotFoundError:
-        return list()
-
-
-def write_pending_users(emails: list[str]) -> int:
-    store = get_kv_store()
-    store.store(KV_PENDING_USERS_KEY, cast(JSON_ro, emails))
-    return len(emails)

backend/onyx/auth/users.py

@@ -100,12 +100,10 @@ from onyx.utils.logger import setup_logger
 from onyx.utils.telemetry import create_milestone_and_report
 from onyx.utils.telemetry import optional_telemetry
 from onyx.utils.telemetry import RecordType
-from onyx.utils.url import add_url_params
 from onyx.utils.variable_functionality import fetch_ee_implementation_or_noop
 from onyx.utils.variable_functionality import fetch_versioned_implementation
 from shared_configs.configs import async_return_default_schema
 from shared_configs.configs import MULTI_TENANT
-from shared_configs.configs import POSTGRES_DEFAULT_SCHEMA
 from shared_configs.contextvars import CURRENT_TENANT_ID_CONTEXTVAR
 from shared_configs.contextvars import get_current_tenant_id
 
@@ -594,7 +592,7 @@ class UserManager(UUIDIDMixin, BaseUserManager[User, uuid.UUID]):
             tenant_id = fetch_ee_implementation_or_noop(
                 "onyx.server.tenants.provisioning",
                 "get_tenant_id_for_email",
-                POSTGRES_DEFAULT_SCHEMA,
+                None,
             )(
                 email=email,
             )
@@ -896,7 +894,7 @@ async def current_limited_user(
     return await double_check_user(user)
 
 
-async def current_chat_accessible_user(
+async def current_chat_accesssible_user(
     user: User | None = Depends(optional_user),
 ) -> User | None:
     tenant_id = get_current_tenant_id()
@@ -1097,12 +1095,6 @@ def get_oauth_router(
 
         next_url = state_data.get("next_url", "/")
         referral_source = state_data.get("referral_source", None)
-        try:
-            tenant_id = fetch_ee_implementation_or_noop(
-                "onyx.server.tenants.user_mapping", "get_tenant_id_for_email", None
-            )(account_email)
-        except exceptions.UserNotExists:
-            tenant_id = None
 
         request.state.referral_source = referral_source
 
@@ -1134,14 +1126,9 @@ def get_oauth_router(
         # Login user
         response = await backend.login(strategy, user)
         await user_manager.on_after_login(user, request, response)
+
         # Prepare redirect response
-        if tenant_id is None:
-            # Use URL utility to add parameters
-            redirect_url = add_url_params(next_url, {"new_team": "true"})
-            redirect_response = RedirectResponse(redirect_url, status_code=302)
-        else:
-            # No parameters to add
-            redirect_response = RedirectResponse(next_url, status_code=302)
+        redirect_response = RedirectResponse(next_url, status_code=302)
 
         # Copy headers and other attributes from 'response' to 'redirect_response'
         for header_name, header_value in response.headers.items():
@@ -1153,7 +1140,6 @@ def get_oauth_router(
             redirect_response.status_code = response.status_code
         if hasattr(response, "media_type"):
             redirect_response.media_type = response.media_type
-
         return redirect_response
 
     return router

backend/onyx/background/celery/apps/light.py

@@ -112,6 +112,5 @@ celery_app.autodiscover_tasks(
         "onyx.background.celery.tasks.connector_deletion",
         "onyx.background.celery.tasks.doc_permission_syncing",
         "onyx.background.celery.tasks.indexing",
-        "onyx.background.celery.tasks.tenant_provisioning",
     ]
 )

backend/onyx/background/celery/apps/monitoring.py

@@ -92,6 +92,5 @@ def on_setup_logging(
 celery_app.autodiscover_tasks(
     [
         "onyx.background.celery.tasks.monitoring",
-        "onyx.background.celery.tasks.tenant_provisioning",
     ]
 )

backend/onyx/background/celery/tasks/beat_schedule.py

@@ -167,16 +167,6 @@ beat_cloud_tasks: list[dict] = [
             "expires": BEAT_EXPIRES_DEFAULT,
         },
     },
-    {
-        "name": f"{ONYX_CLOUD_CELERY_TASK_PREFIX}_check-available-tenants",
-        "task": OnyxCeleryTask.CHECK_AVAILABLE_TENANTS,
-        "schedule": timedelta(minutes=10),
-        "options": {
-            "queue": OnyxCeleryQueues.MONITORING,
-            "priority": OnyxCeleryPriority.HIGH,
-            "expires": BEAT_EXPIRES_DEFAULT,
-        },
-    },
 ]
 
 # tasks that only run self hosted
@@ -194,16 +184,6 @@ if not MULTI_TENANT:
                     "queue": OnyxCeleryQueues.MONITORING,
                 },
             },
-            {
-                "name": "monitor-process-memory",
-                "task": OnyxCeleryTask.MONITOR_PROCESS_MEMORY,
-                "schedule": timedelta(minutes=5),
-                "options": {
-                    "priority": OnyxCeleryPriority.LOW,
-                    "expires": BEAT_EXPIRES_DEFAULT,
-                    "queue": OnyxCeleryQueues.MONITORING,
-                },
-            },
         ]
     )
 

backend/onyx/background/celery/tasks/doc_permission_syncing/tasks.py

@@ -46,6 +46,7 @@ from onyx.configs.constants import OnyxRedisSignals
 from onyx.connectors.factory import validate_ccpair_for_user
 from onyx.db.connector import mark_cc_pair_as_permissions_synced
 from onyx.db.connector_credential_pair import get_connector_credential_pair_from_id
+from onyx.db.connector_credential_pair import update_connector_credential_pair
 from onyx.db.document import upsert_document_by_connector_credential_pair
 from onyx.db.engine import get_session_with_current_tenant
 from onyx.db.enums import AccessType
@@ -419,7 +420,12 @@ def connector_permission_sync_generator_task(
                 task_logger.exception(
                     f"validate_ccpair_permissions_sync exceptioned: cc_pair={cc_pair_id}"
                 )
-                # TODO: add some notification to the admins here
+                update_connector_credential_pair(
+                    db_session=db_session,
+                    connector_id=cc_pair.connector.id,
+                    credential_id=cc_pair.credential.id,
+                    status=ConnectorCredentialPairStatus.INVALID,
+                )
                 raise
 
             source_type = cc_pair.connector.source
@@ -447,23 +453,23 @@ def connector_permission_sync_generator_task(
             redis_connector.permissions.set_fence(new_payload)
 
             callback = PermissionSyncCallback(redis_connector, lock, r)
-            document_external_accesses = doc_sync_func(cc_pair, callback)
+            document_external_accesses: list[DocExternalAccess] = doc_sync_func(
+                cc_pair, callback
+            )
 
             task_logger.info(
                 f"RedisConnector.permissions.generate_tasks starting. cc_pair={cc_pair_id}"
             )
-
-            tasks_generated = 0
-            for doc_external_access in document_external_accesses:
-                redis_connector.permissions.generate_tasks(
-                    celery_app=self.app,
-                    lock=lock,
-                    new_permissions=[doc_external_access],
-                    source_string=source_type,
-                    connector_id=cc_pair.connector.id,
-                    credential_id=cc_pair.credential.id,
-                )
-                tasks_generated += 1
+            tasks_generated = redis_connector.permissions.generate_tasks(
+                celery_app=self.app,
+                lock=lock,
+                new_permissions=document_external_accesses,
+                source_string=source_type,
+                connector_id=cc_pair.connector.id,
+                credential_id=cc_pair.credential.id,
+            )
+            if tasks_generated is None:
+                return None
 
             task_logger.info(
                 f"RedisConnector.permissions.generate_tasks finished. "

backend/onyx/background/celery/tasks/external_group_syncing/tasks.py

@@ -41,6 +41,7 @@ from onyx.connectors.exceptions import ConnectorValidationError
 from onyx.connectors.factory import validate_ccpair_for_user
 from onyx.db.connector import mark_cc_pair_as_external_group_synced
 from onyx.db.connector_credential_pair import get_connector_credential_pair_from_id
+from onyx.db.connector_credential_pair import update_connector_credential_pair
 from onyx.db.engine import get_session_with_current_tenant
 from onyx.db.enums import AccessType
 from onyx.db.enums import ConnectorCredentialPairStatus
@@ -401,7 +402,12 @@ def connector_external_group_sync_generator_task(
                 task_logger.exception(
                     f"validate_ccpair_permissions_sync exceptioned: cc_pair={cc_pair_id}"
                 )
-                # TODO: add some notification to the admins here
+                update_connector_credential_pair(
+                    db_session=db_session,
+                    connector_id=cc_pair.connector.id,
+                    credential_id=cc_pair.credential.id,
+                    status=ConnectorCredentialPairStatus.INVALID,
+                )
                 raise
 
             source_type = cc_pair.connector.source
@@ -419,9 +425,12 @@ def connector_external_group_sync_generator_task(
             try:
                 external_user_groups = ext_group_sync_func(tenant_id, cc_pair)
             except ConnectorValidationError as e:
-                # TODO: add some notification to the admins here
-                logger.exception(
-                    f"Error syncing external groups for {source_type} for cc_pair: {cc_pair_id} {e}"
+                msg = f"Error syncing external groups for {source_type} for cc_pair: {cc_pair_id} {e}"
+                update_connector_credential_pair(
+                    db_session=db_session,
+                    connector_id=cc_pair.connector.id,
+                    credential_id=cc_pair.credential.id,
+                    status=ConnectorCredentialPairStatus.INVALID,
                 )
                 raise e
 

backend/onyx/background/celery/tasks/monitoring/tasks.py

@@ -6,7 +6,6 @@ from itertools import islice
 from typing import Any
 from typing import Literal
 
-import psutil
 from celery import shared_task
 from celery import Task
 from celery.exceptions import SoftTimeLimitExceeded
@@ -20,7 +19,6 @@ from sqlalchemy.orm import Session
 from onyx.background.celery.apps.app_base import task_logger
 from onyx.background.celery.celery_redis import celery_get_queue_length
 from onyx.background.celery.celery_redis import celery_get_unacked_task_ids
-from onyx.background.celery.memory_monitoring import emit_process_memory
 from onyx.configs.constants import CELERY_GENERIC_BEAT_LOCK_TIMEOUT
 from onyx.configs.constants import ONYX_CLOUD_TENANT_ID
 from onyx.configs.constants import OnyxCeleryQueues
@@ -41,10 +39,8 @@ from onyx.db.models import UserGroup
 from onyx.db.search_settings import get_active_search_settings_list
 from onyx.redis.redis_pool import get_redis_client
 from onyx.redis.redis_pool import redis_lock_dump
-from onyx.utils.logger import is_running_in_container
 from onyx.utils.telemetry import optional_telemetry
 from onyx.utils.telemetry import RecordType
-from shared_configs.configs import MULTI_TENANT
 from shared_configs.contextvars import CURRENT_TENANT_ID_CONTEXTVAR
 
 _MONITORING_SOFT_TIME_LIMIT = 60 * 5  # 5 minutes
@@ -908,93 +904,3 @@ def monitor_celery_queues_helper(
         f"external_group_sync={n_external_group_sync} "
         f"permissions_upsert={n_permissions_upsert} "
     )
-
-
-"""Memory monitoring"""
-
-
-def _get_cmdline_for_process(process: psutil.Process) -> str | None:
-    try:
-        return " ".join(process.cmdline())
-    except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess):
-        return None
-
-
-@shared_task(
-    name=OnyxCeleryTask.MONITOR_PROCESS_MEMORY,
-    ignore_result=True,
-    soft_time_limit=_MONITORING_SOFT_TIME_LIMIT,
-    time_limit=_MONITORING_TIME_LIMIT,
-    queue=OnyxCeleryQueues.MONITORING,
-    bind=True,
-)
-def monitor_process_memory(self: Task, *, tenant_id: str) -> None:
-    """
-    Task to monitor memory usage of supervisor-managed processes.
-    This periodically checks the memory usage of processes and logs information
-    in a standardized format.
-
-    The task looks for processes managed by supervisor and logs their
-    memory usage statistics. This is useful for monitoring memory consumption
-    over time and identifying potential memory leaks.
-    """
-    # don't run this task in multi-tenant mode, have other, better means of monitoring
-    if MULTI_TENANT:
-        return
-
-    # Skip memory monitoring if not in container
-    if not is_running_in_container():
-        return
-
-    try:
-        # Get all supervisor-managed processes
-        supervisor_processes: dict[int, str] = {}
-
-        # Map cmd line elements to more readable process names
-        process_type_mapping = {
-            "--hostname=primary": "primary",
-            "--hostname=light": "light",
-            "--hostname=heavy": "heavy",
-            "--hostname=indexing": "indexing",
-            "--hostname=monitoring": "monitoring",
-            "beat": "beat",
-            "slack/listener.py": "slack",
-        }
-
-        # Find all python processes that are likely celery workers
-        for proc in psutil.process_iter():
-            cmdline = _get_cmdline_for_process(proc)
-            if not cmdline:
-                continue
-
-            # Match supervisor-managed processes
-            for process_name, process_type in process_type_mapping.items():
-                if process_name in cmdline:
-                    if process_type in supervisor_processes.values():
-                        task_logger.error(
-                            f"Duplicate process type for type {process_type} "
-                            f"with cmd {cmdline} with pid={proc.pid}."
-                        )
-                        continue
-
-                    supervisor_processes[proc.pid] = process_type
-                    break
-
-        if len(supervisor_processes) != len(process_type_mapping):
-            task_logger.error(
-                "Missing processes: "
-                f"{set(process_type_mapping.keys()).symmetric_difference(supervisor_processes.values())}"
-            )
-
-        # Log memory usage for each process
-        for pid, process_type in supervisor_processes.items():
-            try:
-                emit_process_memory(pid, process_type, {})
-            except psutil.NoSuchProcess:
-                # Process may have terminated since we obtained the list
-                continue
-            except Exception as e:
-                task_logger.exception(f"Error monitoring process {pid}: {str(e)}")
-
-    except Exception:
-        task_logger.exception("Error in monitor_process_memory task")

backend/onyx/background/celery/tasks/tenant_provisioning/tasks.py

@@ -1,199 +0,0 @@
-"""
-Periodic tasks for tenant pre-provisioning.
-"""
-import asyncio
-import datetime
-import uuid
-
-from celery import shared_task
-from celery import Task
-from redis.lock import Lock as RedisLock
-
-from ee.onyx.server.tenants.provisioning import setup_tenant
-from ee.onyx.server.tenants.schema_management import create_schema_if_not_exists
-from ee.onyx.server.tenants.schema_management import get_current_alembic_version
-from onyx.background.celery.apps.app_base import task_logger
-from onyx.configs.app_configs import JOB_TIMEOUT
-from onyx.configs.app_configs import TARGET_AVAILABLE_TENANTS
-from onyx.configs.constants import OnyxCeleryPriority
-from onyx.configs.constants import OnyxCeleryQueues
-from onyx.configs.constants import OnyxCeleryTask
-from onyx.configs.constants import OnyxRedisLocks
-from onyx.db.engine import get_session_with_shared_schema
-from onyx.db.models import AvailableTenant
-from onyx.redis.redis_pool import get_redis_client
-from shared_configs.configs import MULTI_TENANT
-from shared_configs.configs import TENANT_ID_PREFIX
-
-# Default number of pre-provisioned tenants to maintain
-DEFAULT_TARGET_AVAILABLE_TENANTS = 5
-
-# Soft time limit for tenant pre-provisioning tasks (in seconds)
-_TENANT_PROVISIONING_SOFT_TIME_LIMIT = 60 * 5  # 5 minutes
-# Hard time limit for tenant pre-provisioning tasks (in seconds)
-_TENANT_PROVISIONING_TIME_LIMIT = 60 * 10  # 10 minutes
-
-
-@shared_task(
-    name=OnyxCeleryTask.CHECK_AVAILABLE_TENANTS,
-    queue=OnyxCeleryQueues.MONITORING,
-    ignore_result=True,
-    soft_time_limit=JOB_TIMEOUT,
-    trail=False,
-    bind=True,
-)
-def check_available_tenants(self: Task) -> None:
-    """
-    Check if we have enough pre-provisioned tenants available.
-    If not, trigger the pre-provisioning of new tenants.
-    """
-    task_logger.info("STARTING CHECK_AVAILABLE_TENANTS")
-    if not MULTI_TENANT:
-        task_logger.info(
-            "Multi-tenancy is not enabled, skipping tenant pre-provisioning"
-        )
-        return
-
-    r = get_redis_client()
-    lock_check: RedisLock = r.lock(
-        OnyxRedisLocks.CHECK_AVAILABLE_TENANTS_LOCK,
-        timeout=_TENANT_PROVISIONING_SOFT_TIME_LIMIT,
-    )
-
-    # These tasks should never overlap
-    if not lock_check.acquire(blocking=False):
-        task_logger.info(
-            "Skipping check_available_tenants task because it is already running"
-        )
-        return
-
-    try:
-        # Get the current count of available tenants
-        with get_session_with_shared_schema() as db_session:
-            available_tenants_count = db_session.query(AvailableTenant).count()
-
-        # Get the target number of available tenants
-        target_available_tenants = getattr(
-            TARGET_AVAILABLE_TENANTS, "value", DEFAULT_TARGET_AVAILABLE_TENANTS
-        )
-
-        # Calculate how many new tenants we need to provision
-        tenants_to_provision = max(
-            0, target_available_tenants - available_tenants_count
-        )
-
-        task_logger.info(
-            f"Available tenants: {available_tenants_count}, "
-            f"Target: {target_available_tenants}, "
-            f"To provision: {tenants_to_provision}"
-        )
-
-        # Trigger pre-provisioning tasks for each tenant needed
-        for _ in range(tenants_to_provision):
-            from celery import current_app
-
-            current_app.send_task(
-                OnyxCeleryTask.PRE_PROVISION_TENANT,
-                priority=OnyxCeleryPriority.LOW,
-            )
-
-    except Exception:
-        task_logger.exception("Error in check_available_tenants task")
-
-    finally:
-        lock_check.release()
-
-
-@shared_task(
-    name=OnyxCeleryTask.PRE_PROVISION_TENANT,
-    ignore_result=True,
-    soft_time_limit=_TENANT_PROVISIONING_SOFT_TIME_LIMIT,
-    time_limit=_TENANT_PROVISIONING_TIME_LIMIT,
-    queue=OnyxCeleryQueues.MONITORING,
-    bind=True,
-)
-def pre_provision_tenant(self: Task) -> None:
-    """
-    Pre-provision a new tenant and store it in the NewAvailableTenant table.
-    This function fully sets up the tenant with all necessary configurations,
-    so it's ready to be assigned to a user immediately.
-    """
-    # The MULTI_TENANT check is now done at the caller level (check_available_tenants)
-    # rather than inside this function
-
-    r = get_redis_client()
-    lock_provision: RedisLock = r.lock(
-        OnyxRedisLocks.PRE_PROVISION_TENANT_LOCK,
-        timeout=_TENANT_PROVISIONING_SOFT_TIME_LIMIT,
-    )
-
-    # Allow multiple pre-provisioning tasks to run, but ensure they don't overlap
-    if not lock_provision.acquire(blocking=False):
-        task_logger.debug(
-            "Skipping pre_provision_tenant task because it is already running"
-        )
-        return
-
-    tenant_id: str | None = None
-    try:
-        # Generate a new tenant ID
-        tenant_id = TENANT_ID_PREFIX + str(uuid.uuid4())
-        task_logger.info(f"Pre-provisioning tenant: {tenant_id}")
-
-        # Create the schema for the new tenant
-        schema_created = create_schema_if_not_exists(tenant_id)
-        if schema_created:
-            task_logger.debug(f"Created schema for tenant: {tenant_id}")
-        else:
-            task_logger.debug(f"Schema already exists for tenant: {tenant_id}")
-
-        # Set up the tenant with all necessary configurations
-        task_logger.debug(f"Setting up tenant configuration: {tenant_id}")
-        asyncio.run(setup_tenant(tenant_id))
-        task_logger.debug(f"Tenant configuration completed: {tenant_id}")
-
-        # Get the current Alembic version
-        alembic_version = get_current_alembic_version(tenant_id)
-        task_logger.debug(
-            f"Tenant {tenant_id} using Alembic version: {alembic_version}"
-        )
-
-        # Store the pre-provisioned tenant in the database
-        task_logger.debug(f"Storing pre-provisioned tenant in database: {tenant_id}")
-        with get_session_with_shared_schema() as db_session:
-            # Use a transaction to ensure atomicity
-            db_session.begin()
-            try:
-                new_tenant = AvailableTenant(
-                    tenant_id=tenant_id,
-                    alembic_version=alembic_version,
-                    date_created=datetime.datetime.now(),
-                )
-                db_session.add(new_tenant)
-                db_session.commit()
-                task_logger.info(f"Successfully pre-provisioned tenant: {tenant_id}")
-            except Exception:
-                db_session.rollback()
-                task_logger.error(
-                    f"Failed to store pre-provisioned tenant: {tenant_id}",
-                    exc_info=True,
-                )
-                raise
-
-    except Exception:
-        task_logger.error("Error in pre_provision_tenant task", exc_info=True)
-        # If we have a tenant_id, attempt to rollback any partially completed provisioning
-        if tenant_id:
-            task_logger.info(
-                f"Rolling back failed tenant provisioning for: {tenant_id}"
-            )
-            try:
-                from ee.onyx.server.tenants.provisioning import (
-                    rollback_tenant_provisioning,
-                )
-
-                asyncio.run(rollback_tenant_provisioning(tenant_id))
-            except Exception:
-                task_logger.exception(f"Error during rollback for tenant: {tenant_id}")
-    finally:
-        lock_provision.release()

backend/onyx/background/celery/tasks/vespa/tasks.py

@@ -563,7 +563,6 @@ def vespa_metadata_sync_task(self: Task, document_id: str, *, tenant_id: str) ->
                     access=doc_access,
                     boost=doc.boost,
                     hidden=doc.hidden,
-                    # aggregated_boost_factor=doc.aggregated_boost_factor,
                 )
 
                 # update Vespa. OK if doc doesn't exist. Raises exception otherwise.

backend/onyx/background/indexing/checkpointing_utils.py

@@ -6,8 +6,6 @@ from sqlalchemy import and_
 from sqlalchemy.orm import Session
 
 from onyx.configs.constants import FileOrigin
-from onyx.connectors.interfaces import BaseConnector
-from onyx.connectors.interfaces import CheckpointConnector
 from onyx.connectors.models import ConnectorCheckpoint
 from onyx.db.engine import get_db_current_time
 from onyx.db.index_attempt import get_index_attempt
@@ -18,6 +16,7 @@ from onyx.file_store.file_store import get_default_file_store
 from onyx.utils.logger import setup_logger
 from onyx.utils.object_size_check import deep_getsizeof
 
+
 logger = setup_logger()
 
 _NUM_RECENT_ATTEMPTS_TO_CONSIDER = 20
@@ -53,7 +52,7 @@ def save_checkpoint(
 
 
 def load_checkpoint(
-    db_session: Session, index_attempt_id: int, connector: BaseConnector
+    db_session: Session, index_attempt_id: int
 ) -> ConnectorCheckpoint | None:
     """Load a checkpoint for a given index attempt from the file store"""
     checkpoint_pointer = _build_checkpoint_pointer(index_attempt_id)
@@ -61,8 +60,6 @@ def load_checkpoint(
     try:
         checkpoint_io = file_store.read_file(checkpoint_pointer, mode="rb")
         checkpoint_data = checkpoint_io.read().decode("utf-8")
-        if isinstance(connector, CheckpointConnector):
-            return connector.validate_checkpoint_json(checkpoint_data)
         return ConnectorCheckpoint.model_validate_json(checkpoint_data)
     except RuntimeError:
         return None
@@ -74,7 +71,6 @@ def get_latest_valid_checkpoint(
     search_settings_id: int,
     window_start: datetime,
     window_end: datetime,
-    connector: BaseConnector,
 ) -> ConnectorCheckpoint:
     """Get the latest valid checkpoint for a given connector credential pair"""
     checkpoint_candidates = get_recent_completed_attempts_for_cc_pair(
@@ -109,7 +105,7 @@ def get_latest_valid_checkpoint(
             f"for cc_pair={cc_pair_id}. Ignoring checkpoint to let the run start "
             "from scratch."
         )
-        return connector.build_dummy_checkpoint()
+        return ConnectorCheckpoint.build_dummy_checkpoint()
 
     # assumes latest checkpoint is the furthest along. This only isn't true
     # if something else has gone wrong.
@@ -117,13 +113,12 @@ def get_latest_valid_checkpoint(
         checkpoint_candidates[0] if checkpoint_candidates else None
     )
 
-    checkpoint = connector.build_dummy_checkpoint()
+    checkpoint = ConnectorCheckpoint.build_dummy_checkpoint()
     if latest_valid_checkpoint_candidate:
         try:
             previous_checkpoint = load_checkpoint(
                 db_session=db_session,
                 index_attempt_id=latest_valid_checkpoint_candidate.id,
-                connector=connector,
             )
         except Exception:
             logger.exception(
@@ -198,7 +193,7 @@ def cleanup_checkpoint(db_session: Session, index_attempt_id: int) -> None:
 
 def check_checkpoint_size(checkpoint: ConnectorCheckpoint) -> None:
     """Check if the checkpoint content size exceeds the limit (200MB)"""
-    content_size = deep_getsizeof(checkpoint.model_dump())
+    content_size = deep_getsizeof(checkpoint.checkpoint_content)
     if content_size > 200_000_000:  # 200MB in bytes
         raise ValueError(
             f"Checkpoint content size ({content_size} bytes) exceeds 200MB limit"

backend/onyx/background/indexing/run_indexing.py

@@ -24,18 +24,15 @@ from onyx.connectors.connector_runner import ConnectorRunner
 from onyx.connectors.exceptions import ConnectorValidationError
 from onyx.connectors.exceptions import UnexpectedValidationError
 from onyx.connectors.factory import instantiate_connector
+from onyx.connectors.models import ConnectorCheckpoint
 from onyx.connectors.models import ConnectorFailure
 from onyx.connectors.models import Document
 from onyx.connectors.models import IndexAttemptMetadata
-from onyx.connectors.models import TextSection
 from onyx.db.connector_credential_pair import get_connector_credential_pair_from_id
 from onyx.db.connector_credential_pair import get_last_successful_attempt_time
 from onyx.db.connector_credential_pair import update_connector_credential_pair
-from onyx.db.constants import CONNECTOR_VALIDATION_ERROR_MESSAGE_PREFIX
 from onyx.db.engine import get_session_with_current_tenant
 from onyx.db.enums import ConnectorCredentialPairStatus
-from onyx.db.enums import IndexingStatus
-from onyx.db.enums import IndexModelStatus
 from onyx.db.index_attempt import create_index_attempt_error
 from onyx.db.index_attempt import get_index_attempt
 from onyx.db.index_attempt import get_index_attempt_errors_for_cc_pair
@@ -48,14 +45,13 @@ from onyx.db.index_attempt import transition_attempt_to_in_progress
 from onyx.db.index_attempt import update_docs_indexed
 from onyx.db.models import IndexAttempt
 from onyx.db.models import IndexAttemptError
+from onyx.db.models import IndexingStatus
+from onyx.db.models import IndexModelStatus
 from onyx.document_index.factory import get_default_document_index
 from onyx.httpx.httpx_pool import HttpxPool
 from onyx.indexing.embedder import DefaultIndexingEmbedder
 from onyx.indexing.indexing_heartbeat import IndexingHeartbeatInterface
 from onyx.indexing.indexing_pipeline import build_indexing_pipeline
-from onyx.natural_language_processing.search_nlp_models import (
-    InformationContentClassificationModel,
-)
 from onyx.utils.logger import setup_logger
 from onyx.utils.logger import TaskAttemptSingleton
 from onyx.utils.telemetry import create_milestone_and_report
@@ -158,12 +154,14 @@ def strip_null_characters(doc_batch: list[Document]) -> list[Document]:
             )
 
         for section in cleaned_doc.sections:
-            if section.link is not None:
+            if section.link and "\x00" in section.link:
+                logger.warning(
+                    f"NUL characters found in document link for document: {cleaned_doc.id}"
+                )
                 section.link = section.link.replace("\x00", "")
 
             # since text can be longer, just replace to avoid double scan
-            if isinstance(section, TextSection) and section.text is not None:
-                section.text = section.text.replace("\x00", "")
+            section.text = section.text.replace("\x00", "")
 
         cleaned_batch.append(cleaned_doc)
 
@@ -351,8 +349,6 @@ def _run_indexing(
             callback=callback,
         )
 
-    information_content_classification_model = InformationContentClassificationModel()
-
     document_index = get_default_document_index(
         index_attempt_start.search_settings,
         None,
@@ -361,7 +357,6 @@ def _run_indexing(
 
     indexing_pipeline = build_indexing_pipeline(
         embedder=embedding_model,
-        information_content_classification_model=information_content_classification_model,
         document_index=document_index,
         ignore_time_skip=(
             ctx.from_beginning
@@ -387,7 +382,6 @@ def _run_indexing(
     net_doc_change = 0
     document_count = 0
     chunk_count = 0
-    index_attempt: IndexAttempt | None = None
     try:
         with get_session_with_current_tenant() as db_session_temp:
             index_attempt = get_index_attempt(db_session_temp, index_attempt_id)
@@ -406,7 +400,7 @@ def _run_indexing(
             # the beginning in order to avoid weird interactions between
             # checkpointing / failure handling.
             if index_attempt.from_beginning:
-                checkpoint = connector_runner.connector.build_dummy_checkpoint()
+                checkpoint = ConnectorCheckpoint.build_dummy_checkpoint()
             else:
                 checkpoint = get_latest_valid_checkpoint(
                     db_session=db_session_temp,
@@ -414,7 +408,6 @@ def _run_indexing(
                     search_settings_id=index_attempt.search_settings_id,
                     window_start=window_start,
                     window_end=window_end,
-                    connector=connector_runner.connector,
                 )
 
             unresolved_errors = get_index_attempt_errors_for_cc_pair(
@@ -486,11 +479,7 @@ def _run_indexing(
 
                     doc_size = 0
                     for section in doc.sections:
-                        if (
-                            isinstance(section, TextSection)
-                            and section.text is not None
-                        ):
-                            doc_size += len(section.text)
+                        doc_size += len(section.text)
 
                     if doc_size > INDEXING_SIZE_WARNING_THRESHOLD:
                         logger.warning(
@@ -598,44 +587,16 @@ def _run_indexing(
                 mark_attempt_canceled(
                     index_attempt_id,
                     db_session_temp,
-                    reason=f"{CONNECTOR_VALIDATION_ERROR_MESSAGE_PREFIX}{str(e)}",
+                    reason=str(e),
                 )
 
                 if ctx.is_primary:
-                    if not index_attempt:
-                        # should always be set by now
-                        raise RuntimeError("Should never happen.")
-
-                    VALIDATION_ERROR_THRESHOLD = 5
-
-                    recent_index_attempts = get_recent_completed_attempts_for_cc_pair(
-                        cc_pair_id=ctx.cc_pair_id,
-                        search_settings_id=index_attempt.search_settings_id,
-                        limit=VALIDATION_ERROR_THRESHOLD,
+                    update_connector_credential_pair(
                         db_session=db_session_temp,
+                        connector_id=ctx.connector_id,
+                        credential_id=ctx.credential_id,
+                        status=ConnectorCredentialPairStatus.INVALID,
                     )
-                    num_validation_errors = len(
-                        [
-                            index_attempt
-                            for index_attempt in recent_index_attempts
-                            if index_attempt.error_msg
-                            and index_attempt.error_msg.startswith(
-                                CONNECTOR_VALIDATION_ERROR_MESSAGE_PREFIX
-                            )
-                        ]
-                    )
-
-                    if num_validation_errors >= VALIDATION_ERROR_THRESHOLD:
-                        logger.warning(
-                            f"Connector {ctx.connector_id} has {num_validation_errors} consecutive validation"
-                            f" errors. Marking the CC Pair as invalid."
-                        )
-                        update_connector_credential_pair(
-                            db_session=db_session_temp,
-                            connector_id=ctx.connector_id,
-                            credential_id=ctx.credential_id,
-                            status=ConnectorCredentialPairStatus.INVALID,
-                        )
             memory_tracer.stop()
             raise e
 

backend/onyx/chat/answer.py

@@ -30,7 +30,7 @@ from onyx.tools.tool import Tool
 from onyx.tools.tool_implementations.search.search_tool import QUERY_FIELD
 from onyx.tools.tool_implementations.search.search_tool import SearchTool
 from onyx.tools.utils import explicit_tool_calling_supported
-from onyx.utils.gpu_utils import fast_gpu_status_request
+from onyx.utils.gpu_utils import gpu_status_request
 from onyx.utils.logger import setup_logger
 
 logger = setup_logger()
@@ -88,9 +88,7 @@ class Answer:
             rerank_settings is not None
             and rerank_settings.rerank_provider_type is not None
         )
-        allow_agent_reranking = (
-            fast_gpu_status_request(indexing=False) or using_cloud_reranking
-        )
+        allow_agent_reranking = gpu_status_request() or using_cloud_reranking
 
         # TODO: this is a hack to force the query to be used for the search tool
         #       this should be removed once we fully unify graph inputs (i.e.

backend/onyx/chat/models.py

@@ -1,13 +1,10 @@
-from collections import OrderedDict
 from collections.abc import Callable
 from collections.abc import Iterator
-from collections.abc import Mapping
 from datetime import datetime
 from enum import Enum
 from typing import Any
 from typing import Literal
 from typing import TYPE_CHECKING
-from typing import Union
 
 from pydantic import BaseModel
 from pydantic import ConfigDict
@@ -47,44 +44,9 @@ class LlmDoc(BaseModel):
 
 
 class SubQuestionIdentifier(BaseModel):
-    """None represents references to objects in the original flow. To our understanding,
-    these will not be None in the packets returned from agent search.
-    """
-
     level: int | None = None
     level_question_num: int | None = None
 
-    @staticmethod
-    def make_dict_by_level(
-        original_dict: Mapping[tuple[int, int], "SubQuestionIdentifier"]
-    ) -> dict[int, list["SubQuestionIdentifier"]]:
-        """returns a dict of level to object list (sorted by level_question_num)
-        Ordering is asc for readability.
-        """
-
-        # organize by level, then sort ascending by question_index
-        level_dict: dict[int, list[SubQuestionIdentifier]] = {}
-
-        # group by level
-        for k, obj in original_dict.items():
-            level = k[0]
-            if level not in level_dict:
-                level_dict[level] = []
-            level_dict[level].append(obj)
-
-        # for each level, sort the group
-        for k2, value2 in level_dict.items():
-            # we need to handle the none case due to SubQuestionIdentifier typing
-            # level_question_num as int | None, even though it should never be None here.
-            level_dict[k2] = sorted(
-                value2,
-                key=lambda x: (x.level_question_num is None, x.level_question_num),
-            )
-
-        # sort by level
-        sorted_dict = OrderedDict(sorted(level_dict.items()))
-        return sorted_dict
-
 
 # First chunk of info for streaming QA
 class QADocsResponse(RetrievalDocs, SubQuestionIdentifier):
@@ -374,8 +336,6 @@ class AgentAnswerPiece(SubQuestionIdentifier):
 
 
 class SubQuestionPiece(SubQuestionIdentifier):
-    """Refined sub questions generated from the initial user question."""
-
     sub_question: str
 
 
@@ -387,13 +347,13 @@ class RefinedAnswerImprovement(BaseModel):
     refined_answer_improvement: bool
 
 
-AgentSearchPacket = Union[
+AgentSearchPacket = (
     SubQuestionPiece
     | AgentAnswerPiece
     | SubQueryPiece
     | ExtendedToolResponse
     | RefinedAnswerImprovement
-]
+)
 
 AnswerPacket = (
     AnswerQuestionPossibleReturn | AgentSearchPacket | ToolCallKickoff | ToolResponse

backend/onyx/configs/app_configs.py

@@ -8,9 +8,6 @@ from onyx.configs.constants import AuthType
 from onyx.configs.constants import DocumentIndexType
 from onyx.configs.constants import QueryHistoryType
 from onyx.file_processing.enums import HtmlBasedConnectorTransformLinksStrategy
-from onyx.prompts.image_analysis import DEFAULT_IMAGE_ANALYSIS_SYSTEM_PROMPT
-from onyx.prompts.image_analysis import DEFAULT_IMAGE_SUMMARIZATION_SYSTEM_PROMPT
-from onyx.prompts.image_analysis import DEFAULT_IMAGE_SUMMARIZATION_USER_PROMPT
 
 #####
 # App Configs
@@ -33,10 +30,6 @@ GENERATIVE_MODEL_ACCESS_CHECK_FREQ = int(
 )  # 1 day
 DISABLE_GENERATIVE_AI = os.environ.get("DISABLE_GENERATIVE_AI", "").lower() == "true"
 
-# Controls whether to allow admin query history reports with:
-# 1. associated user emails
-# 2. anonymized user emails
-# 3. no queries
 ONYX_QUERY_HISTORY_TYPE = QueryHistoryType(
     (os.environ.get("ONYX_QUERY_HISTORY_TYPE") or QueryHistoryType.NORMAL.value).lower()
 )
@@ -162,8 +155,6 @@ try:
 except ValueError:
     INDEX_BATCH_SIZE = 16
 
-MAX_DRIVE_WORKERS = int(os.environ.get("MAX_DRIVE_WORKERS", 4))
-
 # Below are intended to match the env variables names used by the official postgres docker image
 # https://hub.docker.com/_/postgres
 POSTGRES_USER = os.environ.get("POSTGRES_USER") or "postgres"
@@ -347,8 +338,8 @@ HTML_BASED_CONNECTOR_TRANSFORM_LINKS_STRATEGY = os.environ.get(
     HtmlBasedConnectorTransformLinksStrategy.STRIP,
 )
 
-NOTION_CONNECTOR_DISABLE_RECURSIVE_PAGE_LOOKUP = (
-    os.environ.get("NOTION_CONNECTOR_DISABLE_RECURSIVE_PAGE_LOOKUP", "").lower()
+NOTION_CONNECTOR_ENABLE_RECURSIVE_PAGE_LOOKUP = (
+    os.environ.get("NOTION_CONNECTOR_ENABLE_RECURSIVE_PAGE_LOOKUP", "").lower()
     == "true"
 )
 
@@ -420,9 +411,6 @@ EGNYTE_CLIENT_SECRET = os.getenv("EGNYTE_CLIENT_SECRET")
 LINEAR_CLIENT_ID = os.getenv("LINEAR_CLIENT_ID")
 LINEAR_CLIENT_SECRET = os.getenv("LINEAR_CLIENT_SECRET")
 
-# Slack specific configs
-SLACK_NUM_THREADS = int(os.getenv("SLACK_NUM_THREADS") or 2)
-
 DASK_JOB_CLIENT_ENABLED = (
     os.environ.get("DASK_JOB_CLIENT_ENABLED", "").lower() == "true"
 )
@@ -655,24 +643,3 @@ MOCK_LLM_RESPONSE = (
 
 
 DEFAULT_IMAGE_ANALYSIS_MAX_SIZE_MB = 20
-
-# Number of pre-provisioned tenants to maintain
-TARGET_AVAILABLE_TENANTS = int(os.environ.get("TARGET_AVAILABLE_TENANTS", "5"))
-
-
-# Image summarization configuration
-IMAGE_SUMMARIZATION_SYSTEM_PROMPT = os.environ.get(
-    "IMAGE_SUMMARIZATION_SYSTEM_PROMPT",
-    DEFAULT_IMAGE_SUMMARIZATION_SYSTEM_PROMPT,
-)
-
-# The user prompt for image summarization - the image filename will be automatically prepended
-IMAGE_SUMMARIZATION_USER_PROMPT = os.environ.get(
-    "IMAGE_SUMMARIZATION_USER_PROMPT",
-    DEFAULT_IMAGE_SUMMARIZATION_USER_PROMPT,
-)
-
-IMAGE_ANALYSIS_SYSTEM_PROMPT = os.environ.get(
-    "IMAGE_ANALYSIS_SYSTEM_PROMPT",
-    DEFAULT_IMAGE_ANALYSIS_SYSTEM_PROMPT,
-)

backend/onyx/configs/constants.py

@@ -3,10 +3,6 @@ import socket
 from enum import auto
 from enum import Enum
 
-ONYX_DEFAULT_APPLICATION_NAME = "Onyx"
-ONYX_SLACK_URL = "https://join.slack.com/t/onyx-dot-app/shared_invite/zt-2twesxdr6-5iQitKZQpgq~hYIZ~dv3KA"
-ONYX_EMAILABLE_LOGO_MAX_DIM = 512
-
 SOURCE_TYPE = "source_type"
 # stored in the `metadata` of a chunk. Used to signify that this chunk should
 # not be used for QA. For example, Google Drive file types which can't be parsed
@@ -44,7 +40,6 @@ DISABLED_GEN_AI_MSG = (
     "You can still use Onyx as a search engine."
 )
 
-
 DEFAULT_PERSONA_ID = 0
 
 DEFAULT_CC_PAIR_ID = 1
@@ -81,7 +76,6 @@ KV_REINDEX_KEY = "needs_reindexing"
 KV_SEARCH_SETTINGS = "search_settings"
 KV_UNSTRUCTURED_API_KEY = "unstructured_api_key"
 KV_USER_STORE_KEY = "INVITED_USERS"
-KV_PENDING_USERS_KEY = "PENDING_USERS"
 KV_NO_AUTH_USER_PREFERENCES_KEY = "no_auth_user_preferences"
 KV_CRED_KEY = "credential_id_{}"
 KV_GMAIL_CRED_KEY = "gmail_app_credential"
@@ -179,7 +173,6 @@ class DocumentSource(str, Enum):
     FIREFLIES = "fireflies"
     EGNYTE = "egnyte"
     AIRTABLE = "airtable"
-    HIGHSPOT = "highspot"
 
     # Special case just for integration tests
     MOCK_CONNECTOR = "mock_connector"
@@ -328,8 +321,6 @@ class OnyxRedisLocks:
         "da_lock:check_connector_external_group_sync_beat"
     )
     MONITOR_BACKGROUND_PROCESSES_LOCK = "da_lock:monitor_background_processes"
-    CHECK_AVAILABLE_TENANTS_LOCK = "da_lock:check_available_tenants"
-    PRE_PROVISION_TENANT_LOCK = "da_lock:pre_provision_tenant"
 
     CONNECTOR_DOC_PERMISSIONS_SYNC_LOCK_PREFIX = (
         "da_lock:connector_doc_permissions_sync"
@@ -392,10 +383,6 @@ class OnyxCeleryTask:
     CLOUD_MONITOR_CELERY_QUEUES = (
         f"{ONYX_CLOUD_CELERY_TASK_PREFIX}_monitor_celery_queues"
     )
-    CHECK_AVAILABLE_TENANTS = f"{ONYX_CLOUD_CELERY_TASK_PREFIX}_check_available_tenants"
-
-    # Tenant pre-provisioning
-    PRE_PROVISION_TENANT = f"{ONYX_CLOUD_CELERY_TASK_PREFIX}_pre_provision_tenant"
 
     CHECK_FOR_CONNECTOR_DELETION = "check_for_connector_deletion_task"
     CHECK_FOR_VESPA_SYNC_TASK = "check_for_vespa_sync_task"
@@ -411,7 +398,6 @@ class OnyxCeleryTask:
 
     MONITOR_BACKGROUND_PROCESSES = "monitor_background_processes"
     MONITOR_CELERY_QUEUES = "monitor_celery_queues"
-    MONITOR_PROCESS_MEMORY = "monitor_process_memory"
 
     KOMBU_MESSAGE_CLEANUP_TASK = "kombu_message_cleanup_task"
     CONNECTOR_PERMISSION_SYNC_GENERATOR_TASK = (

backend/onyx/configs/model_configs.py

@@ -132,10 +132,3 @@ if _LITELLM_EXTRA_BODY_RAW:
         LITELLM_EXTRA_BODY = json.loads(_LITELLM_EXTRA_BODY_RAW)
     except Exception:
         pass
-
-# Whether and how to lower scores for short chunks w/o relevant context
-# Evaluated via custom ML model
-
-USE_INFORMATION_CONTENT_CLASSIFICATION = (
-    os.environ.get("USE_INFORMATION_CONTENT_CLASSIFICATION", "false").lower() == "true"
-)

backend/onyx/connectors/airtable/airtable_connector.py

@@ -4,7 +4,6 @@ from concurrent.futures import Future
 from concurrent.futures import ThreadPoolExecutor
 from io import BytesIO
 from typing import Any
-from typing import cast
 
 import requests
 from pyairtable import Api as AirtableApi
@@ -17,8 +16,7 @@ from onyx.configs.constants import DocumentSource
 from onyx.connectors.interfaces import GenerateDocumentsOutput
 from onyx.connectors.interfaces import LoadConnector
 from onyx.connectors.models import Document
-from onyx.connectors.models import ImageSection
-from onyx.connectors.models import TextSection
+from onyx.connectors.models import Section
 from onyx.file_processing.extract_file_text import extract_file_text
 from onyx.file_processing.extract_file_text import get_file_ext
 from onyx.utils.logger import setup_logger
@@ -269,7 +267,7 @@ class AirtableConnector(LoadConnector):
         table_id: str,
         view_id: str | None,
         record_id: str,
-    ) -> tuple[list[TextSection], dict[str, str | list[str]]]:
+    ) -> tuple[list[Section], dict[str, str | list[str]]]:
         """
         Process a single Airtable field and return sections or metadata.
 
@@ -307,7 +305,7 @@ class AirtableConnector(LoadConnector):
 
         # Otherwise, create relevant sections
         sections = [
-            TextSection(
+            Section(
                 link=link,
                 text=(
                     f"{field_name}:\n"
@@ -342,7 +340,7 @@ class AirtableConnector(LoadConnector):
         table_name = table_schema.name
         record_id = record["id"]
         fields = record["fields"]
-        sections: list[TextSection] = []
+        sections: list[Section] = []
         metadata: dict[str, str | list[str]] = {}
 
         # Get primary field value if it exists
@@ -386,7 +384,7 @@ class AirtableConnector(LoadConnector):
 
         return Document(
             id=f"airtable__{record_id}",
-            sections=(cast(list[TextSection | ImageSection], sections)),
+            sections=sections,
             source=DocumentSource.AIRTABLE,
             semantic_identifier=semantic_id,
             metadata=metadata,

backend/onyx/connectors/asana/connector.py

@@ -10,7 +10,7 @@ from onyx.connectors.interfaces import LoadConnector
 from onyx.connectors.interfaces import PollConnector
 from onyx.connectors.interfaces import SecondsSinceUnixEpoch
 from onyx.connectors.models import Document
-from onyx.connectors.models import TextSection
+from onyx.connectors.models import Section
 from onyx.utils.logger import setup_logger
 
 logger = setup_logger()
@@ -82,7 +82,7 @@ class AsanaConnector(LoadConnector, PollConnector):
         logger.debug(f"Converting Asana task {task.id} to Document")
         return Document(
             id=task.id,
-            sections=[TextSection(link=task.link, text=task.text)],
+            sections=[Section(link=task.link, text=task.text)],
             doc_updated_at=task.last_modified,
             source=DocumentSource.ASANA,
             semantic_identifier=task.title,

backend/onyx/connectors/axero/connector.py

@@ -20,7 +20,7 @@ from onyx.connectors.interfaces import PollConnector
 from onyx.connectors.interfaces import SecondsSinceUnixEpoch
 from onyx.connectors.models import ConnectorMissingCredentialError
 from onyx.connectors.models import Document
-from onyx.connectors.models import TextSection
+from onyx.connectors.models import Section
 from onyx.file_processing.html_utils import parse_html_page_basic
 from onyx.utils.logger import setup_logger
 from onyx.utils.retry_wrapper import retry_builder
@@ -221,7 +221,7 @@ def _get_forums(
 def _translate_forum_to_doc(af: AxeroForum) -> Document:
     doc = Document(
         id=af.doc_id,
-        sections=[TextSection(link=af.link, text=reply) for reply in af.responses],
+        sections=[Section(link=af.link, text=reply) for reply in af.responses],
         source=DocumentSource.AXERO,
         semantic_identifier=af.title,
         doc_updated_at=af.last_update,
@@ -244,7 +244,7 @@ def _translate_content_to_doc(content: dict) -> Document:
 
     doc = Document(
         id="AXERO_" + str(content["ContentID"]),
-        sections=[TextSection(link=content["ContentURL"], text=page_text)],
+        sections=[Section(link=content["ContentURL"], text=page_text)],
         source=DocumentSource.AXERO,
         semantic_identifier=content["ContentTitle"],
         doc_updated_at=time_str_to_utc(content["DateUpdated"]),

backend/onyx/connectors/blob/connector.py

@@ -25,7 +25,7 @@ from onyx.connectors.interfaces import PollConnector
 from onyx.connectors.interfaces import SecondsSinceUnixEpoch
 from onyx.connectors.models import ConnectorMissingCredentialError
 from onyx.connectors.models import Document
-from onyx.connectors.models import TextSection
+from onyx.connectors.models import Section
 from onyx.file_processing.extract_file_text import extract_file_text
 from onyx.utils.logger import setup_logger
 
@@ -208,7 +208,7 @@ class BlobStorageConnector(LoadConnector, PollConnector):
                     batch.append(
                         Document(
                             id=f"{self.bucket_type}:{self.bucket_name}:{obj['Key']}",
-                            sections=[TextSection(link=link, text=text)],
+                            sections=[Section(link=link, text=text)],
                             source=DocumentSource(self.bucket_type.value),
                             semantic_identifier=name,
                             doc_updated_at=last_modified,
@@ -341,14 +341,7 @@ if __name__ == "__main__":
                 print("Sections:")
                 for section in doc.sections:
                     print(f"  - Link: {section.link}")
-                    if isinstance(section, TextSection) and section.text is not None:
-                        print(f"  - Text: {section.text[:100]}...")
-                    elif (
-                        hasattr(section, "image_file_name") and section.image_file_name
-                    ):
-                        print(f"  - Image: {section.image_file_name}")
-                    else:
-                        print("Error: Unknown section type")
+                    print(f"  - Text: {section.text[:100]}...")
                 print("---")
             break
 

backend/onyx/connectors/bookstack/connector.py

@@ -18,7 +18,7 @@ from onyx.connectors.interfaces import PollConnector
 from onyx.connectors.interfaces import SecondsSinceUnixEpoch
 from onyx.connectors.models import ConnectorMissingCredentialError
 from onyx.connectors.models import Document
-from onyx.connectors.models import TextSection
+from onyx.connectors.models import Section
 from onyx.file_processing.html_utils import parse_html_page_basic
 
 
@@ -81,7 +81,7 @@ class BookstackConnector(LoadConnector, PollConnector):
         )
         return Document(
             id="book__" + str(book.get("id")),
-            sections=[TextSection(link=url, text=text)],
+            sections=[Section(link=url, text=text)],
             source=DocumentSource.BOOKSTACK,
             semantic_identifier="Book: " + title,
             title=title,
@@ -110,7 +110,7 @@ class BookstackConnector(LoadConnector, PollConnector):
         )
         return Document(
             id="chapter__" + str(chapter.get("id")),
-            sections=[TextSection(link=url, text=text)],
+            sections=[Section(link=url, text=text)],
             source=DocumentSource.BOOKSTACK,
             semantic_identifier="Chapter: " + title,
             title=title,
@@ -134,7 +134,7 @@ class BookstackConnector(LoadConnector, PollConnector):
         )
         return Document(
             id="shelf:" + str(shelf.get("id")),
-            sections=[TextSection(link=url, text=text)],
+            sections=[Section(link=url, text=text)],
             source=DocumentSource.BOOKSTACK,
             semantic_identifier="Shelf: " + title,
             title=title,
@@ -167,7 +167,7 @@ class BookstackConnector(LoadConnector, PollConnector):
         time.sleep(0.1)
         return Document(
             id="page:" + page_id,
-            sections=[TextSection(link=url, text=text)],
+            sections=[Section(link=url, text=text)],
             source=DocumentSource.BOOKSTACK,
             semantic_identifier="Page: " + str(title),
             title=str(title),

backend/onyx/connectors/clickup/connector.py

@@ -17,7 +17,7 @@ from onyx.connectors.interfaces import SecondsSinceUnixEpoch
 from onyx.connectors.models import BasicExpertInfo
 from onyx.connectors.models import ConnectorMissingCredentialError
 from onyx.connectors.models import Document
-from onyx.connectors.models import TextSection
+from onyx.connectors.models import Section
 from onyx.utils.retry_wrapper import retry_builder
 
 
@@ -62,11 +62,11 @@ class ClickupConnector(LoadConnector, PollConnector):
 
         return response.json()
 
-    def _get_task_comments(self, task_id: str) -> list[TextSection]:
+    def _get_task_comments(self, task_id: str) -> list[Section]:
         url_endpoint = f"/task/{task_id}/comment"
         response = self._make_request(url_endpoint)
         comments = [
-            TextSection(
+            Section(
                 link=f'https://app.clickup.com/t/{task_id}?comment={comment_dict["id"]}',
                 text=comment_dict["comment_text"],
             )
@@ -133,7 +133,7 @@ class ClickupConnector(LoadConnector, PollConnector):
                     ],
                     title=task["name"],
                     sections=[
-                        TextSection(
+                        Section(
                             link=task["url"],
                             text=(
                                 task["markdown_description"]

backend/onyx/connectors/confluence/connector.py

@@ -33,9 +33,9 @@ from onyx.connectors.interfaces import SlimConnector
 from onyx.connectors.models import BasicExpertInfo
 from onyx.connectors.models import ConnectorMissingCredentialError
 from onyx.connectors.models import Document
-from onyx.connectors.models import ImageSection
+from onyx.connectors.models import Section
 from onyx.connectors.models import SlimDocument
-from onyx.connectors.models import TextSection
+from onyx.connectors.vision_enabled_connector import VisionEnabledConnector
 from onyx.indexing.indexing_heartbeat import IndexingHeartbeatInterface
 from onyx.utils.logger import setup_logger
 
@@ -85,6 +85,7 @@ class ConfluenceConnector(
     PollConnector,
     SlimConnector,
     CredentialsConnector,
+    VisionEnabledConnector,
 ):
     def __init__(
         self,
@@ -114,7 +115,9 @@ class ConfluenceConnector(
         self.timezone_offset = timezone_offset
         self._confluence_client: OnyxConfluence | None = None
         self._fetched_titles: set[str] = set()
-        self.allow_images = False
+
+        # Initialize vision LLM using the mixin
+        self.initialize_vision_llm()
 
         # Remove trailing slash from wiki_base if present
         self.wiki_base = wiki_base.rstrip("/")
@@ -159,9 +162,6 @@ class ConfluenceConnector(
             "max_backoff_seconds": 60,
         }
 
-    def set_allow_images(self, value: bool) -> None:
-        self.allow_images = value
-
     @property
     def confluence_client(self) -> OnyxConfluence:
         if self._confluence_client is None:
@@ -237,9 +237,7 @@ class ConfluenceConnector(
             # Extract basic page information
             page_id = page["id"]
             page_title = page["title"]
-            page_url = build_confluence_document_id(
-                self.wiki_base, page["_links"]["webui"], self.is_cloud
-            )
+            page_url = f"{self.wiki_base}{page['_links']['webui']}"
 
             # Get the page content
             page_content = extract_text_from_confluence_html(
@@ -247,16 +245,12 @@ class ConfluenceConnector(
             )
 
             # Create the main section for the page content
-            sections: list[TextSection | ImageSection] = [
-                TextSection(text=page_content, link=page_url)
-            ]
+            sections = [Section(text=page_content, link=page_url)]
 
             # Process comments if available
             comment_text = self._get_comment_string_for_page_id(page_id)
             if comment_text:
-                sections.append(
-                    TextSection(text=comment_text, link=f"{page_url}#comments")
-                )
+                sections.append(Section(text=comment_text, link=f"{page_url}#comments"))
 
             # Process attachments
             if "children" in page and "attachment" in page["children"]:
@@ -269,28 +263,21 @@ class ConfluenceConnector(
                     result = process_attachment(
                         self.confluence_client,
                         attachment,
-                        page_id,
-                        self.allow_images,
+                        page_title,
+                        self.image_analysis_llm,
                     )
 
-                    if result and result.text:
+                    if result.text:
                         # Create a section for the attachment text
-                        attachment_section = TextSection(
+                        attachment_section = Section(
                             text=result.text,
                             link=f"{page_url}#attachment-{attachment['id']}",
-                        )
-                        sections.append(attachment_section)
-                    elif result and result.file_name:
-                        # Create an ImageSection for image attachments
-                        image_section = ImageSection(
-                            link=f"{page_url}#attachment-{attachment['id']}",
                             image_file_name=result.file_name,
                         )
-                        sections.append(image_section)
-                    else:
+                        sections.append(attachment_section)
+                    elif result.error:
                         logger.warning(
-                            f"Error processing attachment '{attachment.get('title')}':",
-                            f"{result.error if result else 'Unknown error'}",
+                            f"Error processing attachment '{attachment.get('title')}': {result.error}"
                         )
 
             # Extract metadata
@@ -311,14 +298,13 @@ class ConfluenceConnector(
             if "version" in page and "by" in page["version"]:
                 author = page["version"]["by"]
                 display_name = author.get("displayName", "Unknown")
-                email = author.get("email", "unknown@domain.invalid")
-                primary_owners.append(
-                    BasicExpertInfo(display_name=display_name, email=email)
-                )
+                primary_owners.append(BasicExpertInfo(display_name=display_name))
 
             # Create the document
             return Document(
-                id=page_url,
+                id=build_confluence_document_id(
+                    self.wiki_base, page["_links"]["webui"], self.is_cloud
+                ),
                 sections=sections,
                 source=DocumentSource.CONFLUENCE,
                 semantic_identifier=page_title,
@@ -362,7 +348,7 @@ class ConfluenceConnector(
             # Now get attachments for that page:
             attachment_query = self._construct_attachment_query(page["id"])
             # We'll use the page's XML to provide context if we summarize an image
-            page.get("body", {}).get("storage", {}).get("value", "")
+            confluence_xml = page.get("body", {}).get("storage", {}).get("value", "")
 
             for attachment in self.confluence_client.paginated_cql_retrieval(
                 cql=attachment_query,
@@ -370,7 +356,7 @@ class ConfluenceConnector(
             ):
                 attachment["metadata"].get("mediaType", "")
                 if not validate_attachment_filetype(
-                    attachment,
+                    attachment, self.image_analysis_llm
                 ):
                     continue
 
@@ -380,27 +366,23 @@ class ConfluenceConnector(
                     response = convert_attachment_to_content(
                         confluence_client=self.confluence_client,
                         attachment=attachment,
-                        page_id=page["id"],
-                        allow_images=self.allow_images,
+                        page_context=confluence_xml,
+                        llm=self.image_analysis_llm,
                     )
                     if response is None:
                         continue
 
                     content_text, file_storage_name = response
+
                     object_url = build_confluence_document_id(
                         self.wiki_base, attachment["_links"]["webui"], self.is_cloud
                     )
+
                     if content_text:
                         doc.sections.append(
-                            TextSection(
+                            Section(
                                 text=content_text,
                                 link=object_url,
-                            )
-                        )
-                    elif file_storage_name:
-                        doc.sections.append(
-                            ImageSection(
-                                link=object_url,
                                 image_file_name=file_storage_name,
                             )
                         )
@@ -480,7 +462,7 @@ class ConfluenceConnector(
                 # If you skip images, you'll skip them in the permission sync
                 attachment["metadata"].get("mediaType", "")
                 if not validate_attachment_filetype(
-                    attachment,
+                    attachment, self.image_analysis_llm
                 ):
                     continue
 

backend/onyx/connectors/confluence/onyx_confluence.py

@@ -1,3 +1,4 @@
+import io
 import json
 import time
 from collections.abc import Callable
@@ -18,11 +19,17 @@ from requests import HTTPError
 
 from ee.onyx.configs.app_configs import OAUTH_CONFLUENCE_CLOUD_CLIENT_ID
 from ee.onyx.configs.app_configs import OAUTH_CONFLUENCE_CLOUD_CLIENT_SECRET
+from onyx.configs.app_configs import (
+    CONFLUENCE_CONNECTOR_ATTACHMENT_CHAR_COUNT_THRESHOLD,
+)
+from onyx.configs.app_configs import CONFLUENCE_CONNECTOR_ATTACHMENT_SIZE_THRESHOLD
 from onyx.connectors.confluence.utils import _handle_http_error
 from onyx.connectors.confluence.utils import confluence_refresh_tokens
 from onyx.connectors.confluence.utils import get_start_param_from_url
 from onyx.connectors.confluence.utils import update_param_in_path
+from onyx.connectors.confluence.utils import validate_attachment_filetype
 from onyx.connectors.interfaces import CredentialsProviderInterface
+from onyx.file_processing.extract_file_text import extract_file_text
 from onyx.file_processing.html_utils import format_document_soup
 from onyx.redis.redis_pool import get_redis_client
 from onyx.utils.logger import setup_logger
@@ -801,6 +808,65 @@ def _get_user(confluence_client: OnyxConfluence, user_id: str) -> str:
     return _USER_ID_TO_DISPLAY_NAME_CACHE.get(user_id) or _USER_NOT_FOUND
 
 
+def attachment_to_content(
+    confluence_client: OnyxConfluence,
+    attachment: dict[str, Any],
+    parent_content_id: str | None = None,
+) -> str | None:
+    """If it returns None, assume that we should skip this attachment."""
+    if not validate_attachment_filetype(attachment):
+        return None
+
+    if "api.atlassian.com" in confluence_client.url:
+        # https://developer.atlassian.com/cloud/confluence/rest/v1/api-group-content---attachments/#api-wiki-rest-api-content-id-child-attachment-attachmentid-download-get
+        if not parent_content_id:
+            logger.warning(
+                "parent_content_id is required to download attachments from Confluence Cloud!"
+            )
+            return None
+
+        download_link = (
+            confluence_client.url
+            + f"/rest/api/content/{parent_content_id}/child/attachment/{attachment['id']}/download"
+        )
+    else:
+        download_link = confluence_client.url + attachment["_links"]["download"]
+
+    attachment_size = attachment["extensions"]["fileSize"]
+    if attachment_size > CONFLUENCE_CONNECTOR_ATTACHMENT_SIZE_THRESHOLD:
+        logger.warning(
+            f"Skipping {download_link} due to size. "
+            f"size={attachment_size} "
+            f"threshold={CONFLUENCE_CONNECTOR_ATTACHMENT_SIZE_THRESHOLD}"
+        )
+        return None
+
+    logger.info(f"_attachment_to_content - _session.get: link={download_link}")
+
+    # why are we using session.get here? we probably won't retry these ... is that ok?
+    response = confluence_client._session.get(download_link)
+    if response.status_code != 200:
+        logger.warning(
+            f"Failed to fetch {download_link} with invalid status code {response.status_code}"
+        )
+        return None
+
+    extracted_text = extract_file_text(
+        io.BytesIO(response.content),
+        file_name=attachment["title"],
+        break_on_unprocessable=False,
+    )
+    if len(extracted_text) > CONFLUENCE_CONNECTOR_ATTACHMENT_CHAR_COUNT_THRESHOLD:
+        logger.warning(
+            f"Skipping {download_link} due to char count. "
+            f"char count={len(extracted_text)} "
+            f"threshold={CONFLUENCE_CONNECTOR_ATTACHMENT_CHAR_COUNT_THRESHOLD}"
+        )
+        return None
+
+    return extracted_text
+
+
 def extract_text_from_confluence_html(
     confluence_client: OnyxConfluence,
     confluence_object: dict[str, Any],

backend/onyx/connectors/confluence/utils.py

@@ -22,7 +22,6 @@ from sqlalchemy.orm import Session
 from onyx.configs.app_configs import (
     CONFLUENCE_CONNECTOR_ATTACHMENT_CHAR_COUNT_THRESHOLD,
 )
-from onyx.configs.app_configs import CONFLUENCE_CONNECTOR_ATTACHMENT_SIZE_THRESHOLD
 from onyx.configs.constants import FileOrigin
 
 if TYPE_CHECKING:
@@ -36,6 +35,7 @@ from onyx.db.pg_file_store import upsert_pgfilestore
 from onyx.file_processing.extract_file_text import extract_file_text
 from onyx.file_processing.file_validation import is_valid_image_type
 from onyx.file_processing.image_utils import store_image_and_create_section
+from onyx.llm.interfaces import LLM
 from onyx.utils.logger import setup_logger
 
 logger = setup_logger()
@@ -53,16 +53,17 @@ class TokenResponse(BaseModel):
 
 
 def validate_attachment_filetype(
-    attachment: dict[str, Any],
+    attachment: dict[str, Any], llm: LLM | None = None
 ) -> bool:
     """
     Validates if the attachment is a supported file type.
+    If LLM is provided, also checks if it's an image that can be processed.
     """
     attachment.get("metadata", {})
     media_type = attachment.get("metadata", {}).get("mediaType", "")
 
     if media_type.startswith("image/"):
-        return is_valid_image_type(media_type)
+        return llm is not None and is_valid_image_type(media_type)
 
     # For non-image files, check if we support the extension
     title = attachment.get("title", "")
@@ -83,111 +84,55 @@ class AttachmentProcessingResult(BaseModel):
     error: str | None = None
 
 
-def _make_attachment_link(
-    confluence_client: "OnyxConfluence",
-    attachment: dict[str, Any],
-    parent_content_id: str | None = None,
-) -> str | None:
-    download_link = ""
-
-    if "api.atlassian.com" in confluence_client.url:
-        # https://developer.atlassian.com/cloud/confluence/rest/v1/api-group-content---attachments/#api-wiki-rest-api-content-id-child-attachment-attachmentid-download-get
-        if not parent_content_id:
-            logger.warning(
-                "parent_content_id is required to download attachments from Confluence Cloud!"
-            )
-            return None
-
-        download_link = (
-            confluence_client.url
-            + f"/rest/api/content/{parent_content_id}/child/attachment/{attachment['id']}/download"
+def _download_attachment(
+    confluence_client: "OnyxConfluence", attachment: dict[str, Any]
+) -> bytes | None:
+    """
+    Retrieves the raw bytes of an attachment from Confluence. Returns None on error.
+    """
+    download_link = confluence_client.url + attachment["_links"]["download"]
+    resp = confluence_client._session.get(download_link)
+    if resp.status_code != 200:
+        logger.warning(
+            f"Failed to fetch {download_link} with status code {resp.status_code}"
         )
-    else:
-        download_link = confluence_client.url + attachment["_links"]["download"]
-
-    return download_link
+        return None
+    return resp.content
 
 
 def process_attachment(
     confluence_client: "OnyxConfluence",
     attachment: dict[str, Any],
-    parent_content_id: str | None,
-    allow_images: bool,
+    page_context: str,
+    llm: LLM | None,
 ) -> AttachmentProcessingResult:
     """
     Processes a Confluence attachment. If it's a document, extracts text,
-    or if it's an image, stores it for later analysis. Returns a structured result.
+    or if it's an image and an LLM is available, summarizes it. Returns a structured result.
     """
     try:
         # Get the media type from the attachment metadata
-        media_type: str = attachment.get("metadata", {}).get("mediaType", "")
+        media_type = attachment.get("metadata", {}).get("mediaType", "")
+
         # Validate the attachment type
-        if not validate_attachment_filetype(attachment):
+        if not validate_attachment_filetype(attachment, llm):
             return AttachmentProcessingResult(
                 text=None,
                 file_name=None,
                 error=f"Unsupported file type: {media_type}",
             )
 
-        attachment_link = _make_attachment_link(
-            confluence_client, attachment, parent_content_id
-        )
-        if not attachment_link:
-            return AttachmentProcessingResult(
-                text=None, file_name=None, error="Failed to make attachment link"
-            )
-
-        attachment_size = attachment["extensions"]["fileSize"]
-
-        if media_type.startswith("image/"):
-            if not allow_images:
-                return AttachmentProcessingResult(
-                    text=None,
-                    file_name=None,
-                    error="Image downloading is not enabled",
-                )
-        else:
-            if attachment_size > CONFLUENCE_CONNECTOR_ATTACHMENT_SIZE_THRESHOLD:
-                logger.warning(
-                    f"Skipping {attachment_link} due to size. "
-                    f"size={attachment_size} "
-                    f"threshold={CONFLUENCE_CONNECTOR_ATTACHMENT_SIZE_THRESHOLD}"
-                )
-                return AttachmentProcessingResult(
-                    text=None,
-                    file_name=None,
-                    error=f"Attachment text too long: {attachment_size} chars",
-                )
-
-        logger.info(
-            f"Downloading attachment: "
-            f"title={attachment['title']} "
-            f"length={attachment_size} "
-            f"link={attachment_link}"
-        )
-
         # Download the attachment
-        resp: requests.Response = confluence_client._session.get(attachment_link)
-        if resp.status_code != 200:
-            logger.warning(
-                f"Failed to fetch {attachment_link} with status code {resp.status_code}"
-            )
+        raw_bytes = _download_attachment(confluence_client, attachment)
+        if raw_bytes is None:
             return AttachmentProcessingResult(
-                text=None,
-                file_name=None,
-                error=f"Attachment download status code is {resp.status_code}",
+                text=None, file_name=None, error="Failed to download attachment"
             )
 
-        raw_bytes = resp.content
-        if not raw_bytes:
-            return AttachmentProcessingResult(
-                text=None, file_name=None, error="attachment.content is None"
-            )
-
-        # Process image attachments
-        if media_type.startswith("image/"):
+        # Process image attachments with LLM if available
+        if media_type.startswith("image/") and llm:
             return _process_image_attachment(
-                confluence_client, attachment, raw_bytes, media_type
+                confluence_client, attachment, page_context, llm, raw_bytes, media_type
             )
 
         # Process document attachments
@@ -220,10 +165,12 @@ def process_attachment(
 def _process_image_attachment(
     confluence_client: "OnyxConfluence",
     attachment: dict[str, Any],
+    page_context: str,
+    llm: LLM,
     raw_bytes: bytes,
     media_type: str,
 ) -> AttachmentProcessingResult:
-    """Process an image attachment by saving it without generating a summary."""
+    """Process an image attachment by saving it and generating a summary."""
     try:
         # Use the standardized image storage and section creation
         with get_session_with_current_tenant() as db_session:
@@ -233,14 +180,15 @@ def _process_image_attachment(
                 file_name=Path(attachment["id"]).name,
                 display_name=attachment["title"],
                 media_type=media_type,
+                llm=llm,
                 file_origin=FileOrigin.CONNECTOR,
             )
-            logger.info(f"Stored image attachment with file name: {file_name}")
 
-            # Return empty text but include the file_name for later processing
-            return AttachmentProcessingResult(text="", file_name=file_name, error=None)
+            return AttachmentProcessingResult(
+                text=section.text, file_name=file_name, error=None
+            )
     except Exception as e:
-        msg = f"Image storage failed for {attachment['title']}: {e}"
+        msg = f"Image summarization failed for {attachment['title']}: {e}"
         logger.error(msg, exc_info=e)
         return AttachmentProcessingResult(text=None, file_name=None, error=msg)
 
@@ -301,16 +249,16 @@ def _process_text_attachment(
 def convert_attachment_to_content(
     confluence_client: "OnyxConfluence",
     attachment: dict[str, Any],
-    page_id: str,
-    allow_images: bool,
+    page_context: str,
+    llm: LLM | None,
 ) -> tuple[str | None, str | None] | None:
     """
     Facade function which:
       1. Validates attachment type
-      2. Extracts content or stores image for later processing
+      2. Extracts or summarizes content
       3. Returns (content_text, stored_file_name) or None if we should skip it
     """
-    media_type = attachment.get("metadata", {}).get("mediaType", "")
+    media_type = attachment["metadata"]["mediaType"]
     # Quick check for unsupported types:
     if media_type.startswith("video/") or media_type == "application/gliffy+json":
         logger.warning(
@@ -318,7 +266,7 @@ def convert_attachment_to_content(
         )
         return None
 
-    result = process_attachment(confluence_client, attachment, page_id, allow_images)
+    result = process_attachment(confluence_client, attachment, page_context, llm)
     if result.error is not None:
         logger.warning(
             f"Attachment {attachment['title']} encountered error: {result.error}"
@@ -531,10 +479,6 @@ def attachment_to_file_record(
         download_link, absolute=True, not_json_response=True
     )
 
-    file_type = attachment.get("metadata", {}).get(
-        "mediaType", "application/octet-stream"
-    )
-
     # Save image to file store
     file_name = f"confluence_attachment_{attachment['id']}"
     lobj_oid = create_populate_lobj(BytesIO(image_data), db_session)
@@ -542,7 +486,7 @@ def attachment_to_file_record(
         file_name=file_name,
         display_name=attachment["title"],
         file_origin=FileOrigin.OTHER,
-        file_type=file_type,
+        file_type=attachment["metadata"]["mediaType"],
         lobj_oid=lobj_oid,
         db_session=db_session,
         commit=True,

backend/onyx/connectors/connector_runner.py

@@ -2,8 +2,6 @@ import sys
 import time
 from collections.abc import Generator
 from datetime import datetime
-from typing import Generic
-from typing import TypeVar
 
 from onyx.connectors.interfaces import BaseConnector
 from onyx.connectors.interfaces import CheckpointConnector
@@ -21,10 +19,8 @@ logger = setup_logger()
 
 TimeRange = tuple[datetime, datetime]
 
-CT = TypeVar("CT", bound=ConnectorCheckpoint)
 
-
-class CheckpointOutputWrapper(Generic[CT]):
+class CheckpointOutputWrapper:
     """
     Wraps a CheckpointOutput generator to give things back in a more digestible format.
     The connector format is easier for the connector implementor (e.g. it enforces exactly
@@ -33,20 +29,20 @@ class CheckpointOutputWrapper(Generic[CT]):
     """
 
     def __init__(self) -> None:
-        self.next_checkpoint: CT | None = None
+        self.next_checkpoint: ConnectorCheckpoint | None = None
 
     def __call__(
         self,
-        checkpoint_connector_generator: CheckpointOutput[CT],
+        checkpoint_connector_generator: CheckpointOutput,
     ) -> Generator[
-        tuple[Document | None, ConnectorFailure | None, CT | None],
+        tuple[Document | None, ConnectorFailure | None, ConnectorCheckpoint | None],
         None,
         None,
     ]:
         # grabs the final return value and stores it in the `next_checkpoint` variable
         def _inner_wrapper(
-            checkpoint_connector_generator: CheckpointOutput[CT],
-        ) -> CheckpointOutput[CT]:
+            checkpoint_connector_generator: CheckpointOutput,
+        ) -> CheckpointOutput:
             self.next_checkpoint = yield from checkpoint_connector_generator
             return self.next_checkpoint  # not used
 
@@ -68,7 +64,7 @@ class CheckpointOutputWrapper(Generic[CT]):
         yield None, None, self.next_checkpoint
 
 
-class ConnectorRunner(Generic[CT]):
+class ConnectorRunner:
     """
     Handles:
         - Batching
@@ -89,9 +85,11 @@ class ConnectorRunner(Generic[CT]):
         self.doc_batch: list[Document] = []
 
     def run(
-        self, checkpoint: CT
+        self, checkpoint: ConnectorCheckpoint
     ) -> Generator[
-        tuple[list[Document] | None, ConnectorFailure | None, CT | None],
+        tuple[
+            list[Document] | None, ConnectorFailure | None, ConnectorCheckpoint | None
+        ],
         None,
         None,
     ]:
@@ -107,9 +105,9 @@ class ConnectorRunner(Generic[CT]):
                     end=self.time_range[1].timestamp(),
                     checkpoint=checkpoint,
                 )
-                next_checkpoint: CT | None = None
+                next_checkpoint: ConnectorCheckpoint | None = None
                 # this is guaranteed to always run at least once with next_checkpoint being non-None
-                for document, failure, next_checkpoint in CheckpointOutputWrapper[CT]()(
+                for document, failure, next_checkpoint in CheckpointOutputWrapper()(
                     checkpoint_connector_generator
                 ):
                     if document is not None:
@@ -134,7 +132,7 @@ class ConnectorRunner(Generic[CT]):
                 )
 
             else:
-                finished_checkpoint = self.connector.build_dummy_checkpoint()
+                finished_checkpoint = ConnectorCheckpoint.build_dummy_checkpoint()
                 finished_checkpoint.has_more = False
 
                 if isinstance(self.connector, PollConnector):

backend/onyx/connectors/discord/connector.py

@@ -4,7 +4,6 @@ from collections.abc import Iterable
 from datetime import datetime
 from datetime import timezone
 from typing import Any
-from typing import cast
 
 from discord import Client
 from discord.channel import TextChannel
@@ -21,8 +20,7 @@ from onyx.connectors.interfaces import PollConnector
 from onyx.connectors.interfaces import SecondsSinceUnixEpoch
 from onyx.connectors.models import ConnectorMissingCredentialError
 from onyx.connectors.models import Document
-from onyx.connectors.models import ImageSection
-from onyx.connectors.models import TextSection
+from onyx.connectors.models import Section
 from onyx.utils.logger import setup_logger
 
 logger = setup_logger()
@@ -34,7 +32,7 @@ _SNIPPET_LENGTH = 30
 
 def _convert_message_to_document(
     message: DiscordMessage,
-    sections: list[TextSection],
+    sections: list[Section],
 ) -> Document:
     """
     Convert a discord message to a document
@@ -80,7 +78,7 @@ def _convert_message_to_document(
         semantic_identifier=semantic_identifier,
         doc_updated_at=message.edited_at,
         title=title,
-        sections=(cast(list[TextSection | ImageSection], sections)),
+        sections=sections,
         metadata=metadata,
     )
 
@@ -125,8 +123,8 @@ async def _fetch_documents_from_channel(
         if channel_message.type != MessageType.default:
             continue
 
-        sections: list[TextSection] = [
-            TextSection(
+        sections: list[Section] = [
+            Section(
                 text=channel_message.content,
                 link=channel_message.jump_url,
             )
@@ -144,7 +142,7 @@ async def _fetch_documents_from_channel(
                 continue
 
             sections = [
-                TextSection(
+                Section(
                     text=thread_message.content,
                     link=thread_message.jump_url,
                 )
@@ -162,7 +160,7 @@ async def _fetch_documents_from_channel(
                 continue
 
             sections = [
-                TextSection(
+                Section(
                     text=thread_message.content,
                     link=thread_message.jump_url,
                 )

backend/onyx/connectors/discourse/connector.py

@@ -3,7 +3,6 @@ import urllib.parse
 from datetime import datetime
 from datetime import timezone
 from typing import Any
-from typing import cast
 
 import requests
 from pydantic import BaseModel
@@ -21,8 +20,7 @@ from onyx.connectors.interfaces import SecondsSinceUnixEpoch
 from onyx.connectors.models import BasicExpertInfo
 from onyx.connectors.models import ConnectorMissingCredentialError
 from onyx.connectors.models import Document
-from onyx.connectors.models import ImageSection
-from onyx.connectors.models import TextSection
+from onyx.connectors.models import Section
 from onyx.file_processing.html_utils import parse_html_page_basic
 from onyx.utils.logger import setup_logger
 from onyx.utils.retry_wrapper import retry_builder
@@ -114,7 +112,7 @@ class DiscourseConnector(PollConnector):
                     responders.append(BasicExpertInfo(display_name=responder_name))
 
             sections.append(
-                TextSection(link=topic_url, text=parse_html_page_basic(post["cooked"]))
+                Section(link=topic_url, text=parse_html_page_basic(post["cooked"]))
             )
         category_name = self.category_id_map.get(topic["category_id"])
 
@@ -131,7 +129,7 @@ class DiscourseConnector(PollConnector):
 
         doc = Document(
             id="_".join([DocumentSource.DISCOURSE.value, str(topic["id"])]),
-            sections=cast(list[TextSection | ImageSection], sections),
+            sections=sections,
             source=DocumentSource.DISCOURSE,
             semantic_identifier=topic["title"],
             doc_updated_at=time_str_to_utc(topic["last_posted_at"]),

backend/onyx/connectors/document360/connector.py

@@ -19,7 +19,7 @@ from onyx.connectors.interfaces import SecondsSinceUnixEpoch
 from onyx.connectors.models import BasicExpertInfo
 from onyx.connectors.models import ConnectorMissingCredentialError
 from onyx.connectors.models import Document
-from onyx.connectors.models import TextSection
+from onyx.connectors.models import Section
 from onyx.file_processing.html_utils import parse_html_page_basic
 from onyx.utils.retry_wrapper import retry_builder
 
@@ -158,7 +158,7 @@ class Document360Connector(LoadConnector, PollConnector):
 
             document = Document(
                 id=article_details["id"],
-                sections=[TextSection(link=doc_link, text=doc_text)],
+                sections=[Section(link=doc_link, text=doc_text)],
                 source=DocumentSource.DOCUMENT360,
                 semantic_identifier=article_details["title"],
                 doc_updated_at=updated_at,

backend/onyx/connectors/dropbox/connector.py

@@ -19,7 +19,7 @@ from onyx.connectors.interfaces import PollConnector
 from onyx.connectors.interfaces import SecondsSinceUnixEpoch
 from onyx.connectors.models import ConnectorMissingCredentialError
 from onyx.connectors.models import Document
-from onyx.connectors.models import TextSection
+from onyx.connectors.models import Section
 from onyx.file_processing.extract_file_text import extract_file_text
 from onyx.utils.logger import setup_logger
 
@@ -108,7 +108,7 @@ class DropboxConnector(LoadConnector, PollConnector):
                         batch.append(
                             Document(
                                 id=f"doc:{entry.id}",
-                                sections=[TextSection(link=link, text=text)],
+                                sections=[Section(link=link, text=text)],
                                 source=DocumentSource.DROPBOX,
                                 semantic_identifier=entry.name,
                                 doc_updated_at=modified_time,

backend/onyx/connectors/egnyte/connector.py

@@ -24,7 +24,7 @@ from onyx.connectors.interfaces import SecondsSinceUnixEpoch
 from onyx.connectors.models import BasicExpertInfo
 from onyx.connectors.models import ConnectorMissingCredentialError
 from onyx.connectors.models import Document
-from onyx.connectors.models import TextSection
+from onyx.connectors.models import Section
 from onyx.file_processing.extract_file_text import detect_encoding
 from onyx.file_processing.extract_file_text import extract_file_text
 from onyx.file_processing.extract_file_text import get_file_ext
@@ -111,7 +111,7 @@ def _process_egnyte_file(
     # Create the document
     return Document(
         id=f"egnyte-{file_metadata['entry_id']}",
-        sections=[TextSection(text=file_content_raw.strip(), link=web_url)],
+        sections=[Section(text=file_content_raw.strip(), link=web_url)],
         source=DocumentSource.EGNYTE,
         semantic_identifier=file_name,
         metadata=metadata,

backend/onyx/connectors/factory.py

@@ -5,7 +5,6 @@ from sqlalchemy.orm import Session
 
 from onyx.configs.app_configs import INTEGRATION_TESTS_MODE
 from onyx.configs.constants import DocumentSource
-from onyx.configs.llm_configs import get_image_extraction_and_analysis_enabled
 from onyx.connectors.airtable.airtable_connector import AirtableConnector
 from onyx.connectors.asana.connector import AsanaConnector
 from onyx.connectors.axero.connector import AxeroConnector
@@ -31,7 +30,6 @@ from onyx.connectors.gong.connector import GongConnector
 from onyx.connectors.google_drive.connector import GoogleDriveConnector
 from onyx.connectors.google_site.connector import GoogleSitesConnector
 from onyx.connectors.guru.connector import GuruConnector
-from onyx.connectors.highspot.connector import HighspotConnector
 from onyx.connectors.hubspot.connector import HubSpotConnector
 from onyx.connectors.interfaces import BaseConnector
 from onyx.connectors.interfaces import CheckpointConnector
@@ -119,7 +117,6 @@ def identify_connector_class(
         DocumentSource.FIREFLIES: FirefliesConnector,
         DocumentSource.EGNYTE: EgnyteConnector,
         DocumentSource.AIRTABLE: AirtableConnector,
-        DocumentSource.HIGHSPOT: HighspotConnector,
         # just for integration tests
         DocumentSource.MOCK_CONNECTOR: MockConnector,
     }
@@ -185,8 +182,6 @@ def instantiate_connector(
         if new_credentials is not None:
             backend_update_credential_json(credential, new_credentials, db_session)
 
-    connector.set_allow_images(get_image_extraction_and_analysis_enabled())
-
     return connector
 
 

backend/onyx/connectors/file/connector.py

@@ -16,8 +16,8 @@ from onyx.connectors.interfaces import GenerateDocumentsOutput
 from onyx.connectors.interfaces import LoadConnector
 from onyx.connectors.models import BasicExpertInfo
 from onyx.connectors.models import Document
-from onyx.connectors.models import ImageSection
-from onyx.connectors.models import TextSection
+from onyx.connectors.models import Section
+from onyx.connectors.vision_enabled_connector import VisionEnabledConnector
 from onyx.db.engine import get_session_with_current_tenant
 from onyx.db.pg_file_store import get_pgfilestore_by_file_name
 from onyx.file_processing.extract_file_text import extract_text_and_images
@@ -26,6 +26,7 @@ from onyx.file_processing.extract_file_text import is_valid_file_ext
 from onyx.file_processing.extract_file_text import load_files_from_zip
 from onyx.file_processing.image_utils import store_image_and_create_section
 from onyx.file_store.file_store import get_default_file_store
+from onyx.llm.interfaces import LLM
 from onyx.utils.logger import setup_logger
 
 logger = setup_logger()
@@ -58,44 +59,32 @@ def _read_files_and_metadata(
 
 
 def _create_image_section(
+    llm: LLM | None,
     image_data: bytes,
     db_session: Session,
     parent_file_name: str,
     display_name: str,
-    link: str | None = None,
     idx: int = 0,
-) -> tuple[ImageSection, str | None]:
+) -> tuple[Section, str | None]:
     """
-    Creates an ImageSection for an image file or embedded image.
-    Stores the image in PGFileStore but does not generate a summary.
-
-    Args:
-        image_data: Raw image bytes
-        db_session: Database session
-        parent_file_name: Name of the parent file (for embedded images)
-        display_name: Display name for the image
-        idx: Index for embedded images
+    Create a Section object for a single image and store the image in PGFileStore.
+    If summarization is enabled and we have an LLM, summarize the image.
 
     Returns:
-        Tuple of (ImageSection, stored_file_name or None)
+        tuple: (Section object, file_name in PGFileStore or None if storage failed)
     """
-    # Create a unique identifier for the image
-    file_name = f"{parent_file_name}_embedded_{idx}" if idx > 0 else parent_file_name
+    # Create a unique file name for the embedded image
+    file_name = f"{parent_file_name}_embedded_{idx}"
 
-    # Store the image and create a section
-    try:
-        section, stored_file_name = store_image_and_create_section(
-            db_session=db_session,
-            image_data=image_data,
-            file_name=file_name,
-            display_name=display_name,
-            link=link,
-            file_origin=FileOrigin.CONNECTOR,
-        )
-        return section, stored_file_name
-    except Exception as e:
-        logger.error(f"Failed to store image {display_name}: {e}")
-        raise e
+    # Use the standardized utility to store the image and create a section
+    return store_image_and_create_section(
+        db_session=db_session,
+        image_data=image_data,
+        file_name=file_name,
+        display_name=display_name,
+        llm=llm,
+        file_origin=FileOrigin.OTHER,
+    )
 
 
 def _process_file(
@@ -104,16 +93,12 @@ def _process_file(
     metadata: dict[str, Any] | None,
     pdf_pass: str | None,
     db_session: Session,
+    llm: LLM | None,
 ) -> list[Document]:
     """
-    Process a file and return a list of Documents.
-    For images, creates ImageSection objects without summarization.
-    For documents with embedded images, extracts and stores the images.
+    Processes a single file, returning a list of Documents (typically one).
+    Also handles embedded images if 'EMBEDDED_IMAGE_EXTRACTION_ENABLED' is true.
     """
-    if metadata is None:
-        metadata = {}
-
-    # Get file extension and determine file type
     extension = get_file_ext(file_name)
 
     # Fetch the DB record so we know the ID for internal URL
@@ -129,6 +114,8 @@ def _process_file(
         return []
 
     # Prepare doc metadata
+    if metadata is None:
+        metadata = {}
     file_display_name = metadata.get("file_display_name") or os.path.basename(file_name)
 
     # Timestamps
@@ -171,7 +158,6 @@ def _process_file(
             "title",
             "connector_type",
             "pdf_password",
-            "mime_type",
         ]
     }
 
@@ -184,45 +170,33 @@ def _process_file(
     title = metadata.get("title") or file_display_name
 
     # 1) If the file itself is an image, handle that scenario quickly
-    if extension in LoadConnector.IMAGE_EXTENSIONS:
-        # Read the image data
+    IMAGE_EXTENSIONS = {".jpg", ".jpeg", ".png", ".webp"}
+    if extension in IMAGE_EXTENSIONS:
+        # Summarize or produce empty doc
         image_data = file.read()
-        if not image_data:
-            logger.warning(f"Empty image file: {file_name}")
-            return []
-
-        # Create an ImageSection for the image
-        try:
-            section, _ = _create_image_section(
-                image_data=image_data,
-                db_session=db_session,
-                parent_file_name=pg_record.file_name,
-                display_name=title,
+        image_section, _ = _create_image_section(
+            llm, image_data, db_session, pg_record.file_name, title
+        )
+        return [
+            Document(
+                id=doc_id,
+                sections=[image_section],
+                source=source_type,
+                semantic_identifier=file_display_name,
+                title=title,
+                doc_updated_at=final_time_updated,
+                primary_owners=p_owners,
+                secondary_owners=s_owners,
+                metadata=metadata_tags,
             )
+        ]
 
-            return [
-                Document(
-                    id=doc_id,
-                    sections=[section],
-                    source=source_type,
-                    semantic_identifier=file_display_name,
-                    title=title,
-                    doc_updated_at=final_time_updated,
-                    primary_owners=p_owners,
-                    secondary_owners=s_owners,
-                    metadata=metadata_tags,
-                )
-            ]
-        except Exception as e:
-            logger.error(f"Failed to process image file {file_name}: {e}")
-            return []
-
-    # 2) Otherwise: text-based approach. Possibly with embedded images.
+    # 2) Otherwise: text-based approach. Possibly with embedded images if enabled.
+    #    (For example .docx with inline images).
     file.seek(0)
     text_content = ""
     embedded_images: list[tuple[bytes, str]] = []
 
-    # Extract text and images from the file
     text_content, embedded_images = extract_text_and_images(
         file=file,
         file_name=file_name,
@@ -230,29 +204,24 @@ def _process_file(
     )
 
     # Build sections: first the text as a single Section
-    sections: list[TextSection | ImageSection] = []
+    sections = []
     link_in_meta = metadata.get("link")
     if text_content.strip():
-        sections.append(TextSection(link=link_in_meta, text=text_content.strip()))
+        sections.append(Section(link=link_in_meta, text=text_content.strip()))
 
     # Then any extracted images from docx, etc.
     for idx, (img_data, img_name) in enumerate(embedded_images, start=1):
         # Store each embedded image as a separate file in PGFileStore
-        # and create a section with the image reference
-        try:
-            image_section, _ = _create_image_section(
-                image_data=img_data,
-                db_session=db_session,
-                parent_file_name=pg_record.file_name,
-                display_name=f"{title} - image {idx}",
-                idx=idx,
-            )
-            sections.append(image_section)
-        except Exception as e:
-            logger.warning(
-                f"Failed to process embedded image {idx} in {file_name}: {e}"
-            )
-
+        # and create a section with the image summary
+        image_section, _ = _create_image_section(
+            llm,
+            img_data,
+            db_session,
+            pg_record.file_name,
+            f"{title} - image {idx}",
+            idx,
+        )
+        sections.append(image_section)
     return [
         Document(
             id=doc_id,
@@ -268,10 +237,10 @@ def _process_file(
     ]
 
 
-class LocalFileConnector(LoadConnector):
+class LocalFileConnector(LoadConnector, VisionEnabledConnector):
     """
     Connector that reads files from Postgres and yields Documents, including
-    embedded image extraction without summarization.
+    optional embedded image extraction.
     """
 
     def __init__(
@@ -283,6 +252,9 @@ class LocalFileConnector(LoadConnector):
         self.batch_size = batch_size
         self.pdf_pass: str | None = None
 
+        # Initialize vision LLM using the mixin
+        self.initialize_vision_llm()
+
     def load_credentials(self, credentials: dict[str, Any]) -> dict[str, Any] | None:
         self.pdf_pass = credentials.get("pdf_password")
 
@@ -314,6 +286,7 @@ class LocalFileConnector(LoadConnector):
                         metadata=metadata,
                         pdf_pass=self.pdf_pass,
                         db_session=db_session,
+                        llm=self.image_analysis_llm,
                     )
                     documents.extend(new_docs)
 

backend/onyx/connectors/fireflies/connector.py

@@ -1,7 +1,6 @@
 from collections.abc import Iterator
 from datetime import datetime
 from datetime import timezone
-from typing import cast
 from typing import List
 
 import requests
@@ -15,8 +14,7 @@ from onyx.connectors.interfaces import SecondsSinceUnixEpoch
 from onyx.connectors.models import BasicExpertInfo
 from onyx.connectors.models import ConnectorMissingCredentialError
 from onyx.connectors.models import Document
-from onyx.connectors.models import ImageSection
-from onyx.connectors.models import TextSection
+from onyx.connectors.models import Section
 from onyx.utils.logger import setup_logger
 
 logger = setup_logger()
@@ -47,7 +45,7 @@ _FIREFLIES_API_QUERY = """
 
 
 def _create_doc_from_transcript(transcript: dict) -> Document | None:
-    sections: List[TextSection] = []
+    sections: List[Section] = []
     current_speaker_name = None
     current_link = ""
     current_text = ""
@@ -59,7 +57,7 @@ def _create_doc_from_transcript(transcript: dict) -> Document | None:
         if sentence["speaker_name"] != current_speaker_name:
             if current_speaker_name is not None:
                 sections.append(
-                    TextSection(
+                    Section(
                         link=current_link,
                         text=current_text.strip(),
                     )
@@ -73,7 +71,7 @@ def _create_doc_from_transcript(transcript: dict) -> Document | None:
 
     # Sometimes these links (links with a timestamp) do not work, it is a bug with Fireflies.
     sections.append(
-        TextSection(
+        Section(
             link=current_link,
             text=current_text.strip(),
         )
@@ -96,7 +94,7 @@ def _create_doc_from_transcript(transcript: dict) -> Document | None:
 
     return Document(
         id=fireflies_id,
-        sections=cast(list[TextSection | ImageSection], sections),
+        sections=sections,
         source=DocumentSource.FIREFLIES,
         semantic_identifier=meeting_title,
         metadata={},

backend/onyx/connectors/freshdesk/connector.py

@@ -14,7 +14,7 @@ from onyx.connectors.interfaces import PollConnector
 from onyx.connectors.interfaces import SecondsSinceUnixEpoch
 from onyx.connectors.models import ConnectorMissingCredentialError
 from onyx.connectors.models import Document
-from onyx.connectors.models import TextSection
+from onyx.connectors.models import Section
 from onyx.file_processing.html_utils import parse_html_page_basic
 from onyx.utils.logger import setup_logger
 
@@ -133,7 +133,7 @@ def _create_doc_from_ticket(ticket: dict, domain: str) -> Document:
     return Document(
         id=_FRESHDESK_ID_PREFIX + link,
         sections=[
-            TextSection(
+            Section(
                 link=link,
                 text=text,
             )

backend/onyx/connectors/gitbook/connector.py

@@ -13,7 +13,7 @@ from onyx.connectors.interfaces import PollConnector
 from onyx.connectors.interfaces import SecondsSinceUnixEpoch
 from onyx.connectors.models import ConnectorMissingCredentialError
 from onyx.connectors.models import Document
-from onyx.connectors.models import TextSection
+from onyx.connectors.models import Section
 from onyx.utils.logger import setup_logger
 
 
@@ -183,7 +183,7 @@ def _convert_page_to_document(
     return Document(
         id=f"gitbook-{space_id}-{page_id}",
         sections=[
-            TextSection(
+            Section(
                 link=page.get("urls", {}).get("app", ""),
                 text=_extract_text_from_document(page_content),
             )
@@ -228,15 +228,10 @@ class GitbookConnector(LoadConnector, PollConnector):
             raise ConnectorMissingCredentialError("GitBook")
 
         try:
-            content = self.client.get(f"/spaces/{self.space_id}/content/pages")
+            content = self.client.get(f"/spaces/{self.space_id}/content")
             pages: list[dict[str, Any]] = content.get("pages", [])
             current_batch: list[Document] = []
 
-            logger.info(f"Found {len(pages)} root pages.")
-            logger.info(
-                f"First 20 Page Ids: {[page.get('id', 'Unknown') for page in pages[:20]]}"
-            )
-
             while pages:
                 page = pages.pop(0)
 

backend/onyx/connectors/github/connector.py

@@ -27,7 +27,7 @@ from onyx.connectors.interfaces import PollConnector
 from onyx.connectors.interfaces import SecondsSinceUnixEpoch
 from onyx.connectors.models import ConnectorMissingCredentialError
 from onyx.connectors.models import Document
-from onyx.connectors.models import TextSection
+from onyx.connectors.models import Section
 from onyx.utils.batching import batch_generator
 from onyx.utils.logger import setup_logger
 
@@ -87,9 +87,7 @@ def _batch_github_objects(
 def _convert_pr_to_document(pull_request: PullRequest) -> Document:
     return Document(
         id=pull_request.html_url,
-        sections=[
-            TextSection(link=pull_request.html_url, text=pull_request.body or "")
-        ],
+        sections=[Section(link=pull_request.html_url, text=pull_request.body or "")],
         source=DocumentSource.GITHUB,
         semantic_identifier=pull_request.title,
         # updated_at is UTC time but is timezone unaware, explicitly add UTC
@@ -111,7 +109,7 @@ def _fetch_issue_comments(issue: Issue) -> str:
 def _convert_issue_to_document(issue: Issue) -> Document:
     return Document(
         id=issue.html_url,
-        sections=[TextSection(link=issue.html_url, text=issue.body or "")],
+        sections=[Section(link=issue.html_url, text=issue.body or "")],
         source=DocumentSource.GITHUB,
         semantic_identifier=issue.title,
         # updated_at is UTC time but is timezone unaware

backend/onyx/connectors/gitlab/connector.py

@@ -21,7 +21,7 @@ from onyx.connectors.interfaces import SecondsSinceUnixEpoch
 from onyx.connectors.models import BasicExpertInfo
 from onyx.connectors.models import ConnectorMissingCredentialError
 from onyx.connectors.models import Document
-from onyx.connectors.models import TextSection
+from onyx.connectors.models import Section
 from onyx.utils.logger import setup_logger
 
 
@@ -56,7 +56,7 @@ def get_author(author: Any) -> BasicExpertInfo:
 def _convert_merge_request_to_document(mr: Any) -> Document:
     doc = Document(
         id=mr.web_url,
-        sections=[TextSection(link=mr.web_url, text=mr.description or "")],
+        sections=[Section(link=mr.web_url, text=mr.description or "")],
         source=DocumentSource.GITLAB,
         semantic_identifier=mr.title,
         # updated_at is UTC time but is timezone unaware, explicitly add UTC
@@ -72,7 +72,7 @@ def _convert_merge_request_to_document(mr: Any) -> Document:
 def _convert_issue_to_document(issue: Any) -> Document:
     doc = Document(
         id=issue.web_url,
-        sections=[TextSection(link=issue.web_url, text=issue.description or "")],
+        sections=[Section(link=issue.web_url, text=issue.description or "")],
         source=DocumentSource.GITLAB,
         semantic_identifier=issue.title,
         # updated_at is UTC time but is timezone unaware, explicitly add UTC
@@ -99,7 +99,7 @@ def _convert_code_to_document(
     file_url = f"{url}/{projectOwner}/{projectName}/-/blob/master/{file['path']}"  # Construct the file URL
     doc = Document(
         id=file["id"],
-        sections=[TextSection(link=file_url, text=file_content)],
+        sections=[Section(link=file_url, text=file_content)],
         source=DocumentSource.GITLAB,
         semantic_identifier=file["name"],
         doc_updated_at=datetime.now().replace(

backend/onyx/connectors/gmail/connector.py

@@ -1,6 +1,5 @@
 from base64 import urlsafe_b64decode
 from typing import Any
-from typing import cast
 from typing import Dict
 
 from google.oauth2.credentials import Credentials as OAuthCredentials  # type: ignore
@@ -29,9 +28,8 @@ from onyx.connectors.interfaces import SecondsSinceUnixEpoch
 from onyx.connectors.interfaces import SlimConnector
 from onyx.connectors.models import BasicExpertInfo
 from onyx.connectors.models import Document
-from onyx.connectors.models import ImageSection
+from onyx.connectors.models import Section
 from onyx.connectors.models import SlimDocument
-from onyx.connectors.models import TextSection
 from onyx.indexing.indexing_heartbeat import IndexingHeartbeatInterface
 from onyx.utils.logger import setup_logger
 from onyx.utils.retry_wrapper import retry_builder
@@ -117,7 +115,7 @@ def _get_message_body(payload: dict[str, Any]) -> str:
     return message_body
 
 
-def message_to_section(message: Dict[str, Any]) -> tuple[TextSection, dict[str, str]]:
+def message_to_section(message: Dict[str, Any]) -> tuple[Section, dict[str, str]]:
     link = f"https://mail.google.com/mail/u/0/#inbox/{message['id']}"
 
     payload = message.get("payload", {})
@@ -144,7 +142,7 @@ def message_to_section(message: Dict[str, Any]) -> tuple[TextSection, dict[str,
 
     message_body_text: str = _get_message_body(payload)
 
-    return TextSection(link=link, text=message_body_text + message_data), metadata
+    return Section(link=link, text=message_body_text + message_data), metadata
 
 
 def thread_to_document(full_thread: Dict[str, Any]) -> Document | None:
@@ -194,7 +192,7 @@ def thread_to_document(full_thread: Dict[str, Any]) -> Document | None:
     return Document(
         id=id,
         semantic_identifier=semantic_identifier,
-        sections=cast(list[TextSection | ImageSection], sections),
+        sections=sections,
         source=DocumentSource.GMAIL,
         # This is used to perform permission sync
         primary_owners=primary_owners,

backend/onyx/connectors/gong/connector.py

@@ -18,7 +18,7 @@ from onyx.connectors.interfaces import PollConnector
 from onyx.connectors.interfaces import SecondsSinceUnixEpoch
 from onyx.connectors.models import ConnectorMissingCredentialError
 from onyx.connectors.models import Document
-from onyx.connectors.models import TextSection
+from onyx.connectors.models import Section
 from onyx.utils.logger import setup_logger
 
 
@@ -243,7 +243,7 @@ class GongConnector(LoadConnector, PollConnector):
                     Document(
                         id=call_id,
                         sections=[
-                            TextSection(link=call_metadata["url"], text=transcript_text)
+                            Section(link=call_metadata["url"], text=transcript_text)
                         ],
                         source=DocumentSource.GONG,
                         # Should not ever be Untitled as a call cannot be made without a Title

backend/onyx/connectors/google_drive/doc_conversion.py

@@ -1,53 +1,40 @@
 import io
-from collections.abc import Callable
 from datetime import datetime
-from typing import cast
+from datetime import timezone
+from tempfile import NamedTemporaryFile
 
-from googleapiclient.http import MediaIoBaseDownload  # type: ignore
+import openpyxl  # type: ignore
+from googleapiclient.discovery import build  # type: ignore
+from googleapiclient.errors import HttpError  # type: ignore
 
+from onyx.configs.app_configs import CONTINUE_ON_CONNECTOR_FAILURE
 from onyx.configs.constants import DocumentSource
 from onyx.configs.constants import FileOrigin
 from onyx.connectors.google_drive.constants import DRIVE_FOLDER_TYPE
 from onyx.connectors.google_drive.constants import DRIVE_SHORTCUT_TYPE
+from onyx.connectors.google_drive.constants import UNSUPPORTED_FILE_TYPE_CONTENT
 from onyx.connectors.google_drive.models import GDriveMimeType
 from onyx.connectors.google_drive.models import GoogleDriveFileType
 from onyx.connectors.google_drive.section_extraction import get_document_sections
 from onyx.connectors.google_utils.resources import GoogleDocsService
 from onyx.connectors.google_utils.resources import GoogleDriveService
-from onyx.connectors.models import ConnectorFailure
 from onyx.connectors.models import Document
-from onyx.connectors.models import DocumentFailure
-from onyx.connectors.models import ImageSection
+from onyx.connectors.models import Section
 from onyx.connectors.models import SlimDocument
-from onyx.connectors.models import TextSection
 from onyx.db.engine import get_session_with_current_tenant
 from onyx.file_processing.extract_file_text import docx_to_text_and_images
-from onyx.file_processing.extract_file_text import extract_file_text
 from onyx.file_processing.extract_file_text import pptx_to_text
 from onyx.file_processing.extract_file_text import read_pdf_file
-from onyx.file_processing.extract_file_text import xlsx_to_text
 from onyx.file_processing.file_validation import is_valid_image_type
 from onyx.file_processing.image_summarization import summarize_image_with_error_handling
 from onyx.file_processing.image_utils import store_image_and_create_section
+from onyx.file_processing.unstructured import get_unstructured_api_key
+from onyx.file_processing.unstructured import unstructured_to_text
 from onyx.llm.interfaces import LLM
 from onyx.utils.logger import setup_logger
 
 logger = setup_logger()
 
-# Mapping of Google Drive mime types to export formats
-GOOGLE_MIME_TYPES_TO_EXPORT = {
-    GDriveMimeType.DOC.value: "text/plain",
-    GDriveMimeType.SPREADSHEET.value: "text/csv",
-    GDriveMimeType.PPT.value: "text/plain",
-}
-
-# Define Google MIME types mapping
-GOOGLE_MIME_TYPES = {
-    GDriveMimeType.DOC.value: "text/plain",
-    GDriveMimeType.SPREADSHEET.value: "text/csv",
-    GDriveMimeType.PPT.value: "text/plain",
-}
-
 
 def _summarize_drive_image(
     image_data: bytes, image_name: str, image_analysis_llm: LLM | None
@@ -79,147 +66,260 @@ def is_gdrive_image_mime_type(mime_type: str) -> bool:
 def _extract_sections_basic(
     file: dict[str, str],
     service: GoogleDriveService,
-    allow_images: bool,
-) -> list[TextSection | ImageSection]:
-    """Extract text and images from a Google Drive file."""
-    file_id = file["id"]
-    file_name = file["name"]
+    image_analysis_llm: LLM | None = None,
+) -> list[Section]:
+    """
+    Extends the existing logic to handle either a docx with embedded images
+    or standalone images (PNG, JPG, etc).
+    """
     mime_type = file["mimeType"]
-    link = file.get("webViewLink", "")
+    link = file["webViewLink"]
+    file_name = file.get("name", file["id"])
+    supported_file_types = set(item.value for item in GDriveMimeType)
 
-    try:
-        # skip images if not explicitly enabled
-        if not allow_images and is_gdrive_image_mime_type(mime_type):
-            return []
+    # 1) If the file is an image, retrieve the raw bytes, optionally summarize
+    if is_gdrive_image_mime_type(mime_type):
+        try:
+            response = service.files().get_media(fileId=file["id"]).execute()
 
-        # For Google Docs, Sheets, and Slides, export as plain text
-        if mime_type in GOOGLE_MIME_TYPES_TO_EXPORT:
-            export_mime_type = GOOGLE_MIME_TYPES_TO_EXPORT[mime_type]
-            # Use the correct API call for exporting files
-            request = service.files().export_media(
-                fileId=file_id, mimeType=export_mime_type
-            )
-            response_bytes = io.BytesIO()
-            downloader = MediaIoBaseDownload(response_bytes, request)
-            done = False
-            while not done:
-                _, done = downloader.next_chunk()
-
-            response = response_bytes.getvalue()
-            if not response:
-                logger.warning(f"Failed to export {file_name} as {export_mime_type}")
-                return []
-
-            text = response.decode("utf-8")
-            return [TextSection(link=link, text=text)]
-
-        # For other file types, download the file
-        # Use the correct API call for downloading files
-        request = service.files().get_media(fileId=file_id)
-        response_bytes = io.BytesIO()
-        downloader = MediaIoBaseDownload(response_bytes, request)
-        done = False
-        while not done:
-            _, done = downloader.next_chunk()
-
-        response = response_bytes.getvalue()
-        if not response:
-            logger.warning(f"Failed to download {file_name}")
-            return []
-
-        # Process based on mime type
-        if mime_type == "text/plain":
-            text = response.decode("utf-8")
-            return [TextSection(link=link, text=text)]
-
-        elif (
-            mime_type
-            == "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
-        ):
-            text, _ = docx_to_text_and_images(io.BytesIO(response))
-            return [TextSection(link=link, text=text)]
-
-        elif (
-            mime_type
-            == "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
-        ):
-            text = xlsx_to_text(io.BytesIO(response))
-            return [TextSection(link=link, text=text)]
-
-        elif (
-            mime_type
-            == "application/vnd.openxmlformats-officedocument.presentationml.presentation"
-        ):
-            text = pptx_to_text(io.BytesIO(response))
-            return [TextSection(link=link, text=text)]
-
-        elif is_gdrive_image_mime_type(mime_type):
-            # For images, store them for later processing
-            sections: list[TextSection | ImageSection] = []
-            try:
-                with get_session_with_current_tenant() as db_session:
-                    section, embedded_id = store_image_and_create_section(
-                        db_session=db_session,
-                        image_data=response,
-                        file_name=file_id,
-                        display_name=file_name,
-                        media_type=mime_type,
-                        file_origin=FileOrigin.CONNECTOR,
-                        link=link,
-                    )
-                    sections.append(section)
-            except Exception as e:
-                logger.error(f"Failed to process image {file_name}: {e}")
-            return sections
-
-        elif mime_type == "application/pdf":
-            text, _pdf_meta, images = read_pdf_file(io.BytesIO(response))
-            pdf_sections: list[TextSection | ImageSection] = [
-                TextSection(link=link, text=text)
+            with get_session_with_current_tenant() as db_session:
+                section, _ = store_image_and_create_section(
+                    db_session=db_session,
+                    image_data=response,
+                    file_name=file["id"],
+                    display_name=file_name,
+                    media_type=mime_type,
+                    llm=image_analysis_llm,
+                    file_origin=FileOrigin.CONNECTOR,
+                )
+                return [section]
+        except Exception as e:
+            logger.warning(f"Failed to fetch or summarize image: {e}")
+            return [
+                Section(
+                    link=link,
+                    text="",
+                    image_file_name=link,
+                )
             ]
 
-            # Process embedded images in the PDF
+    if mime_type not in supported_file_types:
+        # Unsupported file types can still have a title, finding this way is still useful
+        return [Section(link=link, text=UNSUPPORTED_FILE_TYPE_CONTENT)]
+
+    try:
+        # ---------------------------
+        # Google Sheets extraction
+        if mime_type == GDriveMimeType.SPREADSHEET.value:
             try:
+                sheets_service = build(
+                    "sheets", "v4", credentials=service._http.credentials
+                )
+                spreadsheet = (
+                    sheets_service.spreadsheets()
+                    .get(spreadsheetId=file["id"])
+                    .execute()
+                )
+
+                sections = []
+                for sheet in spreadsheet["sheets"]:
+                    sheet_name = sheet["properties"]["title"]
+                    sheet_id = sheet["properties"]["sheetId"]
+
+                    # Get sheet dimensions
+                    grid_properties = sheet["properties"].get("gridProperties", {})
+                    row_count = grid_properties.get("rowCount", 1000)
+                    column_count = grid_properties.get("columnCount", 26)
+
+                    # Convert column count to letter (e.g., 26 -> Z, 27 -> AA)
+                    end_column = ""
+                    while column_count:
+                        column_count, remainder = divmod(column_count - 1, 26)
+                        end_column = chr(65 + remainder) + end_column
+
+                    range_name = f"'{sheet_name}'!A1:{end_column}{row_count}"
+
+                    try:
+                        result = (
+                            sheets_service.spreadsheets()
+                            .values()
+                            .get(spreadsheetId=file["id"], range=range_name)
+                            .execute()
+                        )
+                        values = result.get("values", [])
+
+                        if values:
+                            text = f"Sheet: {sheet_name}\n"
+                            for row in values:
+                                text += "\t".join(str(cell) for cell in row) + "\n"
+                            sections.append(
+                                Section(
+                                    link=f"{link}#gid={sheet_id}",
+                                    text=text,
+                                )
+                            )
+                    except HttpError as e:
+                        logger.warning(
+                            f"Error fetching data for sheet '{sheet_name}': {e}"
+                        )
+                        continue
+                return sections
+
+            except Exception as e:
+                logger.warning(
+                    f"Ran into exception '{e}' when pulling data from Google Sheet '{file['name']}'."
+                    " Falling back to basic extraction."
+                )
+        # ---------------------------
+        # Microsoft Excel (.xlsx or .xls) extraction branch
+        elif mime_type in [
+            GDriveMimeType.SPREADSHEET_OPEN_FORMAT.value,
+            GDriveMimeType.SPREADSHEET_MS_EXCEL.value,
+        ]:
+            try:
+                response = service.files().get_media(fileId=file["id"]).execute()
+
+                with NamedTemporaryFile(suffix=".xlsx", delete=True) as tmp:
+                    tmp.write(response)
+                    tmp_path = tmp.name
+
+                    section_separator = "\n\n"
+                    workbook = openpyxl.load_workbook(tmp_path, read_only=True)
+
+                    # Work similarly to the xlsx_to_text function used for file connector
+                    # but returns Sections instead of a string
+                    sections = [
+                        Section(
+                            link=link,
+                            text=(
+                                f"Sheet: {sheet.title}\n\n"
+                                + section_separator.join(
+                                    ",".join(map(str, row))
+                                    for row in sheet.iter_rows(
+                                        min_row=1, values_only=True
+                                    )
+                                    if row
+                                )
+                            ),
+                        )
+                        for sheet in workbook.worksheets
+                    ]
+
+                return sections
+
+            except Exception as e:
+                logger.warning(
+                    f"Error extracting data from Excel file '{file['name']}': {e}"
+                )
+                return [
+                    Section(link=link, text="Error extracting data from Excel file")
+                ]
+
+        # ---------------------------
+        # Export for Google Docs, PPT, and fallback for spreadsheets
+        if mime_type in [
+            GDriveMimeType.DOC.value,
+            GDriveMimeType.PPT.value,
+            GDriveMimeType.SPREADSHEET.value,
+        ]:
+            export_mime_type = (
+                "text/plain"
+                if mime_type != GDriveMimeType.SPREADSHEET.value
+                else "text/csv"
+            )
+            text = (
+                service.files()
+                .export(fileId=file["id"], mimeType=export_mime_type)
+                .execute()
+                .decode("utf-8")
+            )
+            return [Section(link=link, text=text)]
+
+        # ---------------------------
+        # Plain text and Markdown files
+        elif mime_type in [
+            GDriveMimeType.PLAIN_TEXT.value,
+            GDriveMimeType.MARKDOWN.value,
+        ]:
+            text_data = (
+                service.files().get_media(fileId=file["id"]).execute().decode("utf-8")
+            )
+            return [Section(link=link, text=text_data)]
+
+        # ---------------------------
+        # Word, PowerPoint, PDF files
+        elif mime_type in [
+            GDriveMimeType.WORD_DOC.value,
+            GDriveMimeType.POWERPOINT.value,
+            GDriveMimeType.PDF.value,
+        ]:
+            response_bytes = service.files().get_media(fileId=file["id"]).execute()
+
+            # Optionally use Unstructured
+            if get_unstructured_api_key():
+                text = unstructured_to_text(
+                    file=io.BytesIO(response_bytes),
+                    file_name=file_name,
+                )
+                return [Section(link=link, text=text)]
+
+            if mime_type == GDriveMimeType.WORD_DOC.value:
+                # Use docx_to_text_and_images to get text plus embedded images
+                text, embedded_images = docx_to_text_and_images(
+                    file=io.BytesIO(response_bytes),
+                )
+                sections = []
+                if text.strip():
+                    sections.append(Section(link=link, text=text.strip()))
+
+                # Process each embedded image using the standardized function
                 with get_session_with_current_tenant() as db_session:
-                    for idx, (img_data, img_name) in enumerate(images):
-                        section, embedded_id = store_image_and_create_section(
+                    for idx, (img_data, img_name) in enumerate(
+                        embedded_images, start=1
+                    ):
+                        # Create a unique identifier for the embedded image
+                        embedded_id = f"{file['id']}_embedded_{idx}"
+
+                        section, _ = store_image_and_create_section(
                             db_session=db_session,
                             image_data=img_data,
-                            file_name=f"{file_id}_img_{idx}",
+                            file_name=embedded_id,
                             display_name=img_name or f"{file_name} - image {idx}",
+                            llm=image_analysis_llm,
                             file_origin=FileOrigin.CONNECTOR,
                         )
-                        pdf_sections.append(section)
-            except Exception as e:
-                logger.error(f"Failed to process PDF images in {file_name}: {e}")
-            return pdf_sections
+                        sections.append(section)
+                return sections
 
-        else:
-            # For unsupported file types, try to extract text
-            try:
-                text = extract_file_text(io.BytesIO(response), file_name)
-                return [TextSection(link=link, text=text)]
-            except Exception as e:
-                logger.warning(f"Failed to extract text from {file_name}: {e}")
-                return []
+            elif mime_type == GDriveMimeType.PDF.value:
+                text, _pdf_meta, images = read_pdf_file(io.BytesIO(response_bytes))
+                return [Section(link=link, text=text)]
+
+            elif mime_type == GDriveMimeType.POWERPOINT.value:
+                text_data = pptx_to_text(io.BytesIO(response_bytes))
+                return [Section(link=link, text=text_data)]
+
+        # Catch-all case, should not happen since there should be specific handling
+        # for each of the supported file types
+        error_message = f"Unsupported file type: {mime_type}"
+        logger.error(error_message)
+        raise ValueError(error_message)
 
     except Exception as e:
-        logger.error(f"Error processing file {file_name}: {e}")
-        return []
+        logger.exception(f"Error extracting sections from file: {e}")
+        return [Section(link=link, text=UNSUPPORTED_FILE_TYPE_CONTENT)]
 
 
 def convert_drive_item_to_document(
     file: GoogleDriveFileType,
-    drive_service: Callable[[], GoogleDriveService],
-    docs_service: Callable[[], GoogleDocsService],
-    allow_images: bool,
-) -> Document | ConnectorFailure | None:
+    drive_service: GoogleDriveService,
+    docs_service: GoogleDocsService,
+    image_analysis_llm: LLM | None,
+) -> Document | None:
     """
     Main entry point for converting a Google Drive file => Document object.
+    Now we accept an optional `llm` to pass to `_extract_sections_basic`.
     """
-    doc_id = ""
-    sections: list[TextSection | ImageSection] = []
-
     try:
         # skip shortcuts or folders
         if file.get("mimeType") in [DRIVE_SHORTCUT_TYPE, DRIVE_FOLDER_TYPE]:
@@ -227,59 +327,44 @@ def convert_drive_item_to_document(
             return None
 
         # If it's a Google Doc, we might do advanced parsing
+        sections: list[Section] = []
         if file.get("mimeType") == GDriveMimeType.DOC.value:
             try:
                 # get_document_sections is the advanced approach for Google Docs
-                doc_sections = get_document_sections(
-                    docs_service=docs_service(), doc_id=file.get("id", "")
-                )
-                if doc_sections:
-                    sections = cast(list[TextSection | ImageSection], doc_sections)
+                sections = get_document_sections(docs_service, file["id"])
             except Exception as e:
                 logger.warning(
-                    f"Error in advanced parsing: {e}. Falling back to basic extraction."
+                    f"Failed to pull google doc sections from '{file['name']}': {e}. "
+                    "Falling back to basic extraction."
                 )
 
-        # If we don't have sections yet, use the basic extraction method
+        # If not a doc, or if we failed above, do our 'basic' approach
         if not sections:
-            sections = _extract_sections_basic(file, drive_service(), allow_images)
+            sections = _extract_sections_basic(file, drive_service, image_analysis_llm)
 
-        # If we still don't have any sections, skip this file
         if not sections:
-            logger.warning(f"No content extracted from {file.get('name')}. Skipping.")
             return None
 
         doc_id = file["webViewLink"]
+        updated_time = datetime.fromisoformat(file["modifiedTime"]).astimezone(
+            timezone.utc
+        )
 
-        # Create the document
         return Document(
             id=doc_id,
             sections=sections,
             source=DocumentSource.GOOGLE_DRIVE,
-            semantic_identifier=file.get("name", ""),
-            metadata={
-                "owner_names": ", ".join(
-                    owner.get("displayName", "") for owner in file.get("owners", [])
-                ),
-            },
-            doc_updated_at=datetime.fromisoformat(
-                file.get("modifiedTime", "").replace("Z", "+00:00")
-            ),
+            semantic_identifier=file["name"],
+            doc_updated_at=updated_time,
+            metadata={},  # or any metadata from 'file'
+            additional_info=file.get("id"),
         )
+
     except Exception as e:
-        error_str = f"Error converting file '{file.get('name')}' to Document: {e}"
-        logger.exception(error_str)
-        return ConnectorFailure(
-            failed_document=DocumentFailure(
-                document_id=doc_id,
-                document_link=sections[0].link
-                if sections
-                else None,  # TODO: see if this is the best way to get a link
-            ),
-            failed_entity=None,
-            failure_message=error_str,
-            exception=e,
-        )
+        logger.exception(f"Error converting file '{file.get('name')}' to Document: {e}")
+        if not CONTINUE_ON_CONNECTOR_FAILURE:
+            raise
+    return None
 
 
 def build_slim_document(file: GoogleDriveFileType) -> SlimDocument | None:

backend/onyx/connectors/google_drive/file_retrieval.py

@@ -1,23 +1,17 @@
 from collections.abc import Callable
 from collections.abc import Iterator
 from datetime import datetime
-from datetime import timezone
+from typing import Any
 
 from googleapiclient.discovery import Resource  # type: ignore
 
 from onyx.connectors.google_drive.constants import DRIVE_FOLDER_TYPE
 from onyx.connectors.google_drive.constants import DRIVE_SHORTCUT_TYPE
-from onyx.connectors.google_drive.models import DriveRetrievalStage
 from onyx.connectors.google_drive.models import GoogleDriveFileType
-from onyx.connectors.google_drive.models import RetrievedDriveFile
 from onyx.connectors.google_utils.google_utils import execute_paginated_retrieval
-from onyx.connectors.google_utils.google_utils import GoogleFields
-from onyx.connectors.google_utils.google_utils import ORDER_BY_KEY
-from onyx.connectors.google_utils.resources import GoogleDriveService
 from onyx.connectors.interfaces import SecondsSinceUnixEpoch
 from onyx.utils.logger import setup_logger
 
-
 logger = setup_logger()
 
 FILE_FIELDS = (
@@ -37,13 +31,11 @@ def _generate_time_range_filter(
 ) -> str:
     time_range_filter = ""
     if start is not None:
-        time_start = datetime.fromtimestamp(start, tz=timezone.utc).isoformat()
-        time_range_filter += (
-            f" and {GoogleFields.MODIFIED_TIME.value} >= '{time_start}'"
-        )
+        time_start = datetime.utcfromtimestamp(start).isoformat() + "Z"
+        time_range_filter += f" and modifiedTime >= '{time_start}'"
     if end is not None:
-        time_stop = datetime.fromtimestamp(end, tz=timezone.utc).isoformat()
-        time_range_filter += f" and {GoogleFields.MODIFIED_TIME.value} <= '{time_stop}'"
+        time_stop = datetime.utcfromtimestamp(end).isoformat() + "Z"
+        time_range_filter += f" and modifiedTime <= '{time_stop}'"
     return time_range_filter
 
 
@@ -74,9 +66,9 @@ def _get_folders_in_parent(
 def _get_files_in_parent(
     service: Resource,
     parent_id: str,
-    is_slim: bool,
     start: SecondsSinceUnixEpoch | None = None,
     end: SecondsSinceUnixEpoch | None = None,
+    is_slim: bool = False,
 ) -> Iterator[GoogleDriveFileType]:
     query = f"mimeType != '{DRIVE_FOLDER_TYPE}' and '{parent_id}' in parents"
     query += " and trashed = false"
@@ -91,7 +83,6 @@ def _get_files_in_parent(
         includeItemsFromAllDrives=True,
         fields=SLIM_FILE_FIELDS if is_slim else FILE_FIELDS,
         q=query,
-        **({} if is_slim else {ORDER_BY_KEY: GoogleFields.MODIFIED_TIME.value}),
     ):
         yield file
 
@@ -99,50 +90,30 @@ def _get_files_in_parent(
 def crawl_folders_for_files(
     service: Resource,
     parent_id: str,
-    is_slim: bool,
-    user_email: str,
     traversed_parent_ids: set[str],
     update_traversed_ids_func: Callable[[str], None],
     start: SecondsSinceUnixEpoch | None = None,
     end: SecondsSinceUnixEpoch | None = None,
-) -> Iterator[RetrievedDriveFile]:
+) -> Iterator[GoogleDriveFileType]:
     """
     This function starts crawling from any folder. It is slower though.
     """
-    logger.info("Entered crawl_folders_for_files with parent_id: " + parent_id)
-    if parent_id not in traversed_parent_ids:
-        logger.info("Parent id not in traversed parent ids, getting files")
-        found_files = False
-        file = {}
-        try:
-            for file in _get_files_in_parent(
-                service=service,
-                parent_id=parent_id,
-                is_slim=is_slim,
-                start=start,
-                end=end,
-            ):
-                found_files = True
-                logger.info(f"Found file: {file['name']}")
-                yield RetrievedDriveFile(
-                    drive_file=file,
-                    user_email=user_email,
-                    parent_id=parent_id,
-                    completion_stage=DriveRetrievalStage.FOLDER_FILES,
-                )
-        except Exception as e:
-            logger.error(f"Error getting files in parent {parent_id}: {e}")
-            yield RetrievedDriveFile(
-                drive_file=file,
-                user_email=user_email,
-                parent_id=parent_id,
-                completion_stage=DriveRetrievalStage.FOLDER_FILES,
-                error=e,
-            )
-        if found_files:
-            update_traversed_ids_func(parent_id)
-    else:
-        logger.info(f"Skipping subfolder files since already traversed: {parent_id}")
+    if parent_id in traversed_parent_ids:
+        logger.info(f"Skipping subfolder since already traversed: {parent_id}")
+        return
+
+    found_files = False
+    for file in _get_files_in_parent(
+        service=service,
+        start=start,
+        end=end,
+        parent_id=parent_id,
+    ):
+        found_files = True
+        yield file
+
+    if found_files:
+        update_traversed_ids_func(parent_id)
 
     for subfolder in _get_folders_in_parent(
         service=service,
@@ -152,8 +123,6 @@ def crawl_folders_for_files(
         yield from crawl_folders_for_files(
             service=service,
             parent_id=subfolder["id"],
-            is_slim=is_slim,
-            user_email=user_email,
             traversed_parent_ids=traversed_parent_ids,
             update_traversed_ids_func=update_traversed_ids_func,
             start=start,
@@ -164,19 +133,16 @@ def crawl_folders_for_files(
 def get_files_in_shared_drive(
     service: Resource,
     drive_id: str,
-    is_slim: bool,
+    is_slim: bool = False,
     update_traversed_ids_func: Callable[[str], None] = lambda _: None,
     start: SecondsSinceUnixEpoch | None = None,
     end: SecondsSinceUnixEpoch | None = None,
 ) -> Iterator[GoogleDriveFileType]:
-    kwargs = {}
-    if not is_slim:
-        kwargs[ORDER_BY_KEY] = GoogleFields.MODIFIED_TIME.value
-
     # If we know we are going to folder crawl later, we can cache the folders here
     # Get all folders being queried and add them to the traversed set
     folder_query = f"mimeType = '{DRIVE_FOLDER_TYPE}'"
     folder_query += " and trashed = false"
+    found_folders = False
     for file in execute_paginated_retrieval(
         retrieval_function=service.files().list,
         list_key="files",
@@ -189,13 +155,15 @@ def get_files_in_shared_drive(
         q=folder_query,
     ):
         update_traversed_ids_func(file["id"])
+        found_folders = True
+    if found_folders:
+        update_traversed_ids_func(drive_id)
 
     # Get all files in the shared drive
     file_query = f"mimeType != '{DRIVE_FOLDER_TYPE}'"
     file_query += " and trashed = false"
     file_query += _generate_time_range_filter(start, end)
-
-    for file in execute_paginated_retrieval(
+    yield from execute_paginated_retrieval(
         retrieval_function=service.files().list,
         list_key="files",
         continue_on_404_or_403=True,
@@ -205,26 +173,16 @@ def get_files_in_shared_drive(
         includeItemsFromAllDrives=True,
         fields=SLIM_FILE_FIELDS if is_slim else FILE_FIELDS,
         q=file_query,
-        **kwargs,
-    ):
-        # If we found any files, mark this drive as traversed. When a user has access to a drive,
-        # they have access to all the files in the drive. Also not a huge deal if we re-traverse
-        # empty drives.
-        update_traversed_ids_func(drive_id)
-        yield file
+    )
 
 
 def get_all_files_in_my_drive(
-    service: GoogleDriveService,
+    service: Any,
     update_traversed_ids_func: Callable,
-    is_slim: bool,
+    is_slim: bool = False,
     start: SecondsSinceUnixEpoch | None = None,
     end: SecondsSinceUnixEpoch | None = None,
 ) -> Iterator[GoogleDriveFileType]:
-    kwargs = {}
-    if not is_slim:
-        kwargs[ORDER_BY_KEY] = GoogleFields.MODIFIED_TIME.value
-
     # If we know we are going to folder crawl later, we can cache the folders here
     # Get all folders being queried and add them to the traversed set
     folder_query = f"mimeType = '{DRIVE_FOLDER_TYPE}'"
@@ -238,7 +196,7 @@ def get_all_files_in_my_drive(
         fields=SLIM_FILE_FIELDS if is_slim else FILE_FIELDS,
         q=folder_query,
     ):
-        update_traversed_ids_func(file[GoogleFields.ID])
+        update_traversed_ids_func(file["id"])
         found_folders = True
     if found_folders:
         update_traversed_ids_func(get_root_folder_id(service))
@@ -251,28 +209,22 @@ def get_all_files_in_my_drive(
     yield from execute_paginated_retrieval(
         retrieval_function=service.files().list,
         list_key="files",
-        continue_on_404_or_403=False,
         corpora="user",
         fields=SLIM_FILE_FIELDS if is_slim else FILE_FIELDS,
         q=file_query,
-        **kwargs,
     )
 
 
 def get_all_files_for_oauth(
-    service: GoogleDriveService,
+    service: Any,
     include_files_shared_with_me: bool,
     include_my_drives: bool,
     # One of the above 2 should be true
     include_shared_drives: bool,
-    is_slim: bool,
+    is_slim: bool = False,
     start: SecondsSinceUnixEpoch | None = None,
     end: SecondsSinceUnixEpoch | None = None,
 ) -> Iterator[GoogleDriveFileType]:
-    kwargs = {}
-    if not is_slim:
-        kwargs[ORDER_BY_KEY] = GoogleFields.MODIFIED_TIME.value
-
     should_get_all = (
         include_shared_drives and include_my_drives and include_files_shared_with_me
     )
@@ -291,13 +243,11 @@ def get_all_files_for_oauth(
     yield from execute_paginated_retrieval(
         retrieval_function=service.files().list,
         list_key="files",
-        continue_on_404_or_403=False,
         corpora=corpora,
         includeItemsFromAllDrives=should_get_all,
         supportsAllDrives=should_get_all,
         fields=SLIM_FILE_FIELDS if is_slim else FILE_FIELDS,
         q=file_query,
-        **kwargs,
     )
 
 
@@ -305,8 +255,4 @@ def get_all_files_for_oauth(
 def get_root_folder_id(service: Resource) -> str:
     # we dont paginate here because there is only one root folder per user
     # https://developers.google.com/drive/api/guides/v2-to-v3-reference
-    return (
-        service.files()
-        .get(fileId="root", fields=GoogleFields.ID.value)
-        .execute()[GoogleFields.ID.value]
-    )
+    return service.files().get(fileId="root", fields="id").execute()["id"]

backend/onyx/connectors/google_drive/models.py

@@ -1,15 +1,6 @@
 from enum import Enum
 from typing import Any
 
-from pydantic import BaseModel
-from pydantic import ConfigDict
-from pydantic import field_serializer
-from pydantic import field_validator
-
-from onyx.connectors.interfaces import ConnectorCheckpoint
-from onyx.connectors.interfaces import SecondsSinceUnixEpoch
-from onyx.utils.threadpool_concurrency import ThreadSafeDict
-
 
 class GDriveMimeType(str, Enum):
     DOC = "application/vnd.google-apps.document"
@@ -29,128 +20,3 @@ class GDriveMimeType(str, Enum):
 
 
 GoogleDriveFileType = dict[str, Any]
-
-
-TOKEN_EXPIRATION_TIME = 3600  # 1 hour
-
-
-# These correspond to The major stages of retrieval for google drive.
-# The stages for the oauth flow are:
-# get_all_files_for_oauth(),
-# get_all_drive_ids(),
-# get_files_in_shared_drive(),
-# crawl_folders_for_files()
-#
-# The stages for the service account flow are roughly:
-# get_all_user_emails(),
-# get_all_drive_ids(),
-# get_files_in_shared_drive(),
-# Then for each user:
-#   get_files_in_my_drive()
-#   get_files_in_shared_drive()
-#   crawl_folders_for_files()
-class DriveRetrievalStage(str, Enum):
-    START = "start"
-    DONE = "done"
-    # OAuth specific stages
-    OAUTH_FILES = "oauth_files"
-
-    # Service account specific stages
-    USER_EMAILS = "user_emails"
-    MY_DRIVE_FILES = "my_drive_files"
-
-    # Used for both oauth and service account flows
-    DRIVE_IDS = "drive_ids"
-    SHARED_DRIVE_FILES = "shared_drive_files"
-    FOLDER_FILES = "folder_files"
-
-
-class StageCompletion(BaseModel):
-    """
-    Describes the point in the retrieval+indexing process that the
-    connector is at. completed_until is the timestamp of the latest
-    file that has been retrieved or error that has been yielded.
-    Optional fields are used for retrieval stages that need more information
-    for resuming than just the timestamp of the latest file.
-    """
-
-    stage: DriveRetrievalStage
-    completed_until: SecondsSinceUnixEpoch
-    completed_until_parent_id: str | None = None
-
-    # only used for shared drives
-    processed_drive_ids: set[str] = set()
-
-    def update(
-        self,
-        stage: DriveRetrievalStage,
-        completed_until: SecondsSinceUnixEpoch,
-        completed_until_parent_id: str | None = None,
-    ) -> None:
-        self.stage = stage
-        self.completed_until = completed_until
-        self.completed_until_parent_id = completed_until_parent_id
-
-
-class RetrievedDriveFile(BaseModel):
-    """
-    Describes a file that has been retrieved from google drive.
-    user_email is the email of the user that the file was retrieved
-    by impersonating. If an error worthy of being reported is encountered,
-    error should be set and later propagated as a ConnectorFailure.
-    """
-
-    # The stage at which this file was retrieved
-    completion_stage: DriveRetrievalStage
-
-    # The file that was retrieved
-    drive_file: GoogleDriveFileType
-
-    # The email of the user that the file was retrieved by impersonating
-    user_email: str
-
-    # The id of the parent folder or drive of the file
-    parent_id: str | None = None
-
-    # Any unexpected error that occurred while retrieving the file.
-    # In particular, this is not used for 403/404 errors, which are expected
-    # in the context of impersonating all the users to try to retrieve all
-    # files from all their Drives and Folders.
-    error: Exception | None = None
-
-    model_config = ConfigDict(arbitrary_types_allowed=True)
-
-
-class GoogleDriveCheckpoint(ConnectorCheckpoint):
-    # Checkpoint version of _retrieved_ids
-    retrieved_folder_and_drive_ids: set[str]
-
-    # Describes the point in the retrieval+indexing process that the
-    # checkpoint is at. when this is set to a given stage, the connector
-    # has finished yielding all values from the previous stage.
-    completion_stage: DriveRetrievalStage
-
-    # The latest timestamp of a file that has been retrieved per user email.
-    # StageCompletion is used to track the completion of each stage, but the
-    # timestamp part is not used for folder crawling.
-    completion_map: ThreadSafeDict[str, StageCompletion]
-
-    # cached version of the drive and folder ids to retrieve
-    drive_ids_to_retrieve: list[str] | None = None
-    folder_ids_to_retrieve: list[str] | None = None
-
-    # cached user emails
-    user_emails: list[str] | None = None
-
-    @field_serializer("completion_map")
-    def serialize_completion_map(
-        self, completion_map: ThreadSafeDict[str, StageCompletion], _info: Any
-    ) -> dict[str, StageCompletion]:
-        return completion_map._dict
-
-    @field_validator("completion_map", mode="before")
-    def validate_completion_map(cls, v: Any) -> ThreadSafeDict[str, StageCompletion]:
-        assert isinstance(v, dict) or isinstance(v, ThreadSafeDict)
-        return ThreadSafeDict(
-            {k: StageCompletion.model_validate(v) for k, v in v.items()}
-        )

backend/onyx/connectors/google_drive/section_extraction.py

@@ -3,7 +3,7 @@ from typing import Any
 from pydantic import BaseModel
 
 from onyx.connectors.google_utils.resources import GoogleDocsService
-from onyx.connectors.models import TextSection
+from onyx.connectors.models import Section
 
 
 class CurrentHeading(BaseModel):
@@ -37,7 +37,7 @@ def _extract_text_from_paragraph(paragraph: dict[str, Any]) -> str:
 def get_document_sections(
     docs_service: GoogleDocsService,
     doc_id: str,
-) -> list[TextSection]:
+) -> list[Section]:
     """Extracts sections from a Google Doc, including their headings and content"""
     # Fetch the document structure
     doc = docs_service.documents().get(documentId=doc_id).execute()
@@ -45,7 +45,7 @@ def get_document_sections(
     # Get the content
     content = doc.get("body", {}).get("content", [])
 
-    sections: list[TextSection] = []
+    sections: list[Section] = []
     current_section: list[str] = []
     current_heading: CurrentHeading | None = None
 
@@ -70,7 +70,7 @@ def get_document_sections(
                     heading_text = current_heading.text
                     section_text = f"{heading_text}\n" + "\n".join(current_section)
                     sections.append(
-                        TextSection(
+                        Section(
                             text=section_text.strip(),
                             link=_build_gdoc_section_link(doc_id, current_heading.id),
                         )
@@ -96,7 +96,7 @@ def get_document_sections(
     if current_heading is not None and current_section:
         section_text = f"{current_heading.text}\n" + "\n".join(current_section)
         sections.append(
-            TextSection(
+            Section(
                 text=section_text.strip(),
                 link=_build_gdoc_section_link(doc_id, current_heading.id),
             )

backend/onyx/connectors/google_site/connector.py

@@ -12,7 +12,7 @@ from onyx.configs.constants import DocumentSource
 from onyx.connectors.interfaces import GenerateDocumentsOutput
 from onyx.connectors.interfaces import LoadConnector
 from onyx.connectors.models import Document
-from onyx.connectors.models import TextSection
+from onyx.connectors.models import Section
 from onyx.db.engine import get_sqlalchemy_engine
 from onyx.file_processing.extract_file_text import load_files_from_zip
 from onyx.file_processing.extract_file_text import read_text_file
@@ -118,7 +118,7 @@ class GoogleSitesConnector(LoadConnector):
                     source=DocumentSource.GOOGLE_SITES,
                     semantic_identifier=title,
                     sections=[
-                        TextSection(
+                        Section(
                             link=(self.base_url.rstrip("/") + "/" + path.lstrip("/"))
                             if path
                             else "",

backend/onyx/connectors/google_utils/google_utils.py

@@ -4,7 +4,6 @@ from collections.abc import Callable
 from collections.abc import Iterator
 from datetime import datetime
 from datetime import timezone
-from enum import Enum
 from typing import Any
 
 from googleapiclient.errors import HttpError  # type: ignore
@@ -17,37 +16,20 @@ logger = setup_logger()
 
 
 # Google Drive APIs are quite flakey and may 500 for an
-# extended period of time. This is now addressed by checkpointing.
-#
-# NOTE: We previously tried to combat this here by adding a very
-# long retry period (~20 minutes of trying, one request a minute.)
-# This is no longer necessary due to checkpointing.
-add_retries = retry_builder(tries=5, max_delay=10)
-
-NEXT_PAGE_TOKEN_KEY = "nextPageToken"
-PAGE_TOKEN_KEY = "pageToken"
-ORDER_BY_KEY = "orderBy"
-
-
-# See https://developers.google.com/drive/api/reference/rest/v3/files/list for more
-class GoogleFields(str, Enum):
-    ID = "id"
-    CREATED_TIME = "createdTime"
-    MODIFIED_TIME = "modifiedTime"
-    NAME = "name"
-    SIZE = "size"
-    PARENTS = "parents"
+# extended period of time. Trying to combat here by adding a very
+# long retry period (~20 minutes of trying every minute)
+add_retries = retry_builder(tries=50, max_delay=30)
 
 
 def _execute_with_retry(request: Any) -> Any:
-    max_attempts = 6
+    max_attempts = 10
     attempt = 1
 
     while attempt < max_attempts:
         # Note for reasons unknown, the Google API will sometimes return a 429
         # and even after waiting the retry period, it will return another 429.
         # It could be due to a few possibilities:
-        # 1. Other things are also requesting from the Drive/Gmail API with the same key
+        # 1. Other things are also requesting from the Gmail API with the same key
         # 2. It's a rolling rate limit so the moment we get some amount of requests cleared, we hit it again very quickly
         # 3. The retry-after has a maximum and we've already hit the limit for the day
         # or it's something else...
@@ -108,11 +90,11 @@ def execute_paginated_retrieval(
         retrieval_function: The specific list function to call (e.g., service.files().list)
         **kwargs: Arguments to pass to the list function
     """
-    next_page_token = kwargs.get(PAGE_TOKEN_KEY, "")
+    next_page_token = ""
     while next_page_token is not None:
         request_kwargs = kwargs.copy()
         if next_page_token:
-            request_kwargs[PAGE_TOKEN_KEY] = next_page_token
+            request_kwargs["pageToken"] = next_page_token
 
         try:
             results = retrieval_function(**request_kwargs).execute()
@@ -135,7 +117,7 @@ def execute_paginated_retrieval(
                 logger.exception("Error executing request:")
                 raise e
 
-        next_page_token = results.get(NEXT_PAGE_TOKEN_KEY)
+        next_page_token = results.get("nextPageToken")
         if list_key:
             for item in results.get(list_key, []):
                 yield item

backend/onyx/connectors/guru/connector.py

@@ -15,7 +15,7 @@ from onyx.connectors.interfaces import SecondsSinceUnixEpoch
 from onyx.connectors.models import BasicExpertInfo
 from onyx.connectors.models import ConnectorMissingCredentialError
 from onyx.connectors.models import Document
-from onyx.connectors.models import TextSection
+from onyx.connectors.models import Section
 from onyx.file_processing.html_utils import parse_html_page_basic
 from onyx.utils.logger import setup_logger
 
@@ -120,7 +120,7 @@ class GuruConnector(LoadConnector, PollConnector):
                 doc_batch.append(
                     Document(
                         id=card["id"],
-                        sections=[TextSection(link=link, text=content_text)],
+                        sections=[Section(link=link, text=content_text)],
                         source=DocumentSource.GURU,
                         semantic_identifier=title,
                         doc_updated_at=latest_time,

backend/onyx/connectors/highspot/__init__.py

@@ -1,4 +0,0 @@
-"""
-Highspot connector package for Onyx.
-Enables integration with Highspot's knowledge base.
-"""

backend/onyx/connectors/highspot/client.py

@@ -1,280 +0,0 @@
-import base64
-from typing import Any
-from typing import Dict
-from typing import List
-from typing import Optional
-from urllib.parse import urljoin
-
-import requests
-from requests.adapters import HTTPAdapter
-from requests.exceptions import HTTPError
-from requests.exceptions import RequestException
-from requests.exceptions import Timeout
-from urllib3.util.retry import Retry
-
-from onyx.utils.logger import setup_logger
-
-logger = setup_logger()
-
-
-class HighspotClientError(Exception):
-    """Base exception for Highspot API client errors."""
-
-    def __init__(self, message: str, status_code: Optional[int] = None):
-        self.message = message
-        self.status_code = status_code
-        super().__init__(self.message)
-
-
-class HighspotAuthenticationError(HighspotClientError):
-    """Exception raised for authentication errors."""
-
-
-class HighspotRateLimitError(HighspotClientError):
-    """Exception raised when rate limit is exceeded."""
-
-    def __init__(self, message: str, retry_after: Optional[str] = None):
-        self.retry_after = retry_after
-        super().__init__(message)
-
-
-class HighspotClient:
-    """
-    Client for interacting with the Highspot API.
-
-    Uses basic authentication with provided key (username) and secret (password).
-    Implements retry logic, error handling, and connection pooling.
-    """
-
-    BASE_URL = "https://api-su2.highspot.com/v1.0/"
-
-    def __init__(
-        self,
-        key: str,
-        secret: str,
-        base_url: str = BASE_URL,
-        timeout: int = 30,
-        max_retries: int = 3,
-        backoff_factor: float = 0.5,
-        status_forcelist: Optional[List[int]] = None,
-    ):
-        """
-        Initialize the Highspot API client.
-
-        Args:
-            key: API key (used as username)
-            secret: API secret (used as password)
-            base_url: Base URL for the Highspot API
-            timeout: Request timeout in seconds
-            max_retries: Maximum number of retries for failed requests
-            backoff_factor: Backoff factor for retries
-            status_forcelist: HTTP status codes to retry on
-        """
-        if not key or not secret:
-            raise ValueError("API key and secret are required")
-
-        self.key = key
-        self.secret = secret
-        self.base_url = base_url
-        self.timeout = timeout
-
-        # Set up session with retry logic
-        self.session = requests.Session()
-        retry_strategy = Retry(
-            total=max_retries,
-            backoff_factor=backoff_factor,
-            status_forcelist=status_forcelist or [429, 500, 502, 503, 504],
-            allowed_methods=["GET", "POST", "PUT", "DELETE"],
-        )
-        adapter = HTTPAdapter(max_retries=retry_strategy)
-        self.session.mount("http://", adapter)
-        self.session.mount("https://", adapter)
-
-        # Set up authentication
-        self._setup_auth()
-
-    def _setup_auth(self) -> None:
-        """Set up basic authentication for the session."""
-        auth = f"{self.key}:{self.secret}"
-        encoded_auth = base64.b64encode(auth.encode()).decode()
-        self.session.headers.update(
-            {
-                "Authorization": f"Basic {encoded_auth}",
-                "Content-Type": "application/json",
-                "Accept": "application/json",
-            }
-        )
-
-    def _make_request(
-        self,
-        method: str,
-        endpoint: str,
-        params: Optional[Dict[str, Any]] = None,
-        data: Optional[Dict[str, Any]] = None,
-        json_data: Optional[Dict[str, Any]] = None,
-        headers: Optional[Dict[str, str]] = None,
-    ) -> Dict[str, Any]:
-        """
-        Make a request to the Highspot API.
-
-        Args:
-            method: HTTP method (GET, POST, etc.)
-            endpoint: API endpoint
-            params: URL parameters
-            data: Form data
-            json_data: JSON data
-            headers: Additional headers
-
-        Returns:
-            API response as a dictionary
-
-        Raises:
-            HighspotClientError: On API errors
-            HighspotAuthenticationError: On authentication errors
-            HighspotRateLimitError: On rate limiting
-            requests.exceptions.RequestException: On request failures
-        """
-        url = urljoin(self.base_url, endpoint)
-        request_headers = {}
-        if headers:
-            request_headers.update(headers)
-
-        try:
-            logger.debug(f"Making {method} request to {url}")
-            response = self.session.request(
-                method=method,
-                url=url,
-                params=params,
-                data=data,
-                json=json_data,
-                headers=request_headers,
-                timeout=self.timeout,
-            )
-            response.raise_for_status()
-
-            if response.content and response.content.strip():
-                return response.json()
-            return {}
-
-        except HTTPError as e:
-            status_code = e.response.status_code
-            error_msg = str(e)
-
-            try:
-                error_data = e.response.json()
-                if isinstance(error_data, dict):
-                    error_msg = error_data.get("message", str(e))
-            except (ValueError, KeyError):
-                pass
-
-            if status_code == 401:
-                raise HighspotAuthenticationError(f"Authentication failed: {error_msg}")
-            elif status_code == 429:
-                retry_after = e.response.headers.get("Retry-After")
-                raise HighspotRateLimitError(
-                    f"Rate limit exceeded: {error_msg}", retry_after=retry_after
-                )
-            else:
-                raise HighspotClientError(
-                    f"API error {status_code}: {error_msg}", status_code=status_code
-                )
-
-        except Timeout:
-            raise HighspotClientError("Request timed out")
-        except RequestException as e:
-            raise HighspotClientError(f"Request failed: {str(e)}")
-
-    def get_spots(self) -> List[Dict[str, Any]]:
-        """
-        Get all available spots.
-
-        Returns:
-            List of spots with their names and IDs
-        """
-        params = {"right": "view"}
-        response = self._make_request("GET", "spots", params=params)
-        logger.info(f"Received {response} spots")
-        total_counts = response.get("counts_total")
-        # Fix comparison to handle None value
-        if total_counts is not None and total_counts > 0:
-            return response.get("collection", [])
-        return []
-
-    def get_spot(self, spot_id: str) -> Dict[str, Any]:
-        """
-        Get details for a specific spot.
-
-        Args:
-            spot_id: ID of the spot
-
-        Returns:
-            Spot details
-        """
-        if not spot_id:
-            raise ValueError("spot_id is required")
-        return self._make_request("GET", f"spots/{spot_id}")
-
-    def get_spot_items(
-        self, spot_id: str, offset: int = 0, page_size: int = 100
-    ) -> Dict[str, Any]:
-        """
-        Get items in a specific spot.
-
-        Args:
-            spot_id: ID of the spot
-            offset: offset number
-            page_size: Number of items per page
-
-        Returns:
-            Items in the spot
-        """
-        if not spot_id:
-            raise ValueError("spot_id is required")
-
-        params = {"spot": spot_id, "start": offset, "limit": page_size}
-        return self._make_request("GET", "items", params=params)
-
-    def get_item(self, item_id: str) -> Dict[str, Any]:
-        """
-        Get details for a specific item.
-
-        Args:
-            item_id: ID of the item
-
-        Returns:
-            Item details
-        """
-        if not item_id:
-            raise ValueError("item_id is required")
-        return self._make_request("GET", f"items/{item_id}")
-
-    def get_item_content(self, item_id: str) -> bytes:
-        """
-        Get the raw content of an item.
-
-        Args:
-            item_id: ID of the item
-
-        Returns:
-            Raw content bytes
-        """
-        if not item_id:
-            raise ValueError("item_id is required")
-
-        url = urljoin(self.base_url, f"items/{item_id}/content")
-        response = self.session.get(url, timeout=self.timeout)
-        response.raise_for_status()
-        return response.content
-
-    def health_check(self) -> bool:
-        """
-        Check if the API is accessible and credentials are valid.
-
-        Returns:
-            True if API is accessible, False otherwise
-        """
-        try:
-            self._make_request("GET", "spots", params={"limit": 1})
-            return True
-        except (HighspotClientError, HighspotAuthenticationError):
-            return False

backend/onyx/connectors/highspot/connector.py

@@ -1,431 +0,0 @@
-from datetime import datetime
-from io import BytesIO
-from typing import Any
-from typing import Dict
-from typing import List
-from typing import Optional
-
-from onyx.configs.app_configs import INDEX_BATCH_SIZE
-from onyx.configs.constants import DocumentSource
-from onyx.connectors.highspot.client import HighspotClient
-from onyx.connectors.highspot.client import HighspotClientError
-from onyx.connectors.highspot.utils import scrape_url_content
-from onyx.connectors.interfaces import GenerateDocumentsOutput
-from onyx.connectors.interfaces import GenerateSlimDocumentOutput
-from onyx.connectors.interfaces import LoadConnector
-from onyx.connectors.interfaces import PollConnector
-from onyx.connectors.interfaces import SecondsSinceUnixEpoch
-from onyx.connectors.interfaces import SlimConnector
-from onyx.connectors.models import ConnectorMissingCredentialError
-from onyx.connectors.models import Document
-from onyx.connectors.models import SlimDocument
-from onyx.connectors.models import TextSection
-from onyx.file_processing.extract_file_text import extract_file_text
-from onyx.file_processing.extract_file_text import VALID_FILE_EXTENSIONS
-from onyx.indexing.indexing_heartbeat import IndexingHeartbeatInterface
-from onyx.utils.logger import setup_logger
-
-logger = setup_logger()
-_SLIM_BATCH_SIZE = 1000
-
-
-class HighspotConnector(LoadConnector, PollConnector, SlimConnector):
-    """
-    Connector for loading data from Highspot.
-
-    Retrieves content from specified spots using the Highspot API.
-    If no spots are specified, retrieves content from all available spots.
-    """
-
-    def __init__(
-        self,
-        spot_names: List[str] = [],
-        batch_size: int = INDEX_BATCH_SIZE,
-    ):
-        """
-        Initialize the Highspot connector.
-
-        Args:
-            spot_names: List of spot names to retrieve content from (if empty, gets all spots)
-            batch_size: Number of items to retrieve in each batch
-        """
-        self.spot_names = spot_names
-        self.batch_size = batch_size
-        self._client: Optional[HighspotClient] = None
-        self._spot_id_map: Dict[str, str] = {}  # Maps spot names to spot IDs
-        self._all_spots_fetched = False
-        self.highspot_url: Optional[str] = None
-        self.key: Optional[str] = None
-        self.secret: Optional[str] = None
-
-    @property
-    def client(self) -> HighspotClient:
-        if self._client is None:
-            if not self.key or not self.secret:
-                raise ConnectorMissingCredentialError("Highspot")
-            # Ensure highspot_url is a string, use default if None
-            base_url = (
-                self.highspot_url
-                if self.highspot_url is not None
-                else HighspotClient.BASE_URL
-            )
-            self._client = HighspotClient(self.key, self.secret, base_url=base_url)
-        return self._client
-
-    def load_credentials(self, credentials: dict[str, Any]) -> dict[str, Any] | None:
-        logger.info("Loading Highspot credentials")
-        self.highspot_url = credentials.get("highspot_url")
-        self.key = credentials.get("highspot_key")
-        self.secret = credentials.get("highspot_secret")
-        return None
-
-    def _populate_spot_id_map(self) -> None:
-        """
-        Populate the spot ID map with all available spots.
-        Keys are stored as lowercase for case-insensitive lookups.
-        """
-        spots = self.client.get_spots()
-        for spot in spots:
-            if "title" in spot and "id" in spot:
-                spot_name = spot["title"]
-                self._spot_id_map[spot_name.lower()] = spot["id"]
-
-        self._all_spots_fetched = True
-        logger.info(f"Retrieved {len(self._spot_id_map)} spots from Highspot")
-
-    def _get_all_spot_names(self) -> List[str]:
-        """
-        Retrieve all available spot names.
-
-        Returns:
-            List of all spot names
-        """
-        if not self._all_spots_fetched:
-            self._populate_spot_id_map()
-
-        return [spot_name for spot_name in self._spot_id_map.keys()]
-
-    def _get_spot_id_from_name(self, spot_name: str) -> str:
-        """
-        Get spot ID from a spot name.
-
-        Args:
-            spot_name: Name of the spot
-
-        Returns:
-            ID of the spot
-
-        Raises:
-            ValueError: If spot name is not found
-        """
-        if not self._all_spots_fetched:
-            self._populate_spot_id_map()
-
-        spot_name_lower = spot_name.lower()
-        if spot_name_lower not in self._spot_id_map:
-            raise ValueError(f"Spot '{spot_name}' not found")
-
-        return self._spot_id_map[spot_name_lower]
-
-    def load_from_state(self) -> GenerateDocumentsOutput:
-        """
-        Load content from configured spots in Highspot.
-        If no spots are configured, loads from all spots.
-
-        Yields:
-            Batches of Document objects
-        """
-        return self.poll_source(None, None)
-
-    def poll_source(
-        self, start: SecondsSinceUnixEpoch | None, end: SecondsSinceUnixEpoch | None
-    ) -> GenerateDocumentsOutput:
-        """
-        Poll Highspot for content updated since the start time.
-
-        Args:
-            start: Start time as seconds since Unix epoch
-            end: End time as seconds since Unix epoch
-
-        Yields:
-            Batches of Document objects
-        """
-        doc_batch: list[Document] = []
-
-        # If no spots specified, get all spots
-        spot_names_to_process = self.spot_names
-        if not spot_names_to_process:
-            spot_names_to_process = self._get_all_spot_names()
-            logger.info(
-                f"No spots specified, using all {len(spot_names_to_process)} available spots"
-            )
-
-        for spot_name in spot_names_to_process:
-            try:
-                spot_id = self._get_spot_id_from_name(spot_name)
-                if spot_id is None:
-                    logger.warning(f"Spot ID not found for spot {spot_name}")
-                    continue
-                offset = 0
-                has_more = True
-
-                while has_more:
-                    logger.info(
-                        f"Retrieving items from spot {spot_name}, offset {offset}"
-                    )
-                    response = self.client.get_spot_items(
-                        spot_id=spot_id, offset=offset, page_size=self.batch_size
-                    )
-                    items = response.get("collection", [])
-                    logger.info(f"Received Items: {items}")
-                    if not items:
-                        has_more = False
-                        continue
-
-                    for item in items:
-                        try:
-                            item_id = item.get("id")
-                            if not item_id:
-                                logger.warning("Item without ID found, skipping")
-                                continue
-
-                            item_details = self.client.get_item(item_id)
-                            if not item_details:
-                                logger.warning(
-                                    f"Item {item_id} details not found, skipping"
-                                )
-                                continue
-                            # Apply time filter if specified
-                            if start or end:
-                                updated_at = item_details.get("date_updated")
-                                if updated_at:
-                                    # Convert to datetime for comparison
-                                    try:
-                                        updated_time = datetime.fromisoformat(
-                                            updated_at.replace("Z", "+00:00")
-                                        )
-                                        if (
-                                            start and updated_time.timestamp() < start
-                                        ) or (end and updated_time.timestamp() > end):
-                                            continue
-                                    except (ValueError, TypeError):
-                                        # Skip if date cannot be parsed
-                                        logger.warning(
-                                            f"Invalid date format for item {item_id}: {updated_at}"
-                                        )
-                                        continue
-
-                            content = self._get_item_content(item_details)
-                            title = item_details.get("title", "")
-
-                            doc_batch.append(
-                                Document(
-                                    id=f"HIGHSPOT_{item_id}",
-                                    sections=[
-                                        TextSection(
-                                            link=item_details.get(
-                                                "url",
-                                                f"https://www.highspot.com/items/{item_id}",
-                                            ),
-                                            text=content,
-                                        )
-                                    ],
-                                    source=DocumentSource.HIGHSPOT,
-                                    semantic_identifier=title,
-                                    metadata={
-                                        "spot_name": spot_name,
-                                        "type": item_details.get("content_type", ""),
-                                        "created_at": item_details.get(
-                                            "date_added", ""
-                                        ),
-                                        "author": item_details.get("author", ""),
-                                        "language": item_details.get("language", ""),
-                                        "can_download": str(
-                                            item_details.get("can_download", False)
-                                        ),
-                                    },
-                                    doc_updated_at=item_details.get("date_updated"),
-                                )
-                            )
-
-                            if len(doc_batch) >= self.batch_size:
-                                yield doc_batch
-                                doc_batch = []
-
-                        except HighspotClientError as e:
-                            item_id = "ID" if not item_id else item_id
-                            logger.error(f"Error retrieving item {item_id}: {str(e)}")
-
-                    has_more = len(items) >= self.batch_size
-                    offset += self.batch_size
-
-            except (HighspotClientError, ValueError) as e:
-                logger.error(f"Error processing spot {spot_name}: {str(e)}")
-
-        if doc_batch:
-            yield doc_batch
-
-    def _get_item_content(self, item_details: Dict[str, Any]) -> str:
-        """
-        Get the text content of an item.
-
-        Args:
-            item_details: Item details from the API
-
-        Returns:
-            Text content of the item
-        """
-        item_id = item_details.get("id", "")
-        content_name = item_details.get("content_name", "")
-        is_valid_format = content_name and "." in content_name
-        file_extension = content_name.split(".")[-1].lower() if is_valid_format else ""
-        file_extension = "." + file_extension if file_extension else ""
-        can_download = item_details.get("can_download", False)
-        content_type = item_details.get("content_type", "")
-
-        # Extract title and description once at the beginning
-        title, description = self._extract_title_and_description(item_details)
-        default_content = f"{title}\n{description}"
-        logger.info(f"Processing item {item_id} with extension {file_extension}")
-
-        try:
-            if content_type == "WebLink":
-                url = item_details.get("url")
-                if not url:
-                    return default_content
-                content = scrape_url_content(url, True)
-                return content if content else default_content
-
-            elif (
-                is_valid_format
-                and file_extension in VALID_FILE_EXTENSIONS
-                and can_download
-            ):
-                # For documents, try to get the text content
-                if not item_id:  # Ensure item_id is defined
-                    return default_content
-
-                content_response = self.client.get_item_content(item_id)
-                # Process and extract text from binary content based on type
-                if content_response:
-                    text_content = extract_file_text(
-                        BytesIO(content_response), content_name
-                    )
-                    return text_content
-                return default_content
-
-            else:
-                return default_content
-
-        except HighspotClientError as e:
-            # Use item_id safely in the warning message
-            error_context = f"item {item_id}" if item_id else "item"
-            logger.warning(f"Could not retrieve content for {error_context}: {str(e)}")
-            return ""
-
-    def _extract_title_and_description(
-        self, item_details: Dict[str, Any]
-    ) -> tuple[str, str]:
-        """
-        Extract the title and description from item details.
-
-        Args:
-            item_details: Item details from the API
-
-        Returns:
-            Tuple of title and description
-        """
-        title = item_details.get("title", "")
-        description = item_details.get("description", "")
-        return title, description
-
-    def retrieve_all_slim_documents(
-        self,
-        start: SecondsSinceUnixEpoch | None = None,
-        end: SecondsSinceUnixEpoch | None = None,
-        callback: IndexingHeartbeatInterface | None = None,
-    ) -> GenerateSlimDocumentOutput:
-        """
-        Retrieve all document IDs from the configured spots.
-        If no spots are configured, retrieves from all spots.
-
-        Args:
-            start: Optional start time filter
-            end: Optional end time filter
-            callback: Optional indexing heartbeat callback
-
-        Yields:
-            Batches of SlimDocument objects
-        """
-        slim_doc_batch: list[SlimDocument] = []
-
-        # If no spots specified, get all spots
-        spot_names_to_process = self.spot_names
-        if not spot_names_to_process:
-            spot_names_to_process = self._get_all_spot_names()
-            logger.info(
-                f"No spots specified, using all {len(spot_names_to_process)} available spots for slim documents"
-            )
-
-        for spot_name in spot_names_to_process:
-            try:
-                spot_id = self._get_spot_id_from_name(spot_name)
-                offset = 0
-                has_more = True
-
-                while has_more:
-                    logger.info(
-                        f"Retrieving slim documents from spot {spot_name}, offset {offset}"
-                    )
-                    response = self.client.get_spot_items(
-                        spot_id=spot_id, offset=offset, page_size=self.batch_size
-                    )
-
-                    items = response.get("collection", [])
-                    if not items:
-                        has_more = False
-                        continue
-
-                    for item in items:
-                        item_id = item.get("id")
-                        if not item_id:
-                            continue
-
-                        slim_doc_batch.append(SlimDocument(id=f"HIGHSPOT_{item_id}"))
-
-                        if len(slim_doc_batch) >= _SLIM_BATCH_SIZE:
-                            yield slim_doc_batch
-                            slim_doc_batch = []
-
-                    has_more = len(items) >= self.batch_size
-                    offset += self.batch_size
-
-            except (HighspotClientError, ValueError) as e:
-                logger.error(
-                    f"Error retrieving slim documents from spot {spot_name}: {str(e)}"
-                )
-
-        if slim_doc_batch:
-            yield slim_doc_batch
-
-    def validate_credentials(self) -> bool:
-        """
-        Validate that the provided credentials can access the Highspot API.
-
-        Returns:
-            True if credentials are valid, False otherwise
-        """
-        try:
-            return self.client.health_check()
-        except Exception as e:
-            logger.error(f"Failed to validate credentials: {str(e)}")
-            return False
-
-
-if __name__ == "__main__":
-    spot_names: List[str] = []
-    connector = HighspotConnector(spot_names)
-    credentials = {"highspot_key": "", "highspot_secret": ""}
-    connector.load_credentials(credentials=credentials)
-    for doc in connector.load_from_state():
-        print(doc)

backend/onyx/connectors/highspot/utils.py

@@ -1,122 +0,0 @@
-from typing import Optional
-from urllib.parse import urlparse
-
-from bs4 import BeautifulSoup
-from playwright.sync_api import sync_playwright
-
-from onyx.file_processing.html_utils import web_html_cleanup
-from onyx.utils.logger import setup_logger
-
-logger = setup_logger()
-
-# Constants
-WEB_CONNECTOR_MAX_SCROLL_ATTEMPTS = 20
-JAVASCRIPT_DISABLED_MESSAGE = "You have JavaScript disabled in your browser"
-DEFAULT_TIMEOUT = 60000  # 60 seconds
-
-
-def scrape_url_content(
-    url: str, scroll_before_scraping: bool = False, timeout_ms: int = DEFAULT_TIMEOUT
-) -> Optional[str]:
-    """
-    Scrapes content from a given URL and returns the cleaned text.
-
-    Args:
-        url: The URL to scrape
-        scroll_before_scraping: Whether to scroll through the page to load lazy content
-        timeout_ms: Timeout in milliseconds for page navigation and loading
-
-    Returns:
-        The cleaned text content of the page or None if scraping fails
-    """
-    playwright = None
-    browser = None
-    try:
-        validate_url(url)
-        playwright = sync_playwright().start()
-        browser = playwright.chromium.launch(headless=True)
-        context = browser.new_context()
-        page = context.new_page()
-
-        logger.info(f"Navigating to URL: {url}")
-        try:
-            page.goto(url, timeout=timeout_ms)
-        except Exception as e:
-            logger.error(f"Failed to navigate to {url}: {str(e)}")
-            return None
-
-        if scroll_before_scraping:
-            logger.debug("Scrolling page to load lazy content")
-            scroll_attempts = 0
-            previous_height = page.evaluate("document.body.scrollHeight")
-            while scroll_attempts < WEB_CONNECTOR_MAX_SCROLL_ATTEMPTS:
-                page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
-                try:
-                    page.wait_for_load_state("networkidle", timeout=timeout_ms)
-                except Exception as e:
-                    logger.warning(f"Network idle wait timed out: {str(e)}")
-                    break
-
-                new_height = page.evaluate("document.body.scrollHeight")
-                if new_height == previous_height:
-                    break
-                previous_height = new_height
-                scroll_attempts += 1
-
-        content = page.content()
-        soup = BeautifulSoup(content, "html.parser")
-
-        parsed_html = web_html_cleanup(soup)
-
-        if JAVASCRIPT_DISABLED_MESSAGE in parsed_html.cleaned_text:
-            logger.debug("JavaScript disabled message detected, checking iframes")
-            try:
-                iframe_count = page.frame_locator("iframe").locator("html").count()
-                if iframe_count > 0:
-                    iframe_texts = (
-                        page.frame_locator("iframe").locator("html").all_inner_texts()
-                    )
-                    iframe_content = "\n".join(iframe_texts)
-
-                    if len(parsed_html.cleaned_text) < 700:
-                        parsed_html.cleaned_text = iframe_content
-                    else:
-                        parsed_html.cleaned_text += "\n" + iframe_content
-            except Exception as e:
-                logger.warning(f"Error processing iframes: {str(e)}")
-
-        return parsed_html.cleaned_text
-
-    except Exception as e:
-        logger.error(f"Error scraping URL {url}: {str(e)}")
-        return None
-
-    finally:
-        if browser:
-            try:
-                browser.close()
-            except Exception as e:
-                logger.debug(f"Error closing browser: {str(e)}")
-        if playwright:
-            try:
-                playwright.stop()
-            except Exception as e:
-                logger.debug(f"Error stopping playwright: {str(e)}")
-
-
-def validate_url(url: str) -> None:
-    """
-    Validates that a URL is properly formatted.
-
-    Args:
-        url: The URL to validate
-
-    Raises:
-        ValueError: If URL is not valid
-    """
-    parse = urlparse(url)
-    if parse.scheme != "http" and parse.scheme != "https":
-        raise ValueError("URL must be of scheme https?://")
-
-    if not parse.hostname:
-        raise ValueError("URL must include a hostname")

backend/onyx/connectors/hubspot/connector.py

@@ -13,7 +13,7 @@ from onyx.connectors.interfaces import PollConnector
 from onyx.connectors.interfaces import SecondsSinceUnixEpoch
 from onyx.connectors.models import ConnectorMissingCredentialError
 from onyx.connectors.models import Document
-from onyx.connectors.models import TextSection
+from onyx.connectors.models import Section
 from onyx.utils.logger import setup_logger
 
 HUBSPOT_BASE_URL = "https://app.hubspot.com/contacts/"
@@ -108,7 +108,7 @@ class HubSpotConnector(LoadConnector, PollConnector):
             doc_batch.append(
                 Document(
                     id=ticket.id,
-                    sections=[TextSection(link=link, text=content_text)],
+                    sections=[Section(link=link, text=content_text)],
                     source=DocumentSource.HUBSPOT,
                     semantic_identifier=title,
                     # Is already in tzutc, just replacing the timezone format

backend/onyx/connectors/interfaces.py

@@ -4,11 +4,9 @@ from collections.abc import Iterator
 from types import TracebackType
 from typing import Any
 from typing import Generic
-from typing import TypeAlias
 from typing import TypeVar
 
 from pydantic import BaseModel
-from typing_extensions import override
 
 from onyx.configs.constants import DocumentSource
 from onyx.connectors.models import ConnectorCheckpoint
@@ -21,14 +19,11 @@ SecondsSinceUnixEpoch = float
 
 GenerateDocumentsOutput = Iterator[list[Document]]
 GenerateSlimDocumentOutput = Iterator[list[SlimDocument]]
-
-CT = TypeVar("CT", bound=ConnectorCheckpoint)
+CheckpointOutput = Generator[Document | ConnectorFailure, None, ConnectorCheckpoint]
 
 
-class BaseConnector(abc.ABC, Generic[CT]):
+class BaseConnector(abc.ABC):
     REDIS_KEY_PREFIX = "da_connector_data:"
-    # Common image file extensions supported across connectors
-    IMAGE_EXTENSIONS = {".jpg", ".jpeg", ".png", ".webp", ".gif"}
 
     @abc.abstractmethod
     def load_credentials(self, credentials: dict[str, Any]) -> dict[str, Any] | None:
@@ -60,14 +55,6 @@ class BaseConnector(abc.ABC, Generic[CT]):
         Default is a no-op (always successful).
         """
 
-    def set_allow_images(self, value: bool) -> None:
-        """Implement if the underlying connector wants to skip/allow image downloading
-        based on the application level image analysis setting."""
-
-    def build_dummy_checkpoint(self) -> CT:
-        # TODO: find a way to make this work without type: ignore
-        return ConnectorCheckpoint(has_more=True)  # type: ignore
-
 
 # Large set update or reindex, generally pulling a complete state or from a savestate file
 class LoadConnector(BaseConnector):
@@ -85,8 +72,6 @@ class PollConnector(BaseConnector):
         raise NotImplementedError
 
 
-# Slim connectors can retrieve just the ids and
-# permission syncing information for connected documents
 class SlimConnector(BaseConnector):
     @abc.abstractmethod
     def retrieve_all_slim_documents(
@@ -199,17 +184,14 @@ class EventConnector(BaseConnector):
         raise NotImplementedError
 
 
-CheckpointOutput: TypeAlias = Generator[Document | ConnectorFailure, None, CT]
-
-
-class CheckpointConnector(BaseConnector[CT]):
+class CheckpointConnector(BaseConnector):
     @abc.abstractmethod
     def load_from_checkpoint(
         self,
         start: SecondsSinceUnixEpoch,
         end: SecondsSinceUnixEpoch,
-        checkpoint: CT,
-    ) -> CheckpointOutput[CT]:
+        checkpoint: ConnectorCheckpoint,
+    ) -> CheckpointOutput:
         """Yields back documents or failures. Final return is the new checkpoint.
 
         Final return can be access via either:
@@ -230,12 +212,3 @@ class CheckpointConnector(BaseConnector[CT]):
         ```
         """
         raise NotImplementedError
-
-    @override
-    def build_dummy_checkpoint(self) -> CT:
-        raise NotImplementedError
-
-    @abc.abstractmethod
-    def validate_checkpoint_json(self, checkpoint_json: str) -> CT:
-        """Validate the checkpoint json and return the checkpoint object"""
-        raise NotImplementedError

backend/onyx/connectors/linear/connector.py

@@ -21,8 +21,7 @@ from onyx.connectors.interfaces import PollConnector
 from onyx.connectors.interfaces import SecondsSinceUnixEpoch
 from onyx.connectors.models import ConnectorMissingCredentialError
 from onyx.connectors.models import Document
-from onyx.connectors.models import ImageSection
-from onyx.connectors.models import TextSection
+from onyx.connectors.models import Section
 from onyx.utils.logger import setup_logger
 from onyx.utils.retry_wrapper import request_with_retries
 
@@ -238,30 +237,22 @@ class LinearConnector(LoadConnector, PollConnector, OAuthConnector):
             documents: list[Document] = []
             for edge in edges:
                 node = edge["node"]
-                # Create sections for description and comments
-                sections = [
-                    TextSection(
-                        link=node["url"],
-                        text=node["description"] or "",
-                    )
-                ]
-
-                # Add comment sections
-                for comment in node["comments"]["nodes"]:
-                    sections.append(
-                        TextSection(
-                            link=node["url"],
-                            text=comment["body"] or "",
-                        )
-                    )
-
-                # Cast the sections list to the expected type
-                typed_sections = cast(list[TextSection | ImageSection], sections)
-
                 documents.append(
                     Document(
                         id=node["id"],
-                        sections=typed_sections,
+                        sections=[
+                            Section(
+                                link=node["url"],
+                                text=node["description"] or "",
+                            )
+                        ]
+                        + [
+                            Section(
+                                link=node["url"],
+                                text=comment["body"] or "",
+                            )
+                            for comment in node["comments"]["nodes"]
+                        ],
                         source=DocumentSource.LINEAR,
                         semantic_identifier=f"[{node['identifier']}] {node['title']}",
                         title=node["title"],

backend/onyx/connectors/loopio/connector.py

@@ -17,7 +17,7 @@ from onyx.connectors.interfaces import SecondsSinceUnixEpoch
 from onyx.connectors.models import BasicExpertInfo
 from onyx.connectors.models import ConnectorMissingCredentialError
 from onyx.connectors.models import Document
-from onyx.connectors.models import TextSection
+from onyx.connectors.models import Section
 from onyx.file_processing.html_utils import parse_html_page_basic
 from onyx.file_processing.html_utils import strip_excessive_newlines_and_spaces
 from onyx.utils.logger import setup_logger
@@ -162,7 +162,7 @@ class LoopioConnector(LoadConnector, PollConnector):
                 doc_batch.append(
                     Document(
                         id=str(entry["id"]),
-                        sections=[TextSection(link=link, text=content_text)],
+                        sections=[Section(link=link, text=content_text)],
                         source=DocumentSource.LOOPIO,
                         semantic_identifier=questions[0],
                         doc_updated_at=latest_time,

backend/onyx/connectors/mediawiki/wiki.py

@@ -6,7 +6,6 @@ import tempfile
 from collections.abc import Generator
 from collections.abc import Iterator
 from typing import Any
-from typing import cast
 from typing import ClassVar
 
 import pywikibot.time  # type: ignore[import-untyped]
@@ -21,8 +20,7 @@ from onyx.connectors.interfaces import PollConnector
 from onyx.connectors.interfaces import SecondsSinceUnixEpoch
 from onyx.connectors.mediawiki.family import family_class_dispatch
 from onyx.connectors.models import Document
-from onyx.connectors.models import ImageSection
-from onyx.connectors.models import TextSection
+from onyx.connectors.models import Section
 from onyx.utils.logger import setup_logger
 
 
@@ -62,14 +60,14 @@ def get_doc_from_page(
     sections_extracted: textlib.Content = textlib.extract_sections(page_text, site)
 
     sections = [
-        TextSection(
+        Section(
             link=f"{page.full_url()}#" + section.heading.replace(" ", "_"),
             text=section.title + section.content,
         )
         for section in sections_extracted.sections
     ]
     sections.append(
-        TextSection(
+        Section(
             link=page.full_url(),
             text=sections_extracted.header,
         )
@@ -81,7 +79,7 @@ def get_doc_from_page(
         doc_updated_at=pywikibot_timestamp_to_utc_datetime(
             page.latest_revision.timestamp
         ),
-        sections=cast(list[TextSection | ImageSection], sections),
+        sections=sections,
         semantic_identifier=page.title(),
         metadata={"categories": [category.title() for category in page.categories()]},
         id=f"MEDIAWIKI_{page.pageid}_{page.full_url()}",

backend/onyx/connectors/mock_connector/connector.py

@@ -2,7 +2,6 @@ from typing import Any
 
 import httpx
 from pydantic import BaseModel
-from typing_extensions import override
 
 from onyx.connectors.interfaces import CheckpointConnector
 from onyx.connectors.interfaces import CheckpointOutput
@@ -16,18 +15,14 @@ from onyx.utils.logger import setup_logger
 logger = setup_logger()
 
 
-class MockConnectorCheckpoint(ConnectorCheckpoint):
-    last_document_id: str | None = None
-
-
 class SingleConnectorYield(BaseModel):
     documents: list[Document]
-    checkpoint: MockConnectorCheckpoint
+    checkpoint: ConnectorCheckpoint
     failures: list[ConnectorFailure]
     unhandled_exception: str | None = None
 
 
-class MockConnector(CheckpointConnector[MockConnectorCheckpoint]):
+class MockConnector(CheckpointConnector):
     def __init__(
         self,
         mock_server_host: str,
@@ -53,7 +48,7 @@ class MockConnector(CheckpointConnector[MockConnectorCheckpoint]):
     def _get_mock_server_url(self, endpoint: str) -> str:
         return f"http://{self.mock_server_host}:{self.mock_server_port}/{endpoint}"
 
-    def _save_checkpoint(self, checkpoint: MockConnectorCheckpoint) -> None:
+    def _save_checkpoint(self, checkpoint: ConnectorCheckpoint) -> None:
         response = self.client.post(
             self._get_mock_server_url("add-checkpoint"),
             json=checkpoint.model_dump(mode="json"),
@@ -64,8 +59,8 @@ class MockConnector(CheckpointConnector[MockConnectorCheckpoint]):
         self,
         start: SecondsSinceUnixEpoch,
         end: SecondsSinceUnixEpoch,
-        checkpoint: MockConnectorCheckpoint,
-    ) -> CheckpointOutput[MockConnectorCheckpoint]:
+        checkpoint: ConnectorCheckpoint,
+    ) -> CheckpointOutput:
         if self.connector_yields is None:
             raise ValueError("No connector yields configured")
 
@@ -89,13 +84,3 @@ class MockConnector(CheckpointConnector[MockConnectorCheckpoint]):
             yield failure
 
         return current_yield.checkpoint
-
-    @override
-    def build_dummy_checkpoint(self) -> MockConnectorCheckpoint:
-        return MockConnectorCheckpoint(
-            has_more=True,
-            last_document_id=None,
-        )
-
-    def validate_checkpoint_json(self, checkpoint_json: str) -> MockConnectorCheckpoint:
-        return MockConnectorCheckpoint.model_validate_json(checkpoint_json)

backend/onyx/connectors/models.py

@@ -27,25 +27,9 @@ class ConnectorMissingCredentialError(PermissionError):
 
 
 class Section(BaseModel):
-    """Base section class with common attributes"""
-
-    link: str | None = None
-    text: str | None = None
-    image_file_name: str | None = None
-
-
-class TextSection(Section):
-    """Section containing text content"""
-
     text: str
     link: str | None = None
-
-
-class ImageSection(Section):
-    """Section containing an image reference"""
-
-    image_file_name: str
-    link: str | None = None
+    image_file_name: str | None = None
 
 
 class BasicExpertInfo(BaseModel):
@@ -115,7 +99,7 @@ class DocumentBase(BaseModel):
     """Used for Onyx ingestion api, the ID is inferred before use if not provided"""
 
     id: str | None = None
-    sections: list[TextSection | ImageSection]
+    sections: list[Section]
     source: DocumentSource | None = None
     semantic_identifier: str  # displayed in the UI as the main identifier for the doc
     metadata: dict[str, str | list[str]]
@@ -165,11 +149,19 @@ class DocumentBase(BaseModel):
 
 
 class Document(DocumentBase):
-    """Used for Onyx ingestion api, the ID is required"""
-
-    id: str
+    id: str  # This must be unique or during indexing/reindexing, chunks will be overwritten
     source: DocumentSource
 
+    def get_total_char_length(self) -> int:
+        """Calculate the total character length of the document including sections, metadata, and identifiers."""
+        section_length = sum(len(section.text) for section in self.sections)
+        identifier_length = len(self.semantic_identifier) + len(self.title or "")
+        metadata_length = sum(
+            len(k) + len(v) if isinstance(v, str) else len(k) + sum(len(x) for x in v)
+            for k, v in self.metadata.items()
+        )
+        return section_length + identifier_length + metadata_length
+
     def to_short_descriptor(self) -> str:
         """Used when logging the identity of a document"""
         return f"ID: '{self.id}'; Semantic ID: '{self.semantic_identifier}'"
@@ -192,32 +184,6 @@ class Document(DocumentBase):
         )
 
 
-class IndexingDocument(Document):
-    """Document with processed sections for indexing"""
-
-    processed_sections: list[Section] = []
-
-    def get_total_char_length(self) -> int:
-        """Get the total character length of the document including processed sections"""
-        title_len = len(self.title or self.semantic_identifier)
-
-        # Use processed_sections if available, otherwise fall back to original sections
-        if self.processed_sections:
-            section_len = sum(
-                len(section.text) if section.text is not None else 0
-                for section in self.processed_sections
-            )
-        else:
-            section_len = sum(
-                len(section.text)
-                if isinstance(section, TextSection) and section.text is not None
-                else 0
-                for section in self.sections
-            )
-
-        return title_len + section_len
-
-
 class SlimDocument(BaseModel):
     id: str
     perm_sync_data: Any | None = None
@@ -231,16 +197,12 @@ class IndexAttemptMetadata(BaseModel):
 
 class ConnectorCheckpoint(BaseModel):
     # TODO: maybe move this to something disk-based to handle extremely large checkpoints?
+    checkpoint_content: dict
     has_more: bool
 
-    def __str__(self) -> str:
-        """String representation of the checkpoint, with truncation for large checkpoint content."""
-        MAX_CHECKPOINT_CONTENT_CHARS = 1000
-
-        content_str = self.model_dump_json()
-        if len(content_str) > MAX_CHECKPOINT_CONTENT_CHARS:
-            content_str = content_str[: MAX_CHECKPOINT_CONTENT_CHARS - 3] + "..."
-        return content_str
+    @classmethod
+    def build_dummy_checkpoint(cls) -> "ConnectorCheckpoint":
+        return ConnectorCheckpoint(checkpoint_content={}, has_more=True)
 
 
 class DocumentFailure(BaseModel):

backend/onyx/connectors/notion/connector.py

@@ -1,16 +1,17 @@
+import time
 from collections.abc import Generator
+from dataclasses import dataclass
+from dataclasses import fields
 from datetime import datetime
 from datetime import timezone
 from typing import Any
-from typing import cast
 from typing import Optional
 
 import requests
-from pydantic import BaseModel
 from retry import retry
 
 from onyx.configs.app_configs import INDEX_BATCH_SIZE
-from onyx.configs.app_configs import NOTION_CONNECTOR_DISABLE_RECURSIVE_PAGE_LOOKUP
+from onyx.configs.app_configs import NOTION_CONNECTOR_ENABLE_RECURSIVE_PAGE_LOOKUP
 from onyx.configs.constants import DocumentSource
 from onyx.connectors.cross_connector_utils.rate_limit_wrapper import (
     rl_requests,
@@ -25,21 +26,20 @@ from onyx.connectors.interfaces import PollConnector
 from onyx.connectors.interfaces import SecondsSinceUnixEpoch
 from onyx.connectors.models import ConnectorMissingCredentialError
 from onyx.connectors.models import Document
-from onyx.connectors.models import ImageSection
-from onyx.connectors.models import TextSection
+from onyx.connectors.models import Section
 from onyx.utils.batching import batch_generator
 from onyx.utils.logger import setup_logger
 
 logger = setup_logger()
 
-_NOTION_PAGE_SIZE = 100
 _NOTION_CALL_TIMEOUT = 30  # 30 seconds
 
 
 # TODO: Tables need to be ingested, Pages need to have their metadata ingested
 
 
-class NotionPage(BaseModel):
+@dataclass
+class NotionPage:
     """Represents a Notion Page object"""
 
     id: str
@@ -49,10 +49,17 @@ class NotionPage(BaseModel):
     properties: dict[str, Any]
     url: str
 
-    database_name: str | None = None  # Only applicable to the database type page (wiki)
+    database_name: str | None  # Only applicable to the database type page (wiki)
+
+    def __init__(self, **kwargs: dict[str, Any]) -> None:
+        names = set([f.name for f in fields(self)])
+        for k, v in kwargs.items():
+            if k in names:
+                setattr(self, k, v)
 
 
-class NotionBlock(BaseModel):
+@dataclass
+class NotionBlock:
     """Represents a Notion Block object"""
 
     id: str  # Used for the URL
@@ -62,13 +69,20 @@ class NotionBlock(BaseModel):
     prefix: str
 
 
-class NotionSearchResponse(BaseModel):
+@dataclass
+class NotionSearchResponse:
     """Represents the response from the Notion Search API"""
 
     results: list[dict[str, Any]]
     next_cursor: Optional[str]
     has_more: bool = False
 
+    def __init__(self, **kwargs: dict[str, Any]) -> None:
+        names = set([f.name for f in fields(self)])
+        for k, v in kwargs.items():
+            if k in names:
+                setattr(self, k, v)
+
 
 class NotionConnector(LoadConnector, PollConnector):
     """Notion Page connector that reads all Notion pages
@@ -81,7 +95,7 @@ class NotionConnector(LoadConnector, PollConnector):
     def __init__(
         self,
         batch_size: int = INDEX_BATCH_SIZE,
-        recursive_index_enabled: bool = not NOTION_CONNECTOR_DISABLE_RECURSIVE_PAGE_LOOKUP,
+        recursive_index_enabled: bool = NOTION_CONNECTOR_ENABLE_RECURSIVE_PAGE_LOOKUP,
         root_page_id: str | None = None,
     ) -> None:
         """Initialize with parameters."""
@@ -450,53 +464,23 @@ class NotionConnector(LoadConnector, PollConnector):
             page_blocks, child_page_ids = self._read_blocks(page.id)
             all_child_page_ids.extend(child_page_ids)
 
-            # okay to mark here since there's no way for this to not succeed
-            # without a critical failure
-            self.indexed_pages.add(page.id)
-
-            raw_page_title = self._read_page_title(page)
-            page_title = raw_page_title or f"Untitled Page with ID {page.id}"
-
             if not page_blocks:
-                if not raw_page_title:
-                    logger.warning(
-                        f"No blocks OR title found for page with ID '{page.id}'. Skipping."
-                    )
-                    continue
+                continue
 
-                logger.debug(f"No blocks found for page with ID '{page.id}'")
-                """
-                Something like:
-
-                TITLE
-
-                PROP1: PROP1_VALUE
-                PROP2: PROP2_VALUE
-                """
-                text = page_title
-                if page.properties:
-                    text += "\n\n" + "\n".join(
-                        [f"{key}: {value}" for key, value in page.properties.items()]
-                    )
-                sections = [
-                    TextSection(
-                        link=f"{page.url}",
-                        text=text,
-                    )
-                ]
-            else:
-                sections = [
-                    TextSection(
-                        link=f"{page.url}#{block.id.replace('-', '')}",
-                        text=block.prefix + block.text,
-                    )
-                    for block in page_blocks
-                ]
+            page_title = (
+                self._read_page_title(page) or f"Untitled Page with ID {page.id}"
+            )
 
             yield (
                 Document(
                     id=page.id,
-                    sections=cast(list[TextSection | ImageSection], sections),
+                    sections=[
+                        Section(
+                            link=f"{page.url}#{block.id.replace('-', '')}",
+                            text=block.prefix + block.text,
+                        )
+                        for block in page_blocks
+                    ],
                     source=DocumentSource.NOTION,
                     semantic_identifier=page_title,
                     doc_updated_at=datetime.fromisoformat(
@@ -553,9 +537,9 @@ class NotionConnector(LoadConnector, PollConnector):
         """
         filtered_pages: list[NotionPage] = []
         for page in pages:
-            # Parse ISO 8601 timestamp and convert to UTC epoch time
-            timestamp = page[filter_field].replace(".000Z", "+00:00")
-            compare_time = datetime.fromisoformat(timestamp).timestamp()
+            compare_time = time.mktime(
+                time.strptime(page[filter_field], "%Y-%m-%dT%H:%M:%S.000Z")
+            )
             if compare_time > start and compare_time <= end:
                 filtered_pages += [NotionPage(**page)]
         return filtered_pages
@@ -594,7 +578,7 @@ class NotionConnector(LoadConnector, PollConnector):
 
         query_dict = {
             "filter": {"property": "object", "value": "page"},
-            "page_size": _NOTION_PAGE_SIZE,
+            "page_size": self.batch_size,
         }
         while True:
             db_res = self._search_notion(query_dict)
@@ -620,7 +604,7 @@ class NotionConnector(LoadConnector, PollConnector):
             return
 
         query_dict = {
-            "page_size": _NOTION_PAGE_SIZE,
+            "page_size": self.batch_size,
             "sort": {"timestamp": "last_edited_time", "direction": "descending"},
             "filter": {"property": "object", "value": "page"},
         }

backend/onyx/connectors/onyx_jira/connector.py

@@ -6,7 +6,6 @@ from typing import Any
 
 from jira import JIRA
 from jira.resources import Issue
-from typing_extensions import override
 
 from onyx.configs.app_configs import INDEX_BATCH_SIZE
 from onyx.configs.app_configs import JIRA_CONNECTOR_LABELS_TO_SKIP
@@ -16,18 +15,16 @@ from onyx.connectors.cross_connector_utils.miscellaneous_utils import time_str_t
 from onyx.connectors.exceptions import ConnectorValidationError
 from onyx.connectors.exceptions import CredentialExpiredError
 from onyx.connectors.exceptions import InsufficientPermissionsError
-from onyx.connectors.interfaces import CheckpointConnector
-from onyx.connectors.interfaces import CheckpointOutput
+from onyx.connectors.interfaces import GenerateDocumentsOutput
 from onyx.connectors.interfaces import GenerateSlimDocumentOutput
+from onyx.connectors.interfaces import LoadConnector
+from onyx.connectors.interfaces import PollConnector
 from onyx.connectors.interfaces import SecondsSinceUnixEpoch
 from onyx.connectors.interfaces import SlimConnector
-from onyx.connectors.models import ConnectorCheckpoint
-from onyx.connectors.models import ConnectorFailure
 from onyx.connectors.models import ConnectorMissingCredentialError
 from onyx.connectors.models import Document
-from onyx.connectors.models import DocumentFailure
+from onyx.connectors.models import Section
 from onyx.connectors.models import SlimDocument
-from onyx.connectors.models import TextSection
 from onyx.connectors.onyx_jira.utils import best_effort_basic_expert_info
 from onyx.connectors.onyx_jira.utils import best_effort_get_field_from_issue
 from onyx.connectors.onyx_jira.utils import build_jira_client
@@ -45,112 +42,121 @@ _JIRA_SLIM_PAGE_SIZE = 500
 _JIRA_FULL_PAGE_SIZE = 50
 
 
-def _perform_jql_search(
+def _paginate_jql_search(
     jira_client: JIRA,
     jql: str,
-    start: int,
     max_results: int,
     fields: str | None = None,
 ) -> Iterable[Issue]:
-    logger.debug(
-        f"Fetching Jira issues with JQL: {jql}, "
-        f"starting at {start}, max results: {max_results}"
-    )
-    issues = jira_client.search_issues(
-        jql_str=jql,
-        startAt=start,
-        maxResults=max_results,
-        fields=fields,
-    )
+    start = 0
+    while True:
+        logger.debug(
+            f"Fetching Jira issues with JQL: {jql}, "
+            f"starting at {start}, max results: {max_results}"
+        )
+        issues = jira_client.search_issues(
+            jql_str=jql,
+            startAt=start,
+            maxResults=max_results,
+            fields=fields,
+        )
 
-    for issue in issues:
-        if isinstance(issue, Issue):
-            yield issue
-        else:
-            raise RuntimeError(f"Found Jira object not of type Issue: {issue}")
+        for issue in issues:
+            if isinstance(issue, Issue):
+                yield issue
+            else:
+                raise Exception(f"Found Jira object not of type Issue: {issue}")
+
+        if len(issues) < max_results:
+            break
+
+        start += max_results
 
 
-def process_jira_issue(
+def fetch_jira_issues_batch(
     jira_client: JIRA,
-    issue: Issue,
+    jql: str,
+    batch_size: int,
     comment_email_blacklist: tuple[str, ...] = (),
     labels_to_skip: set[str] | None = None,
-) -> Document | None:
-    if labels_to_skip:
-        if any(label in issue.fields.labels for label in labels_to_skip):
-            logger.info(
-                f"Skipping {issue.key} because it has a label to skip. Found "
-                f"labels: {issue.fields.labels}. Labels to skip: {labels_to_skip}."
-            )
-            return None
+) -> Iterable[Document]:
+    for issue in _paginate_jql_search(
+        jira_client=jira_client,
+        jql=jql,
+        max_results=batch_size,
+    ):
+        if labels_to_skip:
+            if any(label in issue.fields.labels for label in labels_to_skip):
+                logger.info(
+                    f"Skipping {issue.key} because it has a label to skip. Found "
+                    f"labels: {issue.fields.labels}. Labels to skip: {labels_to_skip}."
+                )
+                continue
 
-    description = (
-        issue.fields.description
-        if JIRA_API_VERSION == "2"
-        else extract_text_from_adf(issue.raw["fields"]["description"])
-    )
-    comments = get_comment_strs(
-        issue=issue,
-        comment_email_blacklist=comment_email_blacklist,
-    )
-    ticket_content = f"{description}\n" + "\n".join(
-        [f"Comment: {comment}" for comment in comments if comment]
-    )
-
-    # Check ticket size
-    if len(ticket_content.encode("utf-8")) > JIRA_CONNECTOR_MAX_TICKET_SIZE:
-        logger.info(
-            f"Skipping {issue.key} because it exceeds the maximum size of "
-            f"{JIRA_CONNECTOR_MAX_TICKET_SIZE} bytes."
+        description = (
+            issue.fields.description
+            if JIRA_API_VERSION == "2"
+            else extract_text_from_adf(issue.raw["fields"]["description"])
+        )
+        comments = get_comment_strs(
+            issue=issue,
+            comment_email_blacklist=comment_email_blacklist,
+        )
+        ticket_content = f"{description}\n" + "\n".join(
+            [f"Comment: {comment}" for comment in comments if comment]
         )
-        return None
 
-    page_url = build_jira_url(jira_client, issue.key)
+        # Check ticket size
+        if len(ticket_content.encode("utf-8")) > JIRA_CONNECTOR_MAX_TICKET_SIZE:
+            logger.info(
+                f"Skipping {issue.key} because it exceeds the maximum size of "
+                f"{JIRA_CONNECTOR_MAX_TICKET_SIZE} bytes."
+            )
+            continue
 
-    people = set()
-    try:
-        creator = best_effort_get_field_from_issue(issue, "creator")
-        if basic_expert_info := best_effort_basic_expert_info(creator):
-            people.add(basic_expert_info)
-    except Exception:
-        # Author should exist but if not, doesn't matter
-        pass
+        page_url = f"{jira_client.client_info()}/browse/{issue.key}"
 
-    try:
-        assignee = best_effort_get_field_from_issue(issue, "assignee")
-        if basic_expert_info := best_effort_basic_expert_info(assignee):
-            people.add(basic_expert_info)
-    except Exception:
-        # Author should exist but if not, doesn't matter
-        pass
+        people = set()
+        try:
+            creator = best_effort_get_field_from_issue(issue, "creator")
+            if basic_expert_info := best_effort_basic_expert_info(creator):
+                people.add(basic_expert_info)
+        except Exception:
+            # Author should exist but if not, doesn't matter
+            pass
 
-    metadata_dict = {}
-    if priority := best_effort_get_field_from_issue(issue, "priority"):
-        metadata_dict["priority"] = priority.name
-    if status := best_effort_get_field_from_issue(issue, "status"):
-        metadata_dict["status"] = status.name
-    if resolution := best_effort_get_field_from_issue(issue, "resolution"):
-        metadata_dict["resolution"] = resolution.name
-    if labels := best_effort_get_field_from_issue(issue, "labels"):
-        metadata_dict["labels"] = labels
+        try:
+            assignee = best_effort_get_field_from_issue(issue, "assignee")
+            if basic_expert_info := best_effort_basic_expert_info(assignee):
+                people.add(basic_expert_info)
+        except Exception:
+            # Author should exist but if not, doesn't matter
+            pass
 
-    return Document(
-        id=page_url,
-        sections=[TextSection(link=page_url, text=ticket_content)],
-        source=DocumentSource.JIRA,
-        semantic_identifier=f"{issue.key}: {issue.fields.summary}",
-        title=f"{issue.key} {issue.fields.summary}",
-        doc_updated_at=time_str_to_utc(issue.fields.updated),
-        primary_owners=list(people) or None,
-        metadata=metadata_dict,
-    )
+        metadata_dict = {}
+        if priority := best_effort_get_field_from_issue(issue, "priority"):
+            metadata_dict["priority"] = priority.name
+        if status := best_effort_get_field_from_issue(issue, "status"):
+            metadata_dict["status"] = status.name
+        if resolution := best_effort_get_field_from_issue(issue, "resolution"):
+            metadata_dict["resolution"] = resolution.name
+        if labels := best_effort_get_field_from_issue(issue, "labels"):
+            metadata_dict["label"] = labels
+
+        yield Document(
+            id=page_url,
+            sections=[Section(link=page_url, text=ticket_content)],
+            source=DocumentSource.JIRA,
+            semantic_identifier=f"{issue.key}: {issue.fields.summary}",
+            title=f"{issue.key} {issue.fields.summary}",
+            doc_updated_at=time_str_to_utc(issue.fields.updated),
+            primary_owners=list(people) or None,
+            # TODO add secondary_owners (commenters) if needed
+            metadata=metadata_dict,
+        )
 
 
-class JiraConnectorCheckpoint(ConnectorCheckpoint):
-    offset: int | None = None
-
-
-class JiraConnector(CheckpointConnector[JiraConnectorCheckpoint], SlimConnector):
+class JiraConnector(LoadConnector, PollConnector, SlimConnector):
     def __init__(
         self,
         jira_base_url: str,
@@ -194,10 +200,33 @@ class JiraConnector(CheckpointConnector[JiraConnectorCheckpoint], SlimConnector)
         )
         return None
 
-    def _get_jql_query(
+    def _get_jql_query(self) -> str:
+        """Get the JQL query based on whether a specific project is set"""
+        if self.jira_project:
+            return f"project = {self.quoted_jira_project}"
+        return ""  # Empty string means all accessible projects
+
+    def load_from_state(self) -> GenerateDocumentsOutput:
+        jql = self._get_jql_query()
+
+        document_batch = []
+        for doc in fetch_jira_issues_batch(
+            jira_client=self.jira_client,
+            jql=jql,
+            batch_size=_JIRA_FULL_PAGE_SIZE,
+            comment_email_blacklist=self.comment_email_blacklist,
+            labels_to_skip=self.labels_to_skip,
+        ):
+            document_batch.append(doc)
+            if len(document_batch) >= self.batch_size:
+                yield document_batch
+                document_batch = []
+
+        yield document_batch
+
+    def poll_source(
         self, start: SecondsSinceUnixEpoch, end: SecondsSinceUnixEpoch
-    ) -> str:
-        """Get the JQL query based on whether a specific project is set and time range"""
+    ) -> GenerateDocumentsOutput:
         start_date_str = datetime.fromtimestamp(start, tz=timezone.utc).strftime(
             "%Y-%m-%d %H:%M"
         )
@@ -205,61 +234,25 @@ class JiraConnector(CheckpointConnector[JiraConnectorCheckpoint], SlimConnector)
             "%Y-%m-%d %H:%M"
         )
 
-        time_jql = f"updated >= '{start_date_str}' AND updated <= '{end_date_str}'"
+        base_jql = self._get_jql_query()
+        jql = (
+            f"{base_jql} AND " if base_jql else ""
+        ) + f"updated >= '{start_date_str}' AND updated <= '{end_date_str}'"
 
-        if self.jira_project:
-            base_jql = f"project = {self.quoted_jira_project}"
-            return f"{base_jql} AND {time_jql}"
-
-        return time_jql
-
-    def load_from_checkpoint(
-        self,
-        start: SecondsSinceUnixEpoch,
-        end: SecondsSinceUnixEpoch,
-        checkpoint: JiraConnectorCheckpoint,
-    ) -> CheckpointOutput[JiraConnectorCheckpoint]:
-        jql = self._get_jql_query(start, end)
-
-        # Get the current offset from checkpoint or start at 0
-        starting_offset = checkpoint.offset or 0
-        current_offset = starting_offset
-
-        for issue in _perform_jql_search(
+        document_batch = []
+        for doc in fetch_jira_issues_batch(
             jira_client=self.jira_client,
             jql=jql,
-            start=current_offset,
-            max_results=_JIRA_FULL_PAGE_SIZE,
+            batch_size=_JIRA_FULL_PAGE_SIZE,
+            comment_email_blacklist=self.comment_email_blacklist,
+            labels_to_skip=self.labels_to_skip,
         ):
-            issue_key = issue.key
-            try:
-                if document := process_jira_issue(
-                    jira_client=self.jira_client,
-                    issue=issue,
-                    comment_email_blacklist=self.comment_email_blacklist,
-                    labels_to_skip=self.labels_to_skip,
-                ):
-                    yield document
+            document_batch.append(doc)
+            if len(document_batch) >= self.batch_size:
+                yield document_batch
+                document_batch = []
 
-            except Exception as e:
-                yield ConnectorFailure(
-                    failed_document=DocumentFailure(
-                        document_id=issue_key,
-                        document_link=build_jira_url(self.jira_client, issue_key),
-                    ),
-                    failure_message=f"Failed to process Jira issue: {str(e)}",
-                    exception=e,
-                )
-
-            current_offset += 1
-
-        # Update checkpoint
-        checkpoint = JiraConnectorCheckpoint(
-            offset=current_offset,
-            # if we didn't retrieve a full batch, we're done
-            has_more=current_offset - starting_offset == _JIRA_FULL_PAGE_SIZE,
-        )
-        return checkpoint
+        yield document_batch
 
     def retrieve_all_slim_documents(
         self,
@@ -267,13 +260,12 @@ class JiraConnector(CheckpointConnector[JiraConnectorCheckpoint], SlimConnector)
         end: SecondsSinceUnixEpoch | None = None,
         callback: IndexingHeartbeatInterface | None = None,
     ) -> GenerateSlimDocumentOutput:
-        jql = self._get_jql_query(start or 0, end or float("inf"))
+        jql = self._get_jql_query()
 
         slim_doc_batch = []
-        for issue in _perform_jql_search(
+        for issue in _paginate_jql_search(
             jira_client=self.jira_client,
             jql=jql,
-            start=0,
             max_results=_JIRA_SLIM_PAGE_SIZE,
             fields="key",
         ):
@@ -342,16 +334,6 @@ class JiraConnector(CheckpointConnector[JiraConnectorCheckpoint], SlimConnector)
 
                 raise RuntimeError(f"Unexpected Jira error during validation: {e}")
 
-    @override
-    def validate_checkpoint_json(self, checkpoint_json: str) -> JiraConnectorCheckpoint:
-        return JiraConnectorCheckpoint.model_validate_json(checkpoint_json)
-
-    @override
-    def build_dummy_checkpoint(self) -> JiraConnectorCheckpoint:
-        return JiraConnectorCheckpoint(
-            has_more=True,
-        )
-
 
 if __name__ == "__main__":
     import os
@@ -368,7 +350,5 @@ if __name__ == "__main__":
             "jira_api_token": os.environ["JIRA_API_TOKEN"],
         }
     )
-    document_batches = connector.load_from_checkpoint(
-        0, float("inf"), JiraConnectorCheckpoint(has_more=True)
-    )
+    document_batches = connector.load_from_state()
     print(next(document_batches))

backend/onyx/connectors/productboard/connector.py

@@ -16,7 +16,7 @@ from onyx.connectors.interfaces import PollConnector
 from onyx.connectors.interfaces import SecondsSinceUnixEpoch
 from onyx.connectors.models import BasicExpertInfo
 from onyx.connectors.models import Document
-from onyx.connectors.models import TextSection
+from onyx.connectors.models import Section
 from onyx.utils.logger import setup_logger
 
 
@@ -110,7 +110,7 @@ class ProductboardConnector(PollConnector):
             yield Document(
                 id=feature["id"],
                 sections=[
-                    TextSection(
+                    Section(
                         link=feature["links"]["html"],
                         text=self._parse_description_html(feature["description"]),
                     )
@@ -133,7 +133,7 @@ class ProductboardConnector(PollConnector):
             yield Document(
                 id=component["id"],
                 sections=[
-                    TextSection(
+                    Section(
                         link=component["links"]["html"],
                         text=self._parse_description_html(component["description"]),
                     )
@@ -159,7 +159,7 @@ class ProductboardConnector(PollConnector):
             yield Document(
                 id=product["id"],
                 sections=[
-                    TextSection(
+                    Section(
                         link=product["links"]["html"],
                         text=self._parse_description_html(product["description"]),
                     )
@@ -189,7 +189,7 @@ class ProductboardConnector(PollConnector):
             yield Document(
                 id=objective["id"],
                 sections=[
-                    TextSection(
+                    Section(
                         link=objective["links"]["html"],
                         text=self._parse_description_html(objective["description"]),
                     )

backend/onyx/connectors/salesforce/connector.py

@@ -13,7 +13,6 @@ from onyx.connectors.interfaces import SlimConnector
 from onyx.connectors.models import ConnectorMissingCredentialError
 from onyx.connectors.models import Document
 from onyx.connectors.models import SlimDocument
-from onyx.connectors.models import TextSection
 from onyx.connectors.salesforce.doc_conversion import convert_sf_object_to_doc
 from onyx.connectors.salesforce.doc_conversion import ID_PREFIX
 from onyx.connectors.salesforce.salesforce_calls import fetch_all_csvs_in_parallel
@@ -49,12 +48,10 @@ class SalesforceConnector(LoadConnector, PollConnector, SlimConnector):
         self,
         credentials: dict[str, Any],
     ) -> dict[str, Any] | None:
-        domain = "test" if credentials.get("is_sandbox") else None
         self._sf_client = Salesforce(
             username=credentials["sf_username"],
             password=credentials["sf_password"],
             security_token=credentials["sf_security_token"],
-            domain=domain,
         )
         return None
 
@@ -219,8 +216,7 @@ if __name__ == "__main__":
         for doc in doc_batch:
             section_count += len(doc.sections)
             for section in doc.sections:
-                if isinstance(section, TextSection) and section.text is not None:
-                    text_count += len(section.text)
+                text_count += len(section.text)
     end_time = time.time()
 
     print(f"Doc count: {doc_count}")