add some debug logging

.github/workflows/pr-python-connector-tests.yml

@@ -45,8 +45,6 @@ env:
   SHAREPOINT_CLIENT_SECRET: ${{ secrets.SHAREPOINT_CLIENT_SECRET }}
   SHAREPOINT_CLIENT_DIRECTORY_ID: ${{ secrets.SHAREPOINT_CLIENT_DIRECTORY_ID }}
   SHAREPOINT_SITE: ${{ secrets.SHAREPOINT_SITE }}
-  # Github
-  ACCESS_TOKEN_GITHUB: ${{ secrets.ACCESS_TOKEN_GITHUB }}
   # Gitbook
   GITBOOK_SPACE_ID: ${{ secrets.GITBOOK_SPACE_ID }}
   GITBOOK_API_KEY: ${{ secrets.GITBOOK_API_KEY }}

backend/alembic.ini

@@ -84,7 +84,7 @@ keys = console
 keys = generic
 
 [logger_root]
-level = INFO
+level = WARN
 handlers = console
 qualname =
 

backend/alembic/env.py

@@ -25,9 +25,6 @@ from shared_configs.configs import MULTI_TENANT, POSTGRES_DEFAULT_SCHEMA
 from onyx.db.models import Base
 from celery.backends.database.session import ResultModelBase  # type: ignore
 
-# Make sure in alembic.ini [logger_root] level=INFO is set or most logging will be
-# hidden! (defaults to level=WARN)
-
 # Alembic Config object
 config = context.config
 
@@ -39,7 +36,6 @@ if config.config_file_name is not None and config.attributes.get(
 target_metadata = [Base.metadata, ResultModelBase.metadata]
 
 EXCLUDE_TABLES = {"kombu_queue", "kombu_message"}
-
 logger = logging.getLogger(__name__)
 
 ssl_context: ssl.SSLContext | None = None
@@ -68,7 +64,7 @@ def include_object(
     return True
 
 
-def get_schema_options() -> tuple[str, bool, bool, bool]:
+def get_schema_options() -> tuple[str, bool, bool]:
     x_args_raw = context.get_x_argument()
     x_args = {}
     for arg in x_args_raw:
@@ -80,10 +76,6 @@ def get_schema_options() -> tuple[str, bool, bool, bool]:
     create_schema = x_args.get("create_schema", "true").lower() == "true"
     upgrade_all_tenants = x_args.get("upgrade_all_tenants", "false").lower() == "true"
 
-    # continue on error with individual tenant
-    # only applies to online migrations
-    continue_on_error = x_args.get("continue", "false").lower() == "true"
-
     if (
         MULTI_TENANT
         and schema_name == POSTGRES_DEFAULT_SCHEMA
@@ -94,12 +86,14 @@ def get_schema_options() -> tuple[str, bool, bool, bool]:
             "Please specify a tenant-specific schema."
         )
 
-    return schema_name, create_schema, upgrade_all_tenants, continue_on_error
+    return schema_name, create_schema, upgrade_all_tenants
 
 
 def do_run_migrations(
     connection: Connection, schema_name: str, create_schema: bool
 ) -> None:
+    logger.info(f"About to migrate schema: {schema_name}")
+
     if create_schema:
         connection.execute(text(f'CREATE SCHEMA IF NOT EXISTS "{schema_name}"'))
         connection.execute(text("COMMIT"))
@@ -140,12 +134,7 @@ def provide_iam_token_for_alembic(
 
 
 async def run_async_migrations() -> None:
-    (
-        schema_name,
-        create_schema,
-        upgrade_all_tenants,
-        continue_on_error,
-    ) = get_schema_options()
+    schema_name, create_schema, upgrade_all_tenants = get_schema_options()
 
     engine = create_async_engine(
         build_connection_string(),
@@ -162,15 +151,9 @@ async def run_async_migrations() -> None:
 
     if upgrade_all_tenants:
         tenant_schemas = get_all_tenant_ids()
-
-        i_tenant = 0
-        num_tenants = len(tenant_schemas)
         for schema in tenant_schemas:
-            i_tenant += 1
-            logger.info(
-                f"Migrating schema: index={i_tenant} num_tenants={num_tenants} schema={schema}"
-            )
             try:
+                logger.info(f"Migrating schema: {schema}")
                 async with engine.connect() as connection:
                     await connection.run_sync(
                         do_run_migrations,
@@ -179,12 +162,7 @@ async def run_async_migrations() -> None:
                     )
             except Exception as e:
                 logger.error(f"Error migrating schema {schema}: {e}")
-                if not continue_on_error:
-                    logger.error("--continue is not set, raising exception!")
-                    raise
-
-                logger.warning("--continue is set, continuing to next schema.")
-
+                raise
     else:
         try:
             logger.info(f"Migrating schema: {schema_name}")
@@ -202,11 +180,7 @@ async def run_async_migrations() -> None:
 
 
 def run_migrations_offline() -> None:
-    """This doesn't really get used when we migrate in the cloud."""
-
-    logger.info("run_migrations_offline starting.")
-
-    schema_name, _, upgrade_all_tenants, continue_on_error = get_schema_options()
+    schema_name, _, upgrade_all_tenants = get_schema_options()
     url = build_connection_string()
 
     if upgrade_all_tenants:
@@ -256,7 +230,6 @@ def run_migrations_offline() -> None:
 
 
 def run_migrations_online() -> None:
-    logger.info("run_migrations_online starting.")
     asyncio.run(run_async_migrations())
 
 

backend/ee/onyx/server/tenants/provisioning.py

@@ -87,14 +87,11 @@ async def get_or_provision_tenant(
             # If we have a pre-provisioned tenant, assign it to the user
             await assign_tenant_to_user(tenant_id, email, referral_source)
             logger.info(f"Assigned pre-provisioned tenant {tenant_id} to user {email}")
+            return tenant_id
         else:
             # If no pre-provisioned tenant is available, create a new one on-demand
             tenant_id = await create_tenant(email, referral_source)
-
-        # Notify control plane if we have created / assigned a new tenant
-        if not DEV_MODE:
-            await notify_control_plane(tenant_id, email, referral_source)
-        return tenant_id
+            return tenant_id
 
     except Exception as e:
         # If we've encountered an error, log and raise an exception
@@ -119,6 +116,10 @@ async def create_tenant(email: str, referral_source: str | None = None) -> str:
         # Provision tenant on data plane
         await provision_tenant(tenant_id, email)
 
+        # Notify control plane if not already done in provision_tenant
+        if not DEV_MODE and referral_source:
+            await notify_control_plane(tenant_id, email, referral_source)
+
     except Exception as e:
         logger.exception(f"Tenant provisioning failed: {str(e)}")
         # Attempt to rollback the tenant provisioning
@@ -560,3 +561,7 @@ async def assign_tenant_to_user(
     except Exception:
         logger.exception(f"Failed to assign tenant {tenant_id} to user {email}")
         raise Exception("Failed to assign tenant to user")
+
+    # Notify control plane with retry logic
+    if not DEV_MODE:
+        await notify_control_plane(tenant_id, email, referral_source)

backend/onyx/background/celery/tasks/connector_deletion/tasks.py

@@ -30,9 +30,6 @@ from onyx.db.connector_credential_pair import (
 )
 from onyx.db.connector_credential_pair import get_connector_credential_pair_from_id
 from onyx.db.connector_credential_pair import get_connector_credential_pairs
-from onyx.db.document import (
-    delete_all_documents_by_connector_credential_pair__no_commit,
-)
 from onyx.db.document import get_document_ids_for_connector_credential_pair
 from onyx.db.document_set import delete_document_set_cc_pair_relationship__no_commit
 from onyx.db.engine import get_session_with_current_tenant
@@ -443,14 +440,6 @@ def monitor_connector_deletion_taskset(
                 db_session=db_session,
             )
 
-            # Explicitly delete document by connector credential pair records before deleting the connector
-            # This is needed because connector_id is a primary key in that table and cascading deletes won't work
-            delete_all_documents_by_connector_credential_pair__no_commit(
-                db_session=db_session,
-                connector_id=cc_pair.connector_id,
-                credential_id=cc_pair.credential_id,
-            )
-
             # finally, delete the cc-pair
             delete_connector_credential_pair__no_commit(
                 db_session=db_session,

backend/onyx/background/indexing/run_indexing.py

@@ -435,7 +435,7 @@ def _run_indexing(
 
         while checkpoint.has_more:
             logger.info(
-                f"Running '{ctx.source.value}' connector with checkpoint: {checkpoint}"
+                f"Running '{ctx.source}' connector with checkpoint: {checkpoint}"
             )
             for document_batch, failure, next_checkpoint in connector_runner.run(
                 checkpoint

backend/onyx/configs/app_configs.py

@@ -157,7 +157,10 @@ VESPA_CLOUD_CERT_PATH = os.environ.get("VESPA_CLOUD_CERT_PATH")
 VESPA_CLOUD_KEY_PATH = os.environ.get("VESPA_CLOUD_KEY_PATH")
 
 # Number of documents in a batch during indexing (further batching done by chunks before passing to bi-encoder)
-INDEX_BATCH_SIZE = int(os.environ.get("INDEX_BATCH_SIZE") or 16)
+try:
+    INDEX_BATCH_SIZE = int(os.environ.get("INDEX_BATCH_SIZE", 16))
+except ValueError:
+    INDEX_BATCH_SIZE = 16
 
 MAX_DRIVE_WORKERS = int(os.environ.get("MAX_DRIVE_WORKERS", 4))
 
@@ -388,10 +391,6 @@ CONFLUENCE_CONNECTOR_ATTACHMENT_CHAR_COUNT_THRESHOLD = int(
 # connector as some point.
 CONFLUENCE_TIMEZONE_OFFSET = float(os.environ.get("CONFLUENCE_TIMEZONE_OFFSET", 0.0))
 
-GOOGLE_DRIVE_CONNECTOR_SIZE_THRESHOLD = int(
-    os.environ.get("GOOGLE_DRIVE_CONNECTOR_SIZE_THRESHOLD", 10 * 1024 * 1024)
-)
-
 JIRA_CONNECTOR_LABELS_TO_SKIP = [
     ignored_tag
     for ignored_tag in os.environ.get("JIRA_CONNECTOR_LABELS_TO_SKIP", "").split(",")

backend/onyx/connectors/confluence/onyx_confluence.py

@@ -498,12 +498,10 @@ class OnyxConfluence:
                 new_start = get_start_param_from_url(url_suffix)
                 previous_start = get_start_param_from_url(old_url_suffix)
                 if new_start - previous_start > len(results):
-                    logger.debug(
+                    logger.warning(
                         f"Start was updated by more than the amount of results "
-                        f"retrieved for `{url_suffix}`. This is a bug with Confluence, "
-                        "but we have logic to work around it - don't worry this isn't"
-                        f" causing an issue. Start: {new_start}, Previous Start: "
-                        f"{previous_start}, Len Results: {len(results)}."
+                        f"retrieved. This is a bug with Confluence. Start: {new_start}, "
+                        f"Previous Start: {previous_start}, Len Results: {len(results)}."
                     )
 
                     # Update the url_suffix to use the adjusted start

backend/onyx/connectors/file/connector.py

@@ -219,34 +219,24 @@ def _process_file(
 
     # 2) Otherwise: text-based approach. Possibly with embedded images.
     file.seek(0)
+    text_content = ""
+    embedded_images: list[tuple[bytes, str]] = []
 
     # Extract text and images from the file
-    extraction_result = extract_text_and_images(
+    text_content, embedded_images = extract_text_and_images(
         file=file,
         file_name=file_name,
         pdf_pass=pdf_pass,
     )
 
-    # Merge file-specific metadata (from file content) with provided metadata
-    if extraction_result.metadata:
-        logger.debug(
-            f"Found file-specific metadata for {file_name}: {extraction_result.metadata}"
-        )
-        metadata.update(extraction_result.metadata)
-
     # Build sections: first the text as a single Section
     sections: list[TextSection | ImageSection] = []
     link_in_meta = metadata.get("link")
-    if extraction_result.text_content.strip():
-        logger.debug(f"Creating TextSection for {file_name} with link: {link_in_meta}")
-        sections.append(
-            TextSection(link=link_in_meta, text=extraction_result.text_content.strip())
-        )
+    if text_content.strip():
+        sections.append(TextSection(link=link_in_meta, text=text_content.strip()))
 
     # Then any extracted images from docx, etc.
-    for idx, (img_data, img_name) in enumerate(
-        extraction_result.embedded_images, start=1
-    ):
+    for idx, (img_data, img_name) in enumerate(embedded_images, start=1):
         # Store each embedded image as a separate file in PGFileStore
         # and create a section with the image reference
         try:

backend/onyx/connectors/github/connector.py

@@ -1,10 +1,8 @@
-import copy
 import time
-from collections.abc import Generator
+from collections.abc import Iterator
 from datetime import datetime
 from datetime import timedelta
 from datetime import timezone
-from enum import Enum
 from typing import Any
 from typing import cast
 
@@ -15,30 +13,26 @@ from github.GithubException import GithubException
 from github.Issue import Issue
 from github.PaginatedList import PaginatedList
 from github.PullRequest import PullRequest
-from github.Requester import Requester
-from pydantic import BaseModel
-from typing_extensions import override
 
 from onyx.configs.app_configs import GITHUB_CONNECTOR_BASE_URL
+from onyx.configs.app_configs import INDEX_BATCH_SIZE
 from onyx.configs.constants import DocumentSource
 from onyx.connectors.exceptions import ConnectorValidationError
 from onyx.connectors.exceptions import CredentialExpiredError
 from onyx.connectors.exceptions import InsufficientPermissionsError
 from onyx.connectors.exceptions import UnexpectedValidationError
-from onyx.connectors.interfaces import CheckpointConnector
-from onyx.connectors.interfaces import CheckpointOutput
-from onyx.connectors.interfaces import ConnectorCheckpoint
-from onyx.connectors.interfaces import ConnectorFailure
+from onyx.connectors.interfaces import GenerateDocumentsOutput
+from onyx.connectors.interfaces import LoadConnector
+from onyx.connectors.interfaces import PollConnector
 from onyx.connectors.interfaces import SecondsSinceUnixEpoch
 from onyx.connectors.models import ConnectorMissingCredentialError
 from onyx.connectors.models import Document
-from onyx.connectors.models import DocumentFailure
 from onyx.connectors.models import TextSection
+from onyx.utils.batching import batch_generator
 from onyx.utils.logger import setup_logger
 
 logger = setup_logger()
 
-ITEMS_PER_PAGE = 100
 
 _MAX_NUM_RATE_LIMIT_RETRIES = 5
 
@@ -54,7 +48,7 @@ def _sleep_after_rate_limit_exception(github_client: Github) -> None:
 
 def _get_batch_rate_limited(
     git_objs: PaginatedList, page_num: int, github_client: Github, attempt_num: int = 0
-) -> list[PullRequest | Issue]:
+) -> list[Any]:
     if attempt_num > _MAX_NUM_RATE_LIMIT_RETRIES:
         raise RuntimeError(
             "Re-tried fetching batch too many times. Something is going wrong with fetching objects from Github"
@@ -75,6 +69,21 @@ def _get_batch_rate_limited(
         )
 
 
+def _batch_github_objects(
+    git_objs: PaginatedList, github_client: Github, batch_size: int
+) -> Iterator[list[Any]]:
+    page_num = 0
+    while True:
+        batch = _get_batch_rate_limited(git_objs, page_num, github_client)
+        page_num += 1
+
+        if not batch:
+            break
+
+        for mini_batch in batch_generator(batch, batch_size=batch_size):
+            yield mini_batch
+
+
 def _convert_pr_to_document(pull_request: PullRequest) -> Document:
     return Document(
         id=pull_request.html_url,
@@ -86,9 +95,7 @@ def _convert_pr_to_document(pull_request: PullRequest) -> Document:
         # updated_at is UTC time but is timezone unaware, explicitly add UTC
         # as there is logic in indexing to prevent wrong timestamped docs
         # due to local time discrepancies with UTC
-        doc_updated_at=pull_request.updated_at.replace(tzinfo=timezone.utc)
-        if pull_request.updated_at
-        else None,
+        doc_updated_at=pull_request.updated_at.replace(tzinfo=timezone.utc),
         metadata={
             "merged": str(pull_request.merged),
             "state": pull_request.state,
@@ -115,58 +122,31 @@ def _convert_issue_to_document(issue: Issue) -> Document:
     )
 
 
-class SerializedRepository(BaseModel):
-    # id is part of the raw_data as well, just pulled out for convenience
-    id: int
-    headers: dict[str, str | int]
-    raw_data: dict[str, Any]
-
-    def to_Repository(self, requester: Requester) -> Repository.Repository:
-        return Repository.Repository(
-            requester, self.headers, self.raw_data, completed=True
-        )
-
-
-class GithubConnectorStage(Enum):
-    START = "start"
-    PRS = "prs"
-    ISSUES = "issues"
-
-
-class GithubConnectorCheckpoint(ConnectorCheckpoint):
-    stage: GithubConnectorStage
-    curr_page: int
-
-    cached_repo_ids: list[int] | None = None
-    cached_repo: SerializedRepository | None = None
-
-
-class GithubConnector(CheckpointConnector[GithubConnectorCheckpoint]):
+class GithubConnector(LoadConnector, PollConnector):
     def __init__(
         self,
         repo_owner: str,
         repositories: str | None = None,
+        batch_size: int = INDEX_BATCH_SIZE,
         state_filter: str = "all",
         include_prs: bool = True,
         include_issues: bool = False,
     ) -> None:
         self.repo_owner = repo_owner
         self.repositories = repositories
+        self.batch_size = batch_size
         self.state_filter = state_filter
         self.include_prs = include_prs
         self.include_issues = include_issues
         self.github_client: Github | None = None
 
     def load_credentials(self, credentials: dict[str, Any]) -> dict[str, Any] | None:
-        # defaults to 30 items per page, can be set to as high as 100
         self.github_client = (
             Github(
-                credentials["github_access_token"],
-                base_url=GITHUB_CONNECTOR_BASE_URL,
-                per_page=ITEMS_PER_PAGE,
+                credentials["github_access_token"], base_url=GITHUB_CONNECTOR_BASE_URL
             )
             if GITHUB_CONNECTOR_BASE_URL
-            else Github(credentials["github_access_token"], per_page=ITEMS_PER_PAGE)
+            else Github(credentials["github_access_token"])
         )
         return None
 
@@ -237,193 +217,85 @@ class GithubConnector(CheckpointConnector[GithubConnectorCheckpoint]):
             return self._get_all_repos(github_client, attempt_num + 1)
 
     def _fetch_from_github(
-        self,
-        checkpoint: GithubConnectorCheckpoint,
-        start: datetime | None = None,
-        end: datetime | None = None,
-    ) -> Generator[Document | ConnectorFailure, None, GithubConnectorCheckpoint]:
+        self, start: datetime | None = None, end: datetime | None = None
+    ) -> GenerateDocumentsOutput:
         if self.github_client is None:
             raise ConnectorMissingCredentialError("GitHub")
 
-        checkpoint = copy.deepcopy(checkpoint)
-
-        # First run of the connector, fetch all repos and store in checkpoint
-        if checkpoint.cached_repo_ids is None:
-            repos = []
-            if self.repositories:
-                if "," in self.repositories:
-                    # Multiple repositories specified
-                    repos = self._get_github_repos(self.github_client)
-                else:
-                    # Single repository (backward compatibility)
-                    repos = [self._get_github_repo(self.github_client)]
+        repos = []
+        if self.repositories:
+            if "," in self.repositories:
+                # Multiple repositories specified
+                repos = self._get_github_repos(self.github_client)
             else:
-                # All repositories
-                repos = self._get_all_repos(self.github_client)
-            if not repos:
-                checkpoint.has_more = False
-                return checkpoint
+                # Single repository (backward compatibility)
+                repos = [self._get_github_repo(self.github_client)]
+        else:
+            # All repositories
+            repos = self._get_all_repos(self.github_client)
 
-            checkpoint.cached_repo_ids = sorted([repo.id for repo in repos])
-            checkpoint.cached_repo = SerializedRepository(
-                id=checkpoint.cached_repo_ids[0],
-                headers=repos[0].raw_headers,
-                raw_data=repos[0].raw_data,
-            )
-            checkpoint.stage = GithubConnectorStage.PRS
-            checkpoint.curr_page = 0
-            # save checkpoint with repo ids retrieved
-            return checkpoint
+        for repo in repos:
+            if self.include_prs:
+                logger.info(f"Fetching PRs for repo: {repo.name}")
+                pull_requests = repo.get_pulls(
+                    state=self.state_filter, sort="updated", direction="desc"
+                )
 
-        assert checkpoint.cached_repo is not None, "No repo saved in checkpoint"
-        repo = checkpoint.cached_repo.to_Repository(self.github_client.requester)
-
-        if self.include_prs and checkpoint.stage == GithubConnectorStage.PRS:
-            logger.info(f"Fetching PRs for repo: {repo.name}")
-            pull_requests = repo.get_pulls(
-                state=self.state_filter, sort="updated", direction="desc"
-            )
-
-            doc_batch: list[Document] = []
-            pr_batch = _get_batch_rate_limited(
-                pull_requests, checkpoint.curr_page, self.github_client
-            )
-            checkpoint.curr_page += 1
-            done_with_prs = False
-            for pr in pr_batch:
-                # we iterate backwards in time, so at this point we stop processing prs
-                if (
-                    start is not None
-                    and pr.updated_at
-                    and pr.updated_at.replace(tzinfo=timezone.utc) < start
+                for pr_batch in _batch_github_objects(
+                    pull_requests, self.github_client, self.batch_size
                 ):
-                    yield from doc_batch
-                    done_with_prs = True
-                    break
-                # Skip PRs updated after the end date
-                if (
-                    end is not None
-                    and pr.updated_at
-                    and pr.updated_at.replace(tzinfo=timezone.utc) > end
+                    doc_batch: list[Document] = []
+                    for pr in pr_batch:
+                        if start is not None and pr.updated_at < start:
+                            yield doc_batch
+                            break
+                        if end is not None and pr.updated_at > end:
+                            continue
+                        doc_batch.append(_convert_pr_to_document(cast(PullRequest, pr)))
+                    yield doc_batch
+
+            if self.include_issues:
+                logger.info(f"Fetching issues for repo: {repo.name}")
+                issues = repo.get_issues(
+                    state=self.state_filter, sort="updated", direction="desc"
+                )
+
+                for issue_batch in _batch_github_objects(
+                    issues, self.github_client, self.batch_size
                 ):
-                    continue
-                try:
-                    doc_batch.append(_convert_pr_to_document(cast(PullRequest, pr)))
-                except Exception as e:
-                    error_msg = f"Error converting PR to document: {e}"
-                    logger.exception(error_msg)
-                    yield ConnectorFailure(
-                        failed_document=DocumentFailure(
-                            document_id=str(pr.id), document_link=pr.html_url
-                        ),
-                        failure_message=error_msg,
-                        exception=e,
-                    )
-                    continue
+                    doc_batch = []
+                    for issue in issue_batch:
+                        issue = cast(Issue, issue)
+                        if start is not None and issue.updated_at < start:
+                            yield doc_batch
+                            break
+                        if end is not None and issue.updated_at > end:
+                            continue
+                        if issue.pull_request is not None:
+                            # PRs are handled separately
+                            continue
+                        doc_batch.append(_convert_issue_to_document(issue))
+                    yield doc_batch
 
-            # if we found any PRs on the page, yield any associated documents and return the checkpoint
-            if not done_with_prs and len(pr_batch) > 0:
-                yield from doc_batch
-                return checkpoint
+    def load_from_state(self) -> GenerateDocumentsOutput:
+        return self._fetch_from_github()
 
-            # if we went past the start date during the loop or there are no more
-            # prs to get, we move on to issues
-            checkpoint.stage = GithubConnectorStage.ISSUES
-            checkpoint.curr_page = 0
-
-        checkpoint.stage = GithubConnectorStage.ISSUES
-
-        if self.include_issues and checkpoint.stage == GithubConnectorStage.ISSUES:
-            logger.info(f"Fetching issues for repo: {repo.name}")
-            issues = repo.get_issues(
-                state=self.state_filter, sort="updated", direction="desc"
-            )
-
-            doc_batch = []
-            issue_batch = _get_batch_rate_limited(
-                issues, checkpoint.curr_page, self.github_client
-            )
-            checkpoint.curr_page += 1
-            done_with_issues = False
-            for issue in cast(list[Issue], issue_batch):
-                # we iterate backwards in time, so at this point we stop processing prs
-                if (
-                    start is not None
-                    and issue.updated_at.replace(tzinfo=timezone.utc) < start
-                ):
-                    yield from doc_batch
-                    done_with_issues = True
-                    break
-                # Skip PRs updated after the end date
-                if (
-                    end is not None
-                    and issue.updated_at.replace(tzinfo=timezone.utc) > end
-                ):
-                    continue
-
-                if issue.pull_request is not None:
-                    # PRs are handled separately
-                    continue
-
-                try:
-                    doc_batch.append(_convert_issue_to_document(issue))
-                except Exception as e:
-                    error_msg = f"Error converting issue to document: {e}"
-                    logger.exception(error_msg)
-                    yield ConnectorFailure(
-                        failed_document=DocumentFailure(
-                            document_id=str(issue.id),
-                            document_link=issue.html_url,
-                        ),
-                        failure_message=error_msg,
-                        exception=e,
-                    )
-                    continue
-
-            # if we found any issues on the page, yield them and return the checkpoint
-            if not done_with_issues and len(issue_batch) > 0:
-                yield from doc_batch
-                return checkpoint
-
-            # if we went past the start date during the loop or there are no more
-            # issues to get, we move on to the next repo
-            checkpoint.stage = GithubConnectorStage.PRS
-            checkpoint.curr_page = 0
-
-        checkpoint.has_more = len(checkpoint.cached_repo_ids) > 1
-        if checkpoint.cached_repo_ids:
-            next_id = checkpoint.cached_repo_ids.pop()
-            next_repo = self.github_client.get_repo(next_id)
-            checkpoint.cached_repo = SerializedRepository(
-                id=next_id,
-                headers=next_repo.raw_headers,
-                raw_data=next_repo.raw_data,
-            )
-
-        return checkpoint
-
-    @override
-    def load_from_checkpoint(
-        self,
-        start: SecondsSinceUnixEpoch,
-        end: SecondsSinceUnixEpoch,
-        checkpoint: GithubConnectorCheckpoint,
-    ) -> CheckpointOutput[GithubConnectorCheckpoint]:
-        start_datetime = datetime.fromtimestamp(start, tz=timezone.utc)
-        end_datetime = datetime.fromtimestamp(end, tz=timezone.utc)
+    def poll_source(
+        self, start: SecondsSinceUnixEpoch, end: SecondsSinceUnixEpoch
+    ) -> GenerateDocumentsOutput:
+        start_datetime = datetime.utcfromtimestamp(start)
+        end_datetime = datetime.utcfromtimestamp(end)
 
         # Move start time back by 3 hours, since some Issues/PRs are getting dropped
         # Could be due to delayed processing on GitHub side
         # The non-updated issues since last poll will be shortcut-ed and not embedded
         adjusted_start_datetime = start_datetime - timedelta(hours=3)
 
-        epoch = datetime.fromtimestamp(0, tz=timezone.utc)
+        epoch = datetime.utcfromtimestamp(0)
         if adjusted_start_datetime < epoch:
             adjusted_start_datetime = epoch
 
-        return self._fetch_from_github(
-            checkpoint, start=adjusted_start_datetime, end=end_datetime
-        )
+        return self._fetch_from_github(adjusted_start_datetime, end_datetime)
 
     def validate_connector_settings(self) -> None:
         if self.github_client is None:
@@ -525,16 +397,6 @@ class GithubConnector(CheckpointConnector[GithubConnectorCheckpoint]):
                 f"Unexpected error during GitHub settings validation: {exc}"
             )
 
-    def validate_checkpoint_json(
-        self, checkpoint_json: str
-    ) -> GithubConnectorCheckpoint:
-        return GithubConnectorCheckpoint.model_validate_json(checkpoint_json)
-
-    def build_dummy_checkpoint(self) -> GithubConnectorCheckpoint:
-        return GithubConnectorCheckpoint(
-            stage=GithubConnectorStage.PRS, curr_page=0, has_more=True
-        )
-
 
 if __name__ == "__main__":
     import os
@@ -544,9 +406,7 @@ if __name__ == "__main__":
         repositories=os.environ["REPOSITORIES"],
     )
     connector.load_credentials(
-        {"github_access_token": os.environ["ACCESS_TOKEN_GITHUB"]}
-    )
-    document_batches = connector.load_from_checkpoint(
-        0, time.time(), connector.build_dummy_checkpoint()
+        {"github_access_token": os.environ["GITHUB_ACCESS_TOKEN"]}
     )
+    document_batches = connector.load_from_state()
     print(next(document_batches))

backend/onyx/connectors/google_drive/connector.py

@@ -15,7 +15,6 @@ from google.oauth2.service_account import Credentials as ServiceAccountCredentia
 from googleapiclient.errors import HttpError  # type: ignore
 from typing_extensions import override
 
-from onyx.configs.app_configs import GOOGLE_DRIVE_CONNECTOR_SIZE_THRESHOLD
 from onyx.configs.app_configs import INDEX_BATCH_SIZE
 from onyx.configs.app_configs import MAX_DRIVE_WORKERS
 from onyx.configs.constants import DocumentSource
@@ -88,7 +87,6 @@ def _convert_single_file(
     creds: Any,
     primary_admin_email: str,
     allow_images: bool,
-    size_threshold: int,
     file: dict[str, Any],
 ) -> Document | ConnectorFailure | None:
     user_email = file.get("owners", [{}])[0].get("emailAddress") or primary_admin_email
@@ -105,7 +103,6 @@ def _convert_single_file(
         drive_service=user_drive_service,
         docs_service=docs_service,
         allow_images=allow_images,
-        size_threshold=size_threshold,
     )
 
 
@@ -241,8 +238,6 @@ class GoogleDriveConnector(SlimConnector, CheckpointConnector[GoogleDriveCheckpo
         self._retrieved_ids: set[str] = set()
         self.allow_images = False
 
-        self.size_threshold = GOOGLE_DRIVE_CONNECTOR_SIZE_THRESHOLD
-
     def set_allow_images(self, value: bool) -> None:
         self.allow_images = value
 
@@ -912,7 +907,6 @@ class GoogleDriveConnector(SlimConnector, CheckpointConnector[GoogleDriveCheckpo
                     self.creds,
                     self.primary_admin_email,
                     self.allow_images,
-                    self.size_threshold,
                 )
 
                 # Fetch files in batches

backend/onyx/connectors/google_drive/doc_conversion.py

@@ -76,7 +76,7 @@ def is_gdrive_image_mime_type(mime_type: str) -> bool:
     return is_valid_image_type(mime_type)
 
 
-def _download_and_extract_sections_basic(
+def _extract_sections_basic(
     file: dict[str, str],
     service: GoogleDriveService,
     allow_images: bool,
@@ -213,7 +213,6 @@ def convert_drive_item_to_document(
     drive_service: Callable[[], GoogleDriveService],
     docs_service: Callable[[], GoogleDocsService],
     allow_images: bool,
-    size_threshold: int,
 ) -> Document | ConnectorFailure | None:
     """
     Main entry point for converting a Google Drive file => Document object.
@@ -241,24 +240,9 @@ def convert_drive_item_to_document(
                     f"Error in advanced parsing: {e}. Falling back to basic extraction."
                 )
 
-        size_str = file.get("size")
-        if size_str:
-            try:
-                size_int = int(size_str)
-            except ValueError:
-                logger.warning(f"Parsing string to int failed: size_str={size_str}")
-            else:
-                if size_int > size_threshold:
-                    logger.warning(
-                        f"{file.get('name')} exceeds size threshold of {size_threshold}. Skipping."
-                    )
-                    return None
-
         # If we don't have sections yet, use the basic extraction method
         if not sections:
-            sections = _download_and_extract_sections_basic(
-                file, drive_service(), allow_images
-            )
+            sections = _extract_sections_basic(file, drive_service(), allow_images)
 
         # If we still don't have any sections, skip this file
         if not sections:

backend/onyx/connectors/interfaces.py

@@ -8,6 +8,7 @@ from typing import TypeAlias
 from typing import TypeVar
 
 from pydantic import BaseModel
+from typing_extensions import override
 
 from onyx.configs.constants import DocumentSource
 from onyx.connectors.models import ConnectorCheckpoint
@@ -230,7 +231,7 @@ class CheckpointConnector(BaseConnector[CT]):
         """
         raise NotImplementedError
 
-    @abc.abstractmethod
+    @override
     def build_dummy_checkpoint(self) -> CT:
         raise NotImplementedError
 

backend/onyx/connectors/slack/connector.py

@@ -438,11 +438,7 @@ def _get_all_doc_ids(
 
 class ProcessedSlackMessage(BaseModel):
     doc: Document | None
-    # if the message is part of a thread, this is the thread_ts
-    # otherwise, this is the message_ts. Either way, will be a unique identifier.
-    # In the future, if the message becomes a thread, then the thread_ts
-    # will be set to the message_ts.
-    thread_or_message_ts: str
+    thread_ts: str | None
     failure: ConnectorFailure | None
 
 
@@ -456,7 +452,6 @@ def _process_message(
     msg_filter_func: Callable[[MessageType], bool] = default_msg_filter,
 ) -> ProcessedSlackMessage:
     thread_ts = message.get("thread_ts")
-    thread_or_message_ts = thread_ts or message["ts"]
     try:
         # causes random failures for testing checkpointing / continue on failure
         # import random
@@ -472,18 +467,16 @@ def _process_message(
             seen_thread_ts=seen_thread_ts,
             msg_filter_func=msg_filter_func,
         )
-        return ProcessedSlackMessage(
-            doc=doc, thread_or_message_ts=thread_or_message_ts, failure=None
-        )
+        return ProcessedSlackMessage(doc=doc, thread_ts=thread_ts, failure=None)
     except Exception as e:
         logger.exception(f"Error processing message {message['ts']}")
         return ProcessedSlackMessage(
             doc=None,
-            thread_or_message_ts=thread_or_message_ts,
+            thread_ts=thread_ts,
             failure=ConnectorFailure(
                 failed_document=DocumentFailure(
                     document_id=_build_doc_id(
-                        channel_id=channel["id"], thread_ts=thread_or_message_ts
+                        channel_id=channel["id"], thread_ts=(thread_ts or message["ts"])
                     ),
                     document_link=get_message_link(message, client, channel["id"]),
                 ),
@@ -623,7 +616,7 @@ class SlackConnector(SlimConnector, CheckpointConnector[SlackCheckpoint]):
                 for future in as_completed(futures):
                     processed_slack_message = future.result()
                     doc = processed_slack_message.doc
-                    thread_or_message_ts = processed_slack_message.thread_or_message_ts
+                    thread_ts = processed_slack_message.thread_ts
                     failure = processed_slack_message.failure
                     if doc:
                         # handle race conditions here since this is single
@@ -631,13 +624,11 @@ class SlackConnector(SlimConnector, CheckpointConnector[SlackCheckpoint]):
                         # but since this is single threaded, we won't run into simul
                         # writes. At worst, we can duplicate a thread, which will be
                         # deduped later on.
-                        if thread_or_message_ts not in seen_thread_ts:
+                        if thread_ts not in seen_thread_ts:
                             yield doc
 
-                        assert (
-                            thread_or_message_ts
-                        ), "found non-None doc with None thread_or_message_ts"
-                        seen_thread_ts.add(thread_or_message_ts)
+                        assert thread_ts, "found non-None doc with None thread_ts"
+                        seen_thread_ts.add(thread_ts)
                     elif failure:
                         yield failure
 

backend/onyx/connectors/zendesk/connector.py

@@ -1,32 +1,23 @@
-import copy
-import time
 from collections.abc import Iterator
 from typing import Any
 from typing import cast
 
 import requests
-from pydantic import BaseModel
-from requests.exceptions import HTTPError
-from typing_extensions import override
 
+from onyx.configs.app_configs import INDEX_BATCH_SIZE
 from onyx.configs.app_configs import ZENDESK_CONNECTOR_SKIP_ARTICLE_LABELS
 from onyx.configs.constants import DocumentSource
 from onyx.connectors.cross_connector_utils.miscellaneous_utils import (
     time_str_to_utc,
 )
-from onyx.connectors.exceptions import ConnectorValidationError
-from onyx.connectors.exceptions import CredentialExpiredError
-from onyx.connectors.exceptions import InsufficientPermissionsError
-from onyx.connectors.interfaces import CheckpointConnector
-from onyx.connectors.interfaces import CheckpointOutput
-from onyx.connectors.interfaces import ConnectorFailure
+from onyx.connectors.interfaces import GenerateDocumentsOutput
 from onyx.connectors.interfaces import GenerateSlimDocumentOutput
+from onyx.connectors.interfaces import LoadConnector
+from onyx.connectors.interfaces import PollConnector
 from onyx.connectors.interfaces import SecondsSinceUnixEpoch
 from onyx.connectors.interfaces import SlimConnector
 from onyx.connectors.models import BasicExpertInfo
-from onyx.connectors.models import ConnectorCheckpoint
 from onyx.connectors.models import Document
-from onyx.connectors.models import DocumentFailure
 from onyx.connectors.models import SlimDocument
 from onyx.connectors.models import TextSection
 from onyx.file_processing.html_utils import parse_html_page_basic
@@ -35,7 +26,6 @@ from onyx.utils.retry_wrapper import retry_builder
 
 
 MAX_PAGE_SIZE = 30  # Zendesk API maximum
-MAX_AUTHOR_MAP_SIZE = 50_000  # Reset author map cache if it gets too large
 _SLIM_BATCH_SIZE = 1000
 
 
@@ -63,22 +53,10 @@ class ZendeskClient:
                 # Sleep for the duration indicated by the Retry-After header
                 time.sleep(int(retry_after))
 
-        elif (
-            response.status_code == 403
-            and response.json().get("error") == "SupportProductInactive"
-        ):
-            return response.json()
-
         response.raise_for_status()
         return response.json()
 
 
-class ZendeskPageResponse(BaseModel):
-    data: list[dict[str, Any]]
-    meta: dict[str, Any]
-    has_more: bool
-
-
 def _get_content_tag_mapping(client: ZendeskClient) -> dict[str, str]:
     content_tags: dict[str, str] = {}
     params = {"page[size]": MAX_PAGE_SIZE}
@@ -104,9 +82,11 @@ def _get_content_tag_mapping(client: ZendeskClient) -> dict[str, str]:
 def _get_articles(
     client: ZendeskClient, start_time: int | None = None, page_size: int = MAX_PAGE_SIZE
 ) -> Iterator[dict[str, Any]]:
-    params = {"page[size]": page_size, "sort_by": "updated_at", "sort_order": "asc"}
-    if start_time is not None:
-        params["start_time"] = start_time
+    params = (
+        {"start_time": start_time, "page[size]": page_size}
+        if start_time
+        else {"page[size]": page_size}
+    )
 
     while True:
         data = client.make_request("help_center/articles", params)
@@ -118,30 +98,10 @@ def _get_articles(
         params["page[after]"] = data["meta"]["after_cursor"]
 
 
-def _get_article_page(
-    client: ZendeskClient,
-    start_time: int | None = None,
-    after_cursor: str | None = None,
-    page_size: int = MAX_PAGE_SIZE,
-) -> ZendeskPageResponse:
-    params = {"page[size]": page_size, "sort_by": "updated_at", "sort_order": "asc"}
-    if start_time is not None:
-        params["start_time"] = start_time
-    if after_cursor is not None:
-        params["page[after]"] = after_cursor
-
-    data = client.make_request("help_center/articles", params)
-    return ZendeskPageResponse(
-        data=data["articles"],
-        meta=data["meta"],
-        has_more=bool(data["meta"].get("has_more", False)),
-    )
-
-
 def _get_tickets(
     client: ZendeskClient, start_time: int | None = None
 ) -> Iterator[dict[str, Any]]:
-    params = {"start_time": start_time or 0}
+    params = {"start_time": start_time} if start_time else {"start_time": 0}
 
     while True:
         data = client.make_request("incremental/tickets.json", params)
@@ -154,27 +114,6 @@ def _get_tickets(
             break
 
 
-# TODO: maybe these don't need to be their own functions?
-def _get_tickets_page(
-    client: ZendeskClient, start_time: int | None = None
-) -> ZendeskPageResponse:
-    params = {"start_time": start_time or 0}
-
-    # NOTE: for some reason zendesk doesn't seem to be respecting the start_time param
-    # in my local testing with very few tickets. We'll look into it if this becomes an
-    # issue in larger deployments
-    data = client.make_request("incremental/tickets.json", params)
-    if data.get("error") == "SupportProductInactive":
-        raise ValueError(
-            "Zendesk Support Product is not active for this account, No tickets to index"
-        )
-    return ZendeskPageResponse(
-        data=data["tickets"],
-        meta={"end_time": data["end_time"]},
-        has_more=not bool(data.get("end_of_stream", False)),
-    )
-
-
 def _fetch_author(client: ZendeskClient, author_id: str) -> BasicExpertInfo | None:
     # Skip fetching if author_id is invalid
     if not author_id or author_id == "-1":
@@ -339,22 +278,13 @@ def _ticket_to_document(
     )
 
 
-class ZendeskConnectorCheckpoint(ConnectorCheckpoint):
-    # We use cursor-based paginated retrieval for articles
-    after_cursor_articles: str | None
-
-    # We use timestamp-based paginated retrieval for tickets
-    next_start_time_tickets: int | None
-
-    cached_author_map: dict[str, BasicExpertInfo] | None
-    cached_content_tags: dict[str, str] | None
-
-
-class ZendeskConnector(SlimConnector, CheckpointConnector[ZendeskConnectorCheckpoint]):
+class ZendeskConnector(LoadConnector, PollConnector, SlimConnector):
     def __init__(
         self,
+        batch_size: int = INDEX_BATCH_SIZE,
         content_type: str = "articles",
     ) -> None:
+        self.batch_size = batch_size
         self.content_type = content_type
         self.subdomain = ""
         # Fetch all tags ahead of time
@@ -374,50 +304,33 @@ class ZendeskConnector(SlimConnector, CheckpointConnector[ZendeskConnectorCheckp
         )
         return None
 
-    @override
-    def load_from_checkpoint(
-        self,
-        start: SecondsSinceUnixEpoch,
-        end: SecondsSinceUnixEpoch,
-        checkpoint: ZendeskConnectorCheckpoint,
-    ) -> CheckpointOutput[ZendeskConnectorCheckpoint]:
+    def load_from_state(self) -> GenerateDocumentsOutput:
+        return self.poll_source(None, None)
+
+    def poll_source(
+        self, start: SecondsSinceUnixEpoch | None, end: SecondsSinceUnixEpoch | None
+    ) -> GenerateDocumentsOutput:
         if self.client is None:
             raise ZendeskCredentialsNotSetUpError()
 
-        if checkpoint.cached_content_tags is None:
-            checkpoint.cached_content_tags = _get_content_tag_mapping(self.client)
-            return checkpoint  # save the content tags to the checkpoint
-        self.content_tags = checkpoint.cached_content_tags
+        self.content_tags = _get_content_tag_mapping(self.client)
 
         if self.content_type == "articles":
-            checkpoint = yield from self._retrieve_articles(start, end, checkpoint)
-            return checkpoint
+            yield from self._poll_articles(start)
         elif self.content_type == "tickets":
-            checkpoint = yield from self._retrieve_tickets(start, end, checkpoint)
-            return checkpoint
+            yield from self._poll_tickets(start)
         else:
             raise ValueError(f"Unsupported content_type: {self.content_type}")
 
-    def _retrieve_articles(
-        self,
-        start: SecondsSinceUnixEpoch | None,
-        end: SecondsSinceUnixEpoch | None,
-        checkpoint: ZendeskConnectorCheckpoint,
-    ) -> CheckpointOutput[ZendeskConnectorCheckpoint]:
-        checkpoint = copy.deepcopy(checkpoint)
-        # This one is built on the fly as there may be more many more authors than tags
-        author_map: dict[str, BasicExpertInfo] = checkpoint.cached_author_map or {}
-        after_cursor = checkpoint.after_cursor_articles
-        doc_batch: list[Document] = []
+    def _poll_articles(
+        self, start: SecondsSinceUnixEpoch | None
+    ) -> GenerateDocumentsOutput:
+        articles = _get_articles(self.client, start_time=int(start) if start else None)
 
-        response = _get_article_page(
-            self.client,
-            start_time=int(start) if start else None,
-            after_cursor=after_cursor,
-        )
-        articles = response.data
-        has_more = response.has_more
-        after_cursor = response.meta.get("after_cursor")
+        # This one is built on the fly as there may be more many more authors than tags
+        author_map: dict[str, BasicExpertInfo] = {}
+
+        doc_batch = []
         for article in articles:
             if (
                 article.get("body") is None
@@ -429,109 +342,66 @@ class ZendeskConnector(SlimConnector, CheckpointConnector[ZendeskConnectorCheckp
             ):
                 continue
 
-            try:
-                new_author_map, document = _article_to_document(
-                    article, self.content_tags, author_map, self.client
-                )
-            except Exception as e:
-                yield ConnectorFailure(
-                    failed_document=DocumentFailure(
-                        document_id=f"{article.get('id')}",
-                        document_link=article.get("html_url", ""),
-                    ),
-                    failure_message=str(e),
-                    exception=e,
-                )
-                continue
-
+            new_author_map, documents = _article_to_document(
+                article, self.content_tags, author_map, self.client
+            )
             if new_author_map:
                 author_map.update(new_author_map)
 
-            doc_batch.append(document)
+            doc_batch.append(documents)
+            if len(doc_batch) >= self.batch_size:
+                yield doc_batch
+                doc_batch.clear()
 
-        if not has_more:
-            yield from doc_batch
-            checkpoint.has_more = False
-            return checkpoint
+        if doc_batch:
+            yield doc_batch
 
-        # Sometimes no documents are retrieved, but the cursor
-        # is still updated so the connector makes progress.
-        yield from doc_batch
-        checkpoint.after_cursor_articles = after_cursor
-
-        last_doc_updated_at = doc_batch[-1].doc_updated_at if doc_batch else None
-        checkpoint.has_more = bool(
-            end is None
-            or last_doc_updated_at is None
-            or last_doc_updated_at.timestamp() <= end
-        )
-        checkpoint.cached_author_map = (
-            author_map if len(author_map) <= MAX_AUTHOR_MAP_SIZE else None
-        )
-        return checkpoint
-
-    def _retrieve_tickets(
-        self,
-        start: SecondsSinceUnixEpoch | None,
-        end: SecondsSinceUnixEpoch | None,
-        checkpoint: ZendeskConnectorCheckpoint,
-    ) -> CheckpointOutput[ZendeskConnectorCheckpoint]:
-        checkpoint = copy.deepcopy(checkpoint)
+    def _poll_tickets(
+        self, start: SecondsSinceUnixEpoch | None
+    ) -> GenerateDocumentsOutput:
         if self.client is None:
             raise ZendeskCredentialsNotSetUpError()
 
-        author_map: dict[str, BasicExpertInfo] = checkpoint.cached_author_map or {}
+        author_map: dict[str, BasicExpertInfo] = {}
 
-        doc_batch: list[Document] = []
-        next_start_time = int(checkpoint.next_start_time_tickets or start or 0)
-        ticket_response = _get_tickets_page(self.client, start_time=next_start_time)
-        tickets = ticket_response.data
-        has_more = ticket_response.has_more
-        next_start_time = ticket_response.meta["end_time"]
-        for ticket in tickets:
-            if ticket.get("status") == "deleted":
-                continue
-
-            try:
-                new_author_map, document = _ticket_to_document(
-                    ticket=ticket,
-                    author_map=author_map,
-                    client=self.client,
-                    default_subdomain=self.subdomain,
-                )
-            except Exception as e:
-                yield ConnectorFailure(
-                    failed_document=DocumentFailure(
-                        document_id=f"{ticket.get('id')}",
-                        document_link=ticket.get("url", ""),
-                    ),
-                    failure_message=str(e),
-                    exception=e,
-                )
-                continue
-
-            if new_author_map:
-                author_map.update(new_author_map)
-
-            doc_batch.append(document)
-
-        if not has_more:
-            yield from doc_batch
-            checkpoint.has_more = False
-            return checkpoint
-
-        yield from doc_batch
-        checkpoint.next_start_time_tickets = next_start_time
-        last_doc_updated_at = doc_batch[-1].doc_updated_at if doc_batch else None
-        checkpoint.has_more = bool(
-            end is None
-            or last_doc_updated_at is None
-            or last_doc_updated_at.timestamp() <= end
+        ticket_generator = _get_tickets(
+            self.client, start_time=int(start) if start else None
         )
-        checkpoint.cached_author_map = (
-            author_map if len(author_map) <= MAX_AUTHOR_MAP_SIZE else None
-        )
-        return checkpoint
+
+        while True:
+            doc_batch = []
+            for _ in range(self.batch_size):
+                try:
+                    ticket = next(ticket_generator)
+
+                    # Check if the ticket status is deleted and skip it if so
+                    if ticket.get("status") == "deleted":
+                        continue
+
+                    new_author_map, documents = _ticket_to_document(
+                        ticket=ticket,
+                        author_map=author_map,
+                        client=self.client,
+                        default_subdomain=self.subdomain,
+                    )
+
+                    if new_author_map:
+                        author_map.update(new_author_map)
+
+                    doc_batch.append(documents)
+
+                    if len(doc_batch) >= self.batch_size:
+                        yield doc_batch
+                        doc_batch.clear()
+
+                except StopIteration:
+                    # No more tickets to process
+                    if doc_batch:
+                        yield doc_batch
+                    return
+
+            if doc_batch:
+                yield doc_batch
 
     def retrieve_all_slim_documents(
         self,
@@ -571,51 +441,10 @@ class ZendeskConnector(SlimConnector, CheckpointConnector[ZendeskConnectorCheckp
         if slim_doc_batch:
             yield slim_doc_batch
 
-    @override
-    def validate_connector_settings(self) -> None:
-        if self.client is None:
-            raise ZendeskCredentialsNotSetUpError()
-
-        try:
-            _get_article_page(self.client, start_time=0)
-        except HTTPError as e:
-            # Check for HTTP status codes
-            if e.response.status_code == 401:
-                raise CredentialExpiredError(
-                    "Your Zendesk credentials appear to be invalid or expired (HTTP 401)."
-                ) from e
-            elif e.response.status_code == 403:
-                raise InsufficientPermissionsError(
-                    "Your Zendesk token does not have sufficient permissions (HTTP 403)."
-                ) from e
-            elif e.response.status_code == 404:
-                raise ConnectorValidationError(
-                    "Zendesk resource not found (HTTP 404)."
-                ) from e
-            else:
-                raise ConnectorValidationError(
-                    f"Unexpected Zendesk error (status={e.response.status_code}): {e}"
-                ) from e
-
-    @override
-    def validate_checkpoint_json(
-        self, checkpoint_json: str
-    ) -> ZendeskConnectorCheckpoint:
-        return ZendeskConnectorCheckpoint.model_validate_json(checkpoint_json)
-
-    @override
-    def build_dummy_checkpoint(self) -> ZendeskConnectorCheckpoint:
-        return ZendeskConnectorCheckpoint(
-            after_cursor_articles=None,
-            next_start_time_tickets=None,
-            cached_author_map=None,
-            cached_content_tags=None,
-            has_more=True,
-        )
-
 
 if __name__ == "__main__":
     import os
+    import time
 
     connector = ZendeskConnector()
     connector.load_credentials(
@@ -628,8 +457,6 @@ if __name__ == "__main__":
 
     current = time.time()
     one_day_ago = current - 24 * 60 * 60  # 1 day
-    document_batches = connector.load_from_checkpoint(
-        one_day_ago, current, connector.build_dummy_checkpoint()
-    )
+    document_batches = connector.poll_source(one_day_ago, current)
 
     print(next(document_batches))

backend/onyx/db/document.py

@@ -555,28 +555,6 @@ def delete_documents_by_connector_credential_pair__no_commit(
     db_session.execute(stmt)
 
 
-def delete_all_documents_by_connector_credential_pair__no_commit(
-    db_session: Session,
-    connector_id: int,
-    credential_id: int,
-) -> None:
-    """Deletes all document by connector credential pair entries for a specific connector and credential.
-    This is primarily used during connector deletion to ensure all references are removed
-    before deleting the connector itself. This is crucial because connector_id is part of the
-    primary key in DocumentByConnectorCredentialPair, and attempting to delete the Connector
-    would otherwise try to set the foreign key to NULL, which fails for primary keys.
-
-    NOTE: Does not commit the transaction, this must be done by the caller.
-    """
-    stmt = delete(DocumentByConnectorCredentialPair).where(
-        and_(
-            DocumentByConnectorCredentialPair.connector_id == connector_id,
-            DocumentByConnectorCredentialPair.credential_id == credential_id,
-        )
-    )
-    db_session.execute(stmt)
-
-
 def delete_documents__no_commit(db_session: Session, document_ids: list[str]) -> None:
     db_session.execute(delete(DbDocument).where(DbDocument.id.in_(document_ids)))
 

backend/onyx/file_processing/extract_file_text.py

@@ -5,13 +5,13 @@ import re
 import zipfile
 from collections.abc import Callable
 from collections.abc import Iterator
-from collections.abc import Sequence
 from email.parser import Parser as EmailParser
 from io import BytesIO
 from pathlib import Path
 from typing import Any
 from typing import IO
-from typing import NamedTuple
+from typing import List
+from typing import Tuple
 
 import chardet
 import docx  # type: ignore
@@ -219,7 +219,7 @@ def pdf_to_text(file: IO[Any], pdf_pass: str | None = None) -> str:
 
 def read_pdf_file(
     file: IO[Any], pdf_pass: str | None = None, extract_images: bool = False
-) -> tuple[str, dict[str, Any], Sequence[tuple[bytes, str]]]:
+) -> tuple[str, dict, list[tuple[bytes, str]]]:
     """
     Returns the text, basic PDF metadata, and optionally extracted images.
     """
@@ -282,13 +282,13 @@ def read_pdf_file(
 
 def docx_to_text_and_images(
     file: IO[Any],
-) -> tuple[str, Sequence[tuple[bytes, str]]]:
+) -> Tuple[str, List[Tuple[bytes, str]]]:
     """
     Extract text from a docx. If embed_images=True, also extract inline images.
     Return (text_content, list_of_images).
     """
     paragraphs = []
-    embedded_images: list[tuple[bytes, str]] = []
+    embedded_images: List[Tuple[bytes, str]] = []
 
     doc = docx.Document(file)
 
@@ -426,22 +426,14 @@ def extract_file_text(
         return ""
 
 
-class ExtractionResult(NamedTuple):
-    """Structured result from text and image extraction from various file types."""
-
-    text_content: str
-    embedded_images: Sequence[tuple[bytes, str]]
-    metadata: dict[str, Any]
-
-
 def extract_text_and_images(
     file: IO[Any],
     file_name: str,
     pdf_pass: str | None = None,
-) -> ExtractionResult:
+) -> Tuple[str, List[Tuple[bytes, str]]]:
     """
     Primary new function for the updated connector.
-    Returns structured extraction result with text content, embedded images, and metadata.
+    Returns (text_content, [(embedded_img_bytes, embedded_img_name), ...]).
     """
 
     try:
@@ -450,9 +442,7 @@ def extract_text_and_images(
             # If the user doesn't want embedded images, unstructured is fine
             file.seek(0)
             text_content = unstructured_to_text(file, file_name)
-            return ExtractionResult(
-                text_content=text_content, embedded_images=[], metadata={}
-            )
+            return (text_content, [])
 
         extension = get_file_ext(file_name)
 
@@ -460,76 +450,54 @@ def extract_text_and_images(
         if extension == ".docx":
             file.seek(0)
             text_content, images = docx_to_text_and_images(file)
-            return ExtractionResult(
-                text_content=text_content, embedded_images=images, metadata={}
-            )
+            return (text_content, images)
 
         # PDF example: we do not show complicated PDF image extraction here
         # so we simply extract text for now and skip images.
         if extension == ".pdf":
             file.seek(0)
-            text_content, pdf_metadata, images = read_pdf_file(
-                file, pdf_pass, extract_images=True
-            )
-            return ExtractionResult(
-                text_content=text_content, embedded_images=images, metadata=pdf_metadata
-            )
+            text_content, _, images = read_pdf_file(file, pdf_pass, extract_images=True)
+            return (text_content, images)
 
         # For PPTX, XLSX, EML, etc., we do not show embedded image logic here.
         # You can do something similar to docx if needed.
         if extension == ".pptx":
             file.seek(0)
-            return ExtractionResult(
-                text_content=pptx_to_text(file), embedded_images=[], metadata={}
-            )
+            return (pptx_to_text(file), [])
 
         if extension == ".xlsx":
             file.seek(0)
-            return ExtractionResult(
-                text_content=xlsx_to_text(file), embedded_images=[], metadata={}
-            )
+            return (xlsx_to_text(file), [])
 
         if extension == ".eml":
             file.seek(0)
-            return ExtractionResult(
-                text_content=eml_to_text(file), embedded_images=[], metadata={}
-            )
+            return (eml_to_text(file), [])
 
         if extension == ".epub":
             file.seek(0)
-            return ExtractionResult(
-                text_content=epub_to_text(file), embedded_images=[], metadata={}
-            )
+            return (epub_to_text(file), [])
 
         if extension == ".html":
             file.seek(0)
-            return ExtractionResult(
-                text_content=parse_html_page_basic(file),
-                embedded_images=[],
-                metadata={},
-            )
+            return (parse_html_page_basic(file), [])
 
         # If we reach here and it's a recognized text extension
         if is_text_file_extension(file_name):
             file.seek(0)
             encoding = detect_encoding(file)
-            text_content_raw, file_metadata = read_text_file(
+            text_content_raw, _ = read_text_file(
                 file, encoding=encoding, ignore_onyx_metadata=False
             )
-            return ExtractionResult(
-                text_content=text_content_raw,
-                embedded_images=[],
-                metadata=file_metadata,
-            )
+            return (text_content_raw, [])
 
         # If it's an image file or something else, we do not parse embedded images from them
         # just return empty text
         file.seek(0)
-        return ExtractionResult(text_content="", embedded_images=[], metadata={})
+        return ("", [])
 
     except Exception as e:
         logger.exception(f"Failed to extract text/images from {file_name}: {e}")
-        return ExtractionResult(text_content="", embedded_images=[], metadata={})
+        return ("", [])
 
 
 def convert_docx_to_txt(

backend/onyx/onyxbot/slack/handlers/handle_message.py

@@ -170,8 +170,7 @@ def handle_message(
         respond_tag_only = channel_conf.get("respond_tag_only") or False
         respond_member_group_list = channel_conf.get("respond_member_group_list", None)
 
-    # NOTE: always respond in the DMs, as long the default config is not disabled.
-    if respond_tag_only and not bypass_filters and not is_bot_dm:
+    if respond_tag_only and not bypass_filters:
         logger.info(
             "Skipping message since the channel is configured such that "
             "OnyxBot only responds to tags"

backend/onyx/server/manage/slack_bot.py

@@ -261,6 +261,9 @@ def create_bot(
     # Create a default Slack channel config
     default_channel_config = ChannelConfig(
         channel_name=None,
+        respond_member_group_list=[],
+        answer_filters=[],
+        follow_up_tags=[],
         respond_tag_only=True,
     )
     insert_slack_channel_config(
@@ -368,9 +371,7 @@ def get_all_channels_from_slack_api(
     _: User | None = Depends(current_admin_user),
 ) -> list[SlackChannel]:
     """
-    Fetches all channels in the Slack workspace using the conversations_list API.
-    This includes both public and private channels that are visible to the app,
-    not just the ones the bot is a member of.
+    Fetches channels the bot is a member of from the Slack API.
     Handles pagination with a limit to avoid excessive API calls.
     """
     tokens = fetch_slack_bot_tokens(db_session, bot_id)
@@ -385,20 +386,20 @@ def get_all_channels_from_slack_api(
     current_page = 0
 
     try:
-        # Use conversations_list to get all channels in the workspace (including ones the bot is not a member of)
+        # Use users_conversations with limited pagination
         while current_page < MAX_SLACK_PAGES:
             current_page += 1
 
             # Make API call with cursor if we have one
             if next_cursor:
-                response = client.conversations_list(
+                response = client.users_conversations(
                     types="public_channel,private_channel",
                     exclude_archived=True,
                     cursor=next_cursor,
                     limit=SLACK_API_CHANNELS_PER_PAGE,
                 )
             else:
-                response = client.conversations_list(
+                response = client.users_conversations(
                     types="public_channel,private_channel",
                     exclude_archived=True,
                     limit=SLACK_API_CHANNELS_PER_PAGE,

backend/requirements/default.txt

@@ -56,7 +56,7 @@ puremagic==1.28
 pyairtable==3.0.1
 pycryptodome==3.19.1
 pydantic==2.8.2
-PyGithub==2.5.0
+PyGithub==1.58.2
 python-dateutil==2.8.2
 python-gitlab==3.9.0
 python-pptx==0.6.23

backend/tests/daily/connectors/github/test_github_basic.py

@@ -1,54 +0,0 @@
-import os
-import time
-
-import pytest
-
-from onyx.configs.constants import DocumentSource
-from onyx.connectors.github.connector import GithubConnector
-from tests.daily.connectors.utils import load_all_docs_from_checkpoint_connector
-
-
-@pytest.fixture
-def github_connector() -> GithubConnector:
-    connector = GithubConnector(
-        repo_owner="onyx-dot-app",
-        repositories="documentation",
-        include_prs=True,
-        include_issues=True,
-    )
-    connector.load_credentials(
-        {
-            "github_access_token": os.environ["ACCESS_TOKEN_GITHUB"],
-        }
-    )
-    return connector
-
-
-def test_github_connector_basic(github_connector: GithubConnector) -> None:
-    docs = load_all_docs_from_checkpoint_connector(
-        connector=github_connector,
-        start=0,
-        end=time.time(),
-    )
-    assert len(docs) > 0  # We expect at least one PR to exist
-
-    # Test the first document's structure
-    doc = docs[0]
-
-    # Verify basic document properties
-    assert doc.source == DocumentSource.GITHUB
-    assert doc.secondary_owners is None
-    assert doc.from_ingestion_api is False
-    assert doc.additional_info is None
-
-    # Verify GitHub-specific properties
-    assert "github.com" in doc.id  # Should be a GitHub URL
-    assert doc.metadata is not None
-    assert "state" in doc.metadata
-    assert "merged" in doc.metadata
-
-    # Verify sections
-    assert len(doc.sections) == 1
-    section = doc.sections[0]
-    assert section.link == doc.id  # Section link should match document ID
-    assert isinstance(section.text, str)  # Should have some text content

backend/tests/daily/connectors/google_drive/conftest.py

@@ -1,6 +1,5 @@
 import json
 import os
-import resource
 from collections.abc import Callable
 
 import pytest
@@ -137,22 +136,3 @@ def google_drive_service_acct_connector_factory() -> (
         return connector
 
     return _connector_factory
-
-
-@pytest.fixture(scope="session", autouse=True)
-def set_resource_limits() -> None:
-    # the google sdk is aggressive about using up file descriptors and
-    # macos is stingy ... these tests will fail randomly unless the descriptor limit is raised
-    RLIMIT_MINIMUM = 2048
-    soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
-    desired_soft = min(RLIMIT_MINIMUM, hard)  # Pick your target here
-
-    print(f"Open file limit: soft={soft} hard={hard} soft_required={RLIMIT_MINIMUM}")
-
-    if soft < desired_soft:
-        print(f"Raising open file limit: {soft} -> {desired_soft}")
-        resource.setrlimit(resource.RLIMIT_NOFILE, (desired_soft, hard))
-
-    soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
-    print(f"New open file limit: soft={soft} hard={hard}")
-    return

backend/tests/daily/connectors/google_drive/consts_and_utils.py

@@ -161,14 +161,10 @@ def _get_expected_file_content(file_id: int) -> str:
     return file_text_template.format(file_id)
 
 
-def assert_expected_docs_in_retrieved_docs(
+def assert_retrieved_docs_match_expected(
     retrieved_docs: list[Document],
     expected_file_ids: Sequence[int],
 ) -> None:
-    """NOTE: as far as i can tell this does NOT assert for an exact match.
-    it only checks to see if that the expected file id's are IN the retrieved doc list
-    """
-
     expected_file_names = {
         file_name_template.format(file_id) for file_id in expected_file_ids
     }
@@ -179,7 +175,7 @@ def assert_expected_docs_in_retrieved_docs(
     retrieved_docs.sort(key=lambda x: x.semantic_identifier)
 
     for doc in retrieved_docs:
-        print(f"retrieved doc: doc.semantic_identifier={doc.semantic_identifier}")
+        print(f"doc.semantic_identifier: {doc.semantic_identifier}")
 
     # Filter out invalid prefixes to prevent different tests from interfering with each other
     valid_retrieved_docs = [

backend/tests/daily/connectors/google_drive/test_admin_oauth.py

@@ -7,7 +7,7 @@ from tests.daily.connectors.google_drive.consts_and_utils import ADMIN_EMAIL
 from tests.daily.connectors.google_drive.consts_and_utils import ADMIN_FILE_IDS
 from tests.daily.connectors.google_drive.consts_and_utils import ADMIN_FOLDER_3_FILE_IDS
 from tests.daily.connectors.google_drive.consts_and_utils import (
-    assert_expected_docs_in_retrieved_docs,
+    assert_retrieved_docs_match_expected,
 )
 from tests.daily.connectors.google_drive.consts_and_utils import FOLDER_1_1_FILE_IDS
 from tests.daily.connectors.google_drive.consts_and_utils import FOLDER_1_1_URL
@@ -62,7 +62,7 @@ def test_include_all(
         + FOLDER_2_2_FILE_IDS
         + SECTIONS_FILE_IDS
     )
-    assert_expected_docs_in_retrieved_docs(
+    assert_retrieved_docs_match_expected(
         retrieved_docs=retrieved_docs,
         expected_file_ids=expected_file_ids,
     )
@@ -100,7 +100,7 @@ def test_include_shared_drives_only(
         + FOLDER_2_2_FILE_IDS
         + SECTIONS_FILE_IDS
     )
-    assert_expected_docs_in_retrieved_docs(
+    assert_retrieved_docs_match_expected(
         retrieved_docs=retrieved_docs,
         expected_file_ids=expected_file_ids,
     )
@@ -128,7 +128,7 @@ def test_include_my_drives_only(
 
     # Should only get primary_admins My Drive because we are impersonating them
     expected_file_ids = ADMIN_FILE_IDS + ADMIN_FOLDER_3_FILE_IDS
-    assert_expected_docs_in_retrieved_docs(
+    assert_retrieved_docs_match_expected(
         retrieved_docs=retrieved_docs,
         expected_file_ids=expected_file_ids,
     )
@@ -161,7 +161,7 @@ def test_drive_one_only(
         + FOLDER_1_1_FILE_IDS
         + FOLDER_1_2_FILE_IDS
     )
-    assert_expected_docs_in_retrieved_docs(
+    assert_retrieved_docs_match_expected(
         retrieved_docs=retrieved_docs,
         expected_file_ids=expected_file_ids,
     )
@@ -198,7 +198,7 @@ def test_folder_and_shared_drive(
         + FOLDER_2_1_FILE_IDS
         + FOLDER_2_2_FILE_IDS
     )
-    assert_expected_docs_in_retrieved_docs(
+    assert_retrieved_docs_match_expected(
         retrieved_docs=retrieved_docs,
         expected_file_ids=expected_file_ids,
     )
@@ -241,7 +241,7 @@ def test_folders_only(
         + FOLDER_2_2_FILE_IDS
         + ADMIN_FOLDER_3_FILE_IDS
     )
-    assert_expected_docs_in_retrieved_docs(
+    assert_retrieved_docs_match_expected(
         retrieved_docs=retrieved_docs,
         expected_file_ids=expected_file_ids,
     )
@@ -271,7 +271,7 @@ def test_personal_folders_only(
     retrieved_docs = load_all_docs(connector)
 
     expected_file_ids = ADMIN_FOLDER_3_FILE_IDS
-    assert_expected_docs_in_retrieved_docs(
+    assert_retrieved_docs_match_expected(
         retrieved_docs=retrieved_docs,
         expected_file_ids=expected_file_ids,
     )

backend/tests/daily/connectors/google_drive/test_service_acct.py

@@ -7,7 +7,7 @@ from tests.daily.connectors.google_drive.consts_and_utils import ADMIN_EMAIL
 from tests.daily.connectors.google_drive.consts_and_utils import ADMIN_FILE_IDS
 from tests.daily.connectors.google_drive.consts_and_utils import ADMIN_FOLDER_3_FILE_IDS
 from tests.daily.connectors.google_drive.consts_and_utils import (
-    assert_expected_docs_in_retrieved_docs,
+    assert_retrieved_docs_match_expected,
 )
 from tests.daily.connectors.google_drive.consts_and_utils import FOLDER_1_1_FILE_IDS
 from tests.daily.connectors.google_drive.consts_and_utils import FOLDER_1_1_URL
@@ -70,39 +70,12 @@ def test_include_all(
         + FOLDER_2_2_FILE_IDS
         + SECTIONS_FILE_IDS
     )
-    assert_expected_docs_in_retrieved_docs(
+    assert_retrieved_docs_match_expected(
         retrieved_docs=retrieved_docs,
         expected_file_ids=expected_file_ids,
     )
 
 
-@patch(
-    "onyx.file_processing.extract_file_text.get_unstructured_api_key",
-    return_value=None,
-)
-def test_include_shared_drives_only_with_size_threshold(
-    mock_get_api_key: MagicMock,
-    google_drive_service_acct_connector_factory: Callable[..., GoogleDriveConnector],
-) -> None:
-    print("\n\nRunning test_include_shared_drives_only_with_size_threshold")
-    connector = google_drive_service_acct_connector_factory(
-        primary_admin_email=ADMIN_EMAIL,
-        include_shared_drives=True,
-        include_my_drives=False,
-        include_files_shared_with_me=False,
-        shared_folder_urls=None,
-        shared_drive_urls=None,
-        my_drive_emails=None,
-    )
-
-    # this threshold will skip one file
-    connector.size_threshold = 16384
-
-    retrieved_docs = load_all_docs(connector)
-
-    assert len(retrieved_docs) == 50
-
-
 @patch(
     "onyx.file_processing.extract_file_text.get_unstructured_api_key",
     return_value=None,
@@ -121,7 +94,6 @@ def test_include_shared_drives_only(
         shared_drive_urls=None,
         my_drive_emails=None,
     )
-
     retrieved_docs = load_all_docs(connector)
 
     # Should only get shared drives
@@ -136,10 +108,7 @@ def test_include_shared_drives_only(
         + FOLDER_2_2_FILE_IDS
         + SECTIONS_FILE_IDS
     )
-
-    assert len(retrieved_docs) == 51
-
-    assert_expected_docs_in_retrieved_docs(
+    assert_retrieved_docs_match_expected(
         retrieved_docs=retrieved_docs,
         expected_file_ids=expected_file_ids,
     )
@@ -173,7 +142,7 @@ def test_include_my_drives_only(
         + TEST_USER_2_FILE_IDS
         + TEST_USER_3_FILE_IDS
     )
-    assert_expected_docs_in_retrieved_docs(
+    assert_retrieved_docs_match_expected(
         retrieved_docs=retrieved_docs,
         expected_file_ids=expected_file_ids,
     )
@@ -207,7 +176,7 @@ def test_drive_one_only(
         + FOLDER_1_1_FILE_IDS
         + FOLDER_1_2_FILE_IDS
     )
-    assert_expected_docs_in_retrieved_docs(
+    assert_retrieved_docs_match_expected(
         retrieved_docs=retrieved_docs,
         expected_file_ids=expected_file_ids,
     )
@@ -245,7 +214,7 @@ def test_folder_and_shared_drive(
         + FOLDER_2_1_FILE_IDS
         + FOLDER_2_2_FILE_IDS
     )
-    assert_expected_docs_in_retrieved_docs(
+    assert_retrieved_docs_match_expected(
         retrieved_docs=retrieved_docs,
         expected_file_ids=expected_file_ids,
     )
@@ -288,7 +257,7 @@ def test_folders_only(
         + FOLDER_2_2_FILE_IDS
         + ADMIN_FOLDER_3_FILE_IDS
     )
-    assert_expected_docs_in_retrieved_docs(
+    assert_retrieved_docs_match_expected(
         retrieved_docs=retrieved_docs,
         expected_file_ids=expected_file_ids,
     )
@@ -319,7 +288,7 @@ def test_specific_emails(
     retrieved_docs = load_all_docs(connector)
 
     expected_file_ids = TEST_USER_1_FILE_IDS + TEST_USER_3_FILE_IDS
-    assert_expected_docs_in_retrieved_docs(
+    assert_retrieved_docs_match_expected(
         retrieved_docs=retrieved_docs,
         expected_file_ids=expected_file_ids,
     )
@@ -349,7 +318,7 @@ def get_specific_folders_in_my_drive(
     retrieved_docs = load_all_docs(connector)
 
     expected_file_ids = ADMIN_FOLDER_3_FILE_IDS
-    assert_expected_docs_in_retrieved_docs(
+    assert_retrieved_docs_match_expected(
         retrieved_docs=retrieved_docs,
         expected_file_ids=expected_file_ids,
     )

backend/tests/daily/connectors/google_drive/test_user_1_oauth.py

@@ -5,7 +5,7 @@ from unittest.mock import patch
 from onyx.connectors.google_drive.connector import GoogleDriveConnector
 from tests.daily.connectors.google_drive.consts_and_utils import ADMIN_FOLDER_3_FILE_IDS
 from tests.daily.connectors.google_drive.consts_and_utils import (
-    assert_expected_docs_in_retrieved_docs,
+    assert_retrieved_docs_match_expected,
 )
 from tests.daily.connectors.google_drive.consts_and_utils import FOLDER_1_1_FILE_IDS
 from tests.daily.connectors.google_drive.consts_and_utils import FOLDER_1_2_FILE_IDS
@@ -50,7 +50,7 @@ def test_all(
         + ADMIN_FOLDER_3_FILE_IDS
         + list(range(0, 2))
     )
-    assert_expected_docs_in_retrieved_docs(
+    assert_retrieved_docs_match_expected(
         retrieved_docs=retrieved_docs,
         expected_file_ids=expected_file_ids,
     )
@@ -83,7 +83,7 @@ def test_shared_drives_only(
         + FOLDER_1_1_FILE_IDS
         + FOLDER_1_2_FILE_IDS
     )
-    assert_expected_docs_in_retrieved_docs(
+    assert_retrieved_docs_match_expected(
         retrieved_docs=retrieved_docs,
         expected_file_ids=expected_file_ids,
     )
@@ -114,7 +114,7 @@ def test_shared_with_me_only(
         ADMIN_FOLDER_3_FILE_IDS
         + list(range(0, 2))
     )
-    assert_expected_docs_in_retrieved_docs(
+    assert_retrieved_docs_match_expected(
         retrieved_docs=retrieved_docs,
         expected_file_ids=expected_file_ids,
     )
@@ -142,7 +142,7 @@ def test_my_drive_only(
 
     # These are the files from my drive
     expected_file_ids = TEST_USER_1_FILE_IDS
-    assert_expected_docs_in_retrieved_docs(
+    assert_retrieved_docs_match_expected(
         retrieved_docs=retrieved_docs,
         expected_file_ids=expected_file_ids,
     )
@@ -172,7 +172,7 @@ def test_shared_my_drive_folder(
         # this is a folder from admin's drive that is shared with me
         ADMIN_FOLDER_3_FILE_IDS
     )
-    assert_expected_docs_in_retrieved_docs(
+    assert_retrieved_docs_match_expected(
         retrieved_docs=retrieved_docs,
         expected_file_ids=expected_file_ids,
     )
@@ -199,7 +199,7 @@ def test_shared_drive_folder(
     retrieved_docs = load_all_docs(connector)
 
     expected_file_ids = FOLDER_1_FILE_IDS + FOLDER_1_1_FILE_IDS + FOLDER_1_2_FILE_IDS
-    assert_expected_docs_in_retrieved_docs(
+    assert_retrieved_docs_match_expected(
         retrieved_docs=retrieved_docs,
         expected_file_ids=expected_file_ids,
     )

backend/tests/daily/connectors/zendesk/test_zendesk_connector.py

@@ -2,14 +2,12 @@ import json
 import os
 import time
 from pathlib import Path
-from typing import cast
 
 import pytest
 
 from onyx.configs.constants import DocumentSource
 from onyx.connectors.models import Document
 from onyx.connectors.zendesk.connector import ZendeskConnector
-from tests.daily.connectors.utils import load_all_docs_from_checkpoint_connector
 
 
 def load_test_data(file_name: str = "test_zendesk_data.json") -> dict[str, dict]:
@@ -52,7 +50,7 @@ def get_credentials() -> dict[str, str]:
 def test_zendesk_connector_basic(
     request: pytest.FixtureRequest, connector_fixture: str
 ) -> None:
-    connector = cast(ZendeskConnector, request.getfixturevalue(connector_fixture))
+    connector = request.getfixturevalue(connector_fixture)
     test_data = load_test_data()
     all_docs: list[Document] = []
     target_test_doc_id: str
@@ -63,11 +61,12 @@ def test_zendesk_connector_basic(
 
     target_doc: Document | None = None
 
-    for doc in load_all_docs_from_checkpoint_connector(connector, 0, time.time()):
-        all_docs.append(doc)
-        if doc.id == target_test_doc_id:
-            target_doc = doc
-            print(f"target_doc {target_doc}")
+    for doc_batch in connector.poll_source(0, time.time()):
+        for doc in doc_batch:
+            all_docs.append(doc)
+            if doc.id == target_test_doc_id:
+                target_doc = doc
+                print(f"target_doc {target_doc}")
 
     assert len(all_docs) > 0, "No documents were retrieved from the connector"
     assert (
@@ -112,10 +111,8 @@ def test_zendesk_connector_basic(
 def test_zendesk_connector_slim(zendesk_article_connector: ZendeskConnector) -> None:
     # Get full doc IDs
     all_full_doc_ids = set()
-    for doc in load_all_docs_from_checkpoint_connector(
-        zendesk_article_connector, 0, time.time()
-    ):
-        all_full_doc_ids.add(doc.id)
+    for doc_batch in zendesk_article_connector.load_from_state():
+        all_full_doc_ids.update([doc.id for doc in doc_batch])
 
     # Get slim doc IDs
     all_slim_doc_ids = set()

backend/tests/unit/onyx/connectors/github/test_github_checkpointing.py

@@ -1,441 +0,0 @@
-import time
-from collections.abc import Callable
-from collections.abc import Generator
-from datetime import datetime
-from datetime import timezone
-from typing import cast
-from unittest.mock import MagicMock
-from unittest.mock import patch
-
-import pytest
-from github import Github
-from github import GithubException
-from github import RateLimitExceededException
-from github.Issue import Issue
-from github.PullRequest import PullRequest
-from github.RateLimit import RateLimit
-from github.Repository import Repository
-from github.Requester import Requester
-
-from onyx.connectors.exceptions import ConnectorValidationError
-from onyx.connectors.exceptions import CredentialExpiredError
-from onyx.connectors.exceptions import InsufficientPermissionsError
-from onyx.connectors.github.connector import GithubConnector
-from onyx.connectors.github.connector import SerializedRepository
-from onyx.connectors.models import Document
-from tests.unit.onyx.connectors.utils import load_everything_from_checkpoint_connector
-
-
-@pytest.fixture
-def repo_owner() -> str:
-    return "test-org"
-
-
-@pytest.fixture
-def repositories() -> str:
-    return "test-repo"
-
-
-@pytest.fixture
-def mock_github_client() -> MagicMock:
-    """Create a mock GitHub client with proper typing"""
-    mock = MagicMock(spec=Github)
-    # Add proper return typing for get_repo method
-    mock.get_repo = MagicMock(return_value=MagicMock(spec=Repository))
-    # Add proper return typing for get_organization method
-    mock.get_organization = MagicMock()
-    # Add proper return typing for get_user method
-    mock.get_user = MagicMock()
-    # Add proper return typing for get_rate_limit method
-    mock.get_rate_limit = MagicMock(return_value=MagicMock(spec=RateLimit))
-    # Add requester for repository deserialization
-    mock.requester = MagicMock(spec=Requester)
-    return mock
-
-
-@pytest.fixture
-def github_connector(
-    repo_owner: str, repositories: str, mock_github_client: MagicMock
-) -> Generator[GithubConnector, None, None]:
-    connector = GithubConnector(
-        repo_owner=repo_owner,
-        repositories=repositories,
-        include_prs=True,
-        include_issues=True,
-    )
-    connector.github_client = mock_github_client
-    yield connector
-
-
-@pytest.fixture
-def create_mock_pr() -> Callable[..., MagicMock]:
-    def _create_mock_pr(
-        number: int = 1,
-        title: str = "Test PR",
-        body: str = "Test Description",
-        state: str = "open",
-        merged: bool = False,
-        updated_at: datetime = datetime(2023, 1, 1, tzinfo=timezone.utc),
-    ) -> MagicMock:
-        """Helper to create a mock PullRequest object"""
-        mock_pr = MagicMock(spec=PullRequest)
-        mock_pr.number = number
-        mock_pr.title = title
-        mock_pr.body = body
-        mock_pr.state = state
-        mock_pr.merged = merged
-        mock_pr.updated_at = updated_at
-        mock_pr.html_url = f"https://github.com/test-org/test-repo/pull/{number}"
-        return mock_pr
-
-    return _create_mock_pr
-
-
-@pytest.fixture
-def create_mock_issue() -> Callable[..., MagicMock]:
-    def _create_mock_issue(
-        number: int = 1,
-        title: str = "Test Issue",
-        body: str = "Test Description",
-        state: str = "open",
-        updated_at: datetime = datetime(2023, 1, 1, tzinfo=timezone.utc),
-    ) -> MagicMock:
-        """Helper to create a mock Issue object"""
-        mock_issue = MagicMock(spec=Issue)
-        mock_issue.number = number
-        mock_issue.title = title
-        mock_issue.body = body
-        mock_issue.state = state
-        mock_issue.updated_at = updated_at
-        mock_issue.html_url = f"https://github.com/test-org/test-repo/issues/{number}"
-        mock_issue.pull_request = None  # Not a PR
-        return mock_issue
-
-    return _create_mock_issue
-
-
-@pytest.fixture
-def create_mock_repo() -> Callable[..., MagicMock]:
-    def _create_mock_repo(
-        name: str = "test-repo",
-        id: int = 1,
-    ) -> MagicMock:
-        """Helper to create a mock Repository object"""
-        mock_repo = MagicMock(spec=Repository)
-        mock_repo.name = name
-        mock_repo.id = id
-        mock_repo.raw_headers = {"status": "200 OK", "content-type": "application/json"}
-        mock_repo.raw_data = {
-            "id": str(id),
-            "name": name,
-            "full_name": f"test-org/{name}",
-            "private": str(False),
-            "description": "Test repository",
-        }
-        return mock_repo
-
-    return _create_mock_repo
-
-
-def test_load_from_checkpoint_happy_path(
-    github_connector: GithubConnector,
-    mock_github_client: MagicMock,
-    create_mock_repo: Callable[..., MagicMock],
-    create_mock_pr: Callable[..., MagicMock],
-    create_mock_issue: Callable[..., MagicMock],
-) -> None:
-    """Test loading from checkpoint - happy path"""
-    # Set up mocked repo
-    mock_repo = create_mock_repo()
-    github_connector.github_client = mock_github_client
-    mock_github_client.get_repo.return_value = mock_repo
-
-    # Set up mocked PRs and issues
-    mock_pr1 = create_mock_pr(number=1, title="PR 1")
-    mock_pr2 = create_mock_pr(number=2, title="PR 2")
-    mock_issue1 = create_mock_issue(number=1, title="Issue 1")
-    mock_issue2 = create_mock_issue(number=2, title="Issue 2")
-
-    # Mock get_pulls and get_issues methods
-    mock_repo.get_pulls.return_value = MagicMock()
-    mock_repo.get_pulls.return_value.get_page.side_effect = [
-        [mock_pr1, mock_pr2],
-        [],
-    ]
-    mock_repo.get_issues.return_value = MagicMock()
-    mock_repo.get_issues.return_value.get_page.side_effect = [
-        [mock_issue1, mock_issue2],
-        [],
-    ]
-
-    # Mock SerializedRepository.to_Repository to return our mock repo
-    with patch.object(SerializedRepository, "to_Repository", return_value=mock_repo):
-        # Call load_from_checkpoint
-        end_time = time.time()
-        outputs = load_everything_from_checkpoint_connector(
-            github_connector, 0, end_time
-        )
-
-        # Check that we got all documents and final has_more=False
-        assert len(outputs) == 4
-
-        repo_batch = outputs[0]
-        assert len(repo_batch.items) == 0
-        assert repo_batch.next_checkpoint.has_more is True
-
-        # Check first batch (PRs)
-        first_batch = outputs[1]
-        assert len(first_batch.items) == 2
-        assert isinstance(first_batch.items[0], Document)
-        assert first_batch.items[0].id == "https://github.com/test-org/test-repo/pull/1"
-        assert isinstance(first_batch.items[1], Document)
-        assert first_batch.items[1].id == "https://github.com/test-org/test-repo/pull/2"
-        assert first_batch.next_checkpoint.curr_page == 1
-
-        # Check second batch (Issues)
-        second_batch = outputs[2]
-        assert len(second_batch.items) == 2
-        assert isinstance(second_batch.items[0], Document)
-        assert (
-            second_batch.items[0].id == "https://github.com/test-org/test-repo/issues/1"
-        )
-        assert isinstance(second_batch.items[1], Document)
-        assert (
-            second_batch.items[1].id == "https://github.com/test-org/test-repo/issues/2"
-        )
-        assert second_batch.next_checkpoint.has_more
-
-        # Check third batch (finished checkpoint)
-        third_batch = outputs[3]
-        assert len(third_batch.items) == 0
-        assert third_batch.next_checkpoint.has_more is False
-
-
-def test_load_from_checkpoint_with_rate_limit(
-    github_connector: GithubConnector,
-    mock_github_client: MagicMock,
-    create_mock_repo: Callable[..., MagicMock],
-    create_mock_pr: Callable[..., MagicMock],
-) -> None:
-    """Test loading from checkpoint with rate limit handling"""
-    # Set up mocked repo
-    mock_repo = create_mock_repo()
-    github_connector.github_client = mock_github_client
-    mock_github_client.get_repo.return_value = mock_repo
-
-    # Set up mocked PR
-    mock_pr = create_mock_pr()
-
-    # Mock get_pulls to raise RateLimitExceededException on first call
-    mock_repo.get_pulls.return_value = MagicMock()
-    mock_repo.get_pulls.return_value.get_page.side_effect = [
-        RateLimitExceededException(403, {"message": "Rate limit exceeded"}, {}),
-        [mock_pr],
-        [],
-    ]
-
-    # Mock rate limit reset time
-    mock_rate_limit = MagicMock(spec=RateLimit)
-    mock_rate_limit.core.reset = datetime.now(timezone.utc)
-    github_connector.github_client.get_rate_limit.return_value = mock_rate_limit
-
-    # Mock SerializedRepository.to_Repository to return our mock repo
-    with patch.object(SerializedRepository, "to_Repository", return_value=mock_repo):
-        # Call load_from_checkpoint
-        end_time = time.time()
-        with patch(
-            "onyx.connectors.github.connector._sleep_after_rate_limit_exception"
-        ) as mock_sleep:
-            outputs = load_everything_from_checkpoint_connector(
-                github_connector, 0, end_time
-            )
-
-            assert mock_sleep.call_count == 1
-
-        # Check that we got the document after rate limit was handled
-        assert len(outputs) >= 2
-        assert len(outputs[1].items) == 1
-        assert isinstance(outputs[1].items[0], Document)
-        assert outputs[1].items[0].id == "https://github.com/test-org/test-repo/pull/1"
-
-        assert outputs[-1].next_checkpoint.has_more is False
-
-
-def test_load_from_checkpoint_with_empty_repo(
-    github_connector: GithubConnector,
-    mock_github_client: MagicMock,
-    create_mock_repo: Callable[..., MagicMock],
-) -> None:
-    """Test loading from checkpoint with an empty repository"""
-    # Set up mocked repo
-    mock_repo = create_mock_repo()
-    github_connector.github_client = mock_github_client
-    mock_github_client.get_repo.return_value = mock_repo
-
-    # Mock get_pulls and get_issues to return empty lists
-    mock_repo.get_pulls.return_value = MagicMock()
-    mock_repo.get_pulls.return_value.get_page.return_value = []
-    mock_repo.get_issues.return_value = MagicMock()
-    mock_repo.get_issues.return_value.get_page.return_value = []
-
-    # Mock SerializedRepository.to_Repository to return our mock repo
-    with patch.object(SerializedRepository, "to_Repository", return_value=mock_repo):
-        # Call load_from_checkpoint
-        end_time = time.time()
-        outputs = load_everything_from_checkpoint_connector(
-            github_connector, 0, end_time
-        )
-
-        # Check that we got no documents
-        assert len(outputs) == 2
-        assert len(outputs[-1].items) == 0
-        assert not outputs[-1].next_checkpoint.has_more
-
-
-def test_load_from_checkpoint_with_prs_only(
-    github_connector: GithubConnector,
-    mock_github_client: MagicMock,
-    create_mock_repo: Callable[..., MagicMock],
-    create_mock_pr: Callable[..., MagicMock],
-) -> None:
-    """Test loading from checkpoint with only PRs enabled"""
-    # Configure connector to only include PRs
-    github_connector.include_prs = True
-    github_connector.include_issues = False
-
-    # Set up mocked repo
-    mock_repo = create_mock_repo()
-    github_connector.github_client = mock_github_client
-    mock_github_client.get_repo.return_value = mock_repo
-
-    # Set up mocked PRs
-    mock_pr1 = create_mock_pr(number=1, title="PR 1")
-    mock_pr2 = create_mock_pr(number=2, title="PR 2")
-
-    # Mock get_pulls method
-    mock_repo.get_pulls.return_value = MagicMock()
-    mock_repo.get_pulls.return_value.get_page.side_effect = [
-        [mock_pr1, mock_pr2],
-        [],
-    ]
-
-    # Mock SerializedRepository.to_Repository to return our mock repo
-    with patch.object(SerializedRepository, "to_Repository", return_value=mock_repo):
-        # Call load_from_checkpoint
-        end_time = time.time()
-        outputs = load_everything_from_checkpoint_connector(
-            github_connector, 0, end_time
-        )
-
-        # Check that we only got PRs
-        assert len(outputs) >= 2
-        assert len(outputs[1].items) == 2
-        assert all(
-            isinstance(doc, Document) and "pull" in doc.id for doc in outputs[0].items
-        )  # All documents should be PRs
-
-        assert outputs[-1].next_checkpoint.has_more is False
-
-
-def test_load_from_checkpoint_with_issues_only(
-    github_connector: GithubConnector,
-    mock_github_client: MagicMock,
-    create_mock_repo: Callable[..., MagicMock],
-    create_mock_issue: Callable[..., MagicMock],
-) -> None:
-    """Test loading from checkpoint with only issues enabled"""
-    # Configure connector to only include issues
-    github_connector.include_prs = False
-    github_connector.include_issues = True
-
-    # Set up mocked repo
-    mock_repo = create_mock_repo()
-    github_connector.github_client = mock_github_client
-    mock_github_client.get_repo.return_value = mock_repo
-
-    # Set up mocked issues
-    mock_issue1 = create_mock_issue(number=1, title="Issue 1")
-    mock_issue2 = create_mock_issue(number=2, title="Issue 2")
-
-    # Mock get_issues method
-    mock_repo.get_issues.return_value = MagicMock()
-    mock_repo.get_issues.return_value.get_page.side_effect = [
-        [mock_issue1, mock_issue2],
-        [],
-    ]
-
-    # Mock SerializedRepository.to_Repository to return our mock repo
-    with patch.object(SerializedRepository, "to_Repository", return_value=mock_repo):
-        # Call load_from_checkpoint
-        end_time = time.time()
-        outputs = load_everything_from_checkpoint_connector(
-            github_connector, 0, end_time
-        )
-
-        # Check that we only got issues
-        assert len(outputs) >= 2
-        assert len(outputs[1].items) == 2
-        assert all(
-            isinstance(doc, Document) and "issues" in doc.id for doc in outputs[0].items
-        )  # All documents should be issues
-        assert outputs[1].next_checkpoint.has_more
-
-        assert outputs[-1].next_checkpoint.has_more is False
-
-
-@pytest.mark.parametrize(
-    "status_code,expected_exception,expected_message",
-    [
-        (
-            401,
-            CredentialExpiredError,
-            "GitHub credential appears to be invalid or expired",
-        ),
-        (
-            403,
-            InsufficientPermissionsError,
-            "Your GitHub token does not have sufficient permissions",
-        ),
-        (
-            404,
-            ConnectorValidationError,
-            "GitHub repository not found",
-        ),
-    ],
-)
-def test_validate_connector_settings_errors(
-    github_connector: GithubConnector,
-    status_code: int,
-    expected_exception: type[Exception],
-    expected_message: str,
-) -> None:
-    """Test validation with various error scenarios"""
-    error = GithubException(status=status_code, data={}, headers={})
-
-    github_client = cast(Github, github_connector.github_client)
-    get_repo_mock = cast(MagicMock, github_client.get_repo)
-    get_repo_mock.side_effect = error
-
-    with pytest.raises(expected_exception) as excinfo:
-        github_connector.validate_connector_settings()
-    assert expected_message in str(excinfo.value)
-
-
-def test_validate_connector_settings_success(
-    github_connector: GithubConnector,
-    mock_github_client: MagicMock,
-    create_mock_repo: Callable[..., MagicMock],
-) -> None:
-    """Test successful validation"""
-    # Set up mocked repo
-    mock_repo = create_mock_repo()
-    github_connector.github_client = mock_github_client
-    mock_github_client.get_repo.return_value = mock_repo
-
-    # Mock get_contents to simulate successful access
-    mock_repo.get_contents.return_value = MagicMock()
-
-    github_connector.validate_connector_settings()
-    github_connector.github_client.get_repo.assert_called_once_with(
-        f"{github_connector.repo_owner}/{github_connector.repositories}"
-    )

backend/tests/unit/onyx/connectors/zendesk/test_zendesk_checkpointing.py

@@ -1,472 +0,0 @@
-import time
-from collections.abc import Callable
-from collections.abc import Generator
-from typing import Any
-from typing import cast
-from unittest.mock import call
-from unittest.mock import MagicMock
-from unittest.mock import patch
-
-import pytest
-from requests.exceptions import HTTPError
-
-from onyx.configs.constants import DocumentSource
-from onyx.connectors.exceptions import ConnectorValidationError
-from onyx.connectors.exceptions import CredentialExpiredError
-from onyx.connectors.exceptions import InsufficientPermissionsError
-from onyx.connectors.models import Document
-from onyx.connectors.zendesk.connector import ZendeskClient
-from onyx.connectors.zendesk.connector import ZendeskConnector
-from tests.unit.onyx.connectors.utils import load_everything_from_checkpoint_connector
-
-
-@pytest.fixture
-def mock_zendesk_client() -> MagicMock:
-    """Create a mock Zendesk client"""
-    mock = MagicMock(spec=ZendeskClient)
-    mock.base_url = "https://test.zendesk.com/api/v2"
-    mock.auth = ("test@example.com/token", "test_token")
-    return mock
-
-
-@pytest.fixture
-def zendesk_connector(
-    mock_zendesk_client: MagicMock,
-) -> Generator[ZendeskConnector, None, None]:
-    """Create a Zendesk connector with mocked client"""
-    connector = ZendeskConnector(content_type="articles")
-    connector.client = mock_zendesk_client
-    yield connector
-
-
-@pytest.fixture
-def unmocked_zendesk_connector() -> Generator[ZendeskConnector, None, None]:
-    """Create a Zendesk connector with unmocked client"""
-    zendesk_connector = ZendeskConnector(content_type="articles")
-    zendesk_connector.client = ZendeskClient(
-        "test", "test@example.com/token", "test_token"
-    )
-    yield zendesk_connector
-
-
-@pytest.fixture
-def create_mock_article() -> Callable[..., dict[str, Any]]:
-    def _create_mock_article(
-        id: int = 1,
-        title: str = "Test Article",
-        body: str = "Test Content",
-        updated_at: str = "2023-01-01T12:00:00Z",
-        author_id: str = "123",
-        label_names: list[str] | None = None,
-        draft: bool = False,
-    ) -> dict[str, Any]:
-        """Helper to create a mock article"""
-        return {
-            "id": id,
-            "title": title,
-            "body": body,
-            "updated_at": updated_at,
-            "author_id": author_id,
-            "label_names": label_names or [],
-            "draft": draft,
-            "html_url": f"https://test.zendesk.com/hc/en-us/articles/{id}",
-        }
-
-    return _create_mock_article
-
-
-@pytest.fixture
-def create_mock_ticket() -> Callable[..., dict[str, Any]]:
-    def _create_mock_ticket(
-        id: int = 1,
-        subject: str = "Test Ticket",
-        description: str = "Test Description",
-        updated_at: str = "2023-01-01T12:00:00Z",
-        submitter_id: str = "123",
-        status: str = "open",
-        priority: str = "normal",
-        tags: list[str] | None = None,
-        ticket_type: str = "question",
-    ) -> dict[str, Any]:
-        """Helper to create a mock ticket"""
-        return {
-            "id": id,
-            "subject": subject,
-            "description": description,
-            "updated_at": updated_at,
-            "submitter": submitter_id,
-            "status": status,
-            "priority": priority,
-            "tags": tags or [],
-            "type": ticket_type,
-            "url": f"https://test.zendesk.com/agent/tickets/{id}",
-        }
-
-    return _create_mock_ticket
-
-
-@pytest.fixture
-def create_mock_author() -> Callable[..., dict[str, Any]]:
-    def _create_mock_author(
-        id: str = "123",
-        name: str = "Test User",
-        email: str = "test@example.com",
-    ) -> dict[str, Any]:
-        """Helper to create a mock author"""
-        return {
-            "user": {
-                "id": id,
-                "name": name,
-                "email": email,
-            }
-        }
-
-    return _create_mock_author
-
-
-def test_load_from_checkpoint_articles_happy_path(
-    zendesk_connector: ZendeskConnector,
-    mock_zendesk_client: MagicMock,
-    create_mock_article: Callable[..., dict[str, Any]],
-    create_mock_author: Callable[..., dict[str, Any]],
-) -> None:
-    """Test loading articles from checkpoint - happy path"""
-    # Set up mock responses
-    mock_article1 = create_mock_article(id=1, title="Article 1")
-    mock_article2 = create_mock_article(id=2, title="Article 2")
-    mock_author = create_mock_author()
-
-    # Mock API responses
-    mock_zendesk_client.make_request.side_effect = [
-        # First call: content tags
-        {"records": []},
-        # Second call: articles page
-        {
-            "articles": [mock_article1, mock_article2],
-            "meta": {
-                "has_more": False,
-                "after_cursor": None,
-            },
-        },
-        # Third call: author info
-        mock_author,
-    ]
-
-    # Call load_from_checkpoint
-    end_time = time.time()
-    outputs = load_everything_from_checkpoint_connector(zendesk_connector, 0, end_time)
-
-    # Check that we got the documents
-    assert len(outputs) == 2
-    assert outputs[0].next_checkpoint.cached_content_tags is not None
-
-    assert len(outputs[1].items) == 2
-
-    # Check first document
-    doc1 = outputs[1].items[0]
-    assert isinstance(doc1, Document)
-    assert doc1.id == "article:1"
-    assert doc1.semantic_identifier == "Article 1"
-    assert doc1.source == DocumentSource.ZENDESK
-
-    # Check second document
-    doc2 = outputs[1].items[1]
-    assert isinstance(doc2, Document)
-    assert doc2.id == "article:2"
-    assert doc2.semantic_identifier == "Article 2"
-    assert doc2.source == DocumentSource.ZENDESK
-
-    # Check checkpoint state
-    assert not outputs[1].next_checkpoint.has_more
-
-
-def test_load_from_checkpoint_tickets_happy_path(
-    zendesk_connector: ZendeskConnector,
-    mock_zendesk_client: MagicMock,
-    create_mock_ticket: Callable[..., dict[str, Any]],
-    create_mock_author: Callable[..., dict[str, Any]],
-) -> None:
-    """Test loading tickets from checkpoint - happy path"""
-    # Configure connector for tickets
-    zendesk_connector.content_type = "tickets"
-
-    # Set up mock responses
-    mock_ticket1 = create_mock_ticket(id=1, subject="Ticket 1")
-    mock_ticket2 = create_mock_ticket(id=2, subject="Ticket 2")
-    mock_author = create_mock_author()
-
-    # Mock API responses
-    mock_zendesk_client.make_request.side_effect = [
-        # First call: content tags
-        {"records": []},
-        # Second call: tickets page
-        {
-            "tickets": [mock_ticket1, mock_ticket2],
-            "end_of_stream": True,
-            "end_time": int(time.time()),
-        },
-        # Third call: author info
-        mock_author,
-        # Fourth call: comments page
-        {"comments": []},
-        # Fifth call: comments page
-        {"comments": []},
-    ]
-
-    zendesk_connector.client = mock_zendesk_client
-
-    # Call load_from_checkpoint
-    end_time = time.time()
-    outputs = load_everything_from_checkpoint_connector(zendesk_connector, 0, end_time)
-
-    # Check that we got the documents
-    assert len(outputs) == 2
-    assert outputs[0].next_checkpoint.cached_content_tags is not None
-    assert len(outputs[1].items) == 2
-
-    # Check first document
-    doc1 = outputs[1].items[0]
-    print(doc1, type(doc1))
-    assert isinstance(doc1, Document)
-    assert doc1.id == "zendesk_ticket_1"
-    assert doc1.semantic_identifier == "Ticket #1: Ticket 1"
-    assert doc1.source == DocumentSource.ZENDESK
-
-    # Check second document
-    doc2 = outputs[1].items[1]
-    assert isinstance(doc2, Document)
-    assert doc2.id == "zendesk_ticket_2"
-    assert doc2.semantic_identifier == "Ticket #2: Ticket 2"
-    assert doc2.source == DocumentSource.ZENDESK
-
-    # Check checkpoint state
-    assert not outputs[1].next_checkpoint.has_more
-
-
-def test_load_from_checkpoint_with_rate_limit(
-    unmocked_zendesk_connector: ZendeskConnector,
-    create_mock_article: Callable[..., dict[str, Any]],
-    create_mock_author: Callable[..., dict[str, Any]],
-) -> None:
-    """Test loading from checkpoint with rate limit handling"""
-    zendesk_connector = unmocked_zendesk_connector
-    # Set up mock responses
-    mock_article = create_mock_article()
-    mock_author = create_mock_author()
-    author_response = MagicMock()
-    author_response.status_code = 200
-    author_response.json.return_value = mock_author
-
-    # Create mock responses for requests.get
-    rate_limit_response = MagicMock()
-    rate_limit_response.status_code = 429
-    rate_limit_response.headers = {"Retry-After": "60"}
-    rate_limit_response.raise_for_status.side_effect = HTTPError(
-        response=rate_limit_response
-    )
-
-    success_response = MagicMock()
-    success_response.status_code = 200
-    success_response.json.return_value = {
-        "articles": [mock_article],
-        "meta": {
-            "has_more": False,
-            "after_cursor": None,
-        },
-    }
-
-    # Mock requests.get to simulate rate limit then success
-    with patch("onyx.connectors.zendesk.connector.requests.get") as mock_get:
-        mock_get.side_effect = [
-            # First call: content tags
-            MagicMock(
-                status_code=200,
-                json=lambda: {"records": [], "meta": {"has_more": False}},
-            ),
-            # Second call: articles page (rate limited)
-            rate_limit_response,
-            # Third call: articles page (after rate limit)
-            success_response,
-            # Fourth call: author info
-            author_response,
-        ]
-
-        # Call load_from_checkpoint
-        end_time = time.time()
-        with patch("onyx.connectors.zendesk.connector.time.sleep") as mock_sleep:
-            outputs = load_everything_from_checkpoint_connector(
-                zendesk_connector, 0, end_time
-            )
-            mock_sleep.assert_has_calls([call(60), call(0.1)])
-
-        # Check that we got the document after rate limit was handled
-        assert len(outputs) == 2
-        assert outputs[0].next_checkpoint.cached_content_tags is not None
-        assert len(outputs[1].items) == 1
-        assert isinstance(outputs[1].items[0], Document)
-        assert outputs[1].items[0].id == "article:1"
-
-        # Verify the requests were made with correct parameters
-        assert mock_get.call_count == 4
-        # First call should be for content tags
-        args, kwargs = mock_get.call_args_list[0]
-        assert "guide/content_tags" in args[0]
-        # Second call should be for articles (rate limited)
-        args, kwargs = mock_get.call_args_list[1]
-        assert "help_center/articles" in args[0]
-        # Third call should be for articles (success)
-        args, kwargs = mock_get.call_args_list[2]
-        assert "help_center/articles" in args[0]
-        # Fourth call should be for author info
-        args, kwargs = mock_get.call_args_list[3]
-        assert "users/123" in args[0]
-
-
-def test_load_from_checkpoint_with_empty_response(
-    zendesk_connector: ZendeskConnector,
-    mock_zendesk_client: MagicMock,
-) -> None:
-    """Test loading from checkpoint with empty response"""
-    # Mock API responses
-    mock_zendesk_client.make_request.side_effect = [
-        # First call: content tags
-        {"records": []},
-        # Second call: empty articles page
-        {
-            "articles": [],
-            "meta": {
-                "has_more": False,
-                "after_cursor": None,
-            },
-        },
-    ]
-
-    # Call load_from_checkpoint
-    end_time = time.time()
-    outputs = load_everything_from_checkpoint_connector(zendesk_connector, 0, end_time)
-
-    # Check that we got no documents
-    assert len(outputs) == 2
-    assert outputs[0].next_checkpoint.cached_content_tags is not None
-    assert len(outputs[1].items) == 0
-    assert not outputs[1].next_checkpoint.has_more
-
-
-def test_load_from_checkpoint_with_skipped_article(
-    zendesk_connector: ZendeskConnector,
-    mock_zendesk_client: MagicMock,
-    create_mock_article: Callable[..., dict[str, Any]],
-) -> None:
-    """Test loading from checkpoint with an article that should be skipped"""
-    # Set up mock responses with a draft article
-    mock_article = create_mock_article(draft=True)
-    mock_zendesk_client.make_request.side_effect = [
-        # First call: content tags
-        {"records": []},
-        # Second call: articles page with draft article
-        {
-            "articles": [mock_article],
-            "meta": {
-                "has_more": False,
-                "after_cursor": None,
-            },
-        },
-    ]
-
-    # Call load_from_checkpoint
-    end_time = time.time()
-    outputs = load_everything_from_checkpoint_connector(zendesk_connector, 0, end_time)
-
-    # Check that no documents were returned
-    assert len(outputs) == 2
-    assert outputs[0].next_checkpoint.cached_content_tags is not None
-    assert len(outputs[1].items) == 0
-    assert not outputs[1].next_checkpoint.has_more
-
-
-def test_load_from_checkpoint_with_skipped_ticket(
-    zendesk_connector: ZendeskConnector,
-    mock_zendesk_client: MagicMock,
-    create_mock_ticket: Callable[..., dict[str, Any]],
-) -> None:
-    """Test loading from checkpoint with a deleted ticket"""
-    # Configure connector for tickets
-    zendesk_connector.content_type = "tickets"
-
-    # Set up mock responses with a deleted ticket
-    mock_ticket = create_mock_ticket(status="deleted")
-    mock_zendesk_client.make_request.side_effect = [
-        # First call: content tags
-        {"records": []},
-        # Second call: tickets page with deleted ticket
-        {
-            "tickets": [mock_ticket],
-            "end_of_stream": True,
-            "end_time": int(time.time()),
-        },
-    ]
-
-    # Call load_from_checkpoint
-    end_time = time.time()
-    outputs = load_everything_from_checkpoint_connector(zendesk_connector, 0, end_time)
-
-    # Check that no documents were returned
-    assert len(outputs) == 2
-    assert outputs[0].next_checkpoint.cached_content_tags is not None
-    assert len(outputs[1].items) == 0
-    assert not outputs[1].next_checkpoint.has_more
-
-
-@pytest.mark.parametrize(
-    "status_code,expected_exception,expected_message",
-    [
-        (
-            401,
-            CredentialExpiredError,
-            "Your Zendesk credentials appear to be invalid or expired",
-        ),
-        (
-            403,
-            InsufficientPermissionsError,
-            "Your Zendesk token does not have sufficient permissions",
-        ),
-        (
-            404,
-            ConnectorValidationError,
-            "Zendesk resource not found",
-        ),
-    ],
-)
-def test_validate_connector_settings_errors(
-    zendesk_connector: ZendeskConnector,
-    status_code: int,
-    expected_exception: type[Exception],
-    expected_message: str,
-) -> None:
-    """Test validation with various error scenarios"""
-    mock_response = MagicMock()
-    mock_response.status_code = status_code
-    error = HTTPError(response=mock_response)
-
-    mock_zendesk_client = cast(MagicMock, zendesk_connector.client)
-    mock_zendesk_client.make_request.side_effect = error
-
-    with pytest.raises(expected_exception) as excinfo:
-        print("excinfo", excinfo)
-        zendesk_connector.validate_connector_settings()
-
-    assert expected_message in str(excinfo.value)
-
-
-def test_validate_connector_settings_success(
-    zendesk_connector: ZendeskConnector,
-    mock_zendesk_client: MagicMock,
-) -> None:
-    """Test successful validation"""
-    # Mock successful API response
-    mock_zendesk_client.make_request.return_value = {
-        "articles": [],
-        "meta": {"has_more": False},
-    }
-
-    zendesk_connector.validate_connector_settings()

web/src/app/admin/bots/[bot-id]/channels/SlackChannelConfigFormFields.tsx

@@ -254,14 +254,14 @@ export function SlackChannelConfigFormFields({
                       onSearchTermChange={(term) => {
                         form.setFieldValue("channel_name", term);
                       }}
-                      allowCustomValues={true}
                     />
                   )}
                 </Field>
                 <p className="mt-2 text-sm dark:text-neutral-400 text-neutral-600">
-                  Note: This list shows existing public and private channels (up
-                  to 500). You can either select from the list or type any
-                  channel name directly.
+                  Note: This list shows public and private channels where the
+                  bot is a member (up to 500 channels). If you don&apos;t see a
+                  channel, make sure the bot is added to that channel in Slack
+                  first, or type the channel name manually.
                 </p>
               </>
             )}

web/src/app/admin/connectors/[connector]/pages/gdrive/Credential.tsx

@@ -1,3 +1,4 @@
+import { Button } from "@/components/Button";
 import { PopupSpec } from "@/components/admin/connectors/Popup";
 import React, { useState, useEffect } from "react";
 import { useSWRConfig } from "swr";
@@ -7,14 +8,10 @@ import { adminDeleteCredential } from "@/lib/credential";
 import { setupGoogleDriveOAuth } from "@/lib/googleDrive";
 import { GOOGLE_DRIVE_AUTH_IS_ADMIN_COOKIE_NAME } from "@/lib/constants";
 import Cookies from "js-cookie";
-import {
-  TextFormField,
-  SectionHeader,
-  SubLabel,
-} from "@/components/admin/connectors/Field";
+import { TextFormField } from "@/components/admin/connectors/Field";
 import { Form, Formik } from "formik";
 import { User } from "@/lib/types";
-import { Button } from "@/components/ui/button";
+import { Button as TremorButton } from "@/components/ui/button";
 import {
   Credential,
   GoogleDriveCredentialJson,
@@ -23,15 +20,6 @@ import {
 import { refreshAllGoogleData } from "@/lib/googleConnector";
 import { ValidSources } from "@/lib/types";
 import { buildSimilarCredentialInfoURL } from "@/app/admin/connector/[ccPairId]/lib";
-import {
-  FiFile,
-  FiUpload,
-  FiTrash2,
-  FiCheck,
-  FiLink,
-  FiAlertTriangle,
-} from "react-icons/fi";
-import { cn, truncateString } from "@/lib/utils";
 
 type GoogleDriveCredentialJsonTypes = "authorized_user" | "service_account";
 
@@ -43,202 +31,126 @@ export const DriveJsonUpload = ({
   onSuccess?: () => void;
 }) => {
   const { mutate } = useSWRConfig();
-  const [isUploading, setIsUploading] = useState(false);
-  const [fileName, setFileName] = useState<string | undefined>();
-  const [isDragging, setIsDragging] = useState(false);
-
-  const handleFileUpload = async (file: File) => {
-    setIsUploading(true);
-    setFileName(file.name);
-
-    const reader = new FileReader();
-    reader.onload = async (loadEvent) => {
-      if (!loadEvent?.target?.result) {
-        setIsUploading(false);
-        return;
-      }
-
-      const credentialJsonStr = loadEvent.target.result as string;
-
-      // Check credential type
-      let credentialFileType: GoogleDriveCredentialJsonTypes;
-      try {
-        const appCredentialJson = JSON.parse(credentialJsonStr);
-        if (appCredentialJson.web) {
-          credentialFileType = "authorized_user";
-        } else if (appCredentialJson.type === "service_account") {
-          credentialFileType = "service_account";
-        } else {
-          throw new Error(
-            "Unknown credential type, expected one of 'OAuth Web application' or 'Service Account'"
-          );
-        }
-      } catch (e) {
-        setPopup({
-          message: `Invalid file provided - ${e}`,
-          type: "error",
-        });
-        setIsUploading(false);
-        return;
-      }
-
-      if (credentialFileType === "authorized_user") {
-        const response = await fetch(
-          "/api/manage/admin/connector/google-drive/app-credential",
-          {
-            method: "PUT",
-            headers: {
-              "Content-Type": "application/json",
-            },
-            body: credentialJsonStr,
-          }
-        );
-        if (response.ok) {
-          setPopup({
-            message: "Successfully uploaded app credentials",
-            type: "success",
-          });
-          mutate("/api/manage/admin/connector/google-drive/app-credential");
-          if (onSuccess) {
-            onSuccess();
-          }
-        } else {
-          const errorMsg = await response.text();
-          setPopup({
-            message: `Failed to upload app credentials - ${errorMsg}`,
-            type: "error",
-          });
-        }
-      }
-
-      if (credentialFileType === "service_account") {
-        const response = await fetch(
-          "/api/manage/admin/connector/google-drive/service-account-key",
-          {
-            method: "PUT",
-            headers: {
-              "Content-Type": "application/json",
-            },
-            body: credentialJsonStr,
-          }
-        );
-        if (response.ok) {
-          setPopup({
-            message: "Successfully uploaded service account key",
-            type: "success",
-          });
-          mutate(
-            "/api/manage/admin/connector/google-drive/service-account-key"
-          );
-          if (onSuccess) {
-            onSuccess();
-          }
-        } else {
-          const errorMsg = await response.text();
-          setPopup({
-            message: `Failed to upload service account key - ${errorMsg}`,
-            type: "error",
-          });
-        }
-      }
-      setIsUploading(false);
-    };
-
-    reader.readAsText(file);
-  };
-
-  const handleDragEnter = (e: React.DragEvent<HTMLLabelElement>) => {
-    e.preventDefault();
-    e.stopPropagation();
-    if (!isUploading) {
-      setIsDragging(true);
-    }
-  };
-
-  const handleDragLeave = (e: React.DragEvent<HTMLLabelElement>) => {
-    e.preventDefault();
-    e.stopPropagation();
-    setIsDragging(false);
-  };
-
-  const handleDragOver = (e: React.DragEvent<HTMLLabelElement>) => {
-    e.preventDefault();
-    e.stopPropagation();
-  };
-
-  const handleDrop = (e: React.DragEvent<HTMLLabelElement>) => {
-    e.preventDefault();
-    e.stopPropagation();
-    setIsDragging(false);
-
-    if (isUploading) return;
-
-    const files = e.dataTransfer.files;
-    if (files.length > 0) {
-      const file = files[0];
-      if (file.type === "application/json" || file.name.endsWith(".json")) {
-        handleFileUpload(file);
-      } else {
-        setPopup({
-          message: "Please upload a JSON file",
-          type: "error",
-        });
-      }
-    }
-  };
+  const [credentialJsonStr, setCredentialJsonStr] = useState<
+    string | undefined
+  >();
 
   return (
-    <div className="flex flex-col mt-4">
-      <div className="flex items-center">
-        <div className="relative flex flex-1 items-center">
-          <label
-            className={cn(
-              "flex h-10 items-center justify-center w-full px-4 py-2 border border-dashed rounded-md transition-colors",
-              isUploading
-                ? "opacity-70 cursor-not-allowed border-background-400 bg-background-50/30"
-                : isDragging
-                  ? "bg-background-50/50 border-primary dark:border-primary"
-                  : "cursor-pointer hover:bg-background-50/30 hover:border-primary dark:hover:border-primary border-background-300 dark:border-background-600"
-            )}
-            onDragEnter={handleDragEnter}
-            onDragLeave={handleDragLeave}
-            onDragOver={handleDragOver}
-            onDrop={handleDrop}
-          >
-            <div className="flex items-center space-x-2">
-              {isUploading ? (
-                <div className="h-4 w-4 border-t-2 border-b-2 border-primary rounded-full animate-spin"></div>
-              ) : (
-                <FiFile className="h-4 w-4 text-text-500" />
-              )}
-              <span className="text-sm text-text-500">
-                {isUploading
-                  ? `Uploading ${truncateString(fileName || "file", 50)}...`
-                  : isDragging
-                    ? "Drop JSON file here"
-                    : truncateString(
-                        fileName || "Select or drag JSON credentials file...",
-                        50
-                      )}
-              </span>
-            </div>
-            <input
-              className="sr-only"
-              type="file"
-              accept=".json"
-              disabled={isUploading}
-              onChange={(event) => {
-                if (!event.target.files?.length) {
-                  return;
-                }
-                const file = event.target.files[0];
-                handleFileUpload(file);
-              }}
-            />
-          </label>
-        </div>
-      </div>
-    </div>
+    <>
+      <input
+        className={
+          "mr-3 text-sm text-text-900 border border-background-300 " +
+          "cursor-pointer bg-backgrournd dark:text-text-400 focus:outline-none " +
+          "dark:bg-background-700 dark:border-background-600 dark:placeholder-text-400"
+        }
+        type="file"
+        accept=".json"
+        onChange={(event) => {
+          if (!event.target.files) {
+            return;
+          }
+          const file = event.target.files[0];
+          const reader = new FileReader();
+
+          reader.onload = function (loadEvent) {
+            if (!loadEvent?.target?.result) {
+              return;
+            }
+            const fileContents = loadEvent.target.result;
+            setCredentialJsonStr(fileContents as string);
+          };
+
+          reader.readAsText(file);
+        }}
+      />
+
+      <Button
+        disabled={!credentialJsonStr}
+        onClick={async () => {
+          let credentialFileType: GoogleDriveCredentialJsonTypes;
+          try {
+            const appCredentialJson = JSON.parse(credentialJsonStr!);
+            if (appCredentialJson.web) {
+              credentialFileType = "authorized_user";
+            } else if (appCredentialJson.type === "service_account") {
+              credentialFileType = "service_account";
+            } else {
+              throw new Error(
+                "Unknown credential type, expected one of 'OAuth Web application' or 'Service Account'"
+              );
+            }
+          } catch (e) {
+            setPopup({
+              message: `Invalid file provided - ${e}`,
+              type: "error",
+            });
+            return;
+          }
+
+          if (credentialFileType === "authorized_user") {
+            const response = await fetch(
+              "/api/manage/admin/connector/google-drive/app-credential",
+              {
+                method: "PUT",
+                headers: {
+                  "Content-Type": "application/json",
+                },
+                body: credentialJsonStr,
+              }
+            );
+            if (response.ok) {
+              setPopup({
+                message: "Successfully uploaded app credentials",
+                type: "success",
+              });
+              mutate("/api/manage/admin/connector/google-drive/app-credential");
+              if (onSuccess) {
+                onSuccess();
+              }
+            } else {
+              const errorMsg = await response.text();
+              setPopup({
+                message: `Failed to upload app credentials - ${errorMsg}`,
+                type: "error",
+              });
+            }
+          }
+
+          if (credentialFileType === "service_account") {
+            const response = await fetch(
+              "/api/manage/admin/connector/google-drive/service-account-key",
+              {
+                method: "PUT",
+                headers: {
+                  "Content-Type": "application/json",
+                },
+                body: credentialJsonStr,
+              }
+            );
+            if (response.ok) {
+              setPopup({
+                message: "Successfully uploaded service account key",
+                type: "success",
+              });
+              mutate(
+                "/api/manage/admin/connector/google-drive/service-account-key"
+              );
+              if (onSuccess) {
+                onSuccess();
+              }
+            } else {
+              const errorMsg = await response.text();
+              setPopup({
+                message: `Failed to upload service account key - ${errorMsg}`,
+                type: "error",
+              });
+            }
+          }
+        }}
+      >
+        Upload
+      </Button>
+    </>
   );
 };
 
@@ -248,7 +160,6 @@ interface DriveJsonUploadSectionProps {
   serviceAccountCredentialData?: { service_account_email: string };
   isAdmin: boolean;
   onSuccess?: () => void;
-  existingAuthCredential?: boolean;
 }
 
 export const DriveJsonUploadSection = ({
@@ -257,7 +168,6 @@ export const DriveJsonUploadSection = ({
   serviceAccountCredentialData,
   isAdmin,
   onSuccess,
-  existingAuthCredential,
 }: DriveJsonUploadSectionProps) => {
   const { mutate } = useSWRConfig();
   const router = useRouter();
@@ -267,7 +177,6 @@ export const DriveJsonUploadSection = ({
   const [localAppCredentialData, setLocalAppCredentialData] =
     useState(appCredentialData);
 
-  // Update local state when props change
   useEffect(() => {
     setLocalServiceAccountData(serviceAccountCredentialData);
     setLocalAppCredentialData(appCredentialData);
@@ -281,135 +190,153 @@ export const DriveJsonUploadSection = ({
     }
   };
 
-  if (!isAdmin) {
+  if (localServiceAccountData?.service_account_email) {
     return (
-      <div>
-        <div className="flex items-start py-3 px-4 bg-yellow-50/30 dark:bg-yellow-900/5 rounded">
-          <FiAlertTriangle className="text-yellow-500 h-5 w-5 mr-2 mt-0.5 flex-shrink-0" />
-          <p className="text-sm">
-            Curators are unable to set up the Google Drive credentials. To add a
-            Google Drive connector, please contact an administrator.
+      <div className="mt-2 text-sm">
+        <div>
+          Found existing service account key with the following <b>Email:</b>
+          <p className="italic mt-1">
+            {localServiceAccountData.service_account_email}
           </p>
         </div>
+        {isAdmin ? (
+          <>
+            <div className="mt-4 mb-1">
+              If you want to update these credentials, delete the existing
+              credentials through the button below, and then upload a new
+              credentials JSON.
+            </div>
+            <Button
+              onClick={async () => {
+                const response = await fetch(
+                  "/api/manage/admin/connector/google-drive/service-account-key",
+                  {
+                    method: "DELETE",
+                  }
+                );
+                if (response.ok) {
+                  mutate(
+                    "/api/manage/admin/connector/google-drive/service-account-key"
+                  );
+                  mutate(
+                    buildSimilarCredentialInfoURL(ValidSources.GoogleDrive)
+                  );
+                  setPopup({
+                    message: "Successfully deleted service account key",
+                    type: "success",
+                  });
+                  setLocalServiceAccountData(undefined);
+                  handleSuccess();
+                } else {
+                  const errorMsg = await response.text();
+                  setPopup({
+                    message: `Failed to delete service account key - ${errorMsg}`,
+                    type: "error",
+                  });
+                }
+              }}
+            >
+              Delete
+            </Button>
+          </>
+        ) : (
+          <>
+            <div className="mt-4 mb-1">
+              To change these credentials, please contact an administrator.
+            </div>
+          </>
+        )}
+      </div>
+    );
+  }
+
+  if (localAppCredentialData?.client_id) {
+    return (
+      <div className="mt-2 text-sm">
+        <div>
+          Found existing app credentials with the following <b>Client ID:</b>
+          <p className="italic mt-1">{localAppCredentialData.client_id}</p>
+        </div>
+        {isAdmin ? (
+          <>
+            <div className="mt-4 mb-1">
+              If you want to update these credentials, delete the existing
+              credentials through the button below, and then upload a new
+              credentials JSON.
+            </div>
+            <Button
+              onClick={async () => {
+                const response = await fetch(
+                  "/api/manage/admin/connector/google-drive/app-credential",
+                  {
+                    method: "DELETE",
+                  }
+                );
+                if (response.ok) {
+                  mutate(
+                    "/api/manage/admin/connector/google-drive/app-credential"
+                  );
+                  mutate(
+                    buildSimilarCredentialInfoURL(ValidSources.GoogleDrive)
+                  );
+                  setPopup({
+                    message: "Successfully deleted app credentials",
+                    type: "success",
+                  });
+                  setLocalAppCredentialData(undefined);
+                  handleSuccess();
+                } else {
+                  const errorMsg = await response.text();
+                  setPopup({
+                    message: `Failed to delete app credential - ${errorMsg}`,
+                    type: "error",
+                  });
+                }
+              }}
+            >
+              Delete
+            </Button>
+          </>
+        ) : (
+          <div className="mt-4 mb-1">
+            To change these credentials, please contact an administrator.
+          </div>
+        )}
+      </div>
+    );
+  }
+
+  if (!isAdmin) {
+    return (
+      <div className="mt-2">
+        <p className="text-sm mb-2">
+          Curators are unable to set up the google drive credentials. To add a
+          Google Drive connector, please contact an administrator.
+        </p>
       </div>
     );
   }
 
   return (
-    <div>
-      <p className="text-sm mb-3">
-        To connect your Google Drive, create credentials (either OAuth App or
-        Service Account), download the JSON file, and upload it below.
-      </p>
-      <div className="mb-4">
+    <div className="mt-2">
+      <p className="text-sm mb-2">
+        Follow the guide{" "}
         <a
-          className="text-primary hover:text-primary/80 flex items-center gap-1 text-sm"
+          className="text-link"
           target="_blank"
           href="https://docs.onyx.app/connectors/google_drive#authorization"
           rel="noreferrer"
         >
-          <FiLink className="h-3 w-3" />
-          View detailed setup instructions
-        </a>
-      </div>
-
-      {(localServiceAccountData?.service_account_email ||
-        localAppCredentialData?.client_id) && (
-        <div className="mb-4">
-          <div className="relative flex flex-1 items-center">
-            <label
-              className={cn(
-                "flex h-10 items-center justify-center w-full px-4 py-2 border border-dashed rounded-md transition-colors",
-                false
-                  ? "opacity-70 cursor-not-allowed border-background-400 bg-background-50/30"
-                  : "cursor-pointer hover:bg-background-50/30 hover:border-primary dark:hover:border-primary border-background-300 dark:border-background-600"
-              )}
-            >
-              <div className="flex items-center space-x-2">
-                {false ? (
-                  <div className="h-4 w-4 border-t-2 border-b-2 border-primary rounded-full animate-spin"></div>
-                ) : (
-                  <FiFile className="h-4 w-4 text-text-500" />
-                )}
-                <span className="text-sm text-text-500">
-                  {truncateString(
-                    localServiceAccountData?.service_account_email ||
-                      localAppCredentialData?.client_id ||
-                      "",
-                    50
-                  )}
-                </span>
-              </div>
-            </label>
-          </div>
-          {isAdmin && !existingAuthCredential && (
-            <div className="mt-2">
-              <Button
-                variant="destructive"
-                type="button"
-                onClick={async () => {
-                  const endpoint =
-                    localServiceAccountData?.service_account_email
-                      ? "/api/manage/admin/connector/google-drive/service-account-key"
-                      : "/api/manage/admin/connector/google-drive/app-credential";
-
-                  const response = await fetch(endpoint, {
-                    method: "DELETE",
-                  });
-
-                  if (response.ok) {
-                    mutate(endpoint);
-                    // Also mutate the credential endpoints to ensure Step 2 is reset
-                    mutate(
-                      buildSimilarCredentialInfoURL(ValidSources.GoogleDrive)
-                    );
-
-                    // Add additional mutations to refresh all credential-related endpoints
-                    mutate(
-                      "/api/manage/admin/connector/google-drive/credentials"
-                    );
-                    mutate(
-                      "/api/manage/admin/connector/google-drive/public-credential"
-                    );
-                    mutate(
-                      "/api/manage/admin/connector/google-drive/service-account-credential"
-                    );
-
-                    setPopup({
-                      message: `Successfully deleted ${
-                        localServiceAccountData
-                          ? "service account key"
-                          : "app credentials"
-                      }`,
-                      type: "success",
-                    });
-                    // Immediately update local state
-                    if (localServiceAccountData) {
-                      setLocalServiceAccountData(undefined);
-                    } else {
-                      setLocalAppCredentialData(undefined);
-                    }
-                    handleSuccess();
-                  } else {
-                    const errorMsg = await response.text();
-                    setPopup({
-                      message: `Failed to delete credentials - ${errorMsg}`,
-                      type: "error",
-                    });
-                  }
-                }}
-              >
-                Delete Credentials
-              </Button>
-            </div>
-          )}
-        </div>
-      )}
-
-      {!(
-        localServiceAccountData?.service_account_email ||
-        localAppCredentialData?.client_id
-      ) && <DriveJsonUpload setPopup={setPopup} onSuccess={handleSuccess} />}
+          here
+        </a>{" "}
+        to either (1) setup a google OAuth App in your company workspace or (2)
+        create a Service Account.
+        <br />
+        <br />
+        Download the credentials JSON if choosing option (1) or the Service
+        Account key JSON if chooosing option (2), and upload it here.
+      </p>
+      <DriveJsonUpload setPopup={setPopup} onSuccess={handleSuccess} />
     </div>
   );
 };
@@ -464,7 +391,6 @@ export const DriveAuthSection = ({
   user,
 }: DriveCredentialSectionProps) => {
   const router = useRouter();
-  const [isAuthenticating, setIsAuthenticating] = useState(false);
   const [localServiceAccountData, setLocalServiceAccountData] = useState(
     serviceAccountKeyData
   );
@@ -479,7 +405,6 @@ export const DriveAuthSection = ({
     setLocalGoogleDriveServiceAccountCredential,
   ] = useState(googleDriveServiceAccountCredential);
 
-  // Update local state when props change
   useEffect(() => {
     setLocalServiceAccountData(serviceAccountKeyData);
     setLocalAppCredentialData(appCredentialData);
@@ -499,181 +424,126 @@ export const DriveAuthSection = ({
     localGoogleDriveServiceAccountCredential;
   if (existingCredential) {
     return (
-      <div>
-        <div className="mt-4">
-          <div className="py-3 px-4 bg-blue-50/30 dark:bg-blue-900/5 rounded mb-4 flex items-start">
-            <FiCheck className="text-blue-500 h-5 w-5 mr-2 mt-0.5 flex-shrink-0" />
-            <div className="flex-1">
-              <span className="font-medium block">Authentication Complete</span>
-              <p className="text-sm mt-1 text-text-500 dark:text-text-400 break-words">
-                Your Google Drive credentials have been successfully uploaded
-                and authenticated.
-              </p>
-            </div>
-          </div>
-          <Button
-            variant="destructive"
-            type="button"
-            onClick={async () => {
-              handleRevokeAccess(
-                connectorAssociated,
-                setPopup,
-                existingCredential,
-                refreshCredentials
-              );
-            }}
-          >
-            Revoke Access
-          </Button>
-        </div>
-      </div>
-    );
-  }
-
-  // If no credentials are uploaded, show message to complete step 1 first
-  if (
-    !localServiceAccountData?.service_account_email &&
-    !localAppCredentialData?.client_id
-  ) {
-    return (
-      <div>
-        <SectionHeader>Google Drive Authentication</SectionHeader>
-        <div className="mt-4">
-          <div className="flex items-start py-3 px-4 bg-yellow-50/30 dark:bg-yellow-900/5 rounded">
-            <FiAlertTriangle className="text-yellow-500 h-5 w-5 mr-2 mt-0.5 flex-shrink-0" />
-            <p className="text-sm">
-              Please complete Step 1 by uploading either OAuth credentials or a
-              Service Account key before proceeding with authentication.
-            </p>
-          </div>
-        </div>
-      </div>
+      <>
+        <p className="mb-2 text-sm">
+          <i>Uploaded and authenticated credential already exists!</i>
+        </p>
+        <Button
+          onClick={async () => {
+            handleRevokeAccess(
+              connectorAssociated,
+              setPopup,
+              existingCredential,
+              refreshCredentials
+            );
+          }}
+        >
+          Revoke Access
+        </Button>
+      </>
     );
   }
 
   if (localServiceAccountData?.service_account_email) {
     return (
       <div>
-        <div className="mt-4">
-          <Formik
-            initialValues={{
-              google_primary_admin: user?.email || "",
-            }}
-            validationSchema={Yup.object().shape({
-              google_primary_admin: Yup.string()
-                .email("Must be a valid email")
-                .required("Required"),
-            })}
-            onSubmit={async (values, formikHelpers) => {
-              formikHelpers.setSubmitting(true);
-              try {
-                const response = await fetch(
-                  "/api/manage/admin/connector/google-drive/service-account-credential",
-                  {
-                    method: "PUT",
-                    headers: {
-                      "Content-Type": "application/json",
-                    },
-                    body: JSON.stringify({
-                      google_primary_admin: values.google_primary_admin,
-                    }),
-                  }
-                );
-
-                if (response.ok) {
-                  setPopup({
-                    message: "Successfully created service account credential",
-                    type: "success",
-                  });
-                  refreshCredentials();
-                } else {
-                  const errorMsg = await response.text();
-                  setPopup({
-                    message: `Failed to create service account credential - ${errorMsg}`,
-                    type: "error",
-                  });
-                }
-              } catch (error) {
-                setPopup({
-                  message: `Failed to create service account credential - ${error}`,
-                  type: "error",
-                });
-              } finally {
-                formikHelpers.setSubmitting(false);
+        <Formik
+          initialValues={{
+            google_primary_admin: user?.email || "",
+          }}
+          validationSchema={Yup.object().shape({
+            google_primary_admin: Yup.string().required(
+              "User email is required"
+            ),
+          })}
+          onSubmit={async (values, formikHelpers) => {
+            formikHelpers.setSubmitting(true);
+            const response = await fetch(
+              "/api/manage/admin/connector/google-drive/service-account-credential",
+              {
+                method: "PUT",
+                headers: {
+                  "Content-Type": "application/json",
+                },
+                body: JSON.stringify({
+                  google_primary_admin: values.google_primary_admin,
+                }),
               }
-            }}
-          >
-            {({ isSubmitting }) => (
-              <Form>
-                <TextFormField
-                  name="google_primary_admin"
-                  label="Primary Admin Email:"
-                  subtext="Enter the email of an admin/owner of the Google Organization that owns the Google Drive(s) you want to index."
-                />
-                <div className="flex">
-                  <Button type="submit" disabled={isSubmitting}>
-                    {isSubmitting ? "Creating..." : "Create Credential"}
-                  </Button>
-                </div>
-              </Form>
-            )}
-          </Formik>
-        </div>
+            );
+
+            if (response.ok) {
+              setPopup({
+                message: "Successfully created service account credential",
+                type: "success",
+              });
+            } else {
+              const errorMsg = await response.text();
+              setPopup({
+                message: `Failed to create service account credential - ${errorMsg}`,
+                type: "error",
+              });
+            }
+            refreshCredentials();
+          }}
+        >
+          {({ isSubmitting }) => (
+            <Form>
+              <TextFormField
+                name="google_primary_admin"
+                label="Primary Admin Email:"
+                subtext="Enter the email of an admin/owner of the Google Organization that owns the Google Drive(s) you want to index."
+              />
+              <div className="flex">
+                <TremorButton type="submit" disabled={isSubmitting}>
+                  Create Credential
+                </TremorButton>
+              </div>
+            </Form>
+          )}
+        </Formik>
       </div>
     );
   }
 
   if (localAppCredentialData?.client_id) {
     return (
-      <div>
-        <div className="bg-background-50/30 dark:bg-background-900/20 rounded mb-4">
-          <p className="text-sm">
-            Next, you need to authenticate with Google Drive via OAuth. This
-            gives us read access to the documents you have access to in your
-            Google Drive account.
-          </p>
-        </div>
+      <div className="text-sm mb-4">
+        <p className="mb-2">
+          Next, you must provide credentials via OAuth. This gives us read
+          access to the docs you have access to in your google drive account.
+        </p>
         <Button
-          disabled={isAuthenticating}
           onClick={async () => {
-            setIsAuthenticating(true);
-            try {
+            const [authUrl, errorMsg] = await setupGoogleDriveOAuth({
+              isAdmin: true,
+              name: "OAuth (uploaded)",
+            });
+            if (authUrl) {
               // cookie used by callback to determine where to finally redirect to
               Cookies.set(GOOGLE_DRIVE_AUTH_IS_ADMIN_COOKIE_NAME, "true", {
                 path: "/",
               });
-
-              const [authUrl, errorMsg] = await setupGoogleDriveOAuth({
-                isAdmin: true,
-                name: "OAuth (uploaded)",
-              });
-
-              if (authUrl) {
-                router.push(authUrl);
-              } else {
-                setPopup({
-                  message: errorMsg,
-                  type: "error",
-                });
-                setIsAuthenticating(false);
-              }
-            } catch (error) {
-              setPopup({
-                message: `Failed to authenticate with Google Drive - ${error}`,
-                type: "error",
-              });
-              setIsAuthenticating(false);
+              router.push(authUrl);
+              return;
             }
+
+            setPopup({
+              message: errorMsg,
+              type: "error",
+            });
           }}
         >
-          {isAuthenticating
-            ? "Authenticating..."
-            : "Authenticate with Google Drive"}
+          Authenticate with Google Drive
         </Button>
       </div>
     );
   }
 
-  // This code path should not be reached with the new conditions above
-  return null;
+  // case where no keys have been uploaded in step 1
+  return (
+    <p className="text-sm">
+      Please upload either a OAuth Client Credential JSON or a Google Drive
+      Service Account Key JSON in Step 1 before moving onto Step 2.
+    </p>
+  );
 };

web/src/app/admin/connectors/[connector]/pages/gdrive/GoogleDrivePage.tsx

@@ -165,10 +165,6 @@ const GDriveMain = ({
         serviceAccountCredentialData={serviceAccountKeyData}
         isAdmin={isAdmin}
         onSuccess={handleRefresh}
-        existingAuthCredential={Boolean(
-          googleDrivePublicUploadedCredential ||
-            googleDriveServiceAccountCredential
-        )}
       />
 
       {isAdmin &&

web/src/app/admin/connectors/[connector]/pages/gmail/Credential.tsx

@@ -1,4 +1,4 @@
-import { Button } from "@/components/ui/button";
+import { Button } from "@/components/Button";
 import { PopupSpec } from "@/components/admin/connectors/Popup";
 import React, { useState, useEffect } from "react";
 import { useSWRConfig } from "swr";
@@ -8,11 +8,7 @@ import { adminDeleteCredential } from "@/lib/credential";
 import { setupGmailOAuth } from "@/lib/gmail";
 import { GMAIL_AUTH_IS_ADMIN_COOKIE_NAME } from "@/lib/constants";
 import Cookies from "js-cookie";
-import {
-  TextFormField,
-  SectionHeader,
-  SubLabel,
-} from "@/components/admin/connectors/Field";
+import { TextFormField } from "@/components/admin/connectors/Field";
 import { Form, Formik } from "formik";
 import { User } from "@/lib/types";
 import CardSection from "@/components/admin/CardSection";
@@ -24,19 +20,10 @@ import {
 import { refreshAllGoogleData } from "@/lib/googleConnector";
 import { ValidSources } from "@/lib/types";
 import { buildSimilarCredentialInfoURL } from "@/app/admin/connector/[ccPairId]/lib";
-import {
-  FiFile,
-  FiUpload,
-  FiTrash2,
-  FiCheck,
-  FiLink,
-  FiAlertTriangle,
-} from "react-icons/fi";
-import { cn, truncateString } from "@/lib/utils";
 
 type GmailCredentialJsonTypes = "authorized_user" | "service_account";
 
-const GmailCredentialUpload = ({
+const DriveJsonUpload = ({
   setPopup,
   onSuccess,
 }: {
@@ -44,210 +31,134 @@ const GmailCredentialUpload = ({
   onSuccess?: () => void;
 }) => {
   const { mutate } = useSWRConfig();
-  const [isUploading, setIsUploading] = useState(false);
-  const [fileName, setFileName] = useState<string | undefined>();
-  const [isDragging, setIsDragging] = useState(false);
-
-  const handleFileUpload = async (file: File) => {
-    setIsUploading(true);
-    setFileName(file.name);
-
-    const reader = new FileReader();
-    reader.onload = async (loadEvent) => {
-      if (!loadEvent?.target?.result) {
-        setIsUploading(false);
-        return;
-      }
-
-      const credentialJsonStr = loadEvent.target.result as string;
-
-      // Check credential type
-      let credentialFileType: GmailCredentialJsonTypes;
-      try {
-        const appCredentialJson = JSON.parse(credentialJsonStr);
-        if (appCredentialJson.web) {
-          credentialFileType = "authorized_user";
-        } else if (appCredentialJson.type === "service_account") {
-          credentialFileType = "service_account";
-        } else {
-          throw new Error(
-            "Unknown credential type, expected one of 'OAuth Web application' or 'Service Account'"
-          );
-        }
-      } catch (e) {
-        setPopup({
-          message: `Invalid file provided - ${e}`,
-          type: "error",
-        });
-        setIsUploading(false);
-        return;
-      }
-
-      if (credentialFileType === "authorized_user") {
-        const response = await fetch(
-          "/api/manage/admin/connector/gmail/app-credential",
-          {
-            method: "PUT",
-            headers: {
-              "Content-Type": "application/json",
-            },
-            body: credentialJsonStr,
-          }
-        );
-        if (response.ok) {
-          setPopup({
-            message: "Successfully uploaded app credentials",
-            type: "success",
-          });
-          mutate("/api/manage/admin/connector/gmail/app-credential");
-          if (onSuccess) {
-            onSuccess();
-          }
-        } else {
-          const errorMsg = await response.text();
-          setPopup({
-            message: `Failed to upload app credentials - ${errorMsg}`,
-            type: "error",
-          });
-        }
-      }
-
-      if (credentialFileType === "service_account") {
-        const response = await fetch(
-          "/api/manage/admin/connector/gmail/service-account-key",
-          {
-            method: "PUT",
-            headers: {
-              "Content-Type": "application/json",
-            },
-            body: credentialJsonStr,
-          }
-        );
-        if (response.ok) {
-          setPopup({
-            message: "Successfully uploaded service account key",
-            type: "success",
-          });
-          mutate("/api/manage/admin/connector/gmail/service-account-key");
-          if (onSuccess) {
-            onSuccess();
-          }
-        } else {
-          const errorMsg = await response.text();
-          setPopup({
-            message: `Failed to upload service account key - ${errorMsg}`,
-            type: "error",
-          });
-        }
-      }
-      setIsUploading(false);
-    };
-
-    reader.readAsText(file);
-  };
-
-  const handleDragEnter = (e: React.DragEvent<HTMLLabelElement>) => {
-    e.preventDefault();
-    e.stopPropagation();
-    if (!isUploading) {
-      setIsDragging(true);
-    }
-  };
-
-  const handleDragLeave = (e: React.DragEvent<HTMLLabelElement>) => {
-    e.preventDefault();
-    e.stopPropagation();
-    setIsDragging(false);
-  };
-
-  const handleDragOver = (e: React.DragEvent<HTMLLabelElement>) => {
-    e.preventDefault();
-    e.stopPropagation();
-  };
-
-  const handleDrop = (e: React.DragEvent<HTMLLabelElement>) => {
-    e.preventDefault();
-    e.stopPropagation();
-    setIsDragging(false);
-
-    if (isUploading) return;
-
-    const files = e.dataTransfer.files;
-    if (files.length > 0) {
-      const file = files[0];
-      if (file.type === "application/json" || file.name.endsWith(".json")) {
-        handleFileUpload(file);
-      } else {
-        setPopup({
-          message: "Please upload a JSON file",
-          type: "error",
-        });
-      }
-    }
-  };
+  const [credentialJsonStr, setCredentialJsonStr] = useState<
+    string | undefined
+  >();
 
   return (
-    <div className="flex flex-col mt-4">
-      <div className="flex items-center">
-        <div className="relative flex flex-1 items-center">
-          <label
-            className={cn(
-              "flex h-10 items-center justify-center w-full px-4 py-2 border border-dashed rounded-md transition-colors",
-              isUploading
-                ? "opacity-70 cursor-not-allowed border-background-400 bg-background-50/30"
-                : isDragging
-                  ? "bg-background-50/50 border-primary dark:border-primary"
-                  : "cursor-pointer hover:bg-background-50/30 hover:border-primary dark:hover:border-primary border-background-300 dark:border-background-600"
-            )}
-            onDragEnter={handleDragEnter}
-            onDragLeave={handleDragLeave}
-            onDragOver={handleDragOver}
-            onDrop={handleDrop}
-          >
-            <div className="flex items-center space-x-2">
-              {isUploading ? (
-                <div className="h-4 w-4 border-t-2 border-b-2 border-primary rounded-full animate-spin"></div>
-              ) : (
-                <FiFile className="h-4 w-4 text-text-500" />
-              )}
-              <span className="text-sm text-text-500">
-                {isUploading
-                  ? `Uploading ${truncateString(fileName || "file", 50)}...`
-                  : isDragging
-                    ? "Drop JSON file here"
-                    : truncateString(
-                        fileName || "Select or drag JSON credentials file...",
-                        50
-                      )}
-              </span>
-            </div>
-            <input
-              className="sr-only"
-              type="file"
-              accept=".json"
-              disabled={isUploading}
-              onChange={(event) => {
-                if (!event.target.files?.length) {
-                  return;
-                }
-                const file = event.target.files[0];
-                handleFileUpload(file);
-              }}
-            />
-          </label>
-        </div>
-      </div>
-    </div>
+    <>
+      <input
+        className={
+          "mr-3 text-sm text-text-900 border border-background-300 overflow-visible " +
+          "cursor-pointer bg-background dark:text-text-400 focus:outline-none " +
+          "dark:bg-background-700 dark:border-background-600 dark:placeholder-text-400"
+        }
+        type="file"
+        accept=".json"
+        onChange={(event) => {
+          if (!event.target.files) {
+            return;
+          }
+          const file = event.target.files[0];
+          const reader = new FileReader();
+
+          reader.onload = function (loadEvent) {
+            if (!loadEvent?.target?.result) {
+              return;
+            }
+            const fileContents = loadEvent.target.result;
+            setCredentialJsonStr(fileContents as string);
+          };
+
+          reader.readAsText(file);
+        }}
+      />
+
+      <Button
+        disabled={!credentialJsonStr}
+        onClick={async () => {
+          // check if the JSON is a app credential or a service account credential
+          let credentialFileType: GmailCredentialJsonTypes;
+          try {
+            const appCredentialJson = JSON.parse(credentialJsonStr!);
+            if (appCredentialJson.web) {
+              credentialFileType = "authorized_user";
+            } else if (appCredentialJson.type === "service_account") {
+              credentialFileType = "service_account";
+            } else {
+              throw new Error(
+                "Unknown credential type, expected one of 'OAuth Web application' or 'Service Account'"
+              );
+            }
+          } catch (e) {
+            setPopup({
+              message: `Invalid file provided - ${e}`,
+              type: "error",
+            });
+            return;
+          }
+
+          if (credentialFileType === "authorized_user") {
+            const response = await fetch(
+              "/api/manage/admin/connector/gmail/app-credential",
+              {
+                method: "PUT",
+                headers: {
+                  "Content-Type": "application/json",
+                },
+                body: credentialJsonStr,
+              }
+            );
+            if (response.ok) {
+              setPopup({
+                message: "Successfully uploaded app credentials",
+                type: "success",
+              });
+              mutate("/api/manage/admin/connector/gmail/app-credential");
+              if (onSuccess) {
+                onSuccess();
+              }
+            } else {
+              const errorMsg = await response.text();
+              setPopup({
+                message: `Failed to upload app credentials - ${errorMsg}`,
+                type: "error",
+              });
+            }
+          }
+
+          if (credentialFileType === "service_account") {
+            const response = await fetch(
+              "/api/manage/admin/connector/gmail/service-account-key",
+              {
+                method: "PUT",
+                headers: {
+                  "Content-Type": "application/json",
+                },
+                body: credentialJsonStr,
+              }
+            );
+            if (response.ok) {
+              setPopup({
+                message: "Successfully uploaded service account key",
+                type: "success",
+              });
+              mutate("/api/manage/admin/connector/gmail/service-account-key");
+              if (onSuccess) {
+                onSuccess();
+              }
+            } else {
+              const errorMsg = await response.text();
+              setPopup({
+                message: `Failed to upload service account key - ${errorMsg}`,
+                type: "error",
+              });
+            }
+          }
+        }}
+      >
+        Upload
+      </Button>
+    </>
   );
 };
 
-interface GmailJsonUploadSectionProps {
+interface DriveJsonUploadSectionProps {
   setPopup: (popupSpec: PopupSpec | null) => void;
   appCredentialData?: { client_id: string };
   serviceAccountCredentialData?: { service_account_email: string };
   isAdmin: boolean;
   onSuccess?: () => void;
-  existingAuthCredential?: boolean;
 }
 
 export const GmailJsonUploadSection = ({
@@ -256,8 +167,7 @@ export const GmailJsonUploadSection = ({
   serviceAccountCredentialData,
   isAdmin,
   onSuccess,
-  existingAuthCredential,
-}: GmailJsonUploadSectionProps) => {
+}: DriveJsonUploadSectionProps) => {
   const { mutate } = useSWRConfig();
   const router = useRouter();
   const [localServiceAccountData, setLocalServiceAccountData] = useState(
@@ -280,138 +190,156 @@ export const GmailJsonUploadSection = ({
     }
   };
 
-  if (!isAdmin) {
+  if (localServiceAccountData?.service_account_email) {
     return (
-      <div>
-        <div className="flex items-start py-3 px-4 bg-yellow-50/30 dark:bg-yellow-900/5 rounded">
-          <FiAlertTriangle className="text-yellow-500 h-5 w-5 mr-2 mt-0.5 flex-shrink-0" />
-          <p className="text-sm">
-            Curators are unable to set up the Gmail credentials. To add a Gmail
-            connector, please contact an administrator.
+      <div className="mt-2 text-sm">
+        <div>
+          Found existing service account key with the following <b>Email:</b>
+          <p className="italic mt-1">
+            {localServiceAccountData.service_account_email}
           </p>
         </div>
+        {isAdmin ? (
+          <>
+            <div className="mt-4 mb-1">
+              If you want to update these credentials, delete the existing
+              credentials through the button below, and then upload a new
+              credentials JSON.
+            </div>
+            <Button
+              onClick={async () => {
+                const response = await fetch(
+                  "/api/manage/admin/connector/gmail/service-account-key",
+                  {
+                    method: "DELETE",
+                  }
+                );
+                if (response.ok) {
+                  mutate(
+                    "/api/manage/admin/connector/gmail/service-account-key"
+                  );
+                  // Also mutate the credential endpoints to ensure Step 2 is reset
+                  mutate(buildSimilarCredentialInfoURL(ValidSources.Gmail));
+                  setPopup({
+                    message: "Successfully deleted service account key",
+                    type: "success",
+                  });
+                  // Immediately update local state
+                  setLocalServiceAccountData(undefined);
+                  handleSuccess();
+                } else {
+                  const errorMsg = await response.text();
+                  setPopup({
+                    message: `Failed to delete service account key - ${errorMsg}`,
+                    type: "error",
+                  });
+                }
+              }}
+            >
+              Delete
+            </Button>
+          </>
+        ) : (
+          <>
+            <div className="mt-4 mb-1">
+              To change these credentials, please contact an administrator.
+            </div>
+          </>
+        )}
+      </div>
+    );
+  }
+
+  if (localAppCredentialData?.client_id) {
+    return (
+      <div className="mt-2 text-sm">
+        <div>
+          Found existing app credentials with the following <b>Client ID:</b>
+          <p className="italic mt-1">{localAppCredentialData.client_id}</p>
+        </div>
+        {isAdmin ? (
+          <>
+            <div className="mt-4 mb-1">
+              If you want to update these credentials, delete the existing
+              credentials through the button below, and then upload a new
+              credentials JSON.
+            </div>
+            <Button
+              onClick={async () => {
+                const response = await fetch(
+                  "/api/manage/admin/connector/gmail/app-credential",
+                  {
+                    method: "DELETE",
+                  }
+                );
+                if (response.ok) {
+                  mutate("/api/manage/admin/connector/gmail/app-credential");
+                  // Also mutate the credential endpoints to ensure Step 2 is reset
+                  mutate(buildSimilarCredentialInfoURL(ValidSources.Gmail));
+                  setPopup({
+                    message: "Successfully deleted app credentials",
+                    type: "success",
+                  });
+                  // Immediately update local state
+                  setLocalAppCredentialData(undefined);
+                  handleSuccess();
+                } else {
+                  const errorMsg = await response.text();
+                  setPopup({
+                    message: `Failed to delete app credential - ${errorMsg}`,
+                    type: "error",
+                  });
+                }
+              }}
+            >
+              Delete
+            </Button>
+          </>
+        ) : (
+          <div className="mt-4 mb-1">
+            To change these credentials, please contact an administrator.
+          </div>
+        )}
+      </div>
+    );
+  }
+
+  if (!isAdmin) {
+    return (
+      <div className="mt-2">
+        <p className="text-sm mb-2">
+          Curators are unable to set up the Gmail credentials. To add a Gmail
+          connector, please contact an administrator.
+        </p>
       </div>
     );
   }
 
   return (
-    <div>
-      <p className="text-sm mb-3">
-        To connect your Gmail, create credentials (either OAuth App or Service
-        Account), download the JSON file, and upload it below.
-      </p>
-      <div className="mb-4">
+    <div className="mt-2">
+      <p className="text-sm mb-2">
+        Follow the guide{" "}
         <a
-          className="text-primary hover:text-primary/80 flex items-center gap-1 text-sm"
+          className="text-link"
           target="_blank"
           href="https://docs.onyx.app/connectors/gmail#authorization"
           rel="noreferrer"
         >
-          <FiLink className="h-3 w-3" />
-          View detailed setup instructions
-        </a>
-      </div>
-
-      {(localServiceAccountData?.service_account_email ||
-        localAppCredentialData?.client_id) && (
-        <div className="mb-4">
-          <div className="relative flex flex-1 items-center">
-            <label
-              className={cn(
-                "flex h-10 items-center justify-center w-full px-4 py-2 border border-dashed rounded-md transition-colors",
-                false
-                  ? "opacity-70 cursor-not-allowed border-background-400 bg-background-50/30"
-                  : "cursor-pointer hover:bg-background-50/30 hover:border-primary dark:hover:border-primary border-background-300 dark:border-background-600"
-              )}
-            >
-              <div className="flex items-center space-x-2">
-                {false ? (
-                  <div className="h-4 w-4 border-t-2 border-b-2 border-primary rounded-full animate-spin"></div>
-                ) : (
-                  <FiFile className="h-4 w-4 text-text-500" />
-                )}
-                <span className="text-sm text-text-500">
-                  {truncateString(
-                    localServiceAccountData?.service_account_email ||
-                      localAppCredentialData?.client_id ||
-                      "",
-                    50
-                  )}
-                </span>
-              </div>
-            </label>
-          </div>
-          {isAdmin && !existingAuthCredential && (
-            <div className="mt-2">
-              <Button
-                variant="destructive"
-                type="button"
-                onClick={async () => {
-                  const endpoint =
-                    localServiceAccountData?.service_account_email
-                      ? "/api/manage/admin/connector/gmail/service-account-key"
-                      : "/api/manage/admin/connector/gmail/app-credential";
-
-                  const response = await fetch(endpoint, {
-                    method: "DELETE",
-                  });
-
-                  if (response.ok) {
-                    mutate(endpoint);
-                    // Also mutate the credential endpoints to ensure Step 2 is reset
-                    mutate(buildSimilarCredentialInfoURL(ValidSources.Gmail));
-
-                    // Add additional mutations to refresh all credential-related endpoints
-                    mutate("/api/manage/admin/connector/gmail/credentials");
-                    mutate(
-                      "/api/manage/admin/connector/gmail/public-credential"
-                    );
-                    mutate(
-                      "/api/manage/admin/connector/gmail/service-account-credential"
-                    );
-
-                    setPopup({
-                      message: `Successfully deleted ${
-                        localServiceAccountData
-                          ? "service account key"
-                          : "app credentials"
-                      }`,
-                      type: "success",
-                    });
-                    // Immediately update local state
-                    if (localServiceAccountData) {
-                      setLocalServiceAccountData(undefined);
-                    } else {
-                      setLocalAppCredentialData(undefined);
-                    }
-                    handleSuccess();
-                  } else {
-                    const errorMsg = await response.text();
-                    setPopup({
-                      message: `Failed to delete credentials - ${errorMsg}`,
-                      type: "error",
-                    });
-                  }
-                }}
-              >
-                Delete Credentials
-              </Button>
-            </div>
-          )}
-        </div>
-      )}
-
-      {!(
-        localServiceAccountData?.service_account_email ||
-        localAppCredentialData?.client_id
-      ) && (
-        <GmailCredentialUpload setPopup={setPopup} onSuccess={handleSuccess} />
-      )}
+          here
+        </a>{" "}
+        to either (1) setup a Google OAuth App in your company workspace or (2)
+        create a Service Account.
+        <br />
+        <br />
+        Download the credentials JSON if choosing option (1) or the Service
+        Account key JSON if choosing option (2), and upload it here.
+      </p>
+      <DriveJsonUpload setPopup={setPopup} onSuccess={handleSuccess} />
     </div>
   );
 };
 
-interface GmailCredentialSectionProps {
+interface DriveCredentialSectionProps {
   gmailPublicCredential?: Credential<GmailCredentialJson>;
   gmailServiceAccountCredential?: Credential<GmailServiceAccountCredentialJson>;
   serviceAccountKeyData?: { service_account_email: string };
@@ -459,7 +387,7 @@ export const GmailAuthSection = ({
   refreshCredentials,
   connectorExists,
   user,
-}: GmailCredentialSectionProps) => {
+}: DriveCredentialSectionProps) => {
   const router = useRouter();
   const [isAuthenticating, setIsAuthenticating] = useState(false);
   const [localServiceAccountData, setLocalServiceAccountData] = useState(
@@ -492,141 +420,104 @@ export const GmailAuthSection = ({
     localGmailPublicCredential || localGmailServiceAccountCredential;
   if (existingCredential) {
     return (
-      <div>
-        <div className="mt-4">
-          <div className="py-3 px-4 bg-blue-50/30 dark:bg-blue-900/5 rounded mb-4 flex items-start">
-            <FiCheck className="text-blue-500 h-5 w-5 mr-2 mt-0.5 flex-shrink-0" />
-            <div className="flex-1">
-              <span className="font-medium block">Authentication Complete</span>
-              <p className="text-sm mt-1 text-text-500 dark:text-text-400 break-words">
-                Your Gmail credentials have been successfully uploaded and
-                authenticated.
-              </p>
-            </div>
-          </div>
-          <Button
-            variant="destructive"
-            type="button"
-            onClick={async () => {
-              handleRevokeAccess(
-                connectorExists,
-                setPopup,
-                existingCredential,
-                refreshCredentials
-              );
-            }}
-          >
-            Revoke Access
-          </Button>
-        </div>
-      </div>
-    );
-  }
-
-  // If no credentials are uploaded, show message to complete step 1 first
-  if (
-    !localServiceAccountData?.service_account_email &&
-    !localAppCredentialData?.client_id
-  ) {
-    return (
-      <div>
-        <SectionHeader>Gmail Authentication</SectionHeader>
-        <div className="mt-4">
-          <div className="flex items-start py-3 px-4 bg-yellow-50/30 dark:bg-yellow-900/5 rounded">
-            <FiAlertTriangle className="text-yellow-500 h-5 w-5 mr-2 mt-0.5 flex-shrink-0" />
-            <p className="text-sm">
-              Please complete Step 1 by uploading either OAuth credentials or a
-              Service Account key before proceeding with authentication.
-            </p>
-          </div>
-        </div>
-      </div>
+      <>
+        <p className="mb-2 text-sm">
+          <i>Uploaded and authenticated credential already exists!</i>
+        </p>
+        <Button
+          onClick={async () => {
+            handleRevokeAccess(
+              connectorExists,
+              setPopup,
+              existingCredential,
+              refreshCredentials
+            );
+          }}
+        >
+          Revoke Access
+        </Button>
+      </>
     );
   }
 
   if (localServiceAccountData?.service_account_email) {
     return (
       <div>
-        <div className="mt-4">
-          <Formik
-            initialValues={{
-              google_primary_admin: user?.email || "",
-            }}
-            validationSchema={Yup.object().shape({
-              google_primary_admin: Yup.string()
-                .email("Must be a valid email")
-                .required("Required"),
-            })}
-            onSubmit={async (values, formikHelpers) => {
-              formikHelpers.setSubmitting(true);
-              try {
-                const response = await fetch(
-                  "/api/manage/admin/connector/gmail/service-account-credential",
-                  {
-                    method: "PUT",
-                    headers: {
-                      "Content-Type": "application/json",
-                    },
-                    body: JSON.stringify({
-                      google_primary_admin: values.google_primary_admin,
-                    }),
-                  }
-                );
-
-                if (response.ok) {
-                  setPopup({
-                    message: "Successfully created service account credential",
-                    type: "success",
-                  });
-                  refreshCredentials();
-                } else {
-                  const errorMsg = await response.text();
-                  setPopup({
-                    message: `Failed to create service account credential - ${errorMsg}`,
-                    type: "error",
-                  });
+        <Formik
+          initialValues={{
+            google_primary_admin: user?.email || "",
+          }}
+          validationSchema={Yup.object().shape({
+            google_primary_admin: Yup.string()
+              .email("Must be a valid email")
+              .required("Required"),
+          })}
+          onSubmit={async (values, formikHelpers) => {
+            formikHelpers.setSubmitting(true);
+            try {
+              const response = await fetch(
+                "/api/manage/admin/connector/gmail/service-account-credential",
+                {
+                  method: "PUT",
+                  headers: {
+                    "Content-Type": "application/json",
+                  },
+                  body: JSON.stringify({
+                    google_primary_admin: values.google_primary_admin,
+                  }),
                 }
-              } catch (error) {
+              );
+
+              if (response.ok) {
                 setPopup({
-                  message: `Failed to create service account credential - ${error}`,
+                  message: "Successfully created service account credential",
+                  type: "success",
+                });
+                refreshCredentials();
+              } else {
+                const errorMsg = await response.text();
+                setPopup({
+                  message: `Failed to create service account credential - ${errorMsg}`,
                   type: "error",
                 });
-              } finally {
-                formikHelpers.setSubmitting(false);
               }
-            }}
-          >
-            {({ isSubmitting }) => (
-              <Form>
-                <TextFormField
-                  name="google_primary_admin"
-                  label="Primary Admin Email:"
-                  subtext="Enter the email of an admin/owner of the Google Organization that owns the Gmail account(s) you want to index."
-                />
-                <div className="flex">
-                  <Button type="submit" disabled={isSubmitting}>
-                    {isSubmitting ? "Creating..." : "Create Credential"}
-                  </Button>
-                </div>
-              </Form>
-            )}
-          </Formik>
-        </div>
+            } catch (error) {
+              setPopup({
+                message: `Failed to create service account credential - ${error}`,
+                type: "error",
+              });
+            } finally {
+              formikHelpers.setSubmitting(false);
+            }
+          }}
+        >
+          {({ isSubmitting }) => (
+            <Form>
+              <TextFormField
+                name="google_primary_admin"
+                label="Primary Admin Email:"
+                subtext="Enter the email of an admin/owner of the Google Organization that owns the Gmail account(s) you want to index."
+              />
+              <div className="flex">
+                <Button type="submit" disabled={isSubmitting}>
+                  Create Credential
+                </Button>
+              </div>
+            </Form>
+          )}
+        </Formik>
       </div>
     );
   }
 
   if (localAppCredentialData?.client_id) {
     return (
-      <div>
-        <div className="bg-background-50/30 dark:bg-background-900/20 rounded mb-4">
-          <p className="text-sm">
-            Next, you need to authenticate with Gmail via OAuth. This gives us
-            read access to the emails you have access to in your Gmail account.
-          </p>
-        </div>
+      <div className="text-sm mb-4">
+        <p className="mb-2">
+          Next, you must provide credentials via OAuth. This gives us read
+          access to the emails you have access to in your Gmail account.
+        </p>
         <Button
-          disabled={isAuthenticating}
           onClick={async () => {
             setIsAuthenticating(true);
             try {
@@ -654,6 +545,7 @@ export const GmailAuthSection = ({
               setIsAuthenticating(false);
             }
           }}
+          disabled={isAuthenticating}
         >
           {isAuthenticating ? "Authenticating..." : "Authenticate with Gmail"}
         </Button>
@@ -661,6 +553,11 @@ export const GmailAuthSection = ({
     );
   }
 
-  // This code path should not be reached with the new conditions above
-  return null;
+  // case where no keys have been uploaded in step 1
+  return (
+    <p className="text-sm">
+      Please upload either a OAuth Client Credential JSON or a Gmail Service
+      Account Key JSON in Step 1 before moving onto Step 2.
+    </p>
+  );
 };

web/src/app/admin/connectors/[connector]/pages/gmail/GmailPage.tsx

@@ -173,9 +173,6 @@ export const GmailMain = () => {
         serviceAccountCredentialData={serviceAccountKeyData}
         isAdmin={isAdmin}
         onSuccess={handleRefresh}
-        existingAuthCredential={Boolean(
-          gmailPublicUploadedCredential || gmailServiceAccountCredential
-        )}
       />
 
       {isAdmin && hasUploadedCredentials && (

web/src/app/chat/ChatPage.tsx

@@ -1383,6 +1383,7 @@ export function ChatPage({
           if (!packet) {
             continue;
           }
+          console.log("Packet:", JSON.stringify(packet));
 
           if (!initialFetchDetails) {
             if (!Object.hasOwn(packet, "user_message_id")) {
@@ -1728,6 +1729,7 @@ export function ChatPage({
         }
       }
     } catch (e: any) {
+      console.log("Error:", e);
       const errorMsg = e.message;
       upsertToCompleteMessageMap({
         messages: [
@@ -1755,11 +1757,13 @@ export function ChatPage({
         completeMessageMapOverride: currentMessageMap(completeMessageDetail),
       });
     }
+    console.log("Finished streaming");
     setAgenticGenerating(false);
     resetRegenerationState(currentSessionId());
 
     updateChatState("input");
     if (isNewSession) {
+      console.log("Setting up new session");
       if (finalMessage) {
         setSelectedMessageForDocDisplay(finalMessage.message_id);
       }

web/src/app/chat/message/SourcesDisplay.tsx

@@ -54,7 +54,6 @@ export const SourceCard: React.FC<{
 
       <div className="flex items-center gap-1 mt-1">
         <ResultIcon doc={document} size={18} />
-
         <div className="text-text-700 text-xs leading-tight truncate flex-1 min-w-0">
           {truncatedIdentifier}
         </div>

web/src/components/Dropdown.tsx

@@ -54,7 +54,6 @@ export function SearchMultiSelectDropdown({
   onDelete,
   onSearchTermChange,
   initialSearchTerm = "",
-  allowCustomValues = false,
 }: {
   options: StringOrNumberOption[];
   onSelect: (selected: StringOrNumberOption) => void;
@@ -63,7 +62,6 @@ export function SearchMultiSelectDropdown({
   onDelete?: (name: string) => void;
   onSearchTermChange?: (term: string) => void;
   initialSearchTerm?: string;
-  allowCustomValues?: boolean;
 }) {
   const [isOpen, setIsOpen] = useState(false);
   const [searchTerm, setSearchTerm] = useState(initialSearchTerm);
@@ -79,29 +77,12 @@ export function SearchMultiSelectDropdown({
     option.name.toLowerCase().includes(searchTerm.toLowerCase())
   );
 
-  // Handle selecting a custom value not in the options list
-  const handleCustomValueSelect = () => {
-    if (allowCustomValues && searchTerm.trim() !== "") {
-      const customOption: StringOrNumberOption = {
-        name: searchTerm,
-        value: searchTerm,
-      };
-      onSelect(customOption);
-      setIsOpen(false);
-    }
-  };
-
   useEffect(() => {
     const handleClickOutside = (event: MouseEvent) => {
       if (
         dropdownRef.current &&
         !dropdownRef.current.contains(event.target as Node)
       ) {
-        // If allowCustomValues is enabled and there's text in the search field,
-        // treat clicking outside as selecting the custom value
-        if (allowCustomValues && searchTerm.trim() !== "") {
-          handleCustomValueSelect();
-        }
         setIsOpen(false);
       }
     };
@@ -110,7 +91,7 @@ export function SearchMultiSelectDropdown({
     return () => {
       document.removeEventListener("mousedown", handleClickOutside);
     };
-  }, [allowCustomValues, searchTerm]);
+  }, []);
 
   useEffect(() => {
     setSearchTerm(initialSearchTerm);
@@ -121,33 +102,17 @@ export function SearchMultiSelectDropdown({
       <div>
         <input
           type="text"
-          placeholder={
-            allowCustomValues ? "Search or enter custom value..." : "Search..."
-          }
+          placeholder="Search..."
           value={searchTerm}
           onChange={(e: ChangeEvent<HTMLInputElement>) => {
-            const newValue = e.target.value;
-            setSearchTerm(newValue);
-            if (onSearchTermChange) {
-              onSearchTermChange(newValue);
-            }
-            if (newValue) {
+            setSearchTerm(e.target.value);
+            if (e.target.value) {
               setIsOpen(true);
             } else {
               setIsOpen(false);
             }
           }}
           onFocus={() => setIsOpen(true)}
-          onKeyDown={(e) => {
-            if (
-              e.key === "Enter" &&
-              allowCustomValues &&
-              searchTerm.trim() !== ""
-            ) {
-              e.preventDefault();
-              handleCustomValueSelect();
-            }
-          }}
           className="inline-flex justify-between w-full px-4 py-2 text-sm bg-white dark:bg-transparent text-text-800 border border-background-300 rounded-md shadow-sm"
         />
         <button
@@ -188,22 +153,6 @@ export function SearchMultiSelectDropdown({
               )
             )}
 
-            {allowCustomValues &&
-              searchTerm.trim() !== "" &&
-              !filteredOptions.some(
-                (option) =>
-                  option.name.toLowerCase() === searchTerm.toLowerCase()
-              ) && (
-                <button
-                  className="w-full text-left flex items-center px-4 py-2 text-sm text-text-800 hover:bg-background-100"
-                  role="menuitem"
-                  onClick={handleCustomValueSelect}
-                >
-                  <PlusIcon className="w-4 h-4 mr-2 text-text-600" />
-                  Use &quot;{searchTerm}&quot; as custom value
-                </button>
-              )}
-
             {onCreate &&
               searchTerm.trim() !== "" &&
               !filteredOptions.some(
@@ -228,8 +177,7 @@ export function SearchMultiSelectDropdown({
               )}
 
             {filteredOptions.length === 0 &&
-              ((!onCreate && !allowCustomValues) ||
-                searchTerm.trim() === "") && (
+              (!onCreate || searchTerm.trim() === "") && (
                 <div className="px-4 py-2.5 text-sm text-text-500">
                   No matches found
                 </div>

web/src/components/SearchResultIcon.tsx

@@ -49,7 +49,7 @@ export function SearchResultIcon({ url }: { url: string }) {
   if (!faviconUrl) {
     return <SourceIcon sourceType={ValidSources.Web} iconSize={18} />;
   }
-  if (url.includes("onyx.app")) {
+  if (url.includes("docs.onyx.app")) {
     return <OnyxIcon size={18} className="dark:text-[#fff] text-[#000]" />;
   }
 

web/src/components/WebResultIcon.tsx

@@ -17,11 +17,12 @@ export function WebResultIcon({
   try {
     hostname = new URL(url).hostname;
   } catch (e) {
-    hostname = "onyx.app";
+    // console.log(e);
+    hostname = "docs.onyx.app";
   }
   return (
     <>
-      {hostname.includes("onyx.app") ? (
+      {hostname == "docs.onyx.app" ? (
         <OnyxIcon size={size} className="dark:text-[#fff] text-[#000]" />
       ) : !error ? (
         <img

web/src/components/chat/sources/SourceCard.tsx

@@ -26,6 +26,35 @@ export const ResultIcon = ({
   );
 };
 
+// export default function SourceCard({
+//   doc,
+//   setPresentingDocument,
+// }: {
+//   doc: OnyxDocument;
+//   setPresentingDocument?: (document: OnyxDocument) => void;
+// }) {
+//   return (
+//     <div
+//       key={doc.document_id}
+//       onClick={() => openDocument(doc, setPresentingDocument)}
+//       className="cursor-pointer h-[80px] text-left overflow-hidden flex flex-col gap-0.5 rounded-lg px-3 py-2 bg-accent-background hover:bg-accent-background-hovered w-[200px]"
+//     >
+//       <div className="line-clamp-1 font-semibold text-ellipsis text-text-900  flex h-6 items-center gap-2 text-sm">
+//         {doc.is_internet || doc.source_type === "web" ? (
+//           <WebResultIcon url={doc.link} />
+//         ) : (
+//           <SourceIcon sourceType={doc.source_type} iconSize={18} />
+//         )}
+//         <p>{truncateString(doc.semantic_identifier || doc.document_id, 20)}</p>
+//       </div>
+//       <div className="line-clamp-2 text-sm font-semibold"></div>
+//       <div className="line-clamp-2 text-sm font-normal leading-snug text-text-700">
+//         {doc.blurb}
+//       </div>
+//     </div>
+//   );
+// }
+
 interface SeeMoreBlockProps {
   toggleDocumentSelection: () => void;
   docs: OnyxDocument[];