fix: improve check for indexing status (#5042)

* Improve check_for_indexing + check_for_vespa_sync_task

* Remove unused

* Fix

* Simplify query

* Add more logging

* Address bot comments

* Increase # of tasks generated since we're not going cc-pair by cc-pair

* Only index 50 user files at a time

backend/onyx/agents/agent_search/orchestration/nodes/choose_tool.py

@@ -83,6 +83,22 @@ def _expand_query(
     return rephrased_query
 
 
+def _expand_query_non_tool_calling_llm(
+    expanded_keyword_thread: TimeoutThread[str],
+    expanded_semantic_thread: TimeoutThread[str],
+) -> QueryExpansions | None:
+    keyword_expansion: str | None = wait_on_background(expanded_keyword_thread)
+    semantic_expansion: str | None = wait_on_background(expanded_semantic_thread)
+
+    if keyword_expansion is None or semantic_expansion is None:
+        return None
+
+    return QueryExpansions(
+        keywords_expansions=[keyword_expansion],
+        semantic_expansions=[semantic_expansion],
+    )
+
+
 # TODO: break this out into an implementation function
 # and a function that handles extracting the necessary fields
 # from the state and config
@@ -186,6 +202,28 @@ def choose_tool(
             is_keyword, keywords = wait_on_background(keyword_thread)
             override_kwargs.precomputed_is_keyword = is_keyword
             override_kwargs.precomputed_keywords = keywords
+        # dual keyword expansion needs to be added here for non-tool calling LLM case
+        if (
+            USE_SEMANTIC_KEYWORD_EXPANSIONS_BASIC_SEARCH
+            and expanded_keyword_thread
+            and expanded_semantic_thread
+            and tool.name == SearchTool._NAME
+        ):
+            override_kwargs.expanded_queries = _expand_query_non_tool_calling_llm(
+                expanded_keyword_thread=expanded_keyword_thread,
+                expanded_semantic_thread=expanded_semantic_thread,
+            )
+        if (
+            USE_SEMANTIC_KEYWORD_EXPANSIONS_BASIC_SEARCH
+            and tool.name == SearchTool._NAME
+            and override_kwargs.expanded_queries
+        ):
+            if (
+                override_kwargs.expanded_queries.keywords_expansions is None
+                or override_kwargs.expanded_queries.semantic_expansions is None
+            ):
+                raise ValueError("No expanded keyword or semantic threads found.")
+
         return ToolChoiceUpdate(
             tool_choice=ToolChoice(
                 tool=tool,
@@ -283,18 +321,23 @@ def choose_tool(
         and expanded_keyword_thread
         and expanded_semantic_thread
     ):
-        keyword_expansion = wait_on_background(expanded_keyword_thread)
-        semantic_expansion = wait_on_background(expanded_semantic_thread)
-        override_kwargs.expanded_queries = QueryExpansions(
-            keywords_expansions=[keyword_expansion],
-            semantic_expansions=[semantic_expansion],
-        )
 
-        logger.info(
-            f"Original query: {agent_config.inputs.prompt_builder.raw_user_query}"
+        override_kwargs.expanded_queries = _expand_query_non_tool_calling_llm(
+            expanded_keyword_thread=expanded_keyword_thread,
+            expanded_semantic_thread=expanded_semantic_thread,
         )
-        logger.info(f"Expanded keyword queries: {keyword_expansion}")
-        logger.info(f"Expanded semantic queries: {semantic_expansion}")
+    if (
+        USE_SEMANTIC_KEYWORD_EXPANSIONS_BASIC_SEARCH
+        and selected_tool.name == SearchTool._NAME
+        and override_kwargs.expanded_queries
+    ):
+        # TODO: this is a hack to handle the case where the expanded queries are not found.
+        # We should refactor this to be more robust.
+        if (
+            override_kwargs.expanded_queries.keywords_expansions is None
+            or override_kwargs.expanded_queries.semantic_expansions is None
+        ):
+            raise ValueError("No expanded keyword or semantic threads found.")
 
     return ToolChoiceUpdate(
         tool_choice=ToolChoice(

backend/onyx/background/celery/apps/app_base.py

@@ -22,13 +22,14 @@ from sqlalchemy.orm import Session
 from onyx.background.celery.apps.task_formatters import CeleryTaskColoredFormatter
 from onyx.background.celery.apps.task_formatters import CeleryTaskPlainFormatter
 from onyx.background.celery.celery_utils import celery_is_worker_primary
+from onyx.background.celery.tasks.vespa.document_sync import DOCUMENT_SYNC_PREFIX
+from onyx.background.celery.tasks.vespa.document_sync import DOCUMENT_SYNC_TASKSET_KEY
 from onyx.configs.constants import ONYX_CLOUD_CELERY_TASK_PREFIX
 from onyx.configs.constants import OnyxRedisLocks
 from onyx.db.engine import get_sqlalchemy_engine
 from onyx.document_index.vespa.shared_utils.utils import wait_for_vespa_with_timeout
 from onyx.httpx.httpx_pool import HttpxPool
 from onyx.redis.redis_connector import RedisConnector
-from onyx.redis.redis_connector_credential_pair import RedisConnectorCredentialPair
 from onyx.redis.redis_connector_delete import RedisConnectorDelete
 from onyx.redis.redis_connector_doc_perm_sync import RedisConnectorPermissionSync
 from onyx.redis.redis_connector_ext_group_sync import RedisConnectorExternalGroupSync
@@ -143,8 +144,11 @@ def on_task_postrun(
 
     r = get_redis_client(tenant_id=tenant_id)
 
-    if task_id.startswith(RedisConnectorCredentialPair.PREFIX):
-        r.srem(RedisConnectorCredentialPair.get_taskset_key(), task_id)
+    # NOTE: we want to remove the `Redis*` classes, prefer to just have functions to
+    # do these things going forward. In short, things should generally be like the doc
+    # sync task rather than the others below
+    if task_id.startswith(DOCUMENT_SYNC_PREFIX):
+        r.srem(DOCUMENT_SYNC_TASKSET_KEY, task_id)
         return
 
     if task_id.startswith(RedisDocumentSet.PREFIX):

backend/onyx/background/celery/apps/primary.py

@@ -21,6 +21,7 @@ from onyx.background.celery.celery_utils import celery_is_worker_primary
 from onyx.background.celery.tasks.indexing.utils import (
     get_unfenced_index_attempt_ids,
 )
+from onyx.background.celery.tasks.vespa.document_sync import reset_document_sync
 from onyx.configs.constants import CELERY_PRIMARY_WORKER_LOCK_TIMEOUT
 from onyx.configs.constants import OnyxRedisConstants
 from onyx.configs.constants import OnyxRedisLocks
@@ -29,9 +30,6 @@ from onyx.db.engine import get_session_with_current_tenant
 from onyx.db.engine import SqlEngine
 from onyx.db.index_attempt import get_index_attempt
 from onyx.db.index_attempt import mark_attempt_canceled
-from onyx.redis.redis_connector_credential_pair import (
-    RedisGlobalConnectorCredentialPair,
-)
 from onyx.redis.redis_connector_delete import RedisConnectorDelete
 from onyx.redis.redis_connector_doc_perm_sync import RedisConnectorPermissionSync
 from onyx.redis.redis_connector_ext_group_sync import RedisConnectorExternalGroupSync
@@ -156,7 +154,10 @@ def on_worker_init(sender: Worker, **kwargs: Any) -> None:
 
     r.delete(OnyxRedisConstants.ACTIVE_FENCES)
 
-    RedisGlobalConnectorCredentialPair.reset_all(r)
+    # NOTE: we want to remove the `Redis*` classes, prefer to just have functions
+    # This is the preferred way to do this going forward
+    reset_document_sync(r)
+
     RedisDocumentSet.reset_all(r)
     RedisUserGroup.reset_all(r)
     RedisConnectorDelete.reset_all(r)

backend/onyx/background/celery/tasks/indexing/tasks.py

@@ -53,7 +53,10 @@ from onyx.configs.constants import OnyxRedisLocks
 from onyx.configs.constants import OnyxRedisSignals
 from onyx.connectors.exceptions import ConnectorValidationError
 from onyx.db.connector import mark_ccpair_with_indexing_trigger
-from onyx.db.connector_credential_pair import fetch_connector_credential_pairs
+from onyx.db.connector_credential_pair import ConnectorType
+from onyx.db.connector_credential_pair import (
+    fetch_indexable_connector_credential_pair_ids,
+)
 from onyx.db.connector_credential_pair import get_connector_credential_pair_from_id
 from onyx.db.connector_credential_pair import set_cc_pair_repeated_error_state
 from onyx.db.engine import get_session_with_current_tenant
@@ -85,6 +88,8 @@ from shared_configs.configs import SENTRY_DSN
 
 logger = setup_logger()
 
+USER_FILE_INDEXING_LIMIT = 100
+
 
 class IndexingWatchdogTerminalStatus(str, Enum):
     """The different statuses the watchdog can finish with.
@@ -461,20 +466,37 @@ def check_for_indexing(self: Task, *, tenant_id: str) -> int | None:
                         embedding_model=embedding_model,
                     )
 
-        # gather cc_pair_ids
+        # gather cc_pair_ids + current search settings
         lock_beat.reacquire()
-        cc_pair_ids: list[int] = []
         with get_session_with_current_tenant() as db_session:
-            cc_pairs = fetch_connector_credential_pairs(
-                db_session, include_user_files=True
+            standard_cc_pair_ids = fetch_indexable_connector_credential_pair_ids(
+                db_session, connector_type=ConnectorType.STANDARD
             )
-            for cc_pair_entry in cc_pairs:
-                cc_pair_ids.append(cc_pair_entry.id)
+            # only index 50 user files at a time. This makes sense since user files are
+            # indexed only once, and then they are done. In practice, we would rarely
+            # have more than `USER_FILE_INDEXING_LIMIT` user files to index.
+            user_file_cc_pair_ids = fetch_indexable_connector_credential_pair_ids(
+                db_session,
+                connector_type=ConnectorType.USER_FILE,
+                limit=USER_FILE_INDEXING_LIMIT,
+            )
+            cc_pair_ids = standard_cc_pair_ids + user_file_cc_pair_ids
+
+            # NOTE: some potential race conditions here, but the worse case is
+            # kicking off some "invalid" indexing tasks which will just fail
+            search_settings_list = get_active_search_settings_list(db_session)
+
+        current_search_settings = next(
+            search_settings_instance
+            for search_settings_instance in search_settings_list
+            if search_settings_instance.status.is_current()
+        )
 
         # mark CC Pairs that are repeatedly failing as in repeated error state
         with get_session_with_current_tenant() as db_session:
-            current_search_settings = get_current_search_settings(db_session)
             for cc_pair_id in cc_pair_ids:
+                lock_beat.reacquire()
+
                 if is_in_repeated_error_state(
                     cc_pair_id=cc_pair_id,
                     search_settings_id=current_search_settings.id,
@@ -492,7 +514,6 @@ def check_for_indexing(self: Task, *, tenant_id: str) -> int | None:
 
             redis_connector = RedisConnector(tenant_id, cc_pair_id)
             with get_session_with_current_tenant() as db_session:
-                search_settings_list = get_active_search_settings_list(db_session)
                 for search_settings_instance in search_settings_list:
                     # skip non-live search settings that don't have background reindex enabled
                     # those should just auto-change to live shortly after creation without

backend/onyx/background/celery/tasks/vespa/document_sync.py

@@ -0,0 +1,178 @@
+import time
+from typing import cast
+from uuid import uuid4
+
+from celery import Celery
+from redis import Redis
+from redis.lock import Lock as RedisLock
+from sqlalchemy.orm import Session
+
+from onyx.configs.app_configs import DB_YIELD_PER_DEFAULT
+from onyx.configs.constants import CELERY_VESPA_SYNC_BEAT_LOCK_TIMEOUT
+from onyx.configs.constants import OnyxCeleryPriority
+from onyx.configs.constants import OnyxCeleryQueues
+from onyx.configs.constants import OnyxCeleryTask
+from onyx.configs.constants import OnyxRedisConstants
+from onyx.db.document import construct_document_id_select_by_needs_sync
+from onyx.db.document import count_documents_by_needs_sync
+from onyx.utils.logger import setup_logger
+
+# Redis keys for document sync tracking
+DOCUMENT_SYNC_PREFIX = "documentsync"
+DOCUMENT_SYNC_FENCE_KEY = f"{DOCUMENT_SYNC_PREFIX}_fence"
+DOCUMENT_SYNC_TASKSET_KEY = f"{DOCUMENT_SYNC_PREFIX}_taskset"
+
+logger = setup_logger()
+
+
+def is_document_sync_fenced(r: Redis) -> bool:
+    """Check if document sync tasks are currently in progress."""
+    return bool(r.exists(DOCUMENT_SYNC_FENCE_KEY))
+
+
+def get_document_sync_payload(r: Redis) -> int | None:
+    """Get the initial number of tasks that were created."""
+    bytes_result = r.get(DOCUMENT_SYNC_FENCE_KEY)
+    if bytes_result is None:
+        return None
+    return int(cast(int, bytes_result))
+
+
+def get_document_sync_remaining(r: Redis) -> int:
+    """Get the number of tasks still pending completion."""
+    return cast(int, r.scard(DOCUMENT_SYNC_TASKSET_KEY))
+
+
+def set_document_sync_fence(r: Redis, payload: int | None) -> None:
+    """Set up the fence and register with active fences."""
+    if payload is None:
+        r.srem(OnyxRedisConstants.ACTIVE_FENCES, DOCUMENT_SYNC_FENCE_KEY)
+        r.delete(DOCUMENT_SYNC_FENCE_KEY)
+        return
+
+    r.set(DOCUMENT_SYNC_FENCE_KEY, payload)
+    r.sadd(OnyxRedisConstants.ACTIVE_FENCES, DOCUMENT_SYNC_FENCE_KEY)
+
+
+def delete_document_sync_taskset(r: Redis) -> None:
+    """Clear the document sync taskset."""
+    r.delete(DOCUMENT_SYNC_TASKSET_KEY)
+
+
+def reset_document_sync(r: Redis) -> None:
+    """Reset all document sync tracking data."""
+    r.srem(OnyxRedisConstants.ACTIVE_FENCES, DOCUMENT_SYNC_FENCE_KEY)
+    r.delete(DOCUMENT_SYNC_TASKSET_KEY)
+    r.delete(DOCUMENT_SYNC_FENCE_KEY)
+
+
+def generate_document_sync_tasks(
+    r: Redis,
+    max_tasks: int,
+    celery_app: Celery,
+    db_session: Session,
+    lock: RedisLock,
+    tenant_id: str,
+) -> tuple[int, int]:
+    """Generate sync tasks for all documents that need syncing.
+
+    Args:
+        r: Redis client
+        max_tasks: Maximum number of tasks to generate
+        celery_app: Celery application instance
+        db_session: Database session
+        lock: Redis lock for coordination
+        tenant_id: Tenant identifier
+
+    Returns:
+        tuple[int, int]: (tasks_generated, total_docs_found)
+    """
+    last_lock_time = time.monotonic()
+    num_tasks_sent = 0
+    num_docs = 0
+
+    # Get all documents that need syncing
+    stmt = construct_document_id_select_by_needs_sync()
+
+    for doc_id in db_session.scalars(stmt).yield_per(DB_YIELD_PER_DEFAULT):
+        doc_id = cast(str, doc_id)
+        current_time = time.monotonic()
+
+        # Reacquire lock periodically to prevent timeout
+        if current_time - last_lock_time >= (CELERY_VESPA_SYNC_BEAT_LOCK_TIMEOUT / 4):
+            lock.reacquire()
+            last_lock_time = current_time
+
+        num_docs += 1
+
+        # Create a unique task ID
+        custom_task_id = f"{DOCUMENT_SYNC_PREFIX}_{uuid4()}"
+
+        # Add to the tracking taskset in Redis BEFORE creating the celery task
+        r.sadd(DOCUMENT_SYNC_TASKSET_KEY, custom_task_id)
+
+        # Create the Celery task
+        celery_app.send_task(
+            OnyxCeleryTask.VESPA_METADATA_SYNC_TASK,
+            kwargs=dict(document_id=doc_id, tenant_id=tenant_id),
+            queue=OnyxCeleryQueues.VESPA_METADATA_SYNC,
+            task_id=custom_task_id,
+            priority=OnyxCeleryPriority.MEDIUM,
+            ignore_result=True,
+        )
+
+        num_tasks_sent += 1
+
+        if num_tasks_sent >= max_tasks:
+            break
+
+    return num_tasks_sent, num_docs
+
+
+def try_generate_stale_document_sync_tasks(
+    celery_app: Celery,
+    max_tasks: int,
+    db_session: Session,
+    r: Redis,
+    lock_beat: RedisLock,
+    tenant_id: str,
+) -> int | None:
+    # the fence is up, do nothing
+    if is_document_sync_fenced(r):
+        return None
+
+    # add tasks to celery and build up the task set to monitor in redis
+    stale_doc_count = count_documents_by_needs_sync(db_session)
+    if stale_doc_count == 0:
+        logger.info("No stale documents found. Skipping sync tasks generation.")
+        return None
+
+    logger.info(
+        f"Stale documents found (at least {stale_doc_count}). Generating sync tasks in one batch."
+    )
+
+    logger.info("generate_document_sync_tasks starting for all documents.")
+
+    # Generate all tasks in one pass
+    result = generate_document_sync_tasks(
+        r, max_tasks, celery_app, db_session, lock_beat, tenant_id
+    )
+
+    if result is None:
+        return None
+
+    tasks_generated, total_docs = result
+
+    if tasks_generated >= max_tasks:
+        logger.info(
+            f"generate_document_sync_tasks reached the task generation limit: "
+            f"tasks_generated={tasks_generated} max_tasks={max_tasks}"
+        )
+    else:
+        logger.info(
+            f"generate_document_sync_tasks finished for all documents. "
+            f"tasks_generated={tasks_generated} total_docs_found={total_docs}"
+        )
+
+    set_document_sync_fence(r, tasks_generated)
+    return tasks_generated

backend/onyx/background/celery/tasks/vespa/tasks.py

@@ -20,14 +20,19 @@ from onyx.background.celery.tasks.shared.RetryDocumentIndex import RetryDocument
 from onyx.background.celery.tasks.shared.tasks import LIGHT_SOFT_TIME_LIMIT
 from onyx.background.celery.tasks.shared.tasks import LIGHT_TIME_LIMIT
 from onyx.background.celery.tasks.shared.tasks import OnyxCeleryTaskCompletionStatus
+from onyx.background.celery.tasks.vespa.document_sync import DOCUMENT_SYNC_FENCE_KEY
+from onyx.background.celery.tasks.vespa.document_sync import get_document_sync_payload
+from onyx.background.celery.tasks.vespa.document_sync import get_document_sync_remaining
+from onyx.background.celery.tasks.vespa.document_sync import reset_document_sync
+from onyx.background.celery.tasks.vespa.document_sync import (
+    try_generate_stale_document_sync_tasks,
+)
 from onyx.configs.app_configs import JOB_TIMEOUT
 from onyx.configs.app_configs import VESPA_SYNC_MAX_TASKS
 from onyx.configs.constants import CELERY_VESPA_SYNC_BEAT_LOCK_TIMEOUT
 from onyx.configs.constants import OnyxCeleryTask
 from onyx.configs.constants import OnyxRedisConstants
 from onyx.configs.constants import OnyxRedisLocks
-from onyx.db.connector_credential_pair import get_connector_credential_pairs
-from onyx.db.document import count_documents_by_needs_sync
 from onyx.db.document import get_document
 from onyx.db.document import mark_document_as_synced
 from onyx.db.document_set import delete_document_set
@@ -47,10 +52,6 @@ from onyx.db.sync_record import update_sync_record_status
 from onyx.document_index.factory import get_default_document_index
 from onyx.document_index.interfaces import VespaDocumentFields
 from onyx.httpx.httpx_pool import HttpxPool
-from onyx.redis.redis_connector_credential_pair import RedisConnectorCredentialPair
-from onyx.redis.redis_connector_credential_pair import (
-    RedisGlobalConnectorCredentialPair,
-)
 from onyx.redis.redis_document_set import RedisDocumentSet
 from onyx.redis.redis_pool import get_redis_client
 from onyx.redis.redis_pool import get_redis_replica_client
@@ -166,8 +167,11 @@ def check_for_vespa_sync_task(self: Task, *, tenant_id: str) -> bool | None:
                 continue
 
             key_str = key_bytes.decode("utf-8")
-            if key_str == RedisGlobalConnectorCredentialPair.FENCE_KEY:
-                monitor_connector_taskset(r)
+            # NOTE: removing the "Redis*" classes, prefer to just have functions to
+            # do these things going forward. In short, things should generally be like the doc
+            # sync task rather than the others
+            if key_str == DOCUMENT_SYNC_FENCE_KEY:
+                monitor_document_sync_taskset(r)
             elif key_str.startswith(RedisDocumentSet.FENCE_PREFIX):
                 with get_session_with_current_tenant() as db_session:
                     monitor_document_set_taskset(tenant_id, key_bytes, r, db_session)
@@ -203,82 +207,6 @@ def check_for_vespa_sync_task(self: Task, *, tenant_id: str) -> bool | None:
     return True
 
 
-def try_generate_stale_document_sync_tasks(
-    celery_app: Celery,
-    max_tasks: int,
-    db_session: Session,
-    r: Redis,
-    lock_beat: RedisLock,
-    tenant_id: str,
-) -> int | None:
-    # the fence is up, do nothing
-
-    redis_global_ccpair = RedisGlobalConnectorCredentialPair(r)
-    if redis_global_ccpair.fenced:
-        return None
-
-    redis_global_ccpair.delete_taskset()
-
-    # add tasks to celery and build up the task set to monitor in redis
-    stale_doc_count = count_documents_by_needs_sync(db_session)
-    if stale_doc_count == 0:
-        return None
-
-    task_logger.info(
-        f"Stale documents found (at least {stale_doc_count}). Generating sync tasks by cc pair."
-    )
-
-    task_logger.info(
-        "RedisConnector.generate_tasks starting by cc_pair. "
-        "Documents spanning multiple cc_pairs will only be synced once."
-    )
-
-    docs_to_skip: set[str] = set()
-
-    # rkuo: we could technically sync all stale docs in one big pass.
-    # but I feel it's more understandable to group the docs by cc_pair
-    total_tasks_generated = 0
-    tasks_remaining = max_tasks
-    cc_pairs = get_connector_credential_pairs(db_session)
-    for cc_pair in cc_pairs:
-        lock_beat.reacquire()
-
-        rc = RedisConnectorCredentialPair(tenant_id, cc_pair.id)
-        rc.set_skip_docs(docs_to_skip)
-        result = rc.generate_tasks(
-            tasks_remaining, celery_app, db_session, r, lock_beat, tenant_id
-        )
-
-        if result is None:
-            continue
-
-        if result[1] == 0:
-            continue
-
-        task_logger.info(
-            f"RedisConnector.generate_tasks finished for single cc_pair. "
-            f"cc_pair={cc_pair.id} tasks_generated={result[0]} tasks_possible={result[1]}"
-        )
-
-        total_tasks_generated += result[0]
-        tasks_remaining -= result[0]
-        if tasks_remaining <= 0:
-            break
-
-    if tasks_remaining <= 0:
-        task_logger.info(
-            f"RedisConnector.generate_tasks reached the task generation limit: "
-            f"total_tasks_generated={total_tasks_generated} max_tasks={max_tasks}"
-        )
-    else:
-        task_logger.info(
-            f"RedisConnector.generate_tasks finished for all cc_pairs. total_tasks_generated={total_tasks_generated}"
-        )
-
-    redis_global_ccpair.set_fence(total_tasks_generated)
-    return total_tasks_generated
-
-
 def try_generate_document_set_sync_tasks(
     celery_app: Celery,
     document_set_id: int,
@@ -433,19 +361,18 @@ def try_generate_user_group_sync_tasks(
     return tasks_generated
 
 
-def monitor_connector_taskset(r: Redis) -> None:
-    redis_global_ccpair = RedisGlobalConnectorCredentialPair(r)
-    initial_count = redis_global_ccpair.payload
+def monitor_document_sync_taskset(r: Redis) -> None:
+    initial_count = get_document_sync_payload(r)
     if initial_count is None:
         return
 
-    remaining = redis_global_ccpair.get_remaining()
+    remaining = get_document_sync_remaining(r)
     task_logger.info(
-        f"Stale document sync progress: remaining={remaining} initial={initial_count}"
+        f"Document sync progress: remaining={remaining} initial={initial_count}"
     )
     if remaining == 0:
-        redis_global_ccpair.reset()
-        task_logger.info(f"Successfully synced stale documents. count={initial_count}")
+        reset_document_sync(r)
+        task_logger.info(f"Successfully synced all documents. count={initial_count}")
 
 
 def monitor_document_set_taskset(

backend/onyx/configs/app_configs.py

@@ -35,6 +35,9 @@ GENERATIVE_MODEL_ACCESS_CHECK_FREQ = int(
 )  # 1 day
 DISABLE_GENERATIVE_AI = os.environ.get("DISABLE_GENERATIVE_AI", "").lower() == "true"
 
+# Controls whether users can use User Knowledge (personal documents) in assistants
+DISABLE_USER_KNOWLEDGE = os.environ.get("DISABLE_USER_KNOWLEDGE", "").lower() == "true"
+
 # Controls whether to allow admin query history reports with:
 # 1. associated user emails
 # 2. anonymized user emails
@@ -310,7 +313,7 @@ except ValueError:
     CELERY_WORKER_INDEXING_CONCURRENCY = CELERY_WORKER_INDEXING_CONCURRENCY_DEFAULT
 
 # The maximum number of tasks that can be queued up to sync to Vespa in a single pass
-VESPA_SYNC_MAX_TASKS = 1024
+VESPA_SYNC_MAX_TASKS = 8192
 
 DB_YIELD_PER_DEFAULT = 64
 
@@ -746,3 +749,7 @@ IMAGE_ANALYSIS_SYSTEM_PROMPT = os.environ.get(
 DISABLE_AUTO_AUTH_REFRESH = (
     os.environ.get("DISABLE_AUTO_AUTH_REFRESH", "").lower() == "true"
 )
+
+# Forcing Vespa Language
+# English: en, German:de, etc. See: https://docs.vespa.ai/en/linguistics.html
+VESPA_LANGUAGE_OVERRIDE = os.environ.get("VESPA_LANGUAGE_OVERRIDE")

backend/onyx/connectors/confluence/connector.py

@@ -21,6 +21,9 @@ from onyx.connectors.confluence.utils import datetime_from_string
 from onyx.connectors.confluence.utils import process_attachment
 from onyx.connectors.confluence.utils import update_param_in_path
 from onyx.connectors.confluence.utils import validate_attachment_filetype
+from onyx.connectors.cross_connector_utils.miscellaneous_utils import (
+    is_atlassian_date_error,
+)
 from onyx.connectors.exceptions import ConnectorValidationError
 from onyx.connectors.exceptions import CredentialExpiredError
 from onyx.connectors.exceptions import InsufficientPermissionsError
@@ -76,10 +79,6 @@ ONE_DAY = ONE_HOUR * 24
 MAX_CACHED_IDS = 100
 
 
-def _should_propagate_error(e: Exception) -> bool:
-    return "field 'updated' is invalid" in str(e)
-
-
 class ConfluenceCheckpoint(ConnectorCheckpoint):
 
     next_page_url: str | None
@@ -367,7 +366,7 @@ class ConfluenceConnector(
             )
         except Exception as e:
             logger.error(f"Error converting page {page.get('id', 'unknown')}: {e}")
-            if _should_propagate_error(e):
+            if is_atlassian_date_error(e):  # propagate error to be caught and retried
                 raise
             return ConnectorFailure(
                 failed_document=DocumentFailure(
@@ -446,7 +445,9 @@ class ConfluenceConnector(
                     f"Failed to extract/summarize attachment {attachment['title']}",
                     exc_info=e,
                 )
-                if _should_propagate_error(e):
+                if is_atlassian_date_error(
+                    e
+                ):  # propagate error to be caught and retried
                     raise
                 return ConnectorFailure(
                     failed_document=DocumentFailure(
@@ -536,7 +537,7 @@ class ConfluenceConnector(
         try:
             return self._fetch_document_batches(checkpoint, start, end)
         except Exception as e:
-            if _should_propagate_error(e) and start is not None:
+            if is_atlassian_date_error(e) and start is not None:
                 logger.warning(
                     "Confluence says we provided an invalid 'updated' field. This may indicate"
                     "a real issue, but can also appear during edge cases like daylight"

backend/onyx/connectors/cross_connector_utils/miscellaneous_utils.py

@@ -86,3 +86,7 @@ def get_oauth_callback_uri(base_domain: str, connector_id: str) -> str:
         # Used for development
         base_domain = CONNECTOR_LOCALHOST_OVERRIDE
     return f"{base_domain.strip('/')}/connector/oauth/callback/{connector_id}"
+
+
+def is_atlassian_date_error(e: Exception) -> bool:
+    return "field 'updated' is invalid" in str(e)

backend/onyx/connectors/onyx_jira/connector.py

@@ -12,6 +12,9 @@ from onyx.configs.app_configs import INDEX_BATCH_SIZE
 from onyx.configs.app_configs import JIRA_CONNECTOR_LABELS_TO_SKIP
 from onyx.configs.app_configs import JIRA_CONNECTOR_MAX_TICKET_SIZE
 from onyx.configs.constants import DocumentSource
+from onyx.connectors.cross_connector_utils.miscellaneous_utils import (
+    is_atlassian_date_error,
+)
 from onyx.connectors.cross_connector_utils.miscellaneous_utils import time_str_to_utc
 from onyx.connectors.exceptions import ConnectorValidationError
 from onyx.connectors.exceptions import CredentialExpiredError
@@ -40,6 +43,8 @@ from onyx.utils.logger import setup_logger
 
 logger = setup_logger()
 
+ONE_HOUR = 3600
+
 JIRA_API_VERSION = os.environ.get("JIRA_API_VERSION") or "2"
 _JIRA_SLIM_PAGE_SIZE = 500
 _JIRA_FULL_PAGE_SIZE = 50
@@ -240,7 +245,17 @@ class JiraConnector(CheckpointedConnector[JiraConnectorCheckpoint], SlimConnecto
         checkpoint: JiraConnectorCheckpoint,
     ) -> CheckpointOutput[JiraConnectorCheckpoint]:
         jql = self._get_jql_query(start, end)
+        try:
+            return self._load_from_checkpoint(jql, checkpoint)
+        except Exception as e:
+            if is_atlassian_date_error(e):
+                jql = self._get_jql_query(start - ONE_HOUR, end)
+                return self._load_from_checkpoint(jql, checkpoint)
+            raise e
 
+    def _load_from_checkpoint(
+        self, jql: str, checkpoint: JiraConnectorCheckpoint
+    ) -> CheckpointOutput[JiraConnectorCheckpoint]:
         # Get the current offset from checkpoint or start at 0
         starting_offset = checkpoint.offset or 0
         current_offset = starting_offset

backend/onyx/db/connector_credential_pair.py

@@ -1,4 +1,5 @@
 from datetime import datetime
+from enum import Enum
 from typing import TypeVarTuple
 
 from fastapi import HTTPException
@@ -41,6 +42,11 @@ logger = setup_logger()
 R = TypeVarTuple("R")
 
 
+class ConnectorType(str, Enum):
+    STANDARD = "standard"
+    USER_FILE = "user_file"
+
+
 def _add_user_filters(
     stmt: Select[tuple[*R]], user: User | None, get_editable: bool = True
 ) -> Select[tuple[*R]]:
@@ -619,14 +625,24 @@ def remove_credential_from_connector(
     )
 
 
-def fetch_connector_credential_pairs(
+def fetch_indexable_connector_credential_pair_ids(
     db_session: Session,
-    include_user_files: bool = False,
-) -> list[ConnectorCredentialPair]:
-    stmt = select(ConnectorCredentialPair)
-    if not include_user_files:
-        stmt = stmt.where(ConnectorCredentialPair.is_user_file != True)  # noqa: E712
-    return list(db_session.scalars(stmt).unique().all())
+    connector_type: ConnectorType | None = None,
+    limit: int | None = None,
+) -> list[int]:
+    stmt = select(ConnectorCredentialPair.id)
+    stmt = stmt.where(
+        ConnectorCredentialPair.status.in_(
+            ConnectorCredentialPairStatus.active_statuses()
+        )
+    )
+    if connector_type == ConnectorType.USER_FILE:
+        stmt = stmt.where(ConnectorCredentialPair.is_user_file.is_(True))
+    elif connector_type == ConnectorType.STANDARD:
+        stmt = stmt.where(ConnectorCredentialPair.is_user_file.is_(False))
+    if limit:
+        stmt = stmt.limit(limit)
+    return list(db_session.scalars(stmt).all())
 
 
 def resync_cc_pair(

backend/onyx/db/document.py

@@ -66,10 +66,6 @@ def count_documents_by_needs_sync(session: Session) -> int:
 
     return (
         session.query(DbDocument.id)
-        .join(
-            DocumentByConnectorCredentialPair,
-            DbDocument.id == DocumentByConnectorCredentialPair.id,
-        )
         .filter(
             or_(
                 DbDocument.last_modified > DbDocument.last_synced,
@@ -80,67 +76,22 @@ def count_documents_by_needs_sync(session: Session) -> int:
     )
 
 
-def construct_document_select_for_connector_credential_pair_by_needs_sync(
-    connector_id: int, credential_id: int
-) -> Select:
-    return (
-        select(DbDocument)
-        .join(
-            DocumentByConnectorCredentialPair,
-            DbDocument.id == DocumentByConnectorCredentialPair.id,
-        )
-        .where(
-            and_(
-                DocumentByConnectorCredentialPair.connector_id == connector_id,
-                DocumentByConnectorCredentialPair.credential_id == credential_id,
-                or_(
-                    DbDocument.last_modified > DbDocument.last_synced,
-                    DbDocument.last_synced.is_(None),
-                ),
-            )
+def construct_document_id_select_by_needs_sync() -> Select:
+    """Get all document IDs that need syncing across all connector credential pairs.
+
+    Returns a Select statement for documents where:
+    1. last_modified is newer than last_synced
+    2. last_synced is null (meaning we've never synced)
+    AND the document has a relationship with a connector/credential pair
+    """
+    return select(DbDocument.id).where(
+        or_(
+            DbDocument.last_modified > DbDocument.last_synced,
+            DbDocument.last_synced.is_(None),
         )
     )
 
 
-def construct_document_id_select_for_connector_credential_pair_by_needs_sync(
-    connector_id: int, credential_id: int
-) -> Select:
-    return (
-        select(DbDocument.id)
-        .join(
-            DocumentByConnectorCredentialPair,
-            DbDocument.id == DocumentByConnectorCredentialPair.id,
-        )
-        .where(
-            and_(
-                DocumentByConnectorCredentialPair.connector_id == connector_id,
-                DocumentByConnectorCredentialPair.credential_id == credential_id,
-                or_(
-                    DbDocument.last_modified > DbDocument.last_synced,
-                    DbDocument.last_synced.is_(None),
-                ),
-            )
-        )
-    )
-
-
-def get_all_documents_needing_vespa_sync_for_cc_pair(
-    db_session: Session, cc_pair_id: int
-) -> list[DbDocument]:
-    cc_pair = get_connector_credential_pair_from_id(
-        db_session=db_session,
-        cc_pair_id=cc_pair_id,
-    )
-    if not cc_pair:
-        raise ValueError(f"No CC pair found with ID: {cc_pair_id}")
-
-    stmt = construct_document_select_for_connector_credential_pair_by_needs_sync(
-        cc_pair.connector_id, cc_pair.credential_id
-    )
-
-    return list(db_session.scalars(stmt).all())
-
-
 def construct_document_id_select_for_connector_credential_pair(
     connector_id: int, credential_id: int | None = None
 ) -> Select:

backend/onyx/db/enums.py

@@ -86,12 +86,16 @@ class ConnectorCredentialPairStatus(str, PyEnum):
     DELETING = "DELETING"
     INVALID = "INVALID"
 
+    @classmethod
+    def active_statuses(cls) -> list["ConnectorCredentialPairStatus"]:
+        return [
+            ConnectorCredentialPairStatus.ACTIVE,
+            ConnectorCredentialPairStatus.SCHEDULED,
+            ConnectorCredentialPairStatus.INITIAL_INDEXING,
+        ]
+
     def is_active(self) -> bool:
-        return (
-            self == ConnectorCredentialPairStatus.ACTIVE
-            or self == ConnectorCredentialPairStatus.SCHEDULED
-            or self == ConnectorCredentialPairStatus.INITIAL_INDEXING
-        )
+        return self in self.active_statuses()
 
 
 class AccessType(str, PyEnum):

backend/onyx/db/persona.py

@@ -1,5 +1,6 @@
 from collections.abc import Sequence
 from datetime import datetime
+from enum import Enum
 from uuid import UUID
 
 from fastapi import HTTPException
@@ -10,7 +11,6 @@ from sqlalchemy import Select
 from sqlalchemy import select
 from sqlalchemy import update
 from sqlalchemy.orm import aliased
-from sqlalchemy.orm import joinedload
 from sqlalchemy.orm import selectinload
 from sqlalchemy.orm import Session
 
@@ -37,7 +37,9 @@ from onyx.db.models import UserFolder
 from onyx.db.models import UserGroup
 from onyx.db.notification import create_notification
 from onyx.server.features.persona.models import FullPersonaSnapshot
+from onyx.server.features.persona.models import MinimalPersonaSnapshot
 from onyx.server.features.persona.models import PersonaSharedNotificationData
+from onyx.server.features.persona.models import PersonaSnapshot
 from onyx.server.features.persona.models import PersonaUpsertRequest
 from onyx.utils.logger import setup_logger
 from onyx.utils.variable_functionality import fetch_versioned_implementation
@@ -45,9 +47,15 @@ from onyx.utils.variable_functionality import fetch_versioned_implementation
 logger = setup_logger()
 
 
+class PersonaLoadType(Enum):
+    NONE = "none"
+    MINIMAL = "minimal"
+    FULL = "full"
+
+
 def _add_user_filters(
-    stmt: Select, user: User | None, get_editable: bool = True
-) -> Select:
+    stmt: Select[tuple[Persona]], user: User | None, get_editable: bool = True
+) -> Select[tuple[Persona]]:
     # If user is None and auth is disabled, assume the user is an admin
     if (user is None and DISABLE_AUTH) or (user and user.role == UserRole.ADMIN):
         return stmt
@@ -321,7 +329,45 @@ def update_persona_public_status(
     db_session.commit()
 
 
-def get_personas_for_user(
+def _build_persona_filters(
+    stmt: Select[tuple[Persona]],
+    include_default: bool,
+    include_slack_bot_personas: bool,
+    include_deleted: bool,
+) -> Select[tuple[Persona]]:
+    if not include_default:
+        stmt = stmt.where(Persona.builtin_persona.is_(False))
+    if not include_slack_bot_personas:
+        stmt = stmt.where(not_(Persona.name.startswith(SLACK_BOT_PERSONA_PREFIX)))
+    if not include_deleted:
+        stmt = stmt.where(Persona.deleted.is_(False))
+    return stmt
+
+
+def get_minimal_persona_snapshots_for_user(
+    user: User | None,
+    db_session: Session,
+    get_editable: bool = True,
+    include_default: bool = True,
+    include_slack_bot_personas: bool = False,
+    include_deleted: bool = False,
+) -> list[MinimalPersonaSnapshot]:
+    stmt = select(Persona)
+    stmt = _add_user_filters(stmt, user, get_editable)
+    stmt = _build_persona_filters(
+        stmt, include_default, include_slack_bot_personas, include_deleted
+    )
+    stmt = stmt.options(
+        selectinload(Persona.tools),
+        selectinload(Persona.labels),
+        selectinload(Persona.document_sets),
+        selectinload(Persona.user),
+    )
+    results = db_session.scalars(stmt).all()
+    return [MinimalPersonaSnapshot.from_model(persona) for persona in results]
+
+
+def get_persona_snapshots_for_user(
     # if user is `None` assume the user is an admin or auth is disabled
     user: User | None,
     db_session: Session,
@@ -329,32 +375,37 @@ def get_personas_for_user(
     include_default: bool = True,
     include_slack_bot_personas: bool = False,
     include_deleted: bool = False,
-    joinedload_all: bool = False,
+) -> list[PersonaSnapshot]:
+    stmt = select(Persona)
+    stmt = _add_user_filters(stmt, user, get_editable)
+    stmt = _build_persona_filters(
+        stmt, include_default, include_slack_bot_personas, include_deleted
+    )
+    stmt = stmt.options(
+        selectinload(Persona.tools),
+        selectinload(Persona.labels),
+        selectinload(Persona.document_sets),
+        selectinload(Persona.user),
+    )
+
+    results = db_session.scalars(stmt).all()
+    return [PersonaSnapshot.from_model(persona) for persona in results]
+
+
+def get_raw_personas_for_user(
+    user: User | None,
+    db_session: Session,
+    get_editable: bool = True,
+    include_default: bool = True,
+    include_slack_bot_personas: bool = False,
+    include_deleted: bool = False,
 ) -> Sequence[Persona]:
     stmt = select(Persona)
     stmt = _add_user_filters(stmt, user, get_editable)
-
-    if not include_default:
-        stmt = stmt.where(Persona.builtin_persona.is_(False))
-    if not include_slack_bot_personas:
-        stmt = stmt.where(not_(Persona.name.startswith(SLACK_BOT_PERSONA_PREFIX)))
-    if not include_deleted:
-        stmt = stmt.where(Persona.deleted.is_(False))
-
-    if joinedload_all:
-        stmt = stmt.options(
-            selectinload(Persona.prompts),
-            selectinload(Persona.tools),
-            selectinload(Persona.document_sets),
-            selectinload(Persona.groups),
-            selectinload(Persona.users),
-            selectinload(Persona.labels),
-            selectinload(Persona.user_files),
-            selectinload(Persona.user_folders),
-        )
-
-    results = db_session.execute(stmt).scalars().all()
-    return results
+    stmt = _build_persona_filters(
+        stmt, include_default, include_slack_bot_personas, include_deleted
+    )
+    return db_session.scalars(stmt).all()
 
 
 def get_personas(db_session: Session) -> Sequence[Persona]:
@@ -815,7 +866,7 @@ def delete_persona_label(label_id: int, db_session: Session) -> None:
 def persona_has_search_tool(persona_id: int, db_session: Session) -> bool:
     persona = (
         db_session.query(Persona)
-        .options(joinedload(Persona.tools))
+        .options(selectinload(Persona.tools))
         .filter(Persona.id == persona_id)
         .one_or_none()
     )

backend/onyx/document_index/vespa/chunk_retrieval.py

@@ -11,6 +11,7 @@ import httpx
 from retry import retry
 
 from onyx.configs.app_configs import LOG_VESPA_TIMING_INFORMATION
+from onyx.configs.app_configs import VESPA_LANGUAGE_OVERRIDE
 from onyx.context.search.models import IndexFilters
 from onyx.context.search.models import InferenceChunkUncleaned
 from onyx.document_index.interfaces import VespaChunkRequest
@@ -308,6 +309,9 @@ def query_vespa(
         ),
     )
 
+    if VESPA_LANGUAGE_OVERRIDE:
+        params["language"] = VESPA_LANGUAGE_OVERRIDE
+
     try:
         with get_vespa_http_client() as http_client:
             response = http_client.post(SEARCH_ENDPOINT, json=params)

backend/onyx/redis/redis_connector_credential_pair.py

@@ -1,207 +0,0 @@
-import time
-from typing import cast
-from uuid import uuid4
-
-import redis
-from celery import Celery
-from redis import Redis
-from redis.lock import Lock as RedisLock
-from sqlalchemy.orm import Session
-
-from onyx.configs.app_configs import DB_YIELD_PER_DEFAULT
-from onyx.configs.constants import CELERY_VESPA_SYNC_BEAT_LOCK_TIMEOUT
-from onyx.configs.constants import OnyxCeleryPriority
-from onyx.configs.constants import OnyxCeleryQueues
-from onyx.configs.constants import OnyxCeleryTask
-from onyx.configs.constants import OnyxRedisConstants
-from onyx.db.connector_credential_pair import get_connector_credential_pair_from_id
-from onyx.db.document import (
-    construct_document_id_select_for_connector_credential_pair_by_needs_sync,
-)
-from onyx.redis.redis_object_helper import RedisObjectHelper
-
-
-class RedisConnectorCredentialPair(RedisObjectHelper):
-    """This class is used to scan documents by cc_pair in the db and collect them into
-    a unified set for syncing.
-
-    It differs from the other redis helpers in that the taskset used spans
-    all connectors and is not per connector."""
-
-    PREFIX = "connectorsync"
-    TASKSET_PREFIX = PREFIX + "_taskset"
-
-    def __init__(self, tenant_id: str, id: int) -> None:
-        super().__init__(tenant_id, str(id))
-
-        # documents that should be skipped
-        self.skip_docs: set[str] = set()
-
-    @classmethod
-    def get_taskset_key(cls) -> str:
-        return RedisConnectorCredentialPair.TASKSET_PREFIX
-
-    @property
-    def taskset_key(self) -> str:
-        """Notice that this is intentionally reusing the same taskset for all
-        connector syncs"""
-        # example: connectorsync_taskset
-        return f"{self.TASKSET_PREFIX}"
-
-    def set_skip_docs(self, skip_docs: set[str]) -> None:
-        # documents that should be skipped. Note that this class updates
-        # the list on the fly
-        self.skip_docs = skip_docs
-
-    def generate_tasks(
-        self,
-        max_tasks: int,
-        celery_app: Celery,
-        db_session: Session,
-        redis_client: Redis,
-        lock: RedisLock,
-        tenant_id: str,
-    ) -> tuple[int, int] | None:
-        """We can limit the number of tasks generated here, which is useful to prevent
-        one tenant from overwhelming the sync queue.
-
-        This works because the dirty state of a document is in the DB, so more docs
-        get picked up after the limited set of tasks is complete.
-        """
-
-        last_lock_time = time.monotonic()
-
-        num_tasks_sent = 0
-
-        cc_pair = get_connector_credential_pair_from_id(
-            db_session=db_session,
-            cc_pair_id=int(self._id),
-        )
-        if not cc_pair:
-            return None
-
-        stmt = construct_document_id_select_for_connector_credential_pair_by_needs_sync(
-            cc_pair.connector_id, cc_pair.credential_id
-        )
-
-        num_docs = 0
-
-        for doc_id in db_session.scalars(stmt).yield_per(DB_YIELD_PER_DEFAULT):
-            doc_id = cast(str, doc_id)
-            current_time = time.monotonic()
-            if current_time - last_lock_time >= (
-                CELERY_VESPA_SYNC_BEAT_LOCK_TIMEOUT / 4
-            ):
-                lock.reacquire()
-                last_lock_time = current_time
-
-            num_docs += 1
-
-            # check if we should skip the document (typically because it's already syncing)
-            if doc_id in self.skip_docs:
-                continue
-
-            # celery's default task id format is "dd32ded3-00aa-4884-8b21-42f8332e7fac"
-            # the key for the result is "celery-task-meta-dd32ded3-00aa-4884-8b21-42f8332e7fac"
-            # we prefix the task id so it's easier to keep track of who created the task
-            # aka "documentset_1_6dd32ded3-00aa-4884-8b21-42f8332e7fac"
-            custom_task_id = f"{self.task_id_prefix}_{uuid4()}"
-
-            # add to the tracking taskset in redis BEFORE creating the celery task.
-            # note that for the moment we are using a single taskset key, not differentiated by cc_pair id
-            redis_client.sadd(
-                RedisConnectorCredentialPair.get_taskset_key(), custom_task_id
-            )
-
-            # Priority on sync's triggered by new indexing should be medium
-            celery_app.send_task(
-                OnyxCeleryTask.VESPA_METADATA_SYNC_TASK,
-                kwargs=dict(document_id=doc_id, tenant_id=tenant_id),
-                queue=OnyxCeleryQueues.VESPA_METADATA_SYNC,
-                task_id=custom_task_id,
-                priority=OnyxCeleryPriority.MEDIUM,
-                ignore_result=True,
-            )
-
-            num_tasks_sent += 1
-            self.skip_docs.add(doc_id)
-
-            if num_tasks_sent >= max_tasks:
-                break
-
-        return num_tasks_sent, num_docs
-
-
-class RedisGlobalConnectorCredentialPair:
-    """This class is used to scan documents by cc_pair in the db and collect them into
-    a unified set for syncing.
-
-    It differs from the other redis helpers in that the taskset used spans
-    all connectors and is not per connector."""
-
-    PREFIX = "connectorsync"
-    FENCE_KEY = PREFIX + "_fence"
-    TASKSET_KEY = PREFIX + "_taskset"
-
-    def __init__(self, redis: redis.Redis) -> None:
-        self.redis = redis
-
-    @property
-    def fenced(self) -> bool:
-        if self.redis.exists(self.fence_key):
-            return True
-
-        return False
-
-    @property
-    def payload(self) -> int | None:
-        bytes = self.redis.get(self.fence_key)
-        if bytes is None:
-            return None
-
-        progress = int(cast(int, bytes))
-        return progress
-
-    def get_remaining(self) -> int:
-        remaining = cast(int, self.redis.scard(self.taskset_key))
-        return remaining
-
-    @property
-    def fence_key(self) -> str:
-        """Notice that this is intentionally reusing the same fence for all
-        connector syncs"""
-        # example: connectorsync_fence
-        return f"{self.FENCE_KEY}"
-
-    @property
-    def taskset_key(self) -> str:
-        """Notice that this is intentionally reusing the same taskset for all
-        connector syncs"""
-        # example: connectorsync_taskset
-        return f"{self.TASKSET_KEY}"
-
-    def set_fence(self, payload: int | None) -> None:
-        if payload is None:
-            self.redis.srem(OnyxRedisConstants.ACTIVE_FENCES, self.fence_key)
-            self.redis.delete(self.fence_key)
-            return
-
-        self.redis.set(self.fence_key, payload)
-        self.redis.sadd(OnyxRedisConstants.ACTIVE_FENCES, self.fence_key)
-
-    def delete_taskset(self) -> None:
-        self.redis.delete(self.taskset_key)
-
-    def reset(self) -> None:
-        self.redis.srem(OnyxRedisConstants.ACTIVE_FENCES, self.fence_key)
-        self.redis.delete(self.taskset_key)
-        self.redis.delete(self.fence_key)
-
-    @staticmethod
-    def reset_all(r: redis.Redis) -> None:
-        r.srem(
-            OnyxRedisConstants.ACTIVE_FENCES,
-            RedisGlobalConnectorCredentialPair.FENCE_KEY,
-        )
-        r.delete(RedisGlobalConnectorCredentialPair.TASKSET_KEY)
-        r.delete(RedisGlobalConnectorCredentialPair.FENCE_KEY)

backend/onyx/redis/redis_utils.py

@@ -1,6 +1,3 @@
-from onyx.redis.redis_connector_credential_pair import (
-    RedisGlobalConnectorCredentialPair,
-)
 from onyx.redis.redis_connector_delete import RedisConnectorDelete
 from onyx.redis.redis_connector_doc_perm_sync import RedisConnectorPermissionSync
 from onyx.redis.redis_connector_index import RedisConnectorIndex
@@ -11,8 +8,6 @@ from onyx.redis.redis_usergroup import RedisUserGroup
 
 def is_fence(key_bytes: bytes) -> bool:
     key_str = key_bytes.decode("utf-8")
-    if key_str == RedisGlobalConnectorCredentialPair.FENCE_KEY:
-        return True
     if key_str.startswith(RedisDocumentSet.FENCE_PREFIX):
         return True
     if key_str.startswith(RedisUserGroup.FENCE_PREFIX):

backend/onyx/seeding/load_yamls.py

@@ -17,6 +17,9 @@ from onyx.db.persona import upsert_persona
 from onyx.db.prompts import get_prompt_by_name
 from onyx.db.prompts import upsert_prompt
 from onyx.db.user_documents import upsert_user_folder
+from onyx.tools.tool_implementations.images.image_generation_tool import (
+    ImageGenerationTool,
+)
 
 
 def load_user_folders_from_yaml(
@@ -136,7 +139,7 @@ def load_personas_from_yaml(
         if persona.get("image_generation"):
             image_gen_tool = (
                 db_session.query(ToolDBModel)
-                .filter(ToolDBModel.name == "ImageGenerationTool")
+                .filter(ToolDBModel.name == ImageGenerationTool.__name__)
                 .first()
             )
             if image_gen_tool:

backend/onyx/server/documents/models.py

@@ -405,6 +405,26 @@ class ConnectorCredentialPairDescriptor(BaseModel):
     access_type: AccessType
 
 
+class CCPairSummary(BaseModel):
+    """Simplified connector-credential pair information with just essential data"""
+
+    id: int
+    name: str | None
+    source: DocumentSource
+    access_type: AccessType
+
+    @classmethod
+    def from_cc_pair_descriptor(
+        cls, descriptor: ConnectorCredentialPairDescriptor
+    ) -> "CCPairSummary":
+        return cls(
+            id=descriptor.id,
+            name=descriptor.name,
+            source=descriptor.connector.source,
+            access_type=descriptor.access_type,
+        )
+
+
 class RunConnectorRequest(BaseModel):
     connector_id: int
     credential_ids: list[int] | None = None

backend/onyx/server/features/document_set/api.py

@@ -19,8 +19,8 @@ from onyx.db.engine import get_session
 from onyx.db.models import User
 from onyx.server.features.document_set.models import CheckDocSetPublicRequest
 from onyx.server.features.document_set.models import CheckDocSetPublicResponse
-from onyx.server.features.document_set.models import DocumentSet
 from onyx.server.features.document_set.models import DocumentSetCreationRequest
+from onyx.server.features.document_set.models import DocumentSetSummary
 from onyx.server.features.document_set.models import DocumentSetUpdateRequest
 from onyx.utils.variable_functionality import fetch_ee_implementation_or_noop
 
@@ -125,13 +125,11 @@ def list_document_sets_for_user(
     get_editable: bool = Query(
         False, description="If true, return editable document sets"
     ),
-) -> list[DocumentSet]:
-    return [
-        DocumentSet.from_model(ds)
-        for ds in fetch_all_document_sets_for_user(
-            db_session=db_session, user=user, get_editable=get_editable
-        )
-    ]
+) -> list[DocumentSetSummary]:
+    document_sets = fetch_all_document_sets_for_user(
+        db_session=db_session, user=user, get_editable=get_editable
+    )
+    return [DocumentSetSummary.from_model(ds) for ds in document_sets]
 
 
 @router.get("/document-set-public")

backend/onyx/server/features/document_set/models.py

@@ -4,6 +4,7 @@ from pydantic import BaseModel
 from pydantic import Field
 
 from onyx.db.models import DocumentSet as DocumentSetDBModel
+from onyx.server.documents.models import CCPairSummary
 from onyx.server.documents.models import ConnectorCredentialPairDescriptor
 from onyx.server.documents.models import ConnectorSnapshot
 from onyx.server.documents.models import CredentialSnapshot
@@ -77,3 +78,38 @@ class DocumentSet(BaseModel):
             users=[user.id for user in document_set_model.users],
             groups=[group.id for group in document_set_model.groups],
         )
+
+
+class DocumentSetSummary(BaseModel):
+    """Simplified document set model with minimal data for list views"""
+
+    id: int
+    name: str
+    description: str | None
+    cc_pair_summaries: list[CCPairSummary]
+    is_up_to_date: bool
+    is_public: bool
+    users: list[UUID]
+    groups: list[int]
+
+    @classmethod
+    def from_model(cls, document_set: DocumentSetDBModel) -> "DocumentSetSummary":
+        """Create a summary from a DocumentSet database model"""
+        return cls(
+            id=document_set.id,
+            name=document_set.name,
+            description=document_set.description,
+            cc_pair_summaries=[
+                CCPairSummary(
+                    id=cc_pair.id,
+                    name=cc_pair.name,
+                    source=cc_pair.connector.source,
+                    access_type=cc_pair.access_type,
+                )
+                for cc_pair in document_set.connector_credential_pairs
+            ],
+            is_up_to_date=document_set.is_up_to_date,
+            is_public=document_set.is_public,
+            users=[user.id for user in document_set.users],
+            groups=[group.id for group in document_set.groups],
+        )

backend/onyx/server/features/persona/api.py

@@ -26,8 +26,9 @@ from onyx.db.persona import create_assistant_label
 from onyx.db.persona import create_update_persona
 from onyx.db.persona import delete_persona_label
 from onyx.db.persona import get_assistant_labels
+from onyx.db.persona import get_minimal_persona_snapshots_for_user
 from onyx.db.persona import get_persona_by_id
-from onyx.db.persona import get_personas_for_user
+from onyx.db.persona import get_persona_snapshots_for_user
 from onyx.db.persona import mark_persona_as_deleted
 from onyx.db.persona import mark_persona_as_not_deleted
 from onyx.db.persona import update_all_personas_display_priority
@@ -46,6 +47,7 @@ from onyx.secondary_llm_flows.starter_message_creation import (
 from onyx.server.features.persona.models import FullPersonaSnapshot
 from onyx.server.features.persona.models import GenerateStarterMessageRequest
 from onyx.server.features.persona.models import ImageGenerationToolStatus
+from onyx.server.features.persona.models import MinimalPersonaSnapshot
 from onyx.server.features.persona.models import PersonaLabelCreate
 from onyx.server.features.persona.models import PersonaLabelResponse
 from onyx.server.features.persona.models import PersonaSharedNotificationData
@@ -53,6 +55,10 @@ from onyx.server.features.persona.models import PersonaSnapshot
 from onyx.server.features.persona.models import PersonaUpsertRequest
 from onyx.server.features.persona.models import PromptSnapshot
 from onyx.server.models import DisplayPriorityRequest
+from onyx.server.settings.store import load_settings
+from onyx.tools.tool_implementations.images.image_generation_tool import (
+    ImageGenerationTool,
+)
 from onyx.tools.utils import is_image_generation_available
 from onyx.utils.logger import setup_logger
 from onyx.utils.telemetry import create_milestone_and_report
@@ -60,6 +66,23 @@ from shared_configs.contextvars import get_current_tenant_id
 
 logger = setup_logger()
 
+
+def _validate_user_knowledge_enabled(
+    persona_upsert_request: PersonaUpsertRequest, action: str
+) -> None:
+    """Check if user knowledge is enabled when user files/folders are provided."""
+    settings = load_settings()
+    if not settings.user_knowledge_enabled:
+        if (
+            persona_upsert_request.user_file_ids
+            or persona_upsert_request.user_folder_ids
+        ):
+            raise HTTPException(
+                status_code=400,
+                detail=f"User Knowledge is disabled. Cannot {action} assistant with user files or folders.",
+            )
+
+
 admin_router = APIRouter(prefix="/admin/persona")
 basic_router = APIRouter(prefix="/persona")
 
@@ -148,16 +171,12 @@ def list_personas_admin(
     include_deleted: bool = False,
     get_editable: bool = Query(False, description="If true, return editable personas"),
 ) -> list[PersonaSnapshot]:
-    return [
-        PersonaSnapshot.from_model(persona)
-        for persona in get_personas_for_user(
-            db_session=db_session,
-            user=user,
-            get_editable=get_editable,
-            include_deleted=include_deleted,
-            joinedload_all=True,
-        )
-    ]
+    return get_persona_snapshots_for_user(
+        user=user,
+        db_session=db_session,
+        get_editable=get_editable,
+        include_deleted=include_deleted,
+    )
 
 
 @admin_router.patch("/{persona_id}/undelete")
@@ -204,6 +223,8 @@ def create_persona(
 ) -> PersonaSnapshot:
     tenant_id = get_current_tenant_id()
 
+    _validate_user_knowledge_enabled(persona_upsert_request, "create")
+
     prompt_id = (
         persona_upsert_request.prompt_ids[0]
         if persona_upsert_request.prompt_ids
@@ -251,6 +272,7 @@ def update_persona(
     user: User | None = Depends(current_user),
     db_session: Session = Depends(get_session),
 ) -> PersonaSnapshot:
+    _validate_user_knowledge_enabled(persona_upsert_request, "update")
     prompt_id = (
         persona_upsert_request.prompt_ids[0]
         if persona_upsert_request.prompt_ids
@@ -396,13 +418,12 @@ def list_personas(
     db_session: Session = Depends(get_session),
     include_deleted: bool = False,
     persona_ids: list[int] = Query(None),
-) -> list[PersonaSnapshot]:
-    personas = get_personas_for_user(
+) -> list[MinimalPersonaSnapshot]:
+    personas = get_minimal_persona_snapshots_for_user(
         user=user,
         include_deleted=include_deleted,
         db_session=db_session,
         get_editable=False,
-        joinedload_all=True,
     )
 
     if persona_ids:
@@ -413,12 +434,14 @@ def list_personas(
         p
         for p in personas
         if not (
-            any(tool.in_code_tool_id == "ImageGenerationTool" for tool in p.tools)
+            any(
+                tool.in_code_tool_id == ImageGenerationTool.__name__ for tool in p.tools
+            )
             and not is_image_generation_available(db_session=db_session)
         )
     ]
 
-    return [PersonaSnapshot.from_model(p) for p in personas]
+    return personas
 
 
 @basic_router.get("/{persona_id}")

backend/onyx/server/features/persona/models.py

@@ -9,7 +9,7 @@ from onyx.db.models import Persona
 from onyx.db.models import PersonaLabel
 from onyx.db.models import Prompt
 from onyx.db.models import StarterMessage
-from onyx.server.features.document_set.models import DocumentSet
+from onyx.server.features.document_set.models import DocumentSetSummary
 from onyx.server.features.tool.models import ToolSnapshot
 from onyx.server.models import MinimalUserSnapshot
 from onyx.utils.logger import setup_logger
@@ -89,6 +89,70 @@ class PersonaUpsertRequest(BaseModel):
     user_folder_ids: list[int] | None = None
 
 
+class MinimalPersonaSnapshot(BaseModel):
+    """Minimal persona model optimized for ChatPage.tsx - only includes fields actually used"""
+
+    # Core fields used by ChatPage
+    id: int
+    name: str
+    description: str
+    # Used for retrieval capability checking
+    tools: list[ToolSnapshot]
+    starter_messages: list[StarterMessage] | None
+
+    # only show document sets in the UI that the assistant has access to
+    document_sets: list[DocumentSetSummary]
+    llm_model_version_override: str | None
+    llm_model_provider_override: str | None
+
+    uploaded_image_id: str | None
+    icon_shape: int | None
+    icon_color: str | None
+
+    is_public: bool
+    is_visible: bool
+    display_priority: int | None
+    is_default_persona: bool
+    builtin_persona: bool
+
+    # Used for filtering
+    labels: list["PersonaLabelSnapshot"]
+
+    # Used to display ownership
+    owner: MinimalUserSnapshot | None
+
+    @classmethod
+    def from_model(cls, persona: Persona) -> "MinimalPersonaSnapshot":
+        return MinimalPersonaSnapshot(
+            # Core fields actually used by ChatPage
+            id=persona.id,
+            name=persona.name,
+            description=persona.description,
+            tools=[ToolSnapshot.from_model(tool) for tool in persona.tools],
+            starter_messages=persona.starter_messages,
+            document_sets=[
+                DocumentSetSummary.from_model(document_set)
+                for document_set in persona.document_sets
+            ],
+            llm_model_version_override=persona.llm_model_version_override,
+            llm_model_provider_override=persona.llm_model_provider_override,
+            uploaded_image_id=persona.uploaded_image_id,
+            icon_shape=persona.icon_shape,
+            icon_color=persona.icon_color,
+            is_public=persona.is_public,
+            is_visible=persona.is_visible,
+            display_priority=persona.display_priority,
+            is_default_persona=persona.is_default_persona,
+            builtin_persona=persona.builtin_persona,
+            labels=[PersonaLabelSnapshot.from_model(label) for label in persona.labels],
+            owner=(
+                MinimalUserSnapshot(id=persona.user.id, email=persona.user.email)
+                if persona.user
+                else None
+            ),
+        )
+
+
 class PersonaSnapshot(BaseModel):
     id: int
     name: str
@@ -109,7 +173,7 @@ class PersonaSnapshot(BaseModel):
     owner: MinimalUserSnapshot | None
     users: list[MinimalUserSnapshot]
     groups: list[int]
-    document_sets: list[DocumentSet]
+    document_sets: list[DocumentSetSummary]
     llm_model_provider_override: str | None
     llm_model_version_override: str | None
     num_chunks: float | None
@@ -144,7 +208,7 @@ class PersonaSnapshot(BaseModel):
             ],
             groups=[user_group.id for user_group in persona.groups],
             document_sets=[
-                DocumentSet.from_model(document_set_model)
+                DocumentSetSummary.from_model(document_set_model)
                 for document_set_model in persona.document_sets
             ],
             llm_model_provider_override=persona.llm_model_provider_override,
@@ -200,7 +264,7 @@ class FullPersonaSnapshot(PersonaSnapshot):
                 else None
             ),
             document_sets=[
-                DocumentSet.from_model(document_set_model)
+                DocumentSetSummary.from_model(document_set_model)
                 for document_set_model in persona.document_sets
             ],
             num_chunks=persona.num_chunks,

backend/onyx/server/openai_assistants_api/asssistants_api.py

@@ -15,7 +15,7 @@ from onyx.db.engine import get_session
 from onyx.db.models import Persona
 from onyx.db.models import User
 from onyx.db.persona import get_persona_by_id
-from onyx.db.persona import get_personas_for_user
+from onyx.db.persona import get_raw_personas_for_user
 from onyx.db.persona import mark_persona_as_deleted
 from onyx.db.persona import upsert_persona
 from onyx.db.prompts import upsert_prompt
@@ -242,32 +242,31 @@ def list_assistants(
     user: User | None = Depends(current_user),
     db_session: Session = Depends(get_session),
 ) -> ListAssistantsResponse:
-    personas = list(
-        get_personas_for_user(
+    persona_snapshots = list(
+        get_raw_personas_for_user(
             user=user,
             db_session=db_session,
             get_editable=False,
-            joinedload_all=True,
         )
     )
 
     # Apply filtering based on after and before
     if after:
-        personas = [p for p in personas if p.id > int(after)]
+        persona_snapshots = [p for p in persona_snapshots if p.id > int(after)]
     if before:
-        personas = [p for p in personas if p.id < int(before)]
+        persona_snapshots = [p for p in persona_snapshots if p.id < int(before)]
 
     # Apply ordering
-    personas.sort(key=lambda p: p.id, reverse=(order == "desc"))
+    persona_snapshots.sort(key=lambda p: p.id, reverse=(order == "desc"))
 
     # Apply limit
-    personas = personas[:limit]
+    persona_snapshots = persona_snapshots[:limit]
 
-    assistants = [persona_to_assistant(p) for p in personas]
+    assistants = [persona_to_assistant(p) for p in persona_snapshots]
 
     return ListAssistantsResponse(
         data=assistants,
         first_id=assistants[0].id if assistants else None,
         last_id=assistants[-1].id if assistants else None,
-        has_more=len(personas) == limit,
+        has_more=len(persona_snapshots) == limit,
     )

backend/onyx/server/settings/models.py

@@ -59,6 +59,9 @@ class Settings(BaseModel):
     search_time_image_analysis_enabled: bool | None = False
     image_analysis_max_size_mb: int | None = 20
 
+    # User Knowledge settings
+    user_knowledge_enabled: bool | None = True
+
 
 class UserSettings(Settings):
     notifications: list[Notification]

backend/onyx/server/settings/store.py

@@ -1,3 +1,4 @@
+from onyx.configs.app_configs import DISABLE_USER_KNOWLEDGE
 from onyx.configs.app_configs import ONYX_QUERY_HISTORY_TYPE
 from onyx.configs.constants import KV_SETTINGS_KEY
 from onyx.configs.constants import OnyxRedisLocks
@@ -48,6 +49,10 @@ def load_settings() -> Settings:
     settings.anonymous_user_enabled = anonymous_user_enabled
     settings.query_history_type = ONYX_QUERY_HISTORY_TYPE
 
+    # Override user knowledge setting if disabled via environment variable
+    if DISABLE_USER_KNOWLEDGE:
+        settings.user_knowledge_enabled = False
+
     return settings
 
 

backend/tests/integration/common_utils/managers/document_set.py

@@ -1,4 +1,6 @@
 import time
+from typing import Any
+from uuid import UUID
 from uuid import uuid4
 
 import requests
@@ -19,6 +21,7 @@ class DocumentSetManager:
         is_public: bool = True,
         users: list[str] | None = None,
         groups: list[int] | None = None,
+        federated_connectors: list[dict[str, Any]] | None = None,
         user_performing_action: DATestUser | None = None,
     ) -> DATestDocumentSet:
         if name is None:
@@ -29,8 +32,9 @@ class DocumentSetManager:
             "description": description or name,
             "cc_pair_ids": cc_pair_ids or [],
             "is_public": is_public,
-            "users": users or [],
+            "users": [str(UUID(user_id)) for user_id in (users or [])],
             "groups": groups or [],
+            "federated_connectors": federated_connectors or [],
         }
 
         response = requests.post(
@@ -53,6 +57,7 @@ class DocumentSetManager:
             is_up_to_date=True,
             users=users or [],
             groups=groups or [],
+            federated_connectors=federated_connectors or [],
         )
 
     @staticmethod
@@ -65,8 +70,9 @@ class DocumentSetManager:
             "description": document_set.description,
             "cc_pair_ids": document_set.cc_pair_ids,
             "is_public": document_set.is_public,
-            "users": document_set.users,
+            "users": [str(UUID(user_id)) for user_id in document_set.users],
             "groups": document_set.groups,
+            "federated_connectors": document_set.federated_connectors,
         }
         response = requests.patch(
             f"{API_SERVER_URL}/manage/admin/document-set",
@@ -114,13 +120,12 @@ class DocumentSetManager:
                 id=doc_set["id"],
                 name=doc_set["name"],
                 description=doc_set["description"],
-                cc_pair_ids=[
-                    cc_pair["id"] for cc_pair in doc_set["cc_pair_descriptors"]
-                ],
+                cc_pair_ids=[cc_pair["id"] for cc_pair in doc_set["cc_pair_summaries"]],
                 is_public=doc_set["is_public"],
                 is_up_to_date=doc_set["is_up_to_date"],
-                users=doc_set["users"],
+                users=[str(user_id) for user_id in doc_set["users"]],
                 groups=doc_set["groups"],
+                federated_connectors=doc_set["federated_connector_summaries"],
             )
             for doc_set in response.json()
         ]
@@ -186,6 +191,8 @@ class DocumentSetManager:
                     and doc_set.is_public == document_set.is_public
                     and set(doc_set.users) == set(document_set.users)
                     and set(doc_set.groups) == set(document_set.groups)
+                    and doc_set.federated_connectors
+                    == document_set.federated_connectors
                 ):
                     return
         if not verify_deleted:

backend/tests/integration/common_utils/reset.py

@@ -1,4 +1,5 @@
 import logging
+import os
 import time
 from types import SimpleNamespace
 
@@ -398,6 +399,10 @@ def reset_vespa_multitenant() -> None:
 
 
 def reset_all() -> None:
+    if os.environ.get("SKIP_RESET", "").lower() == "true":
+        logger.info("Skipping reset.")
+        return
+
     logger.info("Resetting Postgres...")
     reset_postgres()
     logger.info("Resetting Vespa...")

backend/tests/integration/common_utils/test_models.py

@@ -117,6 +117,7 @@ class DATestDocumentSet(BaseModel):
     is_up_to_date: bool
     users: list[str] = Field(default_factory=list)
     groups: list[int] = Field(default_factory=list)
+    federated_connectors: list[dict[str, Any]] = Field(default_factory=list)
 
 
 class DATestPersona(BaseModel):

deployment/docker_compose/docker-compose.dev.yml

@@ -124,6 +124,11 @@ services:
       # Seeding configuration
       - USE_IAM_AUTH=${USE_IAM_AUTH:-}
       - ONYX_QUERY_HISTORY_TYPE=${ONYX_QUERY_HISTORY_TYPE:-}
+
+      # Vespa Language Forcing
+      # See: https://docs.vespa.ai/en/linguistics.html 
+      - VESPA_LANGUAGE_OVERRIDE=${VESPA_LANGUAGE_OVERRIDE:-}
+
     # Uncomment the line below to use if IAM_AUTH is true and you are using iam auth for postgres
     # volumes:
     #   - ./bundle.pem:/app/bundle.pem:ro

deployment/docker_compose/docker-compose.gpu-dev.yml

@@ -92,6 +92,10 @@ services:
       # Chat Configs
       - HARD_DELETE_CHATS=${HARD_DELETE_CHATS:-}
 
+      # Vespa Language Forcing
+      # See: https://docs.vespa.ai/en/linguistics.html 
+      - VESPA_LANGUAGE_OVERRIDE=${VESPA_LANGUAGE_OVERRIDE:-}
+
       # Enterprise Edition only
       - API_KEY_HASH_ROUNDS=${API_KEY_HASH_ROUNDS:-}
       - ENABLE_PAID_ENTERPRISE_EDITION_FEATURES=${ENABLE_PAID_ENTERPRISE_EDITION_FEATURES:-false}

deployment/docker_compose/docker-compose.multitenant-dev.yml

@@ -106,6 +106,10 @@ services:
       - API_KEY_HASH_ROUNDS=${API_KEY_HASH_ROUNDS:-}
       # Seeding configuration
       - USE_IAM_AUTH=${USE_IAM_AUTH:-}
+
+      # Vespa Language Forcing
+      # See: https://docs.vespa.ai/en/linguistics.html 
+      - VESPA_LANGUAGE_OVERRIDE=${VESPA_LANGUAGE_OVERRIDE:-}
     extra_hosts:
       - "host.docker.internal:host-gateway"
     logging:

deployment/docker_compose/env.prod.template

@@ -55,3 +55,8 @@ SESSION_EXPIRE_TIME_SECONDS=604800
 # Default values here are what Postgres uses by default, feel free to change.
 POSTGRES_USER=postgres
 POSTGRES_PASSWORD=password
+
+
+# If setting the vespa language is required, set this ('en', 'de', etc.).
+# See: https://docs.vespa.ai/en/linguistics.html 
+#VESPA_LANGUAGE_OVERRIDE=

web/src/app/admin/assistants/AssistantEditor.tsx

@@ -5,7 +5,7 @@ import { Option } from "@/components/Dropdown";
 import { generateRandomIconShape } from "@/lib/assistantIconUtils";
 import {
   CCPairBasicInfo,
-  DocumentSet,
+  DocumentSetSummary,
   User,
   UserGroup,
   UserRole,
@@ -40,8 +40,9 @@ import {
 } from "@/components/ui/tooltip";
 import Link from "next/link";
 import { useRouter, useSearchParams } from "next/navigation";
-import { useEffect, useMemo, useState } from "react";
+import { useContext, useEffect, useMemo, useState } from "react";
 import * as Yup from "yup";
+import { SettingsContext } from "@/components/settings/SettingsProvider";
 import { FullPersona, PersonaLabel, StarterMessage } from "./interfaces";
 import {
   PersonaUpsertParameters,
@@ -130,7 +131,7 @@ export function AssistantEditor({
 }: {
   existingPersona?: FullPersona | null;
   ccPairs: CCPairBasicInfo[];
-  documentSets: DocumentSet[];
+  documentSets: DocumentSetSummary[];
   user: User | null;
   defaultPublic: boolean;
   llmProviders: LLMProviderView[];
@@ -147,6 +148,7 @@ export function AssistantEditor({
   const { popup, setPopup } = usePopup();
   const { labels, refreshLabels, createLabel, updateLabel, deleteLabel } =
     useLabels();
+  const settings = useContext(SettingsContext);
 
   const colorOptions = [
     "#FF6FBF",
@@ -237,6 +239,14 @@ export function AssistantEditor({
 
   const [showVisibilityWarning, setShowVisibilityWarning] = useState(false);
 
+  const canShowKnowledgeSource =
+    ccPairs.length > 0 &&
+    searchTool &&
+    !(user?.role === UserRole.BASIC && documentSets.length === 0);
+
+  const userKnowledgeEnabled =
+    settings?.settings?.user_knowledge_enabled ?? true;
+
   const initialValues = {
     name: existingPersona?.name ?? "",
     description: existingPersona?.description ?? "",
@@ -259,7 +269,7 @@ export function AssistantEditor({
       existingPersona?.llm_model_version_override ?? null,
     starter_messages: existingPersona?.starter_messages?.length
       ? existingPersona.starter_messages
-      : [{ message: "" }],
+      : [{ message: "", name: "" }],
     enabled_tools_map: enabledToolsMap,
     icon_color: existingPersona?.icon_color ?? defautIconColor,
     icon_shape: existingPersona?.icon_shape ?? defaultIconShape,
@@ -275,11 +285,14 @@ export function AssistantEditor({
     selectedGroups: existingPersona?.groups ?? [],
     user_file_ids: existingPersona?.user_file_ids ?? [],
     user_folder_ids: existingPersona?.user_folder_ids ?? [],
-    knowledge_source:
-      (existingPersona?.user_file_ids?.length ?? 0) > 0 ||
-      (existingPersona?.user_folder_ids?.length ?? 0) > 0
-        ? "user_files"
-        : "team_knowledge",
+    knowledge_source: !canShowKnowledgeSource
+      ? "user_files"
+      : !userKnowledgeEnabled
+        ? "team_knowledge"
+        : (existingPersona?.user_file_ids?.length ?? 0) > 0 ||
+            (existingPersona?.user_folder_ids?.length ?? 0) > 0
+          ? "user_files"
+          : "team_knowledge",
     is_default_persona: existingPersona?.is_default_persona ?? false,
   };
 
@@ -374,11 +387,6 @@ export function AssistantEditor({
     }
   };
 
-  const canShowKnowledgeSource =
-    ccPairs.length > 0 &&
-    searchTool &&
-    !(user?.role != "admin" && documentSets.length === 0);
-
   return (
     <div className="mx-auto max-w-4xl">
       <style>
@@ -534,10 +542,8 @@ export function AssistantEditor({
           // to tell the backend to not fetch any documents
           const numChunks = searchToolEnabled ? values.num_chunks || 10 : 0;
           const starterMessages = values.starter_messages
-            .filter(
-              (message: { message: string }) => message.message.trim() !== ""
-            )
-            .map((message: { message: string; name?: string }) => ({
+            .filter((message: StarterMessage) => message.message.trim() !== "")
+            .map((message: StarterMessage) => ({
               message: message.message,
               name: message.message,
             }));
@@ -950,26 +956,28 @@ export function AssistantEditor({
                                   </p>
                                 </div>
 
-                                <div
-                                  className={`w-[150px] h-[110px] rounded-lg border flex flex-col items-center justify-center cursor-pointer transition-all ${
-                                    values.knowledge_source === "user_files"
-                                      ? "border-2 border-blue-500 bg-blue-50 dark:bg-blue-950/20"
-                                      : "border-gray-200 hover:border-gray-300 dark:border-gray-700 dark:hover:border-gray-600"
-                                  }`}
-                                  onClick={() =>
-                                    setFieldValue(
-                                      "knowledge_source",
-                                      "user_files"
-                                    )
-                                  }
-                                >
-                                  <div className="text-blue-500 mb-2">
-                                    <FileIcon size={24} />
+                                {userKnowledgeEnabled && (
+                                  <div
+                                    className={`w-[150px] h-[110px] rounded-lg border flex flex-col items-center justify-center cursor-pointer transition-all ${
+                                      values.knowledge_source === "user_files"
+                                        ? "border-2 border-blue-500 bg-blue-50 dark:bg-blue-950/20"
+                                        : "border-gray-200 hover:border-gray-300 dark:border-gray-700 dark:hover:border-gray-600"
+                                    }`}
+                                    onClick={() =>
+                                      setFieldValue(
+                                        "knowledge_source",
+                                        "user_files"
+                                      )
+                                    }
+                                  >
+                                    <div className="text-blue-500 mb-2">
+                                      <FileIcon size={24} />
+                                    </div>
+                                    <p className="font-medium text-xs">
+                                      User Knowledge
+                                    </p>
                                   </div>
-                                  <p className="font-medium text-xs">
-                                    User Knowledge
-                                  </p>
-                                </div>
+                                )}
                               </div>
                             </div>
                           </>

web/src/app/admin/assistants/PersonaTable.tsx

@@ -17,7 +17,6 @@ import {
 import { FiEdit2 } from "react-icons/fi";
 import { TrashIcon } from "@/components/icons/icons";
 import { useUser } from "@/components/user/UserProvider";
-import { useAssistants } from "@/components/context/AssistantsContext";
 import { ConfirmEntityModal } from "@/components/modals/ConfirmEntityModal";
 
 function PersonaTypeDisplay({ persona }: { persona: Persona }) {
@@ -40,15 +39,20 @@ function PersonaTypeDisplay({ persona }: { persona: Persona }) {
   return <Text>Personal {persona.owner && <>({persona.owner.email})</>}</Text>;
 }
 
-export function PersonasTable() {
+export function PersonasTable({
+  personas,
+  refreshPersonas,
+}: {
+  personas: Persona[];
+  refreshPersonas: () => void;
+}) {
   const router = useRouter();
   const { popup, setPopup } = usePopup();
   const { refreshUser, isAdmin } = useUser();
-  const {
-    allAssistants: assistants,
-    refreshAssistants,
-    editablePersonas,
-  } = useAssistants();
+
+  const editablePersonas = useMemo(() => {
+    return personas.filter((p) => !p.builtin_persona);
+  }, [personas]);
 
   const editablePersonaIds = useMemo(() => {
     return new Set(editablePersonas.map((p) => p.id.toString()));
@@ -63,18 +67,18 @@ export function PersonasTable() {
 
   useEffect(() => {
     const editable = editablePersonas.sort(personaComparator);
-    const nonEditable = assistants
+    const nonEditable = personas
       .filter((p) => !editablePersonaIds.has(p.id.toString()))
       .sort(personaComparator);
     setFinalPersonas([...editable, ...nonEditable]);
-  }, [editablePersonas, assistants, editablePersonaIds]);
+  }, [editablePersonas, personas, editablePersonaIds]);
 
   const updatePersonaOrder = async (orderedPersonaIds: UniqueIdentifier[]) => {
-    const reorderedAssistants = orderedPersonaIds.map(
-      (id) => assistants.find((assistant) => assistant.id.toString() === id)!
+    const reorderedPersonas = orderedPersonaIds.map(
+      (id) => personas.find((persona) => persona.id.toString() === id)!
     );
 
-    setFinalPersonas(reorderedAssistants);
+    setFinalPersonas(reorderedPersonas);
 
     const displayPriorityMap = new Map<UniqueIdentifier, number>();
     orderedPersonaIds.forEach((personaId, ind) => {
@@ -96,12 +100,12 @@ export function PersonasTable() {
         type: "error",
         message: `Failed to update persona order - ${await response.text()}`,
       });
-      setFinalPersonas(assistants);
-      await refreshAssistants();
+      setFinalPersonas(personas);
+      await refreshPersonas();
       return;
     }
 
-    await refreshAssistants();
+    await refreshPersonas();
     await refreshUser();
   };
 
@@ -119,7 +123,7 @@ export function PersonasTable() {
     if (personaToDelete) {
       const response = await deletePersona(personaToDelete.id);
       if (response.ok) {
-        await refreshAssistants();
+        refreshPersonas();
         closeDeleteModal();
       } else {
         setPopup({
@@ -147,7 +151,7 @@ export function PersonasTable() {
         personaToToggleDefault.is_default_persona
       );
       if (response.ok) {
-        await refreshAssistants();
+        refreshPersonas();
         closeDefaultModal();
       } else {
         setPopup({
@@ -267,7 +271,7 @@ export function PersonasTable() {
                       persona.is_visible
                     );
                     if (response.ok) {
-                      await refreshAssistants();
+                      refreshPersonas();
                     } else {
                       setPopup({
                         type: "error",

web/src/app/admin/assistants/hooks.ts

@@ -0,0 +1,30 @@
+import useSWR from "swr";
+import { errorHandlingFetcher } from "@/lib/fetcher";
+import { buildApiPath } from "@/lib/urlBuilder";
+import { Persona } from "@/app/admin/assistants/interfaces";
+
+interface UseAdminPersonasOptions {
+  includeDeleted?: boolean;
+  getEditable?: boolean;
+}
+
+export const useAdminPersonas = (options?: UseAdminPersonasOptions) => {
+  const { includeDeleted = false, getEditable = false } = options || {};
+
+  const url = buildApiPath("/api/admin/persona", {
+    include_deleted: includeDeleted,
+    get_editable: getEditable,
+  });
+
+  const { data, error, isLoading, mutate } = useSWR<Persona[]>(
+    url,
+    errorHandlingFetcher
+  );
+
+  return {
+    personas: data || [],
+    error,
+    isLoading,
+    refresh: mutate,
+  };
+};

web/src/app/admin/assistants/interfaces.ts

@@ -1,5 +1,5 @@
 import { ToolSnapshot } from "@/lib/tools/interfaces";
-import { DocumentSet, MinimalUserSnapshot } from "@/lib/types";
+import { DocumentSetSummary, MinimalUserSnapshot } from "@/lib/types";
 
 export interface StarterMessageBase {
   message: string;
@@ -18,29 +18,36 @@ export interface Prompt {
   datetime_aware: boolean;
   default_prompt: boolean;
 }
-export interface Persona {
+
+export interface MinimalPersonaSnapshot {
   id: number;
   name: string;
   description: string;
-  is_public: boolean;
-  is_visible: boolean;
+  tools: ToolSnapshot[];
+  starter_messages: StarterMessage[] | null;
+  document_sets: DocumentSetSummary[];
+  llm_model_version_override?: string;
+  llm_model_provider_override?: string;
+
+  uploaded_image_id?: string;
   icon_shape?: number;
   icon_color?: string;
-  uploaded_image_id?: string;
-  user_file_ids: number[];
-  user_folder_ids: number[];
+
+  is_public: boolean;
+  is_visible: boolean;
   display_priority: number | null;
   is_default_persona: boolean;
   builtin_persona: boolean;
-  starter_messages: StarterMessage[] | null;
-  tools: ToolSnapshot[];
+
   labels?: PersonaLabel[];
   owner: MinimalUserSnapshot | null;
+}
+
+export interface Persona extends MinimalPersonaSnapshot {
+  user_file_ids: number[];
+  user_folder_ids: number[];
   users: MinimalUserSnapshot[];
   groups: number[];
-  document_sets: DocumentSet[];
-  llm_model_provider_override?: string;
-  llm_model_version_override?: string;
   num_chunks?: number;
 }
 

web/src/app/admin/assistants/lib.ts

@@ -1,5 +1,5 @@
 import { LLMProviderView } from "../configuration/llm/interfaces";
-import { Persona, StarterMessage } from "./interfaces";
+import { MinimalPersonaSnapshot, Persona, StarterMessage } from "./interfaces";
 
 interface PersonaUpsertRequest {
   name: string;
@@ -250,7 +250,10 @@ function closerToZeroNegativesFirstComparator(a: number, b: number) {
   return absA > absB ? 1 : -1;
 }
 
-export function personaComparator(a: Persona, b: Persona) {
+export function personaComparator(
+  a: MinimalPersonaSnapshot | Persona,
+  b: MinimalPersonaSnapshot | Persona
+) {
   if (a.display_priority === null && b.display_priority === null) {
     return closerToZeroNegativesFirstComparator(a.id, b.id);
   }

web/src/app/admin/assistants/page.tsx

@@ -1,3 +1,5 @@
+"use client";
+
 import { PersonasTable } from "./PersonaTable";
 import Text from "@/components/ui/text";
 import Title from "@/components/ui/title";
@@ -6,11 +8,20 @@ import { AssistantsIcon } from "@/components/icons/icons";
 import { AdminPageTitle } from "@/components/admin/Title";
 import { SubLabel } from "@/components/admin/connectors/Field";
 import CreateButton from "@/components/ui/createButton";
-export default async function Page() {
-  return (
-    <div className="mx-auto container">
-      <AdminPageTitle icon={<AssistantsIcon size={32} />} title="Assistants" />
+import { useAdminPersonas } from "./hooks";
+import { Persona } from "./interfaces";
+import { ThreeDotsLoader } from "@/components/Loading";
+import { ErrorCallout } from "@/components/ErrorCallout";
 
+function MainContent({
+  personas,
+  refreshPersonas,
+}: {
+  personas: Persona[];
+  refreshPersonas: () => void;
+}) {
+  return (
+    <div>
       <Text className="mb-2">
         Assistants are a way to build custom search/question-answering
         experiences for different use cases.
@@ -40,8 +51,35 @@ export default async function Page() {
           hidden will not be displayed. Editable assistants are shown at the
           top.
         </SubLabel>
-        <PersonasTable />
+        <PersonasTable personas={personas} refreshPersonas={refreshPersonas} />
       </div>
     </div>
   );
 }
+
+export default function Page() {
+  const { personas, isLoading, error, refresh } = useAdminPersonas();
+
+  return (
+    <div className="mx-auto container">
+      <AdminPageTitle icon={<AssistantsIcon size={32} />} title="Assistants" />
+
+      {isLoading && <ThreeDotsLoader />}
+
+      {error && (
+        <ErrorCallout
+          errorTitle="Failed to load assistants"
+          errorMsg={
+            error?.info?.message ||
+            error?.info?.detail ||
+            "An unknown error occurred"
+          }
+        />
+      )}
+
+      {!isLoading && !error && (
+        <MainContent personas={personas} refreshPersonas={refresh} />
+      )}
+    </div>
+  );
+}

web/src/app/admin/bots/SlackBotTable.tsx

@@ -3,7 +3,6 @@
 import { PageSelector } from "@/components/PageSelector";
 import { useRouter } from "next/navigation";
 import { useEffect, useState } from "react";
-import { FiEdit } from "react-icons/fi";
 import {
   Table,
   TableBody,

web/src/app/admin/bots/[bot-id]/channels/SlackChannelConfigCreationForm.tsx

@@ -5,7 +5,7 @@ import { Formik, Form } from "formik";
 import * as Yup from "yup";
 import { usePopup } from "@/components/admin/connectors/Popup";
 import {
-  DocumentSet,
+  DocumentSetSummary,
   SlackChannelConfig,
   SlackBotResponseType,
 } from "@/lib/types";
@@ -16,7 +16,7 @@ import {
 } from "../lib";
 import CardSection from "@/components/admin/CardSection";
 import { useRouter } from "next/navigation";
-import { Persona } from "@/app/admin/assistants/interfaces";
+import { MinimalPersonaSnapshot } from "@/app/admin/assistants/interfaces";
 import { StandardAnswerCategoryResponse } from "@/components/standardAnswers/getStandardAnswerCategoriesIfEE";
 import { SEARCH_TOOL_ID } from "@/app/chat/tools/constants";
 import { SlackChannelConfigFormFields } from "./SlackChannelConfigFormFields";
@@ -29,8 +29,8 @@ export const SlackChannelConfigCreationForm = ({
   existingSlackChannelConfig,
 }: {
   slack_bot_id: number;
-  documentSets: DocumentSet[];
-  personas: Persona[];
+  documentSets: DocumentSetSummary[];
+  personas: MinimalPersonaSnapshot[];
   standardAnswerCategoryResponse: StandardAnswerCategoryResponse;
   existingSlackChannelConfig?: SlackChannelConfig;
 }) => {
@@ -59,7 +59,7 @@ export const SlackChannelConfigCreationForm = ({
         }
         return acc;
       },
-      [[], []] as [Persona[], Persona[]]
+      [[], []] as [MinimalPersonaSnapshot[], MinimalPersonaSnapshot[]]
     );
   }, [personas]);
 

web/src/app/admin/bots/[bot-id]/channels/SlackChannelConfigFormFields.tsx

@@ -2,7 +2,7 @@
 
 import React, { useState, useEffect, useMemo } from "react";
 import { FieldArray, useFormikContext, ErrorMessage, Field } from "formik";
-import { CCPairDescriptor, DocumentSet } from "@/lib/types";
+import { CCPairDescriptor, DocumentSetSummary } from "@/lib/types";
 import {
   Label,
   SelectorFormField,
@@ -11,7 +11,7 @@ import {
   TextFormField,
 } from "@/components/admin/connectors/Field";
 import { Button } from "@/components/ui/button";
-import { Persona } from "@/app/admin/assistants/interfaces";
+import { MinimalPersonaSnapshot } from "@/app/admin/assistants/interfaces";
 import { DocumentSetSelectable } from "@/components/documentSet/DocumentSetSelectable";
 import CollapsibleSection from "@/app/admin/assistants/CollapsibleSection";
 import { StandardAnswerCategoryResponse } from "@/components/standardAnswers/getStandardAnswerCategoriesIfEE";
@@ -47,9 +47,9 @@ import { CheckFormField } from "@/components/ui/CheckField";
 export interface SlackChannelConfigFormFieldsProps {
   isUpdate: boolean;
   isDefault: boolean;
-  documentSets: DocumentSet[];
-  searchEnabledAssistants: Persona[];
-  nonSearchAssistants: Persona[];
+  documentSets: DocumentSetSummary[];
+  searchEnabledAssistants: MinimalPersonaSnapshot[];
+  nonSearchAssistants: MinimalPersonaSnapshot[];
   standardAnswerCategoryResponse: StandardAnswerCategoryResponse;
   setPopup: (popup: {
     message: string;
@@ -76,14 +76,28 @@ export function SlackChannelConfigFormFields({
   const [viewSyncEnabledAssistants, setViewSyncEnabledAssistants] =
     useState(false);
 
-  const documentSetContainsSync = (documentSet: DocumentSet) =>
-    documentSet.cc_pair_descriptors.some(
-      (descriptor) => descriptor.access_type === "sync"
+  // Helper function to check if a document set contains sync connectors
+  const documentSetContainsSync = (documentSet: DocumentSetSummary) => {
+    return documentSet.cc_pair_summaries.some(
+      (summary) => summary.access_type === "sync"
     );
+  };
+
+  // Helper function to check if a document set contains private connectors
+  const documentSetContainsPrivate = (documentSet: DocumentSetSummary) => {
+    return documentSet.cc_pair_summaries.some(
+      (summary) => summary.access_type === "private"
+    );
+  };
+
+  // Helper function to get cc_pair_summaries from DocumentSetSummary
+  const getCcPairSummaries = (documentSet: DocumentSetSummary) => {
+    return documentSet.cc_pair_summaries;
+  };
 
   const [syncEnabledAssistants, availableAssistants] = useMemo(() => {
-    const sync: Persona[] = [];
-    const available: Persona[] = [];
+    const sync: MinimalPersonaSnapshot[] = [];
+    const available: MinimalPersonaSnapshot[] = [];
 
     searchEnabledAssistants.forEach((persona) => {
       const hasSyncSet = persona.document_sets.some(documentSetContainsSync);
@@ -98,21 +112,19 @@ export function SlackChannelConfigFormFields({
   }, [searchEnabledAssistants]);
 
   const unselectableSets = useMemo(() => {
-    return documentSets.filter((ds) =>
-      ds.cc_pair_descriptors.some(
-        (descriptor) => descriptor.access_type === "sync"
-      )
-    );
+    return documentSets.filter(documentSetContainsSync);
   }, [documentSets]);
+
   const memoizedPrivateConnectors = useMemo(() => {
     const uniqueDescriptors = new Map();
-    documentSets.forEach((ds) => {
-      ds.cc_pair_descriptors.forEach((descriptor) => {
+    documentSets.forEach((ds: DocumentSetSummary) => {
+      const ccPairSummaries = getCcPairSummaries(ds);
+      ccPairSummaries.forEach((summary: any) => {
         if (
-          descriptor.access_type === "private" &&
-          !uniqueDescriptors.has(descriptor.id)
+          summary.access_type === "private" &&
+          !uniqueDescriptors.has(summary.id)
         ) {
-          uniqueDescriptors.set(descriptor.id, descriptor);
+          uniqueDescriptors.set(summary.id, summary);
         }
       });
     });
@@ -138,12 +150,6 @@ export function SlackChannelConfigFormFields({
     }
   }, [unselectableSets, values.document_sets, setFieldValue, setPopup]);
 
-  const documentSetContainsPrivate = (documentSet: DocumentSet) => {
-    return documentSet.cc_pair_descriptors.some(
-      (descriptor) => descriptor.access_type === "private"
-    );
-  };
-
   const shouldShowPrivacyAlert = useMemo(() => {
     if (values.knowledge_source === "document_sets") {
       const selectedSets = documentSets.filter((ds) =>
@@ -165,9 +171,7 @@ export function SlackChannelConfigFormFields({
   const selectableSets = useMemo(() => {
     return documentSets.filter(
       (ds) =>
-        !ds.cc_pair_descriptors.some(
-          (descriptor) => descriptor.access_type === "sync"
-        )
+        !ds.cc_pair_summaries.some((summary) => summary.access_type === "sync")
     );
   }, [documentSets]);
 
@@ -462,23 +466,25 @@ export function SlackChannelConfigFormFields({
                   Un-selectable assistants:
                 </p>
                 <div className="mb-3 mt-2 flex gap-2 flex-wrap text-sm">
-                  {syncEnabledAssistants.map((persona: Persona) => (
-                    <button
-                      type="button"
-                      onClick={() =>
-                        router.push(`/admin/assistants/${persona.id}`)
-                      }
-                      key={persona.id}
-                      className="p-2 bg-background-100 cursor-pointer rounded-md flex items-center gap-2"
-                    >
-                      <AssistantIcon
-                        assistant={persona}
-                        size={16}
-                        className="flex-none"
-                      />
-                      {persona.name}
-                    </button>
-                  ))}
+                  {syncEnabledAssistants.map(
+                    (persona: MinimalPersonaSnapshot) => (
+                      <button
+                        type="button"
+                        onClick={() =>
+                          router.push(`/admin/assistants/${persona.id}`)
+                        }
+                        key={persona.id}
+                        className="p-2 bg-background-100 cursor-pointer rounded-md flex items-center gap-2"
+                      >
+                        <AssistantIcon
+                          assistant={persona}
+                          size={16}
+                          className="flex-none"
+                        />
+                        {persona.name}
+                      </button>
+                    )
+                  )}
                 </div>
               </div>
             )}

web/src/app/admin/bots/[bot-id]/channels/[id]/page.tsx

@@ -3,7 +3,11 @@ import { SourceIcon } from "@/components/SourceIcon";
 import { SlackChannelConfigCreationForm } from "../SlackChannelConfigCreationForm";
 import { fetchSS } from "@/lib/utilsSS";
 import { ErrorCallout } from "@/components/ErrorCallout";
-import { DocumentSet, SlackChannelConfig, ValidSources } from "@/lib/types";
+import {
+  DocumentSetSummary,
+  SlackChannelConfig,
+  ValidSources,
+} from "@/lib/types";
 import { BackButton } from "@/components/BackButton";
 import { InstantSSRAutoRefresh } from "@/components/SSRAutoRefresh";
 import {
@@ -68,7 +72,7 @@ async function EditslackChannelConfigPage(props: {
     );
   }
   const response = await documentSetsResponse.json();
-  const documentSets = response as DocumentSet[];
+  const documentSets = response as DocumentSetSummary[];
 
   if (assistantsFetchError) {
     return (

web/src/app/admin/bots/[bot-id]/channels/new/page.tsx

@@ -2,15 +2,11 @@ import { AdminPageTitle } from "@/components/admin/Title";
 import { SlackChannelConfigCreationForm } from "../SlackChannelConfigCreationForm";
 import { fetchSS } from "@/lib/utilsSS";
 import { ErrorCallout } from "@/components/ErrorCallout";
-import { DocumentSet, ValidSources } from "@/lib/types";
+import { DocumentSetSummary, ValidSources } from "@/lib/types";
 import { BackButton } from "@/components/BackButton";
 import { fetchAssistantsSS } from "@/lib/assistants/fetchAssistantsSS";
-import {
-  getStandardAnswerCategoriesIfEE,
-  StandardAnswerCategoryResponse,
-} from "@/components/standardAnswers/getStandardAnswerCategoriesIfEE";
+import { getStandardAnswerCategoriesIfEE } from "@/components/standardAnswers/getStandardAnswerCategoriesIfEE";
 import { redirect } from "next/navigation";
-import { Persona } from "../../../../assistants/interfaces";
 import { SourceIcon } from "@/components/SourceIcon";
 
 async function NewChannelConfigPage(props: {
@@ -32,8 +28,8 @@ async function NewChannelConfigPage(props: {
     standardAnswerCategoryResponse,
   ] = await Promise.all([
     fetchSS("/manage/document-set") as Promise<Response>,
-    fetchAssistantsSS() as Promise<[Persona[], string | null]>,
-    getStandardAnswerCategoriesIfEE() as Promise<StandardAnswerCategoryResponse>,
+    fetchAssistantsSS(),
+    getStandardAnswerCategoriesIfEE(),
   ]);
 
   if (!documentSetsResponse.ok) {
@@ -44,7 +40,8 @@ async function NewChannelConfigPage(props: {
       />
     );
   }
-  const documentSets = (await documentSetsResponse.json()) as DocumentSet[];
+  const documentSets =
+    (await documentSetsResponse.json()) as DocumentSetSummary[];
 
   if (assistantsResponse[1]) {
     return (

web/src/app/admin/connector/[ccPairId]/ReIndexModal.tsx

@@ -2,7 +2,7 @@
 
 import { Button } from "@/components/ui/button";
 import { useState } from "react";
-import { usePopup, PopupSpec } from "@/components/admin/connectors/Popup";
+import { PopupSpec } from "@/components/admin/connectors/Popup";
 import { triggerIndexing } from "./lib";
 import { Modal } from "@/components/Modal";
 import Text from "@/components/ui/text";

web/src/app/admin/documents/explorer/Explorer.tsx

@@ -14,7 +14,7 @@ import { useRouter } from "next/navigation";
 import { useFilters } from "@/lib/hooks";
 import { buildFilters } from "@/lib/search/utils";
 import { DocumentUpdatedAtBadge } from "@/components/search/DocumentUpdatedAtBadge";
-import { DocumentSet } from "@/lib/types";
+import { DocumentSetSummary } from "@/lib/types";
 import { SourceIcon } from "@/components/SourceIcon";
 import { Connector } from "@/lib/connectors/connectors";
 import { HorizontalFilters } from "@/components/filters/SourceSelector";
@@ -110,7 +110,7 @@ export function Explorer({
 }: {
   initialSearchValue: string | undefined;
   connectors: Connector<any>[];
-  documentSets: DocumentSet[];
+  documentSets: DocumentSetSummary[];
 }) {
   const router = useRouter();
   const { popup, setPopup } = usePopup();

web/src/app/admin/documents/sets/DocumentSetCreationForm.tsx

@@ -8,7 +8,12 @@ import {
   updateDocumentSet,
   DocumentSetCreationRequest,
 } from "./lib";
-import { ConnectorStatus, DocumentSet, UserGroup, UserRole } from "@/lib/types";
+import {
+  ConnectorStatus,
+  DocumentSetSummary,
+  UserGroup,
+  UserRole,
+} from "@/lib/types";
 import { TextFormField } from "@/components/admin/connectors/Field";
 import { Separator } from "@/components/ui/separator";
 import { Button } from "@/components/ui/button";
@@ -24,7 +29,7 @@ interface SetCreationPopupProps {
   userGroups: UserGroup[] | undefined;
   onClose: () => void;
   setPopup: (popupSpec: PopupSpec | null) => void;
-  existingDocumentSet?: DocumentSet;
+  existingDocumentSet?: DocumentSetSummary;
 }
 
 export const DocumentSetCreationForm = ({
@@ -52,8 +57,8 @@ export const DocumentSetCreationForm = ({
           name: existingDocumentSet?.name ?? "",
           description: existingDocumentSet?.description ?? "",
           cc_pair_ids:
-            existingDocumentSet?.cc_pair_descriptors.map(
-              (ccPairDescriptor) => ccPairDescriptor.id
+            existingDocumentSet?.cc_pair_summaries.map(
+              (ccPairSummary) => ccPairSummary.id
             ) ?? [],
           is_public: existingDocumentSet?.is_public ?? true,
           users: existingDocumentSet?.users ?? [],

web/src/app/admin/documents/sets/hooks.tsx

@@ -1,5 +1,5 @@
 import { errorHandlingFetcher } from "@/lib/fetcher";
-import { DocumentSet } from "@/lib/types";
+import { DocumentSetSummary } from "@/lib/types";
 import useSWR, { mutate } from "swr";
 
 const DOCUMENT_SETS_URL = "/api/manage/document-set";
@@ -13,7 +13,7 @@ export function refreshDocumentSets() {
 export function useDocumentSets(getEditable: boolean = false) {
   const url = getEditable ? GET_EDITABLE_DOCUMENT_SETS_URL : DOCUMENT_SETS_URL;
 
-  const swrResponse = useSWR<DocumentSet[]>(url, errorHandlingFetcher, {
+  const swrResponse = useSWR<DocumentSetSummary[]>(url, errorHandlingFetcher, {
     refreshInterval: 5000, // 5 seconds
   });
 

web/src/app/admin/documents/sets/page.tsx

@@ -13,7 +13,7 @@ import {
 import Text from "@/components/ui/text";
 import Title from "@/components/ui/title";
 import { Separator } from "@/components/ui/separator";
-import { DocumentSet } from "@/lib/types";
+import { DocumentSetSummary } from "@/lib/types";
 import { useState } from "react";
 import { useDocumentSets } from "./hooks";
 import { ConnectorTitle } from "@/components/admin/connectors/ConnectorTitle";
@@ -39,6 +39,7 @@ import {
   TooltipTrigger,
 } from "@/components/ui/tooltip";
 import CreateButton from "@/components/ui/createButton";
+import { SourceIcon } from "@/components/SourceIcon";
 
 const numToDisplay = 50;
 
@@ -46,7 +47,7 @@ const EditRow = ({
   documentSet,
   isEditable,
 }: {
-  documentSet: DocumentSet;
+  documentSet: DocumentSetSummary;
   isEditable: boolean;
 }) => {
   const router = useRouter();
@@ -96,11 +97,11 @@ const EditRow = ({
 };
 
 interface DocumentFeedbackTableProps {
-  documentSets: DocumentSet[];
+  documentSets: DocumentSetSummary[];
   refresh: () => void;
   refreshEditable: () => void;
   setPopup: (popupSpec: PopupSpec | null) => void;
-  editableDocumentSets: DocumentSet[];
+  editableDocumentSets: DocumentSetSummary[];
 }
 
 const DocumentSetTable = ({
@@ -162,24 +163,27 @@ const DocumentSetTable = ({
                   </TableCell>
                   <TableCell>
                     <div>
-                      {documentSet.cc_pair_descriptors.map(
-                        (ccPairDescriptor, ind) => {
+                      {/* Regular Connectors */}
+                      {documentSet.cc_pair_summaries.map(
+                        (ccPairSummary, ind) => {
                           return (
                             <div
                               className={
-                                ind !==
-                                documentSet.cc_pair_descriptors.length - 1
+                                ind !== documentSet.cc_pair_summaries.length - 1
                                   ? "mb-3"
                                   : ""
                               }
-                              key={ccPairDescriptor.id}
+                              key={ccPairSummary.id}
                             >
-                              <ConnectorTitle
-                                connector={ccPairDescriptor.connector}
-                                ccPairName={ccPairDescriptor.name}
-                                ccPairId={ccPairDescriptor.id}
-                                showMetadata={false}
-                              />
+                              <div className="text-blue-500 dark:text-blue-100 flex w-fit">
+                                <SourceIcon
+                                  sourceType={ccPairSummary.source}
+                                  iconSize={16}
+                                />
+                                <div className="ml-1 my-auto text-xs font-medium truncate">
+                                  {ccPairSummary.name || "Unnamed"}
+                                </div>
+                              </div>
                             </div>
                           );
                         }
@@ -191,7 +195,7 @@ const DocumentSetTable = ({
                       <Badge variant="success" icon={FiCheckCircle}>
                         Up to Date
                       </Badge>
-                    ) : documentSet.cc_pair_descriptors.length > 0 ? (
+                    ) : documentSet.cc_pair_summaries.length > 0 ? (
                       <Badge variant="in_progress" icon={FiClock}>
                         Syncing
                       </Badge>

web/src/app/admin/settings/interfaces.ts

@@ -27,6 +27,9 @@ export interface Settings {
   image_extraction_and_analysis_enabled?: boolean;
   search_time_image_analysis_enabled?: boolean;
   image_analysis_max_size_mb?: number | null;
+
+  // User Knowledge settings
+  user_knowledge_enabled?: boolean;
 }
 
 export enum NotificationType {

web/src/app/assistants/mine/AssistantCard.tsx

@@ -15,7 +15,7 @@ import {
   PopoverContent,
 } from "@/components/ui/popover";
 import { AssistantIcon } from "@/components/assistants/AssistantIcon";
-import { Persona } from "@/app/admin/assistants/interfaces";
+import { MinimalPersonaSnapshot } from "@/app/admin/assistants/interfaces";
 import { useUser } from "@/components/user/UserProvider";
 import { useAssistants } from "@/components/context/AssistantsContext";
 import { checkUserOwnsAssistant } from "@/lib/assistants/utils";
@@ -54,7 +54,7 @@ export const AssistantBadge = ({
 };
 
 const AssistantCard: React.FC<{
-  persona: Persona;
+  persona: MinimalPersonaSnapshot;
   pinned: boolean;
   closeModal: () => void;
 }> = ({ persona, pinned, closeModal }) => {

web/src/app/auth/login/EmailPasswordForm.tsx

@@ -9,7 +9,6 @@ import * as Yup from "yup";
 import { requestEmailVerification } from "../lib";
 import { useState } from "react";
 import { Spinner } from "@/components/Spinner";
-import { NEXT_PUBLIC_FORGOT_PASSWORD_ENABLED } from "@/lib/constants";
 import Link from "next/link";
 import { useUser } from "@/components/user/UserProvider";
 import { useRouter } from "next/navigation";

web/src/app/chat/ChatIntro.tsx

@@ -1,7 +1,11 @@
 import { AssistantIcon } from "@/components/assistants/AssistantIcon";
-import { Persona } from "../admin/assistants/interfaces";
+import { MinimalPersonaSnapshot } from "../admin/assistants/interfaces";
 
-export function ChatIntro({ selectedPersona }: { selectedPersona: Persona }) {
+export function ChatIntro({
+  selectedPersona,
+}: {
+  selectedPersona: MinimalPersonaSnapshot;
+}) {
   return (
     <div data-testid="chat-intro" className="flex flex-col items-center gap-6">
       <div className="relative flex flex-col gap-y-4 w-fit mx-auto justify-center">

web/src/app/chat/ChatPage.tsx

@@ -29,7 +29,7 @@ import {
 import Prism from "prismjs";
 import Cookies from "js-cookie";
 import { HistorySidebar } from "./sessionSidebar/HistorySidebar";
-import { Persona } from "../admin/assistants/interfaces";
+import { MinimalPersonaSnapshot } from "../admin/assistants/interfaces";
 import { HealthCheckBanner } from "@/components/health/healthcheck";
 import {
   buildChatUrl,
@@ -49,7 +49,6 @@ import {
   setMessageAsLatest,
   updateLlmOverrideForChatSession,
   updateParentChildren,
-  uploadFilesForChat,
   useScrollonStream,
 } from "./lib";
 import {
@@ -302,7 +301,7 @@ export function ChatPage({
 
   const existingChatSessionAssistantId = selectedChatSession?.persona_id;
   const [selectedAssistant, setSelectedAssistant] = useState<
-    Persona | undefined
+    MinimalPersonaSnapshot | undefined
   >(
     // NOTE: look through available assistants here, so that even if the user
     // has hidden this assistant it still shows the correct assistant when
@@ -332,7 +331,7 @@ export function ChatPage({
   };
 
   const [alternativeAssistant, setAlternativeAssistant] =
-    useState<Persona | null>(null);
+    useState<MinimalPersonaSnapshot | null>(null);
 
   const [presentingDocument, setPresentingDocument] =
     useState<MinimalOnyxDocument | null>(null);
@@ -343,7 +342,7 @@ export function ChatPage({
   // 3. First pinned assistants (ordered list of pinned assistants)
   // 4. Available assistants (ordered list of available assistants)
   // Relevant test: `live_assistant.spec.ts`
-  const liveAssistant: Persona | undefined = useMemo(
+  const liveAssistant: MinimalPersonaSnapshot | undefined = useMemo(
     () =>
       alternativeAssistant ||
       selectedAssistant ||
@@ -412,7 +411,7 @@ export function ChatPage({
   // 2. we "@"ed the `GPT` assistant and sent a message
   // 3. while the `GPT` assistant message is generating, we "@" the `Paraphrase` assistant
   const [alternativeGeneratingAssistant, setAlternativeGeneratingAssistant] =
-    useState<Persona | null>(null);
+    useState<MinimalPersonaSnapshot | null>(null);
 
   // used to track whether or not the initial "submit on load" has been performed
   // this only applies if `?submit-on-load=true` or `?submit-on-load=1` is in the URL
@@ -1196,7 +1195,7 @@ export function ChatPage({
     queryOverride?: string;
     forceSearch?: boolean;
     isSeededChat?: boolean;
-    alternativeAssistantOverride?: Persona | null;
+    alternativeAssistantOverride?: MinimalPersonaSnapshot | null;
     modelOverride?: LlmDescriptor;
     regenerationRequest?: RegenerationRequest | null;
     overrideFileDescriptors?: FileDescriptor[];
@@ -2081,10 +2080,7 @@ export function ChatPage({
   useEffect(() => {
     if (liveAssistant) {
       const hasSearchTool = liveAssistant.tools.some(
-        (tool) =>
-          tool.in_code_tool_id === SEARCH_TOOL_ID &&
-          liveAssistant.user_file_ids?.length == 0 &&
-          liveAssistant.user_folder_ids?.length == 0
+        (tool) => tool.in_code_tool_id === SEARCH_TOOL_ID
       );
       setRetrievalEnabled(hasSearchTool);
       if (!hasSearchTool) {
@@ -2096,10 +2092,7 @@ export function ChatPage({
   const [retrievalEnabled, setRetrievalEnabled] = useState(() => {
     if (liveAssistant) {
       return liveAssistant.tools.some(
-        (tool) =>
-          tool.in_code_tool_id === SEARCH_TOOL_ID &&
-          liveAssistant.user_file_ids?.length == 0 &&
-          liveAssistant.user_folder_ids?.length == 0
+        (tool) => tool.in_code_tool_id === SEARCH_TOOL_ID
       );
     }
     return false;

web/src/app/chat/RegenerateOption.tsx

@@ -5,7 +5,7 @@ import {
   useLlmManager,
 } from "@/lib/hooks";
 
-import { Persona } from "@/app/admin/assistants/interfaces";
+import { MinimalPersonaSnapshot } from "@/app/admin/assistants/interfaces";
 import { parseLlmDescriptor } from "@/lib/llm/utils";
 import { useState } from "react";
 import { Hoverable } from "@/components/Hoverable";
@@ -19,7 +19,7 @@ export default function RegenerateOption({
   overriddenModel,
   onDropdownVisibleChange,
 }: {
-  selectedAssistant: Persona;
+  selectedAssistant: MinimalPersonaSnapshot;
   regenerate: (modelOverRide: LlmDescriptor) => Promise<void>;
   overriddenModel?: string;
   onDropdownVisibleChange: (isVisible: boolean) => void;

web/src/app/chat/input/ChatInputBar.tsx

@@ -2,7 +2,7 @@ import React, { useContext, useEffect, useMemo, useRef, useState } from "react";
 import { FiPlusCircle, FiPlus, FiX, FiFilter } from "react-icons/fi";
 import { FiLoader } from "react-icons/fi";
 import { ChatInputOption } from "./ChatInputOption";
-import { Persona } from "@/app/admin/assistants/interfaces";
+import { MinimalPersonaSnapshot } from "@/app/admin/assistants/interfaces";
 import LLMPopover from "./LLMPopover";
 import { InputPrompt } from "@/app/chat/interfaces";
 
@@ -29,7 +29,7 @@ import UnconfiguredProviderText from "@/components/chat/UnconfiguredProviderText
 import { useAssistants } from "@/components/context/AssistantsContext";
 import { CalendarIcon, TagIcon, XIcon, FolderIcon } from "lucide-react";
 import { FilterPopup } from "@/components/search/filtering/FilterPopup";
-import { DocumentSet, Tag } from "@/lib/types";
+import { DocumentSetSummary, Tag } from "@/lib/types";
 import { SourceIcon } from "@/components/SourceIcon";
 import { getFormattedDateRangeString } from "@/lib/dateUtils";
 import { truncateString } from "@/lib/utils";
@@ -182,17 +182,19 @@ interface ChatInputBarProps {
   onSubmit: () => void;
   llmManager: LlmManager;
   chatState: ChatState;
-  alternativeAssistant: Persona | null;
+  alternativeAssistant: MinimalPersonaSnapshot | null;
   // assistants
-  selectedAssistant: Persona;
-  setAlternativeAssistant: (alternativeAssistant: Persona | null) => void;
+  selectedAssistant: MinimalPersonaSnapshot;
+  setAlternativeAssistant: (
+    alternativeAssistant: MinimalPersonaSnapshot | null
+  ) => void;
   toggleDocumentSidebar: () => void;
   setFiles: (files: FileDescriptor[]) => void;
   handleFileUpload: (files: File[], intent: UploadIntent) => void;
   textAreaRef: React.RefObject<HTMLTextAreaElement>;
   filterManager: FilterManager;
   availableSources: SourceMetadata[];
-  availableDocumentSets: DocumentSet[];
+  availableDocumentSets: DocumentSetSummary[];
   availableTags: Tag[];
   retrievalEnabled: boolean;
   proSearchEnabled: boolean;
@@ -306,7 +308,7 @@ export function ChatInputBar({
     };
   }, []);
 
-  const updatedTaggedAssistant = (assistant: Persona) => {
+  const updatedTaggedAssistant = (assistant: MinimalPersonaSnapshot) => {
     setAlternativeAssistant(
       assistant.id == selectedAssistant.id ? null : assistant
     );

web/src/app/chat/input/LLMPopover.tsx

@@ -12,7 +12,7 @@ import {
 } from "@/lib/llm/utils";
 import { LLMProviderDescriptor } from "@/app/admin/configuration/llm/interfaces";
 import { getProviderIcon } from "@/app/admin/configuration/llm/utils";
-import { Persona } from "@/app/admin/assistants/interfaces";
+import { MinimalPersonaSnapshot } from "@/app/admin/assistants/interfaces";
 import { LlmManager } from "@/lib/hooks";
 
 import {
@@ -32,7 +32,7 @@ interface LLMPopoverProps {
   llmProviders: LLMProviderDescriptor[];
   llmManager: LlmManager;
   requiresImageGeneration?: boolean;
-  currentAssistant?: Persona;
+  currentAssistant?: MinimalPersonaSnapshot;
   trigger?: React.ReactElement;
   onSelect?: (value: string) => void;
   currentModelName?: string;

web/src/app/chat/lib.tsx

@@ -28,7 +28,7 @@ import {
   AgenticMessageResponseIDInfo,
   UserKnowledgeFilePacket,
 } from "./interfaces";
-import { Persona } from "../admin/assistants/interfaces";
+import { MinimalPersonaSnapshot } from "../admin/assistants/interfaces";
 import { ReadonlyURLSearchParams } from "next/navigation";
 import { SEARCH_PARAM_NAMES } from "./searchParams";
 import { Settings } from "../admin/settings/interfaces";
@@ -634,8 +634,8 @@ export function removeMessage(
 
 export function checkAnyAssistantHasSearch(
   messageHistory: Message[],
-  availableAssistants: Persona[],
-  livePersona: Persona
+  availableAssistants: MinimalPersonaSnapshot[],
+  livePersona: MinimalPersonaSnapshot
 ): boolean {
   const response =
     messageHistory.some((message) => {
@@ -656,19 +656,17 @@ export function checkAnyAssistantHasSearch(
   return response;
 }
 
-export function personaIncludesRetrieval(selectedPersona: Persona) {
+export function personaIncludesRetrieval(
+  selectedPersona: MinimalPersonaSnapshot
+) {
   return selectedPersona.tools.some(
     (tool) =>
       tool.in_code_tool_id &&
-      [SEARCH_TOOL_ID, INTERNET_SEARCH_TOOL_ID].includes(
-        tool.in_code_tool_id
-      ) &&
-      selectedPersona.user_file_ids?.length === 0 &&
-      selectedPersona.user_folder_ids?.length === 0
+      [SEARCH_TOOL_ID, INTERNET_SEARCH_TOOL_ID].includes(tool.in_code_tool_id)
   );
 }
 
-export function personaIncludesImage(selectedPersona: Persona) {
+export function personaIncludesImage(selectedPersona: MinimalPersonaSnapshot) {
   return selectedPersona.tools.some(
     (tool) =>
       tool.in_code_tool_id && tool.in_code_tool_id == IIMAGE_GENERATION_TOOL_ID

web/src/app/chat/message/AgenticMessage.tsx

@@ -27,7 +27,7 @@ import rehypePrism from "rehype-prism-plus";
 
 import "prismjs/themes/prism-tomorrow.css";
 import "./custom-code-styles.css";
-import { Persona } from "@/app/admin/assistants/interfaces";
+import { MinimalPersonaSnapshot } from "@/app/admin/assistants/interfaces";
 import { AssistantIcon } from "@/components/assistants/AssistantIcon";
 
 import { LikeFeedback, DislikeFeedback } from "@/components/icons/icons";
@@ -114,8 +114,8 @@ export const AgenticMessage = ({
   selectedDocuments?: OnyxDocument[] | null;
   toggleDocumentSelection?: (second: boolean) => void;
   docs?: OnyxDocument[] | null;
-  alternativeAssistant?: Persona | null;
-  currentPersona: Persona;
+  alternativeAssistant?: MinimalPersonaSnapshot | null;
+  currentPersona: MinimalPersonaSnapshot;
   messageId: number | null;
   content: string | JSX.Element;
   files?: FileDescriptor[];

web/src/app/chat/message/Messages.tsx

@@ -40,7 +40,7 @@ import { CodeBlock } from "./CodeBlock";
 import rehypePrism from "rehype-prism-plus";
 import "prismjs/themes/prism-tomorrow.css";
 import "./custom-code-styles.css";
-import { Persona } from "@/app/admin/assistants/interfaces";
+import { MinimalPersonaSnapshot } from "@/app/admin/assistants/interfaces";
 import { AssistantIcon } from "@/components/assistants/AssistantIcon";
 import { LikeFeedback, DislikeFeedback } from "@/components/icons/icons";
 import {
@@ -254,8 +254,8 @@ export const AIMessage = ({
   selectedDocuments?: OnyxDocument[] | null;
   toggleDocumentSelection?: () => void;
   docs?: OnyxDocument[] | null;
-  alternativeAssistant?: Persona | null;
-  currentPersona: Persona;
+  alternativeAssistant?: MinimalPersonaSnapshot | null;
+  currentPersona: MinimalPersonaSnapshot;
   messageId: number | null;
   content: string | JSX.Element;
   files?: FileDescriptor[];

web/src/app/chat/modal/configuration/AssistantsTab.tsx

@@ -13,7 +13,7 @@ import {
   sortableKeyboardCoordinates,
   verticalListSortingStrategy,
 } from "@dnd-kit/sortable";
-import { Persona } from "@/app/admin/assistants/interfaces";
+import { MinimalPersonaSnapshot } from "@/app/admin/assistants/interfaces";
 import { LLMProviderDescriptor } from "@/app/admin/configuration/llm/interfaces";
 import { getFinalLLM } from "@/lib/llm/utils";
 import React, { useEffect, useState } from "react";
@@ -27,9 +27,9 @@ export function AssistantsTab({
   llmProviders,
   onSelect,
 }: {
-  selectedAssistant: Persona;
+  selectedAssistant: MinimalPersonaSnapshot;
   llmProviders: LLMProviderDescriptor[];
-  onSelect: (assistant: Persona) => void;
+  onSelect: (assistant: MinimalPersonaSnapshot) => void;
 }) {
   const { refreshUser } = useUser();
   const [_, llmName] = getFinalLLM(llmProviders, null, null);

web/src/app/chat/my-documents/[id]/UserFolderContent.tsx

@@ -6,7 +6,6 @@ import {
   ArrowUp,
   ArrowDown,
   Trash,
-  Upload,
 } from "lucide-react";
 import { useDocumentsContext } from "../DocumentsContext";
 import { useChatContext } from "@/components/context/ChatContext";

web/src/app/chat/my-documents/components/FilePicker.tsx

@@ -34,7 +34,6 @@ import {
   TooltipTrigger,
   TooltipContent,
 } from "@/components/ui/tooltip";
-import { useRouter } from "next/navigation";
 import { usePopup } from "@/components/admin/connectors/Popup";
 import { getFormattedDateTime } from "@/lib/dateUtils";
 import { FileUploadSection } from "../[id]/components/upload/FileUploadSection";

web/src/app/chat/my-documents/components/SelectedItemsList.tsx

@@ -2,7 +2,6 @@ import React from "react";
 import { cn, truncateString } from "@/lib/utils";
 import { Button } from "@/components/ui/button";
 import { X, FolderIcon, Loader2 } from "lucide-react";
-import { ScrollArea } from "@/components/ui/scroll-area";
 import { FolderResponse, FileResponse } from "../DocumentsContext";
 import { getFileIconFromFileNameAndLink } from "@/lib/assistantIconUtils";
 import { MinimalOnyxDocument } from "@/lib/search/interfaces";

web/src/app/chat/sessionSidebar/HistorySidebar.tsx

@@ -27,7 +27,7 @@ import {
 import { PagesTab } from "./PagesTab";
 import { pageType } from "./types";
 import LogoWithText from "@/components/header/LogoWithText";
-import { Persona } from "@/app/admin/assistants/interfaces";
+import { MinimalPersonaSnapshot } from "@/app/admin/assistants/interfaces";
 import { DragEndEvent } from "@dnd-kit/core";
 import { useAssistants } from "@/components/context/AssistantsContext";
 import { AssistantIcon } from "@/components/assistants/AssistantIcon";
@@ -56,7 +56,7 @@ import { restrictToVerticalAxis } from "@dnd-kit/modifiers";
 import { TruncatedText } from "@/components/ui/truncatedText";
 
 interface HistorySidebarProps {
-  liveAssistant?: Persona | null;
+  liveAssistant?: MinimalPersonaSnapshot | null;
   page: pageType;
   existingChats?: ChatSession[];
   currentChatSession?: ChatSession | null | undefined;
@@ -73,7 +73,7 @@ interface HistorySidebarProps {
 }
 
 interface SortableAssistantProps {
-  assistant: Persona;
+  assistant: MinimalPersonaSnapshot;
   active: boolean;
   onClick: () => void;
   onPinAction: (e: React.MouseEvent) => void;
@@ -213,18 +213,22 @@ export const HistorySidebar = forwardRef<HTMLDivElement, HistorySidebarProps>(
         const { active, over } = event;
 
         if (active.id !== over?.id) {
-          setPinnedAssistants((prevAssistants: Persona[]) => {
+          setPinnedAssistants((prevAssistants: MinimalPersonaSnapshot[]) => {
             const oldIndex = prevAssistants.findIndex(
-              (a: Persona) => (a.id === 0 ? "assistant-0" : a.id) === active.id
+              (a: MinimalPersonaSnapshot) =>
+                (a.id === 0 ? "assistant-0" : a.id) === active.id
             );
             const newIndex = prevAssistants.findIndex(
-              (a: Persona) => (a.id === 0 ? "assistant-0" : a.id) === over?.id
+              (a: MinimalPersonaSnapshot) =>
+                (a.id === 0 ? "assistant-0" : a.id) === over?.id
             );
 
             const newOrder = arrayMove(prevAssistants, oldIndex, newIndex);
 
             // Ensure we're sending the correct IDs to the API
-            const reorderedIds = newOrder.map((a: Persona) => a.id);
+            const reorderedIds = newOrder.map(
+              (a: MinimalPersonaSnapshot) => a.id
+            );
             reorderPinnedAssistants(reorderedIds);
 
             return newOrder;
@@ -351,7 +355,7 @@ export const HistorySidebar = forwardRef<HTMLDivElement, HistorySidebarProps>(
                 strategy={verticalListSortingStrategy}
               >
                 <div className="flex px-0  mr-4 flex-col gap-y-1 mt-1">
-                  {pinnedAssistants.map((assistant: Persona) => (
+                  {pinnedAssistants.map((assistant: MinimalPersonaSnapshot) => (
                     <SortableAssistant
                       key={assistant.id === 0 ? "assistant-0" : assistant.id}
                       assistant={assistant}

web/src/app/chat/shared/[chatId]/SharedChatDisplay.tsx

@@ -15,14 +15,12 @@ import { useContext, useEffect, useState } from "react";
 import { SettingsContext } from "@/components/settings/SettingsProvider";
 import { OnyxInitializingLoader } from "@/components/OnyxInitializingLoader";
 import { Persona } from "@/app/admin/assistants/interfaces";
-import { Button } from "@/components/ui/button";
 import { MinimalOnyxDocument } from "@/lib/search/interfaces";
 import TextView from "@/components/chat/TextView";
 import { DocumentResults } from "../../documentSidebar/DocumentResults";
 import { Modal } from "@/components/Modal";
 import FunctionalHeader from "@/components/chat/Header";
 import FixedLogo from "@/components/logo/FixedLogo";
-import { useRouter } from "next/navigation";
 import Link from "next/link";
 
 function BackToOnyxButton({

web/src/app/ee/admin/standard-answer/page.tsx

@@ -25,7 +25,6 @@ import { deleteStandardAnswer } from "./lib";
 import { FilterDropdown } from "@/components/search/filtering/FilterDropdown";
 import { FiTag } from "react-icons/fi";
 import { PageSelector } from "@/components/PageSelector";
-import { CustomCheckbox } from "@/components/CustomCheckbox";
 import Text from "@/components/ui/text";
 import { TableHeader } from "@/components/ui/table";
 import CreateButton from "@/components/ui/createButton";

web/src/components/Loading.tsx

@@ -1,3 +1,5 @@
+"use client";
+
 import React, { useState, useEffect } from "react";
 import "./loading.css";
 import { ThreeDots } from "react-loader-spinner";

web/src/components/assistants/AssistantCards.tsx

@@ -1,4 +1,4 @@
-import { Persona } from "@/app/admin/assistants/interfaces";
+import { MinimalPersonaSnapshot } from "@/app/admin/assistants/interfaces";
 import { AssistantIcon } from "@/components/assistants/AssistantIcon";
 import { useSortable } from "@dnd-kit/sortable";
 import React from "react";
@@ -14,9 +14,9 @@ export const AssistantCard = ({
   isSelected,
   onSelect,
 }: {
-  assistant: Persona;
+  assistant: MinimalPersonaSnapshot;
   isSelected: boolean;
-  onSelect: (assistant: Persona) => void;
+  onSelect: (assistant: MinimalPersonaSnapshot) => void;
 }) => {
   const renderBadgeContent = (tool: { name: string }) => {
     switch (tool.name) {
@@ -73,9 +73,9 @@ export const AssistantCard = ({
 };
 
 export function DraggableAssistantCard(props: {
-  assistant: Persona;
+  assistant: MinimalPersonaSnapshot;
   isSelected: boolean;
-  onSelect: (assistant: Persona) => void;
+  onSelect: (assistant: MinimalPersonaSnapshot) => void;
   llmName: string;
 }) {
   const {

web/src/components/assistants/AssistantIcon.tsx

@@ -1,6 +1,9 @@
 import React from "react";
 import crypto from "crypto";
-import { Persona } from "@/app/admin/assistants/interfaces";
+import {
+  MinimalPersonaSnapshot,
+  Persona,
+} from "@/app/admin/assistants/interfaces";
 import { buildImgUrl } from "@/app/chat/files/images/utils";
 import {
   ArtAsistantIcon,
@@ -92,7 +95,7 @@ export function AssistantIcon({
   className,
   disableToolip,
 }: {
-  assistant: Persona;
+  assistant: MinimalPersonaSnapshot | Persona;
   size?: IconSize;
   className?: string;
   border?: boolean;

web/src/components/assistants/StarterMessage.tsx

@@ -1,12 +1,12 @@
 import { useContext } from "react";
-import { Persona } from "../../app/admin/assistants/interfaces";
+import { MinimalPersonaSnapshot } from "../../app/admin/assistants/interfaces";
 import { SettingsContext } from "../settings/SettingsProvider";
 
 export function StarterMessages({
   currentPersona,
   onSubmit,
 }: {
-  currentPersona: Persona;
+  currentPersona: MinimalPersonaSnapshot;
   onSubmit: (messageOverride: string) => void;
 }) {
   const settings = useContext(SettingsContext);

web/src/components/context/AppProvider.tsx

@@ -4,7 +4,7 @@ import { UserProvider } from "../user/UserProvider";
 import { ProviderContextProvider } from "../chat/ProviderContext";
 import { SettingsProvider } from "../settings/SettingsProvider";
 import { AssistantsProvider } from "./AssistantsContext";
-import { Persona } from "@/app/admin/assistants/interfaces";
+import { MinimalPersonaSnapshot } from "@/app/admin/assistants/interfaces";
 import { User } from "@/lib/types";
 import { ModalProvider } from "./ModalContext";
 import { AuthTypeMetadata } from "@/lib/userSS";
@@ -13,7 +13,7 @@ interface AppProviderProps {
   children: React.ReactNode;
   user: User | null;
   settings: CombinedSettings;
-  assistants: Persona[];
+  assistants: MinimalPersonaSnapshot[];
   hasAnyConnectors: boolean;
   hasImageCompatibleModel: boolean;
   authTypeMetadata: AuthTypeMetadata;

web/src/components/context/AssistantsContext.tsx

@@ -8,7 +8,7 @@ import React, {
   SetStateAction,
   Dispatch,
 } from "react";
-import { Persona } from "@/app/admin/assistants/interfaces";
+import { MinimalPersonaSnapshot } from "@/app/admin/assistants/interfaces";
 import {
   classifyAssistants,
   orderAssistantsForUser,
@@ -18,18 +18,18 @@ import {
 import { useUser } from "../user/UserProvider";
 
 interface AssistantsContextProps {
-  assistants: Persona[];
-  visibleAssistants: Persona[];
-  hiddenAssistants: Persona[];
-  finalAssistants: Persona[];
-  ownedButHiddenAssistants: Persona[];
+  assistants: MinimalPersonaSnapshot[];
+  visibleAssistants: MinimalPersonaSnapshot[];
+  hiddenAssistants: MinimalPersonaSnapshot[];
+  finalAssistants: MinimalPersonaSnapshot[];
+  ownedButHiddenAssistants: MinimalPersonaSnapshot[];
   refreshAssistants: () => Promise<void>;
   isImageGenerationAvailable: boolean;
   // Admin only
-  editablePersonas: Persona[];
-  allAssistants: Persona[];
-  pinnedAssistants: Persona[];
-  setPinnedAssistants: Dispatch<SetStateAction<Persona[]>>;
+  editablePersonas: MinimalPersonaSnapshot[];
+  allAssistants: MinimalPersonaSnapshot[];
+  pinnedAssistants: MinimalPersonaSnapshot[];
+  setPinnedAssistants: Dispatch<SetStateAction<MinimalPersonaSnapshot[]>>;
 }
 
 const AssistantsContext = createContext<AssistantsContextProps | undefined>(
@@ -38,27 +38,36 @@ const AssistantsContext = createContext<AssistantsContextProps | undefined>(
 
 export const AssistantsProvider: React.FC<{
   children: React.ReactNode;
-  initialAssistants: Persona[];
-  hasAnyConnectors: boolean;
-  hasImageCompatibleModel: boolean;
+  initialAssistants: MinimalPersonaSnapshot[];
+  hasAnyConnectors?: boolean;
+  hasImageCompatibleModel?: boolean;
 }> = ({
   children,
   initialAssistants,
   hasAnyConnectors,
   hasImageCompatibleModel,
 }) => {
-  const [assistants, setAssistants] = useState<Persona[]>(
+  const [assistants, setAssistants] = useState<MinimalPersonaSnapshot[]>(
     initialAssistants || []
   );
   const { user, isAdmin, isCurator } = useUser();
-  const [editablePersonas, setEditablePersonas] = useState<Persona[]>([]);
-  const [allAssistants, setAllAssistants] = useState<Persona[]>([]);
+  const [editablePersonas, setEditablePersonas] = useState<
+    MinimalPersonaSnapshot[]
+  >([]);
+  const [allAssistants, setAllAssistants] = useState<MinimalPersonaSnapshot[]>(
+    []
+  );
 
-  const [pinnedAssistants, setPinnedAssistants] = useState<Persona[]>(() => {
+  const [pinnedAssistants, setPinnedAssistants] = useState<
+    MinimalPersonaSnapshot[]
+  >(() => {
     if (user?.preferences.pinned_assistants) {
       return user.preferences.pinned_assistants
         .map((id) => assistants.find((assistant) => assistant.id === id))
-        .filter((assistant): assistant is Persona => assistant !== undefined);
+        .filter(
+          (assistant): assistant is MinimalPersonaSnapshot =>
+            assistant !== undefined
+        );
     } else {
       return assistants.filter((a) => a.is_default_persona);
     }
@@ -69,7 +78,10 @@ export const AssistantsProvider: React.FC<{
       if (user?.preferences.pinned_assistants) {
         return user.preferences.pinned_assistants
           .map((id) => assistants.find((assistant) => assistant.id === id))
-          .filter((assistant): assistant is Persona => assistant !== undefined);
+          .filter(
+            (assistant): assistant is MinimalPersonaSnapshot =>
+              assistant !== undefined
+          );
       } else {
         return assistants.filter((a) => a.is_default_persona);
       }
@@ -135,13 +147,9 @@ export const AssistantsProvider: React.FC<{
         },
       });
       if (!response.ok) throw new Error("Failed to fetch assistants");
-      let assistants: Persona[] = await response.json();
+      let assistants: MinimalPersonaSnapshot[] = await response.json();
 
-      let filteredAssistants = filterAssistants(
-        assistants,
-        hasAnyConnectors,
-        hasImageCompatibleModel
-      );
+      let filteredAssistants = filterAssistants(assistants);
 
       setAssistants(filteredAssistants);
 

web/src/components/context/ChatContext.tsx

@@ -1,7 +1,12 @@
 "use client";
 
 import React, { createContext, useContext, useState } from "react";
-import { CCPairBasicInfo, DocumentSet, Tag, ValidSources } from "@/lib/types";
+import {
+  CCPairBasicInfo,
+  DocumentSetSummary,
+  Tag,
+  ValidSources,
+} from "@/lib/types";
 import { ChatSession, InputPrompt } from "@/app/chat/interfaces";
 import { LLMProviderDescriptor } from "@/app/admin/configuration/llm/interfaces";
 import { Folder } from "@/app/chat/folders/interfaces";
@@ -14,8 +19,8 @@ interface ChatContextProps {
   availableSources: ValidSources[];
   ccPairs: CCPairBasicInfo[];
   tags: Tag[];
-  documentSets: DocumentSet[];
-  availableDocumentSets: DocumentSet[];
+  documentSets: DocumentSetSummary[];
+  availableDocumentSets: DocumentSetSummary[];
   availableTags: Tag[];
   llmProviders: LLMProviderDescriptor[];
   folders: Folder[];

web/src/components/context/SearchContext.tsx

@@ -1,7 +1,7 @@
 "use client";
 
 import React, { createContext, useContext } from "react";
-import { CCPairBasicInfo, DocumentSet, Tag } from "@/lib/types";
+import { CCPairBasicInfo, DocumentSetSummary, Tag } from "@/lib/types";
 import { Persona } from "@/app/admin/assistants/interfaces";
 import { ChatSession } from "@/app/chat/interfaces";
 
@@ -9,7 +9,7 @@ interface SearchContextProps {
   querySessions: ChatSession[];
   ccPairs: CCPairBasicInfo[];
   tags: Tag[];
-  documentSets: DocumentSet[];
+  documentSets: DocumentSetSummary[];
   assistants: Persona[];
   agenticSearchEnabled: boolean;
   disabledAgentic: boolean;

web/src/components/documentSet/DocumentSetSelectable.tsx

@@ -1,6 +1,6 @@
 "use client";
 
-import { DocumentSet, ValidSources } from "@/lib/types";
+import { DocumentSetSummary, ValidSources } from "@/lib/types";
 import { CustomCheckbox } from "../CustomCheckbox";
 import { SourceIcon } from "../SourceIcon";
 import {
@@ -17,7 +17,7 @@ export function DocumentSetSelectable({
   disabled,
   disabledTooltip,
 }: {
-  documentSet: DocumentSet;
+  documentSet: DocumentSetSummary;
   isSelected: boolean;
   onSelect: () => void;
   disabled?: boolean;
@@ -25,8 +25,8 @@ export function DocumentSetSelectable({
 }) {
   // Collect unique connector sources
   const uniqueSources = new Set<ValidSources>();
-  documentSet.cc_pair_descriptors.forEach((ccPairDescriptor) => {
-    uniqueSources.add(ccPairDescriptor.connector.source);
+  documentSet.cc_pair_summaries.forEach((ccPairSummary) => {
+    uniqueSources.add(ccPairSummary.source);
   });
 
   return (

web/src/components/filters/SourceSelector.tsx

@@ -1,5 +1,5 @@
 import React from "react";
-import { DocumentSet, Tag, ValidSources } from "@/lib/types";
+import { DocumentSetSummary, Tag, ValidSources } from "@/lib/types";
 import { SourceMetadata } from "@/lib/search/interfaces";
 import { InfoIcon, defaultTailwindCSS } from "@/components/icons/icons";
 import { HoverPopup } from "@/components/HoverPopup";
@@ -40,7 +40,7 @@ export interface SourceSelectorProps {
   setSelectedDocumentSets: React.Dispatch<React.SetStateAction<string[]>>;
   selectedTags: Tag[];
   setSelectedTags: React.Dispatch<React.SetStateAction<Tag[]>>;
-  availableDocumentSets: DocumentSet[];
+  availableDocumentSets: DocumentSetSummary[];
   existingSources: ValidSources[];
   availableTags: Tag[];
   toggleFilters: () => void;

web/src/components/search/SearchBar.tsx

@@ -9,9 +9,9 @@ interface FullSearchBarProps {
   agentic?: boolean;
   toggleAgentic?: () => void;
   ccPairs: CCPairBasicInfo[];
-  documentSets: DocumentSet[];
+  documentSets: DocumentSetSummary[];
   filterManager: any; // You might want to replace 'any' with a more specific type
-  finalAvailableDocumentSets: DocumentSet[];
+  finalAvailableDocumentSets: DocumentSetSummary[];
   finalAvailableSources: string[];
   tags: Tag[];
   showingSidebar: boolean;
@@ -27,7 +27,7 @@ import { useRef } from "react";
 import { SendIcon } from "../icons/icons";
 import { Separator } from "@/components/ui/separator";
 import KeyboardSymbol from "@/lib/browserUtilities";
-import { CCPairBasicInfo, DocumentSet, Tag } from "@/lib/types";
+import { CCPairBasicInfo, DocumentSetSummary, Tag } from "@/lib/types";
 import { HorizontalSourceSelector } from "./filtering/HorizontalSourceSelector";
 
 export const AnimatedToggle = ({

web/src/components/search/filtering/FilterPopup.tsx

@@ -13,7 +13,7 @@ import {
   FiBook,
 } from "react-icons/fi";
 import { FilterManager } from "@/lib/hooks";
-import { DocumentSet, Tag } from "@/lib/types";
+import { DocumentSetSummary, Tag } from "@/lib/types";
 import { SourceMetadata } from "@/lib/search/interfaces";
 import { Checkbox } from "@/components/ui/checkbox";
 import { Separator } from "@/components/ui/separator";
@@ -26,7 +26,7 @@ interface FilterPopupProps {
   filterManager: FilterManager;
   trigger: React.ReactNode;
   availableSources: SourceMetadata[];
-  availableDocumentSets: DocumentSet[];
+  availableDocumentSets: DocumentSetSummary[];
   availableTags: Tag[];
 }
 
@@ -50,7 +50,7 @@ export function FilterPopup({
   const [currentDate, setCurrentDate] = useState(new Date());
   const [documentSetSearch, setDocumentSetSearch] = useState("");
   const [filteredDocumentSets, setFilteredDocumentSets] = useState<
-    DocumentSet[]
+    DocumentSetSummary[]
   >(availableDocumentSets);
 
   useEffect(() => {
@@ -238,10 +238,10 @@ export function FilterPopup({
     }
   };
 
-  const isDocumentSetSelected = (docSet: DocumentSet) =>
+  const isDocumentSetSelected = (docSet: DocumentSetSummary) =>
     filterManager.selectedDocumentSets.includes(docSet.name);
 
-  const toggleDocumentSet = (docSet: DocumentSet) => {
+  const toggleDocumentSet = (docSet: DocumentSetSummary) => {
     filterManager.setSelectedDocumentSets((prev) =>
       prev.includes(docSet.name)
         ? prev.filter((id) => id !== docSet.name)

web/src/lib/assistants/checkOwnership.ts

@@ -1,14 +1,20 @@
-import { Persona } from "@/app/admin/assistants/interfaces";
+import {
+  MinimalPersonaSnapshot,
+  Persona,
+} from "@/app/admin/assistants/interfaces";
 import { User } from "../types";
 import { checkUserIsNoAuthUser } from "../user";
 
-export function checkUserOwnsAssistant(user: User | null, assistant: Persona) {
+export function checkUserOwnsAssistant(
+  user: User | null,
+  assistant: MinimalPersonaSnapshot | Persona
+) {
   return checkUserIdOwnsAssistant(user?.id, assistant);
 }
 
 export function checkUserIdOwnsAssistant(
   userId: string | undefined,
-  assistant: Persona
+  assistant: MinimalPersonaSnapshot | Persona
 ) {
   return (
     (!userId ||

web/src/lib/assistants/fetchAssistantsSS.ts

@@ -1,12 +1,12 @@
-import { Persona } from "@/app/admin/assistants/interfaces";
+import { MinimalPersonaSnapshot } from "@/app/admin/assistants/interfaces";
 import { fetchSS } from "../utilsSS";
 
-export type FetchAssistantsResponse = [Persona[], string | null];
+export type FetchAssistantsResponse = [MinimalPersonaSnapshot[], string | null];
 
 export async function fetchAssistantsSS(): Promise<FetchAssistantsResponse> {
   const response = await fetchSS("/persona");
   if (response.ok) {
-    return [(await response.json()) as Persona[], null];
+    return [(await response.json()) as MinimalPersonaSnapshot[], null];
   }
   return [[], (await response.json()).detail || "Unknown Error"];
 }

web/src/lib/assistants/fetchPersonaEditorInfoSS.ts

@@ -1,5 +1,5 @@
 import { FullPersona } from "@/app/admin/assistants/interfaces";
-import { CCPairBasicInfo, DocumentSet, User } from "../types";
+import { CCPairBasicInfo, DocumentSetSummary, User } from "../types";
 import { getCurrentUserSS } from "../userSS";
 import { fetchSS } from "../utilsSS";
 import { LLMProviderView } from "@/app/admin/configuration/llm/interfaces";
@@ -12,7 +12,7 @@ export async function fetchAssistantEditorInfoSS(
   | [
       {
         ccPairs: CCPairBasicInfo[];
-        documentSets: DocumentSet[];
+        documentSets: DocumentSetSummary[];
         llmProviders: LLMProviderView[];
         user: User | null;
         existingPersona: FullPersona | null;
@@ -67,7 +67,8 @@ export async function fetchAssistantEditorInfoSS(
       `Failed to fetch document sets - ${await documentSetsResponse.text()}`,
     ];
   }
-  const documentSets = (await documentSetsResponse.json()) as DocumentSet[];
+  const documentSets =
+    (await documentSetsResponse.json()) as DocumentSetSummary[];
 
   if (!toolsResponse) {
     return [null, `Failed to fetch tools`];

web/src/lib/assistants/utils.ts

@@ -1,15 +1,18 @@
-import { Persona } from "@/app/admin/assistants/interfaces";
+import { MinimalPersonaSnapshot } from "@/app/admin/assistants/interfaces";
 import { User } from "../types";
 import { checkUserIsNoAuthUser } from "../user";
 import { personaComparator } from "@/app/admin/assistants/lib";
 
-export function checkUserOwnsAssistant(user: User | null, assistant: Persona) {
+export function checkUserOwnsAssistant(
+  user: User | null,
+  assistant: MinimalPersonaSnapshot
+) {
   return checkUserIdOwnsAssistant(user?.id, assistant);
 }
 
 export function checkUserIdOwnsAssistant(
   userId: string | undefined,
-  assistant: Persona
+  assistant: MinimalPersonaSnapshot
 ) {
   return (
     (!userId ||
@@ -19,7 +22,10 @@ export function checkUserIdOwnsAssistant(
   );
 }
 
-export function classifyAssistants(user: User | null, assistants: Persona[]) {
+export function classifyAssistants(
+  user: User | null,
+  assistants: MinimalPersonaSnapshot[]
+) {
   if (!user) {
     return {
       visibleAssistants: assistants.filter(
@@ -59,7 +65,7 @@ export function classifyAssistants(user: User | null, assistants: Persona[]) {
 }
 
 export function orderAssistantsForUser(
-  assistants: Persona[],
+  assistants: MinimalPersonaSnapshot[],
   user: User | null
 ) {
   let orderedAssistants = [...assistants];
@@ -112,7 +118,7 @@ export function orderAssistantsForUser(
 
 export function getUserCreatedAssistants(
   user: User | null,
-  assistants: Persona[]
+  assistants: MinimalPersonaSnapshot[]
 ) {
   return assistants.filter((assistant) =>
     checkUserOwnsAssistant(user, assistant)
@@ -121,29 +127,10 @@ export function getUserCreatedAssistants(
 
 // Filter assistants based on connector status, image compatibility and visibility
 export function filterAssistants(
-  assistants: Persona[],
-  hasAnyConnectors: boolean,
-  hasImageCompatibleModel: boolean
-): Persona[] {
+  assistants: MinimalPersonaSnapshot[]
+): MinimalPersonaSnapshot[] {
   let filteredAssistants = assistants.filter(
     (assistant) => assistant.is_visible
   );
-
-  if (!hasAnyConnectors) {
-    filteredAssistants = filteredAssistants.filter(
-      (assistant) =>
-        assistant.num_chunks === 0 || assistant.document_sets.length > 0
-    );
-  }
-
-  if (!hasImageCompatibleModel) {
-    filteredAssistants = filteredAssistants.filter(
-      (assistant) =>
-        !assistant.tools.some(
-          (tool) => tool.in_code_tool_id === "ImageGenerationTool"
-        )
-    );
-  }
-
   return filteredAssistants.sort(personaComparator);
 }

web/src/lib/chat/fetchAssistantdata.ts

@@ -1,12 +1,12 @@
 import { fetchSS } from "@/lib/utilsSS";
-import { Persona } from "@/app/admin/assistants/interfaces";
+import { MinimalPersonaSnapshot } from "@/app/admin/assistants/interfaces";
 import { fetchLLMProvidersSS } from "@/lib/llm/fetchLLMs";
 import { fetchAssistantsSS } from "../assistants/fetchAssistantsSS";
 import { modelSupportsImageInput } from "../llm/utils";
 import { filterAssistants } from "../assistants/utils";
 
 interface AssistantData {
-  assistants: Persona[];
+  assistants: MinimalPersonaSnapshot[];
   hasAnyConnectors: boolean;
   hasImageCompatibleModel: boolean;
 }
@@ -51,11 +51,7 @@ export async function fetchAssistantData(): Promise<AssistantData> {
         )
     );
 
-    let filteredAssistants = filterAssistants(
-      assistants,
-      hasAnyConnectors,
-      hasImageCompatibleModel
-    );
+    let filteredAssistants = filterAssistants(assistants);
 
     return {
       assistants: filteredAssistants,

web/src/lib/chat/fetchChatData.ts

@@ -6,7 +6,7 @@ import {
 import { fetchSS } from "@/lib/utilsSS";
 import {
   CCPairBasicInfo,
-  DocumentSet,
+  DocumentSetSummary,
   Tag,
   User,
   ValidSources,
@@ -35,7 +35,7 @@ interface FetchChatDataResult {
   chatSessions: ChatSession[];
   ccPairs: CCPairBasicInfo[];
   availableSources: ValidSources[];
-  documentSets: DocumentSet[];
+  documentSets: DocumentSetSummary[];
   tags: Tag[];
   llmProviders: LLMProviderDescriptor[];
   folders: Folder[];
@@ -167,7 +167,7 @@ export async function fetchChatData(searchParams: {
       new Date(b.time_updated).getTime() - new Date(a.time_updated).getTime()
   );
 
-  let documentSets: DocumentSet[] = [];
+  let documentSets: DocumentSetSummary[] = [];
   if (documentSetsResponse?.ok) {
     documentSets = await documentSetsResponse.json();
   } else {

web/src/lib/chat/fetchSomeChatData.ts

@@ -6,7 +6,7 @@ import {
 import { fetchSS } from "@/lib/utilsSS";
 import {
   CCPairBasicInfo,
-  DocumentSet,
+  DocumentSetSummary,
   Tag,
   User,
   ValidSources,
@@ -31,7 +31,7 @@ interface FetchChatDataResult {
   chatSessions?: ChatSession[];
   ccPairs?: CCPairBasicInfo[];
   availableSources?: ValidSources[];
-  documentSets?: DocumentSet[];
+  documentSets?: DocumentSetSummary[];
   assistants?: Persona[];
   tags?: Tag[];
   llmProviders?: LLMProviderDescriptor[];
@@ -123,7 +123,7 @@ export async function fetchSomeChatData(
         break;
       case "documentSets":
         result.documentSets = result?.ok
-          ? ((await result.json()) as DocumentSet[])
+          ? ((await result.json()) as DocumentSetSummary[])
           : [];
         break;
       case "assistants":

web/src/lib/filters.ts

@@ -1,5 +1,5 @@
 import { Persona } from "@/app/admin/assistants/interfaces";
-import { DocumentSet, ValidSources } from "./types";
+import { DocumentSetSummary, ValidSources } from "./types";
 import { getSourcesForPersona } from "./sources";
 
 export function computeAvailableFilters({
@@ -9,8 +9,8 @@ export function computeAvailableFilters({
 }: {
   selectedPersona: Persona | undefined | null;
   availableSources: ValidSources[];
-  availableDocumentSets: DocumentSet[];
-}): [ValidSources[], DocumentSet[]] {
+  availableDocumentSets: DocumentSetSummary[];
+}): [ValidSources[], DocumentSetSummary[]] {
   const finalAvailableSources =
     selectedPersona && selectedPersona.document_sets.length
       ? getSourcesForPersona(selectedPersona)

web/src/lib/hooks.ts

@@ -22,7 +22,10 @@ import { ChatSession } from "@/app/chat/interfaces";
 import { AllUsersResponse } from "./types";
 import { Credential } from "./connectors/credentials";
 import { SettingsContext } from "@/components/settings/SettingsProvider";
-import { Persona, PersonaLabel } from "@/app/admin/assistants/interfaces";
+import {
+  MinimalPersonaSnapshot,
+  PersonaLabel,
+} from "@/app/admin/assistants/interfaces";
 import { LLMProviderDescriptor } from "@/app/admin/configuration/llm/interfaces";
 import { isAnthropic } from "@/app/admin/configuration/llm/utils";
 import { getSourceMetadata } from "./sources";
@@ -371,7 +374,7 @@ export interface LlmManager {
   updateModelOverrideBasedOnChatSession: (chatSession?: ChatSession) => void;
   imageFilesPresent: boolean;
   updateImageFilesPresent: (present: boolean) => void;
-  liveAssistant: Persona | null;
+  liveAssistant: MinimalPersonaSnapshot | null;
   maxTemperature: number;
 }
 
@@ -419,7 +422,7 @@ providing appropriate defaults for new conversations based on the available tool
 export function useLlmManager(
   llmProviders: LLMProviderDescriptor[],
   currentChatSession?: ChatSession,
-  liveAssistant?: Persona
+  liveAssistant?: MinimalPersonaSnapshot
 ): LlmManager {
   const { user } = useUser();
 

web/src/lib/llm/utils.ts

@@ -1,4 +1,4 @@
-import { Persona } from "@/app/admin/assistants/interfaces";
+import { MinimalPersonaSnapshot } from "@/app/admin/assistants/interfaces";
 import {
   LLMProviderDescriptor,
   ModelConfiguration,
@@ -7,7 +7,7 @@ import { LlmDescriptor } from "@/lib/hooks";
 
 export function getFinalLLM(
   llmProviders: LLMProviderDescriptor[],
-  persona: Persona | null,
+  persona: MinimalPersonaSnapshot | null,
   currentLlm: LlmDescriptor | null
 ): [string, string] {
   const defaultProvider = llmProviders.find(
@@ -38,7 +38,7 @@ export function getFinalLLM(
 }
 
 export function getLLMProviderOverrideForPersona(
-  liveAssistant: Persona,
+  liveAssistant: MinimalPersonaSnapshot,
   llmProviders: LLMProviderDescriptor[]
 ): LlmDescriptor | null {
   const overrideProvider = liveAssistant.llm_model_provider_override;

web/src/lib/search/utilsSS.ts

@@ -1,4 +1,4 @@
-import { DocumentSet } from "../types";
+import { DocumentSetSummary } from "../types";
 import { fetchSS } from "../utilsSS";
 import { Connector } from "../connectors/connectors";
 
@@ -17,9 +17,9 @@ export async function fetchValidFilterInfo() {
     );
   }
 
-  let documentSets = [] as DocumentSet[];
+  let documentSets = [] as DocumentSetSummary[];
   if (documentSetResponse.ok) {
-    documentSets = (await documentSetResponse.json()) as DocumentSet[];
+    documentSets = (await documentSetResponse.json()) as DocumentSetSummary[];
   } else {
     console.log(
       `Failed to fetch document sets - ${documentSetResponse.status} - ${documentSetResponse.statusText}`

web/src/lib/sources.ts

@@ -402,9 +402,9 @@ export function getSourceMetadataForSources(sources: ValidSources[]) {
 export function getSourcesForPersona(persona: Persona): ValidSources[] {
   const personaSources: ValidSources[] = [];
   persona.document_sets.forEach((documentSet) => {
-    documentSet.cc_pair_descriptors.forEach((ccPair) => {
-      if (!personaSources.includes(ccPair.connector.source)) {
-        personaSources.push(ccPair.connector.source);
+    documentSet.cc_pair_summaries.forEach((ccPair) => {
+      if (!personaSources.includes(ccPair.source)) {
+        personaSources.push(ccPair.source);
       }
     });
   });

web/src/lib/types.ts

@@ -243,11 +243,26 @@ export interface CCPairDescriptor<ConnectorType, CredentialType> {
   access_type: AccessType;
 }
 
-export interface DocumentSet {
+// Simplified interfaces with minimal data
+export interface CCPairSummary {
+  id: number;
+  name: string | null;
+  source: ValidSources;
+  access_type: AccessType;
+}
+
+export interface FederatedConnectorSummary {
+  id: number;
+  name: string;
+  source: string;
+  entities: Record<string, any>;
+}
+
+export interface DocumentSetSummary {
   id: number;
   name: string;
   description: string;
-  cc_pair_descriptors: CCPairDescriptor<any, any>[];
+  cc_pair_summaries: CCPairSummary[];
   is_up_to_date: boolean;
   is_public: boolean;
   users: string[];
@@ -339,7 +354,7 @@ export interface UserGroup {
   users: User[];
   curator_ids: string[];
   cc_pairs: CCPairDescriptor<any, any>[];
-  document_sets: DocumentSet[];
+  document_sets: DocumentSetSummary[];
   personas: Persona[];
   is_up_to_date: boolean;
   is_up_for_deletion: boolean;