p

backend/ee/onyx/background/celery/apps/primary.py

@@ -32,7 +32,6 @@ def perform_ttl_management_task(
 
 @celery_app.task(
     name="check_ttl_management_task",
-    ignore_result=True,
     soft_time_limit=JOB_TIMEOUT,
 )
 def check_ttl_management_task(*, tenant_id: str | None) -> None:
@@ -57,7 +56,6 @@ def check_ttl_management_task(*, tenant_id: str | None) -> None:
 
 @celery_app.task(
     name="autogenerate_usage_report_task",
-    ignore_result=True,
     soft_time_limit=JOB_TIMEOUT,
 )
 def autogenerate_usage_report_task(*, tenant_id: str | None) -> None:

backend/ee/onyx/external_permissions/google_drive/doc_sync.py

@@ -42,22 +42,24 @@ def _fetch_permissions_for_permission_ids(
     if not permission_info or not doc_id:
         return []
 
+    # Check cache first for all permission IDs
     permissions = [
         _PERMISSION_ID_PERMISSION_MAP[pid]
         for pid in permission_ids
         if pid in _PERMISSION_ID_PERMISSION_MAP
     ]
 
+    # If we found all permissions in cache, return them
     if len(permissions) == len(permission_ids):
         return permissions
 
     owner_email = permission_info.get("owner_email")
-
     drive_service = get_drive_service(
         creds=google_drive_connector.creds,
         user_email=(owner_email or google_drive_connector.primary_admin_email),
     )
 
+    # Otherwise, fetch all permissions and update cache
     fetched_permissions = execute_paginated_retrieval(
         retrieval_function=drive_service.permissions().list,
         list_key="permissions",
@@ -67,6 +69,7 @@ def _fetch_permissions_for_permission_ids(
     )
 
     permissions_for_doc_id = []
+    # Update cache and return all permissions
     for permission in fetched_permissions:
         permissions_for_doc_id.append(permission)
         _PERMISSION_ID_PERMISSION_MAP[permission["id"]] = permission

backend/onyx/background/celery/apps/beat.py

@@ -1,5 +1,6 @@
 from datetime import timedelta
 from typing import Any
+from typing import cast
 
 from celery import Celery
 from celery import signals
@@ -7,6 +8,7 @@ from celery.beat import PersistentScheduler  # type: ignore
 from celery.signals import beat_init
 
 import onyx.background.celery.apps.app_base as app_base
+from onyx.configs.constants import ONYX_CLOUD_CELERY_TASK_PREFIX
 from onyx.configs.constants import POSTGRES_CELERY_BEAT_APP_NAME
 from onyx.db.engine import get_all_tenant_ids
 from onyx.db.engine import SqlEngine
@@ -130,25 +132,21 @@ class DynamicTenantScheduler(PersistentScheduler):
         # get current schedule and extract current tenants
         current_schedule = self.schedule.items()
 
-        # there are no more per tenant beat tasks, so comment this out
-        # NOTE: we may not actualy need this scheduler any more and should
-        # test reverting to a regular beat schedule implementation
+        current_tenants = set()
+        for task_name, _ in current_schedule:
+            task_name = cast(str, task_name)
+            if task_name.startswith(ONYX_CLOUD_CELERY_TASK_PREFIX):
+                continue
 
-        # current_tenants = set()
-        # for task_name, _ in current_schedule:
-        #     task_name = cast(str, task_name)
-        #     if task_name.startswith(ONYX_CLOUD_CELERY_TASK_PREFIX):
-        #         continue
+            if "_" in task_name:
+                # example: "check-for-condition-tenant_12345678-abcd-efgh-ijkl-12345678"
+                # -> "12345678-abcd-efgh-ijkl-12345678"
+                current_tenants.add(task_name.split("_")[-1])
+        logger.info(f"Found {len(current_tenants)} existing items in schedule")
 
-        #     if "_" in task_name:
-        #         # example: "check-for-condition-tenant_12345678-abcd-efgh-ijkl-12345678"
-        #         # -> "12345678-abcd-efgh-ijkl-12345678"
-        #         current_tenants.add(task_name.split("_")[-1])
-        # logger.info(f"Found {len(current_tenants)} existing items in schedule")
-
-        # for tenant_id in tenant_ids:
-        #     if tenant_id not in current_tenants:
-        #         logger.info(f"Processing new tenant: {tenant_id}")
+        for tenant_id in tenant_ids:
+            if tenant_id not in current_tenants:
+                logger.info(f"Processing new tenant: {tenant_id}")
 
         new_schedule = self._generate_schedule(tenant_ids)
 

backend/onyx/background/celery/tasks/beat_schedule.py

@@ -16,10 +16,6 @@ from shared_configs.configs import MULTI_TENANT
 # it's only important that they run relatively regularly
 BEAT_EXPIRES_DEFAULT = 15 * 60  # 15 minutes (in seconds)
 
-# hack to slow down task dispatch in the cloud until
-# we have a better implementation (backpressure, etc)
-CLOUD_BEAT_SCHEDULE_MULTIPLIER = 8
-
 # tasks that only run in the cloud
 # the name attribute must start with ONYX_CLOUD_CELERY_TASK_PREFIX = "cloud" to be filtered
 # by the DynamicTenantScheduler
@@ -28,7 +24,7 @@ cloud_tasks_to_schedule = [
     {
         "name": f"{ONYX_CLOUD_CELERY_TASK_PREFIX}_check-alembic",
         "task": OnyxCeleryTask.CLOUD_CHECK_ALEMBIC,
-        "schedule": timedelta(hours=1 * CLOUD_BEAT_SCHEDULE_MULTIPLIER),
+        "schedule": timedelta(hours=1),
         "options": {
             "queue": OnyxCeleryQueues.MONITORING,
             "priority": OnyxCeleryPriority.HIGH,
@@ -39,7 +35,7 @@ cloud_tasks_to_schedule = [
     {
         "name": f"{ONYX_CLOUD_CELERY_TASK_PREFIX}_check-for-indexing",
         "task": OnyxCeleryTask.CLOUD_BEAT_TASK_GENERATOR,
-        "schedule": timedelta(seconds=15 * CLOUD_BEAT_SCHEDULE_MULTIPLIER),
+        "schedule": timedelta(seconds=15),
         "options": {
             "priority": OnyxCeleryPriority.HIGHEST,
             "expires": BEAT_EXPIRES_DEFAULT,
@@ -51,7 +47,7 @@ cloud_tasks_to_schedule = [
     {
         "name": f"{ONYX_CLOUD_CELERY_TASK_PREFIX}_check-for-connector-deletion",
         "task": OnyxCeleryTask.CLOUD_BEAT_TASK_GENERATOR,
-        "schedule": timedelta(seconds=20 * CLOUD_BEAT_SCHEDULE_MULTIPLIER),
+        "schedule": timedelta(seconds=20),
         "options": {
             "priority": OnyxCeleryPriority.HIGHEST,
             "expires": BEAT_EXPIRES_DEFAULT,
@@ -63,7 +59,7 @@ cloud_tasks_to_schedule = [
     {
         "name": f"{ONYX_CLOUD_CELERY_TASK_PREFIX}_check-for-vespa-sync",
         "task": OnyxCeleryTask.CLOUD_BEAT_TASK_GENERATOR,
-        "schedule": timedelta(seconds=20 * CLOUD_BEAT_SCHEDULE_MULTIPLIER),
+        "schedule": timedelta(seconds=20),
         "options": {
             "priority": OnyxCeleryPriority.HIGHEST,
             "expires": BEAT_EXPIRES_DEFAULT,
@@ -75,7 +71,7 @@ cloud_tasks_to_schedule = [
     {
         "name": f"{ONYX_CLOUD_CELERY_TASK_PREFIX}_check-for-prune",
         "task": OnyxCeleryTask.CLOUD_BEAT_TASK_GENERATOR,
-        "schedule": timedelta(seconds=15 * CLOUD_BEAT_SCHEDULE_MULTIPLIER),
+        "schedule": timedelta(seconds=15),
         "options": {
             "priority": OnyxCeleryPriority.HIGHEST,
             "expires": BEAT_EXPIRES_DEFAULT,
@@ -87,7 +83,7 @@ cloud_tasks_to_schedule = [
     {
         "name": f"{ONYX_CLOUD_CELERY_TASK_PREFIX}_monitor-vespa-sync",
         "task": OnyxCeleryTask.CLOUD_BEAT_TASK_GENERATOR,
-        "schedule": timedelta(seconds=15 * CLOUD_BEAT_SCHEDULE_MULTIPLIER),
+        "schedule": timedelta(seconds=5),
         "options": {
             "priority": OnyxCeleryPriority.HIGHEST,
             "expires": BEAT_EXPIRES_DEFAULT,
@@ -99,7 +95,7 @@ cloud_tasks_to_schedule = [
     {
         "name": f"{ONYX_CLOUD_CELERY_TASK_PREFIX}_check-for-doc-permissions-sync",
         "task": OnyxCeleryTask.CLOUD_BEAT_TASK_GENERATOR,
-        "schedule": timedelta(seconds=30 * CLOUD_BEAT_SCHEDULE_MULTIPLIER),
+        "schedule": timedelta(seconds=30),
         "options": {
             "priority": OnyxCeleryPriority.HIGHEST,
             "expires": BEAT_EXPIRES_DEFAULT,
@@ -111,7 +107,7 @@ cloud_tasks_to_schedule = [
     {
         "name": f"{ONYX_CLOUD_CELERY_TASK_PREFIX}_check-for-external-group-sync",
         "task": OnyxCeleryTask.CLOUD_BEAT_TASK_GENERATOR,
-        "schedule": timedelta(seconds=20 * CLOUD_BEAT_SCHEDULE_MULTIPLIER),
+        "schedule": timedelta(seconds=20),
         "options": {
             "priority": OnyxCeleryPriority.HIGHEST,
             "expires": BEAT_EXPIRES_DEFAULT,
@@ -123,7 +119,7 @@ cloud_tasks_to_schedule = [
     {
         "name": f"{ONYX_CLOUD_CELERY_TASK_PREFIX}_monitor-background-processes",
         "task": OnyxCeleryTask.CLOUD_BEAT_TASK_GENERATOR,
-        "schedule": timedelta(minutes=5 * CLOUD_BEAT_SCHEDULE_MULTIPLIER),
+        "schedule": timedelta(minutes=5),
         "options": {
             "priority": OnyxCeleryPriority.HIGHEST,
             "expires": BEAT_EXPIRES_DEFAULT,
@@ -141,9 +137,7 @@ if LLM_MODEL_UPDATE_API_URL:
         {
             "name": f"{ONYX_CLOUD_CELERY_TASK_PREFIX}_check-for-llm-model-update",
             "task": OnyxCeleryTask.CLOUD_BEAT_TASK_GENERATOR,
-            "schedule": timedelta(
-                hours=1 * CLOUD_BEAT_SCHEDULE_MULTIPLIER
-            ),  # Check every hour
+            "schedule": timedelta(hours=1),  # Check every hour
             "options": {
                 "priority": OnyxCeleryPriority.HIGHEST,
                 "expires": BEAT_EXPIRES_DEFAULT,
@@ -227,7 +221,7 @@ if not MULTI_TENANT:
             {
                 "name": "monitor-background-processes",
                 "task": OnyxCeleryTask.MONITOR_BACKGROUND_PROCESSES,
-                "schedule": timedelta(minutes=15),
+                "schedule": timedelta(minutes=5),
                 "options": {
                     "priority": OnyxCeleryPriority.LOW,
                     "expires": BEAT_EXPIRES_DEFAULT,

backend/onyx/background/celery/tasks/connector_deletion/tasks.py

@@ -33,7 +33,6 @@ class TaskDependencyError(RuntimeError):
 
 @shared_task(
     name=OnyxCeleryTask.CHECK_FOR_CONNECTOR_DELETION,
-    ignore_result=True,
     soft_time_limit=JOB_TIMEOUT,
     trail=False,
     bind=True,
@@ -140,6 +139,13 @@ def try_generate_document_cc_pair_cleanup_tasks(
         submitted=datetime.now(timezone.utc),
     )
 
+    # create before setting fence to avoid race condition where the monitoring
+    # task updates the sync record before it is created
+    insert_sync_record(
+        db_session=db_session,
+        entity_id=cc_pair_id,
+        sync_type=SyncType.CONNECTOR_DELETION,
+    )
     redis_connector.delete.set_fence(fence_payload)
 
     try:
@@ -178,13 +184,6 @@ def try_generate_document_cc_pair_cleanup_tasks(
         )
         if tasks_generated is None:
             raise ValueError("RedisConnectorDeletion.generate_tasks returned None")
-
-        insert_sync_record(
-            db_session=db_session,
-            entity_id=cc_pair_id,
-            sync_type=SyncType.CONNECTOR_DELETION,
-        )
-
     except TaskDependencyError:
         redis_connector.delete.set_fence(None)
         raise

backend/onyx/background/celery/tasks/doc_permission_syncing/tasks.py

@@ -91,7 +91,6 @@ def _is_external_doc_permissions_sync_due(cc_pair: ConnectorCredentialPair) -> b
 
 @shared_task(
     name=OnyxCeleryTask.CHECK_FOR_DOC_PERMISSIONS_SYNC,
-    ignore_result=True,
     soft_time_limit=JOB_TIMEOUT,
     bind=True,
 )

backend/onyx/background/celery/tasks/external_group_syncing/tasks.py

@@ -91,7 +91,6 @@ def _is_external_group_sync_due(cc_pair: ConnectorCredentialPair) -> bool:
 
 @shared_task(
     name=OnyxCeleryTask.CHECK_FOR_EXTERNAL_GROUP_SYNC,
-    ignore_result=True,
     soft_time_limit=JOB_TIMEOUT,
     bind=True,
 )

backend/onyx/background/celery/tasks/indexing/tasks.py

@@ -45,7 +45,6 @@ from onyx.natural_language_processing.search_nlp_models import EmbeddingModel
 from onyx.natural_language_processing.search_nlp_models import warm_up_bi_encoder
 from onyx.redis.redis_connector import RedisConnector
 from onyx.redis.redis_pool import get_redis_client
-from onyx.redis.redis_pool import get_redis_replica_client
 from onyx.redis.redis_pool import redis_lock_dump
 from onyx.utils.logger import setup_logger
 from onyx.utils.variable_functionality import global_version
@@ -70,7 +69,6 @@ def check_for_indexing(self: Task, *, tenant_id: str | None) -> int | None:
     tasks_created = 0
     locked = False
     redis_client = get_redis_client(tenant_id=tenant_id)
-    redis_client_replica = get_redis_replica_client(tenant_id=tenant_id)
 
     # we need to use celery's redis client to access its redis data
     # (which lives on a different db number)
@@ -229,7 +227,7 @@ def check_for_indexing(self: Task, *, tenant_id: str | None) -> int | None:
             # or be currently executing
             try:
                 validate_indexing_fences(
-                    tenant_id, redis_client_replica, redis_client_celery, lock_beat
+                    tenant_id, self.app, redis_client, redis_client_celery, lock_beat
                 )
             except Exception:
                 task_logger.exception("Exception while validating indexing fences")

backend/onyx/background/celery/tasks/indexing/utils.py

@@ -291,20 +291,17 @@ def validate_indexing_fence(
 
 def validate_indexing_fences(
     tenant_id: str | None,
-    r_replica: Redis,
+    celery_app: Celery,
+    r: Redis,
     r_celery: Redis,
     lock_beat: RedisLock,
 ) -> None:
-    """Validates all indexing fences for this tenant ... aka makes sure
-    indexing tasks sent to celery are still in flight.
-    """
     reserved_indexing_tasks = celery_get_unacked_task_ids(
         OnyxCeleryQueues.CONNECTOR_INDEXING, r_celery
     )
 
-    # Use replica for this because the worst thing that happens
-    # is that we don't run the validation on this pass
-    for key_bytes in r_replica.scan_iter(
+    # validate all existing indexing jobs
+    for key_bytes in r.scan_iter(
         RedisConnectorIndex.FENCE_PREFIX + "*", count=SCAN_ITER_COUNT_DEFAULT
     ):
         lock_beat.reacquire()

backend/onyx/background/celery/tasks/llm_model_update/tasks.py

@@ -54,7 +54,6 @@ def _process_model_list_response(model_list_json: Any) -> list[str]:
 
 @shared_task(
     name=OnyxCeleryTask.CHECK_FOR_LLM_MODEL_UPDATE,
-    ignore_result=True,
     soft_time_limit=JOB_TIMEOUT,
     trail=False,
     bind=True,

backend/onyx/background/celery/tasks/monitoring/tasks.py

@@ -4,7 +4,6 @@ from collections.abc import Callable
 from datetime import timedelta
 from itertools import islice
 from typing import Any
-from typing import Literal
 
 from celery import shared_task
 from celery import Task
@@ -27,7 +26,6 @@ from onyx.db.engine import get_all_tenant_ids
 from onyx.db.engine import get_db_current_time
 from onyx.db.engine import get_session_with_tenant
 from onyx.db.enums import IndexingStatus
-from onyx.db.enums import SyncStatus
 from onyx.db.enums import SyncType
 from onyx.db.models import ConnectorCredentialPair
 from onyx.db.models import DocumentSet
@@ -40,7 +38,6 @@ from onyx.redis.redis_pool import redis_lock_dump
 from onyx.utils.telemetry import optional_telemetry
 from onyx.utils.telemetry import RecordType
 
-
 _MONITORING_SOFT_TIME_LIMIT = 60 * 5  # 5 minutes
 _MONITORING_TIME_LIMIT = _MONITORING_SOFT_TIME_LIMIT + 60  # 6 minutes
 
@@ -52,12 +49,6 @@ _CONNECTOR_INDEX_ATTEMPT_RUN_SUCCESS_KEY_FMT = (
     "monitoring_connector_index_attempt_run_success:{cc_pair_id}:{index_attempt_id}"
 )
 
-_FINAL_METRIC_KEY_FMT = "sync_final_metrics:{sync_type}:{entity_id}:{sync_record_id}"
-
-_SYNC_START_LATENCY_KEY_FMT = (
-    "sync_start_latency:{sync_type}:{entity_id}:{sync_record_id}"
-)
-
 
 def _mark_metric_as_emitted(redis_std: Redis, key: str) -> None:
     """Mark a metric as having been emitted by setting a Redis key with expiration"""
@@ -120,7 +111,6 @@ class Metric(BaseModel):
             }.items()
             if v is not None
         }
-        task_logger.info(f"Emitting metric: {data}")
         optional_telemetry(
             record_type=RecordType.METRIC,
             data=data,
@@ -199,371 +189,239 @@ def _build_connector_start_latency_metric(
         f"Start latency for index attempt {recent_attempt.id}: {start_latency:.2f}s "
         f"(desired: {desired_start_time}, actual: {recent_attempt.time_started})"
     )
-
-    job_id = build_job_id("connector", str(cc_pair.id), str(recent_attempt.id))
-
     return Metric(
         key=metric_key,
         name="connector_start_latency",
         value=start_latency,
-        tags={
-            "job_id": job_id,
-            "connector_id": str(cc_pair.connector.id),
-            "source": str(cc_pair.connector.source),
-        },
+        tags={},
     )
 
 
-def _build_connector_final_metrics(
+def _build_run_success_metrics(
     cc_pair: ConnectorCredentialPair,
     recent_attempts: list[IndexAttempt],
     redis_std: Redis,
 ) -> list[Metric]:
-    """
-    Final metrics for connector index attempts:
-      - Boolean success/fail metric
-      - If success, emit:
-          * duration (seconds)
-          * doc_count
-    """
     metrics = []
     for attempt in recent_attempts:
         metric_key = _CONNECTOR_INDEX_ATTEMPT_RUN_SUCCESS_KEY_FMT.format(
             cc_pair_id=cc_pair.id,
             index_attempt_id=attempt.id,
         )
+
         if _has_metric_been_emitted(redis_std, metric_key):
             task_logger.info(
-                f"Skipping final metrics for connector {cc_pair.connector.id} "
-                f"index attempt {attempt.id}, already emitted."
+                f"Skipping metric for connector {cc_pair.connector.id} "
+                f"index attempt {attempt.id} because it has already been "
+                "emitted"
             )
             continue
 
-        # We only emit final metrics if the attempt is in a terminal state
-        if attempt.status not in [
+        if attempt.status in [
             IndexingStatus.SUCCESS,
             IndexingStatus.FAILED,
             IndexingStatus.CANCELED,
         ]:
-            # Not finished; skip
-            continue
-
-        job_id = build_job_id("connector", str(cc_pair.id), str(attempt.id))
-        success = attempt.status == IndexingStatus.SUCCESS
-        metrics.append(
-            Metric(
-                key=metric_key,  # We'll mark the same key for any final metrics
-                name="connector_run_succeeded",
-                value=success,
-                tags={
-                    "job_id": job_id,
-                    "connector_id": str(cc_pair.connector.id),
-                    "source": str(cc_pair.connector.source),
-                    "status": attempt.status.value,
-                },
+            task_logger.info(
+                f"Adding run success metric for index attempt {attempt.id} with status {attempt.status}"
             )
-        )
-
-        if success:
-            # Make sure we have valid time_started
-            if attempt.time_started and attempt.time_updated:
-                duration_seconds = (
-                    attempt.time_updated - attempt.time_started
-                ).total_seconds()
-                metrics.append(
-                    Metric(
-                        key=None,  # No need for a new key, or you can reuse the same if you prefer
-                        name="connector_index_duration_seconds",
-                        value=duration_seconds,
-                        tags={
-                            "job_id": job_id,
-                            "connector_id": str(cc_pair.connector.id),
-                            "source": str(cc_pair.connector.source),
-                        },
-                    )
-                )
-            else:
-                task_logger.error(
-                    f"Index attempt {attempt.id} succeeded but has missing time "
-                    f"(time_started={attempt.time_started}, time_updated={attempt.time_updated})."
-                )
-
-            # For doc counts, choose whichever field is more relevant
-            doc_count = attempt.total_docs_indexed or 0
             metrics.append(
                 Metric(
-                    key=None,
-                    name="connector_index_doc_count",
-                    value=doc_count,
-                    tags={
-                        "job_id": job_id,
-                        "connector_id": str(cc_pair.connector.id),
-                        "source": str(cc_pair.connector.source),
-                    },
+                    key=metric_key,
+                    name="connector_run_succeeded",
+                    value=attempt.status == IndexingStatus.SUCCESS,
+                    tags={"source": str(cc_pair.connector.source)},
                 )
             )
 
-        _mark_metric_as_emitted(redis_std, metric_key)
-
     return metrics
 
 
 def _collect_connector_metrics(db_session: Session, redis_std: Redis) -> list[Metric]:
     """Collect metrics about connector runs from the past hour"""
+    # NOTE: use get_db_current_time since the IndexAttempt times are set based on DB time
     one_hour_ago = get_db_current_time(db_session) - timedelta(hours=1)
 
     # Get all connector credential pairs
     cc_pairs = db_session.scalars(select(ConnectorCredentialPair)).all()
-    # Might be more than one search setting, or just one
-    active_search_settings = get_active_search_settings(db_session)
 
+    active_search_settings = get_active_search_settings(db_session)
     metrics = []
 
-    # If you want to process each cc_pair against each search setting:
-    for cc_pair in cc_pairs:
-        for search_settings in active_search_settings:
-            recent_attempts = (
-                db_session.query(IndexAttempt)
-                .filter(
-                    IndexAttempt.connector_credential_pair_id == cc_pair.id,
-                    IndexAttempt.search_settings_id == search_settings.id,
-                )
-                .order_by(IndexAttempt.time_created.desc())
-                .limit(2)
-                .all()
+    for cc_pair, search_settings in zip(cc_pairs, active_search_settings):
+        recent_attempts = (
+            db_session.query(IndexAttempt)
+            .filter(
+                IndexAttempt.connector_credential_pair_id == cc_pair.id,
+                IndexAttempt.search_settings_id == search_settings.id,
             )
+            .order_by(IndexAttempt.time_created.desc())
+            .limit(2)
+            .all()
+        )
+        if not recent_attempts:
+            continue
 
-            if not recent_attempts:
-                continue
+        most_recent_attempt = recent_attempts[0]
+        second_most_recent_attempt = (
+            recent_attempts[1] if len(recent_attempts) > 1 else None
+        )
 
-            most_recent_attempt = recent_attempts[0]
-            second_most_recent_attempt = (
-                recent_attempts[1] if len(recent_attempts) > 1 else None
-            )
+        if one_hour_ago > most_recent_attempt.time_created:
+            continue
 
-            if one_hour_ago > most_recent_attempt.time_created:
-                continue
+        # Connector start latency
+        start_latency_metric = _build_connector_start_latency_metric(
+            cc_pair, most_recent_attempt, second_most_recent_attempt, redis_std
+        )
+        if start_latency_metric:
+            metrics.append(start_latency_metric)
 
-            # Connector start latency
-            start_latency_metric = _build_connector_start_latency_metric(
-                cc_pair, most_recent_attempt, second_most_recent_attempt, redis_std
-            )
-
-            if start_latency_metric:
-                metrics.append(start_latency_metric)
-
-            # Connector run success/failure
-            final_metrics = _build_connector_final_metrics(
-                cc_pair, recent_attempts, redis_std
-            )
-            metrics.extend(final_metrics)
+        # Connector run success/failure
+        run_success_metrics = _build_run_success_metrics(
+            cc_pair, recent_attempts, redis_std
+        )
+        metrics.extend(run_success_metrics)
 
     return metrics
 
 
 def _collect_sync_metrics(db_session: Session, redis_std: Redis) -> list[Metric]:
-    """
-    Collect metrics for document set and group syncing:
-      - Success/failure status
-      - Start latency (always)
-      - Duration & doc count (only if success)
-      - Throughput (docs/min) (only if success)
-    """
+    """Collect metrics about document set and group syncing speed"""
+    # NOTE: use get_db_current_time since the SyncRecord times are set based on DB time
     one_hour_ago = get_db_current_time(db_session) - timedelta(hours=1)
 
-    # Get all sync records that ended in the last hour
+    # Get all sync records from the last hour
     recent_sync_records = db_session.scalars(
         select(SyncRecord)
-        .where(SyncRecord.sync_end_time.isnot(None))
-        .where(SyncRecord.sync_end_time >= one_hour_ago)
-        .order_by(SyncRecord.sync_end_time.desc())
+        .where(SyncRecord.sync_start_time >= one_hour_ago)
+        .order_by(SyncRecord.sync_start_time.desc())
     ).all()
 
-    task_logger.info(
-        f"Collecting sync metrics for {len(recent_sync_records)} sync records"
-    )
-
     metrics = []
-
     for sync_record in recent_sync_records:
-        # Build a job_id for correlation
-        job_id = build_job_id("sync_record", str(sync_record.id))
+        # Skip if no end time (sync still in progress)
+        if not sync_record.sync_end_time:
+            continue
 
-        # Emit a SUCCESS/FAIL boolean metric
-        #    Use a single Redis key to avoid re-emitting final metrics
-        final_metric_key = _FINAL_METRIC_KEY_FMT.format(
-            sync_type=sync_record.sync_type,
-            entity_id=sync_record.entity_id,
-            sync_record_id=sync_record.id,
+        # Check if we already emitted a metric for this sync record
+        metric_key = (
+            f"sync_speed:{sync_record.sync_type}:"
+            f"{sync_record.entity_id}:{sync_record.id}"
         )
-        if not _has_metric_been_emitted(redis_std, final_metric_key):
-            # Evaluate success
-            sync_succeeded = sync_record.sync_status == SyncStatus.SUCCESS
-
-            metrics.append(
-                Metric(
-                    key=final_metric_key,
-                    name="sync_run_succeeded",
-                    value=sync_succeeded,
-                    tags={
-                        "job_id": job_id,
-                        "sync_type": str(sync_record.sync_type),
-                        "status": str(sync_record.sync_status),
-                    },
-                )
+        if _has_metric_been_emitted(redis_std, metric_key):
+            task_logger.info(
+                f"Skipping metric for sync record {sync_record.id} "
+                "because it has already been emitted"
             )
+            continue
 
-            # If successful, emit additional metrics
-            if sync_succeeded:
-                if sync_record.sync_end_time and sync_record.sync_start_time:
-                    duration_seconds = (
-                        sync_record.sync_end_time - sync_record.sync_start_time
-                    ).total_seconds()
-                else:
-                    task_logger.error(
-                        f"Invalid times for sync record {sync_record.id}: "
-                        f"start={sync_record.sync_start_time}, end={sync_record.sync_end_time}"
-                    )
-                    duration_seconds = None
+        # Calculate sync duration in minutes
+        sync_duration_mins = (
+            sync_record.sync_end_time - sync_record.sync_start_time
+        ).total_seconds() / 60.0
 
-                doc_count = sync_record.num_docs_synced or 0
-
-                sync_speed = None
-                if duration_seconds and duration_seconds > 0:
-                    duration_mins = duration_seconds / 60.0
-                    sync_speed = (
-                        doc_count / duration_mins if duration_mins > 0 else None
-                    )
-
-                # Emit duration, doc count, speed
-                if duration_seconds is not None:
-                    metrics.append(
-                        Metric(
-                            key=None,
-                            name="sync_duration_seconds",
-                            value=duration_seconds,
-                            tags={
-                                "job_id": job_id,
-                                "sync_type": str(sync_record.sync_type),
-                            },
-                        )
-                    )
-                else:
-                    task_logger.error(
-                        f"Invalid sync record {sync_record.id} with no duration"
-                    )
-
-                metrics.append(
-                    Metric(
-                        key=None,
-                        name="sync_doc_count",
-                        value=doc_count,
-                        tags={
-                            "job_id": job_id,
-                            "sync_type": str(sync_record.sync_type),
-                        },
-                    )
-                )
-
-                if sync_speed is not None:
-                    metrics.append(
-                        Metric(
-                            key=None,
-                            name="sync_speed_docs_per_min",
-                            value=sync_speed,
-                            tags={
-                                "job_id": job_id,
-                                "sync_type": str(sync_record.sync_type),
-                            },
-                        )
-                    )
-                else:
-                    task_logger.error(
-                        f"Invalid sync record {sync_record.id} with no duration"
-                    )
-
-            # Mark final metrics as emitted so we don't re-emit
-            _mark_metric_as_emitted(redis_std, final_metric_key)
-
-        # Emit start latency
-        start_latency_key = _SYNC_START_LATENCY_KEY_FMT.format(
-            sync_type=sync_record.sync_type,
-            entity_id=sync_record.entity_id,
-            sync_record_id=sync_record.id,
+        # Calculate sync speed (docs/min) - avoid division by zero
+        sync_speed = (
+            sync_record.num_docs_synced / sync_duration_mins
+            if sync_duration_mins > 0
+            else None
         )
-        if not _has_metric_been_emitted(redis_std, start_latency_key):
-            # Get the entity's last update time based on sync type
-            entity: DocumentSet | UserGroup | None = None
-            if sync_record.sync_type == SyncType.DOCUMENT_SET:
-                entity = db_session.scalar(
-                    select(DocumentSet).where(DocumentSet.id == sync_record.entity_id)
-                )
-            elif sync_record.sync_type == SyncType.USER_GROUP:
-                entity = db_session.scalar(
-                    select(UserGroup).where(UserGroup.id == sync_record.entity_id)
-                )
-            else:
-                task_logger.info(
-                    f"Skipping sync record {sync_record.id} of type {sync_record.sync_type}."
-                )
-                continue
 
-            if entity is None:
-                task_logger.error(
-                    f"Could not find entity for sync record {sync_record.id} "
-                    f"(type={sync_record.sync_type}, id={sync_record.entity_id})."
-                )
-                continue
+        if sync_speed is None:
+            task_logger.error(
+                f"Something went wrong with sync speed calculation. "
+                f"Sync record: {sync_record.id}, duration: {sync_duration_mins}, "
+                f"docs synced: {sync_record.num_docs_synced}"
+            )
+            continue
 
-            # Calculate start latency in seconds:
-            #    (actual sync start) - (last modified time)
-            if entity.time_last_modified_by_user and sync_record.sync_start_time:
-                start_latency = (
-                    sync_record.sync_start_time - entity.time_last_modified_by_user
-                ).total_seconds()
+        task_logger.info(
+            f"Calculated sync speed for record {sync_record.id}: {sync_speed} docs/min"
+        )
+        metrics.append(
+            Metric(
+                key=metric_key,
+                name="sync_speed_docs_per_min",
+                value=sync_speed,
+                tags={
+                    "sync_type": str(sync_record.sync_type),
+                    "status": str(sync_record.sync_status),
+                },
+            )
+        )
 
-                if start_latency < 0:
-                    task_logger.error(
-                        f"Negative start latency for sync record {sync_record.id} "
-                        f"(start={sync_record.sync_start_time}, entity_modified={entity.time_last_modified_by_user})"
-                    )
-                    continue
+        # Add sync start latency metric
+        start_latency_key = (
+            f"sync_start_latency:{sync_record.sync_type}"
+            f":{sync_record.entity_id}:{sync_record.id}"
+        )
+        if _has_metric_been_emitted(redis_std, start_latency_key):
+            task_logger.info(
+                f"Skipping start latency metric for sync record {sync_record.id} "
+                "because it has already been emitted"
+            )
+            continue
 
-                metrics.append(
-                    Metric(
-                        key=start_latency_key,
-                        name="sync_start_latency_seconds",
-                        value=start_latency,
-                        tags={
-                            "job_id": job_id,
-                            "sync_type": str(sync_record.sync_type),
-                        },
-                    )
-                )
+        # Get the entity's last update time based on sync type
+        entity: DocumentSet | UserGroup | None = None
+        if sync_record.sync_type == SyncType.DOCUMENT_SET:
+            entity = db_session.scalar(
+                select(DocumentSet).where(DocumentSet.id == sync_record.entity_id)
+            )
+        elif sync_record.sync_type == SyncType.USER_GROUP:
+            entity = db_session.scalar(
+                select(UserGroup).where(UserGroup.id == sync_record.entity_id)
+            )
+        else:
+            # Skip other sync types
+            task_logger.info(
+                f"Skipping sync record {sync_record.id} "
+                f"with type {sync_record.sync_type} "
+                f"and id {sync_record.entity_id} "
+                "because it is not a document set or user group"
+            )
+            continue
 
-                _mark_metric_as_emitted(redis_std, start_latency_key)
+        if entity is None:
+            task_logger.error(
+                f"Could not find entity for sync record {sync_record.id} "
+                f"with type {sync_record.sync_type} and id {sync_record.entity_id}"
+            )
+            continue
+
+        # Calculate start latency in seconds
+        start_latency = (
+            sync_record.sync_start_time - entity.time_last_modified_by_user
+        ).total_seconds()
+        task_logger.info(
+            f"Calculated start latency for sync record {sync_record.id}: {start_latency} seconds"
+        )
+        if start_latency < 0:
+            task_logger.error(
+                f"Start latency is negative for sync record {sync_record.id} "
+                f"with type {sync_record.sync_type} and id {sync_record.entity_id}. "
+                f"Sync start time: {sync_record.sync_start_time}, "
+                f"Entity last modified: {entity.time_last_modified_by_user}"
+            )
+            continue
+
+        metrics.append(
+            Metric(
+                key=start_latency_key,
+                name="sync_start_latency_seconds",
+                value=start_latency,
+                tags={
+                    "sync_type": str(sync_record.sync_type),
+                },
+            )
+        )
 
     return metrics
 
 
-def build_job_id(
-    job_type: Literal["connector", "sync_record"],
-    primary_id: str,
-    secondary_id: str | None = None,
-) -> str:
-    if job_type == "connector":
-        if secondary_id is None:
-            raise ValueError(
-                "secondary_id (attempt_id) is required for connector job_type"
-            )
-        return f"connector:{primary_id}:attempt:{secondary_id}"
-    elif job_type == "sync_record":
-        return f"sync_record:{primary_id}"
-
-
 @shared_task(
     name=OnyxCeleryTask.MONITOR_BACKGROUND_PROCESSES,
-    ignore_result=True,
     soft_time_limit=_MONITORING_SOFT_TIME_LIMIT,
     time_limit=_MONITORING_TIME_LIMIT,
     queue=OnyxCeleryQueues.MONITORING,
@@ -601,7 +459,6 @@ def monitor_background_processes(self: Task, *, tenant_id: str | None) -> None:
             lambda: _collect_connector_metrics(db_session, redis_std),
             lambda: _collect_sync_metrics(db_session, redis_std),
         ]
-
         # Collect and log each metric
         with get_session_with_tenant(tenant_id) as db_session:
             for metric_fn in metric_functions:

backend/onyx/background/celery/tasks/pruning/tasks.py

@@ -78,7 +78,6 @@ def _is_pruning_due(cc_pair: ConnectorCredentialPair) -> bool:
 
 @shared_task(
     name=OnyxCeleryTask.CHECK_FOR_PRUNING,
-    ignore_result=True,
     soft_time_limit=JOB_TIMEOUT,
     bind=True,
 )

backend/onyx/background/celery/tasks/shared/tasks.py

@@ -33,7 +33,6 @@ from onyx.document_index.interfaces import VespaDocumentFields
 from onyx.redis.redis_pool import get_redis_client
 from onyx.redis.redis_pool import redis_lock_dump
 from onyx.server.documents.models import ConnectorCredentialPairIdentifier
-from shared_configs.configs import IGNORED_SYNCING_TENANT_LIST
 
 DOCUMENT_BY_CC_PAIR_CLEANUP_MAX_RETRIES = 3
 
@@ -214,7 +213,6 @@ def document_by_cc_pair_cleanup_task(
 
 @shared_task(
     name=OnyxCeleryTask.CLOUD_BEAT_TASK_GENERATOR,
-    ignore_result=True,
     trail=False,
     bind=True,
 )
@@ -249,10 +247,6 @@ def cloud_beat_task_generator(
                 lock_beat.reacquire()
                 last_lock_time = current_time
 
-            # needed in the cloud
-            if IGNORED_SYNCING_TENANT_LIST and tenant_id in IGNORED_SYNCING_TENANT_LIST:
-                continue
-
             self.app.send_task(
                 task_name,
                 kwargs=dict(

backend/onyx/background/celery/tasks/vespa/tasks.py

@@ -78,7 +78,6 @@ from onyx.redis.redis_connector_index import RedisConnectorIndex
 from onyx.redis.redis_connector_prune import RedisConnectorPrune
 from onyx.redis.redis_document_set import RedisDocumentSet
 from onyx.redis.redis_pool import get_redis_client
-from onyx.redis.redis_pool import get_redis_replica_client
 from onyx.redis.redis_pool import redis_lock_dump
 from onyx.redis.redis_pool import SCAN_ITER_COUNT_DEFAULT
 from onyx.redis.redis_usergroup import RedisUserGroup
@@ -98,7 +97,6 @@ logger = setup_logger()
 # which bloats the result metadata considerably. trail=False prevents this.
 @shared_task(
     name=OnyxCeleryTask.CHECK_FOR_VESPA_SYNC_TASK,
-    ignore_result=True,
     soft_time_limit=JOB_TIMEOUT,
     trail=False,
     bind=True,
@@ -873,12 +871,7 @@ def monitor_ccpair_indexing_taskset(
     redis_connector_index.reset()
 
 
-@shared_task(
-    name=OnyxCeleryTask.MONITOR_VESPA_SYNC,
-    ignore_result=True,
-    soft_time_limit=300,
-    bind=True,
-)
+@shared_task(name=OnyxCeleryTask.MONITOR_VESPA_SYNC, soft_time_limit=300, bind=True)
 def monitor_vespa_sync(self: Task, tenant_id: str | None) -> bool | None:
     """This is a celery beat task that monitors and finalizes various long running tasks.
 
@@ -902,17 +895,6 @@ def monitor_vespa_sync(self: Task, tenant_id: str | None) -> bool | None:
 
     r = get_redis_client(tenant_id=tenant_id)
 
-    # Replica usage notes
-    #
-    # False negatives are OK. (aka fail to to see a key that exists on the master).
-    # We simply skip the monitoring work and it will be caught on the next pass.
-    #
-    # False positives are not OK, and are possible if we clear a fence on the master and
-    # then read from the replica. In this case, monitoring work could be done on a fence
-    # that no longer exists. To avoid this, we scan from the replica, but double check
-    # the result on the master.
-    r_replica = get_redis_replica_client(tenant_id=tenant_id)
-
     lock_beat: RedisLock = r.lock(
         OnyxRedisLocks.MONITOR_VESPA_SYNC_BEAT_LOCK,
         timeout=CELERY_VESPA_SYNC_BEAT_LOCK_TIMEOUT,
@@ -972,19 +954,17 @@ def monitor_vespa_sync(self: Task, tenant_id: str | None) -> bool | None:
         # scan and monitor activity to completion
         phase_start = time.monotonic()
         lock_beat.reacquire()
-        if r_replica.exists(RedisConnectorCredentialPair.get_fence_key()):
-            if r.exists(RedisConnectorCredentialPair.get_fence_key()):
-                monitor_connector_taskset(r)
+        if r.exists(RedisConnectorCredentialPair.get_fence_key()):
+            monitor_connector_taskset(r)
         timings["connector"] = time.monotonic() - phase_start
         timings["connector_ttl"] = r.ttl(OnyxRedisLocks.MONITOR_VESPA_SYNC_BEAT_LOCK)
 
         phase_start = time.monotonic()
         lock_beat.reacquire()
-        for key_bytes in r_replica.scan_iter(
+        for key_bytes in r.scan_iter(
             RedisConnectorDelete.FENCE_PREFIX + "*", count=SCAN_ITER_COUNT_DEFAULT
         ):
-            if r.exists(key_bytes):
-                monitor_connector_deletion_taskset(tenant_id, key_bytes, r)
+            monitor_connector_deletion_taskset(tenant_id, key_bytes, r)
             lock_beat.reacquire()
 
         timings["connector_deletion"] = time.monotonic() - phase_start
@@ -994,74 +974,66 @@ def monitor_vespa_sync(self: Task, tenant_id: str | None) -> bool | None:
 
         phase_start = time.monotonic()
         lock_beat.reacquire()
-        for key_bytes in r_replica.scan_iter(
+        for key_bytes in r.scan_iter(
             RedisDocumentSet.FENCE_PREFIX + "*", count=SCAN_ITER_COUNT_DEFAULT
         ):
-            if r.exists(key_bytes):
-                with get_session_with_tenant(tenant_id) as db_session:
-                    monitor_document_set_taskset(tenant_id, key_bytes, r, db_session)
+            with get_session_with_tenant(tenant_id) as db_session:
+                monitor_document_set_taskset(tenant_id, key_bytes, r, db_session)
             lock_beat.reacquire()
         timings["documentset"] = time.monotonic() - phase_start
         timings["documentset_ttl"] = r.ttl(OnyxRedisLocks.MONITOR_VESPA_SYNC_BEAT_LOCK)
 
         phase_start = time.monotonic()
         lock_beat.reacquire()
-        for key_bytes in r_replica.scan_iter(
+        for key_bytes in r.scan_iter(
             RedisUserGroup.FENCE_PREFIX + "*", count=SCAN_ITER_COUNT_DEFAULT
         ):
-            if r.exists(key_bytes):
-                monitor_usergroup_taskset = (
-                    fetch_versioned_implementation_with_fallback(
-                        "onyx.background.celery.tasks.vespa.tasks",
-                        "monitor_usergroup_taskset",
-                        noop_fallback,
-                    )
-                )
-                with get_session_with_tenant(tenant_id) as db_session:
-                    monitor_usergroup_taskset(tenant_id, key_bytes, r, db_session)
+            monitor_usergroup_taskset = fetch_versioned_implementation_with_fallback(
+                "onyx.background.celery.tasks.vespa.tasks",
+                "monitor_usergroup_taskset",
+                noop_fallback,
+            )
+            with get_session_with_tenant(tenant_id) as db_session:
+                monitor_usergroup_taskset(tenant_id, key_bytes, r, db_session)
             lock_beat.reacquire()
         timings["usergroup"] = time.monotonic() - phase_start
         timings["usergroup_ttl"] = r.ttl(OnyxRedisLocks.MONITOR_VESPA_SYNC_BEAT_LOCK)
 
         phase_start = time.monotonic()
         lock_beat.reacquire()
-        for key_bytes in r_replica.scan_iter(
+        for key_bytes in r.scan_iter(
             RedisConnectorPrune.FENCE_PREFIX + "*", count=SCAN_ITER_COUNT_DEFAULT
         ):
-            if r.exists(key_bytes):
-                with get_session_with_tenant(tenant_id) as db_session:
-                    monitor_ccpair_pruning_taskset(tenant_id, key_bytes, r, db_session)
+            with get_session_with_tenant(tenant_id) as db_session:
+                monitor_ccpair_pruning_taskset(tenant_id, key_bytes, r, db_session)
             lock_beat.reacquire()
         timings["pruning"] = time.monotonic() - phase_start
         timings["pruning_ttl"] = r.ttl(OnyxRedisLocks.MONITOR_VESPA_SYNC_BEAT_LOCK)
 
         phase_start = time.monotonic()
         lock_beat.reacquire()
-        for key_bytes in r_replica.scan_iter(
+        for key_bytes in r.scan_iter(
             RedisConnectorIndex.FENCE_PREFIX + "*", count=SCAN_ITER_COUNT_DEFAULT
         ):
-            if r.exists(key_bytes):
-                with get_session_with_tenant(tenant_id) as db_session:
-                    monitor_ccpair_indexing_taskset(tenant_id, key_bytes, r, db_session)
+            with get_session_with_tenant(tenant_id) as db_session:
+                monitor_ccpair_indexing_taskset(tenant_id, key_bytes, r, db_session)
             lock_beat.reacquire()
         timings["indexing"] = time.monotonic() - phase_start
         timings["indexing_ttl"] = r.ttl(OnyxRedisLocks.MONITOR_VESPA_SYNC_BEAT_LOCK)
 
         phase_start = time.monotonic()
         lock_beat.reacquire()
-        for key_bytes in r_replica.scan_iter(
+        for key_bytes in r.scan_iter(
             RedisConnectorPermissionSync.FENCE_PREFIX + "*",
             count=SCAN_ITER_COUNT_DEFAULT,
         ):
-            if r.exists(key_bytes):
-                with get_session_with_tenant(tenant_id) as db_session:
-                    monitor_ccpair_permissions_taskset(
-                        tenant_id, key_bytes, r, db_session
-                    )
+            with get_session_with_tenant(tenant_id) as db_session:
+                monitor_ccpair_permissions_taskset(tenant_id, key_bytes, r, db_session)
             lock_beat.reacquire()
 
         timings["permissions"] = time.monotonic() - phase_start
         timings["permissions_ttl"] = r.ttl(OnyxRedisLocks.MONITOR_VESPA_SYNC_BEAT_LOCK)
+
     except SoftTimeLimitExceeded:
         task_logger.info(
             "Soft time limit exceeded, task is being terminated gracefully."

backend/onyx/configs/app_configs.py

@@ -200,8 +200,6 @@ REDIS_HOST = os.environ.get("REDIS_HOST") or "localhost"
 REDIS_PORT = int(os.environ.get("REDIS_PORT", 6379))
 REDIS_PASSWORD = os.environ.get("REDIS_PASSWORD") or ""
 
-# this assumes that other redis settings remain the same as the primary
-REDIS_REPLICA_HOST = os.environ.get("REDIS_REPLICA_HOST") or REDIS_HOST
 
 REDIS_AUTH_KEY_PREFIX = "fastapi_users_token:"
 

backend/onyx/connectors/confluence/connector.py

@@ -232,29 +232,20 @@ class ConfluenceConnector(LoadConnector, PollConnector, SlimConnector):
         }
 
         # Get labels
-        label_dicts = (
-            confluence_object.get("metadata", {}).get("labels", {}).get("results", [])
-        )
-        page_labels = [label.get("name") for label in label_dicts if label.get("name")]
+        label_dicts = confluence_object["metadata"]["labels"]["results"]
+        page_labels = [label["name"] for label in label_dicts]
         if page_labels:
             doc_metadata["labels"] = page_labels
 
         # Get last modified and author email
-        version_dict = confluence_object.get("version", {})
-        last_modified = (
-            datetime_from_string(version_dict.get("when"))
-            if version_dict.get("when")
-            else None
-        )
-        author_email = version_dict.get("by", {}).get("email")
-
-        title = confluence_object.get("title", "Untitled Document")
+        last_modified = datetime_from_string(confluence_object["version"]["when"])
+        author_email = confluence_object["version"].get("by", {}).get("email")
 
         return Document(
             id=object_url,
             sections=[Section(link=object_url, text=object_text)],
             source=DocumentSource.CONFLUENCE,
-            semantic_identifier=title,
+            semantic_identifier=confluence_object["title"],
             doc_updated_at=last_modified,
             primary_owners=(
                 [BasicExpertInfo(email=author_email)] if author_email else None

backend/onyx/connectors/sharepoint/connector.py

@@ -6,7 +6,6 @@ from datetime import datetime
 from datetime import timezone
 from typing import Any
 from typing import Optional
-from urllib.parse import unquote
 
 import msal  # type: ignore
 from office365.graph_client import GraphClient  # type: ignore
@@ -83,13 +82,8 @@ class SharepointConnector(LoadConnector, PollConnector):
                 sites_index = parts.index("sites")
                 site_url = "/".join(parts[: sites_index + 2])
                 folder = (
-                    "/".join(unquote(part) for part in parts[sites_index + 2 :])
-                    if len(parts) > sites_index + 2
-                    else None
+                    parts[sites_index + 2] if len(parts) > sites_index + 2 else None
                 )
-                # Handling for new URL structure
-                if folder and folder.startswith("Shared Documents/"):
-                    folder = folder[len("Shared Documents/") :]
                 site_data_list.append(
                     SiteData(url=site_url, folder=folder, sites=[], driveitems=[])
                 )
@@ -117,19 +111,11 @@ class SharepointConnector(LoadConnector, PollConnector):
                         query = query.filter(filter_str)
                     driveitems = query.execute_query()
                     if element.folder:
-                        expected_path = f"/root:/{element.folder}"
                         filtered_driveitems = [
                             item
                             for item in driveitems
-                            if item.parent_reference.path.endswith(expected_path)
+                            if element.folder in item.parent_reference.path
                         ]
-                        if len(filtered_driveitems) == 0:
-                            all_paths = [
-                                item.parent_reference.path for item in driveitems
-                            ]
-                            logger.warning(
-                                f"Nothing found for folder '{expected_path}' in any of valid paths: {all_paths}"
-                            )
                         element.driveitems.extend(filtered_driveitems)
                     else:
                         element.driveitems.extend(driveitems)

backend/onyx/db/sync_record.py

@@ -8,64 +8,20 @@ from sqlalchemy.orm import Session
 from onyx.db.enums import SyncStatus
 from onyx.db.enums import SyncType
 from onyx.db.models import SyncRecord
-from onyx.setup import setup_logger
-
-logger = setup_logger()
 
 
 def insert_sync_record(
     db_session: Session,
-    entity_id: int,
+    entity_id: int | None,
     sync_type: SyncType,
 ) -> SyncRecord:
-    """Insert a new sync record into the database, cancelling any existing in-progress records.
+    """Insert a new sync record into the database.
 
     Args:
         db_session: The database session to use
         entity_id: The ID of the entity being synced (document set ID, user group ID, etc.)
         sync_type: The type of sync operation
     """
-    # If an existing in-progress sync record exists, mark as cancelled
-    existing_in_progress_sync_record = fetch_latest_sync_record(
-        db_session, entity_id, sync_type, sync_status=SyncStatus.IN_PROGRESS
-    )
-
-    if existing_in_progress_sync_record is not None:
-        logger.info(
-            f"Cancelling existing in-progress sync record {existing_in_progress_sync_record.id} "
-            f"for entity_id={entity_id} sync_type={sync_type}"
-        )
-        mark_sync_records_as_cancelled(db_session, entity_id, sync_type)
-
-    return _create_sync_record(db_session, entity_id, sync_type)
-
-
-def mark_sync_records_as_cancelled(
-    db_session: Session,
-    entity_id: int | None,
-    sync_type: SyncType,
-) -> None:
-    stmt = (
-        update(SyncRecord)
-        .where(
-            and_(
-                SyncRecord.entity_id == entity_id,
-                SyncRecord.sync_type == sync_type,
-                SyncRecord.sync_status == SyncStatus.IN_PROGRESS,
-            )
-        )
-        .values(sync_status=SyncStatus.CANCELED)
-    )
-    db_session.execute(stmt)
-    db_session.commit()
-
-
-def _create_sync_record(
-    db_session: Session,
-    entity_id: int | None,
-    sync_type: SyncType,
-) -> SyncRecord:
-    """Create and insert a new sync record into the database."""
     sync_record = SyncRecord(
         entity_id=entity_id,
         sync_type=sync_type,
@@ -83,7 +39,6 @@ def fetch_latest_sync_record(
     db_session: Session,
     entity_id: int,
     sync_type: SyncType,
-    sync_status: SyncStatus | None = None,
 ) -> SyncRecord | None:
     """Fetch the most recent sync record for a given entity ID and status.
 
@@ -104,9 +59,6 @@ def fetch_latest_sync_record(
         .limit(1)
     )
 
-    if sync_status is not None:
-        stmt = stmt.where(SyncRecord.sync_status == sync_status)
-
     result = db_session.execute(stmt)
     return result.scalar_one_or_none()
 

backend/onyx/redis/redis_pool.py

@@ -21,7 +21,6 @@ from onyx.configs.app_configs import REDIS_HOST
 from onyx.configs.app_configs import REDIS_PASSWORD
 from onyx.configs.app_configs import REDIS_POOL_MAX_CONNECTIONS
 from onyx.configs.app_configs import REDIS_PORT
-from onyx.configs.app_configs import REDIS_REPLICA_HOST
 from onyx.configs.app_configs import REDIS_SSL
 from onyx.configs.app_configs import REDIS_SSL_CA_CERTS
 from onyx.configs.app_configs import REDIS_SSL_CERT_REQS
@@ -133,32 +132,23 @@ class RedisPool:
     _instance: Optional["RedisPool"] = None
     _lock: threading.Lock = threading.Lock()
     _pool: redis.BlockingConnectionPool
-    _replica_pool: redis.BlockingConnectionPool
 
     def __new__(cls) -> "RedisPool":
         if not cls._instance:
             with cls._lock:
                 if not cls._instance:
                     cls._instance = super(RedisPool, cls).__new__(cls)
-                    cls._instance._init_pools()
+                    cls._instance._init_pool()
         return cls._instance
 
-    def _init_pools(self) -> None:
+    def _init_pool(self) -> None:
         self._pool = RedisPool.create_pool(ssl=REDIS_SSL)
-        self._replica_pool = RedisPool.create_pool(
-            host=REDIS_REPLICA_HOST, ssl=REDIS_SSL
-        )
 
     def get_client(self, tenant_id: str | None) -> Redis:
         if tenant_id is None:
             tenant_id = "public"
         return TenantRedis(tenant_id, connection_pool=self._pool)
 
-    def get_replica_client(self, tenant_id: str | None) -> Redis:
-        if tenant_id is None:
-            tenant_id = "public"
-        return TenantRedis(tenant_id, connection_pool=self._replica_pool)
-
     @staticmethod
     def create_pool(
         host: str = REDIS_HOST,
@@ -222,10 +212,6 @@ def get_redis_client(*, tenant_id: str | None) -> Redis:
     return redis_pool.get_client(tenant_id)
 
 
-def get_redis_replica_client(*, tenant_id: str | None) -> Redis:
-    return redis_pool.get_replica_client(tenant_id)
-
-
 SSL_CERT_REQS_MAP = {
     "none": ssl.CERT_NONE,
     "optional": ssl.CERT_OPTIONAL,

backend/onyx/seeding/initial_docs.json

@@ -6184,7 +6184,7 @@
         "chunk_ind": 0
     },
     {
-        "url": "https://docs.onyx.app/more/use_cases/support",
+        "url": "https://docs.onyx.app/more/use_cases/customer_support",
         "title": "Customer Support",
         "content": "Help your customer support team instantly answer any question across your entire product.\n\nAI Enabled Support\nCustomer support agents have one of the highest breadth jobs. They field requests that cover the entire surface area of the product and need to help your users find success on extremely short timelines. Because they're not the same people who designed or built the system, they often lack the depth of understanding needed - resulting in delays and escalations to other teams. Modern teams are leveraging AI to help their CS team optimize the speed and quality of these critical customer-facing interactions.\n\nThe Importance of Context\nThere are two critical components of AI copilots for customer support. The first is that the AI system needs to be connected with as much information as possible (not just support tools like Zendesk or Intercom) and that the knowledge needs to be as fresh as possible. Sometimes a fix might even be in places rarely checked by CS such as pull requests in a code repository. The second critical component is the ability of the AI system to break down difficult concepts and convoluted processes into more digestible descriptions and for your team members to be able to chat back and forth with the system to build a better understanding.\n\nOnyx takes care of both of these. The system connects up to over 30+ different applications and the knowledge is pulled in constantly so that the information access is always up to date.",
         "title_embedding": [

backend/onyx/seeding/initial_docs_cohere.json

@@ -24,7 +24,7 @@
     "chunk_ind": 0
   },
   {
-    "url": "https://docs.onyx.app/more/use_cases/support",
+    "url": "https://docs.onyx.app/more/use_cases/customer_support",
     "title": "Customer Support",
     "content": "Help your customer support team instantly answer any question across your entire product.\n\nAI Enabled Support\nCustomer support agents have one of the highest breadth jobs. They field requests that cover the entire surface area of the product and need to help your users find success on extremely short timelines. Because they're not the same people who designed or built the system, they often lack the depth of understanding needed - resulting in delays and escalations to other teams. Modern teams are leveraging AI to help their CS team optimize the speed and quality of these critical customer-facing interactions.\n\nThe Importance of Context\nThere are two critical components of AI copilots for customer support. The first is that the AI system needs to be connected with as much information as possible (not just support tools like Zendesk or Intercom) and that the knowledge needs to be as fresh as possible. Sometimes a fix might even be in places rarely checked by CS such as pull requests in a code repository. The second critical component is the ability of the AI system to break down difficult concepts and convoluted processes into more digestible descriptions and for your team members to be able to chat back and forth with the system to build a better understanding.\n\nOnyx takes care of both of these. The system connects up to over 30+ different applications and the knowledge is pulled in constantly so that the information access is always up to date.",
     "chunk_ind": 0

backend/onyx/utils/telemetry.py

@@ -123,7 +123,6 @@ def optional_telemetry(
                     headers={"Content-Type": "application/json"},
                     json=payload,
                 )
-
             except Exception:
                 # This way it silences all thread level logging as well
                 pass

backend/scripts/document_seeding_prep.py

@@ -197,7 +197,7 @@ ai_platform_doc = SeedPresaveDocument(
 )
 
 customer_support_doc = SeedPresaveDocument(
-    url="https://docs.onyx.app/more/use_cases/support",
+    url="https://docs.onyx.app/more/use_cases/customer_support",
     title=customer_support_title,
     content=customer_support,
     title_embedding=model.encode(f"search_document: {customer_support_title}"),

backend/scripts/onyx_vespa.py

@@ -21,144 +21,35 @@ Options:
   --doc-id        : Document ID
   --fields        : Fields to update (JSON)
 
-Example:
+Example: (gets docs for a given tenant id and connector id)
   python vespa_debug_tool.py --action list_docs --tenant-id my_tenant --connector-id 1 --n 5
 """
 import argparse
 import json
-from datetime import datetime
-from datetime import timedelta
-from datetime import timezone
 from typing import Any
 from typing import Dict
 from typing import List
-from typing import Optional
-from uuid import UUID
 
-from pydantic import BaseModel
-from sqlalchemy import and_
-
-from onyx.configs.constants import INDEX_SEPARATOR
-from onyx.context.search.models import IndexFilters
-from onyx.context.search.models import SearchRequest
+from onyx.db.connector_credential_pair import get_connector_credential_pair_from_id
 from onyx.db.engine import get_session_with_tenant
-from onyx.db.models import ConnectorCredentialPair
-from onyx.db.models import Document
-from onyx.db.models import DocumentByConnectorCredentialPair
 from onyx.db.search_settings import get_current_search_settings
-from onyx.document_index.document_index_utils import get_document_chunk_ids
-from onyx.document_index.interfaces import EnrichedDocumentIndexingInfo
-from onyx.document_index.vespa.index import VespaIndex
 from onyx.document_index.vespa.shared_utils.utils import get_vespa_http_client
-from onyx.document_index.vespa_constants import ACCESS_CONTROL_LIST
-from onyx.document_index.vespa_constants import DOC_UPDATED_AT
 from onyx.document_index.vespa_constants import DOCUMENT_ID_ENDPOINT
-from onyx.document_index.vespa_constants import DOCUMENT_SETS
-from onyx.document_index.vespa_constants import HIDDEN
-from onyx.document_index.vespa_constants import METADATA_LIST
 from onyx.document_index.vespa_constants import SEARCH_ENDPOINT
-from onyx.document_index.vespa_constants import SOURCE_TYPE
-from onyx.document_index.vespa_constants import TENANT_ID
 from onyx.document_index.vespa_constants import VESPA_APP_CONTAINER_URL
 from onyx.document_index.vespa_constants import VESPA_APPLICATION_ENDPOINT
-from onyx.utils.logger import setup_logger
-from shared_configs.configs import MULTI_TENANT
-from shared_configs.configs import POSTGRES_DEFAULT_SCHEMA
-
-logger = setup_logger()
-
-
-class DocumentFilter(BaseModel):
-    # Document filter for link matching.
-    link: str | None = None
-
-
-def build_vespa_filters(
-    filters: IndexFilters,
-    *,
-    include_hidden: bool = False,
-    remove_trailing_and: bool = False,
-) -> str:
-    # Build a combined Vespa filter string from the given IndexFilters.
-    def _build_or_filters(key: str, vals: list[str] | None) -> str:
-        if vals is None:
-            return ""
-        valid_vals = [val for val in vals if val]
-        if not key or not valid_vals:
-            return ""
-        eq_elems = [f'{key} contains "{elem}"' for elem in valid_vals]
-        or_clause = " or ".join(eq_elems)
-        return f"({or_clause})"
-
-    def _build_time_filter(
-        cutoff: datetime | None,
-        untimed_doc_cutoff: timedelta = timedelta(days=92),
-    ) -> str:
-        if not cutoff:
-            return ""
-        include_untimed = datetime.now(timezone.utc) - untimed_doc_cutoff > cutoff
-        cutoff_secs = int(cutoff.timestamp())
-        if include_untimed:
-            return f"!({DOC_UPDATED_AT} < {cutoff_secs})"
-        return f"({DOC_UPDATED_AT} >= {cutoff_secs})"
-
-    filter_str = ""
-    if not include_hidden:
-        filter_str += f"AND !({HIDDEN}=true) "
-
-    if filters.tenant_id and MULTI_TENANT:
-        filter_str += f'AND ({TENANT_ID} contains "{filters.tenant_id}") '
-
-    if filters.access_control_list is not None:
-        acl_str = _build_or_filters(ACCESS_CONTROL_LIST, filters.access_control_list)
-        if acl_str:
-            filter_str += f"AND {acl_str} "
-
-    source_strs = (
-        [s.value for s in filters.source_type] if filters.source_type else None
-    )
-    source_str = _build_or_filters(SOURCE_TYPE, source_strs)
-    if source_str:
-        filter_str += f"AND {source_str} "
-
-    tags = filters.tags
-    if tags:
-        tag_attributes = [tag.tag_key + INDEX_SEPARATOR + tag.tag_value for tag in tags]
-    else:
-        tag_attributes = None
-    tag_str = _build_or_filters(METADATA_LIST, tag_attributes)
-    if tag_str:
-        filter_str += f"AND {tag_str} "
-
-    doc_set_str = _build_or_filters(DOCUMENT_SETS, filters.document_set)
-    if doc_set_str:
-        filter_str += f"AND {doc_set_str} "
-
-    time_filter = _build_time_filter(filters.time_cutoff)
-    if time_filter:
-        filter_str += f"AND {time_filter} "
-
-    if remove_trailing_and:
-        while filter_str.endswith(" and "):
-            filter_str = filter_str[:-5]
-        while filter_str.endswith("AND "):
-            filter_str = filter_str[:-4]
-
-    return filter_str.strip()
 
 
+# Print Vespa configuration URLs
 def print_vespa_config() -> None:
-    # Print Vespa configuration.
-    logger.info("Printing Vespa configuration.")
     print(f"Vespa Application Endpoint: {VESPA_APPLICATION_ENDPOINT}")
     print(f"Vespa App Container URL: {VESPA_APP_CONTAINER_URL}")
     print(f"Vespa Search Endpoint: {SEARCH_ENDPOINT}")
     print(f"Vespa Document ID Endpoint: {DOCUMENT_ID_ENDPOINT}")
 
 
+# Check connectivity to Vespa endpoints
 def check_vespa_connectivity() -> None:
-    # Check connectivity to Vespa endpoints.
-    logger.info("Checking Vespa connectivity.")
     endpoints = [
         f"{VESPA_APPLICATION_ENDPOINT}/ApplicationStatus",
         f"{VESPA_APPLICATION_ENDPOINT}/tenant",
@@ -170,21 +61,17 @@ def check_vespa_connectivity() -> None:
         try:
             with get_vespa_http_client() as client:
                 response = client.get(endpoint)
-                logger.info(
-                    f"Connected to Vespa at {endpoint}, status code {response.status_code}"
-                )
                 print(f"Successfully connected to Vespa at {endpoint}")
                 print(f"Status code: {response.status_code}")
                 print(f"Response: {response.text[:200]}...")
         except Exception as e:
-            logger.error(f"Failed to connect to Vespa at {endpoint}: {str(e)}")
             print(f"Failed to connect to Vespa at {endpoint}: {str(e)}")
 
     print("Vespa connectivity check completed.")
 
 
+# Get info about the default Vespa application
 def get_vespa_info() -> Dict[str, Any]:
-    # Get info about the default Vespa application.
     url = f"{VESPA_APPLICATION_ENDPOINT}/tenant/default/application/default"
     with get_vespa_http_client() as client:
         response = client.get(url)
@@ -192,298 +79,121 @@ def get_vespa_info() -> Dict[str, Any]:
         return response.json()
 
 
-def get_index_name(tenant_id: str) -> str:
-    # Return the index name for a given tenant.
+# Get index name for a tenant and connector pair
+def get_index_name(tenant_id: str, connector_id: int) -> str:
     with get_session_with_tenant(tenant_id=tenant_id) as db_session:
+        cc_pair = get_connector_credential_pair_from_id(db_session, connector_id)
+        if not cc_pair:
+            raise ValueError(f"No connector found for id {connector_id}")
         search_settings = get_current_search_settings(db_session)
-        if not search_settings:
-            raise ValueError(f"No search settings found for tenant {tenant_id}")
-        return search_settings.index_name
+        return search_settings.index_name if search_settings else "public"
 
 
-def query_vespa(
-    yql: str, tenant_id: Optional[str] = None, limit: int = 10
-) -> List[Dict[str, Any]]:
-    # Perform a Vespa query using YQL syntax.
-    filters = IndexFilters(tenant_id=tenant_id, access_control_list=[])
-    filter_string = build_vespa_filters(filters, remove_trailing_and=True)
-    full_yql = yql.strip()
-    if filter_string:
-        full_yql = f"{full_yql} {filter_string}"
-    full_yql = f"{full_yql} limit {limit}"
-
-    params = {"yql": full_yql, "timeout": "10s"}
-    search_request = SearchRequest(query="", limit=limit, offset=0)
-    params.update(search_request.model_dump())
-
-    logger.info(f"Executing Vespa query: {full_yql}")
+# Perform a Vespa query using YQL syntax
+def query_vespa(yql: str) -> List[Dict[str, Any]]:
+    params = {
+        "yql": yql,
+        "timeout": "10s",
+    }
     with get_vespa_http_client() as client:
         response = client.get(SEARCH_ENDPOINT, params=params)
         response.raise_for_status()
-        result = response.json()
-        documents = result.get("root", {}).get("children", [])
-        logger.info(f"Found {len(documents)} documents from query.")
-        return documents
+        return response.json()["root"]["children"]
 
 
+# Get first N documents
 def get_first_n_documents(n: int = 10) -> List[Dict[str, Any]]:
-    # Get the first n documents from any source.
-    yql = "select * from sources * where true"
-    return query_vespa(yql, limit=n)
+    yql = f"select * from sources * where true limit {n};"
+    return query_vespa(yql)
 
 
+# Pretty-print a list of documents
 def print_documents(documents: List[Dict[str, Any]]) -> None:
-    # Pretty-print a list of documents.
     for doc in documents:
         print(json.dumps(doc, indent=2))
         print("-" * 80)
 
 
+# Get and print documents for a specific tenant and connector
 def get_documents_for_tenant_connector(
     tenant_id: str, connector_id: int, n: int = 10
 ) -> None:
-    # Get and print documents for a specific tenant and connector.
-    index_name = get_index_name(tenant_id)
-    logger.info(
-        f"Fetching documents for tenant={tenant_id}, connector_id={connector_id}"
-    )
-    yql = f"select * from sources {index_name} where true"
-    documents = query_vespa(yql, tenant_id, limit=n)
-    print(
-        f"First {len(documents)} documents for tenant {tenant_id}, connector {connector_id}:"
-    )
+    get_index_name(tenant_id, connector_id)
+    documents = get_first_n_documents(n)
+    print(f"First {n} documents for tenant {tenant_id}, connector {connector_id}:")
     print_documents(documents)
 
 
+# Search documents for a specific tenant and connector
 def search_documents(
     tenant_id: str, connector_id: int, query: str, n: int = 10
 ) -> None:
-    # Search documents for a specific tenant and connector.
-    index_name = get_index_name(tenant_id)
-    logger.info(
-        f"Searching documents for tenant={tenant_id}, connector_id={connector_id}, query='{query}'"
-    )
-    yql = f"select * from sources {index_name} where userInput(@query)"
-    documents = query_vespa(yql, tenant_id, limit=n)
-    print(f"Search results for query '{query}' in tenant {tenant_id}:")
+    index_name = get_index_name(tenant_id, connector_id)
+    yql = f"select * from sources {index_name} where userInput(@query) limit {n};"
+    documents = query_vespa(yql)
+    print(f"Search results for query '{query}':")
     print_documents(documents)
 
 
+# Update a specific document
 def update_document(
     tenant_id: str, connector_id: int, doc_id: str, fields: Dict[str, Any]
 ) -> None:
-    # Update a specific document.
-    index_name = get_index_name(tenant_id)
-    logger.info(
-        f"Updating document doc_id={doc_id} in tenant={tenant_id}, connector_id={connector_id}"
-    )
+    index_name = get_index_name(tenant_id, connector_id)
     url = DOCUMENT_ID_ENDPOINT.format(index_name=index_name) + f"/{doc_id}"
     update_request = {"fields": {k: {"assign": v} for k, v in fields.items()}}
+
     with get_vespa_http_client() as client:
         response = client.put(url, json=update_request)
         response.raise_for_status()
-        logger.info(f"Document {doc_id} updated successfully.")
         print(f"Document {doc_id} updated successfully")
 
 
+# Delete a specific document
 def delete_document(tenant_id: str, connector_id: int, doc_id: str) -> None:
-    # Delete a specific document.
-    index_name = get_index_name(tenant_id)
-    logger.info(
-        f"Deleting document doc_id={doc_id} in tenant={tenant_id}, connector_id={connector_id}"
-    )
+    index_name = get_index_name(tenant_id, connector_id)
     url = DOCUMENT_ID_ENDPOINT.format(index_name=index_name) + f"/{doc_id}"
+
     with get_vespa_http_client() as client:
         response = client.delete(url)
         response.raise_for_status()
-        logger.info(f"Document {doc_id} deleted successfully.")
         print(f"Document {doc_id} deleted successfully")
 
 
-def list_documents(n: int = 10, tenant_id: Optional[str] = None) -> None:
-    # List documents from any source, filtered by tenant if provided.
-    logger.info(f"Listing up to {n} documents for tenant={tenant_id or 'ALL'}")
-    yql = "select * from sources * where true"
-    if tenant_id:
-        yql += f" and tenant_id contains '{tenant_id}'"
-    documents = query_vespa(yql, tenant_id=tenant_id, limit=n)
-    print(f"Total documents found: {len(documents)}")
-    logger.info(f"Total documents found: {len(documents)}")
-    print(f"First {min(n, len(documents))} documents:")
-    for doc in documents[:n]:
-        print(json.dumps(doc, indent=2))
+# List documents from any source
+def list_documents(n: int = 10) -> None:
+    yql = f"select * from sources * where true limit {n};"
+    url = f"{VESPA_APP_CONTAINER_URL}/search/"
+    params = {
+        "yql": yql,
+        "timeout": "10s",
+    }
+    try:
+        with get_vespa_http_client() as client:
+            response = client.get(url, params=params)
+            response.raise_for_status()
+            documents = response.json()["root"]["children"]
+            print(f"First {n} documents:")
+            print_documents(documents)
+    except Exception as e:
+        print(f"Failed to list documents: {str(e)}")
+
+
+# Get and print ACLs for documents of a specific tenant and connector
+def get_document_acls(tenant_id: str, connector_id: int, n: int = 10) -> None:
+    index_name = get_index_name(tenant_id, connector_id)
+    yql = f"select documentid, access_control_list from sources {index_name} where true limit {n};"
+    documents = query_vespa(yql)
+    print(f"ACLs for {n} documents from tenant {tenant_id}, connector {connector_id}:")
+    for doc in documents:
+        print(f"Document ID: {doc['fields']['documentid']}")
+        print(
+            f"ACL: {json.dumps(doc['fields'].get('access_control_list', {}), indent=2)}"
+        )
         print("-" * 80)
 
 
-def get_document_and_chunk_counts(
-    tenant_id: str, cc_pair_id: int, filter_doc: DocumentFilter | None = None
-) -> Dict[str, int]:
-    # Return a dict mapping each document ID to its chunk count for a given connector.
-    with get_session_with_tenant(tenant_id=tenant_id) as session:
-        doc_ids_data = (
-            session.query(DocumentByConnectorCredentialPair.id, Document.link)
-            .join(
-                ConnectorCredentialPair,
-                and_(
-                    DocumentByConnectorCredentialPair.connector_id
-                    == ConnectorCredentialPair.connector_id,
-                    DocumentByConnectorCredentialPair.credential_id
-                    == ConnectorCredentialPair.credential_id,
-                ),
-            )
-            .join(Document, DocumentByConnectorCredentialPair.id == Document.id)
-            .filter(ConnectorCredentialPair.id == cc_pair_id)
-            .distinct()
-            .all()
-        )
-        doc_ids = []
-        for doc_id, link in doc_ids_data:
-            if filter_doc and filter_doc.link:
-                if link and filter_doc.link.lower() in link.lower():
-                    doc_ids.append(doc_id)
-            else:
-                doc_ids.append(doc_id)
-        chunk_counts_data = (
-            session.query(Document.id, Document.chunk_count)
-            .filter(Document.id.in_(doc_ids))
-            .all()
-        )
-    return {
-        doc_id: chunk_count
-        for doc_id, chunk_count in chunk_counts_data
-        if chunk_count is not None
-    }
-
-
-def get_chunk_ids_for_connector(
-    tenant_id: str,
-    cc_pair_id: int,
-    index_name: str,
-    filter_doc: DocumentFilter | None = None,
-) -> List[UUID]:
-    # Return chunk IDs for a given connector.
-    doc_id_to_new_chunk_cnt = get_document_and_chunk_counts(
-        tenant_id, cc_pair_id, filter_doc
-    )
-    doc_infos: List[EnrichedDocumentIndexingInfo] = [
-        VespaIndex.enrich_basic_chunk_info(
-            index_name=index_name,
-            http_client=get_vespa_http_client(),
-            document_id=doc_id,
-            previous_chunk_count=doc_id_to_new_chunk_cnt.get(doc_id, 0),
-            new_chunk_count=0,
-        )
-        for doc_id in doc_id_to_new_chunk_cnt.keys()
-    ]
-    chunk_ids = get_document_chunk_ids(
-        enriched_document_info_list=doc_infos,
-        tenant_id=tenant_id,
-        large_chunks_enabled=False,
-    )
-    if not isinstance(chunk_ids, list):
-        raise ValueError(f"Expected list of chunk IDs, got {type(chunk_ids)}")
-    return chunk_ids
-
-
-def get_document_acls(
-    tenant_id: str,
-    cc_pair_id: int,
-    n: int | None = 10,
-    filter_doc: DocumentFilter | None = None,
-) -> None:
-    # Fetch document ACLs for the given tenant and connector pair.
-    index_name = get_index_name(tenant_id)
-    logger.info(
-        f"Fetching document ACLs for tenant={tenant_id}, cc_pair_id={cc_pair_id}"
-    )
-    chunk_ids: List[UUID] = get_chunk_ids_for_connector(
-        tenant_id, cc_pair_id, index_name, filter_doc
-    )
-    vespa_client = get_vespa_http_client()
-
-    target_ids = chunk_ids if n is None else chunk_ids[:n]
-    logger.info(
-        f"Found {len(chunk_ids)} chunk IDs, showing ACLs for {len(target_ids)}."
-    )
-    for doc_chunk_id in target_ids:
-        document_url = (
-            f"{DOCUMENT_ID_ENDPOINT.format(index_name=index_name)}/{str(doc_chunk_id)}"
-        )
-        response = vespa_client.get(document_url)
-        if response.status_code == 200:
-            fields = response.json().get("fields", {})
-            document_id = fields.get("document_id") or fields.get(
-                "documentid", "Unknown"
-            )
-            acls = fields.get("access_control_list", {})
-            title = fields.get("title", "")
-            source_type = fields.get("source_type", "")
-            source_links_raw = fields.get("source_links", "{}")
-            try:
-                source_links = json.loads(source_links_raw)
-            except json.JSONDecodeError:
-                source_links = {}
-
-            print(f"Document Chunk ID: {doc_chunk_id}")
-            print(f"Document ID: {document_id}")
-            print(f"ACLs:\n{json.dumps(acls, indent=2)}")
-            print(f"Source Links: {source_links}")
-            print(f"Title: {title}")
-            print(f"Source Type: {source_type}")
-            if MULTI_TENANT:
-                print(f"Tenant ID: {fields.get('tenant_id', 'N/A')}")
-            print("-" * 80)
-        else:
-            logger.error(f"Failed to fetch document for chunk ID: {doc_chunk_id}")
-            print(f"Failed to fetch document for chunk ID: {doc_chunk_id}")
-            print(f"Status Code: {response.status_code}")
-            print("-" * 80)
-
-
-class VespaDebugging:
-    # Class for managing Vespa debugging actions.
-    def __init__(self, tenant_id: str | None = None):
-        self.tenant_id = POSTGRES_DEFAULT_SCHEMA if not tenant_id else tenant_id
-
-    def print_config(self) -> None:
-        # Print Vespa config.
-        print_vespa_config()
-
-    def check_connectivity(self) -> None:
-        # Check Vespa connectivity.
-        check_vespa_connectivity()
-
-    def list_documents(self, n: int = 10) -> None:
-        # List documents for a tenant.
-        list_documents(n, self.tenant_id)
-
-    def search_documents(self, connector_id: int, query: str, n: int = 10) -> None:
-        # Search documents for a tenant and connector.
-        search_documents(self.tenant_id, connector_id, query, n)
-
-    def update_document(
-        self, connector_id: int, doc_id: str, fields: Dict[str, Any]
-    ) -> None:
-        # Update a document.
-        update_document(self.tenant_id, connector_id, doc_id, fields)
-
-    def delete_document(self, connector_id: int, doc_id: str) -> None:
-        # Delete a document.
-        delete_document(self.tenant_id, connector_id, doc_id)
-
-    def acls_by_link(self, cc_pair_id: int, link: str) -> None:
-        # Get ACLs for a document matching a link.
-        get_document_acls(
-            self.tenant_id, cc_pair_id, n=None, filter_doc=DocumentFilter(link=link)
-        )
-
-    def acls(self, cc_pair_id: int, n: int | None = 10) -> None:
-        # Get ACLs for a connector.
-        get_document_acls(self.tenant_id, cc_pair_id, n)
-
-
 def main() -> None:
-    # Main CLI entry point.
     parser = argparse.ArgumentParser(description="Vespa debugging tool")
     parser.add_argument(
         "--action",
@@ -499,45 +209,60 @@ def main() -> None:
         required=True,
         help="Action to perform",
     )
-    parser.add_argument("--tenant-id", help="Tenant ID")
-    parser.add_argument("--connector-id", type=int, help="Connector ID")
     parser.add_argument(
-        "--n", type=int, default=10, help="Number of documents to retrieve"
+        "--tenant-id", help="Tenant ID (for update, delete, and get_acls actions)"
+    )
+    parser.add_argument(
+        "--connector-id",
+        type=int,
+        help="Connector ID (for update, delete, and get_acls actions)",
+    )
+    parser.add_argument(
+        "--n",
+        type=int,
+        default=10,
+        help="Number of documents to retrieve (for list_docs, search, update, and get_acls actions)",
     )
     parser.add_argument("--query", help="Search query (for search action)")
     parser.add_argument("--doc-id", help="Document ID (for update and delete actions)")
     parser.add_argument(
-        "--fields", help="Fields to update, in JSON format (for update)"
+        "--fields", help="Fields to update, in JSON format (for update action)"
     )
 
     args = parser.parse_args()
-    vespa_debug = VespaDebugging(args.tenant_id)
 
     if args.action == "config":
-        vespa_debug.print_config()
+        print_vespa_config()
     elif args.action == "connect":
-        vespa_debug.check_connectivity()
+        check_vespa_connectivity()
     elif args.action == "list_docs":
-        vespa_debug.list_documents(args.n)
-    elif args.action == "search":
-        if not args.query or args.connector_id is None:
-            parser.error("--query and --connector-id are required for search action")
-        vespa_debug.search_documents(args.connector_id, args.query, args.n)
-    elif args.action == "update":
-        if not args.doc_id or not args.fields or args.connector_id is None:
-            parser.error(
-                "--doc-id, --fields, and --connector-id are required for update action"
+        # If tenant_id and connector_id are provided, list docs for that tenant/connector.
+        # Otherwise, list documents from any source.
+        if args.tenant_id and args.connector_id:
+            get_documents_for_tenant_connector(
+                args.tenant_id, args.connector_id, args.n
             )
+        else:
+            list_documents(args.n)
+    elif args.action == "search":
+        if not args.query:
+            parser.error("--query is required for search action")
+        search_documents(args.tenant_id, args.connector_id, args.query, args.n)
+    elif args.action == "update":
+        if not args.doc_id or not args.fields:
+            parser.error("--doc-id and --fields are required for update action")
         fields = json.loads(args.fields)
-        vespa_debug.update_document(args.connector_id, args.doc_id, fields)
+        update_document(args.tenant_id, args.connector_id, args.doc_id, fields)
     elif args.action == "delete":
-        if not args.doc_id or args.connector_id is None:
-            parser.error("--doc-id and --connector-id are required for delete action")
-        vespa_debug.delete_document(args.connector_id, args.doc_id)
+        if not args.doc_id:
+            parser.error("--doc-id is required for delete action")
+        delete_document(args.tenant_id, args.connector_id, args.doc_id)
     elif args.action == "get_acls":
-        if args.connector_id is None:
-            parser.error("--connector-id is required for get_acls action")
-        vespa_debug.acls(args.connector_id, args.n)
+        if not args.tenant_id or args.connector_id is None:
+            parser.error(
+                "--tenant-id and --connector-id are required for get_acls action"
+            )
+        get_document_acls(args.tenant_id, args.connector_id, args.n)
 
 
 if __name__ == "__main__":

web/Dockerfile

@@ -18,9 +18,6 @@ FROM base AS builder
 RUN apk add --no-cache libc6-compat
 WORKDIR /app
 
-# Add NODE_OPTIONS argument
-ARG NODE_OPTIONS
-
 # pull in source code / package.json / package-lock.json
 COPY . .
 
@@ -81,8 +78,7 @@ ENV NEXT_PUBLIC_GTM_ENABLED=${NEXT_PUBLIC_GTM_ENABLED}
 ARG NEXT_PUBLIC_FORGOT_PASSWORD_ENABLED
 ENV NEXT_PUBLIC_FORGOT_PASSWORD_ENABLED=${NEXT_PUBLIC_FORGOT_PASSWORD_ENABLED}
 
-# Use NODE_OPTIONS in the build command
-RUN NODE_OPTIONS="${NODE_OPTIONS}" npx next build
+RUN NODE_OPTIONS="--max-old-space-size=8192" npx next build
 
 # Step 2. Production image, copy all the files and run next
 FROM base AS runner

web/next.config.js

@@ -86,16 +86,14 @@ const sentryWebpackPluginOptions = {
   authToken: process.env.SENTRY_AUTH_TOKEN,
   silent: !sentryEnabled, // Silence output when Sentry is disabled
   dryRun: !sentryEnabled, // Don't upload source maps when Sentry is disabled
-  ...(sentryEnabled && {
-    sourceMaps: {
-      include: ["./.next"],
-      ignore: ["node_modules"],
-      urlPrefix: "~/_next",
-      stripPrefix: ["webpack://_N_E/"],
-      validate: true,
-      cleanArtifacts: true,
-    },
-  }),
+  sourceMaps: {
+    include: ["./.next"],
+    ignore: ["node_modules"],
+    urlPrefix: "~/_next",
+    stripPrefix: ["webpack://_N_E/"],
+    validate: true,
+    cleanArtifacts: true,
+  },
 };
 
 // Export the module with conditional Sentry configuration

web/src/app/chat/ChatPage.tsx

@@ -293,22 +293,19 @@ export function ChatPage({
     );
   };
 
+  const llmOverrideManager = useLlmOverride(
+    llmProviders,
+    user?.preferences.default_model,
+    selectedChatSession
+  );
+
   const [alternativeAssistant, setAlternativeAssistant] =
     useState<Persona | null>(null);
 
   const [presentingDocument, setPresentingDocument] =
     useState<OnyxDocument | null>(null);
 
-  const { recentAssistants, refreshRecentAssistants, assistants } =
-    useAssistants();
-
-  const llmOverrideManager = useLlmOverride(
-    llmProviders,
-    user?.preferences.default_model,
-    selectedChatSession,
-    undefined,
-    assistants
-  );
+  const { recentAssistants, refreshRecentAssistants } = useAssistants();
 
   const liveAssistant: Persona | undefined =
     alternativeAssistant ||
@@ -339,7 +336,7 @@ export function ChatPage({
       );
     }
     // eslint-disable-next-line react-hooks/exhaustive-deps
-  }, [liveAssistant, user?.preferences.default_model, selectedChatSession]);
+  }, [liveAssistant, user?.preferences.default_model]);
 
   const stopGenerating = () => {
     const currentSession = currentSessionId();
@@ -2602,7 +2599,6 @@ export function ChatPage({
                                                 });
                                                 return;
                                               }
-
                                               onSubmit({
                                                 messageIdToResend:
                                                   previousMessage.messageId,

web/src/app/chat/documentSidebar/DocumentResults.tsx

@@ -147,6 +147,24 @@ export const DocumentResults = forwardRef<HTMLDivElement, DocumentResultsProps>(
                 )}
               </div>
             </div>
+            <div
+              className={`sticky bottom-4 w-full left-0 flex justify-center transition-opacity duration-300 ${
+                hasSelectedDocuments
+                  ? "opacity-100"
+                  : "opacity-0 pointer-events-none"
+              }`}
+            >
+              <button
+                className="text-sm font-medium py-2 px-4 rounded-full transition-colors bg-neutral-900 text-white"
+                onClick={clearSelectedDocuments}
+              >
+                {`Remove ${
+                  delayedSelectedDocumentCount > 0
+                    ? delayedSelectedDocumentCount
+                    : ""
+                } Source${delayedSelectedDocumentCount > 1 ? "s" : ""}`}
+              </button>
+            </div>
           </div>
         </div>
       </>

web/src/app/chat/message/SearchSummary.tsx

@@ -198,7 +198,7 @@ export function SearchSummary({
   ) : null;
 
   return (
-    <div className="flex group w-fit items-center">
+    <div className="flex items-center">
       {isEditing ? (
         editInput
       ) : (
@@ -225,7 +225,7 @@ export function SearchSummary({
               <Tooltip>
                 <TooltipTrigger asChild>
                   <button
-                    className="ml-2 -my-2 mobile:hidden hover:bg-hover p-1 rounded flex-shrink-0 group-hover:opacity-100 opacity-0"
+                    className="ml-2 -my-2 mobile:hidden hover:bg-hover p-1 rounded flex-shrink-0"
                     onClick={() => {
                       setIsEditing(true);
                     }}

web/src/app/chat/sessionSidebar/PagesTab.tsx

@@ -445,7 +445,7 @@ export function PagesTab({
         )}
 
         {isHistoryEmpty && (!folders || folders.length === 0) && (
-          <p className="text-sm max-w-full mt-2 w-[250px]">
+          <p className="text-sm mt-2 w-[250px]">
             Try sending a message! Your chat history will appear here.
           </p>
         )}

web/src/app/layout.tsx

@@ -205,9 +205,9 @@ export default async function RootLayout({
     >
       <Suspense fallback={null}>
         <PostHogPageView />
+        {process.env.NEXT_PUBLIC_POSTHOG_KEY && <WebVitals />}
+        {children}
       </Suspense>
-      {children}
-      {process.env.NEXT_PUBLIC_POSTHOG_KEY && <WebVitals />}
     </AppProvider>
   );
 }

web/src/app/page.tsx

@@ -1,5 +1,5 @@
 import { redirect } from "next/navigation";
 
 export default async function Page() {
-  redirect("/chat");
+  redirect("/auth/login");
 }

web/src/components/admin/connectors/Field.tsx

@@ -77,9 +77,7 @@ export function LabelWithTooltip({
 }
 
 export function SubLabel({ children }: { children: string | JSX.Element }) {
-  return (
-    <div className="text-xs text-subtle whitespace-pre-line">{children}</div>
-  );
+  return <div className="text-xs text-subtle">{children}</div>;
 }
 
 export function ManualErrorMessage({ children }: { children: string }) {

web/src/lib/connectors/connectors.tsx

@@ -481,9 +481,7 @@ Hint: Use the singular form of the object name (e.g., 'Opportunity' instead of '
         name: "sites",
         optional: true,
         description: `• If no sites are specified, all sites in your organization will be indexed (Sites.Read.All permission required).
-
 • Specifying 'https://onyxai.sharepoint.com/sites/support' for example will only index documents within this site.
-
 • Specifying 'https://onyxai.sharepoint.com/sites/support/subfolder' for example will only index documents within this folder.
 `,
       },

web/src/lib/hooks.ts

@@ -13,16 +13,12 @@ import { errorHandlingFetcher } from "./fetcher";
 import { useContext, useEffect, useState } from "react";
 import { DateRangePickerValue } from "@/app/ee/admin/performance/DateRangeSelector";
 import { Filters, SourceMetadata } from "./search/interfaces";
-import {
-  destructureValue,
-  getLLMProviderOverrideForPersona,
-  structureValue,
-} from "./llm/utils";
+import { destructureValue, structureValue } from "./llm/utils";
 import { ChatSession } from "@/app/chat/interfaces";
 import { AllUsersResponse } from "./types";
 import { Credential } from "./connectors/credentials";
 import { SettingsContext } from "@/components/settings/SettingsProvider";
-import { Persona, PersonaLabel } from "@/app/admin/assistants/interfaces";
+import { PersonaLabel } from "@/app/admin/assistants/interfaces";
 import { LLMProviderDescriptor } from "@/app/admin/configuration/llm/interfaces";
 import { isAnthropic } from "@/app/admin/configuration/llm/interfaces";
 import { getSourceMetadata } from "./sources";
@@ -371,8 +367,7 @@ export function useLlmOverride(
   llmProviders: LLMProviderDescriptor[],
   globalModel?: string | null,
   currentChatSession?: ChatSession,
-  defaultTemperature?: number,
-  assistants?: Persona[]
+  defaultTemperature?: number
 ): LlmOverrideManager {
   const getValidLlmOverride = (
     overrideModel: string | null | undefined
@@ -429,32 +424,6 @@ export function useLlmOverride(
     defaultTemperature !== undefined ? defaultTemperature : 0
   );
 
-  useEffect(() => {
-    const currentPersona = assistants?.find(
-      (a) => a.id === currentChatSession?.persona_id
-    );
-    const personaDefault = currentPersona
-      ? getLLMProviderOverrideForPersona(currentPersona, llmProviders)
-      : undefined;
-
-    if (personaDefault) {
-      updateLLMOverride(personaDefault);
-    } else {
-      updateLLMOverride(globalDefault);
-    }
-    // eslint-disable-next-line react-hooks/exhaustive-deps
-  }, [currentChatSession]);
-
-  useEffect(() => {
-    if (currentChatSession?.current_alternate_model) {
-      setLlmOverride(
-        getValidLlmOverride(currentChatSession.current_alternate_model)
-      );
-    } else {
-      setLlmOverride(globalDefault);
-    }
-  }, [currentChatSession]);
-
   useEffect(() => {
     setGlobalDefault(getValidLlmOverride(globalModel));
   }, [globalModel, llmProviders]);