k

* WIP

* migrate most beat tasks to fan out strategy

* fix kwargs

* migrate EE tasks

* lock on the task_name level

* typo fix

---------

Co-authored-by: Richard Kuo (Danswer) <rkuo@onyx.app>

.github/pull_request_template.md

@@ -11,5 +11,4 @@
 Note: You have to check that the action passes, otherwise resolve the conflicts manually and tag the patches.
 
 - [ ] This PR should be backported (make sure to check that the backport attempt succeeds)
-- [ ] I have included a link to a Linear ticket in my description.
 - [ ] [Optional] Override Linear Check

.github/workflows/docker-build-push-cloud-web-container-on-tag.yml

@@ -67,6 +67,7 @@ jobs:
             NEXT_PUBLIC_SENTRY_DSN=${{ secrets.SENTRY_DSN }}
             NEXT_PUBLIC_GTM_ENABLED=true
             NEXT_PUBLIC_FORGOT_PASSWORD_ENABLED=true
+            NODE_OPTIONS=--max-old-space-size=8192
           # needed due to weird interactions with the builds for different platforms
           no-cache: true
           labels: ${{ steps.meta.outputs.labels }}

.github/workflows/docker-build-push-web-container-on-tag.yml

@@ -60,6 +60,8 @@ jobs:
           push: true
           build-args: |
             ONYX_VERSION=${{ github.ref_name }}
+            NODE_OPTIONS=--max-old-space-size=8192
+
           # needed due to weird interactions with the builds for different platforms
           no-cache: true
           labels: ${{ steps.meta.outputs.labels }}

README.md

@@ -119,7 +119,7 @@ There are two editions of Onyx:
   - Whitelabeling
   - API key authentication
   - Encryption of secrets
-  - Any many more! Checkout [our website](https://www.onyx.app/) for the latest.
+  - And many more! Checkout [our website](https://www.onyx.app/) for the latest.
 
 To try the Onyx Enterprise Edition:
 

backend/alembic/versions/f1ca58b2f2ec_add_passthrough_auth_to_tool.py

@@ -0,0 +1,33 @@
+"""add passthrough auth to tool
+
+Revision ID: f1ca58b2f2ec
+Revises: c7bf5721733e
+Create Date: 2024-03-19
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = "f1ca58b2f2ec"
+down_revision: Union[str, None] = "c7bf5721733e"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # Add passthrough_auth column to tool table with default value of False
+    op.add_column(
+        "tool",
+        sa.Column(
+            "passthrough_auth", sa.Boolean(), nullable=False, server_default=sa.false()
+        ),
+    )
+
+
+def downgrade() -> None:
+    # Remove passthrough_auth column from tool table
+    op.drop_column("tool", "passthrough_auth")

backend/ee/onyx/background/celery/tasks/beat_schedule.py

@@ -1,30 +1,70 @@
 from datetime import timedelta
 from typing import Any
 
+from onyx.background.celery.tasks.beat_schedule import BEAT_EXPIRES_DEFAULT
 from onyx.background.celery.tasks.beat_schedule import (
     cloud_tasks_to_schedule as base_cloud_tasks_to_schedule,
 )
 from onyx.background.celery.tasks.beat_schedule import (
     tasks_to_schedule as base_tasks_to_schedule,
 )
+from onyx.configs.constants import ONYX_CLOUD_CELERY_TASK_PREFIX
+from onyx.configs.constants import OnyxCeleryPriority
 from onyx.configs.constants import OnyxCeleryTask
+from shared_configs.configs import MULTI_TENANT
 
-ee_tasks_to_schedule = [
+ee_cloud_tasks_to_schedule = [
     {
-        "name": "autogenerate-usage-report",
-        "task": OnyxCeleryTask.AUTOGENERATE_USAGE_REPORT_TASK,
-        "schedule": timedelta(days=30),  # TODO: change this to config flag
+        "name": f"{ONYX_CLOUD_CELERY_TASK_PREFIX}_autogenerate-usage-report",
+        "task": OnyxCeleryTask.CLOUD_BEAT_TASK_GENERATOR,
+        "schedule": timedelta(days=30),
+        "options": {
+            "priority": OnyxCeleryPriority.HIGHEST,
+            "expires": BEAT_EXPIRES_DEFAULT,
+        },
+        "kwargs": {
+            "task_name": OnyxCeleryTask.AUTOGENERATE_USAGE_REPORT_TASK,
+        },
     },
     {
-        "name": "check-ttl-management",
-        "task": OnyxCeleryTask.CHECK_TTL_MANAGEMENT_TASK,
+        "name": f"{ONYX_CLOUD_CELERY_TASK_PREFIX}_check-ttl-management",
+        "task": OnyxCeleryTask.CLOUD_BEAT_TASK_GENERATOR,
         "schedule": timedelta(hours=1),
+        "options": {
+            "priority": OnyxCeleryPriority.HIGHEST,
+            "expires": BEAT_EXPIRES_DEFAULT,
+        },
+        "kwargs": {
+            "task_name": OnyxCeleryTask.CHECK_TTL_MANAGEMENT_TASK,
+        },
     },
 ]
 
+if not MULTI_TENANT:
+    ee_tasks_to_schedule = [
+        {
+            "name": "autogenerate-usage-report",
+            "task": OnyxCeleryTask.AUTOGENERATE_USAGE_REPORT_TASK,
+            "schedule": timedelta(days=30),  # TODO: change this to config flag
+            "options": {
+                "priority": OnyxCeleryPriority.MEDIUM,
+                "expires": BEAT_EXPIRES_DEFAULT,
+            },
+        },
+        {
+            "name": "check-ttl-management",
+            "task": OnyxCeleryTask.CHECK_TTL_MANAGEMENT_TASK,
+            "schedule": timedelta(hours=1),
+            "options": {
+                "priority": OnyxCeleryPriority.MEDIUM,
+                "expires": BEAT_EXPIRES_DEFAULT,
+            },
+        },
+    ]
+
 
 def get_cloud_tasks_to_schedule() -> list[dict[str, Any]]:
-    return base_cloud_tasks_to_schedule
+    return ee_cloud_tasks_to_schedule + base_cloud_tasks_to_schedule
 
 
 def get_tasks_to_schedule() -> list[dict[str, Any]]:

backend/ee/onyx/db/query_history.py

@@ -98,10 +98,9 @@ def get_page_of_chat_sessions(
     conditions = _build_filter_conditions(start_time, end_time, feedback_filter)
 
     subquery = (
-        select(ChatSession.id, ChatSession.time_created)
+        select(ChatSession.id)
         .filter(*conditions)
-        .order_by(ChatSession.id, desc(ChatSession.time_created))
-        .distinct(ChatSession.id)
+        .order_by(desc(ChatSession.time_created), ChatSession.id)
         .limit(page_size)
         .offset(page_num * page_size)
         .subquery()
@@ -118,7 +117,11 @@ def get_page_of_chat_sessions(
                 ChatMessage.chat_message_feedbacks
             ),
         )
-        .order_by(desc(ChatSession.time_created), asc(ChatMessage.id))
+        .order_by(
+            desc(ChatSession.time_created),
+            ChatSession.id,
+            asc(ChatMessage.id),  # Ensure chronological message order
+        )
     )
 
     return db_session.scalars(stmt).unique().all()

backend/onyx/auth/noauth_user.py

@@ -23,7 +23,6 @@ def load_no_auth_user_preferences(store: KeyValueStore) -> UserPreferences:
         preferences_data = cast(
             Mapping[str, Any], store.load(KV_NO_AUTH_USER_PREFERENCES_KEY)
         )
-        print("preferences_data", preferences_data)
         return UserPreferences(**preferences_data)
     except KvKeyNotFoundError:
         return UserPreferences(

backend/onyx/auth/users.py

@@ -55,6 +55,7 @@ from onyx.auth.invited_users import get_invited_users
 from onyx.auth.schemas import UserCreate
 from onyx.auth.schemas import UserRole
 from onyx.auth.schemas import UserUpdate
+from onyx.configs.app_configs import AUTH_COOKIE_EXPIRE_TIME_SECONDS
 from onyx.configs.app_configs import AUTH_TYPE
 from onyx.configs.app_configs import DISABLE_AUTH
 from onyx.configs.app_configs import EMAIL_CONFIGURED
@@ -209,6 +210,7 @@ def verify_email_domain(email: str) -> None:
 class UserManager(UUIDIDMixin, BaseUserManager[User, uuid.UUID]):
     reset_password_token_secret = USER_AUTH_SECRET
     verification_token_secret = USER_AUTH_SECRET
+    verification_token_lifetime_seconds = AUTH_COOKIE_EXPIRE_TIME_SECONDS
 
     user_db: SQLAlchemyUserDatabase[User, uuid.UUID]
 

backend/onyx/background/celery/apps/app_base.py

@@ -23,8 +23,7 @@ from onyx.background.celery.celery_utils import celery_is_worker_primary
 from onyx.configs.constants import ONYX_CLOUD_CELERY_TASK_PREFIX
 from onyx.configs.constants import OnyxRedisLocks
 from onyx.db.engine import get_sqlalchemy_engine
-from onyx.document_index.vespa.shared_utils.utils import get_vespa_http_client
-from onyx.document_index.vespa_constants import VESPA_CONFIG_SERVER_URL
+from onyx.document_index.vespa.shared_utils.utils import wait_for_vespa_with_timeout
 from onyx.redis.redis_connector import RedisConnector
 from onyx.redis.redis_connector_credential_pair import RedisConnectorCredentialPair
 from onyx.redis.redis_connector_delete import RedisConnectorDelete
@@ -280,51 +279,6 @@ def wait_for_db(sender: Any, **kwargs: Any) -> None:
     return
 
 
-def wait_for_vespa(sender: Any, **kwargs: Any) -> None:
-    """Waits for Vespa to become ready subject to a hardcoded timeout.
-    Will raise WorkerShutdown to kill the celery worker if the timeout is reached."""
-
-    WAIT_INTERVAL = 5
-    WAIT_LIMIT = 60
-
-    ready = False
-    time_start = time.monotonic()
-    logger.info("Vespa: Readiness probe starting.")
-    while True:
-        try:
-            client = get_vespa_http_client()
-            response = client.get(f"{VESPA_CONFIG_SERVER_URL}/state/v1/health")
-            response.raise_for_status()
-
-            response_dict = response.json()
-            if response_dict["status"]["code"] == "up":
-                ready = True
-                break
-        except Exception:
-            pass
-
-        time_elapsed = time.monotonic() - time_start
-        if time_elapsed > WAIT_LIMIT:
-            break
-
-        logger.info(
-            f"Vespa: Readiness probe ongoing. elapsed={time_elapsed:.1f} timeout={WAIT_LIMIT:.1f}"
-        )
-
-        time.sleep(WAIT_INTERVAL)
-
-    if not ready:
-        msg = (
-            f"Vespa: Readiness probe did not succeed within the timeout "
-            f"({WAIT_LIMIT} seconds). Exiting..."
-        )
-        logger.error(msg)
-        raise WorkerShutdown(msg)
-
-    logger.info("Vespa: Readiness probe succeeded. Continuing...")
-    return
-
-
 def on_secondary_worker_init(sender: Any, **kwargs: Any) -> None:
     logger.info("Running as a secondary celery worker.")
 
@@ -510,3 +464,13 @@ def reset_tenant_id(
 ) -> None:
     """Signal handler to reset tenant ID in context var after task ends."""
     CURRENT_TENANT_ID_CONTEXTVAR.set(POSTGRES_DEFAULT_SCHEMA)
+
+
+def wait_for_vespa_or_shutdown(sender: Any, **kwargs: Any) -> None:
+    """Waits for Vespa to become ready subject to a timeout.
+    Raises WorkerShutdown if the timeout is reached."""
+
+    if not wait_for_vespa_with_timeout():
+        msg = "Vespa: Readiness probe did not succeed within the timeout. Exiting..."
+        logger.error(msg)
+        raise WorkerShutdown(msg)

backend/onyx/background/celery/apps/beat.py

@@ -81,7 +81,7 @@ class DynamicTenantScheduler(PersistentScheduler):
                 cloud_task = {
                     "task": task["task"],
                     "schedule": task["schedule"],
-                    "kwargs": {},
+                    "kwargs": task.get("kwargs", {}),
                 }
                 if options := task.get("options"):
                     logger.debug(f"Adding options to task {task_name}: {options}")

backend/onyx/background/celery/apps/heavy.py

@@ -62,7 +62,7 @@ def on_worker_init(sender: Worker, **kwargs: Any) -> None:
 
     app_base.wait_for_redis(sender, **kwargs)
     app_base.wait_for_db(sender, **kwargs)
-    app_base.wait_for_vespa(sender, **kwargs)
+    app_base.wait_for_vespa_or_shutdown(sender, **kwargs)
 
     # Less startup checks in multi-tenant case
     if MULTI_TENANT:

backend/onyx/background/celery/apps/indexing.py

@@ -68,7 +68,7 @@ def on_worker_init(sender: Worker, **kwargs: Any) -> None:
 
     app_base.wait_for_redis(sender, **kwargs)
     app_base.wait_for_db(sender, **kwargs)
-    app_base.wait_for_vespa(sender, **kwargs)
+    app_base.wait_for_vespa_or_shutdown(sender, **kwargs)
 
     # Less startup checks in multi-tenant case
     if MULTI_TENANT:

backend/onyx/background/celery/apps/light.py

@@ -63,7 +63,7 @@ def on_worker_init(sender: Worker, **kwargs: Any) -> None:
 
     app_base.wait_for_redis(sender, **kwargs)
     app_base.wait_for_db(sender, **kwargs)
-    app_base.wait_for_vespa(sender, **kwargs)
+    app_base.wait_for_vespa_or_shutdown(sender, **kwargs)
 
     # Less startup checks in multi-tenant case
     if MULTI_TENANT:

backend/onyx/background/celery/apps/primary.py

@@ -86,7 +86,7 @@ def on_worker_init(sender: Worker, **kwargs: Any) -> None:
 
     app_base.wait_for_redis(sender, **kwargs)
     app_base.wait_for_db(sender, **kwargs)
-    app_base.wait_for_vespa(sender, **kwargs)
+    app_base.wait_for_vespa_or_shutdown(sender, **kwargs)
 
     logger.info("Running as the primary celery worker.")
 

backend/onyx/background/celery/tasks/beat_schedule.py

@@ -17,124 +17,234 @@ from shared_configs.configs import MULTI_TENANT
 BEAT_EXPIRES_DEFAULT = 15 * 60  # 15 minutes (in seconds)
 
 # tasks that only run in the cloud
-# the name attribute must start with ONYX_CELERY_CLOUD_PREFIX = "cloud" to be filtered
+# the name attribute must start with ONYX_CLOUD_CELERY_TASK_PREFIX = "cloud" to be filtered
 # by the DynamicTenantScheduler
 cloud_tasks_to_schedule = [
+    # cloud specific tasks
+    {
+        "name": f"{ONYX_CLOUD_CELERY_TASK_PREFIX}_check-alembic",
+        "task": OnyxCeleryTask.CLOUD_CHECK_ALEMBIC,
+        "schedule": timedelta(hours=1),
+        "options": {
+            "queue": OnyxCeleryQueues.MONITORING,
+            "priority": OnyxCeleryPriority.HIGH,
+            "expires": BEAT_EXPIRES_DEFAULT,
+        },
+    },
+    # remaining tasks are cloud generators for per tenant tasks
     {
         "name": f"{ONYX_CLOUD_CELERY_TASK_PREFIX}_check-for-indexing",
-        "task": OnyxCeleryTask.CLOUD_CHECK_FOR_INDEXING,
+        "task": OnyxCeleryTask.CLOUD_BEAT_TASK_GENERATOR,
         "schedule": timedelta(seconds=15),
         "options": {
             "priority": OnyxCeleryPriority.HIGHEST,
             "expires": BEAT_EXPIRES_DEFAULT,
         },
-    },
-]
-
-# tasks that run in either self-hosted on cloud
-tasks_to_schedule = [
-    {
-        "name": "check-for-vespa-sync",
-        "task": OnyxCeleryTask.CHECK_FOR_VESPA_SYNC_TASK,
-        "schedule": timedelta(seconds=20),
-        "options": {
-            "priority": OnyxCeleryPriority.MEDIUM,
-            "expires": BEAT_EXPIRES_DEFAULT,
+        "kwargs": {
+            "task_name": OnyxCeleryTask.CHECK_FOR_INDEXING,
         },
     },
     {
-        "name": "check-for-connector-deletion",
-        "task": OnyxCeleryTask.CHECK_FOR_CONNECTOR_DELETION,
+        "name": f"{ONYX_CLOUD_CELERY_TASK_PREFIX}_check-for-connector-deletion",
+        "task": OnyxCeleryTask.CLOUD_BEAT_TASK_GENERATOR,
         "schedule": timedelta(seconds=20),
         "options": {
-            "priority": OnyxCeleryPriority.MEDIUM,
+            "priority": OnyxCeleryPriority.HIGHEST,
             "expires": BEAT_EXPIRES_DEFAULT,
         },
+        "kwargs": {
+            "task_name": OnyxCeleryTask.CHECK_FOR_CONNECTOR_DELETION,
+        },
     },
     {
-        "name": "check-for-prune",
-        "task": OnyxCeleryTask.CHECK_FOR_PRUNING,
+        "name": f"{ONYX_CLOUD_CELERY_TASK_PREFIX}_check-for-vespa-sync",
+        "task": OnyxCeleryTask.CLOUD_BEAT_TASK_GENERATOR,
+        "schedule": timedelta(seconds=20),
+        "options": {
+            "priority": OnyxCeleryPriority.HIGHEST,
+            "expires": BEAT_EXPIRES_DEFAULT,
+        },
+        "kwargs": {
+            "task_name": OnyxCeleryTask.CHECK_FOR_VESPA_SYNC_TASK,
+        },
+    },
+    {
+        "name": f"{ONYX_CLOUD_CELERY_TASK_PREFIX}_check-for-prune",
+        "task": OnyxCeleryTask.CLOUD_BEAT_TASK_GENERATOR,
         "schedule": timedelta(seconds=15),
         "options": {
-            "priority": OnyxCeleryPriority.MEDIUM,
+            "priority": OnyxCeleryPriority.HIGHEST,
             "expires": BEAT_EXPIRES_DEFAULT,
         },
+        "kwargs": {
+            "task_name": OnyxCeleryTask.CHECK_FOR_PRUNING,
+        },
     },
     {
-        "name": "kombu-message-cleanup",
-        "task": OnyxCeleryTask.KOMBU_MESSAGE_CLEANUP_TASK,
-        "schedule": timedelta(seconds=3600),
-        "options": {
-            "priority": OnyxCeleryPriority.LOWEST,
-            "expires": BEAT_EXPIRES_DEFAULT,
-        },
-    },
-    {
-        "name": "monitor-vespa-sync",
-        "task": OnyxCeleryTask.MONITOR_VESPA_SYNC,
+        "name": f"{ONYX_CLOUD_CELERY_TASK_PREFIX}_monitor-vespa-sync",
+        "task": OnyxCeleryTask.CLOUD_BEAT_TASK_GENERATOR,
         "schedule": timedelta(seconds=5),
         "options": {
-            "priority": OnyxCeleryPriority.MEDIUM,
+            "priority": OnyxCeleryPriority.HIGHEST,
             "expires": BEAT_EXPIRES_DEFAULT,
         },
+        "kwargs": {
+            "task_name": OnyxCeleryTask.MONITOR_VESPA_SYNC,
+        },
     },
     {
-        "name": "monitor-background-processes",
-        "task": OnyxCeleryTask.MONITOR_BACKGROUND_PROCESSES,
-        "schedule": timedelta(minutes=5),
-        "options": {
-            "priority": OnyxCeleryPriority.LOW,
-            "expires": BEAT_EXPIRES_DEFAULT,
-            "queue": OnyxCeleryQueues.MONITORING,
-        },
-    },
-    {
-        "name": "check-for-doc-permissions-sync",
-        "task": OnyxCeleryTask.CHECK_FOR_DOC_PERMISSIONS_SYNC,
+        "name": f"{ONYX_CLOUD_CELERY_TASK_PREFIX}_check-for-doc-permissions-sync",
+        "task": OnyxCeleryTask.CLOUD_BEAT_TASK_GENERATOR,
         "schedule": timedelta(seconds=30),
         "options": {
-            "priority": OnyxCeleryPriority.MEDIUM,
+            "priority": OnyxCeleryPriority.HIGHEST,
             "expires": BEAT_EXPIRES_DEFAULT,
         },
+        "kwargs": {
+            "task_name": OnyxCeleryTask.CHECK_FOR_DOC_PERMISSIONS_SYNC,
+        },
     },
     {
-        "name": "check-for-external-group-sync",
-        "task": OnyxCeleryTask.CHECK_FOR_EXTERNAL_GROUP_SYNC,
+        "name": f"{ONYX_CLOUD_CELERY_TASK_PREFIX}_check-for-external-group-sync",
+        "task": OnyxCeleryTask.CLOUD_BEAT_TASK_GENERATOR,
         "schedule": timedelta(seconds=20),
         "options": {
-            "priority": OnyxCeleryPriority.MEDIUM,
+            "priority": OnyxCeleryPriority.HIGHEST,
             "expires": BEAT_EXPIRES_DEFAULT,
         },
+        "kwargs": {
+            "task_name": OnyxCeleryTask.CHECK_FOR_EXTERNAL_GROUP_SYNC,
+        },
+    },
+    {
+        "name": f"{ONYX_CLOUD_CELERY_TASK_PREFIX}_monitor-background-processes",
+        "task": OnyxCeleryTask.CLOUD_BEAT_TASK_GENERATOR,
+        "schedule": timedelta(minutes=5),
+        "options": {
+            "priority": OnyxCeleryPriority.HIGHEST,
+            "expires": BEAT_EXPIRES_DEFAULT,
+        },
+        "kwargs": {
+            "task_name": OnyxCeleryTask.MONITOR_BACKGROUND_PROCESSES,
+            "queue": OnyxCeleryQueues.MONITORING,
+            "priority": OnyxCeleryPriority.LOW,
+        },
     },
 ]
 
-if not MULTI_TENANT:
-    tasks_to_schedule.append(
+if LLM_MODEL_UPDATE_API_URL:
+    cloud_tasks_to_schedule.append(
         {
-            "name": "check-for-indexing",
-            "task": OnyxCeleryTask.CHECK_FOR_INDEXING,
-            "schedule": timedelta(seconds=15),
+            "name": f"{ONYX_CLOUD_CELERY_TASK_PREFIX}_check-for-llm-model-update",
+            "task": OnyxCeleryTask.CLOUD_BEAT_TASK_GENERATOR,
+            "schedule": timedelta(hours=1),  # Check every hour
             "options": {
-                "priority": OnyxCeleryPriority.MEDIUM,
+                "priority": OnyxCeleryPriority.HIGHEST,
                 "expires": BEAT_EXPIRES_DEFAULT,
             },
+            "kwargs": {
+                "task_name": OnyxCeleryTask.CHECK_FOR_LLM_MODEL_UPDATE,
+                "priority": OnyxCeleryPriority.LOW,
+            },
         }
     )
 
-# Only add the LLM model update task if the API URL is configured
-if LLM_MODEL_UPDATE_API_URL:
-    tasks_to_schedule.append(
-        {
-            "name": "check-for-llm-model-update",
-            "task": OnyxCeleryTask.CHECK_FOR_LLM_MODEL_UPDATE,
-            "schedule": timedelta(hours=1),  # Check every hour
-            "options": {
-                "priority": OnyxCeleryPriority.LOW,
-                "expires": BEAT_EXPIRES_DEFAULT,
+# tasks that run in either self-hosted on cloud
+tasks_to_schedule: list[dict] = []
+
+if not MULTI_TENANT:
+    tasks_to_schedule.extend(
+        [
+            {
+                "name": "check-for-indexing",
+                "task": OnyxCeleryTask.CHECK_FOR_INDEXING,
+                "schedule": timedelta(seconds=15),
+                "options": {
+                    "priority": OnyxCeleryPriority.MEDIUM,
+                    "expires": BEAT_EXPIRES_DEFAULT,
+                },
             },
-        }
+            {
+                "name": "check-for-connector-deletion",
+                "task": OnyxCeleryTask.CHECK_FOR_CONNECTOR_DELETION,
+                "schedule": timedelta(seconds=20),
+                "options": {
+                    "priority": OnyxCeleryPriority.MEDIUM,
+                    "expires": BEAT_EXPIRES_DEFAULT,
+                },
+            },
+            {
+                "name": "check-for-vespa-sync",
+                "task": OnyxCeleryTask.CHECK_FOR_VESPA_SYNC_TASK,
+                "schedule": timedelta(seconds=20),
+                "options": {
+                    "priority": OnyxCeleryPriority.MEDIUM,
+                    "expires": BEAT_EXPIRES_DEFAULT,
+                },
+            },
+            {
+                "name": "check-for-pruning",
+                "task": OnyxCeleryTask.CHECK_FOR_PRUNING,
+                "schedule": timedelta(hours=1),
+                "options": {
+                    "priority": OnyxCeleryPriority.MEDIUM,
+                    "expires": BEAT_EXPIRES_DEFAULT,
+                },
+            },
+            {
+                "name": "monitor-vespa-sync",
+                "task": OnyxCeleryTask.MONITOR_VESPA_SYNC,
+                "schedule": timedelta(seconds=5),
+                "options": {
+                    "priority": OnyxCeleryPriority.MEDIUM,
+                    "expires": BEAT_EXPIRES_DEFAULT,
+                },
+            },
+            {
+                "name": "check-for-doc-permissions-sync",
+                "task": OnyxCeleryTask.CHECK_FOR_DOC_PERMISSIONS_SYNC,
+                "schedule": timedelta(seconds=30),
+                "options": {
+                    "priority": OnyxCeleryPriority.MEDIUM,
+                    "expires": BEAT_EXPIRES_DEFAULT,
+                },
+            },
+            {
+                "name": "check-for-external-group-sync",
+                "task": OnyxCeleryTask.CHECK_FOR_EXTERNAL_GROUP_SYNC,
+                "schedule": timedelta(seconds=20),
+                "options": {
+                    "priority": OnyxCeleryPriority.MEDIUM,
+                    "expires": BEAT_EXPIRES_DEFAULT,
+                },
+            },
+            {
+                "name": "monitor-background-processes",
+                "task": OnyxCeleryTask.MONITOR_BACKGROUND_PROCESSES,
+                "schedule": timedelta(minutes=5),
+                "options": {
+                    "priority": OnyxCeleryPriority.LOW,
+                    "expires": BEAT_EXPIRES_DEFAULT,
+                    "queue": OnyxCeleryQueues.MONITORING,
+                },
+            },
+        ]
     )
 
+    # Only add the LLM model update task if the API URL is configured
+    if LLM_MODEL_UPDATE_API_URL:
+        tasks_to_schedule.append(
+            {
+                "name": "check-for-llm-model-update",
+                "task": OnyxCeleryTask.CHECK_FOR_LLM_MODEL_UPDATE,
+                "schedule": timedelta(hours=1),  # Check every hour
+                "options": {
+                    "priority": OnyxCeleryPriority.LOW,
+                    "expires": BEAT_EXPIRES_DEFAULT,
+                },
+            }
+        )
+
 
 def get_cloud_tasks_to_schedule() -> list[dict[str, Any]]:
     return cloud_tasks_to_schedule

backend/onyx/background/celery/tasks/indexing/tasks.py

@@ -15,7 +15,6 @@ from redis import Redis
 from redis.lock import Lock as RedisLock
 
 from onyx.background.celery.apps.app_base import task_logger
-from onyx.background.celery.tasks.beat_schedule import BEAT_EXPIRES_DEFAULT
 from onyx.background.celery.tasks.indexing.utils import _should_index
 from onyx.background.celery.tasks.indexing.utils import get_unfenced_index_attempt_ids
 from onyx.background.celery.tasks.indexing.utils import IndexingCallback
@@ -26,15 +25,12 @@ from onyx.background.indexing.run_indexing import run_indexing_entrypoint
 from onyx.configs.constants import CELERY_GENERIC_BEAT_LOCK_TIMEOUT
 from onyx.configs.constants import CELERY_INDEXING_LOCK_TIMEOUT
 from onyx.configs.constants import CELERY_TASK_WAIT_FOR_FENCE_TIMEOUT
-from onyx.configs.constants import ONYX_CLOUD_TENANT_ID
-from onyx.configs.constants import OnyxCeleryPriority
 from onyx.configs.constants import OnyxCeleryTask
 from onyx.configs.constants import OnyxRedisLocks
 from onyx.configs.constants import OnyxRedisSignals
 from onyx.db.connector import mark_ccpair_with_indexing_trigger
 from onyx.db.connector_credential_pair import fetch_connector_credential_pairs
 from onyx.db.connector_credential_pair import get_connector_credential_pair_from_id
-from onyx.db.engine import get_all_tenant_ids
 from onyx.db.engine import get_session_with_tenant
 from onyx.db.enums import IndexingMode
 from onyx.db.index_attempt import get_index_attempt
@@ -68,10 +64,6 @@ logger = setup_logger()
 def check_for_indexing(self: Task, *, tenant_id: str | None) -> int | None:
     """a lightweight task used to kick off indexing tasks.
     Occcasionally does some validation of existing state to clear up error conditions"""
-    debug_tenants = {
-        "tenant_i-043470d740845ec56",
-        "tenant_82b497ce-88aa-4fbd-841a-92cae43529c8",
-    }
     time_start = time.monotonic()
 
     tasks_created = 0
@@ -123,16 +115,6 @@ def check_for_indexing(self: Task, *, tenant_id: str | None) -> int | None:
 
         # kick off index attempts
         for cc_pair_id in cc_pair_ids:
-            # debugging logic - remove after we're done
-            if tenant_id in debug_tenants:
-                ttl = redis_client.ttl(OnyxRedisLocks.CHECK_INDEXING_BEAT_LOCK)
-                task_logger.info(
-                    f"check_for_indexing cc_pair lock: "
-                    f"tenant={tenant_id} "
-                    f"cc_pair={cc_pair_id} "
-                    f"ttl={ttl}"
-                )
-
             lock_beat.reacquire()
 
             redis_connector = RedisConnector(tenant_id, cc_pair_id)
@@ -141,30 +123,12 @@ def check_for_indexing(self: Task, *, tenant_id: str | None) -> int | None:
                     db_session
                 )
                 for search_settings_instance in search_settings_list:
-                    if tenant_id in debug_tenants:
-                        ttl = redis_client.ttl(OnyxRedisLocks.CHECK_INDEXING_BEAT_LOCK)
-                        task_logger.info(
-                            f"check_for_indexing cc_pair search settings lock: "
-                            f"tenant={tenant_id} "
-                            f"cc_pair={cc_pair_id} "
-                            f"ttl={ttl}"
-                        )
-
                     redis_connector_index = redis_connector.new_index(
                         search_settings_instance.id
                     )
                     if redis_connector_index.fenced:
                         continue
 
-                    if tenant_id in debug_tenants:
-                        ttl = redis_client.ttl(OnyxRedisLocks.CHECK_INDEXING_BEAT_LOCK)
-                        task_logger.info(
-                            f"check_for_indexing get_connector_credential_pair_from_id: "
-                            f"tenant={tenant_id} "
-                            f"cc_pair={cc_pair_id} "
-                            f"ttl={ttl}"
-                        )
-
                     cc_pair = get_connector_credential_pair_from_id(
                         db_session=db_session,
                         cc_pair_id=cc_pair_id,
@@ -172,28 +136,10 @@ def check_for_indexing(self: Task, *, tenant_id: str | None) -> int | None:
                     if not cc_pair:
                         continue
 
-                    if tenant_id in debug_tenants:
-                        ttl = redis_client.ttl(OnyxRedisLocks.CHECK_INDEXING_BEAT_LOCK)
-                        task_logger.info(
-                            f"check_for_indexing get_last_attempt_for_cc_pair: "
-                            f"tenant={tenant_id} "
-                            f"cc_pair={cc_pair_id} "
-                            f"ttl={ttl}"
-                        )
-
                     last_attempt = get_last_attempt_for_cc_pair(
                         cc_pair.id, search_settings_instance.id, db_session
                     )
 
-                    if tenant_id in debug_tenants:
-                        ttl = redis_client.ttl(OnyxRedisLocks.CHECK_INDEXING_BEAT_LOCK)
-                        task_logger.info(
-                            f"check_for_indexing cc_pair should index: "
-                            f"tenant={tenant_id} "
-                            f"cc_pair={cc_pair_id} "
-                            f"ttl={ttl}"
-                        )
-
                     search_settings_primary = False
                     if search_settings_instance.id == search_settings_list[0].id:
                         search_settings_primary = True
@@ -226,15 +172,6 @@ def check_for_indexing(self: Task, *, tenant_id: str | None) -> int | None:
                                 cc_pair.id, None, db_session
                             )
 
-                    if tenant_id in debug_tenants:
-                        ttl = redis_client.ttl(OnyxRedisLocks.CHECK_INDEXING_BEAT_LOCK)
-                        task_logger.info(
-                            f"check_for_indexing cc_pair try_creating_indexing_task: "
-                            f"tenant={tenant_id} "
-                            f"cc_pair={cc_pair_id} "
-                            f"ttl={ttl}"
-                        )
-
                     # using a task queue and only allowing one task per cc_pair/search_setting
                     # prevents us from starving out certain attempts
                     attempt_id = try_creating_indexing_task(
@@ -255,24 +192,6 @@ def check_for_indexing(self: Task, *, tenant_id: str | None) -> int | None:
                         )
                         tasks_created += 1
 
-                    if tenant_id in debug_tenants:
-                        ttl = redis_client.ttl(OnyxRedisLocks.CHECK_INDEXING_BEAT_LOCK)
-                        task_logger.info(
-                            f"check_for_indexing cc_pair try_creating_indexing_task finished: "
-                            f"tenant={tenant_id} "
-                            f"cc_pair={cc_pair_id} "
-                            f"ttl={ttl}"
-                        )
-
-        # debugging logic - remove after we're done
-        if tenant_id in debug_tenants:
-            ttl = redis_client.ttl(OnyxRedisLocks.CHECK_INDEXING_BEAT_LOCK)
-            task_logger.info(
-                f"check_for_indexing unfenced lock: "
-                f"tenant={tenant_id} "
-                f"ttl={ttl}"
-            )
-
         lock_beat.reacquire()
 
         # Fail any index attempts in the DB that don't have fences
@@ -282,24 +201,7 @@ def check_for_indexing(self: Task, *, tenant_id: str | None) -> int | None:
                 db_session, redis_client
             )
 
-            if tenant_id in debug_tenants:
-                ttl = redis_client.ttl(OnyxRedisLocks.CHECK_INDEXING_BEAT_LOCK)
-                task_logger.info(
-                    f"check_for_indexing after get unfenced lock: "
-                    f"tenant={tenant_id} "
-                    f"ttl={ttl}"
-                )
-
             for attempt_id in unfenced_attempt_ids:
-                # debugging logic - remove after we're done
-                if tenant_id in debug_tenants:
-                    ttl = redis_client.ttl(OnyxRedisLocks.CHECK_INDEXING_BEAT_LOCK)
-                    task_logger.info(
-                        f"check_for_indexing unfenced attempt id lock: "
-                        f"tenant={tenant_id} "
-                        f"ttl={ttl}"
-                    )
-
                 lock_beat.reacquire()
 
                 attempt = get_index_attempt(db_session, attempt_id)
@@ -317,15 +219,6 @@ def check_for_indexing(self: Task, *, tenant_id: str | None) -> int | None:
                     attempt.id, db_session, failure_reason=failure_reason
                 )
 
-        # debugging logic - remove after we're done
-        if tenant_id in debug_tenants:
-            ttl = redis_client.ttl(OnyxRedisLocks.CHECK_INDEXING_BEAT_LOCK)
-            task_logger.info(
-                f"check_for_indexing validate fences lock: "
-                f"tenant={tenant_id} "
-                f"ttl={ttl}"
-            )
-
         lock_beat.reacquire()
         # we want to run this less frequently than the overall task
         if not redis_client.exists(OnyxRedisSignals.VALIDATE_INDEXING_FENCES):
@@ -674,6 +567,9 @@ def connector_indexing_proxy_task(
     while True:
         sleep(5)
 
+        # renew watchdog signal (this has a shorter timeout than set_active)
+        redis_connector_index.set_watchdog(True)
+
         # renew active signal
         redis_connector_index.set_active()
 
@@ -780,67 +676,10 @@ def connector_indexing_proxy_task(
             )
             continue
 
+    redis_connector_index.set_watchdog(False)
     task_logger.info(
         f"Indexing watchdog - finished: attempt={index_attempt_id} "
         f"cc_pair={cc_pair_id} "
         f"search_settings={search_settings_id}"
     )
     return
-
-
-@shared_task(
-    name=OnyxCeleryTask.CLOUD_CHECK_FOR_INDEXING,
-    trail=False,
-    bind=True,
-)
-def cloud_check_for_indexing(self: Task) -> bool | None:
-    """a lightweight task used to kick off individual check tasks for each tenant."""
-    time_start = time.monotonic()
-
-    redis_client = get_redis_client(tenant_id=ONYX_CLOUD_TENANT_ID)
-
-    lock_beat: RedisLock = redis_client.lock(
-        OnyxRedisLocks.CLOUD_CHECK_INDEXING_BEAT_LOCK,
-        timeout=CELERY_GENERIC_BEAT_LOCK_TIMEOUT,
-    )
-
-    # these tasks should never overlap
-    if not lock_beat.acquire(blocking=False):
-        return None
-
-    last_lock_time = time.monotonic()
-
-    try:
-        tenant_ids = get_all_tenant_ids()
-        for tenant_id in tenant_ids:
-            current_time = time.monotonic()
-            if current_time - last_lock_time >= (CELERY_GENERIC_BEAT_LOCK_TIMEOUT / 4):
-                lock_beat.reacquire()
-                last_lock_time = current_time
-
-            self.app.send_task(
-                OnyxCeleryTask.CHECK_FOR_INDEXING,
-                kwargs=dict(
-                    tenant_id=tenant_id,
-                ),
-                priority=OnyxCeleryPriority.HIGH,
-                expires=BEAT_EXPIRES_DEFAULT,
-            )
-    except SoftTimeLimitExceeded:
-        task_logger.info(
-            "Soft time limit exceeded, task is being terminated gracefully."
-        )
-    except Exception:
-        task_logger.exception("Unexpected exception during cloud indexing check")
-    finally:
-        if lock_beat.owned():
-            lock_beat.release()
-        else:
-            task_logger.error("cloud_check_for_indexing - Lock not owned on completion")
-            redis_lock_dump(lock_beat, redis_client)
-
-    time_elapsed = time.monotonic() - time_start
-    task_logger.info(
-        f"cloud_check_for_indexing finished: num_tenants={len(tenant_ids)} elapsed={time_elapsed:.2f}"
-    )
-    return True

backend/onyx/background/celery/tasks/monitoring/tasks.py

@@ -1,6 +1,8 @@
 import json
+import time
 from collections.abc import Callable
 from datetime import timedelta
+from itertools import islice
 from typing import Any
 
 from celery import shared_task
@@ -10,13 +12,17 @@ from pydantic import BaseModel
 from redis import Redis
 from redis.lock import Lock as RedisLock
 from sqlalchemy import select
+from sqlalchemy import text
 from sqlalchemy.orm import Session
 
 from onyx.background.celery.apps.app_base import task_logger
 from onyx.background.celery.tasks.vespa.tasks import celery_get_queue_length
+from onyx.configs.constants import CELERY_GENERIC_BEAT_LOCK_TIMEOUT
+from onyx.configs.constants import ONYX_CLOUD_TENANT_ID
 from onyx.configs.constants import OnyxCeleryQueues
 from onyx.configs.constants import OnyxCeleryTask
 from onyx.configs.constants import OnyxRedisLocks
+from onyx.db.engine import get_all_tenant_ids
 from onyx.db.engine import get_db_current_time
 from onyx.db.engine import get_session_with_tenant
 from onyx.db.enums import IndexingStatus
@@ -26,7 +32,9 @@ from onyx.db.models import DocumentSet
 from onyx.db.models import IndexAttempt
 from onyx.db.models import SyncRecord
 from onyx.db.models import UserGroup
+from onyx.db.search_settings import get_active_search_settings
 from onyx.redis.redis_pool import get_redis_client
+from onyx.redis.redis_pool import redis_lock_dump
 from onyx.utils.telemetry import optional_telemetry
 from onyx.utils.telemetry import RecordType
 
@@ -177,6 +185,10 @@ def _build_connector_start_latency_metric(
 
     start_latency = (recent_attempt.time_started - desired_start_time).total_seconds()
 
+    task_logger.info(
+        f"Start latency for index attempt {recent_attempt.id}: {start_latency:.2f}s "
+        f"(desired: {desired_start_time}, actual: {recent_attempt.time_started})"
+    )
     return Metric(
         key=metric_key,
         name="connector_start_latency",
@@ -210,6 +222,9 @@ def _build_run_success_metrics(
             IndexingStatus.FAILED,
             IndexingStatus.CANCELED,
         ]:
+            task_logger.info(
+                f"Adding run success metric for index attempt {attempt.id} with status {attempt.status}"
+            )
             metrics.append(
                 Metric(
                     key=metric_key,
@@ -230,25 +245,29 @@ def _collect_connector_metrics(db_session: Session, redis_std: Redis) -> list[Me
     # Get all connector credential pairs
     cc_pairs = db_session.scalars(select(ConnectorCredentialPair)).all()
 
+    active_search_settings = get_active_search_settings(db_session)
     metrics = []
-    for cc_pair in cc_pairs:
-        # Get all attempts in the last hour
+
+    for cc_pair, search_settings in zip(cc_pairs, active_search_settings):
         recent_attempts = (
             db_session.query(IndexAttempt)
             .filter(
                 IndexAttempt.connector_credential_pair_id == cc_pair.id,
-                IndexAttempt.time_created >= one_hour_ago,
+                IndexAttempt.search_settings_id == search_settings.id,
             )
             .order_by(IndexAttempt.time_created.desc())
+            .limit(2)
             .all()
         )
-        most_recent_attempt = recent_attempts[0] if recent_attempts else None
+        if not recent_attempts:
+            continue
+
+        most_recent_attempt = recent_attempts[0]
         second_most_recent_attempt = (
             recent_attempts[1] if len(recent_attempts) > 1 else None
         )
 
-        # if no metric to emit, skip
-        if most_recent_attempt is None:
+        if one_hour_ago > most_recent_attempt.time_created:
             continue
 
         # Connector start latency
@@ -291,7 +310,7 @@ def _collect_sync_metrics(db_session: Session, redis_std: Redis) -> list[Metric]
             f"{sync_record.entity_id}:{sync_record.id}"
         )
         if _has_metric_been_emitted(redis_std, metric_key):
-            task_logger.debug(
+            task_logger.info(
                 f"Skipping metric for sync record {sync_record.id} "
                 "because it has already been emitted"
             )
@@ -311,11 +330,15 @@ def _collect_sync_metrics(db_session: Session, redis_std: Redis) -> list[Metric]
 
         if sync_speed is None:
             task_logger.error(
-                "Something went wrong with sync speed calculation. "
-                f"Sync record: {sync_record.id}"
+                f"Something went wrong with sync speed calculation. "
+                f"Sync record: {sync_record.id}, duration: {sync_duration_mins}, "
+                f"docs synced: {sync_record.num_docs_synced}"
             )
             continue
 
+        task_logger.info(
+            f"Calculated sync speed for record {sync_record.id}: {sync_speed} docs/min"
+        )
         metrics.append(
             Metric(
                 key=metric_key,
@@ -334,7 +357,7 @@ def _collect_sync_metrics(db_session: Session, redis_std: Redis) -> list[Metric]
             f":{sync_record.entity_id}:{sync_record.id}"
         )
         if _has_metric_been_emitted(redis_std, start_latency_key):
-            task_logger.debug(
+            task_logger.info(
                 f"Skipping start latency metric for sync record {sync_record.id} "
                 "because it has already been emitted"
             )
@@ -352,7 +375,7 @@ def _collect_sync_metrics(db_session: Session, redis_std: Redis) -> list[Metric]
             )
         else:
             # Skip other sync types
-            task_logger.debug(
+            task_logger.info(
                 f"Skipping sync record {sync_record.id} "
                 f"with type {sync_record.sync_type} "
                 f"and id {sync_record.entity_id} "
@@ -371,12 +394,15 @@ def _collect_sync_metrics(db_session: Session, redis_std: Redis) -> list[Metric]
         start_latency = (
             sync_record.sync_start_time - entity.time_last_modified_by_user
         ).total_seconds()
+        task_logger.info(
+            f"Calculated start latency for sync record {sync_record.id}: {start_latency} seconds"
+        )
         if start_latency < 0:
             task_logger.error(
                 f"Start latency is negative for sync record {sync_record.id} "
-                f"with type {sync_record.sync_type} and id {sync_record.entity_id}."
-                "This is likely because the entity was updated between the time the "
-                "time the sync finished and this job ran. Skipping."
+                f"with type {sync_record.sync_type} and id {sync_record.entity_id}. "
+                f"Sync start time: {sync_record.sync_start_time}, "
+                f"Entity last modified: {entity.time_last_modified_by_user}"
             )
             continue
 
@@ -456,3 +482,116 @@ def monitor_background_processes(self: Task, *, tenant_id: str | None) -> None:
             lock_monitoring.release()
 
         task_logger.info("Background monitoring task finished")
+
+
+@shared_task(
+    name=OnyxCeleryTask.CLOUD_CHECK_ALEMBIC,
+)
+def cloud_check_alembic() -> bool | None:
+    """A task to verify that all tenants are on the same alembic revision.
+
+    This check is expected to fail if a cloud alembic migration is currently running
+    across all tenants.
+
+    TODO: have the cloud migration script set an activity signal that this check
+    uses to know it doesn't make sense to run a check at the present time.
+    """
+    time_start = time.monotonic()
+
+    redis_client = get_redis_client(tenant_id=ONYX_CLOUD_TENANT_ID)
+
+    lock_beat: RedisLock = redis_client.lock(
+        OnyxRedisLocks.CLOUD_CHECK_ALEMBIC_BEAT_LOCK,
+        timeout=CELERY_GENERIC_BEAT_LOCK_TIMEOUT,
+    )
+
+    # these tasks should never overlap
+    if not lock_beat.acquire(blocking=False):
+        return None
+
+    last_lock_time = time.monotonic()
+
+    tenant_to_revision: dict[str, str | None] = {}
+    revision_counts: dict[str, int] = {}
+    out_of_date_tenants: dict[str, str | None] = {}
+    top_revision: str = ""
+
+    try:
+        # map each tenant_id to its revision
+        tenant_ids = get_all_tenant_ids()
+        for tenant_id in tenant_ids:
+            current_time = time.monotonic()
+            if current_time - last_lock_time >= (CELERY_GENERIC_BEAT_LOCK_TIMEOUT / 4):
+                lock_beat.reacquire()
+                last_lock_time = current_time
+
+            if tenant_id is None:
+                continue
+
+            with get_session_with_tenant(tenant_id=None) as session:
+                result = session.execute(
+                    text(f'SELECT * FROM "{tenant_id}".alembic_version LIMIT 1')
+                )
+
+                result_scalar: str | None = result.scalar_one_or_none()
+                tenant_to_revision[tenant_id] = result_scalar
+
+        # get the total count of each revision
+        for k, v in tenant_to_revision.items():
+            if v is None:
+                continue
+
+            revision_counts[v] = revision_counts.get(v, 0) + 1
+
+        # get the revision with the most counts
+        sorted_revision_counts = sorted(
+            revision_counts.items(), key=lambda item: item[1], reverse=True
+        )
+
+        if len(sorted_revision_counts) == 0:
+            task_logger.error(
+                f"cloud_check_alembic - No revisions found for {len(tenant_ids)} tenant ids!"
+            )
+        else:
+            top_revision, _ = sorted_revision_counts[0]
+
+            # build a list of out of date tenants
+            for k, v in tenant_to_revision.items():
+                if v == top_revision:
+                    continue
+
+                out_of_date_tenants[k] = v
+
+    except SoftTimeLimitExceeded:
+        task_logger.info(
+            "Soft time limit exceeded, task is being terminated gracefully."
+        )
+    except Exception:
+        task_logger.exception("Unexpected exception during cloud alembic check")
+        raise
+    finally:
+        if lock_beat.owned():
+            lock_beat.release()
+        else:
+            task_logger.error("cloud_check_alembic - Lock not owned on completion")
+            redis_lock_dump(lock_beat, redis_client)
+
+    if len(out_of_date_tenants) > 0:
+        task_logger.error(
+            f"Found out of date tenants: "
+            f"num_out_of_date_tenants={len(out_of_date_tenants)} "
+            f"num_tenants={len(tenant_ids)} "
+            f"revision={top_revision}"
+        )
+        for k, v in islice(out_of_date_tenants.items(), 5):
+            task_logger.info(f"Out of date tenant: tenant={k} revision={v}")
+    else:
+        task_logger.info(
+            f"All tenants are up to date: num_tenants={len(tenant_ids)} revision={top_revision}"
+        )
+
+    time_elapsed = time.monotonic() - time_start
+    task_logger.info(
+        f"cloud_check_alembic finished: num_tenants={len(tenant_ids)} elapsed={time_elapsed:.2f}"
+    )
+    return True

backend/onyx/background/celery/tasks/shared/tasks.py

@@ -1,15 +1,22 @@
+import time
 from http import HTTPStatus
 
 import httpx
 from celery import shared_task
 from celery import Task
 from celery.exceptions import SoftTimeLimitExceeded
+from redis.lock import Lock as RedisLock
 from tenacity import RetryError
 
 from onyx.access.access import get_access_for_document
 from onyx.background.celery.apps.app_base import task_logger
+from onyx.background.celery.tasks.beat_schedule import BEAT_EXPIRES_DEFAULT
 from onyx.background.celery.tasks.shared.RetryDocumentIndex import RetryDocumentIndex
+from onyx.configs.constants import CELERY_GENERIC_BEAT_LOCK_TIMEOUT
+from onyx.configs.constants import ONYX_CLOUD_TENANT_ID
+from onyx.configs.constants import OnyxCeleryPriority
 from onyx.configs.constants import OnyxCeleryTask
+from onyx.configs.constants import OnyxRedisLocks
 from onyx.db.document import delete_document_by_connector_credential_pair__no_commit
 from onyx.db.document import delete_documents_complete__no_commit
 from onyx.db.document import fetch_chunk_count_for_document
@@ -18,10 +25,13 @@ from onyx.db.document import get_document_connector_count
 from onyx.db.document import mark_document_as_modified
 from onyx.db.document import mark_document_as_synced
 from onyx.db.document_set import fetch_document_sets_for_document
+from onyx.db.engine import get_all_tenant_ids
 from onyx.db.engine import get_session_with_tenant
 from onyx.document_index.document_index_utils import get_both_index_names
 from onyx.document_index.factory import get_default_document_index
 from onyx.document_index.interfaces import VespaDocumentFields
+from onyx.redis.redis_pool import get_redis_client
+from onyx.redis.redis_pool import redis_lock_dump
 from onyx.server.documents.models import ConnectorCredentialPairIdentifier
 
 DOCUMENT_BY_CC_PAIR_CLEANUP_MAX_RETRIES = 3
@@ -199,3 +209,73 @@ def document_by_cc_pair_cleanup_task(
         return False
 
     return True
+
+
+@shared_task(
+    name=OnyxCeleryTask.CLOUD_BEAT_TASK_GENERATOR,
+    trail=False,
+    bind=True,
+)
+def cloud_beat_task_generator(
+    self: Task,
+    task_name: str,
+    queue: str = OnyxCeleryTask.DEFAULT,
+    priority: int = OnyxCeleryPriority.MEDIUM,
+    expires: int = BEAT_EXPIRES_DEFAULT,
+) -> bool | None:
+    """a lightweight task used to kick off individual beat tasks per tenant."""
+    time_start = time.monotonic()
+
+    redis_client = get_redis_client(tenant_id=ONYX_CLOUD_TENANT_ID)
+
+    lock_beat: RedisLock = redis_client.lock(
+        f"{OnyxRedisLocks.CLOUD_BEAT_TASK_GENERATOR_LOCK}:{task_name}",
+        timeout=CELERY_GENERIC_BEAT_LOCK_TIMEOUT,
+    )
+
+    # these tasks should never overlap
+    if not lock_beat.acquire(blocking=False):
+        return None
+
+    last_lock_time = time.monotonic()
+
+    try:
+        tenant_ids = get_all_tenant_ids()
+        for tenant_id in tenant_ids:
+            current_time = time.monotonic()
+            if current_time - last_lock_time >= (CELERY_GENERIC_BEAT_LOCK_TIMEOUT / 4):
+                lock_beat.reacquire()
+                last_lock_time = current_time
+
+            self.app.send_task(
+                task_name,
+                kwargs=dict(
+                    tenant_id=tenant_id,
+                ),
+                queue=queue,
+                priority=priority,
+                expires=expires,
+            )
+    except SoftTimeLimitExceeded:
+        task_logger.info(
+            "Soft time limit exceeded, task is being terminated gracefully."
+        )
+    except Exception:
+        task_logger.exception("Unexpected exception during cloud_beat_task_generator")
+    finally:
+        if not lock_beat.owned():
+            task_logger.error(
+                "cloud_beat_task_generator - Lock not owned on completion"
+            )
+            redis_lock_dump(lock_beat, redis_client)
+        else:
+            lock_beat.release()
+
+    time_elapsed = time.monotonic() - time_start
+    task_logger.info(
+        f"cloud_beat_task_generator finished: "
+        f"task={task_name} "
+        f"num_tenants={len(tenant_ids)} "
+        f"elapsed={time_elapsed:.2f}"
+    )
+    return True

backend/onyx/background/celery/tasks/vespa/tasks.py

@@ -735,7 +735,7 @@ def monitor_ccpair_indexing_taskset(
     composite_id = RedisConnector.get_id_from_fence_key(fence_key)
     if composite_id is None:
         task_logger.warning(
-            f"monitor_ccpair_indexing_taskset: could not parse composite_id from {fence_key}"
+            f"Connector indexing: could not parse composite_id from {fence_key}"
         )
         return
 
@@ -785,6 +785,7 @@ def monitor_ccpair_indexing_taskset(
     # inner/outer/inner double check pattern to avoid race conditions when checking for
     # bad state
 
+    # Verify: if the generator isn't complete, the task must not be in READY state
     # inner = get_completion / generator_complete not signaled
     # outer = result.state in READY state
     status_int = redis_connector_index.get_completion()
@@ -830,7 +831,7 @@ def monitor_ccpair_indexing_taskset(
                             )
                 except Exception:
                     task_logger.exception(
-                        "monitor_ccpair_indexing_taskset - transient exception marking index attempt as failed: "
+                        "Connector indexing - Transient exception marking index attempt as failed: "
                         f"attempt={payload.index_attempt_id} "
                         f"tenant={tenant_id} "
                         f"cc_pair={cc_pair_id} "
@@ -840,6 +841,20 @@ def monitor_ccpair_indexing_taskset(
                 redis_connector_index.reset()
         return
 
+    if redis_connector_index.watchdog_signaled():
+        # if the generator is complete, don't clean up until the watchdog has exited
+        task_logger.info(
+            f"Connector indexing - Delaying finalization until watchdog has exited: "
+            f"attempt={payload.index_attempt_id} "
+            f"cc_pair={cc_pair_id} "
+            f"search_settings={search_settings_id} "
+            f"progress={progress} "
+            f"elapsed_submitted={elapsed_submitted.total_seconds():.2f} "
+            f"elapsed_started={elapsed_started_str}"
+        )
+
+        return
+
     status_enum = HTTPStatus(status_int)
 
     task_logger.info(
@@ -858,9 +873,13 @@ def monitor_ccpair_indexing_taskset(
 
 @shared_task(name=OnyxCeleryTask.MONITOR_VESPA_SYNC, soft_time_limit=300, bind=True)
 def monitor_vespa_sync(self: Task, tenant_id: str | None) -> bool | None:
-    """This is a celery beat task that monitors and finalizes metadata sync tasksets.
+    """This is a celery beat task that monitors and finalizes various long running tasks.
+
+    The name monitor_vespa_sync is a bit of a misnomer since it checks many different tasks
+    now. Should change that at some point.
+
     It scans for fence values and then gets the counts of any associated tasksets.
-    If the count is 0, that means all tasks finished and we should clean up.
+    For many tasks, the count is 0, that means all tasks finished and we should clean up.
 
     This task lock timeout is CELERY_METADATA_SYNC_BEAT_LOCK_TIMEOUT seconds, so don't
     do anything too expensive in this function!
@@ -1045,6 +1064,8 @@ def monitor_vespa_sync(self: Task, tenant_id: str | None) -> bool | None:
 def vespa_metadata_sync_task(
     self: Task, document_id: str, tenant_id: str | None
 ) -> bool:
+    start = time.monotonic()
+
     try:
         with get_session_with_tenant(tenant_id) as db_session:
             curr_ind_name, sec_ind_name = get_both_index_names(db_session)
@@ -1095,7 +1116,13 @@ def vespa_metadata_sync_task(
             # r = get_redis_client(tenant_id=tenant_id)
             # r.delete(redis_syncing_key)
 
-            task_logger.info(f"doc={document_id} action=sync chunks={chunks_affected}")
+            elapsed = time.monotonic() - start
+            task_logger.info(
+                f"doc={document_id} "
+                f"action=sync "
+                f"chunks={chunks_affected} "
+                f"elapsed={elapsed:.2f}"
+            )
     except SoftTimeLimitExceeded:
         task_logger.info(f"SoftTimeLimitExceeded exception. doc={document_id}")
     except Exception as ex:

backend/onyx/chat/prompt_builder/answer_prompt_builder.py

@@ -18,8 +18,8 @@ from onyx.llm.utils import message_to_prompt_and_imgs
 from onyx.natural_language_processing.utils import get_tokenizer
 from onyx.prompts.chat_prompts import CHAT_USER_CONTEXT_FREE_PROMPT
 from onyx.prompts.direct_qa_prompts import HISTORY_BLOCK
-from onyx.prompts.prompt_utils import add_date_time_to_prompt
 from onyx.prompts.prompt_utils import drop_messages_history_overflow
+from onyx.prompts.prompt_utils import handle_onyx_date_awareness
 from onyx.tools.force import ForceUseTool
 from onyx.tools.models import ToolCallFinalResult
 from onyx.tools.models import ToolCallKickoff
@@ -31,15 +31,16 @@ def default_build_system_message(
     prompt_config: PromptConfig,
 ) -> SystemMessage | None:
     system_prompt = prompt_config.system_prompt.strip()
-    if prompt_config.datetime_aware:
-        system_prompt = add_date_time_to_prompt(prompt_str=system_prompt)
+    tag_handled_prompt = handle_onyx_date_awareness(
+        system_prompt,
+        prompt_config,
+        add_additional_info_if_no_tag=prompt_config.datetime_aware,
+    )
 
-    if not system_prompt:
+    if not tag_handled_prompt:
         return None
 
-    system_msg = SystemMessage(content=system_prompt)
-
-    return system_msg
+    return SystemMessage(content=tag_handled_prompt)
 
 
 def default_build_user_message(
@@ -64,8 +65,11 @@ def default_build_user_message(
         else user_query
     )
     user_prompt = user_prompt.strip()
+    tag_handled_prompt = handle_onyx_date_awareness(user_prompt, prompt_config)
     user_msg = HumanMessage(
-        content=build_content_with_imgs(user_prompt, files) if files else user_prompt
+        content=build_content_with_imgs(tag_handled_prompt, files)
+        if files
+        else tag_handled_prompt
     )
     return user_msg
 

backend/onyx/chat/prompt_builder/citations_prompt.py

@@ -21,9 +21,9 @@ from onyx.prompts.constants import DEFAULT_IGNORE_STATEMENT
 from onyx.prompts.direct_qa_prompts import CITATIONS_PROMPT
 from onyx.prompts.direct_qa_prompts import CITATIONS_PROMPT_FOR_TOOL_CALLING
 from onyx.prompts.direct_qa_prompts import HISTORY_BLOCK
-from onyx.prompts.prompt_utils import add_date_time_to_prompt
 from onyx.prompts.prompt_utils import build_complete_context_str
 from onyx.prompts.prompt_utils import build_task_prompt_reminders
+from onyx.prompts.prompt_utils import handle_onyx_date_awareness
 from onyx.prompts.token_counts import ADDITIONAL_INFO_TOKEN_CNT
 from onyx.prompts.token_counts import (
     CHAT_USER_PROMPT_WITH_CONTEXT_OVERHEAD_TOKEN_CNT,
@@ -127,10 +127,11 @@ def build_citations_system_message(
     system_prompt = prompt_config.system_prompt.strip()
     if prompt_config.include_citations:
         system_prompt += REQUIRE_CITATION_STATEMENT
-    if prompt_config.datetime_aware:
-        system_prompt = add_date_time_to_prompt(prompt_str=system_prompt)
+    tag_handled_prompt = handle_onyx_date_awareness(
+        system_prompt, prompt_config, add_additional_info_if_no_tag=True
+    )
 
-    return SystemMessage(content=system_prompt)
+    return SystemMessage(content=tag_handled_prompt)
 
 
 def build_citations_user_message(

backend/onyx/chat/prompt_builder/quotes_prompt.py

@@ -9,8 +9,8 @@ from onyx.llm.utils import message_to_prompt_and_imgs
 from onyx.prompts.direct_qa_prompts import CONTEXT_BLOCK
 from onyx.prompts.direct_qa_prompts import HISTORY_BLOCK
 from onyx.prompts.direct_qa_prompts import JSON_PROMPT
-from onyx.prompts.prompt_utils import add_date_time_to_prompt
 from onyx.prompts.prompt_utils import build_complete_context_str
+from onyx.prompts.prompt_utils import handle_onyx_date_awareness
 
 
 def _build_strong_llm_quotes_prompt(
@@ -39,10 +39,11 @@ def _build_strong_llm_quotes_prompt(
         language_hint_or_none=LANGUAGE_HINT.strip() if use_language_hint else "",
     ).strip()
 
-    if prompt.datetime_aware:
-        full_prompt = add_date_time_to_prompt(prompt_str=full_prompt)
+    tag_handled_prompt = handle_onyx_date_awareness(
+        full_prompt, prompt, add_additional_info_if_no_tag=True
+    )
 
-    return HumanMessage(content=full_prompt)
+    return HumanMessage(content=tag_handled_prompt)
 
 
 def build_quotes_user_message(

backend/onyx/configs/app_configs.py

@@ -92,6 +92,12 @@ OAUTH_CLIENT_SECRET = (
 
 USER_AUTH_SECRET = os.environ.get("USER_AUTH_SECRET", "")
 
+# Duration (in seconds) for which the FastAPI Users JWT token remains valid in the user's browser.
+# By default, this is set to match the Redis expiry time for consistency.
+AUTH_COOKIE_EXPIRE_TIME_SECONDS = int(
+    os.environ.get("AUTH_COOKIE_EXPIRE_TIME_SECONDS") or 86400 * 7
+)  # 7 days
+
 # for basic auth
 REQUIRE_EMAIL_VERIFICATION = (
     os.environ.get("REQUIRE_EMAIL_VERIFICATION", "").lower() == "true"

backend/onyx/configs/constants.py

@@ -294,7 +294,8 @@ class OnyxRedisLocks:
     SLACK_BOT_HEARTBEAT_PREFIX = "da_heartbeat:slack_bot"
     ANONYMOUS_USER_ENABLED = "anonymous_user_enabled"
 
-    CLOUD_CHECK_INDEXING_BEAT_LOCK = "da_lock:cloud_check_indexing_beat"
+    CLOUD_BEAT_TASK_GENERATOR_LOCK = "da_lock:cloud_beat_task_generator"
+    CLOUD_CHECK_ALEMBIC_BEAT_LOCK = "da_lock:cloud_check_alembic"
 
 
 class OnyxRedisSignals:
@@ -317,6 +318,11 @@ ONYX_CLOUD_TENANT_ID = "cloud"
 
 
 class OnyxCeleryTask:
+    DEFAULT = "celery"
+
+    CLOUD_BEAT_TASK_GENERATOR = f"{ONYX_CLOUD_CELERY_TASK_PREFIX}_generate_beat_tasks"
+    CLOUD_CHECK_ALEMBIC = f"{ONYX_CLOUD_CELERY_TASK_PREFIX}_check_alembic"
+
     CHECK_FOR_CONNECTOR_DELETION = "check_for_connector_deletion_task"
     CHECK_FOR_VESPA_SYNC_TASK = "check_for_vespa_sync_task"
     CHECK_FOR_INDEXING = "check_for_indexing"
@@ -324,8 +330,10 @@ class OnyxCeleryTask:
     CHECK_FOR_DOC_PERMISSIONS_SYNC = "check_for_doc_permissions_sync"
     CHECK_FOR_EXTERNAL_GROUP_SYNC = "check_for_external_group_sync"
     CHECK_FOR_LLM_MODEL_UPDATE = "check_for_llm_model_update"
+
     MONITOR_VESPA_SYNC = "monitor_vespa_sync"
     MONITOR_BACKGROUND_PROCESSES = "monitor_background_processes"
+
     KOMBU_MESSAGE_CLEANUP_TASK = "kombu_message_cleanup_task"
     CONNECTOR_PERMISSION_SYNC_GENERATOR_TASK = (
         "connector_permission_sync_generator_task"
@@ -343,8 +351,6 @@ class OnyxCeleryTask:
     CHECK_TTL_MANAGEMENT_TASK = "check_ttl_management_task"
     AUTOGENERATE_USAGE_REPORT_TASK = "autogenerate_usage_report_task"
 
-    CLOUD_CHECK_FOR_INDEXING = f"{ONYX_CLOUD_CELERY_TASK_PREFIX}_check_for_indexing"
-
 
 REDIS_SOCKET_KEEPALIVE_OPTIONS = {}
 REDIS_SOCKET_KEEPALIVE_OPTIONS[socket.TCP_KEEPINTVL] = 15

backend/onyx/connectors/airtable/airtable_connector.py

@@ -71,10 +71,20 @@ class AirtableConnector(LoadConnector):
         self.airtable_client = AirtableApi(credentials["airtable_access_token"])
         return None
 
-    def _get_field_value(self, field_info: Any, field_type: str) -> list[str]:
+    @staticmethod
+    def _extract_field_values(
+        field_id: str,
+        field_info: Any,
+        field_type: str,
+        base_id: str,
+        table_id: str,
+        view_id: str | None,
+        record_id: str,
+    ) -> list[tuple[str, str]]:
         """
-        Extract value(s) from a field regardless of its type.
-        Returns either a single string or list of strings for attachments.
+        Extract value(s) + links from a field regardless of its type.
+        Attachments are represented as multiple sections, and therefore
+        returned as a list of tuples (value, link).
         """
         if field_info is None:
             return []
@@ -85,8 +95,11 @@ class AirtableConnector(LoadConnector):
         if field_type == "multipleRecordLinks":
             return []
 
+        # default link to use for non-attachment fields
+        default_link = f"https://airtable.com/{base_id}/{table_id}/{record_id}"
+
         if field_type == "multipleAttachments":
-            attachment_texts: list[str] = []
+            attachment_texts: list[tuple[str, str]] = []
             for attachment in field_info:
                 url = attachment.get("url")
                 filename = attachment.get("filename", "")
@@ -109,6 +122,7 @@ class AirtableConnector(LoadConnector):
                 if attachment_content:
                     try:
                         file_ext = get_file_ext(filename)
+                        attachment_id = attachment["id"]
                         attachment_text = extract_file_text(
                             BytesIO(attachment_content),
                             filename,
@@ -116,7 +130,20 @@ class AirtableConnector(LoadConnector):
                             extension=file_ext,
                         )
                         if attachment_text:
-                            attachment_texts.append(f"{filename}:\n{attachment_text}")
+                            # slightly nicer loading experience if we can specify the view ID
+                            if view_id:
+                                attachment_link = (
+                                    f"https://airtable.com/{base_id}/{table_id}/{view_id}/{record_id}"
+                                    f"/{field_id}/{attachment_id}?blocks=hide"
+                                )
+                            else:
+                                attachment_link = (
+                                    f"https://airtable.com/{base_id}/{table_id}/{record_id}"
+                                    f"/{field_id}/{attachment_id}?blocks=hide"
+                                )
+                            attachment_texts.append(
+                                (f"{filename}:\n{attachment_text}", attachment_link)
+                            )
                     except Exception as e:
                         logger.warning(
                             f"Failed to process attachment {filename}: {str(e)}"
@@ -131,12 +158,12 @@ class AirtableConnector(LoadConnector):
                 combined.append(collab_name)
             if collab_email:
                 combined.append(f"({collab_email})")
-            return [" ".join(combined) if combined else str(field_info)]
+            return [(" ".join(combined) if combined else str(field_info), default_link)]
 
         if isinstance(field_info, list):
-            return [str(item) for item in field_info]
+            return [(item, default_link) for item in field_info]
 
-        return [str(field_info)]
+        return [(str(field_info), default_link)]
 
     def _should_be_metadata(self, field_type: str) -> bool:
         """Determine if a field type should be treated as metadata."""
@@ -144,10 +171,12 @@ class AirtableConnector(LoadConnector):
 
     def _process_field(
         self,
+        field_id: str,
         field_name: str,
         field_info: Any,
         field_type: str,
         table_id: str,
+        view_id: str | None,
         record_id: str,
     ) -> tuple[list[Section], dict[str, Any]]:
         """
@@ -165,12 +194,21 @@ class AirtableConnector(LoadConnector):
             return [], {}
 
         # Get the value(s) for the field
-        field_values = self._get_field_value(field_info, field_type)
-        if len(field_values) == 0:
+        field_value_and_links = self._extract_field_values(
+            field_id=field_id,
+            field_info=field_info,
+            field_type=field_type,
+            base_id=self.base_id,
+            table_id=table_id,
+            view_id=view_id,
+            record_id=record_id,
+        )
+        if len(field_value_and_links) == 0:
             return [], {}
 
         # Determine if it should be metadata or a section
         if self._should_be_metadata(field_type):
+            field_values = [value for value, _ in field_value_and_links]
             if len(field_values) > 1:
                 return [], {field_name: field_values}
             return [], {field_name: field_values[0]}
@@ -178,7 +216,7 @@ class AirtableConnector(LoadConnector):
         # Otherwise, create relevant sections
         sections = [
             Section(
-                link=f"https://airtable.com/{self.base_id}/{table_id}/{record_id}",
+                link=link,
                 text=(
                     f"{field_name}:\n"
                     "------------------------\n"
@@ -186,7 +224,7 @@ class AirtableConnector(LoadConnector):
                     "------------------------"
                 ),
             )
-            for text in field_values
+            for text, link in field_value_and_links
         ]
         return sections, {}
 
@@ -219,6 +257,7 @@ class AirtableConnector(LoadConnector):
         primary_field_value = (
             fields.get(primary_field_name) if primary_field_name else None
         )
+        view_id = table_schema.views[0].id if table_schema.views else None
 
         for field_schema in table_schema.fields:
             field_name = field_schema.name
@@ -226,10 +265,12 @@ class AirtableConnector(LoadConnector):
             field_type = field_schema.type
 
             field_sections, field_metadata = self._process_field(
+                field_id=field_schema.id,
                 field_name=field_name,
                 field_info=field_val,
                 field_type=field_type,
                 table_id=table_id,
+                view_id=view_id,
                 record_id=record_id,
             )
 

backend/onyx/db/index_attempt.py

@@ -432,7 +432,7 @@ def get_paginated_index_attempts_for_cc_pair_id(
     stmt = stmt.order_by(IndexAttempt.time_started.desc())
 
     # Apply pagination
-    stmt = stmt.offset((page - 1) * page_size).limit(page_size)
+    stmt = stmt.offset(page * page_size).limit(page_size)
 
     return list(db_session.execute(stmt).scalars().all())
 

backend/onyx/db/models.py

@@ -1430,6 +1430,8 @@ class Tool(Base):
     user_id: Mapped[UUID | None] = mapped_column(
         ForeignKey("user.id", ondelete="CASCADE"), nullable=True
     )
+    # whether to pass through the user's OAuth token as Authorization header
+    passthrough_auth: Mapped[bool] = mapped_column(Boolean, default=False)
 
     user: Mapped[User | None] = relationship("User", back_populates="custom_tools")
     # Relationship to Persona through the association table

backend/onyx/db/slack_channel_config.py

@@ -15,6 +15,7 @@ from onyx.db.models import User
 from onyx.db.persona import mark_persona_as_deleted
 from onyx.db.persona import upsert_persona
 from onyx.db.prompts import get_default_prompt
+from onyx.tools.built_in_tools import get_search_tool
 from onyx.utils.errors import EERequiredError
 from onyx.utils.variable_functionality import (
     fetch_versioned_implementation_with_fallback,
@@ -47,6 +48,10 @@ def create_slack_channel_persona(
 ) -> Persona:
     """NOTE: does not commit changes"""
 
+    search_tool = get_search_tool(db_session)
+    if search_tool is None:
+        raise ValueError("Search tool not found")
+
     # create/update persona associated with the Slack channel
     persona_name = _build_persona_name(channel_name)
     default_prompt = get_default_prompt(db_session)
@@ -60,6 +65,7 @@ def create_slack_channel_persona(
         llm_filter_extraction=enable_auto_filters,
         recency_bias=RecencyBiasSetting.AUTO,
         prompt_ids=[default_prompt.id],
+        tool_ids=[search_tool.id],
         document_set_ids=document_set_ids,
         llm_model_provider_override=None,
         llm_model_version_override=None,

backend/onyx/db/tools.py

@@ -38,6 +38,7 @@ def create_tool(
     custom_headers: list[Header] | None,
     user_id: UUID | None,
     db_session: Session,
+    passthrough_auth: bool,
 ) -> Tool:
     new_tool = Tool(
         name=name,
@@ -48,6 +49,7 @@ def create_tool(
         if custom_headers
         else [],
         user_id=user_id,
+        passthrough_auth=passthrough_auth,
     )
     db_session.add(new_tool)
     db_session.commit()
@@ -62,6 +64,7 @@ def update_tool(
     custom_headers: list[Header] | None,
     user_id: UUID | None,
     db_session: Session,
+    passthrough_auth: bool | None,
 ) -> Tool:
     tool = get_tool_by_id(tool_id, db_session)
     if tool is None:
@@ -79,6 +82,8 @@ def update_tool(
         tool.custom_headers = [
             cast(HeaderItemDict, header.model_dump()) for header in custom_headers
         ]
+    if passthrough_auth is not None:
+        tool.passthrough_auth = passthrough_auth
     db_session.commit()
 
     return tool

backend/onyx/document_index/vespa/shared_utils/utils.py

@@ -1,4 +1,5 @@
 import re
+import time
 from typing import cast
 
 import httpx
@@ -7,6 +8,10 @@ from onyx.configs.app_configs import MANAGED_VESPA
 from onyx.configs.app_configs import VESPA_CLOUD_CERT_PATH
 from onyx.configs.app_configs import VESPA_CLOUD_KEY_PATH
 from onyx.configs.app_configs import VESPA_REQUEST_TIMEOUT
+from onyx.document_index.vespa_constants import VESPA_APP_CONTAINER_URL
+from onyx.utils.logger import setup_logger
+
+logger = setup_logger()
 
 # NOTE: This does not seem to be used in reality despite the Vespa Docs pointing to this code
 # See here for reference: https://docs.vespa.ai/en/documents.html
@@ -69,3 +74,37 @@ def get_vespa_http_client(no_timeout: bool = False, http2: bool = True) -> httpx
         timeout=None if no_timeout else VESPA_REQUEST_TIMEOUT,
         http2=http2,
     )
+
+
+def wait_for_vespa_with_timeout(wait_interval: int = 5, wait_limit: int = 60) -> bool:
+    """Waits for Vespa to become ready subject to a timeout.
+    Returns True if Vespa is ready, False otherwise."""
+
+    time_start = time.monotonic()
+    logger.info("Vespa: Readiness probe starting.")
+    while True:
+        try:
+            client = get_vespa_http_client()
+            response = client.get(f"{VESPA_APP_CONTAINER_URL}/state/v1/health")
+            response.raise_for_status()
+
+            response_dict = response.json()
+            if response_dict["status"]["code"] == "up":
+                logger.info("Vespa: Readiness probe succeeded. Continuing...")
+                return True
+        except Exception:
+            pass
+
+        time_elapsed = time.monotonic() - time_start
+        if time_elapsed > wait_limit:
+            logger.info(
+                f"Vespa: Readiness probe did not succeed within the timeout "
+                f"({wait_limit} seconds)."
+            )
+            return False
+
+        logger.info(
+            f"Vespa: Readiness probe ongoing. elapsed={time_elapsed:.1f} timeout={wait_limit:.1f}"
+        )
+
+        time.sleep(wait_interval)

backend/onyx/key_value_store/factory.py

@@ -2,7 +2,7 @@ from onyx.key_value_store.interface import KeyValueStore
 from onyx.key_value_store.store import PgRedisKVStore
 
 
-def get_kv_store() -> KeyValueStore:
+def get_kv_store(tenant_id: str | None = None) -> KeyValueStore:
     # In the Multi Tenant case, the tenant context is picked up automatically, it does not need to be passed in
     # It's read from the global thread level variable
-    return PgRedisKVStore()
+    return PgRedisKVStore(tenant_id=tenant_id)

backend/onyx/key_value_store/store.py

@@ -31,27 +31,27 @@ class PgRedisKVStore(KeyValueStore):
     def __init__(
         self, redis_client: Redis | None = None, tenant_id: str | None = None
     ) -> None:
+        self.tenant_id = tenant_id or CURRENT_TENANT_ID_CONTEXTVAR.get()
+
         # If no redis_client is provided, fall back to the context var
         if redis_client is not None:
             self.redis_client = redis_client
         else:
-            tenant_id = tenant_id or CURRENT_TENANT_ID_CONTEXTVAR.get()
-            self.redis_client = get_redis_client(tenant_id=tenant_id)
+            self.redis_client = get_redis_client(tenant_id=self.tenant_id)
 
     @contextmanager
-    def get_session(self) -> Iterator[Session]:
+    def _get_session(self) -> Iterator[Session]:
         engine = get_sqlalchemy_engine()
         with Session(engine, expire_on_commit=False) as session:
             if MULTI_TENANT:
-                tenant_id = CURRENT_TENANT_ID_CONTEXTVAR.get()
-                if tenant_id == POSTGRES_DEFAULT_SCHEMA:
+                if self.tenant_id == POSTGRES_DEFAULT_SCHEMA:
                     raise HTTPException(
                         status_code=401, detail="User must authenticate"
                     )
-                if not is_valid_schema_name(tenant_id):
+                if not is_valid_schema_name(self.tenant_id):
                     raise HTTPException(status_code=400, detail="Invalid tenant ID")
                 # Set the search_path to the tenant's schema
-                session.execute(text(f'SET search_path = "{tenant_id}"'))
+                session.execute(text(f'SET search_path = "{self.tenant_id}"'))
             yield session
 
     def store(self, key: str, val: JSON_ro, encrypt: bool = False) -> None:
@@ -66,7 +66,7 @@ class PgRedisKVStore(KeyValueStore):
 
         encrypted_val = val if encrypt else None
         plain_val = val if not encrypt else None
-        with self.get_session() as session:
+        with self._get_session() as session:
             obj = session.query(KVStore).filter_by(key=key).first()
             if obj:
                 obj.value = plain_val
@@ -88,7 +88,7 @@ class PgRedisKVStore(KeyValueStore):
         except Exception as e:
             logger.error(f"Failed to get value from Redis for key '{key}': {str(e)}")
 
-        with self.get_session() as session:
+        with self._get_session() as session:
             obj = session.query(KVStore).filter_by(key=key).first()
             if not obj:
                 raise KvKeyNotFoundError
@@ -113,7 +113,7 @@ class PgRedisKVStore(KeyValueStore):
         except Exception as e:
             logger.error(f"Failed to delete value from Redis for key '{key}': {str(e)}")
 
-        with self.get_session() as session:
+        with self._get_session() as session:
             result = session.query(KVStore).filter_by(key=key).delete()  # type: ignore
             if result == 0:
                 raise KvKeyNotFoundError

backend/onyx/llm/chat_llm.py

@@ -275,17 +275,22 @@ class DefaultMultiLLM(LLM):
         # addtional kwargs (and some kwargs MUST be passed in rather than set as
         # env variables)
         if custom_config:
-            # Specifically pass in "vertex_credentials" as a model_kwarg to the
-            # completion call for vertex AI. More details here:
+            # Specifically pass in "vertex_credentials" / "vertex_location" as a
+            # model_kwarg to the completion call for vertex AI. More details here:
             # https://docs.litellm.ai/docs/providers/vertex
             vertex_credentials_key = "vertex_credentials"
-            vertex_credentials = custom_config.get(vertex_credentials_key)
-            if vertex_credentials and model_provider == "vertex_ai":
-                model_kwargs[vertex_credentials_key] = vertex_credentials
-            else:
-                # standard case
-                for k, v in custom_config.items():
-                    os.environ[k] = v
+            vertex_location_key = "vertex_location"
+            for k, v in custom_config.items():
+                if model_provider == "vertex_ai":
+                    if k == vertex_credentials_key:
+                        model_kwargs[k] = v
+                        continue
+                    elif k == vertex_location_key:
+                        model_kwargs[k] = v
+                        continue
+
+                # for all values, set them as env variables
+                os.environ[k] = v
 
         if extra_headers:
             model_kwargs.update({"extra_headers": extra_headers})

backend/onyx/main.py

@@ -212,7 +212,7 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
 
     if not MULTI_TENANT:
         # We cache this at the beginning so there is no delay in the first telemetry
-        get_or_generate_uuid()
+        get_or_generate_uuid(tenant_id=None)
 
         # If we are multi-tenant, we need to only set up initial public tables
         with Session(engine) as db_session:

backend/onyx/onyxbot/slack/listener.py

@@ -14,6 +14,7 @@ from typing import Set
 
 from prometheus_client import Gauge
 from prometheus_client import start_http_server
+from redis.lock import Lock
 from slack_sdk import WebClient
 from slack_sdk.socket_mode.request import SocketModeRequest
 from slack_sdk.socket_mode.response import SocketModeResponse
@@ -122,6 +123,9 @@ class SlackbotHandler:
         self.socket_clients: Dict[tuple[str | None, int], TenantSocketModeClient] = {}
         self.slack_bot_tokens: Dict[tuple[str | None, int], SlackBotTokens] = {}
 
+        # Store Redis lock objects here so we can release them properly
+        self.redis_locks: Dict[str | None, Lock] = {}
+
         self.running = True
         self.pod_id = self.get_pod_id()
         self._shutdown_event = Event()
@@ -159,10 +163,15 @@ class SlackbotHandler:
         while not self._shutdown_event.is_set():
             try:
                 self.acquire_tenants()
+
+                # After we finish acquiring and managing Slack bots,
+                # set the gauge to the number of active tenants (those with Slack bots).
                 active_tenants_gauge.labels(namespace=POD_NAMESPACE, pod=POD_NAME).set(
                     len(self.tenant_ids)
                 )
-                logger.debug(f"Current active tenants: {len(self.tenant_ids)}")
+                logger.debug(
+                    f"Current active tenants with Slack bots: {len(self.tenant_ids)}"
+                )
             except Exception as e:
                 logger.exception(f"Error in Slack acquisition: {e}")
             self._shutdown_event.wait(timeout=TENANT_ACQUISITION_INTERVAL)
@@ -171,7 +180,9 @@ class SlackbotHandler:
         while not self._shutdown_event.is_set():
             try:
                 self.send_heartbeats()
-                logger.debug(f"Sent heartbeats for {len(self.tenant_ids)} tenants")
+                logger.debug(
+                    f"Sent heartbeats for {len(self.tenant_ids)} active tenants"
+                )
             except Exception as e:
                 logger.exception(f"Error in heartbeat loop: {e}")
             self._shutdown_event.wait(timeout=TENANT_HEARTBEAT_INTERVAL)
@@ -179,17 +190,21 @@ class SlackbotHandler:
     def _manage_clients_per_tenant(
         self, db_session: Session, tenant_id: str | None, bot: SlackBot
     ) -> None:
+        """
+        - If the tokens are missing or empty, close the socket client and remove them.
+        - If the tokens have changed, close the existing socket client and reconnect.
+        - If the tokens are new, warm up the model and start a new socket client.
+        """
         slack_bot_tokens = SlackBotTokens(
             bot_token=bot.bot_token,
             app_token=bot.app_token,
         )
         tenant_bot_pair = (tenant_id, bot.id)
 
-        # If the tokens are not set, we need to close the socket client and delete the tokens
-        # for the tenant and app
+        # If the tokens are missing or empty, close the socket client and remove them.
         if not slack_bot_tokens:
             logger.debug(
-                f"No Slack bot token found for tenant {tenant_id}, bot {bot.id}"
+                f"No Slack bot tokens found for tenant={tenant_id}, bot {bot.id}"
             )
             if tenant_bot_pair in self.socket_clients:
                 asyncio.run(self.socket_clients[tenant_bot_pair].close())
@@ -204,9 +219,10 @@ class SlackbotHandler:
         if not tokens_exist or tokens_changed:
             if tokens_exist:
                 logger.info(
-                    f"Slack Bot tokens have changed for tenant {tenant_id}, bot {bot.id} - reconnecting"
+                    f"Slack Bot tokens changed for tenant={tenant_id}, bot {bot.id}; reconnecting"
                 )
             else:
+                # Warm up the model if needed
                 search_settings = get_current_search_settings(db_session)
                 embedding_model = EmbeddingModel.from_db_model(
                     search_settings=search_settings,
@@ -217,77 +233,168 @@ class SlackbotHandler:
 
             self.slack_bot_tokens[tenant_bot_pair] = slack_bot_tokens
 
+            # Close any existing connection first
             if tenant_bot_pair in self.socket_clients:
                 asyncio.run(self.socket_clients[tenant_bot_pair].close())
 
             self.start_socket_client(bot.id, tenant_id, slack_bot_tokens)
 
     def acquire_tenants(self) -> None:
-        tenant_ids = get_all_tenant_ids()
+        """
+        - Attempt to acquire a Redis lock for each tenant.
+        - If acquired, check if that tenant actually has Slack bots.
+        - If yes, store them in self.tenant_ids and manage the socket connections.
+        - If a tenant in self.tenant_ids no longer has Slack bots, remove it (and release the lock in this scope).
+        """
+        all_tenants = get_all_tenant_ids()
 
-        for tenant_id in tenant_ids:
+        # 1) Try to acquire locks for new tenants
+        for tenant_id in all_tenants:
             if (
                 DISALLOWED_SLACK_BOT_TENANT_LIST is not None
                 and tenant_id in DISALLOWED_SLACK_BOT_TENANT_LIST
             ):
-                logger.debug(f"Tenant {tenant_id} is in the disallowed list, skipping")
+                logger.debug(f"Tenant {tenant_id} is disallowed; skipping.")
                 continue
 
+            # Already acquired in a previous loop iteration?
             if tenant_id in self.tenant_ids:
-                logger.debug(f"Tenant {tenant_id} already in self.tenant_ids")
                 continue
 
+            # Respect max tenant limit per pod
             if len(self.tenant_ids) >= MAX_TENANTS_PER_POD:
                 logger.info(
-                    f"Max tenants per pod reached ({MAX_TENANTS_PER_POD}) Not acquiring any more tenants"
+                    f"Max tenants per pod reached ({MAX_TENANTS_PER_POD}); not acquiring more."
                 )
                 break
 
             redis_client = get_redis_client(tenant_id=tenant_id)
-            pod_id = self.pod_id
-            acquired = redis_client.set(
-                OnyxRedisLocks.SLACK_BOT_LOCK,
-                pod_id,
-                nx=True,
-                ex=TENANT_LOCK_EXPIRATION,
+            # Acquire a Redis lock (non-blocking)
+            rlock = redis_client.lock(
+                OnyxRedisLocks.SLACK_BOT_LOCK, timeout=TENANT_LOCK_EXPIRATION
             )
-            if not acquired and not DEV_MODE:
-                logger.debug(f"Another pod holds the lock for tenant {tenant_id}")
+            lock_acquired = rlock.acquire(blocking=False)
+
+            if not lock_acquired and not DEV_MODE:
+                logger.debug(
+                    f"Another pod holds the lock for tenant {tenant_id}, skipping."
+                )
                 continue
 
-            logger.debug(f"Acquired lock for tenant {tenant_id}")
+            if lock_acquired:
+                logger.debug(f"Acquired lock for tenant {tenant_id}.")
+                self.redis_locks[tenant_id] = rlock
+            else:
+                # DEV_MODE will skip the lock acquisition guard
+                logger.debug(
+                    f"Running in DEV_MODE. Not enforcing lock for {tenant_id}."
+                )
 
-            self.tenant_ids.add(tenant_id)
-
-        for tenant_id in self.tenant_ids:
+            # Now check if this tenant actually has Slack bots
             token = CURRENT_TENANT_ID_CONTEXTVAR.set(
                 tenant_id or POSTGRES_DEFAULT_SCHEMA
             )
             try:
                 with get_session_with_tenant(tenant_id) as db_session:
+                    bots: list[SlackBot] = []
                     try:
-                        bots = fetch_slack_bots(db_session=db_session)
+                        bots = list(fetch_slack_bots(db_session=db_session))
+                    except KvKeyNotFoundError:
+                        # No Slackbot tokens, pass
+                        pass
+                    except Exception as e:
+                        logger.exception(
+                            f"Error fetching Slack bots for tenant {tenant_id}: {e}"
+                        )
+
+                    if bots:
+                        # Mark as active tenant
+                        self.tenant_ids.add(tenant_id)
                         for bot in bots:
                             self._manage_clients_per_tenant(
                                 db_session=db_session,
                                 tenant_id=tenant_id,
                                 bot=bot,
                             )
-
-                    except KvKeyNotFoundError:
-                        logger.debug(f"Missing Slack Bot tokens for tenant {tenant_id}")
-                        if (tenant_id, bot.id) in self.socket_clients:
-                            asyncio.run(self.socket_clients[tenant_id, bot.id].close())
-                            del self.socket_clients[tenant_id, bot.id]
-                            del self.slack_bot_tokens[tenant_id, bot.id]
-                    except Exception as e:
-                        logger.exception(f"Error handling tenant {tenant_id}: {e}")
+                    else:
+                        # If no Slack bots, release lock immediately (unless in DEV_MODE)
+                        if lock_acquired and not DEV_MODE:
+                            rlock.release()
+                            del self.redis_locks[tenant_id]
+                        logger.debug(
+                            f"No Slack bots for tenant {tenant_id}; lock released (if held)."
+                        )
             finally:
                 CURRENT_TENANT_ID_CONTEXTVAR.reset(token)
 
+        # 2) Make sure tenants we're handling still have Slack bots
+        for tenant_id in list(self.tenant_ids):
+            token = CURRENT_TENANT_ID_CONTEXTVAR.set(
+                tenant_id or POSTGRES_DEFAULT_SCHEMA
+            )
+            redis_client = get_redis_client(tenant_id=tenant_id)
+
+            try:
+                with get_session_with_tenant(tenant_id) as db_session:
+                    # Attempt to fetch Slack bots
+                    try:
+                        bots = list(fetch_slack_bots(db_session=db_session))
+                    except KvKeyNotFoundError:
+                        # No Slackbot tokens, pass (and remove below)
+                        bots = []
+                    except Exception as e:
+                        logger.exception(f"Error handling tenant {tenant_id}: {e}")
+                        bots = []
+
+                    if not bots:
+                        logger.info(
+                            f"Tenant {tenant_id} no longer has Slack bots. Removing."
+                        )
+                        self._remove_tenant(tenant_id)
+
+                        # NOTE: We release the lock here (in the same scope it was acquired)
+                        if tenant_id in self.redis_locks and not DEV_MODE:
+                            try:
+                                self.redis_locks[tenant_id].release()
+                                del self.redis_locks[tenant_id]
+                                logger.info(f"Released lock for tenant {tenant_id}")
+                            except Exception as e:
+                                logger.error(
+                                    f"Error releasing lock for tenant {tenant_id}: {e}"
+                                )
+                    else:
+                        # Manage or reconnect Slack bot sockets
+                        for bot in bots:
+                            self._manage_clients_per_tenant(
+                                db_session=db_session,
+                                tenant_id=tenant_id,
+                                bot=bot,
+                            )
+            finally:
+                CURRENT_TENANT_ID_CONTEXTVAR.reset(token)
+
+    def _remove_tenant(self, tenant_id: str | None) -> None:
+        """
+        Helper to remove a tenant from `self.tenant_ids` and close any socket clients.
+        (Lock release now happens in `acquire_tenants()`, not here.)
+        """
+        # Close all socket clients for this tenant
+        for (t_id, slack_bot_id), client in list(self.socket_clients.items()):
+            if t_id == tenant_id:
+                asyncio.run(client.close())
+                del self.socket_clients[(t_id, slack_bot_id)]
+                del self.slack_bot_tokens[(t_id, slack_bot_id)]
+                logger.info(
+                    f"Stopped SocketModeClient for tenant: {t_id}, app: {slack_bot_id}"
+                )
+
+        # Remove from active set
+        if tenant_id in self.tenant_ids:
+            self.tenant_ids.remove(tenant_id)
+
     def send_heartbeats(self) -> None:
         current_time = int(time.time())
-        logger.debug(f"Sending heartbeats for {len(self.tenant_ids)} tenants")
+        logger.debug(f"Sending heartbeats for {len(self.tenant_ids)} active tenants")
         for tenant_id in self.tenant_ids:
             redis_client = get_redis_client(tenant_id=tenant_id)
             heartbeat_key = f"{OnyxRedisLocks.SLACK_BOT_HEARTBEAT_PREFIX}:{self.pod_id}"
@@ -315,6 +422,7 @@ class SlackbotHandler:
         )
         socket_client.connect()
         self.socket_clients[tenant_id, slack_bot_id] = socket_client
+        # Ensure tenant is tracked as active
         self.tenant_ids.add(tenant_id)
         logger.info(
             f"Started SocketModeClient for tenant: {tenant_id}, app: {slack_bot_id}"
@@ -322,7 +430,7 @@ class SlackbotHandler:
 
     def stop_socket_clients(self) -> None:
         logger.info(f"Stopping {len(self.socket_clients)} socket clients")
-        for (tenant_id, slack_bot_id), client in self.socket_clients.items():
+        for (tenant_id, slack_bot_id), client in list(self.socket_clients.items()):
             asyncio.run(client.close())
             logger.info(
                 f"Stopped SocketModeClient for tenant: {tenant_id}, app: {slack_bot_id}"
@@ -340,17 +448,19 @@ class SlackbotHandler:
         logger.info(f"Stopping {len(self.socket_clients)} socket clients")
         self.stop_socket_clients()
 
-        # Release locks for all tenants
+        # Release locks for all tenants we currently hold
         logger.info(f"Releasing locks for {len(self.tenant_ids)} tenants")
-        for tenant_id in self.tenant_ids:
-            try:
-                redis_client = get_redis_client(tenant_id=tenant_id)
-                redis_client.delete(OnyxRedisLocks.SLACK_BOT_LOCK)
-                logger.info(f"Released lock for tenant {tenant_id}")
-            except Exception as e:
-                logger.error(f"Error releasing lock for tenant {tenant_id}: {e}")
+        for tenant_id in list(self.tenant_ids):
+            if tenant_id in self.redis_locks:
+                try:
+                    self.redis_locks[tenant_id].release()
+                    logger.info(f"Released lock for tenant {tenant_id}")
+                except Exception as e:
+                    logger.error(f"Error releasing lock for tenant {tenant_id}: {e}")
+                finally:
+                    del self.redis_locks[tenant_id]
 
-        # Wait for background threads to finish (with timeout)
+        # Wait for background threads to finish (with a timeout)
         logger.info("Waiting for background threads to finish...")
         self.acquire_thread.join(timeout=5)
         self.heartbeat_thread.join(timeout=5)

backend/onyx/prompts/prompt_utils.py

@@ -19,9 +19,8 @@ from onyx.utils.logger import setup_logger
 logger = setup_logger()
 
 
-MOST_BASIC_PROMPT = "You are a helpful AI assistant."
-DANSWER_DATETIME_REPLACEMENT = "DANSWER_DATETIME_REPLACEMENT"
-BASIC_TIME_STR = "The current date is {datetime_info}."
+_DANSWER_DATETIME_REPLACEMENT_PAT = "[[CURRENT_DATETIME]]"
+_BASIC_TIME_STR = "The current date is {datetime_info}."
 
 
 def get_current_llm_day_time(
@@ -38,23 +37,36 @@ def get_current_llm_day_time(
     return f"{formatted_datetime}"
 
 
-def add_date_time_to_prompt(prompt_str: str) -> str:
-    if DANSWER_DATETIME_REPLACEMENT in prompt_str:
+def build_date_time_string() -> str:
+    return ADDITIONAL_INFO.format(
+        datetime_info=_BASIC_TIME_STR.format(datetime_info=get_current_llm_day_time())
+    )
+
+
+def handle_onyx_date_awareness(
+    prompt_str: str,
+    prompt_config: PromptConfig,
+    add_additional_info_if_no_tag: bool = False,
+) -> str:
+    """
+    If there is a [[CURRENT_DATETIME]] tag, replace it with the current date and time no matter what.
+    If the prompt is datetime aware, and there are no [[CURRENT_DATETIME]] tags, add it to the prompt.
+    do nothing otherwise.
+    This can later be expanded to support other tags.
+    """
+
+    if _DANSWER_DATETIME_REPLACEMENT_PAT in prompt_str:
         return prompt_str.replace(
-            DANSWER_DATETIME_REPLACEMENT,
+            _DANSWER_DATETIME_REPLACEMENT_PAT,
             get_current_llm_day_time(full_sentence=False, include_day_of_week=True),
         )
-
-    if prompt_str:
-        return prompt_str + ADDITIONAL_INFO.format(
-            datetime_info=get_current_llm_day_time()
-        )
-    else:
-        return (
-            MOST_BASIC_PROMPT
-            + " "
-            + BASIC_TIME_STR.format(datetime_info=get_current_llm_day_time())
-        )
+    any_tag_present = any(
+        _DANSWER_DATETIME_REPLACEMENT_PAT in text
+        for text in [prompt_str, prompt_config.system_prompt, prompt_config.task_prompt]
+    )
+    if add_additional_info_if_no_tag and not any_tag_present:
+        return prompt_str + build_date_time_string()
+    return prompt_str
 
 
 def build_task_prompt_reminders(

backend/onyx/redis/redis_connector_index.py

@@ -30,10 +30,17 @@ class RedisConnectorIndex:
     GENERATOR_LOCK_PREFIX = "da_lock:indexing"
 
     TERMINATE_PREFIX = PREFIX + "_terminate"  # connectorindexing_terminate
+    TERMINATE_TTL = 600
 
     # used to signal the overall workflow is still active
-    # it's difficult to prevent
+    # there are gaps in time between states where we need some slack
+    # to correctly transition
     ACTIVE_PREFIX = PREFIX + "_active"
+    ACTIVE_TTL = 3600
+
+    # used to signal that the watchdog is running
+    WATCHDOG_PREFIX = PREFIX + "_watchdog"
+    WATCHDOG_TTL = 300
 
     def __init__(
         self,
@@ -59,6 +66,7 @@ class RedisConnectorIndex:
         )
         self.terminate_key = f"{self.TERMINATE_PREFIX}_{id}/{search_settings_id}"
         self.active_key = f"{self.ACTIVE_PREFIX}_{id}/{search_settings_id}"
+        self.watchdog_key = f"{self.WATCHDOG_PREFIX}_{id}/{search_settings_id}"
 
     @classmethod
     def fence_key_with_ids(cls, cc_pair_id: int, search_settings_id: int) -> str:
@@ -110,7 +118,24 @@ class RedisConnectorIndex:
         """This sets a signal. It does not block!"""
         # We shouldn't need very long to terminate the spawned task.
         # 10 minute TTL is good.
-        self.redis.set(f"{self.terminate_key}_{celery_task_id}", 0, ex=600)
+        self.redis.set(
+            f"{self.terminate_key}_{celery_task_id}", 0, ex=self.TERMINATE_TTL
+        )
+
+    def set_watchdog(self, value: bool) -> None:
+        """Signal the state of the watchdog."""
+        if not value:
+            self.redis.delete(self.watchdog_key)
+            return
+
+        self.redis.set(self.watchdog_key, 0, ex=self.WATCHDOG_TTL)
+
+    def watchdog_signaled(self) -> bool:
+        """Check the state of the watchdog."""
+        if self.redis.exists(self.watchdog_key):
+            return True
+
+        return False
 
     def set_active(self) -> None:
         """This sets a signal to keep the indexing flow from getting cleaned up within
@@ -118,7 +143,7 @@ class RedisConnectorIndex:
 
         The slack in timing is needed to avoid race conditions where simply checking
         the celery queue and task status could result in race conditions."""
-        self.redis.set(self.active_key, 0, ex=3600)
+        self.redis.set(self.active_key, 0, ex=self.ACTIVE_TTL)
 
     def active(self) -> bool:
         if self.redis.exists(self.active_key):

backend/onyx/seeding/load_docs.py

@@ -26,6 +26,7 @@ from onyx.db.index_attempt import mock_successful_index_attempt
 from onyx.db.search_settings import get_current_search_settings
 from onyx.document_index.factory import get_default_document_index
 from onyx.document_index.interfaces import IndexBatchParams
+from onyx.document_index.vespa.shared_utils.utils import wait_for_vespa_with_timeout
 from onyx.indexing.indexing_pipeline import index_doc_batch_prepare
 from onyx.indexing.models import ChunkEmbedding
 from onyx.indexing.models import DocMetadataAwareIndexChunk
@@ -33,7 +34,6 @@ from onyx.key_value_store.factory import get_kv_store
 from onyx.key_value_store.interface import KvKeyNotFoundError
 from onyx.server.documents.models import ConnectorBase
 from onyx.utils.logger import setup_logger
-from onyx.utils.retry_wrapper import retry_builder
 from onyx.utils.variable_functionality import fetch_versioned_implementation
 
 logger = setup_logger()
@@ -218,9 +218,11 @@ def seed_initial_documents(
 
     # Retries here because the index may take a few seconds to become ready
     # as we just sent over the Vespa schema and there is a slight delay
+    if not wait_for_vespa_with_timeout():
+        logger.error("Vespa did not become ready within the timeout")
+        raise ValueError("Vespa failed to become ready within the timeout")
 
-    index_with_retries = retry_builder(tries=15)(document_index.index)
-    index_with_retries(
+    document_index.index(
         chunks=chunks,
         index_batch_params=IndexBatchParams(
             doc_id_to_previous_chunk_cnt={},

backend/onyx/seeding/prompts.yaml

@@ -8,7 +8,7 @@ prompts:
     # System Prompt (as shown in UI)
     system: >
       You are a question answering system that is constantly learning and improving.
-      The current date is DANSWER_DATETIME_REPLACEMENT.
+      The current date is [[CURRENT_DATETIME]].
 
       You can process and comprehend vast amounts of text and utilize this knowledge to provide
       grounded, accurate, and concise answers to diverse queries.
@@ -24,7 +24,7 @@ prompts:
 
       If there are no relevant documents, refer to the chat history and your internal knowledge.
     # Inject a statement at the end of system prompt to inform the LLM of the current date/time
-    # If the DANSWER_DATETIME_REPLACEMENT is set, the date/time is inserted there instead
+    # If the [[CURRENT_DATETIME]] is set, the date/time is inserted there instead
     # Format looks like: "October 16, 2023 14:30"
     datetime_aware: true
     # Prompts the LLM to include citations in the for [1], [2] etc.
@@ -51,7 +51,7 @@ prompts:
   - name: "OnlyLLM"
     description: "Chat directly with the LLM!"
     system: >
-      You are a helpful AI assistant. The current date is DANSWER_DATETIME_REPLACEMENT
+      You are a helpful AI assistant. The current date is [[CURRENT_DATETIME]]
 
 
       You give concise responses to very simple questions, but provide more thorough responses to
@@ -69,7 +69,7 @@ prompts:
     system: >
       You are a text summarizing assistant that highlights the most important knowledge from the
       context provided, prioritizing the information that relates to the user query.
-      The current date is DANSWER_DATETIME_REPLACEMENT.
+      The current date is [[CURRENT_DATETIME]].
 
       You ARE NOT creative and always stick to the provided documents.
       If there are no documents, refer to the conversation history.
@@ -87,7 +87,7 @@ prompts:
     description: "Recites information from retrieved context! Least creative but most safe!"
     system: >
       Quote and cite relevant information from provided context based on the user query.
-      The current date is DANSWER_DATETIME_REPLACEMENT.
+      The current date is [[CURRENT_DATETIME]].
 
       You only provide quotes that are EXACT substrings from provided documents!
 

backend/onyx/server/documents/cc_pair.py

@@ -62,7 +62,7 @@ router = APIRouter(prefix="/manage")
 @router.get("/admin/cc-pair/{cc_pair_id}/index-attempts")
 def get_cc_pair_index_attempts(
     cc_pair_id: int,
-    page: int = Query(1, ge=1),
+    page_num: int = Query(0, ge=0),
     page_size: int = Query(10, ge=1, le=1000),
     user: User | None = Depends(current_curator_or_admin_user),
     db_session: Session = Depends(get_session),
@@ -81,7 +81,7 @@ def get_cc_pair_index_attempts(
     index_attempts = get_paginated_index_attempts_for_cc_pair_id(
         db_session=db_session,
         connector_id=cc_pair.connector_id,
-        page=page,
+        page=page_num,
         page_size=page_size,
     )
     return PaginatedReturn(

backend/onyx/server/features/persona/api.py

@@ -7,6 +7,7 @@ from fastapi import HTTPException
 from fastapi import Query
 from fastapi import UploadFile
 from pydantic import BaseModel
+from sqlalchemy.exc import IntegrityError
 from sqlalchemy.orm import Session
 
 from onyx.auth.users import current_admin_user
@@ -191,8 +192,7 @@ def create_persona(
         name=build_prompt_name_from_persona_name(persona_upsert_request.name),
         system_prompt=persona_upsert_request.system_prompt,
         task_prompt=persona_upsert_request.task_prompt,
-        # TODO: The PersonaUpsertRequest should provide the value for datetime_aware
-        datetime_aware=False,
+        datetime_aware=persona_upsert_request.datetime_aware,
         include_citations=persona_upsert_request.include_citations,
         prompt_id=prompt_id,
     )
@@ -236,8 +236,7 @@ def update_persona(
         db_session=db_session,
         user=user,
         name=build_prompt_name_from_persona_name(persona_upsert_request.name),
-        # TODO: The PersonaUpsertRequest should provide the value for datetime_aware
-        datetime_aware=False,
+        datetime_aware=persona_upsert_request.datetime_aware,
         system_prompt=persona_upsert_request.system_prompt,
         task_prompt=persona_upsert_request.task_prompt,
         include_citations=persona_upsert_request.include_citations,
@@ -277,8 +276,14 @@ def create_label(
     _: User | None = Depends(current_user),
 ) -> PersonaLabelResponse:
     """Create a new assistant label"""
-    label_model = create_assistant_label(name=label.name, db_session=db)
-    return PersonaLabelResponse.from_model(label_model)
+    try:
+        label_model = create_assistant_label(name=label.name, db_session=db)
+        return PersonaLabelResponse.from_model(label_model)
+    except IntegrityError:
+        raise HTTPException(
+            status_code=400,
+            detail=f"Label with name '{label.name}' already exists. Please choose a different name.",
+        )
 
 
 @admin_router.patch("/label/{label_id}")

backend/onyx/server/features/persona/models.py

@@ -60,6 +60,7 @@ class PersonaUpsertRequest(BaseModel):
     description: str
     system_prompt: str
     task_prompt: str
+    datetime_aware: bool
     document_set_ids: list[int]
     num_chunks: float
     include_citations: bool

backend/onyx/server/features/tool/api.py

@@ -41,6 +41,16 @@ def _validate_tool_definition(definition: dict[str, Any]) -> None:
         raise HTTPException(status_code=400, detail=str(e))
 
 
+def _validate_auth_settings(tool_data: CustomToolCreate | CustomToolUpdate) -> None:
+    if tool_data.passthrough_auth and tool_data.custom_headers:
+        for header in tool_data.custom_headers:
+            if header.key.lower() == "authorization":
+                raise HTTPException(
+                    status_code=400,
+                    detail="Cannot use passthrough auth with custom authorization headers",
+                )
+
+
 @admin_router.post("/custom")
 def create_custom_tool(
     tool_data: CustomToolCreate,
@@ -48,6 +58,7 @@ def create_custom_tool(
     user: User | None = Depends(current_admin_user),
 ) -> ToolSnapshot:
     _validate_tool_definition(tool_data.definition)
+    _validate_auth_settings(tool_data)
     tool = create_tool(
         name=tool_data.name,
         description=tool_data.description,
@@ -55,6 +66,7 @@ def create_custom_tool(
         custom_headers=tool_data.custom_headers,
         user_id=user.id if user else None,
         db_session=db_session,
+        passthrough_auth=tool_data.passthrough_auth,
     )
     return ToolSnapshot.from_model(tool)
 
@@ -68,6 +80,7 @@ def update_custom_tool(
 ) -> ToolSnapshot:
     if tool_data.definition:
         _validate_tool_definition(tool_data.definition)
+    _validate_auth_settings(tool_data)
     updated_tool = update_tool(
         tool_id=tool_id,
         name=tool_data.name,
@@ -76,6 +89,7 @@ def update_custom_tool(
         custom_headers=tool_data.custom_headers,
         user_id=user.id if user else None,
         db_session=db_session,
+        passthrough_auth=tool_data.passthrough_auth,
     )
     return ToolSnapshot.from_model(updated_tool)
 

backend/onyx/server/features/tool/models.py

@@ -13,6 +13,7 @@ class ToolSnapshot(BaseModel):
     display_name: str
     in_code_tool_id: str | None
     custom_headers: list[Any] | None
+    passthrough_auth: bool
 
     @classmethod
     def from_model(cls, tool: Tool) -> "ToolSnapshot":
@@ -24,6 +25,7 @@ class ToolSnapshot(BaseModel):
             display_name=tool.display_name or tool.name,
             in_code_tool_id=tool.in_code_tool_id,
             custom_headers=tool.custom_headers,
+            passthrough_auth=tool.passthrough_auth,
         )
 
 
@@ -37,6 +39,7 @@ class CustomToolCreate(BaseModel):
     description: str | None = None
     definition: dict[str, Any]
     custom_headers: list[Header] | None = None
+    passthrough_auth: bool
 
 
 class CustomToolUpdate(BaseModel):
@@ -44,3 +47,4 @@ class CustomToolUpdate(BaseModel):
     description: str | None = None
     definition: dict[str, Any] | None = None
     custom_headers: list[Header] | None = None
+    passthrough_auth: bool | None = None

backend/onyx/server/manage/users.py

@@ -714,7 +714,6 @@ def update_user_pinned_assistants(
             store = get_kv_store()
             no_auth_user = fetch_no_auth_user(store)
             no_auth_user.preferences.pinned_assistants = ordered_assistant_ids
-            print("ordered_assistant_ids", ordered_assistant_ids)
             set_no_auth_user_preferences(store, no_auth_user.preferences)
             return
         else:

backend/onyx/server/query_and_chat/chat_backend.py

@@ -5,7 +5,6 @@ import os
 import uuid
 from collections.abc import Callable
 from collections.abc import Generator
-from typing import Tuple
 from uuid import UUID
 
 from fastapi import APIRouter
@@ -15,7 +14,6 @@ from fastapi import Request
 from fastapi import Response
 from fastapi import UploadFile
 from fastapi.responses import StreamingResponse
-from PIL import Image
 from pydantic import BaseModel
 from sqlalchemy.orm import Session
 
@@ -595,21 +593,6 @@ def seed_chat_from_slack(
 """File upload"""
 
 
-def convert_to_jpeg(file: UploadFile) -> Tuple[io.BytesIO, str]:
-    try:
-        with Image.open(file.file) as img:
-            if img.mode != "RGB":
-                img = img.convert("RGB")
-            jpeg_io = io.BytesIO()
-            img.save(jpeg_io, format="JPEG", quality=85)
-            jpeg_io.seek(0)
-        return jpeg_io, "image/jpeg"
-    except Exception as e:
-        raise HTTPException(
-            status_code=400, detail=f"Failed to convert image: {str(e)}"
-        )
-
-
 @router.post("/file")
 def upload_files_for_chat(
     files: list[UploadFile],
@@ -645,6 +628,9 @@ def upload_files_for_chat(
     )
 
     for file in files:
+        if not file.content_type:
+            raise HTTPException(status_code=400, detail="File content type is required")
+
         if file.content_type not in allowed_content_types:
             if file.content_type in image_content_types:
                 error_detail = "Unsupported image file type. Supported image types include .jpg, .jpeg, .png, .webp."
@@ -676,22 +662,27 @@ def upload_files_for_chat(
 
     file_info: list[tuple[str, str | None, ChatFileType]] = []
     for file in files:
-        if file.content_type in image_content_types:
-            file_type = ChatFileType.IMAGE
-            # Convert image to JPEG
-            file_content, new_content_type = convert_to_jpeg(file)
-        elif file.content_type in csv_content_types:
-            file_type = ChatFileType.CSV
-            file_content = io.BytesIO(file.file.read())
-            new_content_type = file.content_type or ""
-        elif file.content_type in document_content_types:
-            file_type = ChatFileType.DOC
-            file_content = io.BytesIO(file.file.read())
-            new_content_type = file.content_type or ""
+        file_type = (
+            ChatFileType.IMAGE
+            if file.content_type in image_content_types
+            else ChatFileType.CSV
+            if file.content_type in csv_content_types
+            else ChatFileType.DOC
+            if file.content_type in document_content_types
+            else ChatFileType.PLAIN_TEXT
+        )
+
+        if file_type == ChatFileType.IMAGE:
+            file_content = file.file
+            # NOTE: Image conversion to JPEG used to be enforced here.
+            # This was removed to:
+            # 1. Preserve original file content for downloads
+            # 2. Maintain transparency in formats like PNG
+            # 3. Ameliorate issue with file conversion
         else:
-            file_type = ChatFileType.PLAIN_TEXT
             file_content = io.BytesIO(file.file.read())
-            new_content_type = file.content_type or ""
+
+        new_content_type = file.content_type
 
         # store the file (now JPEG for images)
         file_id = str(uuid.uuid4())

backend/onyx/tools/built_in_tools.py

@@ -104,13 +104,10 @@ def load_builtin_tools(db_session: Session) -> None:
     logger.notice("All built-in tools are loaded/verified.")
 
 
-def auto_add_search_tool_to_personas(db_session: Session) -> None:
+def get_search_tool(db_session: Session) -> ToolDBModel | None:
     """
-    Automatically adds the SearchTool to all Persona objects in the database that have
-    `num_chunks` either unset or set to a value that isn't 0. This is done to migrate
-    Persona objects that were created before the concept of Tools were added.
+    Retrieves for the SearchTool from the BUILT_IN_TOOLS list.
     """
-    # Fetch the SearchTool from the database based on in_code_tool_id from BUILT_IN_TOOLS
     search_tool_id = next(
         (
             tool["in_code_tool_id"]
@@ -119,6 +116,7 @@ def auto_add_search_tool_to_personas(db_session: Session) -> None:
         ),
         None,
     )
+
     if not search_tool_id:
         raise RuntimeError("SearchTool not found in the BUILT_IN_TOOLS list.")
 
@@ -126,6 +124,18 @@ def auto_add_search_tool_to_personas(db_session: Session) -> None:
         select(ToolDBModel).where(ToolDBModel.in_code_tool_id == search_tool_id)
     ).scalar_one_or_none()
 
+    return search_tool
+
+
+def auto_add_search_tool_to_personas(db_session: Session) -> None:
+    """
+    Automatically adds the SearchTool to all Persona objects in the database that have
+    `num_chunks` either unset or set to a value that isn't 0. This is done to migrate
+    Persona objects that were created before the concept of Tools were added.
+    """
+    # Fetch the SearchTool from the database based on in_code_tool_id from BUILT_IN_TOOLS
+    search_tool = get_search_tool(db_session)
+
     if not search_tool:
         raise RuntimeError("SearchTool not found in the database.")
 

backend/onyx/tools/tool_constructor.py

@@ -146,6 +146,11 @@ def construct_tools(
     """Constructs tools based on persona configuration and available APIs"""
     tool_dict: dict[int, list[Tool]] = {}
 
+    # Get user's OAuth token if available
+    user_oauth_token = None
+    if user and user.oauth_accounts:
+        user_oauth_token = user.oauth_accounts[0].access_token
+
     for db_tool_model in persona.tools:
         if db_tool_model.in_code_tool_id:
             tool_cls = get_built_in_tool_by_id(
@@ -236,6 +241,9 @@ def construct_tools(
                             custom_tool_config.additional_headers or {}
                         )
                     ),
+                    user_oauth_token=(
+                        user_oauth_token if db_tool_model.passthrough_auth else None
+                    ),
                 ),
             )
 

backend/onyx/tools/tool_implementations/custom/custom_tool.py

@@ -80,10 +80,12 @@ class CustomTool(BaseTool):
         method_spec: MethodSpec,
         base_url: str,
         custom_headers: list[HeaderItemDict] | None = None,
+        user_oauth_token: str | None = None,
     ) -> None:
         self._base_url = base_url
         self._method_spec = method_spec
         self._tool_definition = self._method_spec.to_tool_definition()
+        self._user_oauth_token = user_oauth_token
 
         self._name = self._method_spec.name
         self._description = self._method_spec.summary
@@ -91,6 +93,20 @@ class CustomTool(BaseTool):
             header_list_to_header_dict(custom_headers) if custom_headers else {}
         )
 
+        # Check for both Authorization header and OAuth token
+        has_auth_header = any(
+            key.lower() == "authorization" for key in self.headers.keys()
+        )
+        if has_auth_header and self._user_oauth_token:
+            logger.warning(
+                f"Tool '{self._name}' has both an Authorization "
+                "header and OAuth token set. This is likely a configuration "
+                "error as the OAuth token will override the custom header."
+            )
+
+        if self._user_oauth_token:
+            self.headers["Authorization"] = f"Bearer {self._user_oauth_token}"
+
     @property
     def name(self) -> str:
         return self._name
@@ -348,6 +364,7 @@ def build_custom_tools_from_openapi_schema_and_headers(
     openapi_schema: dict[str, Any],
     custom_headers: list[HeaderItemDict] | None = None,
     dynamic_schema_info: DynamicSchemaInfo | None = None,
+    user_oauth_token: str | None = None,
 ) -> list[CustomTool]:
     if dynamic_schema_info:
         # Process dynamic schema information
@@ -366,7 +383,13 @@ def build_custom_tools_from_openapi_schema_and_headers(
     url = openapi_to_url(openapi_schema)
     method_specs = openapi_to_method_specs(openapi_schema)
     return [
-        CustomTool(method_spec, url, custom_headers) for method_spec in method_specs
+        CustomTool(
+            method_spec,
+            url,
+            custom_headers,
+            user_oauth_token=user_oauth_token,
+        )
+        for method_spec in method_specs
     ]
 
 

backend/onyx/utils/long_term_log.py

@@ -35,12 +35,16 @@ class LongTermLogger:
     def _cleanup_old_files(self, category_path: Path) -> None:
         try:
             files = sorted(
-                [f for f in category_path.glob("*.json") if f.is_file()],
+                [f for f in category_path.glob("*.json")],
                 key=lambda x: x.stat().st_mtime,  # Sort by modification time
                 reverse=True,
             )
+
             # Delete oldest files that exceed the limit
             for file in files[self.max_files_per_category :]:
+                if not file.is_file():
+                    logger.debug(f"File already deleted: {file}")
+                    continue
                 try:
                     file.unlink()
                 except Exception as e:

backend/onyx/utils/telemetry.py

@@ -11,7 +11,7 @@ from onyx.configs.app_configs import ENTERPRISE_EDITION_ENABLED
 from onyx.configs.constants import KV_CUSTOMER_UUID_KEY
 from onyx.configs.constants import KV_INSTANCE_DOMAIN_KEY
 from onyx.configs.constants import MilestoneRecordType
-from onyx.db.engine import get_sqlalchemy_engine
+from onyx.db.engine import get_session_with_tenant
 from onyx.db.milestone import create_milestone_if_not_exists
 from onyx.db.models import User
 from onyx.key_value_store.factory import get_kv_store
@@ -41,7 +41,7 @@ def _get_or_generate_customer_id_mt(tenant_id: str) -> str:
     return str(uuid.uuid5(uuid.NAMESPACE_X500, tenant_id))
 
 
-def get_or_generate_uuid(tenant_id: str | None = None) -> str:
+def get_or_generate_uuid(tenant_id: str | None) -> str:
     # TODO: split out the whole "instance UUID" generation logic into a separate
     # utility function. Telemetry should not be aware at all of how the UUID is
     # generated/stored.
@@ -52,7 +52,7 @@ def get_or_generate_uuid(tenant_id: str | None = None) -> str:
     if _CACHED_UUID is not None:
         return _CACHED_UUID
 
-    kv_store = get_kv_store()
+    kv_store = get_kv_store(tenant_id=tenant_id)
 
     try:
         _CACHED_UUID = cast(str, kv_store.load(KV_CUSTOMER_UUID_KEY))
@@ -63,18 +63,18 @@ def get_or_generate_uuid(tenant_id: str | None = None) -> str:
     return _CACHED_UUID
 
 
-def _get_or_generate_instance_domain() -> str | None:  #
+def _get_or_generate_instance_domain(tenant_id: str | None = None) -> str | None:  #
     global _CACHED_INSTANCE_DOMAIN
 
     if _CACHED_INSTANCE_DOMAIN is not None:
         return _CACHED_INSTANCE_DOMAIN
 
-    kv_store = get_kv_store()
+    kv_store = get_kv_store(tenant_id=tenant_id)
 
     try:
         _CACHED_INSTANCE_DOMAIN = cast(str, kv_store.load(KV_INSTANCE_DOMAIN_KEY))
     except KvKeyNotFoundError:
-        with Session(get_sqlalchemy_engine()) as db_session:
+        with get_session_with_tenant(tenant_id=tenant_id) as db_session:
             first_user = db_session.query(User).first()
             if first_user:
                 _CACHED_INSTANCE_DOMAIN = first_user.email.split("@")[-1]
@@ -94,16 +94,16 @@ def optional_telemetry(
     if DISABLE_TELEMETRY:
         return
 
+    tenant_id = tenant_id or CURRENT_TENANT_ID_CONTEXTVAR.get()
+
     try:
 
         def telemetry_logic() -> None:
             try:
                 customer_uuid = (
-                    _get_or_generate_customer_id_mt(
-                        tenant_id or CURRENT_TENANT_ID_CONTEXTVAR.get()
-                    )
+                    _get_or_generate_customer_id_mt(tenant_id)
                     if MULTI_TENANT
-                    else get_or_generate_uuid()
+                    else get_or_generate_uuid(tenant_id)
                 )
                 payload = {
                     "data": data,
@@ -115,7 +115,9 @@ def optional_telemetry(
                     "is_cloud": MULTI_TENANT,
                 }
                 if ENTERPRISE_EDITION_ENABLED:
-                    payload["instance_domain"] = _get_or_generate_instance_domain()
+                    payload["instance_domain"] = _get_or_generate_instance_domain(
+                        tenant_id
+                    )
                 requests.post(
                     _DANSWER_TELEMETRY_ENDPOINT,
                     headers={"Content-Type": "application/json"},

backend/scripts/dev_run_background_jobs.py

@@ -72,6 +72,19 @@ def run_jobs() -> None:
         "--queues=connector_indexing",
     ]
 
+    cmd_worker_monitoring = [
+        "celery",
+        "-A",
+        "onyx.background.celery.versioned_apps.monitoring",
+        "worker",
+        "--pool=threads",
+        "--concurrency=1",
+        "--prefetch-multiplier=1",
+        "--loglevel=INFO",
+        "--hostname=monitoring@%n",
+        "--queues=monitoring",
+    ]
+
     cmd_beat = [
         "celery",
         "-A",
@@ -97,6 +110,13 @@ def run_jobs() -> None:
         cmd_worker_indexing, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True
     )
 
+    worker_monitoring_process = subprocess.Popen(
+        cmd_worker_monitoring,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+        text=True,
+    )
+
     beat_process = subprocess.Popen(
         cmd_beat, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True
     )
@@ -114,18 +134,23 @@ def run_jobs() -> None:
     worker_indexing_thread = threading.Thread(
         target=monitor_process, args=("INDEX", worker_indexing_process)
     )
+    worker_monitoring_thread = threading.Thread(
+        target=monitor_process, args=("MONITORING", worker_monitoring_process)
+    )
     beat_thread = threading.Thread(target=monitor_process, args=("BEAT", beat_process))
 
     worker_primary_thread.start()
     worker_light_thread.start()
     worker_heavy_thread.start()
     worker_indexing_thread.start()
+    worker_monitoring_thread.start()
     beat_thread.start()
 
     worker_primary_thread.join()
     worker_light_thread.join()
     worker_heavy_thread.join()
     worker_indexing_thread.join()
+    worker_monitoring_thread.join()
     beat_thread.join()
 
 

backend/tests/daily/connectors/airtable/test_airtable_basic.py

@@ -45,7 +45,7 @@ def create_test_document(
     submitted_by: str,
     assignee: str,
     days_since_status_change: int | None,
-    attachments: list | None = None,
+    attachments: list[tuple[str, str]] | None = None,
 ) -> Document:
     link_base = f"https://airtable.com/{os.environ['AIRTABLE_TEST_BASE_ID']}/{os.environ['AIRTABLE_TEST_TABLE_ID']}"
     sections = [
@@ -60,11 +60,11 @@ def create_test_document(
     ]
 
     if attachments:
-        for attachment in attachments:
+        for attachment_text, attachment_link in attachments:
             sections.append(
                 Section(
-                    text=f"Attachment:\n------------------------\n{attachment}\n------------------------",
-                    link=f"{link_base}/{id}",
+                    text=f"Attachment:\n------------------------\n{attachment_text}\n------------------------",
+                    link=attachment_link,
                 ),
             )
 
@@ -142,7 +142,13 @@ def test_airtable_connector_basic(
             days_since_status_change=0,
             assignee="Chris Weaver (chris@onyx.app)",
             submitted_by="Chris Weaver (chris@onyx.app)",
-            attachments=["Test.pdf:\ntesting!!!"],
+            attachments=[
+                (
+                    "Test.pdf:\ntesting!!!",
+                    # hard code link for now
+                    "https://airtable.com/appCXJqDFS4gea8tn/tblRxFQsTlBBZdRY1/viwVUEJjWPd8XYjh8/reccSlIA4pZEFxPBg/fld1u21zkJACIvAEF/attlj2UBWNEDZngCc?blocks=hide",
+                )
+            ],
         ),
     ]
 

backend/tests/daily/connectors/zendesk/test_zendesk_connector.py

@@ -38,6 +38,12 @@ def get_credentials() -> dict[str, str]:
     }
 
 
+@pytest.mark.xfail(
+    reason=(
+        "Cannot get Zendesk developer account to ensure zendesk account does not "
+        "expire after 2 weeks"
+    )
+)
 @pytest.mark.parametrize(
     "connector_fixture", ["zendesk_article_connector", "zendesk_ticket_connector"]
 )
@@ -96,6 +102,12 @@ def test_zendesk_connector_basic(
         )
 
 
+@pytest.mark.xfail(
+    reason=(
+        "Cannot get Zendesk developer account to ensure zendesk account does not "
+        "expire after 2 weeks"
+    )
+)
 def test_zendesk_connector_slim(zendesk_article_connector: ZendeskConnector) -> None:
     # Get full doc IDs
     all_full_doc_ids = set()

backend/tests/integration/common_utils/managers/index_attempt.py

@@ -0,0 +1,106 @@
+from datetime import datetime
+from datetime import timedelta
+from urllib.parse import urlencode
+
+import requests
+
+from onyx.db.engine import get_session_context_manager
+from onyx.db.enums import IndexModelStatus
+from onyx.db.models import IndexAttempt
+from onyx.db.models import IndexingStatus
+from onyx.db.search_settings import get_current_search_settings
+from onyx.server.documents.models import IndexAttemptSnapshot
+from onyx.server.documents.models import PaginatedReturn
+from tests.integration.common_utils.constants import API_SERVER_URL
+from tests.integration.common_utils.constants import GENERAL_HEADERS
+from tests.integration.common_utils.test_models import DATestIndexAttempt
+from tests.integration.common_utils.test_models import DATestUser
+
+
+class IndexAttemptManager:
+    @staticmethod
+    def create_test_index_attempts(
+        num_attempts: int,
+        cc_pair_id: int,
+        from_beginning: bool = False,
+        status: IndexingStatus = IndexingStatus.SUCCESS,
+        new_docs_indexed: int = 10,
+        total_docs_indexed: int = 10,
+        docs_removed_from_index: int = 0,
+        error_msg: str | None = None,
+        base_time: datetime | None = None,
+    ) -> list[DATestIndexAttempt]:
+        if base_time is None:
+            base_time = datetime.now()
+
+        attempts = []
+        with get_session_context_manager() as db_session:
+            # Get the current search settings
+            search_settings = get_current_search_settings(db_session)
+            if (
+                not search_settings
+                or search_settings.status != IndexModelStatus.PRESENT
+            ):
+                raise ValueError("No current search settings found with PRESENT status")
+
+            for i in range(num_attempts):
+                time_created = base_time - timedelta(hours=i)
+
+                index_attempt = IndexAttempt(
+                    connector_credential_pair_id=cc_pair_id,
+                    from_beginning=from_beginning,
+                    status=status,
+                    new_docs_indexed=new_docs_indexed,
+                    total_docs_indexed=total_docs_indexed,
+                    docs_removed_from_index=docs_removed_from_index,
+                    error_msg=error_msg,
+                    time_created=time_created,
+                    time_started=time_created,
+                    time_updated=time_created,
+                    search_settings_id=search_settings.id,
+                )
+
+                db_session.add(index_attempt)
+                db_session.flush()  # To get the ID
+
+                attempts.append(
+                    DATestIndexAttempt(
+                        id=index_attempt.id,
+                        status=index_attempt.status,
+                        new_docs_indexed=index_attempt.new_docs_indexed,
+                        total_docs_indexed=index_attempt.total_docs_indexed,
+                        docs_removed_from_index=index_attempt.docs_removed_from_index,
+                        error_msg=index_attempt.error_msg,
+                        time_started=index_attempt.time_started,
+                        time_updated=index_attempt.time_updated,
+                    )
+                )
+
+            db_session.commit()
+
+        return attempts
+
+    @staticmethod
+    def get_index_attempt_page(
+        cc_pair_id: int,
+        page: int = 0,
+        page_size: int = 10,
+        user_performing_action: DATestUser | None = None,
+    ) -> PaginatedReturn[IndexAttemptSnapshot]:
+        query_params: dict[str, str | int] = {
+            "page_num": page,
+            "page_size": page_size,
+        }
+
+        response = requests.get(
+            url=f"{API_SERVER_URL}/manage/admin/cc-pair/{cc_pair_id}/index-attempts?{urlencode(query_params, doseq=True)}",
+            headers=user_performing_action.headers
+            if user_performing_action
+            else GENERAL_HEADERS,
+        )
+        response.raise_for_status()
+        data = response.json()
+        return PaginatedReturn(
+            items=[IndexAttemptSnapshot(**item) for item in data["items"]],
+            total_items=data["total_items"],
+        )

backend/tests/integration/common_utils/managers/persona.py

@@ -26,6 +26,7 @@ class PersonaManager:
         is_public: bool = True,
         llm_filter_extraction: bool = True,
         recency_bias: RecencyBiasSetting = RecencyBiasSetting.AUTO,
+        datetime_aware: bool = False,
         prompt_ids: list[int] | None = None,
         document_set_ids: list[int] | None = None,
         tool_ids: list[int] | None = None,
@@ -46,6 +47,7 @@ class PersonaManager:
             description=description,
             system_prompt=system_prompt,
             task_prompt=task_prompt,
+            datetime_aware=datetime_aware,
             include_citations=include_citations,
             num_chunks=num_chunks,
             llm_relevance_filter=llm_relevance_filter,
@@ -104,6 +106,7 @@ class PersonaManager:
         is_public: bool | None = None,
         llm_filter_extraction: bool | None = None,
         recency_bias: RecencyBiasSetting | None = None,
+        datetime_aware: bool = False,
         prompt_ids: list[int] | None = None,
         document_set_ids: list[int] | None = None,
         tool_ids: list[int] | None = None,
@@ -121,6 +124,7 @@ class PersonaManager:
             description=description or persona.description,
             system_prompt=system_prompt,
             task_prompt=task_prompt,
+            datetime_aware=datetime_aware,
             include_citations=include_citations,
             num_chunks=num_chunks or persona.num_chunks,
             llm_relevance_filter=llm_relevance_filter or persona.llm_relevance_filter,

backend/tests/integration/common_utils/test_models.py

@@ -1,3 +1,5 @@
+from dataclasses import dataclass
+from datetime import datetime
 from enum import Enum
 from typing import Any
 from uuid import UUID
@@ -10,6 +12,8 @@ from onyx.configs.constants import QAFeedbackType
 from onyx.context.search.enums import RecencyBiasSetting
 from onyx.db.enums import AccessType
 from onyx.server.documents.models import DocumentSource
+from onyx.server.documents.models import IndexAttemptSnapshot
+from onyx.server.documents.models import IndexingStatus
 from onyx.server.documents.models import InputType
 
 """
@@ -171,3 +175,32 @@ class DATestSettings(BaseModel):
     gpu_enabled: bool | None = None
     product_gating: DATestGatingType = DATestGatingType.NONE
     anonymous_user_enabled: bool | None = None
+
+
+@dataclass
+class DATestIndexAttempt:
+    id: int
+    status: IndexingStatus | None
+    new_docs_indexed: int | None
+    total_docs_indexed: int | None
+    docs_removed_from_index: int | None
+    error_msg: str | None
+    time_started: datetime | None
+    time_updated: datetime | None
+
+    @classmethod
+    def from_index_attempt_snapshot(
+        cls, index_attempt: IndexAttemptSnapshot
+    ) -> "DATestIndexAttempt":
+        return cls(
+            id=index_attempt.id,
+            status=index_attempt.status,
+            new_docs_indexed=index_attempt.new_docs_indexed,
+            total_docs_indexed=index_attempt.total_docs_indexed,
+            docs_removed_from_index=index_attempt.docs_removed_from_index,
+            error_msg=index_attempt.error_msg,
+            time_started=datetime.fromisoformat(index_attempt.time_started)
+            if index_attempt.time_started
+            else None,
+            time_updated=datetime.fromisoformat(index_attempt.time_updated),
+        )

backend/tests/integration/tests/index_attempt/test_index_attempt_pagination.py

@@ -0,0 +1,90 @@
+from datetime import datetime
+
+from onyx.db.models import IndexingStatus
+from tests.integration.common_utils.managers.cc_pair import CCPairManager
+from tests.integration.common_utils.managers.index_attempt import IndexAttemptManager
+from tests.integration.common_utils.managers.user import UserManager
+from tests.integration.common_utils.test_models import DATestIndexAttempt
+from tests.integration.common_utils.test_models import DATestUser
+
+
+def _verify_index_attempt_pagination(
+    cc_pair_id: int,
+    index_attempts: list[DATestIndexAttempt],
+    page_size: int = 5,
+    user_performing_action: DATestUser | None = None,
+) -> None:
+    retrieved_attempts: list[int] = []
+    last_time_started = None  # Track the last time_started seen
+
+    for i in range(0, len(index_attempts), page_size):
+        paginated_result = IndexAttemptManager.get_index_attempt_page(
+            cc_pair_id=cc_pair_id,
+            page=(i // page_size),
+            page_size=page_size,
+            user_performing_action=user_performing_action,
+        )
+
+        # Verify that the total items is equal to the length of the index attempts list
+        assert paginated_result.total_items == len(index_attempts)
+        # Verify that the number of items in the page is equal to the page size
+        assert len(paginated_result.items) == min(page_size, len(index_attempts) - i)
+
+        # Verify time ordering within the page (descending order)
+        for attempt in paginated_result.items:
+            if last_time_started is not None:
+                assert (
+                    attempt.time_started <= last_time_started
+                ), "Index attempts not in descending time order"
+            last_time_started = attempt.time_started
+
+        # Add the retrieved index attempts to the list of retrieved attempts
+        retrieved_attempts.extend([attempt.id for attempt in paginated_result.items])
+
+    # Create a set of all the expected index attempt IDs
+    all_expected_attempts = set(attempt.id for attempt in index_attempts)
+    # Create a set of all the retrieved index attempt IDs
+    all_retrieved_attempts = set(retrieved_attempts)
+
+    # Verify that the set of retrieved attempts is equal to the set of expected attempts
+    assert all_expected_attempts == all_retrieved_attempts
+
+
+def test_index_attempt_pagination(reset: None) -> None:
+    # Create an admin user to perform actions
+    user_performing_action: DATestUser = UserManager.create(
+        name="admin_performing_action",
+        is_first_user=True,
+    )
+
+    # Create a CC pair to attach index attempts to
+    cc_pair = CCPairManager.create_from_scratch(
+        user_performing_action=user_performing_action,
+    )
+
+    # Create 300 successful index attempts
+    base_time = datetime.now()
+    all_attempts = IndexAttemptManager.create_test_index_attempts(
+        num_attempts=300,
+        cc_pair_id=cc_pair.id,
+        status=IndexingStatus.SUCCESS,
+        base_time=base_time,
+    )
+
+    # Verify basic pagination with different page sizes
+    print("Verifying basic pagination with page size 5")
+    _verify_index_attempt_pagination(
+        cc_pair_id=cc_pair.id,
+        index_attempts=all_attempts,
+        page_size=5,
+        user_performing_action=user_performing_action,
+    )
+
+    # Test with a larger page size
+    print("Verifying pagination with page size 100")
+    _verify_index_attempt_pagination(
+        cc_pair_id=cc_pair.id,
+        index_attempts=all_attempts,
+        page_size=100,
+        user_performing_action=user_performing_action,
+    )

backend/tests/integration/tests/llm_provider/test_llm_provider.py

@@ -3,6 +3,7 @@ import uuid
 import requests
 
 from tests.integration.common_utils.constants import API_SERVER_URL
+from tests.integration.common_utils.managers.user import UserManager
 from tests.integration.common_utils.test_models import DATestUser
 
 
@@ -20,14 +21,15 @@ def _get_provider_by_id(admin_user: DATestUser, provider_id: str) -> dict | None
     return next((p for p in providers if p["id"] == provider_id), None)
 
 
-def test_create_llm_provider_without_display_model_names(
-    admin_user: DATestUser,
-) -> None:
+def test_create_llm_provider_without_display_model_names(reset: None) -> None:
     """Test creating an LLM provider without specifying
     display_model_names and verify it's null in response"""
+    # Create admin user
+    admin_user = UserManager.create(name="admin_user")
+
     # Create LLM provider without model_names
     response = requests.put(
-        f"{API_SERVER_URL}/admin/llm/provider",
+        f"{API_SERVER_URL}/admin/llm/provider?is_creation=true",
         headers=admin_user.headers,
         json={
             "name": str(uuid.uuid4()),
@@ -49,12 +51,15 @@ def test_create_llm_provider_without_display_model_names(
     assert provider_data["display_model_names"] is None
 
 
-def test_update_llm_provider_model_names(admin_user: DATestUser) -> None:
+def test_update_llm_provider_model_names(reset: None) -> None:
     """Test updating an LLM provider's model_names"""
+    # Create admin user
+    admin_user = UserManager.create(name="admin_user")
+
     # First create provider without model_names
     name = str(uuid.uuid4())
     response = requests.put(
-        f"{API_SERVER_URL}/admin/llm/provider",
+        f"{API_SERVER_URL}/admin/llm/provider?is_creation=true",
         headers=admin_user.headers,
         json={
             "name": name,
@@ -90,11 +95,14 @@ def test_update_llm_provider_model_names(admin_user: DATestUser) -> None:
     assert provider_data["model_names"] == _DEFAULT_MODELS
 
 
-def test_delete_llm_provider(admin_user: DATestUser) -> None:
+def test_delete_llm_provider(reset: None) -> None:
     """Test deleting an LLM provider"""
+    # Create admin user
+    admin_user = UserManager.create(name="admin_user")
+
     # Create a provider
     response = requests.put(
-        f"{API_SERVER_URL}/admin/llm/provider",
+        f"{API_SERVER_URL}/admin/llm/provider?is_creation=true",
         headers=admin_user.headers,
         json={
             "name": "test-provider-delete",

web/Dockerfile

@@ -18,11 +18,14 @@ FROM base AS builder
 RUN apk add --no-cache libc6-compat
 WORKDIR /app
 
+# Increase Node.js memory limit for the build process
+ENV NODE_OPTIONS="--max-old-space-size=4096"
+
 # pull in source code / package.json / package-lock.json
 COPY . .
 
-# Install dependencies
-RUN npm ci
+# Install dependencies including critters
+RUN npm ci && npm install critters
 
 # needed to get the `standalone` dir we expect later
 ENV NEXT_PRIVATE_STANDALONE=true

web/next.config.js

@@ -22,6 +22,7 @@ const cspHeader = `
 
 /** @type {import('next').NextConfig} */
 const nextConfig = {
+  productionBrowserSourceMaps: false,
   output: "standalone",
   publicRuntimeConfig: {
     version,
@@ -37,6 +38,13 @@ const nextConfig = {
       },
     ],
   },
+  // Modify these options
+  swcMinify: true,
+  experimental: {
+    // Remove the optimizeCss option
+    scrollRestoration: false,
+    legacyBrowsers: false,
+  },
   async headers() {
     return [
       {
@@ -71,26 +79,24 @@ const nextConfig = {
 
 // Sentry configuration for error monitoring:
 // - Without SENTRY_AUTH_TOKEN and NEXT_PUBLIC_SENTRY_DSN: Sentry is completely disabled
-// - With both configured: Only unhandled errors are captured (no performance/session tracking)
+// - With both configured: Capture errors and limited performance data
 
 // Determine if Sentry should be enabled
 const sentryEnabled = Boolean(
   process.env.SENTRY_AUTH_TOKEN && process.env.NEXT_PUBLIC_SENTRY_DSN
 );
 
-// Sentry webpack plugin options
+// Modify the Sentry webpack plugin options
 const sentryWebpackPluginOptions = {
   org: process.env.SENTRY_ORG || "onyx",
   project: process.env.SENTRY_PROJECT || "data-plane-web",
   authToken: process.env.SENTRY_AUTH_TOKEN,
-  silent: !sentryEnabled, // Silence output when Sentry is disabled
-  dryRun: !sentryEnabled, // Don't upload source maps when Sentry is disabled
-  sourceMaps: {
-    include: ["./.next"],
-    validate: false,
-    urlPrefix: "~/_next",
-    skip: !sentryEnabled,
-  },
+  silent: !sentryEnabled,
+  dryRun: !sentryEnabled,
+  sourceMaps: false,
+  // Add this option to disable source map generation
+  disableServerWebpackPlugin: true,
+  disableClientWebpackPlugin: true,
 };
 
 // Export the module with conditional Sentry configuration

web/package.json

@@ -24,13 +24,15 @@
     "@radix-ui/react-label": "^2.1.1",
     "@radix-ui/react-popover": "^1.1.2",
     "@radix-ui/react-radio-group": "^1.2.2",
+    "@radix-ui/react-scroll-area": "^1.2.2",
     "@radix-ui/react-select": "^2.1.2",
     "@radix-ui/react-separator": "^1.1.0",
     "@radix-ui/react-slot": "^1.1.0",
     "@radix-ui/react-switch": "^1.1.1",
     "@radix-ui/react-tabs": "^1.1.1",
     "@radix-ui/react-tooltip": "^1.1.3",
-    "@sentry/nextjs": "^8.34.0",
+    "@sentry/nextjs": "^8.50.0",
+    "@sentry/tracing": "^7.120.3",
     "@stripe/stripe-js": "^4.6.0",
     "@types/js-cookie": "^3.0.3",
     "@types/lodash": "^4.17.0",

web/sentry.client.config.ts

@@ -3,12 +3,10 @@ import * as Sentry from "@sentry/nextjs";
 if (process.env.NEXT_PUBLIC_SENTRY_DSN) {
   Sentry.init({
     dsn: process.env.NEXT_PUBLIC_SENTRY_DSN,
-    // Only capture unhandled exceptions
+
+    // Capture unhandled exceptions and performance data
     enableTracing: false,
     integrations: [],
-    tracesSampleRate: 0,
-    replaysSessionSampleRate: 0,
-    replaysOnErrorSampleRate: 0,
-    autoSessionTracking: false,
+    tracesSampleRate: 0.1,
   });
 }

web/src/app/admin/assistants/AssistantEditor.tsx

@@ -1,6 +1,6 @@
 "use client";
 
-import React, { useCallback } from "react";
+import React from "react";
 import { Option } from "@/components/Dropdown";
 import { generateRandomIconShape } from "@/lib/assistantIconUtils";
 import { CCPairBasicInfo, DocumentSet, User, UserGroup } from "@/lib/types";
@@ -35,7 +35,7 @@ import {
 import Link from "next/link";
 import { useRouter } from "next/navigation";
 import { useEffect, useMemo, useState } from "react";
-import { FiInfo, FiRefreshCcw, FiUsers } from "react-icons/fi";
+import { FiInfo } from "react-icons/fi";
 import * as Yup from "yup";
 import CollapsibleSection from "./CollapsibleSection";
 import { SuccessfulPersonaUpdateRedirectType } from "./enums";
@@ -60,10 +60,11 @@ import { useAssistants } from "@/components/context/AssistantsContext";
 import { debounce } from "lodash";
 import { FullLLMProvider } from "../configuration/llm/interfaces";
 import StarterMessagesList from "./StarterMessageList";
-import { Switch } from "@/components/ui/switch";
+
+import { Switch, SwitchField } from "@/components/ui/switch";
 import { generateIdenticon } from "@/components/assistants/AssistantIcon";
 import { BackButton } from "@/components/BackButton";
-import { Checkbox } from "@/components/ui/checkbox";
+import { Checkbox, CheckboxField } from "@/components/ui/checkbox";
 import { AdvancedOptionsToggle } from "@/components/AdvancedOptionsToggle";
 import { MinimalUserSnapshot } from "@/lib/types";
 import { useUserGroups } from "@/lib/hooks";
@@ -72,11 +73,13 @@ import {
   Option as DropdownOption,
 } from "@/components/Dropdown";
 import { SourceChip } from "@/app/chat/input/ChatInputBar";
-import { TagIcon, UserIcon } from "lucide-react";
+import { TagIcon, UserIcon, XIcon } from "lucide-react";
 import { LLMSelector } from "@/components/llm/LLMSelector";
 import useSWR from "swr";
 import { errorHandlingFetcher } from "@/lib/fetcher";
 import { DeleteEntityModal } from "@/components/modals/DeleteEntityModal";
+import { DeletePersonaButton } from "./[id]/DeletePersonaButton";
+import Title from "@/components/ui/title";
 
 function findSearchTool(tools: ToolSnapshot[]) {
   return tools.find((tool) => tool.in_code_tool_id === "SearchTool");
@@ -128,8 +131,8 @@ export function AssistantEditor({
   const router = useRouter();
 
   const { popup, setPopup } = usePopup();
-  const { data, refreshLabels } = useLabels();
-  const labels = data || [];
+  const { labels, refreshLabels, createLabel, updateLabel, deleteLabel } =
+    useLabels();
 
   const colorOptions = [
     "#FF6FBF",
@@ -141,11 +144,7 @@ export function AssistantEditor({
     "#6FFFFF",
   ];
 
-  const [showSearchTool, setShowSearchTool] = useState(false);
-
   const [showAdvancedOptions, setShowAdvancedOptions] = useState(false);
-  const [hasEditedStarterMessage, setHasEditedStarterMessage] = useState(false);
-  const [showPersonaLabel, setShowPersonaLabel] = useState(!admin);
 
   // state to persist across formik reformatting
   const [defautIconColor, _setDeafultIconColor] = useState(
@@ -222,6 +221,7 @@ export function AssistantEditor({
   const initialValues = {
     name: existingPersona?.name ?? "",
     description: existingPersona?.description ?? "",
+    datetime_aware: existingPrompt?.datetime_aware ?? false,
     system_prompt: existingPrompt?.system_prompt ?? "",
     task_prompt: existingPrompt?.task_prompt ?? "",
     is_public: existingPersona?.is_public ?? defaultPublic,
@@ -330,6 +330,10 @@ export function AssistantEditor({
     }));
   };
 
+  if (!labels) {
+    return <></>;
+  }
+
   return (
     <div className="mx-auto max-w-4xl">
       <style>
@@ -351,7 +355,7 @@ export function AssistantEditor({
           entityName={labelToDelete.name}
           onClose={() => setLabelToDelete(null)}
           onSubmit={async () => {
-            const response = await deletePersonaLabel(labelToDelete.id);
+            const response = await deleteLabel(labelToDelete.id);
             if (response?.ok) {
               setPopup({
                 message: `Label deleted successfully`,
@@ -575,7 +579,7 @@ export function AssistantEditor({
           return (
             <Form className="w-full text-text-950 assistant-editor">
               {/* Refresh starter messages when name or description changes */}
-              <p className="text-base font-normal !text-2xl">
+              <p className="text-base font-normal text-2xl">
                 {existingPersona ? (
                   <>
                     Edit assistant <b>{existingPersona.name}</b>
@@ -744,97 +748,6 @@ export function AssistantEditor({
                 className="[&_input]:placeholder:text-text-muted/50"
               />
 
-              <div className=" w-full max-w-4xl">
-                <Separator />
-                <div className="flex gap-x-2 items-center mt-4 ">
-                  <div className="block font-medium text-sm">Labels</div>
-                </div>
-                <p
-                  className="text-sm text-subtle"
-                  style={{ color: "rgb(113, 114, 121)" }}
-                >
-                  Select labels to categorize this assistant
-                </p>
-                <div className="mt-3">
-                  <SearchMultiSelectDropdown
-                    onCreateLabel={async (name: string) => {
-                      await createPersonaLabel(name);
-                      const currentLabels = await refreshLabels();
-
-                      setTimeout(() => {
-                        const newLabelId = currentLabels.find(
-                          (l: { name: string }) => l.name === name
-                        )?.id;
-                        const updatedLabelIds = [
-                          ...values.label_ids,
-                          newLabelId as number,
-                        ];
-                        setFieldValue("label_ids", updatedLabelIds);
-                      }, 300);
-                    }}
-                    options={Array.from(
-                      new Set(labels.map((label) => label.name))
-                    ).map((name) => ({
-                      name,
-                      value: name,
-                    }))}
-                    onSelect={(selected) => {
-                      const newLabelIds = [
-                        ...values.label_ids,
-                        labels.find((l) => l.name === selected.value)
-                          ?.id as number,
-                      ];
-                      setFieldValue("label_ids", newLabelIds);
-                    }}
-                    itemComponent={({ option }) => (
-                      <div
-                        className="flex items-center px-4 py-2.5 text-sm hover:bg-hover cursor-pointer"
-                        onClick={() => {
-                          const label = labels.find(
-                            (l) => l.name === option.value
-                          );
-                          if (label) {
-                            const isSelected = values.label_ids.includes(
-                              label.id
-                            );
-                            const newLabelIds = isSelected
-                              ? values.label_ids.filter(
-                                  (id: number) => id !== label.id
-                                )
-                              : [...values.label_ids, label.id];
-                            setFieldValue("label_ids", newLabelIds);
-                          }
-                        }}
-                      >
-                        <span className="text-sm font-medium leading-none">
-                          {option.name}
-                        </span>
-                      </div>
-                    )}
-                  />
-                  <div className="mt-2 flex flex-wrap gap-2">
-                    {values.label_ids.map((labelId: number) => {
-                      const label = labels.find((l) => l.id === labelId);
-                      return label ? (
-                        <SourceChip
-                          key={label.id}
-                          onRemove={() => {
-                            setFieldValue(
-                              "label_ids",
-                              values.label_ids.filter(
-                                (id: number) => id !== label.id
-                              )
-                            );
-                          }}
-                          title={label.name}
-                          icon={<TagIcon size={12} />}
-                        />
-                      ) : null;
-                    })}
-                  </div>
-                </div>
-              </div>
-
               <Separator />
 
               <TextFormField
@@ -872,10 +785,9 @@ export function AssistantEditor({
                                           : ""
                                       }`}
                                     >
-                                      <Switch
+                                      <SwitchField
                                         size="sm"
                                         onCheckedChange={(checked) => {
-                                          setShowSearchTool(checked);
                                           setFieldValue("num_chunks", null);
                                           toggleToolInValues(searchTool.id);
                                         }}
@@ -889,8 +801,7 @@ export function AssistantEditor({
                                     <TooltipContent side="top" align="center">
                                       <p className="bg-background-900 max-w-[200px] text-sm rounded-lg p-1.5 text-white">
                                         To use the Knowledge Action, you need to
-                                        have at least one Connector-Credential
-                                        pair configured.
+                                        have at least one Connector configured.
                                       </p>
                                     </TooltipContent>
                                   )}
@@ -910,7 +821,7 @@ export function AssistantEditor({
                   )}
                   {ccPairs.length > 0 &&
                     searchTool &&
-                    showSearchTool &&
+                    values.enabled_tools_map[searchTool.id] &&
                     !(user?.role != "admin" && documentSets.length === 0) && (
                       <CollapsibleSection>
                         <div className="mt-2">
@@ -998,14 +909,10 @@ export function AssistantEditor({
                           <TooltipProvider>
                             <Tooltip>
                               <TooltipTrigger>
-                                <Checkbox
+                                <CheckboxField
                                   size="sm"
                                   id={`enabled_tools_map.${imageGenerationTool.id}`}
-                                  checked={
-                                    values.enabled_tools_map[
-                                      imageGenerationTool.id
-                                    ]
-                                  }
+                                  name={`enabled_tools_map.${imageGenerationTool.id}`}
                                   onCheckedChange={() => {
                                     if (
                                       currentLLMSupportsImageOutput &&
@@ -1061,6 +968,7 @@ export function AssistantEditor({
                             onCheckedChange={() => {
                               toggleToolInValues(internetSearchTool.id);
                             }}
+                            name={`enabled_tools_map.${internetSearchTool.id}`}
                           />
                           <div className="flex flex-col ml-2">
                             <span className="text-sm">
@@ -1080,6 +988,7 @@ export function AssistantEditor({
                         <React.Fragment key={tool.id}>
                           <div className="flex items-center content-start mb-2">
                             <Checkbox
+                              size="sm"
                               id={`enabled_tools_map.${tool.id}`}
                               checked={values.enabled_tools_map[tool.id]}
                               onCheckedChange={() => {
@@ -1114,7 +1023,6 @@ export function AssistantEditor({
                         )
                       : null
                   }
-                  userDefault={user?.preferences?.default_model || null}
                   requiresImageGeneration={
                     imageGenerationTool
                       ? values.enabled_tools_map[imageGenerationTool.id]
@@ -1136,106 +1044,6 @@ export function AssistantEditor({
                 />
               </div>
 
-              {admin && labels && labels.length > 0 && (
-                <div className=" max-w-4xl">
-                  <Separator />
-                  <div className="flex gap-x-2 items-center ">
-                    <div className="block font-medium text-sm">
-                      Manage Labels
-                    </div>
-                    <TooltipProvider>
-                      <Tooltip>
-                        <TooltipTrigger>
-                          <FiInfo size={12} />
-                        </TooltipTrigger>
-                        <TooltipContent side="top" align="center">
-                          Manage existing labels or create new ones to group
-                          similar assistants
-                        </TooltipContent>
-                      </Tooltip>
-                    </TooltipProvider>
-                  </div>
-                  <SubLabel>Edit or delete existing labels</SubLabel>
-                  <div className="grid grid-cols-1 gap-4">
-                    {labels.map((label: PersonaLabel) => (
-                      <div
-                        key={label.id}
-                        className="grid grid-cols-[1fr,2fr,auto] gap-4 items-end"
-                      >
-                        <TextFormField
-                          fontSize="sm"
-                          name={`editLabelName_${label.id}`}
-                          label="Label Name"
-                          value={
-                            values.editLabelId === label.id
-                              ? values.editLabelName
-                              : label.name
-                          }
-                          onChange={(e) => {
-                            setFieldValue("editLabelId", label.id);
-                            setFieldValue("editLabelName", e.target.value);
-                          }}
-                        />
-                        <div className="flex gap-2">
-                          {values.editLabelId === label.id ? (
-                            <>
-                              <Button
-                                onClick={async () => {
-                                  const updatedName =
-                                    values.editLabelName || label.name;
-                                  const response = await updatePersonaLabel(
-                                    label.id,
-                                    updatedName
-                                  );
-                                  if (response?.ok) {
-                                    setPopup({
-                                      message: `Label "${updatedName}" updated successfully`,
-                                      type: "success",
-                                    });
-                                    await refreshLabels();
-                                    setFieldValue("editLabelId", null);
-                                    setFieldValue("editLabelName", "");
-                                    setFieldValue("editLabelDescription", "");
-                                  } else {
-                                    setPopup({
-                                      message: `Failed to update label - ${await response.text()}`,
-                                      type: "error",
-                                    });
-                                  }
-                                }}
-                              >
-                                Save
-                              </Button>
-                              <Button
-                                variant="outline"
-                                onClick={() => {
-                                  setFieldValue("editLabelId", null);
-                                  setFieldValue("editLabelName", "");
-                                  setFieldValue("editLabelDescription", "");
-                                }}
-                              >
-                                Cancel
-                              </Button>
-                            </>
-                          ) : (
-                            <>
-                              <Button
-                                variant="destructive"
-                                onClick={async () => {
-                                  setLabelToDelete(label);
-                                }}
-                              >
-                                Delete
-                              </Button>
-                            </>
-                          )}
-                        </div>
-                      </div>
-                    ))}
-                  </div>
-                </div>
-              )}
-
               <Separator />
               <AdvancedOptionsToggle
                 showAdvancedOptions={showAdvancedOptions}
@@ -1253,9 +1061,9 @@ export function AssistantEditor({
 
                     <div className="min-h-[100px]">
                       <div className="flex items-center mb-2">
-                        <Switch
+                        <SwitchField
+                          name="is_public"
                           size="md"
-                          checked={values.is_public}
                           onCheckedChange={(checked) => {
                             setFieldValue("is_public", checked);
                             if (checked) {
@@ -1397,19 +1205,124 @@ export function AssistantEditor({
                             autoStarterMessageEnabled={
                               autoStarterMessageEnabled
                             }
-                            errors={errors}
                             isRefreshing={isRefreshing}
                             values={values.starter_messages}
                             arrayHelpers={arrayHelpers}
-                            touchStarterMessages={() => {
-                              setHasEditedStarterMessage(true);
-                            }}
                             setFieldValue={setFieldValue}
                           />
                         )}
                       />
                     </div>
                   </div>
+
+                  <div className=" w-full max-w-4xl">
+                    <Separator />
+                    <div className="flex gap-x-2 items-center mt-4 ">
+                      <div className="block font-medium text-sm">Labels</div>
+                    </div>
+                    <p
+                      className="text-sm text-subtle"
+                      style={{ color: "rgb(113, 114, 121)" }}
+                    >
+                      Select labels to categorize this assistant
+                    </p>
+                    <div className="mt-3">
+                      <SearchMultiSelectDropdown
+                        onCreate={async (name: string) => {
+                          await createLabel(name);
+                          const currentLabels = await refreshLabels();
+
+                          setTimeout(() => {
+                            const newLabelId = currentLabels.find(
+                              (l: { name: string }) => l.name === name
+                            )?.id;
+                            const updatedLabelIds = [
+                              ...values.label_ids,
+                              newLabelId as number,
+                            ];
+                            setFieldValue("label_ids", updatedLabelIds);
+                          }, 300);
+                        }}
+                        options={Array.from(
+                          new Set(labels.map((label) => label.name))
+                        ).map((name) => ({
+                          name,
+                          value: name,
+                        }))}
+                        onSelect={(selected) => {
+                          const newLabelIds = [
+                            ...values.label_ids,
+                            labels.find((l) => l.name === selected.value)
+                              ?.id as number,
+                          ];
+                          setFieldValue("label_ids", newLabelIds);
+                        }}
+                        itemComponent={({ option }) => (
+                          <div className="flex items-center justify-between px-4 py-3 text-sm hover:bg-hover cursor-pointer border-b border-border last:border-b-0">
+                            <div
+                              className="flex-grow"
+                              onClick={() => {
+                                const label = labels.find(
+                                  (l) => l.name === option.value
+                                );
+                                if (label) {
+                                  const isSelected = values.label_ids.includes(
+                                    label.id
+                                  );
+                                  const newLabelIds = isSelected
+                                    ? values.label_ids.filter(
+                                        (id: number) => id !== label.id
+                                      )
+                                    : [...values.label_ids, label.id];
+                                  setFieldValue("label_ids", newLabelIds);
+                                }
+                              }}
+                            >
+                              <span className="font-normal leading-none">
+                                {option.name}
+                              </span>
+                            </div>
+                            {admin && (
+                              <button
+                                onClick={(e) => {
+                                  e.stopPropagation();
+                                  const label = labels.find(
+                                    (l) => l.name === option.value
+                                  );
+                                  if (label) {
+                                    deleteLabel(label.id);
+                                  }
+                                }}
+                                className="ml-2 p-1 hover:bg-background-hover rounded"
+                              >
+                                <TrashIcon size={16} />
+                              </button>
+                            )}
+                          </div>
+                        )}
+                      />
+                      <div className="mt-2 flex flex-wrap gap-2">
+                        {values.label_ids.map((labelId: number) => {
+                          const label = labels.find((l) => l.id === labelId);
+                          return label ? (
+                            <SourceChip
+                              key={label.id}
+                              onRemove={() => {
+                                setFieldValue(
+                                  "label_ids",
+                                  values.label_ids.filter(
+                                    (id: number) => id !== label.id
+                                  )
+                                );
+                              }}
+                              title={label.name}
+                              icon={<TagIcon size={12} />}
+                            />
+                          ) : null;
+                        })}
+                      </div>
+                    </div>
+                  </div>
                   <Separator />
 
                   <div className="flex flex-col gap-y-4">
@@ -1435,7 +1348,6 @@ export function AssistantEditor({
                           small
                           subtext="Documents prior to this date will be ignored."
                           label="[Optional] Knowledge Cutoff Date"
-                          value={values.search_start_date}
                           name="search_start_date"
                         />
 
@@ -1461,6 +1373,17 @@ export function AssistantEditor({
                   </div>
                   <Separator />
 
+                  <BooleanFormField
+                    small
+                    removeIndent
+                    alignTop
+                    name="datetime_aware"
+                    label="Date and Time Aware"
+                    subtext='Toggle this option to let the assistant know the current date and time (formatted like: "Thursday Jan 1, 1970 00:01"). To inject it in a specific place in the prompt, use the pattern [[CURRENT_DATETIME]]'
+                  />
+
+                  <Separator />
+
                   <TextFormField
                     maxWidth="max-w-4xl"
                     name="task_prompt"
@@ -1474,6 +1397,14 @@ export function AssistantEditor({
                     explanationLink="https://docs.onyx.app/guides/assistants"
                     className="[&_textarea]:placeholder:text-text-muted/50"
                   />
+                  <div className="flex justify-end">
+                    {existingPersona && (
+                      <DeletePersonaButton
+                        personaId={existingPersona!.id}
+                        redirectType={SuccessfulPersonaUpdateRedirectType.ADMIN}
+                      />
+                    )}
+                  </div>
                 </>
               )}
 

web/src/app/admin/assistants/LabelManagement.tsx

@@ -0,0 +1,182 @@
+"use client";
+
+import React from "react";
+import { Separator } from "@/components/ui/separator";
+import { Button } from "@/components/ui/button";
+import { SubLabel, TextFormField } from "@/components/admin/connectors/Field";
+import { usePopup } from "@/components/admin/connectors/Popup";
+import { useLabels } from "@/lib/hooks";
+import { PersonaLabel } from "./interfaces";
+import { Form, Formik, FormikHelpers } from "formik";
+import Title from "@/components/ui/title";
+
+interface FormValues {
+  newLabelName: string;
+  editLabelId: number | null;
+  editLabelName: string;
+}
+
+export default function LabelManagement() {
+  const { labels, createLabel, updateLabel, deleteLabel } = useLabels();
+  const { setPopup, popup } = usePopup();
+
+  if (!labels) return null;
+
+  const handleSubmit = async (
+    values: FormValues,
+    { setSubmitting, resetForm }: FormikHelpers<FormValues>
+  ) => {
+    if (values.newLabelName.trim()) {
+      const response = await createLabel(values.newLabelName.trim());
+      if (response.ok) {
+        setPopup({
+          message: `Label "${values.newLabelName}" created successfully`,
+          type: "success",
+        });
+        resetForm();
+      } else {
+        const errorMsg = (await response.json()).detail;
+        setPopup({
+          message: `Failed to create label - ${errorMsg}`,
+          type: "error",
+        });
+      }
+    }
+    setSubmitting(false);
+  };
+
+  return (
+    <div>
+      {popup}
+      <div className="max-w-4xl">
+        <div className="flex gap-x-2 items-center">
+          <Title size="lg">Manage Labels</Title>
+        </div>
+
+        <Formik<FormValues>
+          initialValues={{
+            newLabelName: "",
+            editLabelId: null,
+            editLabelName: "",
+          }}
+          onSubmit={handleSubmit}
+        >
+          {({ values, setFieldValue, isSubmitting }) => (
+            <Form>
+              <div className="flex flex-col gap-4 mt-4 mb-6">
+                <div className="flex flex-col">
+                  <Title className="text-lg">Create New Label</Title>
+                  <SubLabel>
+                    Labels are used to categorize personas. You can create a new
+                    label by entering a name below.
+                  </SubLabel>
+                </div>
+                <div className="max-w-3xl w-full justify-start flex gap-4 items-end">
+                  <TextFormField
+                    width="max-w-xs"
+                    fontSize="sm"
+                    name="newLabelName"
+                    label="Label Name"
+                  />
+                  <Button type="submit" disabled={isSubmitting}>
+                    Create
+                  </Button>
+                </div>
+              </div>
+
+              <div className="grid grid-cols-1 w-full gap-4">
+                <div className="flex flex-col">
+                  <Title className="text-lg">Edit Labels</Title>
+                  <SubLabel>
+                    You can edit the name of a label by clicking on the label
+                    name and entering a new name.
+                  </SubLabel>
+                </div>
+
+                {labels.map((label: PersonaLabel) => (
+                  <div key={label.id} className="flex w-full  gap-4 items-end">
+                    <TextFormField
+                      fontSize="sm"
+                      width="w-full max-w-xs"
+                      name={`editLabelName_${label.id}`}
+                      label="Label Name"
+                      value={
+                        values.editLabelId === label.id
+                          ? values.editLabelName
+                          : label.name
+                      }
+                      onChange={(e) => {
+                        setFieldValue("editLabelId", label.id);
+                        setFieldValue("editLabelName", e.target.value);
+                      }}
+                    />
+                    <div className="flex gap-2">
+                      {values.editLabelId === label.id ? (
+                        <>
+                          <Button
+                            onClick={async () => {
+                              const updatedName =
+                                values.editLabelName || label.name;
+                              const response = await updateLabel(
+                                label.id,
+                                updatedName
+                              );
+                              if (response.ok) {
+                                setPopup({
+                                  message: `Label "${updatedName}" updated successfully`,
+                                  type: "success",
+                                });
+                                setFieldValue("editLabelId", null);
+                                setFieldValue("editLabelName", "");
+                              } else {
+                                setPopup({
+                                  message: `Failed to update label - ${await response.text()}`,
+                                  type: "error",
+                                });
+                              }
+                            }}
+                          >
+                            Save
+                          </Button>
+                          <Button
+                            variant="outline"
+                            onClick={() => {
+                              setFieldValue("editLabelId", null);
+                              setFieldValue("editLabelName", "");
+                            }}
+                          >
+                            Cancel
+                          </Button>
+                        </>
+                      ) : (
+                        <Button
+                          variant="destructive"
+                          onClick={async () => {
+                            const response = await deleteLabel(label.id);
+                            if (response.ok) {
+                              setPopup({
+                                message: `Label "${label.name}" deleted successfully`,
+                                type: "success",
+                              });
+                            } else {
+                              setPopup({
+                                message: `Failed to delete label - ${await response.text()}`,
+                                type: "error",
+                              });
+                            }
+                          }}
+                        >
+                          Delete
+                        </Button>
+                      )}
+                    </div>
+                  </div>
+                ))}
+              </div>
+            </Form>
+          )}
+        </Formik>
+      </div>
+    </div>
+  );
+}

web/src/app/admin/assistants/PersonaTable.tsx

@@ -102,12 +102,6 @@ export function PersonasTable() {
     <div>
       {popup}
 
-      <Text className="my-2">
-        Assistants will be displayed as options on the Chat / Search interfaces
-        in the order they are displayed below. Assistants marked as hidden will
-        not be displayed. Editable assistants are shown at the top.
-      </Text>
-
       <DraggableTable
         headers={["Name", "Description", "Type", "Is Visible", "Delete"]}
         isAdmin={isAdmin}

web/src/app/admin/assistants/StarterMessageList.tsx

@@ -18,25 +18,20 @@ export default function StarterMessagesList({
   values,
   arrayHelpers,
   isRefreshing,
-  touchStarterMessages,
   debouncedRefreshPrompts,
   autoStarterMessageEnabled,
-  errors,
   setFieldValue,
 }: {
   values: StarterMessage[];
   arrayHelpers: ArrayHelpers;
   isRefreshing: boolean;
-  touchStarterMessages: () => void;
   debouncedRefreshPrompts: () => void;
   autoStarterMessageEnabled: boolean;
-  errors: any;
   setFieldValue: any;
 }) {
   const [tooltipOpen, setTooltipOpen] = useState(false);
 
   const handleInputChange = (index: number, value: string) => {
-    touchStarterMessages();
     setFieldValue(`starter_messages.${index}.message`, value);
 
     if (value && index === values.length - 1 && values.length < 4) {

web/src/app/admin/assistants/[id]/page.tsx

@@ -29,12 +29,6 @@ export default async function Page(props: { params: Promise<{ id: string }> }) {
             defaultPublic={true}
             redirectType={SuccessfulPersonaUpdateRedirectType.ADMIN}
           />
-          <Title>Delete Assistant</Title>
-
-          <DeletePersonaButton
-            personaId={values.existingPersona!.id}
-            redirectType={SuccessfulPersonaUpdateRedirectType.ADMIN}
-          />
         </CardSection>
       </>
     );

web/src/app/admin/assistants/lib.ts

@@ -6,6 +6,7 @@ interface PersonaUpsertRequest {
   description: string;
   system_prompt: string;
   task_prompt: string;
+  datetime_aware: boolean;
   document_set_ids: number[];
   num_chunks: number | null;
   include_citations: boolean;
@@ -36,6 +37,7 @@ export interface PersonaUpsertParameters {
   system_prompt: string;
   existing_prompt_id: number | null;
   task_prompt: string;
+  datetime_aware: boolean;
   document_set_ids: number[];
   num_chunks: number | null;
   include_citations: boolean;
@@ -105,6 +107,7 @@ function buildPersonaUpsertRequest(
     is_public,
     groups,
     existing_prompt_id,
+    datetime_aware,
     users,
     tool_ids,
     icon_color,
@@ -129,6 +132,7 @@ function buildPersonaUpsertRequest(
     icon_shape,
     remove_image,
     search_start_date,
+    datetime_aware,
     is_default_persona: creationRequest.is_default_persona ?? false,
     recency_bias: "base_decay",
     prompt_ids: existing_prompt_id ? [existing_prompt_id] : [],

web/src/app/admin/assistants/new/page.tsx

@@ -1,32 +1,25 @@
 import { AssistantEditor } from "../AssistantEditor";
 import { ErrorCallout } from "@/components/ErrorCallout";
-import { RobotIcon } from "@/components/icons/icons";
-import { BackButton } from "@/components/BackButton";
-import CardSection from "@/components/admin/CardSection";
-import { AdminPageTitle } from "@/components/admin/Title";
 import { fetchAssistantEditorInfoSS } from "@/lib/assistants/fetchPersonaEditorInfoSS";
 import { SuccessfulPersonaUpdateRedirectType } from "../enums";
 
 export default async function Page() {
   const [values, error] = await fetchAssistantEditorInfoSS();
 
-  let body;
   if (!values) {
-    body = (
+    return (
       <ErrorCallout errorTitle="Something went wrong :(" errorMsg={error} />
     );
   } else {
-    body = (
-      <CardSection className="!border-none !bg-transparent !ring-none">
+    return (
+      <div className="w-full">
         <AssistantEditor
           {...values}
           admin
           defaultPublic={true}
           redirectType={SuccessfulPersonaUpdateRedirectType.ADMIN}
         />
-      </CardSection>
+      </div>
     );
   }
-
-  return <div className="w-full">{body}</div>;
 }

web/src/app/admin/assistants/page.tsx

@@ -1,3 +1,4 @@
+"use client";
 import { PersonasTable } from "./PersonaTable";
 import { FiPlusSquare } from "react-icons/fi";
 import Link from "next/link";
@@ -6,6 +7,8 @@ import Title from "@/components/ui/title";
 import { Separator } from "@/components/ui/separator";
 import { AssistantsIcon } from "@/components/icons/icons";
 import { AdminPageTitle } from "@/components/admin/Title";
+import LabelManagement from "./LabelManagement";
+import { SubLabel } from "@/components/admin/connectors/Field";
 
 export default async function Page() {
   return (
@@ -43,6 +46,12 @@ export default async function Page() {
         <Separator />
 
         <Title>Existing Assistants</Title>
+        <SubLabel>
+          Assistants will be displayed as options on the Chat / Search
+          interfaces in the order they are displayed below. Assistants marked as
+          hidden will not be displayed. Editable assistants are shown at the
+          top.
+        </SubLabel>
         <PersonasTable />
       </div>
     </div>

web/src/app/admin/documents/explorer/Explorer.tsx

@@ -11,13 +11,13 @@ import { PopupSpec, usePopup } from "@/components/admin/connectors/Popup";
 import { getErrorMsg } from "@/lib/fetchUtils";
 import { ScoreSection } from "../ScoreEditor";
 import { useRouter } from "next/navigation";
-import { HorizontalFilters } from "@/components/search/filtering/Filters";
 import { useFilters } from "@/lib/hooks";
 import { buildFilters } from "@/lib/search/utils";
 import { DocumentUpdatedAtBadge } from "@/components/search/DocumentUpdatedAtBadge";
 import { DocumentSet } from "@/lib/types";
 import { SourceIcon } from "@/components/SourceIcon";
 import { Connector } from "@/lib/connectors/connectors";
+import { HorizontalFilters } from "@/app/chat/shared_chat_search/Filters";
 
 const DocumentDisplay = ({
   document,
@@ -200,6 +200,9 @@ export function Explorer({
             availableDocumentSets={documentSets}
             existingSources={connectors.map((connector) => connector.source)}
             availableTags={[]}
+            toggleFilters={() => {}}
+            filtersUntoggled={false}
+            tagsOnLeft={true}
           />
         </div>
       </div>

web/src/app/admin/indexing/status/CCPairIndexingStatusTable.tsx

@@ -59,7 +59,7 @@ function SummaryRow({
   return (
     <TableRow
       onClick={onToggle}
-      className="border-border bg-white py-4 rounded-sm !border cursor-pointer"
+      className="border-border group hover:bg-background-settings-hover bg-background-sidebar py-4 rounded-sm !border cursor-pointer"
     >
       <TableCell>
         <div className="text-xl flex items-center truncate ellipsis gap-x-2 font-semibold">
@@ -86,7 +86,7 @@ function SummaryRow({
           <Tooltip>
             <TooltipTrigger asChild>
               <div className="flex items-center mt-1">
-                <div className="w-full bg-gray-200 rounded-full h-2 mr-2">
+                <div className="w-full bg-white rounded-full h-2 mr-2">
                   <div
                     className="bg-green-500 h-2 rounded-full"
                     style={{ width: `${activePercentage}%` }}

web/src/app/admin/settings/AnonymousUserPath.tsx

@@ -1,21 +1,21 @@
 "use client";
 
 import useSWR from "swr";
-import { useState } from "react";
+import { useContext, useState } from "react";
 
 import { PopupSpec } from "@/components/admin/connectors/Popup";
 import { Button } from "@/components/ui/button";
-import { NEXT_PUBLIC_WEB_DOMAIN } from "@/lib/constants";
 import { ClipboardIcon } from "@/components/icons/icons";
 import { Input } from "@/components/ui/input";
 import { ThreeDotsLoader } from "@/components/Loading";
+import { SettingsContext } from "@/components/settings/SettingsProvider";
 
 export function AnonymousUserPath({
   setPopup,
 }: {
   setPopup: (popup: PopupSpec) => void;
 }) {
-  const fetcher = (url: string) => fetch(url).then((res) => res.json());
+  const settings = useContext(SettingsContext);
   const [customPath, setCustomPath] = useState<string | null>(null);
 
   const {
@@ -116,7 +116,7 @@ export function AnonymousUserPath({
         <div className="flex flex-col gap-2 justify-center items-start">
           <div className="w-full flex-grow  flex items-center rounded-md shadow-sm">
             <span className="inline-flex items-center rounded-l-md border border-r-0 border-gray-300 bg-gray-50 px-3 text-gray-500 sm:text-sm h-10">
-              {NEXT_PUBLIC_WEB_DOMAIN}/anonymous/
+              {settings?.webDomain}/anonymous/
             </span>
             <Input
               type="text"
@@ -141,7 +141,7 @@ export function AnonymousUserPath({
               className="h-10 px-4"
               onClick={() => {
                 navigator.clipboard.writeText(
-                  `${NEXT_PUBLIC_WEB_DOMAIN}/anonymous/${anonymousUserPath}`
+                  `${settings?.webDomain}/anonymous/${anonymousUserPath}`
                 );
                 setPopup({
                   message: "Invite link copied!",

web/src/app/admin/settings/interfaces.ts

@@ -62,4 +62,5 @@ export interface CombinedSettings {
   customAnalyticsScript: string | null;
   isMobile?: boolean;
   webVersion: string | null;
+  webDomain: string | null;
 }

web/src/app/admin/tools/ToolEditor.tsx

@@ -24,6 +24,14 @@ import debounce from "lodash/debounce";
 import { AdvancedOptionsToggle } from "@/components/AdvancedOptionsToggle";
 import Link from "next/link";
 import { Separator } from "@/components/ui/separator";
+import { Checkbox } from "@/components/ui/checkbox";
+import {
+  Tooltip,
+  TooltipContent,
+  TooltipProvider,
+  TooltipTrigger,
+} from "@/components/ui/tooltip";
+import { useAuthType } from "@/lib/hooks";
 
 function parseJsonWithTrailingCommas(jsonString: string) {
   // Regular expression to remove trailing commas before } or ]
@@ -51,7 +59,11 @@ function ToolForm({
 }: {
   existingTool?: ToolSnapshot;
   values: ToolFormValues;
-  setFieldValue: (field: string, value: string) => void;
+  setFieldValue: <T = any>(
+    field: string,
+    value: T,
+    shouldValidate?: boolean
+  ) => void;
   isSubmitting: boolean;
   definitionErrorState: [
     string | null,
@@ -65,6 +77,9 @@ function ToolForm({
   const [definitionError, setDefinitionError] = definitionErrorState;
   const [methodSpecs, setMethodSpecs] = methodSpecsState;
   const [showAdvancedOptions, setShowAdvancedOptions] = useState(false);
+  const authType = useAuthType();
+  const isOAuthEnabled = authType === "oidc" || authType === "google_oauth";
+
   const debouncedValidateDefinition = useCallback(
     (definition: string) => {
       const validateDefinition = async () => {
@@ -218,43 +233,38 @@ function ToolForm({
           </p>
           <FieldArray
             name="customHeaders"
-            render={(arrayHelpers: ArrayHelpers) => (
-              <div className="space-y-4">
-                {values.customHeaders && values.customHeaders.length > 0 && (
-                  <div className="space-y-3">
-                    {values.customHeaders.map(
-                      (
-                        header: { key: string; value: string },
-                        index: number
-                      ) => (
-                        <div
-                          key={index}
-                          className="flex items-center space-x-2 bg-gray-50 p-3 rounded-lg shadow-sm"
+            render={(arrayHelpers) => (
+              <div>
+                <div className="space-y-2">
+                  {values.customHeaders.map(
+                    (header: { key: string; value: string }, index: number) => (
+                      <div
+                        key={index}
+                        className="flex items-center space-x-2 bg-gray-50 p-3 rounded-lg shadow-sm"
+                      >
+                        <Field
+                          name={`customHeaders.${index}.key`}
+                          placeholder="Header Key"
+                          className="flex-1 p-2 border border-gray-300 rounded-md focus:ring-2 focus:ring-primary-500 focus:border-transparent"
+                        />
+                        <Field
+                          name={`customHeaders.${index}.value`}
+                          placeholder="Header Value"
+                          className="flex-1 p-2 border border-gray-300 rounded-md focus:ring-2 focus:ring-primary-500 focus:border-transparent"
+                        />
+                        <Button
+                          type="button"
+                          onClick={() => arrayHelpers.remove(index)}
+                          variant="destructive"
+                          size="sm"
+                          className="transition-colors duration-200 hover:bg-red-600"
                         >
-                          <Field
-                            name={`customHeaders.${index}.key`}
-                            placeholder="Header Key"
-                            className="flex-1 p-2 border border-gray-300 rounded-md focus:ring-2 focus:ring-primary-500 focus:border-transparent"
-                          />
-                          <Field
-                            name={`customHeaders.${index}.value`}
-                            placeholder="Header Value"
-                            className="flex-1 p-2 border border-gray-300 rounded-md focus:ring-2 focus:ring-primary-500 focus:border-transparent"
-                          />
-                          <Button
-                            type="button"
-                            onClick={() => arrayHelpers.remove(index)}
-                            variant="destructive"
-                            size="sm"
-                            className="transition-colors duration-200 hover:bg-red-600"
-                          >
-                            Remove
-                          </Button>
-                        </div>
-                      )
-                    )}
-                  </div>
-                )}
+                          Remove
+                        </Button>
+                      </div>
+                    )
+                  )}
+                </div>
 
                 <Button
                   type="button"
@@ -268,6 +278,75 @@ function ToolForm({
               </div>
             )}
           />
+
+          <div className="mt-6">
+            <h3 className="text-xl font-bold mb-2 text-primary-600">
+              Authentication
+            </h3>
+            {isOAuthEnabled ? (
+              <div className="flex flex-col gap-y-2">
+                <div className="flex items-center space-x-2">
+                  <TooltipProvider>
+                    <Tooltip>
+                      <TooltipTrigger>
+                        <div
+                          className={
+                            values.customHeaders.some(
+                              (header) =>
+                                header.key.toLowerCase() === "authorization"
+                            )
+                              ? "opacity-50"
+                              : ""
+                          }
+                        >
+                          <Checkbox
+                            id="passthrough_auth"
+                            size="sm"
+                            checked={values.passthrough_auth}
+                            disabled={values.customHeaders.some(
+                              (header) =>
+                                header.key.toLowerCase() === "authorization" &&
+                                !values.passthrough_auth
+                            )}
+                            onCheckedChange={(checked) => {
+                              setFieldValue("passthrough_auth", checked, true);
+                            }}
+                          />
+                        </div>
+                      </TooltipTrigger>
+                      {values.customHeaders.some(
+                        (header) => header.key.toLowerCase() === "authorization"
+                      ) && (
+                        <TooltipContent side="top" align="center">
+                          <p className="bg-background-900 max-w-[200px] mb-1 text-sm rounded-lg p-1.5 text-white">
+                            Cannot enable OAuth passthrough when an
+                            Authorization header is already set
+                          </p>
+                        </TooltipContent>
+                      )}
+                    </Tooltip>
+                  </TooltipProvider>
+                  <div className="flex flex-col">
+                    <label
+                      htmlFor="passthrough_auth"
+                      className="text-sm font-medium leading-none peer-disabled:cursor-not-allowed peer-disabled:opacity-70"
+                    >
+                      Pass through user&apos;s OAuth token
+                    </label>
+                    <p className="text-xs text-subtle mt-1">
+                      When enabled, the user&apos;s OAuth token will be passed
+                      as the Authorization header for all API calls
+                    </p>
+                  </div>
+                </div>
+              </div>
+            ) : (
+              <p className="text-sm text-subtle">
+                OAuth passthrough is only available when OIDC or OAuth
+                authentication is enabled
+              </p>
+            )}
+          </div>
         </div>
       )}
 
@@ -291,6 +370,7 @@ function ToolForm({
 interface ToolFormValues {
   definition: string;
   customHeaders: { key: string; value: string }[];
+  passthrough_auth: boolean;
 }
 
 const ToolSchema = Yup.object().shape({
@@ -303,6 +383,7 @@ const ToolSchema = Yup.object().shape({
       })
     )
     .default([]),
+  passthrough_auth: Yup.boolean().default(false),
 });
 
 export function ToolEditor({ tool }: { tool?: ToolSnapshot }) {
@@ -326,9 +407,27 @@ export function ToolEditor({ tool }: { tool?: ToolSnapshot }) {
               key: header.key,
               value: header.value,
             })) ?? [],
+          passthrough_auth: tool?.passthrough_auth ?? false,
         }}
         validationSchema={ToolSchema}
         onSubmit={async (values: ToolFormValues) => {
+          const hasAuthHeader = values.customHeaders?.some(
+            (header) => header.key.toLowerCase() === "authorization"
+          );
+          if (hasAuthHeader && values.passthrough_auth) {
+            setPopup({
+              message:
+                "Cannot enable passthrough auth when Authorization " +
+                "headers are present. Please remove any Authorization " +
+                "headers first.",
+              type: "error",
+            });
+            console.log(
+              "Cannot enable passthrough auth when Authorization headers are present. Please remove any Authorization headers first."
+            );
+            return;
+          }
+
           let definition: any;
           try {
             definition = parseJsonWithTrailingCommas(values.definition);
@@ -344,6 +443,7 @@ export function ToolEditor({ tool }: { tool?: ToolSnapshot }) {
             description: description || "",
             definition: definition,
             custom_headers: values.customHeaders,
+            passthrough_auth: values.passthrough_auth,
           };
           let response;
           if (tool) {

web/src/app/admin/users/page.tsx

@@ -5,7 +5,7 @@ import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card";
 import { Button } from "@/components/ui/button";
 import InvitedUserTable from "@/components/admin/users/InvitedUserTable";
 import SignedUpUserTable from "@/components/admin/users/SignedUpUserTable";
-import { SearchBar } from "@/components/search/SearchBar";
+
 import { FiPlusSquare } from "react-icons/fi";
 import { Modal } from "@/components/Modal";
 import { ThreeDotsLoader } from "@/components/Loading";
@@ -18,6 +18,7 @@ import { ErrorCallout } from "@/components/ErrorCallout";
 import BulkAdd from "@/components/admin/users/BulkAdd";
 import Text from "@/components/ui/text";
 import { InvitedUserSnapshot } from "@/lib/types";
+import { SearchBar } from "@/components/search/SearchBar";
 
 const UsersTables = ({
   q,

web/src/app/assistants/SidebarWrapper.tsx

@@ -117,7 +117,6 @@ export default function SidebarWrapper<T extends object>({
           {" "}
           <HistorySidebar
             setShowAssistantsModal={setShowAssistantsModal}
-            assistants={assistants}
             page={"chat"}
             explicitlyUntoggle={explicitlyUntoggle}
             ref={sidebarElementRef}
@@ -126,7 +125,6 @@ export default function SidebarWrapper<T extends object>({
             existingChats={chatSessions}
             currentChatSession={null}
             folders={folders}
-            openedFolders={openedFolders}
           />
         </div>
       </div>

web/src/app/assistants/mine/AssistantCard.tsx

@@ -1,17 +1,12 @@
-import React, { useState } from "react";
+import React, { useContext, useState, useRef, useLayoutEffect } from "react";
 import { useRouter } from "next/navigation";
 import {
   FiMoreHorizontal,
-  FiShare2,
-  FiEye,
-  FiEyeOff,
   FiTrash,
   FiEdit,
-  FiHash,
   FiBarChart,
   FiLock,
   FiUnlock,
-  FiSearch,
 } from "react-icons/fi";
 import { FaHashtag } from "react-icons/fa";
 import {
@@ -26,33 +21,38 @@ import { Persona } from "@/app/admin/assistants/interfaces";
 import { useUser } from "@/components/user/UserProvider";
 import { useAssistants } from "@/components/context/AssistantsContext";
 import { checkUserOwnsAssistant } from "@/lib/assistants/utils";
-import { toggleAssistantPinnedStatus } from "@/lib/assistants/pinnedAssistants";
 import {
   Tooltip,
   TooltipContent,
   TooltipProvider,
   TooltipTrigger,
 } from "@/components/ui/tooltip";
-import { Button } from "@/components/ui/button";
 import { PinnedIcon } from "@/components/icons/icons";
 import {
   deletePersona,
   togglePersonaPublicStatus,
 } from "@/app/admin/assistants/lib";
-import { HammerIcon } from "lucide-react";
+import { PencilIcon } from "lucide-react";
+import { SettingsContext } from "@/components/settings/SettingsProvider";
+import { usePaidEnterpriseFeaturesEnabled } from "@/components/settings/usePaidEnterpriseFeaturesEnabled";
+import { truncateString } from "@/lib/utils";
 
 export const AssistantBadge = ({
   text,
   className,
+  maxLength,
 }: {
   text: string;
   className?: string;
+  maxLength?: number;
 }) => {
   return (
     <div
-      className={`h-4 px-1.5 py-1 text-[10px]  bg-[#e6e3dd]/50 rounded-lg justify-center items-center gap-1 inline-flex ${className}`}
+      className={`h-4 px-1.5 py-1 text-[10px] flex-none bg-[#e6e3dd]/50 rounded-lg justify-center items-center gap-1 inline-flex ${className}`}
     >
-      <div className="text-[#4a4a4a] font-normal leading-[8px]">{text}</div>
+      <div className="text-[#4a4a4a] font-normal leading-[8px]">
+        {maxLength ? truncateString(text, maxLength) : text}
+      </div>
     </div>
   );
 };
@@ -62,9 +62,9 @@ const AssistantCard: React.FC<{
   pinned: boolean;
   closeModal: () => void;
 }> = ({ persona, pinned, closeModal }) => {
-  const { user, refreshUser } = useUser();
+  const { user, toggleAssistantPinnedStatus } = useUser();
   const router = useRouter();
-  const { refreshAssistants } = useAssistants();
+  const { refreshAssistants, pinnedAssistants } = useAssistants();
 
   const isOwnedByUser = checkUserOwnsAssistant(user, persona);
 
@@ -72,7 +72,8 @@ const AssistantCard: React.FC<{
     undefined
   );
 
-  const handleShare = () => setActivePopover("visibility");
+  const isPaidEnterpriseFeaturesEnabled = usePaidEnterpriseFeaturesEnabled();
+
   const handleDelete = () => setActivePopover("delete");
   const handleEdit = () => {
     router.push(`/assistants/edit/${persona.id}`);
@@ -81,33 +82,74 @@ const AssistantCard: React.FC<{
 
   const closePopover = () => setActivePopover(undefined);
 
+  const nameRef = useRef<HTMLHeadingElement>(null);
+  const hiddenNameRef = useRef<HTMLSpanElement>(null);
+  const [isNameTruncated, setIsNameTruncated] = useState(false);
+
+  useLayoutEffect(() => {
+    const checkTruncation = () => {
+      if (nameRef.current && hiddenNameRef.current) {
+        const visibleWidth = nameRef.current.offsetWidth;
+        const fullTextWidth = hiddenNameRef.current.offsetWidth;
+        setIsNameTruncated(fullTextWidth > visibleWidth);
+      }
+    };
+
+    checkTruncation();
+    window.addEventListener("resize", checkTruncation);
+    return () => window.removeEventListener("resize", checkTruncation);
+  }, [persona.name]);
+
   return (
     <div className="w-full p-2 overflow-visible pb-4 pt-3 bg-[#fefcf9] rounded shadow-[0px_0px_4px_0px_rgba(0,0,0,0.25)] flex flex-col">
       <div className="w-full flex">
-        <div className="ml-2 mr-4 mt-1 w-8 h-8">
+        <div className="ml-2 flex-none mr-2 mt-1 w-10 h-10">
           <AssistantIcon assistant={persona} size="large" />
         </div>
         <div className="flex-1 mt-1 flex flex-col">
           <div className="flex justify-between items-start mb-1">
             <div className="flex items-end gap-x-2 leading-none">
-              <h3 className="text-black leading-none font-semibold text-base lg-normal">
+              <TooltipProvider>
+                <Tooltip>
+                  <TooltipTrigger asChild>
+                    <h3
+                      ref={nameRef}
+                      className={` text-black line-clamp-1 break-all	 text-ellipsis leading-none font-semibold text-base lg-normal w-full overflow-hidden`}
+                    >
+                      {persona.name}
+                    </h3>
+                  </TooltipTrigger>
+                  {isNameTruncated && (
+                    <TooltipContent>{persona.name}</TooltipContent>
+                  )}
+                </Tooltip>
+              </TooltipProvider>
+              <span
+                ref={hiddenNameRef}
+                className="absolute left-0 top-0 invisible whitespace-nowrap"
+                aria-hidden="true"
+              >
                 {persona.name}
-              </h3>
+              </span>
               {persona.labels && persona.labels.length > 0 && (
                 <>
-                  {persona.labels.slice(0, 3).map((label, index) => (
-                    <AssistantBadge key={index} text={label.name} />
-                  ))}
-                  {persona.labels.length > 3 && (
+                  {persona.labels.slice(0, 2).map((label, index) => (
                     <AssistantBadge
-                      text={`+${persona.labels.length - 3} more`}
+                      key={index}
+                      text={label.name}
+                      maxLength={10}
+                    />
+                  ))}
+                  {persona.labels.length > 2 && (
+                    <AssistantBadge
+                      text={`+${persona.labels.length - 2} more`}
                     />
                   )}
                 </>
               )}
             </div>
             {isOwnedByUser && (
-              <div className="flex items-center gap-x-2">
+              <div className="flex ml-2 items-center gap-x-2">
                 <Popover
                   open={activePopover !== undefined}
                   onOpenChange={(open) =>
@@ -141,41 +183,29 @@ const AssistantCard: React.FC<{
                           <FiEdit size={12} className="inline mr-2" />
                           Edit
                         </button>
-                        {/* 
-                        <button
-                          onClick={isOwnedByUser ? handleShare : undefined}
-                          className={`w-full text-left flex items-center px-2 py-1 rounded ${
-                            isOwnedByUser
-                              ? "hover:bg-neutral-100"
-                              : "opacity-50 cursor-not-allowed"
-                          }`}
-                          disabled={!isOwnedByUser}
-                        >
-                          <FiShare2 size={12} className="inline mr-2" />
-                          Share
-                        </button> */}
-
-                        <button
-                          onClick={
-                            isOwnedByUser
-                              ? () => {
-                                  router.push(
-                                    `/assistants/stats/${persona.id}`
-                                  );
-                                  closePopover();
-                                }
-                              : undefined
-                          }
-                          className={`w-full text-left items-center px-2 py-1 rounded ${
-                            isOwnedByUser
-                              ? "hover:bg-neutral-100"
-                              : "opacity-50 cursor-not-allowed"
-                          }`}
-                          disabled={!isOwnedByUser}
-                        >
-                          <FiBarChart size={12} className="inline mr-2" />
-                          Stats
-                        </button>
+                        {isPaidEnterpriseFeaturesEnabled && (
+                          <button
+                            onClick={
+                              isOwnedByUser
+                                ? () => {
+                                    router.push(
+                                      `/assistants/stats/${persona.id}`
+                                    );
+                                    closePopover();
+                                  }
+                                : undefined
+                            }
+                            className={`w-full text-left items-center px-2 py-1 rounded ${
+                              isOwnedByUser
+                                ? "hover:bg-neutral-100"
+                                : "opacity-50 cursor-not-allowed"
+                            }`}
+                            disabled={!isOwnedByUser}
+                          >
+                            <FiBarChart size={12} className="inline mr-2" />
+                            Stats
+                          </button>
+                        )}
                         <button
                           onClick={isOwnedByUser ? handleDelete : undefined}
                           className={`w-full text-left items-center px-2 py-1 rounded ${
@@ -221,33 +251,33 @@ const AssistantCard: React.FC<{
             )}
           </div>
 
-          <p className="text-black font-[350] mt-0 text-sm mb-1 line-clamp-2 h-[2.7em]">
+          <p className="text-black font-[350] mt-0 text-sm line-clamp-2 h-[2.7em]">
             {persona.description || "\u00A0"}
           </p>
 
           <div className="flex flex-col ">
-            {/* <div className="mb-1 mt-1">
-              <div className="flex items-center">
-               
-              </div>
-            </div> */}
-
-            <div className="my-1">
-              <span className="flex items-center text-black text-xs opacity-50">
-                {(persona.owner?.email || persona.builtin_persona) && "By "}
-                {persona.owner?.email || (persona.builtin_persona && "Onyx")}
-                {(persona.owner?.email || persona.builtin_persona) && (
-                  <span className="mx-2">•</span>
-                )}
-                {persona.tools.length > 0 ? (
+            <div className="my-1.5">
+              <p className="flex items-center text-black text-xs opacity-50">
+                {persona.owner?.email || persona.builtin_persona ? (
                   <>
-                    {persona.tools.length}
-                    {" Action"}
-                    {persona.tools.length !== 1 ? "s" : ""}
+                    <span className="truncate">
+                      By {persona.owner?.email || "Onyx"}
+                    </span>
+
+                    <span className="mx-2">•</span>
                   </>
-                ) : (
-                  "No Actions"
-                )}
+                ) : null}
+                <span className="flex-none truncate">
+                  {persona.tools.length > 0 ? (
+                    <>
+                      {persona.tools.length}
+                      {" Action"}
+                      {persona.tools.length !== 1 ? "s" : ""}
+                    </>
+                  ) : (
+                    "No Actions"
+                  )}
+                </span>
                 <span className="mx-2">•</span>
                 {persona.is_public ? (
                   <>
@@ -260,17 +290,7 @@ const AssistantCard: React.FC<{
                     Private
                   </>
                 )}
-              </span>
-            </div>
-
-            <div className="mb-1 flex flex-wrap">
-              {persona.document_sets.slice(0, 5).map((set, index) => (
-                <AssistantBadge
-                  className="!text-base"
-                  key={index}
-                  text={set.name}
-                />
-              ))}
+              </p>
             </div>
           </div>
           <div className="flex gap-2">
@@ -284,7 +304,7 @@ const AssistantCard: React.FC<{
                     }}
                     className="hover:bg-neutral-100 hover:text-text px-2 py-1 gap-x-1 rounded border border-black flex items-center"
                   >
-                    <FaHashtag size={12} className="flex-none" />
+                    <PencilIcon size={12} className="flex-none" />
                     <span className="text-xs">Start Chat</span>
                   </button>
                 </TooltipTrigger>
@@ -296,20 +316,25 @@ const AssistantCard: React.FC<{
             <TooltipProvider>
               <Tooltip>
                 <TooltipTrigger asChild>
-                  <button
+                  <div
                     onClick={async () => {
                       await toggleAssistantPinnedStatus(
-                        user?.preferences.pinned_assistants || [],
+                        pinnedAssistants.map((a) => a.id),
                         persona.id,
                         !pinned
                       );
-                      await refreshUser();
                     }}
-                    className="hover:bg-neutral-100 px-2 py-1 gap-x-1 rounded border border-black flex items-center w-[65px]"
+                    className="hover:bg-neutral-100 px-2 group cursor-pointer py-1 gap-x-1 relative rounded border border-black flex items-center w-[65px]"
                   >
                     <PinnedIcon size={12} />
-                    <p className="text-xs">{pinned ? "Unpin" : "Pin"}</p>
-                  </button>
+                    {!pinned ? (
+                      <p className="absolute w-full left-0 group-hover:text-black w-full text-center transform text-xs">
+                        Pin
+                      </p>
+                    ) : (
+                      <p className="text-xs group-hover:text-black">Unpin</p>
+                    )}
+                  </div>
                 </TooltipTrigger>
                 <TooltipContent>
                   {pinned ? "Remove from" : "Add to"} your pinned list

web/src/app/assistants/mine/AssistantModal.tsx

@@ -1,16 +1,14 @@
 "use client";
 
-import React, { useMemo, useState, useEffect } from "react";
+import React, { useMemo, useState } from "react";
 import { Persona } from "@/app/admin/assistants/interfaces";
 import { useRouter } from "next/navigation";
-
 import { Modal } from "@/components/Modal";
 import AssistantCard from "./AssistantCard";
 import { useAssistants } from "@/components/context/AssistantsContext";
-import { checkUserOwnsAssistant } from "@/lib/assistants/checkOwnership";
 import { useUser } from "@/components/user/UserProvider";
-import { Button } from "@/components/ui/button";
-import { useLabels } from "@/lib/hooks";
+import { FilterIcon } from "lucide-react";
+import { checkUserOwnsAssistant } from "@/lib/assistants/checkOwnership";
 
 export const AssistantBadgeSelector = ({
   text,
@@ -27,7 +25,7 @@ export const AssistantBadgeSelector = ({
         selected
           ? "bg-neutral-900 text-white"
           : "bg-transparent text-neutral-900"
-      } h-5 px-1 py-0.5 rounded-lg cursor-pointer text-[12px] font-normal leading-[10px] border border-black justify-center items-center gap-1 inline-flex`}
+      } w-12 h-5 text-center px-1 py-0.5 rounded-lg cursor-pointer text-[12px] font-normal leading-[10px] border border-black justify-center items-center gap-1 inline-flex`}
       onClick={toggleFilter}
     >
       {text}
@@ -39,6 +37,7 @@ export enum AssistantFilter {
   Pinned = "Pinned",
   Public = "Public",
   Private = "Private",
+  Mine = "Mine",
 }
 
 const useAssistantFilter = () => {
@@ -48,6 +47,7 @@ const useAssistantFilter = () => {
     [AssistantFilter.Pinned]: false,
     [AssistantFilter.Public]: false,
     [AssistantFilter.Private]: false,
+    [AssistantFilter.Mine]: false,
   });
 
   const toggleAssistantFilter = (filter: AssistantFilter) => {
@@ -65,11 +65,8 @@ export default function AssistantModal({
 }: {
   hideModal: () => void;
 }) {
-  const [showAllFeaturedAssistants, setShowAllFeaturedAssistants] =
-    useState(false);
-  const { assistants, visibleAssistants, pinnedAssistants } = useAssistants();
-  const { assistantFilters, toggleAssistantFilter, setAssistantFilters } =
-    useAssistantFilter();
+  const { assistants, pinnedAssistants } = useAssistants();
+  const { assistantFilters, toggleAssistantFilter } = useAssistantFilter();
   const router = useRouter();
   const { user } = useUser();
   const [searchQuery, setSearchQuery] = useState("");
@@ -89,16 +86,21 @@ export default function AssistantModal({
         !assistantFilters[AssistantFilter.Private] || !assistant.is_public;
       const pinnedFilter =
         !assistantFilters[AssistantFilter.Pinned] ||
-        pinnedAssistants.map((a: Persona) => a.id).includes(assistant.id);
+        (user?.preferences?.pinned_assistants?.includes(assistant.id) ?? false);
+
+      const mineFilter =
+        !assistantFilters[AssistantFilter.Mine] ||
+        assistants.map((a: Persona) => checkUserOwnsAssistant(user, a));
 
       return (
         (nameMatches || labelMatches) &&
         publicFilter &&
         privateFilter &&
-        pinnedFilter
+        pinnedFilter &&
+        mineFilter
       );
     });
-  }, [assistants, searchQuery, assistantFilters, pinnedAssistants]);
+  }, [assistants, searchQuery, assistantFilters]);
 
   const featuredAssistants = [
     ...memoizedCurrentlyVisibleAssistants.filter(
@@ -122,10 +124,10 @@ export default function AssistantModal({
       heightOverride={`${height}px`}
       onOutsideClick={hideModal}
       removeBottomPadding
-      className={`max-w-4xl ${height} w-[95%] overflow-hidden`}
+      className={`max-w-4xl max-h-[90vh] ${height} w-[95%] overflow-hidden`}
     >
       <div className="flex flex-col h-full">
-        <div className="flex flex-col sticky top-0 z-10">
+        <div className="flex bg-background flex-col sticky top-0 z-10">
           <div className="flex px-2 justify-between items-center gap-x-2 mb-0">
             <div className="h-12 w-full rounded-lg flex-col justify-center items-start gap-2.5 inline-flex">
               <div className="h-12 rounded-md w-full shadow-[0px_0px_2px_0px_rgba(0,0,0,0.25)] border border-[#dcdad4] flex items-center px-3">
@@ -164,16 +166,18 @@ export default function AssistantModal({
               </div>
             </button>
           </div>
-          <div className="px-2 flex py-2 items-center gap-x-2 mb-2 flex-wrap">
+          <div className="px-2 flex py-4 items-center gap-x-2 flex-wrap">
+            <FilterIcon size={16} />
             <AssistantBadgeSelector
               text="Pinned"
               selected={assistantFilters[AssistantFilter.Pinned]}
               toggleFilter={() => toggleAssistantFilter(AssistantFilter.Pinned)}
             />
+
             <AssistantBadgeSelector
-              text="Public"
-              selected={assistantFilters[AssistantFilter.Public]}
-              toggleFilter={() => toggleAssistantFilter(AssistantFilter.Public)}
+              text="Mine"
+              selected={assistantFilters[AssistantFilter.Mine]}
+              toggleFilter={() => toggleAssistantFilter(AssistantFilter.Mine)}
             />
             <AssistantBadgeSelector
               text="Private"
@@ -182,6 +186,11 @@ export default function AssistantModal({
                 toggleAssistantFilter(AssistantFilter.Private)
               }
             />
+            <AssistantBadgeSelector
+              text="Public"
+              selected={assistantFilters[AssistantFilter.Public]}
+              toggleFilter={() => toggleAssistantFilter(AssistantFilter.Public)}
+            />
           </div>
           <div className="w-full border-t border-neutral-200" />
         </div>
@@ -196,7 +205,9 @@ export default function AssistantModal({
               featuredAssistants.map((assistant, index) => (
                 <div key={index}>
                   <AssistantCard
-                    pinned={pinnedAssistants.includes(assistant)}
+                    pinned={pinnedAssistants
+                      .map((a) => a.id)
+                      .includes(assistant.id)}
                     persona={assistant}
                     closeModal={hideModal}
                   />
@@ -221,7 +232,11 @@ export default function AssistantModal({
                   .map((assistant, index) => (
                     <div key={index}>
                       <AssistantCard
-                        pinned={pinnedAssistants.includes(assistant)}
+                        pinned={
+                          user?.preferences?.pinned_assistants?.includes(
+                            assistant.id
+                          ) ?? false
+                        }
                         persona={assistant}
                         closeModal={hideModal}
                       />

web/src/app/chat/ChatBanner.tsx

@@ -60,7 +60,6 @@ export function ChatBanner() {
       `}
       onMouseLeave={handleMouseLeave}
       aria-expanded={isExpanded}
-      role="region"
     >
       <div className="text-emphasis text-sm w-full">
         {/* Padding for consistent spacing */}

web/src/app/chat/ChatPage.tsx

@@ -25,7 +25,6 @@ import { HealthCheckBanner } from "@/components/health/healthcheck";
 import {
   buildChatUrl,
   buildLatestMessageChain,
-  checkAnyAssistantHasSearch,
   createChatSession,
   deleteAllChatSessions,
   getCitedDocumentsFromMessage,
@@ -305,12 +304,7 @@ export function ChatPage({
   const [presentingDocument, setPresentingDocument] =
     useState<OnyxDocument | null>(null);
 
-  const {
-    visibleAssistants: assistants,
-    recentAssistants,
-    assistants: allAssistants,
-    refreshRecentAssistants,
-  } = useAssistants();
+  const { recentAssistants, refreshRecentAssistants } = useAssistants();
 
   const liveAssistant: Persona | undefined =
     alternativeAssistant ||
@@ -1438,10 +1432,10 @@ export function ChatPage({
               }
             }
 
-            parentMessage =
-              parentMessage || frozenMessageMap?.get(SYSTEM_MESSAGE_ID)!;
             // on initial message send, we insert a dummy system message
             // set this as the parent here if no parent is set
+            parentMessage =
+              parentMessage || frozenMessageMap?.get(SYSTEM_MESSAGE_ID)!;
 
             const updateFn = (messages: Message[]) => {
               const replacementsMap = regenerationRequest
@@ -1890,6 +1884,7 @@ export function ChatPage({
   const [showDeleteAllModal, setShowDeleteAllModal] = useState(false);
 
   const currentPersona = alternativeAssistant || liveAssistant;
+
   useEffect(() => {
     const handleSlackChatRedirect = async () => {
       if (!slackChatId) return;
@@ -2064,9 +2059,9 @@ export function ChatPage({
       {retrievalEnabled && documentSidebarToggled && settings?.isMobile && (
         <div className="md:hidden">
           <Modal
+            hideDividerForTitle
             onOutsideClick={() => setDocumentSidebarToggled(false)}
-            noPadding
-            noScroll
+            title="Sources"
           >
             <DocumentResults
               setPresentingDocument={setPresentingDocument}
@@ -2083,6 +2078,7 @@ export function ChatPage({
               maxTokens={maxTokens}
               initialWidth={400}
               isOpen={true}
+              removeHeader
             />
           </Modal>
         </div>
@@ -2160,20 +2156,16 @@ export function ChatPage({
               <div className="w-full relative">
                 <HistorySidebar
                   setShowAssistantsModal={setShowAssistantsModal}
-                  assistants={assistants}
                   explicitlyUntoggle={explicitlyUntoggle}
-                  stopGenerating={stopGenerating}
                   reset={() => setMessage("")}
                   page="chat"
                   ref={innerSidebarElementRef}
                   toggleSidebar={toggleSidebar}
                   toggled={toggledSidebar}
-                  backgroundToggled={toggledSidebar || showHistorySidebar}
                   currentAssistantId={liveAssistant?.id}
                   existingChats={chatSessions}
                   currentChatSession={selectedChatSession}
                   folders={folders}
-                  openedFolders={openedFolders}
                   removeToggle={removeToggle}
                   showShareModal={showShareModal}
                   showDeleteAllModal={() => setShowDeleteAllModal(true)}
@@ -2191,7 +2183,11 @@ export function ChatPage({
                 bg-opacity-80
                 duration-300
                 ease-in-out
-                ${documentSidebarToggled && "opacity-100 w-[350px]"}`}
+                ${
+                  documentSidebarToggled &&
+                  !settings?.isMobile &&
+                  "opacity-100 w-[350px]"
+                }`}
               ></div>
             </div>
           </div>
@@ -2212,7 +2208,11 @@ export function ChatPage({
                 duration-300
                 ease-in-out
                 h-full
-                ${documentSidebarToggled ? "w-[400px]" : "w-[0px]"}
+                ${
+                  documentSidebarToggled && !settings?.isMobile
+                    ? "w-[400px]"
+                    : "w-[0px]"
+                }
             `}
           >
             <DocumentResults
@@ -2229,12 +2229,13 @@ export function ChatPage({
               selectedDocumentTokens={selectedDocumentTokens}
               maxTokens={maxTokens}
               initialWidth={400}
-              isOpen={documentSidebarToggled}
+              isOpen={documentSidebarToggled && !settings?.isMobile}
             />
           </div>
 
           <BlurBackground
             visible={!untoggled && (showHistorySidebar || toggledSidebar)}
+            onClick={() => toggleSidebar()}
           />
 
           <div
@@ -2253,7 +2254,9 @@ export function ChatPage({
                       ? setSharingModalVisible
                       : undefined
                   }
-                  documentSidebarToggled={documentSidebarToggled}
+                  documentSidebarToggled={
+                    documentSidebarToggled && !settings?.isMobile
+                  }
                   toggleSidebar={toggleSidebar}
                   currentChatSession={selectedChatSession}
                   hideUserDropdown={user?.is_anonymous_user}
@@ -2319,7 +2322,7 @@ export function ChatPage({
                             currentSessionChatState == "input" &&
                             !loadingError &&
                             !submittedMessage && (
-                              <div className="h-full w-[95%] mx-auto flex flex-col justify-center items-center">
+                              <div className="h-full  w-[95%] mx-auto flex flex-col justify-center items-center">
                                 <ChatIntro selectedPersona={liveAssistant} />
 
                                 <StarterMessages
@@ -2340,9 +2343,10 @@ export function ChatPage({
                               (settings?.enterpriseSettings
                                 ?.two_lines_for_chat_header
                                 ? "pt-20 "
-                                : "pt-8") +
-                              (hasPerformedInitialScroll ? "" : "invisible")
+                                : "pt-8 ")
                             }
+                            // NOTE: temporarily removing this to fix the scroll bug
+                            // (hasPerformedInitialScroll ? "" : "invisible")
                           >
                             {(messageHistory.length < BUFFER_COUNT
                               ? messageHistory
@@ -2473,12 +2477,6 @@ export function ChatPage({
                                         setPresentingDocument
                                       }
                                       index={i}
-                                      selectedMessageForDocDisplay={
-                                        selectedMessageForDocDisplay
-                                      }
-                                      documentSelectionToggled={
-                                        documentSidebarToggled
-                                      }
                                       continueGenerating={
                                         i == messageHistory.length - 1 &&
                                         currentCanContinue()
@@ -2598,19 +2596,6 @@ export function ChatPage({
                                             }
                                           : undefined
                                       }
-                                      handleShowRetrieved={(messageNumber) => {
-                                        if (isShowingRetrieved) {
-                                          setSelectedMessageForDocDisplay(null);
-                                        } else {
-                                          if (messageNumber !== null) {
-                                            setSelectedMessageForDocDisplay(
-                                              messageNumber
-                                            );
-                                          } else {
-                                            setSelectedMessageForDocDisplay(-1);
-                                          }
-                                        }
-                                      }}
                                       handleForceSearch={() => {
                                         if (
                                           previousMessage &&
@@ -2817,7 +2802,11 @@ export function ChatPage({
                           duration-300 
                           ease-in-out
                           h-full
-                          ${documentSidebarToggled ? "w-[350px]" : "w-[0px]"}
+                          ${
+                            documentSidebarToggled && !settings?.isMobile
+                              ? "w-[350px]"
+                              : "w-[0px]"
+                          }
                       `}
                       ></div>
                     </div>

web/src/app/chat/RegenerateOption.tsx

@@ -4,10 +4,7 @@ import {
   LlmOverride,
   useLlmOverride,
 } from "@/lib/hooks";
-import {
-  DefaultDropdownElement,
-  StringOrNumberOption,
-} from "@/components/Dropdown";
+import { StringOrNumberOption } from "@/components/Dropdown";
 
 import { Persona } from "@/app/admin/assistants/interfaces";
 import { destructureValue, getFinalLLM, structureValue } from "@/lib/llm/utils";
@@ -15,7 +12,7 @@ import { useState } from "react";
 import { Hoverable } from "@/components/Hoverable";
 import { Popover } from "@/components/popover/Popover";
 import { IconType } from "react-icons";
-import { FiRefreshCw } from "react-icons/fi";
+import { FiRefreshCw, FiCheck } from "react-icons/fi";
 
 export function RegenerateDropdown({
   options,
@@ -43,45 +40,33 @@ export function RegenerateDropdown({
   };
 
   const Dropdown = (
-    <div
-      className={`
-                border 
-                border 
-                rounded-lg 
-                flex 
-                flex-col
-                mx-2
-                bg-background
-                ${maxHeight || "max-h-72"}
-                overflow-y-auto 
-                overscroll-contain relative`}
-    >
-      <p
-        className="
-                sticky 
-                top-0 
-                flex
-                bg-background
-                font-medium
-                px-2
-                text-sm 
-                py-1.5 
-                "
-      >
-        Regenerate with
-      </p>
-      {options.map((option, ind) => {
-        const isSelected = option.value === selected;
-        return (
-          <DefaultDropdownElement
-            key={option.value}
-            name={getDisplayNameForModel(option.name)}
-            description={option.description}
-            onSelect={() => onSelect(option.value)}
-            isSelected={isSelected}
-          />
-        );
-      })}
+    <div className="overflow-y-auto py-2 min-w-fit bg-white dark:bg-gray-800 rounded-md shadow-lg">
+      <div className="mb-1 flex items-center justify-between px-4 pt-2">
+        <span className="text-sm text-text-500 dark:text-text-400">
+          Regenerate with
+        </span>
+      </div>
+      {options.map((option) => (
+        <div
+          key={option.value}
+          role="menuitem"
+          className={`flex items-center m-1.5 p-1.5 text-sm cursor-pointer focus-visible:outline-0 group relative hover:bg-gray-100 dark:hover:bg-gray-700 rounded-md my-0 px-3 mx-2 gap-2.5 py-3 !pr-3 ${
+            option.value === selected ? "bg-gray-100 dark:bg-gray-700" : ""
+          }`}
+          onClick={() => onSelect(option.value)}
+        >
+          <div className="flex grow items-center justify-between gap-2">
+            <div>
+              <div className="flex items-center gap-3">
+                <div>{getDisplayNameForModel(option.name)}</div>
+              </div>
+            </div>
+          </div>
+          {option.value === selected && (
+            <FiCheck className="text-blue-500 dark:text-blue-400" />
+          )}
+        </div>
+      ))}
     </div>
   );
 

web/src/app/chat/documentSidebar/ChatDocumentDisplay.tsx

@@ -79,13 +79,9 @@ export function ChatDocumentDisplay({
     document.updated_at || Object.keys(document.metadata).length > 0;
 
   return (
-    <div
-      className={`desktop:max-w-[400px] opacity-100 ${
-        modal ? "w-[90vw]" : "w-full"
-      }`}
-    >
+    <div className="desktop:max-w-[400px] opacity-100 w-full">
       <div
-        className={`flex relative flex-col px-3 py-2.5 gap-0.5 rounded-xl mx-2 my-1 ${
+        className={`flex relative flex-col px-3 py-2.5 gap-0.5 rounded-xl my-1 ${
           isSelected ? "bg-[#ebe7de]" : "bg- hover:bg-[#ebe7de]/80"
         }`}
       >

web/src/app/chat/documentSidebar/DocumentResults.tsx

@@ -26,6 +26,7 @@ interface DocumentResultsProps {
   isSharedChat?: boolean;
   modal: boolean;
   setPresentingDocument: Dispatch<SetStateAction<OnyxDocument | null>>;
+  removeHeader?: boolean;
 }
 
 export const DocumentResults = forwardRef<HTMLDivElement, DocumentResultsProps>(
@@ -43,10 +44,10 @@ export const DocumentResults = forwardRef<HTMLDivElement, DocumentResultsProps>(
       isSharedChat,
       isOpen,
       setPresentingDocument,
+      removeHeader,
     },
     ref: ForwardedRef<HTMLDivElement>
   ) => {
-    const { popup, setPopup } = usePopup();
     const [delayedSelectedDocumentCount, setDelayedSelectedDocumentCount] =
       useState(0);
 
@@ -98,21 +99,29 @@ export const DocumentResults = forwardRef<HTMLDivElement, DocumentResultsProps>(
             }}
           >
             <div className="flex flex-col h-full">
-              {popup}
-              <div className="p-4 flex items-center justify-between gap-x-2">
-                <div className="flex items-center gap-x-2">
-                  {/* <SourcesIcon size={32} /> */}
-                  <h2 className="text-xl font-bold text-text-900">Sources</h2>
-                </div>
-                <button className="my-auto" onClick={closeSidebar}>
-                  <XIcon size={16} />
-                </button>
-              </div>
-              <div className="border-b border-divider-history-sidebar-bar mx-3" />
-              <div className="overflow-y-auto h-fit mb-8 pb-8 -mx-1 sm:mx-0 flex-grow gap-y-0 default-scrollbar dark-scrollbar flex flex-col">
+              {!removeHeader && (
+                <>
+                  <div className="p-4 flex items-center justify-between gap-x-2">
+                    <div className="flex items-center gap-x-2">
+                      <h2 className="text-xl font-bold text-text-900">
+                        Sources
+                      </h2>
+                    </div>
+                    <button className="my-auto" onClick={closeSidebar}>
+                      <XIcon size={16} />
+                    </button>
+                  </div>
+                  <div className="border-b border-divider-history-sidebar-bar mx-3" />
+                </>
+              )}
+
+              <div className="overflow-y-auto h-fit mb-8 pb-8 sm:mx-0 flex-grow gap-y-0 default-scrollbar dark-scrollbar flex flex-col">
                 {dedupedDocuments.length > 0 ? (
                   dedupedDocuments.map((document, ind) => (
-                    <div key={document.document_id} className="w-full">
+                    <div
+                      key={document.document_id}
+                      className={`desktop:px-2 w-full`}
+                    >
                       <ChatDocumentDisplay
                         setPresentingDocument={setPresentingDocument}
                         closeSidebar={closeSidebar}

web/src/app/chat/folders/FolderDropdown.tsx

@@ -34,6 +34,7 @@ interface FolderDropdownProps {
   onDelete?: (folderId: number) => void;
   onDrop?: (folderId: number, chatSessionId: string) => void;
   children?: ReactNode;
+  index: number;
 }
 
 export const FolderDropdown = forwardRef<HTMLDivElement, FolderDropdownProps>(
@@ -46,6 +47,7 @@ export const FolderDropdown = forwardRef<HTMLDivElement, FolderDropdownProps>(
       onEdit,
       onDrop,
       children,
+      index,
     },
     ref
   ) => {
@@ -155,117 +157,123 @@ export const FolderDropdown = forwardRef<HTMLDivElement, FolderDropdownProps>(
         ref={setNodeRef}
         style={style}
         {...attributes}
-        className="overflow-visible w-full"
+        className="overflow-visible mt-2 w-full"
         onDragOver={handleDragOver}
         onDrop={handleDrop}
       >
         <div
-          ref={ref}
-          className="flex overflow-visible items-center w-full text-[#6c6c6c] rounded-md p-1 relative"
-          onMouseEnter={() => setIsHovered(true)}
-          onMouseLeave={() => setIsHovered(false)}
+          className="sticky top-0 bg-background-sidebar z-10"
+          style={{ zIndex: 1000 - index }}
         >
-          <button
-            className="flex overflow-hidden items-center flex-grow"
-            onClick={() => !isEditing && setIsOpen(!isOpen)}
-            {...(isEditing ? {} : listeners)}
+          <div
+            ref={ref}
+            className="flex overflow-visible items-center w-full text-text-darker rounded-md p-1 relative bg-background-sidebar sticky top-0"
+            style={{ zIndex: 10 - index }}
+            onMouseEnter={() => setIsHovered(true)}
+            onMouseLeave={() => setIsHovered(false)}
           >
-            {isOpen ? (
-              <Caret size={16} className="mr-1" />
-            ) : (
-              <Caret size={16} className="-rotate-90 mr-1" />
-            )}
-            {isEditing ? (
-              <div ref={editingRef} className="flex-grow z-[9999] relative">
-                <input
-                  ref={inputRef}
-                  type="text"
-                  value={newFolderName}
-                  onChange={(e) => setNewFolderName(e.target.value)}
-                  className="text-sm font-medium bg-transparent outline-none w-full pb-1 border-b border-[#6c6c6c] transition-colors duration-200"
-                  onKeyDown={(e) => {
-                    if (e.key === "Enter") {
-                      handleEdit();
-                    }
-                  }}
-                  onClick={(e) => e.stopPropagation()}
-                />
-              </div>
-            ) : (
-              <div className="flex items-center">
-                <span className="text-sm font-medium">
-                  {folder.folder_name}
-                </span>
-              </div>
-            )}
-          </button>
-          {isHovered && !isEditing && folder.folder_id && (
             <button
-              onClick={(e) => {
-                e.stopPropagation();
-                setIsEditing(true);
-              }}
-              className="ml-auto px-1"
+              className="flex overflow-hidden items-center flex-grow"
+              onClick={() => !isEditing && setIsOpen(!isOpen)}
+              {...(isEditing ? {} : listeners)}
             >
-              <PencilIcon size={14} />
-            </button>
-          )}
-          {(isHovered || isDeletePopoverOpen) &&
-            !isEditing &&
-            folder.folder_id && (
-              <Popover
-                open={isDeletePopoverOpen}
-                onOpenChange={setIsDeletePopoverOpen}
-                content={
-                  <button
-                    onClick={(e) => {
-                      e.stopPropagation();
-                      handleDeleteClick();
+              {isOpen ? (
+                <Caret size={16} className="mr-1" />
+              ) : (
+                <Caret size={16} className="-rotate-90 mr-1" />
+              )}
+              {isEditing ? (
+                <div ref={editingRef} className="flex-grow z-[9999] relative">
+                  <input
+                    ref={inputRef}
+                    type="text"
+                    value={newFolderName}
+                    onChange={(e) => setNewFolderName(e.target.value)}
+                    className="text-sm font-medium bg-transparent outline-none w-full pb-1 border-b border-[#6c6c6c] transition-colors duration-200"
+                    onKeyDown={(e) => {
+                      if (e.key === "Enter") {
+                        handleEdit();
+                      }
                     }}
-                    className="px-1"
-                  >
-                    <FiTrash2 size={14} />
-                  </button>
-                }
-                popover={
-                  <div className="p-3 w-64 border border-border rounded-lg bg-background z-50">
-                    <p className="text-sm mb-3">
-                      Are you sure you want to delete this folder?
-                    </p>
-                    <div className="flex justify-center gap-2">
-                      <button
-                        className="px-3 py-1 text-sm bg-gray-200 rounded"
-                        onClick={handleCancelDelete}
-                      >
-                        Cancel
-                      </button>
-                      <button
-                        className="px-3 py-1 text-sm bg-red-500 text-white rounded"
-                        onClick={handleConfirmDelete}
-                      >
-                        Delete
-                      </button>
-                    </div>
-                  </div>
-                }
-                requiresContentPadding
-                sideOffset={6}
-              />
+                    onClick={(e) => e.stopPropagation()}
+                  />
+                </div>
+              ) : (
+                <div className="flex items-center">
+                  <span className="text-sm font-[500]">
+                    {folder.folder_name}
+                  </span>
+                </div>
+              )}
+            </button>
+            {isHovered && !isEditing && folder.folder_id && (
+              <button
+                onClick={(e) => {
+                  e.stopPropagation();
+                  setIsEditing(true);
+                }}
+                className="ml-auto px-1"
+              >
+                <PencilIcon size={14} />
+              </button>
             )}
-          {isEditing && (
-            <div className="flex -my-1 z-[9999]">
-              <button onClick={handleEdit} className="p-1">
-                <FiCheck size={14} />
-              </button>
-              <button onClick={() => setIsEditing(false)} className="p-1">
-                <FiX size={14} />
-              </button>
-            </div>
+            {(isHovered || isDeletePopoverOpen) &&
+              !isEditing &&
+              folder.folder_id && (
+                <Popover
+                  open={isDeletePopoverOpen}
+                  onOpenChange={setIsDeletePopoverOpen}
+                  content={
+                    <button
+                      onClick={(e) => {
+                        e.stopPropagation();
+                        handleDeleteClick();
+                      }}
+                      className="px-1"
+                    >
+                      <FiTrash2 size={14} />
+                    </button>
+                  }
+                  popover={
+                    <div className="p-3 w-64 border border-border rounded-lg bg-background z-50">
+                      <p className="text-sm mb-3">
+                        Are you sure you want to delete this folder?
+                      </p>
+                      <div className="flex justify-center gap-2">
+                        <button
+                          className="px-3 py-1 text-sm bg-gray-200 rounded"
+                          onClick={handleCancelDelete}
+                        >
+                          Cancel
+                        </button>
+                        <button
+                          className="px-3 py-1 text-sm bg-red-500 text-white rounded"
+                          onClick={handleConfirmDelete}
+                        >
+                          Delete
+                        </button>
+                      </div>
+                    </div>
+                  }
+                  requiresContentPadding
+                  sideOffset={6}
+                />
+              )}
+            {isEditing && (
+              <div className="flex -my-1 z-[9999]">
+                <button onClick={handleEdit} className="p-1">
+                  <FiCheck size={14} />
+                </button>
+                <button onClick={() => setIsEditing(false)} className="p-1">
+                  <FiX size={14} />
+                </button>
+              </div>
+            )}
+          </div>
+          {isOpen && (
+            <div className="overflow-visible mr-3 ml-1 mt-1">{children}</div>
           )}
         </div>
-        {isOpen && (
-          <div className="overflow-visible mr-3 ml-1 mt-1">{children}</div>
-        )}
       </div>
     );
   }

web/src/app/chat/input-prompts/InputPrompts.tsx

@@ -268,7 +268,7 @@ export default function InputPrompts() {
           <Title>Prompt Shortcuts</Title>
           <Text>
             Manage and customize prompt shortcuts for your assistants. Use your
-            prompt shortcuts by starting a new message “/” in chat
+            prompt shortcuts by starting a new message “/” in chat.
           </Text>
         </div>
       </div>

web/src/app/chat/input/ChatInputBar.tsx

@@ -328,6 +328,7 @@ export function ChatInputBar({
       <div className="flex  justify-center mx-auto">
         <div
           className="
+            max-w-full
             w-[800px]
             relative
             desktop:px-4
@@ -505,7 +506,10 @@ export function ChatInputBar({
               style={{ scrollbarWidth: "thin" }}
               role="textarea"
               aria-multiline
-              placeholder={`Message ${selectedAssistant.name} assistant...`}
+              placeholder={`Message ${truncateString(
+                selectedAssistant.name,
+                70
+              )} assistant...`}
               value={message}
               onKeyDown={(event) => {
                 if (
@@ -649,92 +653,101 @@ export function ChatInputBar({
               </div>
             )}
 
-            <div className="flex items-center space-x-1 mr-12 px-4 pb-2">
-              <ChatInputOption
-                flexPriority="stiff"
-                name="File"
-                Icon={FiPlusCircle}
-                onClick={() => {
-                  const input = document.createElement("input");
-                  input.type = "file";
-                  input.multiple = true;
-                  input.onchange = (event: any) => {
-                    const files = Array.from(
-                      event?.target?.files || []
-                    ) as File[];
-                    if (files.length > 0) {
-                      handleFileUpload(files);
-                    }
-                  };
-                  input.click();
-                }}
-                tooltipContent={"Upload files"}
-              />
-
-              <LLMPopover
-                llmProviders={llmProviders}
-                llmOverrideManager={llmOverrideManager}
-                requiresImageGeneration={false}
-                currentAssistant={selectedAssistant}
-              />
-
-              {retrievalEnabled && (
-                <FilterPopup
-                  availableSources={availableSources}
-                  availableDocumentSets={availableDocumentSets}
-                  availableTags={availableTags}
-                  filterManager={filterManager}
-                  trigger={
-                    <ChatInputOption
-                      flexPriority="stiff"
-                      name="Filters"
-                      Icon={FiFilter}
-                      tooltipContent="Filter your search"
-                    />
-                  }
+            <div className="flex justify-between items-center overflow-hidden px-4 mb-2">
+              <div className="flex gap-x-1">
+                <ChatInputOption
+                  flexPriority="stiff"
+                  name="File"
+                  Icon={FiPlusCircle}
+                  onClick={() => {
+                    const input = document.createElement("input");
+                    input.type = "file";
+                    input.multiple = true;
+                    input.onchange = (event: any) => {
+                      const files = Array.from(
+                        event?.target?.files || []
+                      ) as File[];
+                      if (files.length > 0) {
+                        handleFileUpload(files);
+                      }
+                    };
+                    input.click();
+                  }}
+                  tooltipContent={"Upload files"}
                 />
-              )}
-            </div>
 
-            <div className="absolute bottom-2.5 mobile:right-4 desktop:right-10">
-              {chatState == "streaming" ||
-              chatState == "toolBuilding" ||
-              chatState == "loading" ? (
+                <LLMPopover
+                  llmProviders={llmProviders}
+                  llmOverrideManager={llmOverrideManager}
+                  requiresImageGeneration={false}
+                  currentAssistant={selectedAssistant}
+                />
+
+                {retrievalEnabled && (
+                  <FilterPopup
+                    availableSources={availableSources}
+                    availableDocumentSets={availableDocumentSets}
+                    availableTags={availableTags}
+                    filterManager={filterManager}
+                    trigger={
+                      <ChatInputOption
+                        flexPriority="stiff"
+                        name="Filters"
+                        Icon={FiFilter}
+                        tooltipContent="Filter your search"
+                      />
+                    }
+                  />
+                )}
+              </div>
+              <div className="flex my-auto">
                 <button
                   className={`cursor-pointer ${
-                    chatState != "streaming"
-                      ? "bg-background-400"
-                      : "bg-background-800"
-                  }  h-[28px] w-[28px] rounded-full`}
-                  onClick={stopGenerating}
-                  disabled={chatState != "streaming"}
-                >
-                  <StopGeneratingIcon
-                    size={10}
-                    className={`text-emphasis m-auto text-white flex-none
-                      }`}
-                  />
-                </button>
-              ) : (
-                <button
-                  className="cursor-pointer"
+                    chatState == "streaming" ||
+                    chatState == "toolBuilding" ||
+                    chatState == "loading"
+                      ? chatState != "streaming"
+                        ? "bg-background-400"
+                        : "bg-background-800"
+                      : ""
+                  } h-[28px] w-[28px] rounded-full`}
                   onClick={() => {
-                    if (message) {
+                    if (
+                      chatState == "streaming" ||
+                      chatState == "toolBuilding" ||
+                      chatState == "loading"
+                    ) {
+                      stopGenerating();
+                    } else if (message) {
                       onSubmit();
                     }
                   }}
-                  disabled={chatState != "input"}
+                  disabled={
+                    (chatState == "streaming" ||
+                      chatState == "toolBuilding" ||
+                      chatState == "loading") &&
+                    chatState != "streaming"
+                  }
                 >
-                  <SendIcon
-                    size={26}
-                    className={`text-emphasis text-white p-1 rounded-full  ${
-                      chatState == "input" && message
-                        ? "bg-submit-background"
-                        : "bg-disabled-submit-background"
-                    } `}
-                  />
+                  {chatState == "streaming" ||
+                  chatState == "toolBuilding" ||
+                  chatState == "loading" ? (
+                    <StopGeneratingIcon
+                      size={10}
+                      className="text-emphasis m-auto text-white flex-none"
+                    />
+                  ) : (
+                    <SendIcon
+                      size={26}
+                      className={`text-emphasis text-white p-1 my-auto rounded-full ${
+                        chatState == "input" && message
+                          ? "bg-submit-background"
+                          : "bg-disabled-submit-background"
+                      }`}
+                    />
+                  )}
                 </button>
-              )}
+              </div>
             </div>
           </div>
         </div>

web/src/app/chat/input/ChatInputOption.tsx

@@ -15,6 +15,7 @@ interface ChatInputOptionProps {
   tooltipContent?: React.ReactNode;
   flexPriority?: "shrink" | "stiff" | "second";
   toggle?: boolean;
+  minimize?: boolean;
 }
 
 export const ChatInputOption: React.FC<ChatInputOptionProps> = ({
@@ -26,28 +27,10 @@ export const ChatInputOption: React.FC<ChatInputOptionProps> = ({
   tooltipContent,
   toggle,
   onClick,
+  minimize,
 }) => {
-  const [isDropupVisible, setDropupVisible] = useState(false);
-  const [isTooltipVisible, setIsTooltipVisible] = useState(false);
   const componentRef = useRef<HTMLButtonElement>(null);
 
-  useEffect(() => {
-    const handleClickOutside = (event: MouseEvent) => {
-      if (
-        componentRef.current &&
-        !componentRef.current.contains(event.target as Node)
-      ) {
-        setIsTooltipVisible(false);
-        setDropupVisible(false);
-      }
-    };
-
-    document.addEventListener("mousedown", handleClickOutside);
-    return () => {
-      document.removeEventListener("mousedown", handleClickOutside);
-    };
-  }, []);
-
   return (
     <TooltipProvider>
       <Tooltip>
@@ -86,7 +69,7 @@ export const ChatInputOption: React.FC<ChatInputOptionProps> = ({
               size={size}
               className="h-4 w-4 my-auto text-[#4a4a4a] group-hover:text-text flex-none"
             />
-            <div className="flex items-center">
+            <div className={`flex items-center ${minimize && "mobile:hidden"}`}>
               {name && (
                 <span className="text-sm text-[#4a4a4a] group-hover:text-text break-all line-clamp-1">
                   {name}

web/src/app/chat/input/LLMPopover.tsx

@@ -1,4 +1,4 @@
-import React from "react";
+import React, { useState } from "react";
 import {
   Popover,
   PopoverContent,
@@ -32,6 +32,7 @@ export default function LLMPopover({
   requiresImageGeneration,
   currentAssistant,
 }: LLMPopoverProps) {
+  const [isOpen, setIsOpen] = useState(false);
   const { llmOverride, updateLLMOverride, globalDefault } = llmOverrideManager;
   const currentLlm = llmOverride.modelName || globalDefault.modelName;
 
@@ -81,10 +82,11 @@ export default function LLMPopover({
     : null;
 
   return (
-    <Popover>
+    <Popover open={isOpen} onOpenChange={setIsOpen}>
       <PopoverTrigger asChild>
         <button className="focus:outline-none">
           <ChatInputOption
+            minimize
             toggle
             flexPriority="stiff"
             name={getDisplayNameForModel(
@@ -119,7 +121,10 @@ export default function LLMPopover({
                       ? "bg-gray-100 text-text"
                       : "text-text-darker"
                   }`}
-                  onClick={() => updateLLMOverride(destructureValue(value))}
+                  onClick={() => {
+                    updateLLMOverride(destructureValue(value));
+                    setIsOpen(false);
+                  }}
                 >
                   {icon({ size: 16, className: "flex-none my-auto " })}
                   <span className="line-clamp-1 ">
@@ -132,14 +137,7 @@ export default function LLMPopover({
                           (assistant)
                         </span>
                       );
-                    } else if (globalDefault.modelName === name) {
-                      return (
-                        <span className="flex-none ml-auto text-xs">
-                          (user default)
-                        </span>
-                      );
                     }
-                    return null;
                   })()}
                 </button>
               );

web/src/app/chat/input/SimplifiedChatInputBar.tsx

@@ -94,7 +94,7 @@ export function SimplifiedChatInputBar({
               rounded-lg
               relative
               text-text-chatbar
-              bg-background-chatbar
+              bg-white
               [&:has(textarea:focus)]::ring-1
               [&:has(textarea:focus)]::ring-black
             "
@@ -146,7 +146,7 @@ export function SimplifiedChatInputBar({
                 resize-none
                 rounded-lg
                 border-0
-                bg-background-chatbar
+                bg-white
                 placeholder:text-text-chatbar-subtle
                 ${
                   textAreaRef.current &&

web/src/app/chat/lib.tsx

@@ -363,8 +363,8 @@ export function groupSessionsByDateRange(chatSessions: ChatSession[]) {
   const groups: Record<string, ChatSession[]> = {
     Today: [],
     "Previous 7 Days": [],
-    "Previous 30 Days": [],
-    "Over 30 days ago": [],
+    "Previous 30 days": [],
+    "Over 30 days": [],
   };
 
   chatSessions.forEach((chatSession) => {
@@ -378,9 +378,9 @@ export function groupSessionsByDateRange(chatSessions: ChatSession[]) {
     } else if (diffDays <= 7) {
       groups["Previous 7 Days"].push(chatSession);
     } else if (diffDays <= 30) {
-      groups["Previous 30 Days"].push(chatSession);
+      groups["Previous 30 days"].push(chatSession);
     } else {
-      groups["Over 30 days ago"].push(chatSession);
+      groups["Over 30 days"].push(chatSession);
     }
   });
 
@@ -424,9 +424,10 @@ export function processRawChatHistory(
       message: messageInfo.message,
       type: messageInfo.message_type as "user" | "assistant",
       files: messageInfo.files,
-      alternateAssistantID: messageInfo.alternate_assistant_id
-        ? Number(messageInfo.alternate_assistant_id)
-        : null,
+      alternateAssistantID:
+        messageInfo.alternate_assistant_id !== null
+          ? Number(messageInfo.alternate_assistant_id)
+          : null,
       // only include these fields if this is an assistant message so that
       // this is identical to what is computed at streaming time
       ...(messageInfo.message_type === "assistant"

web/src/app/chat/message/Messages.tsx

@@ -162,7 +162,6 @@ function FileDisplay({
 export const AIMessage = ({
   regenerate,
   overriddenModel,
-  selectedMessageForDocDisplay,
   continueGenerating,
   shared,
   isActive,
@@ -170,7 +169,6 @@ export const AIMessage = ({
   alternativeAssistant,
   docs,
   messageId,
-  documentSelectionToggled,
   content,
   files,
   selectedDocuments,
@@ -180,7 +178,6 @@ export const AIMessage = ({
   isComplete,
   hasDocs,
   handleFeedback,
-  handleShowRetrieved,
   handleSearchQueryEdit,
   handleForceSearch,
   retrievalDisabled,
@@ -192,7 +189,6 @@ export const AIMessage = ({
   toggledDocumentSidebar,
 }: {
   index?: number;
-  selectedMessageForDocDisplay?: number | null;
   shared?: boolean;
   isActive?: boolean;
   continueGenerating?: () => void;
@@ -205,7 +201,6 @@ export const AIMessage = ({
   currentPersona: Persona;
   messageId: number | null;
   content: string | JSX.Element;
-  documentSelectionToggled?: boolean;
   files?: FileDescriptor[];
   query?: string;
   citedDocuments?: [string, OnyxDocument][] | null;
@@ -214,7 +209,6 @@ export const AIMessage = ({
   toggledDocumentSidebar?: boolean;
   hasDocs?: boolean;
   handleFeedback?: (feedbackType: FeedbackType) => void;
-  handleShowRetrieved?: (messageNumber: number | null) => void;
   handleSearchQueryEdit?: (query: string) => void;
   handleForceSearch?: () => void;
   retrievalDisabled?: boolean;
@@ -602,7 +596,7 @@ export const AIMessage = ({
                         className={`
                         flex md:flex-row gap-x-0.5 mt-1
                         transition-transform duration-300 ease-in-out
-                        transform opacity-100 translate-y-0"
+                        transform opacity-100 "
                   `}
                       >
                         <TooltipGroup>
@@ -692,10 +686,6 @@ export const AIMessage = ({
                             settings?.isMobile) &&
                           "!opacity-100"
                         }
-                        translate-y-2 ${
-                          (isHovering || settings?.isMobile) && "!translate-y-0"
-                        }
-                        transition-transform duration-300 ease-in-out 
                         flex md:flex-row gap-x-0.5 bg-background-125/40 -mx-1.5 p-1.5 rounded-lg
                         `}
                       >

web/src/app/chat/modal/UserSettingsModal.tsx

@@ -183,7 +183,6 @@ export function UserSettingsModal({
               checked={user?.preferences?.shortcut_enabled}
               onCheckedChange={(checked) => {
                 updateUserShortcuts(checked);
-                refreshUser();
               }}
             />
             <Label className="text-sm">Enable Prompt Shortcuts</Label>
@@ -205,6 +204,7 @@ export function UserSettingsModal({
             Scroll to see all options
           </div>
           <LLMSelector
+            userSettings
             llmProviders={llmProviders}
             currentLlm={
               defaultModelDestructured
@@ -215,7 +215,6 @@ export function UserSettingsModal({
                   )
                 : null
             }
-            userDefault={null}
             requiresImageGeneration={false}
             onSelect={(selected) => {
               if (selected === null) {

web/src/app/chat/modal/configuration/LlmTab.tsx

@@ -1,124 +0,0 @@
-import { useChatContext } from "@/components/context/ChatContext";
-import { LlmOverrideManager } from "@/lib/hooks";
-import React, { forwardRef, useCallback, useState } from "react";
-import { debounce } from "lodash";
-import Text from "@/components/ui/text";
-import { Persona } from "@/app/admin/assistants/interfaces";
-import { destructureValue } from "@/lib/llm/utils";
-import { updateModelOverrideForChatSession } from "../../lib";
-import { GearIcon } from "@/components/icons/icons";
-import { LlmList } from "@/components/llm/LLMList";
-import { checkPersonaRequiresImageGeneration } from "@/app/admin/assistants/lib";
-
-interface LlmTabProps {
-  llmOverrideManager: LlmOverrideManager;
-  currentLlm: string;
-  openModelSettings: () => void;
-  chatSessionId?: string;
-  close: () => void;
-  currentAssistant: Persona;
-}
-
-export const LlmTab = forwardRef<HTMLDivElement, LlmTabProps>(
-  (
-    {
-      llmOverrideManager,
-      chatSessionId,
-      currentLlm,
-      close,
-      openModelSettings,
-      currentAssistant,
-    },
-    ref
-  ) => {
-    const requiresImageGeneration =
-      checkPersonaRequiresImageGeneration(currentAssistant);
-
-    const { llmProviders } = useChatContext();
-    const { updateLLMOverride, temperature, updateTemperature } =
-      llmOverrideManager;
-    const [isTemperatureExpanded, setIsTemperatureExpanded] = useState(false);
-
-    return (
-      <div className="w-full">
-        <div className="flex w-full justify-between content-center mb-2 gap-x-2">
-          <label className="block text-sm font-medium">Choose Model</label>
-          <button
-            onClick={() => {
-              close();
-              openModelSettings();
-            }}
-          >
-            <GearIcon />
-          </button>
-        </div>
-        <LlmList
-          requiresImageGeneration={requiresImageGeneration}
-          llmProviders={llmProviders}
-          currentLlm={currentLlm}
-          onSelect={(value: string | null) => {
-            if (value == null) {
-              return;
-            }
-            updateLLMOverride(destructureValue(value));
-            if (chatSessionId) {
-              updateModelOverrideForChatSession(chatSessionId, value as string);
-            }
-            close();
-          }}
-        />
-
-        <div className="mt-4">
-          <button
-            className="flex items-center text-sm font-medium transition-colors duration-200"
-            onClick={() => setIsTemperatureExpanded(!isTemperatureExpanded)}
-          >
-            <span className="mr-2 text-xs text-primary">
-              {isTemperatureExpanded ? "▼" : "►"}
-            </span>
-            <span>Temperature</span>
-          </button>
-
-          {isTemperatureExpanded && (
-            <>
-              <Text className="mt-2 mb-8">
-                Adjust the temperature of the LLM. Higher temperatures will make
-                the LLM generate more creative and diverse responses, while
-                lower temperature will make the LLM generate more conservative
-                and focused responses.
-              </Text>
-
-              <div className="relative w-full">
-                <input
-                  type="range"
-                  onChange={(e) =>
-                    updateTemperature(parseFloat(e.target.value))
-                  }
-                  className="w-full p-2 border border-border rounded-md"
-                  min="0"
-                  max="2"
-                  step="0.01"
-                  value={temperature || 0}
-                />
-                <div
-                  className="absolute text-sm"
-                  style={{
-                    left: `${(temperature || 0) * 50}%`,
-                    transform: `translateX(-${Math.min(
-                      Math.max((temperature || 0) * 50, 10),
-                      90
-                    )}%)`,
-                    top: "-1.5rem",
-                  }}
-                >
-                  {temperature}
-                </div>
-              </div>
-            </>
-          )}
-        </div>
-      </div>
-    );
-  }
-);
-LlmTab.displayName = "LlmTab";

web/src/app/chat/nrf/NRFPage.tsx

@@ -1,5 +1,5 @@
 "use client";
-import React, { useState, useEffect, useRef } from "react";
+import React, { useState, useEffect, useRef, useContext } from "react";
 import { useUser } from "@/components/user/UserProvider";
 import { usePopup } from "@/components/admin/connectors/Popup";
 import {
@@ -31,11 +31,12 @@ import { useNRFPreferences } from "@/components/context/NRFPreferencesContext";
 import { SettingsPanel } from "../../components/nrf/SettingsPanel";
 import { ShortcutsDisplay } from "../../components/nrf/ShortcutsDisplay";
 import LoginPage from "../../auth/login/LoginPage";
-import { AuthType, NEXT_PUBLIC_WEB_DOMAIN } from "@/lib/constants";
+import { AuthType } from "@/lib/constants";
 import { sendSetDefaultNewTabMessage } from "@/lib/extension/utils";
 import { ReadonlyRequestCookies } from "next/dist/server/web/spec-extension/adapters/request-cookies";
 import { CHROME_MESSAGE } from "@/lib/extension/constants";
 import { ApiKeyModal } from "@/components/llm/ApiKeyModal";
+import { SettingsContext } from "@/components/settings/SettingsProvider";
 
 export default function NRFPage({
   requestCookies,
@@ -56,6 +57,7 @@ export default function NRFPage({
   const { isNight } = useNightTime();
   const { user } = useUser();
   const { ccPairs, documentSets, tags, llmProviders } = useChatContext();
+  const settings = useContext(SettingsContext);
 
   const { popup, setPopup } = usePopup();
 
@@ -196,7 +198,7 @@ export default function NRFPage({
     }
 
     const newHref =
-      `${NEXT_PUBLIC_WEB_DOMAIN}/chat?send-on-load=true&user-prompt=` +
+      `${settings?.webDomain}/chat?send-on-load=true&user-prompt=` +
       encodeURIComponent(userMessage) +
       filterString;
 

web/src/app/chat/sessionSidebar/ChatSessionDisplay.tsx

@@ -26,9 +26,7 @@ import { SettingsContext } from "@/components/settings/SettingsProvider";
 import { DragHandle } from "@/components/table/DragHandle";
 import { WarningCircle } from "@phosphor-icons/react";
 import { CustomTooltip } from "@/components/tooltip/CustomTooltip";
-import SlideOverModal from "@/components/ui/SlideOverModal";
 import { useChatContext } from "@/components/context/ChatContext";
-import { Button } from "@/components/ui/button";
 
 export function ChatSessionDisplay({
   chatSession,
@@ -44,8 +42,6 @@ export function ChatSessionDisplay({
   chatSession: ChatSession;
   isSelected: boolean;
   search?: boolean;
-  // needed when the parent is trying to apply some background effect
-  // if not set, the gradient will still be applied and cause weirdness
   skipGradient?: boolean;
   closeSidebar?: () => void;
   showShareModal?: (chatSession: ChatSession) => void;
@@ -55,11 +51,7 @@ export function ChatSessionDisplay({
 }) {
   const router = useRouter();
   const [isHovered, setIsHovered] = useState(false);
-
-  const [isHovering, setIsHovering] = useState(false);
   const [isRenamingChat, setIsRenamingChat] = useState(false);
-  const [isMoreOptionsDropdownOpen, setIsMoreOptionsDropdownOpen] =
-    useState(false);
   const [isShareModalVisible, setIsShareModalVisible] = useState(false);
   const [chatName, setChatName] = useState(chatSession.name);
   const settings = useContext(SettingsContext);
@@ -69,8 +61,9 @@ export function ChatSessionDisplay({
   const inputRef = useRef<HTMLInputElement>(null);
   const renamingRef = useRef<HTMLDivElement>(null);
 
-  const { refreshChatSessions, reorderFolders, refreshFolders } =
-    useChatContext();
+  const { refreshChatSessions, refreshFolders } = useChatContext();
+
+  const isMobile = settings?.isMobile;
   const handlePopoverOpenChange = useCallback(
     (open: boolean) => {
       setPopoverOpen(open);
@@ -105,7 +98,7 @@ export function ChatSessionDisplay({
       setIsDeleteModalOpen(false);
       setPopoverOpen(false);
     },
-    [chatSession, showDeleteModal]
+    [chatSession, showDeleteModal, refreshChatSessions, refreshFolders]
   );
 
   const onRename = useCallback(
@@ -151,6 +144,34 @@ export function ChatSessionDisplay({
     settings?.settings
   );
 
+  const handleDragStart = (event: React.DragEvent<HTMLAnchorElement>) => {
+    event.dataTransfer.setData(CHAT_SESSION_ID_KEY, chatSession.id.toString());
+    event.dataTransfer.setData(
+      FOLDER_ID_KEY,
+      chatSession.folder_id?.toString() || ""
+    );
+  };
+
+  const handleTouchStart = (event: React.TouchEvent<HTMLDivElement>) => {
+    // Prevent default touch behavior
+    event.preventDefault();
+
+    // Create a custom event to mimic drag start
+    const customEvent = new Event("dragstart", { bubbles: true });
+    (customEvent as any).dataTransfer = new DataTransfer();
+    (customEvent as any).dataTransfer.setData(
+      CHAT_SESSION_ID_KEY,
+      chatSession.id.toString()
+    );
+    (customEvent as any).dataTransfer.setData(
+      FOLDER_ID_KEY,
+      chatSession.folder_id?.toString() || ""
+    );
+
+    // Dispatch the custom event
+    event.currentTarget.dispatchEvent(customEvent);
+  };
+
   return (
     <>
       {isShareModalVisible && (
@@ -167,8 +188,6 @@ export function ChatSessionDisplay({
             setIsHovered(true);
           }}
           onMouseLeave={() => {
-            setIsMoreOptionsDropdownOpen(false);
-            setIsHovering(false);
             setIsHovered(false);
           }}
           className="flex group items-center w-full relative"
@@ -184,25 +203,19 @@ export function ChatSessionDisplay({
               : `/chat?chatId=${chatSession.id}`
           }
           scroll={false}
-          draggable="true"
-          onDragStart={(event) => {
-            event.dataTransfer.setData(
-              CHAT_SESSION_ID_KEY,
-              chatSession.id.toString()
-            );
-            event.dataTransfer.setData(
-              FOLDER_ID_KEY,
-              chatSession.folder_id?.toString() || ""
-            );
-          }}
+          draggable={!isMobile}
+          onDragStart={!isMobile ? handleDragStart : undefined}
         >
-          <DragHandle
-            size={16}
-            className={`w-3 ml-[4px] mr-[2px] invisible flex-none ${
-              foldersExisting ? "group-hover:visible" : "invisible"
-            }`}
-          />
+          <div
+            className={`${
+              isMobile ? "visible" : "invisible group-hover:visible"
+            } flex-none`}
+            onTouchStart={isMobile ? handleTouchStart : undefined}
+          >
+            <DragHandle size={16} className="w-3 ml-[4px] mr-[2px]" />
+          </div>
           <BasicSelectable
+            padding="extra"
             isHovered={isHovered}
             isDragging={isDragging}
             fullWidth
@@ -254,7 +267,7 @@ export function ChatSessionDisplay({
                     </div>
                   </div>
                 ) : (
-                  <p className="break-all overflow-hidden whitespace-nowrap w-full mr-3 relative">
+                  <p className="break-all font-normal overflow-hidden whitespace-nowrap w-full mr-3 relative">
                     {chatName || `Unnamed Chat`}
                     <span
                       className={`absolute right-0 top-0 h-full w-8 bg-gradient-to-r from-transparent 
@@ -295,9 +308,7 @@ export function ChatSessionDisplay({
                         <div
                           onClick={(e) => {
                             e.preventDefault();
-                            setIsMoreOptionsDropdownOpen(
-                              !isMoreOptionsDropdownOpen
-                            );
+                            setPopoverOpen(!popoverOpen);
                           }}
                           className="-my-1"
                         >

web/src/app/chat/sessionSidebar/HistorySidebar.tsx

@@ -1,6 +1,5 @@
 "use client";
 
-import { FiEdit, FiFolderPlus, FiMoreHorizontal, FiPlus } from "react-icons/fi";
 import React, {
   ForwardedRef,
   forwardRef,
@@ -16,14 +15,7 @@ import { Folder } from "../folders/interfaces";
 import { usePopup } from "@/components/admin/connectors/Popup";
 import { SettingsContext } from "@/components/settings/SettingsProvider";
 
-import {
-  AssistantsIconSkeleton,
-  DocumentIcon2,
-  NewChatIcon,
-  OnyxIcon,
-  PinnedIcon,
-  PlusIcon,
-} from "@/components/icons/icons";
+import { DocumentIcon2, NewChatIcon } from "@/components/icons/icons";
 import { PagesTab } from "./PagesTab";
 import { pageType } from "./types";
 import LogoWithText from "@/components/header/LogoWithText";
@@ -32,7 +24,7 @@ import { DragEndEvent } from "@dnd-kit/core";
 import { useAssistants } from "@/components/context/AssistantsContext";
 import { AssistantIcon } from "@/components/assistants/AssistantIcon";
 import { buildChatUrl } from "../lib";
-import { toggleAssistantPinnedStatus } from "@/lib/assistants/pinnedAssistants";
+import { reorderPinnedAssistants } from "@/lib/assistants/updateAssistantPreferences";
 import { useUser } from "@/components/user/UserProvider";
 import { DragHandle } from "@/components/table/DragHandle";
 import {
@@ -51,7 +43,6 @@ import {
 } from "@dnd-kit/sortable";
 import { useSortable } from "@dnd-kit/sortable";
 import { CSS } from "@dnd-kit/utilities";
-import { reorderPinnedAssistants } from "@/lib/assistants/pinnedAssistants";
 import { CircleX } from "lucide-react";
 
 interface HistorySidebarProps {
@@ -59,18 +50,14 @@ interface HistorySidebarProps {
   existingChats?: ChatSession[];
   currentChatSession?: ChatSession | null | undefined;
   folders?: Folder[];
-  openedFolders?: { [key: number]: boolean };
   toggleSidebar?: () => void;
   toggled?: boolean;
   removeToggle?: () => void;
   reset?: () => void;
   showShareModal?: (chatSession: ChatSession) => void;
   showDeleteModal?: (chatSession: ChatSession) => void;
-  stopGenerating?: () => void;
   explicitlyUntoggle: () => void;
   showDeleteAllModal?: () => void;
-  backgroundToggled?: boolean;
-  assistants: Persona[];
   currentAssistantId?: number | null;
   setShowAssistantsModal: (show: boolean) => void;
 }
@@ -129,7 +116,9 @@ const SortableAssistant: React.FC<SortableAssistantProps> = ({
         } relative flex items-center gap-x-2 py-1 px-2 rounded-md`}
       >
         <AssistantIcon assistant={assistant} size={16} className="flex-none" />
-        <p className="text-base text-black">{assistant.name}</p>
+        <p className="text-base text-left w-fit line-clamp-1 text-ellipsis text-black">
+          {assistant.name}
+        </p>
         <button
           onClick={(e) => {
             e.stopPropagation();
@@ -153,32 +142,23 @@ export const HistorySidebar = forwardRef<HTMLDivElement, HistorySidebarProps>(
       page,
       existingChats,
       currentChatSession,
-      assistants,
       folders,
-      openedFolders,
       explicitlyUntoggle,
       toggleSidebar,
       removeToggle,
-      stopGenerating = () => null,
       showShareModal,
       showDeleteModal,
       showDeleteAllModal,
-      backgroundToggled,
       currentAssistantId,
     },
     ref: ForwardedRef<HTMLDivElement>
   ) => {
     const searchParams = useSearchParams();
     const router = useRouter();
-    const { refreshUser, user } = useUser();
+    const { user, toggleAssistantPinnedStatus } = useUser();
     const { refreshAssistants, pinnedAssistants, setPinnedAssistants } =
       useAssistants();
 
-    const { popup, setPopup } = usePopup();
-
-    // For determining intial focus state
-    const [newFolderId, setNewFolderId] = useState<number | null>(null);
-
     const currentChatId = currentChatSession?.id;
 
     const sensors = useSensors(
@@ -235,7 +215,6 @@ export const HistorySidebar = forwardRef<HTMLDivElement, HistorySidebarProps>(
 
     return (
       <>
-        {popup}
         <div
           ref={ref}
           className={`
@@ -316,7 +295,7 @@ export const HistorySidebar = forwardRef<HTMLDivElement, HistorySidebarProps>(
             </div>
           )}
 
-          <div>
+          <div className="h-full relative overflow-y-auto">
             <div className="flex px-4 font-normal text-sm gap-x-2 leading-normal text-[#6c6c6c]/80 items-center font-normal leading-normal">
               Assistants
             </div>
@@ -349,7 +328,6 @@ export const HistorySidebar = forwardRef<HTMLDivElement, HistorySidebarProps>(
                           assistant.id,
                           false
                         );
-                        await refreshUser();
                         await refreshAssistants();
                       }}
                     />
@@ -365,19 +343,17 @@ export const HistorySidebar = forwardRef<HTMLDivElement, HistorySidebarProps>(
                 Explore Assistants
               </button>
             </div>
-          </div>
 
-          <PagesTab
-            setNewFolderId={setNewFolderId}
-            newFolderId={newFolderId}
-            showDeleteModal={showDeleteModal}
-            showShareModal={showShareModal}
-            closeSidebar={removeToggle}
-            existingChats={existingChats}
-            currentChatId={currentChatId}
-            folders={folders}
-            showDeleteAllModal={showDeleteAllModal}
-          />
+            <PagesTab
+              showDeleteModal={showDeleteModal}
+              showShareModal={showShareModal}
+              closeSidebar={removeToggle}
+              existingChats={existingChats}
+              currentChatId={currentChatId}
+              folders={folders}
+              showDeleteAllModal={showDeleteAllModal}
+            />
+          </div>
         </div>
       </>
     );