stashing

.github/workflows/pr-integration-tests.yml

@@ -94,19 +94,16 @@ jobs:
           cd deployment/docker_compose
           ENABLE_PAID_ENTERPRISE_EDITION_FEATURES=true \
           MULTI_TENANT=true \
-          AUTH_TYPE=cloud \
+          AUTH_TYPE=basic \
           REQUIRE_EMAIL_VERIFICATION=false \
           DISABLE_TELEMETRY=true \
           IMAGE_TAG=test \
-          DEV_MODE=true \
-          docker compose -f docker-compose.multitenant-dev.yml -p danswer-stack up -d
+          docker compose -f docker-compose.dev.yml -p danswer-stack up -d
         id: start_docker_multi_tenant
 
       # In practice, `cloud` Auth type would require OAUTH credentials to be set.
       - name: Run Multi-Tenant Integration Tests
         run: |
-          echo "Waiting for 3 minutes to ensure API server is ready..."
-          sleep 180
           echo "Running integration tests..."
           docker run --rm --network danswer-stack_default \
             --name test-runner \
@@ -122,10 +119,6 @@ jobs:
             -e TEST_WEB_HOSTNAME=test-runner \
             -e AUTH_TYPE=cloud \
             -e MULTI_TENANT=true \
-            -e REQUIRE_EMAIL_VERIFICATION=false \
-            -e DISABLE_TELEMETRY=true \
-            -e IMAGE_TAG=test \
-            -e DEV_MODE=true \
             onyxdotapp/onyx-integration:test \
             /app/tests/integration/multitenant_tests
         continue-on-error: true
@@ -133,17 +126,17 @@ jobs:
 
       - name: Check multi-tenant test results
         run: |
-          if [ ${{ steps.run_multitenant_tests.outcome }} == 'failure' ]; then
-            echo "Multi-tenant integration tests failed. Exiting with error."
+          if [ ${{ steps.run_tests.outcome }} == 'failure' ]; then
+            echo "Integration tests failed. Exiting with error."
             exit 1
           else
-            echo "All multi-tenant integration tests passed successfully."
+            echo "All integration tests passed successfully."
           fi
 
       - name: Stop multi-tenant Docker containers
         run: |
           cd deployment/docker_compose
-          docker compose -f docker-compose.multitenant-dev.yml -p danswer-stack down -v
+          docker compose -f docker-compose.dev.yml -p danswer-stack down -v
 
       - name: Start Docker containers
         run: |
@@ -223,30 +216,27 @@ jobs:
             echo "All integration tests passed successfully."
           fi
 
-      # ------------------------------------------------------------
-      # Always gather logs BEFORE "down":
-      - name: Dump API server logs
-        if: always()
+      # save before stopping the containers so the logs can be captured
+      - name: Save Docker logs
+        if: success() || failure()
         run: |
           cd deployment/docker_compose
-          docker compose -f docker-compose.dev.yml -p danswer-stack logs --no-color api_server > $GITHUB_WORKSPACE/api_server.log || true
-
-      - name: Dump all-container logs (optional)
-        if: always()
-        run: |
-          cd deployment/docker_compose
-          docker compose -f docker-compose.dev.yml -p danswer-stack logs --no-color > $GITHUB_WORKSPACE/docker-compose.log || true
-
-      - name: Upload logs
-        if: always()
-        uses: actions/upload-artifact@v4
-        with:
-          name: docker-all-logs
-          path: ${{ github.workspace }}/docker-compose.log
-      # ------------------------------------------------------------
+          docker compose -f docker-compose.dev.yml -p danswer-stack logs > docker-compose.log
+          mv docker-compose.log ${{ github.workspace }}/docker-compose.log
+
+      - name: Stop Docker containers
+        run: |
+          cd deployment/docker_compose
+          docker compose -f docker-compose.dev.yml -p danswer-stack down -v
+
+      - name: Upload logs
+        if: success() || failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: docker-logs
+          path: ${{ github.workspace }}/docker-compose.log
 
       - name: Stop Docker containers
-        if: always()
         run: |
           cd deployment/docker_compose
           docker compose -f docker-compose.dev.yml -p danswer-stack down -v

README.md

@@ -133,4 +133,3 @@ Looking to contribute? Please check out the [Contribution Guide](CONTRIBUTING.md
 ## ⭐Star History
 
 [![Star History Chart](https://api.star-history.com/svg?repos=onyx-dot-app/onyx&type=Date)](https://star-history.com/#onyx-dot-app/onyx&Date)
-

backend/alembic/versions/4ee1287bd26a_add_multiple_slack_bot_support.py

@@ -5,6 +5,7 @@ Revises: 47e5bef3a1d7
 Create Date: 2024-11-06 13:15:53.302644
 
 """
+import logging
 from typing import cast
 from alembic import op
 import sqlalchemy as sa
@@ -19,8 +20,13 @@ down_revision = "47e5bef3a1d7"
 branch_labels: None = None
 depends_on: None = None
 
+# Configure logging
+logger = logging.getLogger("alembic.runtime.migration")
+logger.setLevel(logging.INFO)
+
 
 def upgrade() -> None:
+    logger.info(f"{revision}: create_table: slack_bot")
     # Create new slack_bot table
     op.create_table(
         "slack_bot",
@@ -57,6 +63,7 @@ def upgrade() -> None:
     )
 
     # Handle existing Slack bot tokens first
+    logger.info(f"{revision}: Checking for existing Slack bot.")
     bot_token = None
     app_token = None
     first_row_id = None
@@ -64,12 +71,15 @@ def upgrade() -> None:
     try:
         tokens = cast(dict, get_kv_store().load("slack_bot_tokens_config_key"))
     except Exception:
+        logger.warning("No existing Slack bot tokens found.")
         tokens = {}
 
     bot_token = tokens.get("bot_token")
     app_token = tokens.get("app_token")
 
     if bot_token and app_token:
+        logger.info(f"{revision}: Found bot and app tokens.")
+
         session = Session(bind=op.get_bind())
         new_slack_bot = SlackBot(
             name="Slack Bot (Migrated)",
@@ -160,9 +170,10 @@ def upgrade() -> None:
     # Clean up old tokens if they existed
     try:
         if bot_token and app_token:
+            logger.info(f"{revision}: Removing old bot and app tokens.")
             get_kv_store().delete("slack_bot_tokens_config_key")
     except Exception:
-        pass
+        logger.warning("tried to delete tokens in dynamic config but failed")
     # Rename the table
     op.rename_table(
         "slack_bot_config__standard_answer_category",
@@ -179,6 +190,8 @@ def upgrade() -> None:
     # Drop the table with CASCADE to handle dependent objects
     op.execute("DROP TABLE slack_bot_config CASCADE")
 
+    logger.info(f"{revision}: Migration complete.")
+
 
 def downgrade() -> None:
     # Recreate the old slack_bot_config table
@@ -260,7 +273,7 @@ def downgrade() -> None:
             }
             get_kv_store().store("slack_bot_tokens_config_key", tokens)
     except Exception:
-        pass
+        logger.warning("Failed to save tokens back to KV store")
 
     # Drop the new tables in reverse order
     op.drop_table("slack_channel_config")

backend/ee/onyx/external_permissions/confluence/doc_sync.py

@@ -365,9 +365,7 @@ def confluence_doc_sync(
 
     slim_docs = []
     logger.debug("Fetching all slim documents from confluence")
-    for doc_batch in confluence_connector.retrieve_all_slim_documents(
-        callback=callback
-    ):
+    for doc_batch in confluence_connector.retrieve_all_slim_documents():
         logger.debug(f"Got {len(doc_batch)} slim documents from confluence")
         if callback:
             if callback.should_stop():

backend/ee/onyx/external_permissions/gmail/doc_sync.py

@@ -15,7 +15,6 @@ logger = setup_logger()
 def _get_slim_doc_generator(
     cc_pair: ConnectorCredentialPair,
     gmail_connector: GmailConnector,
-    callback: IndexingHeartbeatInterface | None = None,
 ) -> GenerateSlimDocumentOutput:
     current_time = datetime.now(timezone.utc)
     start_time = (
@@ -25,9 +24,7 @@ def _get_slim_doc_generator(
     )
 
     return gmail_connector.retrieve_all_slim_documents(
-        start=start_time,
-        end=current_time.timestamp(),
-        callback=callback,
+        start=start_time, end=current_time.timestamp()
     )
 
 
@@ -43,9 +40,7 @@ def gmail_doc_sync(
     gmail_connector = GmailConnector(**cc_pair.connector.connector_specific_config)
     gmail_connector.load_credentials(cc_pair.credential.credential_json)
 
-    slim_doc_generator = _get_slim_doc_generator(
-        cc_pair, gmail_connector, callback=callback
-    )
+    slim_doc_generator = _get_slim_doc_generator(cc_pair, gmail_connector)
 
     document_external_access: list[DocExternalAccess] = []
     for slim_doc_batch in slim_doc_generator:

backend/ee/onyx/external_permissions/google_drive/doc_sync.py

@@ -21,7 +21,6 @@ _PERMISSION_ID_PERMISSION_MAP: dict[str, dict[str, Any]] = {}
 def _get_slim_doc_generator(
     cc_pair: ConnectorCredentialPair,
     google_drive_connector: GoogleDriveConnector,
-    callback: IndexingHeartbeatInterface | None = None,
 ) -> GenerateSlimDocumentOutput:
     current_time = datetime.now(timezone.utc)
     start_time = (
@@ -31,9 +30,7 @@ def _get_slim_doc_generator(
     )
 
     return google_drive_connector.retrieve_all_slim_documents(
-        start=start_time,
-        end=current_time.timestamp(),
-        callback=callback,
+        start=start_time, end=current_time.timestamp()
     )
 
 

backend/ee/onyx/external_permissions/slack/doc_sync.py

@@ -20,11 +20,19 @@ def _get_slack_document_ids_and_channels(
     slack_connector = SlackPollConnector(**cc_pair.connector.connector_specific_config)
     slack_connector.load_credentials(cc_pair.credential.credential_json)
 
-    slim_doc_generator = slack_connector.retrieve_all_slim_documents(callback=callback)
+    slim_doc_generator = slack_connector.retrieve_all_slim_documents()
 
     channel_doc_map: dict[str, list[str]] = {}
     for doc_metadata_batch in slim_doc_generator:
         for doc_metadata in doc_metadata_batch:
+            if callback:
+                if callback.should_stop():
+                    raise RuntimeError(
+                        "_get_slack_document_ids_and_channels: Stop signal detected"
+                    )
+
+                callback.progress("_get_slack_document_ids_and_channels", 1)
+
             if doc_metadata.perm_sync_data is None:
                 continue
             channel_id = doc_metadata.perm_sync_data["channel_id"]
@@ -32,14 +40,6 @@ def _get_slack_document_ids_and_channels(
                 channel_doc_map[channel_id] = []
             channel_doc_map[channel_id].append(doc_metadata.id)
 
-        if callback:
-            if callback.should_stop():
-                raise RuntimeError(
-                    "_get_slack_document_ids_and_channels: Stop signal detected"
-                )
-
-            callback.progress("_get_slack_document_ids_and_channels", 1)
-
     return channel_doc_map
 
 

backend/ee/onyx/server/middleware/tenant_tracking.py

@@ -64,7 +64,6 @@ async def _get_tenant_id_from_request(
 
     try:
         # Look up token data in Redis
-
         token_data = await retrieve_auth_token_data_from_redis(request)
 
         if not token_data:

backend/ee/onyx/server/tenants/provisioning.py

@@ -24,7 +24,6 @@ from ee.onyx.server.tenants.user_mapping import get_tenant_id_for_email
 from ee.onyx.server.tenants.user_mapping import user_owns_a_tenant
 from onyx.auth.users import exceptions
 from onyx.configs.app_configs import CONTROL_PLANE_API_BASE_URL
-from onyx.configs.app_configs import DEV_MODE
 from onyx.configs.constants import MilestoneRecordType
 from onyx.db.engine import get_session_with_tenant
 from onyx.db.engine import get_sqlalchemy_engine
@@ -86,8 +85,7 @@ async def create_tenant(email: str, referral_source: str | None = None) -> str:
         # Provision tenant on data plane
         await provision_tenant(tenant_id, email)
         # Notify control plane
-        if not DEV_MODE:
-            await notify_control_plane(tenant_id, email, referral_source)
+        await notify_control_plane(tenant_id, email, referral_source)
     except Exception as e:
         logger.error(f"Tenant provisioning failed: {e}")
         await rollback_tenant_provisioning(tenant_id)

backend/onyx/background/celery/apps/primary.py

@@ -84,10 +84,8 @@ def on_celeryd_init(sender: str, conf: Any = None, **kwargs: Any) -> None:
 def on_worker_init(sender: Worker, **kwargs: Any) -> None:
     logger.info("worker_init signal received.")
 
-    EXTRA_CONCURRENCY = 4  # small extra fudge factor for connection limits
-
     SqlEngine.set_app_name(POSTGRES_CELERY_WORKER_PRIMARY_APP_NAME)
-    SqlEngine.init_engine(pool_size=sender.concurrency, max_overflow=EXTRA_CONCURRENCY)  # type: ignore
+    SqlEngine.init_engine(pool_size=8, max_overflow=0)
 
     app_base.wait_for_redis(sender, **kwargs)
     app_base.wait_for_db(sender, **kwargs)

backend/onyx/background/celery/tasks/beat_schedule.py

@@ -18,153 +18,238 @@ BEAT_EXPIRES_DEFAULT = 15 * 60  # 15 minutes (in seconds)
 
 # hack to slow down task dispatch in the cloud until
 # we have a better implementation (backpressure, etc)
-CLOUD_BEAT_SCHEDULE_MULTIPLIER = 8
-
-# tasks that run in either self-hosted on cloud
-beat_task_templates: list[dict] = []
-
-beat_task_templates.extend(
-    [
-        {
-            "name": "check-for-indexing",
-            "task": OnyxCeleryTask.CHECK_FOR_INDEXING,
-            "schedule": timedelta(seconds=15),
-            "options": {
-                "priority": OnyxCeleryPriority.MEDIUM,
-                "expires": BEAT_EXPIRES_DEFAULT,
-            },
-        },
-        {
-            "name": "check-for-connector-deletion",
-            "task": OnyxCeleryTask.CHECK_FOR_CONNECTOR_DELETION,
-            "schedule": timedelta(seconds=20),
-            "options": {
-                "priority": OnyxCeleryPriority.MEDIUM,
-                "expires": BEAT_EXPIRES_DEFAULT,
-            },
-        },
-        {
-            "name": "check-for-vespa-sync",
-            "task": OnyxCeleryTask.CHECK_FOR_VESPA_SYNC_TASK,
-            "schedule": timedelta(seconds=20),
-            "options": {
-                "priority": OnyxCeleryPriority.MEDIUM,
-                "expires": BEAT_EXPIRES_DEFAULT,
-            },
-        },
-        {
-            "name": "check-for-pruning",
-            "task": OnyxCeleryTask.CHECK_FOR_PRUNING,
-            "schedule": timedelta(hours=1),
-            "options": {
-                "priority": OnyxCeleryPriority.MEDIUM,
-                "expires": BEAT_EXPIRES_DEFAULT,
-            },
-        },
-        {
-            "name": "monitor-vespa-sync",
-            "task": OnyxCeleryTask.MONITOR_VESPA_SYNC,
-            "schedule": timedelta(seconds=5),
-            "options": {
-                "priority": OnyxCeleryPriority.MEDIUM,
-                "expires": BEAT_EXPIRES_DEFAULT,
-            },
-        },
-        {
-            "name": "check-for-doc-permissions-sync",
-            "task": OnyxCeleryTask.CHECK_FOR_DOC_PERMISSIONS_SYNC,
-            "schedule": timedelta(seconds=30),
-            "options": {
-                "priority": OnyxCeleryPriority.MEDIUM,
-                "expires": BEAT_EXPIRES_DEFAULT,
-            },
-        },
-        {
-            "name": "check-for-external-group-sync",
-            "task": OnyxCeleryTask.CHECK_FOR_EXTERNAL_GROUP_SYNC,
-            "schedule": timedelta(seconds=20),
-            "options": {
-                "priority": OnyxCeleryPriority.MEDIUM,
-                "expires": BEAT_EXPIRES_DEFAULT,
-            },
-        },
-        {
-            "name": "monitor-background-processes",
-            "task": OnyxCeleryTask.MONITOR_BACKGROUND_PROCESSES,
-            "schedule": timedelta(minutes=5),
-            "options": {
-                "priority": OnyxCeleryPriority.LOW,
-                "expires": BEAT_EXPIRES_DEFAULT,
-                "queue": OnyxCeleryQueues.MONITORING,
-            },
-        },
-    ]
-)
-
-# Only add the LLM model update task if the API URL is configured
-if LLM_MODEL_UPDATE_API_URL:
-    beat_task_templates.append(
-        {
-            "name": "check-for-llm-model-update",
-            "task": OnyxCeleryTask.CHECK_FOR_LLM_MODEL_UPDATE,
-            "schedule": timedelta(hours=1),  # Check every hour
-            "options": {
-                "priority": OnyxCeleryPriority.LOW,
-                "expires": BEAT_EXPIRES_DEFAULT,
-            },
-        }
-    )
-
-
-def make_cloud_generator_task(task: dict[str, Any]) -> dict[str, Any]:
-    cloud_task: dict[str, Any] = {}
-
-    # constant options for cloud beat task generators
-    task_schedule: timedelta = task["schedule"]
-    cloud_task["schedule"] = task_schedule * CLOUD_BEAT_SCHEDULE_MULTIPLIER
-    cloud_task["options"] = {}
-    cloud_task["options"]["priority"] = OnyxCeleryPriority.HIGHEST
-    cloud_task["options"]["expires"] = BEAT_EXPIRES_DEFAULT
-
-    # settings dependent on the original task
-    cloud_task["name"] = f"{ONYX_CLOUD_CELERY_TASK_PREFIX}_{task['name']}"
-    cloud_task["task"] = OnyxCeleryTask.CLOUD_BEAT_TASK_GENERATOR
-    cloud_task["kwargs"] = {}
-    cloud_task["kwargs"]["task_name"] = task["task"]
-
-    optional_fields = ["queue", "priority", "expires"]
-    for field in optional_fields:
-        if field in task["options"]:
-            cloud_task["kwargs"][field] = task["options"][field]
-
-    return cloud_task
-
+CLOUD_BEAT_SCHEDULE_MULTIPLIER = 4
 
 # tasks that only run in the cloud
 # the name attribute must start with ONYX_CLOUD_CELERY_TASK_PREFIX = "cloud" to be filtered
 # by the DynamicTenantScheduler
-cloud_tasks_to_schedule: list[dict] = [
+cloud_tasks_to_schedule = [
     # cloud specific tasks
     {
         "name": f"{ONYX_CLOUD_CELERY_TASK_PREFIX}_check-alembic",
         "task": OnyxCeleryTask.CLOUD_CHECK_ALEMBIC,
-        "schedule": timedelta(hours=1),
+        "schedule": timedelta(hours=1 * CLOUD_BEAT_SCHEDULE_MULTIPLIER),
         "options": {
             "queue": OnyxCeleryQueues.MONITORING,
             "priority": OnyxCeleryPriority.HIGH,
             "expires": BEAT_EXPIRES_DEFAULT,
         },
     },
+    # remaining tasks are cloud generators for per tenant tasks
+    {
+        "name": f"{ONYX_CLOUD_CELERY_TASK_PREFIX}_check-for-indexing",
+        "task": OnyxCeleryTask.CLOUD_BEAT_TASK_GENERATOR,
+        "schedule": timedelta(seconds=15 * CLOUD_BEAT_SCHEDULE_MULTIPLIER),
+        "options": {
+            "priority": OnyxCeleryPriority.HIGHEST,
+            "expires": BEAT_EXPIRES_DEFAULT,
+        },
+        "kwargs": {
+            "task_name": OnyxCeleryTask.CHECK_FOR_INDEXING,
+        },
+    },
+    {
+        "name": f"{ONYX_CLOUD_CELERY_TASK_PREFIX}_check-for-connector-deletion",
+        "task": OnyxCeleryTask.CLOUD_BEAT_TASK_GENERATOR,
+        "schedule": timedelta(seconds=20 * CLOUD_BEAT_SCHEDULE_MULTIPLIER),
+        "options": {
+            "priority": OnyxCeleryPriority.HIGHEST,
+            "expires": BEAT_EXPIRES_DEFAULT,
+        },
+        "kwargs": {
+            "task_name": OnyxCeleryTask.CHECK_FOR_CONNECTOR_DELETION,
+        },
+    },
+    {
+        "name": f"{ONYX_CLOUD_CELERY_TASK_PREFIX}_check-for-vespa-sync",
+        "task": OnyxCeleryTask.CLOUD_BEAT_TASK_GENERATOR,
+        "schedule": timedelta(seconds=20 * CLOUD_BEAT_SCHEDULE_MULTIPLIER),
+        "options": {
+            "priority": OnyxCeleryPriority.HIGHEST,
+            "expires": BEAT_EXPIRES_DEFAULT,
+        },
+        "kwargs": {
+            "task_name": OnyxCeleryTask.CHECK_FOR_VESPA_SYNC_TASK,
+        },
+    },
+    {
+        "name": f"{ONYX_CLOUD_CELERY_TASK_PREFIX}_check-for-prune",
+        "task": OnyxCeleryTask.CLOUD_BEAT_TASK_GENERATOR,
+        "schedule": timedelta(seconds=15 * CLOUD_BEAT_SCHEDULE_MULTIPLIER),
+        "options": {
+            "priority": OnyxCeleryPriority.HIGHEST,
+            "expires": BEAT_EXPIRES_DEFAULT,
+        },
+        "kwargs": {
+            "task_name": OnyxCeleryTask.CHECK_FOR_PRUNING,
+        },
+    },
+    {
+        "name": f"{ONYX_CLOUD_CELERY_TASK_PREFIX}_monitor-vespa-sync",
+        "task": OnyxCeleryTask.CLOUD_BEAT_TASK_GENERATOR,
+        "schedule": timedelta(seconds=15 * CLOUD_BEAT_SCHEDULE_MULTIPLIER),
+        "options": {
+            "priority": OnyxCeleryPriority.HIGHEST,
+            "expires": BEAT_EXPIRES_DEFAULT,
+        },
+        "kwargs": {
+            "task_name": OnyxCeleryTask.MONITOR_VESPA_SYNC,
+        },
+    },
+    {
+        "name": f"{ONYX_CLOUD_CELERY_TASK_PREFIX}_check-for-doc-permissions-sync",
+        "task": OnyxCeleryTask.CLOUD_BEAT_TASK_GENERATOR,
+        "schedule": timedelta(seconds=30 * CLOUD_BEAT_SCHEDULE_MULTIPLIER),
+        "options": {
+            "priority": OnyxCeleryPriority.HIGHEST,
+            "expires": BEAT_EXPIRES_DEFAULT,
+        },
+        "kwargs": {
+            "task_name": OnyxCeleryTask.CHECK_FOR_DOC_PERMISSIONS_SYNC,
+        },
+    },
+    {
+        "name": f"{ONYX_CLOUD_CELERY_TASK_PREFIX}_check-for-external-group-sync",
+        "task": OnyxCeleryTask.CLOUD_BEAT_TASK_GENERATOR,
+        "schedule": timedelta(seconds=20 * CLOUD_BEAT_SCHEDULE_MULTIPLIER),
+        "options": {
+            "priority": OnyxCeleryPriority.HIGHEST,
+            "expires": BEAT_EXPIRES_DEFAULT,
+        },
+        "kwargs": {
+            "task_name": OnyxCeleryTask.CHECK_FOR_EXTERNAL_GROUP_SYNC,
+        },
+    },
+    {
+        "name": f"{ONYX_CLOUD_CELERY_TASK_PREFIX}_monitor-background-processes",
+        "task": OnyxCeleryTask.CLOUD_BEAT_TASK_GENERATOR,
+        "schedule": timedelta(minutes=5 * CLOUD_BEAT_SCHEDULE_MULTIPLIER),
+        "options": {
+            "priority": OnyxCeleryPriority.HIGHEST,
+            "expires": BEAT_EXPIRES_DEFAULT,
+        },
+        "kwargs": {
+            "task_name": OnyxCeleryTask.MONITOR_BACKGROUND_PROCESSES,
+            "queue": OnyxCeleryQueues.MONITORING,
+            "priority": OnyxCeleryPriority.LOW,
+        },
+    },
 ]
 
-# generate our cloud and self-hosted beat tasks from the templates
-for beat_task_template in beat_task_templates:
-    cloud_task = make_cloud_generator_task(beat_task_template)
-    cloud_tasks_to_schedule.append(cloud_task)
+if LLM_MODEL_UPDATE_API_URL:
+    cloud_tasks_to_schedule.append(
+        {
+            "name": f"{ONYX_CLOUD_CELERY_TASK_PREFIX}_check-for-llm-model-update",
+            "task": OnyxCeleryTask.CLOUD_BEAT_TASK_GENERATOR,
+            "schedule": timedelta(
+                hours=1 * CLOUD_BEAT_SCHEDULE_MULTIPLIER
+            ),  # Check every hour
+            "options": {
+                "priority": OnyxCeleryPriority.HIGHEST,
+                "expires": BEAT_EXPIRES_DEFAULT,
+            },
+            "kwargs": {
+                "task_name": OnyxCeleryTask.CHECK_FOR_LLM_MODEL_UPDATE,
+                "priority": OnyxCeleryPriority.LOW,
+            },
+        }
+    )
 
+# tasks that run in either self-hosted on cloud
 tasks_to_schedule: list[dict] = []
+
 if not MULTI_TENANT:
-    tasks_to_schedule = beat_task_templates
+    tasks_to_schedule.extend(
+        [
+            {
+                "name": "check-for-indexing",
+                "task": OnyxCeleryTask.CHECK_FOR_INDEXING,
+                "schedule": timedelta(seconds=15),
+                "options": {
+                    "priority": OnyxCeleryPriority.MEDIUM,
+                    "expires": BEAT_EXPIRES_DEFAULT,
+                },
+            },
+            {
+                "name": "check-for-connector-deletion",
+                "task": OnyxCeleryTask.CHECK_FOR_CONNECTOR_DELETION,
+                "schedule": timedelta(seconds=20),
+                "options": {
+                    "priority": OnyxCeleryPriority.MEDIUM,
+                    "expires": BEAT_EXPIRES_DEFAULT,
+                },
+            },
+            {
+                "name": "check-for-vespa-sync",
+                "task": OnyxCeleryTask.CHECK_FOR_VESPA_SYNC_TASK,
+                "schedule": timedelta(seconds=20),
+                "options": {
+                    "priority": OnyxCeleryPriority.MEDIUM,
+                    "expires": BEAT_EXPIRES_DEFAULT,
+                },
+            },
+            {
+                "name": "check-for-pruning",
+                "task": OnyxCeleryTask.CHECK_FOR_PRUNING,
+                "schedule": timedelta(hours=1),
+                "options": {
+                    "priority": OnyxCeleryPriority.MEDIUM,
+                    "expires": BEAT_EXPIRES_DEFAULT,
+                },
+            },
+            {
+                "name": "monitor-vespa-sync",
+                "task": OnyxCeleryTask.MONITOR_VESPA_SYNC,
+                "schedule": timedelta(seconds=5),
+                "options": {
+                    "priority": OnyxCeleryPriority.MEDIUM,
+                    "expires": BEAT_EXPIRES_DEFAULT,
+                },
+            },
+            {
+                "name": "check-for-doc-permissions-sync",
+                "task": OnyxCeleryTask.CHECK_FOR_DOC_PERMISSIONS_SYNC,
+                "schedule": timedelta(seconds=30),
+                "options": {
+                    "priority": OnyxCeleryPriority.MEDIUM,
+                    "expires": BEAT_EXPIRES_DEFAULT,
+                },
+            },
+            {
+                "name": "check-for-external-group-sync",
+                "task": OnyxCeleryTask.CHECK_FOR_EXTERNAL_GROUP_SYNC,
+                "schedule": timedelta(seconds=20),
+                "options": {
+                    "priority": OnyxCeleryPriority.MEDIUM,
+                    "expires": BEAT_EXPIRES_DEFAULT,
+                },
+            },
+            {
+                "name": "monitor-background-processes",
+                "task": OnyxCeleryTask.MONITOR_BACKGROUND_PROCESSES,
+                "schedule": timedelta(minutes=15),
+                "options": {
+                    "priority": OnyxCeleryPriority.LOW,
+                    "expires": BEAT_EXPIRES_DEFAULT,
+                    "queue": OnyxCeleryQueues.MONITORING,
+                },
+            },
+        ]
+    )
+
+    # Only add the LLM model update task if the API URL is configured
+    if LLM_MODEL_UPDATE_API_URL:
+        tasks_to_schedule.append(
+            {
+                "name": "check-for-llm-model-update",
+                "task": OnyxCeleryTask.CHECK_FOR_LLM_MODEL_UPDATE,
+                "schedule": timedelta(hours=1),  # Check every hour
+                "options": {
+                    "priority": OnyxCeleryPriority.LOW,
+                    "expires": BEAT_EXPIRES_DEFAULT,
+                },
+            }
+        )
 
 
 def get_cloud_tasks_to_schedule() -> list[dict[str, Any]]:

backend/onyx/background/celery/tasks/connector_deletion/tasks.py

@@ -186,7 +186,7 @@ def try_generate_document_cc_pair_cleanup_tasks(
                 sync_type=SyncType.CONNECTOR_DELETION,
             )
         except Exception:
-            task_logger.exception("insert_sync_record exceptioned.")
+            pass
 
     except TaskDependencyError:
         redis_connector.delete.set_fence(None)

backend/onyx/background/celery/tasks/doc_permission_syncing/tasks.py

@@ -228,15 +228,12 @@ def try_creating_permissions_sync_task(
 
         # create before setting fence to avoid race condition where the monitoring
         # task updates the sync record before it is created
-        try:
-            with get_session_with_tenant(tenant_id) as db_session:
-                insert_sync_record(
-                    db_session=db_session,
-                    entity_id=cc_pair_id,
-                    sync_type=SyncType.EXTERNAL_PERMISSIONS,
-                )
-        except Exception:
-            task_logger.exception("insert_sync_record exceptioned.")
+        with get_session_with_tenant(tenant_id) as db_session:
+            insert_sync_record(
+                db_session=db_session,
+                entity_id=cc_pair_id,
+                sync_type=SyncType.EXTERNAL_PERMISSIONS,
+            )
 
         # set a basic fence to start
         redis_connector.permissions.set_active()
@@ -260,10 +257,11 @@ def try_creating_permissions_sync_task(
         )
 
         # fill in the celery task id
+        redis_connector.permissions.set_active()
         payload.celery_task_id = result.id
         redis_connector.permissions.set_fence(payload)
 
-        payload_id = payload.id
+        payload_id = payload.celery_task_id
     except Exception:
         task_logger.exception(f"Unexpected exception: cc_pair={cc_pair_id}")
         return None
@@ -292,8 +290,6 @@ def connector_permission_sync_generator_task(
     This task assumes that the task has already been properly fenced
     """
 
-    payload_id: str | None = None
-
     LoggerContextVars.reset()
 
     doc_permission_sync_ctx_dict = doc_permission_sync_ctx.get()
@@ -336,12 +332,9 @@ def connector_permission_sync_generator_task(
             sleep(1)
             continue
 
-        payload_id = payload.id
-
         logger.info(
             f"connector_permission_sync_generator_task - Fence found, continuing...: "
-            f"fence={redis_connector.permissions.fence_key} "
-            f"payload_id={payload.id}"
+            f"fence={redis_connector.permissions.fence_key}"
         )
         break
 
@@ -420,9 +413,7 @@ def connector_permission_sync_generator_task(
             redis_connector.permissions.generator_complete = tasks_generated
 
     except Exception as e:
-        task_logger.exception(
-            f"Permission sync exceptioned: cc_pair={cc_pair_id} payload_id={payload_id}"
-        )
+        task_logger.exception(f"Failed to run permission sync: cc_pair={cc_pair_id}")
 
         redis_connector.permissions.generator_clear()
         redis_connector.permissions.taskset_clear()
@@ -432,10 +423,6 @@ def connector_permission_sync_generator_task(
         if lock.owned():
             lock.release()
 
-    task_logger.info(
-        f"Permission sync finished: cc_pair={cc_pair_id} payload_id={payload.id}"
-    )
-
 
 @shared_task(
     name=OnyxCeleryTask.UPDATE_EXTERNAL_DOCUMENT_PERMISSIONS_TASK,
@@ -672,7 +659,7 @@ def validate_permission_sync_fence(
         f"tasks_scanned={tasks_scanned} tasks_not_in_celery={tasks_not_in_celery}"
     )
 
-    # we're active if there are still tasks to run and those tasks all exist in celery
+    # we're only active if tasks_scanned > 0 and tasks_not_in_celery == 0
     if tasks_scanned > 0 and tasks_not_in_celery == 0:
         redis_connector.permissions.set_active()
         return
@@ -693,8 +680,7 @@ def validate_permission_sync_fence(
         "validate_permission_sync_fence - "
         "Resetting fence because no associated celery tasks were found: "
         f"cc_pair={cc_pair_id} "
-        f"fence={fence_key} "
-        f"payload_id={payload.id}"
+        f"fence={fence_key}"
     )
 
     redis_connector.permissions.reset()

backend/onyx/background/celery/tasks/external_group_syncing/tasks.py

@@ -2,17 +2,15 @@ import time
 from datetime import datetime
 from datetime import timedelta
 from datetime import timezone
-from typing import Any
-from typing import cast
 from uuid import uuid4
 
 from celery import Celery
 from celery import shared_task
 from celery import Task
 from celery.exceptions import SoftTimeLimitExceeded
-from pydantic import ValidationError
 from redis import Redis
 from redis.lock import Lock as RedisLock
+from sqlalchemy.orm import Session
 
 from ee.onyx.db.connector_credential_pair import get_all_auto_sync_cc_pairs
 from ee.onyx.db.connector_credential_pair import get_cc_pairs_by_source
@@ -34,9 +32,7 @@ from onyx.configs.constants import DANSWER_REDIS_FUNCTION_LOCK_PREFIX
 from onyx.configs.constants import OnyxCeleryPriority
 from onyx.configs.constants import OnyxCeleryQueues
 from onyx.configs.constants import OnyxCeleryTask
-from onyx.configs.constants import OnyxRedisConstants
 from onyx.configs.constants import OnyxRedisLocks
-from onyx.configs.constants import OnyxRedisSignals
 from onyx.db.connector import mark_cc_pair_as_external_group_synced
 from onyx.db.connector_credential_pair import get_connector_credential_pair_from_id
 from onyx.db.engine import get_session_with_tenant
@@ -53,8 +49,7 @@ from onyx.redis.redis_connector_ext_group_sync import (
     RedisConnectorExternalGroupSyncPayload,
 )
 from onyx.redis.redis_pool import get_redis_client
-from onyx.redis.redis_pool import get_redis_replica_client
-from onyx.server.utils import make_short_id
+from onyx.redis.redis_pool import SCAN_ITER_COUNT_DEFAULT
 from onyx.utils.logger import setup_logger
 
 logger = setup_logger()
@@ -112,11 +107,11 @@ def _is_external_group_sync_due(cc_pair: ConnectorCredentialPair) -> bool:
     bind=True,
 )
 def check_for_external_group_sync(self: Task, *, tenant_id: str | None) -> bool | None:
+    r = get_redis_client(tenant_id=tenant_id)
+
     # we need to use celery's redis client to access its redis data
     # (which lives on a different db number)
-    r = get_redis_client(tenant_id=tenant_id)
-    r_replica = get_redis_replica_client(tenant_id=tenant_id)
-    r_celery: Redis = self.app.broker_connection().channel().client  # type: ignore
+    # r_celery: Redis = self.app.broker_connection().channel().client  # type: ignore
 
     lock_beat: RedisLock = r.lock(
         OnyxRedisLocks.CHECK_CONNECTOR_EXTERNAL_GROUP_SYNC_BEAT_LOCK,
@@ -154,32 +149,30 @@ def check_for_external_group_sync(self: Task, *, tenant_id: str | None) -> bool
 
         lock_beat.reacquire()
         for cc_pair_id in cc_pair_ids_to_sync:
-            payload_id = try_creating_external_group_sync_task(
+            tasks_created = try_creating_external_group_sync_task(
                 self.app, cc_pair_id, r, tenant_id
             )
-            if not payload_id:
+            if not tasks_created:
                 continue
 
-            task_logger.info(
-                f"External group sync queued: cc_pair={cc_pair_id} id={payload_id}"
-            )
+            task_logger.info(f"External group sync queued: cc_pair={cc_pair_id}")
 
         # we want to run this less frequently than the overall task
-        lock_beat.reacquire()
-        if not r.exists(OnyxRedisSignals.BLOCK_VALIDATE_EXTERNAL_GROUP_SYNC_FENCES):
-            # clear fences that don't have associated celery tasks in progress
-            # tasks can be in the queue in redis, in reserved tasks (prefetched by the worker),
-            # or be currently executing
-            try:
-                validate_external_group_sync_fences(
-                    tenant_id, self.app, r, r_replica, r_celery, lock_beat
-                )
-            except Exception:
-                task_logger.exception(
-                    "Exception while validating external group sync fences"
-                )
+        # lock_beat.reacquire()
+        # if not r.exists(OnyxRedisSignals.VALIDATE_EXTERNAL_GROUP_SYNC_FENCES):
+        #     # clear any indexing fences that don't have associated celery tasks in progress
+        #     # tasks can be in the queue in redis, in reserved tasks (prefetched by the worker),
+        #     # or be currently executing
+        #     try:
+        #         validate_external_group_sync_fences(
+        #             tenant_id, self.app, r, r_celery, lock_beat
+        #         )
+        #     except Exception:
+        #         task_logger.exception(
+        #             "Exception while validating external group sync fences"
+        #         )
 
-            r.set(OnyxRedisSignals.BLOCK_VALIDATE_EXTERNAL_GROUP_SYNC_FENCES, 1, ex=300)
+        #     r.set(OnyxRedisSignals.VALIDATE_EXTERNAL_GROUP_SYNC_FENCES, 1, ex=60)
     except SoftTimeLimitExceeded:
         task_logger.info(
             "Soft time limit exceeded, task is being terminated gracefully."
@@ -198,11 +191,9 @@ def try_creating_external_group_sync_task(
     cc_pair_id: int,
     r: Redis,
     tenant_id: str | None,
-) -> str | None:
+) -> int | None:
     """Returns an int if syncing is needed. The int represents the number of sync tasks generated.
     Returns None if no syncing is required."""
-    payload_id: str | None = None
-
     redis_connector = RedisConnector(tenant_id, cc_pair_id)
 
     LOCK_TIMEOUT = 30
@@ -224,28 +215,11 @@ def try_creating_external_group_sync_task(
         redis_connector.external_group_sync.generator_clear()
         redis_connector.external_group_sync.taskset_clear()
 
-        # create before setting fence to avoid race condition where the monitoring
-        # task updates the sync record before it is created
-        try:
-            with get_session_with_tenant(tenant_id) as db_session:
-                insert_sync_record(
-                    db_session=db_session,
-                    entity_id=cc_pair_id,
-                    sync_type=SyncType.EXTERNAL_GROUP,
-                )
-        except Exception:
-            task_logger.exception("insert_sync_record exceptioned.")
-
-        # Signal active before creating fence
-        redis_connector.external_group_sync.set_active()
-
         payload = RedisConnectorExternalGroupSyncPayload(
-            id=make_short_id(),
             submitted=datetime.now(timezone.utc),
             started=None,
             celery_task_id=None,
         )
-        redis_connector.external_group_sync.set_fence(payload)
 
         custom_task_id = f"{redis_connector.external_group_sync.taskset_key}_{uuid4()}"
 
@@ -260,10 +234,17 @@ def try_creating_external_group_sync_task(
             priority=OnyxCeleryPriority.HIGH,
         )
 
+        # create before setting fence to avoid race condition where the monitoring
+        # task updates the sync record before it is created
+        with get_session_with_tenant(tenant_id) as db_session:
+            insert_sync_record(
+                db_session=db_session,
+                entity_id=cc_pair_id,
+                sync_type=SyncType.EXTERNAL_GROUP,
+            )
+
         payload.celery_task_id = result.id
         redis_connector.external_group_sync.set_fence(payload)
-
-        payload_id = payload.id
     except Exception:
         task_logger.exception(
             f"Unexpected exception while trying to create external group sync task: cc_pair={cc_pair_id}"
@@ -273,7 +254,7 @@ def try_creating_external_group_sync_task(
         if lock.owned():
             lock.release()
 
-    return payload_id
+    return 1
 
 
 @shared_task(
@@ -331,8 +312,7 @@ def connector_external_group_sync_generator_task(
 
         logger.info(
             f"connector_external_group_sync_generator_task - Fence found, continuing...: "
-            f"fence={redis_connector.external_group_sync.fence_key} "
-            f"payload_id={payload.id}"
+            f"fence={redis_connector.external_group_sync.fence_key}"
         )
         break
 
@@ -401,7 +381,7 @@ def connector_external_group_sync_generator_task(
             )
     except Exception as e:
         task_logger.exception(
-            f"External group sync exceptioned: cc_pair={cc_pair_id} payload_id={payload.id}"
+            f"Failed to run external group sync: cc_pair={cc_pair_id}"
         )
 
         with get_session_with_tenant(tenant_id) as db_session:
@@ -421,41 +401,32 @@ def connector_external_group_sync_generator_task(
         if lock.owned():
             lock.release()
 
-    task_logger.info(
-        f"External group sync finished: cc_pair={cc_pair_id} payload_id={payload.id}"
-    )
-
 
 def validate_external_group_sync_fences(
     tenant_id: str | None,
     celery_app: Celery,
     r: Redis,
-    r_replica: Redis,
     r_celery: Redis,
     lock_beat: RedisLock,
 ) -> None:
-    reserved_tasks = celery_get_unacked_task_ids(
+    reserved_sync_tasks = celery_get_unacked_task_ids(
         OnyxCeleryQueues.CONNECTOR_EXTERNAL_GROUP_SYNC, r_celery
     )
 
-    # validate all existing external group sync tasks
-    lock_beat.reacquire()
-    keys = cast(set[Any], r_replica.smembers(OnyxRedisConstants.ACTIVE_FENCES))
-    for key in keys:
-        key_bytes = cast(bytes, key)
-        key_str = key_bytes.decode("utf-8")
-        if not key_str.startswith(RedisConnectorExternalGroupSync.FENCE_PREFIX):
-            continue
-
-        validate_external_group_sync_fence(
-            tenant_id,
-            key_bytes,
-            reserved_tasks,
-            r_celery,
-        )
-
+    # validate all existing indexing jobs
+    for key_bytes in r.scan_iter(
+        RedisConnectorExternalGroupSync.FENCE_PREFIX + "*",
+        count=SCAN_ITER_COUNT_DEFAULT,
+    ):
         lock_beat.reacquire()
-
+        with get_session_with_tenant(tenant_id) as db_session:
+            validate_external_group_sync_fence(
+                tenant_id,
+                key_bytes,
+                reserved_sync_tasks,
+                r_celery,
+                db_session,
+            )
     return
 
 
@@ -464,6 +435,7 @@ def validate_external_group_sync_fence(
     key_bytes: bytes,
     reserved_tasks: set[str],
     r_celery: Redis,
+    db_session: Session,
 ) -> None:
     """Checks for the error condition where an indexing fence is set but the associated celery tasks don't exist.
     This can happen if the indexing worker hard crashes or is terminated.
@@ -506,26 +478,26 @@ def validate_external_group_sync_fence(
     if not redis_connector.external_group_sync.fenced:
         return
 
-    try:
-        payload = redis_connector.external_group_sync.payload
-    except ValidationError:
-        task_logger.exception(
-            "validate_external_group_sync_fence - "
-            "Resetting fence because fence schema is out of date: "
-            f"cc_pair={cc_pair_id} "
-            f"fence={fence_key}"
-        )
-
-        redis_connector.external_group_sync.reset()
-        return
-
+    payload = redis_connector.external_group_sync.payload
     if not payload:
         return
 
-    if not payload.celery_task_id:
+    # OK, there's actually something for us to validate
+
+    if payload.celery_task_id is None:
+        # the fence is just barely set up.
+        # if redis_connector_index.active():
+        #     return
+
+        # it would be odd to get here as there isn't that much that can go wrong during
+        # initial fence setup, but it's still worth making sure we can recover
+        logger.info(
+            "validate_external_group_sync_fence - "
+            f"Resetting fence in basic state without any activity: fence={fence_key}"
+        )
+        redis_connector.external_group_sync.reset()
         return
 
-    # OK, there's actually something for us to validate
     found = celery_find_task(
         payload.celery_task_id, OnyxCeleryQueues.CONNECTOR_EXTERNAL_GROUP_SYNC, r_celery
     )
@@ -555,8 +527,7 @@ def validate_external_group_sync_fence(
         "validate_external_group_sync_fence - "
         "Resetting fence because no associated celery tasks were found: "
         f"cc_pair={cc_pair_id} "
-        f"fence={fence_key} "
-        f"payload_id={payload.id}"
+        f"fence={fence_key}"
     )
 
     redis_connector.external_group_sync.reset()

backend/onyx/background/celery/tasks/indexing/tasks.py

@@ -423,8 +423,8 @@ def connector_indexing_task(
         # define a callback class
         callback = IndexingCallback(
             os.getppid(),
-            redis_connector,
-            redis_connector_index,
+            redis_connector.stop.fence_key,
+            redis_connector_index.generator_progress_key,
             lock,
             r,
         )

backend/onyx/background/celery/tasks/indexing/utils.py

@@ -99,16 +99,16 @@ class IndexingCallback(IndexingHeartbeatInterface):
     def __init__(
         self,
         parent_pid: int,
-        redis_connector: RedisConnector,
-        redis_connector_index: RedisConnectorIndex,
+        stop_key: str,
+        generator_progress_key: str,
         redis_lock: RedisLock,
         redis_client: Redis,
     ):
         super().__init__()
         self.parent_pid = parent_pid
-        self.redis_connector: RedisConnector = redis_connector
-        self.redis_connector_index: RedisConnectorIndex = redis_connector_index
         self.redis_lock: RedisLock = redis_lock
+        self.stop_key: str = stop_key
+        self.generator_progress_key: str = generator_progress_key
         self.redis_client = redis_client
         self.started: datetime = datetime.now(timezone.utc)
         self.redis_lock.reacquire()
@@ -120,7 +120,7 @@ class IndexingCallback(IndexingHeartbeatInterface):
         self.last_parent_check = time.monotonic()
 
     def should_stop(self) -> bool:
-        if self.redis_connector.stop.fenced:
+        if self.redis_client.exists(self.stop_key):
             return True
 
         return False
@@ -143,8 +143,6 @@ class IndexingCallback(IndexingHeartbeatInterface):
         #         self.last_parent_check = now
 
         try:
-            self.redis_connector.prune.set_active()
-
             current_time = time.monotonic()
             if current_time - self.last_lock_monotonic >= (
                 CELERY_GENERIC_BEAT_LOCK_TIMEOUT / 4
@@ -167,9 +165,7 @@ class IndexingCallback(IndexingHeartbeatInterface):
             redis_lock_dump(self.redis_lock, self.redis_client)
             raise
 
-        self.redis_client.incrby(
-            self.redis_connector_index.generator_progress_key, amount
-        )
+        self.redis_client.incrby(self.generator_progress_key, amount)
 
 
 def validate_indexing_fence(

backend/onyx/background/celery/tasks/pruning/tasks.py

@@ -1,39 +1,28 @@
-import time
 from datetime import datetime
 from datetime import timedelta
 from datetime import timezone
-from typing import Any
-from typing import cast
 from uuid import uuid4
 
 from celery import Celery
 from celery import shared_task
 from celery import Task
 from celery.exceptions import SoftTimeLimitExceeded
-from pydantic import ValidationError
 from redis import Redis
 from redis.lock import Lock as RedisLock
 from sqlalchemy.orm import Session
 
 from onyx.background.celery.apps.app_base import task_logger
-from onyx.background.celery.celery_redis import celery_find_task
-from onyx.background.celery.celery_redis import celery_get_queue_length
-from onyx.background.celery.celery_redis import celery_get_queued_task_ids
-from onyx.background.celery.celery_redis import celery_get_unacked_task_ids
 from onyx.background.celery.celery_utils import extract_ids_from_runnable_connector
 from onyx.background.celery.tasks.indexing.utils import IndexingCallback
 from onyx.configs.app_configs import ALLOW_SIMULTANEOUS_PRUNING
 from onyx.configs.app_configs import JOB_TIMEOUT
 from onyx.configs.constants import CELERY_GENERIC_BEAT_LOCK_TIMEOUT
 from onyx.configs.constants import CELERY_PRUNING_LOCK_TIMEOUT
-from onyx.configs.constants import CELERY_TASK_WAIT_FOR_FENCE_TIMEOUT
 from onyx.configs.constants import DANSWER_REDIS_FUNCTION_LOCK_PREFIX
 from onyx.configs.constants import OnyxCeleryPriority
 from onyx.configs.constants import OnyxCeleryQueues
 from onyx.configs.constants import OnyxCeleryTask
-from onyx.configs.constants import OnyxRedisConstants
 from onyx.configs.constants import OnyxRedisLocks
-from onyx.configs.constants import OnyxRedisSignals
 from onyx.connectors.factory import instantiate_connector
 from onyx.connectors.models import InputType
 from onyx.db.connector import mark_ccpair_as_pruned
@@ -46,15 +35,10 @@ from onyx.db.enums import ConnectorCredentialPairStatus
 from onyx.db.enums import SyncStatus
 from onyx.db.enums import SyncType
 from onyx.db.models import ConnectorCredentialPair
-from onyx.db.search_settings import get_current_search_settings
 from onyx.db.sync_record import insert_sync_record
 from onyx.db.sync_record import update_sync_record_status
 from onyx.redis.redis_connector import RedisConnector
-from onyx.redis.redis_connector_prune import RedisConnectorPrune
-from onyx.redis.redis_connector_prune import RedisConnectorPrunePayload
 from onyx.redis.redis_pool import get_redis_client
-from onyx.redis.redis_pool import get_redis_replica_client
-from onyx.server.utils import make_short_id
 from onyx.utils.logger import LoggerContextVars
 from onyx.utils.logger import pruning_ctx
 from onyx.utils.logger import setup_logger
@@ -109,8 +93,6 @@ def _is_pruning_due(cc_pair: ConnectorCredentialPair) -> bool:
 )
 def check_for_pruning(self: Task, *, tenant_id: str | None) -> bool | None:
     r = get_redis_client(tenant_id=tenant_id)
-    r_replica = get_redis_replica_client(tenant_id=tenant_id)
-    r_celery: Redis = self.app.broker_connection().channel().client  # type: ignore
 
     lock_beat: RedisLock = r.lock(
         OnyxRedisLocks.CHECK_PRUNE_BEAT_LOCK,
@@ -141,28 +123,13 @@ def check_for_pruning(self: Task, *, tenant_id: str | None) -> bool | None:
                 if not _is_pruning_due(cc_pair):
                     continue
 
-                payload_id = try_creating_prune_generator_task(
+                tasks_created = try_creating_prune_generator_task(
                     self.app, cc_pair, db_session, r, tenant_id
                 )
-                if not payload_id:
+                if not tasks_created:
                     continue
 
-                task_logger.info(
-                    f"Pruning queued: cc_pair={cc_pair.id} id={payload_id}"
-                )
-
-        # we want to run this less frequently than the overall task
-        lock_beat.reacquire()
-        if not r.exists(OnyxRedisSignals.BLOCK_VALIDATE_PRUNING_FENCES):
-            # clear any permission fences that don't have associated celery tasks in progress
-            # tasks can be in the queue in redis, in reserved tasks (prefetched by the worker),
-            # or be currently executing
-            try:
-                validate_pruning_fences(tenant_id, r, r_replica, r_celery, lock_beat)
-            except Exception:
-                task_logger.exception("Exception while validating pruning fences")
-
-            r.set(OnyxRedisSignals.BLOCK_VALIDATE_PRUNING_FENCES, 1, ex=300)
+                task_logger.info(f"Pruning queued: cc_pair={cc_pair.id}")
     except SoftTimeLimitExceeded:
         task_logger.info(
             "Soft time limit exceeded, task is being terminated gracefully."
@@ -182,7 +149,7 @@ def try_creating_prune_generator_task(
     db_session: Session,
     r: Redis,
     tenant_id: str | None,
-) -> str | None:
+) -> int | None:
     """Checks for any conditions that should block the pruning generator task from being
     created, then creates the task.
 
@@ -201,7 +168,7 @@ def try_creating_prune_generator_task(
 
     # we need to serialize starting pruning since it can be triggered either via
     # celery beat or manually (API call)
-    lock: RedisLock = r.lock(
+    lock = r.lock(
         DANSWER_REDIS_FUNCTION_LOCK_PREFIX + "try_creating_prune_generator_task",
         timeout=LOCK_TIMEOUT,
     )
@@ -233,30 +200,7 @@ def try_creating_prune_generator_task(
 
         custom_task_id = f"{redis_connector.prune.generator_task_key}_{uuid4()}"
 
-        # create before setting fence to avoid race condition where the monitoring
-        # task updates the sync record before it is created
-        try:
-            insert_sync_record(
-                db_session=db_session,
-                entity_id=cc_pair.id,
-                sync_type=SyncType.PRUNING,
-            )
-        except Exception:
-            task_logger.exception("insert_sync_record exceptioned.")
-
-        # signal active before the fence is set
-        redis_connector.prune.set_active()
-
-        # set a basic fence to start
-        payload = RedisConnectorPrunePayload(
-            id=make_short_id(),
-            submitted=datetime.now(timezone.utc),
-            started=None,
-            celery_task_id=None,
-        )
-        redis_connector.prune.set_fence(payload)
-
-        result = celery_app.send_task(
+        celery_app.send_task(
             OnyxCeleryTask.CONNECTOR_PRUNING_GENERATOR_TASK,
             kwargs=dict(
                 cc_pair_id=cc_pair.id,
@@ -269,11 +213,16 @@ def try_creating_prune_generator_task(
             priority=OnyxCeleryPriority.LOW,
         )
 
-        # fill in the celery task id
-        payload.celery_task_id = result.id
-        redis_connector.prune.set_fence(payload)
+        # create before setting fence to avoid race condition where the monitoring
+        # task updates the sync record before it is created
+        insert_sync_record(
+            db_session=db_session,
+            entity_id=cc_pair.id,
+            sync_type=SyncType.PRUNING,
+        )
 
-        payload_id = payload.id
+        # set this only after all tasks have been added
+        redis_connector.prune.set_fence(True)
     except Exception:
         task_logger.exception(f"Unexpected exception: cc_pair={cc_pair.id}")
         return None
@@ -281,7 +230,7 @@ def try_creating_prune_generator_task(
         if lock.owned():
             lock.release()
 
-    return payload_id
+    return 1
 
 
 @shared_task(
@@ -303,8 +252,6 @@ def connector_pruning_generator_task(
     and compares those IDs to locally stored documents and deletes all locally stored IDs missing
     from the most recently pulled document ID list"""
 
-    payload_id: str | None = None
-
     LoggerContextVars.reset()
 
     pruning_ctx_dict = pruning_ctx.get()
@@ -318,46 +265,6 @@ def connector_pruning_generator_task(
 
     r = get_redis_client(tenant_id=tenant_id)
 
-    # this wait is needed to avoid a race condition where
-    # the primary worker sends the task and it is immediately executed
-    # before the primary worker can finalize the fence
-    start = time.monotonic()
-    while True:
-        if time.monotonic() - start > CELERY_TASK_WAIT_FOR_FENCE_TIMEOUT:
-            raise ValueError(
-                f"connector_prune_generator_task - timed out waiting for fence to be ready: "
-                f"fence={redis_connector.prune.fence_key}"
-            )
-
-        if not redis_connector.prune.fenced:  # The fence must exist
-            raise ValueError(
-                f"connector_prune_generator_task - fence not found: "
-                f"fence={redis_connector.prune.fence_key}"
-            )
-
-        payload = redis_connector.prune.payload  # The payload must exist
-        if not payload:
-            raise ValueError(
-                "connector_prune_generator_task: payload invalid or not found"
-            )
-
-        if payload.celery_task_id is None:
-            logger.info(
-                f"connector_prune_generator_task - Waiting for fence: "
-                f"fence={redis_connector.prune.fence_key}"
-            )
-            time.sleep(1)
-            continue
-
-        payload_id = payload.id
-
-        logger.info(
-            f"connector_prune_generator_task - Fence found, continuing...: "
-            f"fence={redis_connector.prune.fence_key} "
-            f"payload_id={payload.id}"
-        )
-        break
-
     # set thread_local=False since we don't control what thread the indexing/pruning
     # might run our callback with
     lock: RedisLock = r.lock(
@@ -387,18 +294,6 @@ def connector_pruning_generator_task(
                 )
                 return
 
-            payload = redis_connector.prune.payload
-            if not payload:
-                raise ValueError(f"No fence payload found: cc_pair={cc_pair_id}")
-
-            new_payload = RedisConnectorPrunePayload(
-                id=payload.id,
-                submitted=payload.submitted,
-                started=datetime.now(timezone.utc),
-                celery_task_id=payload.celery_task_id,
-            )
-            redis_connector.prune.set_fence(new_payload)
-
             task_logger.info(
                 f"Pruning generator running connector: "
                 f"cc_pair={cc_pair_id} "
@@ -412,13 +307,10 @@ def connector_pruning_generator_task(
                 cc_pair.credential,
             )
 
-            search_settings = get_current_search_settings(db_session)
-            redis_connector_index = redis_connector.new_index(search_settings.id)
-
             callback = IndexingCallback(
                 0,
-                redis_connector,
-                redis_connector_index,
+                redis_connector.stop.fence_key,
+                redis_connector.prune.generator_progress_key,
                 lock,
                 r,
             )
@@ -465,9 +357,7 @@ def connector_pruning_generator_task(
             redis_connector.prune.generator_complete = tasks_generated
     except Exception as e:
         task_logger.exception(
-            f"Pruning exceptioned: cc_pair={cc_pair_id} "
-            f"connector={connector_id} "
-            f"payload_id={payload_id}"
+            f"Failed to run pruning: cc_pair={cc_pair_id} connector={connector_id}"
         )
 
         redis_connector.prune.reset()
@@ -476,9 +366,7 @@ def connector_pruning_generator_task(
         if lock.owned():
             lock.release()
 
-    task_logger.info(
-        f"Pruning generator finished: cc_pair={cc_pair_id} payload_id={payload_id}"
-    )
+        task_logger.info(f"Pruning generator finished: cc_pair={cc_pair_id}")
 
 
 """Monitoring pruning utils, called in monitor_vespa_sync"""
@@ -527,184 +415,4 @@ def monitor_ccpair_pruning_taskset(
 
     redis_connector.prune.taskset_clear()
     redis_connector.prune.generator_clear()
-    redis_connector.prune.set_fence(None)
-
-
-def validate_pruning_fences(
-    tenant_id: str | None,
-    r: Redis,
-    r_replica: Redis,
-    r_celery: Redis,
-    lock_beat: RedisLock,
-) -> None:
-    # building lookup table can be expensive, so we won't bother
-    # validating until the queue is small
-    PERMISSION_SYNC_VALIDATION_MAX_QUEUE_LEN = 1024
-
-    queue_len = celery_get_queue_length(OnyxCeleryQueues.CONNECTOR_DELETION, r_celery)
-    if queue_len > PERMISSION_SYNC_VALIDATION_MAX_QUEUE_LEN:
-        return
-
-    # the queue for a single pruning generator task
-    reserved_generator_tasks = celery_get_unacked_task_ids(
-        OnyxCeleryQueues.CONNECTOR_PRUNING, r_celery
-    )
-
-    # the queue for a reasonably large set of lightweight deletion tasks
-    queued_upsert_tasks = celery_get_queued_task_ids(
-        OnyxCeleryQueues.CONNECTOR_DELETION, r_celery
-    )
-
-    # Use replica for this because the worst thing that happens
-    # is that we don't run the validation on this pass
-    keys = cast(set[Any], r_replica.smembers(OnyxRedisConstants.ACTIVE_FENCES))
-    for key in keys:
-        key_bytes = cast(bytes, key)
-        key_str = key_bytes.decode("utf-8")
-        if not key_str.startswith(RedisConnectorPrune.FENCE_PREFIX):
-            continue
-
-        validate_pruning_fence(
-            tenant_id,
-            key_bytes,
-            reserved_generator_tasks,
-            queued_upsert_tasks,
-            r,
-            r_celery,
-        )
-
-        lock_beat.reacquire()
-
-    return
-
-
-def validate_pruning_fence(
-    tenant_id: str | None,
-    key_bytes: bytes,
-    reserved_tasks: set[str],
-    queued_tasks: set[str],
-    r: Redis,
-    r_celery: Redis,
-) -> None:
-    """See validate_indexing_fence for an overall idea of validation flows.
-
-    queued_tasks: the celery queue of lightweight permission sync tasks
-    reserved_tasks: prefetched tasks for sync task generator
-    """
-    # if the fence doesn't exist, there's nothing to do
-    fence_key = key_bytes.decode("utf-8")
-    cc_pair_id_str = RedisConnector.get_id_from_fence_key(fence_key)
-    if cc_pair_id_str is None:
-        task_logger.warning(
-            f"validate_pruning_fence - could not parse id from {fence_key}"
-        )
-        return
-
-    cc_pair_id = int(cc_pair_id_str)
-    # parse out metadata and initialize the helper class with it
-    redis_connector = RedisConnector(tenant_id, int(cc_pair_id))
-
-    # check to see if the fence/payload exists
-    if not redis_connector.prune.fenced:
-        return
-
-    # in the cloud, the payload format may have changed ...
-    # it's a little sloppy, but just reset the fence for now if that happens
-    # TODO: add intentional cleanup/abort logic
-    try:
-        payload = redis_connector.prune.payload
-    except ValidationError:
-        task_logger.exception(
-            "validate_pruning_fence - "
-            "Resetting fence because fence schema is out of date: "
-            f"cc_pair={cc_pair_id} "
-            f"fence={fence_key}"
-        )
-
-        redis_connector.prune.reset()
-        return
-
-    if not payload:
-        return
-
-    if not payload.celery_task_id:
-        return
-
-    # OK, there's actually something for us to validate
-
-    # either the generator task must be in flight or its subtasks must be
-    found = celery_find_task(
-        payload.celery_task_id,
-        OnyxCeleryQueues.CONNECTOR_PRUNING,
-        r_celery,
-    )
-    if found:
-        # the celery task exists in the redis queue
-        redis_connector.prune.set_active()
-        return
-
-    if payload.celery_task_id in reserved_tasks:
-        # the celery task was prefetched and is reserved within a worker
-        redis_connector.prune.set_active()
-        return
-
-    # look up every task in the current taskset in the celery queue
-    # every entry in the taskset should have an associated entry in the celery task queue
-    # because we get the celery tasks first, the entries in our own pruning taskset
-    # should be roughly a subset of the tasks in celery
-
-    # this check isn't very exact, but should be sufficient over a period of time
-    # A single successful check over some number of attempts is sufficient.
-
-    # TODO: if the number of tasks in celery is much lower than than the taskset length
-    # we might be able to shortcut the lookup since by definition some of the tasks
-    # must not exist in celery.
-
-    tasks_scanned = 0
-    tasks_not_in_celery = 0  # a non-zero number after completing our check is bad
-
-    for member in r.sscan_iter(redis_connector.prune.taskset_key):
-        tasks_scanned += 1
-
-        member_bytes = cast(bytes, member)
-        member_str = member_bytes.decode("utf-8")
-        if member_str in queued_tasks:
-            continue
-
-        if member_str in reserved_tasks:
-            continue
-
-        tasks_not_in_celery += 1
-
-    task_logger.info(
-        "validate_pruning_fence task check: "
-        f"tasks_scanned={tasks_scanned} tasks_not_in_celery={tasks_not_in_celery}"
-    )
-
-    # we're active if there are still tasks to run and those tasks all exist in celery
-    if tasks_scanned > 0 and tasks_not_in_celery == 0:
-        redis_connector.prune.set_active()
-        return
-
-    # we may want to enable this check if using the active task list somehow isn't good enough
-    # if redis_connector_index.generator_locked():
-    #     logger.info(f"{payload.celery_task_id} is currently executing.")
-
-    # if we get here, we didn't find any direct indication that the associated celery tasks exist,
-    # but they still might be there due to gaps in our ability to check states during transitions
-    # Checking the active signal safeguards us against these transition periods
-    # (which has a duration that allows us to bridge those gaps)
-    if redis_connector.prune.active():
-        return
-
-    # celery tasks don't exist and the active signal has expired, possibly due to a crash. Clean it up.
-    task_logger.warning(
-        "validate_pruning_fence - "
-        "Resetting fence because no associated celery tasks were found: "
-        f"cc_pair={cc_pair_id} "
-        f"fence={fence_key} "
-        f"payload_id={payload.id}"
-    )
-
-    redis_connector.prune.reset()
-    return
+    redis_connector.prune.set_fence(False)

backend/onyx/background/celery/tasks/vespa/tasks.py

@@ -339,15 +339,11 @@ def try_generate_document_set_sync_tasks(
 
     # create before setting fence to avoid race condition where the monitoring
     # task updates the sync record before it is created
-    try:
-        insert_sync_record(
-            db_session=db_session,
-            entity_id=document_set_id,
-            sync_type=SyncType.DOCUMENT_SET,
-        )
-    except Exception:
-        task_logger.exception("insert_sync_record exceptioned.")
-
+    insert_sync_record(
+        db_session=db_session,
+        entity_id=document_set_id,
+        sync_type=SyncType.DOCUMENT_SET,
+    )
     # set this only after all tasks have been added
     rds.set_fence(tasks_generated)
     return tasks_generated
@@ -415,15 +411,11 @@ def try_generate_user_group_sync_tasks(
 
     # create before setting fence to avoid race condition where the monitoring
     # task updates the sync record before it is created
-    try:
-        insert_sync_record(
-            db_session=db_session,
-            entity_id=usergroup_id,
-            sync_type=SyncType.USER_GROUP,
-        )
-    except Exception:
-        task_logger.exception("insert_sync_record exceptioned.")
-
+    insert_sync_record(
+        db_session=db_session,
+        entity_id=usergroup_id,
+        sync_type=SyncType.USER_GROUP,
+    )
     # set this only after all tasks have been added
     rug.set_fence(tasks_generated)
 
@@ -912,7 +904,7 @@ def monitor_vespa_sync(self: Task, tenant_id: str | None) -> bool | None:
 
         # use a lookup table to find active fences. We still have to verify the fence
         # exists since it is an optimization and not the source of truth.
-        keys = cast(set[Any], r_replica.smembers(OnyxRedisConstants.ACTIVE_FENCES))
+        keys = cast(set[Any], r.smembers(OnyxRedisConstants.ACTIVE_FENCES))
         for key in keys:
             key_bytes = cast(bytes, key)
 

backend/onyx/chat/prompt_builder/citations_prompt.py

@@ -140,7 +140,6 @@ def build_citations_user_message(
     context_docs: list[LlmDoc] | list[InferenceChunk],
     all_doc_useful: bool,
     history_message: str = "",
-    context_type: str = "context documents",
 ) -> HumanMessage:
     multilingual_expansion = get_multilingual_expansion()
     task_prompt_with_reminder = build_task_prompt_reminders(
@@ -157,7 +156,6 @@ def build_citations_user_message(
         optional_ignore = "" if all_doc_useful else DEFAULT_IGNORE_STATEMENT
 
         user_prompt = CITATIONS_PROMPT.format(
-            context_type=context_type,
             optional_ignore_statement=optional_ignore,
             context_docs_str=context_docs_str,
             task_prompt=task_prompt_with_reminder,
@@ -167,7 +165,6 @@ def build_citations_user_message(
     else:
         # if no context docs provided, assume we're in the tool calling flow
         user_prompt = CITATIONS_PROMPT_FOR_TOOL_CALLING.format(
-            context_type=context_type,
             task_prompt=task_prompt_with_reminder,
             user_query=query,
             history_block=history_block,

backend/onyx/configs/constants.py

@@ -263,11 +263,6 @@ class PostgresAdvisoryLocks(Enum):
 
 
 class OnyxCeleryQueues:
-    # "celery" is the default queue defined by celery and also the queue
-    # we are running in the primary worker to run system tasks
-    # Tasks running in this queue should be designed specifically to run quickly
-    PRIMARY = "celery"
-
     # Light queue
     VESPA_METADATA_SYNC = "vespa_metadata_sync"
     DOC_PERMISSIONS_UPSERT = "doc_permissions_upsert"
@@ -324,7 +319,6 @@ class OnyxRedisSignals:
     BLOCK_VALIDATE_PERMISSION_SYNC_FENCES = (
         "signal:block_validate_permission_sync_fences"
     )
-    BLOCK_VALIDATE_PRUNING_FENCES = "signal:block_validate_pruning_fences"
     BLOCK_BUILD_FENCE_LOOKUP_TABLE = "signal:block_build_fence_lookup_table"
 
 

backend/onyx/connectors/confluence/connector.py

@@ -27,7 +27,6 @@ from onyx.connectors.models import ConnectorMissingCredentialError
 from onyx.connectors.models import Document
 from onyx.connectors.models import Section
 from onyx.connectors.models import SlimDocument
-from onyx.indexing.indexing_heartbeat import IndexingHeartbeatInterface
 from onyx.utils.logger import setup_logger
 
 logger = setup_logger()
@@ -320,7 +319,6 @@ class ConfluenceConnector(LoadConnector, PollConnector, SlimConnector):
         self,
         start: SecondsSinceUnixEpoch | None = None,
         end: SecondsSinceUnixEpoch | None = None,
-        callback: IndexingHeartbeatInterface | None = None,
     ) -> GenerateSlimDocumentOutput:
         doc_metadata_list: list[SlimDocument] = []
 
@@ -388,12 +386,4 @@ class ConfluenceConnector(LoadConnector, PollConnector, SlimConnector):
                 yield doc_metadata_list[:_SLIM_DOC_BATCH_SIZE]
                 doc_metadata_list = doc_metadata_list[_SLIM_DOC_BATCH_SIZE:]
 
-                if callback:
-                    if callback.should_stop():
-                        raise RuntimeError(
-                            "retrieve_all_slim_documents: Stop signal detected"
-                        )
-
-                    callback.progress("retrieve_all_slim_documents", 1)
-
         yield doc_metadata_list

backend/onyx/connectors/gmail/connector.py

@@ -30,7 +30,6 @@ from onyx.connectors.models import BasicExpertInfo
 from onyx.connectors.models import Document
 from onyx.connectors.models import Section
 from onyx.connectors.models import SlimDocument
-from onyx.indexing.indexing_heartbeat import IndexingHeartbeatInterface
 from onyx.utils.logger import setup_logger
 from onyx.utils.retry_wrapper import retry_builder
 
@@ -322,7 +321,6 @@ class GmailConnector(LoadConnector, PollConnector, SlimConnector):
         self,
         time_range_start: SecondsSinceUnixEpoch | None = None,
         time_range_end: SecondsSinceUnixEpoch | None = None,
-        callback: IndexingHeartbeatInterface | None = None,
     ) -> GenerateSlimDocumentOutput:
         query = _build_time_range_query(time_range_start, time_range_end)
         doc_batch = []
@@ -345,15 +343,6 @@ class GmailConnector(LoadConnector, PollConnector, SlimConnector):
                 if len(doc_batch) > SLIM_BATCH_SIZE:
                     yield doc_batch
                     doc_batch = []
-
-                    if callback:
-                        if callback.should_stop():
-                            raise RuntimeError(
-                                "retrieve_all_slim_documents: Stop signal detected"
-                            )
-
-                        callback.progress("retrieve_all_slim_documents", 1)
-
         if doc_batch:
             yield doc_batch
 
@@ -379,10 +368,9 @@ class GmailConnector(LoadConnector, PollConnector, SlimConnector):
         self,
         start: SecondsSinceUnixEpoch | None = None,
         end: SecondsSinceUnixEpoch | None = None,
-        callback: IndexingHeartbeatInterface | None = None,
     ) -> GenerateSlimDocumentOutput:
         try:
-            yield from self._fetch_slim_threads(start, end, callback=callback)
+            yield from self._fetch_slim_threads(start, end)
         except Exception as e:
             if MISSING_SCOPES_ERROR_STR in str(e):
                 raise PermissionError(ONYX_SCOPE_INSTRUCTIONS) from e

backend/onyx/connectors/google_drive/connector.py

@@ -42,7 +42,6 @@ from onyx.connectors.interfaces import LoadConnector
 from onyx.connectors.interfaces import PollConnector
 from onyx.connectors.interfaces import SecondsSinceUnixEpoch
 from onyx.connectors.interfaces import SlimConnector
-from onyx.indexing.indexing_heartbeat import IndexingHeartbeatInterface
 from onyx.utils.logger import setup_logger
 from onyx.utils.retry_wrapper import retry_builder
 
@@ -565,7 +564,6 @@ class GoogleDriveConnector(LoadConnector, PollConnector, SlimConnector):
         self,
         start: SecondsSinceUnixEpoch | None = None,
         end: SecondsSinceUnixEpoch | None = None,
-        callback: IndexingHeartbeatInterface | None = None,
     ) -> GenerateSlimDocumentOutput:
         slim_batch = []
         for file in self._fetch_drive_items(
@@ -578,26 +576,15 @@ class GoogleDriveConnector(LoadConnector, PollConnector, SlimConnector):
             if len(slim_batch) >= SLIM_BATCH_SIZE:
                 yield slim_batch
                 slim_batch = []
-                if callback:
-                    if callback.should_stop():
-                        raise RuntimeError(
-                            "_extract_slim_docs_from_google_drive: Stop signal detected"
-                        )
-
-                    callback.progress("_extract_slim_docs_from_google_drive", 1)
-
         yield slim_batch
 
     def retrieve_all_slim_documents(
         self,
         start: SecondsSinceUnixEpoch | None = None,
         end: SecondsSinceUnixEpoch | None = None,
-        callback: IndexingHeartbeatInterface | None = None,
     ) -> GenerateSlimDocumentOutput:
         try:
-            yield from self._extract_slim_docs_from_google_drive(
-                start, end, callback=callback
-            )
+            yield from self._extract_slim_docs_from_google_drive(start, end)
         except Exception as e:
             if MISSING_SCOPES_ERROR_STR in str(e):
                 raise PermissionError(ONYX_SCOPE_INSTRUCTIONS) from e

backend/onyx/connectors/interfaces.py

@@ -7,7 +7,6 @@ from pydantic import BaseModel
 from onyx.configs.constants import DocumentSource
 from onyx.connectors.models import Document
 from onyx.connectors.models import SlimDocument
-from onyx.indexing.indexing_heartbeat import IndexingHeartbeatInterface
 
 
 SecondsSinceUnixEpoch = float
@@ -64,7 +63,6 @@ class SlimConnector(BaseConnector):
         self,
         start: SecondsSinceUnixEpoch | None = None,
         end: SecondsSinceUnixEpoch | None = None,
-        callback: IndexingHeartbeatInterface | None = None,
     ) -> GenerateSlimDocumentOutput:
         raise NotImplementedError
 

backend/onyx/connectors/linear/connector.py

@@ -91,7 +91,6 @@ class LinearConnector(LoadConnector, PollConnector, OAuthConnector):
             f"&response_type=code"
             f"&scope=read"
             f"&state={state}"
-            f"&prompt=consent"  # prompts user for access; allows choosing workspace
         )
 
     @classmethod

backend/onyx/connectors/onyx_jira/connector.py

@@ -29,7 +29,6 @@ from onyx.connectors.onyx_jira.utils import build_jira_url
 from onyx.connectors.onyx_jira.utils import extract_jira_project
 from onyx.connectors.onyx_jira.utils import extract_text_from_adf
 from onyx.connectors.onyx_jira.utils import get_comment_strs
-from onyx.indexing.indexing_heartbeat import IndexingHeartbeatInterface
 from onyx.utils.logger import setup_logger
 
 
@@ -246,7 +245,6 @@ class JiraConnector(LoadConnector, PollConnector, SlimConnector):
         self,
         start: SecondsSinceUnixEpoch | None = None,
         end: SecondsSinceUnixEpoch | None = None,
-        callback: IndexingHeartbeatInterface | None = None,
     ) -> GenerateSlimDocumentOutput:
         jql = f"project = {self.quoted_jira_project}"
 

backend/onyx/connectors/salesforce/connector.py

@@ -21,7 +21,6 @@ from onyx.connectors.salesforce.sqlite_functions import get_affected_parent_ids_
 from onyx.connectors.salesforce.sqlite_functions import get_record
 from onyx.connectors.salesforce.sqlite_functions import init_db
 from onyx.connectors.salesforce.sqlite_functions import update_sf_db_with_csv
-from onyx.indexing.indexing_heartbeat import IndexingHeartbeatInterface
 from onyx.utils.logger import setup_logger
 
 logger = setup_logger()
@@ -177,7 +176,6 @@ class SalesforceConnector(LoadConnector, PollConnector, SlimConnector):
         self,
         start: SecondsSinceUnixEpoch | None = None,
         end: SecondsSinceUnixEpoch | None = None,
-        callback: IndexingHeartbeatInterface | None = None,
     ) -> GenerateSlimDocumentOutput:
         doc_metadata_list: list[SlimDocument] = []
         for parent_object_type in self.parent_object_list:

backend/onyx/connectors/slab/connector.py

@@ -21,7 +21,6 @@ from onyx.connectors.models import ConnectorMissingCredentialError
 from onyx.connectors.models import Document
 from onyx.connectors.models import Section
 from onyx.connectors.models import SlimDocument
-from onyx.indexing.indexing_heartbeat import IndexingHeartbeatInterface
 from onyx.utils.logger import setup_logger
 
 
@@ -243,7 +242,6 @@ class SlabConnector(LoadConnector, PollConnector, SlimConnector):
         self,
         start: SecondsSinceUnixEpoch | None = None,
         end: SecondsSinceUnixEpoch | None = None,
-        callback: IndexingHeartbeatInterface | None = None,
     ) -> GenerateSlimDocumentOutput:
         slim_doc_batch: list[SlimDocument] = []
         for post_id in get_all_post_ids(self.slab_bot_token):

backend/onyx/connectors/slack/connector.py

@@ -27,7 +27,6 @@ from onyx.connectors.slack.utils import get_message_link
 from onyx.connectors.slack.utils import make_paginated_slack_api_call_w_retries
 from onyx.connectors.slack.utils import make_slack_api_call_w_retries
 from onyx.connectors.slack.utils import SlackTextCleaner
-from onyx.indexing.indexing_heartbeat import IndexingHeartbeatInterface
 from onyx.utils.logger import setup_logger
 
 
@@ -99,7 +98,6 @@ def get_channel_messages(
     channel: dict[str, Any],
     oldest: str | None = None,
     latest: str | None = None,
-    callback: IndexingHeartbeatInterface | None = None,
 ) -> Generator[list[MessageType], None, None]:
     """Get all messages in a channel"""
     # join so that the bot can access messages
@@ -117,11 +115,6 @@ def get_channel_messages(
         oldest=oldest,
         latest=latest,
     ):
-        if callback:
-            if callback.should_stop():
-                raise RuntimeError("get_channel_messages: Stop signal detected")
-
-            callback.progress("get_channel_messages", 0)
         yield cast(list[MessageType], result["messages"])
 
 
@@ -332,7 +325,6 @@ def _get_all_doc_ids(
     channels: list[str] | None = None,
     channel_name_regex_enabled: bool = False,
     msg_filter_func: Callable[[MessageType], bool] = default_msg_filter,
-    callback: IndexingHeartbeatInterface | None = None,
 ) -> GenerateSlimDocumentOutput:
     """
     Get all document ids in the workspace, channel by channel
@@ -350,7 +342,6 @@ def _get_all_doc_ids(
         channel_message_batches = get_channel_messages(
             client=client,
             channel=channel,
-            callback=callback,
         )
 
         message_ts_set: set[str] = set()
@@ -399,7 +390,6 @@ class SlackPollConnector(PollConnector, SlimConnector):
         self,
         start: SecondsSinceUnixEpoch | None = None,
         end: SecondsSinceUnixEpoch | None = None,
-        callback: IndexingHeartbeatInterface | None = None,
     ) -> GenerateSlimDocumentOutput:
         if self.client is None:
             raise ConnectorMissingCredentialError("Slack")
@@ -408,7 +398,6 @@ class SlackPollConnector(PollConnector, SlimConnector):
             client=self.client,
             channels=self.channels,
             channel_name_regex_enabled=self.channel_regex_enabled,
-            callback=callback,
         )
 
     def poll_source(

backend/onyx/connectors/zendesk/connector.py

@@ -20,7 +20,6 @@ from onyx.connectors.models import Document
 from onyx.connectors.models import Section
 from onyx.connectors.models import SlimDocument
 from onyx.file_processing.html_utils import parse_html_page_basic
-from onyx.indexing.indexing_heartbeat import IndexingHeartbeatInterface
 from onyx.utils.retry_wrapper import retry_builder
 
 
@@ -406,7 +405,6 @@ class ZendeskConnector(LoadConnector, PollConnector, SlimConnector):
         self,
         start: SecondsSinceUnixEpoch | None = None,
         end: SecondsSinceUnixEpoch | None = None,
-        callback: IndexingHeartbeatInterface | None = None,
     ) -> GenerateSlimDocumentOutput:
         slim_doc_batch: list[SlimDocument] = []
         if self.content_type == "articles":

backend/onyx/db/models.py

@@ -152,7 +152,7 @@ class User(SQLAlchemyBaseUserTableUUID, Base):
     # if not specified, all assistants are shown
     temperature_override_enabled: Mapped[bool] = mapped_column(Boolean, default=False)
     auto_scroll: Mapped[bool] = mapped_column(Boolean, default=True)
-    shortcut_enabled: Mapped[bool] = mapped_column(Boolean, default=False)
+    shortcut_enabled: Mapped[bool] = mapped_column(Boolean, default=True)
     chosen_assistants: Mapped[list[int] | None] = mapped_column(
         postgresql.JSONB(), nullable=True, default=None
     )

backend/onyx/db/persona.py

@@ -228,7 +228,6 @@ def create_update_persona(
             num_chunks=create_persona_request.num_chunks,
             llm_relevance_filter=create_persona_request.llm_relevance_filter,
             llm_filter_extraction=create_persona_request.llm_filter_extraction,
-            is_default_persona=create_persona_request.is_default_persona,
         )
 
         versioned_make_persona_private = fetch_versioned_implementation(

backend/onyx/llm/chat_llm.py

@@ -27,7 +27,6 @@ from langchain_core.prompt_values import PromptValue
 
 from onyx.configs.app_configs import LOG_DANSWER_MODEL_INTERACTIONS
 from onyx.configs.app_configs import MOCK_LLM_RESPONSE
-from onyx.configs.chat_configs import QA_TIMEOUT
 from onyx.configs.model_configs import (
     DISABLE_LITELLM_STREAMING,
 )
@@ -36,7 +35,6 @@ from onyx.configs.model_configs import LITELLM_EXTRA_BODY
 from onyx.llm.interfaces import LLM
 from onyx.llm.interfaces import LLMConfig
 from onyx.llm.interfaces import ToolChoiceOptions
-from onyx.llm.utils import model_is_reasoning_model
 from onyx.server.utils import mask_string
 from onyx.utils.logger import setup_logger
 from onyx.utils.long_term_log import LongTermLogger
@@ -231,15 +229,15 @@ class DefaultMultiLLM(LLM):
     def __init__(
         self,
         api_key: str | None,
+        timeout: int,
         model_provider: str,
         model_name: str,
-        timeout: int | None = None,
         api_base: str | None = None,
         api_version: str | None = None,
         deployment_name: str | None = None,
         max_output_tokens: int | None = None,
         custom_llm_provider: str | None = None,
-        temperature: float | None = None,
+        temperature: float = GEN_AI_TEMPERATURE,
         custom_config: dict[str, str] | None = None,
         extra_headers: dict[str, str] | None = None,
         extra_body: dict | None = LITELLM_EXTRA_BODY,
@@ -247,16 +245,9 @@ class DefaultMultiLLM(LLM):
         long_term_logger: LongTermLogger | None = None,
     ):
         self._timeout = timeout
-        if timeout is None:
-            if model_is_reasoning_model(model_name):
-                self._timeout = QA_TIMEOUT * 10  # Reasoning models are slow
-            else:
-                self._timeout = QA_TIMEOUT
-
-        self._temperature = GEN_AI_TEMPERATURE if temperature is None else temperature
-
         self._model_provider = model_provider
         self._model_version = model_name
+        self._temperature = temperature
         self._api_key = api_key
         self._deployment_name = deployment_name
         self._api_base = api_base

backend/onyx/llm/factory.py

@@ -2,6 +2,7 @@ from typing import Any
 
 from onyx.chat.models import PersonaOverrideConfig
 from onyx.configs.app_configs import DISABLE_GENERATIVE_AI
+from onyx.configs.chat_configs import QA_TIMEOUT
 from onyx.configs.model_configs import GEN_AI_MODEL_FALLBACK_MAX_TOKENS
 from onyx.configs.model_configs import GEN_AI_TEMPERATURE
 from onyx.db.engine import get_session_context_manager
@@ -87,8 +88,8 @@ def get_llms_for_persona(
 
 
 def get_default_llms(
-    timeout: int | None = None,
-    temperature: float | None = None,
+    timeout: int = QA_TIMEOUT,
+    temperature: float = GEN_AI_TEMPERATURE,
     additional_headers: dict[str, str] | None = None,
     long_term_logger: LongTermLogger | None = None,
 ) -> tuple[LLM, LLM]:
@@ -137,7 +138,7 @@ def get_llm(
     api_version: str | None = None,
     custom_config: dict[str, str] | None = None,
     temperature: float | None = None,
-    timeout: int | None = None,
+    timeout: int = QA_TIMEOUT,
     additional_headers: dict[str, str] | None = None,
     long_term_logger: LongTermLogger | None = None,
 ) -> LLM:

backend/onyx/llm/llm_provider_options.py

@@ -29,11 +29,11 @@ OPENAI_PROVIDER_NAME = "openai"
 OPEN_AI_MODEL_NAMES = [
     "o3-mini",
     "o1-mini",
-    "o1",
+    "o1-preview",
+    "o1-2024-12-17",
     "gpt-4",
     "gpt-4o",
     "gpt-4o-mini",
-    "o1-preview",
     "gpt-4-turbo",
     "gpt-4-turbo-preview",
     "gpt-4-1106-preview",

backend/onyx/llm/utils.py

@@ -543,14 +543,3 @@ def model_supports_image_input(model_name: str, model_provider: str) -> bool:
             f"Failed to get model object for {model_provider}/{model_name}"
         )
         return False
-
-
-def model_is_reasoning_model(model_name: str) -> bool:
-    _REASONING_MODEL_NAMES = [
-        "o1",
-        "o1-mini",
-        "o3-mini",
-        "deepseek-reasoner",
-        "deepseek-r1",
-    ]
-    return model_name.lower() in _REASONING_MODEL_NAMES

backend/onyx/prompts/direct_qa_prompts.py

@@ -91,7 +91,7 @@ SAMPLE RESPONSE:
 # similar to the chat flow, but with the option of including a
 # "conversation history" block
 CITATIONS_PROMPT = f"""
-Refer to the following {{context_type}} when responding to me.{DEFAULT_IGNORE_STATEMENT}
+Refer to the following context documents when responding to me.{DEFAULT_IGNORE_STATEMENT}
 
 CONTEXT:
 {GENERAL_SEP_PAT}
@@ -108,7 +108,7 @@ CONTEXT:
 # NOTE: need to add the extra line about "getting right to the point" since the
 # tool calling models from OpenAI tend to be more verbose
 CITATIONS_PROMPT_FOR_TOOL_CALLING = f"""
-Refer to the provided {{context_type}} when responding to me.{DEFAULT_IGNORE_STATEMENT} \
+Refer to the provided context documents when responding to me.{DEFAULT_IGNORE_STATEMENT} \
 You should always get right to the point, and never use extraneous language.
 
 {{history_block}}{{task_prompt}}

backend/onyx/redis/redis_connector_doc_perm_sync.py

@@ -80,8 +80,7 @@ class RedisConnectorPermissionSync:
     def get_active_task_count(self) -> int:
         """Count of active permission sync tasks"""
         count = 0
-        for _ in self.redis.sscan_iter(
-            OnyxRedisConstants.ACTIVE_FENCES,
+        for _ in self.redis.scan_iter(
             RedisConnectorPermissionSync.FENCE_PREFIX + "*",
             count=SCAN_ITER_COUNT_DEFAULT,
         ):

backend/onyx/redis/redis_connector_ext_group_sync.py

@@ -1,4 +1,5 @@
 from datetime import datetime
+from typing import Any
 from typing import cast
 
 import redis
@@ -7,12 +8,10 @@ from pydantic import BaseModel
 from redis.lock import Lock as RedisLock
 from sqlalchemy.orm import Session
 
-from onyx.configs.constants import OnyxRedisConstants
 from onyx.redis.redis_pool import SCAN_ITER_COUNT_DEFAULT
 
 
 class RedisConnectorExternalGroupSyncPayload(BaseModel):
-    id: str
     submitted: datetime
     started: datetime | None
     celery_task_id: str | None
@@ -38,12 +37,6 @@ class RedisConnectorExternalGroupSync:
     TASKSET_PREFIX = f"{PREFIX}_taskset"  # connectorexternalgroupsync_taskset
     SUBTASK_PREFIX = f"{PREFIX}+sub"  # connectorexternalgroupsync+sub
 
-    # used to signal the overall workflow is still active
-    # it's impossible to get the exact state of the system at a single point in time
-    # so we need a signal with a TTL to bridge gaps in our checks
-    ACTIVE_PREFIX = PREFIX + "_active"
-    ACTIVE_TTL = 3600
-
     def __init__(self, tenant_id: str | None, id: int, redis: redis.Redis) -> None:
         self.tenant_id: str | None = tenant_id
         self.id = id
@@ -57,7 +50,6 @@ class RedisConnectorExternalGroupSync:
         self.taskset_key = f"{self.TASKSET_PREFIX}_{id}"
 
         self.subtask_prefix: str = f"{self.SUBTASK_PREFIX}_{id}"
-        self.active_key = f"{self.ACTIVE_PREFIX}_{id}"
 
     def taskset_clear(self) -> None:
         self.redis.delete(self.taskset_key)
@@ -74,8 +66,7 @@ class RedisConnectorExternalGroupSync:
     def get_active_task_count(self) -> int:
         """Count of active external group syncing tasks"""
         count = 0
-        for _ in self.redis.sscan_iter(
-            OnyxRedisConstants.ACTIVE_FENCES,
+        for _ in self.redis.scan_iter(
             RedisConnectorExternalGroupSync.FENCE_PREFIX + "*",
             count=SCAN_ITER_COUNT_DEFAULT,
         ):
@@ -92,11 +83,10 @@ class RedisConnectorExternalGroupSync:
     @property
     def payload(self) -> RedisConnectorExternalGroupSyncPayload | None:
         # read related data and evaluate/print task progress
-        fence_raw = self.redis.get(self.fence_key)
-        if fence_raw is None:
+        fence_bytes = cast(Any, self.redis.get(self.fence_key))
+        if fence_bytes is None:
             return None
 
-        fence_bytes = cast(bytes, fence_raw)
         fence_str = fence_bytes.decode("utf-8")
         payload = RedisConnectorExternalGroupSyncPayload.model_validate_json(
             cast(str, fence_str)
@@ -109,26 +99,10 @@ class RedisConnectorExternalGroupSync:
         payload: RedisConnectorExternalGroupSyncPayload | None,
     ) -> None:
         if not payload:
-            self.redis.srem(OnyxRedisConstants.ACTIVE_FENCES, self.fence_key)
             self.redis.delete(self.fence_key)
             return
 
         self.redis.set(self.fence_key, payload.model_dump_json())
-        self.redis.sadd(OnyxRedisConstants.ACTIVE_FENCES, self.fence_key)
-
-    def set_active(self) -> None:
-        """This sets a signal to keep the permissioning flow from getting cleaned up within
-        the expiration time.
-
-        The slack in timing is needed to avoid race conditions where simply checking
-        the celery queue and task status could result in race conditions."""
-        self.redis.set(self.active_key, 0, ex=self.ACTIVE_TTL)
-
-    def active(self) -> bool:
-        if self.redis.exists(self.active_key):
-            return True
-
-        return False
 
     @property
     def generator_complete(self) -> int | None:
@@ -164,8 +138,6 @@ class RedisConnectorExternalGroupSync:
         pass
 
     def reset(self) -> None:
-        self.redis.srem(OnyxRedisConstants.ACTIVE_FENCES, self.fence_key)
-        self.redis.delete(self.active_key)
         self.redis.delete(self.generator_progress_key)
         self.redis.delete(self.generator_complete_key)
         self.redis.delete(self.taskset_key)
@@ -180,9 +152,6 @@ class RedisConnectorExternalGroupSync:
     @staticmethod
     def reset_all(r: redis.Redis) -> None:
         """Deletes all redis values for all connectors"""
-        for key in r.scan_iter(RedisConnectorExternalGroupSync.ACTIVE_PREFIX + "*"):
-            r.delete(key)
-
         for key in r.scan_iter(RedisConnectorExternalGroupSync.TASKSET_PREFIX + "*"):
             r.delete(key)
 

backend/onyx/redis/redis_connector_prune.py

@@ -1,11 +1,9 @@
 import time
-from datetime import datetime
 from typing import cast
 from uuid import uuid4
 
 import redis
 from celery import Celery
-from pydantic import BaseModel
 from redis.lock import Lock as RedisLock
 from sqlalchemy.orm import Session
 
@@ -18,13 +16,6 @@ from onyx.db.connector_credential_pair import get_connector_credential_pair_from
 from onyx.redis.redis_pool import SCAN_ITER_COUNT_DEFAULT
 
 
-class RedisConnectorPrunePayload(BaseModel):
-    id: str
-    submitted: datetime
-    started: datetime | None
-    celery_task_id: str | None
-
-
 class RedisConnectorPrune:
     """Manages interactions with redis for pruning tasks. Should only be accessed
     through RedisConnector."""
@@ -45,12 +36,6 @@ class RedisConnectorPrune:
     TASKSET_PREFIX = f"{PREFIX}_taskset"  # connectorpruning_taskset
     SUBTASK_PREFIX = f"{PREFIX}+sub"  # connectorpruning+sub
 
-    # used to signal the overall workflow is still active
-    # it's impossible to get the exact state of the system at a single point in time
-    # so we need a signal with a TTL to bridge gaps in our checks
-    ACTIVE_PREFIX = PREFIX + "_active"
-    ACTIVE_TTL = 3600
-
     def __init__(self, tenant_id: str | None, id: int, redis: redis.Redis) -> None:
         self.tenant_id: str | None = tenant_id
         self.id = id
@@ -64,7 +49,6 @@ class RedisConnectorPrune:
         self.taskset_key = f"{self.TASKSET_PREFIX}_{id}"
 
         self.subtask_prefix: str = f"{self.SUBTASK_PREFIX}_{id}"
-        self.active_key = f"{self.ACTIVE_PREFIX}_{id}"
 
     def taskset_clear(self) -> None:
         self.redis.delete(self.taskset_key)
@@ -81,10 +65,8 @@ class RedisConnectorPrune:
     def get_active_task_count(self) -> int:
         """Count of active pruning tasks"""
         count = 0
-        for _ in self.redis.sscan_iter(
-            OnyxRedisConstants.ACTIVE_FENCES,
-            RedisConnectorPrune.FENCE_PREFIX + "*",
-            count=SCAN_ITER_COUNT_DEFAULT,
+        for key in self.redis.scan_iter(
+            RedisConnectorPrune.FENCE_PREFIX + "*", count=SCAN_ITER_COUNT_DEFAULT
         ):
             count += 1
         return count
@@ -96,44 +78,15 @@ class RedisConnectorPrune:
 
         return False
 
-    @property
-    def payload(self) -> RedisConnectorPrunePayload | None:
-        # read related data and evaluate/print task progress
-        fence_bytes = cast(bytes, self.redis.get(self.fence_key))
-        if fence_bytes is None:
-            return None
-
-        fence_str = fence_bytes.decode("utf-8")
-        payload = RedisConnectorPrunePayload.model_validate_json(cast(str, fence_str))
-
-        return payload
-
-    def set_fence(
-        self,
-        payload: RedisConnectorPrunePayload | None,
-    ) -> None:
-        if not payload:
+    def set_fence(self, value: bool) -> None:
+        if not value:
             self.redis.srem(OnyxRedisConstants.ACTIVE_FENCES, self.fence_key)
             self.redis.delete(self.fence_key)
             return
 
-        self.redis.set(self.fence_key, payload.model_dump_json())
+        self.redis.set(self.fence_key, 0)
         self.redis.sadd(OnyxRedisConstants.ACTIVE_FENCES, self.fence_key)
 
-    def set_active(self) -> None:
-        """This sets a signal to keep the permissioning flow from getting cleaned up within
-        the expiration time.
-
-        The slack in timing is needed to avoid race conditions where simply checking
-        the celery queue and task status could result in race conditions."""
-        self.redis.set(self.active_key, 0, ex=self.ACTIVE_TTL)
-
-    def active(self) -> bool:
-        if self.redis.exists(self.active_key):
-            return True
-
-        return False
-
     @property
     def generator_complete(self) -> int | None:
         """the fence payload is an int representing the starting number of
@@ -209,7 +162,6 @@ class RedisConnectorPrune:
 
     def reset(self) -> None:
         self.redis.srem(OnyxRedisConstants.ACTIVE_FENCES, self.fence_key)
-        self.redis.delete(self.active_key)
         self.redis.delete(self.generator_progress_key)
         self.redis.delete(self.generator_complete_key)
         self.redis.delete(self.taskset_key)
@@ -224,9 +176,6 @@ class RedisConnectorPrune:
     @staticmethod
     def reset_all(r: redis.Redis) -> None:
         """Deletes all redis values for all connectors"""
-        for key in r.scan_iter(RedisConnectorPrune.ACTIVE_PREFIX + "*"):
-            r.delete(key)
-
         for key in r.scan_iter(RedisConnectorPrune.TASKSET_PREFIX + "*"):
             r.delete(key)
 

backend/onyx/server/documents/cc_pair.py

@@ -22,8 +22,6 @@ from onyx.background.celery.tasks.pruning.tasks import (
     try_creating_prune_generator_task,
 )
 from onyx.background.celery.versioned_apps.primary import app as primary_app
-from onyx.configs.constants import OnyxCeleryPriority
-from onyx.configs.constants import OnyxCeleryTask
 from onyx.db.connector_credential_pair import add_credential_to_connector
 from onyx.db.connector_credential_pair import (
     get_connector_credential_pair_from_id_for_user,
@@ -230,13 +228,6 @@ def update_cc_pair_status(
 
     db_session.commit()
 
-    # this speeds up the start of indexing by firing the check immediately
-    primary_app.send_task(
-        OnyxCeleryTask.CHECK_FOR_INDEXING,
-        kwargs=dict(tenant_id=tenant_id),
-        priority=OnyxCeleryPriority.HIGH,
-    )
-
     return JSONResponse(
         status_code=HTTPStatus.OK, content={"message": str(HTTPStatus.OK)}
     )
@@ -368,17 +359,15 @@ def prune_cc_pair(
         f"credential={cc_pair.credential_id} "
         f"{cc_pair.connector.name} connector."
     )
-    payload_id = try_creating_prune_generator_task(
+    tasks_created = try_creating_prune_generator_task(
         primary_app, cc_pair, db_session, r, CURRENT_TENANT_ID_CONTEXTVAR.get()
     )
-    if not payload_id:
+    if not tasks_created:
         raise HTTPException(
             status_code=HTTPStatus.INTERNAL_SERVER_ERROR,
             detail="Pruning task creation failed.",
         )
 
-    logger.info(f"Pruning queued: cc_pair={cc_pair.id} id={payload_id}")
-
     return StatusResponse(
         success=True,
         message="Successfully created the pruning task.",
@@ -516,17 +505,15 @@ def sync_cc_pair_groups(
         f"credential_id={cc_pair.credential_id} "
         f"{cc_pair.connector.name} connector."
     )
-    payload_id = try_creating_external_group_sync_task(
+    tasks_created = try_creating_external_group_sync_task(
         primary_app, cc_pair_id, r, CURRENT_TENANT_ID_CONTEXTVAR.get()
     )
-    if not payload_id:
+    if not tasks_created:
         raise HTTPException(
             status_code=HTTPStatus.INTERNAL_SERVER_ERROR,
             detail="External group sync task creation failed.",
         )
 
-    logger.info(f"External group sync queued: cc_pair={cc_pair_id} id={payload_id}")
-
     return StatusResponse(
         success=True,
         message="Successfully created the external group sync task.",
@@ -553,14 +540,7 @@ def associate_credential_to_connector(
     metadata: ConnectorCredentialPairMetadata,
     user: User | None = Depends(current_curator_or_admin_user),
     db_session: Session = Depends(get_session),
-    tenant_id: str = Depends(get_current_tenant_id),
 ) -> StatusResponse[int]:
-    """NOTE(rkuo): internally discussed and the consensus is this endpoint
-    and create_connector_with_mock_credential should be combined.
-
-    The intent of this endpoint is to handle connectors that actually need credentials.
-    """
-
     fetch_ee_implementation_or_noop(
         "onyx.db.user_group", "validate_object_creation_for_user", None
     )(
@@ -583,18 +563,6 @@ def associate_credential_to_connector(
             groups=metadata.groups,
         )
 
-        # trigger indexing immediately
-        primary_app.send_task(
-            OnyxCeleryTask.CHECK_FOR_INDEXING,
-            priority=OnyxCeleryPriority.HIGH,
-            kwargs={"tenant_id": tenant_id},
-        )
-
-        logger.info(
-            f"associate_credential_to_connector - running check_for_indexing: "
-            f"cc_pair={response.data}"
-        )
-
         return response
     except IntegrityError as e:
         logger.error(f"IntegrityError: {e}")

backend/onyx/server/documents/connector.py

@@ -804,14 +804,6 @@ def create_connector_with_mock_credential(
     db_session: Session = Depends(get_session),
     tenant_id: str = Depends(get_current_tenant_id),
 ) -> StatusResponse:
-    """NOTE(rkuo): internally discussed and the consensus is this endpoint
-    and associate_credential_to_connector should be combined.
-
-    The intent of this endpoint is to handle connectors that don't need credentials,
-    AKA web, file, etc ... but there isn't any reason a single endpoint couldn't
-    server this purpose.
-    """
-
     fetch_ee_implementation_or_noop(
         "onyx.db.user_group", "validate_object_creation_for_user", None
     )(
@@ -849,18 +841,6 @@ def create_connector_with_mock_credential(
             groups=connector_data.groups,
         )
 
-        # trigger indexing immediately
-        primary_app.send_task(
-            OnyxCeleryTask.CHECK_FOR_INDEXING,
-            priority=OnyxCeleryPriority.HIGH,
-            kwargs={"tenant_id": tenant_id},
-        )
-
-        logger.info(
-            f"create_connector_with_mock_credential - running check_for_indexing: "
-            f"cc_pair={response.data}"
-        )
-
         create_milestone_and_report(
             user=user,
             distinct_id=user.email if user else tenant_id or "N/A",
@@ -1025,8 +1005,6 @@ def connector_run_once(
         kwargs={"tenant_id": tenant_id},
     )
 
-    logger.info("connector_run_once - running check_for_indexing")
-
     msg = f"Marked {num_triggers} index attempts with indexing triggers."
     return StatusResponse(
         success=True,

backend/onyx/server/documents/standard_oauth.py

@@ -179,10 +179,12 @@ def oauth_callback(
         db_session=db_session,
     )
 
-    # TODO: use a library for url handling
-    sep = "&" if "?" in desired_return_url else "?"
     return CallbackResponse(
-        redirect_url=f"{desired_return_url}{sep}credentialId={credential.id}"
+        redirect_url=(
+            f"{desired_return_url}?credentialId={credential.id}"
+            if "?" not in desired_return_url
+            else f"{desired_return_url}&credentialId={credential.id}"
+        )
     )
 
 

backend/onyx/server/features/document_set/api.py

@@ -6,15 +6,11 @@ from sqlalchemy.orm import Session
 
 from onyx.auth.users import current_curator_or_admin_user
 from onyx.auth.users import current_user
-from onyx.background.celery.versioned_apps.primary import app as primary_app
-from onyx.configs.constants import OnyxCeleryPriority
-from onyx.configs.constants import OnyxCeleryTask
 from onyx.db.document_set import check_document_sets_are_public
 from onyx.db.document_set import fetch_all_document_sets_for_user
 from onyx.db.document_set import insert_document_set
 from onyx.db.document_set import mark_document_set_as_to_be_deleted
 from onyx.db.document_set import update_document_set
-from onyx.db.engine import get_current_tenant_id
 from onyx.db.engine import get_session
 from onyx.db.models import User
 from onyx.server.features.document_set.models import CheckDocSetPublicRequest
@@ -33,7 +29,6 @@ def create_document_set(
     document_set_creation_request: DocumentSetCreationRequest,
     user: User = Depends(current_curator_or_admin_user),
     db_session: Session = Depends(get_session),
-    tenant_id: str = Depends(get_current_tenant_id),
 ) -> int:
     fetch_ee_implementation_or_noop(
         "onyx.db.user_group", "validate_object_creation_for_user", None
@@ -51,13 +46,6 @@ def create_document_set(
         )
     except Exception as e:
         raise HTTPException(status_code=400, detail=str(e))
-
-    primary_app.send_task(
-        OnyxCeleryTask.CHECK_FOR_VESPA_SYNC_TASK,
-        kwargs={"tenant_id": tenant_id},
-        priority=OnyxCeleryPriority.HIGH,
-    )
-
     return document_set_db_model.id
 
 
@@ -66,7 +54,6 @@ def patch_document_set(
     document_set_update_request: DocumentSetUpdateRequest,
     user: User = Depends(current_curator_or_admin_user),
     db_session: Session = Depends(get_session),
-    tenant_id: str = Depends(get_current_tenant_id),
 ) -> None:
     fetch_ee_implementation_or_noop(
         "onyx.db.user_group", "validate_object_creation_for_user", None
@@ -85,19 +72,12 @@ def patch_document_set(
     except Exception as e:
         raise HTTPException(status_code=400, detail=str(e))
 
-    primary_app.send_task(
-        OnyxCeleryTask.CHECK_FOR_VESPA_SYNC_TASK,
-        kwargs={"tenant_id": tenant_id},
-        priority=OnyxCeleryPriority.HIGH,
-    )
-
 
 @router.delete("/admin/document-set/{document_set_id}")
 def delete_document_set(
     document_set_id: int,
     user: User = Depends(current_curator_or_admin_user),
     db_session: Session = Depends(get_session),
-    tenant_id: str = Depends(get_current_tenant_id),
 ) -> None:
     try:
         mark_document_set_as_to_be_deleted(
@@ -108,12 +88,6 @@ def delete_document_set(
     except Exception as e:
         raise HTTPException(status_code=400, detail=str(e))
 
-    primary_app.send_task(
-        OnyxCeleryTask.CHECK_FOR_VESPA_SYNC_TASK,
-        kwargs={"tenant_id": tenant_id},
-        priority=OnyxCeleryPriority.HIGH,
-    )
-
 
 """Endpoints for non-admins"""
 

backend/onyx/server/manage/administrative.py

@@ -197,11 +197,6 @@ def create_deletion_attempt_for_connector_id(
         kwargs={"tenant_id": tenant_id},
     )
 
-    logger.info(
-        f"create_deletion_attempt_for_connector_id - running check_for_connector_deletion: "
-        f"cc_pair={cc_pair.id}"
-    )
-
     if cc_pair.connector.source == DocumentSource.FILE:
         connector = cc_pair.connector
         file_store = get_default_file_store(db_session)

backend/onyx/server/manage/users.py

@@ -34,7 +34,6 @@ from onyx.auth.users import current_curator_or_admin_user
 from onyx.auth.users import current_user
 from onyx.auth.users import optional_user
 from onyx.configs.app_configs import AUTH_TYPE
-from onyx.configs.app_configs import DEV_MODE
 from onyx.configs.app_configs import ENABLE_EMAIL_INVITES
 from onyx.configs.app_configs import SESSION_EXPIRE_TIME_SECONDS
 from onyx.configs.app_configs import VALID_EMAIL_DOMAINS
@@ -287,7 +286,7 @@ def bulk_invite_users(
             detail=f"Invalid email address: {email} - {str(e)}",
         )
 
-    if MULTI_TENANT and not DEV_MODE:
+    if MULTI_TENANT:
         try:
             fetch_ee_implementation_or_noop(
                 "onyx.server.tenants.provisioning", "add_users_to_tenant", None

backend/onyx/tools/tool_implementations/internet_search/internet_search_tool.py

@@ -279,5 +279,4 @@ class InternetSearchTool(Tool):
             using_tool_calling_llm=using_tool_calling_llm,
             answer_style_config=self.answer_style_config,
             prompt_config=self.prompt_config,
-            context_type="internet search results",
         )

backend/onyx/tools/tool_implementations/search_like_tool_utils.py

@@ -25,7 +25,6 @@ def build_next_prompt_for_search_like_tool(
     using_tool_calling_llm: bool,
     answer_style_config: AnswerStyleConfig,
     prompt_config: PromptConfig,
-    context_type: str = "context documents",
 ) -> AnswerPromptBuilder:
     if not using_tool_calling_llm:
         final_context_docs_response = next(
@@ -59,7 +58,6 @@ def build_next_prompt_for_search_like_tool(
                 else False
             ),
             history_message=prompt_builder.single_message_history or "",
-            context_type=context_type,
         )
     )
 

backend/tests/integration/Dockerfile

@@ -70,8 +70,8 @@ COPY supervisord.conf /etc/supervisor/conf.d/supervisord.conf
 # Set up application files
 COPY ./onyx /app/onyx
 COPY ./shared_configs /app/shared_configs
-COPY ./alembic_tenants /app/alembic_tenants
 COPY ./alembic /app/alembic
+COPY ./alembic_tenants /app/alembic_tenants
 COPY ./alembic.ini /app/alembic.ini
 COPY ./pytest.ini /app/pytest.ini
 COPY supervisord.conf /usr/etc/supervisord.conf

backend/tests/integration/common_utils/managers/tenant.py

@@ -24,6 +24,35 @@ def generate_auth_token() -> str:
 
 
 class TenantManager:
+    @staticmethod
+    def create(
+        tenant_id: str | None = None,
+        initial_admin_email: str | None = None,
+        referral_source: str | None = None,
+    ) -> dict[str, str]:
+        body = {
+            "tenant_id": tenant_id,
+            "initial_admin_email": initial_admin_email,
+            "referral_source": referral_source,
+        }
+
+        token = generate_auth_token()
+        headers = {
+            "Authorization": f"Bearer {token}",
+            "X-API-KEY": "",
+            "Content-Type": "application/json",
+        }
+
+        response = requests.post(
+            url=f"{API_SERVER_URL}/tenants/create",
+            json=body,
+            headers=headers,
+        )
+
+        response.raise_for_status()
+
+        return response.json()
+
     @staticmethod
     def get_all_users(
         user_performing_action: DATestUser | None = None,

backend/tests/integration/common_utils/managers/user.py

@@ -92,7 +92,6 @@ class UserManager:
 
         # Set cookies in the headers
         test_user.headers["Cookie"] = f"fastapiusersauth={session_cookie}; "
-        test_user.cookies = {"fastapiusersauth": session_cookie}
         return test_user
 
     @staticmethod
@@ -103,7 +102,6 @@ class UserManager:
         response = requests.get(
             url=f"{API_SERVER_URL}/me",
             headers=user_to_verify.headers,
-            cookies=user_to_verify.cookies,
         )
 
         if user_to_verify.is_active is False:

backend/tests/integration/common_utils/reset.py

@@ -242,18 +242,6 @@ def reset_postgres_multitenant() -> None:
         schema_name = schema[0]
         cur.execute(f'DROP SCHEMA "{schema_name}" CASCADE')
 
-    # Drop tables in the public schema
-    cur.execute(
-        """
-        SELECT tablename FROM pg_tables
-        WHERE schemaname = 'public'
-        """
-    )
-    public_tables = cur.fetchall()
-    for table in public_tables:
-        table_name = table[0]
-        cur.execute(f'DROP TABLE IF EXISTS public."{table_name}" CASCADE')
-
     cur.close()
     conn.close()
 

backend/tests/integration/common_utils/test_models.py

@@ -44,7 +44,6 @@ class DATestUser(BaseModel):
     headers: dict
     role: UserRole
     is_active: bool
-    cookies: dict = {}
 
 
 class DATestPersonaLabel(BaseModel):

backend/tests/integration/multitenant_tests/syncing/test_search_permissions.py

@@ -4,6 +4,7 @@ from tests.integration.common_utils.managers.cc_pair import CCPairManager
 from tests.integration.common_utils.managers.chat import ChatSessionManager
 from tests.integration.common_utils.managers.document import DocumentManager
 from tests.integration.common_utils.managers.llm_provider import LLMProviderManager
+from tests.integration.common_utils.managers.tenant import TenantManager
 from tests.integration.common_utils.managers.user import UserManager
 from tests.integration.common_utils.test_models import DATestAPIKey
 from tests.integration.common_utils.test_models import DATestCCPair
@@ -12,28 +13,25 @@ from tests.integration.common_utils.test_models import DATestUser
 
 
 def test_multi_tenant_access_control(reset_multitenant: None) -> None:
-    # Creating an admin user (first user created is automatically an admin and also proviions the tenant
-    admin_user1: DATestUser = UserManager.create(
-        email="admin@onyx-test.com",
-    )
-
-    assert UserManager.is_role(admin_user1, UserRole.ADMIN)
+    # Create Tenant 1 and its Admin User
+    TenantManager.create("tenant_dev1", "test1@test.com", "Data Plane Registration")
+    test_user1: DATestUser = UserManager.create(name="test1", email="test1@test.com")
+    assert UserManager.is_role(test_user1, UserRole.ADMIN)
 
     # Create Tenant 2 and its Admin User
-    admin_user2: DATestUser = UserManager.create(
-        email="admin2@onyx-test.com",
-    )
-    assert UserManager.is_role(admin_user2, UserRole.ADMIN)
+    TenantManager.create("tenant_dev2", "test2@test.com", "Data Plane Registration")
+    test_user2: DATestUser = UserManager.create(name="test2", email="test2@test.com")
+    assert UserManager.is_role(test_user2, UserRole.ADMIN)
 
     # Create connectors for Tenant 1
     cc_pair_1: DATestCCPair = CCPairManager.create_from_scratch(
-        user_performing_action=admin_user1,
+        user_performing_action=test_user1,
     )
     api_key_1: DATestAPIKey = APIKeyManager.create(
-        user_performing_action=admin_user1,
+        user_performing_action=test_user1,
     )
-    api_key_1.headers.update(admin_user1.headers)
-    LLMProviderManager.create(user_performing_action=admin_user1)
+    api_key_1.headers.update(test_user1.headers)
+    LLMProviderManager.create(user_performing_action=test_user1)
 
     # Seed documents for Tenant 1
     cc_pair_1.documents = []
@@ -51,13 +49,13 @@ def test_multi_tenant_access_control(reset_multitenant: None) -> None:
 
     # Create connectors for Tenant 2
     cc_pair_2: DATestCCPair = CCPairManager.create_from_scratch(
-        user_performing_action=admin_user2,
+        user_performing_action=test_user2,
     )
     api_key_2: DATestAPIKey = APIKeyManager.create(
-        user_performing_action=admin_user2,
+        user_performing_action=test_user2,
     )
-    api_key_2.headers.update(admin_user2.headers)
-    LLMProviderManager.create(user_performing_action=admin_user2)
+    api_key_2.headers.update(test_user2.headers)
+    LLMProviderManager.create(user_performing_action=test_user2)
 
     # Seed documents for Tenant 2
     cc_pair_2.documents = []
@@ -78,17 +76,17 @@ def test_multi_tenant_access_control(reset_multitenant: None) -> None:
 
     # Create chat sessions for each user
     chat_session1: DATestChatSession = ChatSessionManager.create(
-        user_performing_action=admin_user1
+        user_performing_action=test_user1
     )
     chat_session2: DATestChatSession = ChatSessionManager.create(
-        user_performing_action=admin_user2
+        user_performing_action=test_user2
     )
 
     # User 1 sends a message and gets a response
     response1 = ChatSessionManager.send_message(
         chat_session_id=chat_session1.id,
         message="What is in Tenant 1's documents?",
-        user_performing_action=admin_user1,
+        user_performing_action=test_user1,
     )
     # Assert that the search tool was used
     assert response1.tool_name == "run_search"
@@ -102,16 +100,14 @@ def test_multi_tenant_access_control(reset_multitenant: None) -> None:
     ), "Tenant 2 document IDs should not be in the response"
 
     # Assert that the contents are correct
-    assert any(
-        doc["content"] == "Tenant 1 Document Content"
-        for doc in response1.tool_result or []
-    ), "Tenant 1 Document Content not found in any document"
+    for doc in response1.tool_result or []:
+        assert doc["content"] == "Tenant 1 Document Content"
 
     # User 2 sends a message and gets a response
     response2 = ChatSessionManager.send_message(
         chat_session_id=chat_session2.id,
         message="What is in Tenant 2's documents?",
-        user_performing_action=admin_user2,
+        user_performing_action=test_user2,
     )
     # Assert that the search tool was used
     assert response2.tool_name == "run_search"
@@ -123,18 +119,15 @@ def test_multi_tenant_access_control(reset_multitenant: None) -> None:
     assert not response_doc_ids.intersection(
         tenant1_doc_ids
     ), "Tenant 1 document IDs should not be in the response"
-
     # Assert that the contents are correct
-    assert any(
-        doc["content"] == "Tenant 2 Document Content"
-        for doc in response2.tool_result or []
-    ), "Tenant 2 Document Content not found in any document"
+    for doc in response2.tool_result or []:
+        assert doc["content"] == "Tenant 2 Document Content"
 
     # User 1 tries to access Tenant 2's documents
     response_cross = ChatSessionManager.send_message(
         chat_session_id=chat_session1.id,
         message="What is in Tenant 2's documents?",
-        user_performing_action=admin_user1,
+        user_performing_action=test_user1,
     )
     # Assert that the search tool was used
     assert response_cross.tool_name == "run_search"
@@ -147,7 +140,7 @@ def test_multi_tenant_access_control(reset_multitenant: None) -> None:
     response_cross2 = ChatSessionManager.send_message(
         chat_session_id=chat_session2.id,
         message="What is in Tenant 1's documents?",
-        user_performing_action=admin_user2,
+        user_performing_action=test_user2,
     )
     # Assert that the search tool was used
     assert response_cross2.tool_name == "run_search"

backend/tests/integration/multitenant_tests/tenants/test_tenant_creation.py

@@ -4,12 +4,14 @@ from onyx.db.models import UserRole
 from tests.integration.common_utils.managers.cc_pair import CCPairManager
 from tests.integration.common_utils.managers.connector import ConnectorManager
 from tests.integration.common_utils.managers.credential import CredentialManager
+from tests.integration.common_utils.managers.tenant import TenantManager
 from tests.integration.common_utils.managers.user import UserManager
 from tests.integration.common_utils.test_models import DATestUser
 
 
 # Test flow from creating tenant to registering as a user
 def test_tenant_creation(reset_multitenant: None) -> None:
+    TenantManager.create("tenant_dev", "test@test.com", "Data Plane Registration")
     test_user: DATestUser = UserManager.create(name="test", email="test@test.com")
 
     assert UserManager.is_role(test_user, UserRole.ADMIN)

backend/tests/integration/tests/index_attempt/test_index_attempt_pagination.py

@@ -1,23 +1,23 @@
-import time
 from datetime import datetime
 
 from onyx.db.models import IndexingStatus
 from tests.integration.common_utils.managers.cc_pair import CCPairManager
 from tests.integration.common_utils.managers.index_attempt import IndexAttemptManager
 from tests.integration.common_utils.managers.user import UserManager
+from tests.integration.common_utils.test_models import DATestIndexAttempt
 from tests.integration.common_utils.test_models import DATestUser
 
 
 def _verify_index_attempt_pagination(
     cc_pair_id: int,
-    index_attempt_ids: list[int],
+    index_attempts: list[DATestIndexAttempt],
     page_size: int = 5,
     user_performing_action: DATestUser | None = None,
 ) -> None:
     retrieved_attempts: list[int] = []
     last_time_started = None  # Track the last time_started seen
 
-    for i in range(0, len(index_attempt_ids), page_size):
+    for i in range(0, len(index_attempts), page_size):
         paginated_result = IndexAttemptManager.get_index_attempt_page(
             cc_pair_id=cc_pair_id,
             page=(i // page_size),
@@ -26,9 +26,9 @@ def _verify_index_attempt_pagination(
         )
 
         # Verify that the total items is equal to the length of the index attempts list
-        assert paginated_result.total_items == len(index_attempt_ids)
+        assert paginated_result.total_items == len(index_attempts)
         # Verify that the number of items in the page is equal to the page size
-        assert len(paginated_result.items) == min(page_size, len(index_attempt_ids) - i)
+        assert len(paginated_result.items) == min(page_size, len(index_attempts) - i)
 
         # Verify time ordering within the page (descending order)
         for attempt in paginated_result.items:
@@ -42,7 +42,7 @@ def _verify_index_attempt_pagination(
         retrieved_attempts.extend([attempt.id for attempt in paginated_result.items])
 
     # Create a set of all the expected index attempt IDs
-    all_expected_attempts = set(index_attempt_ids)
+    all_expected_attempts = set(attempt.id for attempt in index_attempts)
     # Create a set of all the retrieved index attempt IDs
     all_retrieved_attempts = set(retrieved_attempts)
 
@@ -51,9 +51,6 @@ def _verify_index_attempt_pagination(
 
 
 def test_index_attempt_pagination(reset: None) -> None:
-    MAX_WAIT = 60
-    all_attempt_ids: list[int] = []
-
     # Create an admin user to perform actions
     user_performing_action: DATestUser = UserManager.create(
         name="admin_performing_action",
@@ -65,49 +62,20 @@ def test_index_attempt_pagination(reset: None) -> None:
         user_performing_action=user_performing_action,
     )
 
-    # Creating a CC pair will create an index attempt as well. wait for it.
-    start = time.monotonic()
-    while True:
-        paginated_result = IndexAttemptManager.get_index_attempt_page(
-            cc_pair_id=cc_pair.id,
-            page=0,
-            page_size=5,
-            user_performing_action=user_performing_action,
-        )
-
-        if paginated_result.total_items == 1:
-            all_attempt_ids.append(paginated_result.items[0].id)
-            print("Initial index attempt from cc_pair creation detected. Continuing...")
-            break
-
-        elapsed = time.monotonic() - start
-        if elapsed > MAX_WAIT:
-            raise TimeoutError(
-                f"Initial index attempt: Not detected within {MAX_WAIT} seconds."
-            )
-
-        print(
-            f"Waiting for initial index attempt: elapsed={elapsed:.2f} timeout={MAX_WAIT}"
-        )
-        time.sleep(1)
-
-    # Create 299 successful index attempts (for 300 total)
+    # Create 300 successful index attempts
     base_time = datetime.now()
-    generated_attempts = IndexAttemptManager.create_test_index_attempts(
-        num_attempts=299,
+    all_attempts = IndexAttemptManager.create_test_index_attempts(
+        num_attempts=300,
         cc_pair_id=cc_pair.id,
         status=IndexingStatus.SUCCESS,
         base_time=base_time,
     )
 
-    for attempt in generated_attempts:
-        all_attempt_ids.append(attempt.id)
-
     # Verify basic pagination with different page sizes
     print("Verifying basic pagination with page size 5")
     _verify_index_attempt_pagination(
         cc_pair_id=cc_pair.id,
-        index_attempt_ids=all_attempt_ids,
+        index_attempts=all_attempts,
         page_size=5,
         user_performing_action=user_performing_action,
     )
@@ -116,7 +84,7 @@ def test_index_attempt_pagination(reset: None) -> None:
     print("Verifying pagination with page size 100")
     _verify_index_attempt_pagination(
         cc_pair_id=cc_pair.id,
-        index_attempt_ids=all_attempt_ids,
+        index_attempts=all_attempts,
         page_size=100,
         user_performing_action=user_performing_action,
     )

deployment/docker_compose/docker-compose.multitenant-dev.yml

@@ -1,423 +0,0 @@
-services:
-  api_server:
-    image: onyxdotapp/onyx-backend:${IMAGE_TAG:-latest}
-    build:
-      context: ../../backend
-      dockerfile: Dockerfile
-    command: >
-      /bin/sh -c "
-      alembic -n schema_private upgrade head &&
-      echo \"Starting Onyx Api Server\" &&
-      uvicorn onyx.main:app --host 0.0.0.0 --port 8080"
-    depends_on:
-      - relational_db
-      - index
-      - cache
-      - inference_model_server
-    restart: always
-    ports:
-      - "8080:8080"
-    environment:
-      - ENABLE_PAID_ENTERPRISE_EDITION_FEATURES=true
-      - MULTI_TENANT=true
-      - LOG_LEVEL=DEBUG
-      - AUTH_TYPE=cloud
-      - REQUIRE_EMAIL_VERIFICATION=false
-      - DISABLE_TELEMETRY=true
-      - IMAGE_TAG=test
-      - DEV_MODE=true
-      # Auth Settings
-      - SESSION_EXPIRE_TIME_SECONDS=${SESSION_EXPIRE_TIME_SECONDS:-}
-      - ENCRYPTION_KEY_SECRET=${ENCRYPTION_KEY_SECRET:-}
-      - VALID_EMAIL_DOMAINS=${VALID_EMAIL_DOMAINS:-}
-      - GOOGLE_OAUTH_CLIENT_ID=${GOOGLE_OAUTH_CLIENT_ID:-}
-      - GOOGLE_OAUTH_CLIENT_SECRET=${GOOGLE_OAUTH_CLIENT_SECRET:-}
-      - SMTP_SERVER=${SMTP_SERVER:-}
-      - SMTP_PORT=${SMTP_PORT:-587}
-      - SMTP_USER=${SMTP_USER:-}
-      - SMTP_PASS=${SMTP_PASS:-}
-      - ENABLE_EMAIL_INVITES=${ENABLE_EMAIL_INVITES:-}
-      - EMAIL_FROM=${EMAIL_FROM:-}
-      - OAUTH_CLIENT_ID=${OAUTH_CLIENT_ID:-}
-      - OAUTH_CLIENT_SECRET=${OAUTH_CLIENT_SECRET:-}
-      - OPENID_CONFIG_URL=${OPENID_CONFIG_URL:-}
-      - TRACK_EXTERNAL_IDP_EXPIRY=${TRACK_EXTERNAL_IDP_EXPIRY:-}
-      - CORS_ALLOWED_ORIGIN=${CORS_ALLOWED_ORIGIN:-}
-      # Gen AI Settings
-      - GEN_AI_MAX_TOKENS=${GEN_AI_MAX_TOKENS:-}
-      - QA_TIMEOUT=${QA_TIMEOUT:-}
-      - MAX_CHUNKS_FED_TO_CHAT=${MAX_CHUNKS_FED_TO_CHAT:-}
-      - DISABLE_LLM_CHOOSE_SEARCH=${DISABLE_LLM_CHOOSE_SEARCH:-}
-      - DISABLE_LLM_QUERY_REPHRASE=${DISABLE_LLM_QUERY_REPHRASE:-}
-      - DISABLE_GENERATIVE_AI=${DISABLE_GENERATIVE_AI:-}
-      - DISABLE_LITELLM_STREAMING=${DISABLE_LITELLM_STREAMING:-}
-      - LITELLM_EXTRA_HEADERS=${LITELLM_EXTRA_HEADERS:-}
-      - BING_API_KEY=${BING_API_KEY:-}
-      - DISABLE_LLM_DOC_RELEVANCE=${DISABLE_LLM_DOC_RELEVANCE:-}
-      - GEN_AI_API_KEY=${GEN_AI_API_KEY:-}
-      - TOKEN_BUDGET_GLOBALLY_ENABLED=${TOKEN_BUDGET_GLOBALLY_ENABLED:-}
-      # Query Options
-      - DOC_TIME_DECAY=${DOC_TIME_DECAY:-}
-      - HYBRID_ALPHA=${HYBRID_ALPHA:-}
-      - EDIT_KEYWORD_QUERY=${EDIT_KEYWORD_QUERY:-}
-      - MULTILINGUAL_QUERY_EXPANSION=${MULTILINGUAL_QUERY_EXPANSION:-}
-      - LANGUAGE_HINT=${LANGUAGE_HINT:-}
-      - LANGUAGE_CHAT_NAMING_HINT=${LANGUAGE_CHAT_NAMING_HINT:-}
-      - QA_PROMPT_OVERRIDE=${QA_PROMPT_OVERRIDE:-}
-      # Other services
-      - POSTGRES_HOST=relational_db
-      - POSTGRES_DEFAULT_SCHEMA=${POSTGRES_DEFAULT_SCHEMA:-}
-      - VESPA_HOST=index
-      - REDIS_HOST=cache
-      - WEB_DOMAIN=${WEB_DOMAIN:-}
-      # Don't change the NLP model configs unless you know what you're doing
-      - EMBEDDING_BATCH_SIZE=${EMBEDDING_BATCH_SIZE:-}
-      - DOCUMENT_ENCODER_MODEL=${DOCUMENT_ENCODER_MODEL:-}
-      - DOC_EMBEDDING_DIM=${DOC_EMBEDDING_DIM:-}
-      - NORMALIZE_EMBEDDINGS=${NORMALIZE_EMBEDDINGS:-}
-      - ASYM_QUERY_PREFIX=${ASYM_QUERY_PREFIX:-}
-      - DISABLE_RERANK_FOR_STREAMING=${DISABLE_RERANK_FOR_STREAMING:-}
-      - MODEL_SERVER_HOST=${MODEL_SERVER_HOST:-inference_model_server}
-      - MODEL_SERVER_PORT=${MODEL_SERVER_PORT:-}
-      - LOG_ALL_MODEL_INTERACTIONS=${LOG_ALL_MODEL_INTERACTIONS:-}
-      - LOG_DANSWER_MODEL_INTERACTIONS=${LOG_DANSWER_MODEL_INTERACTIONS:-}
-      - LOG_INDIVIDUAL_MODEL_TOKENS=${LOG_INDIVIDUAL_MODEL_TOKENS:-}
-      - LOG_VESPA_TIMING_INFORMATION=${LOG_VESPA_TIMING_INFORMATION:-}
-      - LOG_ENDPOINT_LATENCY=${LOG_ENDPOINT_LATENCY:-}
-      - LOG_POSTGRES_LATENCY=${LOG_POSTGRES_LATENCY:-}
-      - LOG_POSTGRES_CONN_COUNTS=${LOG_POSTGRES_CONN_COUNTS:-}
-      - CELERY_BROKER_POOL_LIMIT=${CELERY_BROKER_POOL_LIMIT:-}
-      - LITELLM_CUSTOM_ERROR_MESSAGE_MAPPINGS=${LITELLM_CUSTOM_ERROR_MESSAGE_MAPPINGS:-}
-      # Egnyte OAuth Configs
-      - EGNYTE_CLIENT_ID=${EGNYTE_CLIENT_ID:-}
-      - EGNYTE_CLIENT_SECRET=${EGNYTE_CLIENT_SECRET:-}
-      - EGNYTE_LOCALHOST_OVERRIDE=${EGNYTE_LOCALHOST_OVERRIDE:-}
-      # Linear OAuth Configs
-      - LINEAR_CLIENT_ID=${LINEAR_CLIENT_ID:-}
-      - LINEAR_CLIENT_SECRET=${LINEAR_CLIENT_SECRET:-}
-      # Analytics Configs
-      - SENTRY_DSN=${SENTRY_DSN:-}
-      # Chat Configs
-      - HARD_DELETE_CHATS=${HARD_DELETE_CHATS:-}
-      # Enables the use of bedrock models or IAM Auth
-      - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:-}
-      - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY:-}
-      - AWS_REGION_NAME=${AWS_REGION_NAME:-}
-      - API_KEY_HASH_ROUNDS=${API_KEY_HASH_ROUNDS:-}
-      # Seeding configuration
-      - USE_IAM_AUTH=${USE_IAM_AUTH:-}
-    extra_hosts:
-      - "host.docker.internal:host-gateway"
-    logging:
-      driver: json-file
-      options:
-        max-size: "50m"
-        max-file: "6"
-
-  background:
-    image: onyxdotapp/onyx-backend:${IMAGE_TAG:-latest}
-    build:
-      context: ../../backend
-      dockerfile: Dockerfile
-    command: >
-      /bin/sh -c "
-      if [ -f /etc/ssl/certs/custom-ca.crt ]; then
-        update-ca-certificates;
-      fi &&
-      /usr/bin/supervisord -c /etc/supervisor/conf.d/supervisord.conf"
-    depends_on:
-      - relational_db
-      - index
-      - cache
-      - inference_model_server
-      - indexing_model_server
-    restart: always
-    environment:
-      - ENABLE_PAID_ENTERPRISE_EDITION_FEATURES=true
-      - MULTI_TENANT=true
-      - LOG_LEVEL=DEBUG
-      - AUTH_TYPE=cloud
-      - REQUIRE_EMAIL_VERIFICATION=false
-      - DISABLE_TELEMETRY=true
-      - IMAGE_TAG=test
-      - ENCRYPTION_KEY_SECRET=${ENCRYPTION_KEY_SECRET:-}
-      - JWT_PUBLIC_KEY_URL=${JWT_PUBLIC_KEY_URL:-}
-      # Gen AI Settings (Needed by OnyxBot)
-      - GEN_AI_MAX_TOKENS=${GEN_AI_MAX_TOKENS:-}
-      - QA_TIMEOUT=${QA_TIMEOUT:-}
-      - MAX_CHUNKS_FED_TO_CHAT=${MAX_CHUNKS_FED_TO_CHAT:-}
-      - DISABLE_LLM_CHOOSE_SEARCH=${DISABLE_LLM_CHOOSE_SEARCH:-}
-      - DISABLE_LLM_QUERY_REPHRASE=${DISABLE_LLM_QUERY_REPHRASE:-}
-      - DISABLE_GENERATIVE_AI=${DISABLE_GENERATIVE_AI:-}
-      - GENERATIVE_MODEL_ACCESS_CHECK_FREQ=${GENERATIVE_MODEL_ACCESS_CHECK_FREQ:-}
-      - DISABLE_LITELLM_STREAMING=${DISABLE_LITELLM_STREAMING:-}
-      - LITELLM_EXTRA_HEADERS=${LITELLM_EXTRA_HEADERS:-}
-      - GEN_AI_API_KEY=${GEN_AI_API_KEY:-}
-      - BING_API_KEY=${BING_API_KEY:-}
-      # Query Options
-      - DOC_TIME_DECAY=${DOC_TIME_DECAY:-}
-      - HYBRID_ALPHA=${HYBRID_ALPHA:-}
-      - EDIT_KEYWORD_QUERY=${EDIT_KEYWORD_QUERY:-}
-      - MULTILINGUAL_QUERY_EXPANSION=${MULTILINGUAL_QUERY_EXPANSION:-}
-      - LANGUAGE_HINT=${LANGUAGE_HINT:-}
-      - LANGUAGE_CHAT_NAMING_HINT=${LANGUAGE_CHAT_NAMING_HINT:-}
-      - QA_PROMPT_OVERRIDE=${QA_PROMPT_OVERRIDE:-}
-      # Other Services
-      - POSTGRES_HOST=relational_db
-      - POSTGRES_USER=${POSTGRES_USER:-}
-      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-}
-      - POSTGRES_DB=${POSTGRES_DB:-}
-      - POSTGRES_DEFAULT_SCHEMA=${POSTGRES_DEFAULT_SCHEMA:-}
-      - VESPA_HOST=index
-      - REDIS_HOST=cache
-      - WEB_DOMAIN=${WEB_DOMAIN:-}
-      # Don't change the NLP model configs unless you know what you're doing
-      - DOCUMENT_ENCODER_MODEL=${DOCUMENT_ENCODER_MODEL:-}
-      - DOC_EMBEDDING_DIM=${DOC_EMBEDDING_DIM:-}
-      - NORMALIZE_EMBEDDINGS=${NORMALIZE_EMBEDDINGS:-}
-      - ASYM_QUERY_PREFIX=${ASYM_QUERY_PREFIX:-}
-      - ASYM_PASSAGE_PREFIX=${ASYM_PASSAGE_PREFIX:-}
-      - MODEL_SERVER_HOST=${MODEL_SERVER_HOST:-inference_model_server}
-      - MODEL_SERVER_PORT=${MODEL_SERVER_PORT:-}
-      - INDEXING_MODEL_SERVER_HOST=${INDEXING_MODEL_SERVER_HOST:-indexing_model_server}
-      # Indexing Configs
-      - VESPA_SEARCHER_THREADS=${VESPA_SEARCHER_THREADS:-}
-      - NUM_INDEXING_WORKERS=${NUM_INDEXING_WORKERS:-}
-      - ENABLED_CONNECTOR_TYPES=${ENABLED_CONNECTOR_TYPES:-}
-      - DISABLE_INDEX_UPDATE_ON_SWAP=${DISABLE_INDEX_UPDATE_ON_SWAP:-}
-      - DASK_JOB_CLIENT_ENABLED=${DASK_JOB_CLIENT_ENABLED:-}
-      - CONTINUE_ON_CONNECTOR_FAILURE=${CONTINUE_ON_CONNECTOR_FAILURE:-}
-      - EXPERIMENTAL_CHECKPOINTING_ENABLED=${EXPERIMENTAL_CHECKPOINTING_ENABLED:-}
-      - CONFLUENCE_CONNECTOR_LABELS_TO_SKIP=${CONFLUENCE_CONNECTOR_LABELS_TO_SKIP:-}
-      - JIRA_CONNECTOR_LABELS_TO_SKIP=${JIRA_CONNECTOR_LABELS_TO_SKIP:-}
-      - WEB_CONNECTOR_VALIDATE_URLS=${WEB_CONNECTOR_VALIDATE_URLS:-}
-      - JIRA_API_VERSION=${JIRA_API_VERSION:-}
-      - GONG_CONNECTOR_START_TIME=${GONG_CONNECTOR_START_TIME:-}
-      - NOTION_CONNECTOR_ENABLE_RECURSIVE_PAGE_LOOKUP=${NOTION_CONNECTOR_ENABLE_RECURSIVE_PAGE_LOOKUP:-}
-      - GITHUB_CONNECTOR_BASE_URL=${GITHUB_CONNECTOR_BASE_URL:-}
-      - MAX_DOCUMENT_CHARS=${MAX_DOCUMENT_CHARS:-}
-      - MAX_FILE_SIZE_BYTES=${MAX_FILE_SIZE_BYTES:-}
-      # Egnyte OAuth Configs
-      - EGNYTE_CLIENT_ID=${EGNYTE_CLIENT_ID:-}
-      - EGNYTE_CLIENT_SECRET=${EGNYTE_CLIENT_SECRET:-}
-      - EGNYTE_LOCALHOST_OVERRIDE=${EGNYTE_LOCALHOST_OVERRIDE:-}
-      # Lienar OAuth Configs
-      - LINEAR_CLIENT_ID=${LINEAR_CLIENT_ID:-}
-      - LINEAR_CLIENT_SECRET=${LINEAR_CLIENT_SECRET:-}
-      # Celery Configs (defaults are set in the supervisord.conf file.
-      # prefer doing that to have one source of defaults)
-      - CELERY_WORKER_INDEXING_CONCURRENCY=${CELERY_WORKER_INDEXING_CONCURRENCY:-}
-      - CELERY_WORKER_LIGHT_CONCURRENCY=${CELERY_WORKER_LIGHT_CONCURRENCY:-}
-      - CELERY_WORKER_LIGHT_PREFETCH_MULTIPLIER=${CELERY_WORKER_LIGHT_PREFETCH_MULTIPLIER:-}
-
-      # Onyx SlackBot Configs
-      - DANSWER_BOT_DISABLE_DOCS_ONLY_ANSWER=${DANSWER_BOT_DISABLE_DOCS_ONLY_ANSWER:-}
-      - DANSWER_BOT_FEEDBACK_VISIBILITY=${DANSWER_BOT_FEEDBACK_VISIBILITY:-}
-      - DANSWER_BOT_DISPLAY_ERROR_MSGS=${DANSWER_BOT_DISPLAY_ERROR_MSGS:-}
-      - DANSWER_BOT_RESPOND_EVERY_CHANNEL=${DANSWER_BOT_RESPOND_EVERY_CHANNEL:-}
-      - DANSWER_BOT_DISABLE_COT=${DANSWER_BOT_DISABLE_COT:-} # Currently unused
-      - NOTIFY_SLACKBOT_NO_ANSWER=${NOTIFY_SLACKBOT_NO_ANSWER:-}
-      - DANSWER_BOT_MAX_QPM=${DANSWER_BOT_MAX_QPM:-}
-      - DANSWER_BOT_MAX_WAIT_TIME=${DANSWER_BOT_MAX_WAIT_TIME:-}
-      # Logging
-      # Leave this on pretty please? Nothing sensitive is collected!
-      # https://docs.onyx.app/more/telemetry
-      - DISABLE_TELEMETRY=${DISABLE_TELEMETRY:-}
-      - LOG_LEVEL=${LOG_LEVEL:-info} # Set to debug to get more fine-grained logs
-      - LOG_ALL_MODEL_INTERACTIONS=${LOG_ALL_MODEL_INTERACTIONS:-} # LiteLLM Verbose Logging
-      # Log all of Onyx prompts and interactions with the LLM
-      - LOG_DANSWER_MODEL_INTERACTIONS=${LOG_DANSWER_MODEL_INTERACTIONS:-}
-      - LOG_INDIVIDUAL_MODEL_TOKENS=${LOG_INDIVIDUAL_MODEL_TOKENS:-}
-      - LOG_VESPA_TIMING_INFORMATION=${LOG_VESPA_TIMING_INFORMATION:-}
-
-      # Analytics Configs
-      - SENTRY_DSN=${SENTRY_DSN:-}
-
-      # Enterprise Edition stuff
-      - ENABLE_PAID_ENTERPRISE_EDITION_FEATURES=${ENABLE_PAID_ENTERPRISE_EDITION_FEATURES:-false}
-      - USE_IAM_AUTH=${USE_IAM_AUTH:-}
-      - AWS_REGION_NAME=${AWS_REGION_NAME:-}
-      - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID-}
-      - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY-}
-    # Uncomment the line below to use if IAM_AUTH is true and you are using iam auth for postgres
-    # volumes:
-    #   - ./bundle.pem:/app/bundle.pem:ro
-    extra_hosts:
-      - "host.docker.internal:host-gateway"
-    logging:
-      driver: json-file
-      options:
-        max-size: "50m"
-        max-file: "6"
-    # Uncomment the following lines if you need to include a custom CA certificate
-    # This section enables the use of a custom CA certificate
-    # If present, the custom CA certificate is mounted as a volume
-    # The container checks for its existence and updates the system's CA certificates
-    # This allows for secure communication with services using custom SSL certificates
-    # Optional volume mount for CA certificate
-    # volumes:
-    #   # Maps to the CA_CERT_PATH environment variable in the Dockerfile
-    #   - ${CA_CERT_PATH:-./custom-ca.crt}:/etc/ssl/certs/custom-ca.crt:ro
-
-  web_server:
-    image: onyxdotapp/onyx-web-server:${IMAGE_TAG:-latest}
-    build:
-      context: ../../web
-      dockerfile: Dockerfile
-      args:
-        - NEXT_PUBLIC_DISABLE_STREAMING=${NEXT_PUBLIC_DISABLE_STREAMING:-false}
-        - NEXT_PUBLIC_NEW_CHAT_DIRECTS_TO_SAME_PERSONA=${NEXT_PUBLIC_NEW_CHAT_DIRECTS_TO_SAME_PERSONA:-false}
-        - NEXT_PUBLIC_POSITIVE_PREDEFINED_FEEDBACK_OPTIONS=${NEXT_PUBLIC_POSITIVE_PREDEFINED_FEEDBACK_OPTIONS:-}
-        - NEXT_PUBLIC_NEGATIVE_PREDEFINED_FEEDBACK_OPTIONS=${NEXT_PUBLIC_NEGATIVE_PREDEFINED_FEEDBACK_OPTIONS:-}
-        - NEXT_PUBLIC_DISABLE_LOGOUT=${NEXT_PUBLIC_DISABLE_LOGOUT:-}
-        - NEXT_PUBLIC_DEFAULT_SIDEBAR_OPEN=${NEXT_PUBLIC_DEFAULT_SIDEBAR_OPEN:-}
-        - NEXT_PUBLIC_FORGOT_PASSWORD_ENABLED=${NEXT_PUBLIC_FORGOT_PASSWORD_ENABLED:-}
-        # Enterprise Edition only
-        - NEXT_PUBLIC_THEME=${NEXT_PUBLIC_THEME:-}
-        # DO NOT TURN ON unless you have EXPLICIT PERMISSION from Onyx.
-        - NEXT_PUBLIC_DO_NOT_USE_TOGGLE_OFF_DANSWER_POWERED=${NEXT_PUBLIC_DO_NOT_USE_TOGGLE_OFF_DANSWER_POWERED:-false}
-    depends_on:
-      - api_server
-    restart: always
-    environment:
-      - INTERNAL_URL=http://api_server:8080
-      - WEB_DOMAIN=${WEB_DOMAIN:-}
-      - THEME_IS_DARK=${THEME_IS_DARK:-}
-      - DISABLE_LLM_DOC_RELEVANCE=${DISABLE_LLM_DOC_RELEVANCE:-}
-
-      # Enterprise Edition only
-      - ENABLE_PAID_ENTERPRISE_EDITION_FEATURES=${ENABLE_PAID_ENTERPRISE_EDITION_FEATURES:-false}
-      - NEXT_PUBLIC_CUSTOM_REFRESH_URL=${NEXT_PUBLIC_CUSTOM_REFRESH_URL:-}
-
-  inference_model_server:
-    image: onyxdotapp/onyx-model-server:${IMAGE_TAG:-latest}
-    build:
-      context: ../../backend
-      dockerfile: Dockerfile.model_server
-    command: >
-      /bin/sh -c "if [ \"${DISABLE_MODEL_SERVER:-false}\" = \"True\" ]; then
-        echo 'Skipping service...';
-        exit 0;
-      else
-        exec uvicorn model_server.main:app --host 0.0.0.0 --port 9000;
-      fi"
-    restart: on-failure
-    environment:
-      - MIN_THREADS_ML_MODELS=${MIN_THREADS_ML_MODELS:-}
-      # Set to debug to get more fine-grained logs
-      - LOG_LEVEL=${LOG_LEVEL:-info}
-
-      # Analytics Configs
-      - SENTRY_DSN=${SENTRY_DSN:-}
-    volumes:
-      # Not necessary, this is just to reduce download time during startup
-      - model_cache_huggingface:/root/.cache/huggingface/
-    logging:
-      driver: json-file
-      options:
-        max-size: "50m"
-        max-file: "6"
-
-  indexing_model_server:
-    image: onyxdotapp/onyx-model-server:${IMAGE_TAG:-latest}
-    build:
-      context: ../../backend
-      dockerfile: Dockerfile.model_server
-    command: >
-      /bin/sh -c "if [ \"${DISABLE_MODEL_SERVER:-false}\" = \"True\" ]; then
-        echo 'Skipping service...';
-        exit 0;
-      else
-        exec uvicorn model_server.main:app --host 0.0.0.0 --port 9000;
-      fi"
-    restart: on-failure
-    environment:
-      - INDEX_BATCH_SIZE=${INDEX_BATCH_SIZE:-}
-      - MIN_THREADS_ML_MODELS=${MIN_THREADS_ML_MODELS:-}
-      - INDEXING_ONLY=True
-      # Set to debug to get more fine-grained logs
-      - LOG_LEVEL=${LOG_LEVEL:-info}
-      - CLIENT_EMBEDDING_TIMEOUT=${CLIENT_EMBEDDING_TIMEOUT:-}
-
-      # Analytics Configs
-      - SENTRY_DSN=${SENTRY_DSN:-}
-    volumes:
-      # Not necessary, this is just to reduce download time during startup
-      - indexing_huggingface_model_cache:/root/.cache/huggingface/
-    logging:
-      driver: json-file
-      options:
-        max-size: "50m"
-        max-file: "6"
-
-  relational_db:
-    image: postgres:15.2-alpine
-    command: -c 'max_connections=250'
-    restart: always
-    environment:
-      - POSTGRES_USER=${POSTGRES_USER:-postgres}
-      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-password}
-    ports:
-      - "5432:5432"
-    volumes:
-      - db_volume:/var/lib/postgresql/data
-
-  # This container name cannot have an underscore in it due to Vespa expectations of the URL
-  index:
-    image: vespaengine/vespa:8.277.17
-    restart: always
-    ports:
-      - "19071:19071"
-      - "8081:8081"
-    volumes:
-      - vespa_volume:/opt/vespa/var
-    logging:
-      driver: json-file
-      options:
-        max-size: "50m"
-        max-file: "6"
-
-  nginx:
-    image: nginx:1.23.4-alpine
-    restart: always
-    # nginx will immediately crash with `nginx: [emerg] host not found in upstream`
-    # if api_server / web_server are not up
-    depends_on:
-      - api_server
-      - web_server
-    environment:
-      - DOMAIN=localhost
-    ports:
-      - "80:80"
-      - "3000:80" # allow for localhost:3000 usage, since that is the norm
-    volumes:
-      - ../data/nginx:/etc/nginx/conf.d
-    logging:
-      driver: json-file
-      options:
-        max-size: "50m"
-        max-file: "6"
-    # The specified script waits for the api_server to start up.
-    # Without this we've seen issues where nginx shows no error logs but
-    # does not recieve any traffic
-    # NOTE: we have to use dos2unix to remove Carriage Return chars from the file
-    # in order to make this work on both Unix-like systems and windows
-    command: >
-      /bin/sh -c "dos2unix /etc/nginx/conf.d/run-nginx.sh 
-      && /etc/nginx/conf.d/run-nginx.sh app.conf.template.dev"
-
-  cache:
-    image: redis:7.4-alpine
-    restart: always
-    ports:
-      - "6379:6379"
-    # docker silently mounts /data even without an explicit volume mount, which enables
-    # persistence. explicitly setting save and appendonly forces ephemeral behavior.
-    command: redis-server --save "" --appendonly no
-
-volumes:
-  db_volume:
-  vespa_volume: # Created by the container itself
-
-  model_cache_huggingface:
-  indexing_huggingface_model_cache:

web/components.json

@@ -4,7 +4,7 @@
   "rsc": true,
   "tsx": true,
   "tailwind": {
-    "config": "tailwind-themes/tailwind.config.js",
+    "config": "tailwind.config.js",
     "css": "src/app/globals.css",
     "baseColor": "neutral",
     "cssVariables": false,

web/package.json

@@ -4,7 +4,7 @@
   "version-comment": "version field must be SemVer or chromatic will barf",
   "private": true,
   "scripts": {
-    "dev": "next dev --turbo",
+    "dev": "next dev --turbopack",
     "build": "next build",
     "start": "next start",
     "lint": "next lint",
@@ -21,17 +21,17 @@
     "@radix-ui/react-accordion": "^1.2.2",
     "@radix-ui/react-checkbox": "^1.1.2",
     "@radix-ui/react-collapsible": "^1.1.2",
-    "@radix-ui/react-dialog": "^1.1.6",
-    "@radix-ui/react-dropdown-menu": "^2.1.6",
+    "@radix-ui/react-dialog": "^1.1.2",
+    "@radix-ui/react-dropdown-menu": "^2.1.4",
     "@radix-ui/react-label": "^2.1.1",
-    "@radix-ui/react-popover": "^1.1.6",
+    "@radix-ui/react-popover": "^1.1.2",
     "@radix-ui/react-radio-group": "^1.2.2",
     "@radix-ui/react-scroll-area": "^1.2.2",
-    "@radix-ui/react-select": "^2.1.6",
+    "@radix-ui/react-select": "^2.1.2",
     "@radix-ui/react-separator": "^1.1.0",
     "@radix-ui/react-slider": "^1.2.2",
-    "@radix-ui/react-slot": "^1.1.2",
-    "@radix-ui/react-switch": "^1.1.3",
+    "@radix-ui/react-slot": "^1.1.0",
+    "@radix-ui/react-switch": "^1.1.1",
     "@radix-ui/react-tabs": "^1.1.1",
     "@radix-ui/react-tooltip": "^1.1.3",
     "@sentry/nextjs": "^8.50.0",
@@ -56,7 +56,6 @@
     "lucide-react": "^0.454.0",
     "mdast-util-find-and-replace": "^3.0.1",
     "next": "^15.0.2",
-    "next-themes": "^0.4.4",
     "npm": "^10.8.0",
     "postcss": "^8.4.31",
     "posthog-js": "^1.176.0",

web/src/app/admin/add-connector/page.tsx

@@ -27,12 +27,8 @@ function SourceTile({
         w-40 
         cursor-pointer
         shadow-md
-        hover:bg-accent-background-hovered
-        ${
-          preSelect
-            ? "bg-accent-background-hovered subtle-pulse"
-            : "bg-accent-background"
-        }
+        hover:bg-hover
+        ${preSelect ? "bg-hover subtle-pulse" : "bg-hover-light"}
       `}
       href={sourceMetadata.adminUrl}
     >

web/src/app/admin/api-key/page.tsx

@@ -56,7 +56,7 @@ function NewApiKeyModal({
           <div className="flex mt-2">
             <b className="my-auto break-all">{apiKey}</b>
             <div
-              className="ml-2 my-auto p-2 hover:bg-accent-background-hovered rounded cursor-pointer"
+              className="ml-2 my-auto p-2 hover:bg-hover rounded cursor-pointer"
               onClick={() => {
                 setCopyClicked(true);
                 navigator.clipboard.writeText(apiKey);
@@ -112,10 +112,7 @@ function Main() {
   }
 
   const newApiKeyButton = (
-    <CreateButton
-      onClick={() => setShowCreateUpdateForm(true)}
-      text="Create API Key"
-    />
+    <CreateButton href="/admin/api-key/new" text="Create API Key" />
   );
 
   if (apiKeys.length === 0) {
@@ -182,7 +179,7 @@ function Main() {
                   flex 
                   mb-1 
                   w-fit 
-                  hover:bg-accent-background-hovered cursor-pointer
+                  hover:bg-hover cursor-pointer
                   p-2 
                   rounded-lg
                   border-border
@@ -206,7 +203,7 @@ function Main() {
                   flex 
                   mb-1 
                   w-fit 
-                  hover:bg-accent-background-hovered cursor-pointer
+                  hover:bg-hover cursor-pointer
                   p-2 
                   rounded-lg
                   border-border

web/src/app/admin/assistants/AssistantEditor.tsx

@@ -825,7 +825,10 @@ export function AssistantEditor({
                               </TooltipProvider>
                             </div>
                           </div>
-                          <p className="text-sm text-neutral-700 dark:text-neutral-400">
+                          <p
+                            className="text-sm text-subtle"
+                            style={{ color: "rgb(113, 114, 121)" }}
+                          >
                             Attach additional unique knowledge to this assistant
                           </p>
                         </div>
@@ -1214,7 +1217,7 @@ export function AssistantEditor({
                           setFieldValue("label_ids", newLabelIds);
                         }}
                         itemComponent={({ option }) => (
-                          <div className="flex items-center justify-between px-4 py-3 text-sm hover:bg-accent-background-hovered cursor-pointer border-b border-border last:border-b-0">
+                          <div className="flex items-center justify-between px-4 py-3 text-sm hover:bg-hover cursor-pointer border-b border-border last:border-b-0">
                             <div
                               className="flex-grow"
                               onClick={() => {
@@ -1353,7 +1356,7 @@ export function AssistantEditor({
                 </>
               )}
 
-              <div className="mt-12 gap-x-2 w-full justify-end flex">
+              <div className="mt-12 gap-x-2 w-full  justify-end flex">
                 <Button
                   type="submit"
                   disabled={isSubmitting || isRequestSuccessful}

web/src/app/admin/assistants/HidableSection.tsx

@@ -31,7 +31,7 @@ export function HidableSection({
   return (
     <div>
       <div
-        className="flex hover:bg-accent-background rounded cursor-pointer p-2"
+        className="flex hover:bg-hover-light rounded cursor-pointer p-2"
         onClick={() => setIsHidden(!isHidden)}
       >
         <SectionHeader includeMargin={false}>{sectionTitle}</SectionHeader>

web/src/app/admin/assistants/PersonaTable.tsx

@@ -187,9 +187,7 @@ export function PersonasTable() {
                   }
                 }}
                 className={`px-1 py-0.5 rounded flex ${
-                  isEditable
-                    ? "hover:bg-accent-background-hovered cursor-pointer"
-                    : ""
+                  isEditable ? "hover:bg-hover cursor-pointer" : ""
                 } select-none w-fit`}
               >
                 <div className="my-auto w-12">
@@ -207,7 +205,7 @@ export function PersonasTable() {
                 <div className="mr-auto my-auto">
                   {!persona.builtin_persona && isEditable ? (
                     <div
-                      className="hover:bg-accent-background-hovered rounded p-1 cursor-pointer"
+                      className="hover:bg-hover rounded p-1 cursor-pointer"
                       onClick={() => openDeleteModal(persona)}
                     >
                       <TrashIcon />

web/src/app/admin/assistants/StarterMessageList.tsx

@@ -65,7 +65,7 @@ export default function StarterMessagesList({
             onClick={() => {
               arrayHelpers.remove(index);
             }}
-            className={`text-text-400 hover:text-red-500 ${
+            className={`text-gray-400 hover:text-red-500 ${
               index === values.length - 1 && !starterMessage.message
                 ? "opacity-50 cursor-not-allowed"
                 : ""
@@ -105,7 +105,7 @@ export default function StarterMessagesList({
                       4 ||
                     isRefreshing ||
                     !autoStarterMessageEnabled
-                      ? "bg-background-800 text-text-300 cursor-not-allowed"
+                      ? "bg-neutral-800 text-neutral-300 cursor-not-allowed"
                       : ""
                   }
                 `}

web/src/app/admin/assistants/page.tsx

@@ -1,3 +1,4 @@
+"use client";
 import { PersonasTable } from "./PersonaTable";
 import Text from "@/components/ui/text";
 import Title from "@/components/ui/title";

web/src/app/admin/bots/SlackBotUpdateForm.tsx

@@ -92,9 +92,9 @@ export const ExistingSlackBotForm = ({
 
         <div className="flex flex-col" ref={dropdownRef}>
           <div className="flex items-center gap-4">
-            <div className="border rounded-lg border-background-200">
+            <div className="border rounded-lg border-gray-200">
               <div
-                className="flex items-center gap-2 cursor-pointer hover:bg-background-100 p-2"
+                className="flex items-center gap-2 cursor-pointer hover:bg-gray-100 p-2"
                 onClick={() => setIsExpanded(!isExpanded)}
               >
                 {isExpanded ? (
@@ -117,7 +117,7 @@ export const ExistingSlackBotForm = ({
           </div>
 
           {isExpanded && (
-            <div className="bg-white border rounded-lg border-background-200 shadow-lg absolute mt-12 right-0 z-10 w-full md:w-3/4 lg:w-1/2">
+            <div className="bg-white border rounded-lg border-gray-200 shadow-lg absolute mt-12 right-0 z-10 w-full md:w-3/4 lg:w-1/2">
               <div className="p-4">
                 <SlackTokensForm
                   isUpdate={true}
@@ -134,7 +134,7 @@ export const ExistingSlackBotForm = ({
         </div>
       </div>
       <div className="mt-2">
-        <div className="inline-block border rounded-lg border-background-200 p-2">
+        <div className="inline-block border rounded-lg border-gray-200 p-2">
           <Checkbox
             label="Enabled"
             checked={formValues.enabled}

web/src/app/admin/bots/[bot-id]/channels/SlackChannelConfigFormFields.tsx

@@ -613,7 +613,7 @@ export function SlackChannelConfigFormFields({
                         <Link
                           key={ccpairinfo.id}
                           href={`/admin/connector/${ccpairinfo.id}`}
-                          className="flex items-center p-2 rounded-md hover:bg-background-100 transition-colors"
+                          className="flex items-center p-2 rounded-md hover:bg-gray-100 transition-colors"
                         >
                           <div className="mr-2">
                             <SourceIcon

web/src/app/admin/configuration/document-processing/page.tsx

@@ -84,7 +84,7 @@ function Main() {
             {isApiKeySet ? (
               <div className="w-full p-3 border rounded-md bg-background text-text flex items-center">
                 <span className="flex-grow">••••••••••••••••</span>
-                <Lock className="h-5 w-5 text-text-400" />
+                <Lock className="h-5 w-5 text-gray-400" />
               </div>
             ) : (
               <input

web/src/app/admin/configuration/llm/ConfiguredLLMProviderDisplay.tsx

@@ -25,7 +25,8 @@ function LLMProviderUpdateModal({
 }) {
   const providerName = existingLlmProvider?.name
     ? `"${existingLlmProvider.name}"`
-    : llmProviderDescriptor?.display_name ||
+    : null ||
+      llmProviderDescriptor?.display_name ||
       llmProviderDescriptor?.name ||
       "Custom LLM Provider";
   return (
@@ -74,7 +75,7 @@ function LLMProviderDisplay({
   return (
     <div>
       {popup}
-      <div className="border border-border p-3 dark:bg-neutral-800 dark:border-neutral-700 rounded w-96 flex shadow-md">
+      <div className="border border-border p-3 rounded w-96 flex shadow-md">
         <div className="my-auto">
           <div className="font-bold">{providerName} </div>
           <div className="text-xs italic">({existingLlmProvider.provider})</div>
@@ -112,7 +113,7 @@ function LLMProviderDisplay({
         {existingLlmProvider && (
           <div className="my-auto ml-3">
             {existingLlmProvider.is_default_provider ? (
-              <Badge variant="agent">Default</Badge>
+              <Badge variant="orange">Default</Badge>
             ) : (
               <Badge variant="success">Enabled</Badge>
             )}

web/src/app/admin/configuration/llm/CustomLLMProviderUpdateForm.tsx

@@ -348,7 +348,7 @@ export function CustomLLMProviderUpdateForm({
                           </div>
                           <div className="my-auto">
                             <FiX
-                              className="my-auto w-10 h-10 cursor-pointer hover:bg-accent-background-hovered rounded p-2"
+                              className="my-auto w-10 h-10 cursor-pointer hover:bg-hover rounded p-2"
                               onClick={() => arrayHelpers.remove(index)}
                             />
                           </div>

web/src/app/admin/configuration/llm/LLMConfiguration.tsx

@@ -73,7 +73,7 @@ function DefaultLLMProviderDisplay({
   return (
     <div>
       {popup}
-      <div className="border border-border p-3 dark:bg-neutral-800 dark:border-neutral-700 rounded w-96 flex shadow-md">
+      <div className="border border-border p-3 rounded w-96 flex shadow-md">
         <div className="my-auto">
           <div className="font-bold">{providerName} </div>
         </div>

web/src/app/admin/configuration/llm/interfaces.ts

@@ -12,7 +12,6 @@ import {
   OpenSourceIcon,
   AnthropicSVG,
   IconProps,
-  OpenAIISVG,
 } from "@/components/icons/icons";
 import { FaRobot } from "react-icons/fa";
 
@@ -105,7 +104,7 @@ export const getProviderIcon = (providerName: string, modelName?: string) => {
   switch (providerName) {
     case "openai":
       // Special cases for openai based on modelName
-      return modelNameToIcon(modelName || "", OpenAIISVG);
+      return modelNameToIcon(modelName || "", OpenAIIcon);
     case "anthropic":
       return AnthropicSVG;
     case "bedrock":

web/src/app/admin/configuration/search/page.tsx

@@ -111,14 +111,14 @@ function Main() {
                   <div className="space-y-4">
                     <div>
                       <Text className="font-semibold">Reranking Model</Text>
-                      <Text className="text-text-700">
+                      <Text className="text-gray-700">
                         {searchSettings.rerank_model_name || "Not set"}
                       </Text>
                     </div>
 
                     <div>
                       <Text className="font-semibold">Results to Rerank</Text>
-                      <Text className="text-text-700">
+                      <Text className="text-gray-700">
                         {searchSettings.num_rerank}
                       </Text>
                     </div>
@@ -127,7 +127,7 @@ function Main() {
                       <Text className="font-semibold">
                         Multilingual Expansion
                       </Text>
-                      <Text className="text-text-700">
+                      <Text className="text-gray-700">
                         {searchSettings.multilingual_expansion.length > 0
                           ? searchSettings.multilingual_expansion.join(", ")
                           : "None"}
@@ -136,7 +136,7 @@ function Main() {
 
                     <div>
                       <Text className="font-semibold">Multipass Indexing</Text>
-                      <Text className="text-text-700">
+                      <Text className="text-gray-700">
                         {searchSettings.multipass_indexing
                           ? "Enabled"
                           : "Disabled"}
@@ -147,7 +147,7 @@ function Main() {
                       <Text className="font-semibold">
                         Disable Reranking for Streaming
                       </Text>
-                      <Text className="text-text-700">
+                      <Text className="text-gray-700">
                         {searchSettings.disable_rerank_for_streaming
                           ? "Yes"
                           : "No"}

web/src/app/admin/connector/[ccPairId]/ConfigDisplay.tsx

@@ -149,7 +149,7 @@ export function AdvancedConfigDisplay({
     <>
       <Title className="mt-8 mb-2">Advanced Configuration</Title>
       <CardSection>
-        <ul className="w-full text-sm divide-y divide-neutral-200 dark:divide-neutral-700">
+        <ul className="w-full text-sm divide-y divide-background-200 dark:divide-background-700">
           {pruneFreq && (
             <li
               key={0}

web/src/app/admin/connector/[ccPairId]/DeletionErrorStatus.tsx

@@ -11,7 +11,7 @@ export default function DeletionErrorStatus({
         <h3 className="text-base font-medium">Deletion Error</h3>
         <div className="ml-2 relative group">
           <FiInfo className="h-4 w-4 text-error-600 cursor-help" />
-          <div className="absolute z-10 w-64 p-2 mt-2 text-sm bg-white rounded-md shadow-lg opacity-0 group-hover:opacity-100 transition-opacity duration-300 border border-background-200">
+          <div className="absolute z-10 w-64 p-2 mt-2 text-sm bg-white rounded-md shadow-lg opacity-0 group-hover:opacity-100 transition-opacity duration-300 border border-gray-200">
             This error occurred while attempting to delete the connector. You
             may re-attempt a deletion by clicking the &quot;Delete&quot; button.
           </div>

web/src/app/admin/connector/[ccPairId]/page.tsx

@@ -293,7 +293,7 @@ function Main({ ccPairId }: { ccPairId: number }) {
         <b className="text-emphasis">{ccPair.num_docs_indexed}</b>
       </div>
       {!ccPair.is_editable_for_current_user && (
-        <div className="text-sm mt-2 text-text-500 italic">
+        <div className="text-sm mt-2 text-neutral-500 italic">
           {ccPair.access_type === "public"
             ? "Public connectors are not editable by curators."
             : ccPair.access_type === "sync"

web/src/app/admin/connectors/[connector]/AddConnectorPage.tsx

@@ -18,11 +18,7 @@ import AdvancedFormPage from "./pages/Advanced";
 import DynamicConnectionForm from "./pages/DynamicConnectorCreationForm";
 import CreateCredential from "@/components/credentials/actions/CreateCredential";
 import ModifyCredential from "@/components/credentials/actions/ModifyCredential";
-import {
-  ConfigurableSources,
-  oauthSupportedSources,
-  ValidSources,
-} from "@/lib/types";
+import { ConfigurableSources, oauthSupportedSources } from "@/lib/types";
 import {
   Credential,
   credentialTemplates,
@@ -62,7 +58,6 @@ import {
 } from "@/lib/connectors/oauth";
 import { CreateStdOAuthCredential } from "@/components/credentials/actions/CreateStdOAuthCredential";
 import { Spinner } from "@/components/Spinner";
-import { Button } from "@/components/ui/button";
 export interface AdvancedConfig {
   refreshFreq: number;
   pruneFreq: number;
@@ -449,7 +444,7 @@ export default function AddConnector({
               <CardSection>
                 <Title className="mb-2 text-lg">Select a credential</Title>
 
-                {connector == ValidSources.Gmail ? (
+                {connector == "gmail" ? (
                   <GmailMain />
                 ) : (
                   <>
@@ -465,9 +460,8 @@ export default function AddConnector({
                     {!createCredentialFormToggle && (
                       <div className="mt-6 flex space-x-4">
                         {/* Button to pop up a form to manually enter credentials */}
-                        <Button
-                          variant="secondary"
-                          className="mt-6 text-sm mr-4"
+                        <button
+                          className="mt-6 text-sm bg-background-900 px-2 py-1.5 flex text-text-200 flex-none rounded mr-4"
                           onClick={async () => {
                             if (oauthDetails && oauthDetails.oauth_enabled) {
                               if (oauthDetails.additional_kwargs.length > 0) {
@@ -497,7 +491,7 @@ export default function AddConnector({
                           }}
                         >
                           Create New
-                        </Button>
+                        </button>
                         {/* Button to sign in via OAuth */}
                         {oauthSupportedSources.includes(connector) &&
                           (NEXT_PUBLIC_CLOUD_ENABLED ||

web/src/app/admin/connectors/[connector]/NavigationRow.tsx

@@ -50,10 +50,11 @@ const NavigationRow = ({
           </SquareNavigationButton>
         )}
       </div>
+
       <div className="flex justify-center">
         {(formStep > 0 || noCredentials) && (
           <SquareNavigationButton
-            className="bg-agent text-white py-2.5 px-3.5 disabled:opacity-50"
+            className="bg-accent text-white py-2.5 px-3.5 disabled:opacity-50"
             disabled={!isValid}
             onClick={onSubmit}
           >
@@ -62,6 +63,7 @@ const NavigationRow = ({
           </SquareNavigationButton>
         )}
       </div>
+
       <div className="flex justify-end">
         {formStep === 0 && (
           <SquareNavigationButton

web/src/app/admin/connectors/[connector]/Sidebar.tsx

@@ -96,7 +96,7 @@ export default function Sidebar() {
             <div className="mx-auto w-full max-w-2xl px-4 py-8">
               <div className="relative">
                 {connector != "file" && (
-                  <div className="absolute h-[85%] left-[6px] top-[8px] bottom-0 w-0.5 bg-background-300"></div>
+                  <div className="absolute h-[85%] left-[6px] top-[8px] bottom-0 w-0.5 bg-gray-300"></div>
                 )}
                 {settingSteps.map((step, index) => {
                   const allowed =
@@ -119,7 +119,7 @@ export default function Sidebar() {
                       <div className="flex-shrink-0 mr-4 z-10">
                         <div
                           className={`rounded-full h-3.5 w-3.5 flex items-center justify-center ${
-                            allowed ? "bg-blue-500" : "bg-background-300"
+                            allowed ? "bg-blue-500" : "bg-gray-300"
                           }`}
                         >
                           {formStep === index && (
@@ -129,7 +129,7 @@ export default function Sidebar() {
                       </div>
                       <div
                         className={`${
-                          index <= formStep ? "text-text-800" : "text-text-500"
+                          index <= formStep ? "text-gray-800" : "text-gray-500"
                         }`}
                       >
                         {step}

web/src/app/admin/connectors/[connector]/pages/ConnectorInput/NumberInput.tsx

@@ -16,7 +16,7 @@ export default function NumberInput({
 }) {
   return (
     <div className="w-full flex flex-col">
-      <label className="block text-base font-medium text-text-700 dark:text-neutral-100 mb-1">
+      <label className="block text-base font-medium text-text-700 mb-1">
         {label}
         {optional && <span className="text-text-500 ml-1">(optional)</span>}
       </label>
@@ -27,10 +27,10 @@ export default function NumberInput({
         name={name}
         min="-1"
         className={`mt-2 block w-full px-3 py-2 
-                bg-[#fff] dark:bg-transparent border border-background-300 rounded-md 
-                text-sm shadow-sm placeholder-text-400
+                bg-white border border-gray-300 rounded-md 
+                text-sm shadow-sm placeholder-gray-400
                 focus:outline-none focus:border-sky-500 focus:ring-1 focus:ring-sky-500
-                disabled:bg-background-50 disabled:text-text-500 disabled:border-background-200 disabled:shadow-none
+                disabled:bg-gray-50 disabled:text-gray-500 disabled:border-gray-200 disabled:shadow-none
                 invalid:border-pink-500 invalid:text-pink-600
                 focus:invalid:border-pink-500 focus:invalid:ring-pink-500`}
       />

web/src/app/admin/connectors/[connector]/pages/formelements/NumberInput.tsx

@@ -30,10 +30,10 @@ export default function NumberInput({
         min="-1"
         value={value === 0 && showNeverIfZero ? "Never" : value}
         className={`mt-2 block w-full px-3 py-2 
-                bg-white border border-background-300 rounded-md 
-                text-sm shadow-sm placeholder-text-400
+                bg-white border border-gray-300 rounded-md 
+                text-sm shadow-sm placeholder-gray-400
                 focus:outline-none focus:border-sky-500 focus:ring-1 focus:ring-sky-500
-                disabled:bg-background-50 disabled:text-text-500 disabled:border-background-200 disabled:shadow-none
+                disabled:bg-gray-50 disabled:text-gray-500 disabled:border-gray-200 disabled:shadow-none
                 invalid:border-pink-500 invalid:text-pink-600
                 focus:invalid:border-pink-500 focus:invalid:ring-pink-500`}
       />

web/src/app/admin/connectors/[connector]/pages/gdrive/Credential.tsx

@@ -34,9 +34,9 @@ export const DriveJsonUpload = ({
     <>
       <input
         className={
-          "mr-3 text-sm text-text-900 border border-background-300 " +
-          "cursor-pointer bg-backgrournd dark:text-text-400 focus:outline-none " +
-          "dark:bg-background-700 dark:border-background-600 dark:placeholder-text-400"
+          "mr-3 text-sm text-gray-900 border border-gray-300 " +
+          "cursor-pointer bg-backgrournd dark:text-gray-400 focus:outline-none " +
+          "dark:bg-gray-700 dark:border-gray-600 dark:placeholder-gray-400"
         }
         type="file"
         accept=".json"

web/src/app/admin/connectors/[connector]/pages/gmail/Credential.tsx

@@ -34,9 +34,9 @@ const DriveJsonUpload = ({
     <>
       <input
         className={
-          "mr-3 text-sm text-text-900 border border-background-300 overflow-visible " +
-          "cursor-pointer bg-background dark:text-text-400 focus:outline-none " +
-          "dark:bg-background-700 dark:border-background-600 dark:placeholder-text-400"
+          "mr-3 text-sm text-gray-900 border border-gray-300 overflow-visible " +
+          "cursor-pointer bg-background dark:text-gray-400 focus:outline-none " +
+          "dark:bg-gray-700 dark:border-gray-600 dark:placeholder-gray-400"
         }
         type="file"
         accept=".json"

web/src/app/admin/documents/explorer/Explorer.tsx

@@ -50,7 +50,7 @@ const DocumentDisplay = ({
         </a>
       </div>
       <div className="flex flex-wrap gap-x-2 mt-1 text-xs">
-        <div className="px-1 py-0.5 bg-accent-background-hovered rounded flex">
+        <div className="px-1 py-0.5 bg-hover rounded flex">
           <p className="mr-1 my-auto">Boost:</p>
           <ScoreSection
             documentId={document.document_id}
@@ -77,7 +77,7 @@ const DocumentDisplay = ({
               });
             }
           }}
-          className="px-1 py-0.5 bg-accent-background-hovered hover:bg-accent-background rounded flex cursor-pointer select-none"
+          className="px-1 py-0.5 bg-hover hover:bg-hover-light rounded flex cursor-pointer select-none"
         >
           <div className="my-auto">
             {document.hidden ? (
@@ -169,7 +169,7 @@ export function Explorer({
     <div>
       {popup}
       <div className="justify-center py-2">
-        <div className="flex items-center w-full border-2 border-border rounded-lg px-4 py-2 focus-within:border-accent bg-background-search dark:bg-transparent">
+        <div className="flex items-center w-full border-2 border-border rounded-lg px-4 py-2 focus-within:border-accent bg-background-search">
           <MagnifyingGlass />
           <textarea
             autoFocus
@@ -221,7 +221,7 @@ export function Explorer({
         </div>
       )}
       {!query && (
-        <div className="flex text-text-darker mt-3">
+        <div className="flex text-emphasis mt-3">
           Search for a document above to modify its boost or hide it from
           searches.
         </div>

web/src/app/admin/documents/feedback/DocumentFeedbackTable.tsx

@@ -37,7 +37,7 @@ const IsVisibleSection = ({
               );
               onUpdate(response);
             }}
-            className="flex text-error cursor-pointer hover:bg-accent-background-hovered py-1 px-2 w-fit rounded-full"
+            className="flex text-error cursor-pointer hover:bg-hover py-1 px-2 w-fit rounded-full"
           >
             <div className="select-none">Hidden</div>
             <div className="ml-1 my-auto">
@@ -53,7 +53,7 @@ const IsVisibleSection = ({
               );
               onUpdate(response);
             }}
-            className="flex cursor-pointer hover:bg-accent-background-hovered py-1 px-2 w-fit rounded-full"
+            className="flex cursor-pointer hover:bg-hover py-1 px-2 w-fit rounded-full"
           >
             <div className="my-auto select-none">Visible</div>
             <div className="ml-1 my-auto">

web/src/app/admin/documents/sets/DocumentSetCreationForm.tsx

@@ -189,7 +189,7 @@ export const DocumentSetCreationForm = ({
                                   cursor-pointer ` +
                                     (isSelected
                                       ? " bg-background-strong"
-                                      : " hover:bg-accent-background-hovered")
+                                      : " hover:bg-hover")
                                   }
                                   onClick={() => {
                                     if (isSelected) {
@@ -305,7 +305,7 @@ export const DocumentSetCreationForm = ({
                               cursor-pointer ` +
                                 (isSelected
                                   ? " bg-background-strong"
-                                  : " hover:bg-accent-background-hovered")
+                                  : " hover:bg-hover")
                               }
                               onClick={() => {
                                 if (isSelected) {

web/src/app/admin/documents/sets/page.tsx

@@ -55,7 +55,7 @@ const EditRow = ({
 
   if (!isEditable) {
     return (
-      <div className="text-text-darkerfont-medium my-auto p-1">
+      <div className="text-emphasis font-medium my-auto p-1">
         {documentSet.name}
       </div>
     );
@@ -68,7 +68,7 @@ const EditRow = ({
           <TooltipTrigger asChild>
             <div
               className={`
-              text-text-darkerfont-medium my-auto p-1 hover:bg-accent-background flex items-center select-none
+              text-emphasis font-medium my-auto p-1 hover:bg-hover-light flex items-center select-none
               ${documentSet.is_up_to_date ? "cursor-pointer" : "cursor-default"}
             `}
               style={{ wordBreak: "normal", overflowWrap: "break-word" }}
@@ -213,7 +213,7 @@ const DocumentSetTable = ({
                       </Badge>
                     ) : (
                       <Badge
-                        variant={isEditable ? "private" : "default"}
+                        variant={isEditable ? "in_progress" : "outline"}
                         icon={FiLock}
                       >
                         Private

web/src/app/admin/embeddings/EmbeddingModelSelectionForm.tsx

@@ -235,8 +235,8 @@ export function EmbeddingModelSelection({
           onClick={() => setModelTab(null)}
           className={`mr-4 p-2 font-bold  ${
             !modelTab
-              ? "rounded bg-background-900 dark:bg-neutral-900 text-text-100 dark:text-neutral-100 underline"
-              : " hover:underline bg-background-100 dark:bg-neutral-700"
+              ? "rounded bg-background-900 text-text-100 underline"
+              : " hover:underline bg-background-100"
           }`}
         >
           Current
@@ -246,8 +246,8 @@ export function EmbeddingModelSelection({
             onClick={() => setModelTab("cloud")}
             className={`mx-2 p-2 font-bold  ${
               modelTab == "cloud"
-                ? "rounded bg-background-900 dark:bg-neutral-900 text-text-100 dark:text-neutral-100 underline"
-                : " hover:underline bg-background-100 dark:bg-neutral-700"
+                ? "rounded bg-background-900 text-text-100 underline"
+                : " hover:underline bg-background-100"
             }`}
           >
             Cloud-based
@@ -258,8 +258,8 @@ export function EmbeddingModelSelection({
             onClick={() => setModelTab("open")}
             className={` mx-2 p-2 font-bold  ${
               modelTab == "open"
-                ? "rounded bg-background-900 dark:bg-neutral-900 text-text-100 dark:text-neutral-100 underline"
-                : "hover:underline bg-background-100 dark:bg-neutral-700"
+                ? "rounded bg-background-900 text-text-100 underline"
+                : "hover:underline bg-background-100"
             }`}
           >
             Self-hosted

web/src/app/admin/embeddings/RerankingFormPage.tsx

@@ -177,8 +177,8 @@ const RerankingDetailsForm = forwardRef<
                         key={`${card.rerank_provider_type}-${card.modelName}`}
                         className={`p-4 border rounded-lg cursor-pointer transition-all duration-200 ${
                           isSelected
-                            ? "border-blue-500 bg-blue-50 dark:bg-blue-900 dark:border-blue-700 shadow-md"
-                            : "border-background-200 hover:border-blue-300 hover:shadow-sm dark:border-neutral-700 dark:hover:border-blue-300"
+                            ? "border-blue-500 bg-blue-50 shadow-md"
+                            : "border-gray-200 hover:border-blue-300 hover:shadow-sm"
                         }`}
                         onClick={() => {
                           if (
@@ -240,10 +240,10 @@ const RerankingDetailsForm = forwardRef<
                             </a>
                           )}
                         </div>
-                        <p className="text-sm text-text-600 mb-2">
+                        <p className="text-sm text-gray-600 mb-2">
                           {card.description}
                         </p>
-                        <div className="text-xs text-text-500">
+                        <div className="text-xs text-gray-500">
                           {card.cloud ? "Cloud-based" : "Self-hosted"}
                         </div>
                       </div>

web/src/app/admin/embeddings/pages/CloudEmbeddingPage.tsx

@@ -210,12 +210,12 @@ export default function CloudEmbeddingPage({
             )}
 
             {!liteLLMProvider && (
-              <CardSection className="mt-2 w-full max-w-4xl bg-background-50 border border-background-200">
+              <CardSection className="mt-2 w-full max-w-4xl bg-gray-50 border border-gray-200">
                 <div className="p-4">
                   <Text className="text-lg font-semibold mb-2">
                     API URL Required
                   </Text>
-                  <Text className="text-sm text-text-600 mb-4">
+                  <Text className="text-sm text-gray-600 mb-4">
                     Before you can add models, you need to provide an API URL
                     for your LiteLLM proxy. Click the &quot;Provide API
                     URL&quot; button above to set up your LiteLLM configuration.
@@ -313,16 +313,16 @@ export default function CloudEmbeddingPage({
                 Configure Azure OpenAI
               </button>
               <div className="mt-2 w-full max-w-4xl">
-                <CardSection className="p-4 border border-background-200 rounded-lg shadow-sm">
+                <CardSection className="p-4 border border-gray-200 rounded-lg shadow-sm">
                   <Text className="text-base font-medium mb-2">
                     Configure Azure OpenAI for Embeddings
                   </Text>
-                  <Text className="text-sm text-text-600 mb-3">
+                  <Text className="text-sm text-gray-600 mb-3">
                     Click &quot;Configure Azure OpenAI&quot; to set up Azure
                     OpenAI for embeddings.
                   </Text>
-                  <div className="flex items-center text-sm text-text-700">
-                    <FiInfo className="text-text-400 mr-2" size={16} />
+                  <div className="flex items-center text-sm text-gray-700">
+                    <FiInfo className="text-gray-400 mr-2" size={16} />
                     <Text>
                       You&apos;ll need: API version, base URL, API key, model
                       name, and deployment name.
@@ -339,7 +339,7 @@ export default function CloudEmbeddingPage({
                 </Text>
 
                 {azureProviderDetails ? (
-                  <CardSection className="bg-white shadow-sm border border-background-200 rounded-lg">
+                  <CardSection className="bg-white shadow-sm border border-gray-200 rounded-lg">
                     <div className="p-4 space-y-3">
                       <div className="flex justify-between">
                         <span className="font-medium">API Version:</span>
@@ -364,8 +364,8 @@ export default function CloudEmbeddingPage({
                     </button>
                   </CardSection>
                 ) : (
-                  <CardSection className="bg-background-50 border border-background-200 rounded-lg">
-                    <div className="p-4 text-text-500 text-center">
+                  <CardSection className="bg-gray-50 border border-gray-200 rounded-lg">
+                    <div className="p-4 text-gray-500 text-center">
                       No Azure provider has been configured yet.
                     </div>
                   </CardSection>
@@ -450,8 +450,8 @@ export function CloudModelCard({
     <div
       className={`p-4 w-96 border rounded-lg transition-all duration-200 ${
         enabled
-          ? "border-blue-500 bg-blue-50 dark:bg-blue-950 shadow-md"
-          : "border-background-300 hover:border-blue-300 hover:shadow-sm"
+          ? "border-blue-500 bg-blue-50 shadow-md"
+          : "border-gray-300 hover:border-blue-300 hover:shadow-sm"
       } ${!provider.configured && "opacity-80 hover:opacity-100"}`}
     >
       {popup}
@@ -465,9 +465,7 @@ export function CloudModelCard({
       )}
 
       <div className="flex items-center justify-between mb-3">
-        <h3 className="font-bold dark:text-neutral-100 text-lg">
-          {model.model_name}
-        </h3>
+        <h3 className="font-bold text-lg">{model.model_name}</h3>
         <div className="flex gap-x-2">
           {model.provider_type == EmbeddingProvider.LITELLM.toLowerCase() && (
             <button
@@ -489,12 +487,10 @@ export function CloudModelCard({
           </a>
         </div>
       </div>
-      <p className="text-sm text-text-600 dark:text-neutral-400 mb-2">
-        {model.description}
-      </p>
+      <p className="text-sm text-gray-600 mb-2">{model.description}</p>
       {model?.provider_type?.toLowerCase() !=
         EmbeddingProvider.LITELLM.toLowerCase() && (
-        <div className="text-xs text-text-500 mb-2">
+        <div className="text-xs text-gray-500 mb-2">
           ${model.pricePerMillion}/M tokens
         </div>
       )}
@@ -503,7 +499,7 @@ export function CloudModelCard({
           className={`w-full p-2 rounded-lg text-sm ${
             enabled
               ? "bg-background-125 border border-border cursor-not-allowed"
-              : "bg-background border border-border hover:bg-accent-background-hovered cursor-pointer"
+              : "bg-background border border-border hover:bg-hover cursor-pointer"
           }`}
           onClick={() => {
             if (enabled) {

web/src/app/admin/embeddings/pages/EmbeddingFormPage.tsx

@@ -187,7 +187,7 @@ export default function EmbeddingForm() {
       return needsReIndex ? (
         <div className="flex mx-auto gap-x-1 ml-auto items-center">
           <button
-            className="enabled:cursor-pointer disabled:bg-accent/50 disabled:cursor-not-allowed bg-agent flex gap-x-1 items-center text-white py-2.5 px-3.5 text-sm font-regular rounded-sm"
+            className="enabled:cursor-pointer disabled:bg-accent/50 disabled:cursor-not-allowed bg-accent flex gap-x-1 items-center text-white py-2.5 px-3.5 text-sm font-regular rounded-sm"
             onClick={handleReindex}
           >
             Re-index
@@ -214,7 +214,7 @@ export default function EmbeddingForm() {
         </div>
       ) : (
         <button
-          className="enabled:cursor-pointer ml-auto disabled:bg-accent/50 disabled:cursor-not-allowed bg-agent flex mx-auto gap-x-1 items-center text-white py-2.5 px-3.5 text-sm font-regular rounded-sm"
+          className="enabled:cursor-pointer ml-auto disabled:bg-accent/50 disabled:cursor-not-allowed bg-accent flex mx-auto gap-x-1 items-center text-white py-2.5 px-3.5 text-sm font-regular rounded-sm"
           onClick={async () => {
             updateSearch();
             navigateToEmbeddingPage("search settings");

web/src/app/admin/indexing/[id]/IndexAttemptErrorsTable.tsx

@@ -52,13 +52,13 @@ export function CustomModal({
                   setCopyClicked(true);
                   setTimeout(() => setCopyClicked(false), 2000);
                 }}
-                className="flex w-fit cursor-pointer hover:bg-accent-background p-2 border-border border rounded"
+                className="flex w-fit cursor-pointer hover:bg-hover-light p-2 border-border border rounded"
               >
                 Copy full content
                 <CopyIcon className="ml-2 my-auto" />
               </div>
             ) : (
-              <div className="flex w-fit hover:bg-accent-background p-2 border-border border rounded cursor-default">
+              <div className="flex w-fit hover:bg-hover-light p-2 border-border border rounded cursor-default">
                 Copied to clipboard
                 <CheckmarkIcon
                   className="my-auto ml-2 flex flex-shrink-0 text-success"