StreamWriter w/o default values

README.md

@@ -124,7 +124,7 @@ There are two editions of Onyx:
 To try the Onyx Enterprise Edition:
 
 1. Checkout our [Cloud product](https://cloud.onyx.app/signup).
-2. For self-hosting, contact us at [founders@onyx.app](mailto:founders@onyx.app) or book a call with us on our [Cal](https://cal.com/team/onyx/founders).
+2. For self-hosting, contact us at [founders@onyx.app](mailto:founders@onyx.app) or book a call with us on our [Cal](https://cal.com/team/danswer/founders).
 
 ## 💡 Contributing
 

backend/alembic/versions/eaa3b5593925_add_default_slack_channel_config.py

@@ -1,76 +0,0 @@
-"""add default slack channel config
-
-Revision ID: eaa3b5593925
-Revises: 98a5008d8711
-Create Date: 2025-02-03 18:07:56.552526
-
-"""
-from alembic import op
-import sqlalchemy as sa
-
-
-# revision identifiers, used by Alembic.
-revision = "eaa3b5593925"
-down_revision = "98a5008d8711"
-branch_labels = None
-depends_on = None
-
-
-def upgrade() -> None:
-    # Add is_default column
-    op.add_column(
-        "slack_channel_config",
-        sa.Column("is_default", sa.Boolean(), nullable=False, server_default="false"),
-    )
-
-    op.create_index(
-        "ix_slack_channel_config_slack_bot_id_default",
-        "slack_channel_config",
-        ["slack_bot_id", "is_default"],
-        unique=True,
-        postgresql_where=sa.text("is_default IS TRUE"),
-    )
-
-    # Create default channel configs for existing slack bots without one
-    conn = op.get_bind()
-    slack_bots = conn.execute(sa.text("SELECT id FROM slack_bot")).fetchall()
-
-    for slack_bot in slack_bots:
-        slack_bot_id = slack_bot[0]
-        existing_default = conn.execute(
-            sa.text(
-                "SELECT id FROM slack_channel_config WHERE slack_bot_id = :bot_id AND is_default = TRUE"
-            ),
-            {"bot_id": slack_bot_id},
-        ).fetchone()
-
-        if not existing_default:
-            conn.execute(
-                sa.text(
-                    """
-                    INSERT INTO slack_channel_config (
-                        slack_bot_id, persona_id, channel_config, enable_auto_filters, is_default
-                    ) VALUES (
-                        :bot_id, NULL,
-                        '{"channel_name": null, "respond_member_group_list": [], "answer_filters": [], "follow_up_tags": []}',
-                        FALSE, TRUE
-                    )
-                """
-                ),
-                {"bot_id": slack_bot_id},
-            )
-
-
-def downgrade() -> None:
-    # Delete default slack channel configs
-    conn = op.get_bind()
-    conn.execute(sa.text("DELETE FROM slack_channel_config WHERE is_default = TRUE"))
-
-    # Remove index
-    op.drop_index(
-        "ix_slack_channel_config_slack_bot_id_default",
-        table_name="slack_channel_config",
-    )
-
-    # Remove is_default column
-    op.drop_column("slack_channel_config", "is_default")

backend/ee/onyx/onyxbot/slack/handlers/handle_standard_answers.py

@@ -80,7 +80,7 @@ def oneoff_standard_answers(
 def _handle_standard_answers(
     message_info: SlackMessageInfo,
     receiver_ids: list[str] | None,
-    slack_channel_config: SlackChannelConfig,
+    slack_channel_config: SlackChannelConfig | None,
     prompt: Prompt | None,
     logger: OnyxLoggingAdapter,
     client: WebClient,
@@ -94,10 +94,13 @@ def _handle_standard_answers(
     Returns True if standard answers are found to match the user's message and therefore,
     we still need to respond to the users.
     """
+    # if no channel config, then no standard answers are configured
+    if not slack_channel_config:
+        return False
 
     slack_thread_id = message_info.thread_to_respond
     configured_standard_answer_categories = (
-        slack_channel_config.standard_answer_categories
+        slack_channel_config.standard_answer_categories if slack_channel_config else []
     )
     configured_standard_answers = set(
         [

backend/ee/onyx/server/middleware/tenant_tracking.py

@@ -10,7 +10,6 @@ from fastapi import Response
 from ee.onyx.auth.users import decode_anonymous_user_jwt_token
 from ee.onyx.configs.app_configs import ANONYMOUS_USER_COOKIE_NAME
 from onyx.auth.api_key import extract_tenant_from_api_key_header
-from onyx.configs.constants import TENANT_ID_COOKIE_NAME
 from onyx.db.engine import is_valid_schema_name
 from onyx.redis.redis_pool import retrieve_auth_token_data_from_redis
 from shared_configs.configs import MULTI_TENANT
@@ -44,7 +43,6 @@ async def _get_tenant_id_from_request(
     Attempt to extract tenant_id from:
     1) The API key header
     2) The Redis-based token (stored in Cookie: fastapiusersauth)
-    3)  Reset token cookie
     Fallback: POSTGRES_DEFAULT_SCHEMA
     """
     # Check for API key
@@ -92,12 +90,3 @@ async def _get_tenant_id_from_request(
     except Exception as e:
         logger.error(f"Unexpected error in _get_tenant_id_from_request: {str(e)}")
         raise HTTPException(status_code=500, detail="Internal server error")
-
-    finally:
-        # As a final step, check for explicit tenant_id cookie
-        tenant_id_cookie = request.cookies.get(TENANT_ID_COOKIE_NAME)
-        if tenant_id_cookie and is_valid_schema_name(tenant_id_cookie):
-            return tenant_id_cookie
-
-        # If we've reached this point, return the default schema
-        return POSTGRES_DEFAULT_SCHEMA

backend/ee/onyx/server/oauth.py

@@ -286,7 +286,6 @@ def prepare_authorization_request(
     oauth_state = (
         base64.urlsafe_b64encode(oauth_uuid.bytes).rstrip(b"=").decode("utf-8")
     )
-    session: str
 
     if connector == DocumentSource.SLACK:
         oauth_url = SlackOAuth.generate_oauth_url(oauth_state)
@@ -555,7 +554,6 @@ def handle_google_drive_oauth_callback(
         )
 
     session_json = session_json_bytes.decode("utf-8")
-    session: GoogleDriveOAuth.OAuthSession
     try:
         session = GoogleDriveOAuth.parse_session(session_json)
 

backend/ee/onyx/server/tenants/api.py

@@ -34,7 +34,6 @@ from onyx.auth.users import get_redis_strategy
 from onyx.auth.users import optional_user
 from onyx.auth.users import User
 from onyx.configs.app_configs import WEB_DOMAIN
-from onyx.configs.constants import FASTAPI_USERS_AUTH_COOKIE_NAME
 from onyx.db.auth import get_user_count
 from onyx.db.engine import get_current_tenant_id
 from onyx.db.engine import get_session
@@ -112,7 +111,7 @@ async def login_as_anonymous_user(
     token = generate_anonymous_user_jwt_token(tenant_id)
 
     response = Response()
-    response.delete_cookie(FASTAPI_USERS_AUTH_COOKIE_NAME)
+    response.delete_cookie("fastapiusersauth")
     response.set_cookie(
         key=ANONYMOUS_USER_COOKIE_NAME,
         value=token,

backend/onyx/agents/agent_search/basic/utils.py

@@ -7,7 +7,6 @@ from langgraph.types import StreamWriter
 
 from onyx.agents.agent_search.shared_graph_utils.utils import write_custom_event
 from onyx.chat.models import LlmDoc
-from onyx.chat.models import OnyxContext
 from onyx.chat.stream_processing.answer_response_handler import AnswerResponseHandler
 from onyx.chat.stream_processing.answer_response_handler import CitationResponseHandler
 from onyx.chat.stream_processing.answer_response_handler import (
@@ -24,7 +23,7 @@ def process_llm_stream(
     should_stream_answer: bool,
     writer: StreamWriter,
     final_search_results: list[LlmDoc] | None = None,
-    displayed_search_results: list[OnyxContext] | list[LlmDoc] | None = None,
+    displayed_search_results: list[LlmDoc] | None = None,
 ) -> AIMessageChunk:
     tool_call_chunk = AIMessageChunk(content="")
 

backend/onyx/agents/agent_search/deep_search/initial/generate_individual_sub_answer/nodes/generate_sub_answer.py

@@ -39,7 +39,7 @@ logger = setup_logger()
 def generate_sub_answer(
     state: AnswerQuestionState,
     config: RunnableConfig,
-    writer: StreamWriter = lambda _: None,
+    writer: StreamWriter,
 ) -> SubQuestionAnswerGenerationUpdate:
     """
     LangGraph node to generate a sub-answer.

backend/onyx/agents/agent_search/deep_search/initial/generate_initial_answer/nodes/generate_initial_answer.py

@@ -61,7 +61,7 @@ from onyx.tools.tool_implementations.search.search_tool import yield_search_resp
 def generate_initial_answer(
     state: SubQuestionRetrievalState,
     config: RunnableConfig,
-    writer: StreamWriter = lambda _: None,
+    writer: StreamWriter,
 ) -> InitialAnswerUpdate:
     """
     LangGraph node to generate the initial answer, using the initial sub-questions/sub-answers and the

backend/onyx/agents/agent_search/deep_search/initial/generate_sub_answers/nodes/decompose_orig_question.py

@@ -46,7 +46,7 @@ logger = setup_logger()
 def decompose_orig_question(
     state: SubQuestionRetrievalState,
     config: RunnableConfig,
-    writer: StreamWriter = lambda _: None,
+    writer: StreamWriter,
 ) -> InitialQuestionDecompositionUpdate:
     """
     LangGraph node to decompose the original question into sub-questions.

backend/onyx/agents/agent_search/deep_search/shared/expanded_retrieval/nodes/expand_queries.py

@@ -29,7 +29,7 @@ from onyx.prompts.agent_search import (
 def expand_queries(
     state: ExpandedRetrievalInput,
     config: RunnableConfig,
-    writer: StreamWriter = lambda _: None,
+    writer: StreamWriter,
 ) -> QueryExpansionUpdate:
     """
     LangGraph node to expand a question into multiple search queries.

backend/onyx/agents/agent_search/deep_search/shared/expanded_retrieval/nodes/format_results.py

@@ -28,7 +28,7 @@ from onyx.tools.tool_implementations.search.search_tool import yield_search_resp
 def format_results(
     state: ExpandedRetrievalState,
     config: RunnableConfig,
-    writer: StreamWriter = lambda _: None,
+    writer: StreamWriter,
 ) -> ExpandedRetrievalUpdate:
     """
     LangGraph node that constructs the proper expanded retrieval format.

backend/onyx/agents/agent_search/orchestration/nodes/basic_use_tool_response.py

@@ -9,7 +9,6 @@ from onyx.agents.agent_search.basic.states import BasicState
 from onyx.agents.agent_search.basic.utils import process_llm_stream
 from onyx.agents.agent_search.models import GraphConfig
 from onyx.chat.models import LlmDoc
-from onyx.chat.models import OnyxContexts
 from onyx.tools.tool_implementations.search.search_tool import (
     SEARCH_DOC_CONTENT_ID,
 )
@@ -51,11 +50,13 @@ def basic_use_tool_response(
         if yield_item.id == FINAL_CONTEXT_DOCUMENTS_ID:
             final_search_results = cast(list[LlmDoc], yield_item.response)
         elif yield_item.id == SEARCH_DOC_CONTENT_ID:
-            search_contexts = cast(OnyxContexts, yield_item.response).contexts
+            search_contexts = yield_item.response.contexts
             for doc in search_contexts:
                 if doc.document_id not in initial_search_results:
                     initial_search_results.append(doc)
 
+            initial_search_results = cast(list[LlmDoc], initial_search_results)
+
     new_tool_call_chunk = AIMessageChunk(content="")
     if not agent_config.behavior.skip_gen_ai_answer_generation:
         stream = llm.stream(
@@ -69,9 +70,7 @@ def basic_use_tool_response(
             True,
             writer,
             final_search_results=final_search_results,
-            # when the search tool is called with specific doc ids, initial search
-            # results are not output. But, we still want i.e. citations to be processed.
-            displayed_search_results=initial_search_results or final_search_results,
+            displayed_search_results=initial_search_results,
         )
 
     return BasicOutput(tool_call_chunk=new_tool_call_chunk)

backend/onyx/agents/agent_search/orchestration/nodes/llm_tool_choice.py

@@ -26,9 +26,7 @@ logger = setup_logger()
 # from the state and config
 # TODO: fan-out to multiple tool call nodes? Make this configurable?
 def llm_tool_choice(
-    state: ToolChoiceState,
-    config: RunnableConfig,
-    writer: StreamWriter = lambda _: None,
+    state: ToolChoiceState, config: RunnableConfig, writer: StreamWriter
 ) -> ToolChoiceUpdate:
     """
     This node is responsible for calling the LLM to choose a tool. If no tool is chosen,

backend/onyx/agents/agent_search/orchestration/nodes/tool_call.py

@@ -20,10 +20,6 @@ from onyx.utils.logger import setup_logger
 logger = setup_logger()
 
 
-class ToolCallException(Exception):
-    """Exception raised for errors during tool calls."""
-
-
 def emit_packet(packet: AnswerPacket, writer: StreamWriter) -> None:
     write_custom_event("basic_response", packet, writer)
 
@@ -31,7 +27,7 @@ def emit_packet(packet: AnswerPacket, writer: StreamWriter) -> None:
 def tool_call(
     state: ToolChoiceUpdate,
     config: RunnableConfig,
-    writer: StreamWriter = lambda _: None,
+    writer: StreamWriter,
 ) -> ToolCallUpdate:
     """Calls the tool specified in the state and updates the state with the result"""
 
@@ -49,18 +45,13 @@ def tool_call(
 
     emit_packet(tool_kickoff, writer)
 
-    try:
-        tool_responses = []
-        for response in tool_runner.tool_responses():
-            tool_responses.append(response)
-            emit_packet(response, writer)
+    tool_responses = []
+    for response in tool_runner.tool_responses():
+        tool_responses.append(response)
+        emit_packet(response, writer)
 
-        tool_final_result = tool_runner.tool_final_result()
-        emit_packet(tool_final_result, writer)
-    except Exception as e:
-        raise ToolCallException(
-            f"Error during tool call for {tool.display_name}: {e}"
-        ) from e
+    tool_final_result = tool_runner.tool_final_result()
+    emit_packet(tool_final_result, writer)
 
     tool_call = ToolCall(name=tool.name, args=tool_args, id=tool_id)
     tool_call_summary = ToolCallSummary(

backend/onyx/agents/agent_search/shared_graph_utils/models.py

@@ -12,20 +12,6 @@ from onyx.context.search.models import InferenceSection
 from onyx.tools.models import SearchQueryInfo
 
 
-# Pydantic models for structured outputs
-# class RewrittenQueries(BaseModel):
-#     rewritten_queries: list[str]
-
-
-# class BinaryDecision(BaseModel):
-#     decision: Literal["yes", "no"]
-
-
-# class BinaryDecisionWithReasoning(BaseModel):
-#     reasoning: str
-#     decision: Literal["yes", "no"]
-
-
 class RetrievalFitScoreMetrics(BaseModel):
     scores: dict[str, float]
     chunk_ids: list[str]

backend/onyx/auth/email_utils.py

@@ -10,7 +10,6 @@ from onyx.configs.app_configs import SMTP_PORT
 from onyx.configs.app_configs import SMTP_SERVER
 from onyx.configs.app_configs import SMTP_USER
 from onyx.configs.app_configs import WEB_DOMAIN
-from onyx.configs.constants import TENANT_ID_COOKIE_NAME
 from onyx.db.models import User
 
 
@@ -66,13 +65,9 @@ def send_forgot_password_email(
     user_email: str,
     token: str,
     mail_from: str = EMAIL_FROM,
-    tenant_id: str | None = None,
 ) -> None:
     subject = "Onyx Forgot Password"
     link = f"{WEB_DOMAIN}/auth/reset-password?token={token}"
-    if tenant_id:
-        link += f"&{TENANT_ID_COOKIE_NAME}={tenant_id}"
-        # Keep search param same name as cookie for simplicity
     body = f"Click the following link to reset your password: {link}"
     send_email(user_email, subject, body, mail_from)
 

backend/onyx/auth/users.py

@@ -73,7 +73,6 @@ from onyx.configs.app_configs import WEB_DOMAIN
 from onyx.configs.constants import AuthType
 from onyx.configs.constants import DANSWER_API_KEY_DUMMY_EMAIL_DOMAIN
 from onyx.configs.constants import DANSWER_API_KEY_PREFIX
-from onyx.configs.constants import FASTAPI_USERS_AUTH_COOKIE_NAME
 from onyx.configs.constants import MilestoneRecordType
 from onyx.configs.constants import OnyxRedisLocks
 from onyx.configs.constants import PASSWORD_SPECIAL_CHARS
@@ -219,24 +218,6 @@ class UserManager(UUIDIDMixin, BaseUserManager[User, uuid.UUID]):
     verification_token_lifetime_seconds = AUTH_COOKIE_EXPIRE_TIME_SECONDS
     user_db: SQLAlchemyUserDatabase[User, uuid.UUID]
 
-    async def get_by_email(self, user_email: str) -> User:
-        tenant_id = fetch_ee_implementation_or_noop(
-            "onyx.server.tenants.user_mapping", "get_tenant_id_for_email", None
-        )(user_email)
-        async with get_async_session_with_tenant(tenant_id) as db_session:
-            if MULTI_TENANT:
-                tenant_user_db = SQLAlchemyUserAdminDB[User, uuid.UUID](
-                    db_session, User, OAuthAccount
-                )
-                user = await tenant_user_db.get_by_email(user_email)
-            else:
-                user = await self.user_db.get_by_email(user_email)
-
-        if not user:
-            raise exceptions.UserNotExists()
-
-        return user
-
     async def create(
         self,
         user_create: schemas.UC | UserCreate,
@@ -264,8 +245,6 @@ class UserManager(UUIDIDMixin, BaseUserManager[User, uuid.UUID]):
             referral_source=referral_source,
             request=request,
         )
-        user: User
-
         async with get_async_session_with_tenant(tenant_id) as db_session:
             token = CURRENT_TENANT_ID_CONTEXTVAR.set(tenant_id)
             verify_email_is_invited(user_create.email)
@@ -389,8 +368,6 @@ class UserManager(UUIDIDMixin, BaseUserManager[User, uuid.UUID]):
                 "refresh_token": refresh_token,
             }
 
-            user: User
-
             try:
                 # Attempt to get user by OAuth account
                 user = await self.get_by_oauth_account(oauth_name, account_id)
@@ -523,15 +500,9 @@ class UserManager(UUIDIDMixin, BaseUserManager[User, uuid.UUID]):
             )
             raise HTTPException(
                 status.HTTP_500_INTERNAL_SERVER_ERROR,
-                "Your admin has not enabled this feature.",
+                "Your admin has not enbaled this feature.",
             )
-        tenant_id = await fetch_ee_implementation_or_noop(
-            "onyx.server.tenants.provisioning",
-            "get_or_provision_tenant",
-            async_return_default_schema,
-        )(email=user.email)
-
-        send_forgot_password_email(user.email, token, tenant_id=tenant_id)
+        send_forgot_password_email(user.email, token)
 
     async def on_after_request_verify(
         self, user: User, token: str, request: Optional[Request] = None
@@ -605,7 +576,6 @@ async def get_user_manager(
 cookie_transport = CookieTransport(
     cookie_max_age=SESSION_EXPIRE_TIME_SECONDS,
     cookie_secure=WEB_DOMAIN.startswith("https"),
-    cookie_name=FASTAPI_USERS_AUTH_COOKIE_NAME,
 )
 
 
@@ -1073,8 +1043,6 @@ async def api_key_dep(
     if AUTH_TYPE == AuthType.DISABLED:
         return None
 
-    user: User | None = None
-
     hashed_api_key = get_hashed_api_key_from_request(request)
     if not hashed_api_key:
         raise HTTPException(status_code=401, detail="Missing API key")

backend/onyx/background/celery/apps/primary.py

@@ -21,16 +21,13 @@ from onyx.background.celery.tasks.indexing.utils import (
     get_unfenced_index_attempt_ids,
 )
 from onyx.configs.constants import CELERY_PRIMARY_WORKER_LOCK_TIMEOUT
-from onyx.configs.constants import OnyxRedisConstants
 from onyx.configs.constants import OnyxRedisLocks
 from onyx.configs.constants import POSTGRES_CELERY_WORKER_PRIMARY_APP_NAME
 from onyx.db.engine import get_session_with_default_tenant
 from onyx.db.engine import SqlEngine
 from onyx.db.index_attempt import get_index_attempt
 from onyx.db.index_attempt import mark_attempt_canceled
-from onyx.redis.redis_connector_credential_pair import (
-    RedisGlobalConnectorCredentialPair,
-)
+from onyx.redis.redis_connector_credential_pair import RedisConnectorCredentialPair
 from onyx.redis.redis_connector_delete import RedisConnectorDelete
 from onyx.redis.redis_connector_doc_perm_sync import RedisConnectorPermissionSync
 from onyx.redis.redis_connector_ext_group_sync import RedisConnectorExternalGroupSync
@@ -144,16 +141,23 @@ def on_worker_init(sender: Worker, **kwargs: Any) -> None:
     r.delete(OnyxRedisLocks.CHECK_VESPA_SYNC_BEAT_LOCK)
     r.delete(OnyxRedisLocks.MONITOR_VESPA_SYNC_BEAT_LOCK)
 
-    r.delete(OnyxRedisConstants.ACTIVE_FENCES)
+    r.delete(RedisConnectorCredentialPair.get_taskset_key())
+    r.delete(RedisConnectorCredentialPair.get_fence_key())
 
-    RedisGlobalConnectorCredentialPair.reset_all(r)
     RedisDocumentSet.reset_all(r)
+
     RedisUserGroup.reset_all(r)
+
     RedisConnectorDelete.reset_all(r)
+
     RedisConnectorPrune.reset_all(r)
+
     RedisConnectorIndex.reset_all(r)
+
     RedisConnectorStop.reset_all(r)
+
     RedisConnectorPermissionSync.reset_all(r)
+
     RedisConnectorExternalGroupSync.reset_all(r)
 
     # mark orphaned index attempts as failed

backend/onyx/background/celery/tasks/beat_schedule.py

@@ -18,7 +18,7 @@ BEAT_EXPIRES_DEFAULT = 15 * 60  # 15 minutes (in seconds)
 
 # hack to slow down task dispatch in the cloud until
 # we have a better implementation (backpressure, etc)
-CLOUD_BEAT_SCHEDULE_MULTIPLIER = 4
+CLOUD_BEAT_SCHEDULE_MULTIPLIER = 8
 
 # tasks that only run in the cloud
 # the name attribute must start with ONYX_CLOUD_CELERY_TASK_PREFIX = "cloud" to be filtered

backend/onyx/background/celery/tasks/doc_permission_syncing/tasks.py

@@ -3,7 +3,6 @@ from datetime import datetime
 from datetime import timedelta
 from datetime import timezone
 from time import sleep
-from typing import Any
 from typing import cast
 from uuid import uuid4
 
@@ -39,7 +38,6 @@ from onyx.configs.constants import DocumentSource
 from onyx.configs.constants import OnyxCeleryPriority
 from onyx.configs.constants import OnyxCeleryQueues
 from onyx.configs.constants import OnyxCeleryTask
-from onyx.configs.constants import OnyxRedisConstants
 from onyx.configs.constants import OnyxRedisLocks
 from onyx.configs.constants import OnyxRedisSignals
 from onyx.db.connector import mark_cc_pair_as_permissions_synced
@@ -59,8 +57,8 @@ from onyx.redis.redis_connector import RedisConnector
 from onyx.redis.redis_connector_doc_perm_sync import RedisConnectorPermissionSync
 from onyx.redis.redis_connector_doc_perm_sync import RedisConnectorPermissionSyncPayload
 from onyx.redis.redis_pool import get_redis_client
-from onyx.redis.redis_pool import get_redis_replica_client
 from onyx.redis.redis_pool import redis_lock_dump
+from onyx.redis.redis_pool import SCAN_ITER_COUNT_DEFAULT
 from onyx.server.utils import make_short_id
 from onyx.utils.logger import doc_permission_sync_ctx
 from onyx.utils.logger import LoggerContextVars
@@ -125,7 +123,6 @@ def check_for_doc_permissions_sync(self: Task, *, tenant_id: str | None) -> bool
     # we need to use celery's redis client to access its redis data
     # (which lives on a different db number)
     r = get_redis_client(tenant_id=tenant_id)
-    r_replica = get_redis_replica_client(tenant_id=tenant_id)
     r_celery: Redis = self.app.broker_connection().channel().client  # type: ignore
 
     lock_beat: RedisLock = r.lock(
@@ -161,20 +158,18 @@ def check_for_doc_permissions_sync(self: Task, *, tenant_id: str | None) -> bool
 
         # we want to run this less frequently than the overall task
         lock_beat.reacquire()
-        if not r.exists(OnyxRedisSignals.BLOCK_VALIDATE_PERMISSION_SYNC_FENCES):
+        if not r.exists(OnyxRedisSignals.VALIDATE_PERMISSION_SYNC_FENCES):
             # clear any permission fences that don't have associated celery tasks in progress
             # tasks can be in the queue in redis, in reserved tasks (prefetched by the worker),
             # or be currently executing
             try:
-                validate_permission_sync_fences(
-                    tenant_id, r, r_replica, r_celery, lock_beat
-                )
+                validate_permission_sync_fences(tenant_id, r, r_celery, lock_beat)
             except Exception:
                 task_logger.exception(
                     "Exception while validating permission sync fences"
                 )
 
-            r.set(OnyxRedisSignals.BLOCK_VALIDATE_PERMISSION_SYNC_FENCES, 1, ex=300)
+            r.set(OnyxRedisSignals.VALIDATE_PERMISSION_SYNC_FENCES, 1, ex=60)
     except SoftTimeLimitExceeded:
         task_logger.info(
             "Soft time limit exceeded, task is being terminated gracefully."
@@ -491,7 +486,6 @@ def update_external_document_permissions_task(
 def validate_permission_sync_fences(
     tenant_id: str | None,
     r: Redis,
-    r_replica: Redis,
     r_celery: Redis,
     lock_beat: RedisLock,
 ) -> None:
@@ -512,15 +506,12 @@ def validate_permission_sync_fences(
         OnyxCeleryQueues.CONNECTOR_DOC_PERMISSIONS_SYNC, r_celery
     )
 
-    # validate all existing permission sync jobs
-    lock_beat.reacquire()
-    keys = cast(set[Any], r_replica.smembers(OnyxRedisConstants.ACTIVE_FENCES))
-    for key in keys:
-        key_bytes = cast(bytes, key)
-        key_str = key_bytes.decode("utf-8")
-        if not key_str.startswith(RedisConnectorPermissionSync.FENCE_PREFIX):
-            continue
-
+    # validate all existing indexing jobs
+    for key_bytes in r.scan_iter(
+        RedisConnectorPermissionSync.FENCE_PREFIX + "*",
+        count=SCAN_ITER_COUNT_DEFAULT,
+    ):
+        lock_beat.reacquire()
         validate_permission_sync_fence(
             tenant_id,
             key_bytes,
@@ -529,9 +520,6 @@ def validate_permission_sync_fences(
             r,
             r_celery,
         )
-
-        lock_beat.reacquire()
-
     return
 
 
@@ -659,8 +647,7 @@ def validate_permission_sync_fence(
         f"tasks_scanned={tasks_scanned} tasks_not_in_celery={tasks_not_in_celery}"
     )
 
-    # we're only active if tasks_scanned > 0 and tasks_not_in_celery == 0
-    if tasks_scanned > 0 and tasks_not_in_celery == 0:
+    if tasks_not_in_celery == 0:
         redis_connector.permissions.set_active()
         return
 

backend/onyx/background/celery/tasks/indexing/tasks.py

@@ -224,7 +224,7 @@ def check_for_indexing(self: Task, *, tenant_id: str | None) -> int | None:
 
         lock_beat.reacquire()
         # we want to run this less frequently than the overall task
-        if not redis_client.exists(OnyxRedisSignals.BLOCK_VALIDATE_INDEXING_FENCES):
+        if not redis_client.exists(OnyxRedisSignals.VALIDATE_INDEXING_FENCES):
             # clear any indexing fences that don't have associated celery tasks in progress
             # tasks can be in the queue in redis, in reserved tasks (prefetched by the worker),
             # or be currently executing
@@ -235,7 +235,7 @@ def check_for_indexing(self: Task, *, tenant_id: str | None) -> int | None:
             except Exception:
                 task_logger.exception("Exception while validating indexing fences")
 
-            redis_client.set(OnyxRedisSignals.BLOCK_VALIDATE_INDEXING_FENCES, 1, ex=60)
+            redis_client.set(OnyxRedisSignals.VALIDATE_INDEXING_FENCES, 1, ex=60)
     except SoftTimeLimitExceeded:
         task_logger.info(
             "Soft time limit exceeded, task is being terminated gracefully."
@@ -586,12 +586,11 @@ def connector_indexing_proxy_task(
 
         # if the job is done, clean up and break
         if job.done():
-            exit_code: int | None
             try:
                 if job.status == "error":
                     ignore_exitcode = False
 
-                    exit_code = None
+                    exit_code: int | None = None
                     if job.process:
                         exit_code = job.process.exitcode
 

backend/onyx/background/celery/tasks/indexing/utils.py

@@ -1,8 +1,6 @@
 import time
 from datetime import datetime
 from datetime import timezone
-from typing import Any
-from typing import cast
 
 import redis
 from celery import Celery
@@ -21,7 +19,6 @@ from onyx.configs.constants import DocumentSource
 from onyx.configs.constants import OnyxCeleryPriority
 from onyx.configs.constants import OnyxCeleryQueues
 from onyx.configs.constants import OnyxCeleryTask
-from onyx.configs.constants import OnyxRedisConstants
 from onyx.db.engine import get_db_current_time
 from onyx.db.engine import get_session_with_tenant
 from onyx.db.enums import ConnectorCredentialPairStatus
@@ -40,6 +37,7 @@ from onyx.redis.redis_connector import RedisConnector
 from onyx.redis.redis_connector_index import RedisConnectorIndex
 from onyx.redis.redis_connector_index import RedisConnectorIndexPayload
 from onyx.redis.redis_pool import redis_lock_dump
+from onyx.redis.redis_pool import SCAN_ITER_COUNT_DEFAULT
 from onyx.utils.logger import setup_logger
 
 logger = setup_logger()
@@ -306,13 +304,10 @@ def validate_indexing_fences(
 
     # Use replica for this because the worst thing that happens
     # is that we don't run the validation on this pass
-    keys = cast(set[Any], r_replica.smembers(OnyxRedisConstants.ACTIVE_FENCES))
-    for key in keys:
-        key_bytes = cast(bytes, key)
-        key_str = key_bytes.decode("utf-8")
-        if not key_str.startswith(RedisConnectorIndex.FENCE_PREFIX):
-            continue
-
+    for key_bytes in r_replica.scan_iter(
+        RedisConnectorIndex.FENCE_PREFIX + "*", count=SCAN_ITER_COUNT_DEFAULT
+    ):
+        lock_beat.reacquire()
         with get_session_with_tenant(tenant_id) as db_session:
             validate_indexing_fence(
                 tenant_id,
@@ -321,9 +316,6 @@ def validate_indexing_fences(
                 r_celery,
                 db_session,
             )
-
-        lock_beat.reacquire()
-
     return
 
 
@@ -446,7 +438,6 @@ def try_creating_indexing_task(
     if not acquired:
         return None
 
-    redis_connector_index: RedisConnectorIndex
     try:
         redis_connector = RedisConnector(tenant_id, cc_pair.id)
         redis_connector_index = redis_connector.new_index(search_settings.id)

backend/onyx/background/celery/tasks/monitoring/tasks.py

@@ -728,10 +728,6 @@ def cloud_check_alembic() -> bool | None:
     TODO: have the cloud migration script set an activity signal that this check
     uses to know it doesn't make sense to run a check at the present time.
     """
-
-    # Used as a placeholder if the alembic revision cannot be retrieved
-    ALEMBIC_NULL_REVISION = "000000000000"
-
     time_start = time.monotonic()
 
     redis_client = get_redis_client(tenant_id=ONYX_CLOUD_TENANT_ID)
@@ -747,14 +743,13 @@ def cloud_check_alembic() -> bool | None:
 
     last_lock_time = time.monotonic()
 
-    tenant_to_revision: dict[str, str] = {}
+    tenant_to_revision: dict[str, str | None] = {}
     revision_counts: dict[str, int] = {}
-    out_of_date_tenants: dict[str, str] = {}
+    out_of_date_tenants: dict[str, str | None] = {}
     top_revision: str = ""
-    tenant_ids: list[str] | list[None] = []
 
     try:
-        # map tenant_id to revision (or ALEMBIC_NULL_REVISION if the query fails)
+        # map each tenant_id to its revision
         tenant_ids = get_all_tenant_ids()
         for tenant_id in tenant_ids:
             current_time = time.monotonic()
@@ -766,28 +761,19 @@ def cloud_check_alembic() -> bool | None:
                 continue
 
             with get_session_with_tenant(tenant_id=None) as session:
-                try:
-                    result = session.execute(
-                        text(f'SELECT * FROM "{tenant_id}".alembic_version LIMIT 1')
-                    )
+                result = session.execute(
+                    text(f'SELECT * FROM "{tenant_id}".alembic_version LIMIT 1')
+                )
 
-                    result_scalar: str | None = result.scalar_one_or_none()
-                    if result_scalar is None:
-                        raise ValueError("Alembic version should not be None.")
-
-                    tenant_to_revision[tenant_id] = result_scalar
-                except Exception:
-                    task_logger.warning(f"Tenant {tenant_id} has no revision!")
-                    tenant_to_revision[tenant_id] = ALEMBIC_NULL_REVISION
+                result_scalar: str | None = result.scalar_one_or_none()
+                tenant_to_revision[tenant_id] = result_scalar
 
         # get the total count of each revision
         for k, v in tenant_to_revision.items():
-            revision_counts[v] = revision_counts.get(v, 0) + 1
+            if v is None:
+                continue
 
-        # error if any null revision tenants are found
-        if ALEMBIC_NULL_REVISION in revision_counts:
-            num_null_revisions = revision_counts[ALEMBIC_NULL_REVISION]
-            raise ValueError(f"No revision was found for {num_null_revisions} tenants!")
+            revision_counts[v] = revision_counts.get(v, 0) + 1
 
         # get the revision with the most counts
         sorted_revision_counts = sorted(
@@ -795,24 +781,23 @@ def cloud_check_alembic() -> bool | None:
         )
 
         if len(sorted_revision_counts) == 0:
-            raise ValueError(
+            task_logger.error(
                 f"cloud_check_alembic - No revisions found for {len(tenant_ids)} tenant ids!"
             )
+        else:
+            top_revision, _ = sorted_revision_counts[0]
 
-        top_revision, _ = sorted_revision_counts[0]
+            # build a list of out of date tenants
+            for k, v in tenant_to_revision.items():
+                if v == top_revision:
+                    continue
 
-        # build a list of out of date tenants
-        for k, v in tenant_to_revision.items():
-            if v == top_revision:
-                continue
-
-            out_of_date_tenants[k] = v
+                out_of_date_tenants[k] = v
 
     except SoftTimeLimitExceeded:
         task_logger.info(
             "Soft time limit exceeded, task is being terminated gracefully."
         )
-        raise
     except Exception:
         task_logger.exception("Unexpected exception during cloud alembic check")
         raise
@@ -830,11 +815,6 @@ def cloud_check_alembic() -> bool | None:
             f"num_tenants={len(tenant_ids)} "
             f"revision={top_revision}"
         )
-
-        num_to_log = min(5, len(out_of_date_tenants))
-        task_logger.info(
-            f"Logging {num_to_log}/{len(out_of_date_tenants)} out of date tenants."
-        )
         for k, v in islice(out_of_date_tenants.items(), 5):
             task_logger.info(f"Out of date tenant: tenant={k} revision={v}")
     else:

backend/onyx/background/celery/tasks/shared/tasks.py

@@ -168,7 +168,6 @@ def document_by_cc_pair_cleanup_task(
         task_logger.info(f"SoftTimeLimitExceeded exception. doc={document_id}")
         return False
     except Exception as ex:
-        e: Exception | None = None
         if isinstance(ex, RetryError):
             task_logger.warning(
                 f"Tenacity retry failed: num_attempts={ex.last_attempt.attempt_number}"
@@ -248,7 +247,6 @@ def cloud_beat_task_generator(
         return None
 
     last_lock_time = time.monotonic()
-    tenant_ids: list[str] | list[None] = []
 
     try:
         tenant_ids = get_all_tenant_ids()

backend/onyx/background/celery/tasks/vespa/tasks.py

@@ -36,9 +36,7 @@ from onyx.configs.app_configs import VESPA_SYNC_MAX_TASKS
 from onyx.configs.constants import CELERY_VESPA_SYNC_BEAT_LOCK_TIMEOUT
 from onyx.configs.constants import OnyxCeleryQueues
 from onyx.configs.constants import OnyxCeleryTask
-from onyx.configs.constants import OnyxRedisConstants
 from onyx.configs.constants import OnyxRedisLocks
-from onyx.configs.constants import OnyxRedisSignals
 from onyx.db.connector import fetch_connector_by_id
 from onyx.db.connector_credential_pair import add_deletion_failure_message
 from onyx.db.connector_credential_pair import (
@@ -74,9 +72,6 @@ from onyx.document_index.interfaces import VespaDocumentFields
 from onyx.httpx.httpx_pool import HttpxPool
 from onyx.redis.redis_connector import RedisConnector
 from onyx.redis.redis_connector_credential_pair import RedisConnectorCredentialPair
-from onyx.redis.redis_connector_credential_pair import (
-    RedisGlobalConnectorCredentialPair,
-)
 from onyx.redis.redis_connector_delete import RedisConnectorDelete
 from onyx.redis.redis_connector_doc_perm_sync import RedisConnectorPermissionSync
 from onyx.redis.redis_connector_index import RedisConnectorIndex
@@ -209,12 +204,10 @@ def try_generate_stale_document_sync_tasks(
     tenant_id: str | None,
 ) -> int | None:
     # the fence is up, do nothing
-
-    redis_global_ccpair = RedisGlobalConnectorCredentialPair(r)
-    if redis_global_ccpair.fenced:
+    if r.exists(RedisConnectorCredentialPair.get_fence_key()):
         return None
 
-    redis_global_ccpair.delete_taskset()
+    r.delete(RedisConnectorCredentialPair.get_taskset_key())  # delete the taskset
 
     # add tasks to celery and build up the task set to monitor in redis
     stale_doc_count = count_documents_by_needs_sync(db_session)
@@ -272,7 +265,7 @@ def try_generate_stale_document_sync_tasks(
             f"RedisConnector.generate_tasks finished for all cc_pairs. total_tasks_generated={total_tasks_generated}"
         )
 
-    redis_global_ccpair.set_fence(total_tasks_generated)
+    r.set(RedisConnectorCredentialPair.get_fence_key(), total_tasks_generated)
     return total_tasks_generated
 
 
@@ -423,17 +416,23 @@ def try_generate_user_group_sync_tasks(
 
 
 def monitor_connector_taskset(r: Redis) -> None:
-    redis_global_ccpair = RedisGlobalConnectorCredentialPair(r)
-    initial_count = redis_global_ccpair.payload
-    if initial_count is None:
+    fence_value = r.get(RedisConnectorCredentialPair.get_fence_key())
+    if fence_value is None:
         return
 
-    remaining = redis_global_ccpair.get_remaining()
+    try:
+        initial_count = int(cast(int, fence_value))
+    except ValueError:
+        task_logger.error("The value is not an integer.")
+        return
+
+    count = r.scard(RedisConnectorCredentialPair.get_taskset_key())
     task_logger.info(
-        f"Stale document sync progress: remaining={remaining} initial={initial_count}"
+        f"Stale document sync progress: remaining={count} initial={initial_count}"
     )
-    if remaining == 0:
-        redis_global_ccpair.reset()
+    if count == 0:
+        r.delete(RedisConnectorCredentialPair.get_taskset_key())
+        r.delete(RedisConnectorCredentialPair.get_fence_key())
         task_logger.info(f"Successfully synced stale documents. count={initial_count}")
 
 
@@ -821,6 +820,9 @@ def monitor_vespa_sync(self: Task, tenant_id: str | None) -> bool | None:
 
     time_start = time.monotonic()
 
+    timings: dict[str, Any] = {}
+    timings["start"] = time_start
+
     r = get_redis_client(tenant_id=tenant_id)
 
     # Replica usage notes
@@ -845,7 +847,7 @@ def monitor_vespa_sync(self: Task, tenant_id: str | None) -> bool | None:
 
     try:
         # print current queue lengths
-        time.monotonic()
+        phase_start = time.monotonic()
         # we don't need every tenant polling redis for this info.
         if not MULTI_TENANT or random.randint(1, 10) == 10:
             r_celery = self.app.broker_connection().channel().client  # type: ignore
@@ -887,38 +889,50 @@ def monitor_vespa_sync(self: Task, tenant_id: str | None) -> bool | None:
                 f"external_group_sync={n_external_group_sync} "
                 f"permissions_upsert={n_permissions_upsert} "
             )
+        timings["queues"] = time.monotonic() - phase_start
+        timings["queues_ttl"] = r.ttl(OnyxRedisLocks.MONITOR_VESPA_SYNC_BEAT_LOCK)
 
-        # we want to run this less frequently than the overall task
-        if not r.exists(OnyxRedisSignals.BLOCK_BUILD_FENCE_LOOKUP_TABLE):
-            # build a lookup table of existing fences
-            # this is just a migration concern and should be unnecessary once
-            # lookup tables are rolled out
-            for key_bytes in r_replica.scan_iter(count=SCAN_ITER_COUNT_DEFAULT):
-                if is_fence(key_bytes) and not r.sismember(
-                    OnyxRedisConstants.ACTIVE_FENCES, key_bytes
-                ):
-                    logger.warning(f"Adding {key_bytes} to the lookup table.")
-                    r.sadd(OnyxRedisConstants.ACTIVE_FENCES, key_bytes)
-
-            r.set(OnyxRedisSignals.BLOCK_BUILD_FENCE_LOOKUP_TABLE, 1, ex=300)
-
-        # use a lookup table to find active fences. We still have to verify the fence
-        # exists since it is an optimization and not the source of truth.
-        keys = cast(set[Any], r.smembers(OnyxRedisConstants.ACTIVE_FENCES))
-        for key in keys:
-            key_bytes = cast(bytes, key)
-
-            if not r.exists(key_bytes):
-                r.srem(OnyxRedisConstants.ACTIVE_FENCES, key_bytes)
-                continue
-
-            key_str = key_bytes.decode("utf-8")
-            if key_str == RedisGlobalConnectorCredentialPair.FENCE_KEY:
+        # scan and monitor activity to completion
+        phase_start = time.monotonic()
+        lock_beat.reacquire()
+        if r_replica.exists(RedisConnectorCredentialPair.get_fence_key()):
+            if r.exists(RedisConnectorCredentialPair.get_fence_key()):
                 monitor_connector_taskset(r)
-            elif key_str.startswith(RedisDocumentSet.FENCE_PREFIX):
+        timings["connector"] = time.monotonic() - phase_start
+        timings["connector_ttl"] = r.ttl(OnyxRedisLocks.MONITOR_VESPA_SYNC_BEAT_LOCK)
+
+        phase_start = time.monotonic()
+        lock_beat.reacquire()
+        for key_bytes in r_replica.scan_iter(
+            RedisConnectorDelete.FENCE_PREFIX + "*", count=SCAN_ITER_COUNT_DEFAULT
+        ):
+            if r.exists(key_bytes):
+                monitor_connector_deletion_taskset(tenant_id, key_bytes, r)
+            lock_beat.reacquire()
+
+        timings["connector_deletion"] = time.monotonic() - phase_start
+        timings["connector_deletion_ttl"] = r.ttl(
+            OnyxRedisLocks.MONITOR_VESPA_SYNC_BEAT_LOCK
+        )
+
+        phase_start = time.monotonic()
+        lock_beat.reacquire()
+        for key_bytes in r_replica.scan_iter(
+            RedisDocumentSet.FENCE_PREFIX + "*", count=SCAN_ITER_COUNT_DEFAULT
+        ):
+            if r.exists(key_bytes):
                 with get_session_with_tenant(tenant_id) as db_session:
                     monitor_document_set_taskset(tenant_id, key_bytes, r, db_session)
-            elif key_str.startswith(RedisUserGroup.FENCE_PREFIX):
+            lock_beat.reacquire()
+        timings["documentset"] = time.monotonic() - phase_start
+        timings["documentset_ttl"] = r.ttl(OnyxRedisLocks.MONITOR_VESPA_SYNC_BEAT_LOCK)
+
+        phase_start = time.monotonic()
+        lock_beat.reacquire()
+        for key_bytes in r_replica.scan_iter(
+            RedisUserGroup.FENCE_PREFIX + "*", count=SCAN_ITER_COUNT_DEFAULT
+        ):
+            if r.exists(key_bytes):
                 monitor_usergroup_taskset = (
                     fetch_versioned_implementation_with_fallback(
                         "onyx.background.celery.tasks.vespa.tasks",
@@ -928,21 +942,49 @@ def monitor_vespa_sync(self: Task, tenant_id: str | None) -> bool | None:
                 )
                 with get_session_with_tenant(tenant_id) as db_session:
                     monitor_usergroup_taskset(tenant_id, key_bytes, r, db_session)
-            elif key_str.startswith(RedisConnectorDelete.FENCE_PREFIX):
-                monitor_connector_deletion_taskset(tenant_id, key_bytes, r)
-            elif key_str.startswith(RedisConnectorPrune.FENCE_PREFIX):
+            lock_beat.reacquire()
+        timings["usergroup"] = time.monotonic() - phase_start
+        timings["usergroup_ttl"] = r.ttl(OnyxRedisLocks.MONITOR_VESPA_SYNC_BEAT_LOCK)
+
+        phase_start = time.monotonic()
+        lock_beat.reacquire()
+        for key_bytes in r_replica.scan_iter(
+            RedisConnectorPrune.FENCE_PREFIX + "*", count=SCAN_ITER_COUNT_DEFAULT
+        ):
+            if r.exists(key_bytes):
                 with get_session_with_tenant(tenant_id) as db_session:
                     monitor_ccpair_pruning_taskset(tenant_id, key_bytes, r, db_session)
-            elif key_str.startswith(RedisConnectorIndex.FENCE_PREFIX):
+            lock_beat.reacquire()
+        timings["pruning"] = time.monotonic() - phase_start
+        timings["pruning_ttl"] = r.ttl(OnyxRedisLocks.MONITOR_VESPA_SYNC_BEAT_LOCK)
+
+        phase_start = time.monotonic()
+        lock_beat.reacquire()
+        for key_bytes in r_replica.scan_iter(
+            RedisConnectorIndex.FENCE_PREFIX + "*", count=SCAN_ITER_COUNT_DEFAULT
+        ):
+            if r.exists(key_bytes):
                 with get_session_with_tenant(tenant_id) as db_session:
                     monitor_ccpair_indexing_taskset(tenant_id, key_bytes, r, db_session)
-            elif key_str.startswith(RedisConnectorPermissionSync.FENCE_PREFIX):
+            lock_beat.reacquire()
+        timings["indexing"] = time.monotonic() - phase_start
+        timings["indexing_ttl"] = r.ttl(OnyxRedisLocks.MONITOR_VESPA_SYNC_BEAT_LOCK)
+
+        phase_start = time.monotonic()
+        lock_beat.reacquire()
+        for key_bytes in r_replica.scan_iter(
+            RedisConnectorPermissionSync.FENCE_PREFIX + "*",
+            count=SCAN_ITER_COUNT_DEFAULT,
+        ):
+            if r.exists(key_bytes):
                 with get_session_with_tenant(tenant_id) as db_session:
                     monitor_ccpair_permissions_taskset(
                         tenant_id, key_bytes, r, db_session
                     )
-            else:
-                pass
+            lock_beat.reacquire()
+
+        timings["permissions"] = time.monotonic() - phase_start
+        timings["permissions_ttl"] = r.ttl(OnyxRedisLocks.MONITOR_VESPA_SYNC_BEAT_LOCK)
     except SoftTimeLimitExceeded:
         task_logger.info(
             "Soft time limit exceeded, task is being terminated gracefully."
@@ -957,8 +999,8 @@ def monitor_vespa_sync(self: Task, tenant_id: str | None) -> bool | None:
         else:
             task_logger.error(
                 "monitor_vespa_sync - Lock not owned on completion: "
-                f"tenant={tenant_id}"
-                # f"timings={timings}"
+                f"tenant={tenant_id} "
+                f"timings={timings}"
             )
             redis_lock_dump(lock_beat, r)
 
@@ -1022,6 +1064,15 @@ def vespa_metadata_sync_task(
             # the sync might repeat again later
             mark_document_as_synced(document_id, db_session)
 
+            # this code checks for and removes a per document sync key that is
+            # used to block out the same doc from continualy resyncing
+            # a quick hack that is only needed for production issues
+            # redis_syncing_key = RedisConnectorCredentialPair.make_redis_syncing_key(
+            #     document_id
+            # )
+            # r = get_redis_client(tenant_id=tenant_id)
+            # r.delete(redis_syncing_key)
+
             elapsed = time.monotonic() - start
             task_logger.info(
                 f"doc={document_id} "
@@ -1033,7 +1084,6 @@ def vespa_metadata_sync_task(
         task_logger.info(f"SoftTimeLimitExceeded exception. doc={document_id}")
         return False
     except Exception as ex:
-        e: Exception | None = None
         if isinstance(ex, RetryError):
             task_logger.warning(
                 f"Tenacity retry failed: num_attempts={ex.last_attempt.attempt_number}"
@@ -1064,23 +1114,3 @@ def vespa_metadata_sync_task(
         self.retry(exc=e, countdown=countdown)
 
     return True
-
-
-def is_fence(key_bytes: bytes) -> bool:
-    key_str = key_bytes.decode("utf-8")
-    if key_str == RedisGlobalConnectorCredentialPair.FENCE_KEY:
-        return True
-    if key_str.startswith(RedisDocumentSet.FENCE_PREFIX):
-        return True
-    if key_str.startswith(RedisUserGroup.FENCE_PREFIX):
-        return True
-    if key_str.startswith(RedisConnectorDelete.FENCE_PREFIX):
-        return True
-    if key_str.startswith(RedisConnectorPrune.FENCE_PREFIX):
-        return True
-    if key_str.startswith(RedisConnectorIndex.FENCE_PREFIX):
-        return True
-    if key_str.startswith(RedisConnectorPermissionSync.FENCE_PREFIX):
-        return True
-
-    return False

backend/onyx/background/indexing/run_indexing.py

@@ -11,7 +11,6 @@ from onyx.background.indexing.checkpointing import get_time_windows_for_index_at
 from onyx.background.indexing.tracer import OnyxTracer
 from onyx.configs.app_configs import INDEXING_SIZE_WARNING_THRESHOLD
 from onyx.configs.app_configs import INDEXING_TRACER_INTERVAL
-from onyx.configs.app_configs import LEAVE_CONNECTOR_ACTIVE_ON_INITIALIZATION_FAILURE
 from onyx.configs.app_configs import POLL_CONNECTOR_OFFSET
 from onyx.configs.constants import DocumentSource
 from onyx.configs.constants import MilestoneRecordType
@@ -56,7 +55,6 @@ def _get_connector_runner(
     start_time: datetime,
     end_time: datetime,
     tenant_id: str | None,
-    leave_connector_active: bool = LEAVE_CONNECTOR_ACTIVE_ON_INITIALIZATION_FAILURE,
 ) -> ConnectorRunner:
     """
     NOTE: `start_time` and `end_time` are only used for poll connectors
@@ -78,25 +76,20 @@ def _get_connector_runner(
         )
     except Exception as e:
         logger.exception(f"Unable to instantiate connector due to {e}")
-
         # since we failed to even instantiate the connector, we pause the CCPair since
-        # it will never succeed. Sometimes there are cases where the connector will
-        # intermittently fail to initialize in which case we should pass in
-        # leave_connector_active=True to allow it to continue.
-        # For example, if there is nightly maintenance on a Confluence Server instance,
-        # the connector will fail to initialize every night.
-        if not leave_connector_active:
-            cc_pair = get_connector_credential_pair_from_id(
+        # it will never succeed
+
+        cc_pair = get_connector_credential_pair_from_id(
+            db_session=db_session,
+            cc_pair_id=attempt.connector_credential_pair.id,
+        )
+        if cc_pair and cc_pair.status == ConnectorCredentialPairStatus.ACTIVE:
+            update_connector_credential_pair(
                 db_session=db_session,
-                cc_pair_id=attempt.connector_credential_pair.id,
+                connector_id=attempt.connector_credential_pair.connector.id,
+                credential_id=attempt.connector_credential_pair.credential.id,
+                status=ConnectorCredentialPairStatus.PAUSED,
             )
-            if cc_pair and cc_pair.status == ConnectorCredentialPairStatus.ACTIVE:
-                update_connector_credential_pair(
-                    db_session=db_session,
-                    connector_id=attempt.connector_credential_pair.connector.id,
-                    credential_id=attempt.connector_credential_pair.credential.id,
-                    status=ConnectorCredentialPairStatus.PAUSED,
-                )
         raise e
 
     return ConnectorRunner(
@@ -246,7 +239,6 @@ def _run_indexing(
         callback=callback,
     )
 
-    tracer: OnyxTracer
     if INDEXING_TRACER_INTERVAL > 0:
         logger.debug(f"Memory tracer starting: interval={INDEXING_TRACER_INTERVAL}")
         tracer = OnyxTracer()
@@ -263,8 +255,6 @@ def _run_indexing(
     document_count = 0
     chunk_count = 0
     run_end_dt = None
-    tracer_counter: int
-
     for ind, (window_start, window_end) in enumerate(
         get_time_windows_for_index_attempt(
             last_successful_run=datetime.fromtimestamp(
@@ -275,7 +265,6 @@ def _run_indexing(
     ):
         cc_pair_loop: ConnectorCredentialPair | None = None
         index_attempt_loop: IndexAttempt | None = None
-        tracer_counter = 0
 
         try:
             window_start = max(
@@ -300,6 +289,7 @@ def _run_indexing(
                     tenant_id=tenant_id,
                 )
 
+            tracer_counter = 0
             if INDEXING_TRACER_INTERVAL > 0:
                 tracer.snap()
             for doc_batch in connector_runner.run():

backend/onyx/chat/answer.py

@@ -183,7 +183,6 @@ class Answer:
         citations_by_subquestion: dict[
             SubQuestionKey, list[CitationInfo]
         ] = defaultdict(list)
-        basic_subq_key = SubQuestionKey(level=BASIC_KEY[0], question_num=BASIC_KEY[1])
         for packet in self.processed_streamed_output:
             if isinstance(packet, CitationInfo):
                 if packet.level_question_num is not None and packet.level is not None:
@@ -193,7 +192,7 @@ class Answer:
                         )
                     ].append(packet)
                 elif packet.level is None:
-                    citations_by_subquestion[basic_subq_key].append(packet)
+                    citations_by_subquestion[BASIC_SQ_KEY].append(packet)
         return citations_by_subquestion
 
     def is_cancelled(self) -> bool:

backend/onyx/chat/process_message.py

@@ -7,7 +7,6 @@ from typing import cast
 
 from sqlalchemy.orm import Session
 
-from onyx.agents.agent_search.orchestration.nodes.tool_call import ToolCallException
 from onyx.chat.answer import Answer
 from onyx.chat.chat_utils import create_chat_chain
 from onyx.chat.chat_utils import create_temporary_persona
@@ -88,7 +87,6 @@ from onyx.file_store.utils import save_files
 from onyx.llm.exceptions import GenAIDisabledException
 from onyx.llm.factory import get_llms_for_persona
 from onyx.llm.factory import get_main_llm_from_tuple
-from onyx.llm.interfaces import LLM
 from onyx.llm.models import PreviousMessage
 from onyx.llm.utils import litellm_exception_to_error_msg
 from onyx.natural_language_processing.utils import get_tokenizer
@@ -351,8 +349,7 @@ def stream_chat_message_objects(
     new_msg_req.chunks_above = 0
     new_msg_req.chunks_below = 0
 
-    llm: LLM
-
+    llm = None
     try:
         user_id = user.id if user is not None else None
 
@@ -946,25 +943,19 @@ def stream_chat_message_objects(
         return
 
     except Exception as e:
-        logger.exception(f"Failed to process chat message due to {e}")
+        logger.exception("Failed to process chat message.")
+
         error_msg = str(e)
         stack_trace = traceback.format_exc()
+        if llm:
+            client_error_msg = litellm_exception_to_error_msg(e, llm)
+            if llm.config.api_key and len(llm.config.api_key) > 2:
+                error_msg = error_msg.replace(llm.config.api_key, "[REDACTED_API_KEY]")
+                stack_trace = stack_trace.replace(
+                    llm.config.api_key, "[REDACTED_API_KEY]"
+                )
 
-        if isinstance(e, ToolCallException):
-            yield StreamingError(error=error_msg, stack_trace=stack_trace)
-        else:
-            if llm:
-                client_error_msg = litellm_exception_to_error_msg(e, llm)
-                if llm.config.api_key and len(llm.config.api_key) > 2:
-                    error_msg = error_msg.replace(
-                        llm.config.api_key, "[REDACTED_API_KEY]"
-                    )
-                    stack_trace = stack_trace.replace(
-                        llm.config.api_key, "[REDACTED_API_KEY]"
-                    )
-
-                yield StreamingError(error=client_error_msg, stack_trace=stack_trace)
-
+            yield StreamingError(error=client_error_msg, stack_trace=stack_trace)
         db_session.rollback()
         return
 

backend/onyx/chat/stream_processing/utils.py

@@ -3,7 +3,6 @@ from collections.abc import Sequence
 from pydantic import BaseModel
 
 from onyx.chat.models import LlmDoc
-from onyx.chat.models import OnyxContext
 from onyx.context.search.models import InferenceChunk
 
 
@@ -12,7 +11,7 @@ class DocumentIdOrderMapping(BaseModel):
 
 
 def map_document_id_order(
-    chunks: Sequence[InferenceChunk | LlmDoc | OnyxContext], one_indexed: bool = True
+    chunks: Sequence[InferenceChunk | LlmDoc], one_indexed: bool = True
 ) -> DocumentIdOrderMapping:
     order_mapping = {}
     current = 1 if one_indexed else 0

backend/onyx/configs/app_configs.py

@@ -409,11 +409,6 @@ EXPERIMENTAL_CHECKPOINTING_ENABLED = (
     os.environ.get("EXPERIMENTAL_CHECKPOINTING_ENABLED", "").lower() == "true"
 )
 
-LEAVE_CONNECTOR_ACTIVE_ON_INITIALIZATION_FAILURE = (
-    os.environ.get("LEAVE_CONNECTOR_ACTIVE_ON_INITIALIZATION_FAILURE", "").lower()
-    == "true"
-)
-
 PRUNING_DISABLED = -1
 DEFAULT_PRUNING_FREQ = 60 * 60 * 24  # Once a day
 

backend/onyx/configs/constants.py

@@ -15,12 +15,6 @@ ID_SEPARATOR = ":;:"
 DEFAULT_BOOST = 0
 SESSION_KEY = "session"
 
-# Cookies
-FASTAPI_USERS_AUTH_COOKIE_NAME = (
-    "fastapiusersauth"  # Currently a constant, but logic allows for configuration
-)
-TENANT_ID_COOKIE_NAME = "onyx_tid"  # tenant id - for workaround cases
-
 NO_AUTH_USER_ID = "__no_auth_user__"
 NO_AUTH_USER_EMAIL = "anonymous@onyx.app"
 
@@ -312,18 +306,9 @@ class OnyxRedisLocks:
 
 
 class OnyxRedisSignals:
-    BLOCK_VALIDATE_INDEXING_FENCES = "signal:block_validate_indexing_fences"
-    BLOCK_VALIDATE_EXTERNAL_GROUP_SYNC_FENCES = (
-        "signal:block_validate_external_group_sync_fences"
-    )
-    BLOCK_VALIDATE_PERMISSION_SYNC_FENCES = (
-        "signal:block_validate_permission_sync_fences"
-    )
-    BLOCK_BUILD_FENCE_LOOKUP_TABLE = "signal:block_build_fence_lookup_table"
-
-
-class OnyxRedisConstants:
-    ACTIVE_FENCES = "active_fences"
+    VALIDATE_INDEXING_FENCES = "signal:validate_indexing_fences"
+    VALIDATE_EXTERNAL_GROUP_SYNC_FENCES = "signal:validate_external_group_sync_fences"
+    VALIDATE_PERMISSION_SYNC_FENCES = "signal:validate_permission_sync_fences"
 
 
 class OnyxCeleryPriority(int, Enum):

backend/onyx/connectors/airtable/airtable_connector.py

@@ -369,12 +369,11 @@ class AirtableConnector(LoadConnector):
         # Process records in parallel batches using ThreadPoolExecutor
         PARALLEL_BATCH_SIZE = 8
         max_workers = min(PARALLEL_BATCH_SIZE, len(records))
-        record_documents: list[Document] = []
 
         # Process records in batches
         for i in range(0, len(records), PARALLEL_BATCH_SIZE):
             batch_records = records[i : i + PARALLEL_BATCH_SIZE]
-            record_documents = []
+            record_documents: list[Document] = []
 
             with ThreadPoolExecutor(max_workers=max_workers) as executor:
                 # Submit batch tasks

backend/onyx/connectors/asana/asana_api.py

@@ -99,18 +99,18 @@ class AsanaAPI:
         project = self.project_api.get_project(project_gid, opts={})
         if project["archived"]:
             logger.info(f"Skipping archived project: {project['name']} ({project_gid})")
-            yield from []
+            return []
         if not project["team"] or not project["team"]["gid"]:
             logger.info(
                 f"Skipping project without a team: {project['name']} ({project_gid})"
             )
-            yield from []
+            return []
         if project["privacy_setting"] == "private":
             if self.team_gid and project["team"]["gid"] != self.team_gid:
                 logger.info(
                     f"Skipping private project not in configured team: {project['name']} ({project_gid})"
                 )
-                yield from []
+                return []
             else:
                 logger.info(
                     f"Processing private project in configured team: {project['name']} ({project_gid})"

backend/onyx/connectors/google_utils/resources.py

@@ -26,7 +26,6 @@ def _get_google_service(
     creds: ServiceAccountCredentials | OAuthCredentials,
     user_email: str | None = None,
 ) -> GoogleDriveService | GoogleDocsService | AdminService | GmailService:
-    service: Resource
     if isinstance(creds, ServiceAccountCredentials):
         creds = creds.with_subject(user_email)
         service = build(service_name, service_version, credentials=creds)

backend/onyx/connectors/salesforce/doc_conversion.py

@@ -59,7 +59,6 @@ def _clean_salesforce_dict(data: dict | list) -> dict | list:
     elif isinstance(data, list):
         filtered_list = []
         for item in data:
-            filtered_item: dict | list
             if isinstance(item, (dict, list)):
                 filtered_item = _clean_salesforce_dict(item)
                 # Only add non-empty dictionaries or lists

backend/onyx/db/constants.py

@@ -1,2 +1 @@
 SLACK_BOT_PERSONA_PREFIX = "__slack_bot_persona__"
-DEFAULT_PERSONA_SLACK_CHANNEL_NAME = "DEFAULT_SLACK_CHANNEL"

backend/onyx/db/document_set.py

@@ -198,7 +198,7 @@ def _check_if_cc_pairs_are_owned_by_groups(
             ids=missing_cc_pair_ids,
         )
         for cc_pair in cc_pairs:
-            if cc_pair.access_type == AccessType.PRIVATE:
+            if cc_pair.access_type != AccessType.PUBLIC:
                 raise ValueError(
                     f"Connector Credential Pair with ID: '{cc_pair.id}'"
                     " is not owned by the specified groups"
@@ -221,8 +221,6 @@ def insert_document_set(
             group_ids=document_set_creation_request.groups or [],
         )
 
-    new_document_set_row: DocumentSetDBModel
-    ds_cc_pairs: list[DocumentSet__ConnectorCredentialPair]
     try:
         new_document_set_row = DocumentSetDBModel(
             name=document_set_creation_request.name,

backend/onyx/db/models.py

@@ -1716,7 +1716,7 @@ class ChannelConfig(TypedDict):
     """NOTE: is a `TypedDict` so it can be used as a type hint for a JSONB column
     in Postgres"""
 
-    channel_name: str | None  # None for default channel config
+    channel_name: str
     respond_tag_only: NotRequired[bool]  # defaults to False
     respond_to_bots: NotRequired[bool]  # defaults to False
     respond_member_group_list: NotRequired[list[str]]
@@ -1737,6 +1737,7 @@ class SlackChannelConfig(Base):
     persona_id: Mapped[int | None] = mapped_column(
         ForeignKey("persona.id"), nullable=True
     )
+    # JSON for flexibility. Contains things like: channel name, team members, etc.
     channel_config: Mapped[ChannelConfig] = mapped_column(
         postgresql.JSONB(), nullable=False
     )
@@ -1745,8 +1746,6 @@ class SlackChannelConfig(Base):
         Boolean, nullable=False, default=False
     )
 
-    is_default: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
-
     persona: Mapped[Persona | None] = relationship("Persona")
     slack_bot: Mapped["SlackBot"] = relationship(
         "SlackBot",
@@ -1758,21 +1757,6 @@ class SlackChannelConfig(Base):
         back_populates="slack_channel_configs",
     )
 
-    __table_args__ = (
-        UniqueConstraint(
-            "slack_bot_id",
-            "is_default",
-            name="uq_slack_channel_config_slack_bot_id_default",
-        ),
-        Index(
-            "ix_slack_channel_config_slack_bot_id_default",
-            "slack_bot_id",
-            "is_default",
-            unique=True,
-            postgresql_where=(is_default is True),  #   type: ignore
-        ),
-    )
-
 
 class SlackBot(Base):
     __tablename__ = "slack_bot"

backend/onyx/db/slack_bot.py

@@ -74,15 +74,3 @@ def remove_slack_bot(
 
 def fetch_slack_bots(db_session: Session) -> Sequence[SlackBot]:
     return db_session.scalars(select(SlackBot)).all()
-
-
-def fetch_slack_bot_tokens(
-    db_session: Session, slack_bot_id: int
-) -> dict[str, str] | None:
-    slack_bot = db_session.scalar(select(SlackBot).where(SlackBot.id == slack_bot_id))
-    if not slack_bot:
-        return None
-    return {
-        "app_token": slack_bot.app_token,
-        "bot_token": slack_bot.bot_token,
-    }

backend/onyx/db/slack_channel_config.py

@@ -6,7 +6,6 @@ from sqlalchemy.orm import Session
 
 from onyx.configs.chat_configs import MAX_CHUNKS_FED_TO_CHAT
 from onyx.context.search.enums import RecencyBiasSetting
-from onyx.db.constants import DEFAULT_PERSONA_SLACK_CHANNEL_NAME
 from onyx.db.constants import SLACK_BOT_PERSONA_PREFIX
 from onyx.db.models import ChannelConfig
 from onyx.db.models import Persona
@@ -23,8 +22,8 @@ from onyx.utils.variable_functionality import (
 )
 
 
-def _build_persona_name(channel_name: str | None) -> str:
-    return f"{SLACK_BOT_PERSONA_PREFIX}{channel_name if channel_name else DEFAULT_PERSONA_SLACK_CHANNEL_NAME}"
+def _build_persona_name(channel_name: str) -> str:
+    return f"{SLACK_BOT_PERSONA_PREFIX}{channel_name}"
 
 
 def _cleanup_relationships(db_session: Session, persona_id: int) -> None:
@@ -41,7 +40,7 @@ def _cleanup_relationships(db_session: Session, persona_id: int) -> None:
 
 def create_slack_channel_persona(
     db_session: Session,
-    channel_name: str | None,
+    channel_name: str,
     document_set_ids: list[int],
     existing_persona_id: int | None = None,
     num_chunks: float = MAX_CHUNKS_FED_TO_CHAT,
@@ -91,7 +90,6 @@ def insert_slack_channel_config(
     channel_config: ChannelConfig,
     standard_answer_category_ids: list[int],
     enable_auto_filters: bool,
-    is_default: bool = False,
 ) -> SlackChannelConfig:
     versioned_fetch_standard_answer_categories_by_ids = (
         fetch_versioned_implementation_with_fallback(
@@ -117,26 +115,12 @@ def insert_slack_channel_config(
                 f"Some or all categories with ids {standard_answer_category_ids} do not exist"
             )
 
-    if is_default:
-        existing_default = db_session.scalar(
-            select(SlackChannelConfig).where(
-                SlackChannelConfig.slack_bot_id == slack_bot_id,
-                SlackChannelConfig.is_default is True,  # type: ignore
-            )
-        )
-        if existing_default:
-            raise ValueError("A default config already exists for this Slack bot.")
-    else:
-        if "channel_name" not in channel_config:
-            raise ValueError("Channel name is required for non-default configs.")
-
     slack_channel_config = SlackChannelConfig(
         slack_bot_id=slack_bot_id,
         persona_id=persona_id,
         channel_config=channel_config,
         standard_answer_categories=existing_standard_answer_categories,
         enable_auto_filters=enable_auto_filters,
-        is_default=is_default,
     )
     db_session.add(slack_channel_config)
     db_session.commit()
@@ -180,7 +164,12 @@ def update_slack_channel_config(
             f"Some or all categories with ids {standard_answer_category_ids} do not exist"
         )
 
+    # get the existing persona id before updating the object
+    existing_persona_id = slack_channel_config.persona_id
+
     # update the config
+    # NOTE: need to do this before cleaning up the old persona or else we
+    # will encounter `violates foreign key constraint` errors
     slack_channel_config.persona_id = persona_id
     slack_channel_config.channel_config = channel_config
     slack_channel_config.standard_answer_categories = list(
@@ -188,6 +177,20 @@ def update_slack_channel_config(
     )
     slack_channel_config.enable_auto_filters = enable_auto_filters
 
+    # if the persona has changed, then clean up the old persona
+    if persona_id != existing_persona_id and existing_persona_id:
+        existing_persona = db_session.scalar(
+            select(Persona).where(Persona.id == existing_persona_id)
+        )
+        # if the existing persona was one created just for use with this Slack channel,
+        # then clean it up
+        if existing_persona and existing_persona.name.startswith(
+            SLACK_BOT_PERSONA_PREFIX
+        ):
+            _cleanup_relationships(
+                db_session=db_session, persona_id=existing_persona_id
+            )
+
     db_session.commit()
 
     return slack_channel_config
@@ -250,32 +253,3 @@ def fetch_slack_channel_config(
             SlackChannelConfig.id == slack_channel_config_id
         )
     )
-
-
-def fetch_slack_channel_config_for_channel_or_default(
-    db_session: Session, slack_bot_id: int, channel_name: str | None
-) -> SlackChannelConfig | None:
-    # attempt to find channel-specific config first
-    if channel_name:
-        sc_config = db_session.scalar(
-            select(SlackChannelConfig).where(
-                SlackChannelConfig.slack_bot_id == slack_bot_id,
-                SlackChannelConfig.channel_config["channel_name"].astext
-                == channel_name,
-            )
-        )
-    else:
-        sc_config = None
-
-    if sc_config:
-        return sc_config
-
-    # if none found, see if there is a default
-    default_sc = db_session.scalar(
-        select(SlackChannelConfig).where(
-            SlackChannelConfig.slack_bot_id == slack_bot_id,
-            SlackChannelConfig.is_default == True,  # noqa: E712
-        )
-    )
-
-    return default_sc

backend/onyx/document_index/document_index_utils.py

@@ -142,8 +142,6 @@ def get_uuid_from_chunk_info(
     tenant_id: str | None,
     large_chunk_id: int | None = None,
 ) -> UUID:
-    """NOTE: be VERY carefuly about changing this function. If changed without a migration,
-    this can cause deletion/update/insertion to function incorrectly."""
     doc_str = document_id
 
     # Web parsing URL duplicate catching

backend/onyx/document_index/vespa/index.py

@@ -346,14 +346,6 @@ class VespaIndex(DocumentIndex):
         # IMPORTANT: This must be done one index at a time, do not use secondary index here
         cleaned_chunks = [clean_chunk_id_copy(chunk) for chunk in chunks]
 
-        # needed so the final DocumentInsertionRecord returned can have the original document ID
-        new_document_id_to_original_document_id: dict[str, str] = {}
-        for ind, chunk in enumerate(cleaned_chunks):
-            old_chunk = chunks[ind]
-            new_document_id_to_original_document_id[
-                chunk.source_document.id
-            ] = old_chunk.source_document.id
-
         existing_docs: set[str] = set()
 
         # NOTE: using `httpx` here since `requests` doesn't support HTTP2. This is beneficial for
@@ -409,14 +401,14 @@ class VespaIndex(DocumentIndex):
                     executor=executor,
                 )
 
-        all_cleaned_doc_ids = {chunk.source_document.id for chunk in cleaned_chunks}
+        all_doc_ids = {chunk.source_document.id for chunk in cleaned_chunks}
 
         return {
             DocumentInsertionRecord(
-                document_id=new_document_id_to_original_document_id[cleaned_doc_id],
-                already_existed=cleaned_doc_id in existing_docs,
+                document_id=doc_id,
+                already_existed=doc_id in existing_docs,
             )
-            for cleaned_doc_id in all_cleaned_doc_ids
+            for doc_id in all_doc_ids
         }
 
     @classmethod
@@ -549,7 +541,7 @@ class VespaIndex(DocumentIndex):
             time.monotonic() - update_start,
         )
 
-    def _update_single_chunk(
+    def update_single_chunk(
         self,
         doc_chunk_id: UUID,
         index_name: str,
@@ -613,8 +605,6 @@ class VespaIndex(DocumentIndex):
         """
         doc_chunk_count = 0
 
-        doc_id = replace_invalid_doc_id_characters(doc_id)
-
         with self.httpx_client_context as httpx_client:
             for (
                 index_name,
@@ -637,7 +627,7 @@ class VespaIndex(DocumentIndex):
                 doc_chunk_count += len(doc_chunk_ids)
 
                 for doc_chunk_id in doc_chunk_ids:
-                    self._update_single_chunk(
+                    self.update_single_chunk(
                         doc_chunk_id, index_name, fields, doc_id, httpx_client
                     )
 
@@ -699,18 +689,6 @@ class VespaIndex(DocumentIndex):
         batch_retrieval: bool = False,
         get_large_chunks: bool = False,
     ) -> list[InferenceChunkUncleaned]:
-        # make sure to use the vespa-afied document IDs
-        chunk_requests = [
-            VespaChunkRequest(
-                document_id=replace_invalid_doc_id_characters(
-                    chunk_request.document_id
-                ),
-                min_chunk_ind=chunk_request.min_chunk_ind,
-                max_chunk_ind=chunk_request.max_chunk_ind,
-            )
-            for chunk_request in chunk_requests
-        ]
-
         if batch_retrieval:
             return batch_search_api_retrieval(
                 index_name=self.index_name,

backend/onyx/document_index/vespa/indexing_utils.py

@@ -242,9 +242,9 @@ def batch_index_vespa_chunks(
 def clean_chunk_id_copy(
     chunk: DocMetadataAwareIndexChunk,
 ) -> DocMetadataAwareIndexChunk:
-    clean_chunk = chunk.model_copy(
+    clean_chunk = chunk.copy(
         update={
-            "source_document": chunk.source_document.model_copy(
+            "source_document": chunk.source_document.copy(
                 update={
                     "id": replace_invalid_doc_id_characters(chunk.source_document.id)
                 }

backend/onyx/document_index/vespa/shared_utils/utils.py

@@ -45,9 +45,7 @@ def is_text_character(codepoint: int) -> bool:
 
 
 def replace_invalid_doc_id_characters(text: str) -> str:
-    """Replaces invalid document ID characters in text.
-    NOTE: this must be called at the start of every vespa-related operation or else we
-    risk discrepancies -> silent failures on deletion/update/insertion."""
+    """Replaces invalid document ID characters in text."""
     # There may be a more complete set of replacements that need to be made but Vespa docs are unclear
     # and users only seem to be running into this error with single quotes
     return text.replace("'", "_")

backend/onyx/file_processing/extract_file_text.py

@@ -365,7 +365,7 @@ def extract_file_text(
                     f"Failed to process with Unstructured: {str(unstructured_error)}. Falling back to normal processing."
                 )
                 # Fall through to normal processing
-        final_extension: str
+
         if file_name or extension:
             if extension is not None:
                 final_extension = extension

backend/onyx/indexing/chunker.py

@@ -223,8 +223,6 @@ class Chunker:
                 large_chunk_id=None,
             )
 
-        section_link_text: str
-
         for section_idx, section in enumerate(document.sections):
             section_text = clean_text(section.text)
             section_link_text = section.link or ""

backend/onyx/llm/chat_llm.py

@@ -409,11 +409,7 @@ class DefaultMultiLLM(LLM):
                 # For now, we don't support parallel tool calls
                 # NOTE: we can't pass this in if tools are not specified
                 # or else OpenAI throws an error
-                **(
-                    {"parallel_tool_calls": False}
-                    if tools and self.config.model_name != "o3-mini"
-                    else {}
-                ),  # TODO: remove once LITELLM has patched
+                **({"parallel_tool_calls": False} if tools else {}),
                 **(
                     {"response_format": structured_response_format}
                     if structured_response_format
@@ -473,7 +469,9 @@ class DefaultMultiLLM(LLM):
         if LOG_DANSWER_MODEL_INTERACTIONS:
             self.log_model_configs()
 
-        if DISABLE_LITELLM_STREAMING:
+        if (
+            DISABLE_LITELLM_STREAMING or self.config.model_name == "o1-2024-12-17"
+        ):  # TODO: remove once litellm supports streaming
             yield self.invoke(prompt, tools, tool_choice, structured_response_format)
             return
 

backend/onyx/llm/llm_provider_options.py

@@ -27,7 +27,6 @@ class WellKnownLLMProviderDescriptor(BaseModel):
 
 OPENAI_PROVIDER_NAME = "openai"
 OPEN_AI_MODEL_NAMES = [
-    "o3-mini",
     "o1-mini",
     "o1-preview",
     "o1-2024-12-17",
@@ -92,7 +91,7 @@ def fetch_available_well_known_llms() -> list[WellKnownLLMProviderDescriptor]:
             api_version_required=False,
             custom_config_keys=[],
             llm_names=fetch_models_for_provider(OPENAI_PROVIDER_NAME),
-            default_model="gpt-4o",
+            default_model="gpt-4",
             default_fast_model="gpt-4o-mini",
         ),
         WellKnownLLMProviderDescriptor(

backend/onyx/onyxbot/slack/config.py

@@ -3,11 +3,9 @@ import os
 from sqlalchemy.orm import Session
 
 from onyx.db.models import SlackChannelConfig
-from onyx.db.slack_channel_config import (
-    fetch_slack_channel_config_for_channel_or_default,
-)
 from onyx.db.slack_channel_config import fetch_slack_channel_configs
 
+
 VALID_SLACK_FILTERS = [
     "answerable_prefilter",
     "well_answered_postfilter",
@@ -19,16 +17,18 @@ def get_slack_channel_config_for_bot_and_channel(
     db_session: Session,
     slack_bot_id: int,
     channel_name: str | None,
-) -> SlackChannelConfig:
-    slack_bot_config = fetch_slack_channel_config_for_channel_or_default(
-        db_session=db_session, slack_bot_id=slack_bot_id, channel_name=channel_name
-    )
-    if not slack_bot_config:
-        raise ValueError(
-            "No default configuration has been set for this Slack bot. This should not be possible."
-        )
+) -> SlackChannelConfig | None:
+    if not channel_name:
+        return None
 
-    return slack_bot_config
+    slack_bot_configs = fetch_slack_channel_configs(
+        db_session=db_session, slack_bot_id=slack_bot_id
+    )
+    for config in slack_bot_configs:
+        if channel_name in config.channel_config["channel_name"]:
+            return config
+
+    return None
 
 
 def validate_channel_name(

backend/onyx/onyxbot/slack/handlers/handle_message.py

@@ -106,7 +106,7 @@ def remove_scheduled_feedback_reminder(
 
 def handle_message(
     message_info: SlackMessageInfo,
-    slack_channel_config: SlackChannelConfig,
+    slack_channel_config: SlackChannelConfig | None,
     client: WebClient,
     feedback_reminder_id: str | None,
     tenant_id: str | None,

backend/onyx/onyxbot/slack/handlers/handle_regular_answer.py

@@ -64,7 +64,7 @@ def rate_limits(
 
 def handle_regular_answer(
     message_info: SlackMessageInfo,
-    slack_channel_config: SlackChannelConfig,
+    slack_channel_config: SlackChannelConfig | None,
     receiver_ids: list[str] | None,
     client: WebClient,
     channel: str,
@@ -76,7 +76,7 @@ def handle_regular_answer(
     should_respond_with_error_msgs: bool = DANSWER_BOT_DISPLAY_ERROR_MSGS,
     disable_docs_only_answer: bool = DANSWER_BOT_DISABLE_DOCS_ONLY_ANSWER,
 ) -> bool:
-    channel_conf = slack_channel_config.channel_config
+    channel_conf = slack_channel_config.channel_config if slack_channel_config else None
 
     messages = message_info.thread_messages
 
@@ -92,7 +92,7 @@ def handle_regular_answer(
     prompt = None
     # If no persona is specified, use the default search based persona
     # This way slack flow always has a persona
-    persona = slack_channel_config.persona
+    persona = slack_channel_config.persona if slack_channel_config else None
     if not persona:
         with get_session_with_tenant(tenant_id) as db_session:
             persona = get_persona_by_id(DEFAULT_PERSONA_ID, user, db_session)
@@ -134,7 +134,11 @@ def handle_regular_answer(
     single_message_history = slackify_message_thread(history_messages) or None
 
     bypass_acl = False
-    if slack_channel_config.persona and slack_channel_config.persona.document_sets:
+    if (
+        slack_channel_config
+        and slack_channel_config.persona
+        and slack_channel_config.persona.document_sets
+    ):
         # For Slack channels, use the full document set, admin will be warned when configuring it
         # with non-public document sets
         bypass_acl = True
@@ -186,7 +190,11 @@ def handle_regular_answer(
         # auto_detect_filters = (
         #     persona.llm_filter_extraction if persona is not None else True
         # )
-        auto_detect_filters = slack_channel_config.enable_auto_filters
+        auto_detect_filters = (
+            slack_channel_config.enable_auto_filters
+            if slack_channel_config is not None
+            else False
+        )
         retrieval_details = RetrievalDetails(
             run_search=OptionalSearchSetting.ALWAYS,
             real_time=False,

backend/onyx/onyxbot/slack/handlers/handle_standard_answers.py

@@ -14,7 +14,7 @@ logger = setup_logger()
 def handle_standard_answers(
     message_info: SlackMessageInfo,
     receiver_ids: list[str] | None,
-    slack_channel_config: SlackChannelConfig,
+    slack_channel_config: SlackChannelConfig | None,
     prompt: Prompt | None,
     logger: OnyxLoggingAdapter,
     client: WebClient,
@@ -40,7 +40,7 @@ def handle_standard_answers(
 def _handle_standard_answers(
     message_info: SlackMessageInfo,
     receiver_ids: list[str] | None,
-    slack_channel_config: SlackChannelConfig,
+    slack_channel_config: SlackChannelConfig | None,
     prompt: Prompt | None,
     logger: OnyxLoggingAdapter,
     client: WebClient,

backend/onyx/onyxbot/slack/listener.py

@@ -5,7 +5,6 @@ import sys
 import threading
 import time
 from collections.abc import Callable
-from contextvars import Token
 from threading import Event
 from types import FrameType
 from typing import Any
@@ -251,8 +250,6 @@ class SlackbotHandler:
         """
         all_tenants = get_all_tenant_ids()
 
-        token: Token[str]
-
         # 1) Try to acquire locks for new tenants
         for tenant_id in all_tenants:
             if (
@@ -410,27 +407,13 @@ class SlackbotHandler:
     def start_socket_client(
         self, slack_bot_id: int, tenant_id: str | None, slack_bot_tokens: SlackBotTokens
     ) -> None:
+        logger.info(
+            f"Starting socket client for tenant: {tenant_id}, app: {slack_bot_id}"
+        )
         socket_client: TenantSocketModeClient = _get_socket_client(
             slack_bot_tokens, tenant_id, slack_bot_id
         )
 
-        try:
-            bot_info = socket_client.web_client.auth_test()
-            if bot_info["ok"]:
-                bot_user_id = bot_info["user_id"]
-                user_info = socket_client.web_client.users_info(user=bot_user_id)
-                if user_info["ok"]:
-                    bot_name = (
-                        user_info["user"]["real_name"] or user_info["user"]["name"]
-                    )
-                    logger.info(
-                        f"Started socket client for Slackbot with name '{bot_name}' (tenant: {tenant_id}, app: {slack_bot_id})"
-                    )
-        except Exception as e:
-            logger.warning(
-                f"Could not fetch bot name: {e} for tenant: {tenant_id}, app: {slack_bot_id}"
-            )
-
         # Append the event handler
         process_slack_event = create_process_slack_event()
         socket_client.socket_mode_request_listeners.append(process_slack_event)  # type: ignore
@@ -788,7 +771,6 @@ def process_message(
         client=client.web_client, channel_id=channel
     )
 
-    token: Token[str] | None = None
     # Set the current tenant ID at the beginning for all DB calls within this thread
     if client.tenant_id:
         logger.info(f"Setting tenant ID to {client.tenant_id}")
@@ -804,7 +786,8 @@ def process_message(
             # Be careful about this default, don't want to accidentally spam every channel
             # Users should be able to DM slack bot in their private channels though
             if (
-                not respond_every_channel
+                slack_channel_config is None
+                and not respond_every_channel
                 # Can't have configs for DMs so don't toss them out
                 and not is_dm
                 # If /OnyxBot (is_bot_msg) or @OnyxBot (bypass_filters)
@@ -814,7 +797,8 @@ def process_message(
                 return
 
             follow_up = bool(
-                slack_channel_config.channel_config
+                slack_channel_config
+                and slack_channel_config.channel_config
                 and slack_channel_config.channel_config.get("follow_up_tags")
                 is not None
             )
@@ -841,7 +825,7 @@ def process_message(
                 if notify_no_answer:
                     apologize_for_fail(details, client)
     finally:
-        if token:
+        if client.tenant_id:
             CURRENT_TENANT_ID_CONTEXTVAR.reset(token)
 
 

backend/onyx/onyxbot/slack/utils.py

@@ -518,7 +518,7 @@ def read_slack_thread(
             message_type = MessageType.USER
         else:
             self_slack_bot_id = get_onyx_bot_slack_bot_id(client)
-            blocks: Any
+
             if reply.get("user") == self_slack_bot_id:
                 # OnyxBot response
                 message_type = MessageType.ASSISTANT

backend/onyx/prompts/agent_search.py

@@ -53,8 +53,8 @@ Answer:
 # Step/Utility Prompts
 # Note this one should always be used with the ENTITY_TERM_EXTRACTION_PROMPT_JSON_EXAMPLE
 ENTITY_TERM_EXTRACTION_PROMPT = f"""
-Based on the original question and some context retrieved from a dataset, please generate a list of \
-entities (e.g. companies, organizations, industries, products, locations, etc.), terms and concepts \
+Based on the original question and some context retrieved from a dataset, please generate a list of
+entities (e.g. companies, organizations, industries, products, locations, etc.), terms and concepts
 (e.g. sales, revenue, etc.) that are relevant for the question, plus their relations to each other.
 
 Here is the original question:
@@ -98,541 +98,445 @@ ENTITY_TERM_EXTRACTION_PROMPT_JSON_EXAMPLE = """
 """.strip()
 
 
-HISTORY_CONTEXT_SUMMARY_PROMPT = f"""
-{{persona_specification}}
-
-Your task now is to summarize the key parts of the history of a conversation between a user and an agent. \
-The summary has two purposes:
-  1) providing the suitable context for a new question, and
-  2) To capture the key information that was discussed and that the user may have a follow-up question about.
-
-Here is the question:
-{SEPARATOR_LINE}
-{{question}}
-{SEPARATOR_LINE}
-
-And here is the history:
-{SEPARATOR_LINE}
-{{history}}
-{SEPARATOR_LINE}
-
-Please provide a summarized context from the history so that the question makes sense and can \
-- with suitable extra information - be answered.
-
-Do not use more than three or four sentences.
-
-History summary:
-""".strip()
+HISTORY_CONTEXT_SUMMARY_PROMPT = (
+    "{persona_specification}\n\n"
+    "Your task now is to summarize the key parts of the history of a conversation between a user and an agent."
+    " The summary has two purposes:\n"
+    "  1) providing the suitable context for a new question, and\n"
+    "  2) To capture the key information that was discussed and that the user may have a follow-up question about.\n\n"
+    "Here is the question:\n"
+    f"{SEPARATOR_LINE}\n"
+    "{question}\n"
+    f"{SEPARATOR_LINE}\n\n"
+    "And here is the history:\n"
+    f"{SEPARATOR_LINE}\n"
+    "{history}\n"
+    f"{SEPARATOR_LINE}\n\n"
+    "Please provide a summarized context from the history so that the question makes sense and can"
+    " - with suitable extra information - be answered.\n\n"
+    "Do not use more than three or four sentences.\n\n"
+    "History summary:"
+).strip()
 
 
 # INITIAL PHASE
 # Sub-question
 # Intentionally left a copy in case we want to modify this one differently
-INITIAL_QUESTION_DECOMPOSITION_PROMPT = f"""
-Decompose the initial user question into no more than 3 appropriate sub-questions that help to answer the \
-original question. The purpose for this decomposition may be to:
-  1) isolate individual entities (i.e., 'compare sales of company A and company B' -> \
-['what are sales for company A', 'what are sales for company B'])
-  2) clarify or disambiguate ambiguous terms (i.e., 'what is our success with company A' -> \
-['what are our sales with company A','what is our market share with company A', \
-'is company A a reference customer for us', etc.])
-  3) if a term or a metric is essentially clear, but it could relate to various components of an entity and you \
-are generally familiar with the entity, then you can decompose the question into sub-questions that are more \
-specific to components (i.e., 'what do we do to improve scalability of product X', 'what do we to to improve \
-scalability of product X', 'what do we do to improve stability of product X', ...])
-  4) research an area that could really help to answer the question.
-
-Here is the initial question to decompose:
-{SEPARATOR_LINE}
-{{question}}
-{SEPARATOR_LINE}
-
-{{history}}
-
-Do NOT include any text in your answer outside of the list of sub-questions!
-Please formulate your answer as a newline-separated list of questions like so:
- <sub-question>
- <sub-question>
- <sub-question>
- ...
-
-Answer:
-""".strip()
+INITIAL_QUESTION_DECOMPOSITION_PROMPT = (
+    "Decompose the initial user question into no more than 3 appropriate sub-questions that help to answer the"
+    " original question. The purpose for this decomposition may be to:\n"
+    "  1) isolate individual entities (i.e., 'compare sales of company A and company B' ->"
+    " ['what are sales for company A', 'what are sales for company B'])\n"
+    "  2) clarify or disambiguate ambiguous terms (i.e., 'what is our success with company A' ->"
+    " ['what are our sales with company A','what is our market share with company A',"
+    " 'is company A a reference customer for us', etc.])\n"
+    "  3) if a term or a metric is essentially clear, but it could relate to various components of an entity and you"
+    " are generally familiar with the entity, then you can decompose the question into sub-questions that are more"
+    " specific to components (i.e., 'what do we do to improve scalability of product X', 'what do we to to improve"
+    " scalability of product X', 'what do we do to improve stability of product X', ...])\n"
+    "  4) research an area that could really help to answer the question.\n\n"
+    "Here is the initial question to decompose:\n"
+    f"{SEPARATOR_LINE}\n"
+    "{question}\n"
+    f"{SEPARATOR_LINE}\n\n"
+    "{history}\n\n"
+    "Do NOT include any text in your answer outside of the list of sub-questions!"
+    "Please formulate your answer as a newline-separated list of questions like so:\n"
+    " <sub-question>\n"
+    " <sub-question>\n"
+    " <sub-question>\n"
+    " ...\n\n"
+    "Answer:"
+).strip()
 
 
 # TODO: combine shared pieces with INITIAL_QUESTION_DECOMPOSITION_PROMPT
-INITIAL_DECOMPOSITION_PROMPT_QUESTIONS_AFTER_SEARCH = f"""
-Decompose the initial user question into no more than 3 appropriate sub-questions that help to answer the \
-original question. The purpose for this decomposition may be to:
-  1) isolate individual entities (i.e., 'compare sales of company A and company B' -> \
-['what are sales for company A', 'what are sales for company B'])
-  2) clarify or disambiguate ambiguous terms (i.e., 'what is our success with company A' -> \
-['what are our sales with company A','what is our market share with company A', \
-'is company A a reference customer for us', etc.])
-  3) if a term or a metric is essentially clear, but it could relate to various components of an entity and you \
-are generally familiar with the entity, then you can decompose the question into sub-questions that are more \
-specific to components (i.e., 'what do we do to improve scalability of product X', 'what do we to to improve \
-scalability of product X', 'what do we do to improve stability of product X', ...])
-  4) research an area that could really help to answer the question.
-
-To give you some context, you will see below also some documents that may relate to the question. Please only \
-use this information to learn what the question is approximately asking about, but do not focus on the details \
-to construct the sub-questions! Also, some of the entities, relationships and terms that are in the dataset may \
-not be in these few documents, so DO NOT focussed too much on the documents when constructing the sub-questions! \
-Decomposition and disambiguations are most important!
-
-Here are the sample docs to give you some context:
-{SEPARATOR_LINE}
-{{sample_doc_str}}
-{SEPARATOR_LINE}
-
-And here is the initial question to decompose:
-{SEPARATOR_LINE}
-{{question}}
-{SEPARATOR_LINE}
-
-{{history}}
-
-Do NOT include any text in your answer outside of the list of sub-questions!\
-Please formulate your answer as a newline-separated list of questions like so:
- <sub-question>
- <sub-question>
- <sub-question>
- ...
-
-Answer:
-""".strip()
+INITIAL_DECOMPOSITION_PROMPT_QUESTIONS_AFTER_SEARCH = (
+    "Decompose the initial user question into no more than 3 appropriate sub-questions that help to answer the"
+    " original question. The purpose for this decomposition may be to:\n"
+    "  1) isolate individual entities (i.e., 'compare sales of company A and company B' ->"
+    " ['what are sales for company A', 'what are sales for company B'])\n"
+    "  2) clarify or disambiguate ambiguous terms (i.e., 'what is our success with company A' ->"
+    " ['what are our sales with company A','what is our market share with company A',"
+    " 'is company A a reference customer for us', etc.])\n"
+    "  3) if a term or a metric is essentially clear, but it could relate to various components of an entity and you"
+    " are generally familiar with the entity, then you can decompose the question into sub-questions that are more"
+    " specific to components (i.e., 'what do we do to improve scalability of product X', 'what do we to to improve"
+    " scalability of product X', 'what do we do to improve stability of product X', ...])\n"
+    "  4) research an area that could really help to answer the question.\n\n"
+    "To give you some context, you will see below also some documents that may relate to the question. Please only"
+    " use this information to learn what the question is approximately asking about, but do not focus on the details"
+    " to construct the sub-questions! Also, some of the entities, relationships and terms that are in the dataset may"
+    " not be in these few documents, so DO NOT focussed too much on the documents when constructing the sub-questions!"
+    " Decomposition and disambiguations are most important!\n\n"
+    "Here are the sample docs to give you some context:\n"
+    f"{SEPARATOR_LINE}\n"
+    "{sample_doc_str}\n"
+    f"{SEPARATOR_LINE}\n\n"
+    "And here is the initial question to decompose:\n"
+    f"{SEPARATOR_LINE}\n"
+    "{question}\n"
+    f"{SEPARATOR_LINE}\n\n"
+    "{history}\n\n"
+    "Do NOT include any text in your answer outside of the list of sub-questions!"
+    "Please formulate your answer as a newline-separated list of questions like so:\n"
+    " <sub-question>\n"
+    " <sub-question>\n"
+    " <sub-question>\n"
+    " ...\n\n"
+    "Answer:"
+).strip()
 
 
 # Retrieval
-QUERY_REWRITING_PROMPT = f"""
-Please convert the initial user question into a 2-3 more appropriate short and pointed search queries for \
-retrieval from a document store. Particularly, try to think about resolving ambiguities and make the search \
-queries more specific, enabling the system to search more broadly.
-
-Also, try to make the search queries not redundant, i.e. not too similar!
-
-Here is the initial question:
-{SEPARATOR_LINE}
-{{question}}
-{SEPARATOR_LINE}
-
-Do NOT include any text in your answer outside of the list of queries!\
-Formulate the queries separated by newlines (Do not say 'Query 1: ...', just write the querytext) as follows:
-<query 1>
-<query 2>
-...
-
-Queries:
-""".strip()
+QUERY_REWRITING_PROMPT = (
+    "Please convert the initial user question into a 2-3 more appropriate short and pointed search queries for"
+    " retrieval from a document store. Particularly, try to think about resolving ambiguities and make the search"
+    " queries more specific, enabling the system to search more broadly.\n"
+    "Also, try to make the search queries not redundant, i.e. not too similar!\n\n"
+    "Here is the initial question:\n"
+    f"{SEPARATOR_LINE}\n"
+    "{question}\n"
+    f"{SEPARATOR_LINE}\n\n"
+    "Do NOT include any text in your answer outside of the list of queries!"
+    "Formulate the queries separated by newlines (Do not say 'Query 1: ...', just write the querytext) as follows:\n"
+    "<query 1>\n"
+    "<query 2>\n"
+    "...\n\n"
+    "Queries:"
+)
 
 
-DOCUMENT_VERIFICATION_PROMPT = f"""
-Determine whether the following document text contains data or information that is potentially relevant \
-for a question. It does not have to be fully relevant, but check whether it has some information that \
-would help - possibly in conjunction with other documents - to address the question.
-
-Be careful that you do not use a document where you are not sure whether the text applies to the objects \
-or entities that are relevant for the question. For example, a book about chess could have long passage \
-discussing the psychology of chess without - within the passage - mentioning chess. If now a question \
-is asked about the psychology of football, one could be tempted to use the document as it does discuss \
-psychology in sports. However, it is NOT about football and should not be deemed relevant. Please \
-consider this logic.
-
-DOCUMENT TEXT:
-{SEPARATOR_LINE}
-{{document_content}}
-{SEPARATOR_LINE}
-
-Do you think that this document text is useful and relevant to answer the following question?
-
-QUESTION:
-{SEPARATOR_LINE}
-{{question}}
-{SEPARATOR_LINE}
-
-Please answer with exactly and only a '{YES}' or '{NO}'. Do NOT include any other text in your response:
-
-Answer:
-""".strip()
+DOCUMENT_VERIFICATION_PROMPT = (
+    "Determine whether the following document text contains data or information that is potentially relevant "
+    "for a question. It does not have to be fully relevant, but check whether it has some information that "
+    "would help - possibly in conjunction with other documents - to address the question.\n\n"
+    "Be careful that you do not use a document where you are not sure whether the text applies to the objects "
+    "or entities that are relevant for the question. For example, a book about chess could have long passage "
+    "discussing the psychology of chess without - within the passage - mentioning chess. If now a question "
+    "is asked about the psychology of football, one could be tempted to use the document as it does discuss "
+    "psychology in sports. However, it is NOT about football and should not be deemed relevant. Please "
+    "consider this logic.\n\n"
+    "DOCUMENT TEXT:\n"
+    f"{SEPARATOR_LINE}\n"
+    "{document_content}\n"
+    f"{SEPARATOR_LINE}\n\n"
+    "Do you think that this document text is useful and relevant to answer the following question?\n\n"
+    "QUESTION:\n"
+    f"{SEPARATOR_LINE}\n"
+    "{question}\n"
+    f"{SEPARATOR_LINE}\n\n"
+    "Please answer with exactly and only a 'yes' or 'no'. Do NOT include any other text in your response:\n\n"
+    "Answer:"
+).strip()
 
 
 # Sub-Question Anser Generation
-SUB_QUESTION_RAG_PROMPT = f"""
-Use the context provided below - and only the provided context - to answer the given question. \
-(Note that the answer is in service of answering a broader question, given below as 'motivation').
-
-Again, only use the provided context and do not use your internal knowledge! If you cannot answer the \
-question based on the context, say "{UNKNOWN_ANSWER}". It is a matter of life and death that you do NOT \
-use your internal knowledge, just the provided information!
-
-Make sure that you keep all relevant information, specifically as it concerns to the ultimate goal. \
-(But keep other details as well.)
-
-It is critical that you provide inline citations in the format [D1], [D2], [D3], etc! \
-It is important that the citation is close to the information it supports. \
-Proper citations are very important to the user!
-
-For your general information, here is the ultimate motivation:
-{SEPARATOR_LINE}
-{{original_question}}
-{SEPARATOR_LINE}
-
-And here is the actual question I want you to answer based on the context above (with the motivation in mind):
-{SEPARATOR_LINE}
-{{question}}
-{SEPARATOR_LINE}
-
-Here is the context:
-{SEPARATOR_LINE}
-{{context}}
-{SEPARATOR_LINE}
-
-Please keep your answer brief and concise, and focus on facts and data.
-
-Answer:
-""".strip()
+SUB_QUESTION_RAG_PROMPT = (
+    "Use the context provided below - and only the provided context - to answer the given question. "
+    "(Note that the answer is in service of answering a broader question, given below as 'motivation'.)\n\n"
+    "Again, only use the provided context and do not use your internal knowledge! If you cannot answer the "
+    f'question based on the context, say "{UNKNOWN_ANSWER}". It is a matter of life and death that you do NOT '
+    "use your internal knowledge, just the provided information!\n\n"
+    "Make sure that you keep all relevant information, specifically as it concerns to the ultimate goal. "
+    "(But keep other details as well.)\n\n"
+    "It is critical that you provide inline citations in the format [D1], [D2], [D3], etc! "
+    "It is important that the citation is close to the information it supports. "
+    "Proper citations are very important to the user!\n\n"
+    "For your general information, here is the ultimate motivation:\n"
+    f"{SEPARATOR_LINE}\n"
+    "{original_question}\n"
+    f"{SEPARATOR_LINE}\n\n"
+    "And here is the actual question I want you to answer based on the context above (with the motivation in mind):\n"
+    f"{SEPARATOR_LINE}\n"
+    "{question}\n"
+    f"{SEPARATOR_LINE}\n\n"
+    "Here is the context:\n"
+    f"{SEPARATOR_LINE}\n"
+    "{context}\n"
+    f"{SEPARATOR_LINE}\n\n"
+    "Please keep your answer brief and concise, and focus on facts and data.\n\n"
+    "Answer:"
+).strip()
 
 
-SUB_ANSWER_CHECK_PROMPT = f"""
-Determine whether the given answer addresses the given question. \
-Please do not use any internal knowledge you may have - just focus on whether the answer \
-as given seems to largely address the question as given, or at least addresses part of the question.
-
-Here is the question:
-{SEPARATOR_LINE}
-{{question}}
-{SEPARATOR_LINE}
-
-Here is the suggested answer:
-{SEPARATOR_LINE}
-{{base_answer}}
-{SEPARATOR_LINE}
-
-Does the suggested answer address the question? Please answer with "{YES}" or "{NO}".
-""".strip()
+SUB_ANSWER_CHECK_PROMPT = (
+    "Determine whether the given answer addresses the given question. "
+    "Please do not use any internal knowledge you may have - just focus on whether the answer "
+    "as given seems to largely address the question as given, or at least addresses part of the question.\n\n"
+    "Here is the question:\n"
+    f"{SEPARATOR_LINE}\n"
+    "{question}\n"
+    f"{SEPARATOR_LINE}\n\n"
+    "Here is the suggested answer:\n"
+    f"{SEPARATOR_LINE}\n"
+    "{base_answer}\n"
+    f"{SEPARATOR_LINE}\n\n"
+    f'Does the suggested answer address the question? Please answer with "{YES}" or "{NO}".'
+).strip()
 
 
 # Initial Answer Generation
-INITIAL_ANSWER_PROMPT_W_SUB_QUESTIONS = f"""
-{{persona_specification}}
-
-Use the information provided below - and only the provided information - to answer the provided main question.
-
-The information provided below consists of:
-  1) a number of answered sub-questions - these are very important to help you organize your thoughts and your answer
-  2) a number of documents that deemed relevant for the question.
-
-{{history}}
-
-It is critical that you provide prover inline citations to documents in the format [D1], [D2], [D3], etc.! \
-It is important that the citation is close to the information it supports. If you have multiple citations that support \
-a fact, please cite for example as [D1][D3], or [D2][D4], etc. \
-Feel free to also cite sub-questions in addition to documents, but make sure that you have documents cited with the \
-sub-question citation. If you want to cite both a document and a sub-question, please use [D1][Q3], or [D2][D7][Q4], etc. \
-Again, please NEVER cite sub-questions without a document citation! Proper citations are very important for the user!
-
-IMPORTANT RULES:
- - If you cannot reliably answer the question solely using the provided information, say that you cannot reliably answer. \
-You may give some additional facts you learned, but do not try to invent an answer.
- - If the information is empty or irrelevant, just say "{UNKNOWN_ANSWER}".
- - If the information is relevant but not fully conclusive, specify that the information is not conclusive and say why.
-
-Again, you should be sure that the answer is supported by the information provided!
-
-Try to keep your answer concise. But also highlight uncertainties you may have should there be substantial ones, \
-or assumptions you made.
-
-Here is the contextual information:
-{SEPARATOR_LINE_LONG}
-
-*Answered Sub-questions (these should really matter!):
-{SEPARATOR_LINE}
-{{answered_sub_questions}}
-{SEPARATOR_LINE}
-
-And here are relevant document information that support the sub-question answers, or that are relevant for the actual question:
-{SEPARATOR_LINE}
-{{relevant_docs}}
-{SEPARATOR_LINE}
-
-And here is the question I want you to answer based on the information above:
-{SEPARATOR_LINE}
-{{question}}
-{SEPARATOR_LINE}
-
-Please keep your answer brief and concise, and focus on facts and data.
-
-Answer:
-""".strip()
+INITIAL_ANSWER_PROMPT_W_SUB_QUESTIONS = (
+    "{persona_specification}\n\n"
+    "Use the information provided below - and only the provided information - to answer the provided main question.\n\n"
+    "The information provided below consists of:\n"
+    "  1) a number of answered sub-questions - these are very important to help you organize your thoughts and your answer\n"
+    "  2) a number of documents that deemed relevant for the question.\n\n"
+    "{history}\n\n"
+    "It is critical that you provide prover inline citations to documents in the format [D1], [D2], [D3], etc.!\n"
+    "It is important that the citation is close to the information it supports. If you have multiple citations that support\n"
+    "a fact, please cite for example as [D1][D3], or [D2][D4], etc.\n"
+    "Feel free to also cite sub-questions in addition to documents, but make sure that you have documents cited with the "
+    "sub-question citation. If you want to cite both a document and a sub-question, please use [D1][Q3], or "
+    "[D2][D7][Q4], etc.\n"
+    "Again, please NEVER cite sub-questions without a document citation! "
+    "Proper citations are very important for the user!\n\n"
+    "IMPORTANT RULES:\n"
+    " - If you cannot reliably answer the question solely using the provided information, say that you cannot reliably answer.\n"
+    " You may give some additional facts you learned, but do not try to invent an answer.\n"
+    f' - If the information is empty or irrelevant, just say "{UNKNOWN_ANSWER}".\n'
+    " - If the information is relevant but not fully conclusive, specify that the information is not conclusive and say why.\n\n"
+    "Again, you should be sure that the answer is supported by the information provided!\n\n"
+    "Try to keep your answer concise. But also highlight uncertainties you may have should there be substantial ones,\n"
+    "or assumptions you made.\n\n"
+    "Here is the contextual information:\n"
+    f"{SEPARATOR_LINE_LONG}\n\n"
+    "*Answered Sub-questions (these should really matter!):\n"
+    f"{SEPARATOR_LINE}\n"
+    "{answered_sub_questions}\n"
+    f"{SEPARATOR_LINE}\n\n"
+    "And here are relevant document information that support the sub-question answers, "
+    "or that are relevant for the actual question:\n"
+    f"{SEPARATOR_LINE}\n"
+    "{relevant_docs}\n"
+    f"{SEPARATOR_LINE}\n\n"
+    "And here is the question I want you to answer based on the information above:\n"
+    f"{SEPARATOR_LINE}\n"
+    "{question}\n"
+    f"{SEPARATOR_LINE}\n\n"
+    "Please keep your answer brief and concise, and focus on facts and data.\n\n"
+    "Answer:"
+).strip()
 
 
 # Used if sub_question_answer_str is empty
-INITIAL_ANSWER_PROMPT_WO_SUB_QUESTIONS = f"""
-{{answered_sub_questions}}{{persona_specification}}
-
-Use the information provided below - and only the provided information - to answer the provided question. \
-The information provided below consists of a number of documents that were deemed relevant for the question.
-
-{{history}}
-
-IMPORTANT RULES:
- - If you cannot reliably answer the question solely using the provided information, say that you cannot reliably answer. \
-You may give some additional facts you learned, but do not try to invent an answer.
- - If the information is irrelevant, just say "{UNKNOWN_ANSWER}".
- - If the information is relevant but not fully conclusive, specify that the information is not conclusive and say why.
-
-Again, you should be sure that the answer is supported by the information provided!
-
-It is critical that you provide proper inline citations to documents in the format [D1], [D2], [D3], etc! \
-It is important that the citation is close to the information it supports. \
-If you have multiple citations, please cite for example as [D1][D3], or [D2][D4], etc. \
-Citations are very important for the user!
-
-Here is the relevant context information:
-{SEPARATOR_LINE}
-{{relevant_docs}}
-{SEPARATOR_LINE}
-
-And here is the question I want you to answer based on the context above:
-{SEPARATOR_LINE}
-{{question}}
-{SEPARATOR_LINE}
-
-Please keep your answer brief and concise, and focus on facts and data.
-
-Answer:
-""".strip()
+INITIAL_ANSWER_PROMPT_WO_SUB_QUESTIONS = (
+    "{answered_sub_questions}{persona_specification}\n\n"
+    "Use the information provided below - and only the provided information - to answer the provided question. "
+    "The information provided below consists of a number of documents that were deemed relevant for the question.\n"
+    "{history}\n\n"
+    "IMPORTANT RULES:\n"
+    " - If you cannot reliably answer the question solely using the provided information, say that you cannot reliably answer. "
+    "You may give some additional facts you learned, but do not try to invent an answer.\n"
+    f' - If the information is irrelevant, just say "{UNKNOWN_ANSWER}".\n'
+    " - If the information is relevant but not fully conclusive, specify that the information is not conclusive and say why.\n\n"
+    "Again, you should be sure that the answer is supported by the information provided!\n\n"
+    "It is critical that you provide proper inline citations to documents in the format [D1], [D2], [D3], etc! "
+    "It is important that the citation is close to the information it supports. If you have multiple citations, "
+    "please cite for example as [D1][D3], or [D2][D4], etc. Citations are very important for the user!\n\n"
+    "Here is the relevant context information:\n"
+    f"{SEPARATOR_LINE}\n"
+    "{relevant_docs}\n"
+    f"{SEPARATOR_LINE}\n\n"
+    "And here is the question I want you to answer based on the context above:\n"
+    f"{SEPARATOR_LINE}\n"
+    "{question}\n"
+    f"{SEPARATOR_LINE}\n\n"
+    "Please keep your answer brief and concise, and focus on facts and data.\n\n"
+    "Answer:"
+).strip()
 
 
 # REFINEMENT PHASE
-REFINEMENT_QUESTION_DECOMPOSITION_PROMPT = f"""
-An initial user question needs to be answered. An initial answer has been provided but it wasn't quite good enough. \
-Also, some sub-questions had been answered and this information has been used to provide the initial answer. \
-Some other subquestions may have been suggested based on little knowledge, but they were not directly answerable. \
-Also, some entities, relationships and terms are given to you so that you have an idea of how the available data looks like.
-
-Your role is to generate 2-4 new sub-questions that would help to answer the initial question, considering:
-
-1) The initial question
-2) The initial answer that was found to be unsatisfactory
-3) The sub-questions that were answered
-4) The sub-questions that were suggested but not answered
-5) The entities, relationships and terms that were extracted from the context
-
-The individual questions should be answerable by a good RAG system. So a good idea would be to use the sub-questions to \
-resolve ambiguities and/or to separate the question for different entities that may be involved in the original question, \
-but in a way that does not duplicate questions that were already tried.
-
-Additional Guidelines:
-- The sub-questions should be specific to the question and provide richer context for the question, resolve ambiguities, \
-or address shortcoming of the initial answer
-- Each sub-question - when answered - should be relevant for the answer to the original question
-- The sub-questions should be free from comparisons, ambiguities,judgements, aggregations, or any other complications that \
-may require extra context
-- The sub-questions MUST have the full context of the original question so that it can be executed by a RAG system \
-independently without the original question available
-    Example:
-    - initial question: "What is the capital of France?"
-    - bad sub-question: "What is the name of the river there?"
-    - good sub-question: "What is the name of the river that flows through Paris?"
-- For each sub-question, please also provide a search term that can be used to retrieve relevant documents from a document store.
-- Consider specifically the sub-questions that were suggested but not answered. This is a sign that they are not answerable \
-with the available context, and you should not ask similar questions.
-
-Here is the initial question:
-{SEPARATOR_LINE}
-{{question}}
-{SEPARATOR_LINE}
-{{history}}
-
-Here is the initial sub-optimal answer:
-{SEPARATOR_LINE}
-{{base_answer}}
-{SEPARATOR_LINE}
-
-Here are the sub-questions that were answered:
-{SEPARATOR_LINE}
-{{answered_sub_questions}}
-{SEPARATOR_LINE}
-
-Here are the sub-questions that were suggested but not answered:
-{SEPARATOR_LINE}
-{{failed_sub_questions}}
-{SEPARATOR_LINE}
-
-And here are the entities, relationships and terms extracted from the context:
-{SEPARATOR_LINE}
-{{entity_term_extraction_str}}
-{SEPARATOR_LINE}
-
-Please generate the list of good, fully contextualized sub-questions that would help to address the main question. \
-Specifically pay attention also to the entities, relationships and terms extracted, as these indicate what type of \
-objects/relationships/terms you can ask about! Do not ask about entities, terms or relationships that are not mentioned in the \
-'entities, relationships and terms' section.
-
-Again, please find questions that are NOT overlapping too much with the already answered sub-questions or those that \
-already were suggested and failed. In other words - what can we try in addition to what has been tried so far?
-
-Generate the list of questions separated by one new line like this:
-<sub-question 1>
-<sub-question 2>
-<sub-question 3>
-...""".strip()
+REFINEMENT_QUESTION_DECOMPOSITION_PROMPT = (
+    "An initial user question needs to be answered. An initial answer has been provided but it wasn't quite "
+    "good enough. Also, some sub-questions had been answered and this information has been used to provide "
+    "the initial answer. Some other subquestions may have been suggested based on little knowledge, but they "
+    "were not directly answerable. Also, some entities, relationships and terms are given to you so that "
+    "you have an idea of how the available data looks like.\n\n"
+    "Your role is to generate 2-4 new sub-questions that would help to answer the initial question, considering:\n\n"
+    "1) The initial question\n"
+    "2) The initial answer that was found to be unsatisfactory\n"
+    "3) The sub-questions that were answered\n"
+    "4) The sub-questions that were suggested but not answered\n"
+    "5) The entities, relationships and terms that were extracted from the context\n\n"
+    "The individual questions should be answerable by a good RAG system. "
+    "So a good idea would be to use the sub-questions to resolve ambiguities and/or to separate the "
+    "question for different entities that may be involved in the original question, but in a way that does "
+    "not duplicate questions that were already tried.\n\n"
+    "Additional Guidelines:\n"
+    "- The sub-questions should be specific to the question and provide richer context for the question, "
+    "resolve ambiguities, or address shortcoming of the initial answer\n"
+    "- Each sub-question - when answered - should be relevant for the answer to the original question\n"
+    "- The sub-questions should be free from comparisons, ambiguities,judgements, aggregations, or any "
+    "other complications that may require extra context.\n"
+    "- The sub-questions MUST have the full context of the original question so that it can be executed by "
+    "a RAG system independently without the original question available\n"
+    "    (Example:\n"
+    '    - initial question: "What is the capital of France?"\n'
+    '    - bad sub-question: "What is the name of the river there?"\n'
+    '    - good sub-question: "What is the name of the river that flows through Paris?")\n'
+    "- For each sub-question, please also provide a search term that can be used to retrieve relevant "
+    "documents from a document store.\n"
+    "- Consider specifically the sub-questions that were suggested but not answered. This is a sign that they are not "
+    "answerable with the available context, and you should not ask similar questions.\n\n"
+    "Here is the initial question:\n"
+    f"{SEPARATOR_LINE}\n"
+    "{question}\n"
+    f"{SEPARATOR_LINE}\n"
+    "{history}\n\n"
+    "Here is the initial sub-optimal answer:\n"
+    f"{SEPARATOR_LINE}\n"
+    "{base_answer}\n"
+    f"{SEPARATOR_LINE}\n\n"
+    "Here are the sub-questions that were answered:\n"
+    f"{SEPARATOR_LINE}\n"
+    "{answered_sub_questions}\n"
+    f"{SEPARATOR_LINE}\n\n"
+    "Here are the sub-questions that were suggested but not answered:\n"
+    f"{SEPARATOR_LINE}\n"
+    "{failed_sub_questions}\n"
+    f"{SEPARATOR_LINE}\n\n"
+    "And here are the entities, relationships and terms extracted from the context:\n"
+    f"{SEPARATOR_LINE}\n"
+    "{entity_term_extraction_str}\n"
+    f"{SEPARATOR_LINE}\n\n"
+    "Please generate the list of good, fully contextualized sub-questions that would help to address the main question.\n"
+    "Specifically pay attention also to the entities, relationships and terms extracted, as these indicate what type of "
+    "objects/relationships/terms you can ask about! Do not ask about entities, terms or relationships that are not "
+    "mentioned in the 'entities, relationships and terms' section.\n\n"
+    "Again, please find questions that are NOT overlapping too much with the already answered "
+    "sub-questions or those that already were suggested and failed.\n"
+    "In other words - what can we try in addition to what has been tried so far?\n\n"
+    "Generate the list of questions separated by one new line like this:\n"
+    "<sub-question 1>\n"
+    "<sub-question 2>\n"
+    "<sub-question 3>\n"
+    "..."
+).strip()
 
 
-REFINED_ANSWER_PROMPT_W_SUB_QUESTIONS = f"""
-{{persona_specification}}
-
-Your task is to improve on a given answer to a question, as the initial answer was found to be lacking in some way.
-
-Use the information provided below - and only the provided information - to write your new and improved answer.
-
-The information provided below consists of:
-  1) an initial answer that was given but found to be lacking in some way.
-  2) a number of answered sub-questions - these are very important(!) and definitely should help you to answer the main \
-question. Note that the sub-questions have a type, 'initial' and 'refined'. The 'initial' ones were available for the \
-creation of the initial answer, but the 'refined' were not, they are new. So please use the 'refined' sub-questions in \
-particular to update/extend/correct/enrich the initial answer and to add more details/new facts!
-  3) a number of documents that were deemed relevant for the question. This is the context that you use largely for citations \
-(see below). So consider the answers to the sub-questions as guidelines to construct your new answer, but make sure you cite \
-the relevant document for a fact!
-
-It is critical that you provide proper inline citations to documents in the format [D1], [D2], [D3], etc! \
-It is important that the citation is close to the information it supports. \
-DO NOT just list all of the citations at the very end. \
-Feel free to also cite sub-questions in addition to documents, \
-but make sure that you have documents cited with the sub-question citation. \
-If you want to cite both a document and a sub-question, please use [D1][Q3], or [D2][D7][Q4], etc. and always place the \
-document citation before the sub-question citation. Again, please NEVER cite sub-questions without a document citation! \
-Proper citations are very important for the user!
-
-{{history}}
-
-IMPORTANT RULES:
- - If you cannot reliably answer the question solely using the provided information, say that you cannot reliably answer. \
-You may give some additional facts you learned, but do not try to invent an answer.
- - If the information is empty or irrelevant, just say "{UNKNOWN_ANSWER}".
- - If the information is relevant but not fully conclusive, provide an answer to the extent you can but also specify that \
-the information is not conclusive and why.
- - Ignore any existing citations within the answered sub-questions, like [D1]... and [Q2]! The citations you will need to \
-use will need to refer to the documents (and sub-questions) that you are explicitly presented with below!
-
-Again, you should be sure that the answer is supported by the information provided!
-
-Try to keep your answer concise. But also highlight uncertainties you may have should there be substantial ones, \
-or assumptions you made.
-
-Here is the contextual information:
-{SEPARATOR_LINE_LONG}
-
-*Initial Answer that was found to be lacking:
-{SEPARATOR_LINE}
-{{initial_answer}}
-{SEPARATOR_LINE}
-
-*Answered Sub-questions (these should really help you to research your answer! They also contain questions/answers that \
-were not available when the original answer was constructed):
-{{answered_sub_questions}}
-
-And here are the relevant documents that support the sub-question answers, and that are relevant for the actual question:
-{SEPARATOR_LINE}
-{{relevant_docs}}
-{SEPARATOR_LINE}
-
-Lastly, here is the main question I want you to answer based on the information above:
-{SEPARATOR_LINE}
-{{question}}
-{SEPARATOR_LINE}
-
-Please keep your answer brief and concise, and focus on facts and data.
-
-Answer:
-""".strip()
+REFINED_ANSWER_PROMPT_W_SUB_QUESTIONS = (
+    "{persona_specification}\n\n"
+    "Your task is to improve on a given answer to a question, as the initial answer was found to be lacking in some way.\n\n"
+    "Use the information provided below - and only the provided information - to write your new and improved answer.\n\n"
+    "The information provided below consists of:\n"
+    "  1) an initial answer that was given but found to be lacking in some way.\n"
+    "  2) a number of answered sub-questions - these are very important(!) and definitely should help you to answer "
+    "the main question. Note that the sub-questions have a type, 'initial' and 'refined'. The 'initial' "
+    "ones were available for the creation of the initial answer, but the 'refined' were not, they are new. So please use "
+    "the 'refined' sub-questions in particular to update/extend/correct/enrich the initial answer and to add "
+    "more details/new facts!\n"
+    "  3) a number of documents that were deemed relevant for the question. This is the context that you use largely for "
+    "citations (see below). So consider the answers to the sub-questions as guidelines to construct your new answer, but "
+    "make sure you cite the relevant document for a fact!\n\n"
+    "It is critical that you provide proper inline citations to documents in the format [D1], [D2], [D3], etc! "
+    "It is important that the citation is close to the information it supports. "
+    "DO NOT just list all of the citations at the very end. "
+    "Feel free to also cite sub-questions in addition to documents, but make sure that you have documents cited with the "
+    "sub-question citation. If you want to cite both a document and a sub-question, please use [D1][Q3], or [D2][D7][Q4], etc. "
+    "and always place the document citation before the sub-question citation. "
+    "Again, please NEVER cite sub-questions without a document citation!\n"
+    "Proper citations are very important for the user!\n\n"
+    "{history}\n\n"
+    "IMPORTANT RULES:\n"
+    " - If you cannot reliably answer the question solely using the provided information, say that you cannot reliably answer. "
+    "You may give some additional facts you learned, but do not try to invent an answer.\n"
+    f' - If the information is empty or irrelevant, just say "{UNKNOWN_ANSWER}".\n'
+    " - If the information is relevant but not fully conclusive, provide an answer to the extent you can but also "
+    "specify that the information is not conclusive and why.\n"
+    " - Ignore any existing citations within the answered sub-questions, like [D1]... and [Q2]! "
+    "The citations you will need to use will need to refer to the documents (and sub-questions) that you are explicitly "
+    "presented with below!\n\n"
+    "Again, you should be sure that the answer is supported by the information provided!\n\n"
+    "Try to keep your answer concise. But also highlight uncertainties you may have should there be substantial ones, "
+    "or assumptions you made.\n\n"
+    "Here is the contextual information:\n"
+    f"{SEPARATOR_LINE_LONG}\n\n"
+    "*Initial Answer that was found to be lacking:\n"
+    f"{SEPARATOR_LINE}\n"
+    "{initial_answer}\n"
+    f"{SEPARATOR_LINE}\n\n"
+    "*Answered Sub-questions (these should really help you to research your answer! They also contain questions/answers "
+    "that were not available when the original answer was constructed):\n"
+    "{answered_sub_questions}\n\n"
+    "And here are the relevant documents that support the sub-question answers, and that are relevant for the actual question:\n"
+    f"{SEPARATOR_LINE}\n"
+    "{relevant_docs}\n"
+    f"{SEPARATOR_LINE}\n\n"
+    "Lastly, here is the main question I want you to answer based on the information above:\n"
+    f"{SEPARATOR_LINE}\n"
+    "{question}\n"
+    f"{SEPARATOR_LINE}\n\n"
+    "Please keep your answer brief and concise, and focus on facts and data.\n\n"
+    "Answer:"
+).strip()
 
 # sub_question_answer_str is empty
-REFINED_ANSWER_PROMPT_WO_SUB_QUESTIONS = f"""
-{{answered_sub_questions}}{{persona_specification}}
-
-Use the information provided below - and only the provided information - to answer the provided question.
-
-The information provided below consists of:
-  1) an initial answer that was given but found to be lacking in some way.
-  2) a number of documents that were also deemed relevant for the question.
-
-It is critical that you provide proper inline citations to documents in the format [D1], [D2], [D3], etc! \
-It is important that the citation is close to the information it supports. \
-DO NOT just list all of the citations at the very end of your response. Citations are very important for the user!
-
-{{history}}
-
-IMPORTANT RULES:
- - If you cannot reliably answer the question solely using the provided information, say that you cannot reliably answer. \
-You may give some additional facts you learned, but do not try to invent an answer.
- - If the information is empty or irrelevant, just say "{UNKNOWN_ANSWER}".
- - If the information is relevant but not fully conclusive, provide an answer to the extent you can but also specify that \
-the information is not conclusive and why.
-
-Again, you should be sure that the answer is supported by the information provided!
-
-Try to keep your answer concise. But also highlight uncertainties you may have should there be substantial ones, \
-or assumptions you made.
-
-Here is the contextual information:
-{SEPARATOR_LINE_LONG}
-
-*Initial Answer that was found to be lacking:
-{SEPARATOR_LINE}
-{{initial_answer}}
-{SEPARATOR_LINE}
-
-And here are relevant document information that support the sub-question answers, \
-or that are relevant for the actual question:
-{SEPARATOR_LINE}
-{{relevant_docs}}
-{SEPARATOR_LINE}
-
-Lastly, here is the question I want you to answer based on the information above:
-{SEPARATOR_LINE}
-{{question}}
-{SEPARATOR_LINE}
-
-Please keep your answer brief and concise, and focus on facts and data.
-
-Answer:
-""".strip()
+REFINED_ANSWER_PROMPT_WO_SUB_QUESTIONS = (
+    "{answered_sub_questions}{persona_specification}\n\n"
+    "Use the information provided below - and only the provided information - to answer the provided question.\n\n"
+    "The information provided below consists of:\n"
+    "  1) an initial answer that was given but found to be lacking in some way.\n"
+    "  2) a number of documents that were also deemed relevant for the question.\n\n"
+    "It is critical that you provide proper inline citations to documents in the format [D1], [D2], [D3], etc! "
+    "It is important that the citation is close to the information it supports. "
+    "DO NOT just list all of the citations at the very end of your response. Citations are very important for the user!\n\n"
+    "{history}\n\n"
+    "IMPORTANT RULES:\n"
+    " - If you cannot reliably answer the question solely using the provided information, say that you cannot reliably answer. "
+    "You may give some additional facts you learned, but do not try to invent an answer.\n"
+    f' - If the information is empty or irrelevant, just say "{UNKNOWN_ANSWER}".\n'
+    " - If the information is relevant but not fully conclusive, provide an answer to the extent you can but also "
+    "specify that the information is not conclusive and why.\n\n"
+    "Again, you should be sure that the answer is supported by the information provided!\n\n"
+    "Try to keep your answer concise. But also highlight uncertainties you may have should there be substantial ones, "
+    "or assumptions you made.\n\n"
+    "Here is the contextual information:\n"
+    f"{SEPARATOR_LINE_LONG}\n\n"
+    "*Initial Answer that was found to be lacking:\n"
+    f"{SEPARATOR_LINE}\n"
+    "{initial_answer}\n"
+    f"{SEPARATOR_LINE}\n\n"
+    "And here are relevant document information that support the sub-question answers, "
+    "or that are relevant for the actual question:\n"
+    f"{SEPARATOR_LINE}\n"
+    "{relevant_docs}\n"
+    f"{SEPARATOR_LINE}\n\n"
+    "Lastly, here is the question I want you to answer based on the information above:\n"
+    f"{SEPARATOR_LINE}\n"
+    "{question}\n"
+    f"{SEPARATOR_LINE}\n\n"
+    "Please keep your answer brief and concise, and focus on facts and data.\n\n"
+    "Answer:"
+).strip()
 
 
-INITIAL_REFINED_ANSWER_COMPARISON_PROMPT = f"""
-For the given question, please compare the initial answer and the refined answer and determine if the refined answer is \
-substantially better than the initial answer, not just a bit better. Better could mean:
- - additional information
- - more comprehensive information
- - more concise information
- - more structured information
- - more details
- - new bullet points
- - substantially more document citations ([D1], [D2], [D3], etc.)
-
-Put yourself in the shoes of the user and think about whether the refined answer is really substantially better and \
-delivers really new insights than the initial answer.
-
-Here is the question:
-{SEPARATOR_LINE}
-{{question}}
-{SEPARATOR_LINE}
-
-Here is the initial answer:
-{SEPARATOR_LINE}
-{{initial_answer}}
-{SEPARATOR_LINE}
-
-Here is the refined answer:
-{SEPARATOR_LINE}
-{{refined_answer}}
-{SEPARATOR_LINE}
-
-With these criteria in mind, is the refined answer substantially better than the initial answer?
-
-Please answer with a simple "{YES}" or "{NO}"
-""".strip()
+INITIAL_REFINED_ANSWER_COMPARISON_PROMPT = (
+    "For the given question, please compare the initial answer and the refined answer and determine if "
+    "the refined answer is substantially better than the initial answer, not just a bit better. Better could mean:\n"
+    " - additional information\n"
+    " - more comprehensive information\n"
+    " - more concise information\n"
+    " - more structured information\n"
+    " - more details\n"
+    " - new bullet points\n"
+    " - substantially more document citations ([D1], [D2], [D3], etc.)\n\n"
+    "Put yourself in the shoes of the user and think about whether the refined answer is really substantially "
+    "better and delivers really new insights than the initial answer.\n\n"
+    "Here is the question:\n"
+    f"{SEPARATOR_LINE}\n"
+    "{question}\n"
+    f"{SEPARATOR_LINE}\n\n"
+    "Here is the initial answer:\n"
+    f"{SEPARATOR_LINE}\n"
+    "{initial_answer}\n"
+    f"{SEPARATOR_LINE}\n\n"
+    "Here is the refined answer:\n"
+    f"{SEPARATOR_LINE}\n"
+    "{refined_answer}\n"
+    f"{SEPARATOR_LINE}\n\n"
+    "With these criteria in mind, is the refined answer substantially better than the initial answer?\n\n"
+    f'Please answer with a simple "{YES}" or "{NO}".'
+).strip()

backend/onyx/prompts/chat_prompts.py

@@ -2,10 +2,8 @@ from onyx.prompts.constants import GENERAL_SEP_PAT
 from onyx.prompts.constants import QUESTION_PAT
 
 REQUIRE_CITATION_STATEMENT = """
-Cite relevant statements INLINE using the format [1], [2], [3], etc. to reference the document number. \
-DO NOT provide any links following the citations. In other words, avoid using the format [1](https://example.com). \
-Avoid using double brackets like [[1]]. To cite multiple documents, use [1], [2] format instead of [1, 2]. \
-Try to cite inline as opposed to leaving all citations until the very end of the response.
+Cite relevant statements INLINE using the format [1], [2], [3], etc to reference the document number, \
+DO NOT provide a reference section at the end and DO NOT provide any links following the citations.
 """.rstrip()
 
 NO_CITATION_STATEMENT = """

backend/onyx/redis/redis_connector.py

@@ -17,8 +17,6 @@ class RedisConnector:
     associated background tasks / associated redis interactions."""
 
     def __init__(self, tenant_id: str | None, id: int) -> None:
-        """id: a connector credential pair id"""
-
         self.tenant_id: str | None = tenant_id
         self.id: int = id
         self.redis: redis.Redis = get_redis_client(tenant_id=tenant_id)

backend/onyx/redis/redis_connector_credential_pair.py

@@ -2,7 +2,6 @@ import time
 from typing import cast
 from uuid import uuid4
 
-import redis
 from celery import Celery
 from redis import Redis
 from redis.lock import Lock as RedisLock
@@ -13,7 +12,6 @@ from onyx.configs.constants import CELERY_VESPA_SYNC_BEAT_LOCK_TIMEOUT
 from onyx.configs.constants import OnyxCeleryPriority
 from onyx.configs.constants import OnyxCeleryQueues
 from onyx.configs.constants import OnyxCeleryTask
-from onyx.configs.constants import OnyxRedisConstants
 from onyx.db.connector_credential_pair import get_connector_credential_pair_from_id
 from onyx.db.document import (
     construct_document_select_for_connector_credential_pair_by_needs_sync,
@@ -30,14 +28,21 @@ class RedisConnectorCredentialPair(RedisObjectHelper):
     all connectors and is not per connector."""
 
     PREFIX = "connectorsync"
+    FENCE_PREFIX = PREFIX + "_fence"
     TASKSET_PREFIX = PREFIX + "_taskset"
 
+    SYNCING_PREFIX = PREFIX + ":vespa_syncing"
+
     def __init__(self, tenant_id: str | None, id: int) -> None:
         super().__init__(tenant_id, str(id))
 
         # documents that should be skipped
         self.skip_docs: set[str] = set()
 
+    @classmethod
+    def get_fence_key(cls) -> str:
+        return RedisConnectorCredentialPair.FENCE_PREFIX
+
     @classmethod
     def get_taskset_key(cls) -> str:
         return RedisConnectorCredentialPair.TASKSET_PREFIX
@@ -46,14 +51,19 @@ class RedisConnectorCredentialPair(RedisObjectHelper):
     def taskset_key(self) -> str:
         """Notice that this is intentionally reusing the same taskset for all
         connector syncs"""
-        # example: connectorsync_taskset
+        # example: connector_taskset
         return f"{self.TASKSET_PREFIX}"
 
     def set_skip_docs(self, skip_docs: set[str]) -> None:
-        # documents that should be skipped. Note that this class updates
+        # documents that should be skipped. Note that this classes updates
         # the list on the fly
         self.skip_docs = skip_docs
 
+    @staticmethod
+    def make_redis_syncing_key(doc_id: str) -> str:
+        """used to create a key in redis to block a doc from syncing"""
+        return f"{RedisConnectorCredentialPair.SYNCING_PREFIX}:{doc_id}"
+
     def generate_tasks(
         self,
         max_tasks: int,
@@ -101,6 +111,15 @@ class RedisConnectorCredentialPair(RedisObjectHelper):
             if doc.id in self.skip_docs:
                 continue
 
+            # an arbitrary number in seconds to prevent the same doc from syncing repeatedly
+            # SYNC_EXPIRATION = 24 * 60 * 60
+
+            # a quick hack that can be uncommented to prevent a doc from resyncing over and over
+            # redis_syncing_key = self.make_redis_syncing_key(doc.id)
+            # if redis_client.exists(redis_syncing_key):
+            #     continue
+            # redis_client.set(redis_syncing_key, custom_task_id, ex=SYNC_EXPIRATION)
+
             # celery's default task id format is "dd32ded3-00aa-4884-8b21-42f8332e7fac"
             # the key for the result is "celery-task-meta-dd32ded3-00aa-4884-8b21-42f8332e7fac"
             # we prefix the task id so it's easier to keep track of who created the task
@@ -129,78 +148,3 @@ class RedisConnectorCredentialPair(RedisObjectHelper):
                 break
 
         return len(async_results), num_docs
-
-
-class RedisGlobalConnectorCredentialPair:
-    """This class is used to scan documents by cc_pair in the db and collect them into
-    a unified set for syncing.
-
-    It differs from the other redis helpers in that the taskset used spans
-    all connectors and is not per connector."""
-
-    PREFIX = "connectorsync"
-    FENCE_KEY = PREFIX + "_fence"
-    TASKSET_KEY = PREFIX + "_taskset"
-
-    def __init__(self, redis: redis.Redis) -> None:
-        self.redis = redis
-
-    @property
-    def fenced(self) -> bool:
-        if self.redis.exists(self.fence_key):
-            return True
-
-        return False
-
-    @property
-    def payload(self) -> int | None:
-        bytes = self.redis.get(self.fence_key)
-        if bytes is None:
-            return None
-
-        progress = int(cast(int, bytes))
-        return progress
-
-    def get_remaining(self) -> int:
-        remaining = cast(int, self.redis.scard(self.taskset_key))
-        return remaining
-
-    @property
-    def fence_key(self) -> str:
-        """Notice that this is intentionally reusing the same fence for all
-        connector syncs"""
-        # example: connectorsync_fence
-        return f"{self.FENCE_KEY}"
-
-    @property
-    def taskset_key(self) -> str:
-        """Notice that this is intentionally reusing the same taskset for all
-        connector syncs"""
-        # example: connectorsync_taskset
-        return f"{self.TASKSET_KEY}"
-
-    def set_fence(self, payload: int | None) -> None:
-        if payload is None:
-            self.redis.srem(OnyxRedisConstants.ACTIVE_FENCES, self.fence_key)
-            self.redis.delete(self.fence_key)
-            return
-
-        self.redis.set(self.fence_key, payload)
-        self.redis.sadd(OnyxRedisConstants.ACTIVE_FENCES, self.fence_key)
-
-    def delete_taskset(self) -> None:
-        self.redis.delete(self.taskset_key)
-
-    def reset(self) -> None:
-        self.redis.srem(OnyxRedisConstants.ACTIVE_FENCES, self.fence_key)
-        self.redis.delete(self.taskset_key)
-        self.redis.delete(self.fence_key)
-
-    @staticmethod
-    def reset_all(r: redis.Redis) -> None:
-        r.srem(
-            OnyxRedisConstants.ACTIVE_FENCES,
-            RedisGlobalConnectorCredentialPair.FENCE_KEY,
-        )
-        r.delete(RedisGlobalConnectorCredentialPair.TASKSET_KEY)
-        r.delete(RedisGlobalConnectorCredentialPair.FENCE_KEY)

backend/onyx/redis/redis_connector_delete.py

@@ -14,7 +14,6 @@ from onyx.configs.constants import CELERY_VESPA_SYNC_BEAT_LOCK_TIMEOUT
 from onyx.configs.constants import OnyxCeleryPriority
 from onyx.configs.constants import OnyxCeleryQueues
 from onyx.configs.constants import OnyxCeleryTask
-from onyx.configs.constants import OnyxRedisConstants
 from onyx.db.connector_credential_pair import get_connector_credential_pair_from_id
 from onyx.db.document import construct_document_select_for_connector_credential_pair
 from onyx.db.models import Document as DbDocument
@@ -70,12 +69,10 @@ class RedisConnectorDelete:
 
     def set_fence(self, payload: RedisConnectorDeletePayload | None) -> None:
         if not payload:
-            self.redis.srem(OnyxRedisConstants.ACTIVE_FENCES, self.fence_key)
             self.redis.delete(self.fence_key)
             return
 
         self.redis.set(self.fence_key, payload.model_dump_json())
-        self.redis.sadd(OnyxRedisConstants.ACTIVE_FENCES, self.fence_key)
 
     def _generate_task_id(self) -> str:
         # celery's default task id format is "dd32ded3-00aa-4884-8b21-42f8332e7fac"
@@ -139,7 +136,6 @@ class RedisConnectorDelete:
         return len(async_results)
 
     def reset(self) -> None:
-        self.redis.srem(OnyxRedisConstants.ACTIVE_FENCES, self.fence_key)
         self.redis.delete(self.taskset_key)
         self.redis.delete(self.fence_key)
 

backend/onyx/redis/redis_connector_doc_perm_sync.py

@@ -1,6 +1,5 @@
 import time
 from datetime import datetime
-from typing import Any
 from typing import cast
 from uuid import uuid4
 
@@ -14,7 +13,6 @@ from onyx.configs.constants import CELERY_GENERIC_BEAT_LOCK_TIMEOUT
 from onyx.configs.constants import OnyxCeleryPriority
 from onyx.configs.constants import OnyxCeleryQueues
 from onyx.configs.constants import OnyxCeleryTask
-from onyx.configs.constants import OnyxRedisConstants
 from onyx.redis.redis_pool import SCAN_ITER_COUNT_DEFAULT
 
 
@@ -97,7 +95,7 @@ class RedisConnectorPermissionSync:
     @property
     def payload(self) -> RedisConnectorPermissionSyncPayload | None:
         # read related data and evaluate/print task progress
-        fence_bytes = cast(Any, self.redis.get(self.fence_key))
+        fence_bytes = cast(bytes, self.redis.get(self.fence_key))
         if fence_bytes is None:
             return None
 
@@ -113,12 +111,10 @@ class RedisConnectorPermissionSync:
         payload: RedisConnectorPermissionSyncPayload | None,
     ) -> None:
         if not payload:
-            self.redis.srem(OnyxRedisConstants.ACTIVE_FENCES, self.fence_key)
             self.redis.delete(self.fence_key)
             return
 
         self.redis.set(self.fence_key, payload.model_dump_json())
-        self.redis.sadd(OnyxRedisConstants.ACTIVE_FENCES, self.fence_key)
 
     def set_active(self) -> None:
         """This sets a signal to keep the permissioning flow from getting cleaned up within
@@ -200,7 +196,6 @@ class RedisConnectorPermissionSync:
         return len(async_results)
 
     def reset(self) -> None:
-        self.redis.srem(OnyxRedisConstants.ACTIVE_FENCES, self.fence_key)
         self.redis.delete(self.active_key)
         self.redis.delete(self.generator_progress_key)
         self.redis.delete(self.generator_complete_key)

backend/onyx/redis/redis_connector_ext_group_sync.py

@@ -1,5 +1,4 @@
 from datetime import datetime
-from typing import Any
 from typing import cast
 
 import redis
@@ -83,7 +82,7 @@ class RedisConnectorExternalGroupSync:
     @property
     def payload(self) -> RedisConnectorExternalGroupSyncPayload | None:
         # read related data and evaluate/print task progress
-        fence_bytes = cast(Any, self.redis.get(self.fence_key))
+        fence_bytes = cast(bytes, self.redis.get(self.fence_key))
         if fence_bytes is None:
             return None
 

backend/onyx/redis/redis_connector_index.py

@@ -1,13 +1,10 @@
 from datetime import datetime
-from typing import Any
 from typing import cast
 from uuid import uuid4
 
 import redis
 from pydantic import BaseModel
 
-from onyx.configs.constants import OnyxRedisConstants
-
 
 class RedisConnectorIndexPayload(BaseModel):
     index_attempt_id: int | None
@@ -92,7 +89,7 @@ class RedisConnectorIndex:
     @property
     def payload(self) -> RedisConnectorIndexPayload | None:
         # read related data and evaluate/print task progress
-        fence_bytes = cast(Any, self.redis.get(self.fence_key))
+        fence_bytes = cast(bytes, self.redis.get(self.fence_key))
         if fence_bytes is None:
             return None
 
@@ -106,12 +103,10 @@ class RedisConnectorIndex:
         payload: RedisConnectorIndexPayload | None,
     ) -> None:
         if not payload:
-            self.redis.srem(OnyxRedisConstants.ACTIVE_FENCES, self.fence_key)
             self.redis.delete(self.fence_key)
             return
 
         self.redis.set(self.fence_key, payload.model_dump_json())
-        self.redis.sadd(OnyxRedisConstants.ACTIVE_FENCES, self.fence_key)
 
     def terminating(self, celery_task_id: str) -> bool:
         if self.redis.exists(f"{self.terminate_key}_{celery_task_id}"):
@@ -193,7 +188,6 @@ class RedisConnectorIndex:
         return status
 
     def reset(self) -> None:
-        self.redis.srem(OnyxRedisConstants.ACTIVE_FENCES, self.fence_key)
         self.redis.delete(self.active_key)
         self.redis.delete(self.generator_lock_key)
         self.redis.delete(self.generator_progress_key)

backend/onyx/redis/redis_connector_prune.py

@@ -11,7 +11,6 @@ from onyx.configs.constants import CELERY_GENERIC_BEAT_LOCK_TIMEOUT
 from onyx.configs.constants import OnyxCeleryPriority
 from onyx.configs.constants import OnyxCeleryQueues
 from onyx.configs.constants import OnyxCeleryTask
-from onyx.configs.constants import OnyxRedisConstants
 from onyx.db.connector_credential_pair import get_connector_credential_pair_from_id
 from onyx.redis.redis_pool import SCAN_ITER_COUNT_DEFAULT
 
@@ -80,12 +79,10 @@ class RedisConnectorPrune:
 
     def set_fence(self, value: bool) -> None:
         if not value:
-            self.redis.srem(OnyxRedisConstants.ACTIVE_FENCES, self.fence_key)
             self.redis.delete(self.fence_key)
             return
 
         self.redis.set(self.fence_key, 0)
-        self.redis.sadd(OnyxRedisConstants.ACTIVE_FENCES, self.fence_key)
 
     @property
     def generator_complete(self) -> int | None:
@@ -161,7 +158,6 @@ class RedisConnectorPrune:
         return len(async_results)
 
     def reset(self) -> None:
-        self.redis.srem(OnyxRedisConstants.ACTIVE_FENCES, self.fence_key)
         self.redis.delete(self.generator_progress_key)
         self.redis.delete(self.generator_complete_key)
         self.redis.delete(self.taskset_key)

backend/onyx/redis/redis_document_set.py

@@ -13,7 +13,6 @@ from onyx.configs.constants import CELERY_VESPA_SYNC_BEAT_LOCK_TIMEOUT
 from onyx.configs.constants import OnyxCeleryPriority
 from onyx.configs.constants import OnyxCeleryQueues
 from onyx.configs.constants import OnyxCeleryTask
-from onyx.configs.constants import OnyxRedisConstants
 from onyx.db.document_set import construct_document_select_by_docset
 from onyx.db.models import Document
 from onyx.redis.redis_object_helper import RedisObjectHelper
@@ -36,12 +35,10 @@ class RedisDocumentSet(RedisObjectHelper):
 
     def set_fence(self, payload: int | None) -> None:
         if payload is None:
-            self.redis.srem(OnyxRedisConstants.ACTIVE_FENCES, self.fence_key)
             self.redis.delete(self.fence_key)
             return
 
         self.redis.set(self.fence_key, payload)
-        self.redis.sadd(OnyxRedisConstants.ACTIVE_FENCES, self.fence_key)
 
     @property
     def payload(self) -> int | None:
@@ -99,7 +96,6 @@ class RedisDocumentSet(RedisObjectHelper):
         return len(async_results), len(async_results)
 
     def reset(self) -> None:
-        self.redis.srem(OnyxRedisConstants.ACTIVE_FENCES, self.fence_key)
         self.redis.delete(self.taskset_key)
         self.redis.delete(self.fence_key)
 

backend/onyx/redis/redis_pool.py

@@ -25,7 +25,6 @@ from onyx.configs.app_configs import REDIS_REPLICA_HOST
 from onyx.configs.app_configs import REDIS_SSL
 from onyx.configs.app_configs import REDIS_SSL_CA_CERTS
 from onyx.configs.app_configs import REDIS_SSL_CERT_REQS
-from onyx.configs.constants import FASTAPI_USERS_AUTH_COOKIE_NAME
 from onyx.configs.constants import REDIS_SOCKET_KEEPALIVE_OPTIONS
 from onyx.utils.logger import setup_logger
 
@@ -114,8 +113,6 @@ class TenantRedis(redis.Redis):
             "reacquire",
             "create_lock",
             "startswith",
-            "smembers",
-            "sismember",
             "sadd",
             "srem",
             "scard",
@@ -288,7 +285,7 @@ async def get_async_redis_connection() -> aioredis.Redis:
 
 
 async def retrieve_auth_token_data_from_redis(request: Request) -> dict | None:
-    token = request.cookies.get(FASTAPI_USERS_AUTH_COOKIE_NAME)
+    token = request.cookies.get("fastapiusersauth")
     if not token:
         logger.debug("No auth token cookie found")
         return None

backend/onyx/redis/redis_usergroup.py

@@ -13,7 +13,6 @@ from onyx.configs.constants import CELERY_VESPA_SYNC_BEAT_LOCK_TIMEOUT
 from onyx.configs.constants import OnyxCeleryPriority
 from onyx.configs.constants import OnyxCeleryQueues
 from onyx.configs.constants import OnyxCeleryTask
-from onyx.configs.constants import OnyxRedisConstants
 from onyx.db.models import Document
 from onyx.redis.redis_object_helper import RedisObjectHelper
 from onyx.utils.variable_functionality import fetch_versioned_implementation
@@ -37,12 +36,10 @@ class RedisUserGroup(RedisObjectHelper):
 
     def set_fence(self, payload: int | None) -> None:
         if payload is None:
-            self.redis.srem(OnyxRedisConstants.ACTIVE_FENCES, self.fence_key)
             self.redis.delete(self.fence_key)
             return
 
         self.redis.set(self.fence_key, payload)
-        self.redis.sadd(OnyxRedisConstants.ACTIVE_FENCES, self.fence_key)
 
     @property
     def payload(self) -> int | None:
@@ -112,7 +109,6 @@ class RedisUserGroup(RedisObjectHelper):
         return len(async_results), len(async_results)
 
     def reset(self) -> None:
-        self.redis.srem(OnyxRedisConstants.ACTIVE_FENCES, self.fence_key)
         self.redis.delete(self.taskset_key)
         self.redis.delete(self.fence_key)
 

backend/onyx/seeding/prompts.yaml

@@ -20,7 +20,7 @@ prompts:
       The documents may not all be relevant, ignore any documents that are not directly relevant
       to the most recent user query.
 
-      I have not read or seen any of the documents and do not want to read them. Do not refer to them by Document number.
+      I have not read or seen any of the documents and do not want to read them.
 
       If there are no relevant documents, refer to the chat history and your internal knowledge.
     # Inject a statement at the end of system prompt to inform the LLM of the current date/time

backend/onyx/server/manage/models.py

@@ -215,7 +215,6 @@ class SlackChannelConfig(BaseModel):
     # XXX this is going away soon
     standard_answer_categories: list[StandardAnswerCategory]
     enable_auto_filters: bool
-    is_default: bool
 
     @classmethod
     def from_model(
@@ -238,7 +237,6 @@ class SlackChannelConfig(BaseModel):
                 for standard_answer_category_model in slack_channel_config_model.standard_answer_categories
             ],
             enable_auto_filters=slack_channel_config_model.enable_auto_filters,
-            is_default=slack_channel_config_model.is_default,
         )
 
 
@@ -281,8 +279,3 @@ class AllUsersResponse(BaseModel):
     accepted_pages: int
     invited_pages: int
     slack_users_pages: int
-
-
-class SlackChannel(BaseModel):
-    id: str
-    name: str

backend/onyx/server/manage/slack_bot.py

@@ -1,10 +1,6 @@
-from typing import Any
-
 from fastapi import APIRouter
 from fastapi import Depends
 from fastapi import HTTPException
-from slack_sdk import WebClient
-from slack_sdk.errors import SlackApiError
 from sqlalchemy.orm import Session
 
 from onyx.auth.users import current_admin_user
@@ -16,7 +12,6 @@ from onyx.db.models import ChannelConfig
 from onyx.db.models import User
 from onyx.db.persona import get_persona_by_id
 from onyx.db.slack_bot import fetch_slack_bot
-from onyx.db.slack_bot import fetch_slack_bot_tokens
 from onyx.db.slack_bot import fetch_slack_bots
 from onyx.db.slack_bot import insert_slack_bot
 from onyx.db.slack_bot import remove_slack_bot
@@ -30,7 +25,6 @@ from onyx.db.slack_channel_config import update_slack_channel_config
 from onyx.onyxbot.slack.config import validate_channel_name
 from onyx.server.manage.models import SlackBot
 from onyx.server.manage.models import SlackBotCreationRequest
-from onyx.server.manage.models import SlackChannel
 from onyx.server.manage.models import SlackChannelConfig
 from onyx.server.manage.models import SlackChannelConfigCreationRequest
 from onyx.server.manage.validate_tokens import validate_app_token
@@ -54,6 +48,12 @@ def _form_channel_config(
     answer_filters = slack_channel_config_creation_request.answer_filters
     follow_up_tags = slack_channel_config_creation_request.follow_up_tags
 
+    if not raw_channel_name:
+        raise HTTPException(
+            status_code=400,
+            detail="Must provide at least one channel name",
+        )
+
     try:
         cleaned_channel_name = validate_channel_name(
             db_session=db_session,
@@ -108,12 +108,6 @@ def create_slack_channel_config(
         current_slack_channel_config_id=None,
     )
 
-    if channel_config["channel_name"] is None:
-        raise HTTPException(
-            status_code=400,
-            detail="Channel name is required",
-        )
-
     persona_id = None
     if slack_channel_config_creation_request.persona_id is not None:
         persona_id = slack_channel_config_creation_request.persona_id
@@ -126,11 +120,11 @@ def create_slack_channel_config(
         ).id
 
     slack_channel_config_model = insert_slack_channel_config(
-        db_session=db_session,
         slack_bot_id=slack_channel_config_creation_request.slack_bot_id,
         persona_id=persona_id,
         channel_config=channel_config,
         standard_answer_category_ids=slack_channel_config_creation_request.standard_answer_categories,
+        db_session=db_session,
         enable_auto_filters=slack_channel_config_creation_request.enable_auto_filters,
     )
     return SlackChannelConfig.from_model(slack_channel_config_model)
@@ -241,23 +235,6 @@ def create_bot(
         app_token=slack_bot_creation_request.app_token,
     )
 
-    # Create a default Slack channel config
-    default_channel_config = ChannelConfig(
-        channel_name=None,
-        respond_member_group_list=[],
-        answer_filters=[],
-        follow_up_tags=[],
-    )
-    insert_slack_channel_config(
-        db_session=db_session,
-        slack_bot_id=slack_bot_model.id,
-        persona_id=None,
-        channel_config=default_channel_config,
-        standard_answer_category_ids=[],
-        enable_auto_filters=False,
-        is_default=True,
-    )
-
     create_milestone_and_report(
         user=None,
         distinct_id=tenant_id or "N/A",
@@ -338,48 +315,3 @@ def list_bot_configs(
         SlackChannelConfig.from_model(slack_bot_config_model)
         for slack_bot_config_model in slack_bot_config_models
     ]
-
-
-@router.get(
-    "/admin/slack-app/bots/{bot_id}/channels",
-)
-def get_all_channels_from_slack_api(
-    bot_id: int,
-    db_session: Session = Depends(get_session),
-    _: User | None = Depends(current_admin_user),
-) -> list[SlackChannel]:
-    tokens = fetch_slack_bot_tokens(db_session, bot_id)
-    if not tokens or "bot_token" not in tokens:
-        raise HTTPException(
-            status_code=404, detail="Bot token not found for the given bot ID"
-        )
-
-    bot_token = tokens["bot_token"]
-    client = WebClient(token=bot_token)
-
-    try:
-        channels = []
-        cursor = None
-        while True:
-            response = client.conversations_list(
-                types="public_channel,private_channel",
-                exclude_archived=True,
-                limit=1000,
-                cursor=cursor,
-            )
-            for channel in response["channels"]:
-                channels.append(SlackChannel(id=channel["id"], name=channel["name"]))
-
-            response_metadata: dict[str, Any] = response.get("response_metadata", {})
-            if isinstance(response_metadata, dict):
-                cursor = response_metadata.get("next_cursor")
-                if not cursor:
-                    break
-            else:
-                break
-
-        return channels
-    except SlackApiError as e:
-        raise HTTPException(
-            status_code=500, detail=f"Error fetching channels from Slack API: {str(e)}"
-        )

backend/onyx/server/manage/users.py

@@ -38,7 +38,6 @@ from onyx.configs.app_configs import ENABLE_EMAIL_INVITES
 from onyx.configs.app_configs import SESSION_EXPIRE_TIME_SECONDS
 from onyx.configs.app_configs import VALID_EMAIL_DOMAINS
 from onyx.configs.constants import AuthType
-from onyx.configs.constants import FASTAPI_USERS_AUTH_COOKIE_NAME
 from onyx.db.api_key import is_api_key_email_address
 from onyx.db.auth import get_total_users_count
 from onyx.db.engine import CURRENT_TENANT_ID_CONTEXTVAR
@@ -172,14 +171,13 @@ def list_all_users(
     accepted_page: int | None = None,
     slack_users_page: int | None = None,
     invited_page: int | None = None,
-    include_api_keys: bool = False,
     _: User | None = Depends(current_curator_or_admin_user),
     db_session: Session = Depends(get_session),
 ) -> AllUsersResponse:
     users = [
         user
         for user in get_all_users(db_session, email_filter_string=q)
-        if (include_api_keys or not is_api_key_email_address(user.email))
+        if not is_api_key_email_address(user.email)
     ]
 
     slack_users = [user for user in users if user.role == UserRole.SLACK_USER]
@@ -273,8 +271,6 @@ def bulk_invite_users(
 
     tenant_id = CURRENT_TENANT_ID_CONTEXTVAR.get()
     new_invited_emails = []
-    email: str
-
     try:
         for email in emails:
             email_info = validate_email(email)
@@ -480,7 +476,7 @@ def get_current_token_expiration_jwt(
 
     try:
         # Get the JWT from the cookie
-        jwt_token = request.cookies.get(FASTAPI_USERS_AUTH_COOKIE_NAME)
+        jwt_token = request.cookies.get("fastapiusersauth")
         if not jwt_token:
             logger.error("No JWT token found in cookies")
             return None

backend/onyx/server/query_and_chat/chat_backend.py

@@ -420,6 +420,10 @@ def handle_new_chat_message(
 
     def stream_generator() -> Generator[str, None, None]:
         try:
+            import time
+
+            start_time = time.time()
+            n = 0
             for packet in stream_chat_message(
                 new_msg_req=chat_message_req,
                 user=user,
@@ -431,6 +435,19 @@ def handle_new_chat_message(
                 ),
                 is_connected=is_connected_func,
             ):
+                if "top_documents" in packet:
+                    to_first_docs = time.time() - start_time
+                    print(f"Time to first docs: {to_first_docs}")
+                    print(packet)
+                elif "answer_piece" in packet:
+                    to_answer_piece = time.time() - start_time
+                    if n == 1:
+                        print(f"Time to answer piece: {to_answer_piece}")
+                        print(packet)
+                    n += 1
+
+                # time_since_start = time.time() - start_time
+                # print(f"Time since start: {time_since_start}")
                 yield json.dumps(packet) if isinstance(packet, dict) else packet
 
         except Exception as e:
@@ -717,6 +734,8 @@ def upload_files_for_chat(
             else ChatFileType.PLAIN_TEXT
         )
 
+        file_content = file.file.read()  # Read the file content
+
         if file_type == ChatFileType.IMAGE:
             file_content_io = file.file
             # NOTE: Image conversion to JPEG used to be enforced here.
@@ -725,7 +744,7 @@ def upload_files_for_chat(
             # 2. Maintain transparency in formats like PNG
             # 3. Ameliorate issue with file conversion
         else:
-            file_content_io = io.BytesIO(file.file.read())
+            file_content_io = io.BytesIO(file_content)
 
         new_content_type = file.content_type
 
@@ -743,7 +762,7 @@ def upload_files_for_chat(
         # to re-extract it every time we send a message
         if file_type == ChatFileType.DOC:
             extracted_text = extract_file_text(
-                file=file_content_io,  # use the bytes we already read
+                file=io.BytesIO(file_content),  # use the bytes we already read
                 file_name=file.filename or "",
             )
             text_file_id = str(uuid.uuid4())

backend/onyx/tools/tool_implementations/internet_search/internet_search_tool.py

@@ -218,9 +218,6 @@ class InternetSearchTool(Tool):
             headers=self.headers,
             params={"q": query, "count": self.num_results},
         )
-
-        response.raise_for_status()
-
         results = response.json()
 
         # If no hits, Bing does not include the webPages key

backend/onyx/tools/tool_implementations/search/search_tool.py

@@ -7,6 +7,7 @@ from typing import cast
 from sqlalchemy.orm import Session
 
 from onyx.chat.chat_utils import llm_doc_from_inference_section
+from onyx.chat.llm_response_handler import LLMCall
 from onyx.chat.models import AnswerStyleConfig
 from onyx.chat.models import ContextualPruningConfig
 from onyx.chat.models import DocumentPruningConfig
@@ -370,6 +371,41 @@ class SearchTool(Tool):
             prompt_config=self.prompt_config,
         )
 
+    """Other utility functions"""
+
+    @classmethod
+    def get_search_result(
+        cls, llm_call: LLMCall
+    ) -> tuple[list[LlmDoc], list[LlmDoc]] | None:
+        """
+        Returns the final search results and a map of docs to their original search rank (which is what is displayed to user)
+        """
+        if not llm_call.tool_call_info:
+            return None
+
+        final_search_results = []
+        initial_search_results = []
+
+        for yield_item in llm_call.tool_call_info:
+            if (
+                isinstance(yield_item, ToolResponse)
+                and yield_item.id == FINAL_CONTEXT_DOCUMENTS_ID
+            ):
+                final_search_results = cast(list[LlmDoc], yield_item.response)
+            elif (
+                isinstance(yield_item, ToolResponse)
+                and yield_item.id == SEARCH_DOC_CONTENT_ID
+            ):
+                search_contexts = yield_item.response.contexts
+                # original_doc_search_rank = 1
+                for doc in search_contexts:
+                    if doc.document_id not in initial_search_results:
+                        initial_search_results.append(doc)
+
+                initial_search_results = cast(list[LlmDoc], initial_search_results)
+
+        return final_search_results, initial_search_results
+
 
 # Allows yielding the same responses as a SearchTool without being a SearchTool.
 # SearchTool passed in to allow for access to SearchTool properties.

backend/requirements/default.txt

@@ -37,7 +37,7 @@ langchainhub==0.1.21
 langgraph==0.2.59
 langgraph-checkpoint==2.0.5
 langgraph-sdk==0.1.44
-litellm==1.60.2
+litellm==1.55.4
 lxml==5.3.0
 lxml_html_clean==0.2.2
 llama-index==0.9.45
@@ -46,7 +46,7 @@ msal==1.28.0
 nltk==3.8.1
 Office365-REST-Python-Client==2.5.9
 oauthlib==3.2.2
-openai==1.61.0
+openai==1.55.3
 openpyxl==3.1.2
 playwright==1.41.2
 psutil==5.9.5

backend/requirements/model_server.txt

@@ -3,7 +3,7 @@ cohere==5.6.1
 fastapi==0.109.2
 google-cloud-aiplatform==1.58.0
 numpy==1.26.4
-openai==1.61.0
+openai==1.55.3
 pydantic==2.8.2
 retry==0.9.2
 safetensors==0.4.2
@@ -12,5 +12,5 @@ torch==2.2.0
 transformers==4.39.2
 uvicorn==0.21.1
 voyageai==0.2.3
-litellm==1.60.2
+litellm==1.55.4
 sentry-sdk[fastapi,celery,starlette]==2.14.0

backend/scripts/chat_feedback_dump.py

@@ -198,7 +198,6 @@ def process_all_chat_feedback(onyx_url: str, api_key: str | None) -> None:
         r_sessions = get_chat_sessions(onyx_url, headers, user_id)
         logger.info(f"user={user_id} num_sessions={len(r_sessions.sessions)}")
         for session in r_sessions.sessions:
-            s: ChatSessionSnapshot
             try:
                 s = get_session_history(onyx_url, headers, session.id)
             except requests.exceptions.HTTPError:

backend/scripts/sources_selection_analysis.py

@@ -11,8 +11,6 @@ from typing import Optional
 
 import requests
 
-from onyx.configs.constants import FASTAPI_USERS_AUTH_COOKIE_NAME
-
 parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 sys.path.append(parent_dir)
 
@@ -376,11 +374,7 @@ class SelectionAnalysis:
         Returns:
             dict: The Onyx API response content
         """
-        cookies = (
-            {FASTAPI_USERS_AUTH_COOKIE_NAME: self._auth_cookie}
-            if self._auth_cookie
-            else {}
-        )
+        cookies = {"fastapiusersauth": self._auth_cookie} if self._auth_cookie else {}
 
         endpoint = f"http://127.0.0.1:{self._web_port}/api/direct-qa"
         query_json = {

backend/tests/daily/connectors/confluence/test_confluence_basic.py

@@ -6,7 +6,6 @@ from unittest.mock import patch
 import pytest
 
 from onyx.connectors.confluence.connector import ConfluenceConnector
-from onyx.connectors.models import Document
 
 
 @pytest.fixture
@@ -42,10 +41,6 @@ def test_confluence_connector_basic(
 
     assert len(doc_batch) == 3
 
-    page_within_a_page_doc: Document | None = None
-    page_doc: Document | None = None
-    txt_doc: Document | None = None
-
     for doc in doc_batch:
         if doc.semantic_identifier == "DailyConnectorTestSpace Home":
             page_doc = doc
@@ -54,7 +49,6 @@ def test_confluence_connector_basic(
         elif doc.semantic_identifier == "Page Within A Page":
             page_within_a_page_doc = doc
 
-    assert page_within_a_page_doc is not None
     assert page_within_a_page_doc.semantic_identifier == "Page Within A Page"
     assert page_within_a_page_doc.primary_owners
     assert page_within_a_page_doc.primary_owners[0].email == "hagen@danswer.ai"
@@ -68,7 +62,6 @@ def test_confluence_connector_basic(
         == "https://danswerai.atlassian.net/wiki/spaces/DailyConne/pages/200769540/Page+Within+A+Page"
     )
 
-    assert page_doc is not None
     assert page_doc.semantic_identifier == "DailyConnectorTestSpace Home"
     assert page_doc.metadata["labels"] == ["testlabel"]
     assert page_doc.primary_owners
@@ -82,7 +75,6 @@ def test_confluence_connector_basic(
         == "https://danswerai.atlassian.net/wiki/spaces/DailyConne/overview"
     )
 
-    assert txt_doc is not None
     assert txt_doc.semantic_identifier == "small-file.txt"
     assert len(txt_doc.sections) == 1
     assert txt_doc.sections[0].text == "small"

backend/tests/daily/connectors/gmail/test_gmail_connector.py

@@ -110,8 +110,6 @@ def test_docs_retrieval(
 
     for doc in retrieved_docs:
         id = doc.id
-        retrieved_primary_owner_emails: set[str | None] = set()
-        retrieved_secondary_owner_emails: set[str | None] = set()
         if doc.primary_owners:
             retrieved_primary_owner_emails = set(
                 [owner.email for owner in doc.primary_owners]

backend/tests/integration/common_utils/managers/user.py

@@ -7,7 +7,6 @@ import requests
 from requests import HTTPError
 
 from onyx.auth.schemas import UserRole
-from onyx.configs.constants import FASTAPI_USERS_AUTH_COOKIE_NAME
 from onyx.server.documents.models import PaginatedReturn
 from onyx.server.models import FullUserSnapshot
 from tests.integration.common_utils.constants import API_SERVER_URL
@@ -83,7 +82,7 @@ class UserManager:
         response.raise_for_status()
 
         cookies = response.cookies.get_dict()
-        session_cookie = cookies.get(FASTAPI_USERS_AUTH_COOKIE_NAME)
+        session_cookie = cookies.get("fastapiusersauth")
 
         if not session_cookie:
             raise Exception("Failed to login")
@@ -166,7 +165,6 @@ class UserManager:
         target_status: bool,
         user_performing_action: DATestUser,
     ) -> DATestUser:
-        url_substring: str
         if target_status is True:
             url_substring = "activate"
         elif target_status is False:

backend/tests/integration/connector_job_tests/google/test_google_drive_permission_sync.py

@@ -54,7 +54,6 @@ def google_drive_test_env_setup() -> (
 
     service_account_key = os.environ["FULL_CONTROL_DRIVE_SERVICE_ACCOUNT"]
     drive_id: str | None = None
-    drive_service: GoogleDriveService | None = None
 
     try:
         credentials = {

web/src/app/admin/assistants/AssistantEditor.tsx

@@ -304,7 +304,7 @@ export function AssistantEditor({
   const [isRequestSuccessful, setIsRequestSuccessful] = useState(false);
 
   const { data: userGroups } = useUserGroups();
-  // const { data: allUsers } = useUsers({ includeApiKeys: false }) as {
+  // const { data: allUsers } = useUsers() as {
   //   data: MinimalUserSnapshot[] | undefined;
   // };
 

web/src/app/admin/bots/SlackBotTable.tsx

@@ -1,9 +1,10 @@
 "use client";
 
 import { PageSelector } from "@/components/PageSelector";
+import { SlackBot } from "@/lib/types";
 import { useRouter } from "next/navigation";
 import { useEffect, useState } from "react";
-import { FiEdit } from "react-icons/fi";
+import { FiCheck, FiEdit, FiXCircle } from "react-icons/fi";
 import {
   Table,
   TableBody,
@@ -12,8 +13,6 @@ import {
   TableHeader,
   TableRow,
 } from "@/components/ui/table";
-import { Badge } from "@/components/ui/badge";
-import { SlackBot } from "@/lib/types";
 
 const NUM_IN_PAGE = 20;
 
@@ -43,7 +42,7 @@ function ClickableTableRow({
   );
 }
 
-export const SlackBotTable = ({ slackBots }: { slackBots: SlackBot[] }) => {
+export function SlackBotTable({ slackBots }: { slackBots: SlackBot[] }) {
   const [page, setPage] = useState(1);
 
   // sort by id for consistent ordering
@@ -68,9 +67,8 @@ export const SlackBotTable = ({ slackBots }: { slackBots: SlackBot[] }) => {
         <TableHeader>
           <TableRow>
             <TableHead>Name</TableHead>
-            <TableHead>Status</TableHead>
-            <TableHead>Default Config</TableHead>
             <TableHead>Channel Count</TableHead>
+            <TableHead>Enabled</TableHead>
           </TableRow>
         </TableHeader>
         <TableBody>
@@ -87,19 +85,13 @@ export const SlackBotTable = ({ slackBots }: { slackBots: SlackBot[] }) => {
                     {slackBot.name}
                   </div>
                 </TableCell>
-                <TableCell>
-                  {slackBot.enabled ? (
-                    <Badge variant="success">Enabled</Badge>
-                  ) : (
-                    <Badge variant="destructive">Disabled</Badge>
-                  )}
-                </TableCell>
-                <TableCell>
-                  <Badge variant="secondary">Default Set</Badge>
-                </TableCell>
                 <TableCell>{slackBot.configs_count}</TableCell>
                 <TableCell>
-                  {/* Add any action buttons here if needed */}
+                  {slackBot.enabled ? (
+                    <FiCheck className="text-emerald-600" size="18" />
+                  ) : (
+                    <FiXCircle className="text-red-600" size="18" />
+                  )}
                 </TableCell>
               </ClickableTableRow>
             );
@@ -107,7 +99,7 @@ export const SlackBotTable = ({ slackBots }: { slackBots: SlackBot[] }) => {
           {slackBots.length === 0 && (
             <TableRow>
               <TableCell
-                colSpan={5}
+                colSpan={4}
                 className="text-center text-muted-foreground"
               >
                 Please add a New Slack Bot to begin chatting with Danswer!
@@ -136,4 +128,4 @@ export const SlackBotTable = ({ slackBots }: { slackBots: SlackBot[] }) => {
       )}
     </div>
   );
-};
+}

web/src/app/admin/bots/SlackTokensForm.tsx

@@ -7,7 +7,6 @@ import { createSlackBot, updateSlackBot } from "./new/lib";
 import { Button } from "@/components/ui/button";
 import { Separator } from "@/components/ui/separator";
 import { useEffect } from "react";
-import { Switch } from "@/components/ui/switch";
 
 export const SlackTokensForm = ({
   isUpdate,
@@ -34,9 +33,7 @@ export const SlackTokensForm = ({
 
   return (
     <Formik
-      initialValues={{
-        ...initialValues,
-      }}
+      initialValues={initialValues}
       validationSchema={Yup.object().shape({
         bot_token: Yup.string().required(),
         app_token: Yup.string().required(),

web/src/app/admin/bots/[bot-id]/SlackChannelConfigsTable.tsx

@@ -14,10 +14,8 @@ import {
 } from "@/components/ui/table";
 import Link from "next/link";
 import { useState } from "react";
+import { FiArrowUpRight } from "react-icons/fi";
 import { deleteSlackChannelConfig, isPersonaASlackBotPersona } from "./lib";
-import { Card } from "@/components/ui/card";
-import { Button } from "@/components/ui/button";
-import { FiPlusSquare, FiSettings } from "react-icons/fi";
 
 const numToDisplay = 50;
 
@@ -34,147 +32,128 @@ export function SlackChannelConfigsTable({
 }) {
   const [page, setPage] = useState(1);
 
-  const defaultConfig = slackChannelConfigs.find((config) => config.is_default);
-  const channelConfigs = slackChannelConfigs.filter(
-    (config) => !config.is_default
-  );
+  // sort by name for consistent ordering
+  slackChannelConfigs.sort((a, b) => {
+    if (a.id < b.id) {
+      return -1;
+    } else if (a.id > b.id) {
+      return 1;
+    } else {
+      return 0;
+    }
+  });
 
   return (
-    <div className="space-y-8">
-      <div className="flex justify-between items-center mb-6">
-        <Button
-          variant="outline"
-          onClick={() => {
-            window.location.href = `/admin/bots/${slackBotId}/channels/${defaultConfig?.id}`;
-          }}
-        >
-          <FiSettings />
-          Edit Default Config
-        </Button>
-        <Link href={`/admin/bots/${slackBotId}/channels/new`}>
-          <Button variant="outline">
-            <FiPlusSquare />
-            New Channel Configuration
-          </Button>
-        </Link>
+    <div>
+      <div className="rounded-md border">
+        <Table>
+          <TableHeader>
+            <TableRow>
+              <TableHead>Channel</TableHead>
+              <TableHead>Assistant</TableHead>
+              <TableHead>Document Sets</TableHead>
+              <TableHead>Delete</TableHead>
+            </TableRow>
+          </TableHeader>
+          <TableBody>
+            {slackChannelConfigs
+              .slice(numToDisplay * (page - 1), numToDisplay * page)
+              .map((slackChannelConfig) => {
+                return (
+                  <TableRow
+                    key={slackChannelConfig.id}
+                    className="cursor-pointer hover:bg-gray-100 transition-colors"
+                    onClick={() => {
+                      window.location.href = `/admin/bots/${slackBotId}/channels/${slackChannelConfig.id}`;
+                    }}
+                  >
+                    <TableCell>
+                      <div className="flex gap-x-2">
+                        <div className="my-auto">
+                          <EditIcon />
+                        </div>
+                        <div className="my-auto">
+                          {"#" + slackChannelConfig.channel_config.channel_name}
+                        </div>
+                      </div>
+                    </TableCell>
+                    <TableCell onClick={(e) => e.stopPropagation()}>
+                      {slackChannelConfig.persona &&
+                      !isPersonaASlackBotPersona(slackChannelConfig.persona) ? (
+                        <Link
+                          href={`/admin/assistants/${slackChannelConfig.persona.id}`}
+                          className="text-blue-500 flex hover:underline"
+                        >
+                          {slackChannelConfig.persona.name}
+                        </Link>
+                      ) : (
+                        "-"
+                      )}
+                    </TableCell>
+                    <TableCell>
+                      <div>
+                        {slackChannelConfig.persona &&
+                        slackChannelConfig.persona.document_sets.length > 0
+                          ? slackChannelConfig.persona.document_sets
+                              .map((documentSet) => documentSet.name)
+                              .join(", ")
+                          : "-"}
+                      </div>
+                    </TableCell>
+                    <TableCell onClick={(e) => e.stopPropagation()}>
+                      <div
+                        className="cursor-pointer hover:text-destructive"
+                        onClick={async (e) => {
+                          e.stopPropagation();
+                          const response = await deleteSlackChannelConfig(
+                            slackChannelConfig.id
+                          );
+                          if (response.ok) {
+                            setPopup({
+                              message: `Slack bot config "${slackChannelConfig.id}" deleted`,
+                              type: "success",
+                            });
+                          } else {
+                            const errorMsg = await response.text();
+                            setPopup({
+                              message: `Failed to delete Slack bot config - ${errorMsg}`,
+                              type: "error",
+                            });
+                          }
+                          refresh();
+                        }}
+                      >
+                        <TrashIcon />
+                      </div>
+                    </TableCell>
+                  </TableRow>
+                );
+              })}
+
+            {/* Empty row with message when table has no data */}
+            {slackChannelConfigs.length === 0 && (
+              <TableRow>
+                <TableCell
+                  colSpan={4}
+                  className="text-center text-muted-foreground"
+                >
+                  Please add a New Slack Bot Configuration to begin chatting
+                  with Onyx!
+                </TableCell>
+              </TableRow>
+            )}
+          </TableBody>
+        </Table>
       </div>
 
-      <div>
-        <h2 className="text-2xl font- mb-4">Channel-Specific Configurations</h2>
-        <Card>
-          <Table>
-            <TableHeader>
-              <TableRow>
-                <TableHead>Channel</TableHead>
-                <TableHead>Assistant</TableHead>
-                <TableHead>Document Sets</TableHead>
-                <TableHead>Actions</TableHead>
-              </TableRow>
-            </TableHeader>
-            <TableBody>
-              {channelConfigs
-                .slice(numToDisplay * (page - 1), numToDisplay * page)
-                .map((slackChannelConfig) => {
-                  return (
-                    <TableRow
-                      key={slackChannelConfig.id}
-                      className="cursor-pointer transition-colors"
-                      onClick={() => {
-                        window.location.href = `/admin/bots/${slackBotId}/channels/${slackChannelConfig.id}`;
-                      }}
-                    >
-                      <TableCell>
-                        <div className="flex gap-x-2">
-                          <div className="my-auto">
-                            <EditIcon className="text-muted-foreground" />
-                          </div>
-                          <div className="my-auto">
-                            {"#" +
-                              slackChannelConfig.channel_config.channel_name}
-                          </div>
-                        </div>
-                      </TableCell>
-                      <TableCell onClick={(e) => e.stopPropagation()}>
-                        {slackChannelConfig.persona &&
-                        !isPersonaASlackBotPersona(
-                          slackChannelConfig.persona
-                        ) ? (
-                          <Link
-                            href={`/admin/assistants/${slackChannelConfig.persona.id}`}
-                            className="text-primary hover:underline"
-                          >
-                            {slackChannelConfig.persona.name}
-                          </Link>
-                        ) : (
-                          "-"
-                        )}
-                      </TableCell>
-                      <TableCell>
-                        <div>
-                          {slackChannelConfig.persona &&
-                          slackChannelConfig.persona.document_sets.length > 0
-                            ? slackChannelConfig.persona.document_sets
-                                .map((documentSet) => documentSet.name)
-                                .join(", ")
-                            : "-"}
-                        </div>
-                      </TableCell>
-                      <TableCell onClick={(e) => e.stopPropagation()}>
-                        <Button
-                          variant="ghost"
-                          size="sm"
-                          className="hover:text-destructive"
-                          onClick={async (e) => {
-                            e.stopPropagation();
-                            const response = await deleteSlackChannelConfig(
-                              slackChannelConfig.id
-                            );
-                            if (response.ok) {
-                              setPopup({
-                                message: `Slack bot config "${slackChannelConfig.id}" deleted`,
-                                type: "success",
-                              });
-                            } else {
-                              const errorMsg = await response.text();
-                              setPopup({
-                                message: `Failed to delete Slack bot config - ${errorMsg}`,
-                                type: "error",
-                              });
-                            }
-                            refresh();
-                          }}
-                        >
-                          <TrashIcon />
-                        </Button>
-                      </TableCell>
-                    </TableRow>
-                  );
-                })}
-
-              {channelConfigs.length === 0 && (
-                <TableRow>
-                  <TableCell
-                    colSpan={4}
-                    className="text-center text-muted-foreground"
-                  >
-                    No channel-specific configurations. Add a new configuration
-                    to customize behavior for specific channels.
-                  </TableCell>
-                </TableRow>
-              )}
-            </TableBody>
-          </Table>
-        </Card>
-
-        {channelConfigs.length > numToDisplay && (
-          <div className="mt-4 flex justify-center">
-            <PageSelector
-              totalPages={Math.ceil(channelConfigs.length / numToDisplay)}
-              currentPage={page}
-              onPageChange={(newPage) => setPage(newPage)}
-            />
-          </div>
-        )}
+      <div className="mt-3 flex">
+        <div className="mx-auto">
+          <PageSelector
+            totalPages={Math.ceil(slackChannelConfigs.length / numToDisplay)}
+            currentPage={page}
+            onPageChange={(newPage) => setPage(newPage)}
+          />
+        </div>
       </div>
     </div>
   );

web/src/app/admin/bots/[bot-id]/channels/SlackChannelConfigCreationForm.tsx

@@ -1,29 +1,21 @@
 "use client";
 
-import React, { useMemo, useState, useEffect } from "react";
-import { Formik, Form, Field } from "formik";
+import React, { useMemo } from "react";
+import { Formik } from "formik";
 import * as Yup from "yup";
 import { usePopup } from "@/components/admin/connectors/Popup";
-import {
-  DocumentSet,
-  SlackChannelConfig,
-  SlackBotResponseType,
-} from "@/lib/types";
+import { DocumentSet, SlackChannelConfig } from "@/lib/types";
 import {
   createSlackChannelConfig,
   isPersonaASlackBotPersona,
   updateSlackChannelConfig,
-  fetchSlackChannels,
 } from "../lib";
 import CardSection from "@/components/admin/CardSection";
 import { useRouter } from "next/navigation";
 import { Persona } from "@/app/admin/assistants/interfaces";
 import { StandardAnswerCategoryResponse } from "@/components/standardAnswers/getStandardAnswerCategoriesIfEE";
 import { SEARCH_TOOL_ID, SEARCH_TOOL_NAME } from "@/app/chat/tools/constants";
-import {
-  SlackChannelConfigFormFields,
-  SlackChannelConfigFormFieldsProps,
-} from "./SlackChannelConfigFormFields";
+import { SlackChannelConfigFormFields } from "./SlackChannelConfigFormFields";
 
 export const SlackChannelConfigCreationForm = ({
   slack_bot_id,
@@ -41,7 +33,6 @@ export const SlackChannelConfigCreationForm = ({
   const { popup, setPopup } = usePopup();
   const router = useRouter();
   const isUpdate = Boolean(existingSlackChannelConfig);
-  const isDefault = existingSlackChannelConfig?.is_default || false;
   const existingSlackBotUsesPersona = existingSlackChannelConfig?.persona
     ? !isPersonaASlackBotPersona(existingSlackChannelConfig.persona)
     : false;
@@ -55,16 +46,13 @@ export const SlackChannelConfigCreationForm = ({
   }, [personas]);
 
   return (
-    <CardSection className="!px-12 max-w-4xl">
+    <CardSection className="max-w-4xl">
       {popup}
-
       <Formik
         initialValues={{
           slack_bot_id: slack_bot_id,
-          channel_name: isDefault
-            ? ""
-            : existingSlackChannelConfig?.channel_config.channel_name || "",
-          response_type: "citations" as SlackBotResponseType,
+          channel_name:
+            existingSlackChannelConfig?.channel_config.channel_name || "",
           answer_validity_check_enabled: (
             existingSlackChannelConfig?.channel_config?.answer_filters || []
           ).includes("well_answered_postfilter"),
@@ -102,6 +90,8 @@ export const SlackChannelConfigCreationForm = ({
             !isPersonaASlackBotPersona(existingSlackChannelConfig.persona)
               ? existingSlackChannelConfig.persona.id
               : null,
+          response_type:
+            existingSlackChannelConfig?.response_type || "citations",
           standard_answer_categories:
             existingSlackChannelConfig?.standard_answer_categories || [],
           knowledge_source: existingSlackBotUsesPersona
@@ -112,12 +102,10 @@ export const SlackChannelConfigCreationForm = ({
         }}
         validationSchema={Yup.object().shape({
           slack_bot_id: Yup.number().required(),
-          channel_name: isDefault
-            ? Yup.string()
-            : Yup.string().required("Channel Name is required"),
-          response_type: Yup.mixed<SlackBotResponseType>()
+          channel_name: Yup.string().required("Channel Name is required"),
+          response_type: Yup.string()
             .oneOf(["quotes", "citations"])
-            .required(),
+            .required("Response type is required"),
           answer_validity_check_enabled: Yup.boolean().required(),
           questionmark_prefilter_enabled: Yup.boolean().required(),
           respond_tag_only: Yup.boolean().required(),
@@ -171,7 +159,6 @@ export const SlackChannelConfigCreationForm = ({
             standard_answer_categories: values.standard_answer_categories.map(
               (category: any) => category.id
             ),
-            response_type: values.response_type as SlackBotResponseType,
           };
 
           if (!cleanedValues.still_need_help_enabled) {
@@ -204,22 +191,13 @@ export const SlackChannelConfigCreationForm = ({
           }
         }}
       >
-        {({ isSubmitting, values, setFieldValue }) => (
-          <Form>
-            <div className="pb-6 w-full">
-              <SlackChannelConfigFormFields
-                {...values}
-                isUpdate={isUpdate}
-                isDefault={isDefault}
-                documentSets={documentSets}
-                searchEnabledAssistants={searchEnabledAssistants}
-                standardAnswerCategoryResponse={standardAnswerCategoryResponse}
-                setPopup={setPopup}
-                slack_bot_id={slack_bot_id}
-              />
-            </div>
-          </Form>
-        )}
+        <SlackChannelConfigFormFields
+          isUpdate={isUpdate}
+          documentSets={documentSets}
+          searchEnabledAssistants={searchEnabledAssistants}
+          standardAnswerCategoryResponse={standardAnswerCategoryResponse}
+          setPopup={setPopup}
+        />
       </Formik>
     </CardSection>
   );

web/src/app/admin/bots/[bot-id]/channels/SlackChannelConfigFormFields.tsx

@@ -1,13 +1,7 @@
 "use client";
 
 import React, { useState, useEffect, useMemo } from "react";
-import {
-  FieldArray,
-  Form,
-  useFormikContext,
-  ErrorMessage,
-  Field,
-} from "formik";
+import { FieldArray, Form, useFormikContext, ErrorMessage } from "formik";
 import { CCPairDescriptor, DocumentSet } from "@/lib/types";
 import {
   BooleanFormField,
@@ -37,15 +31,9 @@ import { TooltipProvider } from "@radix-ui/react-tooltip";
 import { SourceIcon } from "@/components/SourceIcon";
 import Link from "next/link";
 import { AssistantIcon } from "@/components/assistants/AssistantIcon";
-import { SearchMultiSelectDropdown } from "@/components/Dropdown";
-import { fetchSlackChannels } from "../lib";
-import { Badge } from "@/components/ui/badge";
-import useSWR from "swr";
-import { ThreeDotsLoader } from "@/components/Loading";
 
-export interface SlackChannelConfigFormFieldsProps {
+interface SlackChannelConfigFormFieldsProps {
   isUpdate: boolean;
-  isDefault: boolean;
   documentSets: DocumentSet[];
   searchEnabledAssistants: Persona[];
   standardAnswerCategoryResponse: StandardAnswerCategoryResponse;
@@ -53,23 +41,19 @@ export interface SlackChannelConfigFormFieldsProps {
     message: string;
     type: "error" | "success" | "warning";
   }) => void;
-  slack_bot_id: number;
 }
 
 export function SlackChannelConfigFormFields({
   isUpdate,
-  isDefault,
   documentSets,
   searchEnabledAssistants,
   standardAnswerCategoryResponse,
   setPopup,
-  slack_bot_id,
 }: SlackChannelConfigFormFieldsProps) {
   const router = useRouter();
   const { values, setFieldValue } = useFormikContext<any>();
   const [showAdvancedOptions, setShowAdvancedOptions] = useState(false);
   const [viewUnselectableSets, setViewUnselectableSets] = useState(false);
-  const [currentSearchTerm, setCurrentSearchTerm] = useState("");
   const [viewSyncEnabledAssistants, setViewSyncEnabledAssistants] =
     useState(false);
 
@@ -168,54 +152,11 @@ export function SlackChannelConfigFormFields({
     );
   }, [documentSets]);
 
-  const { data: channelOptions, isLoading } = useSWR(
-    `/api/manage/admin/slack-app/bots/${slack_bot_id}/channels`,
-    async (url: string) => {
-      const channels = await fetchSlackChannels(slack_bot_id);
-      return channels.map((channel: any) => ({
-        name: channel.name,
-        value: channel.id,
-      }));
-    }
-  );
-  if (isLoading) {
-    return <ThreeDotsLoader />;
-  }
-
   return (
-    <>
-      <div className="w-full">
-        {isDefault && (
-          <Badge variant="agent" className="bg-blue-100 text-blue-800">
-            Default Configuration
-          </Badge>
-        )}
-        {!isDefault && (
-          <>
-            <label
-              htmlFor="channel_name"
-              className="block font-medium text-base mb-2"
-            >
-              Select A Slack Channel:
-            </label>{" "}
-            <Field name="channel_name">
-              {({ field, form }: { field: any; form: any }) => (
-                <SearchMultiSelectDropdown
-                  options={channelOptions || []}
-                  onSelect={(selected) => {
-                    form.setFieldValue("channel_name", selected.name);
-                    setCurrentSearchTerm(selected.name);
-                  }}
-                  initialSearchTerm={field.value}
-                  onSearchTermChange={(term) => {
-                    setCurrentSearchTerm(term);
-                    form.setFieldValue("channel_name", term);
-                  }}
-                />
-              )}
-            </Field>
-          </>
-        )}
+    <Form className="px-6 max-w-4xl">
+      <div className="pt-4 w-full">
+        <TextFormField name="channel_name" label="Slack Channel Name:" />
+
         <div className="space-y-2 mt-4">
           <Label>Knowledge Source</Label>
           <RadioGroup
@@ -229,7 +170,7 @@ export function SlackChannelConfigFormFields({
               value="all_public"
               id="all_public"
               label="All Public Knowledge"
-              sublabel="Let OnyxBot respond based on information from all public connectors"
+              sublabel="Let OnyxBot respond based on information from all public connectors "
             />
             {selectableSets.length + unselectableSets.length > 0 && (
               <RadioGroupItemField
@@ -247,6 +188,7 @@ export function SlackChannelConfigFormFields({
             />
           </RadioGroup>
         </div>
+
         {values.knowledge_source === "document_sets" &&
           documentSets.length > 0 && (
             <div className="mt-4">
@@ -339,6 +281,7 @@ export function SlackChannelConfigFormFields({
               />
             </div>
           )}
+
         {values.knowledge_source === "assistant" && (
           <div className="mt-4">
             <SubLabel>
@@ -410,15 +353,15 @@ export function SlackChannelConfigFormFields({
         )}
       </div>
 
-      <div className="mt-6">
+      <div className="mt-2">
         <AdvancedOptionsToggle
           showAdvancedOptions={showAdvancedOptions}
           setShowAdvancedOptions={setShowAdvancedOptions}
         />
       </div>
       {showAdvancedOptions && (
-        <div className="mt-2 space-y-4">
-          <div className="w-64">
+        <div className="mt-4">
+          <div className="w-64 mb-4">
             <SelectorFormField
               name="response_type"
               label="Answer Type"
@@ -437,79 +380,83 @@ export function SlackChannelConfigFormFields({
             tooltip="If set, will show a button at the bottom of the response that allows the user to continue the conversation in the Onyx Web UI"
           />
 
-          <BooleanFormField
-            name="still_need_help_enabled"
-            removeIndent
-            onChange={(checked: boolean) => {
-              setFieldValue("still_need_help_enabled", checked);
-              if (!checked) {
-                setFieldValue("follow_up_tags", []);
-              }
-            }}
-            label={'Give a "Still need help?" button'}
-            tooltip={`OnyxBot's response will include a button at the bottom 
-                of the response that asks the user if they still need help.`}
-          />
-          {values.still_need_help_enabled && (
-            <CollapsibleSection prompt="Configure Still Need Help Button">
-              <TextArrayField
-                name="follow_up_tags"
-                label="(Optional) Users / Groups to Tag"
-                values={values}
-                subtext={
-                  <div>
-                    The Slack users / groups we should tag if the user clicks
-                    the &quot;Still need help?&quot; button. If no emails are
-                    provided, we will not tag anyone and will just react with a
-                    🆘 emoji to the original message.
-                  </div>
+          <div className="flex flex-col space-y-3 mt-2">
+            <BooleanFormField
+              name="still_need_help_enabled"
+              removeIndent
+              onChange={(checked: boolean) => {
+                setFieldValue("still_need_help_enabled", checked);
+                if (!checked) {
+                  setFieldValue("follow_up_tags", []);
                 }
+              }}
+              label={'Give a "Still need help?" button'}
+              tooltip={`OnyxBot's response will include a button at the bottom 
+                  of the response that asks the user if they still need help.`}
+            />
+            {values.still_need_help_enabled && (
+              <CollapsibleSection prompt="Configure Still Need Help Button">
+                <TextArrayField
+                  name="follow_up_tags"
+                  label="(Optional) Users / Groups to Tag"
+                  values={values}
+                  subtext={
+                    <div>
+                      The Slack users / groups we should tag if the user clicks
+                      the &quot;Still need help?&quot; button. If no emails are
+                      provided, we will not tag anyone and will just react with
+                      a 🆘 emoji to the original message.
+                    </div>
+                  }
+                  placeholder="User email or user group name..."
+                />
+              </CollapsibleSection>
+            )}
+
+            <BooleanFormField
+              name="answer_validity_check_enabled"
+              removeIndent
+              label="Only respond if citations found"
+              tooltip="If set, will only answer questions where the model successfully produces citations"
+            />
+            <BooleanFormField
+              name="questionmark_prefilter_enabled"
+              removeIndent
+              label="Only respond to questions"
+              tooltip="If set, OnyxBot will only respond to messages that contain a question mark"
+            />
+            <BooleanFormField
+              name="respond_tag_only"
+              removeIndent
+              label="Respond to @OnyxBot Only"
+              tooltip="If set, OnyxBot will only respond when directly tagged"
+            />
+            <BooleanFormField
+              name="respond_to_bots"
+              removeIndent
+              label="Respond to Bot messages"
+              tooltip="If not set, OnyxBot will always ignore messages from Bots"
+            />
+            <BooleanFormField
+              name="enable_auto_filters"
+              removeIndent
+              label="Enable LLM Autofiltering"
+              tooltip="If set, the LLM will generate source and time filters based on the user's query"
+            />
+
+            <div className="mt-12">
+              <TextArrayField
+                name="respond_member_group_list"
+                label="(Optional) Respond to Certain Users / Groups"
+                subtext={
+                  "If specified, OnyxBot responses will only " +
+                  "be visible to the members or groups in this list."
+                }
+                values={values}
                 placeholder="User email or user group name..."
               />
-            </CollapsibleSection>
-          )}
-
-          <BooleanFormField
-            name="answer_validity_check_enabled"
-            removeIndent
-            label="Only respond if citations found"
-            tooltip="If set, will only answer questions where the model successfully produces citations"
-          />
-          <BooleanFormField
-            name="questionmark_prefilter_enabled"
-            removeIndent
-            label="Only respond to questions"
-            tooltip="If set, OnyxBot will only respond to messages that contain a question mark"
-          />
-          <BooleanFormField
-            name="respond_tag_only"
-            removeIndent
-            label="Respond to @OnyxBot Only"
-            tooltip="If set, OnyxBot will only respond when directly tagged"
-          />
-          <BooleanFormField
-            name="respond_to_bots"
-            removeIndent
-            label="Respond to Bot messages"
-            tooltip="If not set, OnyxBot will always ignore messages from Bots"
-          />
-          <BooleanFormField
-            name="enable_auto_filters"
-            removeIndent
-            label="Enable LLM Autofiltering"
-            tooltip="If set, the LLM will generate source and time filters based on the user's query"
-          />
-
-          <TextArrayField
-            name="respond_member_group_list"
-            label="(Optional) Respond to Certain Users / Groups"
-            subtext={
-              "If specified, OnyxBot responses will only " +
-              "be visible to the members or groups in this list."
-            }
-            values={values}
-            placeholder="User email or user group name..."
-          />
+            </div>
+          </div>
 
           <StandardAnswerCategoryDropdownField
             standardAnswerCategoryResponse={standardAnswerCategoryResponse}
@@ -521,7 +468,7 @@ export function SlackChannelConfigFormFields({
         </div>
       )}
 
-      <div className="flex mt-8 gap-x-2 w-full justify-end">
+      <div className="flex mt-2 gap-x-2 w-full justify-end flex">
         {shouldShowPrivacyAlert && (
           <TooltipProvider>
             <Tooltip>
@@ -571,11 +518,13 @@ export function SlackChannelConfigFormFields({
             </Tooltip>
           </TooltipProvider>
         )}
-        <Button type="submit">{isUpdate ? "Update" : "Create"}</Button>
+        <Button onClick={() => {}} type="submit">
+          {isUpdate ? "Update" : "Create"}
+        </Button>
         <Button type="button" variant="outline" onClick={() => router.back()}>
           Cancel
         </Button>
       </div>
-    </>
+    </Form>
   );
 }

web/src/app/admin/bots/[bot-id]/lib.ts

@@ -94,17 +94,3 @@ export const deleteSlackChannelConfig = async (id: number) => {
 export function isPersonaASlackBotPersona(persona: Persona) {
   return persona.name.startsWith("__slack_bot_persona__");
 }
-
-export const fetchSlackChannels = async (botId: number) => {
-  return fetch(`/api/manage/admin/slack-app/bots/${botId}/channels`, {
-    method: "GET",
-    headers: {
-      "Content-Type": "application/json",
-    },
-  }).then((response) => {
-    if (!response.ok) {
-      throw new Error("Failed to fetch Slack channels");
-    }
-    return response.json();
-  });
-};

web/src/app/admin/bots/[bot-id]/page.tsx

@@ -78,6 +78,30 @@ function SlackBotEditPage({
       />
       <Separator />
 
+      <div className="my-8" />
+
+      <Link
+        className="
+          flex
+          py-2
+          px-4
+          mt-2
+          border
+          border-border
+          h-fit
+          cursor-pointer
+          hover:bg-hover
+          text-sm
+          w-80
+        "
+        href={`/admin/bots/${unwrappedParams["bot-id"]}/channels/new`}
+      >
+        <div className="mx-auto flex">
+          <FiPlusSquare className="my-auto mr-2" />
+          New Slack Channel Configuration
+        </div>
+      </Link>
+
       <div className="mt-8">
         <SlackChannelConfigsTable
           slackBotId={slackBot.id}

web/src/app/admin/connector/[ccPairId]/page.tsx

@@ -33,6 +33,11 @@ import EditPropertyModal from "@/components/modals/EditPropertyModal";
 
 import * as Yup from "yup";
 
+// since the uploaded files are cleaned up after some period of time
+// re-indexing will not work for the file connector. Also, it would not
+// make sense to re-index, since the files will not have changed.
+const CONNECTOR_TYPES_THAT_CANT_REINDEX: ValidSources[] = [ValidSources.File];
+
 // synchronize these validations with the SQLAlchemy connector class until we have a
 // centralized schema for both frontend and backend
 const RefreshFrequencySchema = Yup.object().shape({
@@ -263,18 +268,21 @@ function Main({ ccPairId }: { ccPairId: number }) {
 
         {ccPair.is_editable_for_current_user && (
           <div className="ml-auto flex gap-x-2">
-            <ReIndexButton
-              ccPairId={ccPair.id}
-              connectorId={ccPair.connector.id}
-              credentialId={ccPair.credential.id}
-              isDisabled={
-                ccPair.indexing ||
-                ccPair.status === ConnectorCredentialPairStatus.PAUSED
-              }
-              isIndexing={ccPair.indexing}
-              isDeleting={isDeleting}
-            />
-
+            {!CONNECTOR_TYPES_THAT_CANT_REINDEX.includes(
+              ccPair.connector.source
+            ) && (
+              <ReIndexButton
+                ccPairId={ccPair.id}
+                connectorId={ccPair.connector.id}
+                credentialId={ccPair.credential.id}
+                isDisabled={
+                  ccPair.indexing ||
+                  ccPair.status === ConnectorCredentialPairStatus.PAUSED
+                }
+                isIndexing={ccPair.indexing}
+                isDeleting={isDeleting}
+              />
+            )}
             {!isDeleting && <ModifyStatusButtonCluster ccPair={ccPair} />}
           </div>
         )}

web/src/app/admin/documents/explorer/Explorer.tsx

@@ -17,7 +17,7 @@ import { DocumentUpdatedAtBadge } from "@/components/search/DocumentUpdatedAtBad
 import { DocumentSet } from "@/lib/types";
 import { SourceIcon } from "@/components/SourceIcon";
 import { Connector } from "@/lib/connectors/connectors";
-import { HorizontalFilters } from "@/components/filters/SourceSelector";
+import { HorizontalFilters } from "@/app/chat/shared_chat_search/Filters";
 
 const DocumentDisplay = ({
   document,

web/src/app/assistants/SidebarWrapper.tsx

@@ -10,17 +10,16 @@ import {
   useRef,
   useState,
 } from "react";
-import { useSidebarVisibility } from "@/components/chat/hooks";
-import FunctionalHeader from "@/components/chat/Header";
+import { useSidebarVisibility } from "@/components/chat_search/hooks";
+import FunctionalHeader from "@/components/chat_search/Header";
 import { useRouter } from "next/navigation";
 import { pageType } from "../chat/sessionSidebar/types";
-import FixedLogo from "../../components/logo/FixedLogo";
+import FixedLogo from "../chat/shared_chat_search/FixedLogo";
 import { SettingsContext } from "@/components/settings/SettingsProvider";
 import { useChatContext } from "@/components/context/ChatContext";
 import { HistorySidebar } from "../chat/sessionSidebar/HistorySidebar";
 import { useAssistants } from "@/components/context/AssistantsContext";
 import AssistantModal from "./mine/AssistantModal";
-import { useSidebarShortcut } from "@/lib/browserUtilities";
 
 interface SidebarWrapperProps<T extends object> {
   initiallyToggled: boolean;
@@ -72,8 +71,23 @@ export default function SidebarWrapper<T extends object>({
 
   const [showAssistantsModal, setShowAssistantsModal] = useState(false);
   const router = useRouter();
+  useEffect(() => {
+    const handleKeyDown = (event: KeyboardEvent) => {
+      if (event.metaKey || event.ctrlKey) {
+        switch (event.key.toLowerCase()) {
+          case "e":
+            event.preventDefault();
+            toggleSidebar();
+            break;
+        }
+      }
+    };
 
-  useSidebarShortcut(router, toggleSidebar);
+    window.addEventListener("keydown", handleKeyDown);
+    return () => {
+      window.removeEventListener("keydown", handleKeyDown);
+    };
+  }, [router]);
 
   return (
     <div className="flex relative overflow-x-hidden overscroll-contain flex-col w-full h-screen">

web/src/app/auth/forgot-password/utils.ts

@@ -28,12 +28,6 @@ export const resetPassword = async (
   });
 
   if (!response.ok) {
-    const error = await response.json();
-    if (error?.detail?.code === "RESET_PASSWORD_INVALID_PASSWORD") {
-      throw new Error(error.detail.reason || "Invalid password");
-    }
-    const errorMessage =
-      error?.detail || "An error occurred during password reset.";
-    throw new Error(errorMessage);
+    throw new Error("Failed to reset password");
   }
 };

web/src/app/auth/reset-password/page.tsx

@@ -1,5 +1,5 @@
 "use client";
-import React, { useState, useEffect } from "react";
+import React, { useState } from "react";
 import { resetPassword } from "../forgot-password/utils";
 import AuthFlowContainer from "@/components/auth/AuthFlowContainer";
 import CardSection from "@/components/admin/CardSection";
@@ -13,28 +13,13 @@ import { TextFormField } from "@/components/admin/connectors/Field";
 import { usePopup } from "@/components/admin/connectors/Popup";
 import { Spinner } from "@/components/Spinner";
 import { redirect, useSearchParams } from "next/navigation";
-import {
-  NEXT_PUBLIC_FORGOT_PASSWORD_ENABLED,
-  TENANT_ID_COOKIE_NAME,
-} from "@/lib/constants";
-import Cookies from "js-cookie";
+import { NEXT_PUBLIC_FORGOT_PASSWORD_ENABLED } from "@/lib/constants";
 
 const ResetPasswordPage: React.FC = () => {
   const { popup, setPopup } = usePopup();
   const [isWorking, setIsWorking] = useState(false);
   const searchParams = useSearchParams();
   const token = searchParams.get("token");
-  const tenantId = searchParams.get(TENANT_ID_COOKIE_NAME);
-  // Keep search param same name as cookie for simplicity
-
-  useEffect(() => {
-    if (tenantId) {
-      Cookies.set(TENANT_ID_COOKIE_NAME, tenantId, {
-        path: "/",
-        expires: 1 / 24,
-      }); // Expires in 1 hour
-    }
-  }, [tenantId]);
 
   if (!NEXT_PUBLIC_FORGOT_PASSWORD_ENABLED) {
     redirect("/auth/login");
@@ -78,18 +63,10 @@ const ResetPasswordPage: React.FC = () => {
                 redirect("/auth/login");
               }, 1000);
             } catch (error) {
-              if (error instanceof Error) {
-                setPopup({
-                  type: "error",
-                  message:
-                    error.message || "An error occurred during password reset.",
-                });
-              } else {
-                setPopup({
-                  type: "error",
-                  message: "An unexpected error occurred. Please try again.",
-                });
-              }
+              setPopup({
+                type: "error",
+                message: "An error occurred. Please try again.",
+              });
             } finally {
               setIsWorking(false);
             }

web/src/app/chat/ChatBanner.tsx

@@ -3,7 +3,7 @@
 import { SettingsContext } from "@/components/settings/SettingsProvider";
 import { useContext, useState, useRef, useLayoutEffect } from "react";
 import { ChevronDownIcon } from "@/components/icons/icons";
-import { MinimalMarkdown } from "@/components/chat/MinimalMarkdown";
+import { MinimalMarkdown } from "@/components/chat_search/MinimalMarkdown";
 
 export function ChatBanner() {
   const settings = useContext(SettingsContext);

web/src/app/chat/ChatPage.tsx

@@ -89,16 +89,16 @@ import { useChatContext } from "@/components/context/ChatContext";
 import { v4 as uuidv4 } from "uuid";
 import { ChatPopup } from "./ChatPopup";
 
-import FunctionalHeader from "@/components/chat/Header";
-import { useSidebarVisibility } from "@/components/chat/hooks";
+import FunctionalHeader from "@/components/chat_search/Header";
+import { useSidebarVisibility } from "@/components/chat_search/hooks";
 import {
   PRO_SEARCH_TOGGLED_COOKIE_NAME,
   SIDEBAR_TOGGLED_COOKIE_NAME,
 } from "@/components/resizable/constants";
-import FixedLogo from "../../components/logo/FixedLogo";
+import FixedLogo from "./shared_chat_search/FixedLogo";
 
 import { DeleteEntityModal } from "../../components/modals/DeleteEntityModal";
-import { MinimalMarkdown } from "@/components/chat/MinimalMarkdown";
+import { MinimalMarkdown } from "@/components/chat_search/MinimalMarkdown";
 import ExceptionTraceModal from "@/components/modals/ExceptionTraceModal";
 
 import {
@@ -108,10 +108,10 @@ import {
 } from "./tools/constants";
 import { useUser } from "@/components/user/UserProvider";
 import { ApiKeyModal } from "@/components/llm/ApiKeyModal";
-import BlurBackground from "../../components/chat/BlurBackground";
+import BlurBackground from "./shared_chat_search/BlurBackground";
 import { NoAssistantModal } from "@/components/modals/NoAssistantModal";
 import { useAssistants } from "@/components/context/AssistantsContext";
-import TextView from "@/components/chat/TextView";
+import TextView from "@/components/chat_search/TextView";
 import { Modal } from "@/components/Modal";
 import { useSendMessageToParent } from "@/lib/extension/utils";
 import {
@@ -124,11 +124,6 @@ import { UserSettingsModal } from "./modal/UserSettingsModal";
 import { AlignStartVertical } from "lucide-react";
 import { AgenticMessage } from "./message/AgenticMessage";
 import AssistantModal from "../assistants/mine/AssistantModal";
-import {
-  OperatingSystem,
-  useOperatingSystem,
-  useSidebarShortcut,
-} from "@/lib/browserUtilities";
 
 const TEMP_USER_MESSAGE_ID = -1;
 const TEMP_ASSISTANT_MESSAGE_ID = -2;
@@ -2057,7 +2052,24 @@ export function ChatPage({
     llmOverrideManager.updateImageFilesPresent(imageFileInMessageHistory);
   }, [imageFileInMessageHistory]);
 
-  useSidebarShortcut(router, toggleSidebar);
+  useEffect(() => {
+    const handleKeyDown = (event: KeyboardEvent) => {
+      if (event.metaKey || event.ctrlKey) {
+        switch (event.key.toLowerCase()) {
+          case "e":
+            event.preventDefault();
+            toggleSidebar();
+            break;
+        }
+      }
+    };
+
+    window.addEventListener("keydown", handleKeyDown);
+    return () => {
+      window.removeEventListener("keydown", handleKeyDown);
+    };
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [router]);
 
   const [sharedChatSession, setSharedChatSession] =
     useState<ChatSession | null>();
@@ -2963,7 +2975,8 @@ export function ChatPage({
                               } else {
                                 return (
                                   <div key={messageReactComponentKey}>
-                                    <AIMessage
+                                    <AgenticMessage
+                                      subQuestions={message.sub_questions || []}
                                       currentPersona={liveAssistant}
                                       messageId={message.messageId}
                                       content={

web/src/app/chat/WrappedChat.tsx

@@ -1,7 +1,7 @@
 "use client";
 import { useChatContext } from "@/components/context/ChatContext";
 import { ChatPage } from "./ChatPage";
-import FunctionalWrapper from "../../components/chat/FunctionalWrapper";
+import FunctionalWrapper from "./shared_chat_search/FunctionalWrapper";
 
 export default function WrappedChat({
   firstMessage,

web/src/app/chat/documentSidebar/ChatDocumentDisplay.tsx

@@ -39,6 +39,7 @@ export function DocumentMetadataBlock({
 
       {metadataEntries.length > 0 && (
         <>
+          <div className="mx-1 h-4 border-l border-border" />
           <div className="flex items-center overflow-hidden">
             {metadataEntries
               .slice(0, MAX_METADATA_ITEMS)

web/src/app/chat/input/ChatInputBar.tsx

@@ -24,7 +24,7 @@ import {
 } from "@/components/ui/tooltip";
 import { Hoverable } from "@/components/Hoverable";
 import { ChatState } from "../types";
-import UnconfiguredProviderText from "@/components/chat/UnconfiguredProviderText";
+import UnconfiguredProviderText from "@/components/chat_search/UnconfiguredProviderText";
 import { useAssistants } from "@/components/context/AssistantsContext";
 import { CalendarIcon, TagIcon, XIcon } from "lucide-react";
 import { FilterPopup } from "@/components/search/filtering/FilterPopup";