stashing

* try adding back some params

* raise timeout

* update chromatic version

* fix typo

* use chromatic imports

* update gitignore

* slim down the config file

* update readme

---------

Co-authored-by: Richard Kuo (Danswer) <rkuo@onyx.app>

.github/workflows/docker-build-push-cloud-web-container-on-tag.yml

@@ -67,6 +67,7 @@ jobs:
             NEXT_PUBLIC_SENTRY_DSN=${{ secrets.SENTRY_DSN }}
             NEXT_PUBLIC_GTM_ENABLED=true
             NEXT_PUBLIC_FORGOT_PASSWORD_ENABLED=true
+            NEXT_PUBLIC_INCLUDE_ERROR_POPUP_SUPPORT_LINK=true
             NODE_OPTIONS=--max-old-space-size=8192
           # needed due to weird interactions with the builds for different platforms
           no-cache: true

README.md

@@ -124,7 +124,7 @@ There are two editions of Onyx:
 To try the Onyx Enterprise Edition:
 
 1. Checkout our [Cloud product](https://cloud.onyx.app/signup).
-2. For self-hosting, contact us at [founders@onyx.app](mailto:founders@onyx.app) or book a call with us on our [Cal](https://cal.com/team/danswer/founders).
+2. For self-hosting, contact us at [founders@onyx.app](mailto:founders@onyx.app) or book a call with us on our [Cal](https://cal.com/team/onyx/founders).
 
 ## 💡 Contributing
 

backend/Dockerfile

@@ -101,7 +101,8 @@ COPY ./alembic_tenants /app/alembic_tenants
 COPY ./alembic.ini /app/alembic.ini
 COPY supervisord.conf /usr/etc/supervisord.conf
 
-# Escape hatch
+# Escape hatch scripts
+COPY ./scripts/debugging /app/scripts/debugging
 COPY ./scripts/force_delete_connector_by_id.py /app/scripts/force_delete_connector_by_id.py
 
 # Put logo in assets

backend/alembic/versions/eaa3b5593925_add_default_slack_channel_config.py

@@ -0,0 +1,80 @@
+"""add default slack channel config
+
+Revision ID: eaa3b5593925
+Revises: 98a5008d8711
+Create Date: 2025-02-03 18:07:56.552526
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = "eaa3b5593925"
+down_revision = "98a5008d8711"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # Add is_default column
+    op.add_column(
+        "slack_channel_config",
+        sa.Column("is_default", sa.Boolean(), nullable=False, server_default="false"),
+    )
+
+    op.create_index(
+        "ix_slack_channel_config_slack_bot_id_default",
+        "slack_channel_config",
+        ["slack_bot_id", "is_default"],
+        unique=True,
+        postgresql_where=sa.text("is_default IS TRUE"),
+    )
+
+    # Create default channel configs for existing slack bots without one
+    conn = op.get_bind()
+    slack_bots = conn.execute(sa.text("SELECT id FROM slack_bot")).fetchall()
+
+    for slack_bot in slack_bots:
+        slack_bot_id = slack_bot[0]
+        existing_default = conn.execute(
+            sa.text(
+                "SELECT id FROM slack_channel_config WHERE slack_bot_id = :bot_id AND is_default = TRUE"
+            ),
+            {"bot_id": slack_bot_id},
+        ).fetchone()
+
+        if not existing_default:
+            conn.execute(
+                sa.text(
+                    """
+                    INSERT INTO slack_channel_config (
+                        slack_bot_id, persona_id, channel_config, enable_auto_filters, is_default
+                    ) VALUES (
+                        :bot_id, NULL,
+                        '{"channel_name": null, '
+                        '"respond_member_group_list": [], '
+                        '"answer_filters": [], '
+                        '"follow_up_tags": [], '
+                        '"respond_tag_only": true}',
+                        FALSE, TRUE
+                    )
+                """
+                ),
+                {"bot_id": slack_bot_id},
+            )
+
+
+def downgrade() -> None:
+    # Delete default slack channel configs
+    conn = op.get_bind()
+    conn.execute(sa.text("DELETE FROM slack_channel_config WHERE is_default = TRUE"))
+
+    # Remove index
+    op.drop_index(
+        "ix_slack_channel_config_slack_bot_id_default",
+        table_name="slack_channel_config",
+    )
+
+    # Remove is_default column
+    op.drop_column("slack_channel_config", "is_default")

backend/alembic/versions/f5437cc136c5_delete_non_search_assistants.py

@@ -0,0 +1,53 @@
+"""delete non-search assistants
+
+Revision ID: f5437cc136c5
+Revises: eaa3b5593925
+Create Date: 2025-02-04 16:17:15.677256
+
+"""
+from alembic import op
+
+
+# revision identifiers, used by Alembic.
+revision = "f5437cc136c5"
+down_revision = "eaa3b5593925"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    pass
+
+
+def downgrade() -> None:
+    # Fix: split the statements into multiple op.execute() calls
+    op.execute(
+        """
+        WITH personas_without_search AS (
+            SELECT p.id
+            FROM persona p
+            LEFT JOIN persona__tool pt ON p.id = pt.persona_id
+            LEFT JOIN tool t ON pt.tool_id = t.id
+            GROUP BY p.id
+            HAVING COUNT(CASE WHEN t.in_code_tool_id = 'run_search' THEN 1 END) = 0
+        )
+        UPDATE slack_channel_config
+        SET persona_id = NULL
+        WHERE is_default = TRUE AND persona_id IN (SELECT id FROM personas_without_search)
+        """
+    )
+
+    op.execute(
+        """
+        WITH personas_without_search AS (
+            SELECT p.id
+            FROM persona p
+            LEFT JOIN persona__tool pt ON p.id = pt.persona_id
+            LEFT JOIN tool t ON pt.tool_id = t.id
+            GROUP BY p.id
+            HAVING COUNT(CASE WHEN t.in_code_tool_id = 'run_search' THEN 1 END) = 0
+        )
+        DELETE FROM slack_channel_config
+        WHERE is_default = FALSE AND persona_id IN (SELECT id FROM personas_without_search)
+        """
+    )

backend/ee/onyx/db/persona.py

@@ -2,8 +2,11 @@ from uuid import UUID
 
 from sqlalchemy.orm import Session
 
+from onyx.configs.constants import NotificationType
 from onyx.db.models import Persona__User
 from onyx.db.models import Persona__UserGroup
+from onyx.db.notification import create_notification
+from onyx.server.features.persona.models import PersonaSharedNotificationData
 
 
 def make_persona_private(
@@ -23,6 +26,14 @@ def make_persona_private(
         for user_uuid in user_ids:
             db_session.add(Persona__User(persona_id=persona_id, user_id=user_uuid))
 
+            create_notification(
+                user_id=user_uuid,
+                notif_type=NotificationType.PERSONA_SHARED,
+                db_session=db_session,
+                additional_data=PersonaSharedNotificationData(
+                    persona_id=persona_id,
+                ).model_dump(),
+            )
     if group_ids:
         for group_id in group_ids:
             db_session.add(

backend/ee/onyx/db/user_group.py

@@ -218,14 +218,14 @@ def fetch_user_groups_for_user(
     return db_session.scalars(stmt).all()
 
 
-def construct_document_select_by_usergroup(
+def construct_document_id_select_by_usergroup(
     user_group_id: int,
 ) -> Select:
     """This returns a statement that should be executed using
     .yield_per() to minimize overhead. The primary consumers of this function
     are background processing task generators."""
     stmt = (
-        select(Document)
+        select(Document.id)
         .join(
             DocumentByConnectorCredentialPair,
             Document.id == DocumentByConnectorCredentialPair.id,

backend/ee/onyx/onyxbot/slack/handlers/handle_standard_answers.py

@@ -80,7 +80,7 @@ def oneoff_standard_answers(
 def _handle_standard_answers(
     message_info: SlackMessageInfo,
     receiver_ids: list[str] | None,
-    slack_channel_config: SlackChannelConfig | None,
+    slack_channel_config: SlackChannelConfig,
     prompt: Prompt | None,
     logger: OnyxLoggingAdapter,
     client: WebClient,
@@ -94,13 +94,10 @@ def _handle_standard_answers(
     Returns True if standard answers are found to match the user's message and therefore,
     we still need to respond to the users.
     """
-    # if no channel config, then no standard answers are configured
-    if not slack_channel_config:
-        return False
 
     slack_thread_id = message_info.thread_to_respond
     configured_standard_answer_categories = (
-        slack_channel_config.standard_answer_categories if slack_channel_config else []
+        slack_channel_config.standard_answer_categories
     )
     configured_standard_answers = set(
         [

backend/ee/onyx/server/middleware/tenant_tracking.py

@@ -10,6 +10,7 @@ from fastapi import Response
 from ee.onyx.auth.users import decode_anonymous_user_jwt_token
 from ee.onyx.configs.app_configs import ANONYMOUS_USER_COOKIE_NAME
 from onyx.auth.api_key import extract_tenant_from_api_key_header
+from onyx.configs.constants import TENANT_ID_COOKIE_NAME
 from onyx.db.engine import is_valid_schema_name
 from onyx.redis.redis_pool import retrieve_auth_token_data_from_redis
 from shared_configs.configs import MULTI_TENANT
@@ -43,6 +44,7 @@ async def _get_tenant_id_from_request(
     Attempt to extract tenant_id from:
     1) The API key header
     2) The Redis-based token (stored in Cookie: fastapiusersauth)
+    3)  Reset token cookie
     Fallback: POSTGRES_DEFAULT_SCHEMA
     """
     # Check for API key
@@ -85,8 +87,18 @@ async def _get_tenant_id_from_request(
         if not is_valid_schema_name(tenant_id):
             raise HTTPException(status_code=400, detail="Invalid tenant ID format")
 
-        return tenant_id
-
     except Exception as e:
         logger.error(f"Unexpected error in _get_tenant_id_from_request: {str(e)}")
         raise HTTPException(status_code=500, detail="Internal server error")
+
+    finally:
+        if tenant_id:
+            return tenant_id
+
+        # As a final step, check for explicit tenant_id cookie
+        tenant_id_cookie = request.cookies.get(TENANT_ID_COOKIE_NAME)
+        if tenant_id_cookie and is_valid_schema_name(tenant_id_cookie):
+            return tenant_id_cookie
+
+        # If we've reached this point, return the default schema
+        return POSTGRES_DEFAULT_SCHEMA

backend/ee/onyx/server/oauth.py

@@ -286,6 +286,7 @@ def prepare_authorization_request(
     oauth_state = (
         base64.urlsafe_b64encode(oauth_uuid.bytes).rstrip(b"=").decode("utf-8")
     )
+    session: str
 
     if connector == DocumentSource.SLACK:
         oauth_url = SlackOAuth.generate_oauth_url(oauth_state)
@@ -554,6 +555,7 @@ def handle_google_drive_oauth_callback(
         )
 
     session_json = session_json_bytes.decode("utf-8")
+    session: GoogleDriveOAuth.OAuthSession
     try:
         session = GoogleDriveOAuth.parse_session(session_json)
 

backend/ee/onyx/server/tenants/api.py

@@ -34,6 +34,7 @@ from onyx.auth.users import get_redis_strategy
 from onyx.auth.users import optional_user
 from onyx.auth.users import User
 from onyx.configs.app_configs import WEB_DOMAIN
+from onyx.configs.constants import FASTAPI_USERS_AUTH_COOKIE_NAME
 from onyx.db.auth import get_user_count
 from onyx.db.engine import get_current_tenant_id
 from onyx.db.engine import get_session
@@ -111,7 +112,7 @@ async def login_as_anonymous_user(
     token = generate_anonymous_user_jwt_token(tenant_id)
 
     response = Response()
-    response.delete_cookie("fastapiusersauth")
+    response.delete_cookie(FASTAPI_USERS_AUTH_COOKIE_NAME)
     response.set_cookie(
         key=ANONYMOUS_USER_COOKIE_NAME,
         value=token,

backend/onyx/agents/agent_search/basic/utils.py

@@ -7,6 +7,7 @@ from langgraph.types import StreamWriter
 
 from onyx.agents.agent_search.shared_graph_utils.utils import write_custom_event
 from onyx.chat.models import LlmDoc
+from onyx.chat.models import OnyxContext
 from onyx.chat.stream_processing.answer_response_handler import AnswerResponseHandler
 from onyx.chat.stream_processing.answer_response_handler import CitationResponseHandler
 from onyx.chat.stream_processing.answer_response_handler import (
@@ -23,7 +24,7 @@ def process_llm_stream(
     should_stream_answer: bool,
     writer: StreamWriter,
     final_search_results: list[LlmDoc] | None = None,
-    displayed_search_results: list[LlmDoc] | None = None,
+    displayed_search_results: list[OnyxContext] | list[LlmDoc] | None = None,
 ) -> AIMessageChunk:
     tool_call_chunk = AIMessageChunk(content="")
 

backend/onyx/agents/agent_search/deep_search/initial/generate_individual_sub_answer/nodes/generate_sub_answer.py

@@ -39,7 +39,7 @@ logger = setup_logger()
 def generate_sub_answer(
     state: AnswerQuestionState,
     config: RunnableConfig,
-    writer: StreamWriter,
+    writer: StreamWriter = lambda _: None,
 ) -> SubQuestionAnswerGenerationUpdate:
     """
     LangGraph node to generate a sub-answer.

backend/onyx/agents/agent_search/deep_search/initial/generate_initial_answer/nodes/generate_initial_answer.py

@@ -61,7 +61,7 @@ from onyx.tools.tool_implementations.search.search_tool import yield_search_resp
 def generate_initial_answer(
     state: SubQuestionRetrievalState,
     config: RunnableConfig,
-    writer: StreamWriter,
+    writer: StreamWriter = lambda _: None,
 ) -> InitialAnswerUpdate:
     """
     LangGraph node to generate the initial answer, using the initial sub-questions/sub-answers and the

backend/onyx/agents/agent_search/deep_search/initial/generate_sub_answers/nodes/decompose_orig_question.py

@@ -12,8 +12,9 @@ from onyx.agents.agent_search.deep_search.initial.generate_initial_answer.states
 from onyx.agents.agent_search.deep_search.main.models import (
     AgentRefinedMetrics,
 )
+from onyx.agents.agent_search.deep_search.main.operations import dispatch_subquestion
 from onyx.agents.agent_search.deep_search.main.operations import (
-    dispatch_subquestion,
+    dispatch_subquestion_sep,
 )
 from onyx.agents.agent_search.deep_search.main.states import (
     InitialQuestionDecompositionUpdate,
@@ -46,7 +47,7 @@ logger = setup_logger()
 def decompose_orig_question(
     state: SubQuestionRetrievalState,
     config: RunnableConfig,
-    writer: StreamWriter,
+    writer: StreamWriter = lambda _: None,
 ) -> InitialQuestionDecompositionUpdate:
     """
     LangGraph node to decompose the original question into sub-questions.
@@ -109,9 +110,12 @@ def decompose_orig_question(
         ),
         writer,
     )
+
     # dispatches custom events for subquestion tokens, adding in subquestion ids.
     streamed_tokens = dispatch_separated(
-        model.stream(msg), dispatch_subquestion(0, writer)
+        model.stream(msg),
+        dispatch_subquestion(0, writer),
+        sep_callback=dispatch_subquestion_sep(0, writer),
     )
 
     stop_event = StreamStopInfo(

backend/onyx/agents/agent_search/deep_search/main/nodes/create_refined_sub_questions.py

@@ -9,8 +9,9 @@ from langgraph.types import StreamWriter
 from onyx.agents.agent_search.deep_search.main.models import (
     RefinementSubQuestion,
 )
+from onyx.agents.agent_search.deep_search.main.operations import dispatch_subquestion
 from onyx.agents.agent_search.deep_search.main.operations import (
-    dispatch_subquestion,
+    dispatch_subquestion_sep,
 )
 from onyx.agents.agent_search.deep_search.main.states import MainState
 from onyx.agents.agent_search.deep_search.main.states import (
@@ -96,7 +97,9 @@ def create_refined_sub_questions(
     model = graph_config.tooling.fast_llm
 
     streamed_tokens = dispatch_separated(
-        model.stream(msg), dispatch_subquestion(1, writer)
+        model.stream(msg),
+        dispatch_subquestion(1, writer),
+        sep_callback=dispatch_subquestion_sep(1, writer),
     )
     response = merge_content(*streamed_tokens)
 

backend/onyx/agents/agent_search/deep_search/main/operations.py

@@ -9,6 +9,9 @@ from onyx.agents.agent_search.shared_graph_utils.models import (
     SubQuestionAnswerResults,
 )
 from onyx.agents.agent_search.shared_graph_utils.utils import write_custom_event
+from onyx.chat.models import StreamStopInfo
+from onyx.chat.models import StreamStopReason
+from onyx.chat.models import StreamType
 from onyx.chat.models import SubQuestionPiece
 from onyx.context.search.models import IndexFilters
 from onyx.tools.models import SearchQueryInfo
@@ -34,6 +37,22 @@ def dispatch_subquestion(
     return _helper
 
 
+def dispatch_subquestion_sep(level: int, writer: StreamWriter) -> Callable[[int], None]:
+    def _helper(sep_num: int) -> None:
+        write_custom_event(
+            "stream_finished",
+            StreamStopInfo(
+                stop_reason=StreamStopReason.FINISHED,
+                stream_type=StreamType.SUB_QUESTIONS,
+                level=level,
+                level_question_num=sep_num,
+            ),
+            writer,
+        )
+
+    return _helper
+
+
 def calculate_initial_agent_stats(
     decomp_answer_results: list[SubQuestionAnswerResults],
     original_question_stats: AgentChunkRetrievalStats,

backend/onyx/agents/agent_search/deep_search/shared/expanded_retrieval/nodes/expand_queries.py

@@ -29,7 +29,7 @@ from onyx.prompts.agent_search import (
 def expand_queries(
     state: ExpandedRetrievalInput,
     config: RunnableConfig,
-    writer: StreamWriter,
+    writer: StreamWriter = lambda _: None,
 ) -> QueryExpansionUpdate:
     """
     LangGraph node to expand a question into multiple search queries.

backend/onyx/agents/agent_search/deep_search/shared/expanded_retrieval/nodes/format_results.py

@@ -28,7 +28,7 @@ from onyx.tools.tool_implementations.search.search_tool import yield_search_resp
 def format_results(
     state: ExpandedRetrievalState,
     config: RunnableConfig,
-    writer: StreamWriter,
+    writer: StreamWriter = lambda _: None,
 ) -> ExpandedRetrievalUpdate:
     """
     LangGraph node that constructs the proper expanded retrieval format.

backend/onyx/agents/agent_search/orchestration/nodes/basic_use_tool_response.py

@@ -9,6 +9,7 @@ from onyx.agents.agent_search.basic.states import BasicState
 from onyx.agents.agent_search.basic.utils import process_llm_stream
 from onyx.agents.agent_search.models import GraphConfig
 from onyx.chat.models import LlmDoc
+from onyx.chat.models import OnyxContexts
 from onyx.tools.tool_implementations.search.search_tool import (
     SEARCH_DOC_CONTENT_ID,
 )
@@ -50,13 +51,11 @@ def basic_use_tool_response(
         if yield_item.id == FINAL_CONTEXT_DOCUMENTS_ID:
             final_search_results = cast(list[LlmDoc], yield_item.response)
         elif yield_item.id == SEARCH_DOC_CONTENT_ID:
-            search_contexts = yield_item.response.contexts
+            search_contexts = cast(OnyxContexts, yield_item.response).contexts
             for doc in search_contexts:
                 if doc.document_id not in initial_search_results:
                     initial_search_results.append(doc)
 
-            initial_search_results = cast(list[LlmDoc], initial_search_results)
-
     new_tool_call_chunk = AIMessageChunk(content="")
     if not agent_config.behavior.skip_gen_ai_answer_generation:
         stream = llm.stream(
@@ -70,7 +69,9 @@ def basic_use_tool_response(
             True,
             writer,
             final_search_results=final_search_results,
-            displayed_search_results=initial_search_results,
+            # when the search tool is called with specific doc ids, initial search
+            # results are not output. But, we still want i.e. citations to be processed.
+            displayed_search_results=initial_search_results or final_search_results,
         )
 
     return BasicOutput(tool_call_chunk=new_tool_call_chunk)

backend/onyx/agents/agent_search/orchestration/nodes/llm_tool_choice.py

@@ -26,7 +26,9 @@ logger = setup_logger()
 # from the state and config
 # TODO: fan-out to multiple tool call nodes? Make this configurable?
 def llm_tool_choice(
-    state: ToolChoiceState, config: RunnableConfig, writer: StreamWriter
+    state: ToolChoiceState,
+    config: RunnableConfig,
+    writer: StreamWriter = lambda _: None,
 ) -> ToolChoiceUpdate:
     """
     This node is responsible for calling the LLM to choose a tool. If no tool is chosen,

backend/onyx/agents/agent_search/orchestration/nodes/tool_call.py

@@ -20,6 +20,10 @@ from onyx.utils.logger import setup_logger
 logger = setup_logger()
 
 
+class ToolCallException(Exception):
+    """Exception raised for errors during tool calls."""
+
+
 def emit_packet(packet: AnswerPacket, writer: StreamWriter) -> None:
     write_custom_event("basic_response", packet, writer)
 
@@ -27,7 +31,7 @@ def emit_packet(packet: AnswerPacket, writer: StreamWriter) -> None:
 def tool_call(
     state: ToolChoiceUpdate,
     config: RunnableConfig,
-    writer: StreamWriter,
+    writer: StreamWriter = lambda _: None,
 ) -> ToolCallUpdate:
     """Calls the tool specified in the state and updates the state with the result"""
 
@@ -45,13 +49,18 @@ def tool_call(
 
     emit_packet(tool_kickoff, writer)
 
-    tool_responses = []
-    for response in tool_runner.tool_responses():
-        tool_responses.append(response)
-        emit_packet(response, writer)
+    try:
+        tool_responses = []
+        for response in tool_runner.tool_responses():
+            tool_responses.append(response)
+            emit_packet(response, writer)
 
-    tool_final_result = tool_runner.tool_final_result()
-    emit_packet(tool_final_result, writer)
+        tool_final_result = tool_runner.tool_final_result()
+        emit_packet(tool_final_result, writer)
+    except Exception as e:
+        raise ToolCallException(
+            f"Error during tool call for {tool.display_name}: {e}"
+        ) from e
 
     tool_call = ToolCall(name=tool.name, args=tool_args, id=tool_id)
     tool_call_summary = ToolCallSummary(

backend/onyx/agents/agent_search/shared_graph_utils/models.py

@@ -12,6 +12,20 @@ from onyx.context.search.models import InferenceSection
 from onyx.tools.models import SearchQueryInfo
 
 
+# Pydantic models for structured outputs
+# class RewrittenQueries(BaseModel):
+#     rewritten_queries: list[str]
+
+
+# class BinaryDecision(BaseModel):
+#     decision: Literal["yes", "no"]
+
+
+# class BinaryDecisionWithReasoning(BaseModel):
+#     reasoning: str
+#     decision: Literal["yes", "no"]
+
+
 class RetrievalFitScoreMetrics(BaseModel):
     scores: dict[str, float]
     chunk_ids: list[str]

backend/onyx/agents/agent_search/shared_graph_utils/utils.py

@@ -295,6 +295,7 @@ def _dispatch_nonempty(
 def dispatch_separated(
     tokens: Iterator[BaseMessage],
     dispatch_event: Callable[[str, int], None],
+    sep_callback: Callable[[int], None] | None = None,
     sep: str = DISPATCH_SEP_CHAR,
 ) -> list[BaseMessage_Content]:
     num = 1
@@ -304,6 +305,10 @@ def dispatch_separated(
         if sep in content:
             sub_question_parts = content.split(sep)
             _dispatch_nonempty(sub_question_parts[0], dispatch_event, num)
+
+            if sep_callback:
+                sep_callback(num)
+
             num += 1
             _dispatch_nonempty(
                 "".join(sub_question_parts[1:]).strip(), dispatch_event, num
@@ -312,6 +317,9 @@ def dispatch_separated(
             _dispatch_nonempty(content, dispatch_event, num)
         streamed_tokens.append(content)
 
+    if sep_callback:
+        sep_callback(num)
+
     return streamed_tokens
 
 

backend/onyx/auth/email_utils.py

@@ -10,6 +10,7 @@ from onyx.configs.app_configs import SMTP_PORT
 from onyx.configs.app_configs import SMTP_SERVER
 from onyx.configs.app_configs import SMTP_USER
 from onyx.configs.app_configs import WEB_DOMAIN
+from onyx.configs.constants import TENANT_ID_COOKIE_NAME
 from onyx.db.models import User
 
 
@@ -65,9 +66,13 @@ def send_forgot_password_email(
     user_email: str,
     token: str,
     mail_from: str = EMAIL_FROM,
+    tenant_id: str | None = None,
 ) -> None:
     subject = "Onyx Forgot Password"
     link = f"{WEB_DOMAIN}/auth/reset-password?token={token}"
+    if tenant_id:
+        link += f"&{TENANT_ID_COOKIE_NAME}={tenant_id}"
+        # Keep search param same name as cookie for simplicity
     body = f"Click the following link to reset your password: {link}"
     send_email(user_email, subject, body, mail_from)
 

backend/onyx/auth/users.py

@@ -73,6 +73,7 @@ from onyx.configs.app_configs import WEB_DOMAIN
 from onyx.configs.constants import AuthType
 from onyx.configs.constants import DANSWER_API_KEY_DUMMY_EMAIL_DOMAIN
 from onyx.configs.constants import DANSWER_API_KEY_PREFIX
+from onyx.configs.constants import FASTAPI_USERS_AUTH_COOKIE_NAME
 from onyx.configs.constants import MilestoneRecordType
 from onyx.configs.constants import OnyxRedisLocks
 from onyx.configs.constants import PASSWORD_SPECIAL_CHARS
@@ -218,6 +219,24 @@ class UserManager(UUIDIDMixin, BaseUserManager[User, uuid.UUID]):
     verification_token_lifetime_seconds = AUTH_COOKIE_EXPIRE_TIME_SECONDS
     user_db: SQLAlchemyUserDatabase[User, uuid.UUID]
 
+    async def get_by_email(self, user_email: str) -> User:
+        tenant_id = fetch_ee_implementation_or_noop(
+            "onyx.server.tenants.user_mapping", "get_tenant_id_for_email", None
+        )(user_email)
+        async with get_async_session_with_tenant(tenant_id) as db_session:
+            if MULTI_TENANT:
+                tenant_user_db = SQLAlchemyUserAdminDB[User, uuid.UUID](
+                    db_session, User, OAuthAccount
+                )
+                user = await tenant_user_db.get_by_email(user_email)
+            else:
+                user = await self.user_db.get_by_email(user_email)
+
+        if not user:
+            raise exceptions.UserNotExists()
+
+        return user
+
     async def create(
         self,
         user_create: schemas.UC | UserCreate,
@@ -245,6 +264,8 @@ class UserManager(UUIDIDMixin, BaseUserManager[User, uuid.UUID]):
             referral_source=referral_source,
             request=request,
         )
+        user: User
+
         async with get_async_session_with_tenant(tenant_id) as db_session:
             token = CURRENT_TENANT_ID_CONTEXTVAR.set(tenant_id)
             verify_email_is_invited(user_create.email)
@@ -368,6 +389,8 @@ class UserManager(UUIDIDMixin, BaseUserManager[User, uuid.UUID]):
                 "refresh_token": refresh_token,
             }
 
+            user: User
+
             try:
                 # Attempt to get user by OAuth account
                 user = await self.get_by_oauth_account(oauth_name, account_id)
@@ -500,9 +523,15 @@ class UserManager(UUIDIDMixin, BaseUserManager[User, uuid.UUID]):
             )
             raise HTTPException(
                 status.HTTP_500_INTERNAL_SERVER_ERROR,
-                "Your admin has not enbaled this feature.",
+                "Your admin has not enabled this feature.",
             )
-        send_forgot_password_email(user.email, token)
+        tenant_id = await fetch_ee_implementation_or_noop(
+            "onyx.server.tenants.provisioning",
+            "get_or_provision_tenant",
+            async_return_default_schema,
+        )(email=user.email)
+
+        send_forgot_password_email(user.email, token, tenant_id=tenant_id)
 
     async def on_after_request_verify(
         self, user: User, token: str, request: Optional[Request] = None
@@ -576,6 +605,7 @@ async def get_user_manager(
 cookie_transport = CookieTransport(
     cookie_max_age=SESSION_EXPIRE_TIME_SECONDS,
     cookie_secure=WEB_DOMAIN.startswith("https"),
+    cookie_name=FASTAPI_USERS_AUTH_COOKIE_NAME,
 )
 
 
@@ -1043,6 +1073,8 @@ async def api_key_dep(
     if AUTH_TYPE == AuthType.DISABLED:
         return None
 
+    user: User | None = None
+
     hashed_api_key = get_hashed_api_key_from_request(request)
     if not hashed_api_key:
         raise HTTPException(status_code=401, detail="Missing API key")

backend/onyx/background/celery/apps/primary.py

@@ -21,13 +21,16 @@ from onyx.background.celery.tasks.indexing.utils import (
     get_unfenced_index_attempt_ids,
 )
 from onyx.configs.constants import CELERY_PRIMARY_WORKER_LOCK_TIMEOUT
+from onyx.configs.constants import OnyxRedisConstants
 from onyx.configs.constants import OnyxRedisLocks
 from onyx.configs.constants import POSTGRES_CELERY_WORKER_PRIMARY_APP_NAME
 from onyx.db.engine import get_session_with_default_tenant
 from onyx.db.engine import SqlEngine
 from onyx.db.index_attempt import get_index_attempt
 from onyx.db.index_attempt import mark_attempt_canceled
-from onyx.redis.redis_connector_credential_pair import RedisConnectorCredentialPair
+from onyx.redis.redis_connector_credential_pair import (
+    RedisGlobalConnectorCredentialPair,
+)
 from onyx.redis.redis_connector_delete import RedisConnectorDelete
 from onyx.redis.redis_connector_doc_perm_sync import RedisConnectorPermissionSync
 from onyx.redis.redis_connector_ext_group_sync import RedisConnectorExternalGroupSync
@@ -141,23 +144,16 @@ def on_worker_init(sender: Worker, **kwargs: Any) -> None:
     r.delete(OnyxRedisLocks.CHECK_VESPA_SYNC_BEAT_LOCK)
     r.delete(OnyxRedisLocks.MONITOR_VESPA_SYNC_BEAT_LOCK)
 
-    r.delete(RedisConnectorCredentialPair.get_taskset_key())
-    r.delete(RedisConnectorCredentialPair.get_fence_key())
+    r.delete(OnyxRedisConstants.ACTIVE_FENCES)
 
+    RedisGlobalConnectorCredentialPair.reset_all(r)
     RedisDocumentSet.reset_all(r)
-
     RedisUserGroup.reset_all(r)
-
     RedisConnectorDelete.reset_all(r)
-
     RedisConnectorPrune.reset_all(r)
-
     RedisConnectorIndex.reset_all(r)
-
     RedisConnectorStop.reset_all(r)
-
     RedisConnectorPermissionSync.reset_all(r)
-
     RedisConnectorExternalGroupSync.reset_all(r)
 
     # mark orphaned index attempts as failed

backend/onyx/background/celery/tasks/beat_schedule.py

@@ -18,7 +18,7 @@ BEAT_EXPIRES_DEFAULT = 15 * 60  # 15 minutes (in seconds)
 
 # hack to slow down task dispatch in the cloud until
 # we have a better implementation (backpressure, etc)
-CLOUD_BEAT_SCHEDULE_MULTIPLIER = 8
+CLOUD_BEAT_SCHEDULE_MULTIPLIER = 4
 
 # tasks that only run in the cloud
 # the name attribute must start with ONYX_CLOUD_CELERY_TASK_PREFIX = "cloud" to be filtered

backend/onyx/background/celery/tasks/connector_deletion/tasks.py

@@ -179,11 +179,14 @@ def try_generate_document_cc_pair_cleanup_tasks(
         if tasks_generated is None:
             raise ValueError("RedisConnectorDeletion.generate_tasks returned None")
 
-        insert_sync_record(
-            db_session=db_session,
-            entity_id=cc_pair_id,
-            sync_type=SyncType.CONNECTOR_DELETION,
-        )
+        try:
+            insert_sync_record(
+                db_session=db_session,
+                entity_id=cc_pair_id,
+                sync_type=SyncType.CONNECTOR_DELETION,
+            )
+        except Exception:
+            pass
 
     except TaskDependencyError:
         redis_connector.delete.set_fence(None)

backend/onyx/background/celery/tasks/doc_permission_syncing/tasks.py

@@ -3,6 +3,7 @@ from datetime import datetime
 from datetime import timedelta
 from datetime import timezone
 from time import sleep
+from typing import Any
 from typing import cast
 from uuid import uuid4
 
@@ -38,6 +39,7 @@ from onyx.configs.constants import DocumentSource
 from onyx.configs.constants import OnyxCeleryPriority
 from onyx.configs.constants import OnyxCeleryQueues
 from onyx.configs.constants import OnyxCeleryTask
+from onyx.configs.constants import OnyxRedisConstants
 from onyx.configs.constants import OnyxRedisLocks
 from onyx.configs.constants import OnyxRedisSignals
 from onyx.db.connector import mark_cc_pair_as_permissions_synced
@@ -57,8 +59,8 @@ from onyx.redis.redis_connector import RedisConnector
 from onyx.redis.redis_connector_doc_perm_sync import RedisConnectorPermissionSync
 from onyx.redis.redis_connector_doc_perm_sync import RedisConnectorPermissionSyncPayload
 from onyx.redis.redis_pool import get_redis_client
+from onyx.redis.redis_pool import get_redis_replica_client
 from onyx.redis.redis_pool import redis_lock_dump
-from onyx.redis.redis_pool import SCAN_ITER_COUNT_DEFAULT
 from onyx.server.utils import make_short_id
 from onyx.utils.logger import doc_permission_sync_ctx
 from onyx.utils.logger import LoggerContextVars
@@ -123,6 +125,7 @@ def check_for_doc_permissions_sync(self: Task, *, tenant_id: str | None) -> bool
     # we need to use celery's redis client to access its redis data
     # (which lives on a different db number)
     r = get_redis_client(tenant_id=tenant_id)
+    r_replica = get_redis_replica_client(tenant_id=tenant_id)
     r_celery: Redis = self.app.broker_connection().channel().client  # type: ignore
 
     lock_beat: RedisLock = r.lock(
@@ -158,18 +161,20 @@ def check_for_doc_permissions_sync(self: Task, *, tenant_id: str | None) -> bool
 
         # we want to run this less frequently than the overall task
         lock_beat.reacquire()
-        if not r.exists(OnyxRedisSignals.VALIDATE_PERMISSION_SYNC_FENCES):
+        if not r.exists(OnyxRedisSignals.BLOCK_VALIDATE_PERMISSION_SYNC_FENCES):
             # clear any permission fences that don't have associated celery tasks in progress
             # tasks can be in the queue in redis, in reserved tasks (prefetched by the worker),
             # or be currently executing
             try:
-                validate_permission_sync_fences(tenant_id, r, r_celery, lock_beat)
+                validate_permission_sync_fences(
+                    tenant_id, r, r_replica, r_celery, lock_beat
+                )
             except Exception:
                 task_logger.exception(
                     "Exception while validating permission sync fences"
                 )
 
-            r.set(OnyxRedisSignals.VALIDATE_PERMISSION_SYNC_FENCES, 1, ex=60)
+            r.set(OnyxRedisSignals.BLOCK_VALIDATE_PERMISSION_SYNC_FENCES, 1, ex=300)
     except SoftTimeLimitExceeded:
         task_logger.info(
             "Soft time limit exceeded, task is being terminated gracefully."
@@ -486,6 +491,7 @@ def update_external_document_permissions_task(
 def validate_permission_sync_fences(
     tenant_id: str | None,
     r: Redis,
+    r_replica: Redis,
     r_celery: Redis,
     lock_beat: RedisLock,
 ) -> None:
@@ -506,12 +512,15 @@ def validate_permission_sync_fences(
         OnyxCeleryQueues.CONNECTOR_DOC_PERMISSIONS_SYNC, r_celery
     )
 
-    # validate all existing indexing jobs
-    for key_bytes in r.scan_iter(
-        RedisConnectorPermissionSync.FENCE_PREFIX + "*",
-        count=SCAN_ITER_COUNT_DEFAULT,
-    ):
-        lock_beat.reacquire()
+    # validate all existing permission sync jobs
+    lock_beat.reacquire()
+    keys = cast(set[Any], r_replica.smembers(OnyxRedisConstants.ACTIVE_FENCES))
+    for key in keys:
+        key_bytes = cast(bytes, key)
+        key_str = key_bytes.decode("utf-8")
+        if not key_str.startswith(RedisConnectorPermissionSync.FENCE_PREFIX):
+            continue
+
         validate_permission_sync_fence(
             tenant_id,
             key_bytes,
@@ -520,6 +529,9 @@ def validate_permission_sync_fences(
             r,
             r_celery,
         )
+
+        lock_beat.reacquire()
+
     return
 
 
@@ -647,7 +659,8 @@ def validate_permission_sync_fence(
         f"tasks_scanned={tasks_scanned} tasks_not_in_celery={tasks_not_in_celery}"
     )
 
-    if tasks_not_in_celery == 0:
+    # we're only active if tasks_scanned > 0 and tasks_not_in_celery == 0
+    if tasks_scanned > 0 and tasks_not_in_celery == 0:
         redis_connector.permissions.set_active()
         return
 

backend/onyx/background/celery/tasks/indexing/tasks.py

@@ -224,7 +224,7 @@ def check_for_indexing(self: Task, *, tenant_id: str | None) -> int | None:
 
         lock_beat.reacquire()
         # we want to run this less frequently than the overall task
-        if not redis_client.exists(OnyxRedisSignals.VALIDATE_INDEXING_FENCES):
+        if not redis_client.exists(OnyxRedisSignals.BLOCK_VALIDATE_INDEXING_FENCES):
             # clear any indexing fences that don't have associated celery tasks in progress
             # tasks can be in the queue in redis, in reserved tasks (prefetched by the worker),
             # or be currently executing
@@ -235,7 +235,7 @@ def check_for_indexing(self: Task, *, tenant_id: str | None) -> int | None:
             except Exception:
                 task_logger.exception("Exception while validating indexing fences")
 
-            redis_client.set(OnyxRedisSignals.VALIDATE_INDEXING_FENCES, 1, ex=60)
+            redis_client.set(OnyxRedisSignals.BLOCK_VALIDATE_INDEXING_FENCES, 1, ex=60)
     except SoftTimeLimitExceeded:
         task_logger.info(
             "Soft time limit exceeded, task is being terminated gracefully."
@@ -586,11 +586,12 @@ def connector_indexing_proxy_task(
 
         # if the job is done, clean up and break
         if job.done():
+            exit_code: int | None
             try:
                 if job.status == "error":
                     ignore_exitcode = False
 
-                    exit_code: int | None = None
+                    exit_code = None
                     if job.process:
                         exit_code = job.process.exitcode
 

backend/onyx/background/celery/tasks/indexing/utils.py

@@ -1,6 +1,8 @@
 import time
 from datetime import datetime
 from datetime import timezone
+from typing import Any
+from typing import cast
 
 import redis
 from celery import Celery
@@ -19,6 +21,7 @@ from onyx.configs.constants import DocumentSource
 from onyx.configs.constants import OnyxCeleryPriority
 from onyx.configs.constants import OnyxCeleryQueues
 from onyx.configs.constants import OnyxCeleryTask
+from onyx.configs.constants import OnyxRedisConstants
 from onyx.db.engine import get_db_current_time
 from onyx.db.engine import get_session_with_tenant
 from onyx.db.enums import ConnectorCredentialPairStatus
@@ -37,7 +40,6 @@ from onyx.redis.redis_connector import RedisConnector
 from onyx.redis.redis_connector_index import RedisConnectorIndex
 from onyx.redis.redis_connector_index import RedisConnectorIndexPayload
 from onyx.redis.redis_pool import redis_lock_dump
-from onyx.redis.redis_pool import SCAN_ITER_COUNT_DEFAULT
 from onyx.utils.logger import setup_logger
 
 logger = setup_logger()
@@ -304,10 +306,13 @@ def validate_indexing_fences(
 
     # Use replica for this because the worst thing that happens
     # is that we don't run the validation on this pass
-    for key_bytes in r_replica.scan_iter(
-        RedisConnectorIndex.FENCE_PREFIX + "*", count=SCAN_ITER_COUNT_DEFAULT
-    ):
-        lock_beat.reacquire()
+    keys = cast(set[Any], r_replica.smembers(OnyxRedisConstants.ACTIVE_FENCES))
+    for key in keys:
+        key_bytes = cast(bytes, key)
+        key_str = key_bytes.decode("utf-8")
+        if not key_str.startswith(RedisConnectorIndex.FENCE_PREFIX):
+            continue
+
         with get_session_with_tenant(tenant_id) as db_session:
             validate_indexing_fence(
                 tenant_id,
@@ -316,6 +321,9 @@ def validate_indexing_fences(
                 r_celery,
                 db_session,
             )
+
+        lock_beat.reacquire()
+
     return
 
 
@@ -438,6 +446,7 @@ def try_creating_indexing_task(
     if not acquired:
         return None
 
+    redis_connector_index: RedisConnectorIndex
     try:
         redis_connector = RedisConnector(tenant_id, cc_pair.id)
         redis_connector_index = redis_connector.new_index(search_settings.id)

backend/onyx/background/celery/tasks/monitoring/tasks.py

@@ -728,6 +728,10 @@ def cloud_check_alembic() -> bool | None:
     TODO: have the cloud migration script set an activity signal that this check
     uses to know it doesn't make sense to run a check at the present time.
     """
+
+    # Used as a placeholder if the alembic revision cannot be retrieved
+    ALEMBIC_NULL_REVISION = "000000000000"
+
     time_start = time.monotonic()
 
     redis_client = get_redis_client(tenant_id=ONYX_CLOUD_TENANT_ID)
@@ -743,13 +747,14 @@ def cloud_check_alembic() -> bool | None:
 
     last_lock_time = time.monotonic()
 
-    tenant_to_revision: dict[str, str | None] = {}
+    tenant_to_revision: dict[str, str] = {}
     revision_counts: dict[str, int] = {}
-    out_of_date_tenants: dict[str, str | None] = {}
+    out_of_date_tenants: dict[str, str] = {}
     top_revision: str = ""
+    tenant_ids: list[str] | list[None] = []
 
     try:
-        # map each tenant_id to its revision
+        # map tenant_id to revision (or ALEMBIC_NULL_REVISION if the query fails)
         tenant_ids = get_all_tenant_ids()
         for tenant_id in tenant_ids:
             current_time = time.monotonic()
@@ -761,43 +766,53 @@ def cloud_check_alembic() -> bool | None:
                 continue
 
             with get_session_with_tenant(tenant_id=None) as session:
-                result = session.execute(
-                    text(f'SELECT * FROM "{tenant_id}".alembic_version LIMIT 1')
-                )
+                try:
+                    result = session.execute(
+                        text(f'SELECT * FROM "{tenant_id}".alembic_version LIMIT 1')
+                    )
 
-                result_scalar: str | None = result.scalar_one_or_none()
-                tenant_to_revision[tenant_id] = result_scalar
+                    result_scalar: str | None = result.scalar_one_or_none()
+                    if result_scalar is None:
+                        raise ValueError("Alembic version should not be None.")
+
+                    tenant_to_revision[tenant_id] = result_scalar
+                except Exception:
+                    task_logger.warning(f"Tenant {tenant_id} has no revision!")
+                    tenant_to_revision[tenant_id] = ALEMBIC_NULL_REVISION
 
         # get the total count of each revision
         for k, v in tenant_to_revision.items():
-            if v is None:
-                continue
-
             revision_counts[v] = revision_counts.get(v, 0) + 1
 
+        # error if any null revision tenants are found
+        if ALEMBIC_NULL_REVISION in revision_counts:
+            num_null_revisions = revision_counts[ALEMBIC_NULL_REVISION]
+            raise ValueError(f"No revision was found for {num_null_revisions} tenants!")
+
         # get the revision with the most counts
         sorted_revision_counts = sorted(
             revision_counts.items(), key=lambda item: item[1], reverse=True
         )
 
         if len(sorted_revision_counts) == 0:
-            task_logger.error(
+            raise ValueError(
                 f"cloud_check_alembic - No revisions found for {len(tenant_ids)} tenant ids!"
             )
-        else:
-            top_revision, _ = sorted_revision_counts[0]
 
-            # build a list of out of date tenants
-            for k, v in tenant_to_revision.items():
-                if v == top_revision:
-                    continue
+        top_revision, _ = sorted_revision_counts[0]
 
-                out_of_date_tenants[k] = v
+        # build a list of out of date tenants
+        for k, v in tenant_to_revision.items():
+            if v == top_revision:
+                continue
+
+            out_of_date_tenants[k] = v
 
     except SoftTimeLimitExceeded:
         task_logger.info(
             "Soft time limit exceeded, task is being terminated gracefully."
         )
+        raise
     except Exception:
         task_logger.exception("Unexpected exception during cloud alembic check")
         raise
@@ -815,6 +830,11 @@ def cloud_check_alembic() -> bool | None:
             f"num_tenants={len(tenant_ids)} "
             f"revision={top_revision}"
         )
+
+        num_to_log = min(5, len(out_of_date_tenants))
+        task_logger.info(
+            f"Logging {num_to_log}/{len(out_of_date_tenants)} out of date tenants."
+        )
         for k, v in islice(out_of_date_tenants.items(), 5):
             task_logger.info(f"Out of date tenant: tenant={k} revision={v}")
     else:

backend/onyx/background/celery/tasks/shared/tasks.py

@@ -168,6 +168,7 @@ def document_by_cc_pair_cleanup_task(
         task_logger.info(f"SoftTimeLimitExceeded exception. doc={document_id}")
         return False
     except Exception as ex:
+        e: Exception | None = None
         if isinstance(ex, RetryError):
             task_logger.warning(
                 f"Tenacity retry failed: num_attempts={ex.last_attempt.attempt_number}"
@@ -247,6 +248,7 @@ def cloud_beat_task_generator(
         return None
 
     last_lock_time = time.monotonic()
+    tenant_ids: list[str] | list[None] = []
 
     try:
         tenant_ids = get_all_tenant_ids()

backend/onyx/background/celery/tasks/vespa/tasks.py

@@ -36,7 +36,9 @@ from onyx.configs.app_configs import VESPA_SYNC_MAX_TASKS
 from onyx.configs.constants import CELERY_VESPA_SYNC_BEAT_LOCK_TIMEOUT
 from onyx.configs.constants import OnyxCeleryQueues
 from onyx.configs.constants import OnyxCeleryTask
+from onyx.configs.constants import OnyxRedisConstants
 from onyx.configs.constants import OnyxRedisLocks
+from onyx.configs.constants import OnyxRedisSignals
 from onyx.db.connector import fetch_connector_by_id
 from onyx.db.connector_credential_pair import add_deletion_failure_message
 from onyx.db.connector_credential_pair import (
@@ -72,6 +74,9 @@ from onyx.document_index.interfaces import VespaDocumentFields
 from onyx.httpx.httpx_pool import HttpxPool
 from onyx.redis.redis_connector import RedisConnector
 from onyx.redis.redis_connector_credential_pair import RedisConnectorCredentialPair
+from onyx.redis.redis_connector_credential_pair import (
+    RedisGlobalConnectorCredentialPair,
+)
 from onyx.redis.redis_connector_delete import RedisConnectorDelete
 from onyx.redis.redis_connector_doc_perm_sync import RedisConnectorPermissionSync
 from onyx.redis.redis_connector_index import RedisConnectorIndex
@@ -204,10 +209,12 @@ def try_generate_stale_document_sync_tasks(
     tenant_id: str | None,
 ) -> int | None:
     # the fence is up, do nothing
-    if r.exists(RedisConnectorCredentialPair.get_fence_key()):
+
+    redis_global_ccpair = RedisGlobalConnectorCredentialPair(r)
+    if redis_global_ccpair.fenced:
         return None
 
-    r.delete(RedisConnectorCredentialPair.get_taskset_key())  # delete the taskset
+    redis_global_ccpair.delete_taskset()
 
     # add tasks to celery and build up the task set to monitor in redis
     stale_doc_count = count_documents_by_needs_sync(db_session)
@@ -265,7 +272,7 @@ def try_generate_stale_document_sync_tasks(
             f"RedisConnector.generate_tasks finished for all cc_pairs. total_tasks_generated={total_tasks_generated}"
         )
 
-    r.set(RedisConnectorCredentialPair.get_fence_key(), total_tasks_generated)
+    redis_global_ccpair.set_fence(total_tasks_generated)
     return total_tasks_generated
 
 
@@ -416,23 +423,17 @@ def try_generate_user_group_sync_tasks(
 
 
 def monitor_connector_taskset(r: Redis) -> None:
-    fence_value = r.get(RedisConnectorCredentialPair.get_fence_key())
-    if fence_value is None:
+    redis_global_ccpair = RedisGlobalConnectorCredentialPair(r)
+    initial_count = redis_global_ccpair.payload
+    if initial_count is None:
         return
 
-    try:
-        initial_count = int(cast(int, fence_value))
-    except ValueError:
-        task_logger.error("The value is not an integer.")
-        return
-
-    count = r.scard(RedisConnectorCredentialPair.get_taskset_key())
+    remaining = redis_global_ccpair.get_remaining()
     task_logger.info(
-        f"Stale document sync progress: remaining={count} initial={initial_count}"
+        f"Stale document sync progress: remaining={remaining} initial={initial_count}"
     )
-    if count == 0:
-        r.delete(RedisConnectorCredentialPair.get_taskset_key())
-        r.delete(RedisConnectorCredentialPair.get_fence_key())
+    if remaining == 0:
+        redis_global_ccpair.reset()
         task_logger.info(f"Successfully synced stale documents. count={initial_count}")
 
 
@@ -820,9 +821,6 @@ def monitor_vespa_sync(self: Task, tenant_id: str | None) -> bool | None:
 
     time_start = time.monotonic()
 
-    timings: dict[str, Any] = {}
-    timings["start"] = time_start
-
     r = get_redis_client(tenant_id=tenant_id)
 
     # Replica usage notes
@@ -847,7 +845,7 @@ def monitor_vespa_sync(self: Task, tenant_id: str | None) -> bool | None:
 
     try:
         # print current queue lengths
-        phase_start = time.monotonic()
+        time.monotonic()
         # we don't need every tenant polling redis for this info.
         if not MULTI_TENANT or random.randint(1, 10) == 10:
             r_celery = self.app.broker_connection().channel().client  # type: ignore
@@ -889,50 +887,38 @@ def monitor_vespa_sync(self: Task, tenant_id: str | None) -> bool | None:
                 f"external_group_sync={n_external_group_sync} "
                 f"permissions_upsert={n_permissions_upsert} "
             )
-        timings["queues"] = time.monotonic() - phase_start
-        timings["queues_ttl"] = r.ttl(OnyxRedisLocks.MONITOR_VESPA_SYNC_BEAT_LOCK)
 
-        # scan and monitor activity to completion
-        phase_start = time.monotonic()
-        lock_beat.reacquire()
-        if r_replica.exists(RedisConnectorCredentialPair.get_fence_key()):
-            if r.exists(RedisConnectorCredentialPair.get_fence_key()):
+        # we want to run this less frequently than the overall task
+        if not r.exists(OnyxRedisSignals.BLOCK_BUILD_FENCE_LOOKUP_TABLE):
+            # build a lookup table of existing fences
+            # this is just a migration concern and should be unnecessary once
+            # lookup tables are rolled out
+            for key_bytes in r_replica.scan_iter(count=SCAN_ITER_COUNT_DEFAULT):
+                if is_fence(key_bytes) and not r.sismember(
+                    OnyxRedisConstants.ACTIVE_FENCES, key_bytes
+                ):
+                    logger.warning(f"Adding {key_bytes} to the lookup table.")
+                    r.sadd(OnyxRedisConstants.ACTIVE_FENCES, key_bytes)
+
+            r.set(OnyxRedisSignals.BLOCK_BUILD_FENCE_LOOKUP_TABLE, 1, ex=300)
+
+        # use a lookup table to find active fences. We still have to verify the fence
+        # exists since it is an optimization and not the source of truth.
+        keys = cast(set[Any], r.smembers(OnyxRedisConstants.ACTIVE_FENCES))
+        for key in keys:
+            key_bytes = cast(bytes, key)
+
+            if not r.exists(key_bytes):
+                r.srem(OnyxRedisConstants.ACTIVE_FENCES, key_bytes)
+                continue
+
+            key_str = key_bytes.decode("utf-8")
+            if key_str == RedisGlobalConnectorCredentialPair.FENCE_KEY:
                 monitor_connector_taskset(r)
-        timings["connector"] = time.monotonic() - phase_start
-        timings["connector_ttl"] = r.ttl(OnyxRedisLocks.MONITOR_VESPA_SYNC_BEAT_LOCK)
-
-        phase_start = time.monotonic()
-        lock_beat.reacquire()
-        for key_bytes in r_replica.scan_iter(
-            RedisConnectorDelete.FENCE_PREFIX + "*", count=SCAN_ITER_COUNT_DEFAULT
-        ):
-            if r.exists(key_bytes):
-                monitor_connector_deletion_taskset(tenant_id, key_bytes, r)
-            lock_beat.reacquire()
-
-        timings["connector_deletion"] = time.monotonic() - phase_start
-        timings["connector_deletion_ttl"] = r.ttl(
-            OnyxRedisLocks.MONITOR_VESPA_SYNC_BEAT_LOCK
-        )
-
-        phase_start = time.monotonic()
-        lock_beat.reacquire()
-        for key_bytes in r_replica.scan_iter(
-            RedisDocumentSet.FENCE_PREFIX + "*", count=SCAN_ITER_COUNT_DEFAULT
-        ):
-            if r.exists(key_bytes):
+            elif key_str.startswith(RedisDocumentSet.FENCE_PREFIX):
                 with get_session_with_tenant(tenant_id) as db_session:
                     monitor_document_set_taskset(tenant_id, key_bytes, r, db_session)
-            lock_beat.reacquire()
-        timings["documentset"] = time.monotonic() - phase_start
-        timings["documentset_ttl"] = r.ttl(OnyxRedisLocks.MONITOR_VESPA_SYNC_BEAT_LOCK)
-
-        phase_start = time.monotonic()
-        lock_beat.reacquire()
-        for key_bytes in r_replica.scan_iter(
-            RedisUserGroup.FENCE_PREFIX + "*", count=SCAN_ITER_COUNT_DEFAULT
-        ):
-            if r.exists(key_bytes):
+            elif key_str.startswith(RedisUserGroup.FENCE_PREFIX):
                 monitor_usergroup_taskset = (
                     fetch_versioned_implementation_with_fallback(
                         "onyx.background.celery.tasks.vespa.tasks",
@@ -942,49 +928,21 @@ def monitor_vespa_sync(self: Task, tenant_id: str | None) -> bool | None:
                 )
                 with get_session_with_tenant(tenant_id) as db_session:
                     monitor_usergroup_taskset(tenant_id, key_bytes, r, db_session)
-            lock_beat.reacquire()
-        timings["usergroup"] = time.monotonic() - phase_start
-        timings["usergroup_ttl"] = r.ttl(OnyxRedisLocks.MONITOR_VESPA_SYNC_BEAT_LOCK)
-
-        phase_start = time.monotonic()
-        lock_beat.reacquire()
-        for key_bytes in r_replica.scan_iter(
-            RedisConnectorPrune.FENCE_PREFIX + "*", count=SCAN_ITER_COUNT_DEFAULT
-        ):
-            if r.exists(key_bytes):
+            elif key_str.startswith(RedisConnectorDelete.FENCE_PREFIX):
+                monitor_connector_deletion_taskset(tenant_id, key_bytes, r)
+            elif key_str.startswith(RedisConnectorPrune.FENCE_PREFIX):
                 with get_session_with_tenant(tenant_id) as db_session:
                     monitor_ccpair_pruning_taskset(tenant_id, key_bytes, r, db_session)
-            lock_beat.reacquire()
-        timings["pruning"] = time.monotonic() - phase_start
-        timings["pruning_ttl"] = r.ttl(OnyxRedisLocks.MONITOR_VESPA_SYNC_BEAT_LOCK)
-
-        phase_start = time.monotonic()
-        lock_beat.reacquire()
-        for key_bytes in r_replica.scan_iter(
-            RedisConnectorIndex.FENCE_PREFIX + "*", count=SCAN_ITER_COUNT_DEFAULT
-        ):
-            if r.exists(key_bytes):
+            elif key_str.startswith(RedisConnectorIndex.FENCE_PREFIX):
                 with get_session_with_tenant(tenant_id) as db_session:
                     monitor_ccpair_indexing_taskset(tenant_id, key_bytes, r, db_session)
-            lock_beat.reacquire()
-        timings["indexing"] = time.monotonic() - phase_start
-        timings["indexing_ttl"] = r.ttl(OnyxRedisLocks.MONITOR_VESPA_SYNC_BEAT_LOCK)
-
-        phase_start = time.monotonic()
-        lock_beat.reacquire()
-        for key_bytes in r_replica.scan_iter(
-            RedisConnectorPermissionSync.FENCE_PREFIX + "*",
-            count=SCAN_ITER_COUNT_DEFAULT,
-        ):
-            if r.exists(key_bytes):
+            elif key_str.startswith(RedisConnectorPermissionSync.FENCE_PREFIX):
                 with get_session_with_tenant(tenant_id) as db_session:
                     monitor_ccpair_permissions_taskset(
                         tenant_id, key_bytes, r, db_session
                     )
-            lock_beat.reacquire()
-
-        timings["permissions"] = time.monotonic() - phase_start
-        timings["permissions_ttl"] = r.ttl(OnyxRedisLocks.MONITOR_VESPA_SYNC_BEAT_LOCK)
+            else:
+                pass
     except SoftTimeLimitExceeded:
         task_logger.info(
             "Soft time limit exceeded, task is being terminated gracefully."
@@ -999,8 +957,8 @@ def monitor_vespa_sync(self: Task, tenant_id: str | None) -> bool | None:
         else:
             task_logger.error(
                 "monitor_vespa_sync - Lock not owned on completion: "
-                f"tenant={tenant_id} "
-                f"timings={timings}"
+                f"tenant={tenant_id}"
+                # f"timings={timings}"
             )
             redis_lock_dump(lock_beat, r)
 
@@ -1064,15 +1022,6 @@ def vespa_metadata_sync_task(
             # the sync might repeat again later
             mark_document_as_synced(document_id, db_session)
 
-            # this code checks for and removes a per document sync key that is
-            # used to block out the same doc from continualy resyncing
-            # a quick hack that is only needed for production issues
-            # redis_syncing_key = RedisConnectorCredentialPair.make_redis_syncing_key(
-            #     document_id
-            # )
-            # r = get_redis_client(tenant_id=tenant_id)
-            # r.delete(redis_syncing_key)
-
             elapsed = time.monotonic() - start
             task_logger.info(
                 f"doc={document_id} "
@@ -1084,6 +1033,7 @@ def vespa_metadata_sync_task(
         task_logger.info(f"SoftTimeLimitExceeded exception. doc={document_id}")
         return False
     except Exception as ex:
+        e: Exception | None = None
         if isinstance(ex, RetryError):
             task_logger.warning(
                 f"Tenacity retry failed: num_attempts={ex.last_attempt.attempt_number}"
@@ -1114,3 +1064,23 @@ def vespa_metadata_sync_task(
         self.retry(exc=e, countdown=countdown)
 
     return True
+
+
+def is_fence(key_bytes: bytes) -> bool:
+    key_str = key_bytes.decode("utf-8")
+    if key_str == RedisGlobalConnectorCredentialPair.FENCE_KEY:
+        return True
+    if key_str.startswith(RedisDocumentSet.FENCE_PREFIX):
+        return True
+    if key_str.startswith(RedisUserGroup.FENCE_PREFIX):
+        return True
+    if key_str.startswith(RedisConnectorDelete.FENCE_PREFIX):
+        return True
+    if key_str.startswith(RedisConnectorPrune.FENCE_PREFIX):
+        return True
+    if key_str.startswith(RedisConnectorIndex.FENCE_PREFIX):
+        return True
+    if key_str.startswith(RedisConnectorPermissionSync.FENCE_PREFIX):
+        return True
+
+    return False

backend/onyx/background/indexing/run_indexing.py

@@ -11,6 +11,7 @@ from onyx.background.indexing.checkpointing import get_time_windows_for_index_at
 from onyx.background.indexing.tracer import OnyxTracer
 from onyx.configs.app_configs import INDEXING_SIZE_WARNING_THRESHOLD
 from onyx.configs.app_configs import INDEXING_TRACER_INTERVAL
+from onyx.configs.app_configs import LEAVE_CONNECTOR_ACTIVE_ON_INITIALIZATION_FAILURE
 from onyx.configs.app_configs import POLL_CONNECTOR_OFFSET
 from onyx.configs.constants import DocumentSource
 from onyx.configs.constants import MilestoneRecordType
@@ -55,6 +56,7 @@ def _get_connector_runner(
     start_time: datetime,
     end_time: datetime,
     tenant_id: str | None,
+    leave_connector_active: bool = LEAVE_CONNECTOR_ACTIVE_ON_INITIALIZATION_FAILURE,
 ) -> ConnectorRunner:
     """
     NOTE: `start_time` and `end_time` are only used for poll connectors
@@ -76,20 +78,25 @@ def _get_connector_runner(
         )
     except Exception as e:
         logger.exception(f"Unable to instantiate connector due to {e}")
-        # since we failed to even instantiate the connector, we pause the CCPair since
-        # it will never succeed
 
-        cc_pair = get_connector_credential_pair_from_id(
-            db_session=db_session,
-            cc_pair_id=attempt.connector_credential_pair.id,
-        )
-        if cc_pair and cc_pair.status == ConnectorCredentialPairStatus.ACTIVE:
-            update_connector_credential_pair(
+        # since we failed to even instantiate the connector, we pause the CCPair since
+        # it will never succeed. Sometimes there are cases where the connector will
+        # intermittently fail to initialize in which case we should pass in
+        # leave_connector_active=True to allow it to continue.
+        # For example, if there is nightly maintenance on a Confluence Server instance,
+        # the connector will fail to initialize every night.
+        if not leave_connector_active:
+            cc_pair = get_connector_credential_pair_from_id(
                 db_session=db_session,
-                connector_id=attempt.connector_credential_pair.connector.id,
-                credential_id=attempt.connector_credential_pair.credential.id,
-                status=ConnectorCredentialPairStatus.PAUSED,
+                cc_pair_id=attempt.connector_credential_pair.id,
             )
+            if cc_pair and cc_pair.status == ConnectorCredentialPairStatus.ACTIVE:
+                update_connector_credential_pair(
+                    db_session=db_session,
+                    connector_id=attempt.connector_credential_pair.connector.id,
+                    credential_id=attempt.connector_credential_pair.credential.id,
+                    status=ConnectorCredentialPairStatus.PAUSED,
+                )
         raise e
 
     return ConnectorRunner(
@@ -239,6 +246,7 @@ def _run_indexing(
         callback=callback,
     )
 
+    tracer: OnyxTracer
     if INDEXING_TRACER_INTERVAL > 0:
         logger.debug(f"Memory tracer starting: interval={INDEXING_TRACER_INTERVAL}")
         tracer = OnyxTracer()
@@ -255,6 +263,8 @@ def _run_indexing(
     document_count = 0
     chunk_count = 0
     run_end_dt = None
+    tracer_counter: int
+
     for ind, (window_start, window_end) in enumerate(
         get_time_windows_for_index_attempt(
             last_successful_run=datetime.fromtimestamp(
@@ -265,6 +275,7 @@ def _run_indexing(
     ):
         cc_pair_loop: ConnectorCredentialPair | None = None
         index_attempt_loop: IndexAttempt | None = None
+        tracer_counter = 0
 
         try:
             window_start = max(
@@ -289,7 +300,6 @@ def _run_indexing(
                     tenant_id=tenant_id,
                 )
 
-            tracer_counter = 0
             if INDEXING_TRACER_INTERVAL > 0:
                 tracer.snap()
             for doc_batch in connector_runner.run():

backend/onyx/chat/answer.py

@@ -183,6 +183,7 @@ class Answer:
         citations_by_subquestion: dict[
             SubQuestionKey, list[CitationInfo]
         ] = defaultdict(list)
+        basic_subq_key = SubQuestionKey(level=BASIC_KEY[0], question_num=BASIC_KEY[1])
         for packet in self.processed_streamed_output:
             if isinstance(packet, CitationInfo):
                 if packet.level_question_num is not None and packet.level is not None:
@@ -192,7 +193,7 @@ class Answer:
                         )
                     ].append(packet)
                 elif packet.level is None:
-                    citations_by_subquestion[BASIC_SQ_KEY].append(packet)
+                    citations_by_subquestion[basic_subq_key].append(packet)
         return citations_by_subquestion
 
     def is_cancelled(self) -> bool:

backend/onyx/chat/process_message.py

@@ -7,6 +7,7 @@ from typing import cast
 
 from sqlalchemy.orm import Session
 
+from onyx.agents.agent_search.orchestration.nodes.tool_call import ToolCallException
 from onyx.chat.answer import Answer
 from onyx.chat.chat_utils import create_chat_chain
 from onyx.chat.chat_utils import create_temporary_persona
@@ -87,6 +88,7 @@ from onyx.file_store.utils import save_files
 from onyx.llm.exceptions import GenAIDisabledException
 from onyx.llm.factory import get_llms_for_persona
 from onyx.llm.factory import get_main_llm_from_tuple
+from onyx.llm.interfaces import LLM
 from onyx.llm.models import PreviousMessage
 from onyx.llm.utils import litellm_exception_to_error_msg
 from onyx.natural_language_processing.utils import get_tokenizer
@@ -349,7 +351,8 @@ def stream_chat_message_objects(
     new_msg_req.chunks_above = 0
     new_msg_req.chunks_below = 0
 
-    llm = None
+    llm: LLM
+
     try:
         user_id = user.id if user is not None else None
 
@@ -943,19 +946,25 @@ def stream_chat_message_objects(
         return
 
     except Exception as e:
-        logger.exception("Failed to process chat message.")
-
+        logger.exception(f"Failed to process chat message due to {e}")
         error_msg = str(e)
         stack_trace = traceback.format_exc()
-        if llm:
-            client_error_msg = litellm_exception_to_error_msg(e, llm)
-            if llm.config.api_key and len(llm.config.api_key) > 2:
-                error_msg = error_msg.replace(llm.config.api_key, "[REDACTED_API_KEY]")
-                stack_trace = stack_trace.replace(
-                    llm.config.api_key, "[REDACTED_API_KEY]"
-                )
 
-            yield StreamingError(error=client_error_msg, stack_trace=stack_trace)
+        if isinstance(e, ToolCallException):
+            yield StreamingError(error=error_msg, stack_trace=stack_trace)
+        else:
+            if llm:
+                client_error_msg = litellm_exception_to_error_msg(e, llm)
+                if llm.config.api_key and len(llm.config.api_key) > 2:
+                    error_msg = error_msg.replace(
+                        llm.config.api_key, "[REDACTED_API_KEY]"
+                    )
+                    stack_trace = stack_trace.replace(
+                        llm.config.api_key, "[REDACTED_API_KEY]"
+                    )
+
+                yield StreamingError(error=client_error_msg, stack_trace=stack_trace)
+
         db_session.rollback()
         return
 

backend/onyx/chat/stream_processing/utils.py

@@ -3,6 +3,7 @@ from collections.abc import Sequence
 from pydantic import BaseModel
 
 from onyx.chat.models import LlmDoc
+from onyx.chat.models import OnyxContext
 from onyx.context.search.models import InferenceChunk
 
 
@@ -11,7 +12,7 @@ class DocumentIdOrderMapping(BaseModel):
 
 
 def map_document_id_order(
-    chunks: Sequence[InferenceChunk | LlmDoc], one_indexed: bool = True
+    chunks: Sequence[InferenceChunk | LlmDoc | OnyxContext], one_indexed: bool = True
 ) -> DocumentIdOrderMapping:
     order_mapping = {}
     current = 1 if one_indexed else 0

backend/onyx/configs/app_configs.py

@@ -409,6 +409,11 @@ EXPERIMENTAL_CHECKPOINTING_ENABLED = (
     os.environ.get("EXPERIMENTAL_CHECKPOINTING_ENABLED", "").lower() == "true"
 )
 
+LEAVE_CONNECTOR_ACTIVE_ON_INITIALIZATION_FAILURE = (
+    os.environ.get("LEAVE_CONNECTOR_ACTIVE_ON_INITIALIZATION_FAILURE", "").lower()
+    == "true"
+)
+
 PRUNING_DISABLED = -1
 DEFAULT_PRUNING_FREQ = 60 * 60 * 24  # Once a day
 

backend/onyx/configs/constants.py

@@ -15,6 +15,12 @@ ID_SEPARATOR = ":;:"
 DEFAULT_BOOST = 0
 SESSION_KEY = "session"
 
+# Cookies
+FASTAPI_USERS_AUTH_COOKIE_NAME = (
+    "fastapiusersauth"  # Currently a constant, but logic allows for configuration
+)
+TENANT_ID_COOKIE_NAME = "onyx_tid"  # tenant id - for workaround cases
+
 NO_AUTH_USER_ID = "__no_auth_user__"
 NO_AUTH_USER_EMAIL = "anonymous@onyx.app"
 
@@ -306,9 +312,18 @@ class OnyxRedisLocks:
 
 
 class OnyxRedisSignals:
-    VALIDATE_INDEXING_FENCES = "signal:validate_indexing_fences"
-    VALIDATE_EXTERNAL_GROUP_SYNC_FENCES = "signal:validate_external_group_sync_fences"
-    VALIDATE_PERMISSION_SYNC_FENCES = "signal:validate_permission_sync_fences"
+    BLOCK_VALIDATE_INDEXING_FENCES = "signal:block_validate_indexing_fences"
+    BLOCK_VALIDATE_EXTERNAL_GROUP_SYNC_FENCES = (
+        "signal:block_validate_external_group_sync_fences"
+    )
+    BLOCK_VALIDATE_PERMISSION_SYNC_FENCES = (
+        "signal:block_validate_permission_sync_fences"
+    )
+    BLOCK_BUILD_FENCE_LOOKUP_TABLE = "signal:block_build_fence_lookup_table"
+
+
+class OnyxRedisConstants:
+    ACTIVE_FENCES = "active_fences"
 
 
 class OnyxCeleryPriority(int, Enum):

backend/onyx/connectors/airtable/airtable_connector.py

@@ -369,11 +369,12 @@ class AirtableConnector(LoadConnector):
         # Process records in parallel batches using ThreadPoolExecutor
         PARALLEL_BATCH_SIZE = 8
         max_workers = min(PARALLEL_BATCH_SIZE, len(records))
+        record_documents: list[Document] = []
 
         # Process records in batches
         for i in range(0, len(records), PARALLEL_BATCH_SIZE):
             batch_records = records[i : i + PARALLEL_BATCH_SIZE]
-            record_documents: list[Document] = []
+            record_documents = []
 
             with ThreadPoolExecutor(max_workers=max_workers) as executor:
                 # Submit batch tasks

backend/onyx/connectors/asana/asana_api.py

@@ -99,18 +99,18 @@ class AsanaAPI:
         project = self.project_api.get_project(project_gid, opts={})
         if project["archived"]:
             logger.info(f"Skipping archived project: {project['name']} ({project_gid})")
-            return []
+            yield from []
         if not project["team"] or not project["team"]["gid"]:
             logger.info(
                 f"Skipping project without a team: {project['name']} ({project_gid})"
             )
-            return []
+            yield from []
         if project["privacy_setting"] == "private":
             if self.team_gid and project["team"]["gid"] != self.team_gid:
                 logger.info(
                     f"Skipping private project not in configured team: {project['name']} ({project_gid})"
                 )
-                return []
+                yield from []
             else:
                 logger.info(
                     f"Processing private project in configured team: {project['name']} ({project_gid})"

backend/onyx/connectors/google_utils/resources.py

@@ -26,6 +26,7 @@ def _get_google_service(
     creds: ServiceAccountCredentials | OAuthCredentials,
     user_email: str | None = None,
 ) -> GoogleDriveService | GoogleDocsService | AdminService | GmailService:
+    service: Resource
     if isinstance(creds, ServiceAccountCredentials):
         creds = creds.with_subject(user_email)
         service = build(service_name, service_version, credentials=creds)

backend/onyx/connectors/salesforce/doc_conversion.py

@@ -59,6 +59,7 @@ def _clean_salesforce_dict(data: dict | list) -> dict | list:
     elif isinstance(data, list):
         filtered_list = []
         for item in data:
+            filtered_item: dict | list
             if isinstance(item, (dict, list)):
                 filtered_item = _clean_salesforce_dict(item)
                 # Only add non-empty dictionaries or lists

backend/onyx/db/constants.py

@@ -1 +1,2 @@
 SLACK_BOT_PERSONA_PREFIX = "__slack_bot_persona__"
+DEFAULT_PERSONA_SLACK_CHANNEL_NAME = "DEFAULT_SLACK_CHANNEL"

backend/onyx/db/document.py

@@ -105,6 +105,32 @@ def construct_document_select_for_connector_credential_pair_by_needs_sync(
     return stmt
 
 
+def construct_document_id_select_for_connector_credential_pair_by_needs_sync(
+    connector_id: int, credential_id: int
+) -> Select:
+    initial_doc_ids_stmt = select(DocumentByConnectorCredentialPair.id).where(
+        and_(
+            DocumentByConnectorCredentialPair.connector_id == connector_id,
+            DocumentByConnectorCredentialPair.credential_id == credential_id,
+        )
+    )
+
+    stmt = (
+        select(DbDocument.id)
+        .where(
+            DbDocument.id.in_(initial_doc_ids_stmt),
+            or_(
+                DbDocument.last_modified
+                > DbDocument.last_synced,  # last_modified is newer than last_synced
+                DbDocument.last_synced.is_(None),  # never synced
+            ),
+        )
+        .distinct()
+    )
+
+    return stmt
+
+
 def get_all_documents_needing_vespa_sync_for_cc_pair(
     db_session: Session, cc_pair_id: int
 ) -> list[DbDocument]:

backend/onyx/db/document_set.py

@@ -198,7 +198,7 @@ def _check_if_cc_pairs_are_owned_by_groups(
             ids=missing_cc_pair_ids,
         )
         for cc_pair in cc_pairs:
-            if cc_pair.access_type != AccessType.PUBLIC:
+            if cc_pair.access_type == AccessType.PRIVATE:
                 raise ValueError(
                     f"Connector Credential Pair with ID: '{cc_pair.id}'"
                     " is not owned by the specified groups"
@@ -221,6 +221,8 @@ def insert_document_set(
             group_ids=document_set_creation_request.groups or [],
         )
 
+    new_document_set_row: DocumentSetDBModel
+    ds_cc_pairs: list[DocumentSet__ConnectorCredentialPair]
     try:
         new_document_set_row = DocumentSetDBModel(
             name=document_set_creation_request.name,
@@ -543,7 +545,7 @@ def fetch_documents_for_document_set_paginated(
     return documents, documents[-1].id if documents else None
 
 
-def construct_document_select_by_docset(
+def construct_document_id_select_by_docset(
     document_set_id: int,
     current_only: bool = True,
 ) -> Select:
@@ -552,7 +554,7 @@ def construct_document_select_by_docset(
     are background processing task generators."""
 
     stmt = (
-        select(Document)
+        select(Document.id)
         .join(
             DocumentByConnectorCredentialPair,
             DocumentByConnectorCredentialPair.id == Document.id,

backend/onyx/db/models.py

@@ -1716,7 +1716,7 @@ class ChannelConfig(TypedDict):
     """NOTE: is a `TypedDict` so it can be used as a type hint for a JSONB column
     in Postgres"""
 
-    channel_name: str
+    channel_name: str | None  # None for default channel config
     respond_tag_only: NotRequired[bool]  # defaults to False
     respond_to_bots: NotRequired[bool]  # defaults to False
     respond_member_group_list: NotRequired[list[str]]
@@ -1737,7 +1737,6 @@ class SlackChannelConfig(Base):
     persona_id: Mapped[int | None] = mapped_column(
         ForeignKey("persona.id"), nullable=True
     )
-    # JSON for flexibility. Contains things like: channel name, team members, etc.
     channel_config: Mapped[ChannelConfig] = mapped_column(
         postgresql.JSONB(), nullable=False
     )
@@ -1746,6 +1745,8 @@ class SlackChannelConfig(Base):
         Boolean, nullable=False, default=False
     )
 
+    is_default: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
+
     persona: Mapped[Persona | None] = relationship("Persona")
     slack_bot: Mapped["SlackBot"] = relationship(
         "SlackBot",
@@ -1757,6 +1758,21 @@ class SlackChannelConfig(Base):
         back_populates="slack_channel_configs",
     )
 
+    __table_args__ = (
+        UniqueConstraint(
+            "slack_bot_id",
+            "is_default",
+            name="uq_slack_channel_config_slack_bot_id_default",
+        ),
+        Index(
+            "ix_slack_channel_config_slack_bot_id_default",
+            "slack_bot_id",
+            "is_default",
+            unique=True,
+            postgresql_where=(is_default is True),  #   type: ignore
+        ),
+    )
+
 
 class SlackBot(Base):
     __tablename__ = "slack_bot"

backend/onyx/db/persona.py

@@ -11,6 +11,7 @@ from sqlalchemy import Select
 from sqlalchemy import select
 from sqlalchemy import update
 from sqlalchemy.orm import aliased
+from sqlalchemy.orm import joinedload
 from sqlalchemy.orm import selectinload
 from sqlalchemy.orm import Session
 
@@ -19,6 +20,7 @@ from onyx.configs.app_configs import DISABLE_AUTH
 from onyx.configs.chat_configs import BING_API_KEY
 from onyx.configs.chat_configs import CONTEXT_CHUNKS_ABOVE
 from onyx.configs.chat_configs import CONTEXT_CHUNKS_BELOW
+from onyx.configs.constants import NotificationType
 from onyx.context.search.enums import RecencyBiasSetting
 from onyx.db.constants import SLACK_BOT_PERSONA_PREFIX
 from onyx.db.models import DocumentSet
@@ -32,6 +34,8 @@ from onyx.db.models import Tool
 from onyx.db.models import User
 from onyx.db.models import User__UserGroup
 from onyx.db.models import UserGroup
+from onyx.db.notification import create_notification
+from onyx.server.features.persona.models import PersonaSharedNotificationData
 from onyx.server.features.persona.models import PersonaSnapshot
 from onyx.server.features.persona.models import PersonaUpsertRequest
 from onyx.utils.logger import setup_logger
@@ -169,6 +173,15 @@ def make_persona_private(
         for user_uuid in user_ids:
             db_session.add(Persona__User(persona_id=persona_id, user_id=user_uuid))
 
+            create_notification(
+                user_id=user_uuid,
+                notif_type=NotificationType.PERSONA_SHARED,
+                db_session=db_session,
+                additional_data=PersonaSharedNotificationData(
+                    persona_id=persona_id,
+                ).model_dump(),
+            )
+
         db_session.commit()
 
     # May cause error if someone switches down to MIT from EE
@@ -708,3 +721,15 @@ def update_persona_label(
 def delete_persona_label(label_id: int, db_session: Session) -> None:
     db_session.query(PersonaLabel).filter(PersonaLabel.id == label_id).delete()
     db_session.commit()
+
+
+def persona_has_search_tool(persona_id: int, db_session: Session) -> bool:
+    persona = (
+        db_session.query(Persona)
+        .options(joinedload(Persona.tools))
+        .filter(Persona.id == persona_id)
+        .one_or_none()
+    )
+    if persona is None:
+        raise ValueError(f"Persona with ID {persona_id} does not exist")
+    return any(tool.in_code_tool_id == "run_search" for tool in persona.tools)

backend/onyx/db/slack_bot.py

@@ -74,3 +74,15 @@ def remove_slack_bot(
 
 def fetch_slack_bots(db_session: Session) -> Sequence[SlackBot]:
     return db_session.scalars(select(SlackBot)).all()
+
+
+def fetch_slack_bot_tokens(
+    db_session: Session, slack_bot_id: int
+) -> dict[str, str] | None:
+    slack_bot = db_session.scalar(select(SlackBot).where(SlackBot.id == slack_bot_id))
+    if not slack_bot:
+        return None
+    return {
+        "app_token": slack_bot.app_token,
+        "bot_token": slack_bot.bot_token,
+    }

backend/onyx/db/slack_channel_config.py

@@ -6,6 +6,7 @@ from sqlalchemy.orm import Session
 
 from onyx.configs.chat_configs import MAX_CHUNKS_FED_TO_CHAT
 from onyx.context.search.enums import RecencyBiasSetting
+from onyx.db.constants import DEFAULT_PERSONA_SLACK_CHANNEL_NAME
 from onyx.db.constants import SLACK_BOT_PERSONA_PREFIX
 from onyx.db.models import ChannelConfig
 from onyx.db.models import Persona
@@ -22,8 +23,8 @@ from onyx.utils.variable_functionality import (
 )
 
 
-def _build_persona_name(channel_name: str) -> str:
-    return f"{SLACK_BOT_PERSONA_PREFIX}{channel_name}"
+def _build_persona_name(channel_name: str | None) -> str:
+    return f"{SLACK_BOT_PERSONA_PREFIX}{channel_name if channel_name else DEFAULT_PERSONA_SLACK_CHANNEL_NAME}"
 
 
 def _cleanup_relationships(db_session: Session, persona_id: int) -> None:
@@ -40,7 +41,7 @@ def _cleanup_relationships(db_session: Session, persona_id: int) -> None:
 
 def create_slack_channel_persona(
     db_session: Session,
-    channel_name: str,
+    channel_name: str | None,
     document_set_ids: list[int],
     existing_persona_id: int | None = None,
     num_chunks: float = MAX_CHUNKS_FED_TO_CHAT,
@@ -90,6 +91,7 @@ def insert_slack_channel_config(
     channel_config: ChannelConfig,
     standard_answer_category_ids: list[int],
     enable_auto_filters: bool,
+    is_default: bool = False,
 ) -> SlackChannelConfig:
     versioned_fetch_standard_answer_categories_by_ids = (
         fetch_versioned_implementation_with_fallback(
@@ -115,12 +117,26 @@ def insert_slack_channel_config(
                 f"Some or all categories with ids {standard_answer_category_ids} do not exist"
             )
 
+    if is_default:
+        existing_default = db_session.scalar(
+            select(SlackChannelConfig).where(
+                SlackChannelConfig.slack_bot_id == slack_bot_id,
+                SlackChannelConfig.is_default is True,  # type: ignore
+            )
+        )
+        if existing_default:
+            raise ValueError("A default config already exists for this Slack bot.")
+    else:
+        if "channel_name" not in channel_config:
+            raise ValueError("Channel name is required for non-default configs.")
+
     slack_channel_config = SlackChannelConfig(
         slack_bot_id=slack_bot_id,
         persona_id=persona_id,
         channel_config=channel_config,
         standard_answer_categories=existing_standard_answer_categories,
         enable_auto_filters=enable_auto_filters,
+        is_default=is_default,
     )
     db_session.add(slack_channel_config)
     db_session.commit()
@@ -164,12 +180,7 @@ def update_slack_channel_config(
             f"Some or all categories with ids {standard_answer_category_ids} do not exist"
         )
 
-    # get the existing persona id before updating the object
-    existing_persona_id = slack_channel_config.persona_id
-
     # update the config
-    # NOTE: need to do this before cleaning up the old persona or else we
-    # will encounter `violates foreign key constraint` errors
     slack_channel_config.persona_id = persona_id
     slack_channel_config.channel_config = channel_config
     slack_channel_config.standard_answer_categories = list(
@@ -177,20 +188,6 @@ def update_slack_channel_config(
     )
     slack_channel_config.enable_auto_filters = enable_auto_filters
 
-    # if the persona has changed, then clean up the old persona
-    if persona_id != existing_persona_id and existing_persona_id:
-        existing_persona = db_session.scalar(
-            select(Persona).where(Persona.id == existing_persona_id)
-        )
-        # if the existing persona was one created just for use with this Slack channel,
-        # then clean it up
-        if existing_persona and existing_persona.name.startswith(
-            SLACK_BOT_PERSONA_PREFIX
-        ):
-            _cleanup_relationships(
-                db_session=db_session, persona_id=existing_persona_id
-            )
-
     db_session.commit()
 
     return slack_channel_config
@@ -253,3 +250,32 @@ def fetch_slack_channel_config(
             SlackChannelConfig.id == slack_channel_config_id
         )
     )
+
+
+def fetch_slack_channel_config_for_channel_or_default(
+    db_session: Session, slack_bot_id: int, channel_name: str | None
+) -> SlackChannelConfig | None:
+    # attempt to find channel-specific config first
+    if channel_name is not None:
+        sc_config = db_session.scalar(
+            select(SlackChannelConfig).where(
+                SlackChannelConfig.slack_bot_id == slack_bot_id,
+                SlackChannelConfig.channel_config["channel_name"].astext
+                == channel_name,
+            )
+        )
+    else:
+        sc_config = None
+
+    if sc_config:
+        return sc_config
+
+    # if none found, see if there is a default
+    default_sc = db_session.scalar(
+        select(SlackChannelConfig).where(
+            SlackChannelConfig.slack_bot_id == slack_bot_id,
+            SlackChannelConfig.is_default == True,  # noqa: E712
+        )
+    )
+
+    return default_sc

backend/onyx/document_index/document_index_utils.py

@@ -142,6 +142,8 @@ def get_uuid_from_chunk_info(
     tenant_id: str | None,
     large_chunk_id: int | None = None,
 ) -> UUID:
+    """NOTE: be VERY carefuly about changing this function. If changed without a migration,
+    this can cause deletion/update/insertion to function incorrectly."""
     doc_str = document_id
 
     # Web parsing URL duplicate catching

backend/onyx/document_index/vespa/index.py

@@ -346,6 +346,14 @@ class VespaIndex(DocumentIndex):
         # IMPORTANT: This must be done one index at a time, do not use secondary index here
         cleaned_chunks = [clean_chunk_id_copy(chunk) for chunk in chunks]
 
+        # needed so the final DocumentInsertionRecord returned can have the original document ID
+        new_document_id_to_original_document_id: dict[str, str] = {}
+        for ind, chunk in enumerate(cleaned_chunks):
+            old_chunk = chunks[ind]
+            new_document_id_to_original_document_id[
+                chunk.source_document.id
+            ] = old_chunk.source_document.id
+
         existing_docs: set[str] = set()
 
         # NOTE: using `httpx` here since `requests` doesn't support HTTP2. This is beneficial for
@@ -401,14 +409,14 @@ class VespaIndex(DocumentIndex):
                     executor=executor,
                 )
 
-        all_doc_ids = {chunk.source_document.id for chunk in cleaned_chunks}
+        all_cleaned_doc_ids = {chunk.source_document.id for chunk in cleaned_chunks}
 
         return {
             DocumentInsertionRecord(
-                document_id=doc_id,
-                already_existed=doc_id in existing_docs,
+                document_id=new_document_id_to_original_document_id[cleaned_doc_id],
+                already_existed=cleaned_doc_id in existing_docs,
             )
-            for doc_id in all_doc_ids
+            for cleaned_doc_id in all_cleaned_doc_ids
         }
 
     @classmethod
@@ -541,7 +549,7 @@ class VespaIndex(DocumentIndex):
             time.monotonic() - update_start,
         )
 
-    def update_single_chunk(
+    def _update_single_chunk(
         self,
         doc_chunk_id: UUID,
         index_name: str,
@@ -605,6 +613,8 @@ class VespaIndex(DocumentIndex):
         """
         doc_chunk_count = 0
 
+        doc_id = replace_invalid_doc_id_characters(doc_id)
+
         with self.httpx_client_context as httpx_client:
             for (
                 index_name,
@@ -627,7 +637,7 @@ class VespaIndex(DocumentIndex):
                 doc_chunk_count += len(doc_chunk_ids)
 
                 for doc_chunk_id in doc_chunk_ids:
-                    self.update_single_chunk(
+                    self._update_single_chunk(
                         doc_chunk_id, index_name, fields, doc_id, httpx_client
                     )
 
@@ -689,6 +699,18 @@ class VespaIndex(DocumentIndex):
         batch_retrieval: bool = False,
         get_large_chunks: bool = False,
     ) -> list[InferenceChunkUncleaned]:
+        # make sure to use the vespa-afied document IDs
+        chunk_requests = [
+            VespaChunkRequest(
+                document_id=replace_invalid_doc_id_characters(
+                    chunk_request.document_id
+                ),
+                min_chunk_ind=chunk_request.min_chunk_ind,
+                max_chunk_ind=chunk_request.max_chunk_ind,
+            )
+            for chunk_request in chunk_requests
+        ]
+
         if batch_retrieval:
             return batch_search_api_retrieval(
                 index_name=self.index_name,

backend/onyx/document_index/vespa/indexing_utils.py

@@ -242,9 +242,9 @@ def batch_index_vespa_chunks(
 def clean_chunk_id_copy(
     chunk: DocMetadataAwareIndexChunk,
 ) -> DocMetadataAwareIndexChunk:
-    clean_chunk = chunk.copy(
+    clean_chunk = chunk.model_copy(
         update={
-            "source_document": chunk.source_document.copy(
+            "source_document": chunk.source_document.model_copy(
                 update={
                     "id": replace_invalid_doc_id_characters(chunk.source_document.id)
                 }

backend/onyx/document_index/vespa/shared_utils/utils.py

@@ -45,7 +45,9 @@ def is_text_character(codepoint: int) -> bool:
 
 
 def replace_invalid_doc_id_characters(text: str) -> str:
-    """Replaces invalid document ID characters in text."""
+    """Replaces invalid document ID characters in text.
+    NOTE: this must be called at the start of every vespa-related operation or else we
+    risk discrepancies -> silent failures on deletion/update/insertion."""
     # There may be a more complete set of replacements that need to be made but Vespa docs are unclear
     # and users only seem to be running into this error with single quotes
     return text.replace("'", "_")

backend/onyx/file_processing/extract_file_text.py

@@ -365,7 +365,7 @@ def extract_file_text(
                     f"Failed to process with Unstructured: {str(unstructured_error)}. Falling back to normal processing."
                 )
                 # Fall through to normal processing
-
+        final_extension: str
         if file_name or extension:
             if extension is not None:
                 final_extension = extension

backend/onyx/indexing/chunker.py

@@ -223,6 +223,8 @@ class Chunker:
                 large_chunk_id=None,
             )
 
+        section_link_text: str
+
         for section_idx, section in enumerate(document.sections):
             section_text = clean_text(section.text)
             section_link_text = section.link or ""

backend/onyx/llm/chat_llm.py

@@ -409,7 +409,11 @@ class DefaultMultiLLM(LLM):
                 # For now, we don't support parallel tool calls
                 # NOTE: we can't pass this in if tools are not specified
                 # or else OpenAI throws an error
-                **({"parallel_tool_calls": False} if tools else {}),
+                **(
+                    {"parallel_tool_calls": False}
+                    if tools and self.config.model_name != "o3-mini"
+                    else {}
+                ),  # TODO: remove once LITELLM has patched
                 **(
                     {"response_format": structured_response_format}
                     if structured_response_format
@@ -469,9 +473,7 @@ class DefaultMultiLLM(LLM):
         if LOG_DANSWER_MODEL_INTERACTIONS:
             self.log_model_configs()
 
-        if (
-            DISABLE_LITELLM_STREAMING or self.config.model_name == "o1-2024-12-17"
-        ):  # TODO: remove once litellm supports streaming
+        if DISABLE_LITELLM_STREAMING:
             yield self.invoke(prompt, tools, tool_choice, structured_response_format)
             return
 

backend/onyx/llm/llm_provider_options.py

@@ -27,6 +27,7 @@ class WellKnownLLMProviderDescriptor(BaseModel):
 
 OPENAI_PROVIDER_NAME = "openai"
 OPEN_AI_MODEL_NAMES = [
+    "o3-mini",
     "o1-mini",
     "o1-preview",
     "o1-2024-12-17",
@@ -91,7 +92,7 @@ def fetch_available_well_known_llms() -> list[WellKnownLLMProviderDescriptor]:
             api_version_required=False,
             custom_config_keys=[],
             llm_names=fetch_models_for_provider(OPENAI_PROVIDER_NAME),
-            default_model="gpt-4",
+            default_model="gpt-4o",
             default_fast_model="gpt-4o-mini",
         ),
         WellKnownLLMProviderDescriptor(

backend/onyx/onyxbot/slack/blocks.py

@@ -1,4 +1,5 @@
 from datetime import datetime
+from typing import cast
 
 import pytz
 import timeago  # type: ignore
@@ -338,6 +339,23 @@ def _build_citations_blocks(
     return citations_block
 
 
+def _build_answer_blocks(
+    answer: ChatOnyxBotResponse, fallback_answer: str
+) -> list[SectionBlock]:
+    if not answer.answer:
+        answer_blocks = [SectionBlock(text=fallback_answer)]
+    else:
+        # replaces markdown links with slack format links
+        formatted_answer = format_slack_message(answer.answer)
+        answer_processed = decode_escapes(
+            remove_slack_text_interactions(formatted_answer)
+        )
+        answer_blocks = [
+            SectionBlock(text=text) for text in _split_text(answer_processed)
+        ]
+    return answer_blocks
+
+
 def _build_qa_response_blocks(
     answer: ChatOnyxBotResponse,
 ) -> list[Block]:
@@ -376,21 +394,10 @@ def _build_qa_response_blocks(
 
         filter_block = SectionBlock(text=f"_{filter_text}_")
 
-    if not answer.answer:
-        answer_blocks = [
-            SectionBlock(
-                text="Sorry, I was unable to find an answer, but I did find some potentially relevant docs 🤓"
-            )
-        ]
-    else:
-        # replaces markdown links with slack format links
-        formatted_answer = format_slack_message(answer.answer)
-        answer_processed = decode_escapes(
-            remove_slack_text_interactions(formatted_answer)
-        )
-        answer_blocks = [
-            SectionBlock(text=text) for text in _split_text(answer_processed)
-        ]
+    answer_blocks = _build_answer_blocks(
+        answer=answer,
+        fallback_answer="Sorry, I was unable to find an answer, but I did find some potentially relevant docs 🤓",
+    )
 
     response_blocks: list[Block] = []
 
@@ -481,6 +488,7 @@ def build_slack_response_blocks(
     use_citations: bool,
     feedback_reminder_id: str | None,
     skip_ai_feedback: bool = False,
+    expecting_search_result: bool = False,
 ) -> list[Block]:
     """
     This function is a top level function that builds all the blocks for the Slack response.
@@ -491,9 +499,19 @@ def build_slack_response_blocks(
         message_info.thread_messages[-1].message, message_info.is_bot_msg
     )
 
-    answer_blocks = _build_qa_response_blocks(
-        answer=answer,
-    )
+    if expecting_search_result:
+        answer_blocks = _build_qa_response_blocks(
+            answer=answer,
+        )
+
+    else:
+        answer_blocks = cast(
+            list[Block],
+            _build_answer_blocks(
+                answer=answer,
+                fallback_answer="Sorry, I was unable to generate an answer.",
+            ),
+        )
 
     web_follow_up_block = []
     if channel_conf and channel_conf.get("show_continue_in_web_ui"):

backend/onyx/onyxbot/slack/config.py

@@ -3,9 +3,11 @@ import os
 from sqlalchemy.orm import Session
 
 from onyx.db.models import SlackChannelConfig
+from onyx.db.slack_channel_config import (
+    fetch_slack_channel_config_for_channel_or_default,
+)
 from onyx.db.slack_channel_config import fetch_slack_channel_configs
 
-
 VALID_SLACK_FILTERS = [
     "answerable_prefilter",
     "well_answered_postfilter",
@@ -17,18 +19,16 @@ def get_slack_channel_config_for_bot_and_channel(
     db_session: Session,
     slack_bot_id: int,
     channel_name: str | None,
-) -> SlackChannelConfig | None:
-    if not channel_name:
-        return None
-
-    slack_bot_configs = fetch_slack_channel_configs(
-        db_session=db_session, slack_bot_id=slack_bot_id
+) -> SlackChannelConfig:
+    slack_bot_config = fetch_slack_channel_config_for_channel_or_default(
+        db_session=db_session, slack_bot_id=slack_bot_id, channel_name=channel_name
     )
-    for config in slack_bot_configs:
-        if channel_name in config.channel_config["channel_name"]:
-            return config
+    if not slack_bot_config:
+        raise ValueError(
+            "No default configuration has been set for this Slack bot. This should not be possible."
+        )
 
-    return None
+    return slack_bot_config
 
 
 def validate_channel_name(

backend/onyx/onyxbot/slack/handlers/handle_message.py

@@ -106,7 +106,7 @@ def remove_scheduled_feedback_reminder(
 
 def handle_message(
     message_info: SlackMessageInfo,
-    slack_channel_config: SlackChannelConfig | None,
+    slack_channel_config: SlackChannelConfig,
     client: WebClient,
     feedback_reminder_id: str | None,
     tenant_id: str | None,

backend/onyx/onyxbot/slack/handlers/handle_regular_answer.py

@@ -27,6 +27,7 @@ from onyx.db.engine import get_session_with_tenant
 from onyx.db.models import SlackChannelConfig
 from onyx.db.models import User
 from onyx.db.persona import get_persona_by_id
+from onyx.db.persona import persona_has_search_tool
 from onyx.db.users import get_user_by_email
 from onyx.onyxbot.slack.blocks import build_slack_response_blocks
 from onyx.onyxbot.slack.handlers.utils import send_team_member_message
@@ -64,7 +65,7 @@ def rate_limits(
 
 def handle_regular_answer(
     message_info: SlackMessageInfo,
-    slack_channel_config: SlackChannelConfig | None,
+    slack_channel_config: SlackChannelConfig,
     receiver_ids: list[str] | None,
     client: WebClient,
     channel: str,
@@ -76,7 +77,7 @@ def handle_regular_answer(
     should_respond_with_error_msgs: bool = DANSWER_BOT_DISPLAY_ERROR_MSGS,
     disable_docs_only_answer: bool = DANSWER_BOT_DISABLE_DOCS_ONLY_ANSWER,
 ) -> bool:
-    channel_conf = slack_channel_config.channel_config if slack_channel_config else None
+    channel_conf = slack_channel_config.channel_config
 
     messages = message_info.thread_messages
 
@@ -92,7 +93,7 @@ def handle_regular_answer(
     prompt = None
     # If no persona is specified, use the default search based persona
     # This way slack flow always has a persona
-    persona = slack_channel_config.persona if slack_channel_config else None
+    persona = slack_channel_config.persona
     if not persona:
         with get_session_with_tenant(tenant_id) as db_session:
             persona = get_persona_by_id(DEFAULT_PERSONA_ID, user, db_session)
@@ -106,7 +107,8 @@ def handle_regular_answer(
         ]
         prompt = persona.prompts[0] if persona.prompts else None
 
-    should_respond_even_with_no_docs = persona.num_chunks == 0 if persona else False
+    with get_session_with_tenant(tenant_id) as db_session:
+        expecting_search_result = persona_has_search_tool(persona.id, db_session)
 
     # TODO: Add in support for Slack to truncate messages based on max LLM context
     # llm, _ = get_llms_for_persona(persona)
@@ -134,11 +136,7 @@ def handle_regular_answer(
     single_message_history = slackify_message_thread(history_messages) or None
 
     bypass_acl = False
-    if (
-        slack_channel_config
-        and slack_channel_config.persona
-        and slack_channel_config.persona.document_sets
-    ):
+    if slack_channel_config.persona and slack_channel_config.persona.document_sets:
         # For Slack channels, use the full document set, admin will be warned when configuring it
         # with non-public document sets
         bypass_acl = True
@@ -190,11 +188,7 @@ def handle_regular_answer(
         # auto_detect_filters = (
         #     persona.llm_filter_extraction if persona is not None else True
         # )
-        auto_detect_filters = (
-            slack_channel_config.enable_auto_filters
-            if slack_channel_config is not None
-            else False
-        )
+        auto_detect_filters = slack_channel_config.enable_auto_filters
         retrieval_details = RetrievalDetails(
             run_search=OptionalSearchSetting.ALWAYS,
             real_time=False,
@@ -311,12 +305,12 @@ def handle_regular_answer(
         return True
 
     retrieval_info = answer.docs
-    if not retrieval_info:
+    if not retrieval_info and expecting_search_result:
         # This should not happen, even with no docs retrieved, there is still info returned
         raise RuntimeError("Failed to retrieve docs, cannot answer question.")
 
-    top_docs = retrieval_info.top_documents
-    if not top_docs and not should_respond_even_with_no_docs:
+    top_docs = retrieval_info.top_documents if retrieval_info else []
+    if not top_docs and expecting_search_result:
         logger.error(
             f"Unable to answer question: '{user_message}' - no documents found"
         )
@@ -345,7 +339,8 @@ def handle_regular_answer(
     )
 
     if (
-        only_respond_if_citations
+        expecting_search_result
+        and only_respond_if_citations
         and not answer.citations
         and not message_info.bypass_filters
     ):
@@ -371,6 +366,7 @@ def handle_regular_answer(
         channel_conf=channel_conf,
         use_citations=True,  # No longer supporting quotes
         feedback_reminder_id=feedback_reminder_id,
+        expecting_search_result=expecting_search_result,
     )
 
     try:

backend/onyx/onyxbot/slack/handlers/handle_standard_answers.py

@@ -14,7 +14,7 @@ logger = setup_logger()
 def handle_standard_answers(
     message_info: SlackMessageInfo,
     receiver_ids: list[str] | None,
-    slack_channel_config: SlackChannelConfig | None,
+    slack_channel_config: SlackChannelConfig,
     prompt: Prompt | None,
     logger: OnyxLoggingAdapter,
     client: WebClient,
@@ -40,7 +40,7 @@ def handle_standard_answers(
 def _handle_standard_answers(
     message_info: SlackMessageInfo,
     receiver_ids: list[str] | None,
-    slack_channel_config: SlackChannelConfig | None,
+    slack_channel_config: SlackChannelConfig,
     prompt: Prompt | None,
     logger: OnyxLoggingAdapter,
     client: WebClient,

backend/onyx/onyxbot/slack/listener.py

@@ -5,6 +5,7 @@ import sys
 import threading
 import time
 from collections.abc import Callable
+from contextvars import Token
 from threading import Event
 from types import FrameType
 from typing import Any
@@ -250,6 +251,8 @@ class SlackbotHandler:
         """
         all_tenants = get_all_tenant_ids()
 
+        token: Token[str]
+
         # 1) Try to acquire locks for new tenants
         for tenant_id in all_tenants:
             if (
@@ -407,13 +410,27 @@ class SlackbotHandler:
     def start_socket_client(
         self, slack_bot_id: int, tenant_id: str | None, slack_bot_tokens: SlackBotTokens
     ) -> None:
-        logger.info(
-            f"Starting socket client for tenant: {tenant_id}, app: {slack_bot_id}"
-        )
         socket_client: TenantSocketModeClient = _get_socket_client(
             slack_bot_tokens, tenant_id, slack_bot_id
         )
 
+        try:
+            bot_info = socket_client.web_client.auth_test()
+            if bot_info["ok"]:
+                bot_user_id = bot_info["user_id"]
+                user_info = socket_client.web_client.users_info(user=bot_user_id)
+                if user_info["ok"]:
+                    bot_name = (
+                        user_info["user"]["real_name"] or user_info["user"]["name"]
+                    )
+                    logger.info(
+                        f"Started socket client for Slackbot with name '{bot_name}' (tenant: {tenant_id}, app: {slack_bot_id})"
+                    )
+        except Exception as e:
+            logger.warning(
+                f"Could not fetch bot name: {e} for tenant: {tenant_id}, app: {slack_bot_id}"
+            )
+
         # Append the event handler
         process_slack_event = create_process_slack_event()
         socket_client.socket_mode_request_listeners.append(process_slack_event)  # type: ignore
@@ -771,6 +788,7 @@ def process_message(
         client=client.web_client, channel_id=channel
     )
 
+    token: Token[str] | None = None
     # Set the current tenant ID at the beginning for all DB calls within this thread
     if client.tenant_id:
         logger.info(f"Setting tenant ID to {client.tenant_id}")
@@ -783,22 +801,8 @@ def process_message(
                 channel_name=channel_name,
             )
 
-            # Be careful about this default, don't want to accidentally spam every channel
-            # Users should be able to DM slack bot in their private channels though
-            if (
-                slack_channel_config is None
-                and not respond_every_channel
-                # Can't have configs for DMs so don't toss them out
-                and not is_dm
-                # If /OnyxBot (is_bot_msg) or @OnyxBot (bypass_filters)
-                # always respond with the default configs
-                and not (details.is_bot_msg or details.bypass_filters)
-            ):
-                return
-
             follow_up = bool(
-                slack_channel_config
-                and slack_channel_config.channel_config
+                slack_channel_config.channel_config
                 and slack_channel_config.channel_config.get("follow_up_tags")
                 is not None
             )
@@ -825,7 +829,7 @@ def process_message(
                 if notify_no_answer:
                     apologize_for_fail(details, client)
     finally:
-        if client.tenant_id:
+        if token:
             CURRENT_TENANT_ID_CONTEXTVAR.reset(token)
 
 

backend/onyx/onyxbot/slack/utils.py

@@ -518,7 +518,7 @@ def read_slack_thread(
             message_type = MessageType.USER
         else:
             self_slack_bot_id = get_onyx_bot_slack_bot_id(client)
-
+            blocks: Any
             if reply.get("user") == self_slack_bot_id:
                 # OnyxBot response
                 message_type = MessageType.ASSISTANT

backend/onyx/prompts/agent_search.py

@@ -53,8 +53,8 @@ Answer:
 # Step/Utility Prompts
 # Note this one should always be used with the ENTITY_TERM_EXTRACTION_PROMPT_JSON_EXAMPLE
 ENTITY_TERM_EXTRACTION_PROMPT = f"""
-Based on the original question and some context retrieved from a dataset, please generate a list of
-entities (e.g. companies, organizations, industries, products, locations, etc.), terms and concepts
+Based on the original question and some context retrieved from a dataset, please generate a list of \
+entities (e.g. companies, organizations, industries, products, locations, etc.), terms and concepts \
 (e.g. sales, revenue, etc.) that are relevant for the question, plus their relations to each other.
 
 Here is the original question:
@@ -98,445 +98,541 @@ ENTITY_TERM_EXTRACTION_PROMPT_JSON_EXAMPLE = """
 """.strip()
 
 
-HISTORY_CONTEXT_SUMMARY_PROMPT = (
-    "{persona_specification}\n\n"
-    "Your task now is to summarize the key parts of the history of a conversation between a user and an agent."
-    " The summary has two purposes:\n"
-    "  1) providing the suitable context for a new question, and\n"
-    "  2) To capture the key information that was discussed and that the user may have a follow-up question about.\n\n"
-    "Here is the question:\n"
-    f"{SEPARATOR_LINE}\n"
-    "{question}\n"
-    f"{SEPARATOR_LINE}\n\n"
-    "And here is the history:\n"
-    f"{SEPARATOR_LINE}\n"
-    "{history}\n"
-    f"{SEPARATOR_LINE}\n\n"
-    "Please provide a summarized context from the history so that the question makes sense and can"
-    " - with suitable extra information - be answered.\n\n"
-    "Do not use more than three or four sentences.\n\n"
-    "History summary:"
-).strip()
+HISTORY_CONTEXT_SUMMARY_PROMPT = f"""
+{{persona_specification}}
+
+Your task now is to summarize the key parts of the history of a conversation between a user and an agent. \
+The summary has two purposes:
+  1) providing the suitable context for a new question, and
+  2) To capture the key information that was discussed and that the user may have a follow-up question about.
+
+Here is the question:
+{SEPARATOR_LINE}
+{{question}}
+{SEPARATOR_LINE}
+
+And here is the history:
+{SEPARATOR_LINE}
+{{history}}
+{SEPARATOR_LINE}
+
+Please provide a summarized context from the history so that the question makes sense and can \
+- with suitable extra information - be answered.
+
+Do not use more than three or four sentences.
+
+History summary:
+""".strip()
 
 
 # INITIAL PHASE
 # Sub-question
 # Intentionally left a copy in case we want to modify this one differently
-INITIAL_QUESTION_DECOMPOSITION_PROMPT = (
-    "Decompose the initial user question into no more than 3 appropriate sub-questions that help to answer the"
-    " original question. The purpose for this decomposition may be to:\n"
-    "  1) isolate individual entities (i.e., 'compare sales of company A and company B' ->"
-    " ['what are sales for company A', 'what are sales for company B'])\n"
-    "  2) clarify or disambiguate ambiguous terms (i.e., 'what is our success with company A' ->"
-    " ['what are our sales with company A','what is our market share with company A',"
-    " 'is company A a reference customer for us', etc.])\n"
-    "  3) if a term or a metric is essentially clear, but it could relate to various components of an entity and you"
-    " are generally familiar with the entity, then you can decompose the question into sub-questions that are more"
-    " specific to components (i.e., 'what do we do to improve scalability of product X', 'what do we to to improve"
-    " scalability of product X', 'what do we do to improve stability of product X', ...])\n"
-    "  4) research an area that could really help to answer the question.\n\n"
-    "Here is the initial question to decompose:\n"
-    f"{SEPARATOR_LINE}\n"
-    "{question}\n"
-    f"{SEPARATOR_LINE}\n\n"
-    "{history}\n\n"
-    "Do NOT include any text in your answer outside of the list of sub-questions!"
-    "Please formulate your answer as a newline-separated list of questions like so:\n"
-    " <sub-question>\n"
-    " <sub-question>\n"
-    " <sub-question>\n"
-    " ...\n\n"
-    "Answer:"
-).strip()
+INITIAL_QUESTION_DECOMPOSITION_PROMPT = f"""
+Decompose the initial user question into no more than 3 appropriate sub-questions that help to answer the \
+original question. The purpose for this decomposition may be to:
+  1) isolate individual entities (i.e., 'compare sales of company A and company B' -> \
+['what are sales for company A', 'what are sales for company B'])
+  2) clarify or disambiguate ambiguous terms (i.e., 'what is our success with company A' -> \
+['what are our sales with company A','what is our market share with company A', \
+'is company A a reference customer for us', etc.])
+  3) if a term or a metric is essentially clear, but it could relate to various components of an entity and you \
+are generally familiar with the entity, then you can decompose the question into sub-questions that are more \
+specific to components (i.e., 'what do we do to improve scalability of product X', 'what do we to to improve \
+scalability of product X', 'what do we do to improve stability of product X', ...])
+  4) research an area that could really help to answer the question.
+
+Here is the initial question to decompose:
+{SEPARATOR_LINE}
+{{question}}
+{SEPARATOR_LINE}
+
+{{history}}
+
+Do NOT include any text in your answer outside of the list of sub-questions!
+Please formulate your answer as a newline-separated list of questions like so:
+ <sub-question>
+ <sub-question>
+ <sub-question>
+ ...
+
+Answer:
+""".strip()
 
 
 # TODO: combine shared pieces with INITIAL_QUESTION_DECOMPOSITION_PROMPT
-INITIAL_DECOMPOSITION_PROMPT_QUESTIONS_AFTER_SEARCH = (
-    "Decompose the initial user question into no more than 3 appropriate sub-questions that help to answer the"
-    " original question. The purpose for this decomposition may be to:\n"
-    "  1) isolate individual entities (i.e., 'compare sales of company A and company B' ->"
-    " ['what are sales for company A', 'what are sales for company B'])\n"
-    "  2) clarify or disambiguate ambiguous terms (i.e., 'what is our success with company A' ->"
-    " ['what are our sales with company A','what is our market share with company A',"
-    " 'is company A a reference customer for us', etc.])\n"
-    "  3) if a term or a metric is essentially clear, but it could relate to various components of an entity and you"
-    " are generally familiar with the entity, then you can decompose the question into sub-questions that are more"
-    " specific to components (i.e., 'what do we do to improve scalability of product X', 'what do we to to improve"
-    " scalability of product X', 'what do we do to improve stability of product X', ...])\n"
-    "  4) research an area that could really help to answer the question.\n\n"
-    "To give you some context, you will see below also some documents that may relate to the question. Please only"
-    " use this information to learn what the question is approximately asking about, but do not focus on the details"
-    " to construct the sub-questions! Also, some of the entities, relationships and terms that are in the dataset may"
-    " not be in these few documents, so DO NOT focussed too much on the documents when constructing the sub-questions!"
-    " Decomposition and disambiguations are most important!\n\n"
-    "Here are the sample docs to give you some context:\n"
-    f"{SEPARATOR_LINE}\n"
-    "{sample_doc_str}\n"
-    f"{SEPARATOR_LINE}\n\n"
-    "And here is the initial question to decompose:\n"
-    f"{SEPARATOR_LINE}\n"
-    "{question}\n"
-    f"{SEPARATOR_LINE}\n\n"
-    "{history}\n\n"
-    "Do NOT include any text in your answer outside of the list of sub-questions!"
-    "Please formulate your answer as a newline-separated list of questions like so:\n"
-    " <sub-question>\n"
-    " <sub-question>\n"
-    " <sub-question>\n"
-    " ...\n\n"
-    "Answer:"
-).strip()
+INITIAL_DECOMPOSITION_PROMPT_QUESTIONS_AFTER_SEARCH = f"""
+Decompose the initial user question into no more than 3 appropriate sub-questions that help to answer the \
+original question. The purpose for this decomposition may be to:
+  1) isolate individual entities (i.e., 'compare sales of company A and company B' -> \
+['what are sales for company A', 'what are sales for company B'])
+  2) clarify or disambiguate ambiguous terms (i.e., 'what is our success with company A' -> \
+['what are our sales with company A','what is our market share with company A', \
+'is company A a reference customer for us', etc.])
+  3) if a term or a metric is essentially clear, but it could relate to various components of an entity and you \
+are generally familiar with the entity, then you can decompose the question into sub-questions that are more \
+specific to components (i.e., 'what do we do to improve scalability of product X', 'what do we to to improve \
+scalability of product X', 'what do we do to improve stability of product X', ...])
+  4) research an area that could really help to answer the question.
+
+To give you some context, you will see below also some documents that may relate to the question. Please only \
+use this information to learn what the question is approximately asking about, but do not focus on the details \
+to construct the sub-questions! Also, some of the entities, relationships and terms that are in the dataset may \
+not be in these few documents, so DO NOT focussed too much on the documents when constructing the sub-questions! \
+Decomposition and disambiguations are most important!
+
+Here are the sample docs to give you some context:
+{SEPARATOR_LINE}
+{{sample_doc_str}}
+{SEPARATOR_LINE}
+
+And here is the initial question to decompose:
+{SEPARATOR_LINE}
+{{question}}
+{SEPARATOR_LINE}
+
+{{history}}
+
+Do NOT include any text in your answer outside of the list of sub-questions!\
+Please formulate your answer as a newline-separated list of questions like so:
+ <sub-question>
+ <sub-question>
+ <sub-question>
+ ...
+
+Answer:
+""".strip()
 
 
 # Retrieval
-QUERY_REWRITING_PROMPT = (
-    "Please convert the initial user question into a 2-3 more appropriate short and pointed search queries for"
-    " retrieval from a document store. Particularly, try to think about resolving ambiguities and make the search"
-    " queries more specific, enabling the system to search more broadly.\n"
-    "Also, try to make the search queries not redundant, i.e. not too similar!\n\n"
-    "Here is the initial question:\n"
-    f"{SEPARATOR_LINE}\n"
-    "{question}\n"
-    f"{SEPARATOR_LINE}\n\n"
-    "Do NOT include any text in your answer outside of the list of queries!"
-    "Formulate the queries separated by newlines (Do not say 'Query 1: ...', just write the querytext) as follows:\n"
-    "<query 1>\n"
-    "<query 2>\n"
-    "...\n\n"
-    "Queries:"
-)
+QUERY_REWRITING_PROMPT = f"""
+Please convert the initial user question into a 2-3 more appropriate short and pointed search queries for \
+retrieval from a document store. Particularly, try to think about resolving ambiguities and make the search \
+queries more specific, enabling the system to search more broadly.
+
+Also, try to make the search queries not redundant, i.e. not too similar!
+
+Here is the initial question:
+{SEPARATOR_LINE}
+{{question}}
+{SEPARATOR_LINE}
+
+Do NOT include any text in your answer outside of the list of queries!\
+Formulate the queries separated by newlines (Do not say 'Query 1: ...', just write the querytext) as follows:
+<query 1>
+<query 2>
+...
+
+Queries:
+""".strip()
 
 
-DOCUMENT_VERIFICATION_PROMPT = (
-    "Determine whether the following document text contains data or information that is potentially relevant "
-    "for a question. It does not have to be fully relevant, but check whether it has some information that "
-    "would help - possibly in conjunction with other documents - to address the question.\n\n"
-    "Be careful that you do not use a document where you are not sure whether the text applies to the objects "
-    "or entities that are relevant for the question. For example, a book about chess could have long passage "
-    "discussing the psychology of chess without - within the passage - mentioning chess. If now a question "
-    "is asked about the psychology of football, one could be tempted to use the document as it does discuss "
-    "psychology in sports. However, it is NOT about football and should not be deemed relevant. Please "
-    "consider this logic.\n\n"
-    "DOCUMENT TEXT:\n"
-    f"{SEPARATOR_LINE}\n"
-    "{document_content}\n"
-    f"{SEPARATOR_LINE}\n\n"
-    "Do you think that this document text is useful and relevant to answer the following question?\n\n"
-    "QUESTION:\n"
-    f"{SEPARATOR_LINE}\n"
-    "{question}\n"
-    f"{SEPARATOR_LINE}\n\n"
-    "Please answer with exactly and only a 'yes' or 'no'. Do NOT include any other text in your response:\n\n"
-    "Answer:"
-).strip()
+DOCUMENT_VERIFICATION_PROMPT = f"""
+Determine whether the following document text contains data or information that is potentially relevant \
+for a question. It does not have to be fully relevant, but check whether it has some information that \
+would help - possibly in conjunction with other documents - to address the question.
+
+Be careful that you do not use a document where you are not sure whether the text applies to the objects \
+or entities that are relevant for the question. For example, a book about chess could have long passage \
+discussing the psychology of chess without - within the passage - mentioning chess. If now a question \
+is asked about the psychology of football, one could be tempted to use the document as it does discuss \
+psychology in sports. However, it is NOT about football and should not be deemed relevant. Please \
+consider this logic.
+
+DOCUMENT TEXT:
+{SEPARATOR_LINE}
+{{document_content}}
+{SEPARATOR_LINE}
+
+Do you think that this document text is useful and relevant to answer the following question?
+
+QUESTION:
+{SEPARATOR_LINE}
+{{question}}
+{SEPARATOR_LINE}
+
+Please answer with exactly and only a '{YES}' or '{NO}'. Do NOT include any other text in your response:
+
+Answer:
+""".strip()
 
 
 # Sub-Question Anser Generation
-SUB_QUESTION_RAG_PROMPT = (
-    "Use the context provided below - and only the provided context - to answer the given question. "
-    "(Note that the answer is in service of answering a broader question, given below as 'motivation'.)\n\n"
-    "Again, only use the provided context and do not use your internal knowledge! If you cannot answer the "
-    f'question based on the context, say "{UNKNOWN_ANSWER}". It is a matter of life and death that you do NOT '
-    "use your internal knowledge, just the provided information!\n\n"
-    "Make sure that you keep all relevant information, specifically as it concerns to the ultimate goal. "
-    "(But keep other details as well.)\n\n"
-    "It is critical that you provide inline citations in the format [D1], [D2], [D3], etc! "
-    "It is important that the citation is close to the information it supports. "
-    "Proper citations are very important to the user!\n\n"
-    "For your general information, here is the ultimate motivation:\n"
-    f"{SEPARATOR_LINE}\n"
-    "{original_question}\n"
-    f"{SEPARATOR_LINE}\n\n"
-    "And here is the actual question I want you to answer based on the context above (with the motivation in mind):\n"
-    f"{SEPARATOR_LINE}\n"
-    "{question}\n"
-    f"{SEPARATOR_LINE}\n\n"
-    "Here is the context:\n"
-    f"{SEPARATOR_LINE}\n"
-    "{context}\n"
-    f"{SEPARATOR_LINE}\n\n"
-    "Please keep your answer brief and concise, and focus on facts and data.\n\n"
-    "Answer:"
-).strip()
+SUB_QUESTION_RAG_PROMPT = f"""
+Use the context provided below - and only the provided context - to answer the given question. \
+(Note that the answer is in service of answering a broader question, given below as 'motivation').
+
+Again, only use the provided context and do not use your internal knowledge! If you cannot answer the \
+question based on the context, say "{UNKNOWN_ANSWER}". It is a matter of life and death that you do NOT \
+use your internal knowledge, just the provided information!
+
+Make sure that you keep all relevant information, specifically as it concerns to the ultimate goal. \
+(But keep other details as well.)
+
+It is critical that you provide inline citations in the format [D1], [D2], [D3], etc! \
+It is important that the citation is close to the information it supports. \
+Proper citations are very important to the user!
+
+For your general information, here is the ultimate motivation:
+{SEPARATOR_LINE}
+{{original_question}}
+{SEPARATOR_LINE}
+
+And here is the actual question I want you to answer based on the context above (with the motivation in mind):
+{SEPARATOR_LINE}
+{{question}}
+{SEPARATOR_LINE}
+
+Here is the context:
+{SEPARATOR_LINE}
+{{context}}
+{SEPARATOR_LINE}
+
+Please keep your answer brief and concise, and focus on facts and data.
+
+Answer:
+""".strip()
 
 
-SUB_ANSWER_CHECK_PROMPT = (
-    "Determine whether the given answer addresses the given question. "
-    "Please do not use any internal knowledge you may have - just focus on whether the answer "
-    "as given seems to largely address the question as given, or at least addresses part of the question.\n\n"
-    "Here is the question:\n"
-    f"{SEPARATOR_LINE}\n"
-    "{question}\n"
-    f"{SEPARATOR_LINE}\n\n"
-    "Here is the suggested answer:\n"
-    f"{SEPARATOR_LINE}\n"
-    "{base_answer}\n"
-    f"{SEPARATOR_LINE}\n\n"
-    f'Does the suggested answer address the question? Please answer with "{YES}" or "{NO}".'
-).strip()
+SUB_ANSWER_CHECK_PROMPT = f"""
+Determine whether the given answer addresses the given question. \
+Please do not use any internal knowledge you may have - just focus on whether the answer \
+as given seems to largely address the question as given, or at least addresses part of the question.
+
+Here is the question:
+{SEPARATOR_LINE}
+{{question}}
+{SEPARATOR_LINE}
+
+Here is the suggested answer:
+{SEPARATOR_LINE}
+{{base_answer}}
+{SEPARATOR_LINE}
+
+Does the suggested answer address the question? Please answer with "{YES}" or "{NO}".
+""".strip()
 
 
 # Initial Answer Generation
-INITIAL_ANSWER_PROMPT_W_SUB_QUESTIONS = (
-    "{persona_specification}\n\n"
-    "Use the information provided below - and only the provided information - to answer the provided main question.\n\n"
-    "The information provided below consists of:\n"
-    "  1) a number of answered sub-questions - these are very important to help you organize your thoughts and your answer\n"
-    "  2) a number of documents that deemed relevant for the question.\n\n"
-    "{history}\n\n"
-    "It is critical that you provide prover inline citations to documents in the format [D1], [D2], [D3], etc.!\n"
-    "It is important that the citation is close to the information it supports. If you have multiple citations that support\n"
-    "a fact, please cite for example as [D1][D3], or [D2][D4], etc.\n"
-    "Feel free to also cite sub-questions in addition to documents, but make sure that you have documents cited with the "
-    "sub-question citation. If you want to cite both a document and a sub-question, please use [D1][Q3], or "
-    "[D2][D7][Q4], etc.\n"
-    "Again, please NEVER cite sub-questions without a document citation! "
-    "Proper citations are very important for the user!\n\n"
-    "IMPORTANT RULES:\n"
-    " - If you cannot reliably answer the question solely using the provided information, say that you cannot reliably answer.\n"
-    " You may give some additional facts you learned, but do not try to invent an answer.\n"
-    f' - If the information is empty or irrelevant, just say "{UNKNOWN_ANSWER}".\n'
-    " - If the information is relevant but not fully conclusive, specify that the information is not conclusive and say why.\n\n"
-    "Again, you should be sure that the answer is supported by the information provided!\n\n"
-    "Try to keep your answer concise. But also highlight uncertainties you may have should there be substantial ones,\n"
-    "or assumptions you made.\n\n"
-    "Here is the contextual information:\n"
-    f"{SEPARATOR_LINE_LONG}\n\n"
-    "*Answered Sub-questions (these should really matter!):\n"
-    f"{SEPARATOR_LINE}\n"
-    "{answered_sub_questions}\n"
-    f"{SEPARATOR_LINE}\n\n"
-    "And here are relevant document information that support the sub-question answers, "
-    "or that are relevant for the actual question:\n"
-    f"{SEPARATOR_LINE}\n"
-    "{relevant_docs}\n"
-    f"{SEPARATOR_LINE}\n\n"
-    "And here is the question I want you to answer based on the information above:\n"
-    f"{SEPARATOR_LINE}\n"
-    "{question}\n"
-    f"{SEPARATOR_LINE}\n\n"
-    "Please keep your answer brief and concise, and focus on facts and data.\n\n"
-    "Answer:"
-).strip()
+INITIAL_ANSWER_PROMPT_W_SUB_QUESTIONS = f"""
+{{persona_specification}}
+
+Use the information provided below - and only the provided information - to answer the provided main question.
+
+The information provided below consists of:
+  1) a number of answered sub-questions - these are very important to help you organize your thoughts and your answer
+  2) a number of documents that deemed relevant for the question.
+
+{{history}}
+
+It is critical that you provide prover inline citations to documents in the format [D1], [D2], [D3], etc.! \
+It is important that the citation is close to the information it supports. If you have multiple citations that support \
+a fact, please cite for example as [D1][D3], or [D2][D4], etc. \
+Feel free to also cite sub-questions in addition to documents, but make sure that you have documents cited with the \
+sub-question citation. If you want to cite both a document and a sub-question, please use [D1][Q3], or [D2][D7][Q4], etc. \
+Again, please NEVER cite sub-questions without a document citation! Proper citations are very important for the user!
+
+IMPORTANT RULES:
+ - If you cannot reliably answer the question solely using the provided information, say that you cannot reliably answer. \
+You may give some additional facts you learned, but do not try to invent an answer.
+ - If the information is empty or irrelevant, just say "{UNKNOWN_ANSWER}".
+ - If the information is relevant but not fully conclusive, specify that the information is not conclusive and say why.
+
+Again, you should be sure that the answer is supported by the information provided!
+
+Try to keep your answer concise. But also highlight uncertainties you may have should there be substantial ones, \
+or assumptions you made.
+
+Here is the contextual information:
+{SEPARATOR_LINE_LONG}
+
+*Answered Sub-questions (these should really matter!):
+{SEPARATOR_LINE}
+{{answered_sub_questions}}
+{SEPARATOR_LINE}
+
+And here are relevant document information that support the sub-question answers, or that are relevant for the actual question:
+{SEPARATOR_LINE}
+{{relevant_docs}}
+{SEPARATOR_LINE}
+
+And here is the question I want you to answer based on the information above:
+{SEPARATOR_LINE}
+{{question}}
+{SEPARATOR_LINE}
+
+Please keep your answer brief and concise, and focus on facts and data.
+
+Answer:
+""".strip()
 
 
 # Used if sub_question_answer_str is empty
-INITIAL_ANSWER_PROMPT_WO_SUB_QUESTIONS = (
-    "{answered_sub_questions}{persona_specification}\n\n"
-    "Use the information provided below - and only the provided information - to answer the provided question. "
-    "The information provided below consists of a number of documents that were deemed relevant for the question.\n"
-    "{history}\n\n"
-    "IMPORTANT RULES:\n"
-    " - If you cannot reliably answer the question solely using the provided information, say that you cannot reliably answer. "
-    "You may give some additional facts you learned, but do not try to invent an answer.\n"
-    f' - If the information is irrelevant, just say "{UNKNOWN_ANSWER}".\n'
-    " - If the information is relevant but not fully conclusive, specify that the information is not conclusive and say why.\n\n"
-    "Again, you should be sure that the answer is supported by the information provided!\n\n"
-    "It is critical that you provide proper inline citations to documents in the format [D1], [D2], [D3], etc! "
-    "It is important that the citation is close to the information it supports. If you have multiple citations, "
-    "please cite for example as [D1][D3], or [D2][D4], etc. Citations are very important for the user!\n\n"
-    "Here is the relevant context information:\n"
-    f"{SEPARATOR_LINE}\n"
-    "{relevant_docs}\n"
-    f"{SEPARATOR_LINE}\n\n"
-    "And here is the question I want you to answer based on the context above:\n"
-    f"{SEPARATOR_LINE}\n"
-    "{question}\n"
-    f"{SEPARATOR_LINE}\n\n"
-    "Please keep your answer brief and concise, and focus on facts and data.\n\n"
-    "Answer:"
-).strip()
+INITIAL_ANSWER_PROMPT_WO_SUB_QUESTIONS = f"""
+{{answered_sub_questions}}{{persona_specification}}
+
+Use the information provided below - and only the provided information - to answer the provided question. \
+The information provided below consists of a number of documents that were deemed relevant for the question.
+
+{{history}}
+
+IMPORTANT RULES:
+ - If you cannot reliably answer the question solely using the provided information, say that you cannot reliably answer. \
+You may give some additional facts you learned, but do not try to invent an answer.
+ - If the information is irrelevant, just say "{UNKNOWN_ANSWER}".
+ - If the information is relevant but not fully conclusive, specify that the information is not conclusive and say why.
+
+Again, you should be sure that the answer is supported by the information provided!
+
+It is critical that you provide proper inline citations to documents in the format [D1], [D2], [D3], etc! \
+It is important that the citation is close to the information it supports. \
+If you have multiple citations, please cite for example as [D1][D3], or [D2][D4], etc. \
+Citations are very important for the user!
+
+Here is the relevant context information:
+{SEPARATOR_LINE}
+{{relevant_docs}}
+{SEPARATOR_LINE}
+
+And here is the question I want you to answer based on the context above:
+{SEPARATOR_LINE}
+{{question}}
+{SEPARATOR_LINE}
+
+Please keep your answer brief and concise, and focus on facts and data.
+
+Answer:
+""".strip()
 
 
 # REFINEMENT PHASE
-REFINEMENT_QUESTION_DECOMPOSITION_PROMPT = (
-    "An initial user question needs to be answered. An initial answer has been provided but it wasn't quite "
-    "good enough. Also, some sub-questions had been answered and this information has been used to provide "
-    "the initial answer. Some other subquestions may have been suggested based on little knowledge, but they "
-    "were not directly answerable. Also, some entities, relationships and terms are given to you so that "
-    "you have an idea of how the available data looks like.\n\n"
-    "Your role is to generate 2-4 new sub-questions that would help to answer the initial question, considering:\n\n"
-    "1) The initial question\n"
-    "2) The initial answer that was found to be unsatisfactory\n"
-    "3) The sub-questions that were answered\n"
-    "4) The sub-questions that were suggested but not answered\n"
-    "5) The entities, relationships and terms that were extracted from the context\n\n"
-    "The individual questions should be answerable by a good RAG system. "
-    "So a good idea would be to use the sub-questions to resolve ambiguities and/or to separate the "
-    "question for different entities that may be involved in the original question, but in a way that does "
-    "not duplicate questions that were already tried.\n\n"
-    "Additional Guidelines:\n"
-    "- The sub-questions should be specific to the question and provide richer context for the question, "
-    "resolve ambiguities, or address shortcoming of the initial answer\n"
-    "- Each sub-question - when answered - should be relevant for the answer to the original question\n"
-    "- The sub-questions should be free from comparisons, ambiguities,judgements, aggregations, or any "
-    "other complications that may require extra context.\n"
-    "- The sub-questions MUST have the full context of the original question so that it can be executed by "
-    "a RAG system independently without the original question available\n"
-    "    (Example:\n"
-    '    - initial question: "What is the capital of France?"\n'
-    '    - bad sub-question: "What is the name of the river there?"\n'
-    '    - good sub-question: "What is the name of the river that flows through Paris?")\n'
-    "- For each sub-question, please also provide a search term that can be used to retrieve relevant "
-    "documents from a document store.\n"
-    "- Consider specifically the sub-questions that were suggested but not answered. This is a sign that they are not "
-    "answerable with the available context, and you should not ask similar questions.\n\n"
-    "Here is the initial question:\n"
-    f"{SEPARATOR_LINE}\n"
-    "{question}\n"
-    f"{SEPARATOR_LINE}\n"
-    "{history}\n\n"
-    "Here is the initial sub-optimal answer:\n"
-    f"{SEPARATOR_LINE}\n"
-    "{base_answer}\n"
-    f"{SEPARATOR_LINE}\n\n"
-    "Here are the sub-questions that were answered:\n"
-    f"{SEPARATOR_LINE}\n"
-    "{answered_sub_questions}\n"
-    f"{SEPARATOR_LINE}\n\n"
-    "Here are the sub-questions that were suggested but not answered:\n"
-    f"{SEPARATOR_LINE}\n"
-    "{failed_sub_questions}\n"
-    f"{SEPARATOR_LINE}\n\n"
-    "And here are the entities, relationships and terms extracted from the context:\n"
-    f"{SEPARATOR_LINE}\n"
-    "{entity_term_extraction_str}\n"
-    f"{SEPARATOR_LINE}\n\n"
-    "Please generate the list of good, fully contextualized sub-questions that would help to address the main question.\n"
-    "Specifically pay attention also to the entities, relationships and terms extracted, as these indicate what type of "
-    "objects/relationships/terms you can ask about! Do not ask about entities, terms or relationships that are not "
-    "mentioned in the 'entities, relationships and terms' section.\n\n"
-    "Again, please find questions that are NOT overlapping too much with the already answered "
-    "sub-questions or those that already were suggested and failed.\n"
-    "In other words - what can we try in addition to what has been tried so far?\n\n"
-    "Generate the list of questions separated by one new line like this:\n"
-    "<sub-question 1>\n"
-    "<sub-question 2>\n"
-    "<sub-question 3>\n"
-    "..."
-).strip()
+REFINEMENT_QUESTION_DECOMPOSITION_PROMPT = f"""
+An initial user question needs to be answered. An initial answer has been provided but it wasn't quite good enough. \
+Also, some sub-questions had been answered and this information has been used to provide the initial answer. \
+Some other subquestions may have been suggested based on little knowledge, but they were not directly answerable. \
+Also, some entities, relationships and terms are given to you so that you have an idea of how the available data looks like.
+
+Your role is to generate 2-4 new sub-questions that would help to answer the initial question, considering:
+
+1) The initial question
+2) The initial answer that was found to be unsatisfactory
+3) The sub-questions that were answered
+4) The sub-questions that were suggested but not answered
+5) The entities, relationships and terms that were extracted from the context
+
+The individual questions should be answerable by a good RAG system. So a good idea would be to use the sub-questions to \
+resolve ambiguities and/or to separate the question for different entities that may be involved in the original question, \
+but in a way that does not duplicate questions that were already tried.
+
+Additional Guidelines:
+- The sub-questions should be specific to the question and provide richer context for the question, resolve ambiguities, \
+or address shortcoming of the initial answer
+- Each sub-question - when answered - should be relevant for the answer to the original question
+- The sub-questions should be free from comparisons, ambiguities,judgements, aggregations, or any other complications that \
+may require extra context
+- The sub-questions MUST have the full context of the original question so that it can be executed by a RAG system \
+independently without the original question available
+    Example:
+    - initial question: "What is the capital of France?"
+    - bad sub-question: "What is the name of the river there?"
+    - good sub-question: "What is the name of the river that flows through Paris?"
+- For each sub-question, please also provide a search term that can be used to retrieve relevant documents from a document store.
+- Consider specifically the sub-questions that were suggested but not answered. This is a sign that they are not answerable \
+with the available context, and you should not ask similar questions.
+
+Here is the initial question:
+{SEPARATOR_LINE}
+{{question}}
+{SEPARATOR_LINE}
+{{history}}
+
+Here is the initial sub-optimal answer:
+{SEPARATOR_LINE}
+{{base_answer}}
+{SEPARATOR_LINE}
+
+Here are the sub-questions that were answered:
+{SEPARATOR_LINE}
+{{answered_sub_questions}}
+{SEPARATOR_LINE}
+
+Here are the sub-questions that were suggested but not answered:
+{SEPARATOR_LINE}
+{{failed_sub_questions}}
+{SEPARATOR_LINE}
+
+And here are the entities, relationships and terms extracted from the context:
+{SEPARATOR_LINE}
+{{entity_term_extraction_str}}
+{SEPARATOR_LINE}
+
+Please generate the list of good, fully contextualized sub-questions that would help to address the main question. \
+Specifically pay attention also to the entities, relationships and terms extracted, as these indicate what type of \
+objects/relationships/terms you can ask about! Do not ask about entities, terms or relationships that are not mentioned in the \
+'entities, relationships and terms' section.
+
+Again, please find questions that are NOT overlapping too much with the already answered sub-questions or those that \
+already were suggested and failed. In other words - what can we try in addition to what has been tried so far?
+
+Generate the list of questions separated by one new line like this:
+<sub-question 1>
+<sub-question 2>
+<sub-question 3>
+...""".strip()
 
 
-REFINED_ANSWER_PROMPT_W_SUB_QUESTIONS = (
-    "{persona_specification}\n\n"
-    "Your task is to improve on a given answer to a question, as the initial answer was found to be lacking in some way.\n\n"
-    "Use the information provided below - and only the provided information - to write your new and improved answer.\n\n"
-    "The information provided below consists of:\n"
-    "  1) an initial answer that was given but found to be lacking in some way.\n"
-    "  2) a number of answered sub-questions - these are very important(!) and definitely should help you to answer "
-    "the main question. Note that the sub-questions have a type, 'initial' and 'refined'. The 'initial' "
-    "ones were available for the creation of the initial answer, but the 'refined' were not, they are new. So please use "
-    "the 'refined' sub-questions in particular to update/extend/correct/enrich the initial answer and to add "
-    "more details/new facts!\n"
-    "  3) a number of documents that were deemed relevant for the question. This is the context that you use largely for "
-    "citations (see below). So consider the answers to the sub-questions as guidelines to construct your new answer, but "
-    "make sure you cite the relevant document for a fact!\n\n"
-    "It is critical that you provide proper inline citations to documents in the format [D1], [D2], [D3], etc! "
-    "It is important that the citation is close to the information it supports. "
-    "DO NOT just list all of the citations at the very end. "
-    "Feel free to also cite sub-questions in addition to documents, but make sure that you have documents cited with the "
-    "sub-question citation. If you want to cite both a document and a sub-question, please use [D1][Q3], or [D2][D7][Q4], etc. "
-    "and always place the document citation before the sub-question citation. "
-    "Again, please NEVER cite sub-questions without a document citation!\n"
-    "Proper citations are very important for the user!\n\n"
-    "{history}\n\n"
-    "IMPORTANT RULES:\n"
-    " - If you cannot reliably answer the question solely using the provided information, say that you cannot reliably answer. "
-    "You may give some additional facts you learned, but do not try to invent an answer.\n"
-    f' - If the information is empty or irrelevant, just say "{UNKNOWN_ANSWER}".\n'
-    " - If the information is relevant but not fully conclusive, provide an answer to the extent you can but also "
-    "specify that the information is not conclusive and why.\n"
-    " - Ignore any existing citations within the answered sub-questions, like [D1]... and [Q2]! "
-    "The citations you will need to use will need to refer to the documents (and sub-questions) that you are explicitly "
-    "presented with below!\n\n"
-    "Again, you should be sure that the answer is supported by the information provided!\n\n"
-    "Try to keep your answer concise. But also highlight uncertainties you may have should there be substantial ones, "
-    "or assumptions you made.\n\n"
-    "Here is the contextual information:\n"
-    f"{SEPARATOR_LINE_LONG}\n\n"
-    "*Initial Answer that was found to be lacking:\n"
-    f"{SEPARATOR_LINE}\n"
-    "{initial_answer}\n"
-    f"{SEPARATOR_LINE}\n\n"
-    "*Answered Sub-questions (these should really help you to research your answer! They also contain questions/answers "
-    "that were not available when the original answer was constructed):\n"
-    "{answered_sub_questions}\n\n"
-    "And here are the relevant documents that support the sub-question answers, and that are relevant for the actual question:\n"
-    f"{SEPARATOR_LINE}\n"
-    "{relevant_docs}\n"
-    f"{SEPARATOR_LINE}\n\n"
-    "Lastly, here is the main question I want you to answer based on the information above:\n"
-    f"{SEPARATOR_LINE}\n"
-    "{question}\n"
-    f"{SEPARATOR_LINE}\n\n"
-    "Please keep your answer brief and concise, and focus on facts and data.\n\n"
-    "Answer:"
-).strip()
+REFINED_ANSWER_PROMPT_W_SUB_QUESTIONS = f"""
+{{persona_specification}}
+
+Your task is to improve on a given answer to a question, as the initial answer was found to be lacking in some way.
+
+Use the information provided below - and only the provided information - to write your new and improved answer.
+
+The information provided below consists of:
+  1) an initial answer that was given but found to be lacking in some way.
+  2) a number of answered sub-questions - these are very important(!) and definitely should help you to answer the main \
+question. Note that the sub-questions have a type, 'initial' and 'refined'. The 'initial' ones were available for the \
+creation of the initial answer, but the 'refined' were not, they are new. So please use the 'refined' sub-questions in \
+particular to update/extend/correct/enrich the initial answer and to add more details/new facts!
+  3) a number of documents that were deemed relevant for the question. This is the context that you use largely for citations \
+(see below). So consider the answers to the sub-questions as guidelines to construct your new answer, but make sure you cite \
+the relevant document for a fact!
+
+It is critical that you provide proper inline citations to documents in the format [D1], [D2], [D3], etc! \
+It is important that the citation is close to the information it supports. \
+DO NOT just list all of the citations at the very end. \
+Feel free to also cite sub-questions in addition to documents, \
+but make sure that you have documents cited with the sub-question citation. \
+If you want to cite both a document and a sub-question, please use [D1][Q3], or [D2][D7][Q4], etc. and always place the \
+document citation before the sub-question citation. Again, please NEVER cite sub-questions without a document citation! \
+Proper citations are very important for the user!
+
+{{history}}
+
+IMPORTANT RULES:
+ - If you cannot reliably answer the question solely using the provided information, say that you cannot reliably answer. \
+You may give some additional facts you learned, but do not try to invent an answer.
+ - If the information is empty or irrelevant, just say "{UNKNOWN_ANSWER}".
+ - If the information is relevant but not fully conclusive, provide an answer to the extent you can but also specify that \
+the information is not conclusive and why.
+ - Ignore any existing citations within the answered sub-questions, like [D1]... and [Q2]! The citations you will need to \
+use will need to refer to the documents (and sub-questions) that you are explicitly presented with below!
+
+Again, you should be sure that the answer is supported by the information provided!
+
+Try to keep your answer concise. But also highlight uncertainties you may have should there be substantial ones, \
+or assumptions you made.
+
+Here is the contextual information:
+{SEPARATOR_LINE_LONG}
+
+*Initial Answer that was found to be lacking:
+{SEPARATOR_LINE}
+{{initial_answer}}
+{SEPARATOR_LINE}
+
+*Answered Sub-questions (these should really help you to research your answer! They also contain questions/answers that \
+were not available when the original answer was constructed):
+{{answered_sub_questions}}
+
+And here are the relevant documents that support the sub-question answers, and that are relevant for the actual question:
+{SEPARATOR_LINE}
+{{relevant_docs}}
+{SEPARATOR_LINE}
+
+Lastly, here is the main question I want you to answer based on the information above:
+{SEPARATOR_LINE}
+{{question}}
+{SEPARATOR_LINE}
+
+Please keep your answer brief and concise, and focus on facts and data.
+
+Answer:
+""".strip()
 
 # sub_question_answer_str is empty
-REFINED_ANSWER_PROMPT_WO_SUB_QUESTIONS = (
-    "{answered_sub_questions}{persona_specification}\n\n"
-    "Use the information provided below - and only the provided information - to answer the provided question.\n\n"
-    "The information provided below consists of:\n"
-    "  1) an initial answer that was given but found to be lacking in some way.\n"
-    "  2) a number of documents that were also deemed relevant for the question.\n\n"
-    "It is critical that you provide proper inline citations to documents in the format [D1], [D2], [D3], etc! "
-    "It is important that the citation is close to the information it supports. "
-    "DO NOT just list all of the citations at the very end of your response. Citations are very important for the user!\n\n"
-    "{history}\n\n"
-    "IMPORTANT RULES:\n"
-    " - If you cannot reliably answer the question solely using the provided information, say that you cannot reliably answer. "
-    "You may give some additional facts you learned, but do not try to invent an answer.\n"
-    f' - If the information is empty or irrelevant, just say "{UNKNOWN_ANSWER}".\n'
-    " - If the information is relevant but not fully conclusive, provide an answer to the extent you can but also "
-    "specify that the information is not conclusive and why.\n\n"
-    "Again, you should be sure that the answer is supported by the information provided!\n\n"
-    "Try to keep your answer concise. But also highlight uncertainties you may have should there be substantial ones, "
-    "or assumptions you made.\n\n"
-    "Here is the contextual information:\n"
-    f"{SEPARATOR_LINE_LONG}\n\n"
-    "*Initial Answer that was found to be lacking:\n"
-    f"{SEPARATOR_LINE}\n"
-    "{initial_answer}\n"
-    f"{SEPARATOR_LINE}\n\n"
-    "And here are relevant document information that support the sub-question answers, "
-    "or that are relevant for the actual question:\n"
-    f"{SEPARATOR_LINE}\n"
-    "{relevant_docs}\n"
-    f"{SEPARATOR_LINE}\n\n"
-    "Lastly, here is the question I want you to answer based on the information above:\n"
-    f"{SEPARATOR_LINE}\n"
-    "{question}\n"
-    f"{SEPARATOR_LINE}\n\n"
-    "Please keep your answer brief and concise, and focus on facts and data.\n\n"
-    "Answer:"
-).strip()
+REFINED_ANSWER_PROMPT_WO_SUB_QUESTIONS = f"""
+{{answered_sub_questions}}{{persona_specification}}
+
+Use the information provided below - and only the provided information - to answer the provided question.
+
+The information provided below consists of:
+  1) an initial answer that was given but found to be lacking in some way.
+  2) a number of documents that were also deemed relevant for the question.
+
+It is critical that you provide proper inline citations to documents in the format [D1], [D2], [D3], etc! \
+It is important that the citation is close to the information it supports. \
+DO NOT just list all of the citations at the very end of your response. Citations are very important for the user!
+
+{{history}}
+
+IMPORTANT RULES:
+ - If you cannot reliably answer the question solely using the provided information, say that you cannot reliably answer. \
+You may give some additional facts you learned, but do not try to invent an answer.
+ - If the information is empty or irrelevant, just say "{UNKNOWN_ANSWER}".
+ - If the information is relevant but not fully conclusive, provide an answer to the extent you can but also specify that \
+the information is not conclusive and why.
+
+Again, you should be sure that the answer is supported by the information provided!
+
+Try to keep your answer concise. But also highlight uncertainties you may have should there be substantial ones, \
+or assumptions you made.
+
+Here is the contextual information:
+{SEPARATOR_LINE_LONG}
+
+*Initial Answer that was found to be lacking:
+{SEPARATOR_LINE}
+{{initial_answer}}
+{SEPARATOR_LINE}
+
+And here are relevant document information that support the sub-question answers, \
+or that are relevant for the actual question:
+{SEPARATOR_LINE}
+{{relevant_docs}}
+{SEPARATOR_LINE}
+
+Lastly, here is the question I want you to answer based on the information above:
+{SEPARATOR_LINE}
+{{question}}
+{SEPARATOR_LINE}
+
+Please keep your answer brief and concise, and focus on facts and data.
+
+Answer:
+""".strip()
 
 
-INITIAL_REFINED_ANSWER_COMPARISON_PROMPT = (
-    "For the given question, please compare the initial answer and the refined answer and determine if "
-    "the refined answer is substantially better than the initial answer, not just a bit better. Better could mean:\n"
-    " - additional information\n"
-    " - more comprehensive information\n"
-    " - more concise information\n"
-    " - more structured information\n"
-    " - more details\n"
-    " - new bullet points\n"
-    " - substantially more document citations ([D1], [D2], [D3], etc.)\n\n"
-    "Put yourself in the shoes of the user and think about whether the refined answer is really substantially "
-    "better and delivers really new insights than the initial answer.\n\n"
-    "Here is the question:\n"
-    f"{SEPARATOR_LINE}\n"
-    "{question}\n"
-    f"{SEPARATOR_LINE}\n\n"
-    "Here is the initial answer:\n"
-    f"{SEPARATOR_LINE}\n"
-    "{initial_answer}\n"
-    f"{SEPARATOR_LINE}\n\n"
-    "Here is the refined answer:\n"
-    f"{SEPARATOR_LINE}\n"
-    "{refined_answer}\n"
-    f"{SEPARATOR_LINE}\n\n"
-    "With these criteria in mind, is the refined answer substantially better than the initial answer?\n\n"
-    f'Please answer with a simple "{YES}" or "{NO}".'
-).strip()
+INITIAL_REFINED_ANSWER_COMPARISON_PROMPT = f"""
+For the given question, please compare the initial answer and the refined answer and determine if the refined answer is \
+substantially better than the initial answer, not just a bit better. Better could mean:
+ - additional information
+ - more comprehensive information
+ - more concise information
+ - more structured information
+ - more details
+ - new bullet points
+ - substantially more document citations ([D1], [D2], [D3], etc.)
+
+Put yourself in the shoes of the user and think about whether the refined answer is really substantially better and \
+delivers really new insights than the initial answer.
+
+Here is the question:
+{SEPARATOR_LINE}
+{{question}}
+{SEPARATOR_LINE}
+
+Here is the initial answer:
+{SEPARATOR_LINE}
+{{initial_answer}}
+{SEPARATOR_LINE}
+
+Here is the refined answer:
+{SEPARATOR_LINE}
+{{refined_answer}}
+{SEPARATOR_LINE}
+
+With these criteria in mind, is the refined answer substantially better than the initial answer?
+
+Please answer with a simple "{YES}" or "{NO}"
+""".strip()

backend/onyx/prompts/chat_prompts.py

@@ -2,8 +2,10 @@ from onyx.prompts.constants import GENERAL_SEP_PAT
 from onyx.prompts.constants import QUESTION_PAT
 
 REQUIRE_CITATION_STATEMENT = """
-Cite relevant statements INLINE using the format [1], [2], [3], etc to reference the document number, \
-DO NOT provide a reference section at the end and DO NOT provide any links following the citations.
+Cite relevant statements INLINE using the format [1], [2], [3], etc. to reference the document number. \
+DO NOT provide any links following the citations. In other words, avoid using the format [1](https://example.com). \
+Avoid using double brackets like [[1]]. To cite multiple documents, use [1], [2] format instead of [1, 2]. \
+Try to cite inline as opposed to leaving all citations until the very end of the response.
 """.rstrip()
 
 NO_CITATION_STATEMENT = """

backend/onyx/redis/redis_connector.py

@@ -17,6 +17,8 @@ class RedisConnector:
     associated background tasks / associated redis interactions."""
 
     def __init__(self, tenant_id: str | None, id: int) -> None:
+        """id: a connector credential pair id"""
+
         self.tenant_id: str | None = tenant_id
         self.id: int = id
         self.redis: redis.Redis = get_redis_client(tenant_id=tenant_id)

backend/onyx/redis/redis_connector_credential_pair.py

@@ -2,6 +2,7 @@ import time
 from typing import cast
 from uuid import uuid4
 
+import redis
 from celery import Celery
 from redis import Redis
 from redis.lock import Lock as RedisLock
@@ -12,11 +13,11 @@ from onyx.configs.constants import CELERY_VESPA_SYNC_BEAT_LOCK_TIMEOUT
 from onyx.configs.constants import OnyxCeleryPriority
 from onyx.configs.constants import OnyxCeleryQueues
 from onyx.configs.constants import OnyxCeleryTask
+from onyx.configs.constants import OnyxRedisConstants
 from onyx.db.connector_credential_pair import get_connector_credential_pair_from_id
 from onyx.db.document import (
-    construct_document_select_for_connector_credential_pair_by_needs_sync,
+    construct_document_id_select_for_connector_credential_pair_by_needs_sync,
 )
-from onyx.db.models import Document
 from onyx.redis.redis_object_helper import RedisObjectHelper
 
 
@@ -28,21 +29,14 @@ class RedisConnectorCredentialPair(RedisObjectHelper):
     all connectors and is not per connector."""
 
     PREFIX = "connectorsync"
-    FENCE_PREFIX = PREFIX + "_fence"
     TASKSET_PREFIX = PREFIX + "_taskset"
 
-    SYNCING_PREFIX = PREFIX + ":vespa_syncing"
-
     def __init__(self, tenant_id: str | None, id: int) -> None:
         super().__init__(tenant_id, str(id))
 
         # documents that should be skipped
         self.skip_docs: set[str] = set()
 
-    @classmethod
-    def get_fence_key(cls) -> str:
-        return RedisConnectorCredentialPair.FENCE_PREFIX
-
     @classmethod
     def get_taskset_key(cls) -> str:
         return RedisConnectorCredentialPair.TASKSET_PREFIX
@@ -51,19 +45,14 @@ class RedisConnectorCredentialPair(RedisObjectHelper):
     def taskset_key(self) -> str:
         """Notice that this is intentionally reusing the same taskset for all
         connector syncs"""
-        # example: connector_taskset
+        # example: connectorsync_taskset
         return f"{self.TASKSET_PREFIX}"
 
     def set_skip_docs(self, skip_docs: set[str]) -> None:
-        # documents that should be skipped. Note that this classes updates
+        # documents that should be skipped. Note that this class updates
         # the list on the fly
         self.skip_docs = skip_docs
 
-    @staticmethod
-    def make_redis_syncing_key(doc_id: str) -> str:
-        """used to create a key in redis to block a doc from syncing"""
-        return f"{RedisConnectorCredentialPair.SYNCING_PREFIX}:{doc_id}"
-
     def generate_tasks(
         self,
         max_tasks: int,
@@ -82,7 +71,8 @@ class RedisConnectorCredentialPair(RedisObjectHelper):
 
         last_lock_time = time.monotonic()
 
-        async_results = []
+        num_tasks_sent = 0
+
         cc_pair = get_connector_credential_pair_from_id(
             db_session=db_session,
             cc_pair_id=int(self._id),
@@ -90,14 +80,14 @@ class RedisConnectorCredentialPair(RedisObjectHelper):
         if not cc_pair:
             return None
 
-        stmt = construct_document_select_for_connector_credential_pair_by_needs_sync(
+        stmt = construct_document_id_select_for_connector_credential_pair_by_needs_sync(
             cc_pair.connector_id, cc_pair.credential_id
         )
 
         num_docs = 0
 
-        for doc in db_session.scalars(stmt).yield_per(DB_YIELD_PER_DEFAULT):
-            doc = cast(Document, doc)
+        for doc_id in db_session.scalars(stmt).yield_per(DB_YIELD_PER_DEFAULT):
+            doc_id = cast(str, doc_id)
             current_time = time.monotonic()
             if current_time - last_lock_time >= (
                 CELERY_VESPA_SYNC_BEAT_LOCK_TIMEOUT / 4
@@ -108,18 +98,9 @@ class RedisConnectorCredentialPair(RedisObjectHelper):
             num_docs += 1
 
             # check if we should skip the document (typically because it's already syncing)
-            if doc.id in self.skip_docs:
+            if doc_id in self.skip_docs:
                 continue
 
-            # an arbitrary number in seconds to prevent the same doc from syncing repeatedly
-            # SYNC_EXPIRATION = 24 * 60 * 60
-
-            # a quick hack that can be uncommented to prevent a doc from resyncing over and over
-            # redis_syncing_key = self.make_redis_syncing_key(doc.id)
-            # if redis_client.exists(redis_syncing_key):
-            #     continue
-            # redis_client.set(redis_syncing_key, custom_task_id, ex=SYNC_EXPIRATION)
-
             # celery's default task id format is "dd32ded3-00aa-4884-8b21-42f8332e7fac"
             # the key for the result is "celery-task-meta-dd32ded3-00aa-4884-8b21-42f8332e7fac"
             # we prefix the task id so it's easier to keep track of who created the task
@@ -133,18 +114,93 @@ class RedisConnectorCredentialPair(RedisObjectHelper):
             )
 
             # Priority on sync's triggered by new indexing should be medium
-            result = celery_app.send_task(
+            celery_app.send_task(
                 OnyxCeleryTask.VESPA_METADATA_SYNC_TASK,
-                kwargs=dict(document_id=doc.id, tenant_id=tenant_id),
+                kwargs=dict(document_id=doc_id, tenant_id=tenant_id),
                 queue=OnyxCeleryQueues.VESPA_METADATA_SYNC,
                 task_id=custom_task_id,
                 priority=OnyxCeleryPriority.MEDIUM,
             )
 
-            async_results.append(result)
-            self.skip_docs.add(doc.id)
+            num_tasks_sent += 1
+            self.skip_docs.add(doc_id)
 
-            if len(async_results) >= max_tasks:
+            if num_tasks_sent >= max_tasks:
                 break
 
-        return len(async_results), num_docs
+        return num_tasks_sent, num_docs
+
+
+class RedisGlobalConnectorCredentialPair:
+    """This class is used to scan documents by cc_pair in the db and collect them into
+    a unified set for syncing.
+
+    It differs from the other redis helpers in that the taskset used spans
+    all connectors and is not per connector."""
+
+    PREFIX = "connectorsync"
+    FENCE_KEY = PREFIX + "_fence"
+    TASKSET_KEY = PREFIX + "_taskset"
+
+    def __init__(self, redis: redis.Redis) -> None:
+        self.redis = redis
+
+    @property
+    def fenced(self) -> bool:
+        if self.redis.exists(self.fence_key):
+            return True
+
+        return False
+
+    @property
+    def payload(self) -> int | None:
+        bytes = self.redis.get(self.fence_key)
+        if bytes is None:
+            return None
+
+        progress = int(cast(int, bytes))
+        return progress
+
+    def get_remaining(self) -> int:
+        remaining = cast(int, self.redis.scard(self.taskset_key))
+        return remaining
+
+    @property
+    def fence_key(self) -> str:
+        """Notice that this is intentionally reusing the same fence for all
+        connector syncs"""
+        # example: connectorsync_fence
+        return f"{self.FENCE_KEY}"
+
+    @property
+    def taskset_key(self) -> str:
+        """Notice that this is intentionally reusing the same taskset for all
+        connector syncs"""
+        # example: connectorsync_taskset
+        return f"{self.TASKSET_KEY}"
+
+    def set_fence(self, payload: int | None) -> None:
+        if payload is None:
+            self.redis.srem(OnyxRedisConstants.ACTIVE_FENCES, self.fence_key)
+            self.redis.delete(self.fence_key)
+            return
+
+        self.redis.set(self.fence_key, payload)
+        self.redis.sadd(OnyxRedisConstants.ACTIVE_FENCES, self.fence_key)
+
+    def delete_taskset(self) -> None:
+        self.redis.delete(self.taskset_key)
+
+    def reset(self) -> None:
+        self.redis.srem(OnyxRedisConstants.ACTIVE_FENCES, self.fence_key)
+        self.redis.delete(self.taskset_key)
+        self.redis.delete(self.fence_key)
+
+    @staticmethod
+    def reset_all(r: redis.Redis) -> None:
+        r.srem(
+            OnyxRedisConstants.ACTIVE_FENCES,
+            RedisGlobalConnectorCredentialPair.FENCE_KEY,
+        )
+        r.delete(RedisGlobalConnectorCredentialPair.TASKSET_KEY)
+        r.delete(RedisGlobalConnectorCredentialPair.FENCE_KEY)

backend/onyx/redis/redis_connector_delete.py

@@ -14,6 +14,7 @@ from onyx.configs.constants import CELERY_VESPA_SYNC_BEAT_LOCK_TIMEOUT
 from onyx.configs.constants import OnyxCeleryPriority
 from onyx.configs.constants import OnyxCeleryQueues
 from onyx.configs.constants import OnyxCeleryTask
+from onyx.configs.constants import OnyxRedisConstants
 from onyx.db.connector_credential_pair import get_connector_credential_pair_from_id
 from onyx.db.document import construct_document_select_for_connector_credential_pair
 from onyx.db.models import Document as DbDocument
@@ -69,10 +70,12 @@ class RedisConnectorDelete:
 
     def set_fence(self, payload: RedisConnectorDeletePayload | None) -> None:
         if not payload:
+            self.redis.srem(OnyxRedisConstants.ACTIVE_FENCES, self.fence_key)
             self.redis.delete(self.fence_key)
             return
 
         self.redis.set(self.fence_key, payload.model_dump_json())
+        self.redis.sadd(OnyxRedisConstants.ACTIVE_FENCES, self.fence_key)
 
     def _generate_task_id(self) -> str:
         # celery's default task id format is "dd32ded3-00aa-4884-8b21-42f8332e7fac"
@@ -136,6 +139,7 @@ class RedisConnectorDelete:
         return len(async_results)
 
     def reset(self) -> None:
+        self.redis.srem(OnyxRedisConstants.ACTIVE_FENCES, self.fence_key)
         self.redis.delete(self.taskset_key)
         self.redis.delete(self.fence_key)
 

backend/onyx/redis/redis_connector_doc_perm_sync.py

@@ -1,5 +1,6 @@
 import time
 from datetime import datetime
+from typing import Any
 from typing import cast
 from uuid import uuid4
 
@@ -13,6 +14,7 @@ from onyx.configs.constants import CELERY_GENERIC_BEAT_LOCK_TIMEOUT
 from onyx.configs.constants import OnyxCeleryPriority
 from onyx.configs.constants import OnyxCeleryQueues
 from onyx.configs.constants import OnyxCeleryTask
+from onyx.configs.constants import OnyxRedisConstants
 from onyx.redis.redis_pool import SCAN_ITER_COUNT_DEFAULT
 
 
@@ -95,7 +97,7 @@ class RedisConnectorPermissionSync:
     @property
     def payload(self) -> RedisConnectorPermissionSyncPayload | None:
         # read related data and evaluate/print task progress
-        fence_bytes = cast(bytes, self.redis.get(self.fence_key))
+        fence_bytes = cast(Any, self.redis.get(self.fence_key))
         if fence_bytes is None:
             return None
 
@@ -111,10 +113,12 @@ class RedisConnectorPermissionSync:
         payload: RedisConnectorPermissionSyncPayload | None,
     ) -> None:
         if not payload:
+            self.redis.srem(OnyxRedisConstants.ACTIVE_FENCES, self.fence_key)
             self.redis.delete(self.fence_key)
             return
 
         self.redis.set(self.fence_key, payload.model_dump_json())
+        self.redis.sadd(OnyxRedisConstants.ACTIVE_FENCES, self.fence_key)
 
     def set_active(self) -> None:
         """This sets a signal to keep the permissioning flow from getting cleaned up within
@@ -196,6 +200,7 @@ class RedisConnectorPermissionSync:
         return len(async_results)
 
     def reset(self) -> None:
+        self.redis.srem(OnyxRedisConstants.ACTIVE_FENCES, self.fence_key)
         self.redis.delete(self.active_key)
         self.redis.delete(self.generator_progress_key)
         self.redis.delete(self.generator_complete_key)

backend/onyx/redis/redis_connector_ext_group_sync.py

@@ -1,4 +1,5 @@
 from datetime import datetime
+from typing import Any
 from typing import cast
 
 import redis
@@ -82,7 +83,7 @@ class RedisConnectorExternalGroupSync:
     @property
     def payload(self) -> RedisConnectorExternalGroupSyncPayload | None:
         # read related data and evaluate/print task progress
-        fence_bytes = cast(bytes, self.redis.get(self.fence_key))
+        fence_bytes = cast(Any, self.redis.get(self.fence_key))
         if fence_bytes is None:
             return None
 

backend/onyx/redis/redis_connector_index.py

@@ -1,10 +1,13 @@
 from datetime import datetime
+from typing import Any
 from typing import cast
 from uuid import uuid4
 
 import redis
 from pydantic import BaseModel
 
+from onyx.configs.constants import OnyxRedisConstants
+
 
 class RedisConnectorIndexPayload(BaseModel):
     index_attempt_id: int | None
@@ -89,7 +92,7 @@ class RedisConnectorIndex:
     @property
     def payload(self) -> RedisConnectorIndexPayload | None:
         # read related data and evaluate/print task progress
-        fence_bytes = cast(bytes, self.redis.get(self.fence_key))
+        fence_bytes = cast(Any, self.redis.get(self.fence_key))
         if fence_bytes is None:
             return None
 
@@ -103,10 +106,12 @@ class RedisConnectorIndex:
         payload: RedisConnectorIndexPayload | None,
     ) -> None:
         if not payload:
+            self.redis.srem(OnyxRedisConstants.ACTIVE_FENCES, self.fence_key)
             self.redis.delete(self.fence_key)
             return
 
         self.redis.set(self.fence_key, payload.model_dump_json())
+        self.redis.sadd(OnyxRedisConstants.ACTIVE_FENCES, self.fence_key)
 
     def terminating(self, celery_task_id: str) -> bool:
         if self.redis.exists(f"{self.terminate_key}_{celery_task_id}"):
@@ -188,6 +193,7 @@ class RedisConnectorIndex:
         return status
 
     def reset(self) -> None:
+        self.redis.srem(OnyxRedisConstants.ACTIVE_FENCES, self.fence_key)
         self.redis.delete(self.active_key)
         self.redis.delete(self.generator_lock_key)
         self.redis.delete(self.generator_progress_key)

backend/onyx/redis/redis_connector_prune.py

@@ -11,6 +11,7 @@ from onyx.configs.constants import CELERY_GENERIC_BEAT_LOCK_TIMEOUT
 from onyx.configs.constants import OnyxCeleryPriority
 from onyx.configs.constants import OnyxCeleryQueues
 from onyx.configs.constants import OnyxCeleryTask
+from onyx.configs.constants import OnyxRedisConstants
 from onyx.db.connector_credential_pair import get_connector_credential_pair_from_id
 from onyx.redis.redis_pool import SCAN_ITER_COUNT_DEFAULT
 
@@ -79,10 +80,12 @@ class RedisConnectorPrune:
 
     def set_fence(self, value: bool) -> None:
         if not value:
+            self.redis.srem(OnyxRedisConstants.ACTIVE_FENCES, self.fence_key)
             self.redis.delete(self.fence_key)
             return
 
         self.redis.set(self.fence_key, 0)
+        self.redis.sadd(OnyxRedisConstants.ACTIVE_FENCES, self.fence_key)
 
     @property
     def generator_complete(self) -> int | None:
@@ -158,6 +161,7 @@ class RedisConnectorPrune:
         return len(async_results)
 
     def reset(self) -> None:
+        self.redis.srem(OnyxRedisConstants.ACTIVE_FENCES, self.fence_key)
         self.redis.delete(self.generator_progress_key)
         self.redis.delete(self.generator_complete_key)
         self.redis.delete(self.taskset_key)

backend/onyx/redis/redis_document_set.py

@@ -13,8 +13,8 @@ from onyx.configs.constants import CELERY_VESPA_SYNC_BEAT_LOCK_TIMEOUT
 from onyx.configs.constants import OnyxCeleryPriority
 from onyx.configs.constants import OnyxCeleryQueues
 from onyx.configs.constants import OnyxCeleryTask
-from onyx.db.document_set import construct_document_select_by_docset
-from onyx.db.models import Document
+from onyx.configs.constants import OnyxRedisConstants
+from onyx.db.document_set import construct_document_id_select_by_docset
 from onyx.redis.redis_object_helper import RedisObjectHelper
 
 
@@ -35,10 +35,12 @@ class RedisDocumentSet(RedisObjectHelper):
 
     def set_fence(self, payload: int | None) -> None:
         if payload is None:
+            self.redis.srem(OnyxRedisConstants.ACTIVE_FENCES, self.fence_key)
             self.redis.delete(self.fence_key)
             return
 
         self.redis.set(self.fence_key, payload)
+        self.redis.sadd(OnyxRedisConstants.ACTIVE_FENCES, self.fence_key)
 
     @property
     def payload(self) -> int | None:
@@ -63,10 +65,11 @@ class RedisDocumentSet(RedisObjectHelper):
         """
         last_lock_time = time.monotonic()
 
-        async_results = []
-        stmt = construct_document_select_by_docset(int(self._id), current_only=False)
-        for doc in db_session.scalars(stmt).yield_per(DB_YIELD_PER_DEFAULT):
-            doc = cast(Document, doc)
+        num_tasks_sent = 0
+
+        stmt = construct_document_id_select_by_docset(int(self._id), current_only=False)
+        for doc_id in db_session.scalars(stmt).yield_per(DB_YIELD_PER_DEFAULT):
+            doc_id = cast(str, doc_id)
             current_time = time.monotonic()
             if current_time - last_lock_time >= (
                 CELERY_VESPA_SYNC_BEAT_LOCK_TIMEOUT / 4
@@ -83,19 +86,20 @@ class RedisDocumentSet(RedisObjectHelper):
             # add to the set BEFORE creating the task.
             redis_client.sadd(self.taskset_key, custom_task_id)
 
-            result = celery_app.send_task(
+            celery_app.send_task(
                 OnyxCeleryTask.VESPA_METADATA_SYNC_TASK,
-                kwargs=dict(document_id=doc.id, tenant_id=tenant_id),
+                kwargs=dict(document_id=doc_id, tenant_id=tenant_id),
                 queue=OnyxCeleryQueues.VESPA_METADATA_SYNC,
                 task_id=custom_task_id,
                 priority=OnyxCeleryPriority.LOW,
             )
 
-            async_results.append(result)
+            num_tasks_sent += 1
 
-        return len(async_results), len(async_results)
+        return num_tasks_sent, num_tasks_sent
 
     def reset(self) -> None:
+        self.redis.srem(OnyxRedisConstants.ACTIVE_FENCES, self.fence_key)
         self.redis.delete(self.taskset_key)
         self.redis.delete(self.fence_key)
 

backend/onyx/redis/redis_pool.py

@@ -25,6 +25,7 @@ from onyx.configs.app_configs import REDIS_REPLICA_HOST
 from onyx.configs.app_configs import REDIS_SSL
 from onyx.configs.app_configs import REDIS_SSL_CA_CERTS
 from onyx.configs.app_configs import REDIS_SSL_CERT_REQS
+from onyx.configs.constants import FASTAPI_USERS_AUTH_COOKIE_NAME
 from onyx.configs.constants import REDIS_SOCKET_KEEPALIVE_OPTIONS
 from onyx.utils.logger import setup_logger
 
@@ -113,6 +114,8 @@ class TenantRedis(redis.Redis):
             "reacquire",
             "create_lock",
             "startswith",
+            "smembers",
+            "sismember",
             "sadd",
             "srem",
             "scard",
@@ -285,7 +288,7 @@ async def get_async_redis_connection() -> aioredis.Redis:
 
 
 async def retrieve_auth_token_data_from_redis(request: Request) -> dict | None:
-    token = request.cookies.get("fastapiusersauth")
+    token = request.cookies.get(FASTAPI_USERS_AUTH_COOKIE_NAME)
     if not token:
         logger.debug("No auth token cookie found")
         return None

backend/onyx/redis/redis_usergroup.py

@@ -13,7 +13,7 @@ from onyx.configs.constants import CELERY_VESPA_SYNC_BEAT_LOCK_TIMEOUT
 from onyx.configs.constants import OnyxCeleryPriority
 from onyx.configs.constants import OnyxCeleryQueues
 from onyx.configs.constants import OnyxCeleryTask
-from onyx.db.models import Document
+from onyx.configs.constants import OnyxRedisConstants
 from onyx.redis.redis_object_helper import RedisObjectHelper
 from onyx.utils.variable_functionality import fetch_versioned_implementation
 from onyx.utils.variable_functionality import global_version
@@ -36,10 +36,12 @@ class RedisUserGroup(RedisObjectHelper):
 
     def set_fence(self, payload: int | None) -> None:
         if payload is None:
+            self.redis.srem(OnyxRedisConstants.ACTIVE_FENCES, self.fence_key)
             self.redis.delete(self.fence_key)
             return
 
         self.redis.set(self.fence_key, payload)
+        self.redis.sadd(OnyxRedisConstants.ACTIVE_FENCES, self.fence_key)
 
     @property
     def payload(self) -> int | None:
@@ -63,23 +65,22 @@ class RedisUserGroup(RedisObjectHelper):
         user group up to date over multiple batches.
         """
         last_lock_time = time.monotonic()
-
-        async_results = []
+        num_tasks_sent = 0
 
         if not global_version.is_ee_version():
             return 0, 0
 
         try:
-            construct_document_select_by_usergroup = fetch_versioned_implementation(
+            construct_document_id_select_by_usergroup = fetch_versioned_implementation(
                 "onyx.db.user_group",
-                "construct_document_select_by_usergroup",
+                "construct_document_id_select_by_usergroup",
             )
         except ModuleNotFoundError:
             return 0, 0
 
-        stmt = construct_document_select_by_usergroup(int(self._id))
-        for doc in db_session.scalars(stmt).yield_per(DB_YIELD_PER_DEFAULT):
-            doc = cast(Document, doc)
+        stmt = construct_document_id_select_by_usergroup(int(self._id))
+        for doc_id in db_session.scalars(stmt).yield_per(DB_YIELD_PER_DEFAULT):
+            doc_id = cast(str, doc_id)
             current_time = time.monotonic()
             if current_time - last_lock_time >= (
                 CELERY_VESPA_SYNC_BEAT_LOCK_TIMEOUT / 4
@@ -96,19 +97,20 @@ class RedisUserGroup(RedisObjectHelper):
             # add to the set BEFORE creating the task.
             redis_client.sadd(self.taskset_key, custom_task_id)
 
-            result = celery_app.send_task(
+            celery_app.send_task(
                 OnyxCeleryTask.VESPA_METADATA_SYNC_TASK,
-                kwargs=dict(document_id=doc.id, tenant_id=tenant_id),
+                kwargs=dict(document_id=doc_id, tenant_id=tenant_id),
                 queue=OnyxCeleryQueues.VESPA_METADATA_SYNC,
                 task_id=custom_task_id,
                 priority=OnyxCeleryPriority.LOW,
             )
 
-            async_results.append(result)
+            num_tasks_sent += 1
 
-        return len(async_results), len(async_results)
+        return num_tasks_sent, num_tasks_sent
 
     def reset(self) -> None:
+        self.redis.srem(OnyxRedisConstants.ACTIVE_FENCES, self.fence_key)
         self.redis.delete(self.taskset_key)
         self.redis.delete(self.fence_key)
 

backend/onyx/seeding/prompts.yaml

@@ -20,7 +20,7 @@ prompts:
       The documents may not all be relevant, ignore any documents that are not directly relevant
       to the most recent user query.
 
-      I have not read or seen any of the documents and do not want to read them.
+      I have not read or seen any of the documents and do not want to read them. Do not refer to them by Document number.
 
       If there are no relevant documents, refer to the chat history and your internal knowledge.
     # Inject a statement at the end of system prompt to inform the LLM of the current date/time

backend/onyx/server/manage/models.py

@@ -215,6 +215,7 @@ class SlackChannelConfig(BaseModel):
     # XXX this is going away soon
     standard_answer_categories: list[StandardAnswerCategory]
     enable_auto_filters: bool
+    is_default: bool
 
     @classmethod
     def from_model(
@@ -237,6 +238,7 @@ class SlackChannelConfig(BaseModel):
                 for standard_answer_category_model in slack_channel_config_model.standard_answer_categories
             ],
             enable_auto_filters=slack_channel_config_model.enable_auto_filters,
+            is_default=slack_channel_config_model.is_default,
         )
 
 
@@ -279,3 +281,8 @@ class AllUsersResponse(BaseModel):
     accepted_pages: int
     invited_pages: int
     slack_users_pages: int
+
+
+class SlackChannel(BaseModel):
+    id: str
+    name: str

backend/onyx/server/manage/slack_bot.py

@@ -1,6 +1,10 @@
+from typing import Any
+
 from fastapi import APIRouter
 from fastapi import Depends
 from fastapi import HTTPException
+from slack_sdk import WebClient
+from slack_sdk.errors import SlackApiError
 from sqlalchemy.orm import Session
 
 from onyx.auth.users import current_admin_user
@@ -12,6 +16,7 @@ from onyx.db.models import ChannelConfig
 from onyx.db.models import User
 from onyx.db.persona import get_persona_by_id
 from onyx.db.slack_bot import fetch_slack_bot
+from onyx.db.slack_bot import fetch_slack_bot_tokens
 from onyx.db.slack_bot import fetch_slack_bots
 from onyx.db.slack_bot import insert_slack_bot
 from onyx.db.slack_bot import remove_slack_bot
@@ -25,6 +30,7 @@ from onyx.db.slack_channel_config import update_slack_channel_config
 from onyx.onyxbot.slack.config import validate_channel_name
 from onyx.server.manage.models import SlackBot
 from onyx.server.manage.models import SlackBotCreationRequest
+from onyx.server.manage.models import SlackChannel
 from onyx.server.manage.models import SlackChannelConfig
 from onyx.server.manage.models import SlackChannelConfigCreationRequest
 from onyx.server.manage.validate_tokens import validate_app_token
@@ -48,12 +54,6 @@ def _form_channel_config(
     answer_filters = slack_channel_config_creation_request.answer_filters
     follow_up_tags = slack_channel_config_creation_request.follow_up_tags
 
-    if not raw_channel_name:
-        raise HTTPException(
-            status_code=400,
-            detail="Must provide at least one channel name",
-        )
-
     try:
         cleaned_channel_name = validate_channel_name(
             db_session=db_session,
@@ -108,6 +108,12 @@ def create_slack_channel_config(
         current_slack_channel_config_id=None,
     )
 
+    if channel_config["channel_name"] is None:
+        raise HTTPException(
+            status_code=400,
+            detail="Channel name is required",
+        )
+
     persona_id = None
     if slack_channel_config_creation_request.persona_id is not None:
         persona_id = slack_channel_config_creation_request.persona_id
@@ -120,11 +126,11 @@ def create_slack_channel_config(
         ).id
 
     slack_channel_config_model = insert_slack_channel_config(
+        db_session=db_session,
         slack_bot_id=slack_channel_config_creation_request.slack_bot_id,
         persona_id=persona_id,
         channel_config=channel_config,
         standard_answer_category_ids=slack_channel_config_creation_request.standard_answer_categories,
-        db_session=db_session,
         enable_auto_filters=slack_channel_config_creation_request.enable_auto_filters,
     )
     return SlackChannelConfig.from_model(slack_channel_config_model)
@@ -235,6 +241,24 @@ def create_bot(
         app_token=slack_bot_creation_request.app_token,
     )
 
+    # Create a default Slack channel config
+    default_channel_config = ChannelConfig(
+        channel_name=None,
+        respond_member_group_list=[],
+        answer_filters=[],
+        follow_up_tags=[],
+        respond_tag_only=True,
+    )
+    insert_slack_channel_config(
+        db_session=db_session,
+        slack_bot_id=slack_bot_model.id,
+        persona_id=None,
+        channel_config=default_channel_config,
+        standard_answer_category_ids=[],
+        enable_auto_filters=False,
+        is_default=True,
+    )
+
     create_milestone_and_report(
         user=None,
         distinct_id=tenant_id or "N/A",
@@ -315,3 +339,48 @@ def list_bot_configs(
         SlackChannelConfig.from_model(slack_bot_config_model)
         for slack_bot_config_model in slack_bot_config_models
     ]
+
+
+@router.get(
+    "/admin/slack-app/bots/{bot_id}/channels",
+)
+def get_all_channels_from_slack_api(
+    bot_id: int,
+    db_session: Session = Depends(get_session),
+    _: User | None = Depends(current_admin_user),
+) -> list[SlackChannel]:
+    tokens = fetch_slack_bot_tokens(db_session, bot_id)
+    if not tokens or "bot_token" not in tokens:
+        raise HTTPException(
+            status_code=404, detail="Bot token not found for the given bot ID"
+        )
+
+    bot_token = tokens["bot_token"]
+    client = WebClient(token=bot_token)
+
+    try:
+        channels = []
+        cursor = None
+        while True:
+            response = client.conversations_list(
+                types="public_channel,private_channel",
+                exclude_archived=True,
+                limit=1000,
+                cursor=cursor,
+            )
+            for channel in response["channels"]:
+                channels.append(SlackChannel(id=channel["id"], name=channel["name"]))
+
+            response_metadata: dict[str, Any] = response.get("response_metadata", {})
+            if isinstance(response_metadata, dict):
+                cursor = response_metadata.get("next_cursor")
+                if not cursor:
+                    break
+            else:
+                break
+
+        return channels
+    except SlackApiError as e:
+        raise HTTPException(
+            status_code=500, detail=f"Error fetching channels from Slack API: {str(e)}"
+        )

backend/onyx/server/manage/users.py

@@ -38,6 +38,7 @@ from onyx.configs.app_configs import ENABLE_EMAIL_INVITES
 from onyx.configs.app_configs import SESSION_EXPIRE_TIME_SECONDS
 from onyx.configs.app_configs import VALID_EMAIL_DOMAINS
 from onyx.configs.constants import AuthType
+from onyx.configs.constants import FASTAPI_USERS_AUTH_COOKIE_NAME
 from onyx.db.api_key import is_api_key_email_address
 from onyx.db.auth import get_total_users_count
 from onyx.db.engine import CURRENT_TENANT_ID_CONTEXTVAR
@@ -171,13 +172,14 @@ def list_all_users(
     accepted_page: int | None = None,
     slack_users_page: int | None = None,
     invited_page: int | None = None,
+    include_api_keys: bool = False,
     _: User | None = Depends(current_curator_or_admin_user),
     db_session: Session = Depends(get_session),
 ) -> AllUsersResponse:
     users = [
         user
         for user in get_all_users(db_session, email_filter_string=q)
-        if not is_api_key_email_address(user.email)
+        if (include_api_keys or not is_api_key_email_address(user.email))
     ]
 
     slack_users = [user for user in users if user.role == UserRole.SLACK_USER]
@@ -271,6 +273,8 @@ def bulk_invite_users(
 
     tenant_id = CURRENT_TENANT_ID_CONTEXTVAR.get()
     new_invited_emails = []
+    email: str
+
     try:
         for email in emails:
             email_info = validate_email(email)
@@ -476,7 +480,7 @@ def get_current_token_expiration_jwt(
 
     try:
         # Get the JWT from the cookie
-        jwt_token = request.cookies.get("fastapiusersauth")
+        jwt_token = request.cookies.get(FASTAPI_USERS_AUTH_COOKIE_NAME)
         if not jwt_token:
             logger.error("No JWT token found in cookies")
             return None

backend/onyx/server/query_and_chat/chat_backend.py

@@ -420,10 +420,6 @@ def handle_new_chat_message(
 
     def stream_generator() -> Generator[str, None, None]:
         try:
-            import time
-
-            start_time = time.time()
-            n = 0
             for packet in stream_chat_message(
                 new_msg_req=chat_message_req,
                 user=user,
@@ -435,19 +431,6 @@ def handle_new_chat_message(
                 ),
                 is_connected=is_connected_func,
             ):
-                if "top_documents" in packet:
-                    to_first_docs = time.time() - start_time
-                    print(f"Time to first docs: {to_first_docs}")
-                    print(packet)
-                elif "answer_piece" in packet:
-                    to_answer_piece = time.time() - start_time
-                    if n == 1:
-                        print(f"Time to answer piece: {to_answer_piece}")
-                        print(packet)
-                    n += 1
-
-                # time_since_start = time.time() - start_time
-                # print(f"Time since start: {time_since_start}")
                 yield json.dumps(packet) if isinstance(packet, dict) else packet
 
         except Exception as e:
@@ -736,15 +719,12 @@ def upload_files_for_chat(
 
         file_content = file.file.read()  # Read the file content
 
-        if file_type == ChatFileType.IMAGE:
-            file_content_io = file.file
-            # NOTE: Image conversion to JPEG used to be enforced here.
-            # This was removed to:
-            # 1. Preserve original file content for downloads
-            # 2. Maintain transparency in formats like PNG
-            # 3. Ameliorate issue with file conversion
-        else:
-            file_content_io = io.BytesIO(file_content)
+        # NOTE: Image conversion to JPEG used to be enforced here.
+        # This was removed to:
+        # 1. Preserve original file content for downloads
+        # 2. Maintain transparency in formats like PNG
+        # 3. Ameliorate issue with file conversion
+        file_content_io = io.BytesIO(file_content)
 
         new_content_type = file.content_type
 
@@ -762,10 +742,11 @@ def upload_files_for_chat(
         # to re-extract it every time we send a message
         if file_type == ChatFileType.DOC:
             extracted_text = extract_file_text(
-                file=io.BytesIO(file_content),  # use the bytes we already read
+                file=file_content_io,  # use the bytes we already read
                 file_name=file.filename or "",
             )
             text_file_id = str(uuid.uuid4())
+
             file_store.save_file(
                 file_name=text_file_id,
                 content=io.BytesIO(extracted_text.encode()),

backend/onyx/tools/tool_implementations/internet_search/internet_search_tool.py

@@ -218,6 +218,9 @@ class InternetSearchTool(Tool):
             headers=self.headers,
             params={"q": query, "count": self.num_results},
         )
+
+        response.raise_for_status()
+
         results = response.json()
 
         # If no hits, Bing does not include the webPages key

backend/onyx/tools/tool_implementations/search/search_tool.py

@@ -7,7 +7,6 @@ from typing import cast
 from sqlalchemy.orm import Session
 
 from onyx.chat.chat_utils import llm_doc_from_inference_section
-from onyx.chat.llm_response_handler import LLMCall
 from onyx.chat.models import AnswerStyleConfig
 from onyx.chat.models import ContextualPruningConfig
 from onyx.chat.models import DocumentPruningConfig
@@ -371,41 +370,6 @@ class SearchTool(Tool):
             prompt_config=self.prompt_config,
         )
 
-    """Other utility functions"""
-
-    @classmethod
-    def get_search_result(
-        cls, llm_call: LLMCall
-    ) -> tuple[list[LlmDoc], list[LlmDoc]] | None:
-        """
-        Returns the final search results and a map of docs to their original search rank (which is what is displayed to user)
-        """
-        if not llm_call.tool_call_info:
-            return None
-
-        final_search_results = []
-        initial_search_results = []
-
-        for yield_item in llm_call.tool_call_info:
-            if (
-                isinstance(yield_item, ToolResponse)
-                and yield_item.id == FINAL_CONTEXT_DOCUMENTS_ID
-            ):
-                final_search_results = cast(list[LlmDoc], yield_item.response)
-            elif (
-                isinstance(yield_item, ToolResponse)
-                and yield_item.id == SEARCH_DOC_CONTENT_ID
-            ):
-                search_contexts = yield_item.response.contexts
-                # original_doc_search_rank = 1
-                for doc in search_contexts:
-                    if doc.document_id not in initial_search_results:
-                        initial_search_results.append(doc)
-
-                initial_search_results = cast(list[LlmDoc], initial_search_results)
-
-        return final_search_results, initial_search_results
-
 
 # Allows yielding the same responses as a SearchTool without being a SearchTool.
 # SearchTool passed in to allow for access to SearchTool properties.

backend/requirements/default.txt

@@ -37,7 +37,7 @@ langchainhub==0.1.21
 langgraph==0.2.59
 langgraph-checkpoint==2.0.5
 langgraph-sdk==0.1.44
-litellm==1.55.4
+litellm==1.60.2
 lxml==5.3.0
 lxml_html_clean==0.2.2
 llama-index==0.9.45
@@ -46,7 +46,7 @@ msal==1.28.0
 nltk==3.8.1
 Office365-REST-Python-Client==2.5.9
 oauthlib==3.2.2
-openai==1.55.3
+openai==1.61.0
 openpyxl==3.1.2
 playwright==1.41.2
 psutil==5.9.5

backend/requirements/model_server.txt

@@ -3,7 +3,7 @@ cohere==5.6.1
 fastapi==0.109.2
 google-cloud-aiplatform==1.58.0
 numpy==1.26.4
-openai==1.55.3
+openai==1.61.0
 pydantic==2.8.2
 retry==0.9.2
 safetensors==0.4.2
@@ -12,5 +12,5 @@ torch==2.2.0
 transformers==4.39.2
 uvicorn==0.21.1
 voyageai==0.2.3
-litellm==1.55.4
+litellm==1.60.2
 sentry-sdk[fastapi,celery,starlette]==2.14.0

backend/scripts/chat_feedback_dump.py

@@ -198,6 +198,7 @@ def process_all_chat_feedback(onyx_url: str, api_key: str | None) -> None:
         r_sessions = get_chat_sessions(onyx_url, headers, user_id)
         logger.info(f"user={user_id} num_sessions={len(r_sessions.sessions)}")
         for session in r_sessions.sessions:
+            s: ChatSessionSnapshot
             try:
                 s = get_session_history(onyx_url, headers, session.id)
             except requests.exceptions.HTTPError:

backend/scripts/debugging/onyx_redis.py

@@ -10,6 +10,8 @@ from uuid import UUID
 from redis import Redis
 
 from ee.onyx.server.tenants.user_mapping import get_tenant_id_for_email
+from onyx.auth.invited_users import get_invited_users
+from onyx.auth.invited_users import write_invited_users
 from onyx.configs.app_configs import REDIS_AUTH_KEY_PREFIX
 from onyx.configs.app_configs import REDIS_DB_NUMBER
 from onyx.configs.app_configs import REDIS_HOST
@@ -21,6 +23,7 @@ from onyx.db.users import get_user_by_email
 from onyx.redis.redis_pool import RedisPool
 from shared_configs.configs import MULTI_TENANT
 from shared_configs.configs import POSTGRES_DEFAULT_SCHEMA
+from shared_configs.contextvars import CURRENT_TENANT_ID_CONTEXTVAR
 
 # Tool to run helpful operations on Redis in production
 # This is targeted for internal usage and may not have all the necessary parameters
@@ -310,6 +313,13 @@ if __name__ == "__main__":
         required=False,
     )
 
+    parser.add_argument(
+        "--tenant-id",
+        type=str,
+        help="Tenant ID for get, delete user token, or add to invited users",
+        required=False,
+    )
+
     parser.add_argument(
         "--batch",
         type=int,
@@ -328,11 +338,32 @@ if __name__ == "__main__":
     parser.add_argument(
         "--user-email",
         type=str,
-        help="User email for get or delete user token",
+        help="User email for get, delete user token, or add to invited users",
         required=False,
     )
 
     args = parser.parse_args()
+
+    if args.tenant_id:
+        CURRENT_TENANT_ID_CONTEXTVAR.set(args.tenant_id)
+
+    if args.command == "add_invited_user":
+        if not args.user_email:
+            print("Error: --user-email is required for add_invited_user command")
+            sys.exit(1)
+
+        current_invited_users = get_invited_users()
+        if args.user_email not in current_invited_users:
+            current_invited_users.append(args.user_email)
+            if args.dry_run:
+                print(f"(DRY-RUN) Would add {args.user_email} to invited users")
+            else:
+                write_invited_users(current_invited_users)
+                print(f"Added {args.user_email} to invited users")
+        else:
+            print(f"{args.user_email} is already in the invited users list")
+        sys.exit(0)
+
     exitcode = onyx_redis(
         command=args.command,
         batch=args.batch,

backend/scripts/debugging/onyx_vespa.py

@@ -255,6 +255,24 @@ def get_documents_for_tenant_connector(
     print_documents(documents)
 
 
+def search_for_document(
+    index_name: str, document_id: str, max_hits: int | None = 10
+) -> List[Dict[str, Any]]:
+    yql_query = (
+        f'select * from sources {index_name} where document_id contains "{document_id}"'
+    )
+    params: dict[str, Any] = {"yql": yql_query}
+    if max_hits is not None:
+        params["hits"] = max_hits
+    with get_vespa_http_client() as client:
+        response = client.get(f"{SEARCH_ENDPOINT}/search/", params=params)
+        response.raise_for_status()
+        result = response.json()
+        documents = result.get("root", {}).get("children", [])
+        logger.info(f"Found {len(documents)} documents from query.")
+        return documents
+
+
 def search_documents(
     tenant_id: str, connector_id: int, query: str, n: int = 10
 ) -> None:
@@ -440,10 +458,98 @@ def get_document_acls(
             print("-" * 80)
 
 
+def get_current_chunk_count(
+    document_id: str, index_name: str, tenant_id: str
+) -> int | None:
+    with get_session_with_tenant(tenant_id=tenant_id) as session:
+        return (
+            session.query(Document.chunk_count)
+            .filter(Document.id == document_id)
+            .scalar()
+        )
+
+
+def get_number_of_chunks_we_think_exist(
+    document_id: str, index_name: str, tenant_id: str
+) -> int:
+    current_chunk_count = get_current_chunk_count(document_id, index_name, tenant_id)
+    print(f"Current chunk count: {current_chunk_count}")
+
+    doc_info = VespaIndex.enrich_basic_chunk_info(
+        index_name=index_name,
+        http_client=get_vespa_http_client(),
+        document_id=document_id,
+        previous_chunk_count=current_chunk_count,
+        new_chunk_count=0,
+    )
+
+    chunk_ids = get_document_chunk_ids(
+        enriched_document_info_list=[doc_info],
+        tenant_id=tenant_id,
+        large_chunks_enabled=False,
+    )
+    return len(chunk_ids)
+
+
 class VespaDebugging:
     # Class for managing Vespa debugging actions.
     def __init__(self, tenant_id: str | None = None):
         self.tenant_id = POSTGRES_DEFAULT_SCHEMA if not tenant_id else tenant_id
+        self.index_name = get_index_name(self.tenant_id)
+
+    def sample_document_counts(self) -> None:
+        # Sample random documents and compare chunk counts
+        mismatches = []
+        no_chunks = []
+        with get_session_with_tenant(tenant_id=self.tenant_id) as session:
+            # Get a sample of random documents
+            from sqlalchemy import func
+
+            sample_docs = (
+                session.query(Document.id, Document.link, Document.semantic_id)
+                .order_by(func.random())
+                .limit(1000)
+                .all()
+            )
+
+            for doc in sample_docs:
+                document_id, link, semantic_id = doc
+                (
+                    number_of_chunks_in_vespa,
+                    number_of_chunks_we_think_exist,
+                ) = self.compare_chunk_count(document_id)
+                if number_of_chunks_in_vespa != number_of_chunks_we_think_exist:
+                    mismatches.append(
+                        (
+                            document_id,
+                            link,
+                            semantic_id,
+                            number_of_chunks_in_vespa,
+                            number_of_chunks_we_think_exist,
+                        )
+                    )
+                elif number_of_chunks_in_vespa == 0:
+                    no_chunks.append((document_id, link, semantic_id))
+
+        # Print results
+        print("\nDocuments with mismatched chunk counts:")
+        for doc_id, link, semantic_id, vespa_count, expected_count in mismatches:
+            print(f"Document ID: {doc_id}")
+            print(f"Link: {link}")
+            print(f"Semantic ID: {semantic_id}")
+            print(f"Chunks in Vespa: {vespa_count}")
+            print(f"Expected chunks: {expected_count}")
+            print("-" * 80)
+
+        print("\nDocuments with no chunks in Vespa:")
+        for doc_id, link, semantic_id in no_chunks:
+            print(f"Document ID: {doc_id}")
+            print(f"Link: {link}")
+            print(f"Semantic ID: {semantic_id}")
+            print("-" * 80)
+
+        print(f"\nTotal mismatches: {len(mismatches)}")
+        print(f"Total documents with no chunks: {len(no_chunks)}")
 
     def print_config(self) -> None:
         # Print Vespa config.
@@ -457,6 +563,16 @@ class VespaDebugging:
         # List documents for a tenant.
         list_documents(n, self.tenant_id)
 
+    def compare_chunk_count(self, document_id: str) -> tuple[int, int]:
+        docs = search_for_document(self.index_name, document_id, max_hits=None)
+        number_of_chunks_we_think_exist = get_number_of_chunks_we_think_exist(
+            document_id, self.index_name, self.tenant_id
+        )
+        print(
+            f"Number of chunks in Vespa: {len(docs)}, Number of chunks we think exist: {number_of_chunks_we_think_exist}"
+        )
+        return len(docs), number_of_chunks_we_think_exist
+
     def search_documents(self, connector_id: int, query: str, n: int = 10) -> None:
         # Search documents for a tenant and connector.
         search_documents(self.tenant_id, connector_id, query, n)
@@ -464,9 +580,11 @@ class VespaDebugging:
     def update_document(
         self, connector_id: int, doc_id: str, fields: Dict[str, Any]
     ) -> None:
-        # Update a document.
         update_document(self.tenant_id, connector_id, doc_id, fields)
 
+    def search_for_document(self, document_id: str) -> List[Dict[str, Any]]:
+        return search_for_document(self.index_name, document_id)
+
     def delete_document(self, connector_id: int, doc_id: str) -> None:
         # Delete a document.
         delete_document(self.tenant_id, connector_id, doc_id)
@@ -483,7 +601,6 @@ class VespaDebugging:
 
 
 def main() -> None:
-    # Main CLI entry point.
     parser = argparse.ArgumentParser(description="Vespa debugging tool")
     parser.add_argument(
         "--action",

backend/scripts/sources_selection_analysis.py

@@ -11,6 +11,8 @@ from typing import Optional
 
 import requests
 
+from onyx.configs.constants import FASTAPI_USERS_AUTH_COOKIE_NAME
+
 parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 sys.path.append(parent_dir)
 
@@ -374,7 +376,11 @@ class SelectionAnalysis:
         Returns:
             dict: The Onyx API response content
         """
-        cookies = {"fastapiusersauth": self._auth_cookie} if self._auth_cookie else {}
+        cookies = (
+            {FASTAPI_USERS_AUTH_COOKIE_NAME: self._auth_cookie}
+            if self._auth_cookie
+            else {}
+        )
 
         endpoint = f"http://127.0.0.1:{self._web_port}/api/direct-qa"
         query_json = {

backend/tests/daily/connectors/confluence/test_confluence_basic.py

@@ -6,6 +6,7 @@ from unittest.mock import patch
 import pytest
 
 from onyx.connectors.confluence.connector import ConfluenceConnector
+from onyx.connectors.models import Document
 
 
 @pytest.fixture
@@ -41,6 +42,10 @@ def test_confluence_connector_basic(
 
     assert len(doc_batch) == 3
 
+    page_within_a_page_doc: Document | None = None
+    page_doc: Document | None = None
+    txt_doc: Document | None = None
+
     for doc in doc_batch:
         if doc.semantic_identifier == "DailyConnectorTestSpace Home":
             page_doc = doc
@@ -49,6 +54,7 @@ def test_confluence_connector_basic(
         elif doc.semantic_identifier == "Page Within A Page":
             page_within_a_page_doc = doc
 
+    assert page_within_a_page_doc is not None
     assert page_within_a_page_doc.semantic_identifier == "Page Within A Page"
     assert page_within_a_page_doc.primary_owners
     assert page_within_a_page_doc.primary_owners[0].email == "hagen@danswer.ai"
@@ -62,6 +68,7 @@ def test_confluence_connector_basic(
         == "https://danswerai.atlassian.net/wiki/spaces/DailyConne/pages/200769540/Page+Within+A+Page"
     )
 
+    assert page_doc is not None
     assert page_doc.semantic_identifier == "DailyConnectorTestSpace Home"
     assert page_doc.metadata["labels"] == ["testlabel"]
     assert page_doc.primary_owners
@@ -75,6 +82,7 @@ def test_confluence_connector_basic(
         == "https://danswerai.atlassian.net/wiki/spaces/DailyConne/overview"
     )
 
+    assert txt_doc is not None
     assert txt_doc.semantic_identifier == "small-file.txt"
     assert len(txt_doc.sections) == 1
     assert txt_doc.sections[0].text == "small"

backend/tests/daily/connectors/gmail/test_gmail_connector.py

@@ -110,6 +110,8 @@ def test_docs_retrieval(
 
     for doc in retrieved_docs:
         id = doc.id
+        retrieved_primary_owner_emails: set[str | None] = set()
+        retrieved_secondary_owner_emails: set[str | None] = set()
         if doc.primary_owners:
             retrieved_primary_owner_emails = set(
                 [owner.email for owner in doc.primary_owners]

backend/tests/integration/Dockerfile

@@ -71,6 +71,7 @@ COPY supervisord.conf /etc/supervisor/conf.d/supervisord.conf
 COPY ./onyx /app/onyx
 COPY ./shared_configs /app/shared_configs
 COPY ./alembic /app/alembic
+COPY ./alembic_tenants /app/alembic_tenants
 COPY ./alembic.ini /app/alembic.ini
 COPY ./pytest.ini /app/pytest.ini
 COPY supervisord.conf /usr/etc/supervisord.conf

backend/tests/integration/common_utils/managers/user.py

@@ -7,6 +7,7 @@ import requests
 from requests import HTTPError
 
 from onyx.auth.schemas import UserRole
+from onyx.configs.constants import FASTAPI_USERS_AUTH_COOKIE_NAME
 from onyx.server.documents.models import PaginatedReturn
 from onyx.server.models import FullUserSnapshot
 from tests.integration.common_utils.constants import API_SERVER_URL
@@ -82,7 +83,7 @@ class UserManager:
         response.raise_for_status()
 
         cookies = response.cookies.get_dict()
-        session_cookie = cookies.get("fastapiusersauth")
+        session_cookie = cookies.get(FASTAPI_USERS_AUTH_COOKIE_NAME)
 
         if not session_cookie:
             raise Exception("Failed to login")
@@ -165,6 +166,7 @@ class UserManager:
         target_status: bool,
         user_performing_action: DATestUser,
     ) -> DATestUser:
+        url_substring: str
         if target_status is True:
             url_substring = "activate"
         elif target_status is False:

backend/tests/integration/connector_job_tests/google/test_google_drive_permission_sync.py

@@ -54,6 +54,7 @@ def google_drive_test_env_setup() -> (
 
     service_account_key = os.environ["FULL_CONTROL_DRIVE_SERVICE_ACCOUNT"]
     drive_id: str | None = None
+    drive_service: GoogleDriveService | None = None
 
     try:
         credentials = {

deployment/helm/charts/onyx/Chart.lock

@@ -4,12 +4,12 @@ dependencies:
   version: 14.3.1
 - name: vespa
   repository: https://onyx-dot-app.github.io/vespa-helm-charts
-  version: 0.2.18
+  version: 0.2.20
 - name: nginx
   repository: oci://registry-1.docker.io/bitnamicharts
   version: 15.14.0
 - name: redis
   repository: https://charts.bitnami.com/bitnami
   version: 20.1.0
-digest: sha256:5c9eb3d55d5f8e3beb64f26d26f686c8d62755daa10e2e6d87530bdf2fbbf957
-generated: "2024-12-10T10:47:35.812483-08:00"
+digest: sha256:4615c033064a987e3f66a48f4744d2e88bd1cc932c79453c4928455695a72778
+generated: "2025-02-04T11:45:05.39228-08:00"

deployment/helm/charts/onyx/Chart.yaml

@@ -23,7 +23,7 @@ dependencies:
     repository: https://charts.bitnami.com/bitnami
     condition: postgresql.enabled
   - name: vespa
-    version: 0.2.18
+    version: 0.2.20
     repository: https://onyx-dot-app.github.io/vespa-helm-charts
     condition: vespa.enabled
   - name: nginx

deployment/helm/charts/onyx/templates/configmap.yaml

@@ -7,10 +7,10 @@ metadata:
 data:
   INTERNAL_URL: "http://{{ include "onyx-stack.fullname" . }}-api-service:{{ .Values.api.service.port | default 8080 }}"
   POSTGRES_HOST: {{ .Release.Name }}-postgresql
-  VESPA_HOST: da-vespa-0.vespa-service
+  VESPA_HOST: {{ .Values.vespa.name }}.{{ .Values.vespa.service.name }}.{{ .Release.Namespace }}.svc.cluster.local
   REDIS_HOST: {{ .Release.Name }}-redis-master
   MODEL_SERVER_HOST: "{{ include "onyx-stack.fullname" . }}-inference-model-service"
   INDEXING_MODEL_SERVER_HOST: "{{ include "onyx-stack.fullname" . }}-indexing-model-service"
 {{- range $key, $value := .Values.configMap }}
   {{ $key }}: "{{ $value }}"
-{{- end }}
+{{- end }}