fix

* Confluence fixes

* Small tweak

* Address greptile comments

backend/alembic/versions/3934b1bc7b62_update_github_connector_repo_name_to_.py

@@ -0,0 +1,125 @@
+"""Update GitHub connector repo_name to repositories
+
+Revision ID: 3934b1bc7b62
+Revises: b7c2b63c4a03
+Create Date: 2025-03-05 10:50:30.516962
+
+"""
+from alembic import op
+import sqlalchemy as sa
+import json
+import logging
+
+# revision identifiers, used by Alembic.
+revision = "3934b1bc7b62"
+down_revision = "b7c2b63c4a03"
+branch_labels = None
+depends_on = None
+
+logger = logging.getLogger("alembic.runtime.migration")
+
+
+def upgrade() -> None:
+    # Get all GitHub connectors
+    conn = op.get_bind()
+
+    # First get all GitHub connectors
+    github_connectors = conn.execute(
+        sa.text(
+            """
+            SELECT id, connector_specific_config
+            FROM connector
+            WHERE source = 'GITHUB'
+            """
+        )
+    ).fetchall()
+
+    # Update each connector's config
+    updated_count = 0
+    for connector_id, config in github_connectors:
+        try:
+            if not config:
+                logger.warning(f"Connector {connector_id} has no config, skipping")
+                continue
+
+            # Parse the config if it's a string
+            if isinstance(config, str):
+                config = json.loads(config)
+
+            if "repo_name" not in config:
+                continue
+
+            # Create new config with repositories instead of repo_name
+            new_config = dict(config)
+            repo_name_value = new_config.pop("repo_name")
+            new_config["repositories"] = repo_name_value
+
+            # Update the connector with the new config
+            conn.execute(
+                sa.text(
+                    """
+                    UPDATE connector
+                    SET connector_specific_config = :new_config
+                    WHERE id = :connector_id
+                    """
+                ),
+                {"connector_id": connector_id, "new_config": json.dumps(new_config)},
+            )
+            updated_count += 1
+        except Exception as e:
+            logger.error(f"Error updating connector {connector_id}: {str(e)}")
+
+
+def downgrade() -> None:
+    # Get all GitHub connectors
+    conn = op.get_bind()
+
+    logger.debug(
+        "Starting rollback of GitHub connectors from repositories to repo_name"
+    )
+
+    github_connectors = conn.execute(
+        sa.text(
+            """
+            SELECT id, connector_specific_config
+            FROM connector
+            WHERE source = 'GITHUB'
+            """
+        )
+    ).fetchall()
+
+    logger.debug(f"Found {len(github_connectors)} GitHub connectors to rollback")
+
+    # Revert each GitHub connector to use repo_name instead of repositories
+    reverted_count = 0
+    for connector_id, config in github_connectors:
+        try:
+            if not config:
+                continue
+
+            # Parse the config if it's a string
+            if isinstance(config, str):
+                config = json.loads(config)
+
+            if "repositories" not in config:
+                continue
+
+            # Create new config with repo_name instead of repositories
+            new_config = dict(config)
+            repositories_value = new_config.pop("repositories")
+            new_config["repo_name"] = repositories_value
+
+            # Update the connector with the new config
+            conn.execute(
+                sa.text(
+                    """
+                    UPDATE connector
+                    SET connector_specific_config = :new_config
+                    WHERE id = :connector_id
+                    """
+                ),
+                {"new_config": json.dumps(new_config), "connector_id": connector_id},
+            )
+            reverted_count += 1
+        except Exception as e:
+            logger.error(f"Error reverting connector {connector_id}: {str(e)}")

backend/ee/onyx/background/celery/apps/primary.py

@@ -4,7 +4,8 @@ from ee.onyx.server.reporting.usage_export_generation import create_new_usage_re
 from onyx.background.celery.apps.primary import celery_app
 from onyx.background.task_utils import build_celery_task_wrapper
 from onyx.configs.app_configs import JOB_TIMEOUT
-from onyx.db.chat import delete_chat_sessions_older_than
+from onyx.db.chat import delete_chat_session
+from onyx.db.chat import get_chat_sessions_older_than
 from onyx.db.engine import get_session_with_current_tenant
 from onyx.server.settings.store import load_settings
 from onyx.utils.logger import setup_logger
@@ -18,7 +19,26 @@ logger = setup_logger()
 @celery_app.task(soft_time_limit=JOB_TIMEOUT)
 def perform_ttl_management_task(retention_limit_days: int, *, tenant_id: str) -> None:
     with get_session_with_current_tenant() as db_session:
-        delete_chat_sessions_older_than(retention_limit_days, db_session)
+        old_chat_sessions = get_chat_sessions_older_than(
+            retention_limit_days, db_session
+        )
+
+    for user_id, session_id in old_chat_sessions:
+        # one session per delete so that we don't blow up if a deletion fails.
+        with get_session_with_current_tenant() as db_session:
+            try:
+                delete_chat_session(
+                    user_id,
+                    session_id,
+                    db_session,
+                    include_deleted=True,
+                    hard_delete=True,
+                )
+            except Exception:
+                logger.exception(
+                    "delete_chat_session exceptioned. "
+                    f"user_id={user_id} session_id={session_id}"
+                )
 
 
 #####

backend/ee/onyx/db/query_history.py

@@ -134,7 +134,9 @@ def fetch_chat_sessions_eagerly_by_time(
     limit: int | None = 500,
     initial_time: datetime | None = None,
 ) -> list[ChatSession]:
-    time_order: UnaryExpression = desc(ChatSession.time_created)
+    """Sorted by oldest to newest, then by message id"""
+
+    asc_time_order: UnaryExpression = asc(ChatSession.time_created)
     message_order: UnaryExpression = asc(ChatMessage.id)
 
     filters: list[ColumnElement | BinaryExpression] = [
@@ -147,8 +149,7 @@ def fetch_chat_sessions_eagerly_by_time(
     subquery = (
         db_session.query(ChatSession.id, ChatSession.time_created)
         .filter(*filters)
-        .order_by(ChatSession.id, time_order)
-        .distinct(ChatSession.id)
+        .order_by(asc_time_order)
         .limit(limit)
         .subquery()
     )
@@ -164,7 +165,7 @@ def fetch_chat_sessions_eagerly_by_time(
                 ChatMessage.chat_message_feedbacks
             ),
         )
-        .order_by(time_order, message_order)
+        .order_by(asc_time_order, message_order)
     )
 
     chat_sessions = query.all()

backend/ee/onyx/db/usage_export.py

@@ -16,13 +16,18 @@ from onyx.db.models import UsageReport
 from onyx.file_store.file_store import get_default_file_store
 
 
-# Gets skeletons of all message
+# Gets skeletons of all messages in the given range
 def get_empty_chat_messages_entries__paginated(
     db_session: Session,
     period: tuple[datetime, datetime],
     limit: int | None = 500,
     initial_time: datetime | None = None,
 ) -> tuple[Optional[datetime], list[ChatMessageSkeleton]]:
+    """Returns a tuple where:
+    first element is the most recent timestamp out of the sessions iterated
+    - this timestamp can be used to paginate forward in time
+    second element is a list of messages belonging to all the sessions iterated
+    """
     chat_sessions = fetch_chat_sessions_eagerly_by_time(
         start=period[0],
         end=period[1],
@@ -52,18 +57,17 @@ def get_empty_chat_messages_entries__paginated(
     if len(chat_sessions) == 0:
         return None, []
 
-    return chat_sessions[0].time_created, message_skeletons
+    return chat_sessions[-1].time_created, message_skeletons
 
 
 def get_all_empty_chat_message_entries(
     db_session: Session,
     period: tuple[datetime, datetime],
 ) -> Generator[list[ChatMessageSkeleton], None, None]:
+    """period is the range of time over which to fetch messages."""
     initial_time: Optional[datetime] = period[0]
-    ind = 0
     while True:
-        ind += 1
-
+        # iterate from oldest to newest
         time_created, message_skeletons = get_empty_chat_messages_entries__paginated(
             db_session,
             period,

backend/ee/onyx/main.py

@@ -15,7 +15,7 @@ from ee.onyx.server.enterprise_settings.api import (
 )
 from ee.onyx.server.manage.standard_answer import router as standard_answer_router
 from ee.onyx.server.middleware.tenant_tracking import add_tenant_id_middleware
-from ee.onyx.server.oauth.api import router as oauth_router
+from ee.onyx.server.oauth.api import router as ee_oauth_router
 from ee.onyx.server.query_and_chat.chat_backend import (
     router as chat_router,
 )
@@ -128,7 +128,7 @@ def get_application() -> FastAPI:
     include_router_with_global_prefix_prepended(application, query_router)
     include_router_with_global_prefix_prepended(application, chat_router)
     include_router_with_global_prefix_prepended(application, standard_answer_router)
-    include_router_with_global_prefix_prepended(application, oauth_router)
+    include_router_with_global_prefix_prepended(application, ee_oauth_router)
 
     # Enterprise-only global settings
     include_router_with_global_prefix_prepended(

backend/ee/onyx/onyxbot/slack/handlers/handle_standard_answers.py

@@ -22,7 +22,7 @@ from onyx.onyxbot.slack.blocks import get_restate_blocks
 from onyx.onyxbot.slack.constants import GENERATE_ANSWER_BUTTON_ACTION_ID
 from onyx.onyxbot.slack.handlers.utils import send_team_member_message
 from onyx.onyxbot.slack.models import SlackMessageInfo
-from onyx.onyxbot.slack.utils import respond_in_thread
+from onyx.onyxbot.slack.utils import respond_in_thread_or_channel
 from onyx.onyxbot.slack.utils import update_emote_react
 from onyx.utils.logger import OnyxLoggingAdapter
 from onyx.utils.logger import setup_logger
@@ -216,7 +216,7 @@ def _handle_standard_answers(
         all_blocks = restate_question_blocks + answer_blocks
 
         try:
-            respond_in_thread(
+            respond_in_thread_or_channel(
                 client=client,
                 channel=message_info.channel_to_respond,
                 receiver_ids=receiver_ids,
@@ -231,6 +231,7 @@ def _handle_standard_answers(
                     client=client,
                     channel=message_info.channel_to_respond,
                     thread_ts=slack_thread_id,
+                    receiver_ids=receiver_ids,
                 )
 
             return True

backend/ee/onyx/server/oauth/confluence_cloud.py

@@ -80,6 +80,7 @@ class ConfluenceCloudOAuth:
         "search:confluence%20"
         # granular scope
         "read:attachment:confluence%20"  # possibly unneeded unless calling v2 attachments api
+        "read:content-details:confluence%20"  # for permission sync
         "offline_access"
     )
 

backend/ee/onyx/server/tenants/product_gating.py

@@ -48,4 +48,5 @@ def store_product_gating(tenant_id: str, application_status: ApplicationStatus)
 
 def get_gated_tenants() -> set[str]:
     redis_client = get_redis_replica_client(tenant_id=ONYX_CLOUD_TENANT_ID)
-    return cast(set[str], redis_client.smembers(GATED_TENANTS_KEY))
+    gated_tenants_bytes = cast(set[bytes], redis_client.smembers(GATED_TENANTS_KEY))
+    return {tenant_id.decode("utf-8") for tenant_id in gated_tenants_bytes}

backend/ee/onyx/server/tenants/provisioning.py

@@ -55,7 +55,11 @@ logger = logging.getLogger(__name__)
 async def get_or_provision_tenant(
     email: str, referral_source: str | None = None, request: Request | None = None
 ) -> str:
-    """Get existing tenant ID for an email or create a new tenant if none exists."""
+    """
+    Get existing tenant ID for an email or create a new tenant if none exists.
+    This function should only be called after we have verified we want this user's tenant to exist.
+    It returns the tenant ID associated with the email, creating a new tenant if necessary.
+    """
     if not MULTI_TENANT:
         return POSTGRES_DEFAULT_SCHEMA
 
@@ -104,14 +108,14 @@ async def provision_tenant(tenant_id: str, email: str) -> None:
             status_code=409, detail="User already belongs to an organization"
         )
 
-    logger.info(f"Provisioning tenant: {tenant_id}")
+    logger.debug(f"Provisioning tenant {tenant_id} for user {email}")
     token = None
 
     try:
         if not create_schema_if_not_exists(tenant_id):
-            logger.info(f"Created schema for tenant {tenant_id}")
+            logger.debug(f"Created schema for tenant {tenant_id}")
         else:
-            logger.info(f"Schema already exists for tenant {tenant_id}")
+            logger.debug(f"Schema already exists for tenant {tenant_id}")
 
         token = CURRENT_TENANT_ID_CONTEXTVAR.set(tenant_id)
 

backend/model_server/constants.py

@@ -6,7 +6,7 @@ MODEL_WARM_UP_STRING = "hi " * 512
 DEFAULT_OPENAI_MODEL = "text-embedding-3-small"
 DEFAULT_COHERE_MODEL = "embed-english-light-v3.0"
 DEFAULT_VOYAGE_MODEL = "voyage-large-2-instruct"
-DEFAULT_VERTEX_MODEL = "text-embedding-004"
+DEFAULT_VERTEX_MODEL = "text-embedding-005"
 
 
 class EmbeddingModelTextType:

backend/model_server/encoders.py

@@ -5,6 +5,7 @@ from types import TracebackType
 from typing import cast
 from typing import Optional
 
+import aioboto3  # type: ignore
 import httpx
 import openai
 import vertexai  # type: ignore
@@ -28,11 +29,13 @@ from model_server.constants import DEFAULT_VERTEX_MODEL
 from model_server.constants import DEFAULT_VOYAGE_MODEL
 from model_server.constants import EmbeddingModelTextType
 from model_server.constants import EmbeddingProvider
+from model_server.utils import pass_aws_key
 from model_server.utils import simple_log_function_time
 from onyx.utils.logger import setup_logger
 from shared_configs.configs import API_BASED_EMBEDDING_TIMEOUT
 from shared_configs.configs import INDEXING_ONLY
 from shared_configs.configs import OPENAI_EMBEDDING_TIMEOUT
+from shared_configs.configs import VERTEXAI_EMBEDDING_LOCAL_BATCH_SIZE
 from shared_configs.enums import EmbedTextType
 from shared_configs.enums import RerankerProvider
 from shared_configs.model_server_models import Embedding
@@ -182,17 +185,24 @@ class CloudEmbedding:
         vertexai.init(project=project_id, credentials=credentials)
         client = TextEmbeddingModel.from_pretrained(model)
 
-        embeddings = await client.get_embeddings_async(
-            [
-                TextEmbeddingInput(
-                    text,
-                    embedding_type,
-                )
-                for text in texts
-            ],
-            auto_truncate=True,  # This is the default
-        )
-        return [embedding.values for embedding in embeddings]
+        inputs = [TextEmbeddingInput(text, embedding_type) for text in texts]
+
+        # Split into batches of 25 texts
+        max_texts_per_batch = VERTEXAI_EMBEDDING_LOCAL_BATCH_SIZE
+        batches = [
+            inputs[i : i + max_texts_per_batch]
+            for i in range(0, len(inputs), max_texts_per_batch)
+        ]
+
+        # Dispatch all embedding calls asynchronously at once
+        tasks = [
+            client.get_embeddings_async(batch, auto_truncate=True) for batch in batches
+        ]
+
+        # Wait for all tasks to complete in parallel
+        results = await asyncio.gather(*tasks)
+
+        return [embedding.values for batch in results for embedding in batch]
 
     async def _embed_litellm_proxy(
         self, texts: list[str], model_name: str | None
@@ -447,7 +457,7 @@ async def local_rerank(query: str, docs: list[str], model_name: str) -> list[flo
     )
 
 
-async def cohere_rerank(
+async def cohere_rerank_api(
     query: str, docs: list[str], model_name: str, api_key: str
 ) -> list[float]:
     cohere_client = CohereAsyncClient(api_key=api_key)
@@ -457,6 +467,45 @@ async def cohere_rerank(
     return [result.relevance_score for result in sorted_results]
 
 
+async def cohere_rerank_aws(
+    query: str,
+    docs: list[str],
+    model_name: str,
+    region_name: str,
+    aws_access_key_id: str,
+    aws_secret_access_key: str,
+) -> list[float]:
+    session = aioboto3.Session(
+        aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key
+    )
+    async with session.client(
+        "bedrock-runtime", region_name=region_name
+    ) as bedrock_client:
+        body = json.dumps(
+            {
+                "query": query,
+                "documents": docs,
+                "api_version": 2,
+            }
+        )
+        # Invoke the Bedrock model asynchronously
+        response = await bedrock_client.invoke_model(
+            modelId=model_name,
+            accept="application/json",
+            contentType="application/json",
+            body=body,
+        )
+
+        # Read the response asynchronously
+        response_body = json.loads(await response["body"].read())
+
+        # Extract and sort the results
+        results = response_body.get("results", [])
+        sorted_results = sorted(results, key=lambda item: item["index"])
+
+        return [result["relevance_score"] for result in sorted_results]
+
+
 async def litellm_rerank(
     query: str, docs: list[str], api_url: str, model_name: str, api_key: str | None
 ) -> list[float]:
@@ -572,15 +621,32 @@ async def process_rerank_request(rerank_request: RerankRequest) -> RerankRespons
         elif rerank_request.provider_type == RerankerProvider.COHERE:
             if rerank_request.api_key is None:
                 raise RuntimeError("Cohere Rerank Requires an API Key")
-            sim_scores = await cohere_rerank(
+            sim_scores = await cohere_rerank_api(
                 query=rerank_request.query,
                 docs=rerank_request.documents,
                 model_name=rerank_request.model_name,
                 api_key=rerank_request.api_key,
             )
             return RerankResponse(scores=sim_scores)
+
+        elif rerank_request.provider_type == RerankerProvider.BEDROCK:
+            if rerank_request.api_key is None:
+                raise RuntimeError("Bedrock Rerank Requires an API Key")
+            aws_access_key_id, aws_secret_access_key, aws_region = pass_aws_key(
+                rerank_request.api_key
+            )
+            sim_scores = await cohere_rerank_aws(
+                query=rerank_request.query,
+                docs=rerank_request.documents,
+                model_name=rerank_request.model_name,
+                region_name=aws_region,
+                aws_access_key_id=aws_access_key_id,
+                aws_secret_access_key=aws_secret_access_key,
+            )
+            return RerankResponse(scores=sim_scores)
         else:
             raise ValueError(f"Unsupported provider: {rerank_request.provider_type}")
+
     except Exception as e:
         logger.exception(f"Error during reranking process:\n{str(e)}")
         raise HTTPException(

backend/model_server/utils.py

@@ -70,3 +70,32 @@ def get_gpu_type() -> str:
         return GPUStatus.MAC_MPS
 
     return GPUStatus.NONE
+
+
+def pass_aws_key(api_key: str) -> tuple[str, str, str]:
+    """Parse AWS API key string into components.
+
+    Args:
+        api_key: String in format 'aws_ACCESSKEY_SECRETKEY_REGION'
+
+    Returns:
+        Tuple of (access_key, secret_key, region)
+
+    Raises:
+        ValueError: If key format is invalid
+    """
+    if not api_key.startswith("aws"):
+        raise ValueError("API key must start with 'aws' prefix")
+
+    parts = api_key.split("_")
+    if len(parts) != 4:
+        raise ValueError(
+            f"API key must be in format 'aws_ACCESSKEY_SECRETKEY_REGION', got {len(parts) - 1} parts"
+            "this is an onyx specific format for formatting the aws secrets for bedrock"
+        )
+
+    try:
+        _, aws_access_key_id, aws_secret_access_key, aws_region = parts
+        return aws_access_key_id, aws_secret_access_key, aws_region
+    except Exception as e:
+        raise ValueError(f"Failed to parse AWS key components: {str(e)}")

backend/onyx/agents/agent_search/orchestration/nodes/choose_tool.py

@@ -98,8 +98,16 @@ def choose_tool(
         # For tool calling LLMs, we want to insert the task prompt as part of this flow, this is because the LLM
         # may choose to not call any tools and just generate the answer, in which case the task prompt is needed.
         prompt=built_prompt,
-        tools=[tool.tool_definition() for tool in tools] or None,
-        tool_choice=("required" if tools and force_use_tool.force_use else None),
+        tools=(
+            [tool.tool_definition() for tool in tools] or None
+            if using_tool_calling_llm
+            else None
+        ),
+        tool_choice=(
+            "required"
+            if tools and force_use_tool.force_use and using_tool_calling_llm
+            else None
+        ),
         structured_response_format=structured_response_format,
     )
 

backend/onyx/auth/users.py

@@ -523,6 +523,7 @@ class UserManager(UUIDIDMixin, BaseUserManager[User, uuid.UUID]):
         token = CURRENT_TENANT_ID_CONTEXTVAR.set(tenant_id)
         try:
             user_count = await get_user_count()
+            logger.debug(f"Current tenant user count: {user_count}")
 
             with get_session_with_tenant(tenant_id=tenant_id) as db_session:
                 if user_count == 1:
@@ -544,7 +545,7 @@ class UserManager(UUIDIDMixin, BaseUserManager[User, uuid.UUID]):
         finally:
             CURRENT_TENANT_ID_CONTEXTVAR.reset(token)
 
-        logger.notice(f"User {user.id} has registered.")
+        logger.debug(f"User {user.id} has registered.")
         optional_telemetry(
             record_type=RecordType.SIGN_UP,
             data={"action": "create"},
@@ -586,14 +587,20 @@ class UserManager(UUIDIDMixin, BaseUserManager[User, uuid.UUID]):
     ) -> Optional[User]:
         email = credentials.username
 
-        # Get tenant_id from mapping table
-        tenant_id = await fetch_ee_implementation_or_noop(
-            "onyx.server.tenants.provisioning",
-            "get_or_provision_tenant",
-            async_return_default_schema,
-        )(
-            email=email,
-        )
+        tenant_id: str | None = None
+        try:
+            tenant_id = fetch_ee_implementation_or_noop(
+                "onyx.server.tenants.provisioning",
+                "get_tenant_id_for_email",
+                None,
+            )(
+                email=email,
+            )
+        except Exception as e:
+            logger.warning(
+                f"User attempted to login with invalid credentials: {str(e)}"
+            )
+
         if not tenant_id:
             # User not found in mapping
             self.password_helper.hash(credentials.password)

backend/onyx/background/indexing/run_indexing.py

@@ -22,6 +22,7 @@ from onyx.configs.constants import DocumentSource
 from onyx.configs.constants import MilestoneRecordType
 from onyx.connectors.connector_runner import ConnectorRunner
 from onyx.connectors.exceptions import ConnectorValidationError
+from onyx.connectors.exceptions import UnexpectedValidationError
 from onyx.connectors.factory import instantiate_connector
 from onyx.connectors.models import ConnectorCheckpoint
 from onyx.connectors.models import ConnectorFailure
@@ -92,8 +93,13 @@ def _get_connector_runner(
         if not INTEGRATION_TESTS_MODE:
             runnable_connector.validate_connector_settings()
 
+    except UnexpectedValidationError as e:
+        logger.exception(
+            "Unable to instantiate connector due to an unexpected temporary issue."
+        )
+        raise e
     except Exception as e:
-        logger.exception("Unable to instantiate connector.")
+        logger.exception("Unable to instantiate connector. Pausing until fixed.")
         # since we failed to even instantiate the connector, we pause the CCPair since
         # it will never succeed
 

backend/onyx/chat/process_message.py

@@ -756,6 +756,7 @@ def stream_chat_message_objects(
         )
 
         # LLM prompt building, response capturing, etc.
+
         answer = Answer(
             prompt_builder=prompt_builder,
             is_connected=is_connected,

backend/onyx/configs/app_configs.py

@@ -640,3 +640,6 @@ TEST_ENV = os.environ.get("TEST_ENV", "").lower() == "true"
 MOCK_LLM_RESPONSE = (
     os.environ.get("MOCK_LLM_RESPONSE") if os.environ.get("MOCK_LLM_RESPONSE") else None
 )
+
+
+DEFAULT_IMAGE_ANALYSIS_MAX_SIZE_MB = 20

backend/onyx/configs/llm_configs.py

@@ -0,0 +1,38 @@
+from onyx.configs.app_configs import DEFAULT_IMAGE_ANALYSIS_MAX_SIZE_MB
+from onyx.server.settings.store import load_settings
+
+
+def get_image_extraction_and_analysis_enabled() -> bool:
+    """Get image extraction and analysis enabled setting from workspace settings or fallback to False"""
+    try:
+        settings = load_settings()
+        if settings.image_extraction_and_analysis_enabled is not None:
+            return settings.image_extraction_and_analysis_enabled
+    except Exception:
+        pass
+
+    return False
+
+
+def get_search_time_image_analysis_enabled() -> bool:
+    """Get search time image analysis enabled setting from workspace settings or fallback to False"""
+    try:
+        settings = load_settings()
+        if settings.search_time_image_analysis_enabled is not None:
+            return settings.search_time_image_analysis_enabled
+    except Exception:
+        pass
+
+    return False
+
+
+def get_image_analysis_max_size_mb() -> int:
+    """Get image analysis max size MB setting from workspace settings or fallback to environment variable"""
+    try:
+        settings = load_settings()
+        if settings.image_analysis_max_size_mb is not None:
+            return settings.image_analysis_max_size_mb
+    except Exception:
+        pass
+
+    return DEFAULT_IMAGE_ANALYSIS_MAX_SIZE_MB

backend/onyx/connectors/airtable/airtable_connector.py

@@ -200,7 +200,6 @@ class AirtableConnector(LoadConnector):
                                         return attachment_response.content
 
                             logger.error(f"Failed to refresh attachment for {filename}")
-
                         raise
 
                 attachment_content = get_attachment_with_retry(url, record_id)

backend/onyx/connectors/blob/connector.py

@@ -18,7 +18,7 @@ from onyx.configs.constants import DocumentSource
 from onyx.connectors.exceptions import ConnectorValidationError
 from onyx.connectors.exceptions import CredentialExpiredError
 from onyx.connectors.exceptions import InsufficientPermissionsError
-from onyx.connectors.exceptions import UnexpectedError
+from onyx.connectors.exceptions import UnexpectedValidationError
 from onyx.connectors.interfaces import GenerateDocumentsOutput
 from onyx.connectors.interfaces import LoadConnector
 from onyx.connectors.interfaces import PollConnector
@@ -310,7 +310,7 @@ class BlobStorageConnector(LoadConnector, PollConnector):
             # Catch-all for anything not captured by the above
             # Since we are unsure of the error and it may not disable the connector,
             #  raise an unexpected error (does not disable connector)
-            raise UnexpectedError(
+            raise UnexpectedValidationError(
                 f"Unexpected error during blob storage settings validation: {e}"
             )
 

backend/onyx/connectors/confluence/connector.py

@@ -11,18 +11,17 @@ from onyx.configs.app_configs import CONFLUENCE_TIMEZONE_OFFSET
 from onyx.configs.app_configs import CONTINUE_ON_CONNECTOR_FAILURE
 from onyx.configs.app_configs import INDEX_BATCH_SIZE
 from onyx.configs.constants import DocumentSource
-from onyx.connectors.confluence.onyx_confluence import attachment_to_content
-from onyx.connectors.confluence.onyx_confluence import (
-    extract_text_from_confluence_html,
-)
+from onyx.connectors.confluence.onyx_confluence import extract_text_from_confluence_html
 from onyx.connectors.confluence.onyx_confluence import OnyxConfluence
 from onyx.connectors.confluence.utils import build_confluence_document_id
+from onyx.connectors.confluence.utils import convert_attachment_to_content
 from onyx.connectors.confluence.utils import datetime_from_string
+from onyx.connectors.confluence.utils import process_attachment
 from onyx.connectors.confluence.utils import validate_attachment_filetype
 from onyx.connectors.exceptions import ConnectorValidationError
 from onyx.connectors.exceptions import CredentialExpiredError
 from onyx.connectors.exceptions import InsufficientPermissionsError
-from onyx.connectors.exceptions import UnexpectedError
+from onyx.connectors.exceptions import UnexpectedValidationError
 from onyx.connectors.interfaces import CredentialsConnector
 from onyx.connectors.interfaces import CredentialsProviderInterface
 from onyx.connectors.interfaces import GenerateDocumentsOutput
@@ -36,28 +35,26 @@ from onyx.connectors.models import ConnectorMissingCredentialError
 from onyx.connectors.models import Document
 from onyx.connectors.models import Section
 from onyx.connectors.models import SlimDocument
+from onyx.connectors.vision_enabled_connector import VisionEnabledConnector
 from onyx.indexing.indexing_heartbeat import IndexingHeartbeatInterface
 from onyx.utils.logger import setup_logger
 
 logger = setup_logger()
-
 # Potential Improvements
-# 1. Include attachments, etc
-# 2. Segment into Sections for more accurate linking, can split by headers but make sure no text/ordering is lost
-
+# 1. Segment into Sections for more accurate linking, can split by headers but make sure no text/ordering is lost
 _COMMENT_EXPANSION_FIELDS = ["body.storage.value"]
 _PAGE_EXPANSION_FIELDS = [
     "body.storage.value",
     "version",
     "space",
     "metadata.labels",
+    "history.lastUpdated",
 ]
 _ATTACHMENT_EXPANSION_FIELDS = [
     "version",
     "space",
     "metadata.labels",
 ]
-
 _RESTRICTIONS_EXPANSION_FIELDS = [
     "space",
     "restrictions.read.restrictions.user",
@@ -87,7 +84,11 @@ _FULL_EXTENSION_FILTER_STRING = "".join(
 
 
 class ConfluenceConnector(
-    LoadConnector, PollConnector, SlimConnector, CredentialsConnector
+    LoadConnector,
+    PollConnector,
+    SlimConnector,
+    CredentialsConnector,
+    VisionEnabledConnector,
 ):
     def __init__(
         self,
@@ -105,13 +106,24 @@ class ConfluenceConnector(
         labels_to_skip: list[str] = CONFLUENCE_CONNECTOR_LABELS_TO_SKIP,
         timezone_offset: float = CONFLUENCE_TIMEZONE_OFFSET,
     ) -> None:
+        self.wiki_base = wiki_base
+        self.is_cloud = is_cloud
+        self.space = space
+        self.page_id = page_id
+        self.index_recursively = index_recursively
+        self.cql_query = cql_query
         self.batch_size = batch_size
         self.continue_on_failure = continue_on_failure
-        self.is_cloud = is_cloud
+        self.labels_to_skip = labels_to_skip
+        self.timezone_offset = timezone_offset
+        self._confluence_client: OnyxConfluence | None = None
+        self._fetched_titles: set[str] = set()
+
+        # Initialize vision LLM using the mixin
+        self.initialize_vision_llm()
 
         # Remove trailing slash from wiki_base if present
         self.wiki_base = wiki_base.rstrip("/")
-
         """
         If nothing is provided, we default to fetching all pages
         Only one or none of the following options should be specified so
@@ -153,8 +165,6 @@ class ConfluenceConnector(
             "max_backoff_seconds": 60,
         }
 
-        self._confluence_client: OnyxConfluence | None = None
-
     @property
     def confluence_client(self) -> OnyxConfluence:
         if self._confluence_client is None:
@@ -184,7 +194,6 @@ class ConfluenceConnector(
         end: SecondsSinceUnixEpoch | None = None,
     ) -> str:
         page_query = self.base_cql_page_query + self.cql_label_filter
-
         # Add time filters
         if start:
             formatted_start_time = datetime.fromtimestamp(
@@ -196,7 +205,6 @@ class ConfluenceConnector(
                 "%Y-%m-%d %H:%M"
             )
             page_query += f" and lastmodified <= '{formatted_end_time}'"
-
         return page_query
 
     def _construct_attachment_query(self, confluence_page_id: str) -> str:
@@ -207,11 +215,10 @@ class ConfluenceConnector(
 
     def _get_comment_string_for_page_id(self, page_id: str) -> str:
         comment_string = ""
-
         comment_cql = f"type=comment and container='{page_id}'"
         comment_cql += self.cql_label_filter
-
         expand = ",".join(_COMMENT_EXPANSION_FIELDS)
+
         for comment in self.confluence_client.paginated_cql_retrieval(
             cql=comment_cql,
             expand=expand,
@@ -222,123 +229,177 @@ class ConfluenceConnector(
                 confluence_object=comment,
                 fetched_titles=set(),
             )
-
         return comment_string
 
-    def _convert_object_to_document(
-        self,
-        confluence_object: dict[str, Any],
-        parent_content_id: str | None = None,
-    ) -> Document | None:
+    def _convert_page_to_document(self, page: dict[str, Any]) -> Document | None:
         """
-        Takes in a confluence object, extracts all metadata, and converts it into a document.
-        If its a page, it extracts the text, adds the comments for the document text.
-        If its an attachment, it just downloads the attachment and converts that into a document.
-
-        parent_content_id: if the object is an attachment, specifies the content id that
-        the attachment is attached to
+        Converts a Confluence page to a Document object.
+        Includes the page content, comments, and attachments.
         """
-        # The url and the id are the same
-        object_url = build_confluence_document_id(
-            self.wiki_base, confluence_object["_links"]["webui"], self.is_cloud
-        )
+        try:
+            # Extract basic page information
+            page_id = page["id"]
+            page_title = page["title"]
+            page_url = f"{self.wiki_base}{page['_links']['webui']}"
 
-        object_text = None
-        # Extract text from page
-        if confluence_object["type"] == "page":
-            object_text = extract_text_from_confluence_html(
-                confluence_client=self.confluence_client,
-                confluence_object=confluence_object,
-                fetched_titles={confluence_object.get("title", "")},
-            )
-            # Add comments to text
-            object_text += self._get_comment_string_for_page_id(confluence_object["id"])
-        elif confluence_object["type"] == "attachment":
-            object_text = attachment_to_content(
-                confluence_client=self.confluence_client,
-                attachment=confluence_object,
-                parent_content_id=parent_content_id,
+            # Get the page content
+            page_content = extract_text_from_confluence_html(
+                self.confluence_client, page, self._fetched_titles
             )
 
-        if object_text is None:
-            # This only happens for attachments that are not parseable
+            # Create the main section for the page content
+            sections = [Section(text=page_content, link=page_url)]
+
+            # Process comments if available
+            comment_text = self._get_comment_string_for_page_id(page_id)
+            if comment_text:
+                sections.append(Section(text=comment_text, link=f"{page_url}#comments"))
+
+            # Process attachments
+            if "children" in page and "attachment" in page["children"]:
+                attachments = self.confluence_client.get_attachments_for_page(
+                    page_id, expand="metadata"
+                )
+
+                for attachment in attachments.get("results", []):
+                    # Process each attachment
+                    result = process_attachment(
+                        self.confluence_client,
+                        attachment,
+                        page_title,
+                        self.image_analysis_llm,
+                    )
+
+                    if result.text:
+                        # Create a section for the attachment text
+                        attachment_section = Section(
+                            text=result.text,
+                            link=f"{page_url}#attachment-{attachment['id']}",
+                            image_file_name=result.file_name,
+                        )
+                        sections.append(attachment_section)
+                    elif result.error:
+                        logger.warning(
+                            f"Error processing attachment '{attachment.get('title')}': {result.error}"
+                        )
+
+            # Extract metadata
+            metadata = {}
+            if "space" in page:
+                metadata["space"] = page["space"].get("name", "")
+
+            # Extract labels
+            labels = []
+            if "metadata" in page and "labels" in page["metadata"]:
+                for label in page["metadata"]["labels"].get("results", []):
+                    labels.append(label.get("name", ""))
+            if labels:
+                metadata["labels"] = labels
+
+            # Extract owners
+            primary_owners = []
+            if "version" in page and "by" in page["version"]:
+                author = page["version"]["by"]
+                display_name = author.get("displayName", "Unknown")
+                primary_owners.append(BasicExpertInfo(display_name=display_name))
+
+            # Create the document
+            return Document(
+                id=build_confluence_document_id(self.wiki_base, page_id, self.is_cloud),
+                sections=sections,
+                source=DocumentSource.CONFLUENCE,
+                semantic_identifier=page_title,
+                metadata=metadata,
+                doc_updated_at=datetime_from_string(page["version"]["when"]),
+                primary_owners=primary_owners if primary_owners else None,
+            )
+        except Exception as e:
+            logger.error(f"Error converting page {page.get('id', 'unknown')}: {e}")
+            if not self.continue_on_failure:
+                raise
             return None
 
-        # Get space name
-        doc_metadata: dict[str, str | list[str]] = {
-            "Wiki Space Name": confluence_object["space"]["name"]
-        }
-
-        # Get labels
-        label_dicts = (
-            confluence_object.get("metadata", {}).get("labels", {}).get("results", [])
-        )
-        page_labels = [label.get("name") for label in label_dicts if label.get("name")]
-        if page_labels:
-            doc_metadata["labels"] = page_labels
-
-        # Get last modified and author email
-        version_dict = confluence_object.get("version", {})
-        last_modified = (
-            datetime_from_string(version_dict.get("when"))
-            if version_dict.get("when")
-            else None
-        )
-        author_email = version_dict.get("by", {}).get("email")
-
-        title = confluence_object.get("title", "Untitled Document")
-
-        return Document(
-            id=object_url,
-            sections=[Section(link=object_url, text=object_text)],
-            source=DocumentSource.CONFLUENCE,
-            semantic_identifier=title,
-            doc_updated_at=last_modified,
-            primary_owners=(
-                [BasicExpertInfo(email=author_email)] if author_email else None
-            ),
-            metadata=doc_metadata,
-        )
-
     def _fetch_document_batches(
         self,
         start: SecondsSinceUnixEpoch | None = None,
         end: SecondsSinceUnixEpoch | None = None,
     ) -> GenerateDocumentsOutput:
+        """
+        Yields batches of Documents. For each page:
+         - Create a Document with 1 Section for the page text/comments
+         - Then fetch attachments. For each attachment:
+             - Attempt to convert it with convert_attachment_to_content(...)
+             - If successful, create a new Section with the extracted text or summary.
+        """
         doc_batch: list[Document] = []
-        confluence_page_ids: list[str] = []
 
         page_query = self._construct_page_query(start, end)
         logger.debug(f"page_query: {page_query}")
-        # Fetch pages as Documents
+
         for page in self.confluence_client.paginated_cql_retrieval(
             cql=page_query,
             expand=",".join(_PAGE_EXPANSION_FIELDS),
             limit=self.batch_size,
         ):
-            logger.debug(f"_fetch_document_batches: {page['id']}")
-            confluence_page_ids.append(page["id"])
-            doc = self._convert_object_to_document(page)
-            if doc is not None:
-                doc_batch.append(doc)
-            if len(doc_batch) >= self.batch_size:
-                yield doc_batch
-                doc_batch = []
+            # Build doc from page
+            doc = self._convert_page_to_document(page)
+            if not doc:
+                continue
+
+            # Now get attachments for that page:
+            attachment_query = self._construct_attachment_query(page["id"])
+            # We'll use the page's XML to provide context if we summarize an image
+            confluence_xml = page.get("body", {}).get("storage", {}).get("value", "")
 
-        # Fetch attachments as Documents
-        for confluence_page_id in confluence_page_ids:
-            attachment_query = self._construct_attachment_query(confluence_page_id)
-            # TODO: maybe should add time filter as well?
             for attachment in self.confluence_client.paginated_cql_retrieval(
                 cql=attachment_query,
                 expand=",".join(_ATTACHMENT_EXPANSION_FIELDS),
             ):
-                doc = self._convert_object_to_document(attachment, confluence_page_id)
-                if doc is not None:
-                    doc_batch.append(doc)
-                if len(doc_batch) >= self.batch_size:
-                    yield doc_batch
-                    doc_batch = []
+                attachment["metadata"].get("mediaType", "")
+                if not validate_attachment_filetype(
+                    attachment, self.image_analysis_llm
+                ):
+                    continue
+
+                # Attempt to get textual content or image summarization:
+                try:
+                    logger.info(f"Processing attachment: {attachment['title']}")
+                    response = convert_attachment_to_content(
+                        confluence_client=self.confluence_client,
+                        attachment=attachment,
+                        page_context=confluence_xml,
+                        llm=self.image_analysis_llm,
+                    )
+                    if response is None:
+                        continue
+
+                    content_text, file_storage_name = response
+
+                    object_url = build_confluence_document_id(
+                        self.wiki_base, page["_links"]["webui"], self.is_cloud
+                    )
+
+                    if content_text:
+                        doc.sections.append(
+                            Section(
+                                text=content_text,
+                                link=object_url,
+                                image_file_name=file_storage_name,
+                            )
+                        )
+                except Exception as e:
+                    logger.error(
+                        f"Failed to extract/summarize attachment {attachment['title']}",
+                        exc_info=e,
+                    )
+                    if not self.continue_on_failure:
+                        raise
+
+            doc_batch.append(doc)
+
+            if len(doc_batch) >= self.batch_size:
+                yield doc_batch
+                doc_batch = []
 
         if doc_batch:
             yield doc_batch
@@ -359,55 +420,63 @@ class ConfluenceConnector(
         end: SecondsSinceUnixEpoch | None = None,
         callback: IndexingHeartbeatInterface | None = None,
     ) -> GenerateSlimDocumentOutput:
+        """
+        Return 'slim' docs (IDs + minimal permission data).
+        Does not fetch actual text. Used primarily for incremental permission sync.
+        """
         doc_metadata_list: list[SlimDocument] = []
-
         restrictions_expand = ",".join(_RESTRICTIONS_EXPANSION_FIELDS)
 
+        # Query pages
         page_query = self.base_cql_page_query + self.cql_label_filter
         for page in self.confluence_client.cql_paginate_all_expansions(
             cql=page_query,
             expand=restrictions_expand,
             limit=_SLIM_DOC_BATCH_SIZE,
         ):
-            # If the page has restrictions, add them to the perm_sync_data
-            # These will be used by doc_sync.py to sync permissions
             page_restrictions = page.get("restrictions")
             page_space_key = page.get("space", {}).get("key")
             page_ancestors = page.get("ancestors", [])
+
             page_perm_sync_data = {
                 "restrictions": page_restrictions or {},
                 "space_key": page_space_key,
-                "ancestors": page_ancestors or [],
+                "ancestors": page_ancestors,
             }
 
             doc_metadata_list.append(
                 SlimDocument(
                     id=build_confluence_document_id(
-                        self.wiki_base,
-                        page["_links"]["webui"],
-                        self.is_cloud,
+                        self.wiki_base, page["_links"]["webui"], self.is_cloud
                     ),
                     perm_sync_data=page_perm_sync_data,
                 )
             )
+
+            # Query attachments for each page
             attachment_query = self._construct_attachment_query(page["id"])
             for attachment in self.confluence_client.cql_paginate_all_expansions(
                 cql=attachment_query,
                 expand=restrictions_expand,
                 limit=_SLIM_DOC_BATCH_SIZE,
             ):
-                if not validate_attachment_filetype(attachment):
+                # If you skip images, you'll skip them in the permission sync
+                attachment["metadata"].get("mediaType", "")
+                if not validate_attachment_filetype(
+                    attachment, self.image_analysis_llm
+                ):
                     continue
-                attachment_restrictions = attachment.get("restrictions")
+
+                attachment_restrictions = attachment.get("restrictions", {})
                 if not attachment_restrictions:
-                    attachment_restrictions = page_restrictions
+                    attachment_restrictions = page_restrictions or {}
 
                 attachment_space_key = attachment.get("space", {}).get("key")
                 if not attachment_space_key:
                     attachment_space_key = page_space_key
 
                 attachment_perm_sync_data = {
-                    "restrictions": attachment_restrictions or {},
+                    "restrictions": attachment_restrictions,
                     "space_key": attachment_space_key,
                 }
 
@@ -421,16 +490,16 @@ class ConfluenceConnector(
                         perm_sync_data=attachment_perm_sync_data,
                     )
                 )
+
             if len(doc_metadata_list) > _SLIM_DOC_BATCH_SIZE:
                 yield doc_metadata_list[:_SLIM_DOC_BATCH_SIZE]
                 doc_metadata_list = doc_metadata_list[_SLIM_DOC_BATCH_SIZE:]
 
+                if callback and callback.should_stop():
+                    raise RuntimeError(
+                        "retrieve_all_slim_documents: Stop signal detected"
+                    )
                 if callback:
-                    if callback.should_stop():
-                        raise RuntimeError(
-                            "retrieve_all_slim_documents: Stop signal detected"
-                        )
-
                     callback.progress("retrieve_all_slim_documents", 1)
 
         yield doc_metadata_list
@@ -451,11 +520,11 @@ class ConfluenceConnector(
                 raise InsufficientPermissionsError(
                     "Insufficient permissions to access Confluence resources (HTTP 403)."
                 )
-            raise UnexpectedError(
+            raise UnexpectedValidationError(
                 f"Unexpected Confluence error (status={status_code}): {e}"
             )
         except Exception as e:
-            raise UnexpectedError(
+            raise UnexpectedValidationError(
                 f"Unexpected error while validating Confluence settings: {e}"
             )
 

backend/onyx/connectors/confluence/onyx_confluence.py

@@ -144,6 +144,12 @@ class OnyxConfluence:
             self.static_credentials = credential_json
             return credential_json, False
 
+        if not OAUTH_CONFLUENCE_CLOUD_CLIENT_ID:
+            raise RuntimeError("OAUTH_CONFLUENCE_CLOUD_CLIENT_ID must be set!")
+
+        if not OAUTH_CONFLUENCE_CLOUD_CLIENT_SECRET:
+            raise RuntimeError("OAUTH_CONFLUENCE_CLOUD_CLIENT_SECRET must be set!")
+
         # check if we should refresh tokens. we're deciding to refresh halfway
         # to expiration
         now = datetime.now(timezone.utc)

backend/onyx/connectors/confluence/utils.py

@@ -1,9 +1,12 @@
+import io
 import math
 import time
 from collections.abc import Callable
 from datetime import datetime
 from datetime import timedelta
 from datetime import timezone
+from io import BytesIO
+from pathlib import Path
 from typing import Any
 from typing import cast
 from typing import TYPE_CHECKING
@@ -12,14 +15,28 @@ from urllib.parse import parse_qs
 from urllib.parse import quote
 from urllib.parse import urlparse
 
-import bs4
 import requests
 from pydantic import BaseModel
+from sqlalchemy.orm import Session
 
-from onyx.utils.logger import setup_logger
+from onyx.configs.app_configs import (
+    CONFLUENCE_CONNECTOR_ATTACHMENT_CHAR_COUNT_THRESHOLD,
+)
+from onyx.configs.constants import FileOrigin
 
 if TYPE_CHECKING:
-    pass
+    from onyx.connectors.confluence.onyx_confluence import OnyxConfluence
+
+from onyx.db.engine import get_session_with_current_tenant
+from onyx.db.models import PGFileStore
+from onyx.db.pg_file_store import create_populate_lobj
+from onyx.db.pg_file_store import save_bytes_to_pgfilestore
+from onyx.db.pg_file_store import upsert_pgfilestore
+from onyx.file_processing.extract_file_text import extract_file_text
+from onyx.file_processing.file_validation import is_valid_image_type
+from onyx.file_processing.image_utils import store_image_and_create_section
+from onyx.llm.interfaces import LLM
+from onyx.utils.logger import setup_logger
 
 logger = setup_logger()
 
@@ -35,15 +52,229 @@ class TokenResponse(BaseModel):
     scope: str
 
 
-def validate_attachment_filetype(attachment: dict[str, Any]) -> bool:
-    return attachment["metadata"]["mediaType"] not in [
-        "image/jpeg",
-        "image/png",
-        "image/gif",
-        "image/svg+xml",
-        "video/mp4",
-        "video/quicktime",
-    ]
+def validate_attachment_filetype(
+    attachment: dict[str, Any], llm: LLM | None = None
+) -> bool:
+    """
+    Validates if the attachment is a supported file type.
+    If LLM is provided, also checks if it's an image that can be processed.
+    """
+    attachment.get("metadata", {})
+    media_type = attachment.get("metadata", {}).get("mediaType", "")
+
+    if media_type.startswith("image/"):
+        return llm is not None and is_valid_image_type(media_type)
+
+    # For non-image files, check if we support the extension
+    title = attachment.get("title", "")
+    extension = Path(title).suffix.lstrip(".").lower() if "." in title else ""
+    return extension in ["pdf", "doc", "docx", "txt", "md", "rtf"]
+
+
+class AttachmentProcessingResult(BaseModel):
+    """
+    A container for results after processing a Confluence attachment.
+    'text' is the textual content of the attachment.
+    'file_name' is the final file name used in PGFileStore to store the content.
+    'error' holds an exception or string if something failed.
+    """
+
+    text: str | None
+    file_name: str | None
+    error: str | None = None
+
+
+def _download_attachment(
+    confluence_client: "OnyxConfluence", attachment: dict[str, Any]
+) -> bytes | None:
+    """
+    Retrieves the raw bytes of an attachment from Confluence. Returns None on error.
+    """
+    download_link = confluence_client.url + attachment["_links"]["download"]
+    resp = confluence_client._session.get(download_link)
+    if resp.status_code != 200:
+        logger.warning(
+            f"Failed to fetch {download_link} with status code {resp.status_code}"
+        )
+        return None
+    return resp.content
+
+
+def process_attachment(
+    confluence_client: "OnyxConfluence",
+    attachment: dict[str, Any],
+    page_context: str,
+    llm: LLM | None,
+) -> AttachmentProcessingResult:
+    """
+    Processes a Confluence attachment. If it's a document, extracts text,
+    or if it's an image and an LLM is available, summarizes it. Returns a structured result.
+    """
+    try:
+        # Get the media type from the attachment metadata
+        media_type = attachment.get("metadata", {}).get("mediaType", "")
+
+        # Validate the attachment type
+        if not validate_attachment_filetype(attachment, llm):
+            return AttachmentProcessingResult(
+                text=None,
+                file_name=None,
+                error=f"Unsupported file type: {media_type}",
+            )
+
+        # Download the attachment
+        raw_bytes = _download_attachment(confluence_client, attachment)
+        if raw_bytes is None:
+            return AttachmentProcessingResult(
+                text=None, file_name=None, error="Failed to download attachment"
+            )
+
+        # Process image attachments with LLM if available
+        if media_type.startswith("image/") and llm:
+            return _process_image_attachment(
+                confluence_client, attachment, page_context, llm, raw_bytes, media_type
+            )
+
+        # Process document attachments
+        try:
+            text = extract_file_text(
+                file=BytesIO(raw_bytes),
+                file_name=attachment["title"],
+            )
+
+            # Skip if the text is too long
+            if len(text) > CONFLUENCE_CONNECTOR_ATTACHMENT_CHAR_COUNT_THRESHOLD:
+                return AttachmentProcessingResult(
+                    text=None,
+                    file_name=None,
+                    error=f"Attachment text too long: {len(text)} chars",
+                )
+
+            return AttachmentProcessingResult(text=text, file_name=None, error=None)
+        except Exception as e:
+            return AttachmentProcessingResult(
+                text=None, file_name=None, error=f"Failed to extract text: {e}"
+            )
+
+    except Exception as e:
+        return AttachmentProcessingResult(
+            text=None, file_name=None, error=f"Failed to process attachment: {e}"
+        )
+
+
+def _process_image_attachment(
+    confluence_client: "OnyxConfluence",
+    attachment: dict[str, Any],
+    page_context: str,
+    llm: LLM,
+    raw_bytes: bytes,
+    media_type: str,
+) -> AttachmentProcessingResult:
+    """Process an image attachment by saving it and generating a summary."""
+    try:
+        # Use the standardized image storage and section creation
+        with get_session_with_current_tenant() as db_session:
+            section, file_name = store_image_and_create_section(
+                db_session=db_session,
+                image_data=raw_bytes,
+                file_name=Path(attachment["id"]).name,
+                display_name=attachment["title"],
+                media_type=media_type,
+                llm=llm,
+                file_origin=FileOrigin.CONNECTOR,
+            )
+
+            return AttachmentProcessingResult(
+                text=section.text, file_name=file_name, error=None
+            )
+    except Exception as e:
+        msg = f"Image summarization failed for {attachment['title']}: {e}"
+        logger.error(msg, exc_info=e)
+        return AttachmentProcessingResult(text=None, file_name=None, error=msg)
+
+
+def _process_text_attachment(
+    attachment: dict[str, Any],
+    raw_bytes: bytes,
+    media_type: str,
+) -> AttachmentProcessingResult:
+    """Process a text-based attachment by extracting its content."""
+    try:
+        extracted_text = extract_file_text(
+            io.BytesIO(raw_bytes),
+            file_name=attachment["title"],
+            break_on_unprocessable=False,
+        )
+    except Exception as e:
+        msg = f"Failed to extract text for '{attachment['title']}': {e}"
+        logger.error(msg, exc_info=e)
+        return AttachmentProcessingResult(text=None, file_name=None, error=msg)
+
+    # Check length constraints
+    if extracted_text is None or len(extracted_text) == 0:
+        msg = f"No text extracted for {attachment['title']}"
+        logger.warning(msg)
+        return AttachmentProcessingResult(text=None, file_name=None, error=msg)
+
+    if len(extracted_text) > CONFLUENCE_CONNECTOR_ATTACHMENT_CHAR_COUNT_THRESHOLD:
+        msg = (
+            f"Skipping attachment {attachment['title']} due to char count "
+            f"({len(extracted_text)} > {CONFLUENCE_CONNECTOR_ATTACHMENT_CHAR_COUNT_THRESHOLD})"
+        )
+        logger.warning(msg)
+        return AttachmentProcessingResult(text=None, file_name=None, error=msg)
+
+    # Save the attachment
+    try:
+        with get_session_with_current_tenant() as db_session:
+            saved_record = save_bytes_to_pgfilestore(
+                db_session=db_session,
+                raw_bytes=raw_bytes,
+                media_type=media_type,
+                identifier=attachment["id"],
+                display_name=attachment["title"],
+            )
+    except Exception as e:
+        msg = f"Failed to save attachment '{attachment['title']}' to PG: {e}"
+        logger.error(msg, exc_info=e)
+        return AttachmentProcessingResult(
+            text=extracted_text, file_name=None, error=msg
+        )
+
+    return AttachmentProcessingResult(
+        text=extracted_text, file_name=saved_record.file_name, error=None
+    )
+
+
+def convert_attachment_to_content(
+    confluence_client: "OnyxConfluence",
+    attachment: dict[str, Any],
+    page_context: str,
+    llm: LLM | None,
+) -> tuple[str | None, str | None] | None:
+    """
+    Facade function which:
+      1. Validates attachment type
+      2. Extracts or summarizes content
+      3. Returns (content_text, stored_file_name) or None if we should skip it
+    """
+    media_type = attachment["metadata"]["mediaType"]
+    # Quick check for unsupported types:
+    if media_type.startswith("video/") or media_type == "application/gliffy+json":
+        logger.warning(
+            f"Skipping unsupported attachment type: '{media_type}' for {attachment['title']}"
+        )
+        return None
+
+    result = process_attachment(confluence_client, attachment, page_context, llm)
+    if result.error is not None:
+        logger.warning(
+            f"Attachment {attachment['title']} encountered error: {result.error}"
+        )
+        return None
+
+    # Return the text and the file name
+    return result.text, result.file_name
 
 
 def build_confluence_document_id(
@@ -64,23 +295,6 @@ def build_confluence_document_id(
     return f"{base_url}{content_url}"
 
 
-def _extract_referenced_attachment_names(page_text: str) -> list[str]:
-    """Parse a Confluence html page to generate a list of current
-        attachments in use
-
-    Args:
-        text (str): The page content
-
-    Returns:
-        list[str]: List of filenames currently in use by the page text
-    """
-    referenced_attachment_filenames = []
-    soup = bs4.BeautifulSoup(page_text, "html.parser")
-    for attachment in soup.findAll("ri:attachment"):
-        referenced_attachment_filenames.append(attachment.attrs["ri:filename"])
-    return referenced_attachment_filenames
-
-
 def datetime_from_string(datetime_string: str) -> datetime:
     datetime_object = datetime.fromisoformat(datetime_string)
 
@@ -252,3 +466,37 @@ def update_param_in_path(path: str, param: str, value: str) -> str:
         + "?"
         + "&".join(f"{k}={quote(v[0])}" for k, v in query_params.items())
     )
+
+
+def attachment_to_file_record(
+    confluence_client: "OnyxConfluence",
+    attachment: dict[str, Any],
+    db_session: Session,
+) -> tuple[PGFileStore, bytes]:
+    """Save an attachment to the file store and return the file record."""
+    download_link = _attachment_to_download_link(confluence_client, attachment)
+    image_data = confluence_client.get(
+        download_link, absolute=True, not_json_response=True
+    )
+
+    # Save image to file store
+    file_name = f"confluence_attachment_{attachment['id']}"
+    lobj_oid = create_populate_lobj(BytesIO(image_data), db_session)
+    pgfilestore = upsert_pgfilestore(
+        file_name=file_name,
+        display_name=attachment["title"],
+        file_origin=FileOrigin.OTHER,
+        file_type=attachment["metadata"]["mediaType"],
+        lobj_oid=lobj_oid,
+        db_session=db_session,
+        commit=True,
+    )
+
+    return pgfilestore, image_data
+
+
+def _attachment_to_download_link(
+    confluence_client: "OnyxConfluence", attachment: dict[str, Any]
+) -> str:
+    """Extracts the download link to images."""
+    return confluence_client.url + attachment["_links"]["download"]

backend/onyx/connectors/exceptions.py

@@ -14,12 +14,15 @@ class ConnectorValidationError(ValidationError):
         super().__init__(self.message)
 
 
-class UnexpectedError(ValidationError):
+class UnexpectedValidationError(ValidationError):
     """Raised when an unexpected error occurs during connector validation.
 
     Unexpected errors don't necessarily mean the credential is invalid,
     but rather that there was an error during the validation process
     or we encountered a currently unhandled error case.
+
+    Currently, unexpected validation errors are defined as transient and should not be
+    used to disable the connector.
     """
 
     def __init__(self, message: str = "Unexpected error during connector validation"):

backend/onyx/connectors/file/connector.py

@@ -10,22 +10,23 @@ from sqlalchemy.orm import Session
 
 from onyx.configs.app_configs import INDEX_BATCH_SIZE
 from onyx.configs.constants import DocumentSource
+from onyx.configs.constants import FileOrigin
 from onyx.connectors.cross_connector_utils.miscellaneous_utils import time_str_to_utc
 from onyx.connectors.interfaces import GenerateDocumentsOutput
 from onyx.connectors.interfaces import LoadConnector
 from onyx.connectors.models import BasicExpertInfo
 from onyx.connectors.models import Document
 from onyx.connectors.models import Section
+from onyx.connectors.vision_enabled_connector import VisionEnabledConnector
 from onyx.db.engine import get_session_with_current_tenant
-from onyx.file_processing.extract_file_text import detect_encoding
-from onyx.file_processing.extract_file_text import extract_file_text
+from onyx.db.pg_file_store import get_pgfilestore_by_file_name
+from onyx.file_processing.extract_file_text import extract_text_and_images
 from onyx.file_processing.extract_file_text import get_file_ext
-from onyx.file_processing.extract_file_text import is_text_file_extension
 from onyx.file_processing.extract_file_text import is_valid_file_ext
 from onyx.file_processing.extract_file_text import load_files_from_zip
-from onyx.file_processing.extract_file_text import read_pdf_file
-from onyx.file_processing.extract_file_text import read_text_file
+from onyx.file_processing.image_utils import store_image_and_create_section
 from onyx.file_store.file_store import get_default_file_store
+from onyx.llm.interfaces import LLM
 from onyx.utils.logger import setup_logger
 
 logger = setup_logger()
@@ -35,81 +36,115 @@ def _read_files_and_metadata(
     file_name: str,
     db_session: Session,
 ) -> Iterator[tuple[str, IO, dict[str, Any]]]:
-    """Reads the file into IO, in the case of a zip file, yields each individual
-    file contained within, also includes the metadata dict if packaged in the zip"""
+    """
+    Reads the file from Postgres. If the file is a .zip, yields subfiles.
+    """
     extension = get_file_ext(file_name)
     metadata: dict[str, Any] = {}
     directory_path = os.path.dirname(file_name)
 
+    # Read file from Postgres store
     file_content = get_default_file_store(db_session).read_file(file_name, mode="b")
 
+    # If it's a zip, expand it
     if extension == ".zip":
-        for file_info, file, metadata in load_files_from_zip(
+        for file_info, subfile, metadata in load_files_from_zip(
             file_content, ignore_dirs=True
         ):
-            yield os.path.join(directory_path, file_info.filename), file, metadata
+            yield os.path.join(directory_path, file_info.filename), subfile, metadata
     elif is_valid_file_ext(extension):
         yield file_name, file_content, metadata
     else:
         logger.warning(f"Skipping file '{file_name}' with extension '{extension}'")
 
 
+def _create_image_section(
+    llm: LLM | None,
+    image_data: bytes,
+    db_session: Session,
+    parent_file_name: str,
+    display_name: str,
+    idx: int = 0,
+) -> tuple[Section, str | None]:
+    """
+    Create a Section object for a single image and store the image in PGFileStore.
+    If summarization is enabled and we have an LLM, summarize the image.
+
+    Returns:
+        tuple: (Section object, file_name in PGFileStore or None if storage failed)
+    """
+    # Create a unique file name for the embedded image
+    file_name = f"{parent_file_name}_embedded_{idx}"
+
+    # Use the standardized utility to store the image and create a section
+    return store_image_and_create_section(
+        db_session=db_session,
+        image_data=image_data,
+        file_name=file_name,
+        display_name=display_name,
+        llm=llm,
+        file_origin=FileOrigin.OTHER,
+    )
+
+
 def _process_file(
     file_name: str,
     file: IO[Any],
-    metadata: dict[str, Any] | None = None,
-    pdf_pass: str | None = None,
+    metadata: dict[str, Any] | None,
+    pdf_pass: str | None,
+    db_session: Session,
+    llm: LLM | None,
 ) -> list[Document]:
+    """
+    Processes a single file, returning a list of Documents (typically one).
+    Also handles embedded images if 'EMBEDDED_IMAGE_EXTRACTION_ENABLED' is true.
+    """
     extension = get_file_ext(file_name)
-    if not is_valid_file_ext(extension):
-        logger.warning(f"Skipping file '{file_name}' with extension '{extension}'")
+
+    # Fetch the DB record so we know the ID for internal URL
+    pg_record = get_pgfilestore_by_file_name(file_name=file_name, db_session=db_session)
+    if not pg_record:
+        logger.warning(f"No file record found for '{file_name}' in PG; skipping.")
         return []
 
-    file_metadata: dict[str, Any] = {}
-
-    if is_text_file_extension(file_name):
-        encoding = detect_encoding(file)
-        file_content_raw, file_metadata = read_text_file(
-            file, encoding=encoding, ignore_onyx_metadata=False
+    if not is_valid_file_ext(extension):
+        logger.warning(
+            f"Skipping file '{file_name}' with unrecognized extension '{extension}'"
         )
+        return []
 
-    # Using the PDF reader function directly to pass in password cleanly
-    elif extension == ".pdf" and pdf_pass is not None:
-        file_content_raw, file_metadata = read_pdf_file(file=file, pdf_pass=pdf_pass)
+    # Prepare doc metadata
+    if metadata is None:
+        metadata = {}
+    file_display_name = metadata.get("file_display_name") or os.path.basename(file_name)
 
-    else:
-        file_content_raw = extract_file_text(
-            file=file,
-            file_name=file_name,
-            break_on_unprocessable=True,
-        )
-
-    all_metadata = {**metadata, **file_metadata} if metadata else file_metadata
-
-    # add a prefix to avoid conflicts with other connectors
-    doc_id = f"FILE_CONNECTOR__{file_name}"
-    if metadata:
-        doc_id = metadata.get("document_id") or doc_id
-
-    # If this is set, we will show this in the UI as the "name" of the file
-    file_display_name = all_metadata.get("file_display_name") or os.path.basename(
-        file_name
-    )
-    title = (
-        all_metadata["title"] or "" if "title" in all_metadata else file_display_name
-    )
-
-    time_updated = all_metadata.get("time_updated", datetime.now(timezone.utc))
+    # Timestamps
+    current_datetime = datetime.now(timezone.utc)
+    time_updated = metadata.get("time_updated", current_datetime)
     if isinstance(time_updated, str):
         time_updated = time_str_to_utc(time_updated)
 
-    dt_str = all_metadata.get("doc_updated_at")
+    dt_str = metadata.get("doc_updated_at")
     final_time_updated = time_str_to_utc(dt_str) if dt_str else time_updated
 
-    # Metadata tags separate from the Onyx specific fields
+    # Collect owners
+    p_owner_names = metadata.get("primary_owners")
+    s_owner_names = metadata.get("secondary_owners")
+    p_owners = (
+        [BasicExpertInfo(display_name=name) for name in p_owner_names]
+        if p_owner_names
+        else None
+    )
+    s_owners = (
+        [BasicExpertInfo(display_name=name) for name in s_owner_names]
+        if s_owner_names
+        else None
+    )
+
+    # Additional tags we store as doc metadata
     metadata_tags = {
         k: v
-        for k, v in all_metadata.items()
+        for k, v in metadata.items()
         if k
         not in [
             "document_id",
@@ -122,77 +157,142 @@ def _process_file(
             "file_display_name",
             "title",
             "connector_type",
+            "pdf_password",
         ]
     }
 
-    source_type_str = all_metadata.get("connector_type")
-    source_type = DocumentSource(source_type_str) if source_type_str else None
-
-    p_owner_names = all_metadata.get("primary_owners")
-    s_owner_names = all_metadata.get("secondary_owners")
-    p_owners = (
-        [BasicExpertInfo(display_name=name) for name in p_owner_names]
-        if p_owner_names
-        else None
-    )
-    s_owners = (
-        [BasicExpertInfo(display_name=name) for name in s_owner_names]
-        if s_owner_names
-        else None
+    source_type_str = metadata.get("connector_type")
+    source_type = (
+        DocumentSource(source_type_str) if source_type_str else DocumentSource.FILE
     )
 
+    doc_id = metadata.get("document_id") or f"FILE_CONNECTOR__{file_name}"
+    title = metadata.get("title") or file_display_name
+
+    # 1) If the file itself is an image, handle that scenario quickly
+    IMAGE_EXTENSIONS = {".jpg", ".jpeg", ".png", ".webp"}
+    if extension in IMAGE_EXTENSIONS:
+        # Summarize or produce empty doc
+        image_data = file.read()
+        image_section, _ = _create_image_section(
+            llm, image_data, db_session, pg_record.file_name, title
+        )
+        return [
+            Document(
+                id=doc_id,
+                sections=[image_section],
+                source=source_type,
+                semantic_identifier=file_display_name,
+                title=title,
+                doc_updated_at=final_time_updated,
+                primary_owners=p_owners,
+                secondary_owners=s_owners,
+                metadata=metadata_tags,
+            )
+        ]
+
+    # 2) Otherwise: text-based approach. Possibly with embedded images if enabled.
+    #    (For example .docx with inline images).
+    file.seek(0)
+    text_content = ""
+    embedded_images: list[tuple[bytes, str]] = []
+
+    text_content, embedded_images = extract_text_and_images(
+        file=file,
+        file_name=file_name,
+        pdf_pass=pdf_pass,
+    )
+
+    # Build sections: first the text as a single Section
+    sections = []
+    link_in_meta = metadata.get("link")
+    if text_content.strip():
+        sections.append(Section(link=link_in_meta, text=text_content.strip()))
+
+    # Then any extracted images from docx, etc.
+    for idx, (img_data, img_name) in enumerate(embedded_images, start=1):
+        # Store each embedded image as a separate file in PGFileStore
+        # and create a section with the image summary
+        image_section, _ = _create_image_section(
+            llm,
+            img_data,
+            db_session,
+            pg_record.file_name,
+            f"{title} - image {idx}",
+            idx,
+        )
+        sections.append(image_section)
     return [
         Document(
             id=doc_id,
-            sections=[
-                Section(link=all_metadata.get("link"), text=file_content_raw.strip())
-            ],
-            source=source_type or DocumentSource.FILE,
+            sections=sections,
+            source=source_type,
             semantic_identifier=file_display_name,
             title=title,
             doc_updated_at=final_time_updated,
             primary_owners=p_owners,
             secondary_owners=s_owners,
-            # currently metadata just houses tags, other stuff like owners / updated at have dedicated fields
             metadata=metadata_tags,
         )
     ]
 
 
-class LocalFileConnector(LoadConnector):
+class LocalFileConnector(LoadConnector, VisionEnabledConnector):
+    """
+    Connector that reads files from Postgres and yields Documents, including
+    optional embedded image extraction.
+    """
+
     def __init__(
         self,
         file_locations: list[Path | str],
         batch_size: int = INDEX_BATCH_SIZE,
     ) -> None:
-        self.file_locations = [Path(file_location) for file_location in file_locations]
+        self.file_locations = [str(loc) for loc in file_locations]
         self.batch_size = batch_size
         self.pdf_pass: str | None = None
 
+        # Initialize vision LLM using the mixin
+        self.initialize_vision_llm()
+
     def load_credentials(self, credentials: dict[str, Any]) -> dict[str, Any] | None:
         self.pdf_pass = credentials.get("pdf_password")
+
         return None
 
     def load_from_state(self) -> GenerateDocumentsOutput:
+        """
+        Iterates over each file path, fetches from Postgres, tries to parse text
+        or images, and yields Document batches.
+        """
         documents: list[Document] = []
 
         with get_session_with_current_tenant() as db_session:
             for file_path in self.file_locations:
                 current_datetime = datetime.now(timezone.utc)
-                files = _read_files_and_metadata(
-                    file_name=str(file_path), db_session=db_session
+
+                files_iter = _read_files_and_metadata(
+                    file_name=file_path,
+                    db_session=db_session,
                 )
 
-                for file_name, file, metadata in files:
+                for actual_file_name, file, metadata in files_iter:
                     metadata["time_updated"] = metadata.get(
                         "time_updated", current_datetime
                     )
-                    documents.extend(
-                        _process_file(file_name, file, metadata, self.pdf_pass)
+                    new_docs = _process_file(
+                        file_name=actual_file_name,
+                        file=file,
+                        metadata=metadata,
+                        pdf_pass=self.pdf_pass,
+                        db_session=db_session,
+                        llm=self.image_analysis_llm,
                     )
+                    documents.extend(new_docs)
 
                     if len(documents) >= self.batch_size:
                         yield documents
+
                         documents = []
 
             if documents:
@@ -201,7 +301,7 @@ class LocalFileConnector(LoadConnector):
 
 if __name__ == "__main__":
     connector = LocalFileConnector(file_locations=[os.environ["TEST_FILE"]])
-    connector.load_credentials({"pdf_password": os.environ["PDF_PASSWORD"]})
-
-    document_batches = connector.load_from_state()
-    print(next(document_batches))
+    connector.load_credentials({"pdf_password": os.environ.get("PDF_PASSWORD")})
+    doc_batches = connector.load_from_state()
+    for batch in doc_batches:
+        print("BATCH:", batch)

backend/onyx/connectors/github/connector.py

@@ -20,7 +20,7 @@ from onyx.configs.constants import DocumentSource
 from onyx.connectors.exceptions import ConnectorValidationError
 from onyx.connectors.exceptions import CredentialExpiredError
 from onyx.connectors.exceptions import InsufficientPermissionsError
-from onyx.connectors.exceptions import UnexpectedError
+from onyx.connectors.exceptions import UnexpectedValidationError
 from onyx.connectors.interfaces import GenerateDocumentsOutput
 from onyx.connectors.interfaces import LoadConnector
 from onyx.connectors.interfaces import PollConnector
@@ -124,14 +124,14 @@ class GithubConnector(LoadConnector, PollConnector):
     def __init__(
         self,
         repo_owner: str,
-        repo_name: str | None = None,
+        repositories: str | None = None,
         batch_size: int = INDEX_BATCH_SIZE,
         state_filter: str = "all",
         include_prs: bool = True,
         include_issues: bool = False,
     ) -> None:
         self.repo_owner = repo_owner
-        self.repo_name = repo_name
+        self.repositories = repositories
         self.batch_size = batch_size
         self.state_filter = state_filter
         self.include_prs = include_prs
@@ -157,11 +157,42 @@ class GithubConnector(LoadConnector, PollConnector):
             )
 
         try:
-            return github_client.get_repo(f"{self.repo_owner}/{self.repo_name}")
+            return github_client.get_repo(f"{self.repo_owner}/{self.repositories}")
         except RateLimitExceededException:
             _sleep_after_rate_limit_exception(github_client)
             return self._get_github_repo(github_client, attempt_num + 1)
 
+    def _get_github_repos(
+        self, github_client: Github, attempt_num: int = 0
+    ) -> list[Repository.Repository]:
+        """Get specific repositories based on comma-separated repo_name string."""
+        if attempt_num > _MAX_NUM_RATE_LIMIT_RETRIES:
+            raise RuntimeError(
+                "Re-tried fetching repos too many times. Something is going wrong with fetching objects from Github"
+            )
+
+        try:
+            repos = []
+            # Split repo_name by comma and strip whitespace
+            repo_names = [
+                name.strip() for name in (cast(str, self.repositories)).split(",")
+            ]
+
+            for repo_name in repo_names:
+                if repo_name:  # Skip empty strings
+                    try:
+                        repo = github_client.get_repo(f"{self.repo_owner}/{repo_name}")
+                        repos.append(repo)
+                    except GithubException as e:
+                        logger.warning(
+                            f"Could not fetch repo {self.repo_owner}/{repo_name}: {e}"
+                        )
+
+            return repos
+        except RateLimitExceededException:
+            _sleep_after_rate_limit_exception(github_client)
+            return self._get_github_repos(github_client, attempt_num + 1)
+
     def _get_all_repos(
         self, github_client: Github, attempt_num: int = 0
     ) -> list[Repository.Repository]:
@@ -189,11 +220,17 @@ class GithubConnector(LoadConnector, PollConnector):
         if self.github_client is None:
             raise ConnectorMissingCredentialError("GitHub")
 
-        repos = (
-            [self._get_github_repo(self.github_client)]
-            if self.repo_name
-            else self._get_all_repos(self.github_client)
-        )
+        repos = []
+        if self.repositories:
+            if "," in self.repositories:
+                # Multiple repositories specified
+                repos = self._get_github_repos(self.github_client)
+            else:
+                # Single repository (backward compatibility)
+                repos = [self._get_github_repo(self.github_client)]
+        else:
+            # All repositories
+            repos = self._get_all_repos(self.github_client)
 
         for repo in repos:
             if self.include_prs:
@@ -268,11 +305,48 @@ class GithubConnector(LoadConnector, PollConnector):
             )
 
         try:
-            if self.repo_name:
-                test_repo = self.github_client.get_repo(
-                    f"{self.repo_owner}/{self.repo_name}"
-                )
-                test_repo.get_contents("")
+            if self.repositories:
+                if "," in self.repositories:
+                    # Multiple repositories specified
+                    repo_names = [name.strip() for name in self.repositories.split(",")]
+                    if not repo_names:
+                        raise ConnectorValidationError(
+                            "Invalid connector settings: No valid repository names provided."
+                        )
+
+                    # Validate at least one repository exists and is accessible
+                    valid_repos = False
+                    validation_errors = []
+
+                    for repo_name in repo_names:
+                        if not repo_name:
+                            continue
+
+                        try:
+                            test_repo = self.github_client.get_repo(
+                                f"{self.repo_owner}/{repo_name}"
+                            )
+                            test_repo.get_contents("")
+                            valid_repos = True
+                            # If at least one repo is valid, we can proceed
+                            break
+                        except GithubException as e:
+                            validation_errors.append(
+                                f"Repository '{repo_name}': {e.data.get('message', str(e))}"
+                            )
+
+                    if not valid_repos:
+                        error_msg = (
+                            "None of the specified repositories could be accessed: "
+                        )
+                        error_msg += ", ".join(validation_errors)
+                        raise ConnectorValidationError(error_msg)
+                else:
+                    # Single repository (backward compatibility)
+                    test_repo = self.github_client.get_repo(
+                        f"{self.repo_owner}/{self.repositories}"
+                    )
+                    test_repo.get_contents("")
             else:
                 # Try to get organization first
                 try:
@@ -284,7 +358,7 @@ class GithubConnector(LoadConnector, PollConnector):
                     user.get_repos().totalCount  # Just check if we can access repos
 
         except RateLimitExceededException:
-            raise UnexpectedError(
+            raise UnexpectedValidationError(
                 "Validation failed due to GitHub rate-limits being exceeded. Please try again later."
             )
 
@@ -298,10 +372,15 @@ class GithubConnector(LoadConnector, PollConnector):
                     "Your GitHub token does not have sufficient permissions for this repository (HTTP 403)."
                 )
             elif e.status == 404:
-                if self.repo_name:
-                    raise ConnectorValidationError(
-                        f"GitHub repository not found with name: {self.repo_owner}/{self.repo_name}"
-                    )
+                if self.repositories:
+                    if "," in self.repositories:
+                        raise ConnectorValidationError(
+                            f"None of the specified GitHub repositories could be found for owner: {self.repo_owner}"
+                        )
+                    else:
+                        raise ConnectorValidationError(
+                            f"GitHub repository not found with name: {self.repo_owner}/{self.repositories}"
+                        )
                 else:
                     raise ConnectorValidationError(
                         f"GitHub user or organization not found: {self.repo_owner}"
@@ -310,6 +389,7 @@ class GithubConnector(LoadConnector, PollConnector):
                 raise ConnectorValidationError(
                     f"Unexpected GitHub error (status={e.status}): {e.data}"
                 )
+
         except Exception as exc:
             raise Exception(
                 f"Unexpected error during GitHub settings validation: {exc}"
@@ -321,7 +401,7 @@ if __name__ == "__main__":
 
     connector = GithubConnector(
         repo_owner=os.environ["REPO_OWNER"],
-        repo_name=os.environ["REPO_NAME"],
+        repositories=os.environ["REPOSITORIES"],
     )
     connector.load_credentials(
         {"github_access_token": os.environ["GITHUB_ACCESS_TOKEN"]}

backend/onyx/connectors/google_drive/connector.py

@@ -4,14 +4,12 @@ from concurrent.futures import as_completed
 from concurrent.futures import ThreadPoolExecutor
 from functools import partial
 from typing import Any
-from typing import cast
 
 from google.oauth2.credentials import Credentials as OAuthCredentials  # type: ignore
 from google.oauth2.service_account import Credentials as ServiceAccountCredentials  # type: ignore
 from googleapiclient.errors import HttpError  # type: ignore
 
 from onyx.configs.app_configs import INDEX_BATCH_SIZE
-from onyx.configs.app_configs import MAX_FILE_SIZE_BYTES
 from onyx.configs.constants import DocumentSource
 from onyx.connectors.exceptions import ConnectorValidationError
 from onyx.connectors.exceptions import CredentialExpiredError
@@ -36,7 +34,6 @@ from onyx.connectors.google_utils.shared_constants import (
 )
 from onyx.connectors.google_utils.shared_constants import MISSING_SCOPES_ERROR_STR
 from onyx.connectors.google_utils.shared_constants import ONYX_SCOPE_INSTRUCTIONS
-from onyx.connectors.google_utils.shared_constants import SCOPE_DOC_URL
 from onyx.connectors.google_utils.shared_constants import SLIM_BATCH_SIZE
 from onyx.connectors.google_utils.shared_constants import USER_FIELDS
 from onyx.connectors.interfaces import GenerateDocumentsOutput
@@ -46,7 +43,9 @@ from onyx.connectors.interfaces import PollConnector
 from onyx.connectors.interfaces import SecondsSinceUnixEpoch
 from onyx.connectors.interfaces import SlimConnector
 from onyx.connectors.models import ConnectorMissingCredentialError
+from onyx.connectors.vision_enabled_connector import VisionEnabledConnector
 from onyx.indexing.indexing_heartbeat import IndexingHeartbeatInterface
+from onyx.llm.interfaces import LLM
 from onyx.utils.logger import setup_logger
 from onyx.utils.retry_wrapper import retry_builder
 
@@ -66,7 +65,10 @@ def _extract_ids_from_urls(urls: list[str]) -> list[str]:
 
 
 def _convert_single_file(
-    creds: Any, primary_admin_email: str, file: dict[str, Any]
+    creds: Any,
+    primary_admin_email: str,
+    file: dict[str, Any],
+    image_analysis_llm: LLM | None,
 ) -> Any:
     user_email = file.get("owners", [{}])[0].get("emailAddress") or primary_admin_email
     user_drive_service = get_drive_service(creds, user_email=user_email)
@@ -75,11 +77,14 @@ def _convert_single_file(
         file=file,
         drive_service=user_drive_service,
         docs_service=docs_service,
+        image_analysis_llm=image_analysis_llm,  # pass the LLM so doc_conversion can summarize images
     )
 
 
 def _process_files_batch(
-    files: list[GoogleDriveFileType], convert_func: Callable, batch_size: int
+    files: list[GoogleDriveFileType],
+    convert_func: Callable[[GoogleDriveFileType], Any],
+    batch_size: int,
 ) -> GenerateDocumentsOutput:
     doc_batch = []
     with ThreadPoolExecutor(max_workers=min(16, len(files))) as executor:
@@ -111,7 +116,9 @@ def _clean_requested_drive_ids(
     return valid_requested_drive_ids, filtered_folder_ids
 
 
-class GoogleDriveConnector(LoadConnector, PollConnector, SlimConnector):
+class GoogleDriveConnector(
+    LoadConnector, PollConnector, SlimConnector, VisionEnabledConnector
+):
     def __init__(
         self,
         include_shared_drives: bool = False,
@@ -129,23 +136,23 @@ class GoogleDriveConnector(LoadConnector, PollConnector, SlimConnector):
         continue_on_failure: bool | None = None,
     ) -> None:
         # Check for old input parameters
-        if (
-            folder_paths is not None
-            or include_shared is not None
-            or follow_shortcuts is not None
-            or only_org_public is not None
-            or continue_on_failure is not None
-        ):
-            logger.exception(
-                "Google Drive connector received old input parameters. "
-                "Please visit the docs for help with the new setup: "
-                f"{SCOPE_DOC_URL}"
+        if folder_paths is not None:
+            logger.warning(
+                "The 'folder_paths' parameter is deprecated. Use 'shared_folder_urls' instead."
             )
-            raise ConnectorValidationError(
-                "Google Drive connector received old input parameters. "
-                "Please visit the docs for help with the new setup: "
-                f"{SCOPE_DOC_URL}"
+        if include_shared is not None:
+            logger.warning(
+                "The 'include_shared' parameter is deprecated. Use 'include_files_shared_with_me' instead."
             )
+        if follow_shortcuts is not None:
+            logger.warning("The 'follow_shortcuts' parameter is deprecated.")
+        if only_org_public is not None:
+            logger.warning("The 'only_org_public' parameter is deprecated.")
+        if continue_on_failure is not None:
+            logger.warning("The 'continue_on_failure' parameter is deprecated.")
+
+        # Initialize vision LLM using the mixin
+        self.initialize_vision_llm()
 
         if (
             not include_shared_drives
@@ -237,6 +244,7 @@ class GoogleDriveConnector(LoadConnector, PollConnector, SlimConnector):
             credentials=credentials,
             source=DocumentSource.GOOGLE_DRIVE,
         )
+
         return new_creds_dict
 
     def _update_traversed_parent_ids(self, folder_id: str) -> None:
@@ -523,37 +531,53 @@ class GoogleDriveConnector(LoadConnector, PollConnector, SlimConnector):
         end: SecondsSinceUnixEpoch | None = None,
     ) -> GenerateDocumentsOutput:
         # Create a larger process pool for file conversion
-        convert_func = partial(
-            _convert_single_file, self.creds, self.primary_admin_email
-        )
-
-        # Process files in larger batches
-        LARGE_BATCH_SIZE = self.batch_size * 4
-        files_to_process = []
-        # Gather the files into batches to be processed in parallel
-        for file in self._fetch_drive_items(is_slim=False, start=start, end=end):
-            if (
-                file.get("size")
-                and int(cast(str, file.get("size"))) > MAX_FILE_SIZE_BYTES
-            ):
-                logger.warning(
-                    f"Skipping file {file.get('name', 'Unknown')} as it is too large: {file.get('size')} bytes"
-                )
-                continue
-
-            files_to_process.append(file)
-            if len(files_to_process) >= LARGE_BATCH_SIZE:
-                yield from _process_files_batch(
-                    files_to_process, convert_func, self.batch_size
-                )
-                files_to_process = []
-
-        # Process any remaining files
-        if files_to_process:
-            yield from _process_files_batch(
-                files_to_process, convert_func, self.batch_size
+        with ThreadPoolExecutor(max_workers=8) as executor:
+            # Prepare a partial function with the credentials and admin email
+            convert_func = partial(
+                _convert_single_file,
+                self.creds,
+                self.primary_admin_email,
+                image_analysis_llm=self.image_analysis_llm,  # Use the mixin's LLM
             )
 
+            # Fetch files in batches
+            files_batch: list[GoogleDriveFileType] = []
+            for file in self._fetch_drive_items(is_slim=False, start=start, end=end):
+                files_batch.append(file)
+
+                if len(files_batch) >= self.batch_size:
+                    # Process the batch
+                    futures = [
+                        executor.submit(convert_func, file) for file in files_batch
+                    ]
+                    documents = []
+                    for future in as_completed(futures):
+                        try:
+                            doc = future.result()
+                            if doc is not None:
+                                documents.append(doc)
+                        except Exception as e:
+                            logger.error(f"Error converting file: {e}")
+
+                    if documents:
+                        yield documents
+                    files_batch = []
+
+            # Process any remaining files
+            if files_batch:
+                futures = [executor.submit(convert_func, file) for file in files_batch]
+                documents = []
+                for future in as_completed(futures):
+                    try:
+                        doc = future.result()
+                        if doc is not None:
+                            documents.append(doc)
+                    except Exception as e:
+                        logger.error(f"Error converting file: {e}")
+
+                if documents:
+                    yield documents
+
     def load_from_state(self) -> GenerateDocumentsOutput:
         try:
             yield from self._extract_docs_from_google_drive()

backend/onyx/connectors/google_drive/doc_conversion.py

@@ -9,7 +9,7 @@ from googleapiclient.errors import HttpError  # type: ignore
 
 from onyx.configs.app_configs import CONTINUE_ON_CONNECTOR_FAILURE
 from onyx.configs.constants import DocumentSource
-from onyx.configs.constants import IGNORE_FOR_QA
+from onyx.configs.constants import FileOrigin
 from onyx.connectors.google_drive.constants import DRIVE_FOLDER_TYPE
 from onyx.connectors.google_drive.constants import DRIVE_SHORTCUT_TYPE
 from onyx.connectors.google_drive.constants import UNSUPPORTED_FILE_TYPE_CONTENT
@@ -21,32 +21,88 @@ from onyx.connectors.google_utils.resources import GoogleDriveService
 from onyx.connectors.models import Document
 from onyx.connectors.models import Section
 from onyx.connectors.models import SlimDocument
-from onyx.file_processing.extract_file_text import docx_to_text
+from onyx.db.engine import get_session_with_current_tenant
+from onyx.file_processing.extract_file_text import docx_to_text_and_images
 from onyx.file_processing.extract_file_text import pptx_to_text
 from onyx.file_processing.extract_file_text import read_pdf_file
+from onyx.file_processing.file_validation import is_valid_image_type
+from onyx.file_processing.image_summarization import summarize_image_with_error_handling
+from onyx.file_processing.image_utils import store_image_and_create_section
 from onyx.file_processing.unstructured import get_unstructured_api_key
 from onyx.file_processing.unstructured import unstructured_to_text
+from onyx.llm.interfaces import LLM
 from onyx.utils.logger import setup_logger
 
 logger = setup_logger()
 
 
-# these errors don't represent a failure in the connector, but simply files
-# that can't / shouldn't be indexed
-ERRORS_TO_CONTINUE_ON = [
-    "cannotExportFile",
-    "exportSizeLimitExceeded",
-    "cannotDownloadFile",
-]
+def _summarize_drive_image(
+    image_data: bytes, image_name: str, image_analysis_llm: LLM | None
+) -> str:
+    """
+    Summarize the given image using the provided LLM.
+    """
+    if not image_analysis_llm:
+        return ""
+
+    return (
+        summarize_image_with_error_handling(
+            llm=image_analysis_llm,
+            image_data=image_data,
+            context_name=image_name,
+        )
+        or ""
+    )
+
+
+def is_gdrive_image_mime_type(mime_type: str) -> bool:
+    """
+    Return True if the mime_type is a common image type in GDrive.
+    (e.g. 'image/png', 'image/jpeg')
+    """
+    return is_valid_image_type(mime_type)
 
 
 def _extract_sections_basic(
-    file: dict[str, str], service: GoogleDriveService
+    file: dict[str, str],
+    service: GoogleDriveService,
+    image_analysis_llm: LLM | None = None,
 ) -> list[Section]:
+    """
+    Extends the existing logic to handle either a docx with embedded images
+    or standalone images (PNG, JPG, etc).
+    """
     mime_type = file["mimeType"]
     link = file["webViewLink"]
+    file_name = file.get("name", file["id"])
     supported_file_types = set(item.value for item in GDriveMimeType)
 
+    # 1) If the file is an image, retrieve the raw bytes, optionally summarize
+    if is_gdrive_image_mime_type(mime_type):
+        try:
+            response = service.files().get_media(fileId=file["id"]).execute()
+
+            with get_session_with_current_tenant() as db_session:
+                section, _ = store_image_and_create_section(
+                    db_session=db_session,
+                    image_data=response,
+                    file_name=file["id"],
+                    display_name=file_name,
+                    media_type=mime_type,
+                    llm=image_analysis_llm,
+                    file_origin=FileOrigin.CONNECTOR,
+                )
+                return [section]
+        except Exception as e:
+            logger.warning(f"Failed to fetch or summarize image: {e}")
+            return [
+                Section(
+                    link=link,
+                    text="",
+                    image_file_name=link,
+                )
+            ]
+
     if mime_type not in supported_file_types:
         # Unsupported file types can still have a title, finding this way is still useful
         return [Section(link=link, text=UNSUPPORTED_FILE_TYPE_CONTENT)]
@@ -185,45 +241,63 @@ def _extract_sections_basic(
             GDriveMimeType.PLAIN_TEXT.value,
             GDriveMimeType.MARKDOWN.value,
         ]:
-            return [
-                Section(
-                    link=link,
-                    text=service.files()
-                    .get_media(fileId=file["id"])
-                    .execute()
-                    .decode("utf-8"),
-                )
-            ]
+            text_data = (
+                service.files().get_media(fileId=file["id"]).execute().decode("utf-8")
+            )
+            return [Section(link=link, text=text_data)]
+
         # ---------------------------
         # Word, PowerPoint, PDF files
-        if mime_type in [
+        elif mime_type in [
             GDriveMimeType.WORD_DOC.value,
             GDriveMimeType.POWERPOINT.value,
             GDriveMimeType.PDF.value,
         ]:
-            response = service.files().get_media(fileId=file["id"]).execute()
+            response_bytes = service.files().get_media(fileId=file["id"]).execute()
+
+            # Optionally use Unstructured
             if get_unstructured_api_key():
-                return [
-                    Section(
-                        link=link,
-                        text=unstructured_to_text(
-                            file=io.BytesIO(response),
-                            file_name=file.get("name", file["id"]),
-                        ),
-                    )
-                ]
+                text = unstructured_to_text(
+                    file=io.BytesIO(response_bytes),
+                    file_name=file_name,
+                )
+                return [Section(link=link, text=text)]
 
             if mime_type == GDriveMimeType.WORD_DOC.value:
-                return [
-                    Section(link=link, text=docx_to_text(file=io.BytesIO(response)))
-                ]
+                # Use docx_to_text_and_images to get text plus embedded images
+                text, embedded_images = docx_to_text_and_images(
+                    file=io.BytesIO(response_bytes),
+                )
+                sections = []
+                if text.strip():
+                    sections.append(Section(link=link, text=text.strip()))
+
+                # Process each embedded image using the standardized function
+                with get_session_with_current_tenant() as db_session:
+                    for idx, (img_data, img_name) in enumerate(
+                        embedded_images, start=1
+                    ):
+                        # Create a unique identifier for the embedded image
+                        embedded_id = f"{file['id']}_embedded_{idx}"
+
+                        section, _ = store_image_and_create_section(
+                            db_session=db_session,
+                            image_data=img_data,
+                            file_name=embedded_id,
+                            display_name=img_name or f"{file_name} - image {idx}",
+                            llm=image_analysis_llm,
+                            file_origin=FileOrigin.CONNECTOR,
+                        )
+                        sections.append(section)
+                return sections
+
             elif mime_type == GDriveMimeType.PDF.value:
-                text, _ = read_pdf_file(file=io.BytesIO(response))
+                text, _pdf_meta, images = read_pdf_file(io.BytesIO(response_bytes))
                 return [Section(link=link, text=text)]
+
             elif mime_type == GDriveMimeType.POWERPOINT.value:
-                return [
-                    Section(link=link, text=pptx_to_text(file=io.BytesIO(response)))
-                ]
+                text_data = pptx_to_text(io.BytesIO(response_bytes))
+                return [Section(link=link, text=text_data)]
 
         # Catch-all case, should not happen since there should be specific handling
         # for each of the supported file types
@@ -231,7 +305,8 @@ def _extract_sections_basic(
         logger.error(error_message)
         raise ValueError(error_message)
 
-    except Exception:
+    except Exception as e:
+        logger.exception(f"Error extracting sections from file: {e}")
         return [Section(link=link, text=UNSUPPORTED_FILE_TYPE_CONTENT)]
 
 
@@ -239,74 +314,62 @@ def convert_drive_item_to_document(
     file: GoogleDriveFileType,
     drive_service: GoogleDriveService,
     docs_service: GoogleDocsService,
+    image_analysis_llm: LLM | None,
 ) -> Document | None:
+    """
+    Main entry point for converting a Google Drive file => Document object.
+    Now we accept an optional `llm` to pass to `_extract_sections_basic`.
+    """
     try:
-        # Skip files that are shortcuts
-        if file.get("mimeType") == DRIVE_SHORTCUT_TYPE:
-            logger.info("Ignoring Drive Shortcut Filetype")
-            return None
-        # Skip files that are folders
-        if file.get("mimeType") == DRIVE_FOLDER_TYPE:
-            logger.info("Ignoring Drive Folder Filetype")
+        # skip shortcuts or folders
+        if file.get("mimeType") in [DRIVE_SHORTCUT_TYPE, DRIVE_FOLDER_TYPE]:
+            logger.info("Skipping shortcut/folder.")
             return None
 
+        # If it's a Google Doc, we might do advanced parsing
         sections: list[Section] = []
-
-        # Special handling for Google Docs to preserve structure, link
-        # to headers
         if file.get("mimeType") == GDriveMimeType.DOC.value:
             try:
+                # get_document_sections is the advanced approach for Google Docs
                 sections = get_document_sections(docs_service, file["id"])
             except Exception as e:
                 logger.warning(
-                    f"Ran into exception '{e}' when pulling sections from Google Doc '{file['name']}'."
-                    " Falling back to basic extraction."
+                    f"Failed to pull google doc sections from '{file['name']}': {e}. "
+                    "Falling back to basic extraction."
                 )
-        # NOTE: this will run for either (1) the above failed or (2) the file is not a Google Doc
+
+        # If not a doc, or if we failed above, do our 'basic' approach
         if not sections:
-            try:
-                # For all other file types just extract the text
-                sections = _extract_sections_basic(file, drive_service)
+            sections = _extract_sections_basic(file, drive_service, image_analysis_llm)
 
-            except HttpError as e:
-                reason = e.error_details[0]["reason"] if e.error_details else e.reason
-                message = e.error_details[0]["message"] if e.error_details else e.reason
-                if e.status_code == 403 and reason in ERRORS_TO_CONTINUE_ON:
-                    logger.warning(
-                        f"Could not export file '{file['name']}' due to '{message}', skipping..."
-                    )
-                    return None
-
-                raise
         if not sections:
             return None
 
+        doc_id = file["webViewLink"]
+        updated_time = datetime.fromisoformat(file["modifiedTime"]).astimezone(
+            timezone.utc
+        )
+
         return Document(
-            id=file["webViewLink"],
+            id=doc_id,
             sections=sections,
             source=DocumentSource.GOOGLE_DRIVE,
             semantic_identifier=file["name"],
-            doc_updated_at=datetime.fromisoformat(file["modifiedTime"]).astimezone(
-                timezone.utc
-            ),
-            metadata={}
-            if any(section.text for section in sections)
-            else {IGNORE_FOR_QA: "True"},
+            doc_updated_at=updated_time,
+            metadata={},  # or any metadata from 'file'
             additional_info=file.get("id"),
         )
-    except Exception as e:
-        if not CONTINUE_ON_CONNECTOR_FAILURE:
-            raise e
 
-        logger.exception("Ran into exception when pulling a file from Google Drive")
+    except Exception as e:
+        logger.exception(f"Error converting file '{file.get('name')}' to Document: {e}")
+        if not CONTINUE_ON_CONNECTOR_FAILURE:
+            raise
     return None
 
 
 def build_slim_document(file: GoogleDriveFileType) -> SlimDocument | None:
-    # Skip files that are folders or shortcuts
     if file.get("mimeType") in [DRIVE_FOLDER_TYPE, DRIVE_SHORTCUT_TYPE]:
         return None
-
     return SlimDocument(
         id=file["webViewLink"],
         perm_sync_data={

backend/onyx/connectors/models.py

@@ -28,7 +28,8 @@ class ConnectorMissingCredentialError(PermissionError):
 
 class Section(BaseModel):
     text: str
-    link: str | None
+    link: str | None = None
+    image_file_name: str | None = None
 
 
 class BasicExpertInfo(BaseModel):

backend/onyx/connectors/notion/connector.py

@@ -19,7 +19,7 @@ from onyx.connectors.cross_connector_utils.rate_limit_wrapper import (
 from onyx.connectors.exceptions import ConnectorValidationError
 from onyx.connectors.exceptions import CredentialExpiredError
 from onyx.connectors.exceptions import InsufficientPermissionsError
-from onyx.connectors.exceptions import UnexpectedError
+from onyx.connectors.exceptions import UnexpectedValidationError
 from onyx.connectors.interfaces import GenerateDocumentsOutput
 from onyx.connectors.interfaces import LoadConnector
 from onyx.connectors.interfaces import PollConnector
@@ -671,12 +671,12 @@ class NotionConnector(LoadConnector, PollConnector):
                     "Please try again later."
                 )
             else:
-                raise UnexpectedError(
+                raise UnexpectedValidationError(
                     f"Unexpected Notion HTTP error (status={status_code}): {http_err}"
                 ) from http_err
 
         except Exception as exc:
-            raise UnexpectedError(
+            raise UnexpectedValidationError(
                 f"Unexpected error during Notion settings validation: {exc}"
             )
 

backend/onyx/connectors/slack/connector.py

@@ -21,7 +21,7 @@ from onyx.configs.constants import DocumentSource
 from onyx.connectors.exceptions import ConnectorValidationError
 from onyx.connectors.exceptions import CredentialExpiredError
 from onyx.connectors.exceptions import InsufficientPermissionsError
-from onyx.connectors.exceptions import UnexpectedError
+from onyx.connectors.exceptions import UnexpectedValidationError
 from onyx.connectors.interfaces import CheckpointConnector
 from onyx.connectors.interfaces import CheckpointOutput
 from onyx.connectors.interfaces import GenerateSlimDocumentOutput
@@ -702,7 +702,9 @@ class SlackConnector(SlimConnector, CheckpointConnector):
                     raise CredentialExpiredError(
                         f"Invalid or expired Slack bot token ({error_msg})."
                     )
-                raise UnexpectedError(f"Slack API returned a failure: {error_msg}")
+                raise UnexpectedValidationError(
+                    f"Slack API returned a failure: {error_msg}"
+                )
 
             # 3) If channels are specified, verify each is accessible
             if self.channels:
@@ -740,13 +742,13 @@ class SlackConnector(SlimConnector, CheckpointConnector):
                 raise CredentialExpiredError(
                     f"Invalid or expired Slack bot token ({slack_error})."
                 )
-            raise UnexpectedError(
+            raise UnexpectedValidationError(
                 f"Unexpected Slack error '{slack_error}' during settings validation."
             )
         except ConnectorValidationError as e:
             raise e
         except Exception as e:
-            raise UnexpectedError(
+            raise UnexpectedValidationError(
                 f"Unexpected error during Slack settings validation: {e}"
             )
 

backend/onyx/connectors/slack/utils.py

@@ -72,6 +72,7 @@ def make_slack_api_rate_limited(
     @wraps(call)
     def rate_limited_call(**kwargs: Any) -> SlackResponse:
         last_exception = None
+
         for _ in range(max_retries):
             try:
                 # Make the API call

backend/onyx/connectors/teams/connector.py

@@ -16,7 +16,7 @@ from onyx.connectors.cross_connector_utils.miscellaneous_utils import time_str_t
 from onyx.connectors.exceptions import ConnectorValidationError
 from onyx.connectors.exceptions import CredentialExpiredError
 from onyx.connectors.exceptions import InsufficientPermissionsError
-from onyx.connectors.exceptions import UnexpectedError
+from onyx.connectors.exceptions import UnexpectedValidationError
 from onyx.connectors.interfaces import GenerateDocumentsOutput
 from onyx.connectors.interfaces import LoadConnector
 from onyx.connectors.interfaces import PollConnector
@@ -302,7 +302,7 @@ class TeamsConnector(LoadConnector, PollConnector):
                 raise InsufficientPermissionsError(
                     "Your app lacks sufficient permissions to read Teams (403 Forbidden)."
                 )
-            raise UnexpectedError(f"Unexpected error retrieving teams: {e}")
+            raise UnexpectedValidationError(f"Unexpected error retrieving teams: {e}")
 
         except Exception as e:
             error_str = str(e).lower()

backend/onyx/connectors/vision_enabled_connector.py

@@ -0,0 +1,45 @@
+"""
+Mixin for connectors that need vision capabilities.
+"""
+from onyx.configs.llm_configs import get_image_extraction_and_analysis_enabled
+from onyx.llm.factory import get_default_llm_with_vision
+from onyx.llm.interfaces import LLM
+from onyx.utils.logger import setup_logger
+
+logger = setup_logger()
+
+
+class VisionEnabledConnector:
+    """
+    Mixin for connectors that need vision capabilities.
+
+    This mixin provides a standard way to initialize a vision-capable LLM
+    for image analysis during indexing.
+
+    Usage:
+        class MyConnector(LoadConnector, VisionEnabledConnector):
+            def __init__(self, ...):
+                super().__init__(...)
+                self.initialize_vision_llm()
+    """
+
+    def initialize_vision_llm(self) -> None:
+        """
+        Initialize a vision-capable LLM if enabled by configuration.
+
+        Sets self.image_analysis_llm to the LLM instance or None if disabled.
+        """
+        self.image_analysis_llm: LLM | None = None
+        if get_image_extraction_and_analysis_enabled():
+            try:
+                self.image_analysis_llm = get_default_llm_with_vision()
+                if self.image_analysis_llm is None:
+                    logger.warning(
+                        "No LLM with vision found; image summarization will be disabled"
+                    )
+            except Exception as e:
+                logger.warning(
+                    f"Failed to initialize vision LLM due to an error: {str(e)}. "
+                    "Image summarization will be disabled."
+                )
+                self.image_analysis_llm = None

backend/onyx/connectors/web/connector.py

@@ -28,7 +28,7 @@ from onyx.configs.constants import DocumentSource
 from onyx.connectors.exceptions import ConnectorValidationError
 from onyx.connectors.exceptions import CredentialExpiredError
 from onyx.connectors.exceptions import InsufficientPermissionsError
-from onyx.connectors.exceptions import UnexpectedError
+from onyx.connectors.exceptions import UnexpectedValidationError
 from onyx.connectors.interfaces import GenerateDocumentsOutput
 from onyx.connectors.interfaces import LoadConnector
 from onyx.connectors.models import Document
@@ -157,6 +157,7 @@ def get_internal_links(
 
 def start_playwright() -> Tuple[Playwright, BrowserContext]:
     playwright = sync_playwright().start()
+
     browser = playwright.chromium.launch(headless=True)
 
     context = browser.new_context()
@@ -320,7 +321,8 @@ class WebConnector(LoadConnector):
                 logger.warning(last_error)
                 continue
 
-            logger.info(f"{len(visited_links)}: Visiting {initial_url}")
+            index = len(visited_links)
+            logger.info(f"{index}: Visiting {initial_url}")
 
             try:
                 check_internet_connection(initial_url)
@@ -331,7 +333,7 @@ class WebConnector(LoadConnector):
                 if initial_url.split(".")[-1] == "pdf":
                     # PDF files are not checked for links
                     response = requests.get(initial_url)
-                    page_text, metadata = read_pdf_file(
+                    page_text, metadata, images = read_pdf_file(
                         file=io.BytesIO(response.content)
                     )
                     last_modified = response.headers.get("Last-Modified")
@@ -371,12 +373,10 @@ class WebConnector(LoadConnector):
                     initial_url = final_url
                     if initial_url in visited_links:
                         logger.info(
-                            f"{len(visited_links)}: {initial_url} redirected to {final_url} - already indexed"
+                            f"{index}: {initial_url} redirected to {final_url} - already indexed"
                         )
                         continue
-                    logger.info(
-                        f"{len(visited_links)}: {initial_url} redirected to {final_url}"
-                    )
+                    logger.info(f"{index}: {initial_url} redirected to {final_url}")
                     visited_links.add(initial_url)
 
                 if self.scroll_before_scraping:
@@ -410,7 +410,9 @@ class WebConnector(LoadConnector):
                 """For websites containing iframes that need to be scraped,
                 the code below can extract text from within these iframes.
                 """
-                logger.info(f"Length of cleaned text {len(parsed_html.cleaned_text)}")
+                logger.debug(
+                    f"{index}: Length of cleaned text {len(parsed_html.cleaned_text)}"
+                )
                 if JAVASCRIPT_DISABLED_MESSAGE in parsed_html.cleaned_text:
                     iframe_count = page.frame_locator("iframe").locator("html").count()
                     if iframe_count > 0:
@@ -427,11 +429,13 @@ class WebConnector(LoadConnector):
                         else:
                             parsed_html.cleaned_text += "\n" + document_text
 
-                # Sometimes pages with #! will server duplicate content
+                # Sometimes pages with #! will serve duplicate content
                 # There are also just other ways this can happen
-                hashed_text = hash(parsed_html.cleaned_text)
+                hashed_text = hash((parsed_html.title, parsed_html.cleaned_text))
                 if hashed_text in content_hashes:
-                    logger.info(f"Skipping duplicate content for {initial_url}")
+                    logger.info(
+                        f"{index}: Skipping duplicate title + content for {initial_url}"
+                    )
                     continue
                 content_hashes.add(hashed_text)
 
@@ -525,7 +529,9 @@ class WebConnector(LoadConnector):
                 )
             else:
                 # Could be a 5xx or another error, treat as unexpected
-                raise UnexpectedError(f"Unexpected error validating '{test_url}': {e}")
+                raise UnexpectedValidationError(
+                    f"Unexpected error validating '{test_url}': {e}"
+                )
 
 
 if __name__ == "__main__":

backend/onyx/context/search/postprocessing/postprocessing.py

@@ -1,12 +1,17 @@
+import base64
 from collections.abc import Callable
 from collections.abc import Iterator
 from typing import cast
 
 import numpy
+from langchain_core.messages import BaseMessage
+from langchain_core.messages import HumanMessage
+from langchain_core.messages import SystemMessage
 
 from onyx.chat.models import SectionRelevancePiece
 from onyx.configs.app_configs import BLURB_SIZE
 from onyx.configs.constants import RETURN_SEPARATOR
+from onyx.configs.llm_configs import get_search_time_image_analysis_enabled
 from onyx.configs.model_configs import CROSS_ENCODER_RANGE_MAX
 from onyx.configs.model_configs import CROSS_ENCODER_RANGE_MIN
 from onyx.context.search.enums import LLMEvaluationType
@@ -18,11 +23,15 @@ from onyx.context.search.models import MAX_METRICS_CONTENT
 from onyx.context.search.models import RerankingDetails
 from onyx.context.search.models import RerankMetricsContainer
 from onyx.context.search.models import SearchQuery
+from onyx.db.engine import get_session_with_current_tenant
 from onyx.document_index.document_index_utils import (
     translate_boost_count_to_multiplier,
 )
+from onyx.file_store.file_store import get_default_file_store
 from onyx.llm.interfaces import LLM
+from onyx.llm.utils import message_to_string
 from onyx.natural_language_processing.search_nlp_models import RerankingModel
+from onyx.prompts.image_analysis import IMAGE_ANALYSIS_SYSTEM_PROMPT
 from onyx.secondary_llm_flows.chunk_usefulness import llm_batch_eval_sections
 from onyx.utils.logger import setup_logger
 from onyx.utils.threadpool_concurrency import FunctionCall
@@ -30,6 +39,124 @@ from onyx.utils.threadpool_concurrency import run_functions_in_parallel
 from onyx.utils.timing import log_function_time
 
 
+def update_image_sections_with_query(
+    sections: list[InferenceSection],
+    query: str,
+    llm: LLM,
+) -> None:
+    """
+    For each chunk in each section that has an image URL, call an LLM to produce
+    a new 'content' string that directly addresses the user's query about that image.
+    This implementation uses parallel processing for efficiency.
+    """
+    logger = setup_logger()
+    logger.debug(f"Starting image section update with query: {query}")
+
+    chunks_with_images = []
+    for section in sections:
+        for chunk in section.chunks:
+            if chunk.image_file_name:
+                chunks_with_images.append(chunk)
+
+    if not chunks_with_images:
+        logger.debug("No images to process in the sections")
+        return  # No images to process
+
+    logger.info(f"Found {len(chunks_with_images)} chunks with images to process")
+
+    def process_image_chunk(chunk: InferenceChunk) -> tuple[str, str]:
+        try:
+            logger.debug(
+                f"Processing image chunk with ID: {chunk.unique_id}, image: {chunk.image_file_name}"
+            )
+            with get_session_with_current_tenant() as db_session:
+                file_record = get_default_file_store(db_session).read_file(
+                    cast(str, chunk.image_file_name), mode="b"
+                )
+                if not file_record:
+                    logger.error(f"Image file not found: {chunk.image_file_name}")
+                    raise Exception("File not found")
+                file_content = file_record.read()
+                image_base64 = base64.b64encode(file_content).decode()
+                logger.debug(
+                    f"Successfully loaded image data for {chunk.image_file_name}"
+                )
+
+            messages: list[BaseMessage] = [
+                SystemMessage(content=IMAGE_ANALYSIS_SYSTEM_PROMPT),
+                HumanMessage(
+                    content=[
+                        {
+                            "type": "text",
+                            "text": (
+                                f"The user's question is: '{query}'. "
+                                "Please analyze the following image in that context:\n"
+                            ),
+                        },
+                        {
+                            "type": "image_url",
+                            "image_url": {
+                                "url": f"data:image/jpeg;base64,{image_base64}",
+                            },
+                        },
+                    ]
+                ),
+            ]
+
+            raw_response = llm.invoke(messages)
+
+            answer_text = message_to_string(raw_response).strip()
+            return (
+                chunk.unique_id,
+                answer_text if answer_text else "No relevant info found.",
+            )
+
+        except Exception:
+            logger.exception(
+                f"Error updating image section with query source image url: {chunk.image_file_name}"
+            )
+            return chunk.unique_id, "Error analyzing image."
+
+    image_processing_tasks = [
+        FunctionCall(process_image_chunk, (chunk,)) for chunk in chunks_with_images
+    ]
+
+    logger.info(
+        f"Starting parallel processing of {len(image_processing_tasks)} image tasks"
+    )
+    image_processing_results = run_functions_in_parallel(image_processing_tasks)
+    logger.info(
+        f"Completed parallel processing with {len(image_processing_results)} results"
+    )
+
+    # Create a mapping of chunk IDs to their processed content
+    chunk_id_to_content = {}
+    success_count = 0
+    for task_id, result in image_processing_results.items():
+        if result:
+            chunk_id, content = result
+            chunk_id_to_content[chunk_id] = content
+            success_count += 1
+        else:
+            logger.error(f"Task {task_id} failed to return a valid result")
+
+    logger.info(
+        f"Successfully processed {success_count}/{len(image_processing_results)} images"
+    )
+
+    # Update the chunks with the processed content
+    updated_count = 0
+    for section in sections:
+        for chunk in section.chunks:
+            if chunk.unique_id in chunk_id_to_content:
+                chunk.content = chunk_id_to_content[chunk.unique_id]
+                updated_count += 1
+
+    logger.info(
+        f"Updated content for {updated_count} chunks with image analysis results"
+    )
+
+
 logger = setup_logger()
 
 
@@ -286,6 +413,10 @@ def search_postprocessing(
         # NOTE: if we don't rerank, we can return the chunks immediately
         # since we know this is the final order.
         # This way the user experience isn't delayed by the LLM step
+        if get_search_time_image_analysis_enabled():
+            update_image_sections_with_query(
+                retrieved_sections, search_query.query, llm
+            )
         _log_top_section_links(search_query.search_type.value, retrieved_sections)
         yield retrieved_sections
         sections_yielded = True
@@ -323,6 +454,13 @@ def search_postprocessing(
             )
         else:
             _log_top_section_links(search_query.search_type.value, reranked_sections)
+
+            # Add the image processing step here
+            if get_search_time_image_analysis_enabled():
+                update_image_sections_with_query(
+                    reranked_sections, search_query.query, llm
+                )
+
             yield reranked_sections
 
     llm_selected_section_ids = (

backend/onyx/db/chat.py

@@ -3,6 +3,7 @@ from datetime import datetime
 from datetime import timedelta
 from typing import Any
 from typing import cast
+from typing import Tuple
 from uuid import UUID
 
 from fastapi import HTTPException
@@ -11,6 +12,7 @@ from sqlalchemy import desc
 from sqlalchemy import func
 from sqlalchemy import nullsfirst
 from sqlalchemy import or_
+from sqlalchemy import Row
 from sqlalchemy import select
 from sqlalchemy import update
 from sqlalchemy.exc import MultipleResultsFound
@@ -375,24 +377,33 @@ def delete_chat_session(
     db_session.commit()
 
 
-def delete_chat_sessions_older_than(days_old: int, db_session: Session) -> None:
+def get_chat_sessions_older_than(
+    days_old: int, db_session: Session
+) -> list[tuple[UUID | None, UUID]]:
+    """
+    Retrieves chat sessions older than a specified number of days.
+
+    Args:
+        days_old: The number of days to consider as "old".
+        db_session: The database session.
+
+    Returns:
+        A list of tuples, where each tuple contains the user_id (can be None) and the chat_session_id of an old chat session.
+    """
+
     cutoff_time = datetime.utcnow() - timedelta(days=days_old)
-    old_sessions = db_session.execute(
+    old_sessions: Sequence[Row[Tuple[UUID | None, UUID]]] = db_session.execute(
         select(ChatSession.user_id, ChatSession.id).where(
             ChatSession.time_created < cutoff_time
         )
     ).fetchall()
 
-    for user_id, session_id in old_sessions:
-        try:
-            delete_chat_session(
-                user_id, session_id, db_session, include_deleted=True, hard_delete=True
-            )
-        except Exception:
-            logger.exception(
-                "delete_chat_session exceptioned. "
-                f"user_id={user_id} session_id={session_id}"
-            )
+    # convert old_sessions to a conventional list of tuples
+    returned_sessions: list[tuple[UUID | None, UUID]] = [
+        (user_id, session_id) for user_id, session_id in old_sessions
+    ]
+
+    return returned_sessions
 
 
 def get_chat_message(

backend/onyx/db/models.py

@@ -1790,6 +1790,7 @@ class ChannelConfig(TypedDict):
     channel_name: str | None  # None for default channel config
     respond_tag_only: NotRequired[bool]  # defaults to False
     respond_to_bots: NotRequired[bool]  # defaults to False
+    is_ephemeral: NotRequired[bool]  # defaults to False
     respond_member_group_list: NotRequired[list[str]]
     answer_filters: NotRequired[list[AllowedAnswerFilters]]
     # If None then no follow up

backend/onyx/db/pg_file_store.py

@@ -148,3 +148,28 @@ def upsert_pgfilestore(
         db_session.commit()
 
     return pgfilestore
+
+
+def save_bytes_to_pgfilestore(
+    db_session: Session,
+    raw_bytes: bytes,
+    media_type: str,
+    identifier: str,
+    display_name: str,
+    file_origin: FileOrigin = FileOrigin.OTHER,
+) -> PGFileStore:
+    """
+    Saves raw bytes to PGFileStore and returns the resulting record.
+    """
+    file_name = f"{file_origin.name.lower()}_{identifier}"
+    lobj_oid = create_populate_lobj(BytesIO(raw_bytes), db_session)
+    pgfilestore = upsert_pgfilestore(
+        file_name=file_name,
+        display_name=display_name,
+        file_origin=file_origin,
+        file_type=media_type,
+        lobj_oid=lobj_oid,
+        db_session=db_session,
+        commit=True,
+    )
+    return pgfilestore

backend/onyx/db/seeding/chat_history_seeding.py

@@ -0,0 +1,53 @@
+import random
+from datetime import datetime
+from datetime import timedelta
+
+from onyx.configs.constants import MessageType
+from onyx.db.chat import create_chat_session
+from onyx.db.chat import create_new_chat_message
+from onyx.db.chat import get_or_create_root_message
+from onyx.db.engine import get_session_with_current_tenant
+from onyx.db.models import ChatSession
+
+
+def seed_chat_history(num_sessions: int, num_messages: int, days: int) -> None:
+    """Utility function to seed chat history for testing.
+
+    num_sessions: the number of sessions to seed
+    num_messages: the number of messages to seed per sessions
+    days: the number of days looking backwards from the current time over which to randomize
+    the times.
+    """
+    with get_session_with_current_tenant() as db_session:
+        for y in range(0, num_sessions):
+            create_chat_session(db_session, f"pytest_session_{y}", None, None)
+
+        # randomize all session times
+        rows = db_session.query(ChatSession).all()
+        for row in rows:
+            row.time_created = datetime.utcnow() - timedelta(
+                days=random.randint(0, days)
+            )
+            row.time_updated = row.time_created + timedelta(
+                minutes=random.randint(0, 10)
+            )
+
+            root_message = get_or_create_root_message(row.id, db_session)
+
+            for x in range(0, num_messages):
+                chat_message = create_new_chat_message(
+                    row.id,
+                    root_message,
+                    f"pytest_message_{x}",
+                    None,
+                    0,
+                    MessageType.USER,
+                    db_session,
+                )
+
+                chat_message.time_sent = row.time_created + timedelta(
+                    minutes=random.randint(0, 10)
+                )
+            db_session.commit()
+
+        db_session.commit()

backend/onyx/document_index/vespa/app_config/schemas/danswer_chunk.sd

@@ -55,6 +55,9 @@ schema DANSWER_CHUNK_NAME {
         field blurb type string {
             indexing: summary | attribute
         }
+        field image_file_name type string {
+            indexing: summary | attribute
+        }
         # https://docs.vespa.ai/en/attributes.html potential enum store for speed, but probably not worth it
         field source_type type string {
             indexing: summary | attribute
@@ -149,7 +152,7 @@ schema DANSWER_CHUNK_NAME {
 
     rank-profile hybrid_searchVARIABLE_DIM inherits default, default_rank {
         inputs {
-            query(query_embedding) tensor<EMBEDDING_PRECISION>(x[VARIABLE_DIM])
+            query(query_embedding) tensor<float>(x[VARIABLE_DIM])
         }
 
         function title_vector_score() {

backend/onyx/document_index/vespa/chunk_retrieval.py

@@ -31,6 +31,7 @@ from onyx.document_index.vespa_constants import DOC_UPDATED_AT
 from onyx.document_index.vespa_constants import DOCUMENT_ID
 from onyx.document_index.vespa_constants import DOCUMENT_ID_ENDPOINT
 from onyx.document_index.vespa_constants import HIDDEN
+from onyx.document_index.vespa_constants import IMAGE_FILE_NAME
 from onyx.document_index.vespa_constants import LARGE_CHUNK_REFERENCE_IDS
 from onyx.document_index.vespa_constants import MAX_ID_SEARCH_QUERY_SIZE
 from onyx.document_index.vespa_constants import MAX_OR_CONDITIONS
@@ -130,6 +131,7 @@ def _vespa_hit_to_inference_chunk(
         section_continuation=fields[SECTION_CONTINUATION],
         document_id=fields[DOCUMENT_ID],
         source_type=fields[SOURCE_TYPE],
+        image_file_name=fields.get(IMAGE_FILE_NAME),
         title=fields.get(TITLE),
         semantic_identifier=fields[SEMANTIC_IDENTIFIER],
         boost=fields.get(BOOST, 1),
@@ -211,6 +213,7 @@ def _get_chunks_via_visit_api(
 
         # Check if the response contains any documents
         response_data = response.json()
+
         if "documents" in response_data:
             for document in response_data["documents"]:
                 if filters.access_control_list:

backend/onyx/document_index/vespa/indexing_utils.py

@@ -32,6 +32,7 @@ from onyx.document_index.vespa_constants import DOCUMENT_ID
 from onyx.document_index.vespa_constants import DOCUMENT_ID_ENDPOINT
 from onyx.document_index.vespa_constants import DOCUMENT_SETS
 from onyx.document_index.vespa_constants import EMBEDDINGS
+from onyx.document_index.vespa_constants import IMAGE_FILE_NAME
 from onyx.document_index.vespa_constants import LARGE_CHUNK_REFERENCE_IDS
 from onyx.document_index.vespa_constants import METADATA
 from onyx.document_index.vespa_constants import METADATA_LIST
@@ -198,13 +199,13 @@ def _index_vespa_chunk(
         # which only calls VespaIndex.update
         ACCESS_CONTROL_LIST: {acl_entry: 1 for acl_entry in chunk.access.to_acl()},
         DOCUMENT_SETS: {document_set: 1 for document_set in chunk.document_sets},
+        IMAGE_FILE_NAME: chunk.image_file_name,
         BOOST: chunk.boost,
     }
 
     if multitenant:
         if chunk.tenant_id:
             vespa_document_fields[TENANT_ID] = chunk.tenant_id
-
     vespa_url = f"{DOCUMENT_ID_ENDPOINT.format(index_name=index_name)}/{vespa_chunk_id}"
     logger.debug(f'Indexing to URL "{vespa_url}"')
     res = http_client.post(

backend/onyx/document_index/vespa_constants.py

@@ -77,6 +77,7 @@ PRIMARY_OWNERS = "primary_owners"
 SECONDARY_OWNERS = "secondary_owners"
 RECENCY_BIAS = "recency_bias"
 HIDDEN = "hidden"
+IMAGE_FILE_NAME = "image_file_name"
 
 # Specific to Vespa, needed for highlighting matching keywords / section
 CONTENT_SUMMARY = "content_summary"
@@ -94,6 +95,7 @@ YQL_BASE = (
     f"{SEMANTIC_IDENTIFIER}, "
     f"{TITLE}, "
     f"{SECTION_CONTINUATION}, "
+    f"{IMAGE_FILE_NAME}, "
     f"{BOOST}, "
     f"{HIDDEN}, "
     f"{DOC_UPDATED_AT}, "

backend/onyx/file_processing/extract_file_text.py

@@ -9,15 +9,17 @@ from email.parser import Parser as EmailParser
 from io import BytesIO
 from pathlib import Path
 from typing import Any
-from typing import Dict
 from typing import IO
+from typing import List
+from typing import Tuple
 
 import chardet
 import docx  # type: ignore
 import openpyxl  # type: ignore
 import pptx  # type: ignore
-from docx import Document
+from docx import Document as DocxDocument
 from fastapi import UploadFile
+from PIL import Image
 from pypdf import PdfReader
 from pypdf.errors import PdfStreamError
 
@@ -31,10 +33,8 @@ from onyx.utils.logger import setup_logger
 
 logger = setup_logger()
 
-
 TEXT_SECTION_SEPARATOR = "\n\n"
 
-
 PLAIN_TEXT_FILE_EXTENSIONS = [
     ".txt",
     ".md",
@@ -49,7 +49,6 @@ PLAIN_TEXT_FILE_EXTENSIONS = [
     ".yaml",
 ]
 
-
 VALID_FILE_EXTENSIONS = PLAIN_TEXT_FILE_EXTENSIONS + [
     ".pdf",
     ".docx",
@@ -58,6 +57,16 @@ VALID_FILE_EXTENSIONS = PLAIN_TEXT_FILE_EXTENSIONS + [
     ".eml",
     ".epub",
     ".html",
+    ".png",
+    ".jpg",
+    ".jpeg",
+    ".webp",
+]
+
+IMAGE_MEDIA_TYPES = [
+    "image/png",
+    "image/jpeg",
+    "image/webp",
 ]
 
 
@@ -67,11 +76,13 @@ def is_text_file_extension(file_name: str) -> bool:
 
 def get_file_ext(file_path_or_name: str | Path) -> str:
     _, extension = os.path.splitext(file_path_or_name)
-    # standardize all extensions to be lowercase so that checks against
-    # VALID_FILE_EXTENSIONS and similar will work as intended
     return extension.lower()
 
 
+def is_valid_media_type(media_type: str) -> bool:
+    return media_type in IMAGE_MEDIA_TYPES
+
+
 def is_valid_file_ext(ext: str) -> bool:
     return ext in VALID_FILE_EXTENSIONS
 
@@ -79,17 +90,18 @@ def is_valid_file_ext(ext: str) -> bool:
 def is_text_file(file: IO[bytes]) -> bool:
     """
     checks if the first 1024 bytes only contain printable or whitespace characters
-    if it does, then we say its a plaintext file
+    if it does, then we say it's a plaintext file
     """
     raw_data = file.read(1024)
+    file.seek(0)
     text_chars = bytearray({7, 8, 9, 10, 12, 13, 27} | set(range(0x20, 0x100)) - {0x7F})
     return all(c in text_chars for c in raw_data)
 
 
 def detect_encoding(file: IO[bytes]) -> str:
     raw_data = file.read(50000)
-    encoding = chardet.detect(raw_data)["encoding"] or "utf-8"
     file.seek(0)
+    encoding = chardet.detect(raw_data)["encoding"] or "utf-8"
     return encoding
 
 
@@ -99,14 +111,14 @@ def is_macos_resource_fork_file(file_name: str) -> bool:
     )
 
 
-# To include additional metadata in the search index, add a .onyx_metadata.json file
-# to the zip file. This file should contain a list of objects with the following format:
-# [{ "filename": "file1.txt", "link": "https://example.com/file1.txt" }]
 def load_files_from_zip(
     zip_file_io: IO,
     ignore_macos_resource_fork_files: bool = True,
     ignore_dirs: bool = True,
 ) -> Iterator[tuple[zipfile.ZipInfo, IO[Any], dict[str, Any]]]:
+    """
+    If there's a .onyx_metadata.json in the zip, attach those metadata to each subfile.
+    """
     with zipfile.ZipFile(zip_file_io, "r") as zip_file:
         zip_metadata = {}
         try:
@@ -118,24 +130,31 @@ def load_files_from_zip(
                         # convert list of dicts to dict of dicts
                         zip_metadata = {d["filename"]: d for d in zip_metadata}
                 except json.JSONDecodeError:
-                    logger.warn(f"Unable to load {DANSWER_METADATA_FILENAME}")
+                    logger.warning(f"Unable to load {DANSWER_METADATA_FILENAME}")
         except KeyError:
             logger.info(f"No {DANSWER_METADATA_FILENAME} file")
 
         for file_info in zip_file.infolist():
-            with zip_file.open(file_info.filename, "r") as file:
-                if ignore_dirs and file_info.is_dir():
-                    continue
+            if ignore_dirs and file_info.is_dir():
+                continue
 
-                if (
-                    ignore_macos_resource_fork_files
-                    and is_macos_resource_fork_file(file_info.filename)
-                ) or file_info.filename == DANSWER_METADATA_FILENAME:
-                    continue
-                yield file_info, file, zip_metadata.get(file_info.filename, {})
+            if (
+                ignore_macos_resource_fork_files
+                and is_macos_resource_fork_file(file_info.filename)
+            ) or file_info.filename == DANSWER_METADATA_FILENAME:
+                continue
+
+            with zip_file.open(file_info.filename, "r") as subfile:
+                yield file_info, subfile, zip_metadata.get(file_info.filename, {})
 
 
 def _extract_onyx_metadata(line: str) -> dict | None:
+    """
+    Example: first line has:
+        <!-- DANSWER_METADATA={"title": "..."} -->
+      or
+        #DANSWER_METADATA={"title":"..."}
+    """
     html_comment_pattern = r"<!--\s*DANSWER_METADATA=\{(.*?)\}\s*-->"
     hashtag_pattern = r"#DANSWER_METADATA=\{(.*?)\}"
 
@@ -161,9 +180,13 @@ def read_text_file(
     errors: str = "replace",
     ignore_onyx_metadata: bool = True,
 ) -> tuple[str, dict]:
+    """
+    For plain text files. Optionally extracts Onyx metadata from the first line.
+    """
     metadata = {}
     file_content_raw = ""
     for ind, line in enumerate(file):
+        # decode
         try:
             line = line.decode(encoding) if isinstance(line, bytes) else line
         except UnicodeDecodeError:
@@ -173,131 +196,132 @@ def read_text_file(
                 else line
             )
 
-        if ind == 0:
-            metadata_or_none = (
-                None if ignore_onyx_metadata else _extract_onyx_metadata(line)
-            )
-            if metadata_or_none is not None:
-                metadata = metadata_or_none
-            else:
-                file_content_raw += line
-        else:
-            file_content_raw += line
+        # optionally parse metadata in the first line
+        if ind == 0 and not ignore_onyx_metadata:
+            potential_meta = _extract_onyx_metadata(line)
+            if potential_meta is not None:
+                metadata = potential_meta
+                continue
+
+        file_content_raw += line
 
     return file_content_raw, metadata
 
 
 def pdf_to_text(file: IO[Any], pdf_pass: str | None = None) -> str:
-    """Extract text from a PDF file."""
-    # Return only the extracted text from read_pdf_file
-    text, _ = read_pdf_file(file, pdf_pass)
+    """
+    Extract text from a PDF. For embedded images, a more complex approach is needed.
+    This is a minimal approach returning text only.
+    """
+    text, _, _ = read_pdf_file(file, pdf_pass)
     return text
 
 
 def read_pdf_file(
-    file: IO[Any],
-    pdf_pass: str | None = None,
-) -> tuple[str, dict]:
-    metadata: Dict[str, Any] = {}
+    file: IO[Any], pdf_pass: str | None = None, extract_images: bool = False
+) -> tuple[str, dict, list[tuple[bytes, str]]]:
+    """
+    Returns the text, basic PDF metadata, and optionally extracted images.
+    """
+    metadata: dict[str, Any] = {}
+    extracted_images: list[tuple[bytes, str]] = []
     try:
         pdf_reader = PdfReader(file)
 
-        # If marked as encrypted and a password is provided, try to decrypt
         if pdf_reader.is_encrypted and pdf_pass is not None:
             decrypt_success = False
-            if pdf_pass is not None:
-                try:
-                    decrypt_success = pdf_reader.decrypt(pdf_pass) != 0
-                except Exception:
-                    logger.error("Unable to decrypt pdf")
+            try:
+                decrypt_success = pdf_reader.decrypt(pdf_pass) != 0
+            except Exception:
+                logger.error("Unable to decrypt pdf")
 
             if not decrypt_success:
-                # By user request, keep files that are unreadable just so they
-                # can be discoverable by title.
-                return "", metadata
+                return "", metadata, []
         elif pdf_reader.is_encrypted:
-            logger.warning("No Password available to decrypt pdf, returning empty")
-            return "", metadata
+            logger.warning("No Password for an encrypted PDF, returning empty text.")
+            return "", metadata, []
 
-        # Extract metadata from the PDF, removing leading '/' from keys if present
-        # This standardizes the metadata keys for consistency
-        metadata = {}
+        # Basic PDF metadata
         if pdf_reader.metadata is not None:
             for key, value in pdf_reader.metadata.items():
                 clean_key = key.lstrip("/")
                 if isinstance(value, str) and value.strip():
                     metadata[clean_key] = value
-
                 elif isinstance(value, list) and all(
                     isinstance(item, str) for item in value
                 ):
                     metadata[clean_key] = ", ".join(value)
 
-        return (
-            TEXT_SECTION_SEPARATOR.join(
-                page.extract_text() for page in pdf_reader.pages
-            ),
-            metadata,
+        text = TEXT_SECTION_SEPARATOR.join(
+            page.extract_text() for page in pdf_reader.pages
         )
+
+        if extract_images:
+            for page_num, page in enumerate(pdf_reader.pages):
+                for image_file_object in page.images:
+                    image = Image.open(io.BytesIO(image_file_object.data))
+                    img_byte_arr = io.BytesIO()
+                    image.save(img_byte_arr, format=image.format)
+                    img_bytes = img_byte_arr.getvalue()
+
+                    image_name = (
+                        f"page_{page_num + 1}_image_{image_file_object.name}."
+                        f"{image.format.lower() if image.format else 'png'}"
+                    )
+                    extracted_images.append((img_bytes, image_name))
+
+        return text, metadata, extracted_images
+
     except PdfStreamError:
-        logger.exception("PDF file is not a valid PDF")
+        logger.exception("Invalid PDF file")
     except Exception:
         logger.exception("Failed to read PDF")
 
-    # File is still discoverable by title
-    # but the contents are not included as they cannot be parsed
-    return "", metadata
+    return "", metadata, []
 
 
-def docx_to_text(file: IO[Any]) -> str:
-    def is_simple_table(table: docx.table.Table) -> bool:
-        for row in table.rows:
-            # No omitted cells
-            if row.grid_cols_before > 0 or row.grid_cols_after > 0:
-                return False
-
-            # No nested tables
-            if any(cell.tables for cell in row.cells):
-                return False
-
-        return True
-
-    def extract_cell_text(cell: docx.table._Cell) -> str:
-        cell_paragraphs = [para.text.strip() for para in cell.paragraphs]
-        return " ".join(p for p in cell_paragraphs if p) or "N/A"
-
+def docx_to_text_and_images(
+    file: IO[Any],
+) -> Tuple[str, List[Tuple[bytes, str]]]:
+    """
+    Extract text from a docx. If embed_images=True, also extract inline images.
+    Return (text_content, list_of_images).
+    """
     paragraphs = []
+    embedded_images: List[Tuple[bytes, str]] = []
+
     doc = docx.Document(file)
-    for item in doc.iter_inner_content():
-        if isinstance(item, docx.text.paragraph.Paragraph):
-            paragraphs.append(item.text)
 
-        elif isinstance(item, docx.table.Table):
-            if not item.rows or not is_simple_table(item):
-                continue
+    # Grab text from paragraphs
+    for paragraph in doc.paragraphs:
+        paragraphs.append(paragraph.text)
 
-            # Every row is a new line, joined with a single newline
-            table_content = "\n".join(
-                [
-                    ",\t".join(extract_cell_text(cell) for cell in row.cells)
-                    for row in item.rows
-                ]
-            )
-            paragraphs.append(table_content)
+    # Reset position so we can re-load the doc (python-docx has read the stream)
+    # Note: if python-docx has fully consumed the stream, you may need to open it again from memory.
+    # For large docs, a more robust approach is needed.
+    # This is a simplified example.
 
-    # Docx already has good spacing between paragraphs
-    return "\n".join(paragraphs)
+    for rel_id, rel in doc.part.rels.items():
+        if "image" in rel.reltype:
+            # image is typically in rel.target_part.blob
+            image_bytes = rel.target_part.blob
+            image_name = rel.target_part.partname
+            # store
+            embedded_images.append((image_bytes, os.path.basename(str(image_name))))
+
+    text_content = "\n".join(paragraphs)
+    return text_content, embedded_images
 
 
 def pptx_to_text(file: IO[Any]) -> str:
     presentation = pptx.Presentation(file)
     text_content = []
     for slide_number, slide in enumerate(presentation.slides, start=1):
-        extracted_text = f"\nSlide {slide_number}:\n"
+        slide_text = f"\nSlide {slide_number}:\n"
         for shape in slide.shapes:
             if hasattr(shape, "text"):
-                extracted_text += shape.text + "\n"
-        text_content.append(extracted_text)
+                slide_text += shape.text + "\n"
+        text_content.append(slide_text)
     return TEXT_SECTION_SEPARATOR.join(text_content)
 
 
@@ -305,18 +329,21 @@ def xlsx_to_text(file: IO[Any]) -> str:
     workbook = openpyxl.load_workbook(file, read_only=True)
     text_content = []
     for sheet in workbook.worksheets:
-        sheet_string = "\n".join(
-            ",".join(map(str, row))
-            for row in sheet.iter_rows(min_row=1, values_only=True)
-        )
-        text_content.append(sheet_string)
+        rows = []
+        for row in sheet.iter_rows(min_row=1, values_only=True):
+            row_str = ",".join(str(cell) if cell is not None else "" for cell in row)
+            rows.append(row_str)
+        sheet_str = "\n".join(rows)
+        text_content.append(sheet_str)
     return TEXT_SECTION_SEPARATOR.join(text_content)
 
 
 def eml_to_text(file: IO[Any]) -> str:
-    text_file = io.TextIOWrapper(file, encoding=detect_encoding(file))
+    encoding = detect_encoding(file)
+    text_file = io.TextIOWrapper(file, encoding=encoding)
     parser = EmailParser()
     message = parser.parse(text_file)
+
     text_content = []
     for part in message.walk():
         if part.get_content_type().startswith("text/plain"):
@@ -342,8 +369,8 @@ def epub_to_text(file: IO[Any]) -> str:
 
 def file_io_to_text(file: IO[Any]) -> str:
     encoding = detect_encoding(file)
-    file_content_raw, _ = read_text_file(file, encoding=encoding)
-    return file_content_raw
+    file_content, _ = read_text_file(file, encoding=encoding)
+    return file_content
 
 
 def extract_file_text(
@@ -352,9 +379,13 @@ def extract_file_text(
     break_on_unprocessable: bool = True,
     extension: str | None = None,
 ) -> str:
+    """
+    Legacy function that returns *only text*, ignoring embedded images.
+    For backward-compatibility in code that only wants text.
+    """
     extension_to_function: dict[str, Callable[[IO[Any]], str]] = {
         ".pdf": pdf_to_text,
-        ".docx": docx_to_text,
+        ".docx": lambda f: docx_to_text_and_images(f)[0],  # no images
         ".pptx": pptx_to_text,
         ".xlsx": xlsx_to_text,
         ".eml": eml_to_text,
@@ -368,24 +399,23 @@ def extract_file_text(
                 return unstructured_to_text(file, file_name)
             except Exception as unstructured_error:
                 logger.error(
-                    f"Failed to process with Unstructured: {str(unstructured_error)}. Falling back to normal processing."
+                    f"Failed to process with Unstructured: {str(unstructured_error)}. "
+                    "Falling back to normal processing."
                 )
-                # Fall through to normal processing
-        final_extension: str
-        if file_name or extension:
-            if extension is not None:
-                final_extension = extension
-            elif file_name is not None:
-                final_extension = get_file_ext(file_name)
+        if extension is None:
+            extension = get_file_ext(file_name)
 
-            if is_valid_file_ext(final_extension):
-                return extension_to_function.get(final_extension, file_io_to_text)(file)
+        if is_valid_file_ext(extension):
+            func = extension_to_function.get(extension, file_io_to_text)
+            file.seek(0)
+            return func(file)
 
-        # Either the file somehow has no name or the extension is not one that we recognize
+        # If unknown extension, maybe it's a text file
+        file.seek(0)
         if is_text_file(file):
             return file_io_to_text(file)
 
-        raise ValueError("Unknown file extension and unknown text encoding")
+        raise ValueError("Unknown file extension or not recognized as text data")
 
     except Exception as e:
         if break_on_unprocessable:
@@ -396,20 +426,93 @@ def extract_file_text(
         return ""
 
 
+def extract_text_and_images(
+    file: IO[Any],
+    file_name: str,
+    pdf_pass: str | None = None,
+) -> Tuple[str, List[Tuple[bytes, str]]]:
+    """
+    Primary new function for the updated connector.
+    Returns (text_content, [(embedded_img_bytes, embedded_img_name), ...]).
+    """
+
+    try:
+        # Attempt unstructured if env var is set
+        if get_unstructured_api_key():
+            # If the user doesn't want embedded images, unstructured is fine
+            file.seek(0)
+            text_content = unstructured_to_text(file, file_name)
+            return (text_content, [])
+
+        extension = get_file_ext(file_name)
+
+        # docx example for embedded images
+        if extension == ".docx":
+            file.seek(0)
+            text_content, images = docx_to_text_and_images(file)
+            return (text_content, images)
+
+        # PDF example: we do not show complicated PDF image extraction here
+        # so we simply extract text for now and skip images.
+        if extension == ".pdf":
+            file.seek(0)
+            text_content, _, images = read_pdf_file(file, pdf_pass, extract_images=True)
+            return (text_content, images)
+
+        # For PPTX, XLSX, EML, etc., we do not show embedded image logic here.
+        # You can do something similar to docx if needed.
+        if extension == ".pptx":
+            file.seek(0)
+            return (pptx_to_text(file), [])
+
+        if extension == ".xlsx":
+            file.seek(0)
+            return (xlsx_to_text(file), [])
+
+        if extension == ".eml":
+            file.seek(0)
+            return (eml_to_text(file), [])
+
+        if extension == ".epub":
+            file.seek(0)
+            return (epub_to_text(file), [])
+
+        if extension == ".html":
+            file.seek(0)
+            return (parse_html_page_basic(file), [])
+
+        # If we reach here and it's a recognized text extension
+        if is_text_file_extension(file_name):
+            file.seek(0)
+            encoding = detect_encoding(file)
+            text_content_raw, _ = read_text_file(
+                file, encoding=encoding, ignore_onyx_metadata=False
+            )
+            return (text_content_raw, [])
+
+        # If it's an image file or something else, we do not parse embedded images from them
+        # just return empty text
+        file.seek(0)
+        return ("", [])
+
+    except Exception as e:
+        logger.exception(f"Failed to extract text/images from {file_name}: {e}")
+        return ("", [])
+
+
 def convert_docx_to_txt(
     file: UploadFile, file_store: FileStore, file_path: str
 ) -> None:
+    """
+    Helper to convert docx to a .txt file in the same filestore.
+    """
     file.file.seek(0)
     docx_content = file.file.read()
-    doc = Document(BytesIO(docx_content))
+    doc = DocxDocument(BytesIO(docx_content))
 
     # Extract text from the document
-    full_text = []
-    for para in doc.paragraphs:
-        full_text.append(para.text)
-
-    # Join the extracted text
-    text_content = "\n".join(full_text)
+    all_paras = [p.text for p in doc.paragraphs]
+    text_content = "\n".join(all_paras)
 
     txt_file_path = docx_to_txt_filename(file_path)
     file_store.save_file(
@@ -422,7 +525,4 @@ def convert_docx_to_txt(
 
 
 def docx_to_txt_filename(file_path: str) -> str:
-    """
-    Convert a .docx file path to its corresponding .txt file path.
-    """
     return file_path.rsplit(".", 1)[0] + ".txt"

backend/onyx/file_processing/file_validation.py

@@ -0,0 +1,46 @@
+"""
+Centralized file type validation utilities.
+"""
+# Standard image MIME types supported by most vision LLMs
+IMAGE_MIME_TYPES = [
+    "image/png",
+    "image/jpeg",
+    "image/jpg",
+    "image/webp",
+]
+
+# Image types that should be excluded from processing
+EXCLUDED_IMAGE_TYPES = [
+    "image/bmp",
+    "image/tiff",
+    "image/gif",
+    "image/svg+xml",
+]
+
+
+def is_valid_image_type(mime_type: str) -> bool:
+    """
+    Check if mime_type is a valid image type.
+
+    Args:
+        mime_type: The MIME type to check
+
+    Returns:
+        True if the MIME type is a valid image type, False otherwise
+    """
+    if not mime_type:
+        return False
+    return mime_type.startswith("image/") and mime_type not in EXCLUDED_IMAGE_TYPES
+
+
+def is_supported_by_vision_llm(mime_type: str) -> bool:
+    """
+    Check if this image type can be processed by vision LLMs.
+
+    Args:
+        mime_type: The MIME type to check
+
+    Returns:
+        True if the MIME type is supported by vision LLMs, False otherwise
+    """
+    return mime_type in IMAGE_MIME_TYPES

backend/onyx/file_processing/image_summarization.py

@@ -0,0 +1,129 @@
+import base64
+from io import BytesIO
+
+from langchain_core.messages import BaseMessage
+from langchain_core.messages import HumanMessage
+from langchain_core.messages import SystemMessage
+from PIL import Image
+
+from onyx.llm.interfaces import LLM
+from onyx.llm.utils import message_to_string
+from onyx.prompts.image_analysis import IMAGE_SUMMARIZATION_SYSTEM_PROMPT
+from onyx.prompts.image_analysis import IMAGE_SUMMARIZATION_USER_PROMPT
+from onyx.utils.logger import setup_logger
+
+logger = setup_logger()
+
+
+def prepare_image_bytes(image_data: bytes) -> str:
+    """Prepare image bytes for summarization.
+    Resizes image if it's larger than 20MB. Encodes image as a base64 string."""
+    image_data = _resize_image_if_needed(image_data)
+
+    # encode image (base64)
+    encoded_image = _encode_image_for_llm_prompt(image_data)
+
+    return encoded_image
+
+
+def summarize_image_pipeline(
+    llm: LLM,
+    image_data: bytes,
+    query: str | None = None,
+    system_prompt: str | None = None,
+) -> str:
+    """Pipeline to generate a summary of an image.
+    Resizes images if it is bigger than 20MB. Encodes image as a base64 string.
+    And finally uses the Default LLM to generate a textual summary of the image."""
+    # resize image if it's bigger than 20MB
+    encoded_image = prepare_image_bytes(image_data)
+
+    summary = _summarize_image(
+        encoded_image,
+        llm,
+        query,
+        system_prompt,
+    )
+
+    return summary
+
+
+def summarize_image_with_error_handling(
+    llm: LLM | None,
+    image_data: bytes,
+    context_name: str,
+    system_prompt: str = IMAGE_SUMMARIZATION_SYSTEM_PROMPT,
+    user_prompt_template: str = IMAGE_SUMMARIZATION_USER_PROMPT,
+) -> str | None:
+    """Wrapper function that handles error cases and configuration consistently.
+
+    Args:
+        llm: The LLM with vision capabilities to use for summarization
+        image_data: The raw image bytes
+        context_name: Name or title of the image for context
+        system_prompt: System prompt to use for the LLM
+        user_prompt_template: Template for the user prompt, should contain {title} placeholder
+
+    Returns:
+        The image summary text, or None if summarization failed or is disabled
+    """
+    if llm is None:
+        return None
+
+    user_prompt = user_prompt_template.format(title=context_name)
+    return summarize_image_pipeline(llm, image_data, user_prompt, system_prompt)
+
+
+def _summarize_image(
+    encoded_image: str,
+    llm: LLM,
+    query: str | None = None,
+    system_prompt: str | None = None,
+) -> str:
+    """Use default LLM (if it is multimodal) to generate a summary of an image."""
+
+    messages: list[BaseMessage] = []
+
+    if system_prompt:
+        messages.append(SystemMessage(content=system_prompt))
+
+    messages.append(
+        HumanMessage(
+            content=[
+                {"type": "text", "text": query},
+                {"type": "image_url", "image_url": {"url": encoded_image}},
+            ],
+        ),
+    )
+
+    try:
+        return message_to_string(llm.invoke(messages))
+
+    except Exception as e:
+        raise ValueError(f"Summarization failed. Messages: {messages}") from e
+
+
+def _encode_image_for_llm_prompt(image_data: bytes) -> str:
+    """Getting the base64 string."""
+    base64_encoded_data = base64.b64encode(image_data).decode("utf-8")
+
+    return f"data:image/jpeg;base64,{base64_encoded_data}"
+
+
+def _resize_image_if_needed(image_data: bytes, max_size_mb: int = 20) -> bytes:
+    """Resize image if it's larger than the specified max size in MB."""
+    max_size_bytes = max_size_mb * 1024 * 1024
+
+    if len(image_data) > max_size_bytes:
+        with Image.open(BytesIO(image_data)) as img:
+            # Reduce dimensions for better size reduction
+            img.thumbnail((1024, 1024), Image.Resampling.LANCZOS)
+            output = BytesIO()
+
+            # Save with lower quality for compression
+            img.save(output, format="JPEG", quality=85)
+            resized_data = output.getvalue()
+
+            return resized_data
+
+    return image_data

backend/onyx/file_processing/image_utils.py

@@ -0,0 +1,70 @@
+from typing import Tuple
+
+from sqlalchemy.orm import Session
+
+from onyx.configs.app_configs import CONTINUE_ON_CONNECTOR_FAILURE
+from onyx.configs.constants import FileOrigin
+from onyx.connectors.models import Section
+from onyx.db.pg_file_store import save_bytes_to_pgfilestore
+from onyx.file_processing.image_summarization import summarize_image_with_error_handling
+from onyx.llm.interfaces import LLM
+from onyx.utils.logger import setup_logger
+
+logger = setup_logger()
+
+
+def store_image_and_create_section(
+    db_session: Session,
+    image_data: bytes,
+    file_name: str,
+    display_name: str,
+    media_type: str = "image/unknown",
+    llm: LLM | None = None,
+    file_origin: FileOrigin = FileOrigin.OTHER,
+) -> Tuple[Section, str | None]:
+    """
+    Stores an image in PGFileStore and creates a Section object with optional summarization.
+
+    Args:
+        db_session: Database session
+        image_data: Raw image bytes
+        file_name: Base identifier for the file
+        display_name: Human-readable name for the image
+        media_type: MIME type of the image
+        llm: Optional LLM with vision capabilities for summarization
+        file_origin: Origin of the file (e.g., CONFLUENCE, GOOGLE_DRIVE, etc.)
+
+    Returns:
+        Tuple containing:
+        - Section object with image reference and optional summary text
+        - The file_name in PGFileStore or None if storage failed
+    """
+    # Storage logic
+    stored_file_name = None
+    try:
+        pgfilestore = save_bytes_to_pgfilestore(
+            db_session=db_session,
+            raw_bytes=image_data,
+            media_type=media_type,
+            identifier=file_name,
+            display_name=display_name,
+            file_origin=file_origin,
+        )
+        stored_file_name = pgfilestore.file_name
+    except Exception as e:
+        logger.error(f"Failed to store image: {e}")
+        if not CONTINUE_ON_CONNECTOR_FAILURE:
+            raise
+        return Section(text=""), None
+
+    # Summarization logic
+    summary_text = ""
+    if llm:
+        summary_text = (
+            summarize_image_with_error_handling(llm, image_data, display_name) or ""
+        )
+
+    return (
+        Section(text=summary_text, image_file_name=stored_file_name),
+        stored_file_name,
+    )

backend/onyx/indexing/chunker.py

@@ -23,12 +23,9 @@ from shared_configs.configs import STRICT_CHUNK_TOKEN_LIMIT
 CHUNK_OVERLAP = 0
 # Fairly arbitrary numbers but the general concept is we don't want the title/metadata to
 # overwhelm the actual contents of the chunk
-# For example in a rare case, this could be 128 tokens for the 512 chunk and title prefix
-# could be another 128 tokens leaving 256 for the actual contents
 MAX_METADATA_PERCENTAGE = 0.25
 CHUNK_MIN_CONTENT = 256
 
-
 logger = setup_logger()
 
 
@@ -36,16 +33,8 @@ def _get_metadata_suffix_for_document_index(
     metadata: dict[str, str | list[str]], include_separator: bool = False
 ) -> tuple[str, str]:
     """
-    Returns the metadata as a natural language string representation with all of the keys and values for the vector embedding
-    and a string of all of the values for the keyword search
-
-    For example, if we have the following metadata:
-    {
-        "author": "John Doe",
-        "space": "Engineering"
-    }
-    The vector embedding string should include the relation between the key and value wheres as for keyword we only want John Doe
-    and Engineering. The keys are repeat and much more noisy.
+    Returns the metadata as a natural language string representation with all of the keys and values
+    for the vector embedding and a string of all of the values for the keyword search.
     """
     if not metadata:
         return "", ""
@@ -74,12 +63,17 @@ def _get_metadata_suffix_for_document_index(
 
 
 def _combine_chunks(chunks: list[DocAwareChunk], large_chunk_id: int) -> DocAwareChunk:
+    """
+    Combines multiple DocAwareChunks into one large chunk (for “multipass” mode),
+    appending the content and adjusting source_links accordingly.
+    """
     merged_chunk = DocAwareChunk(
         source_document=chunks[0].source_document,
         chunk_id=chunks[0].chunk_id,
         blurb=chunks[0].blurb,
         content=chunks[0].content,
         source_links=chunks[0].source_links or {},
+        image_file_name=None,
         section_continuation=(chunks[0].chunk_id > 0),
         title_prefix=chunks[0].title_prefix,
         metadata_suffix_semantic=chunks[0].metadata_suffix_semantic,
@@ -103,6 +97,9 @@ def _combine_chunks(chunks: list[DocAwareChunk], large_chunk_id: int) -> DocAwar
 
 
 def generate_large_chunks(chunks: list[DocAwareChunk]) -> list[DocAwareChunk]:
+    """
+    Generates larger “grouped” chunks by combining sets of smaller chunks.
+    """
     large_chunks = []
     for idx, i in enumerate(range(0, len(chunks), LARGE_CHUNK_RATIO)):
         chunk_group = chunks[i : i + LARGE_CHUNK_RATIO]
@@ -172,23 +169,60 @@ class Chunker:
         while start < total_tokens:
             end = min(start + content_token_limit, total_tokens)
             token_chunk = tokens[start:end]
-            # Join the tokens to reconstruct the text
             chunk_text = " ".join(token_chunk)
             chunks.append(chunk_text)
             start = end
         return chunks
 
     def _extract_blurb(self, text: str) -> str:
+        """
+        Extract a short blurb from the text (first chunk of size `blurb_size`).
+        """
         texts = self.blurb_splitter.split_text(text)
         if not texts:
             return ""
         return texts[0]
 
     def _get_mini_chunk_texts(self, chunk_text: str) -> list[str] | None:
+        """
+        For “multipass” mode: additional sub-chunks (mini-chunks) for use in certain embeddings.
+        """
         if self.mini_chunk_splitter and chunk_text.strip():
             return self.mini_chunk_splitter.split_text(chunk_text)
         return None
 
+    # ADDED: extra param image_url to store in the chunk
+    def _create_chunk(
+        self,
+        document: Document,
+        chunks_list: list[DocAwareChunk],
+        text: str,
+        links: dict[int, str],
+        is_continuation: bool = False,
+        title_prefix: str = "",
+        metadata_suffix_semantic: str = "",
+        metadata_suffix_keyword: str = "",
+        image_file_name: str | None = None,
+    ) -> None:
+        """
+        Helper to create a new DocAwareChunk, append it to chunks_list.
+        """
+        new_chunk = DocAwareChunk(
+            source_document=document,
+            chunk_id=len(chunks_list),
+            blurb=self._extract_blurb(text),
+            content=text,
+            source_links=links or {0: ""},
+            image_file_name=image_file_name,
+            section_continuation=is_continuation,
+            title_prefix=title_prefix,
+            metadata_suffix_semantic=metadata_suffix_semantic,
+            metadata_suffix_keyword=metadata_suffix_keyword,
+            mini_chunk_texts=self._get_mini_chunk_texts(text),
+            large_chunk_id=None,
+        )
+        chunks_list.append(new_chunk)
+
     def _chunk_document(
         self,
         document: Document,
@@ -198,122 +232,156 @@ class Chunker:
         content_token_limit: int,
     ) -> list[DocAwareChunk]:
         """
-        Loops through sections of the document, adds metadata and converts them into chunks.
+        Loops through sections of the document, converting them into one or more chunks.
+        If a section has an image_link, we treat it as a dedicated chunk.
         """
+
         chunks: list[DocAwareChunk] = []
         link_offsets: dict[int, str] = {}
         chunk_text = ""
 
-        def _create_chunk(
-            text: str,
-            links: dict[int, str],
-            is_continuation: bool = False,
-        ) -> DocAwareChunk:
-            return DocAwareChunk(
-                source_document=document,
-                chunk_id=len(chunks),
-                blurb=self._extract_blurb(text),
-                content=text,
-                source_links=links or {0: ""},
-                section_continuation=is_continuation,
-                title_prefix=title_prefix,
-                metadata_suffix_semantic=metadata_suffix_semantic,
-                metadata_suffix_keyword=metadata_suffix_keyword,
-                mini_chunk_texts=self._get_mini_chunk_texts(text),
-                large_chunk_id=None,
-            )
-
-        section_link_text: str
-
         for section_idx, section in enumerate(document.sections):
             section_text = clean_text(section.text)
             section_link_text = section.link or ""
-            # If there is no useful content, not even the title, just drop it
+            # ADDED: if the Section has an image link
+            image_url = section.image_file_name
+
+            # If there is no useful content, skip
             if not section_text and (not document.title or section_idx > 0):
-                # If a section is empty and the document has no title, we can just drop it. We return a list of
-                # DocAwareChunks where each one contains the necessary information needed down the line for indexing.
-                # There is no concern about dropping whole documents from this list, it should not cause any indexing failures.
                 logger.warning(
-                    f"Skipping section {section.text} from document "
-                    f"{document.semantic_identifier} due to empty text after cleaning "
-                    f"with link {section_link_text}"
+                    f"Skipping empty or irrelevant section in doc "
+                    f"{document.semantic_identifier}, link={section_link_text}"
                 )
                 continue
 
+            # CASE 1: If this is an image section, force a separate chunk
+            if image_url:
+                # First, if we have any partially built text chunk, finalize it
+                if chunk_text.strip():
+                    self._create_chunk(
+                        document,
+                        chunks,
+                        chunk_text,
+                        link_offsets,
+                        is_continuation=False,
+                        title_prefix=title_prefix,
+                        metadata_suffix_semantic=metadata_suffix_semantic,
+                        metadata_suffix_keyword=metadata_suffix_keyword,
+                    )
+                    chunk_text = ""
+                    link_offsets = {}
+
+                # Create a chunk specifically for this image
+                # (If the section has text describing the image, use that as content)
+                self._create_chunk(
+                    document,
+                    chunks,
+                    section_text,
+                    links={0: section_link_text}
+                    if section_link_text
+                    else {},  # No text offsets needed for images
+                    image_file_name=image_url,
+                    title_prefix=title_prefix,
+                    metadata_suffix_semantic=metadata_suffix_semantic,
+                    metadata_suffix_keyword=metadata_suffix_keyword,
+                )
+                # Continue to next section
+                continue
+
+            # CASE 2: Normal text section
             section_token_count = len(self.tokenizer.tokenize(section_text))
 
-            # Large sections are considered self-contained/unique
-            # Therefore, they start a new chunk and are not concatenated
-            # at the end by other sections
+            # If the section is large on its own, split it separately
             if section_token_count > content_token_limit:
-                if chunk_text:
-                    chunks.append(_create_chunk(chunk_text, link_offsets))
-                    link_offsets = {}
+                if chunk_text.strip():
+                    self._create_chunk(
+                        document,
+                        chunks,
+                        chunk_text,
+                        link_offsets,
+                        False,
+                        title_prefix,
+                        metadata_suffix_semantic,
+                        metadata_suffix_keyword,
+                    )
                     chunk_text = ""
+                    link_offsets = {}
 
                 split_texts = self.chunk_splitter.split_text(section_text)
-
                 for i, split_text in enumerate(split_texts):
+                    # If even the split_text is bigger than strict limit, further split
                     if (
                         STRICT_CHUNK_TOKEN_LIMIT
-                        and
-                        # Tokenizer only runs if STRICT_CHUNK_TOKEN_LIMIT is true
-                        len(self.tokenizer.tokenize(split_text)) > content_token_limit
+                        and len(self.tokenizer.tokenize(split_text))
+                        > content_token_limit
                     ):
-                        # If STRICT_CHUNK_TOKEN_LIMIT is true, manually check
-                        # the token count of each split text to ensure it is
-                        # not larger than the content_token_limit
                         smaller_chunks = self._split_oversized_chunk(
                             split_text, content_token_limit
                         )
-                        for i, small_chunk in enumerate(smaller_chunks):
-                            chunks.append(
-                                _create_chunk(
-                                    text=small_chunk,
-                                    links={0: section_link_text},
-                                    is_continuation=(i != 0),
-                                )
+                        for j, small_chunk in enumerate(smaller_chunks):
+                            self._create_chunk(
+                                document,
+                                chunks,
+                                small_chunk,
+                                {0: section_link_text},
+                                is_continuation=(j != 0),
+                                title_prefix=title_prefix,
+                                metadata_suffix_semantic=metadata_suffix_semantic,
+                                metadata_suffix_keyword=metadata_suffix_keyword,
                             )
                     else:
-                        chunks.append(
-                            _create_chunk(
-                                text=split_text,
-                                links={0: section_link_text},
-                                is_continuation=(i != 0),
-                            )
+                        self._create_chunk(
+                            document,
+                            chunks,
+                            split_text,
+                            {0: section_link_text},
+                            is_continuation=(i != 0),
+                            title_prefix=title_prefix,
+                            metadata_suffix_semantic=metadata_suffix_semantic,
+                            metadata_suffix_keyword=metadata_suffix_keyword,
                         )
-
                 continue
 
+            # If we can still fit this section into the current chunk, do so
             current_token_count = len(self.tokenizer.tokenize(chunk_text))
             current_offset = len(shared_precompare_cleanup(chunk_text))
-            # In the case where the whole section is shorter than a chunk, either add
-            # to chunk or start a new one
             next_section_tokens = (
                 len(self.tokenizer.tokenize(SECTION_SEPARATOR)) + section_token_count
             )
+
             if next_section_tokens + current_token_count <= content_token_limit:
                 if chunk_text:
                     chunk_text += SECTION_SEPARATOR
                 chunk_text += section_text
                 link_offsets[current_offset] = section_link_text
             else:
-                chunks.append(_create_chunk(chunk_text, link_offsets))
+                # finalize the existing chunk
+                self._create_chunk(
+                    document,
+                    chunks,
+                    chunk_text,
+                    link_offsets,
+                    False,
+                    title_prefix,
+                    metadata_suffix_semantic,
+                    metadata_suffix_keyword,
+                )
+                # start a new chunk
                 link_offsets = {0: section_link_text}
                 chunk_text = section_text
 
-        # Once we hit the end, if we're still in the process of building a chunk, add what we have.
-        # If there is only whitespace left then don't include it. If there are no chunks at all
-        # from the doc, we can just create a single chunk with the title.
+        # finalize any leftover text chunk
         if chunk_text.strip() or not chunks:
-            chunks.append(
-                _create_chunk(
-                    chunk_text,
-                    link_offsets or {0: section_link_text},
-                )
+            self._create_chunk(
+                document,
+                chunks,
+                chunk_text,
+                link_offsets or {0: ""},  # safe default
+                False,
+                title_prefix,
+                metadata_suffix_semantic,
+                metadata_suffix_keyword,
             )
-
-        # If the chunk does not have any useable content, it will not be indexed
         return chunks
 
     def _handle_single_document(self, document: Document) -> list[DocAwareChunk]:
@@ -321,10 +389,12 @@ class Chunker:
         if document.source == DocumentSource.GMAIL:
             logger.debug(f"Chunking {document.semantic_identifier}")
 
+        # Title prep
         title = self._extract_blurb(document.get_title_for_document_index() or "")
         title_prefix = title + RETURN_SEPARATOR if title else ""
         title_tokens = len(self.tokenizer.tokenize(title_prefix))
 
+        # Metadata prep
         metadata_suffix_semantic = ""
         metadata_suffix_keyword = ""
         metadata_tokens = 0
@@ -337,19 +407,20 @@ class Chunker:
             )
             metadata_tokens = len(self.tokenizer.tokenize(metadata_suffix_semantic))
 
+        # If metadata is too large, skip it in the semantic content
         if metadata_tokens >= self.chunk_token_limit * MAX_METADATA_PERCENTAGE:
-            # Note: we can keep the keyword suffix even if the semantic suffix is too long to fit in the model
-            # context, there is no limit for the keyword component
             metadata_suffix_semantic = ""
             metadata_tokens = 0
 
+        # Adjust content token limit to accommodate title + metadata
         content_token_limit = self.chunk_token_limit - title_tokens - metadata_tokens
-        # If there is not enough context remaining then just index the chunk with no prefix/suffix
         if content_token_limit <= CHUNK_MIN_CONTENT:
+            # Not enough space left, so revert to full chunk without the prefix
             content_token_limit = self.chunk_token_limit
             title_prefix = ""
             metadata_suffix_semantic = ""
 
+        # Chunk the document
         normal_chunks = self._chunk_document(
             document,
             title_prefix,
@@ -358,6 +429,7 @@ class Chunker:
             content_token_limit,
         )
 
+        # Optional “multipass” large chunk creation
         if self.enable_multipass and self.enable_large_chunks:
             large_chunks = generate_large_chunks(normal_chunks)
             normal_chunks.extend(large_chunks)
@@ -371,9 +443,8 @@ class Chunker:
         """
         final_chunks: list[DocAwareChunk] = []
         for document in documents:
-            if self.callback:
-                if self.callback.should_stop():
-                    raise RuntimeError("Chunker.chunk: Stop signal detected")
+            if self.callback and self.callback.should_stop():
+                raise RuntimeError("Chunker.chunk: Stop signal detected")
 
             chunks = self._handle_single_document(document)
             final_chunks.extend(chunks)

backend/onyx/indexing/indexing_pipeline.py

@@ -464,12 +464,29 @@ def index_doc_batch(
             ),
         )
 
-        successful_doc_ids = {record.document_id for record in insertion_records}
-        if successful_doc_ids != set(updatable_ids):
+        all_returned_doc_ids = (
+            {record.document_id for record in insertion_records}
+            .union(
+                {
+                    record.failed_document.document_id
+                    for record in vector_db_write_failures
+                    if record.failed_document
+                }
+            )
+            .union(
+                {
+                    record.failed_document.document_id
+                    for record in embedding_failures
+                    if record.failed_document
+                }
+            )
+        )
+        if all_returned_doc_ids != set(updatable_ids):
             raise RuntimeError(
                 f"Some documents were not successfully indexed. "
                 f"Updatable IDs: {updatable_ids}, "
-                f"Successful IDs: {successful_doc_ids}"
+                f"Returned IDs: {all_returned_doc_ids}. "
+                "This should never happen."
             )
 
         last_modified_ids = []

backend/onyx/indexing/models.py

@@ -29,6 +29,7 @@ class BaseChunk(BaseModel):
     content: str
     # Holds the link and the offsets into the raw Chunk text
     source_links: dict[int, str] | None
+    image_file_name: str | None
     # True if this Chunk's start is not at the start of a Section
     section_continuation: bool
 

backend/onyx/llm/factory.py

@@ -6,12 +6,14 @@ from onyx.configs.model_configs import GEN_AI_MODEL_FALLBACK_MAX_TOKENS
 from onyx.configs.model_configs import GEN_AI_TEMPERATURE
 from onyx.db.engine import get_session_context_manager
 from onyx.db.llm import fetch_default_provider
+from onyx.db.llm import fetch_existing_llm_providers
 from onyx.db.llm import fetch_provider
 from onyx.db.models import Persona
 from onyx.llm.chat_llm import DefaultMultiLLM
 from onyx.llm.exceptions import GenAIDisabledException
 from onyx.llm.interfaces import LLM
 from onyx.llm.override_models import LLMOverride
+from onyx.llm.utils import model_supports_image_input
 from onyx.utils.headers import build_llm_extra_headers
 from onyx.utils.logger import setup_logger
 from onyx.utils.long_term_log import LongTermLogger
@@ -86,6 +88,48 @@ def get_llms_for_persona(
     return _create_llm(model), _create_llm(fast_model)
 
 
+def get_default_llm_with_vision(
+    timeout: int | None = None,
+    temperature: float | None = None,
+    additional_headers: dict[str, str] | None = None,
+    long_term_logger: LongTermLogger | None = None,
+) -> LLM | None:
+    if DISABLE_GENERATIVE_AI:
+        raise GenAIDisabledException()
+
+    with get_session_context_manager() as db_session:
+        llm_providers = fetch_existing_llm_providers(db_session)
+
+    if not llm_providers:
+        return None
+
+    for provider in llm_providers:
+        model_name = provider.default_model_name
+        fast_model_name = (
+            provider.fast_default_model_name or provider.default_model_name
+        )
+
+        if not model_name or not fast_model_name:
+            continue
+
+        if model_supports_image_input(model_name, provider.provider):
+            return get_llm(
+                provider=provider.provider,
+                model=model_name,
+                deployment_name=provider.deployment_name,
+                api_key=provider.api_key,
+                api_base=provider.api_base,
+                api_version=provider.api_version,
+                custom_config=provider.custom_config,
+                timeout=timeout,
+                temperature=temperature,
+                additional_headers=additional_headers,
+                long_term_logger=long_term_logger,
+            )
+
+    raise ValueError("No LLM provider found that supports image input")
+
+
 def get_default_llms(
     timeout: int | None = None,
     temperature: float | None = None,

backend/onyx/main.py

@@ -51,6 +51,7 @@ from onyx.server.documents.cc_pair import router as cc_pair_router
 from onyx.server.documents.connector import router as connector_router
 from onyx.server.documents.credential import router as credential_router
 from onyx.server.documents.document import router as document_router
+from onyx.server.documents.standard_oauth import router as standard_oauth_router
 from onyx.server.features.document_set.api import router as document_set_router
 from onyx.server.features.folder.api import router as folder_router
 from onyx.server.features.input_prompt.api import (
@@ -322,6 +323,7 @@ def get_application() -> FastAPI:
     )
     include_router_with_global_prefix_prepended(application, long_term_logs_router)
     include_router_with_global_prefix_prepended(application, api_key_router)
+    include_router_with_global_prefix_prepended(application, standard_oauth_router)
 
     if AUTH_TYPE == AuthType.DISABLED:
         # Server logs this during auth setup verification step

backend/onyx/onyxbot/slack/blocks.py

@@ -31,12 +31,18 @@ from onyx.onyxbot.slack.constants import FEEDBACK_DOC_BUTTON_BLOCK_ACTION_ID
 from onyx.onyxbot.slack.constants import FOLLOWUP_BUTTON_ACTION_ID
 from onyx.onyxbot.slack.constants import FOLLOWUP_BUTTON_RESOLVED_ACTION_ID
 from onyx.onyxbot.slack.constants import IMMEDIATE_RESOLVED_BUTTON_ACTION_ID
+from onyx.onyxbot.slack.constants import KEEP_TO_YOURSELF_ACTION_ID
 from onyx.onyxbot.slack.constants import LIKE_BLOCK_ACTION_ID
+from onyx.onyxbot.slack.constants import SHOW_EVERYONE_ACTION_ID
 from onyx.onyxbot.slack.formatting import format_slack_message
 from onyx.onyxbot.slack.icons import source_to_github_img_link
+from onyx.onyxbot.slack.models import ActionValuesEphemeralMessage
+from onyx.onyxbot.slack.models import ActionValuesEphemeralMessageChannelConfig
+from onyx.onyxbot.slack.models import ActionValuesEphemeralMessageMessageInfo
 from onyx.onyxbot.slack.models import SlackMessageInfo
 from onyx.onyxbot.slack.utils import build_continue_in_web_ui_id
 from onyx.onyxbot.slack.utils import build_feedback_id
+from onyx.onyxbot.slack.utils import build_publish_ephemeral_message_id
 from onyx.onyxbot.slack.utils import remove_slack_text_interactions
 from onyx.onyxbot.slack.utils import translate_vespa_highlight_to_slack
 from onyx.utils.text_processing import decode_escapes
@@ -105,6 +111,77 @@ def _build_qa_feedback_block(
     )
 
 
+def _build_ephemeral_publication_block(
+    channel_id: str,
+    chat_message_id: int,
+    message_info: SlackMessageInfo,
+    original_question_ts: str,
+    channel_conf: ChannelConfig,
+    feedback_reminder_id: str | None = None,
+) -> Block:
+    # check whether the message is in a thread
+    if (
+        message_info is not None
+        and message_info.msg_to_respond is not None
+        and message_info.thread_to_respond is not None
+        and (message_info.msg_to_respond == message_info.thread_to_respond)
+    ):
+        respond_ts = None
+    else:
+        respond_ts = original_question_ts
+
+    action_values_ephemeral_message_channel_config = (
+        ActionValuesEphemeralMessageChannelConfig(
+            channel_name=channel_conf.get("channel_name"),
+            respond_tag_only=channel_conf.get("respond_tag_only"),
+            respond_to_bots=channel_conf.get("respond_to_bots"),
+            is_ephemeral=channel_conf.get("is_ephemeral", False),
+            respond_member_group_list=channel_conf.get("respond_member_group_list"),
+            answer_filters=channel_conf.get("answer_filters"),
+            follow_up_tags=channel_conf.get("follow_up_tags"),
+            show_continue_in_web_ui=channel_conf.get("show_continue_in_web_ui", False),
+        )
+    )
+
+    action_values_ephemeral_message_message_info = (
+        ActionValuesEphemeralMessageMessageInfo(
+            bypass_filters=message_info.bypass_filters,
+            channel_to_respond=message_info.channel_to_respond,
+            msg_to_respond=message_info.msg_to_respond,
+            email=message_info.email,
+            sender_id=message_info.sender_id,
+            thread_messages=[],
+            is_bot_msg=message_info.is_bot_msg,
+            is_bot_dm=message_info.is_bot_dm,
+            thread_to_respond=respond_ts,
+        )
+    )
+
+    action_values_ephemeral_message = ActionValuesEphemeralMessage(
+        original_question_ts=original_question_ts,
+        feedback_reminder_id=feedback_reminder_id,
+        chat_message_id=chat_message_id,
+        message_info=action_values_ephemeral_message_message_info,
+        channel_conf=action_values_ephemeral_message_channel_config,
+    )
+
+    return ActionsBlock(
+        block_id=build_publish_ephemeral_message_id(original_question_ts),
+        elements=[
+            ButtonElement(
+                action_id=SHOW_EVERYONE_ACTION_ID,
+                text="📢 Share with Everyone",
+                value=action_values_ephemeral_message.model_dump_json(),
+            ),
+            ButtonElement(
+                action_id=KEEP_TO_YOURSELF_ACTION_ID,
+                text="🤫  Keep to Yourself",
+                value=action_values_ephemeral_message.model_dump_json(),
+            ),
+        ],
+    )
+
+
 def get_document_feedback_blocks() -> Block:
     return SectionBlock(
         text=(
@@ -486,16 +563,21 @@ def build_slack_response_blocks(
     use_citations: bool,
     feedback_reminder_id: str | None,
     skip_ai_feedback: bool = False,
+    offer_ephemeral_publication: bool = False,
     expecting_search_result: bool = False,
+    skip_restated_question: bool = False,
 ) -> list[Block]:
     """
     This function is a top level function that builds all the blocks for the Slack response.
     It also handles combining all the blocks together.
     """
     # If called with the OnyxBot slash command, the question is lost so we have to reshow it
-    restate_question_block = get_restate_blocks(
-        message_info.thread_messages[-1].message, message_info.is_bot_msg
-    )
+    if not skip_restated_question:
+        restate_question_block = get_restate_blocks(
+            message_info.thread_messages[-1].message, message_info.is_bot_msg
+        )
+    else:
+        restate_question_block = []
 
     if expecting_search_result:
         answer_blocks = _build_qa_response_blocks(
@@ -520,12 +602,36 @@ def build_slack_response_blocks(
         )
 
     follow_up_block = []
-    if channel_conf and channel_conf.get("follow_up_tags") is not None:
+    if (
+        channel_conf
+        and channel_conf.get("follow_up_tags") is not None
+        and not channel_conf.get("is_ephemeral", False)
+    ):
         follow_up_block.append(
             _build_follow_up_block(message_id=answer.chat_message_id)
         )
 
-    ai_feedback_block = []
+    publish_ephemeral_message_block = []
+
+    if (
+        offer_ephemeral_publication
+        and answer.chat_message_id is not None
+        and message_info.msg_to_respond is not None
+        and channel_conf is not None
+    ):
+        publish_ephemeral_message_block.append(
+            _build_ephemeral_publication_block(
+                channel_id=message_info.channel_to_respond,
+                chat_message_id=answer.chat_message_id,
+                original_question_ts=message_info.msg_to_respond,
+                message_info=message_info,
+                channel_conf=channel_conf,
+                feedback_reminder_id=feedback_reminder_id,
+            )
+        )
+
+    ai_feedback_block: list[Block] = []
+
     if answer.chat_message_id is not None and not skip_ai_feedback:
         ai_feedback_block.append(
             _build_qa_feedback_block(
@@ -547,6 +653,7 @@ def build_slack_response_blocks(
     all_blocks = (
         restate_question_block
         + answer_blocks
+        + publish_ephemeral_message_block
         + ai_feedback_block
         + citations_divider
         + citations_blocks

backend/onyx/onyxbot/slack/constants.py

@@ -2,6 +2,8 @@ from enum import Enum
 
 LIKE_BLOCK_ACTION_ID = "feedback-like"
 DISLIKE_BLOCK_ACTION_ID = "feedback-dislike"
+SHOW_EVERYONE_ACTION_ID = "show-everyone"
+KEEP_TO_YOURSELF_ACTION_ID = "keep-to-yourself"
 CONTINUE_IN_WEB_UI_ACTION_ID = "continue-in-web-ui"
 FEEDBACK_DOC_BUTTON_BLOCK_ACTION_ID = "feedback-doc-button"
 IMMEDIATE_RESOLVED_BUTTON_ACTION_ID = "immediate-resolved-button"

backend/onyx/onyxbot/slack/handlers/handle_buttons.py

@@ -1,3 +1,4 @@
+import json
 from typing import Any
 from typing import cast
 
@@ -5,21 +6,32 @@ from slack_sdk import WebClient
 from slack_sdk.models.blocks import SectionBlock
 from slack_sdk.models.views import View
 from slack_sdk.socket_mode.request import SocketModeRequest
+from slack_sdk.webhook import WebhookClient
 
+from onyx.chat.models import ChatOnyxBotResponse
+from onyx.chat.models import CitationInfo
+from onyx.chat.models import QADocsResponse
 from onyx.configs.constants import MessageType
 from onyx.configs.constants import SearchFeedbackType
 from onyx.configs.onyxbot_configs import DANSWER_FOLLOWUP_EMOJI
 from onyx.connectors.slack.utils import expert_info_from_slack_id
 from onyx.connectors.slack.utils import make_slack_api_rate_limited
+from onyx.context.search.models import SavedSearchDoc
+from onyx.db.chat import get_chat_message
+from onyx.db.chat import translate_db_message_to_chat_message_detail
 from onyx.db.engine import get_session_with_current_tenant
 from onyx.db.feedback import create_chat_message_feedback
 from onyx.db.feedback import create_doc_retrieval_feedback
+from onyx.db.users import get_user_by_email
 from onyx.onyxbot.slack.blocks import build_follow_up_resolved_blocks
+from onyx.onyxbot.slack.blocks import build_slack_response_blocks
 from onyx.onyxbot.slack.blocks import get_document_feedback_blocks
 from onyx.onyxbot.slack.config import get_slack_channel_config_for_bot_and_channel
 from onyx.onyxbot.slack.constants import DISLIKE_BLOCK_ACTION_ID
 from onyx.onyxbot.slack.constants import FeedbackVisibility
+from onyx.onyxbot.slack.constants import KEEP_TO_YOURSELF_ACTION_ID
 from onyx.onyxbot.slack.constants import LIKE_BLOCK_ACTION_ID
+from onyx.onyxbot.slack.constants import SHOW_EVERYONE_ACTION_ID
 from onyx.onyxbot.slack.constants import VIEW_DOC_FEEDBACK_ID
 from onyx.onyxbot.slack.handlers.handle_message import (
     remove_scheduled_feedback_reminder,
@@ -35,15 +47,48 @@ from onyx.onyxbot.slack.utils import fetch_slack_user_ids_from_emails
 from onyx.onyxbot.slack.utils import get_channel_name_from_id
 from onyx.onyxbot.slack.utils import get_feedback_visibility
 from onyx.onyxbot.slack.utils import read_slack_thread
-from onyx.onyxbot.slack.utils import respond_in_thread
+from onyx.onyxbot.slack.utils import respond_in_thread_or_channel
 from onyx.onyxbot.slack.utils import TenantSocketModeClient
 from onyx.onyxbot.slack.utils import update_emote_react
+from onyx.server.query_and_chat.models import ChatMessageDetail
 from onyx.utils.logger import setup_logger
 
 
 logger = setup_logger()
 
 
+def _convert_db_doc_id_to_document_ids(
+    citation_dict: dict[int, int], top_documents: list[SavedSearchDoc]
+) -> list[CitationInfo]:
+    citation_list_with_document_id = []
+    for citation_num, db_doc_id in citation_dict.items():
+        if db_doc_id is not None:
+            matching_doc = next(
+                (d for d in top_documents if d.db_doc_id == db_doc_id), None
+            )
+            if matching_doc:
+                citation_list_with_document_id.append(
+                    CitationInfo(
+                        citation_num=citation_num, document_id=matching_doc.document_id
+                    )
+                )
+    return citation_list_with_document_id
+
+
+def _build_citation_list(chat_message_detail: ChatMessageDetail) -> list[CitationInfo]:
+    citation_dict = chat_message_detail.citations
+    if citation_dict is None:
+        return []
+    else:
+        top_documents = (
+            chat_message_detail.context_docs.top_documents
+            if chat_message_detail.context_docs
+            else []
+        )
+        citation_list = _convert_db_doc_id_to_document_ids(citation_dict, top_documents)
+        return citation_list
+
+
 def handle_doc_feedback_button(
     req: SocketModeRequest,
     client: TenantSocketModeClient,
@@ -58,7 +103,7 @@ def handle_doc_feedback_button(
     external_id = build_feedback_id(query_event_id, doc_id, doc_rank)
 
     channel_id = req.payload["container"]["channel_id"]
-    thread_ts = req.payload["container"]["thread_ts"]
+    thread_ts = req.payload["container"].get("thread_ts", None)
 
     data = View(
         type="modal",
@@ -84,7 +129,7 @@ def handle_generate_answer_button(
     channel_id = req.payload["channel"]["id"]
     channel_name = req.payload["channel"]["name"]
     message_ts = req.payload["message"]["ts"]
-    thread_ts = req.payload["container"]["thread_ts"]
+    thread_ts = req.payload["container"].get("thread_ts", None)
     user_id = req.payload["user"]["id"]
     expert_info = expert_info_from_slack_id(user_id, client.web_client, user_cache={})
     email = expert_info.email if expert_info else None
@@ -106,7 +151,7 @@ def handle_generate_answer_button(
 
     # tell the user that we're working on it
     # Send an ephemeral message to the user that we're generating the answer
-    respond_in_thread(
+    respond_in_thread_or_channel(
         client=client.web_client,
         channel=channel_id,
         receiver_ids=[user_id],
@@ -142,6 +187,178 @@ def handle_generate_answer_button(
         )
 
 
+def handle_publish_ephemeral_message_button(
+    req: SocketModeRequest,
+    client: TenantSocketModeClient,
+    action_id: str,
+) -> None:
+    """
+    This function handles the Share with Everyone/Keep for Yourself buttons
+    for ephemeral messages.
+    """
+    channel_id = req.payload["channel"]["id"]
+    ephemeral_message_ts = req.payload["container"]["message_ts"]
+
+    slack_sender_id = req.payload["user"]["id"]
+    response_url = req.payload["response_url"]
+    webhook = WebhookClient(url=response_url)
+
+    # The additional data required that was added to buttons.
+    # Specifically, this contains the message_info, channel_conf information
+    # and some additional attributes.
+    value_dict = json.loads(req.payload["actions"][0]["value"])
+
+    original_question_ts = value_dict.get("original_question_ts")
+    if not original_question_ts:
+        raise ValueError("Missing original_question_ts in the payload")
+    if not ephemeral_message_ts:
+        raise ValueError("Missing ephemeral_message_ts in the payload")
+
+    feedback_reminder_id = value_dict.get("feedback_reminder_id")
+
+    slack_message_info = SlackMessageInfo(**value_dict["message_info"])
+    channel_conf = value_dict.get("channel_conf")
+
+    user_email = value_dict.get("message_info", {}).get("email")
+
+    chat_message_id = value_dict.get("chat_message_id")
+
+    # Obtain onyx_user and chat_message information
+    if not chat_message_id:
+        raise ValueError("Missing chat_message_id in the payload")
+
+    with get_session_with_current_tenant() as db_session:
+        onyx_user = get_user_by_email(user_email, db_session)
+        if not onyx_user:
+            raise ValueError("Cannot determine onyx_user_id from email in payload")
+        try:
+            chat_message = get_chat_message(chat_message_id, onyx_user.id, db_session)
+        except ValueError:
+            chat_message = get_chat_message(
+                chat_message_id, None, db_session
+            )  # is this good idea?
+        except Exception as e:
+            logger.error(f"Failed to get chat message: {e}")
+            raise e
+
+        chat_message_detail = translate_db_message_to_chat_message_detail(chat_message)
+
+        # construct the proper citation format and then the answer in the suitable format
+        # we need to construct the blocks.
+        citation_list = _build_citation_list(chat_message_detail)
+
+        onyx_bot_answer = ChatOnyxBotResponse(
+            answer=chat_message_detail.message,
+            citations=citation_list,
+            chat_message_id=chat_message_id,
+            docs=QADocsResponse(
+                top_documents=chat_message_detail.context_docs.top_documents
+                if chat_message_detail.context_docs
+                else [],
+                predicted_flow=None,
+                predicted_search=None,
+                applied_source_filters=None,
+                applied_time_cutoff=None,
+                recency_bias_multiplier=1.0,
+            ),
+            llm_selected_doc_indices=None,
+            error_msg=None,
+        )
+
+    # Note: we need to use the webhook and the respond_url to update/delete ephemeral messages
+    if action_id == SHOW_EVERYONE_ACTION_ID:
+        # Convert to non-ephemeral message in thread
+        try:
+            webhook.send(
+                response_type="ephemeral",
+                text="",
+                blocks=[],
+                replace_original=True,
+                delete_original=True,
+            )
+        except Exception as e:
+            logger.error(f"Failed to send webhook: {e}")
+
+        # remove handling of empheremal block and add AI feedback.
+        all_blocks = build_slack_response_blocks(
+            answer=onyx_bot_answer,
+            message_info=slack_message_info,
+            channel_conf=channel_conf,
+            use_citations=True,
+            feedback_reminder_id=feedback_reminder_id,
+            skip_ai_feedback=False,
+            offer_ephemeral_publication=False,
+            skip_restated_question=True,
+        )
+        try:
+            # Post in thread as non-ephemeral message
+            respond_in_thread_or_channel(
+                client=client.web_client,
+                channel=channel_id,
+                receiver_ids=None,  # If respond_member_group_list is set, send to them. TODO: check!
+                text="Hello! Onyx has some results for you!",
+                blocks=all_blocks,
+                thread_ts=original_question_ts,
+                # don't unfurl, since otherwise we will have 5+ previews which makes the message very long
+                unfurl=False,
+                send_as_ephemeral=False,
+            )
+        except Exception as e:
+            logger.error(f"Failed to publish ephemeral message: {e}")
+            raise e
+
+    elif action_id == KEEP_TO_YOURSELF_ACTION_ID:
+        # Keep as ephemeral message in channel or thread, but remove the publish button and add feedback button
+
+        changed_blocks = build_slack_response_blocks(
+            answer=onyx_bot_answer,
+            message_info=slack_message_info,
+            channel_conf=channel_conf,
+            use_citations=True,
+            feedback_reminder_id=feedback_reminder_id,
+            skip_ai_feedback=False,
+            offer_ephemeral_publication=False,
+            skip_restated_question=True,
+        )
+
+        try:
+            if slack_message_info.thread_to_respond is not None:
+                # There seems to be a bug in slack where an update within the thread
+                # actually leads to the update to be posted in the channel. Therefore,
+                # for now we delete the original ephemeral message and post a new one
+                # if the ephemeral message is in a thread.
+                webhook.send(
+                    response_type="ephemeral",
+                    text="",
+                    blocks=[],
+                    replace_original=True,
+                    delete_original=True,
+                )
+
+                respond_in_thread_or_channel(
+                    client=client.web_client,
+                    channel=channel_id,
+                    receiver_ids=[slack_sender_id],
+                    text="Your personal response, sent as an ephemeral message.",
+                    blocks=changed_blocks,
+                    thread_ts=original_question_ts,
+                    # don't unfurl, since otherwise we will have 5+ previews which makes the message very long
+                    unfurl=False,
+                    send_as_ephemeral=True,
+                )
+            else:
+                # This works fine if the ephemeral message is in the channel
+                webhook.send(
+                    response_type="ephemeral",
+                    text="Your personal response, sent as an ephemeral message.",
+                    blocks=changed_blocks,
+                    replace_original=True,
+                    delete_original=False,
+                )
+        except Exception as e:
+            logger.error(f"Failed to send webhook: {e}")
+
+
 def handle_slack_feedback(
     feedback_id: str,
     feedback_type: str,
@@ -153,13 +370,20 @@ def handle_slack_feedback(
 ) -> None:
     message_id, doc_id, doc_rank = decompose_action_id(feedback_id)
 
+    # Get Onyx user from Slack ID
+    expert_info = expert_info_from_slack_id(
+        user_id_to_post_confirmation, client, user_cache={}
+    )
+    email = expert_info.email if expert_info else None
+
     with get_session_with_current_tenant() as db_session:
+        onyx_user = get_user_by_email(email, db_session) if email else None
         if feedback_type in [LIKE_BLOCK_ACTION_ID, DISLIKE_BLOCK_ACTION_ID]:
             create_chat_message_feedback(
                 is_positive=feedback_type == LIKE_BLOCK_ACTION_ID,
                 feedback_text="",
                 chat_message_id=message_id,
-                user_id=None,  # no "user" for Slack bot for now
+                user_id=onyx_user.id if onyx_user else None,
                 db_session=db_session,
             )
             remove_scheduled_feedback_reminder(
@@ -213,7 +437,7 @@ def handle_slack_feedback(
         else:
             msg = f"<@{user_id_to_post_confirmation}> has {feedback_response_txt} the AI Answer"
 
-        respond_in_thread(
+        respond_in_thread_or_channel(
             client=client,
             channel=channel_id_to_post_confirmation,
             text=msg,
@@ -232,7 +456,7 @@ def handle_followup_button(
         action_id = cast(str, action.get("block_id"))
 
     channel_id = req.payload["container"]["channel_id"]
-    thread_ts = req.payload["container"]["thread_ts"]
+    thread_ts = req.payload["container"].get("thread_ts", None)
 
     update_emote_react(
         emoji=DANSWER_FOLLOWUP_EMOJI,
@@ -265,7 +489,7 @@ def handle_followup_button(
 
     blocks = build_follow_up_resolved_blocks(tag_ids=tag_ids, group_ids=group_ids)
 
-    respond_in_thread(
+    respond_in_thread_or_channel(
         client=client.web_client,
         channel=channel_id,
         text="Received your request for more help",
@@ -315,7 +539,7 @@ def handle_followup_resolved_button(
 ) -> None:
     channel_id = req.payload["container"]["channel_id"]
     message_ts = req.payload["container"]["message_ts"]
-    thread_ts = req.payload["container"]["thread_ts"]
+    thread_ts = req.payload["container"].get("thread_ts", None)
 
     clicker_name = get_clicker_name(req, client)
 
@@ -349,7 +573,7 @@ def handle_followup_resolved_button(
 
     resolved_block = SectionBlock(text=msg_text)
 
-    respond_in_thread(
+    respond_in_thread_or_channel(
         client=client.web_client,
         channel=channel_id,
         text="Your request for help as been addressed!",

backend/onyx/onyxbot/slack/handlers/handle_message.py

@@ -18,7 +18,7 @@ from onyx.onyxbot.slack.handlers.handle_standard_answers import (
 from onyx.onyxbot.slack.models import SlackMessageInfo
 from onyx.onyxbot.slack.utils import fetch_slack_user_ids_from_emails
 from onyx.onyxbot.slack.utils import fetch_user_ids_from_groups
-from onyx.onyxbot.slack.utils import respond_in_thread
+from onyx.onyxbot.slack.utils import respond_in_thread_or_channel
 from onyx.onyxbot.slack.utils import slack_usage_report
 from onyx.onyxbot.slack.utils import update_emote_react
 from onyx.utils.logger import setup_logger
@@ -29,7 +29,7 @@ logger_base = setup_logger()
 
 def send_msg_ack_to_user(details: SlackMessageInfo, client: WebClient) -> None:
     if details.is_bot_msg and details.sender_id:
-        respond_in_thread(
+        respond_in_thread_or_channel(
             client=client,
             channel=details.channel_to_respond,
             thread_ts=details.msg_to_respond,
@@ -202,7 +202,7 @@ def handle_message(
     # which would just respond to the sender
     if send_to and is_bot_msg:
         if sender_id:
-            respond_in_thread(
+            respond_in_thread_or_channel(
                 client=client,
                 channel=channel,
                 receiver_ids=[sender_id],
@@ -220,6 +220,7 @@ def handle_message(
             add_slack_user_if_not_exists(db_session, message_info.email)
 
         # first check if we need to respond with a standard answer
+        # standard answers should be published in a thread
         used_standard_answer = handle_standard_answers(
             message_info=message_info,
             receiver_ids=send_to,

backend/onyx/onyxbot/slack/handlers/handle_regular_answer.py

@@ -33,7 +33,7 @@ from onyx.onyxbot.slack.blocks import build_slack_response_blocks
 from onyx.onyxbot.slack.handlers.utils import send_team_member_message
 from onyx.onyxbot.slack.handlers.utils import slackify_message_thread
 from onyx.onyxbot.slack.models import SlackMessageInfo
-from onyx.onyxbot.slack.utils import respond_in_thread
+from onyx.onyxbot.slack.utils import respond_in_thread_or_channel
 from onyx.onyxbot.slack.utils import SlackRateLimiter
 from onyx.onyxbot.slack.utils import update_emote_react
 from onyx.server.query_and_chat.models import CreateChatMessageRequest
@@ -82,12 +82,38 @@ def handle_regular_answer(
 
     message_ts_to_respond_to = message_info.msg_to_respond
     is_bot_msg = message_info.is_bot_msg
+
+    # Capture whether response mode for channel is ephemeral. Even if the channel is set
+    # to respond with an ephemeral message, we still send as non-ephemeral if
+    # the message is a dm with the Onyx bot.
+    send_as_ephemeral = (
+        slack_channel_config.channel_config.get("is_ephemeral", False)
+        and not message_info.is_bot_dm
+    )
+
+    # If the channel mis configured to respond with an ephemeral message,
+    # or the message is a dm to the Onyx bot, we should use the proper onyx user from the email.
+    # This will make documents privately accessible to the user available to Onyx Bot answers.
+    # Otherwise - if not ephemeral or DM to Onyx Bot - we must use None as the user to restrict
+    # to public docs.
+
     user = None
-    if message_info.is_bot_dm:
+    if message_info.is_bot_dm or send_as_ephemeral:
         if message_info.email:
             with get_session_with_current_tenant() as db_session:
                 user = get_user_by_email(message_info.email, db_session)
 
+    target_thread_ts = (
+        None
+        if send_as_ephemeral and len(message_info.thread_messages) < 2
+        else message_ts_to_respond_to
+    )
+    target_receiver_ids = (
+        [message_info.sender_id]
+        if message_info.sender_id and send_as_ephemeral
+        else receiver_ids
+    )
+
     document_set_names: list[str] | None = None
     prompt = None
     # If no persona is specified, use the default search based persona
@@ -134,11 +160,10 @@ def handle_regular_answer(
     history_messages = messages[:-1]
     single_message_history = slackify_message_thread(history_messages) or None
 
+    # Always check for ACL permissions, also for documnt sets that were explicitly added
+    # to the Bot by the Administrator. (Change relative to earlier behavior where all documents
+    # in an attached document set were available to all users in the channel.)
     bypass_acl = False
-    if slack_channel_config.persona and slack_channel_config.persona.document_sets:
-        # For Slack channels, use the full document set, admin will be warned when configuring it
-        # with non-public document sets
-        bypass_acl = True
 
     if not message_ts_to_respond_to and not is_bot_msg:
         # if the message is not "/onyx" command, then it should have a message ts to respond to
@@ -219,12 +244,13 @@ def handle_regular_answer(
         # Optionally, respond in thread with the error message, Used primarily
         # for debugging purposes
         if should_respond_with_error_msgs:
-            respond_in_thread(
+            respond_in_thread_or_channel(
                 client=client,
                 channel=channel,
-                receiver_ids=None,
+                receiver_ids=target_receiver_ids,
                 text=f"Encountered exception when trying to answer: \n\n```{e}```",
-                thread_ts=message_ts_to_respond_to,
+                thread_ts=target_thread_ts,
+                send_as_ephemeral=send_as_ephemeral,
             )
 
         # In case of failures, don't keep the reaction there permanently
@@ -242,32 +268,36 @@ def handle_regular_answer(
     if answer is None:
         assert DISABLE_GENERATIVE_AI is True
         try:
-            respond_in_thread(
+            respond_in_thread_or_channel(
                 client=client,
                 channel=channel,
-                receiver_ids=receiver_ids,
+                receiver_ids=target_receiver_ids,
                 text="Hello! Onyx has some results for you!",
                 blocks=[
                     SectionBlock(
                         text="Onyx is down for maintenance.\nWe're working hard on recharging the AI!"
                     )
                 ],
-                thread_ts=message_ts_to_respond_to,
+                thread_ts=target_thread_ts,
+                send_as_ephemeral=send_as_ephemeral,
                 # don't unfurl, since otherwise we will have 5+ previews which makes the message very long
                 unfurl=False,
             )
 
             # For DM (ephemeral message), we need to create a thread via a normal message so the user can see
             # the ephemeral message. This also will give the user a notification which ephemeral message does not.
-            if receiver_ids:
-                respond_in_thread(
+
+            # If the channel is ephemeral, we don't need to send a message to the user since they will already see the message
+            if target_receiver_ids and not send_as_ephemeral:
+                respond_in_thread_or_channel(
                     client=client,
                     channel=channel,
                     text=(
                         "👋 Hi, we've just gathered and forwarded the relevant "
                         + "information to the team. They'll get back to you shortly!"
                     ),
-                    thread_ts=message_ts_to_respond_to,
+                    thread_ts=target_thread_ts,
+                    send_as_ephemeral=send_as_ephemeral,
                 )
 
             return False
@@ -316,12 +346,13 @@ def handle_regular_answer(
         # Optionally, respond in thread with the error message
         # Used primarily for debugging purposes
         if should_respond_with_error_msgs:
-            respond_in_thread(
+            respond_in_thread_or_channel(
                 client=client,
                 channel=channel,
-                receiver_ids=None,
+                receiver_ids=target_receiver_ids,
                 text="Found no documents when trying to answer. Did you index any documents?",
-                thread_ts=message_ts_to_respond_to,
+                thread_ts=target_thread_ts,
+                send_as_ephemeral=send_as_ephemeral,
             )
         return True
 
@@ -349,15 +380,27 @@ def handle_regular_answer(
         # Optionally, respond in thread with the error message
         # Used primarily for debugging purposes
         if should_respond_with_error_msgs:
-            respond_in_thread(
+            respond_in_thread_or_channel(
                 client=client,
                 channel=channel,
-                receiver_ids=None,
+                receiver_ids=target_receiver_ids,
                 text="Found no citations or quotes when trying to answer.",
-                thread_ts=message_ts_to_respond_to,
+                thread_ts=target_thread_ts,
+                send_as_ephemeral=send_as_ephemeral,
             )
         return True
 
+    if (
+        send_as_ephemeral
+        and target_receiver_ids is not None
+        and len(target_receiver_ids) == 1
+    ):
+        offer_ephemeral_publication = True
+        skip_ai_feedback = True
+    else:
+        offer_ephemeral_publication = False
+        skip_ai_feedback = False if feedback_reminder_id else True
+
     all_blocks = build_slack_response_blocks(
         message_info=message_info,
         answer=answer,
@@ -365,31 +408,39 @@ def handle_regular_answer(
         use_citations=True,  # No longer supporting quotes
         feedback_reminder_id=feedback_reminder_id,
         expecting_search_result=expecting_search_result,
+        offer_ephemeral_publication=offer_ephemeral_publication,
+        skip_ai_feedback=skip_ai_feedback,
     )
 
     try:
-        respond_in_thread(
+        respond_in_thread_or_channel(
             client=client,
             channel=channel,
-            receiver_ids=[message_info.sender_id]
-            if message_info.is_bot_msg and message_info.sender_id
-            else receiver_ids,
+            receiver_ids=target_receiver_ids,
             text="Hello! Onyx has some results for you!",
             blocks=all_blocks,
-            thread_ts=message_ts_to_respond_to,
+            thread_ts=target_thread_ts,
             # don't unfurl, since otherwise we will have 5+ previews which makes the message very long
             unfurl=False,
+            send_as_ephemeral=send_as_ephemeral,
         )
 
         # For DM (ephemeral message), we need to create a thread via a normal message so the user can see
         # the ephemeral message. This also will give the user a notification which ephemeral message does not.
         # if there is no message_ts_to_respond_to, and we have made it this far, then this is a /onyx message
         # so we shouldn't send_team_member_message
-        if receiver_ids and message_ts_to_respond_to is not None:
+        if (
+            target_receiver_ids
+            and message_ts_to_respond_to is not None
+            and not send_as_ephemeral
+            and target_thread_ts is not None
+        ):
             send_team_member_message(
                 client=client,
                 channel=channel,
-                thread_ts=message_ts_to_respond_to,
+                thread_ts=target_thread_ts,
+                receiver_ids=target_receiver_ids,
+                send_as_ephemeral=send_as_ephemeral,
             )
 
         return False

backend/onyx/onyxbot/slack/handlers/utils.py

@@ -2,7 +2,7 @@ from slack_sdk import WebClient
 
 from onyx.chat.models import ThreadMessage
 from onyx.configs.constants import MessageType
-from onyx.onyxbot.slack.utils import respond_in_thread
+from onyx.onyxbot.slack.utils import respond_in_thread_or_channel
 
 
 def slackify_message_thread(messages: list[ThreadMessage]) -> str:
@@ -32,8 +32,10 @@ def send_team_member_message(
     client: WebClient,
     channel: str,
     thread_ts: str,
+    receiver_ids: list[str] | None = None,
+    send_as_ephemeral: bool = False,
 ) -> None:
-    respond_in_thread(
+    respond_in_thread_or_channel(
         client=client,
         channel=channel,
         text=(
@@ -41,4 +43,6 @@ def send_team_member_message(
             + "information to the team. They'll get back to you shortly!"
         ),
         thread_ts=thread_ts,
+        receiver_ids=None,
+        send_as_ephemeral=send_as_ephemeral,
     )

backend/onyx/onyxbot/slack/listener.py

@@ -57,7 +57,9 @@ from onyx.onyxbot.slack.constants import FOLLOWUP_BUTTON_ACTION_ID
 from onyx.onyxbot.slack.constants import FOLLOWUP_BUTTON_RESOLVED_ACTION_ID
 from onyx.onyxbot.slack.constants import GENERATE_ANSWER_BUTTON_ACTION_ID
 from onyx.onyxbot.slack.constants import IMMEDIATE_RESOLVED_BUTTON_ACTION_ID
+from onyx.onyxbot.slack.constants import KEEP_TO_YOURSELF_ACTION_ID
 from onyx.onyxbot.slack.constants import LIKE_BLOCK_ACTION_ID
+from onyx.onyxbot.slack.constants import SHOW_EVERYONE_ACTION_ID
 from onyx.onyxbot.slack.constants import VIEW_DOC_FEEDBACK_ID
 from onyx.onyxbot.slack.handlers.handle_buttons import handle_doc_feedback_button
 from onyx.onyxbot.slack.handlers.handle_buttons import handle_followup_button
@@ -67,6 +69,9 @@ from onyx.onyxbot.slack.handlers.handle_buttons import (
 from onyx.onyxbot.slack.handlers.handle_buttons import (
     handle_generate_answer_button,
 )
+from onyx.onyxbot.slack.handlers.handle_buttons import (
+    handle_publish_ephemeral_message_button,
+)
 from onyx.onyxbot.slack.handlers.handle_buttons import handle_slack_feedback
 from onyx.onyxbot.slack.handlers.handle_message import handle_message
 from onyx.onyxbot.slack.handlers.handle_message import (
@@ -81,7 +86,7 @@ from onyx.onyxbot.slack.utils import get_onyx_bot_slack_bot_id
 from onyx.onyxbot.slack.utils import read_slack_thread
 from onyx.onyxbot.slack.utils import remove_onyx_bot_tag
 from onyx.onyxbot.slack.utils import rephrase_slack_message
-from onyx.onyxbot.slack.utils import respond_in_thread
+from onyx.onyxbot.slack.utils import respond_in_thread_or_channel
 from onyx.onyxbot.slack.utils import TenantSocketModeClient
 from onyx.redis.redis_pool import get_redis_client
 from onyx.server.manage.models import SlackBotTokens
@@ -667,7 +672,11 @@ def process_feedback(req: SocketModeRequest, client: TenantSocketModeClient) ->
         feedback_msg_reminder = cast(str, action.get("value"))
         feedback_id = cast(str, action.get("block_id"))
         channel_id = cast(str, req.payload["container"]["channel_id"])
-        thread_ts = cast(str, req.payload["container"]["thread_ts"])
+        thread_ts = cast(
+            str,
+            req.payload["container"].get("thread_ts")
+            or req.payload["container"].get("message_ts"),
+        )
     else:
         logger.error("Unable to process feedback. Action not found")
         return
@@ -783,7 +792,7 @@ def apologize_for_fail(
     details: SlackMessageInfo,
     client: TenantSocketModeClient,
 ) -> None:
-    respond_in_thread(
+    respond_in_thread_or_channel(
         client=client.web_client,
         channel=details.channel_to_respond,
         thread_ts=details.msg_to_respond,
@@ -859,6 +868,14 @@ def action_routing(req: SocketModeRequest, client: TenantSocketModeClient) -> No
         if action["action_id"] in [DISLIKE_BLOCK_ACTION_ID, LIKE_BLOCK_ACTION_ID]:
             # AI Answer feedback
             return process_feedback(req, client)
+        elif action["action_id"] in [
+            SHOW_EVERYONE_ACTION_ID,
+            KEEP_TO_YOURSELF_ACTION_ID,
+        ]:
+            # Publish ephemeral message or keep hidden in main channel
+            return handle_publish_ephemeral_message_button(
+                req, client, action["action_id"]
+            )
         elif action["action_id"] == FEEDBACK_DOC_BUTTON_BLOCK_ACTION_ID:
             # Activation of the "source feedback" button
             return handle_doc_feedback_button(req, client)

backend/onyx/onyxbot/slack/models.py

@@ -1,3 +1,5 @@
+from typing import Literal
+
 from pydantic import BaseModel
 
 from onyx.chat.models import ThreadMessage
@@ -13,3 +15,37 @@ class SlackMessageInfo(BaseModel):
     bypass_filters: bool  # User has tagged @OnyxBot
     is_bot_msg: bool  # User is using /OnyxBot
     is_bot_dm: bool  # User is direct messaging to OnyxBot
+
+
+# Models used to encode the relevant data for the ephemeral message actions
+class ActionValuesEphemeralMessageMessageInfo(BaseModel):
+    bypass_filters: bool | None
+    channel_to_respond: str | None
+    msg_to_respond: str | None
+    email: str | None
+    sender_id: str | None
+    thread_messages: list[ThreadMessage] | None
+    is_bot_msg: bool | None
+    is_bot_dm: bool | None
+    thread_to_respond: str | None
+
+
+class ActionValuesEphemeralMessageChannelConfig(BaseModel):
+    channel_name: str | None
+    respond_tag_only: bool | None
+    respond_to_bots: bool | None
+    is_ephemeral: bool
+    respond_member_group_list: list[str] | None
+    answer_filters: list[
+        Literal["well_answered_postfilter", "questionmark_prefilter"]
+    ] | None
+    follow_up_tags: list[str] | None
+    show_continue_in_web_ui: bool
+
+
+class ActionValuesEphemeralMessage(BaseModel):
+    original_question_ts: str | None
+    feedback_reminder_id: str | None
+    chat_message_id: int
+    message_info: ActionValuesEphemeralMessageMessageInfo
+    channel_conf: ActionValuesEphemeralMessageChannelConfig

backend/onyx/onyxbot/slack/utils.py

@@ -184,7 +184,7 @@ def _build_error_block(error_message: str) -> Block:
     backoff=2,
     logger=cast(logging.Logger, logger),
 )
-def respond_in_thread(
+def respond_in_thread_or_channel(
     client: WebClient,
     channel: str,
     thread_ts: str | None,
@@ -193,6 +193,7 @@ def respond_in_thread(
     receiver_ids: list[str] | None = None,
     metadata: Metadata | None = None,
     unfurl: bool = True,
+    send_as_ephemeral: bool | None = True,
 ) -> list[str]:
     if not text and not blocks:
         raise ValueError("One of `text` or `blocks` must be provided")
@@ -236,6 +237,7 @@ def respond_in_thread(
         message_ids.append(response["message_ts"])
     else:
         slack_call = make_slack_api_rate_limited(client.chat_postEphemeral)
+
         for receiver in receiver_ids:
             try:
                 response = slack_call(
@@ -299,6 +301,12 @@ def build_feedback_id(
     return unique_prefix + ID_SEPARATOR + feedback_id
 
 
+def build_publish_ephemeral_message_id(
+    original_question_ts: str,
+) -> str:
+    return "publish_ephemeral_message__" + original_question_ts
+
+
 def build_continue_in_web_ui_id(
     message_id: int,
 ) -> str:
@@ -539,7 +547,7 @@ def read_slack_thread(
 
                 # If auto-detected filters are on, use the second block for the actual answer
                 # The first block is the auto-detected filters
-                if message.startswith("_Filters"):
+                if message is not None and message.startswith("_Filters"):
                     if len(blocks) < 2:
                         logger.warning(f"Only filter blocks found: {reply}")
                         continue
@@ -611,7 +619,7 @@ class SlackRateLimiter:
     def notify(
         self, client: WebClient, channel: str, position: int, thread_ts: str | None
     ) -> None:
-        respond_in_thread(
+        respond_in_thread_or_channel(
             client=client,
             channel=channel,
             receiver_ids=None,

backend/onyx/prompts/image_analysis.py

@@ -0,0 +1,22 @@
+# Used for creating embeddings of images for vector search
+IMAGE_SUMMARIZATION_SYSTEM_PROMPT = """
+You are an assistant for summarizing images for retrieval.
+Summarize the content of the following image and be as precise as possible.
+The summary will be embedded and used to retrieve the original image.
+Therefore, write a concise summary of the image that is optimized for retrieval.
+"""
+
+# Prompt for generating image descriptions with filename context
+IMAGE_SUMMARIZATION_USER_PROMPT = """
+The image has the file name '{title}'.
+Describe precisely and concisely what the image shows.
+"""
+
+
+# Used for analyzing images in response to user queries at search time
+IMAGE_ANALYSIS_SYSTEM_PROMPT = (
+    "You are an AI assistant specialized in describing images.\n"
+    "You will receive a user question plus an image URL. Provide a concise textual answer.\n"
+    "Focus on aspects of the image that are relevant to the user's question.\n"
+    "Be specific and detailed about visual elements that directly address the query.\n"
+)

backend/onyx/seeding/load_docs.py

@@ -55,7 +55,11 @@ def _create_indexable_chunks(
             # The section is not really used past this point since we have already done the other processing
             # for the chunking and embedding.
             sections=[
-                Section(text=preprocessed_doc["content"], link=preprocessed_doc["url"])
+                Section(
+                    text=preprocessed_doc["content"],
+                    link=preprocessed_doc["url"],
+                    image_file_name=None,
+                )
             ],
             source=DocumentSource.WEB,
             semantic_identifier=preprocessed_doc["title"],
@@ -93,6 +97,7 @@ def _create_indexable_chunks(
             document_sets=set(),
             boost=DEFAULT_BOOST,
             large_chunk_id=None,
+            image_file_name=None,
         )
 
         chunks.append(chunk)

backend/onyx/server/manage/models.py

@@ -181,6 +181,7 @@ class SlackChannelConfigCreationRequest(BaseModel):
     channel_name: str
     respond_tag_only: bool = False
     respond_to_bots: bool = False
+    is_ephemeral: bool = False
     show_continue_in_web_ui: bool = False
     enable_auto_filters: bool = False
     # If no team members, assume respond in the channel to everyone

backend/onyx/server/manage/slack_bot.py

@@ -71,6 +71,15 @@ def _form_channel_config(
             "also respond to a predetermined set of users."
         )
 
+    if (
+        slack_channel_config_creation_request.is_ephemeral
+        and slack_channel_config_creation_request.respond_member_group_list
+    ):
+        raise ValueError(
+            "Cannot set OnyxBot to respond to users in a private (ephemeral) message "
+            "and also respond to a selected list of users."
+        )
+
     channel_config: ChannelConfig = {
         "channel_name": cleaned_channel_name,
     }
@@ -91,6 +100,8 @@ def _form_channel_config(
         "respond_to_bots"
     ] = slack_channel_config_creation_request.respond_to_bots
 
+    channel_config["is_ephemeral"] = slack_channel_config_creation_request.is_ephemeral
+
     channel_config["disabled"] = slack_channel_config_creation_request.disabled
 
     return channel_config

backend/onyx/server/settings/models.py

@@ -53,6 +53,11 @@ class Settings(BaseModel):
     auto_scroll: bool | None = False
     query_history_type: QueryHistoryType | None = None
 
+    # Image processing settings
+    image_extraction_and_analysis_enabled: bool | None = False
+    search_time_image_analysis_enabled: bool | None = False
+    image_analysis_max_size_mb: int | None = 20
+
 
 class UserSettings(Settings):
     notifications: list[Notification]

backend/onyx/server/settings/store.py

@@ -47,6 +47,7 @@ def load_settings() -> Settings:
 
     settings.anonymous_user_enabled = anonymous_user_enabled
     settings.query_history_type = ONYX_QUERY_HISTORY_TYPE
+
     return settings
 
 

backend/onyx/utils/error_handling.py

@@ -0,0 +1,23 @@
+"""
+Standardized error handling utilities.
+"""
+from onyx.configs.app_configs import CONTINUE_ON_CONNECTOR_FAILURE
+from onyx.utils.logger import setup_logger
+
+logger = setup_logger()
+
+
+def handle_connector_error(e: Exception, context: str) -> None:
+    """
+    Standard error handling for connectors.
+
+    Args:
+        e: The exception that was raised
+        context: A description of where the error occurred
+
+    Raises:
+        The original exception if CONTINUE_ON_CONNECTOR_FAILURE is False
+    """
+    logger.error(f"Error in {context}: {e}", exc_info=e)
+    if not CONTINUE_ON_CONNECTOR_FAILURE:
+        raise

backend/requirements/default.txt

@@ -1,9 +1,10 @@
+aioboto3==14.0.0
 aiohttp==3.10.2
 alembic==1.10.4
 asyncpg==0.27.0
 atlassian-python-api==3.41.16
 beautifulsoup4==4.12.3
-boto3==1.34.84
+boto3==1.36.23
 celery==5.5.0b4
 chardet==5.2.0
 dask==2023.8.1

backend/requirements/model_server.txt

@@ -13,4 +13,5 @@ transformers==4.39.2
 uvicorn==0.21.1
 voyageai==0.2.3
 litellm==1.61.16
-sentry-sdk[fastapi,celery,starlette]==2.14.0
+sentry-sdk[fastapi,celery,starlette]==2.14.0
+aioboto3==13.4.0

backend/scripts/chat_history_seeding.py

@@ -0,0 +1,45 @@
+import argparse
+import logging
+from logging import getLogger
+
+from onyx.db.seeding.chat_history_seeding import seed_chat_history
+
+# Configure the logger
+logging.basicConfig(
+    level=logging.INFO,  # Set the log level (DEBUG, INFO, WARNING, ERROR, CRITICAL)
+    format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",  # Log format
+    handlers=[logging.StreamHandler()],  # Output logs to console
+)
+
+logger = getLogger(__name__)
+
+
+def go_main(num_sessions: int, num_messages: int, num_days: int) -> None:
+    seed_chat_history(num_sessions, num_messages, num_days)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Seed chat history")
+    parser.add_argument(
+        "--sessions",
+        type=int,
+        default=2048,
+        help="Number of chat sessions to seed",
+    )
+
+    parser.add_argument(
+        "--messages",
+        type=int,
+        default=4,
+        help="Number of chat messages to seed per session",
+    )
+
+    parser.add_argument(
+        "--days",
+        type=int,
+        default=90,
+        help="Number of days looking backwards over which to seed the timestamps with",
+    )
+
+    args = parser.parse_args()
+    go_main(args.sessions, args.messages, args.days)

backend/scripts/debugging/onyx_vespa.py

@@ -207,7 +207,7 @@ def query_vespa(
     yql: str, tenant_id: Optional[str] = None, limit: int = 10
 ) -> List[Dict[str, Any]]:
     # Perform a Vespa query using YQL syntax.
-    filters = IndexFilters(tenant_id=tenant_id, access_control_list=[])
+    filters = IndexFilters(tenant_id=None, access_control_list=[])
     filter_string = build_vespa_filters(filters, remove_trailing_and=True)
     full_yql = yql.strip()
     if filter_string:
@@ -472,9 +472,7 @@ def get_document_acls(
             print("-" * 80)
 
 
-def get_current_chunk_count(
-    document_id: str, index_name: str, tenant_id: str
-) -> int | None:
+def get_current_chunk_count(document_id: str) -> int | None:
     with get_session_with_current_tenant() as session:
         return (
             session.query(Document.chunk_count)
@@ -486,7 +484,7 @@ def get_current_chunk_count(
 def get_number_of_chunks_we_think_exist(
     document_id: str, index_name: str, tenant_id: str
 ) -> int:
-    current_chunk_count = get_current_chunk_count(document_id, index_name, tenant_id)
+    current_chunk_count = get_current_chunk_count(document_id)
     print(f"Current chunk count: {current_chunk_count}")
 
     doc_info = VespaIndex.enrich_basic_chunk_info(
@@ -636,6 +634,7 @@ def delete_where(
     Removes visited documents in `cluster` where the given selection
     is true, using Vespa's 'delete where' endpoint.
 
+
     :param index_name: Typically <namespace>/<document-type> from your schema
     :param selection:  The selection string, e.g., "true" or "foo contains 'bar'"
     :param cluster:    The name of the cluster where documents reside
@@ -799,7 +798,7 @@ def main() -> None:
     args = parser.parse_args()
     vespa_debug = VespaDebugging(args.tenant_id)
 
-    CURRENT_TENANT_ID_CONTEXTVAR.set(args.tenant_id)
+    CURRENT_TENANT_ID_CONTEXTVAR.set(args.tenant_id or "public")
     if args.action == "delete-all-documents":
         if not args.tenant_id:
             parser.error("--tenant-id is required for delete-all-documents action")

backend/scripts/query_time_check/seed_dummy_docs.py

@@ -71,6 +71,7 @@ def generate_dummy_chunk(
         title_embedding=generate_random_embedding(embedding_dim),
         large_chunk_id=None,
         large_chunk_reference_ids=[],
+        image_file_name=None,
     )
 
     document_set_names = []

backend/shared_configs/configs.py

@@ -68,6 +68,12 @@ LOG_LEVEL = os.environ.get("LOG_LEVEL", "info")
 # allow us to specify a custom timeout
 API_BASED_EMBEDDING_TIMEOUT = int(os.environ.get("API_BASED_EMBEDDING_TIMEOUT", "600"))
 
+# Local batch size for VertexAI embedding models currently calibrated for item size of 512 tokens
+# NOTE: increasing this value may lead to API errors due to token limit exhaustion per call.
+VERTEXAI_EMBEDDING_LOCAL_BATCH_SIZE = int(
+    os.environ.get("VERTEXAI_EMBEDDING_LOCAL_BATCH_SIZE", "25")
+)
+
 # Only used for OpenAI
 OPENAI_EMBEDDING_TIMEOUT = int(
     os.environ.get("OPENAI_EMBEDDING_TIMEOUT", API_BASED_EMBEDDING_TIMEOUT)
@@ -200,12 +206,12 @@ SUPPORTED_EMBEDDING_MODELS = [
         index_name="danswer_chunk_text_embedding_3_small",
     ),
     SupportedEmbeddingModel(
-        name="google/text-embedding-004",
+        name="google/text-embedding-005",
         dim=768,
         index_name="danswer_chunk_google_text_embedding_004",
     ),
     SupportedEmbeddingModel(
-        name="google/text-embedding-004",
+        name="google/text-embedding-005",
         dim=768,
         index_name="danswer_chunk_text_embedding_004",
     ),

backend/shared_configs/enums.py

@@ -13,6 +13,7 @@ class EmbeddingProvider(str, Enum):
 class RerankerProvider(str, Enum):
     COHERE = "cohere"
     LITELLM = "litellm"
+    BEDROCK = "bedrock"
 
 
 class EmbedTextType(str, Enum):

backend/tests/daily/connectors/confluence/test_confluence_basic.py

@@ -45,7 +45,7 @@ def test_confluence_connector_basic(
     with pytest.raises(StopIteration):
         next(doc_batch_generator)
 
-    assert len(doc_batch) == 3
+    assert len(doc_batch) == 2
 
     page_within_a_page_doc: Document | None = None
     page_doc: Document | None = None

backend/tests/integration/tests/pruning/test_pruning.py

@@ -142,8 +142,12 @@ def test_web_pruning(reset: None, vespa_client: vespa_fixture) -> None:
             selected_cc_pair = CCPairManager.get_indexing_status_by_id(
                 cc_pair_1.id, user_performing_action=admin_user
             )
+
             assert selected_cc_pair is not None, "cc_pair not found after indexing!"
-            assert selected_cc_pair.docs_indexed == 15
+
+            # used to be 15, but now
+            # localhost:8889/ and localhost:8889/index.html are deduped
+            assert selected_cc_pair.docs_indexed == 14
 
             logger.info("Removing about.html.")
             os.remove(os.path.join(website_tgt, "about.html"))
@@ -160,24 +164,29 @@ def test_web_pruning(reset: None, vespa_client: vespa_fixture) -> None:
                 cc_pair_1.id, user_performing_action=admin_user
             )
             assert selected_cc_pair is not None, "cc_pair not found after pruning!"
-            assert selected_cc_pair.docs_indexed == 13
+            assert selected_cc_pair.docs_indexed == 12
 
             # check vespa
+            root_id = f"http://{hostname}:{port}/"
             index_id = f"http://{hostname}:{port}/index.html"
             about_id = f"http://{hostname}:{port}/about.html"
             courses_id = f"http://{hostname}:{port}/courses.html"
 
-            doc_ids = [index_id, about_id, courses_id]
+            doc_ids = [root_id, index_id, about_id, courses_id]
             retrieved_docs_dict = vespa_client.get_documents_by_id(doc_ids)["documents"]
             retrieved_docs = {
                 doc["fields"]["document_id"]: doc["fields"]
                 for doc in retrieved_docs_dict
             }
 
-            # verify index.html exists in Vespa
-            retrieved_doc = retrieved_docs.get(index_id)
+            # verify root exists in Vespa
+            retrieved_doc = retrieved_docs.get(root_id)
             assert retrieved_doc
 
+            # verify index.html does not exist in Vespa since it is a duplicate of root
+            retrieved_doc = retrieved_docs.get(index_id)
+            assert not retrieved_doc
+
             # verify about and courses do not exist
             retrieved_doc = retrieved_docs.get(about_id)
             assert not retrieved_doc

backend/tests/integration/tests/query_history/test_usage_reports.py

@@ -0,0 +1,46 @@
+from datetime import datetime
+from datetime import timedelta
+from datetime import timezone
+
+from ee.onyx.db.usage_export import get_all_empty_chat_message_entries
+from onyx.db.engine import get_session_with_current_tenant
+from onyx.db.seeding.chat_history_seeding import seed_chat_history
+
+
+def test_usage_reports(reset: None) -> None:
+    EXPECTED_SESSIONS = 2048
+    MESSAGES_PER_SESSION = 4
+    EXPECTED_MESSAGES = EXPECTED_SESSIONS * MESSAGES_PER_SESSION
+
+    seed_chat_history(EXPECTED_SESSIONS, MESSAGES_PER_SESSION, 90)
+
+    with get_session_with_current_tenant() as db_session:
+        # count of all entries should be exact
+        period = (
+            datetime.fromtimestamp(0, tz=timezone.utc),
+            datetime.now(tz=timezone.utc),
+        )
+
+        count = 0
+        for entry_batch in get_all_empty_chat_message_entries(db_session, period):
+            for entry in entry_batch:
+                count += 1
+
+        assert count == EXPECTED_MESSAGES
+
+        # count in a one month time range should be within a certain range statistically
+        # this can be improved if we seed the chat history data deterministically
+        period = (
+            datetime.now(tz=timezone.utc) - timedelta(days=30),
+            datetime.now(tz=timezone.utc),
+        )
+
+        count = 0
+        for entry_batch in get_all_empty_chat_message_entries(db_session, period):
+            for entry in entry_batch:
+                count += 1
+
+        lower = EXPECTED_MESSAGES // 3 - (EXPECTED_MESSAGES // (3 * 3))
+        upper = EXPECTED_MESSAGES // 3 + (EXPECTED_MESSAGES // (3 * 3))
+        assert count > lower
+        assert count < upper

backend/tests/unit/ee/onyx/external_permissions/salesforce/test_postprocessing.py

@@ -31,6 +31,7 @@ def create_test_chunk(
         metadata={},
         match_highlights=[],
         updated_at=datetime.now(),
+        image_file_name=None,
     )
 
 

backend/tests/unit/onyx/chat/conftest.py

@@ -80,6 +80,7 @@ def mock_inference_sections() -> list[InferenceSection]:
                 updated_at=datetime(2023, 1, 1),
                 source_links={0: "https://example.com/doc1"},
                 match_highlights=[],
+                image_file_name=None,
             ),
             chunks=MagicMock(),
         ),
@@ -102,6 +103,7 @@ def mock_inference_sections() -> list[InferenceSection]:
                 updated_at=datetime(2023, 1, 2),
                 source_links={0: "https://example.com/doc2"},
                 match_highlights=[],
+                image_file_name=None,
             ),
             chunks=MagicMock(),
         ),

backend/tests/unit/onyx/chat/stream_processing/test_quotes_processing.py

@@ -150,6 +150,7 @@ def test_fuzzy_match_quotes_to_docs() -> None:
         metadata={},
         match_highlights=[],
         updated_at=None,
+        image_file_name=None,
     )
     test_chunk_1 = InferenceChunk(
         document_id="test doc 1",
@@ -168,6 +169,7 @@ def test_fuzzy_match_quotes_to_docs() -> None:
         metadata={},
         match_highlights=[],
         updated_at=None,
+        image_file_name=None,
     )
 
     test_quotes = [

backend/tests/unit/onyx/chat/test_prune_and_merge.py

@@ -37,6 +37,7 @@ def create_inference_chunk(
         metadata={},
         match_highlights=[],
         updated_at=None,
+        image_file_name=None,
     )
 
 

backend/tests/unit/onyx/indexing/test_embedder.py

@@ -62,6 +62,7 @@ def test_default_indexing_embedder_embed_chunks(mock_embedding_model: Mock) -> N
             mini_chunk_texts=None,
             large_chunk_reference_ids=[],
             large_chunk_id=None,
+            image_file_name=None,
         )
     ]
 

deployment/README.md

@@ -80,3 +80,13 @@ prod cluster**
    - `kubectl delete -f .`
    - To not delete the persistent volumes (Document indexes and Users), specify the specific `.yaml` files instead of
      `.` without specifying delete on persistent-volumes.yaml.
+
+### Using Helm to deploy to an existing cluster
+
+Onyx has a helm chart that is convenient to install all services to an existing Kubernetes cluster. To install:
+
+* Currently the helm chart is not published so to install, clone the repo.
+* Configure access to the cluster via kubectl. Ensure the kubectl context is set to the cluster that you want to use
+* The default secrets, environment variables and other service level configuration are stored in `deployment/helm/charts/onyx/values.yml`. You may create another `override.yml`
+* `cd deployment/helm/charts/onyx` and run `helm install onyx -n onyx -f override.yaml .`. This will install onyx on the cluster under the `onyx` namespace.
+* Check the status of the deploy using `kubectl get pods -n onyx`

deployment/docker_compose/docker-compose.dev.yml

@@ -118,6 +118,7 @@ services:
       - API_KEY_HASH_ROUNDS=${API_KEY_HASH_ROUNDS:-}
       # Seeding configuration
       - USE_IAM_AUTH=${USE_IAM_AUTH:-}
+      - ONYX_QUERY_HISTORY_TYPE=${ONYX_QUERY_HISTORY_TYPE:-}
     # Uncomment the line below to use if IAM_AUTH is true and you are using iam auth for postgres
     # volumes:
     #   - ./bundle.pem:/app/bundle.pem:ro

deployment/docker_compose/docker-compose.gpu-dev.yml

@@ -95,6 +95,7 @@ services:
       # Enterprise Edition only
       - API_KEY_HASH_ROUNDS=${API_KEY_HASH_ROUNDS:-}
       - ENABLE_PAID_ENTERPRISE_EDITION_FEATURES=${ENABLE_PAID_ENTERPRISE_EDITION_FEATURES:-false}
+      - ONYX_QUERY_HISTORY_TYPE=${ONYX_QUERY_HISTORY_TYPE:-}
     # Uncomment the line below to use if IAM_AUTH is true and you are using iam auth for postgres
     # volumes:
     #   - ./bundle.pem:/app/bundle.pem:ro

deployment/helm/charts/onyx/templates/ingress-api.yaml

@@ -0,0 +1,27 @@
+{{- if .Values.ingress.enabled -}}
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: {{ include "onyx-stack.fullname" . }}-ingress-api
+  annotations:
+    kubernetes.io/ingress.class: nginx
+    nginx.ingress.kubernetes.io/rewrite-target: /$2
+    nginx.ingress.kubernetes.io/use-regex: "true"
+    cert-manager.io/cluster-issuer: {{ include "onyx-stack.fullname" . }}-letsencrypt
+spec:
+  rules:
+    - host: {{ .Values.ingress.api.host }}
+      http:
+        paths:
+          - path: /api(/|$)(.*)
+            pathType: Prefix
+            backend:
+              service:
+                name: {{ include "onyx-stack.fullname" . }}-api-service
+                port:
+                  number: {{ .Values.api.service.servicePort }}
+  tls:
+    - hosts:
+        - {{ .Values.ingress.api.host }}
+      secretName: {{ include "onyx-stack.fullname" . }}-ingress-api-tls
+{{- end }}

deployment/helm/charts/onyx/templates/ingress-webserver.yaml

@@ -0,0 +1,26 @@
+{{- if .Values.ingress.enabled -}}
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: {{ include "onyx-stack.fullname" . }}-ingress-webserver
+  annotations:
+    kubernetes.io/ingress.class: nginx
+    cert-manager.io/cluster-issuer: {{ include "onyx-stack.fullname" . }}-letsencrypt
+    kubernetes.io/tls-acme: "true"
+spec:
+  rules:
+    - host: {{ .Values.ingress.webserver.host }}
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: {{ include "onyx-stack.fullname" . }}-webserver
+                port:
+                  number: {{ .Values.webserver.service.servicePort }}
+  tls:
+    - hosts:
+        - {{ .Values.ingress.webserver.host }}
+      secretName: {{ include "onyx-stack.fullname" . }}-ingress-webserver-tls
+{{- end }}

deployment/helm/charts/onyx/templates/lets-encrypt.yaml

@@ -0,0 +1,20 @@
+{{- if .Values.letsencrypt.enabled -}}
+apiVersion: cert-manager.io/v1
+kind: ClusterIssuer
+metadata:
+  name: {{ include "onyx-stack.fullname" . }}-letsencrypt
+spec:
+  acme:
+    # The ACME server URL
+    server: https://acme-v02.api.letsencrypt.org/directory
+    # Email address used for ACME registration
+    email: {{ .Values.letsencrypt.email }}
+    # Name of a secret used to store the ACME account private key
+    privateKeySecretRef:
+      name: {{ include "onyx-stack.fullname" . }}-letsencrypt
+    # Enable the HTTP-01 challenge provider
+    solvers:
+      - http01:
+          ingress:
+            class: nginx
+{{- end }}

deployment/helm/charts/onyx/values.yaml

@@ -376,22 +376,17 @@ redis:
     existingSecret: onyx-secrets
     existingSecretPasswordKey: redis_password
 
-# ingress:
-#  enabled: false
-#  className: ""
-#  annotations: {}
-#    # kubernetes.io/ingress.class: nginx
-#    # kubernetes.io/tls-acme: "true"
-#  hosts:
-#    - host: chart-example.local
-#      paths:
-#        - path: /
-#          pathType: ImplementationSpecific
-#  tls: []
-#  #  - secretName: chart-example-tls
-#  #    hosts:
-#  #      - chart-example.local
+ingress:
+  enabled: false
+  className: ""
+  api:
+    host: onyx.local
+  webserver:
+    host: onyx.local
 
+letsencrypt:
+  enabled: false
+  email: "abc@abc.com"
 
 auth:
   # existingSecret onyx-secret for storing smtp, oauth, slack, and other secrets

web/package-lock.json

@@ -44,6 +44,7 @@
         "autoprefixer": "^10.4.14",
         "class-variance-authority": "^0.7.0",
         "clsx": "^2.1.1",
+        "cmdk": "^1.0.0",
         "date-fns": "^3.6.0",
         "favicon-fetch": "^1.0.0",
         "formik": "^2.2.9",
@@ -9313,6 +9314,438 @@
         "node": ">=6"
       }
     },
+    "node_modules/cmdk": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/cmdk/-/cmdk-1.0.0.tgz",
+      "integrity": "sha512-gDzVf0a09TvoJ5jnuPvygTB77+XdOSwEmJ88L6XPFPlv7T3RxbP9jgenfylrAMD0+Le1aO0nVjQUzl2g+vjz5Q==",
+      "license": "MIT",
+      "dependencies": {
+        "@radix-ui/react-dialog": "1.0.5",
+        "@radix-ui/react-primitive": "1.0.3"
+      },
+      "peerDependencies": {
+        "react": "^18.0.0",
+        "react-dom": "^18.0.0"
+      }
+    },
+    "node_modules/cmdk/node_modules/@radix-ui/primitive": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/@radix-ui/primitive/-/primitive-1.0.1.tgz",
+      "integrity": "sha512-yQ8oGX2GVsEYMWGxcovu1uGWPCxV5BFfeeYxqPmuAzUyLT9qmaMXSAhXpb0WrspIeqYzdJpkh2vHModJPgRIaw==",
+      "license": "MIT",
+      "dependencies": {
+        "@babel/runtime": "^7.13.10"
+      }
+    },
+    "node_modules/cmdk/node_modules/@radix-ui/react-compose-refs": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-compose-refs/-/react-compose-refs-1.0.1.tgz",
+      "integrity": "sha512-fDSBgd44FKHa1FRMU59qBMPFcl2PZE+2nmqunj+BWFyYYjnhIDWL2ItDs3rrbJDQOtzt5nIebLCQc4QRfz6LJw==",
+      "license": "MIT",
+      "dependencies": {
+        "@babel/runtime": "^7.13.10"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "react": "^16.8 || ^17.0 || ^18.0"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/cmdk/node_modules/@radix-ui/react-context": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-context/-/react-context-1.0.1.tgz",
+      "integrity": "sha512-ebbrdFoYTcuZ0v4wG5tedGnp9tzcV8awzsxYph7gXUyvnNLuTIcCk1q17JEbnVhXAKG9oX3KtchwiMIAYp9NLg==",
+      "license": "MIT",
+      "dependencies": {
+        "@babel/runtime": "^7.13.10"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "react": "^16.8 || ^17.0 || ^18.0"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/cmdk/node_modules/@radix-ui/react-dialog": {
+      "version": "1.0.5",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-dialog/-/react-dialog-1.0.5.tgz",
+      "integrity": "sha512-GjWJX/AUpB703eEBanuBnIWdIXg6NvJFCXcNlSZk4xdszCdhrJgBoUd1cGk67vFO+WdA2pfI/plOpqz/5GUP6Q==",
+      "license": "MIT",
+      "dependencies": {
+        "@babel/runtime": "^7.13.10",
+        "@radix-ui/primitive": "1.0.1",
+        "@radix-ui/react-compose-refs": "1.0.1",
+        "@radix-ui/react-context": "1.0.1",
+        "@radix-ui/react-dismissable-layer": "1.0.5",
+        "@radix-ui/react-focus-guards": "1.0.1",
+        "@radix-ui/react-focus-scope": "1.0.4",
+        "@radix-ui/react-id": "1.0.1",
+        "@radix-ui/react-portal": "1.0.4",
+        "@radix-ui/react-presence": "1.0.1",
+        "@radix-ui/react-primitive": "1.0.3",
+        "@radix-ui/react-slot": "1.0.2",
+        "@radix-ui/react-use-controllable-state": "1.0.1",
+        "aria-hidden": "^1.1.1",
+        "react-remove-scroll": "2.5.5"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "@types/react-dom": "*",
+        "react": "^16.8 || ^17.0 || ^18.0",
+        "react-dom": "^16.8 || ^17.0 || ^18.0"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        },
+        "@types/react-dom": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/cmdk/node_modules/@radix-ui/react-dismissable-layer": {
+      "version": "1.0.5",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-dismissable-layer/-/react-dismissable-layer-1.0.5.tgz",
+      "integrity": "sha512-aJeDjQhywg9LBu2t/At58hCvr7pEm0o2Ke1x33B+MhjNmmZ17sy4KImo0KPLgsnc/zN7GPdce8Cnn0SWvwZO7g==",
+      "license": "MIT",
+      "dependencies": {
+        "@babel/runtime": "^7.13.10",
+        "@radix-ui/primitive": "1.0.1",
+        "@radix-ui/react-compose-refs": "1.0.1",
+        "@radix-ui/react-primitive": "1.0.3",
+        "@radix-ui/react-use-callback-ref": "1.0.1",
+        "@radix-ui/react-use-escape-keydown": "1.0.3"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "@types/react-dom": "*",
+        "react": "^16.8 || ^17.0 || ^18.0",
+        "react-dom": "^16.8 || ^17.0 || ^18.0"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        },
+        "@types/react-dom": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/cmdk/node_modules/@radix-ui/react-dismissable-layer/node_modules/@radix-ui/react-use-callback-ref": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-callback-ref/-/react-use-callback-ref-1.0.1.tgz",
+      "integrity": "sha512-D94LjX4Sp0xJFVaoQOd3OO9k7tpBYNOXdVhkltUbGv2Qb9OXdrg/CpsjlZv7ia14Sylv398LswWBVVu5nqKzAQ==",
+      "license": "MIT",
+      "dependencies": {
+        "@babel/runtime": "^7.13.10"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "react": "^16.8 || ^17.0 || ^18.0"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/cmdk/node_modules/@radix-ui/react-dismissable-layer/node_modules/@radix-ui/react-use-escape-keydown": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-escape-keydown/-/react-use-escape-keydown-1.0.3.tgz",
+      "integrity": "sha512-vyL82j40hcFicA+M4Ex7hVkB9vHgSse1ZWomAqV2Je3RleKGO5iM8KMOEtfoSB0PnIelMd2lATjTGMYqN5ylTg==",
+      "license": "MIT",
+      "dependencies": {
+        "@babel/runtime": "^7.13.10",
+        "@radix-ui/react-use-callback-ref": "1.0.1"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "react": "^16.8 || ^17.0 || ^18.0"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/cmdk/node_modules/@radix-ui/react-focus-guards": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-focus-guards/-/react-focus-guards-1.0.1.tgz",
+      "integrity": "sha512-Rect2dWbQ8waGzhMavsIbmSVCgYxkXLxxR3ZvCX79JOglzdEy4JXMb98lq4hPxUbLr77nP0UOGf4rcMU+s1pUA==",
+      "license": "MIT",
+      "dependencies": {
+        "@babel/runtime": "^7.13.10"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "react": "^16.8 || ^17.0 || ^18.0"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/cmdk/node_modules/@radix-ui/react-focus-scope": {
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-focus-scope/-/react-focus-scope-1.0.4.tgz",
+      "integrity": "sha512-sL04Mgvf+FmyvZeYfNu1EPAaaxD+aw7cYeIB9L9Fvq8+urhltTRaEo5ysKOpHuKPclsZcSUMKlN05x4u+CINpA==",
+      "license": "MIT",
+      "dependencies": {
+        "@babel/runtime": "^7.13.10",
+        "@radix-ui/react-compose-refs": "1.0.1",
+        "@radix-ui/react-primitive": "1.0.3",
+        "@radix-ui/react-use-callback-ref": "1.0.1"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "@types/react-dom": "*",
+        "react": "^16.8 || ^17.0 || ^18.0",
+        "react-dom": "^16.8 || ^17.0 || ^18.0"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        },
+        "@types/react-dom": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/cmdk/node_modules/@radix-ui/react-focus-scope/node_modules/@radix-ui/react-use-callback-ref": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-callback-ref/-/react-use-callback-ref-1.0.1.tgz",
+      "integrity": "sha512-D94LjX4Sp0xJFVaoQOd3OO9k7tpBYNOXdVhkltUbGv2Qb9OXdrg/CpsjlZv7ia14Sylv398LswWBVVu5nqKzAQ==",
+      "license": "MIT",
+      "dependencies": {
+        "@babel/runtime": "^7.13.10"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "react": "^16.8 || ^17.0 || ^18.0"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/cmdk/node_modules/@radix-ui/react-id": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-id/-/react-id-1.0.1.tgz",
+      "integrity": "sha512-tI7sT/kqYp8p96yGWY1OAnLHrqDgzHefRBKQ2YAkBS5ja7QLcZ9Z/uY7bEjPUatf8RomoXM8/1sMj1IJaE5UzQ==",
+      "license": "MIT",
+      "dependencies": {
+        "@babel/runtime": "^7.13.10",
+        "@radix-ui/react-use-layout-effect": "1.0.1"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "react": "^16.8 || ^17.0 || ^18.0"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/cmdk/node_modules/@radix-ui/react-id/node_modules/@radix-ui/react-use-layout-effect": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-layout-effect/-/react-use-layout-effect-1.0.1.tgz",
+      "integrity": "sha512-v/5RegiJWYdoCvMnITBkNNx6bCj20fiaJnWtRkU18yITptraXjffz5Qbn05uOiQnOvi+dbkznkoaMltz1GnszQ==",
+      "license": "MIT",
+      "dependencies": {
+        "@babel/runtime": "^7.13.10"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "react": "^16.8 || ^17.0 || ^18.0"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/cmdk/node_modules/@radix-ui/react-portal": {
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-portal/-/react-portal-1.0.4.tgz",
+      "integrity": "sha512-Qki+C/EuGUVCQTOTD5vzJzJuMUlewbzuKyUy+/iHM2uwGiru9gZeBJtHAPKAEkB5KWGi9mP/CHKcY0wt1aW45Q==",
+      "license": "MIT",
+      "dependencies": {
+        "@babel/runtime": "^7.13.10",
+        "@radix-ui/react-primitive": "1.0.3"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "@types/react-dom": "*",
+        "react": "^16.8 || ^17.0 || ^18.0",
+        "react-dom": "^16.8 || ^17.0 || ^18.0"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        },
+        "@types/react-dom": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/cmdk/node_modules/@radix-ui/react-presence": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-presence/-/react-presence-1.0.1.tgz",
+      "integrity": "sha512-UXLW4UAbIY5ZjcvzjfRFo5gxva8QirC9hF7wRE4U5gz+TP0DbRk+//qyuAQ1McDxBt1xNMBTaciFGvEmJvAZCg==",
+      "license": "MIT",
+      "dependencies": {
+        "@babel/runtime": "^7.13.10",
+        "@radix-ui/react-compose-refs": "1.0.1",
+        "@radix-ui/react-use-layout-effect": "1.0.1"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "@types/react-dom": "*",
+        "react": "^16.8 || ^17.0 || ^18.0",
+        "react-dom": "^16.8 || ^17.0 || ^18.0"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        },
+        "@types/react-dom": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/cmdk/node_modules/@radix-ui/react-presence/node_modules/@radix-ui/react-use-layout-effect": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-layout-effect/-/react-use-layout-effect-1.0.1.tgz",
+      "integrity": "sha512-v/5RegiJWYdoCvMnITBkNNx6bCj20fiaJnWtRkU18yITptraXjffz5Qbn05uOiQnOvi+dbkznkoaMltz1GnszQ==",
+      "license": "MIT",
+      "dependencies": {
+        "@babel/runtime": "^7.13.10"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "react": "^16.8 || ^17.0 || ^18.0"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/cmdk/node_modules/@radix-ui/react-primitive": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-primitive/-/react-primitive-1.0.3.tgz",
+      "integrity": "sha512-yi58uVyoAcK/Nq1inRY56ZSjKypBNKTa/1mcL8qdl6oJeEaDbOldlzrGn7P6Q3Id5d+SYNGc5AJgc4vGhjs5+g==",
+      "license": "MIT",
+      "dependencies": {
+        "@babel/runtime": "^7.13.10",
+        "@radix-ui/react-slot": "1.0.2"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "@types/react-dom": "*",
+        "react": "^16.8 || ^17.0 || ^18.0",
+        "react-dom": "^16.8 || ^17.0 || ^18.0"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        },
+        "@types/react-dom": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/cmdk/node_modules/@radix-ui/react-slot": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-slot/-/react-slot-1.0.2.tgz",
+      "integrity": "sha512-YeTpuq4deV+6DusvVUW4ivBgnkHwECUu0BiN43L5UCDFgdhsRUWAghhTF5MbvNTPzmiFOx90asDSUjWuCNapwg==",
+      "license": "MIT",
+      "dependencies": {
+        "@babel/runtime": "^7.13.10",
+        "@radix-ui/react-compose-refs": "1.0.1"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "react": "^16.8 || ^17.0 || ^18.0"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/cmdk/node_modules/@radix-ui/react-use-controllable-state": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-controllable-state/-/react-use-controllable-state-1.0.1.tgz",
+      "integrity": "sha512-Svl5GY5FQeN758fWKrjM6Qb7asvXeiZltlT4U2gVfl8Gx5UAv2sMR0LWo8yhsIZh2oQ0eFdZ59aoOOMV7b47VA==",
+      "license": "MIT",
+      "dependencies": {
+        "@babel/runtime": "^7.13.10",
+        "@radix-ui/react-use-callback-ref": "1.0.1"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "react": "^16.8 || ^17.0 || ^18.0"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/cmdk/node_modules/@radix-ui/react-use-controllable-state/node_modules/@radix-ui/react-use-callback-ref": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-callback-ref/-/react-use-callback-ref-1.0.1.tgz",
+      "integrity": "sha512-D94LjX4Sp0xJFVaoQOd3OO9k7tpBYNOXdVhkltUbGv2Qb9OXdrg/CpsjlZv7ia14Sylv398LswWBVVu5nqKzAQ==",
+      "license": "MIT",
+      "dependencies": {
+        "@babel/runtime": "^7.13.10"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "react": "^16.8 || ^17.0 || ^18.0"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/cmdk/node_modules/react-remove-scroll": {
+      "version": "2.5.5",
+      "resolved": "https://registry.npmjs.org/react-remove-scroll/-/react-remove-scroll-2.5.5.tgz",
+      "integrity": "sha512-ImKhrzJJsyXJfBZ4bzu8Bwpka14c/fQt0k+cyFp/PBhTfyDnU5hjOtM4AG/0AMyy8oKzOTR0lDgJIM7pYXI0kw==",
+      "license": "MIT",
+      "dependencies": {
+        "react-remove-scroll-bar": "^2.3.3",
+        "react-style-singleton": "^2.2.1",
+        "tslib": "^2.1.0",
+        "use-callback-ref": "^1.3.0",
+        "use-sidecar": "^1.1.2"
+      },
+      "engines": {
+        "node": ">=10"
+      },
+      "peerDependencies": {
+        "@types/react": "^16.8.0 || ^17.0.0 || ^18.0.0",
+        "react": "^16.8.0 || ^17.0.0 || ^18.0.0"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        }
+      }
+    },
     "node_modules/co": {
       "version": "4.6.0",
       "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz",

web/package.json

@@ -47,6 +47,7 @@
     "autoprefixer": "^10.4.14",
     "class-variance-authority": "^0.7.0",
     "clsx": "^2.1.1",
+    "cmdk": "^1.0.0",
     "date-fns": "^3.6.0",
     "favicon-fetch": "^1.0.0",
     "formik": "^2.2.9",

web/public/Anthropic.svg

@@ -1,8 +1 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<svg width="256px" height="176px" viewBox="0 0 256 176" version="1.1" xmlns="http://www.w3.org/2000/svg" preserveAspectRatio="xMidYMid">
-    <title>Anthropic</title>
-    <g fill="#181818">
-        <path d="M147.486878,0 C147.486878,0 217.568251,175.780074 217.568251,175.780074 C217.568251,175.780074 256,175.780074 256,175.780074 C256,175.780074 185.918621,0 185.918621,0 C185.918621,0 147.486878,0 147.486878,0 C147.486878,0 147.486878,0 147.486878,0 Z"></path>
-        <path d="M66.1828124,106.221191 C66.1828124,106.221191 90.1624677,44.4471185 90.1624677,44.4471185 C90.1624677,44.4471185 114.142128,106.221191 114.142128,106.221191 C114.142128,106.221191 66.1828124,106.221191 66.1828124,106.221191 C66.1828124,106.221191 66.1828124,106.221191 66.1828124,106.221191 Z M70.0705318,0 C70.0705318,0 0,175.780074 0,175.780074 C0,175.780074 39.179211,175.780074 39.179211,175.780074 C39.179211,175.780074 53.5097704,138.86606 53.5097704,138.86606 C53.5097704,138.86606 126.817544,138.86606 126.817544,138.86606 C126.817544,138.86606 141.145724,175.780074 141.145724,175.780074 C141.145724,175.780074 180.324935,175.780074 180.324935,175.780074 C180.324935,175.780074 110.254409,0 110.254409,0 C110.254409,0 70.0705318,0 70.0705318,0 C70.0705318,0 70.0705318,0 70.0705318,0 Z"></path>
-    </g>
-</svg>
+<svg xmlns="http://www.w3.org/2000/svg" shape-rendering="geometricPrecision" text-rendering="geometricPrecision" image-rendering="optimizeQuality" fill-rule="evenodd" clip-rule="evenodd" viewBox="0 0 512 509.64"><path fill="#D77655" d="M115.612 0h280.775C459.974 0 512 52.026 512 115.612v278.415c0 63.587-52.026 115.612-115.613 115.612H115.612C52.026 509.639 0 457.614 0 394.027V115.612C0 52.026 52.026 0 115.612 0z"/><path fill="#FCF2EE" fill-rule="nonzero" d="M142.27 316.619l73.655-41.326 1.238-3.589-1.238-1.996-3.589-.001-12.31-.759-42.084-1.138-36.498-1.516-35.361-1.896-8.897-1.895-8.34-10.995.859-5.484 7.482-5.03 10.717.935 23.683 1.617 35.537 2.452 25.782 1.517 38.193 3.968h6.064l.86-2.451-2.073-1.517-1.618-1.517-36.776-24.922-39.81-26.338-20.852-15.166-11.273-7.683-5.687-7.204-2.451-15.721 10.237-11.273 13.75.935 3.513.936 13.928 10.716 29.749 23.027 38.848 28.612 5.687 4.727 2.275-1.617.278-1.138-2.553-4.271-21.13-38.193-22.546-38.848-10.035-16.101-2.654-9.655c-.935-3.968-1.617-7.304-1.617-11.374l11.652-15.823 6.445-2.073 15.545 2.073 6.547 5.687 9.655 22.092 15.646 34.78 24.265 47.291 7.103 14.028 3.791 12.992 1.416 3.968 2.449-.001v-2.275l1.997-26.641 3.69-32.707 3.589-42.084 1.239-11.854 5.863-14.206 11.652-7.683 9.099 4.348 7.482 10.716-1.036 6.926-4.449 28.915-8.72 45.294-5.687 30.331h3.313l3.792-3.791 15.342-20.372 25.782-32.227 11.374-12.789 13.27-14.129 8.517-6.724 16.1-.001 11.854 17.617-5.307 18.199-16.581 21.029-13.75 17.819-19.716 26.54-12.309 21.231 1.138 1.694 2.932-.278 44.536-9.479 24.062-4.347 28.714-4.928 12.992 6.066 1.416 6.167-5.106 12.613-30.71 7.583-36.018 7.204-53.636 12.689-.657.48.758.935 24.164 2.275 10.337.556h25.301l47.114 3.514 12.309 8.139 7.381 9.959-1.238 7.583-18.957 9.655-25.579-6.066-59.702-14.205-20.474-5.106-2.83-.001v1.694l17.061 16.682 31.266 28.233 39.152 36.397 1.997 8.999-5.03 7.102-5.307-.758-34.401-25.883-13.27-11.651-30.053-25.302-1.996-.001v2.654l6.926 10.136 36.574 54.975 1.895 16.859-2.653 5.485-9.479 3.311-10.414-1.895-21.408-30.054-22.092-33.844-17.819-30.331-2.173 1.238-10.515 113.261-4.929 5.788-11.374 4.348-9.478-7.204-5.03-11.652 5.03-23.027 6.066-30.052 4.928-23.886 4.449-29.674 2.654-9.858-.177-.657-2.173.278-22.37 30.71-34.021 45.977-26.919 28.815-6.445 2.553-11.173-5.789 1.037-10.337 6.243-9.2 37.257-47.392 22.47-29.371 14.508-16.961-.101-2.451h-.859l-98.954 64.251-17.618 2.275-7.583-7.103.936-11.652 3.589-3.791 29.749-20.474-.101.102.024.101z"/></svg>

web/public/Deepseek.svg

@@ -0,0 +1 @@
+<svg height="1em" style="flex:none;line-height:1" viewBox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><title>DeepSeek</title><path d="M23.748 4.482c-.254-.124-.364.113-.512.234-.051.039-.094.09-.137.136-.372.397-.806.657-1.373.626-.829-.046-1.537.214-2.163.848-.133-.782-.575-1.248-1.247-1.548-.352-.156-.708-.311-.955-.65-.172-.241-.219-.51-.305-.774-.055-.16-.11-.323-.293-.35-.2-.031-.278.136-.356.276-.313.572-.434 1.202-.422 1.84.027 1.436.633 2.58 1.838 3.393.137.093.172.187.129.323-.082.28-.18.552-.266.833-.055.179-.137.217-.329.14a5.526 5.526 0 01-1.736-1.18c-.857-.828-1.631-1.742-2.597-2.458a11.365 11.365 0 00-.689-.471c-.985-.957.13-1.743.388-1.836.27-.098.093-.432-.779-.428-.872.004-1.67.295-2.687.684a3.055 3.055 0 01-.465.137 9.597 9.597 0 00-2.883-.102c-1.885.21-3.39 1.102-4.497 2.623C.082 8.606-.231 10.684.152 12.85c.403 2.284 1.569 4.175 3.36 5.653 1.858 1.533 3.997 2.284 6.438 2.14 1.482-.085 3.133-.284 4.994-1.86.47.234.962.327 1.78.397.63.059 1.236-.03 1.705-.128.735-.156.684-.837.419-.961-2.155-1.004-1.682-.595-2.113-.926 1.096-1.296 2.746-2.642 3.392-7.003.05-.347.007-.565 0-.845-.004-.17.035-.237.23-.256a4.173 4.173 0 001.545-.475c1.396-.763 1.96-2.015 2.093-3.517.02-.23-.004-.467-.247-.588zM11.581 18c-2.089-1.642-3.102-2.183-3.52-2.16-.392.024-.321.471-.235.763.09.288.207.486.371.739.114.167.192.416-.113.603-.673.416-1.842-.14-1.897-.167-1.361-.802-2.5-1.86-3.301-3.307-.774-1.393-1.224-2.887-1.298-4.482-.02-.386.093-.522.477-.592a4.696 4.696 0 011.529-.039c2.132.312 3.946 1.265 5.468 2.774.868.86 1.525 1.887 2.202 2.891.72 1.066 1.494 2.082 2.48 2.914.348.292.625.514.891.677-.802.09-2.14.11-3.054-.614zm1-6.44a.306.306 0 01.415-.287.302.302 0 01.2.288.306.306 0 01-.31.307.303.303 0 01-.304-.308zm3.11 1.596c-.2.081-.399.151-.59.16a1.245 1.245 0 01-.798-.254c-.274-.23-.47-.358-.552-.758a1.73 1.73 0 01.016-.588c.07-.327-.008-.537-.239-.727-.187-.156-.426-.199-.688-.199a.559.559 0 01-.254-.078c-.11-.054-.2-.19-.114-.358.028-.054.16-.186.192-.21.356-.202.767-.136 1.146.016.352.144.618.408 1.001.782.391.451.462.576.685.914.176.265.336.537.445.848.067.195-.019.354-.25.452z" fill="#4D6BFE"></path></svg>

web/src/app/admin/bots/[bot-id]/channels/SlackChannelConfigCreationForm.tsx

@@ -83,6 +83,8 @@ export const SlackChannelConfigCreationForm = ({
           respond_tag_only:
             existingSlackChannelConfig?.channel_config?.respond_tag_only ||
             false,
+          is_ephemeral:
+            existingSlackChannelConfig?.channel_config?.is_ephemeral || false,
           respond_to_bots:
             existingSlackChannelConfig?.channel_config?.respond_to_bots ||
             false,
@@ -135,6 +137,7 @@ export const SlackChannelConfigCreationForm = ({
           questionmark_prefilter_enabled: Yup.boolean().required(),
           respond_tag_only: Yup.boolean().required(),
           respond_to_bots: Yup.boolean().required(),
+          is_ephemeral: Yup.boolean().required(),
           show_continue_in_web_ui: Yup.boolean().required(),
           enable_auto_filters: Yup.boolean().required(),
           respond_member_group_list: Yup.array().of(Yup.string()).required(),

web/src/app/admin/bots/[bot-id]/channels/SlackChannelConfigFormFields.tsx

@@ -597,6 +597,13 @@ export function SlackChannelConfigFormFields({
                 label="Respond to Bot messages"
                 tooltip="If not set, OnyxBot will always ignore messages from Bots"
               />
+              <CheckFormField
+                name="is_ephemeral"
+                label="Respond to user in a private (ephemeral) message"
+                tooltip="If set, OnyxBot will respond only to the user in a private (ephemeral) message. If you also 
+                chose 'Search' Assistant above, selecting this option will make documents that are private to the user 
+                available for their queries."
+              />
 
               <TextArrayField
                 name="respond_member_group_list"
@@ -635,11 +642,14 @@ export function SlackChannelConfigFormFields({
                   Privacy Alert
                 </Label>
                 <p className="text-sm text-text-darker mb-4">
-                  Please note that at least one of the documents accessible by
-                  your OnyxBot is marked as private and may contain sensitive
-                  information. These documents will be accessible to all users
-                  of this OnyxBot. Ensure this aligns with your intended
-                  document sharing policy.
+                  Please note that if the private (ephemeral) response is *not
+                  selected*, only public documents within the selected document
+                  sets will be accessible for user queries. If the private
+                  (ephemeral) response *is selected*, user quries can also
+                  leverage documents that the user has already been granted
+                  access to. Note that users will be able to share the response
+                  with others in the channel, so please ensure that this is
+                  aligned with your company sharing policies.
                 </p>
                 <div className="space-y-2">
                   <h4 className="text-sm text-text font-medium">

web/src/app/admin/bots/[bot-id]/lib.ts

@@ -14,6 +14,7 @@ interface SlackChannelConfigCreationRequest {
   answer_validity_check_enabled: boolean;
   questionmark_prefilter_enabled: boolean;
   respond_tag_only: boolean;
+  is_ephemeral: boolean;
   respond_to_bots: boolean;
   show_continue_in_web_ui: boolean;
   respond_member_group_list: string[];
@@ -45,6 +46,7 @@ const buildRequestBodyFromCreationRequest = (
     channel_name: creationRequest.channel_name,
     respond_tag_only: creationRequest.respond_tag_only,
     respond_to_bots: creationRequest.respond_to_bots,
+    is_ephemeral: creationRequest.is_ephemeral,
     show_continue_in_web_ui: creationRequest.show_continue_in_web_ui,
     enable_auto_filters: creationRequest.enable_auto_filters,
     respond_member_group_list: creationRequest.respond_member_group_list,

web/src/app/admin/configuration/llm/interfaces.ts

@@ -7,14 +7,12 @@ import {
   MicrosoftIconSVG,
   MistralIcon,
   MetaIcon,
-  OpenAIIcon,
   GeminiIcon,
-  OpenSourceIcon,
   AnthropicSVG,
   IconProps,
   OpenAIISVG,
+  DeepseekIcon,
 } from "@/components/icons/icons";
-import { FaRobot } from "react-icons/fa";
 
 export interface CustomConfigKey {
   name: string;
@@ -76,30 +74,31 @@ export interface LLMProviderDescriptor {
 }
 
 export const getProviderIcon = (providerName: string, modelName?: string) => {
+  const modelIconMap: Record<
+    string,
+    ({ size, className }: IconProps) => JSX.Element
+  > = {
+    amazon: AmazonIcon,
+    phi: MicrosoftIconSVG,
+    mistral: MistralIcon,
+    ministral: MistralIcon,
+    llama: MetaIcon,
+    gemini: GeminiIcon,
+    deepseek: DeepseekIcon,
+    claude: AnthropicIcon,
+  };
+
   const modelNameToIcon = (
     modelName: string,
     fallbackIcon: ({ size, className }: IconProps) => JSX.Element
   ): (({ size, className }: IconProps) => JSX.Element) => {
-    if (modelName?.toLowerCase().includes("amazon")) {
-      return AmazonIcon;
-    }
-    if (modelName?.toLowerCase().includes("phi")) {
-      return MicrosoftIconSVG;
-    }
-    if (modelName?.toLowerCase().includes("mistral")) {
-      return MistralIcon;
-    }
-    if (modelName?.toLowerCase().includes("llama")) {
-      return MetaIcon;
-    }
-    if (modelName?.toLowerCase().includes("gemini")) {
-      return GeminiIcon;
-    }
-    if (modelName?.toLowerCase().includes("claude")) {
-      return AnthropicIcon;
-    } else {
-      return fallbackIcon;
+    const lowerModelName = modelName?.toLowerCase();
+    for (const [key, icon] of Object.entries(modelIconMap)) {
+      if (lowerModelName?.includes(key)) {
+        return icon;
+      }
     }
+    return fallbackIcon;
   };
 
   switch (providerName) {