add shadow, all background-code, TODO rm Spacer, TODO fix shadow w/ codeVariant

.github/workflows/storybook-deploy.yml

@@ -48,7 +48,7 @@ jobs:
 
       - name: Deploy to Vercel (Production)
         working-directory: web
-        run: npx --yes "$VERCEL_CLI" deploy storybook-static/ --prod --yes
+        run: npx --yes "$VERCEL_CLI" deploy storybook-static/ --prod --yes --token="$VERCEL_TOKEN"
 
   notify-slack-on-failure:
     needs: Deploy-Storybook

backend/onyx/connectors/sharepoint/connector.py

@@ -258,6 +258,10 @@ class SharepointConnectorCheckpoint(ConnectorCheckpoint):
     # Track yielded hierarchy nodes by their raw_node_id (URLs) to avoid duplicates
     seen_hierarchy_node_raw_ids: set[str] = Field(default_factory=set)
 
+    # Track yielded document IDs to avoid processing the same document twice.
+    # The Microsoft Graph delta API can return the same item on multiple pages.
+    seen_document_ids: set[str] = Field(default_factory=set)
+
 
 class SharepointAuthMethod(Enum):
     CLIENT_SECRET = "client_secret"
@@ -1557,6 +1561,7 @@ class SharepointConnector(
         checkpoint.current_drive_id = None
         checkpoint.current_drive_web_url = None
         checkpoint.current_drive_delta_next_link = None
+        checkpoint.seen_document_ids.clear()
 
     def _fetch_slim_documents_from_sharepoint(self) -> GenerateSlimDocumentOutput:
         site_descriptors = self.site_descriptors or self.fetch_sites()
@@ -2137,6 +2142,14 @@ class SharepointConnector(
             item_count = 0
             for driveitem in driveitems:
                 item_count += 1
+
+                if driveitem.id and driveitem.id in checkpoint.seen_document_ids:
+                    logger.debug(
+                        f"Skipping duplicate document {driveitem.id} "
+                        f"({driveitem.name})"
+                    )
+                    continue
+
                 driveitem_extension = get_file_ext(driveitem.name)
                 if driveitem_extension not in OnyxFileExtensions.ALL_ALLOWED_EXTENSIONS:
                     logger.warning(
@@ -2189,11 +2202,13 @@ class SharepointConnector(
 
                     if isinstance(doc_or_failure, Document):
                         if doc_or_failure.sections:
+                            checkpoint.seen_document_ids.add(doc_or_failure.id)
                             yield doc_or_failure
                         elif should_yield_if_empty:
                             doc_or_failure.sections = [
                                 TextSection(link=driveitem.web_url, text="")
                             ]
+                            checkpoint.seen_document_ids.add(doc_or_failure.id)
                             yield doc_or_failure
                         else:
                             logger.warning(

backend/onyx/db/llm.py

@@ -25,6 +25,7 @@ from onyx.server.manage.embedding.models import CloudEmbeddingProvider
 from onyx.server.manage.embedding.models import CloudEmbeddingProviderCreationRequest
 from onyx.server.manage.llm.models import LLMProviderUpsertRequest
 from onyx.server.manage.llm.models import LLMProviderView
+from onyx.server.manage.llm.models import SyncModelEntry
 from onyx.utils.logger import setup_logger
 from shared_configs.enums import EmbeddingProvider
 
@@ -369,9 +370,9 @@ def upsert_llm_provider(
 def sync_model_configurations(
     db_session: Session,
     provider_name: str,
-    models: list[dict],
+    models: list[SyncModelEntry],
 ) -> int:
-    """Sync model configurations for a dynamic provider (OpenRouter, Bedrock, Ollama).
+    """Sync model configurations for a dynamic provider (OpenRouter, Bedrock, Ollama, etc.).
 
     This inserts NEW models from the source API without overwriting existing ones.
     User preferences (is_visible, max_input_tokens) are preserved for existing models.
@@ -379,7 +380,7 @@ def sync_model_configurations(
     Args:
         db_session: Database session
         provider_name: Name of the LLM provider
-        models: List of model dicts with keys: name, display_name, max_input_tokens, supports_image_input
+        models: List of SyncModelEntry objects describing the fetched models
 
     Returns:
         Number of new models added
@@ -393,21 +394,20 @@ def sync_model_configurations(
 
     new_count = 0
     for model in models:
-        model_name = model["name"]
-        if model_name not in existing_names:
+        if model.name not in existing_names:
             # Insert new model with is_visible=False (user must explicitly enable)
             supported_flows = [LLMModelFlowType.CHAT]
-            if model.get("supports_image_input", False):
+            if model.supports_image_input:
                 supported_flows.append(LLMModelFlowType.VISION)
 
             insert_new_model_configuration__no_commit(
                 db_session=db_session,
                 llm_provider_id=provider.id,
-                model_name=model_name,
+                model_name=model.name,
                 supported_flows=supported_flows,
                 is_visible=False,
-                max_input_tokens=model.get("max_input_tokens"),
-                display_name=model.get("display_name"),
+                max_input_tokens=model.max_input_tokens,
+                display_name=model.display_name,
             )
             new_count += 1
 

backend/onyx/db/models.py

@@ -163,6 +163,8 @@ class _EncryptedBase(TypeDecorator):
 
 
 class EncryptedString(_EncryptedBase):
+    # Must redeclare cache_ok in this child class since we explicitly redeclare _is_json
+    cache_ok = True
     _is_json: bool = False
 
     def process_bind_param(
@@ -189,6 +191,7 @@ class EncryptedString(_EncryptedBase):
 
 
 class EncryptedJson(_EncryptedBase):
+    cache_ok = True
     _is_json: bool = True
 
     def process_bind_param(

backend/onyx/document_index/opensearch/opensearch_document_index.py

@@ -433,12 +433,16 @@ class OpenSearchOldDocumentIndex(OldDocumentIndex):
             hidden=fields.hidden if fields else None,
             project_ids=(
                 set(user_fields.user_projects)
-                if user_fields and user_fields.user_projects
+                # NOTE: Empty user_projects is semantically different from None
+                # user_projects.
+                if user_fields and user_fields.user_projects is not None
                 else None
             ),
             persona_ids=(
                 set(user_fields.personas)
-                if user_fields and user_fields.personas
+                # NOTE: Empty personas is semantically different from None
+                # personas.
+                if user_fields and user_fields.personas is not None
                 else None
             ),
         )

backend/onyx/document_index/opensearch/search.py

@@ -255,8 +255,12 @@ class DocumentQuery:
                 f"result window ({DEFAULT_OPENSEARCH_MAX_RESULT_WINDOW})."
             )
 
+        # TODO(andrei, yuhong): We can tune this more dynamically based on
+        # num_hits.
+        max_results_per_subquery = DEFAULT_NUM_HYBRID_SEARCH_CANDIDATES
+
         hybrid_search_subqueries = DocumentQuery._get_hybrid_search_subqueries(
-            query_text, query_vector
+            query_text, query_vector, vector_candidates=max_results_per_subquery
         )
         hybrid_search_filters = DocumentQuery._get_search_filters(
             tenant_state=tenant_state,
@@ -291,7 +295,7 @@ class DocumentQuery:
                 # Sources:
                 # https://docs.opensearch.org/latest/vector-search/ai-search/hybrid-search/pagination/
                 # https://opensearch.org/blog/navigating-pagination-in-hybrid-queries-with-the-pagination_depth-parameter/
-                "pagination_depth": DEFAULT_NUM_HYBRID_SEARCH_CANDIDATES,
+                "pagination_depth": max_results_per_subquery,
                 # Applied to all the sub-queries independently (this avoids
                 # subqueries having a lot of results thrown out during
                 # aggregation).
@@ -734,14 +738,13 @@ class DocumentQuery:
             # document's metadata list.
             filter_clauses.append(_get_tag_filter(tags))
 
-        # Knowledge scope: explicit knowledge attachments restrict what
-        # an assistant can see.  When none are set the assistant
-        # searches everything.
+        # Knowledge scope: explicit knowledge attachments restrict what an
+        # assistant can see. When none are set the assistant searches
+        # everything.
         #
-        # project_id / persona_id are additive: they make overflowing
-        # user files findable but must NOT trigger the restriction on
-        # their own (an agent with no explicit knowledge should search
-        # everything).
+        # project_id / persona_id are additive: they make overflowing user files
+        # findable but must NOT trigger the restriction on their own (an agent
+        # with no explicit knowledge should search everything).
         has_knowledge_scope = (
             attached_document_ids
             or hierarchy_node_ids
@@ -769,9 +772,8 @@ class DocumentQuery:
                 knowledge_filter["bool"]["should"].append(
                     _get_document_set_filter(document_sets)
                 )
-            # Additive: widen scope to also cover overflowing user
-            # files, but only when an explicit restriction is already
-            # in effect.
+            # Additive: widen scope to also cover overflowing user files, but
+            # only when an explicit restriction is already in effect.
             if project_id is not None:
                 knowledge_filter["bool"]["should"].append(
                     _get_user_project_filter(project_id)

backend/onyx/document_index/vespa/index.py

@@ -690,9 +690,12 @@ class VespaIndex(DocumentIndex):
             )
 
         project_ids: set[int] | None = None
+        # NOTE: Empty user_projects is semantically different from None
+        # user_projects.
         if user_fields is not None and user_fields.user_projects is not None:
             project_ids = set(user_fields.user_projects)
         persona_ids: set[int] | None = None
+        # NOTE: Empty personas is semantically different from None personas.
         if user_fields is not None and user_fields.personas is not None:
             persona_ids = set(user_fields.personas)
         update_request = MetadataUpdateRequest(

backend/onyx/server/manage/llm/api.py

@@ -58,6 +58,9 @@ from onyx.llm.well_known_providers.llm_provider_options import (
 from onyx.server.manage.llm.models import BedrockFinalModelResponse
 from onyx.server.manage.llm.models import BedrockModelsRequest
 from onyx.server.manage.llm.models import DefaultModel
+from onyx.server.manage.llm.models import LitellmFinalModelResponse
+from onyx.server.manage.llm.models import LitellmModelDetails
+from onyx.server.manage.llm.models import LitellmModelsRequest
 from onyx.server.manage.llm.models import LLMCost
 from onyx.server.manage.llm.models import LLMProviderDescriptor
 from onyx.server.manage.llm.models import LLMProviderResponse
@@ -72,6 +75,7 @@ from onyx.server.manage.llm.models import OllamaModelsRequest
 from onyx.server.manage.llm.models import OpenRouterFinalModelResponse
 from onyx.server.manage.llm.models import OpenRouterModelDetails
 from onyx.server.manage.llm.models import OpenRouterModelsRequest
+from onyx.server.manage.llm.models import SyncModelEntry
 from onyx.server.manage.llm.models import TestLLMRequest
 from onyx.server.manage.llm.models import VisionProviderResponse
 from onyx.server.manage.llm.utils import generate_bedrock_display_name
@@ -98,6 +102,34 @@ def _mask_string(value: str) -> str:
     return value[:4] + "****" + value[-4:]
 
 
+def _sync_fetched_models(
+    db_session: Session,
+    provider_name: str,
+    models: list[SyncModelEntry],
+    source_label: str,
+) -> None:
+    """Sync fetched models to DB for the given provider.
+
+    Args:
+        db_session: Database session
+        provider_name: Name of the LLM provider
+        models: List of SyncModelEntry objects describing the fetched models
+        source_label: Human-readable label for log messages (e.g. "Bedrock", "LiteLLM")
+    """
+    try:
+        new_count = sync_model_configurations(
+            db_session=db_session,
+            provider_name=provider_name,
+            models=models,
+        )
+        if new_count > 0:
+            logger.info(
+                f"Added {new_count} new {source_label} models to provider '{provider_name}'"
+            )
+    except ValueError as e:
+        logger.warning(f"Failed to sync {source_label} models to DB: {e}")
+
+
 # Keys in custom_config that contain sensitive credentials
 _SENSITIVE_CONFIG_KEYS = {
     "vertex_credentials",
@@ -963,27 +995,20 @@ def get_bedrock_available_models(
 
         # Sync new models to DB if provider_name is specified
         if request.provider_name:
-            try:
-                models_to_sync = [
-                    {
-                        "name": r.name,
-                        "display_name": r.display_name,
-                        "max_input_tokens": r.max_input_tokens,
-                        "supports_image_input": r.supports_image_input,
-                    }
-                    for r in results
-                ]
-                new_count = sync_model_configurations(
-                    db_session=db_session,
-                    provider_name=request.provider_name,
-                    models=models_to_sync,
-                )
-                if new_count > 0:
-                    logger.info(
-                        f"Added {new_count} new Bedrock models to provider '{request.provider_name}'"
+            _sync_fetched_models(
+                db_session=db_session,
+                provider_name=request.provider_name,
+                models=[
+                    SyncModelEntry(
+                        name=r.name,
+                        display_name=r.display_name,
+                        max_input_tokens=r.max_input_tokens,
+                        supports_image_input=r.supports_image_input,
                     )
-            except ValueError as e:
-                logger.warning(f"Failed to sync Bedrock models to DB: {e}")
+                    for r in results
+                ],
+                source_label="Bedrock",
+            )
 
         return results
 
@@ -1101,27 +1126,20 @@ def get_ollama_available_models(
 
     # Sync new models to DB if provider_name is specified
     if request.provider_name:
-        try:
-            models_to_sync = [
-                {
-                    "name": r.name,
-                    "display_name": r.display_name,
-                    "max_input_tokens": r.max_input_tokens,
-                    "supports_image_input": r.supports_image_input,
-                }
-                for r in sorted_results
-            ]
-            new_count = sync_model_configurations(
-                db_session=db_session,
-                provider_name=request.provider_name,
-                models=models_to_sync,
-            )
-            if new_count > 0:
-                logger.info(
-                    f"Added {new_count} new Ollama models to provider '{request.provider_name}'"
+        _sync_fetched_models(
+            db_session=db_session,
+            provider_name=request.provider_name,
+            models=[
+                SyncModelEntry(
+                    name=r.name,
+                    display_name=r.display_name,
+                    max_input_tokens=r.max_input_tokens,
+                    supports_image_input=r.supports_image_input,
                 )
-        except ValueError as e:
-            logger.warning(f"Failed to sync Ollama models to DB: {e}")
+                for r in sorted_results
+            ],
+            source_label="Ollama",
+        )
 
     return sorted_results
 
@@ -1210,27 +1228,20 @@ def get_openrouter_available_models(
 
     # Sync new models to DB if provider_name is specified
     if request.provider_name:
-        try:
-            models_to_sync = [
-                {
-                    "name": r.name,
-                    "display_name": r.display_name,
-                    "max_input_tokens": r.max_input_tokens,
-                    "supports_image_input": r.supports_image_input,
-                }
-                for r in sorted_results
-            ]
-            new_count = sync_model_configurations(
-                db_session=db_session,
-                provider_name=request.provider_name,
-                models=models_to_sync,
-            )
-            if new_count > 0:
-                logger.info(
-                    f"Added {new_count} new OpenRouter models to provider '{request.provider_name}'"
+        _sync_fetched_models(
+            db_session=db_session,
+            provider_name=request.provider_name,
+            models=[
+                SyncModelEntry(
+                    name=r.name,
+                    display_name=r.display_name,
+                    max_input_tokens=r.max_input_tokens,
+                    supports_image_input=r.supports_image_input,
                 )
-        except ValueError as e:
-            logger.warning(f"Failed to sync OpenRouter models to DB: {e}")
+                for r in sorted_results
+            ],
+            source_label="OpenRouter",
+        )
 
     return sorted_results
 
@@ -1324,26 +1335,119 @@ def get_lm_studio_available_models(
 
     # Sync new models to DB if provider_name is specified
     if request.provider_name:
-        try:
-            models_to_sync = [
-                {
-                    "name": r.name,
-                    "display_name": r.display_name,
-                    "max_input_tokens": r.max_input_tokens,
-                    "supports_image_input": r.supports_image_input,
-                }
-                for r in sorted_results
-            ]
-            new_count = sync_model_configurations(
-                db_session=db_session,
-                provider_name=request.provider_name,
-                models=models_to_sync,
-            )
-            if new_count > 0:
-                logger.info(
-                    f"Added {new_count} new LM Studio models to provider '{request.provider_name}'"
+        _sync_fetched_models(
+            db_session=db_session,
+            provider_name=request.provider_name,
+            models=[
+                SyncModelEntry(
+                    name=r.name,
+                    display_name=r.display_name,
+                    max_input_tokens=r.max_input_tokens,
+                    supports_image_input=r.supports_image_input,
                 )
-        except ValueError as e:
-            logger.warning(f"Failed to sync LM Studio models to DB: {e}")
+                for r in sorted_results
+            ],
+            source_label="LM Studio",
+        )
 
     return sorted_results
+
+
+@admin_router.post("/litellm/available-models")
+def get_litellm_available_models(
+    request: LitellmModelsRequest,
+    _: User = Depends(current_admin_user),
+    db_session: Session = Depends(get_session),
+) -> list[LitellmFinalModelResponse]:
+    """Fetch available models from Litellm proxy /v1/models endpoint."""
+    response_json = _get_litellm_models_response(
+        api_key=request.api_key, api_base=request.api_base
+    )
+
+    models = response_json.get("data", [])
+    if not isinstance(models, list) or len(models) == 0:
+        raise OnyxError(
+            OnyxErrorCode.VALIDATION_ERROR,
+            "No models found from your Litellm endpoint",
+        )
+
+    results: list[LitellmFinalModelResponse] = []
+    for model in models:
+        try:
+            model_details = LitellmModelDetails.model_validate(model)
+
+            results.append(
+                LitellmFinalModelResponse(
+                    provider_name=model_details.owned_by,
+                    model_name=model_details.id,
+                )
+            )
+        except Exception as e:
+            logger.warning(
+                "Failed to parse Litellm model entry",
+                extra={"error": str(e), "item": str(model)[:1000]},
+            )
+
+    if not results:
+        raise OnyxError(
+            OnyxErrorCode.VALIDATION_ERROR,
+            "No compatible models found from Litellm",
+        )
+
+    sorted_results = sorted(results, key=lambda m: m.model_name.lower())
+
+    # Sync new models to DB if provider_name is specified
+    if request.provider_name:
+        _sync_fetched_models(
+            db_session=db_session,
+            provider_name=request.provider_name,
+            models=[
+                SyncModelEntry(
+                    name=r.model_name,
+                    display_name=r.model_name,
+                )
+                for r in sorted_results
+            ],
+            source_label="LiteLLM",
+        )
+
+    return sorted_results
+
+
+def _get_litellm_models_response(api_key: str, api_base: str) -> dict:
+    """Perform GET to Litellm proxy /api/v1/models and return parsed JSON."""
+    cleaned_api_base = api_base.strip().rstrip("/")
+    url = f"{cleaned_api_base}/v1/models"
+
+    headers = {
+        "Authorization": f"Bearer {api_key}",
+        "HTTP-Referer": "https://onyx.app",
+        "X-Title": "Onyx",
+    }
+
+    try:
+        response = httpx.get(url, headers=headers, timeout=10.0)
+        response.raise_for_status()
+        return response.json()
+    except httpx.HTTPStatusError as e:
+        if e.response.status_code == 401:
+            raise OnyxError(
+                OnyxErrorCode.VALIDATION_ERROR,
+                "Authentication failed: invalid or missing API key for LiteLLM proxy.",
+            )
+        elif e.response.status_code == 404:
+            raise OnyxError(
+                OnyxErrorCode.VALIDATION_ERROR,
+                f"LiteLLM models endpoint not found at {url}. "
+                "Please verify the API base URL.",
+            )
+        else:
+            raise OnyxError(
+                OnyxErrorCode.BAD_GATEWAY,
+                f"Failed to fetch LiteLLM models: {e}",
+            )
+    except Exception as e:
+        raise OnyxError(
+            OnyxErrorCode.BAD_GATEWAY,
+            f"Failed to fetch LiteLLM models: {e}",
+        )

backend/onyx/server/manage/llm/models.py

@@ -420,3 +420,32 @@ class LLMProviderResponse(BaseModel, Generic[T]):
             default_text=default_text,
             default_vision=default_vision,
         )
+
+
+class SyncModelEntry(BaseModel):
+    """Typed model for syncing fetched models to the DB."""
+
+    name: str
+    display_name: str
+    max_input_tokens: int | None = None
+    supports_image_input: bool = False
+
+
+class LitellmModelsRequest(BaseModel):
+    api_key: str
+    api_base: str
+    provider_name: str | None = None  # Optional: to save models to existing provider
+
+
+class LitellmModelDetails(BaseModel):
+    """Response model for Litellm proxy /api/v1/models endpoint"""
+
+    id: str  # Model ID (e.g. "gpt-4o")
+    object: str  # "model"
+    created: int  # Unix timestamp in seconds
+    owned_by: str  # Provider name (e.g. "openai")
+
+
+class LitellmFinalModelResponse(BaseModel):
+    provider_name: str  # Provider name (e.g. "openai")
+    model_name: str  # Model ID (e.g. "gpt-4o")

backend/tests/external_dependency_unit/document_index/test_document_index_old.py

@@ -0,0 +1,398 @@
+"""External dependency tests for the old DocumentIndex interface.
+
+These tests assume Vespa and OpenSearch are running.
+
+TODO(ENG-3764)(andrei): Consolidate some of these test fixtures.
+"""
+
+import os
+import time
+import uuid
+from collections.abc import Generator
+from unittest.mock import patch
+
+import httpx
+import pytest
+
+from onyx.access.models import DocumentAccess
+from onyx.configs.constants import DocumentSource
+from onyx.connectors.models import Document
+from onyx.context.search.models import IndexFilters
+from onyx.db.enums import EmbeddingPrecision
+from onyx.document_index.interfaces import DocumentIndex
+from onyx.document_index.interfaces import IndexBatchParams
+from onyx.document_index.interfaces import VespaChunkRequest
+from onyx.document_index.interfaces import VespaDocumentUserFields
+from onyx.document_index.opensearch.client import wait_for_opensearch_with_timeout
+from onyx.document_index.opensearch.opensearch_document_index import (
+    OpenSearchOldDocumentIndex,
+)
+from onyx.document_index.vespa.index import VespaIndex
+from onyx.document_index.vespa.shared_utils.utils import get_vespa_http_client
+from onyx.document_index.vespa.shared_utils.utils import wait_for_vespa_with_timeout
+from onyx.indexing.models import ChunkEmbedding
+from onyx.indexing.models import DocMetadataAwareIndexChunk
+from shared_configs.configs import MULTI_TENANT
+from shared_configs.contextvars import CURRENT_TENANT_ID_CONTEXTVAR
+from shared_configs.contextvars import get_current_tenant_id
+from tests.external_dependency_unit.constants import TEST_TENANT_ID
+
+
+@pytest.fixture(scope="module")
+def opensearch_available() -> Generator[None, None, None]:
+    """Verifies OpenSearch is running, fails the test if not."""
+    if not wait_for_opensearch_with_timeout():
+        pytest.fail("OpenSearch is not available.")
+    yield  # Test runs here.
+
+
+@pytest.fixture(scope="module")
+def test_index_name() -> Generator[str, None, None]:
+    yield f"test_index_{uuid.uuid4().hex[:8]}"  # Test runs here.
+
+
+@pytest.fixture(scope="module")
+def tenant_context() -> Generator[None, None, None]:
+    """Sets up tenant context for testing."""
+    token = CURRENT_TENANT_ID_CONTEXTVAR.set(TEST_TENANT_ID)
+    try:
+        yield  # Test runs here.
+    finally:
+        # Reset the tenant context after the test
+        CURRENT_TENANT_ID_CONTEXTVAR.reset(token)
+
+
+@pytest.fixture(scope="module")
+def httpx_client() -> Generator[httpx.Client, None, None]:
+    client = get_vespa_http_client()
+    try:
+        yield client
+    finally:
+        client.close()
+
+
+@pytest.fixture(scope="module")
+def vespa_document_index(
+    httpx_client: httpx.Client,
+    tenant_context: None,  # noqa: ARG001
+    test_index_name: str,
+) -> Generator[VespaIndex, None, None]:
+    vespa_index = VespaIndex(
+        index_name=test_index_name,
+        secondary_index_name=None,
+        large_chunks_enabled=False,
+        secondary_large_chunks_enabled=None,
+        multitenant=MULTI_TENANT,
+        httpx_client=httpx_client,
+    )
+    backend_dir = os.path.abspath(
+        os.path.join(os.path.dirname(__file__), "..", "..", "..")
+    )
+    with patch("os.getcwd", return_value=backend_dir):
+        vespa_index.ensure_indices_exist(
+            primary_embedding_dim=128,
+            primary_embedding_precision=EmbeddingPrecision.FLOAT,
+            secondary_index_embedding_dim=None,
+            secondary_index_embedding_precision=None,
+        )
+    # Verify Vespa is running, fails the test if not. Try 90 seconds for testing
+    # in CI. We have to do this here because this endpoint only becomes live
+    # once we create an index.
+    if not wait_for_vespa_with_timeout(wait_limit=90):
+        pytest.fail("Vespa is not available.")
+
+    # Wait until the schema is actually ready for writes on content nodes. We
+    # probe by attempting a PUT; 200 means the schema is live, 400 means not
+    # yet. This is so scuffed but running the test is really flakey otherwise;
+    # this is only temporary until we entirely move off of Vespa.
+    probe_doc = {
+        "fields": {
+            "document_id": "__probe__",
+            "chunk_id": 0,
+            "blurb": "",
+            "title": "",
+            "skip_title": True,
+            "content": "",
+            "content_summary": "",
+            "source_type": "file",
+            "source_links": "null",
+            "semantic_identifier": "",
+            "section_continuation": False,
+            "large_chunk_reference_ids": [],
+            "metadata": "{}",
+            "metadata_list": [],
+            "metadata_suffix": "",
+            "chunk_context": "",
+            "doc_summary": "",
+            "embeddings": {"full_chunk": [1.0] + [0.0] * 127},
+            "access_control_list": {},
+            "document_sets": {},
+            "image_file_name": None,
+            "user_project": [],
+            "personas": [],
+            "boost": 0.0,
+            "aggregated_chunk_boost_factor": 0.0,
+            "primary_owners": [],
+            "secondary_owners": [],
+        }
+    }
+    schema_ready = False
+    probe_url = (
+        f"http://localhost:8081/document/v1/default/{test_index_name}/docid/__probe__"
+    )
+    for _ in range(60):
+        resp = httpx_client.post(probe_url, json=probe_doc)
+        if resp.status_code == 200:
+            schema_ready = True
+            # Clean up the probe document.
+            httpx_client.delete(probe_url)
+            break
+        time.sleep(1)
+    if not schema_ready:
+        pytest.fail(f"Vespa schema '{test_index_name}' did not become ready in time.")
+
+    yield vespa_index  # Test runs here.
+
+    # TODO(ENG-3765)(andrei): Explicitly cleanup index. Not immediately
+    # pressing; in CI we should be using fresh instances of dependencies each
+    # time anyway.
+
+
+@pytest.fixture(scope="module")
+def opensearch_document_index(
+    opensearch_available: None,  # noqa: ARG001
+    tenant_context: None,  # noqa: ARG001
+    test_index_name: str,
+) -> Generator[OpenSearchOldDocumentIndex, None, None]:
+    opensearch_index = OpenSearchOldDocumentIndex(
+        index_name=test_index_name,
+        embedding_dim=128,
+        embedding_precision=EmbeddingPrecision.FLOAT,
+        secondary_index_name=None,
+        secondary_embedding_dim=None,
+        secondary_embedding_precision=None,
+        large_chunks_enabled=False,
+        secondary_large_chunks_enabled=None,
+        multitenant=MULTI_TENANT,
+    )
+    opensearch_index.ensure_indices_exist(
+        primary_embedding_dim=128,
+        primary_embedding_precision=EmbeddingPrecision.FLOAT,
+        secondary_index_embedding_dim=None,
+        secondary_index_embedding_precision=None,
+    )
+
+    yield opensearch_index  # Test runs here.
+
+    # TODO(ENG-3765)(andrei): Explicitly cleanup index. Not immediately
+    # pressing; in CI we should be using fresh instances of dependencies each
+    # time anyway.
+
+
+@pytest.fixture(scope="module")
+def document_indices(
+    vespa_document_index: VespaIndex,
+    opensearch_document_index: OpenSearchOldDocumentIndex,
+) -> Generator[list[DocumentIndex], None, None]:
+    # Ideally these are parametrized; doing so with pytest fixtures is tricky.
+    yield [opensearch_document_index, vespa_document_index]  # Test runs here.
+
+
+@pytest.fixture(scope="function")
+def chunks(
+    tenant_context: None,  # noqa: ARG001
+) -> Generator[list[DocMetadataAwareIndexChunk], None, None]:
+    result = []
+    chunk_count = 5
+    doc_id = "test_doc"
+    tenant_id = get_current_tenant_id()
+    access = DocumentAccess.build(
+        user_emails=[],
+        user_groups=[],
+        external_user_emails=[],
+        external_user_group_ids=[],
+        is_public=True,
+    )
+    document_sets: set[str] = set()
+    user_project: list[int] = list()
+    personas: list[int] = list()
+    boost = 0
+    blurb = "blurb"
+    content = "content"
+    title_prefix = ""
+    doc_summary = ""
+    chunk_context = ""
+    title_embedding = [1.0] + [0] * 127
+    # Full 0 vectors are not supported for cos similarity.
+    embeddings = ChunkEmbedding(
+        full_embedding=[1.0] + [0] * 127, mini_chunk_embeddings=[]
+    )
+    source_document = Document(
+        id=doc_id,
+        semantic_identifier="semantic identifier",
+        source=DocumentSource.FILE,
+        sections=[],
+        metadata={},
+        title="title",
+    )
+    metadata_suffix_keyword = ""
+    image_file_id = None
+    source_links: dict[int, str] = {0: ""}
+    ancestor_hierarchy_node_ids: list[int] = []
+    for i in range(chunk_count):
+        result.append(
+            DocMetadataAwareIndexChunk(
+                tenant_id=tenant_id,
+                access=access,
+                document_sets=document_sets,
+                user_project=user_project,
+                personas=personas,
+                boost=boost,
+                aggregated_chunk_boost_factor=0,
+                ancestor_hierarchy_node_ids=ancestor_hierarchy_node_ids,
+                embeddings=embeddings,
+                title_embedding=title_embedding,
+                source_document=source_document,
+                title_prefix=title_prefix,
+                metadata_suffix_keyword=metadata_suffix_keyword,
+                metadata_suffix_semantic="",
+                contextual_rag_reserved_tokens=0,
+                doc_summary=doc_summary,
+                chunk_context=chunk_context,
+                mini_chunk_texts=None,
+                large_chunk_id=None,
+                chunk_id=i,
+                blurb=blurb,
+                content=content,
+                source_links=source_links,
+                image_file_id=image_file_id,
+                section_continuation=False,
+            )
+        )
+    yield result  # Test runs here.
+
+
+@pytest.fixture(scope="function")
+def index_batch_params(
+    tenant_context: None,  # noqa: ARG001
+) -> Generator[IndexBatchParams, None, None]:
+    # WARNING: doc_id_to_previous_chunk_cnt={"test_doc": 0} is hardcoded to 0,
+    # which is only correct on the very first index call. The document_indices
+    # fixture is scope="module", meaning the same OpenSearch and Vespa backends
+    # persist across all test functions in this module. When a second test
+    # function uses this fixture and calls document_index.index(...), the
+    # backend already has 5 chunks for "test_doc" from the previous test run,
+    # but the batch params still claim 0 prior chunks exist. This can lead to
+    # orphaned/duplicate chunks that make subsequent assertions incorrect.
+    # TODO: Whenever adding a second test, either change this or cleanup the
+    # index between test cases.
+    yield IndexBatchParams(
+        doc_id_to_previous_chunk_cnt={"test_doc": 0},
+        doc_id_to_new_chunk_cnt={"test_doc": 5},
+        tenant_id=get_current_tenant_id(),
+        large_chunks_enabled=False,
+    )
+
+
+class TestDocumentIndexOld:
+    """Tests the old DocumentIndex interface."""
+
+    def test_update_single_can_clear_user_projects_and_personas(
+        self,
+        document_indices: list[DocumentIndex],
+        # This test case assumes all these chunks correspond to one document.
+        chunks: list[DocMetadataAwareIndexChunk],
+        index_batch_params: IndexBatchParams,
+    ) -> None:
+        """
+        Tests that update_single can clear user_projects and personas.
+        """
+        for document_index in document_indices:
+            # Precondition.
+            # Ensure there is some non-empty value for user project and
+            # personas.
+            for chunk in chunks:
+                chunk.user_project = [1]
+                chunk.personas = [2]
+            document_index.index(chunks, index_batch_params)
+
+            # Ensure that we can get chunks as expected with filters.
+            doc_id = chunks[0].source_document.id
+            chunk_count = len(chunks)
+            tenant_id = get_current_tenant_id()
+            # We need to specify the chunk index range and specify
+            # batch_retrieval=True below to trigger the codepath for Vespa's
+            # search API, which uses the expected additive filtering for
+            # project_id and persona_id. Otherwise we would use the codepath for
+            # the visit API, which does not have this kind of filtering
+            # implemented.
+            chunk_request = VespaChunkRequest(
+                document_id=doc_id, min_chunk_ind=0, max_chunk_ind=chunk_count - 1
+            )
+            project_persona_filters = IndexFilters(
+                access_control_list=None,
+                tenant_id=tenant_id,
+                project_id=1,
+                persona_id=2,
+                # We need this even though none of the chunks belong to a
+                # document set because project_id and persona_id are only
+                # additive filters in the event the agent has knowledge scope;
+                # if the agent does not, it is implied that it can see
+                # everything it is allowed to.
+                document_set=["1"],
+            )
+            # Not best practice here but the API for refreshing the index to
+            # ensure that the latest data is present is not exposed in this
+            # class and is not the same for Vespa and OpenSearch, so we just
+            # tolerate a sleep for now. As a consequence the number of tests in
+            # this suite should be small. We only need to tolerate this for as
+            # long as we continue to use Vespa, we can consider exposing
+            # something for OpenSearch later.
+            time.sleep(1)
+            inference_chunks = document_index.id_based_retrieval(
+                chunk_requests=[chunk_request],
+                filters=project_persona_filters,
+                batch_retrieval=True,
+            )
+            assert len(inference_chunks) == chunk_count
+            # Sort by chunk id to easily test if we have all chunks.
+            for i, inference_chunk in enumerate(
+                sorted(inference_chunks, key=lambda x: x.chunk_id)
+            ):
+                assert inference_chunk.chunk_id == i
+                assert inference_chunk.document_id == doc_id
+
+            # Under test.
+            # Explicitly set empty fields here.
+            user_fields = VespaDocumentUserFields(user_projects=[], personas=[])
+            document_index.update_single(
+                doc_id=doc_id,
+                chunk_count=chunk_count,
+                tenant_id=tenant_id,
+                fields=None,
+                user_fields=user_fields,
+            )
+
+            # Postcondition.
+            filters = IndexFilters(access_control_list=None, tenant_id=tenant_id)
+            # We should expect to get back all expected chunks with no filters.
+            # Again, not best practice here.
+            time.sleep(1)
+            inference_chunks = document_index.id_based_retrieval(
+                chunk_requests=[chunk_request], filters=filters, batch_retrieval=True
+            )
+            assert len(inference_chunks) == chunk_count
+            # Sort by chunk id to easily test if we have all chunks.
+            for i, inference_chunk in enumerate(
+                sorted(inference_chunks, key=lambda x: x.chunk_id)
+            ):
+                assert inference_chunk.chunk_id == i
+                assert inference_chunk.document_id == doc_id
+            # Now, we should expect to not get any chunks if we specify the user
+            # project and personas filters.
+            inference_chunks = document_index.id_based_retrieval(
+                chunk_requests=[chunk_request],
+                filters=project_persona_filters,
+                batch_retrieval=True,
+            )
+            assert len(inference_chunks) == 0

backend/tests/external_dependency_unit/opensearch_migration/test_opensearch_migration_tasks.py

@@ -239,6 +239,8 @@ def full_deployment_setup() -> Generator[None, None, None]:
     NOTE: We deliberately duplicate this logic from
     backend/tests/external_dependency_unit/conftest.py because we need to set
     opensearch_available just for this module, not the entire test session.
+
+    TODO(ENG-3764)(andrei): Consolidate some of these test fixtures.
     """
     # Patch ENABLE_OPENSEARCH_INDEXING_FOR_ONYX just for this test because we
     # don't yet want that enabled for all tests.

backend/tests/unit/onyx/connectors/sharepoint/test_delta_checkpointing.py

@@ -6,6 +6,7 @@ Validates that:
 - Crash + resume skips already-processed pages
 - BFS (folder-scoped) drives process all items in one call
 - 410 Gone triggers a full-resync URL in the checkpoint
+- Duplicate document IDs across delta pages are deduplicated
 """
 
 from __future__ import annotations
@@ -457,3 +458,228 @@ class TestDeltaPageFetchFailure:
         assert final_cp.current_drive_name is None
         assert final_cp.current_drive_id is None
         assert final_cp.current_drive_delta_next_link is None
+
+
+class TestDeltaDuplicateDocumentDedup:
+    """The Microsoft Graph delta API can return the same item on multiple
+    pages.  Documents already yielded should be skipped via
+    checkpoint.seen_document_ids."""
+
+    def test_duplicate_across_pages_is_skipped(
+        self, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        """Item 'dup' appears on both page 1 and page 2.  It should only be
+        yielded once."""
+        connector = _setup_connector(monkeypatch)
+        _mock_convert(monkeypatch)
+
+        call_count = 0
+
+        def fake_fetch_page(
+            self: SharepointConnector,  # noqa: ARG001
+            page_url: str,  # noqa: ARG001
+            drive_id: str,  # noqa: ARG001
+            start: datetime | None = None,  # noqa: ARG001
+            end: datetime | None = None,  # noqa: ARG001
+            page_size: int = 200,  # noqa: ARG001
+        ) -> tuple[list[DriveItemData], str | None]:
+            nonlocal call_count
+            call_count += 1
+            if call_count == 1:
+                return [_make_item("a"), _make_item("dup")], "https://next2"
+            return [_make_item("dup"), _make_item("b")], None
+
+        monkeypatch.setattr(
+            SharepointConnector, "_fetch_one_delta_page", fake_fetch_page
+        )
+
+        checkpoint = _build_ready_checkpoint()
+
+        # Page 1: yields a, dup
+        gen = connector._load_from_checkpoint(
+            _START_TS, _END_TS, checkpoint, include_permissions=False
+        )
+        yielded, checkpoint = _consume_generator(gen)
+        docs = _docs_from(yielded)
+        assert [d.id for d in docs] == ["a", "dup"]
+        assert "dup" in checkpoint.seen_document_ids
+
+        # Page 2: dup should be skipped, only b yielded
+        gen = connector._load_from_checkpoint(
+            _START_TS, _END_TS, checkpoint, include_permissions=False
+        )
+        yielded, checkpoint = _consume_generator(gen)
+        docs = _docs_from(yielded)
+        assert [d.id for d in docs] == ["b"]
+
+    def test_duplicate_within_same_page_is_skipped(
+        self, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        """If the same item appears twice on a single delta page, only the
+        first occurrence should be yielded."""
+        connector = _setup_connector(monkeypatch)
+        _mock_convert(monkeypatch)
+
+        def fake_fetch_page(
+            self: SharepointConnector,  # noqa: ARG001
+            page_url: str,  # noqa: ARG001
+            drive_id: str,  # noqa: ARG001
+            start: datetime | None = None,  # noqa: ARG001
+            end: datetime | None = None,  # noqa: ARG001
+            page_size: int = 200,  # noqa: ARG001
+        ) -> tuple[list[DriveItemData], str | None]:
+            return [_make_item("x"), _make_item("x"), _make_item("y")], None
+
+        monkeypatch.setattr(
+            SharepointConnector, "_fetch_one_delta_page", fake_fetch_page
+        )
+
+        checkpoint = _build_ready_checkpoint()
+        gen = connector._load_from_checkpoint(
+            _START_TS, _END_TS, checkpoint, include_permissions=False
+        )
+        yielded, checkpoint = _consume_generator(gen)
+        docs = _docs_from(yielded)
+        assert [d.id for d in docs] == ["x", "y"]
+
+    def test_seen_ids_survive_checkpoint_serialization(
+        self, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        """seen_document_ids must survive JSON serialization so that
+        dedup works across crash + resume."""
+        connector = _setup_connector(monkeypatch)
+        _mock_convert(monkeypatch)
+
+        call_count = 0
+
+        def fake_fetch_page(
+            self: SharepointConnector,  # noqa: ARG001
+            page_url: str,  # noqa: ARG001
+            drive_id: str,  # noqa: ARG001
+            start: datetime | None = None,  # noqa: ARG001
+            end: datetime | None = None,  # noqa: ARG001
+            page_size: int = 200,  # noqa: ARG001
+        ) -> tuple[list[DriveItemData], str | None]:
+            nonlocal call_count
+            call_count += 1
+            if call_count == 1:
+                return [_make_item("a")], "https://next2"
+            return [_make_item("a"), _make_item("b")], None
+
+        monkeypatch.setattr(
+            SharepointConnector, "_fetch_one_delta_page", fake_fetch_page
+        )
+
+        checkpoint = _build_ready_checkpoint()
+
+        # Page 1
+        gen = connector._load_from_checkpoint(
+            _START_TS, _END_TS, checkpoint, include_permissions=False
+        )
+        _, checkpoint = _consume_generator(gen)
+        assert "a" in checkpoint.seen_document_ids
+
+        # Simulate crash: round-trip through JSON
+        restored = SharepointConnectorCheckpoint.model_validate_json(
+            checkpoint.model_dump_json()
+        )
+        assert "a" in restored.seen_document_ids
+
+        # Page 2 with restored checkpoint: 'a' should be skipped
+        connector2 = _setup_connector(monkeypatch)
+        _mock_convert(monkeypatch)
+        monkeypatch.setattr(
+            SharepointConnector, "_fetch_one_delta_page", fake_fetch_page
+        )
+
+        gen = connector2._load_from_checkpoint(
+            _START_TS, _END_TS, restored, include_permissions=False
+        )
+        yielded, final_cp = _consume_generator(gen)
+        docs = _docs_from(yielded)
+        assert [d.id for d in docs] == ["b"]
+
+    def test_no_dedup_across_separate_indexing_runs(
+        self, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        """A fresh checkpoint (new indexing run) should have an empty
+        seen_document_ids, so previously-indexed docs are re-processed."""
+        connector = _setup_connector(monkeypatch)
+        _mock_convert(monkeypatch)
+
+        def fake_fetch_page(
+            self: SharepointConnector,  # noqa: ARG001
+            page_url: str,  # noqa: ARG001
+            drive_id: str,  # noqa: ARG001
+            start: datetime | None = None,  # noqa: ARG001
+            end: datetime | None = None,  # noqa: ARG001
+            page_size: int = 200,  # noqa: ARG001
+        ) -> tuple[list[DriveItemData], str | None]:
+            return [_make_item("a")], None
+
+        monkeypatch.setattr(
+            SharepointConnector, "_fetch_one_delta_page", fake_fetch_page
+        )
+
+        # First run
+        cp1 = _build_ready_checkpoint()
+        gen = connector._load_from_checkpoint(
+            _START_TS, _END_TS, cp1, include_permissions=False
+        )
+        yielded, _ = _consume_generator(gen)
+        assert len(_docs_from(yielded)) == 1
+
+        # Second run with a fresh checkpoint — same doc should appear again
+        cp2 = _build_ready_checkpoint()
+        assert len(cp2.seen_document_ids) == 0
+        gen = connector._load_from_checkpoint(
+            _START_TS, _END_TS, cp2, include_permissions=False
+        )
+        yielded, _ = _consume_generator(gen)
+        assert len(_docs_from(yielded)) == 1
+
+    def test_same_id_across_drives_not_skipped(
+        self, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        """Graph item IDs are only unique within a drive.  An item in drive B
+        that happens to share an ID with an item already seen in drive A must
+        NOT be skipped."""
+        connector = _setup_connector(monkeypatch)
+        _mock_convert(monkeypatch)
+
+        def fake_fetch_page(
+            self: SharepointConnector,  # noqa: ARG001
+            page_url: str,  # noqa: ARG001
+            drive_id: str,  # noqa: ARG001
+            start: datetime | None = None,  # noqa: ARG001
+            end: datetime | None = None,  # noqa: ARG001
+            page_size: int = 200,  # noqa: ARG001
+        ) -> tuple[list[DriveItemData], str | None]:
+            return [_make_item("shared-id")], None
+
+        monkeypatch.setattr(
+            SharepointConnector, "_fetch_one_delta_page", fake_fetch_page
+        )
+
+        checkpoint = _build_ready_checkpoint(drive_names=["DriveA", "DriveB"])
+
+        # Drive A: yields the item
+        gen = connector._load_from_checkpoint(
+            _START_TS, _END_TS, checkpoint, include_permissions=False
+        )
+        yielded, checkpoint = _consume_generator(gen)
+        docs = _docs_from(yielded)
+        assert len(docs) == 1
+        assert docs[0].id == "shared-id"
+
+        # seen_document_ids should have been cleared when drive A finished
+        assert len(checkpoint.seen_document_ids) == 0
+
+        # Drive B: same ID must be yielded again (different drive)
+        gen = connector._load_from_checkpoint(
+            _START_TS, _END_TS, checkpoint, include_permissions=False
+        )
+        yielded, checkpoint = _consume_generator(gen)
+        docs = _docs_from(yielded)
+        assert len(docs) == 1
+        assert docs[0].id == "shared-id"

backend/tests/unit/onyx/db/test_llm_sync.py

@@ -7,6 +7,7 @@ import pytest
 
 from onyx.db.llm import sync_model_configurations
 from onyx.llm.constants import LlmProviderNames
+from onyx.server.manage.llm.models import SyncModelEntry
 
 
 class TestSyncModelConfigurations:
@@ -25,18 +26,18 @@ class TestSyncModelConfigurations:
             "onyx.db.llm.fetch_existing_llm_provider", return_value=mock_provider
         ):
             models = [
-                {
-                    "name": "gpt-4",
-                    "display_name": "GPT-4",
-                    "max_input_tokens": 128000,
-                    "supports_image_input": True,
-                },
-                {
-                    "name": "gpt-4o",
-                    "display_name": "GPT-4o",
-                    "max_input_tokens": 128000,
-                    "supports_image_input": True,
-                },
+                SyncModelEntry(
+                    name="gpt-4",
+                    display_name="GPT-4",
+                    max_input_tokens=128000,
+                    supports_image_input=True,
+                ),
+                SyncModelEntry(
+                    name="gpt-4o",
+                    display_name="GPT-4o",
+                    max_input_tokens=128000,
+                    supports_image_input=True,
+                ),
             ]
 
             result = sync_model_configurations(
@@ -67,18 +68,18 @@ class TestSyncModelConfigurations:
             "onyx.db.llm.fetch_existing_llm_provider", return_value=mock_provider
         ):
             models = [
-                {
-                    "name": "gpt-4",  # Existing - should be skipped
-                    "display_name": "GPT-4",
-                    "max_input_tokens": 128000,
-                    "supports_image_input": True,
-                },
-                {
-                    "name": "gpt-4o",  # New - should be inserted
-                    "display_name": "GPT-4o",
-                    "max_input_tokens": 128000,
-                    "supports_image_input": True,
-                },
+                SyncModelEntry(
+                    name="gpt-4",  # Existing - should be skipped
+                    display_name="GPT-4",
+                    max_input_tokens=128000,
+                    supports_image_input=True,
+                ),
+                SyncModelEntry(
+                    name="gpt-4o",  # New - should be inserted
+                    display_name="GPT-4o",
+                    max_input_tokens=128000,
+                    supports_image_input=True,
+                ),
             ]
 
             result = sync_model_configurations(
@@ -105,12 +106,12 @@ class TestSyncModelConfigurations:
             "onyx.db.llm.fetch_existing_llm_provider", return_value=mock_provider
         ):
             models = [
-                {
-                    "name": "gpt-4",  # Already exists
-                    "display_name": "GPT-4",
-                    "max_input_tokens": 128000,
-                    "supports_image_input": True,
-                },
+                SyncModelEntry(
+                    name="gpt-4",  # Already exists
+                    display_name="GPT-4",
+                    max_input_tokens=128000,
+                    supports_image_input=True,
+                ),
             ]
 
             result = sync_model_configurations(
@@ -131,7 +132,7 @@ class TestSyncModelConfigurations:
                 sync_model_configurations(
                     db_session=mock_session,
                     provider_name="nonexistent",
-                    models=[{"name": "model", "display_name": "Model"}],
+                    models=[SyncModelEntry(name="model", display_name="Model")],
                 )
 
     def test_handles_missing_optional_fields(self) -> None:
@@ -145,12 +146,12 @@ class TestSyncModelConfigurations:
         with patch(
             "onyx.db.llm.fetch_existing_llm_provider", return_value=mock_provider
         ):
-            # Model with only required fields
+            # Model with only required fields (max_input_tokens and supports_image_input default)
             models = [
-                {
-                    "name": "model-1",
-                    # No display_name, max_input_tokens, or supports_image_input
-                },
+                SyncModelEntry(
+                    name="model-1",
+                    display_name="Model 1",
+                ),
             ]
 
             result = sync_model_configurations(

backend/tests/unit/onyx/server/manage/llm/test_fetch_models_api.py

@@ -1,15 +1,19 @@
 """Tests for LLM model fetch endpoints.
 
 These tests verify the full request/response flow for fetching models
-from dynamic providers (Ollama, OpenRouter), including the
+from dynamic providers (Ollama, OpenRouter, Litellm), including the
 sync-to-DB behavior when provider_name is specified.
 """
 
 from unittest.mock import MagicMock
 from unittest.mock import patch
 
+import httpx
 import pytest
 
+from onyx.error_handling.exceptions import OnyxError
+from onyx.server.manage.llm.models import LitellmFinalModelResponse
+from onyx.server.manage.llm.models import LitellmModelsRequest
 from onyx.server.manage.llm.models import LMStudioFinalModelResponse
 from onyx.server.manage.llm.models import LMStudioModelsRequest
 from onyx.server.manage.llm.models import OllamaFinalModelResponse
@@ -614,3 +618,283 @@ class TestGetLMStudioAvailableModels:
             request = LMStudioModelsRequest(api_base="http://localhost:1234")
             with pytest.raises(OnyxError):
                 get_lm_studio_available_models(request, MagicMock(), mock_session)
+
+
+class TestGetLitellmAvailableModels:
+    """Tests for the Litellm proxy model fetch endpoint."""
+
+    @pytest.fixture
+    def mock_litellm_response(self) -> dict:
+        """Mock response from Litellm /v1/models endpoint."""
+        return {
+            "data": [
+                {
+                    "id": "gpt-4o",
+                    "object": "model",
+                    "created": 1700000000,
+                    "owned_by": "openai",
+                },
+                {
+                    "id": "claude-3-5-sonnet",
+                    "object": "model",
+                    "created": 1700000001,
+                    "owned_by": "anthropic",
+                },
+                {
+                    "id": "gemini-pro",
+                    "object": "model",
+                    "created": 1700000002,
+                    "owned_by": "google",
+                },
+            ]
+        }
+
+    def test_returns_model_list(self, mock_litellm_response: dict) -> None:
+        """Test that endpoint returns properly formatted model list."""
+        from onyx.server.manage.llm.api import get_litellm_available_models
+
+        mock_session = MagicMock()
+
+        with patch("onyx.server.manage.llm.api.httpx.get") as mock_get:
+            mock_response = MagicMock()
+            mock_response.json.return_value = mock_litellm_response
+            mock_response.raise_for_status = MagicMock()
+            mock_get.return_value = mock_response
+
+            request = LitellmModelsRequest(
+                api_base="http://localhost:4000",
+                api_key="test-key",
+            )
+            results = get_litellm_available_models(request, MagicMock(), mock_session)
+
+            assert len(results) == 3
+            assert all(isinstance(r, LitellmFinalModelResponse) for r in results)
+
+    def test_model_fields_parsed_correctly(self, mock_litellm_response: dict) -> None:
+        """Test that provider_name and model_name are correctly extracted."""
+        from onyx.server.manage.llm.api import get_litellm_available_models
+
+        mock_session = MagicMock()
+
+        with patch("onyx.server.manage.llm.api.httpx.get") as mock_get:
+            mock_response = MagicMock()
+            mock_response.json.return_value = mock_litellm_response
+            mock_response.raise_for_status = MagicMock()
+            mock_get.return_value = mock_response
+
+            request = LitellmModelsRequest(
+                api_base="http://localhost:4000",
+                api_key="test-key",
+            )
+            results = get_litellm_available_models(request, MagicMock(), mock_session)
+
+            gpt = next(r for r in results if r.model_name == "gpt-4o")
+            assert gpt.provider_name == "openai"
+
+            claude = next(r for r in results if r.model_name == "claude-3-5-sonnet")
+            assert claude.provider_name == "anthropic"
+
+    def test_results_sorted_by_model_name(self, mock_litellm_response: dict) -> None:
+        """Test that results are alphabetically sorted by model_name."""
+        from onyx.server.manage.llm.api import get_litellm_available_models
+
+        mock_session = MagicMock()
+
+        with patch("onyx.server.manage.llm.api.httpx.get") as mock_get:
+            mock_response = MagicMock()
+            mock_response.json.return_value = mock_litellm_response
+            mock_response.raise_for_status = MagicMock()
+            mock_get.return_value = mock_response
+
+            request = LitellmModelsRequest(
+                api_base="http://localhost:4000",
+                api_key="test-key",
+            )
+            results = get_litellm_available_models(request, MagicMock(), mock_session)
+
+            model_names = [r.model_name for r in results]
+            assert model_names == sorted(model_names, key=str.lower)
+
+    def test_empty_data_raises_onyx_error(self) -> None:
+        """Test that empty model list raises OnyxError."""
+        from onyx.server.manage.llm.api import get_litellm_available_models
+
+        mock_session = MagicMock()
+
+        with patch("onyx.server.manage.llm.api.httpx.get") as mock_get:
+            mock_response = MagicMock()
+            mock_response.json.return_value = {"data": []}
+            mock_response.raise_for_status = MagicMock()
+            mock_get.return_value = mock_response
+
+            request = LitellmModelsRequest(
+                api_base="http://localhost:4000",
+                api_key="test-key",
+            )
+            with pytest.raises(OnyxError, match="No models found"):
+                get_litellm_available_models(request, MagicMock(), mock_session)
+
+    def test_missing_data_key_raises_onyx_error(self) -> None:
+        """Test that response without 'data' key raises OnyxError."""
+        from onyx.server.manage.llm.api import get_litellm_available_models
+
+        mock_session = MagicMock()
+
+        with patch("onyx.server.manage.llm.api.httpx.get") as mock_get:
+            mock_response = MagicMock()
+            mock_response.json.return_value = {}
+            mock_response.raise_for_status = MagicMock()
+            mock_get.return_value = mock_response
+
+            request = LitellmModelsRequest(
+                api_base="http://localhost:4000",
+                api_key="test-key",
+            )
+            with pytest.raises(OnyxError):
+                get_litellm_available_models(request, MagicMock(), mock_session)
+
+    def test_skips_unparseable_entries(self) -> None:
+        """Test that malformed model entries are skipped without failing."""
+        from onyx.server.manage.llm.api import get_litellm_available_models
+
+        mock_session = MagicMock()
+        response_with_bad_entry = {
+            "data": [
+                {
+                    "id": "gpt-4o",
+                    "object": "model",
+                    "created": 1700000000,
+                    "owned_by": "openai",
+                },
+                # Missing required fields
+                {"bad_field": "bad_value"},
+            ]
+        }
+
+        with patch("onyx.server.manage.llm.api.httpx.get") as mock_get:
+            mock_response = MagicMock()
+            mock_response.json.return_value = response_with_bad_entry
+            mock_response.raise_for_status = MagicMock()
+            mock_get.return_value = mock_response
+
+            request = LitellmModelsRequest(
+                api_base="http://localhost:4000",
+                api_key="test-key",
+            )
+            results = get_litellm_available_models(request, MagicMock(), mock_session)
+
+            assert len(results) == 1
+            assert results[0].model_name == "gpt-4o"
+
+    def test_all_entries_unparseable_raises_onyx_error(self) -> None:
+        """Test that OnyxError is raised when all entries fail to parse."""
+        from onyx.server.manage.llm.api import get_litellm_available_models
+
+        mock_session = MagicMock()
+        response_all_bad = {
+            "data": [
+                {"bad_field": "bad_value"},
+                {"another_bad": 123},
+            ]
+        }
+
+        with patch("onyx.server.manage.llm.api.httpx.get") as mock_get:
+            mock_response = MagicMock()
+            mock_response.json.return_value = response_all_bad
+            mock_response.raise_for_status = MagicMock()
+            mock_get.return_value = mock_response
+
+            request = LitellmModelsRequest(
+                api_base="http://localhost:4000",
+                api_key="test-key",
+            )
+            with pytest.raises(OnyxError, match="No compatible models"):
+                get_litellm_available_models(request, MagicMock(), mock_session)
+
+    def test_api_base_trailing_slash_handled(self) -> None:
+        """Test that trailing slashes in api_base are handled correctly."""
+        from onyx.server.manage.llm.api import get_litellm_available_models
+
+        mock_session = MagicMock()
+        mock_litellm_response = {
+            "data": [
+                {
+                    "id": "gpt-4o",
+                    "object": "model",
+                    "created": 1700000000,
+                    "owned_by": "openai",
+                },
+            ]
+        }
+
+        with patch("onyx.server.manage.llm.api.httpx.get") as mock_get:
+            mock_response = MagicMock()
+            mock_response.json.return_value = mock_litellm_response
+            mock_response.raise_for_status = MagicMock()
+            mock_get.return_value = mock_response
+
+            request = LitellmModelsRequest(
+                api_base="http://localhost:4000/",
+                api_key="test-key",
+            )
+            get_litellm_available_models(request, MagicMock(), mock_session)
+
+            # Should call /v1/models without double slashes
+            call_args = mock_get.call_args
+            assert call_args[0][0] == "http://localhost:4000/v1/models"
+
+    def test_connection_failure_raises_onyx_error(self) -> None:
+        """Test that connection failures are wrapped in OnyxError."""
+        from onyx.server.manage.llm.api import get_litellm_available_models
+
+        mock_session = MagicMock()
+
+        with patch("onyx.server.manage.llm.api.httpx.get") as mock_get:
+            mock_get.side_effect = Exception("Connection refused")
+
+            request = LitellmModelsRequest(
+                api_base="http://localhost:4000",
+                api_key="test-key",
+            )
+            with pytest.raises(OnyxError, match="Failed to fetch LiteLLM models"):
+                get_litellm_available_models(request, MagicMock(), mock_session)
+
+    def test_401_raises_authentication_error(self) -> None:
+        """Test that a 401 response raises OnyxError with authentication message."""
+        from onyx.server.manage.llm.api import get_litellm_available_models
+
+        mock_session = MagicMock()
+
+        with patch("onyx.server.manage.llm.api.httpx.get") as mock_get:
+            mock_response = MagicMock()
+            mock_response.status_code = 401
+            mock_get.side_effect = httpx.HTTPStatusError(
+                "Unauthorized", request=MagicMock(), response=mock_response
+            )
+
+            request = LitellmModelsRequest(
+                api_base="http://localhost:4000",
+                api_key="bad-key",
+            )
+            with pytest.raises(OnyxError, match="Authentication failed"):
+                get_litellm_available_models(request, MagicMock(), mock_session)
+
+    def test_404_raises_not_found_error(self) -> None:
+        """Test that a 404 response raises OnyxError with endpoint not found message."""
+        from onyx.server.manage.llm.api import get_litellm_available_models
+
+        mock_session = MagicMock()
+
+        with patch("onyx.server.manage.llm.api.httpx.get") as mock_get:
+            mock_response = MagicMock()
+            mock_response.status_code = 404
+            mock_get.side_effect = httpx.HTTPStatusError(
+                "Not Found", request=MagicMock(), response=mock_response
+            )
+
+            request = LitellmModelsRequest(
+                api_base="http://localhost:4000",
+                api_key="test-key",
+            )
+            with pytest.raises(OnyxError, match="endpoint not found"):
+                get_litellm_available_models(request, MagicMock(), mock_session)

deployment/docker_compose/docker-compose.onyx-lite.yml

@@ -15,9 +15,8 @@
 #                  -f docker-compose.dev.yml up -d --wait
 #
 # This overlay:
-#   - Moves Vespa (index), both model servers, code-interpreter, OpenSearch,
-#     MinIO, Redis (cache), and the background worker to profiles so they do
-#     not start by default
+#   - Moves Vespa (index), both model servers, code-interpreter, Redis (cache),
+#     and the background worker to profiles so they do not start by default
 #   - Makes depends_on references to removed services optional
 #   - Sets DISABLE_VECTOR_DB=true on the api_server
 #   - Uses PostgreSQL for caching and auth instead of Redis
@@ -28,8 +27,6 @@
 #   --profile inference         Inference model server
 #   --profile background        Background worker (Celery) — also needs redis
 #   --profile redis             Redis cache
-#   --profile opensearch        OpenSearch
-#   --profile s3-filestore      MinIO (S3-compatible file store)
 #   --profile code-interpreter  Code interpreter
 # =============================================================================
 
@@ -41,9 +38,6 @@ services:
       index:
         condition: service_started
         required: false
-      opensearch:
-        condition: service_started
-        required: false
       cache:
         condition: service_started
         required: false
@@ -90,13 +84,4 @@ services:
   inference_model_server:
     profiles: ["inference"]
 
-  # OpenSearch is not needed in lite mode (no indexing).
-  opensearch:
-    profiles: ["opensearch"]
-
-  # MinIO is not needed in lite mode (Postgres handles file storage).
-  minio:
-    profiles: ["s3-filestore"]
-
-  code-interpreter:
-    profiles: ["code-interpreter"]
+  code-interpreter: {}

deployment/docker_compose/install.sh

@@ -1,8 +1,8 @@
 #!/bin/bash
 
-set -euo pipefail
+set -e
 
-# Expected resource requirements (overridden below if --lite)
+# Expected resource requirements
 EXPECTED_DOCKER_RAM_GB=10
 EXPECTED_DISK_GB=32
 
@@ -10,10 +10,6 @@ EXPECTED_DISK_GB=32
 SHUTDOWN_MODE=false
 DELETE_DATA_MODE=false
 INCLUDE_CRAFT=false  # Disabled by default, use --include-craft to enable
-LITE_MODE=false       # Disabled by default, use --lite to enable
-NO_PROMPT=false
-DRY_RUN=false
-VERBOSE=false
 
 while [[ $# -gt 0 ]]; do
     case $1 in
@@ -29,22 +25,6 @@ while [[ $# -gt 0 ]]; do
             INCLUDE_CRAFT=true
             shift
             ;;
-        --lite)
-            LITE_MODE=true
-            shift
-            ;;
-        --no-prompt)
-            NO_PROMPT=true
-            shift
-            ;;
-        --dry-run)
-            DRY_RUN=true
-            shift
-            ;;
-        --verbose)
-            VERBOSE=true
-            shift
-            ;;
         --help|-h)
             echo "Onyx Installation Script"
             echo ""
@@ -52,21 +32,15 @@ while [[ $# -gt 0 ]]; do
             echo ""
             echo "Options:"
             echo "  --include-craft  Enable Onyx Craft (AI-powered web app building)"
-            echo "  --lite           Deploy Onyx Lite (no Vespa, Redis, or model servers)"
             echo "  --shutdown       Stop (pause) Onyx containers"
             echo "  --delete-data    Remove all Onyx data (containers, volumes, and files)"
-            echo "  --no-prompt      Run non-interactively with defaults (for CI/automation)"
-            echo "  --dry-run        Show what would be done without making changes"
-            echo "  --verbose        Show detailed output for debugging"
             echo "  --help, -h       Show this help message"
             echo ""
             echo "Examples:"
             echo "  $0                    # Install Onyx"
-            echo "  $0 --lite             # Install Onyx Lite (minimal deployment)"
             echo "  $0 --include-craft    # Install Onyx with Craft enabled"
             echo "  $0 --shutdown         # Pause Onyx services"
             echo "  $0 --delete-data      # Completely remove Onyx and all data"
-            echo "  $0 --no-prompt        # Non-interactive install with defaults"
             exit 0
             ;;
         *)
@@ -77,102 +51,8 @@ while [[ $# -gt 0 ]]; do
     esac
 done
 
-if [[ "$VERBOSE" = true ]]; then
-    set -x
-fi
-
-if [[ "$LITE_MODE" = true ]] && [[ "$INCLUDE_CRAFT" = true ]]; then
-    echo "ERROR: --lite and --include-craft cannot be used together."
-    echo "Craft requires services (Vespa, Redis, background workers) that lite mode disables."
-    exit 1
-fi
-
-# When --lite is passed as a flag, lower resource thresholds early (before the
-# resource check). When lite is chosen interactively, the thresholds are adjusted
-# inside the new-deployment flow, after the resource check has already passed
-# with the standard thresholds — which is the safer direction.
-if [[ "$LITE_MODE" = true ]]; then
-    EXPECTED_DOCKER_RAM_GB=4
-    EXPECTED_DISK_GB=16
-fi
-
 INSTALL_ROOT="${INSTALL_PREFIX:-onyx_data}"
 
-LITE_COMPOSE_FILE="docker-compose.onyx-lite.yml"
-
-# Build the -f flags for docker compose.
-# Pass "true" as $1 to auto-detect a previously-downloaded lite overlay
-# (used by shutdown/delete-data so users don't need to remember --lite).
-# Without the argument, the lite overlay is only included when --lite was
-# explicitly passed — preventing install/start from silently staying in
-# lite mode just because the file exists on disk from a prior run.
-compose_file_args() {
-    local auto_detect="${1:-false}"
-    local args="-f docker-compose.yml"
-    if [[ "$LITE_MODE" = true ]] || { [[ "$auto_detect" = true ]] && [[ -f "${INSTALL_ROOT}/deployment/${LITE_COMPOSE_FILE}" ]]; }; then
-        args="$args -f ${LITE_COMPOSE_FILE}"
-    fi
-    echo "$args"
-}
-
-# --- Downloader detection (curl with wget fallback) ---
-DOWNLOADER=""
-detect_downloader() {
-    if command -v curl &> /dev/null; then
-        DOWNLOADER="curl"
-        return 0
-    fi
-    if command -v wget &> /dev/null; then
-        DOWNLOADER="wget"
-        return 0
-    fi
-    echo "ERROR: Neither curl nor wget found. Please install one and retry."
-    exit 1
-}
-detect_downloader
-
-download_file() {
-    local url="$1"
-    local output="$2"
-    if [[ "$DOWNLOADER" == "curl" ]]; then
-        curl -fsSL --retry 3 --retry-delay 2 --retry-connrefused -o "$output" "$url"
-    else
-        wget -q --tries=3 --timeout=20 -O "$output" "$url"
-    fi
-}
-
-# --- Interactive prompt helpers ---
-is_interactive() {
-    [[ "$NO_PROMPT" = false ]] && [[ -t 0 ]]
-}
-
-prompt_or_default() {
-    local prompt_text="$1"
-    local default_value="$2"
-    if is_interactive; then
-        read -p "$prompt_text" -r REPLY
-        if [[ -z "$REPLY" ]]; then
-            REPLY="$default_value"
-        fi
-    else
-        REPLY="$default_value"
-    fi
-}
-
-prompt_yn_or_default() {
-    local prompt_text="$1"
-    local default_value="$2"
-    if is_interactive; then
-        read -p "$prompt_text" -n 1 -r
-        echo ""
-        if [[ -z "$REPLY" ]]; then
-            REPLY="$default_value"
-        fi
-    else
-        REPLY="$default_value"
-    fi
-}
-
 # Colors for output
 RED='\033[0;31m'
 GREEN='\033[0;32m'
@@ -231,7 +111,7 @@ if [ "$SHUTDOWN_MODE" = true ]; then
             fi
 
             # Stop containers (without removing them)
-            (cd "${INSTALL_ROOT}/deployment" && $COMPOSE_CMD $(compose_file_args true) stop)
+            (cd "${INSTALL_ROOT}/deployment" && $COMPOSE_CMD -f docker-compose.yml stop)
             if [ $? -eq 0 ]; then
                 print_success "Onyx containers stopped (paused)"
             else
@@ -260,17 +140,12 @@ if [ "$DELETE_DATA_MODE" = true ]; then
     echo "  • All downloaded files and configurations"
     echo "  • All user data and documents"
     echo ""
-    if is_interactive; then
-        read -p "Are you sure you want to continue? Type 'DELETE' to confirm: " -r
-        echo ""
-        if [ "$REPLY" != "DELETE" ]; then
-            print_info "Operation cancelled."
-            exit 0
-        fi
-    else
-        print_error "Cannot confirm destructive operation in non-interactive mode."
-        print_info "Run interactively or remove the ${INSTALL_ROOT} directory manually."
-        exit 1
+    read -p "Are you sure you want to continue? Type 'DELETE' to confirm: " -r
+    echo ""
+
+    if [ "$REPLY" != "DELETE" ]; then
+        print_info "Operation cancelled."
+        exit 0
     fi
 
     print_info "Removing Onyx containers and volumes..."
@@ -289,7 +164,7 @@ if [ "$DELETE_DATA_MODE" = true ]; then
             fi
 
             # Stop and remove containers with volumes
-            (cd "${INSTALL_ROOT}/deployment" && $COMPOSE_CMD $(compose_file_args true) down -v)
+            (cd "${INSTALL_ROOT}/deployment" && $COMPOSE_CMD -f docker-compose.yml down -v)
             if [ $? -eq 0 ]; then
                 print_success "Onyx containers and volumes removed"
             else
@@ -334,7 +209,8 @@ echo "2. Check your system resources (Docker, memory, disk space)"
 echo "3. Guide you through deployment options (version, authentication)"
 echo ""
 
-if is_interactive; then
+# Only prompt for acknowledgment if running interactively
+if [ -t 0 ]; then
     echo -e "${YELLOW}${BOLD}Please acknowledge and press Enter to continue...${NC}"
     read -r
     echo ""
@@ -343,25 +219,6 @@ else
     echo ""
 fi
 
-# Detect OS (including WSL)
-IS_WSL=false
-if [[ -n "${WSL_DISTRO_NAME:-}" ]] || grep -qi microsoft /proc/version 2>/dev/null; then
-    IS_WSL=true
-fi
-
-# Dry-run: show plan and exit
-if [[ "$DRY_RUN" = true ]]; then
-    print_info "Dry run mode — showing what would happen:"
-    echo "  • Install root: ${INSTALL_ROOT}"
-    echo "  • Lite mode: ${LITE_MODE}"
-    echo "  • Include Craft: ${INCLUDE_CRAFT}"
-    echo "  • OS type: ${OSTYPE:-unknown} (WSL: ${IS_WSL})"
-    echo "  • Downloader: ${DOWNLOADER}"
-    echo ""
-    print_success "Dry run complete (no changes made)"
-    exit 0
-fi
-
 # GitHub repo base URL - using main branch
 GITHUB_RAW_URL="https://raw.githubusercontent.com/onyx-dot-app/onyx/main/deployment/docker_compose"
 
@@ -403,35 +260,41 @@ else
     exit 1
 fi
 
-# Returns 0 if $1 <= $2, 1 if $1 > $2
-# Handles missing or non-numeric parts gracefully (treats them as 0)
+# Function to compare version numbers
 version_compare() {
-    local version1="${1:-0.0.0}"
-    local version2="${2:-0.0.0}"
+    # Returns 0 if $1 <= $2, 1 if $1 > $2
+    local version1=$1
+    local version2=$2
 
-    local v1_major v1_minor v1_patch v2_major v2_minor v2_patch
-    v1_major=$(echo "$version1" | cut -d. -f1)
-    v1_minor=$(echo "$version1" | cut -d. -f2)
-    v1_patch=$(echo "$version1" | cut -d. -f3)
-    v2_major=$(echo "$version2" | cut -d. -f1)
-    v2_minor=$(echo "$version2" | cut -d. -f2)
-    v2_patch=$(echo "$version2" | cut -d. -f3)
+    # Split versions into components
+    local v1_major=$(echo $version1 | cut -d. -f1)
+    local v1_minor=$(echo $version1 | cut -d. -f2)
+    local v1_patch=$(echo $version1 | cut -d. -f3)
 
-    # Default non-numeric or empty parts to 0
-    [[ "$v1_major" =~ ^[0-9]+$ ]] || v1_major=0
-    [[ "$v1_minor" =~ ^[0-9]+$ ]] || v1_minor=0
-    [[ "$v1_patch" =~ ^[0-9]+$ ]] || v1_patch=0
-    [[ "$v2_major" =~ ^[0-9]+$ ]] || v2_major=0
-    [[ "$v2_minor" =~ ^[0-9]+$ ]] || v2_minor=0
-    [[ "$v2_patch" =~ ^[0-9]+$ ]] || v2_patch=0
+    local v2_major=$(echo $version2 | cut -d. -f1)
+    local v2_minor=$(echo $version2 | cut -d. -f2)
+    local v2_patch=$(echo $version2 | cut -d. -f3)
 
-    if [ "$v1_major" -lt "$v2_major" ]; then return 0
-    elif [ "$v1_major" -gt "$v2_major" ]; then return 1; fi
+    # Compare major version
+    if [ "$v1_major" -lt "$v2_major" ]; then
+        return 0
+    elif [ "$v1_major" -gt "$v2_major" ]; then
+        return 1
+    fi
 
-    if [ "$v1_minor" -lt "$v2_minor" ]; then return 0
-    elif [ "$v1_minor" -gt "$v2_minor" ]; then return 1; fi
+    # Compare minor version
+    if [ "$v1_minor" -lt "$v2_minor" ]; then
+        return 0
+    elif [ "$v1_minor" -gt "$v2_minor" ]; then
+        return 1
+    fi
 
-    [ "$v1_patch" -le "$v2_patch" ]
+    # Compare patch version
+    if [ "$v1_patch" -le "$v2_patch" ]; then
+        return 0
+    else
+        return 1
+    fi
 }
 
 # Check Docker daemon
@@ -506,10 +369,10 @@ fi
 
 if [ "$RESOURCE_WARNING" = true ]; then
     echo ""
-    print_warning "Onyx recommends at least ${EXPECTED_DOCKER_RAM_GB}GB RAM and ${EXPECTED_DISK_GB}GB disk space for optimal performance in standard mode."
-    print_warning "Lite mode requires less resources, but does not include a vector database."
+    print_warning "Onyx recommends at least ${EXPECTED_DOCKER_RAM_GB}GB RAM and ${EXPECTED_DISK_GB}GB disk space for optimal performance."
+    echo ""
+    read -p "Do you want to continue anyway? (y/N): " -n 1 -r
     echo ""
-    prompt_yn_or_default "Do you want to continue anyway? (Y/n): " "y"
     if [[ ! $REPLY =~ ^[Yy]$ ]]; then
         print_info "Installation cancelled. Please allocate more resources and try again."
         exit 1
@@ -534,9 +397,6 @@ print_info "This step downloads all necessary configuration files from GitHub...
 echo ""
 print_info "Downloading the following files:"
 echo "  • docker-compose.yml - Main Docker Compose configuration"
-if [[ "$LITE_MODE" = true ]]; then
-    echo "  • ${LITE_COMPOSE_FILE} - Lite mode overlay"
-fi
 echo "  • env.template - Environment variables template"
 echo "  • nginx/app.conf.template - Nginx web server configuration"
 echo "  • nginx/run-nginx.sh - Nginx startup script"
@@ -546,7 +406,7 @@ echo ""
 # Download Docker Compose file
 COMPOSE_FILE="${INSTALL_ROOT}/deployment/docker-compose.yml"
 print_info "Downloading docker-compose.yml..."
-if download_file "${GITHUB_RAW_URL}/docker-compose.yml" "$COMPOSE_FILE" 2>/dev/null; then
+if curl -fsSL -o "$COMPOSE_FILE" "${GITHUB_RAW_URL}/docker-compose.yml" 2>/dev/null; then
     print_success "Docker Compose file downloaded successfully"
 
     # Check if Docker Compose version is older than 2.24.0 and show warning
@@ -571,7 +431,8 @@ if download_file "${GITHUB_RAW_URL}/docker-compose.yml" "$COMPOSE_FILE" 2>/dev/n
         echo ""
         print_warning "The installation will continue, but may fail if Docker Compose cannot parse the file."
         echo ""
-        prompt_yn_or_default "Do you want to continue anyway? (Y/n): " "y"
+        read -p "Do you want to continue anyway? (y/N): " -n 1 -r
+        echo ""
         if [[ ! $REPLY =~ ^[Yy]$ ]]; then
             print_info "Installation cancelled. Please upgrade Docker Compose or manually edit the docker-compose.yml file."
             exit 1
@@ -584,40 +445,10 @@ else
     exit 1
 fi
 
-# Download lite overlay if --lite was requested, otherwise remove any stale
-# overlay from a previous lite install so shutdown/delete-data auto-detection
-# doesn't mistakenly include it later.
-if [[ "$LITE_MODE" = true ]]; then
-    LITE_FILE="${INSTALL_ROOT}/deployment/${LITE_COMPOSE_FILE}"
-    print_info "Downloading ${LITE_COMPOSE_FILE} (lite overlay)..."
-    if download_file "${GITHUB_RAW_URL}/${LITE_COMPOSE_FILE}" "$LITE_FILE" 2>/dev/null; then
-        print_success "Lite overlay downloaded successfully"
-    else
-        print_error "Failed to download lite overlay"
-        print_info "Please ensure you have internet connection and try again"
-        exit 1
-    fi
-elif [[ -f "${INSTALL_ROOT}/deployment/${LITE_COMPOSE_FILE}" ]]; then
-    if [[ -f "${INSTALL_ROOT}/deployment/.env" ]]; then
-        print_warning "Existing lite overlay found but --lite was not passed."
-        prompt_yn_or_default "Remove lite overlay and switch to standard mode? (y/N): " "n"
-        if [[ ! $REPLY =~ ^[Yy]$ ]]; then
-            print_info "Keeping existing lite overlay. Pass --lite to keep using lite mode."
-            LITE_MODE=true
-        else
-            rm -f "${INSTALL_ROOT}/deployment/${LITE_COMPOSE_FILE}"
-            print_info "Removed lite overlay (switching to standard mode)"
-        fi
-    else
-        rm -f "${INSTALL_ROOT}/deployment/${LITE_COMPOSE_FILE}"
-        print_info "Removed previous lite overlay (switching to standard mode)"
-    fi
-fi
-
 # Download env.template file
 ENV_TEMPLATE="${INSTALL_ROOT}/deployment/env.template"
 print_info "Downloading env.template..."
-if download_file "${GITHUB_RAW_URL}/env.template" "$ENV_TEMPLATE" 2>/dev/null; then
+if curl -fsSL -o "$ENV_TEMPLATE" "${GITHUB_RAW_URL}/env.template" 2>/dev/null; then
     print_success "Environment template downloaded successfully"
 else
     print_error "Failed to download env.template"
@@ -631,7 +462,7 @@ NGINX_BASE_URL="https://raw.githubusercontent.com/onyx-dot-app/onyx/main/deploym
 # Download app.conf.template
 NGINX_CONFIG="${INSTALL_ROOT}/data/nginx/app.conf.template"
 print_info "Downloading nginx configuration template..."
-if download_file "$NGINX_BASE_URL/app.conf.template" "$NGINX_CONFIG" 2>/dev/null; then
+if curl -fsSL -o "$NGINX_CONFIG" "$NGINX_BASE_URL/app.conf.template" 2>/dev/null; then
     print_success "Nginx configuration template downloaded"
 else
     print_error "Failed to download nginx configuration template"
@@ -642,7 +473,7 @@ fi
 # Download run-nginx.sh script
 NGINX_RUN_SCRIPT="${INSTALL_ROOT}/data/nginx/run-nginx.sh"
 print_info "Downloading nginx startup script..."
-if download_file "$NGINX_BASE_URL/run-nginx.sh" "$NGINX_RUN_SCRIPT" 2>/dev/null; then
+if curl -fsSL -o "$NGINX_RUN_SCRIPT" "$NGINX_BASE_URL/run-nginx.sh" 2>/dev/null; then
     chmod +x "$NGINX_RUN_SCRIPT"
     print_success "Nginx startup script downloaded and made executable"
 else
@@ -654,7 +485,7 @@ fi
 # Download README file
 README_FILE="${INSTALL_ROOT}/README.md"
 print_info "Downloading README.md..."
-if download_file "${GITHUB_RAW_URL}/README.md" "$README_FILE" 2>/dev/null; then
+if curl -fsSL -o "$README_FILE" "${GITHUB_RAW_URL}/README.md" 2>/dev/null; then
     print_success "README.md downloaded successfully"
 else
     print_error "Failed to download README.md"
@@ -682,7 +513,7 @@ if [ -d "${INSTALL_ROOT}/deployment" ] && [ -f "${INSTALL_ROOT}/deployment/docke
 
     if [ -n "$COMPOSE_CMD" ]; then
         # Check if any containers are running
-        RUNNING_CONTAINERS=$(cd "${INSTALL_ROOT}/deployment" && $COMPOSE_CMD $(compose_file_args true) ps -q 2>/dev/null | wc -l)
+        RUNNING_CONTAINERS=$(cd "${INSTALL_ROOT}/deployment" && $COMPOSE_CMD -f docker-compose.yml ps -q 2>/dev/null | wc -l)
         if [ "$RUNNING_CONTAINERS" -gt 0 ]; then
             print_error "Onyx services are currently running!"
             echo ""
@@ -703,7 +534,7 @@ if [ -f "$ENV_FILE" ]; then
     echo "• Press Enter to restart with current configuration"
     echo "• Type 'update' to update to a newer version"
     echo ""
-    prompt_or_default "Choose an option [default: restart]: " ""
+    read -p "Choose an option [default: restart]: " -r
     echo ""
 
     if [ "$REPLY" = "update" ]; then
@@ -712,30 +543,26 @@ if [ -f "$ENV_FILE" ]; then
         echo "• Press Enter for latest (recommended)"
         echo "• Type a specific tag (e.g., v0.1.0)"
         echo ""
+        # If --include-craft was passed, default to craft-latest
         if [ "$INCLUDE_CRAFT" = true ]; then
-            prompt_or_default "Enter tag [default: craft-latest]: " "craft-latest"
-            VERSION="$REPLY"
+            read -p "Enter tag [default: craft-latest]: " -r VERSION
         else
-            prompt_or_default "Enter tag [default: latest]: " "latest"
-            VERSION="$REPLY"
+            read -p "Enter tag [default: latest]: " -r VERSION
         fi
         echo ""
 
-        if [ "$INCLUDE_CRAFT" = true ] && [ "$VERSION" = "craft-latest" ]; then
-            print_info "Selected: craft-latest (Craft enabled)"
-        elif [ "$VERSION" = "latest" ]; then
-            print_info "Selected: Latest version"
+        if [ -z "$VERSION" ]; then
+            if [ "$INCLUDE_CRAFT" = true ]; then
+                VERSION="craft-latest"
+                print_info "Selected: craft-latest (Craft enabled)"
+            else
+                VERSION="latest"
+                print_info "Selected: Latest version"
+            fi
         else
             print_info "Selected: $VERSION"
         fi
 
-        # Reject craft image tags when running in lite mode
-        if [[ "$LITE_MODE" = true ]] && [[ "${VERSION:-}" == craft-* ]]; then
-            print_error "Cannot use a craft image tag (${VERSION}) with --lite."
-            print_info "Craft requires services (Vespa, Redis, background workers) that lite mode disables."
-            exit 1
-        fi
-
         # Update .env file with new version
         print_info "Updating configuration for version $VERSION..."
         if grep -q "^IMAGE_TAG=" "$ENV_FILE"; then
@@ -754,73 +581,13 @@ if [ -f "$ENV_FILE" ]; then
         fi
         print_success "Configuration updated for upgrade"
     else
-        # Reject restarting a craft deployment in lite mode
-        EXISTING_TAG=$(grep "^IMAGE_TAG=" "$ENV_FILE" | head -1 | cut -d'=' -f2 | tr -d ' "'"'"'')
-        if [[ "$LITE_MODE" = true ]] && [[ "${EXISTING_TAG:-}" == craft-* ]]; then
-            print_error "Cannot restart a craft deployment (${EXISTING_TAG}) with --lite."
-            print_info "Craft requires services (Vespa, Redis, background workers) that lite mode disables."
-            exit 1
-        fi
-
         print_info "Keeping existing configuration..."
         print_success "Will restart with current settings"
     fi
-
-    # Ensure COMPOSE_PROFILES is cleared when running in lite mode on an
-    # existing .env (the template ships with s3-filestore enabled).
-    if [[ "$LITE_MODE" = true ]] && grep -q "^COMPOSE_PROFILES=.*s3-filestore" "$ENV_FILE" 2>/dev/null; then
-        sed -i.bak 's/^COMPOSE_PROFILES=.*/COMPOSE_PROFILES=/' "$ENV_FILE" 2>/dev/null || true
-        print_success "Cleared COMPOSE_PROFILES for lite mode"
-    fi
 else
     print_info "No existing .env file found. Setting up new deployment..."
     echo ""
 
-    # Ask for deployment mode (standard vs lite) unless already set via --lite flag
-    if [[ "$LITE_MODE" = false ]]; then
-        print_info "Which deployment mode would you like?"
-        echo ""
-        echo "  1) Standard  - Full deployment with search, connectors, and RAG"
-        echo "  2) Lite      - Minimal deployment (no Vespa, Redis, or model servers)"
-        echo "                  LLM chat, tools, file uploads, and Projects still work"
-        echo ""
-        prompt_or_default "Choose a mode (1 or 2) [default: 1]: " "1"
-        echo ""
-
-        case "$REPLY" in
-            2)
-                LITE_MODE=true
-                print_info "Selected: Lite mode"
-                LITE_FILE="${INSTALL_ROOT}/deployment/${LITE_COMPOSE_FILE}"
-                print_info "Downloading ${LITE_COMPOSE_FILE} (lite overlay)..."
-                if download_file "${GITHUB_RAW_URL}/${LITE_COMPOSE_FILE}" "$LITE_FILE"; then
-                    print_success "Lite overlay downloaded successfully"
-                else
-                    print_error "Failed to download lite overlay"
-                    exit 1
-                fi
-                ;;
-            *)
-                print_info "Selected: Standard mode"
-                ;;
-        esac
-    else
-        print_info "Deployment mode: Lite (set via --lite flag)"
-    fi
-
-    # Validate lite + craft combination (could now be set interactively)
-    if [[ "$LITE_MODE" = true ]] && [[ "$INCLUDE_CRAFT" = true ]]; then
-        print_error "--include-craft cannot be used with Lite mode."
-        print_info "Craft requires services (Vespa, Redis, background workers) that lite mode disables."
-        exit 1
-    fi
-
-    # Adjust resource expectations for lite mode
-    if [[ "$LITE_MODE" = true ]]; then
-        EXPECTED_DOCKER_RAM_GB=4
-        EXPECTED_DISK_GB=16
-    fi
-
     # Ask for version
     print_info "Which tag would you like to deploy?"
     echo ""
@@ -828,21 +595,23 @@ else
         echo "• Press Enter for craft-latest (recommended for Craft)"
         echo "• Type a specific tag (e.g., craft-v1.0.0)"
         echo ""
-        prompt_or_default "Enter tag [default: craft-latest]: " "craft-latest"
-        VERSION="$REPLY"
+        read -p "Enter tag [default: craft-latest]: " -r VERSION
     else
         echo "• Press Enter for latest (recommended)"
         echo "• Type a specific tag (e.g., v0.1.0)"
         echo ""
-        prompt_or_default "Enter tag [default: latest]: " "latest"
-        VERSION="$REPLY"
+        read -p "Enter tag [default: latest]: " -r VERSION
     fi
     echo ""
 
-    if [ "$INCLUDE_CRAFT" = true ] && [ "$VERSION" = "craft-latest" ]; then
-        print_info "Selected: craft-latest (Craft enabled)"
-    elif [ "$VERSION" = "latest" ]; then
-        print_info "Selected: Latest tag"
+    if [ -z "$VERSION" ]; then
+        if [ "$INCLUDE_CRAFT" = true ]; then
+            VERSION="craft-latest"
+            print_info "Selected: craft-latest (Craft enabled)"
+        else
+            VERSION="latest"
+            print_info "Selected: Latest tag"
+        fi
     else
         print_info "Selected: $VERSION"
     fi
@@ -876,13 +645,6 @@ else
     # Use basic auth by default
     AUTH_SCHEMA="basic"
 
-    # Reject craft image tags when running in lite mode (must check before writing .env)
-    if [[ "$LITE_MODE" = true ]] && [[ "${VERSION:-}" == craft-* ]]; then
-        print_error "Cannot use a craft image tag (${VERSION}) with --lite."
-        print_info "Craft requires services (Vespa, Redis, background workers) that lite mode disables."
-        exit 1
-    fi
-
     # Create .env file from template
     print_info "Creating .env file with your selections..."
     cp "$ENV_TEMPLATE" "$ENV_FILE"
@@ -892,13 +654,6 @@ else
     sed -i.bak "s/^IMAGE_TAG=.*/IMAGE_TAG=$VERSION/" "$ENV_FILE"
     print_success "IMAGE_TAG set to $VERSION"
 
-    # In lite mode, clear COMPOSE_PROFILES so profiled services (MinIO, etc.)
-    # stay disabled — the template ships with s3-filestore enabled by default.
-    if [[ "$LITE_MODE" = true ]]; then
-        sed -i.bak 's/^COMPOSE_PROFILES=.*/COMPOSE_PROFILES=/' "$ENV_FILE" 2>/dev/null || true
-        print_success "Cleared COMPOSE_PROFILES for lite mode"
-    fi
-
     # Configure basic authentication (default)
     sed -i.bak 's/^AUTH_TYPE=.*/AUTH_TYPE=basic/' "$ENV_FILE" 2>/dev/null || true
     print_success "Basic authentication enabled in configuration"
@@ -1019,7 +774,7 @@ print_step "Pulling Docker images"
 print_info "This may take several minutes depending on your internet connection..."
 echo ""
 print_info "Downloading Docker images (this may take a while)..."
-(cd "${INSTALL_ROOT}/deployment" && $COMPOSE_CMD $(compose_file_args) pull --quiet)
+(cd "${INSTALL_ROOT}/deployment" && $COMPOSE_CMD -f docker-compose.yml pull --quiet)
 if [ $? -eq 0 ]; then
     print_success "Docker images downloaded successfully"
 else
@@ -1033,9 +788,9 @@ print_info "Launching containers..."
 echo ""
 if [ "$USE_LATEST" = true ]; then
     print_info "Force pulling latest images and recreating containers..."
-    (cd "${INSTALL_ROOT}/deployment" && $COMPOSE_CMD $(compose_file_args) up -d --pull always --force-recreate)
+    (cd "${INSTALL_ROOT}/deployment" && $COMPOSE_CMD -f docker-compose.yml up -d --pull always --force-recreate)
 else
-    (cd "${INSTALL_ROOT}/deployment" && $COMPOSE_CMD $(compose_file_args) up -d)
+    (cd "${INSTALL_ROOT}/deployment" && $COMPOSE_CMD -f docker-compose.yml up -d)
 fi
 if [ $? -ne 0 ]; then
     print_error "Failed to start Onyx services"
@@ -1057,7 +812,7 @@ echo ""
 # Check for restart loops
 print_info "Checking container health status..."
 RESTART_ISSUES=false
-CONTAINERS=$(cd "${INSTALL_ROOT}/deployment" && $COMPOSE_CMD $(compose_file_args) ps -q 2>/dev/null)
+CONTAINERS=$(cd "${INSTALL_ROOT}/deployment" && $COMPOSE_CMD -f docker-compose.yml ps -q 2>/dev/null)
 
 for CONTAINER in $CONTAINERS; do
     PROJECT_NAME="$(basename "$INSTALL_ROOT")_deployment_"
@@ -1086,7 +841,7 @@ if [ "$RESTART_ISSUES" = true ]; then
     print_error "Some containers are experiencing issues!"
     echo ""
     print_info "Please check the logs for more information:"
-    echo "  (cd \"${INSTALL_ROOT}/deployment\" && $COMPOSE_CMD $(compose_file_args) logs)"
+    echo "  (cd \"${INSTALL_ROOT}/deployment\" && $COMPOSE_CMD -f docker-compose.yml logs)"
 
     echo ""
     print_info "If the issue persists, please contact: founders@onyx.app"
@@ -1105,12 +860,8 @@ check_onyx_health() {
     echo ""
 
     while [ $attempt -le $max_attempts ]; do
-        local http_code=""
-        if [[ "$DOWNLOADER" == "curl" ]]; then
-            http_code=$(curl -s -o /dev/null -w "%{http_code}" "http://localhost:$port" 2>/dev/null || echo "000")
-        else
-            http_code=$(wget -q --spider -S "http://localhost:$port" 2>&1 | grep "HTTP/" | tail -1 | awk '{print $2}' || echo "000")
-        fi
+        # Check for successful HTTP responses (200, 301, 302, etc.)
+        local http_code=$(curl -s -o /dev/null -w "%{http_code}" "http://localhost:$port")
         if echo "$http_code" | grep -qE "^(200|301|302|303|307|308)$"; then
             return 0
         fi
@@ -1166,18 +917,6 @@ print_info "If authentication is enabled, you can create your admin account here
 echo "   • Visit http://localhost:${HOST_PORT}/auth/signup to create your admin account"
 echo "   • The first user created will automatically have admin privileges"
 echo ""
-if [[ "$LITE_MODE" = true ]]; then
-    echo ""
-    print_info "Running in Lite mode — the following services are NOT started:"
-    echo "  • Vespa (vector database)"
-    echo "  • Redis (cache)"
-    echo "  • Model servers (embedding/inference)"
-    echo "  • Background workers (Celery)"
-    echo ""
-    print_info "Connectors and RAG search are disabled. LLM chat, tools, user file"
-    print_info "uploads, Projects, Agent knowledge, and code interpreter still work."
-fi
-echo ""
 print_info "Refer to the README in the ${INSTALL_ROOT} directory for more information."
 echo ""
 print_info "For help or issues, contact: founders@onyx.app"

web/jest.config.js

@@ -144,6 +144,7 @@ module.exports = {
         "**/src/app/**/hooks/*.test.ts", // Pure packet processor tests
         "**/src/refresh-components/**/*.test.ts",
         "**/src/sections/**/*.test.ts",
+        "**/src/components/**/*.test.ts",
         // Add more patterns here as you add more unit tests
       ],
     },

web/lib/opal/src/core/interactive/Interactive.stories.tsx

@@ -1,5 +1,5 @@
 import type { Meta, StoryObj } from "@storybook/react";
-import { Interactive } from "@opal/core";
+import { Interactive, Disabled } from "@opal/core";
 
 // ---------------------------------------------------------------------------
 // Variant / Prominence mappings for the matrix story
@@ -9,8 +9,6 @@ const VARIANT_PROMINENCE_MAP: Record<string, string[]> = {
   default: ["primary", "secondary", "tertiary", "internal"],
   action: ["primary", "secondary", "tertiary", "internal"],
   danger: ["primary", "secondary", "tertiary", "internal"],
-  select: ["light", "heavy"],
-  sidebar: ["light"],
   none: [],
 };
 
@@ -35,39 +33,39 @@ export default meta;
 // Stories
 // ---------------------------------------------------------------------------
 
-/** Basic Interactive.Base + Container with text content. */
+/** Basic Interactive.Stateless + Container with text content. */
 export const Default: StoryObj = {
   render: () => (
     <div style={{ display: "flex", gap: "0.75rem", alignItems: "center" }}>
-      <Interactive.Base
+      <Interactive.Stateless
         variant="default"
         prominence="secondary"
         onClick={() => {}}
       >
         <Interactive.Container border>
-          <span>Secondary</span>
+          <span className="interactive-foreground">Secondary</span>
         </Interactive.Container>
-      </Interactive.Base>
+      </Interactive.Stateless>
 
-      <Interactive.Base
+      <Interactive.Stateless
         variant="default"
         prominence="primary"
         onClick={() => {}}
       >
         <Interactive.Container border>
-          <span>Primary</span>
+          <span className="interactive-foreground">Primary</span>
         </Interactive.Container>
-      </Interactive.Base>
+      </Interactive.Stateless>
 
-      <Interactive.Base
+      <Interactive.Stateless
         variant="default"
         prominence="tertiary"
         onClick={() => {}}
       >
         <Interactive.Container border>
-          <span>Tertiary</span>
+          <span className="interactive-foreground">Tertiary</span>
         </Interactive.Container>
-      </Interactive.Base>
+      </Interactive.Stateless>
     </div>
   ),
 };
@@ -91,11 +89,13 @@ export const VariantMatrix: StoryObj = {
           </div>
 
           {prominences.length === 0 ? (
-            <Interactive.Base variant="none" onClick={() => {}}>
+            <Interactive.Stateless variant="none" onClick={() => {}}>
               <Interactive.Container border>
-                <span>none (no prominence)</span>
+                <span style={{ color: "var(--text-01)" }}>
+                  none (no prominence)
+                </span>
               </Interactive.Container>
-            </Interactive.Base>
+            </Interactive.Stateless>
           ) : (
             <div style={{ display: "flex", gap: "0.5rem", flexWrap: "wrap" }}>
               {prominences.map((prominence) => (
@@ -108,16 +108,18 @@ export const VariantMatrix: StoryObj = {
                     gap: "0.25rem",
                   }}
                 >
-                  <Interactive.Base
+                  <Interactive.Stateless
                     // Cast required because the discriminated union can't be
                     // resolved from dynamic strings at the type level.
                     {...({ variant, prominence } as any)}
                     onClick={() => {}}
                   >
                     <Interactive.Container border>
-                      <span>{prominence}</span>
+                      <span className="interactive-foreground">
+                        {prominence}
+                      </span>
                     </Interactive.Container>
-                  </Interactive.Base>
+                  </Interactive.Stateless>
                   <span
                     style={{
                       fontSize: "0.625rem",
@@ -141,16 +143,16 @@ export const Sizes: StoryObj = {
   render: () => (
     <div style={{ display: "flex", alignItems: "center", gap: "0.75rem" }}>
       {SIZE_VARIANTS.map((size) => (
-        <Interactive.Base
+        <Interactive.Stateless
           key={size}
           variant="default"
           prominence="secondary"
           onClick={() => {}}
         >
           <Interactive.Container border heightVariant={size}>
-            <span>{size}</span>
+            <span className="interactive-foreground">{size}</span>
           </Interactive.Container>
-        </Interactive.Base>
+        </Interactive.Stateless>
       ))}
     </div>
   ),
@@ -160,15 +162,15 @@ export const Sizes: StoryObj = {
 export const WidthFull: StoryObj = {
   render: () => (
     <div style={{ width: 400 }}>
-      <Interactive.Base
+      <Interactive.Stateless
         variant="default"
         prominence="secondary"
         onClick={() => {}}
       >
         <Interactive.Container border widthVariant="full">
-          <span>Full width container</span>
+          <span className="interactive-foreground">Full width container</span>
         </Interactive.Container>
-      </Interactive.Base>
+      </Interactive.Stateless>
     </div>
   ),
 };
@@ -178,73 +180,86 @@ export const Rounding: StoryObj = {
   render: () => (
     <div style={{ display: "flex", gap: "0.75rem" }}>
       {ROUNDING_VARIANTS.map((rounding) => (
-        <Interactive.Base
+        <Interactive.Stateless
           key={rounding}
           variant="default"
           prominence="secondary"
           onClick={() => {}}
         >
           <Interactive.Container border roundingVariant={rounding}>
-            <span>{rounding}</span>
+            <span className="interactive-foreground">{rounding}</span>
           </Interactive.Container>
-        </Interactive.Base>
+        </Interactive.Stateless>
       ))}
     </div>
   ),
 };
 
 /** Disabled state prevents clicks and shows disabled styling. */
-export const Disabled: StoryObj = {
+export const DisabledStory: StoryObj = {
+  name: "Disabled",
   render: () => (
     <div style={{ display: "flex", gap: "0.75rem" }}>
-      <Interactive.Base
-        variant="default"
-        prominence="secondary"
-        onClick={() => {}}
-        disabled
-      >
-        <Interactive.Container border>
-          <span>Disabled</span>
-        </Interactive.Container>
-      </Interactive.Base>
+      <Disabled disabled>
+        <Interactive.Stateless
+          variant="default"
+          prominence="secondary"
+          onClick={() => {}}
+        >
+          <Interactive.Container border>
+            <span className="interactive-foreground">Disabled</span>
+          </Interactive.Container>
+        </Interactive.Stateless>
+      </Disabled>
 
-      <Interactive.Base
+      <Interactive.Stateless
         variant="default"
         prominence="secondary"
         onClick={() => {}}
       >
         <Interactive.Container border>
-          <span>Enabled</span>
+          <span className="interactive-foreground">Enabled</span>
         </Interactive.Container>
-      </Interactive.Base>
+      </Interactive.Stateless>
     </div>
   ),
 };
 
-/** Transient prop forces the hover/active visual state. */
-export const Transient: StoryObj = {
+/** Interaction override forces the hover/active visual state. */
+export const Interaction: StoryObj = {
   render: () => (
     <div style={{ display: "flex", gap: "0.75rem" }}>
-      <Interactive.Base
+      <Interactive.Stateless
         variant="default"
         prominence="secondary"
+        interaction="hover"
         onClick={() => {}}
-        transient
       >
         <Interactive.Container border>
-          <span>Forced hover</span>
+          <span className="interactive-foreground">Forced hover</span>
         </Interactive.Container>
-      </Interactive.Base>
+      </Interactive.Stateless>
 
-      <Interactive.Base
+      <Interactive.Stateless
+        variant="default"
+        prominence="secondary"
+        interaction="active"
+        onClick={() => {}}
+      >
+        <Interactive.Container border>
+          <span className="interactive-foreground">Forced active</span>
+        </Interactive.Container>
+      </Interactive.Stateless>
+
+      <Interactive.Stateless
         variant="default"
         prominence="secondary"
         onClick={() => {}}
       >
         <Interactive.Container border>
-          <span>Normal</span>
+          <span className="interactive-foreground">Normal (rest)</span>
         </Interactive.Container>
-      </Interactive.Base>
+      </Interactive.Stateless>
     </div>
   ),
 };
@@ -253,25 +268,25 @@ export const Transient: StoryObj = {
 export const WithBorder: StoryObj = {
   render: () => (
     <div style={{ display: "flex", gap: "0.75rem" }}>
-      <Interactive.Base
+      <Interactive.Stateless
         variant="default"
         prominence="secondary"
         onClick={() => {}}
       >
         <Interactive.Container border>
-          <span>With border</span>
+          <span className="interactive-foreground">With border</span>
         </Interactive.Container>
-      </Interactive.Base>
+      </Interactive.Stateless>
 
-      <Interactive.Base
+      <Interactive.Stateless
         variant="default"
         prominence="secondary"
         onClick={() => {}}
       >
         <Interactive.Container>
-          <span>Without border</span>
+          <span className="interactive-foreground">Without border</span>
         </Interactive.Container>
-      </Interactive.Base>
+      </Interactive.Stateless>
     </div>
   ),
 };
@@ -279,51 +294,57 @@ export const WithBorder: StoryObj = {
 /** Using href to render as a link. */
 export const AsLink: StoryObj = {
   render: () => (
-    <Interactive.Base variant="action" href="/settings">
+    <Interactive.Stateless variant="action" href="/settings">
       <Interactive.Container border>
-        <span>Go to Settings</span>
+        <span className="interactive-foreground">Go to Settings</span>
       </Interactive.Container>
-    </Interactive.Base>
+    </Interactive.Stateless>
   ),
 };
 
-/** Select variant with selected and unselected states. */
+/** Stateful select variant with selected and unselected states. */
 export const SelectVariant: StoryObj = {
   render: () => (
     <div style={{ display: "flex", gap: "0.75rem" }}>
-      <Interactive.Base
-        variant="select"
-        prominence="light"
-        selected
+      <Interactive.Stateful
+        variant="select-light"
+        state="selected"
         onClick={() => {}}
       >
         <Interactive.Container border>
-          <span>Selected (light)</span>
+          <span className="interactive-foreground">Selected (light)</span>
         </Interactive.Container>
-      </Interactive.Base>
+      </Interactive.Stateful>
 
-      <Interactive.Base variant="select" prominence="light" onClick={() => {}}>
-        <Interactive.Container border>
-          <span>Unselected (light)</span>
-        </Interactive.Container>
-      </Interactive.Base>
-
-      <Interactive.Base
-        variant="select"
-        prominence="heavy"
-        selected
+      <Interactive.Stateful
+        variant="select-light"
+        state="empty"
         onClick={() => {}}
       >
         <Interactive.Container border>
-          <span>Selected (heavy)</span>
+          <span className="interactive-foreground">Unselected (light)</span>
         </Interactive.Container>
-      </Interactive.Base>
+      </Interactive.Stateful>
 
-      <Interactive.Base variant="select" prominence="heavy" onClick={() => {}}>
+      <Interactive.Stateful
+        variant="select-heavy"
+        state="selected"
+        onClick={() => {}}
+      >
         <Interactive.Container border>
-          <span>Unselected (heavy)</span>
+          <span className="interactive-foreground">Selected (heavy)</span>
         </Interactive.Container>
-      </Interactive.Base>
+      </Interactive.Stateful>
+
+      <Interactive.Stateful
+        variant="select-heavy"
+        state="empty"
+        onClick={() => {}}
+      >
+        <Interactive.Container border>
+          <span className="interactive-foreground">Unselected (heavy)</span>
+        </Interactive.Container>
+      </Interactive.Stateful>
     </div>
   ),
 };

web/lib/opal/src/icons/index.ts

@@ -89,7 +89,7 @@ export { default as SvgHistory } from "@opal/icons/history";
 export { default as SvgHourglass } from "@opal/icons/hourglass";
 export { default as SvgImage } from "@opal/icons/image";
 export { default as SvgImageSmall } from "@opal/icons/image-small";
-export { default as SvgImport } from "@opal/icons/import";
+export { default as SvgImport } from "@opal/icons/import-icon";
 export { default as SvgInfo } from "@opal/icons/info";
 export { default as SvgInfoSmall } from "@opal/icons/info-small";
 export { default as SvgKey } from "@opal/icons/key";

web/src/components/chat/MinimalMarkdown.tsx

@@ -11,14 +11,13 @@ import rehypeHighlight from "rehype-highlight";
 import remarkMath from "remark-math";
 import rehypeKatex from "rehype-katex";
 import "katex/dist/katex.min.css";
-import { transformLinkUri } from "@/lib/utils";
+import { cn, transformLinkUri } from "@/lib/utils";
 
 type MinimalMarkdownComponentOverrides = Partial<Components>;
 
 interface MinimalMarkdownProps {
   content: string;
   className?: string;
-  style?: CSSProperties;
   showHeader?: boolean;
   /**
    * Override specific markdown renderers.
@@ -30,7 +29,6 @@ interface MinimalMarkdownProps {
 export default function MinimalMarkdown({
   content,
   className = "",
-  style,
   showHeader = true,
   components,
 }: MinimalMarkdownProps) {
@@ -63,19 +61,17 @@ export default function MinimalMarkdown({
   }, [content, components, showHeader]);
 
   return (
-    <div style={style || {}} className={`${className}`}>
-      <ReactMarkdown
-        className="prose dark:prose-invert max-w-full text-sm break-words"
-        components={markdownComponents}
-        rehypePlugins={[rehypeHighlight, rehypeKatex]}
-        remarkPlugins={[
-          remarkGfm,
-          [remarkMath, { singleDollarTextMath: false }],
-        ]}
-        urlTransform={transformLinkUri}
-      >
-        {content}
-      </ReactMarkdown>
-    </div>
+    <ReactMarkdown
+      className={cn(
+        "prose dark:prose-invert max-w-full text-sm break-words",
+        className
+      )}
+      components={markdownComponents}
+      rehypePlugins={[rehypeHighlight, rehypeKatex]}
+      remarkPlugins={[remarkGfm, [remarkMath, { singleDollarTextMath: false }]]}
+      urlTransform={transformLinkUri}
+    >
+      {content}
+    </ReactMarkdown>
   );
 }

web/src/components/tools/CSVContent.tsx

@@ -60,27 +60,28 @@ const CsvContent: React.FC<ContentComponentProps> = ({
       }
 
       const csvData = await response.text();
-      const rows = csvData.trim().split("\n");
+      const rows = parseCSV(csvData.trim());
       const firstRow = rows[0];
       if (!firstRow) {
         throw new Error("CSV file is empty");
       }
-      const parsedHeaders = firstRow.split(",");
+      const parsedHeaders = firstRow;
       setHeaders(parsedHeaders);
 
-      const parsedData: Record<string, string>[] = rows.slice(1).map((row) => {
-        const values = row.split(",");
-        return parsedHeaders.reduce<Record<string, string>>(
-          (obj, header, index) => {
-            const val = values[index];
-            if (val !== undefined) {
-              obj[header] = val;
-            }
-            return obj;
-          },
-          {}
-        );
-      });
+      const parsedData: Record<string, string>[] = rows
+        .slice(1)
+        .map((fields) => {
+          return parsedHeaders.reduce<Record<string, string>>(
+            (obj, header, index) => {
+              const val = fields[index];
+              if (val !== undefined) {
+                obj[header] = val;
+              }
+              return obj;
+            },
+            {}
+          );
+        });
       setData(parsedData);
       csvCache.set(id, { headers: parsedHeaders, data: parsedData });
     } catch (error) {
@@ -173,3 +174,53 @@ const csvCache = new Map<
   string,
   { headers: string[]; data: Record<string, string>[] }
 >();
+
+export function parseCSV(text: string): string[][] {
+  const rows: string[][] = [];
+  let field = "";
+  let fields: string[] = [];
+  let inQuotes = false;
+
+  for (let i = 0; i < text.length; i++) {
+    const char = text[i];
+
+    if (inQuotes) {
+      if (char === '"') {
+        if (i + 1 < text.length && text[i + 1] === '"') {
+          field += '"';
+          i++;
+        } else {
+          inQuotes = false;
+        }
+      } else {
+        field += char;
+      }
+    } else if (char === '"') {
+      inQuotes = true;
+    } else if (char === ",") {
+      fields.push(field);
+      field = "";
+    } else if (char === "\n" || char === "\r") {
+      if (char === "\r" && i + 1 < text.length && text[i + 1] === "\n") {
+        i++;
+      }
+      fields.push(field);
+      field = "";
+      rows.push(fields);
+      fields = [];
+    } else {
+      field += char;
+    }
+  }
+
+  if (inQuotes) {
+    throw new Error("Malformed CSV: unterminated quoted field");
+  }
+
+  if (field.length > 0 || fields.length > 0) {
+    fields.push(field);
+    rows.push(fields);
+  }
+
+  return rows;
+}

web/src/components/tools/parseCSV.test.ts

@@ -0,0 +1,84 @@
+import { parseCSV } from "./CSVContent";
+
+describe("parseCSV", () => {
+  it("parses simple comma-separated rows", () => {
+    expect(parseCSV("a,b,c\n1,2,3")).toEqual([
+      ["a", "b", "c"],
+      ["1", "2", "3"],
+    ]);
+  });
+
+  it("preserves commas inside quoted fields", () => {
+    expect(parseCSV('name,address\nAlice,"123 Main St, Apt 4"')).toEqual([
+      ["name", "address"],
+      ["Alice", "123 Main St, Apt 4"],
+    ]);
+  });
+
+  it("handles escaped double quotes inside quoted fields", () => {
+    expect(parseCSV('a,b\n"say ""hello""",world')).toEqual([
+      ["a", "b"],
+      ['say "hello"', "world"],
+    ]);
+  });
+
+  it("handles newlines inside quoted fields", () => {
+    expect(parseCSV('a,b\n"line1\nline2",val')).toEqual([
+      ["a", "b"],
+      ["line1\nline2", "val"],
+    ]);
+  });
+
+  it("handles CRLF line endings", () => {
+    expect(parseCSV("a,b\r\n1,2\r\n3,4")).toEqual([
+      ["a", "b"],
+      ["1", "2"],
+      ["3", "4"],
+    ]);
+  });
+
+  it("handles empty fields", () => {
+    expect(parseCSV("a,b,c\n1,,3")).toEqual([
+      ["a", "b", "c"],
+      ["1", "", "3"],
+    ]);
+  });
+
+  it("handles a single element", () => {
+    expect(parseCSV("a")).toEqual([["a"]]);
+  });
+
+  it("handles a single row with no newline", () => {
+    expect(parseCSV("a,b,c")).toEqual([["a", "b", "c"]]);
+  });
+
+  it("handles quoted fields that are entirely empty", () => {
+    expect(parseCSV('a,b\n"",val')).toEqual([
+      ["a", "b"],
+      ["", "val"],
+    ]);
+  });
+
+  it("handles multiple quoted fields with commas", () => {
+    expect(parseCSV('"foo, bar","baz, qux"\n"1, 2","3, 4"')).toEqual([
+      ["foo, bar", "baz, qux"],
+      ["1, 2", "3, 4"],
+    ]);
+  });
+
+  it("throws on unterminated quoted field", () => {
+    expect(() => parseCSV('a,b\n"foo,bar')).toThrow(
+      "Malformed CSV: unterminated quoted field"
+    );
+  });
+
+  it("throws on unterminated quote at end of input", () => {
+    expect(() => parseCSV('"unterminated')).toThrow(
+      "Malformed CSV: unterminated quoted field"
+    );
+  });
+
+  it("returns empty array for empty input", () => {
+    expect(parseCSV("")).toEqual([]);
+  });
+});

web/src/hooks/useContainerCenter.ts

@@ -38,7 +38,7 @@ function measure(el: HTMLElement): { x: number; y: number } | null {
  */
 export default function useContainerCenter(): ContainerCenter {
   const pathname = usePathname();
-  const { isSmallScreen } = useScreenSize();
+  const { isMediumScreen } = useScreenSize();
   const [center, setCenter] = useState<{ x: number | null; y: number | null }>(
     () => {
       if (typeof document === "undefined") return NULL_CENTER;
@@ -68,9 +68,9 @@ export default function useContainerCenter(): ContainerCenter {
   }, [pathname]);
 
   return {
-    centerX: isSmallScreen ? null : center.x,
-    centerY: isSmallScreen ? null : center.y,
-    hasContainerCenter: isSmallScreen
+    centerX: isMediumScreen ? null : center.x,
+    centerY: isMediumScreen ? null : center.y,
+    hasContainerCenter: isMediumScreen
       ? false
       : center.x !== null && center.y !== null,
   };

web/src/hooks/useScreenSize.ts

@@ -2,6 +2,7 @@
 
 import {
   DESKTOP_SMALL_BREAKPOINT_PX,
+  DESKTOP_MEDIUM_BREAKPOINT_PX,
   MOBILE_SIDEBAR_BREAKPOINT_PX,
 } from "@/lib/constants";
 import { useState, useCallback } from "react";
@@ -12,6 +13,7 @@ export interface ScreenSize {
   width: number;
   isMobile: boolean;
   isSmallScreen: boolean;
+  isMediumScreen: boolean;
 }
 
 export default function useScreenSize(): ScreenSize {
@@ -34,11 +36,13 @@ export default function useScreenSize(): ScreenSize {
 
   const isMobile = sizes.width <= MOBILE_SIDEBAR_BREAKPOINT_PX;
   const isSmall = sizes.width <= DESKTOP_SMALL_BREAKPOINT_PX;
+  const isMedium = sizes.width <= DESKTOP_MEDIUM_BREAKPOINT_PX;
 
   return {
     height: sizes.height,
     width: sizes.width,
     isMobile: isMounted && isMobile,
     isSmallScreen: isMounted && isSmall,
+    isMediumScreen: isMounted && isMedium,
   };
 }

web/src/lib/constants.ts

@@ -123,6 +123,7 @@ export const MAX_FILES_TO_SHOW = 3;
 // SIZES
 export const MOBILE_SIDEBAR_BREAKPOINT_PX = 640;
 export const DESKTOP_SMALL_BREAKPOINT_PX = 912;
+export const DESKTOP_MEDIUM_BREAKPOINT_PX = 1232;
 export const DEFAULT_AGENT_AVATAR_SIZE_PX = 18;
 export const HORIZON_DISTANCE_PX = 800;
 export const LOGO_FOLDED_SIZE_PX = 24;

web/src/refresh-components/ScrollIndicatorDiv.tsx

@@ -42,6 +42,12 @@ export default function ScrollIndicatorDiv({
     if (!container) return;
 
     const { scrollTop, scrollHeight, clientHeight } = container;
+    console.log(
+      "scrollHeight: ",
+      scrollHeight,
+      " clientHeight: ",
+      clientHeight
+    );
     const isScrollable = scrollHeight > clientHeight;
 
     // Show top indicator if scrolled down from top

web/src/refresh-components/buttons/ButtonRenaming.stories.tsx

@@ -2,6 +2,8 @@ import React from "react";
 import type { Meta, StoryObj } from "@storybook/react";
 import ButtonRenaming from "./ButtonRenaming";
 
+const noop = () => {};
+
 const meta: Meta<typeof ButtonRenaming> = {
   title: "refresh-components/buttons/ButtonRenaming",
   component: ButtonRenaming,
@@ -28,35 +30,23 @@ type Story = StoryObj<typeof ButtonRenaming>;
 export const Default: Story = {
   args: {
     initialName: "My Chat Session",
-    onRename: async (name: string) => {
-      console.log("Renamed to:", name);
-    },
-    onClose: () => {
-      console.log("Closed");
-    },
+    onRename: async () => {},
+    onClose: noop,
   },
 };
 
 export const EmptyName: Story = {
   args: {
     initialName: null,
-    onRename: async (name: string) => {
-      console.log("Renamed to:", name);
-    },
-    onClose: () => {
-      console.log("Closed");
-    },
+    onRename: async () => {},
+    onClose: noop,
   },
 };
 
 export const LongName: Story = {
   args: {
     initialName: "This is a very long chat session name that should overflow",
-    onRename: async (name: string) => {
-      console.log("Renamed to:", name);
-    },
-    onClose: () => {
-      console.log("Closed");
-    },
+    onRename: async () => {},
+    onClose: noop,
   },
 };

web/src/sections/modals/PreviewModal/PreviewModal.tsx

@@ -167,7 +167,7 @@ export default function PreviewModal({
         />
 
         {/* Body + floating footer wrapper */}
-        <Modal.Body padding={0} gap={0}>
+        <Modal.Body padding={0} gap={0} height="full">
           <Section padding={0} gap={0}>
             {isLoading ? (
               <Section>

web/src/sections/modals/PreviewModal/variants/CodePreview.tsx

@@ -1,22 +1,36 @@
 "use client";
 
 import MinimalMarkdown from "@/components/chat/MinimalMarkdown";
+import Spacer from "@/refresh-components/Spacer";
+import ScrollIndicatorDiv from "@/refresh-components/ScrollIndicatorDiv";
 import "@/app/app/message/custom-code-styles.css";
 
 interface CodePreviewProps {
   content: string;
   language?: string | null;
+  normalize?: boolean;
 }
 
-export function CodePreview({ content, language }: CodePreviewProps) {
-  const normalizedContent = content.replace(/~~~/g, "\\~\\~\\~");
-  const fenceHeader = language ? `~~~${language}` : "~~~";
+export function CodePreview({
+  content,
+  language,
+  normalize,
+}: CodePreviewProps) {
+  const markdownContent = normalize
+    ? `~~~${language || ""}\n${content.replace(/~~~/g, "\\~\\~\\~")}\n~~~`
+    : content;
 
   return (
-    <MinimalMarkdown
-      content={`${fenceHeader}\n${normalizedContent}\n\n~~~`}
-      className="w-full h-full"
-      showHeader={false}
-    />
+    <ScrollIndicatorDiv
+      className="h-full bg-background-code-01"
+      backgroundColor="var(--background-code-01)"
+      variant="shadow"
+    >
+      <MinimalMarkdown
+        content={markdownContent}
+        className="w-full h-full p-4 pb-8"
+        showHeader={false}
+      />
+    </ScrollIndicatorDiv>
   );
 }

web/src/sections/modals/PreviewModal/variants/codeVariant.tsx

@@ -22,7 +22,7 @@ export const codeVariant: PreviewVariant = {
       : "",
 
   renderContent: (ctx) => (
-    <CodePreview content={ctx.fileContent} language={ctx.language} />
+    <CodePreview normalize content={ctx.fileContent} language={ctx.language} />
   ),
 
   renderFooterLeft: (ctx) => (

web/src/sections/modals/PreviewModal/variants/dataVariant.tsx

@@ -36,7 +36,9 @@ export const dataVariant: PreviewVariant = {
 
   renderContent: (ctx) => {
     const formatted = formatContent(ctx.language, ctx.fileContent);
-    return <CodePreview content={formatted} language={ctx.language} />;
+    return (
+      <CodePreview normalize content={formatted} language={ctx.language} />
+    );
   },
 
   renderFooterLeft: (ctx) => (

web/src/sections/modals/PreviewModal/variants/markdownVariant.tsx

@@ -1,8 +1,7 @@
-import MinimalMarkdown from "@/components/chat/MinimalMarkdown";
-import ScrollIndicatorDiv from "@/refresh-components/ScrollIndicatorDiv";
 import { Section } from "@/layouts/general-layouts";
 import { isMarkdownFile } from "@/lib/languages";
 import { PreviewVariant } from "@/sections/modals/PreviewModal/interfaces";
+import { CodePreview } from "@/sections/modals/PreviewModal/variants/CodePreview";
 import {
   CopyButton,
   DownloadButton,
@@ -26,12 +25,7 @@ export const markdownVariant: PreviewVariant = {
   headerDescription: () => "",
 
   renderContent: (ctx) => (
-    <ScrollIndicatorDiv className="flex-1 min-h-0 p-4" variant="shadow">
-      <MinimalMarkdown
-        content={ctx.fileContent}
-        className="w-full pb-4 text-lg break-words"
-      />
-    </ScrollIndicatorDiv>
+    <CodePreview content={ctx.fileContent} language={ctx.language} />
   ),
 
   renderFooterLeft: () => null,

web/src/sections/modals/PreviewModal/variants/textVariant.tsx

@@ -36,7 +36,7 @@ export const textVariant: PreviewVariant = {
       : "",
 
   renderContent: (ctx) => (
-    <CodePreview content={ctx.fileContent} language={ctx.language} />
+    <CodePreview normalize content={ctx.fileContent} language={ctx.language} />
   ),
 
   renderFooterLeft: (ctx) => (

web/tailwind-themes/tailwind.config.js

@@ -260,6 +260,7 @@ module.exports = {
         "code-string": "var(--code-string)",
         "code-number": "var(--code-number)",
         "code-definition": "var(--code-definition)",
+        "background-code-01": "var(--background-code-01)",
 
         // Shimmer colors for loading animations
         "shimmer-base": "var(--shimmer-base)",