build org

.github/workflows/pr-python-connector-tests.yml

@@ -24,8 +24,6 @@ env:
   GOOGLE_DRIVE_OAUTH_CREDENTIALS_JSON_STR: ${{ secrets.GOOGLE_DRIVE_OAUTH_CREDENTIALS_JSON_STR }}
   GOOGLE_GMAIL_SERVICE_ACCOUNT_JSON_STR: ${{ secrets.GOOGLE_GMAIL_SERVICE_ACCOUNT_JSON_STR }}
   GOOGLE_GMAIL_OAUTH_CREDENTIALS_JSON_STR: ${{ secrets.GOOGLE_GMAIL_OAUTH_CREDENTIALS_JSON_STR }}
-  # Slab
-  SLAB_BOT_TOKEN: ${{ secrets.SLAB_BOT_TOKEN }}
 
 jobs:
   connectors-check:

backend/Dockerfile

@@ -73,7 +73,6 @@ RUN apt-get update && \
     rm -rf /var/lib/apt/lists/* && \
     rm -f /usr/local/lib/python3.11/site-packages/tornado/test/test.key
 
-
 # Pre-downloading models for setups with limited egress
 RUN python -c "from tokenizers import Tokenizer; \
 Tokenizer.from_pretrained('nomic-ai/nomic-embed-text-v1')"

backend/danswer/configs/app_configs.py

@@ -493,6 +493,10 @@ CONTROL_PLANE_API_BASE_URL = os.environ.get(
 # JWT configuration
 JWT_ALGORITHM = "HS256"
 
+# Super Users
+SUPER_USERS = json.loads(os.environ.get("SUPER_USERS", '["pablo@danswer.ai"]'))
+SUPER_CLOUD_API_KEY = os.environ.get("SUPER_CLOUD_API_KEY", "api_key")
+
 
 #####
 # API Key Configs

backend/danswer/connectors/README.md

@@ -11,16 +11,11 @@ Connectors come in 3 different flows:
 - Load Connector:
   - Bulk indexes documents to reflect a point in time. This type of connector generally works by either pulling all
   documents via a connector's API or loads the documents from some sort of a dump file.
-- Poll Connector:
+- Poll connector:
   - Incrementally updates documents based on a provided time range. It is used by the background job to pull the latest
   changes and additions since the last round of polling. This connector helps keep the document index up to date
   without needing to fetch/embed/index every document which would be too slow to do frequently on large sets of
   documents.
-- Slim Connector:
-  - This connector should be a lighter weight method of checking all documents in the source to see if they still exist.
-  - This connector should be identical to the Poll or Load Connector except that it only fetches the IDs of the documents, not the documents themselves.
-  - This is used by our pruning job which removes old documents from the index.
-  - The optional start and end datetimes can be ignored.
 - Event Based connectors:
   - Connectors that listen to events and update documents accordingly.
   - Currently not used by the background job, this exists for future design purposes.
@@ -31,14 +26,8 @@ Refer to [interfaces.py](https://github.com/danswer-ai/danswer/blob/main/backend
 and this first contributor created Pull Request for a new connector (Shoutout to Dan Brown):
 [Reference Pull Request](https://github.com/danswer-ai/danswer/pull/139)
 
-For implementing a Slim Connector, refer to the comments in this PR:
-[Slim Connector PR](https://github.com/danswer-ai/danswer/pull/3303/files)
-
-All new connectors should have tests added to the `backend/tests/daily/connectors` directory. Refer to the above PR for an example of adding tests for a new connector.
-
-
 #### Implementing the new Connector
-The connector must subclass one or more of LoadConnector, PollConnector, SlimConnector, or EventConnector.
+The connector must subclass one or more of LoadConnector, PollConnector, or EventConnector.
 
 The `__init__` should take arguments for configuring what documents the connector will and where it finds those
 documents. For example, if you have a wiki site, it may include the configuration for the team, topic, folder, etc. of

backend/danswer/connectors/slab/connector.py

@@ -12,15 +12,12 @@ from dateutil import parser
 from danswer.configs.app_configs import INDEX_BATCH_SIZE
 from danswer.configs.constants import DocumentSource
 from danswer.connectors.interfaces import GenerateDocumentsOutput
-from danswer.connectors.interfaces import GenerateSlimDocumentOutput
 from danswer.connectors.interfaces import LoadConnector
 from danswer.connectors.interfaces import PollConnector
 from danswer.connectors.interfaces import SecondsSinceUnixEpoch
-from danswer.connectors.interfaces import SlimConnector
 from danswer.connectors.models import ConnectorMissingCredentialError
 from danswer.connectors.models import Document
 from danswer.connectors.models import Section
-from danswer.connectors.models import SlimDocument
 from danswer.utils.logger import setup_logger
 
 
@@ -31,8 +28,6 @@ logger = setup_logger()
 SLAB_GRAPHQL_MAX_TRIES = 10
 SLAB_API_URL = "https://api.slab.com/v1/graphql"
 
-_SLIM_BATCH_SIZE = 1000
-
 
 def run_graphql_request(
     graphql_query: dict, bot_token: str, max_tries: int = SLAB_GRAPHQL_MAX_TRIES
@@ -163,26 +158,21 @@ def get_slab_url_from_title_id(base_url: str, title: str, page_id: str) -> str:
     return urljoin(urljoin(base_url, "posts/"), url_id)
 
 
-class SlabConnector(LoadConnector, PollConnector, SlimConnector):
+class SlabConnector(LoadConnector, PollConnector):
     def __init__(
         self,
         base_url: str,
         batch_size: int = INDEX_BATCH_SIZE,
+        slab_bot_token: str | None = None,
     ) -> None:
         self.base_url = base_url
         self.batch_size = batch_size
-        self._slab_bot_token: str | None = None
+        self.slab_bot_token = slab_bot_token
 
     def load_credentials(self, credentials: dict[str, Any]) -> dict[str, Any] | None:
-        self._slab_bot_token = credentials["slab_bot_token"]
+        self.slab_bot_token = credentials["slab_bot_token"]
         return None
 
-    @property
-    def slab_bot_token(self) -> str:
-        if self._slab_bot_token is None:
-            raise ConnectorMissingCredentialError("Slab")
-        return self._slab_bot_token
-
     def _iterate_posts(
         self, time_filter: Callable[[datetime], bool] | None = None
     ) -> GenerateDocumentsOutput:
@@ -237,21 +227,3 @@ class SlabConnector(LoadConnector, PollConnector, SlimConnector):
         yield from self._iterate_posts(
             time_filter=lambda t: start_time <= t <= end_time
         )
-
-    def retrieve_all_slim_documents(
-        self,
-        start: SecondsSinceUnixEpoch | None = None,
-        end: SecondsSinceUnixEpoch | None = None,
-    ) -> GenerateSlimDocumentOutput:
-        slim_doc_batch: list[SlimDocument] = []
-        for post_id in get_all_post_ids(self.slab_bot_token):
-            slim_doc_batch.append(
-                SlimDocument(
-                    id=post_id,
-                )
-            )
-            if len(slim_doc_batch) >= _SLIM_BATCH_SIZE:
-                yield slim_doc_batch
-                slim_doc_batch = []
-        if slim_doc_batch:
-            yield slim_doc_batch

backend/danswer/file_store/file_store.py

@@ -59,12 +59,6 @@ class FileStore(ABC):
             Contents of the file and metadata dict
         """
 
-    @abstractmethod
-    def read_file_record(self, file_name: str) -> PGFileStore:
-        """
-        Read the file record by the name
-        """
-
     @abstractmethod
     def delete_file(self, file_name: str) -> None:
         """

backend/danswer/llm/answering/stream_processing/citation_processing.py

@@ -67,9 +67,9 @@ class CitationProcessor:
                 if piece_that_comes_after == "\n" and in_code_block(self.llm_out):
                     self.curr_segment = self.curr_segment.replace("```", "```plaintext")
 
-        citation_pattern = r"\[(\d+)\]|\[\[(\d+)\]\]"  # [1], [[1]], etc.
+        citation_pattern = r"\[(\d+)\]"
         citations_found = list(re.finditer(citation_pattern, self.curr_segment))
-        possible_citation_pattern = r"(\[+\d*$)"  # [1, [, [[, [[2, etc.
+        possible_citation_pattern = r"(\[\d*$)"  # [1, [, etc
         possible_citation_found = re.search(
             possible_citation_pattern, self.curr_segment
         )
@@ -77,15 +77,13 @@ class CitationProcessor:
         if len(citations_found) == 0 and len(self.llm_out) - self.past_cite_count > 5:
             self.current_citations = []
 
-        result = ""
+        result = ""  # Initialize result here
         if citations_found and not in_code_block(self.llm_out):
             last_citation_end = 0
             length_to_add = 0
             while len(citations_found) > 0:
                 citation = citations_found.pop(0)
-                numerical_value = int(
-                    next(group for group in citation.groups() if group is not None)
-                )
+                numerical_value = int(citation.group(1))
 
                 if 1 <= numerical_value <= self.max_citation_num:
                     context_llm_doc = self.context_docs[numerical_value - 1]
@@ -133,6 +131,14 @@ class CitationProcessor:
 
                     link = context_llm_doc.link
 
+                    # Replace the citation in the current segment
+                    start, end = citation.span()
+                    self.curr_segment = (
+                        self.curr_segment[: start + length_to_add]
+                        + f"[{target_citation_num}]"
+                        + self.curr_segment[end + length_to_add :]
+                    )
+
                     self.past_cite_count = len(self.llm_out)
                     self.current_citations.append(target_citation_num)
 
@@ -143,7 +149,6 @@ class CitationProcessor:
                             document_id=context_llm_doc.document_id,
                         )
 
-                    start, end = citation.span()
                     if link:
                         prev_length = len(self.curr_segment)
                         self.curr_segment = (

backend/danswer/server/manage/users.py

@@ -34,6 +34,7 @@ from danswer.auth.users import optional_user
 from danswer.configs.app_configs import AUTH_TYPE
 from danswer.configs.app_configs import ENABLE_EMAIL_INVITES
 from danswer.configs.app_configs import SESSION_EXPIRE_TIME_SECONDS
+from danswer.configs.app_configs import SUPER_USERS
 from danswer.configs.app_configs import VALID_EMAIL_DOMAINS
 from danswer.configs.constants import AuthType
 from danswer.db.api_key import is_api_key_email_address
@@ -63,7 +64,6 @@ from danswer.server.models import MinimalUserSnapshot
 from danswer.server.utils import send_user_email_invite
 from danswer.utils.logger import setup_logger
 from danswer.utils.variable_functionality import fetch_ee_implementation_or_noop
-from ee.danswer.configs.app_configs import SUPER_USERS
 from shared_configs.configs import MULTI_TENANT
 
 logger = setup_logger()

backend/danswer/server/query_and_chat/chat_backend.py

@@ -707,18 +707,14 @@ def upload_files_for_chat(
     }
 
 
-@router.get("/file/{file_id:path}")
+@router.get("/file/{file_id}")
 def fetch_chat_file(
     file_id: str,
     db_session: Session = Depends(get_session),
     _: User | None = Depends(current_user),
 ) -> Response:
     file_store = get_default_file_store(db_session)
-    file_record = file_store.read_file_record(file_id)
-    if not file_record:
-        raise HTTPException(status_code=404, detail="File not found")
-
-    media_type = file_record.file_type
     file_io = file_store.read_file(file_id, mode="b")
-
-    return StreamingResponse(file_io, media_type=media_type)
+    # NOTE: specifying "image/jpeg" here, but it still works for pngs
+    # TODO: do this properly
+    return Response(content=file_io.read(), media_type="image/jpeg")

backend/ee/danswer/auth/users.py

@@ -1,72 +1,23 @@
-from functools import lru_cache
-
-import requests
 from fastapi import Depends
 from fastapi import HTTPException
 from fastapi import Request
 from fastapi import status
-from jwt import decode as jwt_decode
-from jwt import InvalidTokenError
-from jwt import PyJWTError
-from sqlalchemy import func
-from sqlalchemy import select
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from danswer.auth.users import current_admin_user
 from danswer.configs.app_configs import AUTH_TYPE
+from danswer.configs.app_configs import SUPER_CLOUD_API_KEY
+from danswer.configs.app_configs import SUPER_USERS
 from danswer.configs.constants import AuthType
 from danswer.db.models import User
 from danswer.utils.logger import setup_logger
-from ee.danswer.configs.app_configs import JWT_PUBLIC_KEY_URL
-from ee.danswer.configs.app_configs import SUPER_CLOUD_API_KEY
-from ee.danswer.configs.app_configs import SUPER_USERS
 from ee.danswer.db.saml import get_saml_account
 from ee.danswer.server.seeding import get_seed_config
 from ee.danswer.utils.secrets import extract_hashed_cookie
 
-
 logger = setup_logger()
 
 
-@lru_cache()
-def get_public_key() -> str | None:
-    if JWT_PUBLIC_KEY_URL is None:
-        logger.error("JWT_PUBLIC_KEY_URL is not set")
-        return None
-
-    response = requests.get(JWT_PUBLIC_KEY_URL)
-    response.raise_for_status()
-    return response.text
-
-
-async def verify_jwt_token(token: str, async_db_session: AsyncSession) -> User | None:
-    try:
-        public_key_pem = get_public_key()
-        if public_key_pem is None:
-            logger.error("Failed to retrieve public key")
-            return None
-
-        payload = jwt_decode(
-            token,
-            public_key_pem,
-            algorithms=["RS256"],
-            audience=None,
-        )
-        email = payload.get("email")
-        if email:
-            result = await async_db_session.execute(
-                select(User).where(func.lower(User.email) == func.lower(email))
-            )
-            return result.scalars().first()
-    except InvalidTokenError:
-        logger.error("Invalid JWT token")
-        get_public_key.cache_clear()
-    except PyJWTError as e:
-        logger.error(f"JWT decoding error: {str(e)}")
-        get_public_key.cache_clear()
-    return None
-
-
 def verify_auth_setting() -> None:
     # All the Auth flows are valid for EE version
     logger.notice(f"Using Auth Type: {AUTH_TYPE.value}")
@@ -87,13 +38,6 @@ async def optional_user_(
             )
             user = saml_account.user if saml_account else None
 
-    # If user is still None, check for JWT in Authorization header
-    if user is None and JWT_PUBLIC_KEY_URL is not None:
-        auth_header = request.headers.get("Authorization")
-        if auth_header and auth_header.startswith("Bearer "):
-            token = auth_header[len("Bearer ") :].strip()
-            user = await verify_jwt_token(token, async_db_session)
-
     return user
 
 

backend/ee/danswer/configs/app_configs.py

@@ -1,4 +1,3 @@
-import json
 import os
 
 # Applicable for OIDC Auth
@@ -20,11 +19,3 @@ STRIPE_PRICE_ID = os.environ.get("STRIPE_PRICE")
 OPENAI_DEFAULT_API_KEY = os.environ.get("OPENAI_DEFAULT_API_KEY")
 ANTHROPIC_DEFAULT_API_KEY = os.environ.get("ANTHROPIC_DEFAULT_API_KEY")
 COHERE_DEFAULT_API_KEY = os.environ.get("COHERE_DEFAULT_API_KEY")
-
-# JWT Public Key URL
-JWT_PUBLIC_KEY_URL: str | None = os.getenv("JWT_PUBLIC_KEY_URL", None)
-
-
-# Super Users
-SUPER_USERS = json.loads(os.environ.get("SUPER_USERS", '["pablo@danswer.ai"]'))
-SUPER_CLOUD_API_KEY = os.environ.get("SUPER_CLOUD_API_KEY", "api_key")

backend/shared_configs/configs.py

@@ -163,92 +163,47 @@ SUPPORTED_EMBEDDING_MODELS = [
         dim=1024,
         index_name="danswer_chunk_cohere_embed_english_v3_0",
     ),
-    SupportedEmbeddingModel(
-        name="cohere/embed-english-v3.0",
-        dim=1024,
-        index_name="danswer_chunk_embed_english_v3_0",
-    ),
     SupportedEmbeddingModel(
         name="cohere/embed-english-light-v3.0",
         dim=384,
         index_name="danswer_chunk_cohere_embed_english_light_v3_0",
     ),
-    SupportedEmbeddingModel(
-        name="cohere/embed-english-light-v3.0",
-        dim=384,
-        index_name="danswer_chunk_embed_english_light_v3_0",
-    ),
     SupportedEmbeddingModel(
         name="openai/text-embedding-3-large",
         dim=3072,
         index_name="danswer_chunk_openai_text_embedding_3_large",
     ),
-    SupportedEmbeddingModel(
-        name="openai/text-embedding-3-large",
-        dim=3072,
-        index_name="danswer_chunk_text_embedding_3_large",
-    ),
     SupportedEmbeddingModel(
         name="openai/text-embedding-3-small",
         dim=1536,
         index_name="danswer_chunk_openai_text_embedding_3_small",
     ),
-    SupportedEmbeddingModel(
-        name="openai/text-embedding-3-small",
-        dim=1536,
-        index_name="danswer_chunk_text_embedding_3_small",
-    ),
     SupportedEmbeddingModel(
         name="google/text-embedding-004",
         dim=768,
         index_name="danswer_chunk_google_text_embedding_004",
     ),
-    SupportedEmbeddingModel(
-        name="google/text-embedding-004",
-        dim=768,
-        index_name="danswer_chunk_text_embedding_004",
-    ),
     SupportedEmbeddingModel(
         name="google/textembedding-gecko@003",
         dim=768,
         index_name="danswer_chunk_google_textembedding_gecko_003",
     ),
-    SupportedEmbeddingModel(
-        name="google/textembedding-gecko@003",
-        dim=768,
-        index_name="danswer_chunk_textembedding_gecko_003",
-    ),
     SupportedEmbeddingModel(
         name="voyage/voyage-large-2-instruct",
         dim=1024,
         index_name="danswer_chunk_voyage_large_2_instruct",
     ),
-    SupportedEmbeddingModel(
-        name="voyage/voyage-large-2-instruct",
-        dim=1024,
-        index_name="danswer_chunk_large_2_instruct",
-    ),
     SupportedEmbeddingModel(
         name="voyage/voyage-light-2-instruct",
         dim=384,
         index_name="danswer_chunk_voyage_light_2_instruct",
     ),
-    SupportedEmbeddingModel(
-        name="voyage/voyage-light-2-instruct",
-        dim=384,
-        index_name="danswer_chunk_light_2_instruct",
-    ),
     # Self-hosted models
     SupportedEmbeddingModel(
         name="nomic-ai/nomic-embed-text-v1",
         dim=768,
         index_name="danswer_chunk_nomic_ai_nomic_embed_text_v1",
     ),
-    SupportedEmbeddingModel(
-        name="nomic-ai/nomic-embed-text-v1",
-        dim=768,
-        index_name="danswer_chunk_nomic_embed_text_v1",
-    ),
     SupportedEmbeddingModel(
         name="intfloat/e5-base-v2",
         dim=768,

backend/tests/daily/connectors/slab/test_slab_connector.py

@@ -1,88 +0,0 @@
-import json
-import os
-import time
-from pathlib import Path
-
-import pytest
-
-from danswer.configs.constants import DocumentSource
-from danswer.connectors.models import Document
-from danswer.connectors.slab.connector import SlabConnector
-
-
-def load_test_data(file_name: str = "test_slab_data.json") -> dict[str, str]:
-    current_dir = Path(__file__).parent
-    with open(current_dir / file_name, "r") as f:
-        return json.load(f)
-
-
-@pytest.fixture
-def slab_connector() -> SlabConnector:
-    connector = SlabConnector(
-        base_url="https://onyx-test.slab.com/",
-    )
-    connector.load_credentials(
-        {
-            "slab_bot_token": os.environ["SLAB_BOT_TOKEN"],
-        }
-    )
-    return connector
-
-
-@pytest.mark.xfail(
-    reason=(
-        "Need a test account with a slab subscription to run this test."
-        "Trial only lasts 14 days."
-    )
-)
-def test_slab_connector_basic(slab_connector: SlabConnector) -> None:
-    all_docs: list[Document] = []
-    target_test_doc_id = "jcp6cohu"
-    target_test_doc: Document | None = None
-    for doc_batch in slab_connector.poll_source(0, time.time()):
-        for doc in doc_batch:
-            all_docs.append(doc)
-            if doc.id == target_test_doc_id:
-                target_test_doc = doc
-
-    assert len(all_docs) == 6
-    assert target_test_doc is not None
-
-    desired_test_data = load_test_data()
-    assert (
-        target_test_doc.semantic_identifier == desired_test_data["semantic_identifier"]
-    )
-    assert target_test_doc.source == DocumentSource.SLAB
-    assert target_test_doc.metadata == {}
-    assert target_test_doc.primary_owners is None
-    assert target_test_doc.secondary_owners is None
-    assert target_test_doc.title is None
-    assert target_test_doc.from_ingestion_api is False
-    assert target_test_doc.additional_info is None
-
-    assert len(target_test_doc.sections) == 1
-    section = target_test_doc.sections[0]
-    # Need to replace the weird apostrophe with a normal one
-    assert section.text.replace("\u2019", "'") == desired_test_data["section_text"]
-    assert section.link == desired_test_data["link"]
-
-
-@pytest.mark.xfail(
-    reason=(
-        "Need a test account with a slab subscription to run this test."
-        "Trial only lasts 14 days."
-    )
-)
-def test_slab_connector_slim(slab_connector: SlabConnector) -> None:
-    # Get all doc IDs from the full connector
-    all_full_doc_ids = set()
-    for doc_batch in slab_connector.load_from_state():
-        all_full_doc_ids.update([doc.id for doc in doc_batch])
-
-    # Get all doc IDs from the slim connector
-    all_slim_doc_ids = set()
-    for slim_doc_batch in slab_connector.retrieve_all_slim_documents():
-        all_slim_doc_ids.update([doc.id for doc in slim_doc_batch])
-
-    # The set of full doc IDs should be always be a subset of the slim doc IDs
-    assert all_full_doc_ids.issubset(all_slim_doc_ids)

backend/tests/daily/connectors/slab/test_slab_data.json

@@ -1,5 +0,0 @@
-{
-    "section_text": "Learn about Posts\nWelcome\nThis is a post, where you can edit, share, and collaborate in real time with your team. We'd love to show you how it works!\nReading and editing\nClick the mode button to toggle between read and edit modes. You can only make changes to a post when editing.\nOrganize your posts\nWhen in edit mode, you can add topics to a post, which will keep it organized for the right 👀 to see.\nSmart mentions\nMentions are references to users, posts, topics and third party tools that show details on hover. Paste in a link for automatic conversion.\nLook back in time\nYou are ready to begin writing. You can always bring back this tour in the help menu.\nGreat job!\nYou are ready to begin writing. You can always bring back this tour in the help menu.\n\n",
-    "link": "https://onyx-test.slab.com/posts/learn-about-posts-jcp6cohu",
-    "semantic_identifier": "Learn about Posts"
-} 

backend/tests/integration/common_utils/managers/file.py

@@ -1,62 +0,0 @@
-import mimetypes
-from typing import cast
-from typing import IO
-from typing import List
-from typing import Tuple
-
-import requests
-
-from danswer.file_store.models import FileDescriptor
-from tests.integration.common_utils.constants import API_SERVER_URL
-from tests.integration.common_utils.constants import GENERAL_HEADERS
-from tests.integration.common_utils.test_models import DATestUser
-
-
-class FileManager:
-    @staticmethod
-    def upload_files(
-        files: List[Tuple[str, IO]],
-        user_performing_action: DATestUser | None = None,
-    ) -> Tuple[List[FileDescriptor], str]:
-        headers = (
-            user_performing_action.headers
-            if user_performing_action
-            else GENERAL_HEADERS
-        )
-        headers.pop("Content-Type", None)
-
-        files_param = []
-        for filename, file_obj in files:
-            mime_type, _ = mimetypes.guess_type(filename)
-            if mime_type is None:
-                mime_type = "application/octet-stream"
-            files_param.append(("files", (filename, file_obj, mime_type)))
-
-        response = requests.post(
-            f"{API_SERVER_URL}/chat/file",
-            files=files_param,
-            headers=headers,
-        )
-
-        if not response.ok:
-            return (
-                cast(List[FileDescriptor], []),
-                f"Failed to upload files - {response.json().get('detail', 'Unknown error')}",
-            )
-
-        response_json = response.json()
-        return response_json.get("files", cast(List[FileDescriptor], [])), ""
-
-    @staticmethod
-    def fetch_uploaded_file(
-        file_id: str,
-        user_performing_action: DATestUser | None = None,
-    ) -> bytes:
-        response = requests.get(
-            f"{API_SERVER_URL}/chat/file/{file_id}",
-            headers=user_performing_action.headers
-            if user_performing_action
-            else GENERAL_HEADERS,
-        )
-        response.raise_for_status()
-        return response.content

backend/tests/unit/danswer/llm/answering/stream_processing/test_citation_processing.py

@@ -385,16 +385,6 @@ def process_text(
             "Here is some text[[1]](https://0.com). Some other text",
             ["doc_0"],
         ),
-        # ['To', ' set', ' up', ' D', 'answer', ',', ' if', ' you', ' are', ' running', ' it', ' yourself', ' and',
-        # ' need', ' access', ' to', ' certain', ' features', ' like', ' auto', '-sync', 'ing', ' document',
-        # '-level', ' access', ' permissions', ',', ' you', ' should', ' reach', ' out', ' to', ' the', ' D',
-        # 'answer', ' team', ' to', ' receive', ' access', ' [[', '4', ']].', '']
-        (
-            "Unique tokens with double brackets and a single token that ends the citation and has characters after it.",
-            ["... to receive access", " [[", "1", "]].", ""],
-            "... to receive access [[1]](https://0.com).",
-            ["doc_0"],
-        ),
     ],
 )
 def test_citation_extraction(

deployment/docker_compose/docker-compose.dev.yml

@@ -130,7 +130,6 @@ services:
     restart: always
     environment:
       - ENCRYPTION_KEY_SECRET=${ENCRYPTION_KEY_SECRET:-}
-      - JWT_PUBLIC_KEY_URL=${JWT_PUBLIC_KEY_URL:-} # used for JWT authentication of users via API
       # Gen AI Settings (Needed by DanswerBot)
       - GEN_AI_MAX_TOKENS=${GEN_AI_MAX_TOKENS:-}
       - QA_TIMEOUT=${QA_TIMEOUT:-}

node_modules/.package-lock.json

@@ -1,6 +0,0 @@
-{
-  "name": "danswer",
-  "lockfileVersion": 3,
-  "requires": true,
-  "packages": {}
-}

web/src/app/admin/assistants/AssistantEditor.tsx

@@ -24,6 +24,13 @@ import {
   TextFormField,
 } from "@/components/admin/connectors/Field";
 
+import {
+  Card,
+  CardHeader,
+  CardTitle,
+  CardContent,
+  CardFooter,
+} from "@/components/ui/card";
 import { usePopup } from "@/components/admin/connectors/Popup";
 import { getDisplayNameForModel, useCategories } from "@/lib/hooks";
 import { DocumentSetSelectable } from "@/components/documentSet/DocumentSetSelectable";

web/src/app/admin/embeddings/pages/EmbeddingFormPage.tsx

@@ -28,8 +28,7 @@ import { Modal } from "@/components/Modal";
 
 import { useRouter } from "next/navigation";
 import CardSection from "@/components/admin/CardSection";
-import { combineSearchSettings } from "./utils";
-
+import { CardDescription } from "@/components/ui/card";
 export default function EmbeddingForm() {
   const { formStep, nextFormStep, prevFormStep } = useEmbeddingFormContext();
   const { popup, setPopup } = usePopup();
@@ -223,14 +222,15 @@ export default function EmbeddingForm() {
   };
 
   const updateSearch = async () => {
-    const searchSettings = combineSearchSettings(
-      selectedProvider,
-      advancedEmbeddingDetails,
-      rerankingDetails,
-      selectedProvider.provider_type?.toLowerCase() as EmbeddingProvider | null
-    );
+    const values: SavedSearchSettings = {
+      ...rerankingDetails,
+      ...advancedEmbeddingDetails,
+      ...selectedProvider,
+      provider_type:
+        selectedProvider.provider_type?.toLowerCase() as EmbeddingProvider | null,
+    };
 
-    const response = await updateSearchSettings(searchSettings);
+    const response = await updateSearchSettings(values);
     if (response.ok) {
       return true;
     } else {
@@ -247,35 +247,39 @@ export default function EmbeddingForm() {
     if (!selectedProvider) {
       return;
     }
-    let searchSettings: SavedSearchSettings;
+    let newModel: SavedSearchSettings;
 
+    // We use a spread operation to merge properties from multiple objects into a single object.
+    // Advanced embedding details may update default values.
+    // Do NOT modify the order unless you are positive the new hierarchy is correct.
     if (selectedProvider.provider_type != null) {
       // This is a cloud model
-      searchSettings = combineSearchSettings(
-        selectedProvider,
-        advancedEmbeddingDetails,
-        rerankingDetails,
-        selectedProvider.provider_type
-          ?.toLowerCase()
-          .split(" ")[0] as EmbeddingProvider | null
-      );
+      newModel = {
+        ...selectedProvider,
+        ...advancedEmbeddingDetails,
+        ...rerankingDetails,
+        provider_type:
+          (selectedProvider.provider_type
+            ?.toLowerCase()
+            .split(" ")[0] as EmbeddingProvider) || null,
+      };
     } else {
       // This is a locally hosted model
-      searchSettings = combineSearchSettings(
-        selectedProvider,
-        advancedEmbeddingDetails,
-        rerankingDetails,
-        null
-      );
+      newModel = {
+        ...selectedProvider,
+        ...advancedEmbeddingDetails,
+        ...rerankingDetails,
+        provider_type: null,
+      };
     }
 
-    searchSettings.index_name = null;
+    newModel.index_name = null;
 
     const response = await fetch(
       "/api/search-settings/set-new-search-settings",
       {
         method: "POST",
-        body: JSON.stringify(searchSettings),
+        body: JSON.stringify(newModel),
         headers: {
           "Content-Type": "application/json",
         },

web/src/app/admin/embeddings/pages/utils.ts

@@ -1,16 +1,3 @@
-import {
-  CloudEmbeddingProvider,
-  HostedEmbeddingModel,
-} from "@/components/embedding/interfaces";
-
-import {
-  AdvancedSearchConfiguration,
-  SavedSearchSettings,
-} from "../interfaces";
-
-import { EmbeddingProvider } from "@/components/embedding/interfaces";
-import { RerankingDetails } from "../interfaces";
-
 export const deleteSearchSettings = async (search_settings_id: number) => {
   const response = await fetch(`/api/search-settings/delete-search-settings`, {
     method: "DELETE",
@@ -55,20 +42,3 @@ export const testEmbedding = async ({
 
   return testResponse;
 };
-
-// We use a spread operation to merge properties from multiple objects into a single object.
-// Advanced embedding details may update default values.
-// Do NOT modify the order unless you are positive the new hierarchy is correct.
-export const combineSearchSettings = (
-  selectedProvider: CloudEmbeddingProvider | HostedEmbeddingModel,
-  advancedEmbeddingDetails: AdvancedSearchConfiguration,
-  rerankingDetails: RerankingDetails,
-  provider_type: EmbeddingProvider | null
-): SavedSearchSettings => {
-  return {
-    ...selectedProvider,
-    ...advancedEmbeddingDetails,
-    ...rerankingDetails,
-    provider_type: provider_type,
-  };
-};

web/src/app/auth/login/EmailPasswordForm.tsx

@@ -5,6 +5,7 @@ import { usePopup } from "@/components/admin/connectors/Popup";
 import { basicLogin, basicSignup } from "@/lib/user";
 import { Button } from "@/components/ui/button";
 import { Form, Formik } from "formik";
+import { useRouter } from "next/navigation";
 import * as Yup from "yup";
 import { requestEmailVerification } from "../lib";
 import { useState } from "react";
@@ -21,8 +22,10 @@ export function EmailPasswordForm({
   referralSource?: string;
   nextUrl?: string | null;
 }) {
+  const router = useRouter();
   const { popup, setPopup } = usePopup();
   const [isWorking, setIsWorking] = useState(false);
+
   return (
     <>
       {isWorking && <Spinner />}
@@ -66,13 +69,9 @@ export function EmailPasswordForm({
           if (loginResponse.ok) {
             if (isSignup && shouldVerify) {
               await requestEmailVerification(values.email);
-              // Use window.location.href to force a full page reload,
-              // ensuring app re-initializes with the new state (including
-              // server-side provider values)
-              window.location.href = "/auth/waiting-on-verification";
+              router.push("/auth/waiting-on-verification");
             } else {
-              // See above comment
-              window.location.href = nextUrl ? encodeURI(nextUrl) : "/";
+              router.push(nextUrl ? encodeURI(nextUrl) : "/");
             }
           } else {
             setIsWorking(false);

web/src/app/chat/ChatPage.tsx

@@ -106,7 +106,6 @@ import { NoAssistantModal } from "@/components/modals/NoAssistantModal";
 import { useAssistants } from "@/components/context/AssistantsContext";
 import { Separator } from "@/components/ui/separator";
 import AssistantBanner from "../../components/assistants/AssistantBanner";
-import TextView from "@/components/chat_search/TextView";
 import AssistantSelector from "@/components/chat_search/AssistantSelector";
 import { Modal } from "@/components/Modal";
 
@@ -280,9 +279,6 @@ export function ChatPage({
   const [alternativeAssistant, setAlternativeAssistant] =
     useState<Persona | null>(null);
 
-  const [presentingDocument, setPresentingDocument] =
-    useState<DanswerDocument | null>(null);
-
   const {
     visibleAssistants: assistants,
     recentAssistants,
@@ -494,7 +490,6 @@ export function ChatPage({
           clientScrollToBottom(true);
         }
       }
-
       setIsFetchingChatMessages(false);
 
       // if this is a seeded chat, then kick off the AI message generation
@@ -1654,6 +1649,7 @@ export function ChatPage({
     scrollDist,
     endDivRef,
     debounceNumber,
+    waitForScrollRef,
     mobile: settings?.isMobile,
     enableAutoScroll: autoScrollEnabled,
   });
@@ -1950,7 +1946,6 @@ export function ChatPage({
       {popup}
 
       <ChatPopup />
-
       {currentFeedback && (
         <FeedbackModal
           feedbackType={currentFeedback[0]}
@@ -1984,7 +1979,6 @@ export function ChatPage({
         <div className="md:hidden">
           <Modal noPadding noScroll>
             <ChatFilters
-              setPresentingDocument={setPresentingDocument}
               modal={true}
               filterManager={filterManager}
               ccPairs={ccPairs}
@@ -2030,13 +2024,6 @@ export function ChatPage({
         />
       )}
 
-      {presentingDocument && (
-        <TextView
-          presentingDocument={presentingDocument}
-          onClose={() => setPresentingDocument(null)}
-        />
-      )}
-
       {stackTraceModalContent && (
         <ExceptionTraceModal
           onOutsideClick={() => setStackTraceModalContent(null)}
@@ -2140,7 +2127,6 @@ export function ChatPage({
             `}
             >
               <ChatFilters
-                setPresentingDocument={setPresentingDocument}
                 modal={false}
                 filterManager={filterManager}
                 ccPairs={ccPairs}
@@ -2438,9 +2424,6 @@ export function ChatPage({
                                     }
                                   >
                                     <AIMessage
-                                      setPresentingDocument={
-                                        setPresentingDocument
-                                      }
                                       index={i}
                                       selectedMessageForDocDisplay={
                                         selectedMessageForDocDisplay

web/src/app/chat/documentSidebar/ChatDocumentDisplay.tsx

@@ -6,16 +6,13 @@ import { buildDocumentSummaryDisplay } from "@/components/search/DocumentDisplay
 import { DocumentUpdatedAtBadge } from "@/components/search/DocumentUpdatedAtBadge";
 import { MetadataBadge } from "@/components/MetadataBadge";
 import { WebResultIcon } from "@/components/WebResultIcon";
-import { Dispatch, SetStateAction } from "react";
 
 interface DocumentDisplayProps {
-  closeSidebar: () => void;
   document: DanswerDocument;
   modal?: boolean;
   isSelected: boolean;
   handleSelect: (documentId: string) => void;
   tokenLimitReached: boolean;
-  setPresentingDocument: Dispatch<SetStateAction<DanswerDocument | null>>;
 }
 
 export function DocumentMetadataBlock({
@@ -58,13 +55,11 @@ export function DocumentMetadataBlock({
 }
 
 export function ChatDocumentDisplay({
-  closeSidebar,
   document,
   modal,
   isSelected,
   handleSelect,
   tokenLimitReached,
-  setPresentingDocument,
 }: DocumentDisplayProps) {
   const isInternet = document.is_internet;
 
@@ -72,18 +67,6 @@ export function ChatDocumentDisplay({
     return null;
   }
 
-  const handleViewFile = async () => {
-    if (document.link) {
-      window.open(document.link, "_blank");
-    } else {
-      closeSidebar();
-
-      setTimeout(async () => {
-        setPresentingDocument(document);
-      }, 100);
-    }
-  };
-
   return (
     <div className={`opacity-100   ${modal ? "w-[90vw]" : "w-full"}`}>
       <div
@@ -91,9 +74,11 @@ export function ChatDocumentDisplay({
           isSelected ? "bg-gray-200" : "hover:bg-background-125"
         }`}
       >
-        <button
-          onClick={handleViewFile}
-          className="cursor-pointer text-left flex flex-col px-2 py-1.5"
+        <a
+          href={document.link}
+          target="_blank"
+          rel="noopener noreferrer"
+          className="cursor-pointer flex flex-col px-2 py-1.5"
         >
           <div className="line-clamp-1 mb-1 flex h-6 items-center gap-2 text-xs">
             {document.is_internet || document.source_type === "web" ? (
@@ -126,7 +111,7 @@ export function ChatDocumentDisplay({
               />
             )}
           </div>
-        </button>
+        </a>
       </div>
     </div>
   );

web/src/app/chat/documentSidebar/ChatFilters.tsx

@@ -3,14 +3,7 @@ import { ChatDocumentDisplay } from "./ChatDocumentDisplay";
 import { usePopup } from "@/components/admin/connectors/Popup";
 import { removeDuplicateDocs } from "@/lib/documentUtils";
 import { Message } from "../interfaces";
-import {
-  Dispatch,
-  ForwardedRef,
-  forwardRef,
-  SetStateAction,
-  useEffect,
-  useState,
-} from "react";
+import { ForwardedRef, forwardRef, useEffect, useState } from "react";
 import { FilterManager } from "@/lib/hooks";
 import { CCPairBasicInfo, DocumentSet, Tag } from "@/lib/types";
 import { SourceSelector } from "../shared_chat_search/SearchFilters";
@@ -32,7 +25,6 @@ interface ChatFiltersProps {
   tags: Tag[];
   documentSets: DocumentSet[];
   showFilters: boolean;
-  setPresentingDocument: Dispatch<SetStateAction<DanswerDocument | null>>;
 }
 
 export const ChatFilters = forwardRef<HTMLDivElement, ChatFiltersProps>(
@@ -51,7 +43,6 @@ export const ChatFilters = forwardRef<HTMLDivElement, ChatFiltersProps>(
       isOpen,
       ccPairs,
       tags,
-      setPresentingDocument,
       documentSets,
       showFilters,
     },
@@ -143,8 +134,6 @@ export const ChatFilters = forwardRef<HTMLDivElement, ChatFiltersProps>(
                         }`}
                       >
                         <ChatDocumentDisplay
-                          setPresentingDocument={setPresentingDocument}
-                          closeSidebar={closeSidebar}
                           modal={modal}
                           document={document}
                           isSelected={selectedDocumentIds.includes(

web/src/app/chat/lib.tsx

@@ -644,6 +644,7 @@ export async function useScrollonStream({
 }: {
   chatState: ChatState;
   scrollableDivRef: RefObject<HTMLDivElement>;
+  waitForScrollRef: RefObject<boolean>;
   scrollDist: MutableRefObject<number>;
   endDivRef: RefObject<HTMLDivElement>;
   debounceNumber: number;

web/src/app/chat/message/MemoizedTextComponents.tsx

@@ -6,53 +6,45 @@ import { ValidSources } from "@/lib/types";
 import React, { memo } from "react";
 import isEqual from "lodash/isEqual";
 
-export const MemoizedAnchor = memo(
-  ({ docs, updatePresentingDocument, children }: any) => {
-    const value = children?.toString();
-    if (value?.startsWith("[") && value?.endsWith("]")) {
-      const match = value.match(/\[(\d+)\]/);
-      if (match) {
-        const index = parseInt(match[1], 10) - 1;
-        const associatedDoc = docs && docs[index];
+export const MemoizedAnchor = memo(({ docs, children }: any) => {
+  console.log(children);
+  const value = children?.toString();
+  if (value?.startsWith("[") && value?.endsWith("]")) {
+    const match = value.match(/\[(\d+)\]/);
+    if (match) {
+      const index = parseInt(match[1], 10) - 1;
+      const associatedDoc = docs && docs[index];
 
-        const url = associatedDoc?.link
-          ? new URL(associatedDoc.link).origin + "/favicon.ico"
-          : "";
+      const url = associatedDoc?.link
+        ? new URL(associatedDoc.link).origin + "/favicon.ico"
+        : "";
 
-        const getIcon = (sourceType: ValidSources, link: string) => {
-          return getSourceMetadata(sourceType).icon({ size: 18 });
-        };
+      const getIcon = (sourceType: ValidSources, link: string) => {
+        return getSourceMetadata(sourceType).icon({ size: 18 });
+      };
 
-        const icon =
-          associatedDoc?.source_type === "web" ? (
-            <WebResultIcon url={associatedDoc.link} />
-          ) : (
-            getIcon(
-              associatedDoc?.source_type || "web",
-              associatedDoc?.link || ""
-            )
-          );
-
-        return (
-          <MemoizedLink
-            updatePresentingDocument={updatePresentingDocument}
-            document={{ ...associatedDoc, icon, url }}
-          >
-            {children}
-          </MemoizedLink>
+      const icon =
+        associatedDoc?.source_type === "web" ? (
+          <WebResultIcon url={associatedDoc.link} />
+        ) : (
+          getIcon(
+            associatedDoc?.source_type || "web",
+            associatedDoc?.link || ""
+          )
         );
-      }
+
+      return (
+        <MemoizedLink document={{ ...associatedDoc, icon, url }}>
+          {children}
+        </MemoizedLink>
+      );
     }
-    return (
-      <MemoizedLink updatePresentingDocument={updatePresentingDocument}>
-        {children}
-      </MemoizedLink>
-    );
   }
-);
+  return <MemoizedLink>{children}</MemoizedLink>;
+});
 
 export const MemoizedLink = memo((props: any) => {
-  const { node, document, updatePresentingDocument, ...rest } = props;
+  const { node, document, ...rest } = props;
   const value = rest.children;
 
   if (value?.toString().startsWith("*")) {
@@ -66,21 +58,22 @@ export const MemoizedLink = memo((props: any) => {
         icon={document?.icon as React.ReactNode}
         link={rest?.href}
         document={document as LoadedDanswerDocument}
-        updatePresentingDocument={updatePresentingDocument}
       >
         {rest.children}
       </Citation>
     );
+  } else {
+    return (
+      <a
+        onMouseDown={() =>
+          rest.href ? window.open(rest.href, "_blank") : undefined
+        }
+        className="cursor-pointer text-link hover:text-link-hover"
+      >
+        {rest.children}
+      </a>
+    );
   }
-
-  return (
-    <a
-      onMouseDown={() => rest.href && window.open(rest.href, "_blank")}
-      className="cursor-pointer text-link hover:text-link-hover"
-    >
-      {rest.children}
-    </a>
-  );
 });
 
 export const MemoizedParagraph = memo(

web/src/app/chat/message/Messages.tsx

@@ -10,7 +10,6 @@ import {
 import { FeedbackType } from "../types";
 import React, {
   memo,
-  ReactNode,
   useCallback,
   useContext,
   useEffect,
@@ -22,7 +21,6 @@ import ReactMarkdown from "react-markdown";
 import {
   DanswerDocument,
   FilteredDanswerDocument,
-  LoadedDanswerDocument,
 } from "@/lib/search/interfaces";
 import { SearchSummary } from "./SearchSummary";
 
@@ -190,7 +188,6 @@ export const AIMessage = ({
   currentPersona,
   otherMessagesCanSwitchTo,
   onMessageSelection,
-  setPresentingDocument,
   index,
 }: {
   index?: number;
@@ -221,7 +218,6 @@ export const AIMessage = ({
   retrievalDisabled?: boolean;
   overriddenModel?: string;
   regenerate?: (modelOverRide: LlmOverride) => Promise<void>;
-  setPresentingDocument?: (document: DanswerDocument) => void;
 }) => {
   const toolCallGenerating = toolCall && !toolCall.tool_result;
   const processContent = (content: string | JSX.Element) => {
@@ -312,12 +308,7 @@ export const AIMessage = ({
 
   const anchorCallback = useCallback(
     (props: any) => (
-      <MemoizedAnchor
-        updatePresentingDocument={setPresentingDocument}
-        docs={docs}
-      >
-        {props.children}
-      </MemoizedAnchor>
+      <MemoizedAnchor docs={docs}>{props.children}</MemoizedAnchor>
     ),
     [docs]
   );

web/src/app/chat/shared/[chatId]/SharedChatDisplay.tsx

@@ -17,8 +17,6 @@ import { SettingsContext } from "@/components/settings/SettingsProvider";
 import { DanswerInitializingLoader } from "@/components/DanswerInitializingLoader";
 import { Persona } from "@/app/admin/assistants/interfaces";
 import { Button } from "@/components/ui/button";
-import { DanswerDocument } from "@/lib/search/interfaces";
-import TextView from "@/components/chat_search/TextView";
 
 function BackToDanswerButton() {
   const router = useRouter();
@@ -43,9 +41,6 @@ export function SharedChatDisplay({
   persona: Persona;
 }) {
   const [isReady, setIsReady] = useState(false);
-  const [presentingDocument, setPresentingDocument] =
-    useState<DanswerDocument | null>(null);
-
   useEffect(() => {
     Prism.highlightAll();
     setIsReady(true);
@@ -68,70 +63,61 @@ export function SharedChatDisplay({
   );
 
   return (
-    <>
-      {presentingDocument && (
-        <TextView
-          presentingDocument={presentingDocument}
-          onClose={() => setPresentingDocument(null)}
-        />
-      )}
-      <div className="w-full h-[100dvh] overflow-hidden">
-        <div className="flex max-h-full overflow-hidden pb-[72px]">
-          <div className="flex w-full overflow-hidden overflow-y-scroll">
-            <div className="w-full h-full flex-col flex max-w-message-max mx-auto">
-              <div className="px-5 pt-8">
-                <h1 className="text-3xl text-strong font-bold">
-                  {chatSession.description ||
-                    `Chat ${chatSession.chat_session_id}`}
-                </h1>
-                <p className="text-emphasis">
-                  {humanReadableFormat(chatSession.time_created)}
-                </p>
+    <div className="w-full h-[100dvh] overflow-hidden">
+      <div className="flex max-h-full overflow-hidden pb-[72px]">
+        <div className="flex w-full overflow-hidden overflow-y-scroll">
+          <div className="w-full h-full flex-col flex max-w-message-max mx-auto">
+            <div className="px-5 pt-8">
+              <h1 className="text-3xl text-strong font-bold">
+                {chatSession.description ||
+                  `Chat ${chatSession.chat_session_id}`}
+              </h1>
+              <p className="text-emphasis">
+                {humanReadableFormat(chatSession.time_created)}
+              </p>
 
-                <Separator />
-              </div>
-              {isReady ? (
-                <div className="w-full pb-16">
-                  {messages.map((message) => {
-                    if (message.type === "user") {
-                      return (
-                        <HumanMessage
-                          shared
-                          key={message.messageId}
-                          content={message.message}
-                          files={message.files}
-                        />
-                      );
-                    } else {
-                      return (
-                        <AIMessage
-                          shared
-                          setPresentingDocument={setPresentingDocument}
-                          currentPersona={persona}
-                          key={message.messageId}
-                          messageId={message.messageId}
-                          content={message.message}
-                          files={message.files || []}
-                          citedDocuments={getCitedDocumentsFromMessage(message)}
-                          isComplete
-                        />
-                      );
-                    }
-                  })}
-                </div>
-              ) : (
-                <div className="grow flex-0 h-screen w-full flex items-center justify-center">
-                  <div className="mb-[33vh]">
-                    <DanswerInitializingLoader />
-                  </div>
-                </div>
-              )}
+              <Separator />
             </div>
+            {isReady ? (
+              <div className="w-full pb-16">
+                {messages.map((message) => {
+                  if (message.type === "user") {
+                    return (
+                      <HumanMessage
+                        shared
+                        key={message.messageId}
+                        content={message.message}
+                        files={message.files}
+                      />
+                    );
+                  } else {
+                    return (
+                      <AIMessage
+                        shared
+                        currentPersona={persona}
+                        key={message.messageId}
+                        messageId={message.messageId}
+                        content={message.message}
+                        files={message.files || []}
+                        citedDocuments={getCitedDocumentsFromMessage(message)}
+                        isComplete
+                      />
+                    );
+                  }
+                })}
+              </div>
+            ) : (
+              <div className="grow flex-0 h-screen w-full flex items-center justify-center">
+                <div className="mb-[33vh]">
+                  <DanswerInitializingLoader />
+                </div>
+              </div>
+            )}
           </div>
         </div>
-
-        <BackToDanswerButton />
       </div>
-    </>
+
+      <BackToDanswerButton />
+    </div>
   );
 }

web/src/components/chat_search/AssistantSelector.tsx

@@ -28,13 +28,12 @@ import { DraggableAssistantCard } from "@/components/assistants/AssistantCards";
 import { updateUserAssistantList } from "@/lib/assistants/updateAssistantPreferences";
 
 import Text from "@/components/ui/text";
-import { getDisplayNameForModel, LlmOverrideManager } from "@/lib/hooks";
+import { LlmOverrideManager } from "@/lib/hooks";
 import { Tab } from "@headlessui/react";
 import { AssistantIcon } from "../assistants/AssistantIcon";
 import { restrictToVerticalAxis } from "@dnd-kit/modifiers";
 import { restrictToParentElement } from "@dnd-kit/modifiers";
 import { Drawer, DrawerContent, DrawerHeader, DrawerTitle } from "../ui/drawer";
-import { truncateString } from "@/lib/utils";
 
 const AssistantSelector = ({
   liveAssistant,
@@ -311,9 +310,7 @@ const AssistantSelector = ({
             <span className="font-bold">{liveAssistant.name}</span>
           </div>
           <div className="flex items-center">
-            <span className="mr-2 text-xs">
-              {truncateString(getDisplayNameForModel(currentLlm), 30)}
-            </span>
+            <span className="mr-2 text-xs">{currentLlm}</span>
             <FiChevronDown
               className={`w-5 h-5 text-white transition-transform duration-300 transform ${
                 isOpen ? "rotate-180" : ""

web/src/components/chat_search/TextView.tsx

@@ -1,173 +0,0 @@
-"use client";
-
-import { useState, useEffect, useCallback } from "react";
-import { Button } from "@/components/ui/button";
-import {
-  Dialog,
-  DialogContent,
-  DialogHeader,
-  DialogTitle,
-} from "@/components/ui/dialog";
-import { Download, XIcon, ZoomIn, ZoomOut } from "lucide-react";
-import { DanswerDocument } from "@/lib/search/interfaces";
-import { MinimalMarkdown } from "./MinimalMarkdown";
-
-interface TextViewProps {
-  presentingDocument: DanswerDocument;
-  onClose: () => void;
-}
-export default function TextView({
-  presentingDocument,
-  onClose,
-}: TextViewProps) {
-  const [zoom, setZoom] = useState(100);
-  const [fileContent, setFileContent] = useState<string>("");
-  const [fileUrl, setFileUrl] = useState<string>("");
-  const [fileName, setFileName] = useState<string>("");
-  const [isLoading, setIsLoading] = useState(true);
-  const [fileType, setFileType] = useState<string>("application/octet-stream");
-
-  const isMarkdownFormat = (mimeType: string): boolean => {
-    const markdownFormats = [
-      "text/markdown",
-      "text/x-markdown",
-      "text/plain",
-      "text/x-rst",
-      "text/x-org",
-    ];
-    return markdownFormats.some((format) => mimeType.startsWith(format));
-  };
-
-  const isSupportedIframeFormat = (mimeType: string): boolean => {
-    const supportedFormats = [
-      "application/pdf",
-      "image/png",
-      "image/jpeg",
-      "image/gif",
-      "image/svg+xml",
-    ];
-    return supportedFormats.some((format) => mimeType.startsWith(format));
-  };
-
-  const fetchFile = useCallback(async () => {
-    setIsLoading(true);
-    const fileId = presentingDocument.document_id.split("__")[1];
-    try {
-      const response = await fetch(
-        `/api/chat/file/${encodeURIComponent(fileId)}`,
-        {
-          method: "GET",
-        }
-      );
-      const blob = await response.blob();
-      const url = window.URL.createObjectURL(blob);
-      setFileUrl(url);
-      setFileName(presentingDocument.semantic_identifier || "document");
-      const contentType =
-        response.headers.get("Content-Type") || "application/octet-stream";
-      setFileType(contentType);
-
-      if (isMarkdownFormat(blob.type)) {
-        const text = await blob.text();
-        setFileContent(text);
-      }
-    } catch (error) {
-      console.error("Error fetching file:", error);
-    } finally {
-      setTimeout(() => {
-        setIsLoading(false);
-      }, 1000);
-    }
-  }, [presentingDocument]);
-
-  useEffect(() => {
-    fetchFile();
-  }, [fetchFile]);
-
-  const handleDownload = () => {
-    const link = document.createElement("a");
-    link.href = fileUrl;
-    link.download = fileName;
-    document.body.appendChild(link);
-    link.click();
-    document.body.removeChild(link);
-  };
-
-  const handleZoomIn = () => setZoom((prev) => Math.min(prev + 25, 200));
-  const handleZoomOut = () => setZoom((prev) => Math.max(prev - 25, 100));
-
-  return (
-    <Dialog open={true} onOpenChange={onClose}>
-      <DialogContent
-        hideCloseIcon
-        className="max-w-5xl w-[90vw] flex flex-col justify-between gap-y-0 h-full max-h-[80vh] p-0"
-      >
-        <DialogHeader className="px-4 mb-0 pt-2 pb-3 flex flex-row items-center justify-between border-b">
-          <DialogTitle className="text-lg font-medium truncate">
-            {fileName}
-          </DialogTitle>
-          <div className="flex items-center space-x-2">
-            <Button variant="ghost" size="icon" onClick={handleZoomOut}>
-              <ZoomOut className="h-4 w-4" />
-              <span className="sr-only">Zoom Out</span>
-            </Button>
-            <span className="text-sm">{zoom}%</span>
-            <Button variant="ghost" size="icon" onClick={handleZoomIn}>
-              <ZoomIn className="h-4 w-4" />
-              <span className="sr-only">Zoom In</span>
-            </Button>
-            <Button variant="ghost" size="icon" onClick={handleDownload}>
-              <Download className="h-4 w-4" />
-              <span className="sr-only">Download</span>
-            </Button>
-            <Button variant="ghost" size="icon" onClick={() => onClose()}>
-              <XIcon className="h-4 w-4" />
-              <span className="sr-only">Close</span>
-            </Button>
-          </div>
-        </DialogHeader>
-        <div className="mt-0 rounded-b-lg flex-1 overflow-hidden">
-          <div className="flex items-center justify-center w-full h-full">
-            {isLoading ? (
-              <div className="flex flex-col items-center justify-center h-full">
-                <div className="animate-spin rounded-full h-16 w-16 border-t-4 border-b-4 border-primary"></div>
-                <p className="mt-6 text-lg font-medium text-muted-foreground">
-                  Loading document...
-                </p>
-              </div>
-            ) : (
-              <div
-                className={`w-full h-full transform origin-center transition-transform duration-300 ease-in-out`}
-                style={{ transform: `scale(${zoom / 100})` }}
-              >
-                {isSupportedIframeFormat(fileType) ? (
-                  <iframe
-                    src={`${fileUrl}#toolbar=0`}
-                    className="w-full h-full border-none"
-                    title="File Viewer"
-                  />
-                ) : isMarkdownFormat(fileType) ? (
-                  <div className="w-full h-full p-6 overflow-y-scroll overflow-x-hidden">
-                    <MinimalMarkdown
-                      content={fileContent}
-                      className="w-full pb-4 h-full text-lg text-wrap break-words"
-                    />
-                  </div>
-                ) : (
-                  <div className="flex flex-col items-center justify-center h-full">
-                    <p className="text-lg font-medium text-muted-foreground">
-                      This file format is not supported for preview.
-                    </p>
-                    <Button className="mt-4" onClick={handleDownload}>
-                      Download File
-                    </Button>
-                  </div>
-                )}
-              </div>
-            )}
-          </div>
-        </div>
-      </DialogContent>
-    </Dialog>
-  );
-}

web/src/components/chat_search/sources/SourceCard.tsx

@@ -1,7 +1,6 @@
 import { WebResultIcon } from "@/components/WebResultIcon";
 import { SourceIcon } from "@/components/SourceIcon";
 import { DanswerDocument } from "@/lib/search/interfaces";
-import { truncateString } from "@/lib/utils";
 
 export default function SourceCard({ doc }: { doc: DanswerDocument }) {
   return (
@@ -18,7 +17,12 @@ export default function SourceCard({ doc }: { doc: DanswerDocument }) {
         ) : (
           <SourceIcon sourceType={doc.source_type} iconSize={18} />
         )}
-        <p>{truncateString(doc.semantic_identifier || doc.document_id, 12)}</p>
+        <p>
+          {(doc.semantic_identifier || doc.document_id).slice(0, 12).trim()}
+          {(doc.semantic_identifier || doc.document_id).length > 12 && (
+            <span className="text-text-500">...</span>
+          )}
+        </p>
       </div>
       <div className="line-clamp-2 text-sm font-semibold"></div>
       <div className="line-clamp-2 text-sm font-normal leading-snug text-text-700">

web/src/components/search/DocumentDisplay.tsx

@@ -19,7 +19,6 @@ import { FiTag } from "react-icons/fi";
 import { SettingsContext } from "../settings/SettingsProvider";
 import { CustomTooltip, TooltipGroup } from "../tooltip/CustomTooltip";
 import { WarningCircle } from "@phosphor-icons/react";
-import TextView from "../chat_search/TextView";
 import { SearchResultIcon } from "../SearchResultIcon";
 
 export const buildDocumentSummaryDisplay = (
@@ -189,12 +188,6 @@ export const DocumentDisplay = ({
   const relevance_explanation =
     document.relevance_explanation ?? additional_relevance?.content;
   const settings = useContext(SettingsContext);
-  const [presentingDocument, setPresentingDocument] =
-    useState<DanswerDocument | null>(null);
-
-  const handleViewFile = async () => {
-    setPresentingDocument(document);
-  };
 
   return (
     <div
@@ -226,22 +219,19 @@ export const DocumentDisplay = ({
         }`}
       >
         <div className="flex relative">
-          <button
-            type="button"
-            className={`rounded-lg flex font-bold text-link max-w-full`}
-            onClick={() => {
-              if (document.link) {
-                window.open(document.link, "_blank");
-              } else {
-                handleViewFile();
-              }
-            }}
+          <a
+            className={`rounded-lg flex font-bold text-link max-w-full ${
+              document.link ? "" : "pointer-events-none"
+            }`}
+            href={document.link}
+            target="_blank"
+            rel="noopener noreferrer"
           >
             <SourceIcon sourceType={document.source_type} iconSize={22} />
             <p className="truncate text-wrap break-all ml-2 my-auto line-clamp-1 text-base max-w-full">
               {document.semantic_identifier || document.document_id}
             </p>
-          </button>
+          </a>
           <div className="ml-auto flex items-center">
             <TooltipGroup>
               {isHovered && messageId && (
@@ -280,13 +270,6 @@ export const DocumentDisplay = ({
           <DocumentMetadataBlock document={document} />
         </div>
 
-        {presentingDocument && (
-          <TextView
-            presentingDocument={presentingDocument}
-            onClose={() => setPresentingDocument(null)}
-          />
-        )}
-
         <p
           style={{ transition: "height 0.30s ease-in-out" }}
           className="pl-1 pt-2 pb-3 break-words text-wrap"
@@ -314,14 +297,11 @@ export const AgenticDocumentDisplay = ({
   setPopup,
 }: DocumentDisplayProps) => {
   const [isHovered, setIsHovered] = useState(false);
-  const [presentingDocument, setPresentingDocument] =
-    useState<DanswerDocument | null>(null);
 
   const [alternativeToggled, setAlternativeToggled] = useState(false);
 
   const relevance_explanation =
     document.relevance_explanation ?? additional_relevance?.content;
-
   return (
     <div
       key={document.semantic_identifier}
@@ -340,24 +320,19 @@ export const AgenticDocumentDisplay = ({
         }`}
       >
         <div className="flex relative">
-          <button
-            type="button"
+          <a
             className={`rounded-lg flex font-bold text-link max-w-full ${
               document.link ? "" : "pointer-events-none"
             }`}
-            onClick={() => {
-              if (document.link) {
-                window.open(document.link, "_blank");
-              } else {
-                setPresentingDocument(document);
-              }
-            }}
+            href={document.link}
+            target="_blank"
+            rel="noopener noreferrer"
           >
             <SourceIcon sourceType={document.source_type} iconSize={22} />
             <p className="truncate text-wrap break-all ml-2 my-auto line-clamp-1 text-base max-w-full">
               {document.semantic_identifier || document.document_id}
             </p>
-          </button>
+          </a>
 
           <div className="ml-auto items-center flex">
             <TooltipGroup>
@@ -390,12 +365,6 @@ export const AgenticDocumentDisplay = ({
         <div className="mt-1">
           <DocumentMetadataBlock document={document} />
         </div>
-        {presentingDocument && (
-          <TextView
-            presentingDocument={presentingDocument}
-            onClose={() => setPresentingDocument(null)}
-          />
-        )}
 
         <div className="pt-2 break-words flex gap-x-2">
           <p

web/src/components/search/results/Citation.tsx

@@ -13,14 +13,12 @@ export function Citation({
   link,
   document,
   index,
-  updatePresentingDocument,
   icon,
   url,
 }: {
   link?: string;
   children?: JSX.Element | string | null | ReactNode;
   index?: number;
-  updatePresentingDocument: (documentIndex: LoadedDanswerDocument) => void;
   document: LoadedDanswerDocument;
   icon?: React.ReactNode;
   url?: string;
@@ -35,13 +33,7 @@ export function Citation({
         <Tooltip>
           <TooltipTrigger asChild>
             <div
-              onMouseDown={() => {
-                if (!link) {
-                  updatePresentingDocument(document);
-                } else {
-                  window.open(link, "_blank");
-                }
-              }}
+              onMouseDown={() => window.open(link, "_blank")}
               className="inline-flex items-center ml-1 cursor-pointer transition-all duration-200 ease-in-out"
             >
               <span className="relative min-w-[1.4rem] text-center no-underline -top-0.5 px-1.5 py-0.5 text-xs font-medium text-gray-700 bg-gray-100 rounded-full border border-gray-300 hover:bg-gray-200 hover:text-gray-900 shadow-sm no-underline">
@@ -61,13 +53,7 @@ export function Citation({
         <Tooltip>
           <TooltipTrigger asChild>
             <div
-              onMouseDown={() => {
-                if (!link) {
-                  updatePresentingDocument(document);
-                } else {
-                  window.open(link, "_blank");
-                }
-              }}
+              onMouseDown={() => window.open(link, "_blank")}
               className="inline-flex items-center ml-1 cursor-pointer transition-all duration-200 ease-in-out"
             >
               <span className="relative min-w-[1.4rem]  pchatno-underline -top-0.5 px-1.5 py-0.5 text-xs font-medium text-gray-700 bg-gray-100 rounded-full border border-gray-300 hover:bg-gray-200 hover:text-gray-900 shadow-sm no-underline">

web/src/components/ui/dialog.tsx

@@ -1,125 +0,0 @@
-"use client";
-
-import * as React from "react";
-import * as DialogPrimitive from "@radix-ui/react-dialog";
-import { X } from "lucide-react";
-
-import { cn } from "@/lib/utils";
-
-const Dialog = DialogPrimitive.Root;
-
-const DialogTrigger = DialogPrimitive.Trigger;
-
-const DialogPortal = DialogPrimitive.Portal;
-
-const DialogClose = DialogPrimitive.Close;
-
-const DialogOverlay = React.forwardRef<
-  React.ElementRef<typeof DialogPrimitive.Overlay>,
-  React.ComponentPropsWithoutRef<typeof DialogPrimitive.Overlay>
->(({ className, ...props }, ref) => (
-  <DialogPrimitive.Overlay
-    ref={ref}
-    className={cn(
-      "fixed inset-0 z-50 bg-black/80  data-[state=open]:animate-in data-[state=closed]:animate-out data-[state=closed]:fade-out-0 data-[state=open]:fade-in-0",
-      className
-    )}
-    {...props}
-  />
-));
-DialogOverlay.displayName = DialogPrimitive.Overlay.displayName;
-const DialogContent = React.forwardRef<
-  React.ElementRef<typeof DialogPrimitive.Content>,
-  React.ComponentPropsWithoutRef<typeof DialogPrimitive.Content> & {
-    hideCloseIcon?: boolean;
-  }
->(({ className, children, hideCloseIcon = false, ...props }, ref) => (
-  <DialogPortal>
-    <DialogOverlay />
-    <DialogPrimitive.Content
-      ref={ref}
-      className={cn(
-        "fixed left-[50%] top-[50%] z-50 grid w-full max-w-lg translate-x-[-50%] translate-y-[-50%] gap-4 border border-neutral-200 bg-white p-6 shadow-lg duration-200 data-[state=open]:animate-in data-[state=closed]:animate-out data-[state=closed]:fade-out-0 data-[state=open]:fade-in-0 data-[state=closed]:zoom-out-95 data-[state=open]:zoom-in-95 data-[state=closed]:slide-out-to-left-1/2 data-[state=closed]:slide-out-to-top-[48%] data-[state=open]:slide-in-from-left-1/2 data-[state=open]:slide-in-from-top-[48%] sm:rounded-lg dark:border-neutral-800 dark:bg-neutral-950",
-        className
-      )}
-      {...props}
-    >
-      {children}
-      {!hideCloseIcon && (
-        <DialogPrimitive.Close className="absolute right-4 top-4 rounded-sm opacity-70 ring-offset-white transition-opacity hover:opacity-100 focus:outline-none focus:ring-2 focus:ring-neutral-950 focus:ring-offset-2 disabled:pointer-events-none data-[state=open]:bg-neutral-100 data-[state=open]:text-neutral-500 dark:ring-offset-neutral-950 dark:focus:ring-neutral-300 dark:data-[state=open]:bg-neutral-800 dark:data-[state=open]:text-neutral-400">
-          <X className="h-4 w-4" />
-          <span className="sr-only">Close</span>
-        </DialogPrimitive.Close>
-      )}
-    </DialogPrimitive.Content>
-  </DialogPortal>
-));
-DialogContent.displayName = DialogPrimitive.Content.displayName;
-
-const DialogHeader = ({
-  className,
-  ...props
-}: React.HTMLAttributes<HTMLDivElement>) => (
-  <div
-    className={cn(
-      "flex flex-col space-y-1.5 text-center sm:text-left",
-      className
-    )}
-    {...props}
-  />
-);
-DialogHeader.displayName = "DialogHeader";
-
-const DialogFooter = ({
-  className,
-  ...props
-}: React.HTMLAttributes<HTMLDivElement>) => (
-  <div
-    className={cn(
-      "flex flex-col-reverse sm:flex-row sm:justify-end sm:space-x-2",
-      className
-    )}
-    {...props}
-  />
-);
-DialogFooter.displayName = "DialogFooter";
-
-const DialogTitle = React.forwardRef<
-  React.ElementRef<typeof DialogPrimitive.Title>,
-  React.ComponentPropsWithoutRef<typeof DialogPrimitive.Title>
->(({ className, ...props }, ref) => (
-  <DialogPrimitive.Title
-    ref={ref}
-    className={cn(
-      "text-lg font-semibold leading-none tracking-tight",
-      className
-    )}
-    {...props}
-  />
-));
-DialogTitle.displayName = DialogPrimitive.Title.displayName;
-
-const DialogDescription = React.forwardRef<
-  React.ElementRef<typeof DialogPrimitive.Description>,
-  React.ComponentPropsWithoutRef<typeof DialogPrimitive.Description>
->(({ className, ...props }, ref) => (
-  <DialogPrimitive.Description
-    ref={ref}
-    className={cn("text-sm text-neutral-500 dark:text-neutral-400", className)}
-    {...props}
-  />
-));
-DialogDescription.displayName = DialogPrimitive.Description.displayName;
-
-export {
-  Dialog,
-  DialogPortal,
-  DialogOverlay,
-  DialogClose,
-  DialogTrigger,
-  DialogContent,
-  DialogHeader,
-  DialogFooter,
-  DialogTitle,
-  DialogDescription,
-};

web/src/lib/utils.ts

@@ -4,7 +4,3 @@ import { twMerge } from "tailwind-merge";
 export function cn(...inputs: ClassValue[]) {
   return twMerge(clsx(inputs));
 }
-
-export const truncateString = (str: string, maxLength: number) => {
-  return str.length > maxLength ? str.slice(0, maxLength - 1) + "..." : str;
-};