build org

backend/danswer/llm/answering/stream_processing/citation_processing.py

@@ -67,9 +67,9 @@ class CitationProcessor:
                 if piece_that_comes_after == "\n" and in_code_block(self.llm_out):
                     self.curr_segment = self.curr_segment.replace("```", "```plaintext")
 
-        citation_pattern = r"\[(\d+)\]|\[\[(\d+)\]\]"  # [1], [[1]], etc.
+        citation_pattern = r"\[(\d+)\]"
         citations_found = list(re.finditer(citation_pattern, self.curr_segment))
-        possible_citation_pattern = r"(\[+\d*$)"  # [1, [, [[, [[2, etc.
+        possible_citation_pattern = r"(\[\d*$)"  # [1, [, etc
         possible_citation_found = re.search(
             possible_citation_pattern, self.curr_segment
         )
@@ -77,15 +77,13 @@ class CitationProcessor:
         if len(citations_found) == 0 and len(self.llm_out) - self.past_cite_count > 5:
             self.current_citations = []
 
-        result = ""
+        result = ""  # Initialize result here
         if citations_found and not in_code_block(self.llm_out):
             last_citation_end = 0
             length_to_add = 0
             while len(citations_found) > 0:
                 citation = citations_found.pop(0)
-                numerical_value = int(
-                    next(group for group in citation.groups() if group is not None)
-                )
+                numerical_value = int(citation.group(1))
 
                 if 1 <= numerical_value <= self.max_citation_num:
                     context_llm_doc = self.context_docs[numerical_value - 1]
@@ -133,6 +131,14 @@ class CitationProcessor:
 
                     link = context_llm_doc.link
 
+                    # Replace the citation in the current segment
+                    start, end = citation.span()
+                    self.curr_segment = (
+                        self.curr_segment[: start + length_to_add]
+                        + f"[{target_citation_num}]"
+                        + self.curr_segment[end + length_to_add :]
+                    )
+
                     self.past_cite_count = len(self.llm_out)
                     self.current_citations.append(target_citation_num)
 
@@ -143,7 +149,6 @@ class CitationProcessor:
                             document_id=context_llm_doc.document_id,
                         )
 
-                    start, end = citation.span()
                     if link:
                         prev_length = len(self.curr_segment)
                         self.curr_segment = (

backend/tests/unit/danswer/llm/answering/stream_processing/test_citation_processing.py

@@ -385,16 +385,6 @@ def process_text(
             "Here is some text[[1]](https://0.com). Some other text",
             ["doc_0"],
         ),
-        # ['To', ' set', ' up', ' D', 'answer', ',', ' if', ' you', ' are', ' running', ' it', ' yourself', ' and',
-        # ' need', ' access', ' to', ' certain', ' features', ' like', ' auto', '-sync', 'ing', ' document',
-        # '-level', ' access', ' permissions', ',', ' you', ' should', ' reach', ' out', ' to', ' the', ' D',
-        # 'answer', ' team', ' to', ' receive', ' access', ' [[', '4', ']].', '']
-        (
-            "Unique tokens with double brackets and a single token that ends the citation and has characters after it.",
-            ["... to receive access", " [[", "1", "]].", ""],
-            "... to receive access [[1]](https://0.com).",
-            ["doc_0"],
-        ),
     ],
 )
 def test_citation_extraction(

web/src/app/admin/assistants/AssistantEditor.tsx

@@ -24,6 +24,13 @@ import {
   TextFormField,
 } from "@/components/admin/connectors/Field";
 
+import {
+  Card,
+  CardHeader,
+  CardTitle,
+  CardContent,
+  CardFooter,
+} from "@/components/ui/card";
 import { usePopup } from "@/components/admin/connectors/Popup";
 import { getDisplayNameForModel, useCategories } from "@/lib/hooks";
 import { DocumentSetSelectable } from "@/components/documentSet/DocumentSetSelectable";

web/src/app/admin/embeddings/pages/EmbeddingFormPage.tsx

@@ -28,8 +28,7 @@ import { Modal } from "@/components/Modal";
 
 import { useRouter } from "next/navigation";
 import CardSection from "@/components/admin/CardSection";
-import { combineSearchSettings } from "./utils";
-
+import { CardDescription } from "@/components/ui/card";
 export default function EmbeddingForm() {
   const { formStep, nextFormStep, prevFormStep } = useEmbeddingFormContext();
   const { popup, setPopup } = usePopup();
@@ -223,14 +222,15 @@ export default function EmbeddingForm() {
   };
 
   const updateSearch = async () => {
-    const searchSettings = combineSearchSettings(
-      selectedProvider,
-      advancedEmbeddingDetails,
-      rerankingDetails,
-      selectedProvider.provider_type?.toLowerCase() as EmbeddingProvider | null
-    );
+    const values: SavedSearchSettings = {
+      ...rerankingDetails,
+      ...advancedEmbeddingDetails,
+      ...selectedProvider,
+      provider_type:
+        selectedProvider.provider_type?.toLowerCase() as EmbeddingProvider | null,
+    };
 
-    const response = await updateSearchSettings(searchSettings);
+    const response = await updateSearchSettings(values);
     if (response.ok) {
       return true;
     } else {
@@ -247,35 +247,39 @@ export default function EmbeddingForm() {
     if (!selectedProvider) {
       return;
     }
-    let searchSettings: SavedSearchSettings;
+    let newModel: SavedSearchSettings;
 
+    // We use a spread operation to merge properties from multiple objects into a single object.
+    // Advanced embedding details may update default values.
+    // Do NOT modify the order unless you are positive the new hierarchy is correct.
     if (selectedProvider.provider_type != null) {
       // This is a cloud model
-      searchSettings = combineSearchSettings(
-        selectedProvider,
-        advancedEmbeddingDetails,
-        rerankingDetails,
-        selectedProvider.provider_type
-          ?.toLowerCase()
-          .split(" ")[0] as EmbeddingProvider | null
-      );
+      newModel = {
+        ...selectedProvider,
+        ...advancedEmbeddingDetails,
+        ...rerankingDetails,
+        provider_type:
+          (selectedProvider.provider_type
+            ?.toLowerCase()
+            .split(" ")[0] as EmbeddingProvider) || null,
+      };
     } else {
       // This is a locally hosted model
-      searchSettings = combineSearchSettings(
-        selectedProvider,
-        advancedEmbeddingDetails,
-        rerankingDetails,
-        null
-      );
+      newModel = {
+        ...selectedProvider,
+        ...advancedEmbeddingDetails,
+        ...rerankingDetails,
+        provider_type: null,
+      };
     }
 
-    searchSettings.index_name = null;
+    newModel.index_name = null;
 
     const response = await fetch(
       "/api/search-settings/set-new-search-settings",
       {
         method: "POST",
-        body: JSON.stringify(searchSettings),
+        body: JSON.stringify(newModel),
         headers: {
           "Content-Type": "application/json",
         },

web/src/app/admin/embeddings/pages/utils.ts

@@ -1,16 +1,3 @@
-import {
-  CloudEmbeddingProvider,
-  HostedEmbeddingModel,
-} from "@/components/embedding/interfaces";
-
-import {
-  AdvancedSearchConfiguration,
-  SavedSearchSettings,
-} from "../interfaces";
-
-import { EmbeddingProvider } from "@/components/embedding/interfaces";
-import { RerankingDetails } from "../interfaces";
-
 export const deleteSearchSettings = async (search_settings_id: number) => {
   const response = await fetch(`/api/search-settings/delete-search-settings`, {
     method: "DELETE",
@@ -55,20 +42,3 @@ export const testEmbedding = async ({
 
   return testResponse;
 };
-
-// We use a spread operation to merge properties from multiple objects into a single object.
-// Advanced embedding details may update default values.
-// Do NOT modify the order unless you are positive the new hierarchy is correct.
-export const combineSearchSettings = (
-  selectedProvider: CloudEmbeddingProvider | HostedEmbeddingModel,
-  advancedEmbeddingDetails: AdvancedSearchConfiguration,
-  rerankingDetails: RerankingDetails,
-  provider_type: EmbeddingProvider | null
-): SavedSearchSettings => {
-  return {
-    ...selectedProvider,
-    ...advancedEmbeddingDetails,
-    ...rerankingDetails,
-    provider_type: provider_type,
-  };
-};

web/src/components/chat_search/AssistantSelector.tsx

@@ -28,13 +28,12 @@ import { DraggableAssistantCard } from "@/components/assistants/AssistantCards";
 import { updateUserAssistantList } from "@/lib/assistants/updateAssistantPreferences";
 
 import Text from "@/components/ui/text";
-import { getDisplayNameForModel, LlmOverrideManager } from "@/lib/hooks";
+import { LlmOverrideManager } from "@/lib/hooks";
 import { Tab } from "@headlessui/react";
 import { AssistantIcon } from "../assistants/AssistantIcon";
 import { restrictToVerticalAxis } from "@dnd-kit/modifiers";
 import { restrictToParentElement } from "@dnd-kit/modifiers";
 import { Drawer, DrawerContent, DrawerHeader, DrawerTitle } from "../ui/drawer";
-import { truncateString } from "@/lib/utils";
 
 const AssistantSelector = ({
   liveAssistant,
@@ -311,9 +310,7 @@ const AssistantSelector = ({
             <span className="font-bold">{liveAssistant.name}</span>
           </div>
           <div className="flex items-center">
-            <span className="mr-2 text-xs">
-              {truncateString(getDisplayNameForModel(currentLlm), 30)}
-            </span>
+            <span className="mr-2 text-xs">{currentLlm}</span>
             <FiChevronDown
               className={`w-5 h-5 text-white transition-transform duration-300 transform ${
                 isOpen ? "rotate-180" : ""

web/src/components/chat_search/sources/SourceCard.tsx

@@ -1,7 +1,6 @@
 import { WebResultIcon } from "@/components/WebResultIcon";
 import { SourceIcon } from "@/components/SourceIcon";
 import { DanswerDocument } from "@/lib/search/interfaces";
-import { truncateString } from "@/lib/utils";
 
 export default function SourceCard({ doc }: { doc: DanswerDocument }) {
   return (
@@ -18,7 +17,12 @@ export default function SourceCard({ doc }: { doc: DanswerDocument }) {
         ) : (
           <SourceIcon sourceType={doc.source_type} iconSize={18} />
         )}
-        <p>{truncateString(doc.semantic_identifier || doc.document_id, 12)}</p>
+        <p>
+          {(doc.semantic_identifier || doc.document_id).slice(0, 12).trim()}
+          {(doc.semantic_identifier || doc.document_id).length > 12 && (
+            <span className="text-text-500">...</span>
+          )}
+        </p>
       </div>
       <div className="line-clamp-2 text-sm font-semibold"></div>
       <div className="line-clamp-2 text-sm font-normal leading-snug text-text-700">

web/src/lib/utils.ts

@@ -4,7 +4,3 @@ import { twMerge } from "tailwind-merge";
 export function cn(...inputs: ClassValue[]) {
   return twMerge(clsx(inputs));
 }
-
-export const truncateString = (str: string, maxLength: number) => {
-  return str.length > maxLength ? str.slice(0, maxLength - 1) + "..." : str;
-};