order seeding

* validate dropdown

* validate

* update organization

* move to utils

backend/danswer/context/search/pipeline.py

@@ -135,7 +135,6 @@ class SearchPipeline:
 
     """Retrieval and Postprocessing"""
 
-    @log_function_time(print_only=True)
     def _get_chunks(self) -> list[InferenceChunk]:
         if self._retrieved_chunks is not None:
             return self._retrieved_chunks
@@ -307,7 +306,6 @@ class SearchPipeline:
         return expanded_inference_sections
 
     @property
-    @log_function_time(print_only=True)
     def reranked_sections(self) -> list[InferenceSection]:
         """Reranking is always done at the chunk level since section merging could create arbitrarily
         long sections which could be:
@@ -333,7 +331,6 @@ class SearchPipeline:
         return self._reranked_sections
 
     @property
-    @log_function_time(print_only=True)
     def final_context_sections(self) -> list[InferenceSection]:
         if self._final_context_sections is not None:
             return self._final_context_sections
@@ -342,7 +339,6 @@ class SearchPipeline:
         return self._final_context_sections
 
     @property
-    @log_function_time(print_only=True)
     def section_relevance(self) -> list[SectionRelevancePiece] | None:
         if self._section_relevance is not None:
             return self._section_relevance
@@ -397,7 +393,6 @@ class SearchPipeline:
         return self._section_relevance
 
     @property
-    @log_function_time(print_only=True)
     def section_relevance_list(self) -> list[bool]:
         llm_indices = relevant_sections_to_indices(
             relevance_sections=self.section_relevance,

backend/danswer/context/search/postprocessing/postprocessing.py

@@ -42,7 +42,6 @@ def _log_top_section_links(search_flow: str, sections: list[InferenceSection]) -
     logger.debug(f"Top links from {search_flow} search: {', '.join(top_links)}")
 
 
-@log_function_time(print_only=True)
 def cleanup_chunks(chunks: list[InferenceChunkUncleaned]) -> list[InferenceChunk]:
     def _remove_title(chunk: InferenceChunkUncleaned) -> str:
         if not chunk.title or not chunk.content:
@@ -245,7 +244,6 @@ def filter_sections(
     ]
 
 
-@log_function_time(print_only=True)
 def search_postprocessing(
     search_query: SearchQuery,
     retrieved_sections: list[InferenceSection],

backend/danswer/context/search/retrieval/search_runner.py

@@ -1,5 +1,4 @@
 import string
-import time
 from collections.abc import Callable
 
 import nltk  # type:ignore
@@ -86,7 +85,6 @@ def remove_stop_words_and_punctuation(keywords: list[str]) -> list[str]:
         return keywords
 
 
-@log_function_time(print_only=True)
 def combine_retrieval_results(
     chunk_sets: list[list[InferenceChunk]],
 ) -> list[InferenceChunk]:
@@ -258,13 +256,7 @@ def retrieve_chunks(
                     (q_copy, document_index, db_session),
                 )
             )
-
-        start_time = time.time()
         parallel_search_results = run_functions_tuples_in_parallel(run_queries)
-        end_time = time.time()
-        logger.info(
-            f"Parallel search execution took {end_time - start_time:.2f} seconds"
-        )
         top_chunks = combine_retrieval_results(parallel_search_results)
 
     if not top_chunks:

backend/danswer/context/search/utils.py

@@ -8,7 +8,6 @@ from danswer.context.search.models import SavedSearchDoc
 from danswer.context.search.models import SavedSearchDocWithContent
 from danswer.context.search.models import SearchDoc
 from danswer.db.models import SearchDoc as DBSearchDoc
-from danswer.utils.timing import log_function_time
 
 
 T = TypeVar(
@@ -89,7 +88,6 @@ def drop_llm_indices(
     return [i for i, val in enumerate(llm_bools) if val]
 
 
-@log_function_time(print_only=True)
 def inference_section_from_chunks(
     center_chunk: InferenceChunk,
     chunks: list[InferenceChunk],

backend/danswer/document_index/vespa/chunk_retrieval.py

@@ -48,7 +48,6 @@ from danswer.document_index.vespa_constants import TITLE
 from danswer.document_index.vespa_constants import YQL_BASE
 from danswer.utils.logger import setup_logger
 from danswer.utils.threadpool_concurrency import run_functions_tuples_in_parallel
-from danswer.utils.timing import log_function_time
 
 logger = setup_logger()
 
@@ -147,7 +146,6 @@ def _vespa_hit_to_inference_chunk(
     )
 
 
-@log_function_time(print_only=True)
 def _get_chunks_via_visit_api(
     chunk_request: VespaChunkRequest,
     index_name: str,
@@ -234,7 +232,6 @@ def _get_chunks_via_visit_api(
 
 
 @retry(tries=10, delay=1, backoff=2)
-@log_function_time(print_only=True)
 def get_all_vespa_ids_for_document_id(
     document_id: str,
     index_name: str,
@@ -251,7 +248,6 @@ def get_all_vespa_ids_for_document_id(
     return [chunk["id"].split("::", 1)[-1] for chunk in document_chunks]
 
 
-@log_function_time(print_only=True)
 def parallel_visit_api_retrieval(
     index_name: str,
     chunk_requests: list[VespaChunkRequest],
@@ -266,12 +262,9 @@ def parallel_visit_api_retrieval(
         for chunk_request in chunk_requests
     ]
 
-    start_time = datetime.now()
     parallel_results = run_functions_tuples_in_parallel(
         functions_with_args, allow_failures=True
     )
-    duration = datetime.now() - start_time
-    print(f"Parallel visit API retrieval took {duration.total_seconds():.2f} seconds")
 
     # Any failures to retrieve would give a None, drop the Nones and empty lists
     vespa_chunk_sets = [res for res in parallel_results if res]
@@ -289,11 +282,9 @@ def parallel_visit_api_retrieval(
 
 
 @retry(tries=3, delay=1, backoff=2)
-@log_function_time(print_only=True)
 def query_vespa(
     query_params: Mapping[str, str | int | float]
 ) -> list[InferenceChunkUncleaned]:
-    print(f"query_params: {query_params}")
     if "query" in query_params and not cast(str, query_params["query"]).strip():
         raise ValueError("No/empty query received")
 
@@ -349,7 +340,6 @@ def query_vespa(
     return inference_chunks
 
 
-@log_function_time(print_only=True)
 def _get_chunks_via_batch_search(
     index_name: str,
     chunk_requests: list[VespaChunkRequest],
@@ -384,7 +374,6 @@ def _get_chunks_via_batch_search(
     return inference_chunks
 
 
-@log_function_time(print_only=True)
 def batch_search_api_retrieval(
     index_name: str,
     chunk_requests: list[VespaChunkRequest],

backend/danswer/document_index/vespa/index.py

@@ -72,7 +72,6 @@ from danswer.indexing.models import DocMetadataAwareIndexChunk
 from danswer.key_value_store.factory import get_kv_store
 from danswer.utils.batching import batch_generator
 from danswer.utils.logger import setup_logger
-from danswer.utils.timing import log_function_time
 from shared_configs.configs import MULTI_TENANT
 from shared_configs.model_server_models import Embedding
 
@@ -661,7 +660,6 @@ class VespaIndex(DocumentIndex):
 
         return total_chunks_deleted
 
-    @log_function_time(print_only=True)
     def id_based_retrieval(
         self,
         chunk_requests: list[VespaChunkRequest],
@@ -683,7 +681,6 @@ class VespaIndex(DocumentIndex):
             get_large_chunks=get_large_chunks,
         )
 
-    @log_function_time(print_only=True)
     def hybrid_retrieval(
         self,
         query: str,

backend/danswer/llm/answering/prune_and_merge.py

@@ -21,7 +21,6 @@ from danswer.natural_language_processing.utils import tokenizer_trim_content
 from danswer.prompts.prompt_utils import build_doc_context_str
 from danswer.tools.tool_implementations.search.search_utils import section_to_dict
 from danswer.utils.logger import setup_logger
-from danswer.utils.timing import log_function_time
 
 
 logger = setup_logger()
@@ -44,7 +43,6 @@ class ChunkRange(BaseModel):
     end: int
 
 
-@log_function_time(print_only=True)
 def merge_chunk_intervals(chunk_ranges: list[ChunkRange]) -> list[ChunkRange]:
     """
     This acts on a single document to merge the overlapping ranges of chunks
@@ -302,7 +300,6 @@ def prune_sections(
     )
 
 
-@log_function_time(print_only=True)
 def _merge_doc_chunks(chunks: list[InferenceChunk]) -> InferenceSection:
     # Assuming there are no duplicates by this point
     sorted_chunks = sorted(chunks, key=lambda x: x.chunk_id)
@@ -330,7 +327,6 @@ def _merge_doc_chunks(chunks: list[InferenceChunk]) -> InferenceSection:
     )
 
 
-@log_function_time(print_only=True)
 def _merge_sections(sections: list[InferenceSection]) -> list[InferenceSection]:
     docs_map: dict[str, dict[int, InferenceChunk]] = defaultdict(dict)
     doc_order: dict[str, int] = {}

backend/danswer/secondary_llm_flows/query_expansion.py

@@ -15,7 +15,6 @@ from danswer.prompts.miscellaneous_prompts import LANGUAGE_REPHRASE_PROMPT
 from danswer.utils.logger import setup_logger
 from danswer.utils.text_processing import count_punctuation
 from danswer.utils.threadpool_concurrency import run_functions_tuples_in_parallel
-from danswer.utils.timing import log_function_time
 
 logger = setup_logger()
 
@@ -49,7 +48,6 @@ def llm_multilingual_query_expansion(query: str, language: str) -> str:
     return model_output
 
 
-@log_function_time(print_only=True)
 def multilingual_query_expansion(
     query: str,
     expansion_languages: list[str],

backend/danswer/seeding/load_yamls.py

@@ -81,6 +81,7 @@ def load_personas_from_yaml(
 
         p_id = persona.get("id")
         tool_ids = []
+
         if persona.get("image_generation"):
             image_gen_tool = (
                 db_session.query(ToolDBModel)

backend/danswer/setup.py

@@ -38,6 +38,7 @@ from danswer.key_value_store.interface import KvKeyNotFoundError
 from danswer.natural_language_processing.search_nlp_models import EmbeddingModel
 from danswer.natural_language_processing.search_nlp_models import warm_up_bi_encoder
 from danswer.natural_language_processing.search_nlp_models import warm_up_cross_encoder
+from danswer.seeding.load_docs import seed_initial_documents
 from danswer.seeding.load_yamls import load_chat_yamls
 from danswer.server.manage.llm.models import LLMProviderUpsertRequest
 from danswer.server.settings.store import load_settings
@@ -149,7 +150,7 @@ def setup_danswer(
     # update multipass indexing setting based on GPU availability
     update_default_multipass_indexing(db_session)
 
-    # seed_initial_documents(db_session, tenant_id, cohere_enabled)
+    seed_initial_documents(db_session, tenant_id, cohere_enabled)
 
 
 def translate_saved_search_settings(db_session: Session) -> None:
@@ -253,13 +254,14 @@ def setup_postgres(db_session: Session) -> None:
     create_initial_public_credential(db_session)
     create_initial_default_connector(db_session)
     associate_default_cc_pair(db_session)
-
-    logger.notice("Loading default Prompts and Personas")
     delete_old_default_personas(db_session)
-    load_chat_yamls(db_session)
 
     logger.notice("Loading built-in tools")
     load_builtin_tools(db_session)
+
+    logger.notice("Loading default Prompts and Personas")
+    load_chat_yamls(db_session)
+
     refresh_built_in_tools_cache(db_session)
     auto_add_search_tool_to_personas(db_session)
 

web/src/app/chat/ChatPage.tsx

@@ -259,7 +259,7 @@ export function ChatPage({
     refreshRecentAssistants,
   } = useAssistants();
 
-  const liveAssistant =
+  const liveAssistant: Persona | undefined =
     alternativeAssistant ||
     selectedAssistant ||
     recentAssistants[0] ||
@@ -269,6 +269,7 @@ export function ChatPage({
   const noAssistants = liveAssistant == null || liveAssistant == undefined;
   // always set the model override for the chat session, when an assistant, llm provider, or user preference exists
   useEffect(() => {
+    if (noAssistants) return;
     const personaDefault = getLLMProviderOverrideForPersona(
       liveAssistant,
       llmProviders
@@ -753,7 +754,7 @@ export function ChatPage({
   useEffect(() => {
     async function fetchMaxTokens() {
       const response = await fetch(
-        `/api/chat/max-selected-document-tokens?persona_id=${liveAssistant.id}`
+        `/api/chat/max-selected-document-tokens?persona_id=${liveAssistant?.id}`
       );
       if (response.ok) {
         const maxTokens = (await response.json()).max_tokens as number;
@@ -1809,18 +1810,23 @@ export function ChatPage({
       });
     };
   }
+  if (noAssistants)
+    return (
+      <>
+        <HealthCheckBanner />
+        <NoAssistantModal isAdmin={isAdmin} />
+      </>
+    );
 
   return (
     <>
       <HealthCheckBanner />
 
-      {showApiKeyModal && !shouldShowWelcomeModal ? (
+      {showApiKeyModal && !shouldShowWelcomeModal && (
         <ApiKeyModal
           hide={() => setShowApiKeyModal(false)}
           setPopup={setPopup}
         />
-      ) : (
-        noAssistants && <NoAssistantModal isAdmin={isAdmin} />
       )}
 
       {/* ChatPopup is a custom popup that displays a admin-specified message on initial user visit. 

web/src/components/Dropdown.tsx

@@ -10,6 +10,8 @@ import {
 import { ChevronDownIcon } from "./icons/icons";
 import { FiCheck, FiChevronDown } from "react-icons/fi";
 import { Popover } from "./popover/Popover";
+import { createPortal } from "react-dom";
+import { useDropdownPosition } from "@/lib/dropdown";
 
 export interface Option<T> {
   name: string;
@@ -60,6 +62,7 @@ export function SearchMultiSelectDropdown({
   const [isOpen, setIsOpen] = useState(false);
   const [searchTerm, setSearchTerm] = useState("");
   const dropdownRef = useRef<HTMLDivElement>(null);
+  const dropdownMenuRef = useRef<HTMLDivElement>(null);
 
   const handleSelect = (option: StringOrNumberOption) => {
     onSelect(option);
@@ -75,7 +78,9 @@ export function SearchMultiSelectDropdown({
     const handleClickOutside = (event: MouseEvent) => {
       if (
         dropdownRef.current &&
-        !dropdownRef.current.contains(event.target as Node)
+        !dropdownRef.current.contains(event.target as Node) &&
+        dropdownMenuRef.current &&
+        !dropdownMenuRef.current.contains(event.target as Node)
       ) {
         setIsOpen(false);
       }
@@ -87,105 +92,103 @@ export function SearchMultiSelectDropdown({
     };
   }, []);
 
+  useDropdownPosition({ isOpen, dropdownRef, dropdownMenuRef });
+
   return (
-    <div className="relative inline-block text-left w-full" ref={dropdownRef}>
+    <div className="relative text-left w-full" ref={dropdownRef}>
       <div>
         <input
           type="text"
           placeholder="Search..."
           value={searchTerm}
           onChange={(e: ChangeEvent<HTMLInputElement>) => {
-            if (!searchTerm) {
+            setSearchTerm(e.target.value);
+            if (e.target.value) {
               setIsOpen(true);
-            }
-            if (!e.target.value) {
+            } else {
               setIsOpen(false);
             }
-            setSearchTerm(e.target.value);
           }}
           onFocus={() => setIsOpen(true)}
           className={`inline-flex 
-          justify-between 
-          w-full 
-          px-4 
-          py-2 
-          text-sm 
-          bg-background
-          border
-          border-border
-          rounded-md 
-          shadow-sm 
-          `}
-          onClick={(e) => e.stopPropagation()}
+            justify-between 
+            w-full 
+            px-4 
+            py-2 
+            text-sm 
+            bg-background
+            border
+            border-border
+            rounded-md 
+            shadow-sm 
+            `}
         />
         <button
           type="button"
           className={`absolute top-0 right-0 
-            text-sm 
-            h-full px-2 border-l border-border`}
-          aria-expanded="true"
+              text-sm 
+              h-full px-2 border-l border-border`}
+          aria-expanded={isOpen}
           aria-haspopup="true"
           onClick={() => setIsOpen(!isOpen)}
         >
-          <ChevronDownIcon className="my-auto" />
+          <ChevronDownIcon className="my-auto w-4 h-4" />
         </button>
       </div>
 
-      {isOpen && (
-        <div
-          className={`origin-top-right
-            absolute
-            left-0
-            mt-3
-            w-full
-            rounded-md
-            shadow-lg
-            bg-background
-            border
-            border-border
-            max-h-80
-            overflow-y-auto
-            overscroll-contain`}
-        >
+      {isOpen &&
+        createPortal(
           <div
-            role="menu"
-            aria-orientation="vertical"
-            aria-labelledby="options-menu"
+            ref={dropdownMenuRef}
+            className={`origin-top-right
+                rounded-md
+                shadow-lg
+                bg-background
+                border
+                border-border
+                max-h-80
+                overflow-y-auto
+                overscroll-contain`}
           >
-            {filteredOptions.length ? (
-              filteredOptions.map((option, index) =>
-                itemComponent ? (
-                  <div
-                    key={option.name}
-                    onClick={() => {
-                      setIsOpen(false);
-                      handleSelect(option);
-                    }}
-                  >
-                    {itemComponent({ option })}
-                  </div>
-                ) : (
-                  <StandardDropdownOption
-                    key={index}
-                    option={option}
-                    index={index}
-                    handleSelect={handleSelect}
-                  />
+            <div
+              role="menu"
+              aria-orientation="vertical"
+              aria-labelledby="options-menu"
+            >
+              {filteredOptions.length ? (
+                filteredOptions.map((option, index) =>
+                  itemComponent ? (
+                    <div
+                      key={option.name}
+                      onClick={() => {
+                        handleSelect(option);
+                      }}
+                    >
+                      {itemComponent({ option })}
+                    </div>
+                  ) : (
+                    <StandardDropdownOption
+                      key={index}
+                      option={option}
+                      index={index}
+                      handleSelect={handleSelect}
+                    />
+                  )
                 )
-              )
-            ) : (
-              <button
-                key={0}
-                className={`w-full text-left block px-4 py-2.5 text-sm hover:bg-hover`}
-                role="menuitem"
-                onClick={() => setIsOpen(false)}
-              >
-                No matches found...
-              </button>
-            )}
-          </div>
-        </div>
-      )}
+              ) : (
+                <button
+                  key={0}
+                  className={`w-full text-left block px-4 py-2.5 text-sm hover:bg-hover`}
+                  role="menuitem"
+                  onClick={() => setIsOpen(false)}
+                >
+                  No matches found...
+                </button>
+              )}
+            </div>
+          </div>,
+          document.body
+        )}
     </div>
   );
 }

web/src/components/Modal.tsx

@@ -66,11 +66,21 @@ export function Modal({
             e.stopPropagation();
           }
         }}
-        className={`bg-background text-emphasis rounded shadow-2xl 
-          transform transition-all duration-300 ease-in-out
+        className={`
+          bg-background 
+          text-emphasis 
+          rounded 
+          shadow-2xl 
+          transform 
+          transition-all 
+          duration-300 
+          ease-in-out
+          relative
+          overflow-visible
           ${width ?? "w-11/12 max-w-4xl"}
           ${noPadding ? "" : "p-10"}
-          ${className || ""}`}
+          ${className || ""}
+        `}
       >
         {onOutsideClick && !hideCloseButton && (
           <div className="absolute top-2 right-2">

web/src/lib/dropdown.ts

@@ -0,0 +1,49 @@
+import { RefObject, useCallback, useEffect } from "react";
+
+interface DropdownPositionProps {
+  isOpen: boolean;
+  dropdownRef: RefObject<HTMLElement>;
+  dropdownMenuRef: RefObject<HTMLElement>;
+}
+
+// This hook manages the positioning of a dropdown menu relative to its trigger element.
+// It ensures the menu is positioned correctly, adjusting for viewport boundaries and scroll position.
+// Also adds event listeners for window resize and scroll to update the position dynamically.
+export const useDropdownPosition = ({
+  isOpen,
+  dropdownRef,
+  dropdownMenuRef,
+}: DropdownPositionProps) => {
+  const updateMenuPosition = useCallback(() => {
+    if (isOpen && dropdownRef.current && dropdownMenuRef.current) {
+      const rect = dropdownRef.current.getBoundingClientRect();
+      const menuRect = dropdownMenuRef.current.getBoundingClientRect();
+      const viewportHeight = window.innerHeight;
+
+      let top = rect.bottom + window.scrollY;
+
+      if (top + menuRect.height > viewportHeight) {
+        top = rect.top + window.scrollY - menuRect.height;
+      }
+
+      dropdownMenuRef.current.style.position = "absolute";
+      dropdownMenuRef.current.style.top = `${top}px`;
+      dropdownMenuRef.current.style.left = `${rect.left + window.scrollX}px`;
+      dropdownMenuRef.current.style.width = `${rect.width}px`;
+      dropdownMenuRef.current.style.zIndex = "10000";
+    }
+  }, [isOpen, dropdownRef, dropdownMenuRef]);
+
+  useEffect(() => {
+    updateMenuPosition();
+    window.addEventListener("resize", updateMenuPosition);
+    window.addEventListener("scroll", updateMenuPosition);
+
+    return () => {
+      window.removeEventListener("resize", updateMenuPosition);
+      window.removeEventListener("scroll", updateMenuPosition);
+    };
+  }, [isOpen, updateMenuPosition]);
+
+  return updateMenuPosition;
+};