order seeding

* validate dropdown

* validate

* update organization

* move to utils

backend/danswer/db/users.py

@@ -103,17 +103,6 @@ def list_users(
     return db_session.scalars(stmt).unique().all()
 
 
-def get_users_by_emails(
-    db_session: Session, emails: list[str]
-) -> tuple[list[User], list[str]]:
-    # Use distinct to avoid duplicates
-    stmt = select(User).filter(User.email.in_(emails))  # type: ignore
-    found_users = list(db_session.scalars(stmt).unique().all())  # Convert to list
-    found_users_emails = [user.email for user in found_users]
-    missing_user_emails = [email for email in emails if email not in found_users_emails]
-    return found_users, missing_user_emails
-
-
 def get_user_by_email(email: str, db_session: Session) -> User | None:
     user = (
         db_session.query(User)
@@ -128,7 +117,7 @@ def fetch_user_by_id(db_session: Session, user_id: UUID) -> User | None:
     return db_session.query(User).filter(User.id == user_id).first()  # type: ignore
 
 
-def _generate_non_web_slack_user(email: str) -> User:
+def _generate_slack_user(email: str) -> User:
     fastapi_users_pw_helper = PasswordHelper()
     password = fastapi_users_pw_helper.generate()
     hashed_pass = fastapi_users_pw_helper.hash(password)
@@ -149,13 +138,29 @@ def add_slack_user_if_not_exists(db_session: Session, email: str) -> User:
             db_session.commit()
         return user
 
-    user = _generate_non_web_slack_user(email=email)
+    user = _generate_slack_user(email=email)
     db_session.add(user)
     db_session.commit()
     return user
 
 
-def _generate_non_web_permissioned_user(email: str) -> User:
+def _get_users_by_emails(
+    db_session: Session, lower_emails: list[str]
+) -> tuple[list[User], list[str]]:
+    stmt = select(User).filter(func.lower(User.email).in_(lower_emails))  # type: ignore
+    found_users = list(db_session.scalars(stmt).unique().all())  # Convert to list
+
+    # Extract found emails and convert to lowercase to avoid case sensitivity issues
+    found_users_emails = [user.email.lower() for user in found_users]
+
+    # Separate emails for users that were not found
+    missing_user_emails = [
+        email for email in lower_emails if email not in found_users_emails
+    ]
+    return found_users, missing_user_emails
+
+
+def _generate_ext_permissioned_user(email: str) -> User:
     fastapi_users_pw_helper = PasswordHelper()
     password = fastapi_users_pw_helper.generate()
     hashed_pass = fastapi_users_pw_helper.hash(password)
@@ -169,12 +174,12 @@ def _generate_non_web_permissioned_user(email: str) -> User:
 def batch_add_ext_perm_user_if_not_exists(
     db_session: Session, emails: list[str]
 ) -> list[User]:
-    emails = [email.lower() for email in emails]
-    found_users, missing_user_emails = get_users_by_emails(db_session, emails)
+    lower_emails = [email.lower() for email in emails]
+    found_users, missing_lower_emails = _get_users_by_emails(db_session, lower_emails)
 
     new_users: list[User] = []
-    for email in missing_user_emails:
-        new_users.append(_generate_non_web_permissioned_user(email=email))
+    for email in missing_lower_emails:
+        new_users.append(_generate_ext_permissioned_user(email=email))
 
     db_session.add_all(new_users)
     db_session.commit()

backend/danswer/seeding/load_yamls.py

@@ -81,6 +81,7 @@ def load_personas_from_yaml(
 
         p_id = persona.get("id")
         tool_ids = []
+
         if persona.get("image_generation"):
             image_gen_tool = (
                 db_session.query(ToolDBModel)

backend/danswer/setup.py

@@ -254,13 +254,14 @@ def setup_postgres(db_session: Session) -> None:
     create_initial_public_credential(db_session)
     create_initial_default_connector(db_session)
     associate_default_cc_pair(db_session)
-
-    logger.notice("Loading default Prompts and Personas")
     delete_old_default_personas(db_session)
-    load_chat_yamls(db_session)
 
     logger.notice("Loading built-in tools")
     load_builtin_tools(db_session)
+
+    logger.notice("Loading default Prompts and Personas")
+    load_chat_yamls(db_session)
+
     refresh_built_in_tools_cache(db_session)
     auto_add_search_tool_to_personas(db_session)
 

backend/ee/danswer/external_permissions/confluence/doc_sync.py

@@ -228,6 +228,16 @@ def _fetch_all_page_restrictions_for_space(
                     external_access=space_permissions,
                 )
             )
+            if (
+                not space_permissions.is_public
+                and not space_permissions.external_user_emails
+                and not space_permissions.external_user_group_ids
+            ):
+                logger.warning(
+                    f"Permissions are empty for document: {slim_doc.id}\n"
+                    "This means space permissions are may be wrong for"
+                    f" Space key: {space_key}"
+                )
             continue
 
         logger.warning(f"No permissions found for document {slim_doc.id}")

deployment/data/flower/celeryconfig.py

@@ -1,7 +0,0 @@
-broker_url = "redis://cache:6379/15"
-broker_result_backend = "redis://cache:6379/14"
-broker_transport_options = {
-    "priority_steps": [0, 1, 2, 3, 4],
-    "sep": ":",
-    "queue_order_strategy": "priority",
-}

deployment/data/flower/tasks.py

@@ -1,4 +0,0 @@
-from celery import Celery
-
-app = Celery("flower")
-app.config_from_object("celeryconfig")

deployment/docker_compose/docker-compose.dev.yml

@@ -386,18 +386,6 @@ services:
     # persistence. explicitly setting save and appendonly forces ephemeral behavior.
     command: redis-server --save "" --appendonly no
 
-  # web app to monitor and perform basic management of celery
-  flower:
-    profiles:
-      - beta
-    image: mher/flower:master
-    command: celery -A tasks.app flower
-    volumes:
-      - ../data/flower:/data
-    working_dir: /data
-    ports:
-      - 5555:5555
-      
 volumes:
   db_volume:
   vespa_volume: # Created by the container itself

web/public/Gemini.svg

@@ -0,0 +1 @@
+<svg fill="none" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 16"><path d="M16 8.016A8.522 8.522 0 008.016 16h-.032A8.521 8.521 0 000 8.016v-.032A8.521 8.521 0 007.984 0h.032A8.522 8.522 0 0016 7.984v.032z" fill="url(#prefix__paint0_radial_980_20147)"/><defs><radialGradient id="prefix__paint0_radial_980_20147" cx="0" cy="0" r="1" gradientUnits="userSpaceOnUse" gradientTransform="matrix(16.1326 5.4553 -43.70045 129.2322 1.588 6.503)"><stop offset=".067" stop-color="#9168C0"/><stop offset=".343" stop-color="#5684D1"/><stop offset=".672" stop-color="#1BA1E3"/></radialGradient></defs></svg>

web/src/app/admin/configuration/llm/interfaces.ts

@@ -4,6 +4,7 @@ import {
   AzureIcon,
   CPUIcon,
   OpenAIIcon,
+  GeminiIcon,
   OpenSourceIcon,
 } from "@/components/icons/icons";
 import { FaRobot } from "react-icons/fa";
@@ -67,10 +68,17 @@ export interface LLMProviderDescriptor {
   display_model_names: string[] | null;
 }
 
-export const getProviderIcon = (providerName: string) => {
+export const getProviderIcon = (providerName: string, modelName?: string) => {
   switch (providerName) {
     case "openai":
-      return OpenAIIcon;
+      // Special cases for openai based on modelName
+      if (modelName?.toLowerCase().includes("gemini")) {
+        return GeminiIcon;
+      }
+      if (modelName?.toLowerCase().includes("claude")) {
+        return AnthropicIcon;
+      }
+      return OpenAIIcon; // Default for openai
     case "anthropic":
       return AnthropicIcon;
     case "bedrock":

web/src/app/chat/ChatPage.tsx

@@ -259,7 +259,7 @@ export function ChatPage({
     refreshRecentAssistants,
   } = useAssistants();
 
-  const liveAssistant =
+  const liveAssistant: Persona | undefined =
     alternativeAssistant ||
     selectedAssistant ||
     recentAssistants[0] ||
@@ -269,6 +269,7 @@ export function ChatPage({
   const noAssistants = liveAssistant == null || liveAssistant == undefined;
   // always set the model override for the chat session, when an assistant, llm provider, or user preference exists
   useEffect(() => {
+    if (noAssistants) return;
     const personaDefault = getLLMProviderOverrideForPersona(
       liveAssistant,
       llmProviders
@@ -753,7 +754,7 @@ export function ChatPage({
   useEffect(() => {
     async function fetchMaxTokens() {
       const response = await fetch(
-        `/api/chat/max-selected-document-tokens?persona_id=${liveAssistant.id}`
+        `/api/chat/max-selected-document-tokens?persona_id=${liveAssistant?.id}`
       );
       if (response.ok) {
         const maxTokens = (await response.json()).max_tokens as number;
@@ -1809,18 +1810,23 @@ export function ChatPage({
       });
     };
   }
+  if (noAssistants)
+    return (
+      <>
+        <HealthCheckBanner />
+        <NoAssistantModal isAdmin={isAdmin} />
+      </>
+    );
 
   return (
     <>
       <HealthCheckBanner />
 
-      {showApiKeyModal && !shouldShowWelcomeModal ? (
+      {showApiKeyModal && !shouldShowWelcomeModal && (
         <ApiKeyModal
           hide={() => setShowApiKeyModal(false)}
           setPopup={setPopup}
         />
-      ) : (
-        noAssistants && <NoAssistantModal isAdmin={isAdmin} />
       )}
 
       {/* ChatPopup is a custom popup that displays a admin-specified message on initial user visit. 

web/src/app/ee/admin/cloud-settings/BillingInformationPage.tsx

@@ -17,25 +17,13 @@ import { useEffect } from "react";
 export default function BillingInformationPage() {
   const router = useRouter();
   const { popup, setPopup } = usePopup();
-  const stripePromise = loadStripe(
-    process.env.NEXT_PUBLIC_STRIPE_PUBLISHABLE_KEY!
-  );
 
   const {
     data: billingInformation,
     error,
     isLoading,
-    refreshBillingInformation,
   } = useBillingInformation();
 
-  const [seats, setSeats] = useState<number>(1);
-
-  useEffect(() => {
-    if (billingInformation?.seats) {
-      setSeats(billingInformation.seats);
-    }
-  }, [billingInformation?.seats]);
-
   if (error) {
     console.error("Failed to fetch billing information:", error);
   }
@@ -66,7 +54,9 @@ export default function BillingInformationPage() {
       if (!response.ok) {
         const errorData = await response.json();
         throw new Error(
-          `Failed to create customer portal session: ${errorData.message || response.statusText}`
+          `Failed to create customer portal session: ${
+            errorData.message || response.statusText
+          }`
         );
       }
 

web/src/components/Dropdown.tsx

@@ -10,6 +10,8 @@ import {
 import { ChevronDownIcon } from "./icons/icons";
 import { FiCheck, FiChevronDown } from "react-icons/fi";
 import { Popover } from "./popover/Popover";
+import { createPortal } from "react-dom";
+import { useDropdownPosition } from "@/lib/dropdown";
 
 export interface Option<T> {
   name: string;
@@ -60,6 +62,7 @@ export function SearchMultiSelectDropdown({
   const [isOpen, setIsOpen] = useState(false);
   const [searchTerm, setSearchTerm] = useState("");
   const dropdownRef = useRef<HTMLDivElement>(null);
+  const dropdownMenuRef = useRef<HTMLDivElement>(null);
 
   const handleSelect = (option: StringOrNumberOption) => {
     onSelect(option);
@@ -75,7 +78,9 @@ export function SearchMultiSelectDropdown({
     const handleClickOutside = (event: MouseEvent) => {
       if (
         dropdownRef.current &&
-        !dropdownRef.current.contains(event.target as Node)
+        !dropdownRef.current.contains(event.target as Node) &&
+        dropdownMenuRef.current &&
+        !dropdownMenuRef.current.contains(event.target as Node)
       ) {
         setIsOpen(false);
       }
@@ -87,105 +92,103 @@ export function SearchMultiSelectDropdown({
     };
   }, []);
 
+  useDropdownPosition({ isOpen, dropdownRef, dropdownMenuRef });
+
   return (
-    <div className="relative inline-block text-left w-full" ref={dropdownRef}>
+    <div className="relative text-left w-full" ref={dropdownRef}>
       <div>
         <input
           type="text"
           placeholder="Search..."
           value={searchTerm}
           onChange={(e: ChangeEvent<HTMLInputElement>) => {
-            if (!searchTerm) {
+            setSearchTerm(e.target.value);
+            if (e.target.value) {
               setIsOpen(true);
-            }
-            if (!e.target.value) {
+            } else {
               setIsOpen(false);
             }
-            setSearchTerm(e.target.value);
           }}
           onFocus={() => setIsOpen(true)}
           className={`inline-flex 
-          justify-between 
-          w-full 
-          px-4 
-          py-2 
-          text-sm 
-          bg-background
-          border
-          border-border
-          rounded-md 
-          shadow-sm 
-          `}
-          onClick={(e) => e.stopPropagation()}
+            justify-between 
+            w-full 
+            px-4 
+            py-2 
+            text-sm 
+            bg-background
+            border
+            border-border
+            rounded-md 
+            shadow-sm 
+            `}
         />
         <button
           type="button"
           className={`absolute top-0 right-0 
-            text-sm 
-            h-full px-2 border-l border-border`}
-          aria-expanded="true"
+              text-sm 
+              h-full px-2 border-l border-border`}
+          aria-expanded={isOpen}
           aria-haspopup="true"
           onClick={() => setIsOpen(!isOpen)}
         >
-          <ChevronDownIcon className="my-auto" />
+          <ChevronDownIcon className="my-auto w-4 h-4" />
         </button>
       </div>
 
-      {isOpen && (
-        <div
-          className={`origin-top-right
-            absolute
-            left-0
-            mt-3
-            w-full
-            rounded-md
-            shadow-lg
-            bg-background
-            border
-            border-border
-            max-h-80
-            overflow-y-auto
-            overscroll-contain`}
-        >
+      {isOpen &&
+        createPortal(
           <div
-            role="menu"
-            aria-orientation="vertical"
-            aria-labelledby="options-menu"
+            ref={dropdownMenuRef}
+            className={`origin-top-right
+                rounded-md
+                shadow-lg
+                bg-background
+                border
+                border-border
+                max-h-80
+                overflow-y-auto
+                overscroll-contain`}
           >
-            {filteredOptions.length ? (
-              filteredOptions.map((option, index) =>
-                itemComponent ? (
-                  <div
-                    key={option.name}
-                    onClick={() => {
-                      setIsOpen(false);
-                      handleSelect(option);
-                    }}
-                  >
-                    {itemComponent({ option })}
-                  </div>
-                ) : (
-                  <StandardDropdownOption
-                    key={index}
-                    option={option}
-                    index={index}
-                    handleSelect={handleSelect}
-                  />
+            <div
+              role="menu"
+              aria-orientation="vertical"
+              aria-labelledby="options-menu"
+            >
+              {filteredOptions.length ? (
+                filteredOptions.map((option, index) =>
+                  itemComponent ? (
+                    <div
+                      key={option.name}
+                      onClick={() => {
+                        handleSelect(option);
+                      }}
+                    >
+                      {itemComponent({ option })}
+                    </div>
+                  ) : (
+                    <StandardDropdownOption
+                      key={index}
+                      option={option}
+                      index={index}
+                      handleSelect={handleSelect}
+                    />
+                  )
                 )
-              )
-            ) : (
-              <button
-                key={0}
-                className={`w-full text-left block px-4 py-2.5 text-sm hover:bg-hover`}
-                role="menuitem"
-                onClick={() => setIsOpen(false)}
-              >
-                No matches found...
-              </button>
-            )}
-          </div>
-        </div>
-      )}
+              ) : (
+                <button
+                  key={0}
+                  className={`w-full text-left block px-4 py-2.5 text-sm hover:bg-hover`}
+                  role="menuitem"
+                  onClick={() => setIsOpen(false)}
+                >
+                  No matches found...
+                </button>
+              )}
+            </div>
+          </div>,
+          document.body
+        )}
     </div>
   );
 }

web/src/components/Modal.tsx

@@ -66,11 +66,21 @@ export function Modal({
             e.stopPropagation();
           }
         }}
-        className={`bg-background text-emphasis rounded shadow-2xl 
-          transform transition-all duration-300 ease-in-out
+        className={`
+          bg-background 
+          text-emphasis 
+          rounded 
+          shadow-2xl 
+          transform 
+          transition-all 
+          duration-300 
+          ease-in-out
+          relative
+          overflow-visible
           ${width ?? "w-11/12 max-w-4xl"}
           ${noPadding ? "" : "p-10"}
-          ${className || ""}`}
+          ${className || ""}
+        `}
       >
         {onOutsideClick && !hideCloseButton && (
           <div className="absolute top-2 right-2">

web/src/components/icons/icons.tsx

@@ -39,6 +39,7 @@ import Image, { StaticImageData } from "next/image";
 import jiraSVG from "../../../public/Jira.svg";
 import confluenceSVG from "../../../public/Confluence.svg";
 import openAISVG from "../../../public/Openai.svg";
+import geminiSVG from "../../../public/Gemini.svg";
 import openSourceIcon from "../../../public/OpenSource.png";
 import litellmIcon from "../../../public/LiteLLM.jpg";
 
@@ -1096,6 +1097,11 @@ export const OpenAIIcon = ({
   className = defaultTailwindCSS,
 }: IconProps) => <LogoIcon size={size} className={className} src={openAISVG} />;
 
+export const GeminiIcon = ({
+  size = 16,
+  className = defaultTailwindCSS,
+}: IconProps) => <LogoIcon size={size} className={className} src={geminiSVG} />;
+
 export const VoyageIcon = ({
   size = 16,
   className = defaultTailwindCSS,

web/src/components/llm/LLMList.tsx

@@ -49,7 +49,7 @@ export const LlmList: React.FC<LlmListProps> = ({
               llmProvider.provider,
               modelName
             ),
-            icon: getProviderIcon(llmProvider.provider),
+            icon: getProviderIcon(llmProvider.provider, modelName),
           });
         }
       }

web/src/lib/dropdown.ts

@@ -0,0 +1,49 @@
+import { RefObject, useCallback, useEffect } from "react";
+
+interface DropdownPositionProps {
+  isOpen: boolean;
+  dropdownRef: RefObject<HTMLElement>;
+  dropdownMenuRef: RefObject<HTMLElement>;
+}
+
+// This hook manages the positioning of a dropdown menu relative to its trigger element.
+// It ensures the menu is positioned correctly, adjusting for viewport boundaries and scroll position.
+// Also adds event listeners for window resize and scroll to update the position dynamically.
+export const useDropdownPosition = ({
+  isOpen,
+  dropdownRef,
+  dropdownMenuRef,
+}: DropdownPositionProps) => {
+  const updateMenuPosition = useCallback(() => {
+    if (isOpen && dropdownRef.current && dropdownMenuRef.current) {
+      const rect = dropdownRef.current.getBoundingClientRect();
+      const menuRect = dropdownMenuRef.current.getBoundingClientRect();
+      const viewportHeight = window.innerHeight;
+
+      let top = rect.bottom + window.scrollY;
+
+      if (top + menuRect.height > viewportHeight) {
+        top = rect.top + window.scrollY - menuRect.height;
+      }
+
+      dropdownMenuRef.current.style.position = "absolute";
+      dropdownMenuRef.current.style.top = `${top}px`;
+      dropdownMenuRef.current.style.left = `${rect.left + window.scrollX}px`;
+      dropdownMenuRef.current.style.width = `${rect.width}px`;
+      dropdownMenuRef.current.style.zIndex = "10000";
+    }
+  }, [isOpen, dropdownRef, dropdownMenuRef]);
+
+  useEffect(() => {
+    updateMenuPosition();
+    window.addEventListener("resize", updateMenuPosition);
+    window.addEventListener("scroll", updateMenuPosition);
+
+    return () => {
+      window.removeEventListener("resize", updateMenuPosition);
+      window.removeEventListener("scroll", updateMenuPosition);
+    };
+  }, [isOpen, updateMenuPosition]);
+
+  return updateMenuPosition;
+};

web/src/lib/hooks.ts

@@ -292,6 +292,16 @@ const MODEL_DISPLAY_NAMES: { [key: string]: string } = {
   "claude-instant-1.2": "Claude Instant 1.2",
   "claude-3-5-sonnet-20240620": "Claude 3.5 Sonnet",
   "claude-3-5-sonnet-20241022": "Claude 3.5 Sonnet (New)",
+  "claude-3-5-sonnet-v2@20241022": "Claude 3.5 Sonnet (New)",
+  "claude-3.5-sonnet-v2@20241022": "Claude 3.5 Sonnet (New)",
+
+  // Google Models
+  "gemini-1.5-pro": "Gemini 1.5 Pro",
+  "gemini-1.5-flash": "Gemini 1.5 Flash",
+  "gemini-1.5-pro-001": "Gemini 1.5 Pro",
+  "gemini-1.5-flash-001": "Gemini 1.5 Flash",
+  "gemini-1.5-pro-002": "Gemini 1.5 Pro (v2)",
+  "gemini-1.5-flash-002": "Gemini 1.5 Flash (v2)",
 
   // Bedrock models
   "meta.llama3-1-70b-instruct-v1:0": "Llama 3.1 70B",

web/src/lib/llm/utils.ts

@@ -86,6 +86,13 @@ const MODEL_NAMES_SUPPORTING_IMAGE_INPUT = [
   "anthropic.claude-3-haiku-20240307-v1:0",
   "anthropic.claude-3-5-sonnet-20240620-v1:0",
   "anthropic.claude-3-5-sonnet-20241022-v2:0",
+  // google gemini model names
+  "gemini-1.5-pro",
+  "gemini-1.5-flash",
+  "gemini-1.5-pro-001",
+  "gemini-1.5-flash-001",
+  "gemini-1.5-pro-002",
+  "gemini-1.5-flash-002",
 ];
 
 export function checkLLMSupportsImageInput(model: string) {