Merge branch 'main' into yash-changes

backend/onyx/agents/agent_search/shared_graph_utils/llm.py

@@ -154,7 +154,7 @@ def invoke_llm_json(
         or []
     ) and supports_response_schema(llm.config.model_name, llm.config.model_provider)
 
-    response_content = str(
+    raw_response_content = str(
         llm.invoke(
             prompt,
             tools=tools,
@@ -167,6 +167,8 @@ def invoke_llm_json(
         ).content
     )
 
+    response_content = raw_response_content
+
     if not supports_json:
         # remove newlines as they often lead to json decoding errors
         response_content = response_content.replace("\n", " ")
@@ -177,9 +179,38 @@ def invoke_llm_json(
         else:
             first_bracket = response_content.find("{")
             last_bracket = response_content.rfind("}")
-            response_content = response_content[first_bracket : last_bracket + 1]
+            # Guard against missing braces to avoid creating an empty string
+            if first_bracket == -1 or last_bracket == -1 or last_bracket < first_bracket:
+                response_content = ""
+            else:
+                response_content = response_content[first_bracket : last_bracket + 1]
+    # Final validation with a robust fallback for common schemas
+    try:
+        if response_content and response_content.strip():
+            return schema.model_validate_json(response_content)
+    except Exception:
+        # If JSON parsing fails below, we'll attempt a fallback
+        pass
 
-    return schema.model_validate_json(response_content)
+    # Fallback: if schema expects DecisionResponse and model didn't return JSON,
+    # create a sensible default using raw content as reasoning
+    try:
+        if schema.__name__ == "DecisionResponse":
+            from onyx.agents.agent_search.dr.models import DecisionResponse  # local import to avoid cycles
+
+            return cast(SchemaType, DecisionResponse(
+                reasoning=(raw_response_content or ""),
+                decision="LLM",
+            ))
+    except Exception:
+        # If even fallback construction fails, raise the original error
+        pass
+
+    # If we get here, raise a descriptive error including a snippet of the content
+    snippet = (raw_response_content or "").strip()[:200]
+    raise ValueError(
+        f"Failed to parse JSON response for schema {schema.__name__}. Content snippet: {snippet}"
+    )
 
 
 def get_answer_from_llm(

backend/onyx/llm/llm_provider_options.py

@@ -5,6 +5,8 @@ from pydantic import BaseModel
 
 from onyx.llm.chat_llm import VERTEX_CREDENTIALS_FILE_KWARG
 from onyx.llm.chat_llm import VERTEX_LOCATION_KWARG
+import requests
+from typing import List, Optional
 from onyx.llm.utils import model_supports_image_input
 from onyx.server.manage.llm.models import ModelConfigurationView
 
@@ -154,12 +156,40 @@ VERTEXAI_VISIBLE_MODEL_NAMES = [
     VERTEXAI_DEFAULT_FAST_MODEL,
 ]
 
+# Ollama Provider Configuration
+OLLAMA_PROVIDER_NAME = "ollama"
+OLLAMA_DEFAULT_MODEL = "gemma3:1b"
+OLLAMA_DEFAULT_FAST_MODEL = "gemma3:1b"
+OLLAMA_MODEL_NAMES = [
+    "llama3",
+    "llama3:8b",
+    "llama3:70b",
+    "mistral",
+    "mixtral",
+    "codellama",
+    "gemma:2b",
+    "gemma:7b",
+    "gemma3:1b",
+    "smollm"
+]
+OLLAMA_VISIBLE_MODEL_NAMES = [
+    "llama3",
+    "llama3:8b",
+    "mistral",
+    "mixtral",
+    "gemma:2b",
+    "gemma:7b",
+    "gemma3:1b",
+    "smollm"
+]
+
 
 _PROVIDER_TO_MODELS_MAP = {
     OPENAI_PROVIDER_NAME: OPEN_AI_MODEL_NAMES,
     BEDROCK_PROVIDER_NAME: BEDROCK_MODEL_NAMES,
     ANTHROPIC_PROVIDER_NAME: ANTHROPIC_MODEL_NAMES,
     VERTEXAI_PROVIDER_NAME: VERTEXAI_MODEL_NAMES,
+    OLLAMA_PROVIDER_NAME: OLLAMA_MODEL_NAMES,
 }
 
 _PROVIDER_TO_VISIBLE_MODELS_MAP = {
@@ -167,11 +197,54 @@ _PROVIDER_TO_VISIBLE_MODELS_MAP = {
     BEDROCK_PROVIDER_NAME: [BEDROCK_DEFAULT_MODEL],
     ANTHROPIC_PROVIDER_NAME: ANTHROPIC_VISIBLE_MODEL_NAMES,
     VERTEXAI_PROVIDER_NAME: VERTEXAI_VISIBLE_MODEL_NAMES,
+    OLLAMA_PROVIDER_NAME: OLLAMA_VISIBLE_MODEL_NAMES,
 }
 
 
+def fetch_available_ollama_models(api_base: Optional[str] = None) -> List[str]:
+    """Fetch available models from Ollama API.
+    
+    Args:
+        api_base: Base URL of the Ollama server. If None, uses default localhost:11434.
+        
+    Returns:
+        List of available model names.
+    """
+    base_url = (api_base or "http://localhost:11434").rstrip('/')
+    try:
+        response = requests.get(f"{base_url}/api/tags", timeout=10)
+        response.raise_for_status()
+        models_data = response.json()
+        return [model["name"] for model in models_data.get("models", [])]
+    except requests.exceptions.RequestException as e:
+        print(f"Error connecting to Ollama API at {base_url}: {e}")
+    except Exception as e:
+        print(f"Unexpected error fetching Ollama models: {e}")
+    # Return default models if API call fails
+    return OLLAMA_MODEL_NAMES
+
 def fetch_available_well_known_llms() -> list[WellKnownLLMProviderDescriptor]:
+    """Fetch all well-known LLM provider configurations."""
     return [
+        # Ollama Provider
+        WellKnownLLMProviderDescriptor(
+            name=OLLAMA_PROVIDER_NAME,
+            display_name="Ollama",
+            api_key_required=False,
+            api_base_required=True,
+            api_version_required=False,
+            # api_base is already a top-level field for providers. Do not also
+            # surface it as a custom_config key to avoid duplicate fields /
+            # validation in the UI.
+            custom_config_keys=[],
+            model_configurations=fetch_model_configurations_for_provider(
+                OLLAMA_PROVIDER_NAME
+            ),
+            default_model=OLLAMA_DEFAULT_MODEL,
+            default_fast_model=OLLAMA_DEFAULT_FAST_MODEL,
+            deployment_name_required=False,
+            single_model_supported=False,
+        ),
         WellKnownLLMProviderDescriptor(
             name=OPENAI_PROVIDER_NAME,
             display_name="OpenAI",
@@ -307,9 +380,28 @@ def fetch_visible_model_names_for_provider_as_set(
 def fetch_model_configurations_for_provider(
     provider_name: str,
 ) -> list[ModelConfigurationView]:
-    # if there are no explicitly listed visible model names,
-    # then we won't mark any of them as "visible". This will get taken
-    # care of by the logic to make default models visible.
+    # For Ollama, we want to fetch the actual available models
+    if provider_name == OLLAMA_PROVIDER_NAME:
+        try:
+            # Get the actual models from the Ollama server
+            ollama_models = fetch_available_ollama_models()
+            if ollama_models:
+                return [
+                    ModelConfigurationView(
+                        name=model_name,
+                        is_visible=True,  # Make all Ollama models visible
+                        max_input_tokens=4096,  # Default context window
+                        supports_image_input=model_supports_image_input(
+                            model_name=model_name,
+                            model_provider=provider_name,
+                        ),
+                    )
+                    for model_name in ollama_models
+                ]
+        except Exception as e:
+            print(f"Error fetching Ollama models: {e}")
+    
+    # Fallback for other providers
     visible_model_names = (
         fetch_visible_model_names_for_provider_as_set(provider_name) or set()
     )

backend/onyx/server/manage/llm/api.py

@@ -30,6 +30,7 @@ from onyx.llm.factory import get_max_input_tokens_from_llm_provider
 from onyx.llm.llm_provider_options import BEDROCK_MODEL_NAMES
 from onyx.llm.llm_provider_options import fetch_available_well_known_llms
 from onyx.llm.llm_provider_options import WellKnownLLMProviderDescriptor
+from onyx.llm.llm_provider_options import fetch_available_ollama_models
 from onyx.llm.utils import get_llm_contextual_cost
 from onyx.llm.utils import litellm_exception_to_error_msg
 from onyx.llm.utils import model_supports_image_input
@@ -325,6 +326,33 @@ def get_vision_capable_providers(
 """Endpoints for all"""
 
 
+@admin_router.get("/ollama/models")
+async def fetch_ollama_models(
+    api_base: str = Query(
+        "http://localhost:11434",
+        description="URL of the Ollama server",
+    ),
+    _: User | None = Depends(current_admin_user),
+) -> list[str]:
+    """
+    Fetch available models from an Ollama server.
+    
+    Args:
+        api_base: Base URL of the Ollama server (e.g., http://localhost:11434)
+        
+    Returns:
+        List of available model names on the Ollama server
+    """
+    try:
+        return fetch_available_ollama_models(api_base=api_base)
+    except Exception as e:
+        logger.error(f"Error fetching Ollama models: {str(e)}")
+        raise HTTPException(
+            status_code=400,
+            detail=f"Failed to fetch models from Ollama server at {api_base}: {str(e)}",
+        )
+
+
 @basic_router.get("/provider")
 def list_llm_provider_basics(
     user: User | None = Depends(current_chat_accessible_user),

web/src/app/admin/configuration/llm/FetchOllamaModelsButton.tsx

@@ -0,0 +1,79 @@
+import { Button } from "@/components/ui/button";
+import { FiRefreshCw } from "react-icons/fi";
+import { useState } from "react";
+import { PopupSpec } from "@/components/admin/connectors/Popup";
+import { LoadingAnimation } from "@/components/Loading";
+
+interface FetchOllamaModelsButtonProps {
+  apiBase: string;
+  setModels: (models: string[]) => void;
+  setPopup: (popup: PopupSpec) => void;
+  disabled?: boolean;
+}
+
+export function FetchOllamaModelsButton({
+  apiBase,
+  setModels,
+  setPopup,
+  disabled = false,
+}: FetchOllamaModelsButtonProps) {
+  const [isFetching, setIsFetching] = useState(false);
+
+  const fetchModels = async () => {
+    // Normalize API base: trim and ensure scheme
+    let base = (apiBase || "").trim();
+    if (base && !/^https?:\/\//i.test(base)) {
+      base = `http://${base}`;
+    }
+
+    if (!base) {
+      setPopup({
+        message: "Please enter an Ollama server URL first",
+        type: "error",
+      });
+      return;
+    }
+
+    setIsFetching(true);
+    try {
+      const response = await fetch(`/api/admin/llm/ollama/models?api_base=${encodeURIComponent(base)}`);
+      if (!response.ok) {
+        const errorData = await response.json();
+        throw new Error(errorData.detail || "Failed to fetch models");
+      }
+      const models = await response.json();
+      setModels(models);
+      setPopup({
+        message: `Found ${models.length} model(s)`,
+        type: "success",
+      });
+    } catch (error) {
+      console.error("Error fetching Ollama models:", error);
+      setPopup({
+        message: `Error fetching models: ${error instanceof Error ? error.message : String(error)}`,
+        type: "error",
+      });
+    } finally {
+      setIsFetching(false);
+    }
+  };
+
+  return (
+    <Button
+      onClick={fetchModels}
+      disabled={disabled || isFetching}
+      className="flex items-center gap-2"
+      type="button"
+      variant="outline"
+    >
+      {isFetching ? (
+        <LoadingAnimation text="" />
+      ) : (
+        <>
+          <FiRefreshCw className="h-4 w-4" />
+          Fetch Available Models
+        </>
+      )}
+    </Button>
+  );
+}

web/src/app/admin/configuration/llm/LLMProviderUpdateForm.tsx

@@ -24,6 +24,7 @@ import { PopupSpec } from "@/components/admin/connectors/Popup";
 import * as Yup from "yup";
 import isEqual from "lodash/isEqual";
 import { IsPublicGroupSelector } from "@/components/IsPublicGroupSelector";
+import { FetchOllamaModelsButton } from "./FetchOllamaModelsButton";
 
 export function LLMProviderUpdateForm({
   llmProviderDescriptor,
@@ -48,9 +49,12 @@ export function LLMProviderUpdateForm({
 
   const [isTesting, setIsTesting] = useState(false);
   const [testError, setTestError] = useState<string>("");
+
+  const [availableModels, setAvailableModels] = useState<string[]>([]);
   const [isFetchingModels, setIsFetchingModels] = useState(false);
   const [fetchModelsError, setFetchModelsError] = useState<string>("");
 
+
   const [showAdvancedOptions, setShowAdvancedOptions] = useState(false);
 
   // Define the initial values based on the provider's requirements
@@ -294,7 +298,7 @@ export function LLMProviderUpdateForm({
         } = values;
 
         // For Azure OpenAI, parse target_uri to extract api_base and api_version
-        let finalApiBase = rest.api_base;
+        let finalApiBase = rest.api_base?.trim();
         let finalApiVersion = rest.api_version;
 
         if (llmProviderDescriptor.name === "azure" && target_uri) {
@@ -308,16 +312,30 @@ export function LLMProviderUpdateForm({
           }
         }
 
+        // Normalize api_base: ensure scheme for providers like Ollama
+        if (finalApiBase) {
+          const hasScheme = /^https?:\/\//i.test(finalApiBase);
+          if (!hasScheme) {
+            finalApiBase = `http://${finalApiBase}`;
+          }
+        }
+
         // Create the final payload with proper typing
+        // Persist models from fetched list when available (e.g., Ollama), otherwise use descriptor
+        const modelNamesForSave: string[] =
+          availableModels.length > 0
+            ? availableModels
+            : llmProviderDescriptor.model_configurations.map((m) => m.name);
+
         const finalValues = {
           ...rest,
           api_base: finalApiBase,
           api_version: finalApiVersion,
           api_key_changed: values.api_key !== initialValues.api_key,
-          model_configurations: llmProviderDescriptor.model_configurations.map(
-            (modelConfiguration): ModelConfigurationUpsertRequest => ({
-              name: modelConfiguration.name,
-              is_visible: visibleModels.includes(modelConfiguration.name),
+          model_configurations: modelNamesForSave.map(
+            (name): ModelConfigurationUpsertRequest => ({
+              name,
+              is_visible: visibleModels.includes(name),
               max_input_tokens: null,
             })
           ),
@@ -557,14 +575,18 @@ export function LLMProviderUpdateForm({
                   name="default_model_name"
                   subtext="The model to use by default for this provider unless otherwise specified."
                   label="Default Model"
-                  options={llmProviderDescriptor.model_configurations.map(
-                    (modelConfiguration) => ({
-                      // don't clean up names here to give admins descriptive names / handle duplicates
-                      // like us.anthropic.claude-3-7-sonnet-20250219-v1:0 and anthropic.claude-3-7-sonnet-20250219-v1:0
-                      name: modelConfiguration.name,
-                      value: modelConfiguration.name,
-                    })
-                  )}
+                  options={(
+                    availableModels.length > 0
+                      ? availableModels
+                      : llmProviderDescriptor.model_configurations.map(
+                          (m) => m.name
+                        )
+                  ).map((name) => ({
+                    // don't clean up names here to give admins descriptive names / handle duplicates
+                    // like us.anthropic.claude-3-7-sonnet-20250219-v1:0 and anthropic.claude-3-7-sonnet-20250219-v1:0
+                    name,
+                    value: name,
+                  }))}
                   maxHeight="max-h-56"
                 />
               ) : (
@@ -576,6 +598,31 @@ export function LLMProviderUpdateForm({
                 />
               )}
 
+              {llmProviderDescriptor.name === "ollama" && (
+                <div className="mt-2">
+                  <FetchOllamaModelsButton
+                    apiBase={
+                      // Use the api_base field; if empty, default to localhost
+                      (formikProps.values as any).api_base || "http://localhost:11434"
+                    }
+                    setModels={(models) => {
+                      setAvailableModels(models);
+                      const values: any = formikProps.values;
+                      // Ensure default_model_name is part of the list
+                      if (!models.includes(values.default_model_name)) {
+                        formikProps.setFieldValue(
+                          "default_model_name",
+                          models[0] || ""
+                        );
+                      }
+                      // Update selected_model_names to the fetched models by default
+                      formikProps.setFieldValue("selected_model_names", models);
+                    }}
+                    setPopup={(p) => (setPopup ? setPopup(p) : undefined)}
+                  />
+                </div>
+              )}
+
               {llmProviderDescriptor.deployment_name_required && (
                 <TextFormField
                   name="deployment_name"
@@ -592,14 +639,18 @@ export function LLMProviderUpdateForm({
             for this provider. If \`Default\` is specified, will use
             the Default Model configured above.`}
                     label="[Optional] Fast Model"
-                    options={llmProviderDescriptor.model_configurations.map(
-                      (modelConfiguration) => ({
-                        // don't clean up names here to give admins descriptive names / handle duplicates
-                        // like us.anthropic.claude-3-7-sonnet-20250219-v1:0 and anthropic.claude-3-7-sonnet-20250219-v1:0
-                        name: modelConfiguration.name,
-                        value: modelConfiguration.name,
-                      })
-                    )}
+                    options={(
+                      availableModels.length > 0
+                        ? availableModels
+                        : llmProviderDescriptor.model_configurations.map(
+                            (m) => m.name
+                          )
+                    ).map((name) => ({
+                      // don't clean up names here to give admins descriptive names / handle duplicates
+                      // like us.anthropic.claude-3-7-sonnet-20250219-v1:0 and anthropic.claude-3-7-sonnet-20250219-v1:0
+                      name,
+                      value: name,
+                    }))}
                     includeDefault
                     maxHeight="max-h-56"
                   />
@@ -626,19 +677,23 @@ export function LLMProviderUpdateForm({
                       <div className="w-full">
                         <MultiSelectField
                           selectedInitially={
-                            formikProps.values.selected_model_names ?? []
+                            (formikProps.values as any).selected_model_names ?? []
                           }
                           name="selected_model_names"
                           label="Display Models"
                           subtext="Select the models to make available to users. Unselected models will not be available."
-                          options={llmProviderDescriptor.model_configurations.map(
-                            (modelConfiguration) => ({
-                              value: modelConfiguration.name,
-                              // don't clean up names here to give admins descriptive names / handle duplicates
-                              // like us.anthropic.claude-3-7-sonnet-20250219-v1:0 and anthropic.claude-3-7-sonnet-20250219-v1:0
-                              label: modelConfiguration.name,
-                            })
-                          )}
+                          options={(
+                            availableModels.length > 0
+                              ? availableModels
+                              : llmProviderDescriptor.model_configurations.map(
+                                  (m) => m.name
+                                )
+                          ).map((name) => ({
+                            value: name,
+                            // don't clean up names here to give admins descriptive names / handle duplicates
+                            // like us.anthropic.claude-3-7-sonnet-20250219-v1:0 and anthropic.claude-3-7-sonnet-20250219-v1:0
+                            label: name,
+                          }))}
                           onChange={(selected) =>
                             formikProps.setFieldValue(
                               "selected_model_names",

web/src/app/chat/components/ChatPage.tsx

@@ -773,7 +773,8 @@ export function ChatPage({
     <>
       <HealthCheckBanner />
 
-      {showApiKeyModal && !shouldShowWelcomeModal && (
+     
+    {showApiKeyModal && (
         <ApiKeyModal
           hide={() => setShowApiKeyModal(false)}
           setPopup={setPopup}

web/src/app/chat/components/input/ChatInputBar.tsx

@@ -660,7 +660,11 @@ export const ChatInputBar = React.memo(function ChatInputBar({
                 <LLMPopover
                   llmProviders={llmProviders}
                   llmManager={llmManager}
+
+                  requiresImageGeneration={llmManager.imageFilesPresent}
+
                   requiresImageGeneration={false}
+
                   currentAssistant={selectedAssistant}
                 />