feat(opensearch): Enable by default (#9211) to release v3.0 (#9217)

Co-authored-by: acaprau <48705707+acaprau@users.noreply.github.com>

.github/workflows/pr-integration-tests.yml

@@ -316,6 +316,7 @@ jobs:
           # Base config shared by both editions
           cat <<EOF > deployment/docker_compose/.env
           COMPOSE_PROFILES=s3-filestore
+          OPENSEARCH_FOR_ONYX_ENABLED=false
           AUTH_TYPE=basic
           POSTGRES_POOL_PRE_PING=true
           POSTGRES_USE_NULL_POOL=true
@@ -418,6 +419,7 @@ jobs:
               -e POSTGRES_POOL_PRE_PING=true \
               -e POSTGRES_USE_NULL_POOL=true \
               -e VESPA_HOST=index \
+              -e ENABLE_OPENSEARCH_INDEXING_FOR_ONYX=false \
               -e REDIS_HOST=cache \
               -e API_SERVER_HOST=api_server \
               -e OPENAI_API_KEY=${OPENAI_API_KEY} \
@@ -637,6 +639,7 @@ jobs:
           ONYX_BACKEND_IMAGE=${ECR_CACHE}:integration-test-backend-test-${RUN_ID} \
           ONYX_MODEL_SERVER_IMAGE=${ECR_CACHE}:integration-test-model-server-test-${RUN_ID} \
           DEV_MODE=true \
+          OPENSEARCH_FOR_ONYX_ENABLED=false \
           docker compose -f docker-compose.multitenant-dev.yml up \
             relational_db \
             index \
@@ -691,6 +694,7 @@ jobs:
             -e POSTGRES_DB=postgres \
             -e POSTGRES_USE_NULL_POOL=true \
             -e VESPA_HOST=index \
+            -e ENABLE_OPENSEARCH_INDEXING_FOR_ONYX=false \
             -e REDIS_HOST=cache \
             -e API_SERVER_HOST=api_server \
             -e OPENAI_API_KEY=${OPENAI_API_KEY} \

backend/onyx/configs/app_configs.py

@@ -288,8 +288,9 @@ OPENSEARCH_TEXT_ANALYZER = os.environ.get("OPENSEARCH_TEXT_ANALYZER") or "englis
 # environments we always want to be dual indexing into both OpenSearch and Vespa
 # to stress test the new codepaths. Only enable this if there is some instance
 # of OpenSearch running for the relevant Onyx instance.
+# NOTE: Now enabled on by default, unless the env indicates otherwise.
 ENABLE_OPENSEARCH_INDEXING_FOR_ONYX = (
-    os.environ.get("ENABLE_OPENSEARCH_INDEXING_FOR_ONYX", "").lower() == "true"
+    os.environ.get("ENABLE_OPENSEARCH_INDEXING_FOR_ONYX", "true").lower() == "true"
 )
 # NOTE: This effectively does nothing anymore, admins can now toggle whether
 # retrieval is through OpenSearch. This value is only used as a final fallback

deployment/docker_compose/docker-compose.multitenant-dev.yml

@@ -61,6 +61,9 @@ services:
       - POSTGRES_HOST=relational_db
       - POSTGRES_DEFAULT_SCHEMA=${POSTGRES_DEFAULT_SCHEMA:-}
       - VESPA_HOST=index
+      - OPENSEARCH_HOST=${OPENSEARCH_HOST:-opensearch}
+      - OPENSEARCH_ADMIN_PASSWORD=${OPENSEARCH_ADMIN_PASSWORD:-StrongPassword123!}
+      - ENABLE_OPENSEARCH_INDEXING_FOR_ONYX=${OPENSEARCH_FOR_ONYX_ENABLED:-true}
       - REDIS_HOST=cache
       - WEB_DOMAIN=${WEB_DOMAIN:-}
       # MinIO configuration
@@ -168,6 +171,9 @@ services:
       - POSTGRES_DB=${POSTGRES_DB:-}
       - POSTGRES_DEFAULT_SCHEMA=${POSTGRES_DEFAULT_SCHEMA:-}
       - VESPA_HOST=index
+      - OPENSEARCH_HOST=${OPENSEARCH_HOST:-opensearch}
+      - OPENSEARCH_ADMIN_PASSWORD=${OPENSEARCH_ADMIN_PASSWORD:-StrongPassword123!}
+      - ENABLE_OPENSEARCH_INDEXING_FOR_ONYX=${OPENSEARCH_FOR_ONYX_ENABLED:-true}
       - REDIS_HOST=cache
       - WEB_DOMAIN=${WEB_DOMAIN:-}
       # MinIO configuration
@@ -424,6 +430,50 @@ services:
         max-size: "50m"
         max-file: "6"
 
+  opensearch:
+    image: opensearchproject/opensearch:3.4.0
+    restart: unless-stopped
+    # Controls whether this service runs. In order to enable it, add
+    # opensearch-enabled to COMPOSE_PROFILES in the environment for this
+    # docker-compose.
+    # NOTE: Now enabled on by default. To explicitly disable this service,
+    # uncomment this profile and ensure COMPOSE_PROFILES in your env does not
+    # list the profile, or when running docker compose, include all desired
+    # service names but this one. Additionally set
+    # OPENSEARCH_FOR_ONYX_ENABLED=false in your env.
+    # profiles: ["opensearch-enabled"]
+    environment:
+      # We need discovery.type=single-node so that OpenSearch doesn't try
+      # forming a cluster and waiting for other nodes to become live.
+      - discovery.type=single-node
+      - OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_ADMIN_PASSWORD:-StrongPassword123!}
+      # This and the JVM config below come from the example in https://docs.opensearch.org/latest/install-and-configure/install-opensearch/docker/
+      # We do this to avoid unstable performance from page swaps.
+      - bootstrap.memory_lock=true # Disable JVM heap memory swapping.
+      # Java heap should be ~50% of memory limit. For now we assume a limit of
+      # 4g although in practice the container can request more than this.
+      # See https://opster.com/guides/opensearch/opensearch-basics/opensearch-heap-size-usage-and-jvm-garbage-collection/
+      # Xms is the starting size, Xmx is the maximum size. These should be the
+      # same.
+      - "OPENSEARCH_JAVA_OPTS=-Xms2g -Xmx2g"
+    volumes:
+      - opensearch-data:/usr/share/opensearch/data
+    # These come from the example in https://docs.opensearch.org/latest/install-and-configure/install-opensearch/docker/
+    ulimits:
+      # Similarly to bootstrap.memory_lock, we don't want to impose limits on
+      # how much memory a process can lock from being swapped.
+      memlock:
+        soft: -1 # Set memlock to unlimited (no soft or hard limit).
+        hard: -1
+      nofile:
+        soft: 65536 # Maximum number of open files for the opensearch user - set to at least 65536.
+        hard: 65536
+    logging:
+      driver: json-file
+      options:
+        max-size: "50m"
+        max-file: "6"
+
   nginx:
     image: nginx:1.25.5-alpine
     restart: unless-stopped
@@ -508,3 +558,5 @@ volumes:
   model_cache_huggingface:
   indexing_huggingface_model_cache:
   # mcp_server_logs:
+  # Persistent data for OpenSearch.
+  opensearch-data:

deployment/docker_compose/docker-compose.prod-cloud.yml

@@ -21,6 +21,9 @@ services:
       - AUTH_TYPE=${AUTH_TYPE:-oidc}
       - POSTGRES_HOST=relational_db
       - VESPA_HOST=index
+      - OPENSEARCH_HOST=${OPENSEARCH_HOST:-opensearch}
+      - OPENSEARCH_ADMIN_PASSWORD=${OPENSEARCH_ADMIN_PASSWORD:-StrongPassword123!}
+      - ENABLE_OPENSEARCH_INDEXING_FOR_ONYX=${OPENSEARCH_FOR_ONYX_ENABLED:-true}
       - REDIS_HOST=cache
       - MODEL_SERVER_HOST=${MODEL_SERVER_HOST:-inference_model_server}
       # MinIO configuration
@@ -55,6 +58,9 @@ services:
       - AUTH_TYPE=${AUTH_TYPE:-oidc}
       - POSTGRES_HOST=relational_db
       - VESPA_HOST=index
+      - OPENSEARCH_HOST=${OPENSEARCH_HOST:-opensearch}
+      - OPENSEARCH_ADMIN_PASSWORD=${OPENSEARCH_ADMIN_PASSWORD:-StrongPassword123!}
+      - ENABLE_OPENSEARCH_INDEXING_FOR_ONYX=${OPENSEARCH_FOR_ONYX_ENABLED:-true}
       - REDIS_HOST=cache
       - MODEL_SERVER_HOST=${MODEL_SERVER_HOST:-inference_model_server}
       - INDEXING_MODEL_SERVER_HOST=${INDEXING_MODEL_SERVER_HOST:-indexing_model_server}
@@ -228,6 +234,50 @@ services:
         max-size: "50m"
         max-file: "6"
 
+  opensearch:
+    image: opensearchproject/opensearch:3.4.0
+    restart: unless-stopped
+    # Controls whether this service runs. In order to enable it, add
+    # opensearch-enabled to COMPOSE_PROFILES in the environment for this
+    # docker-compose.
+    # NOTE: Now enabled on by default. To explicitly disable this service,
+    # uncomment this profile and ensure COMPOSE_PROFILES in your env does not
+    # list the profile, or when running docker compose, include all desired
+    # service names but this one. Additionally set
+    # OPENSEARCH_FOR_ONYX_ENABLED=false in your env.
+    # profiles: ["opensearch-enabled"]
+    environment:
+      # We need discovery.type=single-node so that OpenSearch doesn't try
+      # forming a cluster and waiting for other nodes to become live.
+      - discovery.type=single-node
+      - OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_ADMIN_PASSWORD:-StrongPassword123!}
+      # This and the JVM config below come from the example in https://docs.opensearch.org/latest/install-and-configure/install-opensearch/docker/
+      # We do this to avoid unstable performance from page swaps.
+      - bootstrap.memory_lock=true # Disable JVM heap memory swapping.
+      # Java heap should be ~50% of memory limit. For now we assume a limit of
+      # 4g although in practice the container can request more than this.
+      # See https://opster.com/guides/opensearch/opensearch-basics/opensearch-heap-size-usage-and-jvm-garbage-collection/
+      # Xms is the starting size, Xmx is the maximum size. These should be the
+      # same.
+      - "OPENSEARCH_JAVA_OPTS=-Xms2g -Xmx2g"
+    volumes:
+      - opensearch-data:/usr/share/opensearch/data
+    # These come from the example in https://docs.opensearch.org/latest/install-and-configure/install-opensearch/docker/
+    ulimits:
+      # Similarly to bootstrap.memory_lock, we don't want to impose limits on
+      # how much memory a process can lock from being swapped.
+      memlock:
+        soft: -1 # Set memlock to unlimited (no soft or hard limit).
+        hard: -1
+      nofile:
+        soft: 65536 # Maximum number of open files for the opensearch user - set to at least 65536.
+        hard: 65536
+    logging:
+      driver: json-file
+      options:
+        max-size: "50m"
+        max-file: "6"
+
   nginx:
     image: nginx:1.25.5-alpine
     restart: unless-stopped
@@ -315,3 +365,5 @@ volumes:
   model_cache_huggingface:
   indexing_huggingface_model_cache:
   # mcp_server_logs:
+  # Persistent data for OpenSearch.
+  opensearch-data:

deployment/docker_compose/docker-compose.prod-no-letsencrypt.yml

@@ -21,6 +21,9 @@ services:
       - AUTH_TYPE=${AUTH_TYPE:-oidc}
       - POSTGRES_HOST=relational_db
       - VESPA_HOST=index
+      - OPENSEARCH_HOST=${OPENSEARCH_HOST:-opensearch}
+      - OPENSEARCH_ADMIN_PASSWORD=${OPENSEARCH_ADMIN_PASSWORD:-StrongPassword123!}
+      - ENABLE_OPENSEARCH_INDEXING_FOR_ONYX=${OPENSEARCH_FOR_ONYX_ENABLED:-true}
       - REDIS_HOST=cache
       - MODEL_SERVER_HOST=${MODEL_SERVER_HOST:-inference_model_server}
       - USE_IAM_AUTH=${USE_IAM_AUTH}
@@ -68,6 +71,9 @@ services:
       - AUTH_TYPE=${AUTH_TYPE:-oidc}
       - POSTGRES_HOST=relational_db
       - VESPA_HOST=index
+      - OPENSEARCH_HOST=${OPENSEARCH_HOST:-opensearch}
+      - OPENSEARCH_ADMIN_PASSWORD=${OPENSEARCH_ADMIN_PASSWORD:-StrongPassword123!}
+      - ENABLE_OPENSEARCH_INDEXING_FOR_ONYX=${OPENSEARCH_FOR_ONYX_ENABLED:-true}
       - REDIS_HOST=cache
       - MODEL_SERVER_HOST=${MODEL_SERVER_HOST:-inference_model_server}
       - INDEXING_MODEL_SERVER_HOST=${INDEXING_MODEL_SERVER_HOST:-indexing_model_server}
@@ -251,6 +257,50 @@ services:
         max-size: "50m"
         max-file: "6"
 
+  opensearch:
+    image: opensearchproject/opensearch:3.4.0
+    restart: unless-stopped
+    # Controls whether this service runs. In order to enable it, add
+    # opensearch-enabled to COMPOSE_PROFILES in the environment for this
+    # docker-compose.
+    # NOTE: Now enabled on by default. To explicitly disable this service,
+    # uncomment this profile and ensure COMPOSE_PROFILES in your env does not
+    # list the profile, or when running docker compose, include all desired
+    # service names but this one. Additionally set
+    # OPENSEARCH_FOR_ONYX_ENABLED=false in your env.
+    # profiles: ["opensearch-enabled"]
+    environment:
+      # We need discovery.type=single-node so that OpenSearch doesn't try
+      # forming a cluster and waiting for other nodes to become live.
+      - discovery.type=single-node
+      - OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_ADMIN_PASSWORD:-StrongPassword123!}
+      # This and the JVM config below come from the example in https://docs.opensearch.org/latest/install-and-configure/install-opensearch/docker/
+      # We do this to avoid unstable performance from page swaps.
+      - bootstrap.memory_lock=true # Disable JVM heap memory swapping.
+      # Java heap should be ~50% of memory limit. For now we assume a limit of
+      # 4g although in practice the container can request more than this.
+      # See https://opster.com/guides/opensearch/opensearch-basics/opensearch-heap-size-usage-and-jvm-garbage-collection/
+      # Xms is the starting size, Xmx is the maximum size. These should be the
+      # same.
+      - "OPENSEARCH_JAVA_OPTS=-Xms2g -Xmx2g"
+    volumes:
+      - opensearch-data:/usr/share/opensearch/data
+    # These come from the example in https://docs.opensearch.org/latest/install-and-configure/install-opensearch/docker/
+    ulimits:
+      # Similarly to bootstrap.memory_lock, we don't want to impose limits on
+      # how much memory a process can lock from being swapped.
+      memlock:
+        soft: -1 # Set memlock to unlimited (no soft or hard limit).
+        hard: -1
+      nofile:
+        soft: 65536 # Maximum number of open files for the opensearch user - set to at least 65536.
+        hard: 65536
+    logging:
+      driver: json-file
+      options:
+        max-size: "50m"
+        max-file: "6"
+
   nginx:
     image: nginx:1.25.5-alpine
     restart: unless-stopped
@@ -343,3 +393,5 @@ volumes:
   # mcp_server_logs:
   # Shared volume for persistent document storage (Craft file-system mode)
   file-system:
+  # Persistent data for OpenSearch.
+  opensearch-data:

deployment/docker_compose/docker-compose.prod.yml

@@ -22,6 +22,9 @@ services:
       - AUTH_TYPE=${AUTH_TYPE:-oidc}
       - POSTGRES_HOST=relational_db
       - VESPA_HOST=index
+      - OPENSEARCH_HOST=${OPENSEARCH_HOST:-opensearch}
+      - OPENSEARCH_ADMIN_PASSWORD=${OPENSEARCH_ADMIN_PASSWORD:-StrongPassword123!}
+      - ENABLE_OPENSEARCH_INDEXING_FOR_ONYX=${OPENSEARCH_FOR_ONYX_ENABLED:-true}
       - REDIS_HOST=cache
       - MODEL_SERVER_HOST=${MODEL_SERVER_HOST:-inference_model_server}
       - USE_IAM_AUTH=${USE_IAM_AUTH}
@@ -73,6 +76,9 @@ services:
       - AUTH_TYPE=${AUTH_TYPE:-oidc}
       - POSTGRES_HOST=relational_db
       - VESPA_HOST=index
+      - OPENSEARCH_HOST=${OPENSEARCH_HOST:-opensearch}
+      - OPENSEARCH_ADMIN_PASSWORD=${OPENSEARCH_ADMIN_PASSWORD:-StrongPassword123!}
+      - ENABLE_OPENSEARCH_INDEXING_FOR_ONYX=${OPENSEARCH_FOR_ONYX_ENABLED:-true}
       - REDIS_HOST=cache
       - MODEL_SERVER_HOST=${MODEL_SERVER_HOST:-inference_model_server}
       - INDEXING_MODEL_SERVER_HOST=${INDEXING_MODEL_SERVER_HOST:-indexing_model_server}
@@ -270,6 +276,50 @@ services:
         max-size: "50m"
         max-file: "6"
 
+  opensearch:
+    image: opensearchproject/opensearch:3.4.0
+    restart: unless-stopped
+    # Controls whether this service runs. In order to enable it, add
+    # opensearch-enabled to COMPOSE_PROFILES in the environment for this
+    # docker-compose.
+    # NOTE: Now enabled on by default. To explicitly disable this service,
+    # uncomment this profile and ensure COMPOSE_PROFILES in your env does not
+    # list the profile, or when running docker compose, include all desired
+    # service names but this one. Additionally set
+    # OPENSEARCH_FOR_ONYX_ENABLED=false in your env.
+    # profiles: ["opensearch-enabled"]
+    environment:
+      # We need discovery.type=single-node so that OpenSearch doesn't try
+      # forming a cluster and waiting for other nodes to become live.
+      - discovery.type=single-node
+      - OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_ADMIN_PASSWORD:-StrongPassword123!}
+      # This and the JVM config below come from the example in https://docs.opensearch.org/latest/install-and-configure/install-opensearch/docker/
+      # We do this to avoid unstable performance from page swaps.
+      - bootstrap.memory_lock=true # Disable JVM heap memory swapping.
+      # Java heap should be ~50% of memory limit. For now we assume a limit of
+      # 4g although in practice the container can request more than this.
+      # See https://opster.com/guides/opensearch/opensearch-basics/opensearch-heap-size-usage-and-jvm-garbage-collection/
+      # Xms is the starting size, Xmx is the maximum size. These should be the
+      # same.
+      - "OPENSEARCH_JAVA_OPTS=-Xms2g -Xmx2g"
+    volumes:
+      - opensearch-data:/usr/share/opensearch/data
+    # These come from the example in https://docs.opensearch.org/latest/install-and-configure/install-opensearch/docker/
+    ulimits:
+      # Similarly to bootstrap.memory_lock, we don't want to impose limits on
+      # how much memory a process can lock from being swapped.
+      memlock:
+        soft: -1 # Set memlock to unlimited (no soft or hard limit).
+        hard: -1
+      nofile:
+        soft: 65536 # Maximum number of open files for the opensearch user - set to at least 65536.
+        hard: 65536
+    logging:
+      driver: json-file
+      options:
+        max-size: "50m"
+        max-file: "6"
+
   nginx:
     image: nginx:1.25.5-alpine
     restart: unless-stopped
@@ -380,3 +430,5 @@ volumes:
   # mcp_server_logs:
   # Shared volume for persistent document storage (Craft file-system mode)
   file-system:
+  # Persistent data for OpenSearch.
+  opensearch-data:

deployment/docker_compose/docker-compose.yml

@@ -57,6 +57,9 @@ services:
         condition: service_started
       index:
         condition: service_started
+      opensearch:
+        condition: service_started
+        required: false
       cache:
         condition: service_started
       inference_model_server:
@@ -78,7 +81,7 @@ services:
       - VESPA_HOST=${VESPA_HOST:-index}
       - OPENSEARCH_HOST=${OPENSEARCH_HOST:-opensearch}
       - OPENSEARCH_ADMIN_PASSWORD=${OPENSEARCH_ADMIN_PASSWORD:-StrongPassword123!}
-      - ENABLE_OPENSEARCH_INDEXING_FOR_ONYX=${OPENSEARCH_FOR_ONYX_ENABLED:-false}
+      - ENABLE_OPENSEARCH_INDEXING_FOR_ONYX=${OPENSEARCH_FOR_ONYX_ENABLED:-true}
       - REDIS_HOST=${REDIS_HOST:-cache}
       - MODEL_SERVER_HOST=${MODEL_SERVER_HOST:-inference_model_server}
       - S3_ENDPOINT_URL=${S3_ENDPOINT_URL:-http://minio:9000}
@@ -139,11 +142,19 @@ services:
       - path: .env
         required: false
     depends_on:
-      - relational_db
-      - index
-      - cache
-      - inference_model_server
-      - indexing_model_server
+      relational_db:
+        condition: service_started
+      index:
+        condition: service_started
+      opensearch:
+        condition: service_started
+        required: false
+      cache:
+        condition: service_started
+      inference_model_server:
+        condition: service_started
+      indexing_model_server:
+        condition: service_started
     restart: unless-stopped
     environment:
       - FILE_STORE_BACKEND=${FILE_STORE_BACKEND:-s3}
@@ -151,7 +162,7 @@ services:
       - VESPA_HOST=${VESPA_HOST:-index}
       - OPENSEARCH_HOST=${OPENSEARCH_HOST:-opensearch}
       - OPENSEARCH_ADMIN_PASSWORD=${OPENSEARCH_ADMIN_PASSWORD:-StrongPassword123!}
-      - ENABLE_OPENSEARCH_INDEXING_FOR_ONYX=${OPENSEARCH_FOR_ONYX_ENABLED:-false}
+      - ENABLE_OPENSEARCH_INDEXING_FOR_ONYX=${OPENSEARCH_FOR_ONYX_ENABLED:-true}
       - REDIS_HOST=${REDIS_HOST:-cache}
       - MODEL_SERVER_HOST=${MODEL_SERVER_HOST:-inference_model_server}
       - INDEXING_MODEL_SERVER_HOST=${INDEXING_MODEL_SERVER_HOST:-indexing_model_server}
@@ -406,7 +417,12 @@ services:
     # Controls whether this service runs. In order to enable it, add
     # opensearch-enabled to COMPOSE_PROFILES in the environment for this
     # docker-compose.
-    profiles: ["opensearch-enabled"]
+    # NOTE: Now enabled on by default. To explicitly disable this service,
+    # uncomment this profile and ensure COMPOSE_PROFILES in your env does not
+    # list the profile, or when running docker compose, include all desired
+    # service names but this one. Additionally set
+    # OPENSEARCH_FOR_ONYX_ENABLED=false in your env.
+    # profiles: ["opensearch-enabled"]
     environment:
       # We need discovery.type=single-node so that OpenSearch doesn't try
       # forming a cluster and waiting for other nodes to become live.
@@ -416,11 +432,11 @@ services:
       # We do this to avoid unstable performance from page swaps.
       - bootstrap.memory_lock=true # Disable JVM heap memory swapping.
       # Java heap should be ~50% of memory limit. For now we assume a limit of
-      # 2g although in practice the container can request more than this.
+      # 4g although in practice the container can request more than this.
       # See https://opster.com/guides/opensearch/opensearch-basics/opensearch-heap-size-usage-and-jvm-garbage-collection/
       # Xms is the starting size, Xmx is the maximum size. These should be the
       # same.
-      - "OPENSEARCH_JAVA_OPTS=-Xms1g -Xmx1g"
+      - "OPENSEARCH_JAVA_OPTS=-Xms2g -Xmx2g"
     volumes:
       - opensearch-data:/usr/share/opensearch/data
     # These come from the example in https://docs.opensearch.org/latest/install-and-configure/install-opensearch/docker/

deployment/docker_compose/env.template

@@ -67,10 +67,8 @@ POSTGRES_PASSWORD=password
 ## remove s3-filestore from COMPOSE_PROFILES and set FILE_STORE_BACKEND=postgres.
 COMPOSE_PROFILES=s3-filestore
 FILE_STORE_BACKEND=s3
-## Settings for enabling OpenSearch. Uncomment these and comment out
-## COMPOSE_PROFILES above.
-# COMPOSE_PROFILES=s3-filestore,opensearch-enabled
-# OPENSEARCH_FOR_ONYX_ENABLED=true
+## Setting for enabling OpenSearch.
+OPENSEARCH_FOR_ONYX_ENABLED=true
 
 ## MinIO/S3 Configuration (only needed when FILE_STORE_BACKEND=s3)
 S3_ENDPOINT_URL=http://minio:9000

deployment/helm/charts/onyx/Chart.yaml

@@ -5,7 +5,7 @@ home: https://www.onyx.app/
 sources:
   - "https://github.com/onyx-dot-app/onyx"
 type: application
-version: 0.4.32
+version: 0.4.33
 appVersion: latest
 annotations:
   category: Productivity

deployment/helm/charts/onyx/values.yaml

@@ -76,7 +76,10 @@ vespa:
       memory: 32000Mi
 
 opensearch:
-  enabled: false
+  # Enabled by default. Override to false and set the appropriate env vars in
+  # the instance-specific values yaml if using AWS-managed OpenSearch, or simply
+  # override to false to entirely disable.
+  enabled: true
   # These values are passed to the opensearch subchart.
   # See https://github.com/opensearch-project/helm-charts/blob/main/charts/opensearch/values.yaml
 
@@ -1158,8 +1161,10 @@ auth:
   opensearch:
     # Enable or disable this secret entirely. Will remove from env var
     # configurations and remove any created secrets.
-    # Set to true when opensearch.enabled is true.
-    enabled: false
+    # Enabled by default. Override to false and set the appropriate env vars in
+    # the instance-specific values yaml if using AWS-managed OpenSearch, or
+    # simply override to false to entirely disable.
+    enabled: true
     # Overwrite the default secret name, ignored if existingSecret is defined.
     secretName: 'onyx-opensearch'
     # Use a secret specified elsewhere.

web/src/app/admin/bots/SlackBotCreationForm.tsx

@@ -7,7 +7,7 @@ import { SlackTokensForm } from "./SlackTokensForm";
 import * as SettingsLayouts from "@/layouts/settings-layouts";
 import { SvgSlack } from "@opal/icons";
 
-export const NewSlackBotForm = () => {
+export function NewSlackBotForm() {
   const [formValues] = useState({
     name: "",
     enabled: true,
@@ -19,7 +19,12 @@ export const NewSlackBotForm = () => {
 
   return (
     <SettingsLayouts.Root>
-      <SettingsLayouts.Header icon={SvgSlack} title="New Slack Bot" separator />
+      <SettingsLayouts.Header
+        icon={SvgSlack}
+        title="New Slack Bot"
+        separator
+        backButton
+      />
       <SettingsLayouts.Body>
         <CardSection>
           <div className="p-4">
@@ -33,4 +38,4 @@ export const NewSlackBotForm = () => {
       </SettingsLayouts.Body>
     </SettingsLayouts.Root>
   );
-};
+}

web/src/app/admin/bots/SlackBotUpdateForm.tsx

@@ -1,12 +1,12 @@
 "use client";
 
 import { toast } from "@/hooks/useToast";
-import { SlackBot, ValidSources } from "@/lib/types";
+import { SlackBot } from "@/lib/types";
 import { useRouter } from "next/navigation";
 import { useState, useEffect, useRef } from "react";
 import { updateSlackBotField } from "@/lib/updateSlackBotField";
 import { SlackTokensForm } from "./SlackTokensForm";
-import { SourceIcon } from "@/components/SourceIcon";
+
 import { EditableStringFieldDisplay } from "@/components/EditableStringFieldDisplay";
 import { deleteSlackBot } from "./new/lib";
 import GenericConfirmModal from "@/components/modals/GenericConfirmModal";
@@ -90,10 +90,7 @@ export const ExistingSlackBotForm = ({
     <div>
       <div className="flex items-center justify-between h-14">
         <div className="flex items-center gap-2">
-          <div className="my-auto">
-            <SourceIcon iconSize={32} sourceType={ValidSources.Slack} />
-          </div>
-          <div className="ml-1">
+          <div>
             <EditableStringFieldDisplay
               value={formValues.name}
               isEditable={true}

web/src/app/admin/bots/[bot-id]/channels/[id]/page.tsx

@@ -1,100 +1,122 @@
-import { SlackChannelConfigCreationForm } from "../SlackChannelConfigCreationForm";
-import { fetchSS } from "@/lib/utilsSS";
+"use client";
+
+import { use } from "react";
+import { SlackChannelConfigCreationForm } from "@/app/admin/bots/[bot-id]/channels/SlackChannelConfigCreationForm";
 import { ErrorCallout } from "@/components/ErrorCallout";
-import { DocumentSetSummary, SlackChannelConfig } from "@/lib/types";
-import { InstantSSRAutoRefresh } from "@/components/SSRAutoRefresh";
+import SimpleLoader from "@/refresh-components/loaders/SimpleLoader";
 import * as SettingsLayouts from "@/layouts/settings-layouts";
 import { SvgSlack } from "@opal/icons";
-import { FetchAgentsResponse, fetchAgentsSS } from "@/lib/agentsSS";
-import { getStandardAnswerCategoriesIfEE } from "@/components/standardAnswers/getStandardAnswerCategoriesIfEE";
+import { useSlackChannelConfigs } from "@/app/admin/bots/[bot-id]/hooks";
+import { useDocumentSets } from "@/app/admin/documents/sets/hooks";
+import { useAgents } from "@/hooks/useAgents";
+import { useStandardAnswerCategories } from "@/app/ee/admin/standard-answer/hooks";
+import { usePaidEnterpriseFeaturesEnabled } from "@/components/settings/usePaidEnterpriseFeaturesEnabled";
+import type { StandardAnswerCategoryResponse } from "@/components/standardAnswers/getStandardAnswerCategoriesIfEE";
 
-async function EditslackChannelConfigPage(props: {
-  params: Promise<{ id: number }>;
-}) {
-  const params = await props.params;
-  const tasks = [
-    fetchSS("/manage/admin/slack-app/channel"),
-    fetchSS("/manage/document-set"),
-    fetchAgentsSS(),
-  ];
+function EditSlackChannelConfigContent({ id }: { id: string }) {
+  const isPaidEnterprise = usePaidEnterpriseFeaturesEnabled();
 
-  const [
-    slackChannelsResponse,
-    documentSetsResponse,
-    [assistants, agentsFetchError],
-  ] = (await Promise.all(tasks)) as [Response, Response, FetchAgentsResponse];
+  const {
+    data: slackChannelConfigs,
+    isLoading: isChannelsLoading,
+    error: channelsError,
+  } = useSlackChannelConfigs();
 
-  const eeStandardAnswerCategoryResponse =
-    await getStandardAnswerCategoriesIfEE();
+  const {
+    data: documentSets,
+    isLoading: isDocSetsLoading,
+    error: docSetsError,
+  } = useDocumentSets();
 
-  if (!slackChannelsResponse.ok) {
-    return (
-      <ErrorCallout
-        errorTitle="Something went wrong :("
-        errorMsg={`Failed to fetch Slack Channels - ${await slackChannelsResponse.text()}`}
-      />
-    );
-  }
-  const allslackChannelConfigs =
-    (await slackChannelsResponse.json()) as SlackChannelConfig[];
+  const {
+    agents,
+    isLoading: isAgentsLoading,
+    error: agentsError,
+  } = useAgents();
 
-  const slackChannelConfig = allslackChannelConfigs.find(
-    (config) => config.id === Number(params.id)
+  const {
+    data: standardAnswerCategories,
+    isLoading: isStdAnswerLoading,
+    error: stdAnswerError,
+  } = useStandardAnswerCategories();
+
+  const isLoading =
+    isChannelsLoading ||
+    isDocSetsLoading ||
+    isAgentsLoading ||
+    (isPaidEnterprise && isStdAnswerLoading);
+
+  const slackChannelConfig = slackChannelConfigs?.find(
+    (config) => config.id === Number(id)
   );
 
-  if (!slackChannelConfig) {
-    return (
-      <ErrorCallout
-        errorTitle="Something went wrong :("
-        errorMsg={`Did not find Slack Channel config with ID: ${params.id}`}
-      />
-    );
-  }
-
-  if (!documentSetsResponse.ok) {
-    return (
-      <ErrorCallout
-        errorTitle="Something went wrong :("
-        errorMsg={`Failed to fetch document sets - ${await documentSetsResponse.text()}`}
-      />
-    );
-  }
-  const response = await documentSetsResponse.json();
-  const documentSets = response as DocumentSetSummary[];
-
-  if (agentsFetchError) {
-    return (
-      <ErrorCallout
-        errorTitle="Something went wrong :("
-        errorMsg={`Failed to fetch personas - ${agentsFetchError}`}
-      />
-    );
-  }
+  const title = slackChannelConfig?.is_default
+    ? "Edit Default Slack Config"
+    : "Edit Slack Channel Config";
 
   return (
     <SettingsLayouts.Root>
-      <InstantSSRAutoRefresh />
       <SettingsLayouts.Header
         icon={SvgSlack}
-        title={
-          slackChannelConfig.is_default
-            ? "Edit Default Slack Config"
-            : "Edit Slack Channel Config"
-        }
+        title={title}
         separator
         backButton
       />
       <SettingsLayouts.Body>
-        <SlackChannelConfigCreationForm
-          slack_bot_id={slackChannelConfig.slack_bot_id}
-          documentSets={documentSets}
-          personas={assistants}
-          standardAnswerCategoryResponse={eeStandardAnswerCategoryResponse}
-          existingSlackChannelConfig={slackChannelConfig}
-        />
+        {isLoading ? (
+          <SimpleLoader />
+        ) : channelsError || !slackChannelConfigs ? (
+          <ErrorCallout
+            errorTitle="Something went wrong :("
+            errorMsg={`Failed to fetch Slack Channels - ${
+              channelsError?.message ?? "unknown error"
+            }`}
+          />
+        ) : !slackChannelConfig ? (
+          <ErrorCallout
+            errorTitle="Something went wrong :("
+            errorMsg={`Did not find Slack Channel config with ID: ${id}`}
+          />
+        ) : docSetsError || !documentSets ? (
+          <ErrorCallout
+            errorTitle="Something went wrong :("
+            errorMsg={`Failed to fetch document sets - ${
+              docSetsError?.message ?? "unknown error"
+            }`}
+          />
+        ) : agentsError ? (
+          <ErrorCallout
+            errorTitle="Something went wrong :("
+            errorMsg={`Failed to fetch agents - ${
+              agentsError?.message ?? "unknown error"
+            }`}
+          />
+        ) : (
+          <SlackChannelConfigCreationForm
+            slack_bot_id={slackChannelConfig.slack_bot_id}
+            documentSets={documentSets}
+            personas={agents}
+            standardAnswerCategoryResponse={
+              isPaidEnterprise
+                ? {
+                    paidEnterpriseFeaturesEnabled: true,
+                    categories: standardAnswerCategories ?? [],
+                    ...(stdAnswerError
+                      ? { error: { message: String(stdAnswerError) } }
+                      : {}),
+                  }
+                : { paidEnterpriseFeaturesEnabled: false }
+            }
+            existingSlackChannelConfig={slackChannelConfig}
+          />
+        )}
       </SettingsLayouts.Body>
     </SettingsLayouts.Root>
   );
 }
 
-export default EditslackChannelConfigPage;
+export default function Page(props: { params: Promise<{ id: string }> }) {
+  const params = use(props.params);
+
+  return <EditSlackChannelConfigContent id={params.id} />;
+}

web/src/app/admin/bots/[bot-id]/channels/new/page.tsx

@@ -1,53 +1,109 @@
-import { SlackChannelConfigCreationForm } from "../SlackChannelConfigCreationForm";
-import { fetchSS } from "@/lib/utilsSS";
+"use client";
+
+import { use, useEffect } from "react";
+import { SlackChannelConfigCreationForm } from "@/app/admin/bots/[bot-id]/channels/SlackChannelConfigCreationForm";
 import { ErrorCallout } from "@/components/ErrorCallout";
-import { DocumentSetSummary } from "@/lib/types";
-import { fetchAgentsSS } from "@/lib/agentsSS";
-import { getStandardAnswerCategoriesIfEE } from "@/components/standardAnswers/getStandardAnswerCategoriesIfEE";
-import { redirect } from "next/navigation";
+import SimpleLoader from "@/refresh-components/loaders/SimpleLoader";
 import * as SettingsLayouts from "@/layouts/settings-layouts";
 import { SvgSlack } from "@opal/icons";
+import { useDocumentSets } from "@/app/admin/documents/sets/hooks";
+import { useAgents } from "@/hooks/useAgents";
+import { useStandardAnswerCategories } from "@/app/ee/admin/standard-answer/hooks";
+import { usePaidEnterpriseFeaturesEnabled } from "@/components/settings/usePaidEnterpriseFeaturesEnabled";
+import type { StandardAnswerCategoryResponse } from "@/components/standardAnswers/getStandardAnswerCategoriesIfEE";
+import { useRouter } from "next/navigation";
+
+function NewChannelConfigContent({ slackBotId }: { slackBotId: number }) {
+  const isPaidEnterprise = usePaidEnterpriseFeaturesEnabled();
+
+  const {
+    data: documentSets,
+    isLoading: isDocSetsLoading,
+    error: docSetsError,
+  } = useDocumentSets();
+
+  const {
+    agents,
+    isLoading: isAgentsLoading,
+    error: agentsError,
+  } = useAgents();
+
+  const {
+    data: standardAnswerCategories,
+    isLoading: isStdAnswerLoading,
+    error: stdAnswerError,
+  } = useStandardAnswerCategories();
+
+  if (
+    isDocSetsLoading ||
+    isAgentsLoading ||
+    (isPaidEnterprise && isStdAnswerLoading)
+  ) {
+    return <SimpleLoader />;
+  }
+
+  if (docSetsError || !documentSets) {
+    return (
+      <ErrorCallout
+        errorTitle="Something went wrong :("
+        errorMsg={`Failed to fetch document sets - ${
+          docSetsError?.message ?? "unknown error"
+        }`}
+      />
+    );
+  }
+
+  if (agentsError) {
+    return (
+      <ErrorCallout
+        errorTitle="Something went wrong :("
+        errorMsg={`Failed to fetch agents - ${
+          agentsError?.message ?? "unknown error"
+        }`}
+      />
+    );
+  }
+
+  const standardAnswerCategoryResponse: StandardAnswerCategoryResponse =
+    isPaidEnterprise
+      ? {
+          paidEnterpriseFeaturesEnabled: true,
+          categories: standardAnswerCategories ?? [],
+          ...(stdAnswerError
+            ? { error: { message: String(stdAnswerError) } }
+            : {}),
+        }
+      : { paidEnterpriseFeaturesEnabled: false };
+
+  return (
+    <SlackChannelConfigCreationForm
+      slack_bot_id={slackBotId}
+      documentSets={documentSets}
+      personas={agents}
+      standardAnswerCategoryResponse={standardAnswerCategoryResponse}
+    />
+  );
+}
+
+export default function Page(props: { params: Promise<{ "bot-id": string }> }) {
+  const unwrappedParams = use(props.params);
+  const router = useRouter();
 
-async function NewChannelConfigPage(props: {
-  params: Promise<{ "bot-id": string }>;
-}) {
-  const unwrappedParams = await props.params;
   const slack_bot_id_raw = unwrappedParams?.["bot-id"] || null;
   const slack_bot_id = slack_bot_id_raw
     ? parseInt(slack_bot_id_raw as string, 10)
     : null;
+
+  useEffect(() => {
+    if (!slack_bot_id || isNaN(slack_bot_id)) {
+      router.replace("/admin/bots");
+    }
+  }, [slack_bot_id, router]);
+
   if (!slack_bot_id || isNaN(slack_bot_id)) {
-    redirect("/admin/bots");
     return null;
   }
 
-  const [documentSetsResponse, agentsResponse, standardAnswerCategoryResponse] =
-    await Promise.all([
-      fetchSS("/manage/document-set") as Promise<Response>,
-      fetchAgentsSS(),
-      getStandardAnswerCategoriesIfEE(),
-    ]);
-
-  if (!documentSetsResponse.ok) {
-    return (
-      <ErrorCallout
-        errorTitle="Something went wrong :("
-        errorMsg={`Failed to fetch document sets - ${await documentSetsResponse.text()}`}
-      />
-    );
-  }
-  const documentSets =
-    (await documentSetsResponse.json()) as DocumentSetSummary[];
-
-  if (agentsResponse[1]) {
-    return (
-      <ErrorCallout
-        errorTitle="Something went wrong :("
-        errorMsg={`Failed to fetch agents - ${agentsResponse[1]}`}
-      />
-    );
-  }
-
   return (
     <SettingsLayouts.Root>
       <SettingsLayouts.Header
@@ -57,15 +113,8 @@ async function NewChannelConfigPage(props: {
         backButton
       />
       <SettingsLayouts.Body>
-        <SlackChannelConfigCreationForm
-          slack_bot_id={slack_bot_id}
-          documentSets={documentSets}
-          personas={agentsResponse[0]}
-          standardAnswerCategoryResponse={standardAnswerCategoryResponse}
-        />
+        <NewChannelConfigContent slackBotId={slack_bot_id} />
       </SettingsLayouts.Body>
     </SettingsLayouts.Root>
   );
 }
-
-export default NewChannelConfigPage;

web/src/app/admin/bots/[bot-id]/page.tsx

@@ -1,82 +1,62 @@
 "use client";
 
 import { use } from "react";
-import BackButton from "@/refresh-components/buttons/BackButton";
 import { ErrorCallout } from "@/components/ErrorCallout";
-import { ThreeDotsLoader } from "@/components/Loading";
-import { InstantSSRAutoRefresh } from "@/components/SSRAutoRefresh";
+import SimpleLoader from "@/refresh-components/loaders/SimpleLoader";
 import SlackChannelConfigsTable from "./SlackChannelConfigsTable";
 import { useSlackBot, useSlackChannelConfigsByBot } from "./hooks";
 import { ExistingSlackBotForm } from "../SlackBotUpdateForm";
-import Separator from "@/refresh-components/Separator";
-
-function SlackBotEditPage({
-  params,
-}: {
-  params: Promise<{ "bot-id": string }>;
-}) {
-  // Unwrap the params promise
-  const unwrappedParams = use(params);
+import * as SettingsLayouts from "@/layouts/settings-layouts";
+import { SvgSlack } from "@opal/icons";
+import { getErrorMsg } from "@/lib/error";
 
+function SlackBotEditContent({ botId }: { botId: string }) {
   const {
     data: slackBot,
     isLoading: isSlackBotLoading,
     error: slackBotError,
     refreshSlackBot,
-  } = useSlackBot(Number(unwrappedParams["bot-id"]));
+  } = useSlackBot(Number(botId));
 
   const {
     data: slackChannelConfigs,
     isLoading: isSlackChannelConfigsLoading,
     error: slackChannelConfigsError,
     refreshSlackChannelConfigs,
-  } = useSlackChannelConfigsByBot(Number(unwrappedParams["bot-id"]));
+  } = useSlackChannelConfigsByBot(Number(botId));
 
   if (isSlackBotLoading || isSlackChannelConfigsLoading) {
-    return (
-      <div className="flex justify-center items-center h-screen">
-        <ThreeDotsLoader />
-      </div>
-    );
+    return <SimpleLoader />;
   }
 
   if (slackBotError || !slackBot) {
-    const errorMsg =
-      slackBotError?.info?.message ||
-      slackBotError?.info?.detail ||
-      "An unknown error occurred";
     return (
       <ErrorCallout
         errorTitle="Something went wrong :("
-        errorMsg={`Failed to fetch Slack Bot ${unwrappedParams["bot-id"]}: ${errorMsg}`}
+        errorMsg={`Failed to fetch Slack Bot ${botId}: ${getErrorMsg(
+          slackBotError
+        )}`}
       />
     );
   }
 
   if (slackChannelConfigsError || !slackChannelConfigs) {
-    const errorMsg =
-      slackChannelConfigsError?.info?.message ||
-      slackChannelConfigsError?.info?.detail ||
-      "An unknown error occurred";
     return (
       <ErrorCallout
         errorTitle="Something went wrong :("
-        errorMsg={`Failed to fetch Slack Bot ${unwrappedParams["bot-id"]}: ${errorMsg}`}
+        errorMsg={`Failed to fetch Slack Bot ${botId}: ${getErrorMsg(
+          slackChannelConfigsError
+        )}`}
       />
     );
   }
 
   return (
     <>
-      <InstantSSRAutoRefresh />
-
-      <BackButton routerOverride="/admin/bots" />
-
       <ExistingSlackBotForm
         existingSlackBot={slackBot}
         refreshSlackBot={refreshSlackBot}
       />
-      <Separator />
 
       <div className="mt-8">
         <SlackChannelConfigsTable
@@ -94,9 +74,19 @@ export default function Page({
 }: {
   params: Promise<{ "bot-id": string }>;
 }) {
+  const unwrappedParams = use(params);
+
   return (
-    <>
-      <SlackBotEditPage params={params} />
-    </>
+    <SettingsLayouts.Root>
+      <SettingsLayouts.Header
+        icon={SvgSlack}
+        title="Edit Slack Bot"
+        backButton
+        separator
+      />
+      <SettingsLayouts.Body>
+        <SlackBotEditContent botId={unwrappedParams["bot-id"]} />
+      </SettingsLayouts.Body>
+    </SettingsLayouts.Root>
   );
 }

web/src/app/admin/bots/new/page.tsx

@@ -1,12 +1,7 @@
-import BackButton from "@/refresh-components/buttons/BackButton";
+"use client";
+
 import { NewSlackBotForm } from "../SlackBotCreationForm";
 
-export default async function NewSlackBotPage() {
-  return (
-    <>
-      <BackButton routerOverride="/admin/bots" />
-
-      <NewSlackBotForm />
-    </>
-  );
+export default function Page() {
+  return <NewSlackBotForm />;
 }

web/src/lib/error.ts

@@ -0,0 +1,10 @@
+/**
+ * Extract a human-readable error message from an SWR error object.
+ * SWR errors from `errorHandlingFetcher` attach `info.message` or `info.detail`.
+ */
+export function getErrorMsg(
+  error: { info?: { message?: string; detail?: string } } | null | undefined,
+  fallback = "An unknown error occurred"
+): string {
+  return error?.info?.message || error?.info?.detail || fallback;
+}

web/src/refresh-pages/AppPage.tsx

@@ -716,7 +716,7 @@ export default function AppPage({ firstMessage }: ChatPageProps) {
             >
               {/* Main content grid — 3 rows, animated */}
               <div
-                className="flex-1 w-full grid min-h-0 px-4 transition-[grid-template-rows] duration-150 ease-in-out"
+                className="flex-1 w-full grid min-h-0 transition-[grid-template-rows] duration-150 ease-in-out"
                 style={gridStyle}
               >
                 {/* ── Top row: ChatUI / WelcomeMessage / ProjectUI ── */}
@@ -781,7 +781,7 @@ export default function AppPage({ firstMessage }: ChatPageProps) {
                 </div>
 
                 {/* ── Middle-center: AppInputBar ── */}
-                <div className="row-start-2 flex flex-col items-center">
+                <div className="row-start-2 flex flex-col items-center px-4">
                   <div className="relative w-full max-w-[var(--app-page-main-content-width)] flex flex-col">
                     {/* Scroll to bottom button - positioned absolutely above AppInputBar */}
                     {appFocus.isChat() && showScrollButton && (
@@ -881,7 +881,7 @@ export default function AppPage({ firstMessage }: ChatPageProps) {
                 </div>
 
                 {/* ── Bottom: SearchResults + SourceFilter / Suggestions / ProjectChatList ── */}
-                <div className="row-start-3 min-h-0 overflow-hidden flex flex-col items-center w-full">
+                <div className="row-start-3 min-h-0 overflow-hidden flex flex-col items-center w-full px-4">
                   {/* Agent description below input */}
                   {(appFocus.isNewSession() || appFocus.isAgent()) &&
                     !isDefaultAgent && (

web/src/sections/chat/ChatUI.tsx

@@ -115,7 +115,7 @@ const ChatUI = React.memo(
 
     return (
       <>
-        <div className="flex flex-col w-full max-w-[var(--app-page-main-content-width)] h-full pt-4 pb-8 pr-1 gap-12">
+        <div className="flex flex-col w-full max-w-[var(--app-page-main-content-width)] h-full p-4 pb-8 pr-5 gap-12">
           {messages.map((message, i) => {
             const messageReactComponentKey = `message-${message.nodeId}`;
             const parentMessage = message.parentNodeId