fix(embedding): Fix Deletion of Same Name (#6149)

.github/actionlint.yml

@@ -1,43 +0,0 @@
-self-hosted-runner:
-  # Labels of self-hosted runner in array of strings.
-  labels:
-    - extras=ecr-cache
-    - extras=s3-cache
-    - hdd=256
-    - runs-on
-    - runner=1cpu-linux-arm64
-    - runner=1cpu-linux-x64
-    - runner=2cpu-linux-arm64
-    - runner=2cpu-linux-x64
-    - runner=4cpu-linux-arm64
-    - runner=4cpu-linux-x64
-    - runner=8cpu-linux-arm64
-    - runner=8cpu-linux-x64
-    - runner=16cpu-linux-arm64
-    - runner=16cpu-linux-x64
-    - ubuntu-slim # Currently in public preview
-    - volume=40gb
-    - volume=50gb
-
-# Configuration variables in array of strings defined in your repository or
-# organization. `null` means disabling configuration variables check.
-# Empty array means no configuration variable is allowed.
-config-variables: null
-
-# Configuration for file paths. The keys are glob patterns to match to file
-# paths relative to the repository root. The values are the configurations for
-# the file paths. Note that the path separator is always '/'.
-# The following configurations are available.
-#
-# "ignore" is an array of regular expression patterns. Matched error messages
-# are ignored. This is similar to the "-ignore" command line option.
-paths:
-  # Glob pattern relative to the repository root for matching files. The path separator is always '/'.
-  # This example configures any YAML file under the '.github/workflows/' directory.
-  .github/workflows/**/*.{yml,yaml}:
-    # TODO: These are real and should be fixed eventually.
-    ignore:
-      - 'shellcheck reported issue in this script: SC2038:.+'
-      - 'shellcheck reported issue in this script: SC2046:.+'
-      - 'shellcheck reported issue in this script: SC2086:.+'
-      - 'shellcheck reported issue in this script: SC2193:.+'

.github/actions/custom-build-and-push/action.yml

@@ -0,0 +1,135 @@
+name: 'Build and Push Docker Image with Retry'
+description: 'Attempts to build and push a Docker image, with a retry on failure'
+inputs:
+  context:
+    description: 'Build context'
+    required: true
+  file:
+    description: 'Dockerfile location'
+    required: true
+  platforms:
+    description: 'Target platforms'
+    required: true
+  pull:
+    description: 'Always attempt to pull a newer version of the image'
+    required: false
+    default: 'true'
+  push:
+    description: 'Push the image to registry'
+    required: false
+    default: 'true'
+  load:
+    description: 'Load the image into Docker daemon'
+    required: false
+    default: 'true'
+  tags:
+    description: 'Image tags'
+    required: true
+  no-cache:
+    description: 'Read from cache'
+    required: false
+    default: 'false'
+  cache-from:
+    description: 'Cache sources'
+    required: false
+  cache-to:
+    description: 'Cache destinations'
+    required: false
+  outputs:
+    description: 'Output destinations'
+    required: false
+  provenance:
+    description: 'Generate provenance attestation'
+    required: false
+    default: 'false'
+  build-args:
+    description: 'Build arguments'
+    required: false
+  retry-wait-time:
+    description: 'Time to wait before attempt 2 in seconds'
+    required: false
+    default: '60'
+  retry-wait-time-2:
+    description: 'Time to wait before attempt 3 in seconds'
+    required: false
+    default: '120'
+
+runs:
+  using: "composite"
+  steps:
+    - name: Build and push Docker image (Attempt 1 of 3)
+      id: buildx1
+      uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # ratchet:docker/build-push-action@v6
+      continue-on-error: true
+      with:
+        context: ${{ inputs.context }}
+        file: ${{ inputs.file }}
+        platforms: ${{ inputs.platforms }}
+        pull: ${{ inputs.pull }}
+        push: ${{ inputs.push }}
+        load: ${{ inputs.load }}
+        tags: ${{ inputs.tags }}
+        no-cache: ${{ inputs.no-cache }}
+        cache-from: ${{ inputs.cache-from }}
+        cache-to: ${{ inputs.cache-to }}
+        outputs: ${{ inputs.outputs }}
+        provenance: ${{ inputs.provenance }}
+        build-args: ${{ inputs.build-args }}
+
+    - name: Wait before attempt 2
+      if: steps.buildx1.outcome != 'success'
+      run: |
+        echo "First attempt failed. Waiting ${{ inputs.retry-wait-time }} seconds before retry..."
+        sleep ${{ inputs.retry-wait-time }}
+      shell: bash
+
+    - name: Build and push Docker image (Attempt 2 of 3)
+      id: buildx2
+      if: steps.buildx1.outcome != 'success'
+      uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # ratchet:docker/build-push-action@v6
+      with:
+        context: ${{ inputs.context }}
+        file: ${{ inputs.file }}
+        platforms: ${{ inputs.platforms }}
+        pull: ${{ inputs.pull }}
+        push: ${{ inputs.push }}
+        load: ${{ inputs.load }}
+        tags: ${{ inputs.tags }}
+        no-cache: ${{ inputs.no-cache }}
+        cache-from: ${{ inputs.cache-from }}
+        cache-to: ${{ inputs.cache-to }}
+        outputs: ${{ inputs.outputs }}
+        provenance: ${{ inputs.provenance }}
+        build-args: ${{ inputs.build-args }}
+
+    - name: Wait before attempt 3
+      if: steps.buildx1.outcome != 'success' && steps.buildx2.outcome != 'success'
+      run: |
+        echo "Second attempt failed. Waiting ${{ inputs.retry-wait-time-2 }} seconds before retry..."
+        sleep ${{ inputs.retry-wait-time-2 }}
+      shell: bash
+
+    - name: Build and push Docker image (Attempt 3 of 3)
+      id: buildx3
+      if: steps.buildx1.outcome != 'success' && steps.buildx2.outcome != 'success'
+      uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # ratchet:docker/build-push-action@v6
+      with:
+        context: ${{ inputs.context }}
+        file: ${{ inputs.file }}
+        platforms: ${{ inputs.platforms }}
+        pull: ${{ inputs.pull }}
+        push: ${{ inputs.push }}
+        load: ${{ inputs.load }}
+        tags: ${{ inputs.tags }}
+        no-cache: ${{ inputs.no-cache }}
+        cache-from: ${{ inputs.cache-from }}
+        cache-to: ${{ inputs.cache-to }}
+        outputs: ${{ inputs.outputs }}
+        provenance: ${{ inputs.provenance }}
+        build-args: ${{ inputs.build-args }}
+
+    - name: Report failure
+      if: steps.buildx1.outcome != 'success' && steps.buildx2.outcome != 'success' && steps.buildx3.outcome != 'success'
+      run: |
+        echo "All attempts failed. Possible transient infrastucture issues? Try again later or inspect logs for details."
+      shell: bash

.github/actions/prepare-build/action.yml

@@ -0,0 +1,50 @@
+name: "Prepare Build (OpenAPI generation)"
+description: "Sets up Python with uv, installs deps, generates OpenAPI schema and Python client, uploads artifact"
+runs:
+  using: "composite"
+  steps:
+    - name: Checkout code
+      uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # ratchet:actions/checkout@v4
+
+    - name: Setup uv
+      uses: astral-sh/setup-uv@caf0cab7a618c569241d31dcd442f54681755d39 # ratchet:astral-sh/setup-uv@v3
+
+    - name: Setup Python
+      uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # ratchet:actions/setup-python@v5
+      with:
+        python-version: "3.11"
+
+    - name: Install Python dependencies with uv
+      shell: bash
+      run: |
+        uv pip install --system \
+          -r backend/requirements/default.txt \
+          -r backend/requirements/dev.txt
+
+    - name: Generate OpenAPI schema
+      shell: bash
+      working-directory: backend
+      env:
+        PYTHONPATH: "."
+      run: |
+        python scripts/onyx_openapi_schema.py --filename generated/openapi.json
+
+    - name: Generate OpenAPI Python client
+      shell: bash
+      run: |
+        docker run --rm \
+          -v "${{ github.workspace }}/backend/generated:/local" \
+          openapitools/openapi-generator-cli generate \
+          -i /local/openapi.json \
+          -g python \
+          -o /local/onyx_openapi_client \
+          --package-name onyx_openapi_client \
+          --skip-validate-spec \
+          --openapi-normalizer "SIMPLIFY_ONEOF_ANYOF=true,SET_OAS3_NULLABLE=true"
+
+    - name: Upload OpenAPI artifacts
+      uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # ratchet:actions/upload-artifact@v4
+      with:
+        name: openapi-artifacts
+        path: backend/generated/
+

.github/actions/setup-playwright/action.yml

@@ -1,17 +0,0 @@
-name: "Setup Playwright"
-description: "Sets up Playwright and system deps (assumes Python and Playwright are installed)"
-runs:
-  using: "composite"
-  steps:
-    - name: Cache playwright cache
-      uses: runs-on/cache@50350ad4242587b6c8c2baa2e740b1bc11285ff4 # ratchet:runs-on/cache@v4
-      with:
-        path: ~/.cache/ms-playwright
-        key: ${{ runner.os }}-${{ runner.arch }}-playwright-${{ hashFiles('backend/requirements/default.txt') }}
-        restore-keys: |
-          ${{ runner.os }}-${{ runner.arch }}-playwright-
-
-    - name: Install playwright
-      shell: bash
-      run: |
-        playwright install chromium --with-deps

.github/actions/setup-python-and-install-dependencies/action.yml

@@ -1,72 +0,0 @@
-name: "Setup Python and Install Dependencies"
-description: "Sets up Python with uv and installs deps"
-inputs:
-  requirements:
-    description: "Newline-separated list of requirement files to install (relative to repo root)"
-    required: true
-runs:
-  using: "composite"
-  steps:
-    - name: Setup uv
-      uses: astral-sh/setup-uv@caf0cab7a618c569241d31dcd442f54681755d39 # ratchet:astral-sh/setup-uv@v3
-      # TODO: Enable caching once there is a uv.lock file checked in.
-      # with:
-      #   enable-cache: true
-
-    - name: Compute requirements hash
-      id: req-hash
-      shell: bash
-      env:
-        REQUIREMENTS: ${{ inputs.requirements }}
-      run: |
-        # Hash the contents of the specified requirement files
-        hash=""
-        while IFS= read -r req; do
-          if [ -n "$req" ] && [ -f "$req" ]; then
-            hash="$hash$(sha256sum "$req")"
-          fi
-        done <<< "$REQUIREMENTS"
-        echo "hash=$(echo "$hash" | sha256sum | cut -d' ' -f1)" >> "$GITHUB_OUTPUT"
-
-    - name: Cache uv cache directory
-      uses: runs-on/cache@50350ad4242587b6c8c2baa2e740b1bc11285ff4 # ratchet:runs-on/cache@v4
-      with:
-        path: ~/.cache/uv
-        key: ${{ runner.os }}-uv-${{ steps.req-hash.outputs.hash }}
-        restore-keys: |
-          ${{ runner.os }}-uv-
-
-    - name: Setup Python
-      uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # ratchet:actions/setup-python@v5
-      with:
-        python-version: "3.11"
-
-    - name: Create virtual environment
-      shell: bash
-      env:
-        VENV_DIR: ${{ runner.temp }}/venv
-      run: | # zizmor: ignore[github-env]
-        uv venv "$VENV_DIR"
-        # Validate path before adding to GITHUB_PATH to prevent code injection
-        if [ -d "$VENV_DIR/bin" ]; then
-          realpath "$VENV_DIR/bin" >> "$GITHUB_PATH"
-        else
-          echo "Error: $VENV_DIR/bin does not exist"
-          exit 1
-        fi
-
-    - name: Install Python dependencies with uv
-      shell: bash
-      env:
-        REQUIREMENTS: ${{ inputs.requirements }}
-      run: |
-        # Build the uv pip install command with each requirement file as array elements
-        cmd=("uv" "pip" "install")
-        while IFS= read -r req; do
-          # Skip empty lines
-          if [ -n "$req" ]; then
-            cmd+=("-r" "$req")
-          fi
-        done <<< "$REQUIREMENTS"
-        echo "Running: ${cmd[*]}"
-        "${cmd[@]}"

.github/actions/slack-notify/action.yml

@@ -1,102 +0,0 @@
-name: "Slack Notify on Failure"
-description: "Sends a Slack notification when a workflow fails"
-inputs:
-  webhook-url:
-    description: "Slack webhook URL (can also use SLACK_WEBHOOK_URL env var)"
-    required: false
-  failed-jobs:
-    description: "List of failed job names (newline-separated)"
-    required: false
-  title:
-    description: "Title for the notification"
-    required: false
-    default: "🚨 Workflow Failed"
-  ref-name:
-    description: "Git ref name (tag/branch)"
-    required: false
-runs:
-  using: "composite"
-  steps:
-    - name: Send Slack notification
-      shell: bash
-      env:
-        SLACK_WEBHOOK_URL: ${{ inputs.webhook-url }}
-        FAILED_JOBS: ${{ inputs.failed-jobs }}
-        TITLE: ${{ inputs.title }}
-        REF_NAME: ${{ inputs.ref-name }}
-        REPO: ${{ github.repository }}
-        WORKFLOW: ${{ github.workflow }}
-        RUN_NUMBER: ${{ github.run_number }}
-        RUN_ID: ${{ github.run_id }}
-        SERVER_URL: ${{ github.server_url }}
-        GITHUB_REF_NAME: ${{ github.ref_name }}
-      run: |
-        if [ -z "$SLACK_WEBHOOK_URL" ]; then
-          echo "webhook-url input or SLACK_WEBHOOK_URL env var is not set, skipping notification"
-          exit 0
-        fi
-
-        # Build workflow URL
-        WORKFLOW_URL="${SERVER_URL}/${REPO}/actions/runs/${RUN_ID}"
-
-        # Use ref_name from input or fall back to github.ref_name
-        if [ -z "$REF_NAME" ]; then
-          REF_NAME="$GITHUB_REF_NAME"
-        fi
-
-        # Escape JSON special characters
-        escape_json() {
-          local input="$1"
-          # Escape backslashes first (but preserve \n sequences)
-          # Protect \n sequences temporarily
-          input=$(printf '%s' "$input" | sed 's/\\n/\x01NL\x01/g')
-          # Escape remaining backslashes
-          input=$(printf '%s' "$input" | sed 's/\\/\\\\/g')
-          # Restore \n sequences (single backslash, will be correct in JSON)
-          input=$(printf '%s' "$input" | sed 's/\x01NL\x01/\\n/g')
-          # Escape quotes
-          printf '%s' "$input" | sed 's/"/\\"/g'
-        }
-
-        REF_NAME_ESC=$(escape_json "$REF_NAME")
-        FAILED_JOBS_ESC=$(escape_json "$FAILED_JOBS")
-        WORKFLOW_URL_ESC=$(escape_json "$WORKFLOW_URL")
-        TITLE_ESC=$(escape_json "$TITLE")
-
-        # Build JSON payload piece by piece
-        # Note: FAILED_JOBS_ESC already contains \n sequences that should remain as \n in JSON
-        PAYLOAD="{"
-        PAYLOAD="${PAYLOAD}\"text\":\"${TITLE_ESC}\","
-        PAYLOAD="${PAYLOAD}\"blocks\":[{"
-        PAYLOAD="${PAYLOAD}\"type\":\"header\","
-        PAYLOAD="${PAYLOAD}\"text\":{\"type\":\"plain_text\",\"text\":\"${TITLE_ESC}\"}"
-        PAYLOAD="${PAYLOAD}},{"
-        PAYLOAD="${PAYLOAD}\"type\":\"section\","
-        PAYLOAD="${PAYLOAD}\"fields\":["
-        if [ -n "$REF_NAME" ]; then
-          PAYLOAD="${PAYLOAD}{\"type\":\"mrkdwn\",\"text\":\"*Ref:*\\n${REF_NAME_ESC}\"},"
-        fi
-        PAYLOAD="${PAYLOAD}{\"type\":\"mrkdwn\",\"text\":\"*Run ID:*\\n#${RUN_NUMBER}\"}"
-        PAYLOAD="${PAYLOAD}]"
-        PAYLOAD="${PAYLOAD}}"
-        if [ -n "$FAILED_JOBS" ]; then
-          PAYLOAD="${PAYLOAD},{"
-          PAYLOAD="${PAYLOAD}\"type\":\"section\","
-          PAYLOAD="${PAYLOAD}\"text\":{\"type\":\"mrkdwn\",\"text\":\"*Failed Jobs:*\\n${FAILED_JOBS_ESC}\"}"
-          PAYLOAD="${PAYLOAD}}"
-        fi
-        PAYLOAD="${PAYLOAD},{"
-        PAYLOAD="${PAYLOAD}\"type\":\"actions\","
-        PAYLOAD="${PAYLOAD}\"elements\":[{"
-        PAYLOAD="${PAYLOAD}\"type\":\"button\","
-        PAYLOAD="${PAYLOAD}\"text\":{\"type\":\"plain_text\",\"text\":\"View Workflow Run\"},"
-        PAYLOAD="${PAYLOAD}\"url\":\"${WORKFLOW_URL_ESC}\""
-        PAYLOAD="${PAYLOAD}}]"
-        PAYLOAD="${PAYLOAD}}"
-        PAYLOAD="${PAYLOAD}]"
-        PAYLOAD="${PAYLOAD}}"
-
-        curl -X POST -H 'Content-type: application/json' \
-          --data "$PAYLOAD" \
-          "$SLACK_WEBHOOK_URL"
-

.github/dependabot.yml

@@ -4,21 +4,8 @@ updates:
     directory: "/"
     schedule:
       interval: "weekly"
-    cooldown:
-      default-days: 7
     open-pull-requests-limit: 3
     assignees:
       - "jmelahman"
     labels:
       - "dependabot:actions"
-  - package-ecosystem: "pip"
-    directory: "/backend"
-    schedule:
-      interval: "weekly"
-    cooldown:
-      default-days: 7
-    open-pull-requests-limit: 3
-    assignees:
-      - "jmelahman"
-    labels:
-      - "dependabot:python"

.github/runs-on.yml

@@ -1 +0,0 @@
-_extend: .github-private

.github/workflows/check-lazy-imports.yml

@@ -0,0 +1,24 @@
+name: Check Lazy Imports
+
+on:
+  merge_group:
+  pull_request:
+    branches:
+      - main
+      - 'release/**'
+
+jobs:
+  check-lazy-imports:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # ratchet:actions/checkout@v4
+
+    - name: Set up Python
+      uses: actions/setup-python@7f4fc3e22c37d6ff65e88745f38bd3157c663f7c # ratchet:actions/setup-python@v4
+      with:
+        python-version: '3.11'
+
+    - name: Check lazy imports
+      run: python3 backend/scripts/check_lazy_imports.py

.github/workflows/docker-build-push-backend-container-on-tag.yml

@@ -0,0 +1,198 @@
+name: Build and Push Backend Image on Tag
+
+on:
+  push:
+    tags:
+      - "*"
+
+env:
+  REGISTRY_IMAGE: ${{ contains(github.ref_name, 'cloud') && 'onyxdotapp/onyx-backend-cloud' || 'onyxdotapp/onyx-backend' }}
+  DEPLOYMENT: ${{ contains(github.ref_name, 'cloud') && 'cloud' || 'standalone' }}
+
+  # tag nightly builds with "edge"
+  EDGE_TAG: ${{ startsWith(github.ref_name, 'nightly-latest') }}
+
+jobs:
+  build-and-push:
+    # TODO: investigate a matrix build like the web container
+    # See https://runs-on.com/runners/linux/
+    runs-on:
+      - runs-on
+      - runner=${{ matrix.platform == 'linux/amd64' && '8cpu-linux-x64' || '8cpu-linux-arm64' }}
+      - run-id=${{ github.run_id }}
+      - tag=platform-${{ matrix.platform }}
+    strategy:
+      fail-fast: false
+      matrix:
+        platform:
+          - linux/amd64
+          - linux/arm64
+          
+    steps:
+      - name: Prepare
+        run: |
+          platform=${{ matrix.platform }}
+          echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV
+
+      - name: Check if stable release version
+        id: check_version
+        run: |
+          if [[ "${{ github.ref_name }}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]] && [[ "${{ github.ref_name }}" != *"cloud"* ]]; then
+            echo "is_stable=true" >> $GITHUB_OUTPUT
+          else
+            echo "is_stable=false" >> $GITHUB_OUTPUT
+          fi
+          if [[ "${{ github.ref_name }}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+-beta\.[0-9]+$ ]] && [[ "${{ github.ref_name }}" != *"cloud"* ]]; then
+            echo "is_beta=true" >> $GITHUB_OUTPUT
+          else
+            echo "is_beta=false" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Checkout code
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # ratchet:actions/checkout@v4
+
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@318604b99e75e41977312d83839a89be02ca4893 # ratchet:docker/metadata-action@v5
+        with:
+          images: ${{ env.REGISTRY_IMAGE }}
+          flavor: |
+            latest=false
+          tags: |
+            type=raw,value=${{ github.ref_name }}
+            type=raw,value=${{ steps.check_version.outputs.is_stable == 'true' && 'latest' || '' }}
+            type=raw,value=${{ env.EDGE_TAG == 'true' && 'edge' || '' }}
+            type=raw,value=${{ steps.check_version.outputs.is_beta == 'true' && 'beta' || '' }}
+            
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # ratchet:docker/setup-buildx-action@v3
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # ratchet:docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_TOKEN }}
+
+      - name: Install build-essential
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y build-essential
+
+      - name: Backend Image Docker Build and Push
+        id: build
+        uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # ratchet:docker/build-push-action@v6
+        with:
+          context: ./backend
+          file: ./backend/Dockerfile
+          platforms: ${{ matrix.platform }}
+          push: true
+          build-args: |
+            ONYX_VERSION=${{ github.ref_name }}
+          labels: ${{ steps.meta.outputs.labels }}
+          outputs: type=image,name=${{ env.REGISTRY_IMAGE }},push-by-digest=true,name-canonical=true,push=true
+          cache-from: type=s3,prefix=cache/${{ github.repository }}/${{ env.DEPLOYMENT }}/backend-${{ env.PLATFORM_PAIR }}/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }}
+          cache-to: type=s3,prefix=cache/${{ github.repository }}/${{ env.DEPLOYMENT }}/backend-${{ env.PLATFORM_PAIR }}/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }},mode=max
+
+      - name: Export digest      
+        run: |
+          mkdir -p /tmp/digests
+          digest="${{ steps.build.outputs.digest }}"
+          touch "/tmp/digests/${digest#sha256:}"
+
+      - name: Upload digest
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # ratchet:actions/upload-artifact@v4
+        with:
+          name: backend-digests-${{ env.PLATFORM_PAIR }}-${{ github.run_id }}
+          path: /tmp/digests/*
+          if-no-files-found: error
+          retention-days: 1
+          
+  merge:
+    runs-on: ubuntu-latest
+    needs:
+      - build-and-push
+    steps:
+      # Needed for trivyignore
+      - name: Checkout
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # ratchet:actions/checkout@v4
+      
+      - name: Check if stable release version
+        id: check_version
+        run: |
+          if [[ "${{ github.ref_name }}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]] && [[ "${{ github.ref_name }}" != *"cloud"* ]]; then
+            echo "is_stable=true" >> $GITHUB_OUTPUT
+          else
+            echo "is_stable=false" >> $GITHUB_OUTPUT
+          fi
+          if [[ "${{ github.ref_name }}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+-beta\.[0-9]+$ ]] && [[ "${{ github.ref_name }}" != *"cloud"* ]]; then
+            echo "is_beta=true" >> $GITHUB_OUTPUT
+          else
+            echo "is_beta=false" >> $GITHUB_OUTPUT
+          fi
+        
+      - name: Download digests
+        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # ratchet:actions/download-artifact@v4
+        with:
+          path: /tmp/digests
+          pattern: backend-digests-*-${{ github.run_id }}
+          merge-multiple: true
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # ratchet:docker/setup-buildx-action@v3
+
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@318604b99e75e41977312d83839a89be02ca4893 # ratchet:docker/metadata-action@v5
+        with:
+          images: ${{ env.REGISTRY_IMAGE }}
+          flavor: |
+            latest=false
+          tags: |
+            type=raw,value=${{ github.ref_name }}
+            type=raw,value=${{ steps.check_version.outputs.is_stable == 'true' && 'latest' || '' }}
+            type=raw,value=${{ env.EDGE_TAG == 'true' && 'edge' || '' }}
+            type=raw,value=${{ steps.check_version.outputs.is_beta == 'true' && 'beta' || '' }}
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # ratchet:docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_TOKEN }}
+
+      - name: Create manifest list and push
+        working-directory: /tmp/digests
+        run: |
+          docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
+            $(printf '${{ env.REGISTRY_IMAGE }}@sha256:%s ' *)
+
+      - name: Inspect image
+        run: |
+          docker buildx imagetools inspect ${{ env.REGISTRY_IMAGE }}:${{ steps.meta.outputs.version }}
+          
+      # trivy has their own rate limiting issues causing this action to flake
+      # we worked around it by hardcoding to different db repos in env
+      # can re-enable when they figure it out
+      # https://github.com/aquasecurity/trivy/discussions/7538
+      # https://github.com/aquasecurity/trivy-action/issues/389
+      # Security: Using pinned digest (0.65.0@sha256:a22415a38938a56c379387a8163fcb0ce38b10ace73e593475d3658d578b2436)
+      # Security: No Docker socket mount needed for remote registry scanning
+      - name: Run Trivy vulnerability scanner
+        uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # ratchet:nick-fields/retry@v3
+        with:
+          timeout_minutes: 30
+          max_attempts: 3
+          retry_wait_seconds: 10
+          command: |
+            docker run --rm -v $HOME/.cache/trivy:/root/.cache/trivy \
+              -v ${{ github.workspace }}/backend/.trivyignore:/tmp/.trivyignore:ro \
+              -e TRIVY_DB_REPOSITORY="public.ecr.aws/aquasecurity/trivy-db:2" \
+              -e TRIVY_JAVA_DB_REPOSITORY="public.ecr.aws/aquasecurity/trivy-java-db:1" \
+              -e TRIVY_USERNAME="${{ secrets.DOCKER_USERNAME }}" \
+              -e TRIVY_PASSWORD="${{ secrets.DOCKER_TOKEN }}" \
+              aquasec/trivy@sha256:a22415a38938a56c379387a8163fcb0ce38b10ace73e593475d3658d578b2436 \
+              image \
+              --skip-version-check \
+              --timeout 20m \
+              --severity CRITICAL,HIGH \
+              --ignorefile /tmp/.trivyignore \
+              docker.io/${{ env.REGISTRY_IMAGE }}:${{ github.ref_name }}

.github/workflows/docker-build-push-cloud-web-container-on-tag.yml

@@ -0,0 +1,158 @@
+name: Build and Push Cloud Web Image on Tag
+# Identical to the web container build, but with correct image tag and build args
+
+on:
+  push:
+    tags:
+      - "*cloud*"
+
+env:
+  REGISTRY_IMAGE: onyxdotapp/onyx-web-server-cloud
+  DEPLOYMENT: cloud
+  
+jobs:
+  build:
+    runs-on:
+      - runs-on
+      - runner=${{ matrix.platform == 'linux/amd64' && '8cpu-linux-x64' || '8cpu-linux-arm64' }}
+      - run-id=${{ github.run_id }}
+      - tag=platform-${{ matrix.platform }}
+    strategy:
+      fail-fast: false
+      matrix:
+        platform:
+          - linux/amd64
+          - linux/arm64
+
+    steps:
+      - name: Prepare
+        run: |
+          platform=${{ matrix.platform }}
+          echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV
+
+      - name: Checkout
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # ratchet:actions/checkout@v4
+
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@318604b99e75e41977312d83839a89be02ca4893 # ratchet:docker/metadata-action@v5
+        with:
+          images: ${{ env.REGISTRY_IMAGE }}
+          flavor: |
+            latest=false
+          tags: |
+            type=raw,value=${{ github.ref_name }}
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # ratchet:docker/setup-buildx-action@v3
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # ratchet:docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_TOKEN }}
+
+      - name: Build and push by digest
+        id: build
+        uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # ratchet:docker/build-push-action@v6
+        with:
+          context: ./web
+          file: ./web/Dockerfile
+          platforms: ${{ matrix.platform }}
+          push: true
+          build-args: |
+            ONYX_VERSION=${{ github.ref_name }}
+            NEXT_PUBLIC_CLOUD_ENABLED=true
+            NEXT_PUBLIC_POSTHOG_KEY=${{ secrets.POSTHOG_KEY }}
+            NEXT_PUBLIC_POSTHOG_HOST=${{ secrets.POSTHOG_HOST }}
+            NEXT_PUBLIC_SENTRY_DSN=${{ secrets.SENTRY_DSN }}
+            NEXT_PUBLIC_STRIPE_PUBLISHABLE_KEY=${{ secrets.STRIPE_PUBLISHABLE_KEY }}
+            NEXT_PUBLIC_GTM_ENABLED=true
+            NEXT_PUBLIC_FORGOT_PASSWORD_ENABLED=true
+            NEXT_PUBLIC_INCLUDE_ERROR_POPUP_SUPPORT_LINK=true
+            NODE_OPTIONS=--max-old-space-size=8192
+          labels: ${{ steps.meta.outputs.labels }}
+          outputs: type=image,name=${{ env.REGISTRY_IMAGE }},push-by-digest=true,name-canonical=true,push=true
+          cache-from: type=s3,prefix=cache/${{ github.repository }}/${{ env.DEPLOYMENT }}/cloudweb-${{ env.PLATFORM_PAIR }}/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }}
+          cache-to: type=s3,prefix=cache/${{ github.repository }}/${{ env.DEPLOYMENT }}/cloudweb-${{ env.PLATFORM_PAIR }}/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }},mode=max
+          # no-cache needed due to weird interactions with the builds for different platforms
+          # NOTE(rkuo): this may not be true any more with the proper cache prefixing by architecture - currently testing with it off
+
+      - name: Export digest
+        run: |
+          mkdir -p /tmp/digests
+          digest="${{ steps.build.outputs.digest }}"
+          touch "/tmp/digests/${digest#sha256:}"
+
+      - name: Upload digest
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # ratchet:actions/upload-artifact@v4
+        with:
+          name: cloudweb-digests-${{ env.PLATFORM_PAIR }}-${{ github.run_id }}
+          path: /tmp/digests/*
+          if-no-files-found: error
+          retention-days: 1
+
+  merge:
+    runs-on: ubuntu-latest
+    needs:
+      - build
+    steps:
+      - name: Download digests
+        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # ratchet:actions/download-artifact@v4
+        with:
+          path: /tmp/digests
+          pattern: cloudweb-digests-*-${{ github.run_id }}
+          merge-multiple: true
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # ratchet:docker/setup-buildx-action@v3
+
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@318604b99e75e41977312d83839a89be02ca4893 # ratchet:docker/metadata-action@v5
+        with:
+          images: ${{ env.REGISTRY_IMAGE }}
+          flavor: |
+            latest=false
+          tags: |
+            type=raw,value=${{ github.ref_name }}
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # ratchet:docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_TOKEN }}
+
+      - name: Create manifest list and push
+        working-directory: /tmp/digests
+        run: |
+          docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
+            $(printf '${{ env.REGISTRY_IMAGE }}@sha256:%s ' *)
+
+      - name: Inspect image
+        run: |
+          docker buildx imagetools inspect ${{ env.REGISTRY_IMAGE }}:${{ steps.meta.outputs.version }}
+
+      # trivy has their own rate limiting issues causing this action to flake
+      # we worked around it by hardcoding to different db repos in env
+      # can re-enable when they figure it out
+      # https://github.com/aquasecurity/trivy/discussions/7538
+      # https://github.com/aquasecurity/trivy-action/issues/389
+      - name: Run Trivy vulnerability scanner
+        uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # ratchet:nick-fields/retry@v3
+        with:
+          timeout_minutes: 30
+          max_attempts: 3
+          retry_wait_seconds: 10
+          command: |
+            docker run --rm -v $HOME/.cache/trivy:/root/.cache/trivy \
+              -e TRIVY_DB_REPOSITORY="public.ecr.aws/aquasecurity/trivy-db:2" \
+              -e TRIVY_JAVA_DB_REPOSITORY="public.ecr.aws/aquasecurity/trivy-java-db:1" \
+              -e TRIVY_USERNAME="${{ secrets.DOCKER_USERNAME }}" \
+              -e TRIVY_PASSWORD="${{ secrets.DOCKER_TOKEN }}" \
+              aquasec/trivy@sha256:a22415a38938a56c379387a8163fcb0ce38b10ace73e593475d3658d578b2436 \
+              image \
+              --skip-version-check \
+              --timeout 20m \
+              --severity CRITICAL,HIGH \
+              docker.io/${{ env.REGISTRY_IMAGE }}:${{ github.ref_name }}

.github/workflows/docker-build-push-model-server-container-on-tag.yml

@@ -0,0 +1,207 @@
+name: Build and Push Model Server Image on Tag
+
+on:
+  push:
+    tags:
+      - "*"
+
+env:
+  REGISTRY_IMAGE: ${{ contains(github.ref_name, 'cloud') && 'onyxdotapp/onyx-model-server-cloud' || 'onyxdotapp/onyx-model-server' }}
+  DOCKER_BUILDKIT: 1
+  BUILDKIT_PROGRESS: plain
+  DEPLOYMENT: ${{ contains(github.ref_name, 'cloud') && 'cloud' || 'standalone' }}
+
+  # tag nightly builds with "edge"
+  EDGE_TAG: ${{ startsWith(github.ref_name, 'nightly-latest') }}
+  
+jobs:
+
+#   Bypassing this for now as the idea of not building is glitching
+#   releases and builds that depends on everything being tagged in docker
+#   1) Preliminary job to check if the changed files are relevant
+#   check_model_server_changes:
+#     runs-on: ubuntu-latest
+#     outputs:
+#       changed: ${{ steps.check.outputs.changed }}
+#     steps:
+#       - name: Checkout code
+#         uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # ratchet:actions/checkout@v4
+# 
+#       - name: Check if relevant files changed
+#         id: check
+#         run: |
+#           # Default to "false"
+#           echo "changed=false" >> $GITHUB_OUTPUT
+# 
+#           # Compare the previous commit (github.event.before) to the current one (github.sha)
+#           # If any file in backend/model_server/** or backend/Dockerfile.model_server is changed,
+#           # set changed=true
+#           if git diff --name-only ${{ github.event.before }} ${{ github.sha }} \
+#              | grep -E '^backend/model_server/|^backend/Dockerfile.model_server'; then
+#             echo "changed=true" >> $GITHUB_OUTPUT
+#           fi
+
+  check_model_server_changes:
+    runs-on: ubuntu-latest
+    outputs:
+      changed: "true"
+    steps:
+      - name: Bypass check and set output
+        run: echo "changed=true" >> $GITHUB_OUTPUT
+        
+  build-amd64:
+    needs: [check_model_server_changes]
+    if: needs.check_model_server_changes.outputs.changed == 'true'
+    runs-on:
+      [runs-on, runner=8cpu-linux-x64, "run-id=${{ github.run_id }}-amd64"]
+    env:
+      PLATFORM_PAIR: linux-amd64
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # ratchet:actions/checkout@v4
+
+      - name: System Info
+        run: |
+          df -h
+          free -h
+          docker system prune -af --volumes
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # ratchet:docker/setup-buildx-action@v3
+        with:
+          driver-opts: |
+            image=moby/buildkit:latest
+            network=host
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # ratchet:docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_TOKEN }}
+
+      - name: Build and Push AMD64
+        uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # ratchet:docker/build-push-action@v6
+        with:
+          context: ./backend
+          file: ./backend/Dockerfile.model_server
+          platforms: linux/amd64
+          push: true
+          tags: ${{ env.REGISTRY_IMAGE }}:${{ github.ref_name }}-amd64
+          build-args: |
+            ONYX_VERSION=${{ github.ref_name }}
+          outputs: type=registry
+          provenance: false
+          cache-from: type=s3,prefix=cache/${{ github.repository }}/${{ env.DEPLOYMENT }}/model-server-${{ env.PLATFORM_PAIR }}/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }}
+          cache-to: type=s3,prefix=cache/${{ github.repository }}/${{ env.DEPLOYMENT }}/model-server-${{ env.PLATFORM_PAIR }}/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }},mode=max
+#           no-cache: true
+
+  build-arm64:
+    needs: [check_model_server_changes]
+    if: needs.check_model_server_changes.outputs.changed == 'true'
+    runs-on:
+      [runs-on, runner=8cpu-linux-arm64, "run-id=${{ github.run_id }}-arm64"]
+    env:
+      PLATFORM_PAIR: linux-arm64
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # ratchet:actions/checkout@v4
+
+      - name: System Info
+        run: |
+          df -h
+          free -h
+          docker system prune -af --volumes
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # ratchet:docker/setup-buildx-action@v3
+        with:
+          driver-opts: |
+            image=moby/buildkit:latest
+            network=host
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # ratchet:docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_TOKEN }}
+
+      - name: Build and Push ARM64
+        uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # ratchet:docker/build-push-action@v6
+        with:
+          context: ./backend
+          file: ./backend/Dockerfile.model_server
+          platforms: linux/arm64
+          push: true
+          tags: ${{ env.REGISTRY_IMAGE }}:${{ github.ref_name }}-arm64
+          build-args: |
+            ONYX_VERSION=${{ github.ref_name }}
+          outputs: type=registry
+          provenance: false
+          cache-from: type=s3,prefix=cache/${{ github.repository }}/${{ env.DEPLOYMENT }}/model-server-${{ env.PLATFORM_PAIR }}/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }}
+          cache-to: type=s3,prefix=cache/${{ github.repository }}/${{ env.DEPLOYMENT }}/model-server-${{ env.PLATFORM_PAIR }}/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }},mode=max
+
+  merge-and-scan:
+    needs: [build-amd64, build-arm64, check_model_server_changes]
+    if: needs.check_model_server_changes.outputs.changed == 'true'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check if stable release version
+        id: check_version
+        run: |
+          if [[ "${{ github.ref_name }}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]] && [[ "${{ github.ref_name }}" != *"cloud"* ]]; then
+            echo "is_stable=true" >> $GITHUB_OUTPUT
+          else
+            echo "is_stable=false" >> $GITHUB_OUTPUT
+          fi
+          if [[ "${{ github.ref_name }}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+-beta\.[0-9]+$ ]] && [[ "${{ github.ref_name }}" != *"cloud"* ]]; then
+            echo "is_beta=true" >> $GITHUB_OUTPUT
+          else
+            echo "is_beta=false" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # ratchet:docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_TOKEN }}
+
+      - name: Create and Push Multi-arch Manifest
+        run: |
+          docker buildx create --use
+          docker buildx imagetools create -t ${{ env.REGISTRY_IMAGE }}:${{ github.ref_name }} \
+            ${{ env.REGISTRY_IMAGE }}:${{ github.ref_name }}-amd64 \
+            ${{ env.REGISTRY_IMAGE }}:${{ github.ref_name }}-arm64
+          if [[ "${{ steps.check_version.outputs.is_stable }}" == "true" ]]; then
+            docker buildx imagetools create -t ${{ env.REGISTRY_IMAGE }}:latest \
+              ${{ env.REGISTRY_IMAGE }}:${{ github.ref_name }}-amd64 \
+              ${{ env.REGISTRY_IMAGE }}:${{ github.ref_name }}-arm64
+          fi
+          if [[ "${{ env.EDGE_TAG }}" == "true" ]]; then
+            docker buildx imagetools create -t ${{ env.REGISTRY_IMAGE }}:edge \
+              ${{ env.REGISTRY_IMAGE }}:${{ github.ref_name }}-amd64 \
+              ${{ env.REGISTRY_IMAGE }}:${{ github.ref_name }}-arm64
+          fi
+          if [[ "${{ steps.check_version.outputs.is_beta }}" == "true" ]]; then
+            docker buildx imagetools create -t ${{ env.REGISTRY_IMAGE }}:beta \
+              ${{ env.REGISTRY_IMAGE }}:${{ github.ref_name }}-amd64 \
+              ${{ env.REGISTRY_IMAGE }}:${{ github.ref_name }}-arm64
+          fi
+
+      - name: Run Trivy vulnerability scanner
+        uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # ratchet:nick-fields/retry@v3
+        with:
+          timeout_minutes: 30
+          max_attempts: 3
+          retry_wait_seconds: 10
+          command: |
+            docker run --rm -v $HOME/.cache/trivy:/root/.cache/trivy \
+              -e TRIVY_DB_REPOSITORY="public.ecr.aws/aquasecurity/trivy-db:2" \
+              -e TRIVY_JAVA_DB_REPOSITORY="public.ecr.aws/aquasecurity/trivy-java-db:1" \
+              -e TRIVY_USERNAME="${{ secrets.DOCKER_USERNAME }}" \
+              -e TRIVY_PASSWORD="${{ secrets.DOCKER_TOKEN }}" \
+              aquasec/trivy@sha256:a22415a38938a56c379387a8163fcb0ce38b10ace73e593475d3658d578b2436 \
+              image \
+              --skip-version-check \
+              --timeout 20m \
+              --severity CRITICAL,HIGH \
+              docker.io/${{ env.REGISTRY_IMAGE }}:${{ github.ref_name }}

.github/workflows/docker-build-push-web-container-on-tag.yml

@@ -0,0 +1,204 @@
+name: Build and Push Web Image on Tag
+
+on:
+  push:
+    tags:
+      - "*"
+
+env:
+  REGISTRY_IMAGE: onyxdotapp/onyx-web-server
+
+  # tag nightly builds with "edge"
+  EDGE_TAG: ${{ startsWith(github.ref_name, 'nightly-latest') }}
+
+  DEPLOYMENT: standalone
+
+jobs:
+  precheck:
+    runs-on: [runs-on, runner=2cpu-linux-x64, "run-id=${{ github.run_id }}"]
+    outputs:
+      should-run: ${{ steps.set-output.outputs.should-run }}
+    steps:
+      - name: Check if tag contains "cloud"
+        id: set-output
+        run: |
+          if [[ "${{ github.ref_name }}" == *cloud* ]]; then
+            echo "should-run=false" >> "$GITHUB_OUTPUT"
+          else
+            echo "should-run=true" >> "$GITHUB_OUTPUT"
+          fi
+  build:
+    needs: precheck
+    if: needs.precheck.outputs.should-run == 'true'
+    runs-on:
+      - runs-on
+      - runner=${{ matrix.platform == 'linux/amd64' && '8cpu-linux-x64' || '8cpu-linux-arm64' }}
+      - run-id=${{ github.run_id }}
+      - tag=platform-${{ matrix.platform }}
+    strategy:
+      fail-fast: false
+      matrix:
+        platform:
+          - linux/amd64
+          - linux/arm64
+
+    steps:
+      - name: Prepare
+        run: |
+          platform=${{ matrix.platform }}
+          echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV
+
+      - name: Check if stable release version
+        id: check_version
+        run: |
+          if [[ "${{ github.ref_name }}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+            echo "is_stable=true" >> $GITHUB_OUTPUT
+          else
+            echo "is_stable=false" >> $GITHUB_OUTPUT
+          fi
+          if [[ "${{ github.ref_name }}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+-beta\.[0-9]+$ ]]; then
+            echo "is_beta=true" >> $GITHUB_OUTPUT
+          else
+            echo "is_beta=false" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Checkout
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # ratchet:actions/checkout@v4
+
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@318604b99e75e41977312d83839a89be02ca4893 # ratchet:docker/metadata-action@v5
+        with:
+          images: ${{ env.REGISTRY_IMAGE }}
+          flavor: |
+            latest=false
+          tags: |
+            type=raw,value=${{ github.ref_name }}
+            type=raw,value=${{ steps.check_version.outputs.is_stable == 'true' && 'latest' || '' }}
+            type=raw,value=${{ env.EDGE_TAG == 'true' && 'edge' || '' }}
+            type=raw,value=${{ steps.check_version.outputs.is_beta == 'true' && 'beta' || '' }}
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # ratchet:docker/setup-buildx-action@v3
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # ratchet:docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_TOKEN }}
+
+      - name: Build and push by digest
+        id: build
+        uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # ratchet:docker/build-push-action@v6
+        with:
+          context: ./web
+          file: ./web/Dockerfile
+          platforms: ${{ matrix.platform }}
+          push: true
+          build-args: |
+            ONYX_VERSION=${{ github.ref_name }}
+            NODE_OPTIONS=--max-old-space-size=8192
+
+          labels: ${{ steps.meta.outputs.labels }}
+          outputs: type=image,name=${{ env.REGISTRY_IMAGE }},push-by-digest=true,name-canonical=true,push=true
+          cache-from: type=s3,prefix=cache/${{ github.repository }}/${{ env.DEPLOYMENT }}/web-${{ env.PLATFORM_PAIR }}/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }}
+          cache-to: type=s3,prefix=cache/${{ github.repository }}/${{ env.DEPLOYMENT }}/web-${{ env.PLATFORM_PAIR }}/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }},mode=max
+          # no-cache needed due to weird interactions with the builds for different platforms
+          # NOTE(rkuo): this may not be true any more with the proper cache prefixing by architecture - currently testing with it off
+          
+      - name: Export digest
+        run: |
+          mkdir -p /tmp/digests
+          digest="${{ steps.build.outputs.digest }}"
+          touch "/tmp/digests/${digest#sha256:}"
+
+      - name: Upload digest
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # ratchet:actions/upload-artifact@v4
+        with:
+          name: web-digests-${{ env.PLATFORM_PAIR }}-${{ github.run_id }}
+          path: /tmp/digests/*
+          if-no-files-found: error
+          retention-days: 1
+
+  merge:
+    needs:
+      - build
+    if: needs.precheck.outputs.should-run == 'true'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check if stable release version
+        id: check_version
+        run: |
+          if [[ "${{ github.ref_name }}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]] && [[ "${{ github.ref_name }}" != *"cloud"* ]]; then
+            echo "is_stable=true" >> $GITHUB_OUTPUT
+          else
+            echo "is_stable=false" >> $GITHUB_OUTPUT
+          fi
+          if [[ "${{ github.ref_name }}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+-beta\.[0-9]+$ ]]; then
+            echo "is_beta=true" >> $GITHUB_OUTPUT
+          else
+            echo "is_beta=false" >> $GITHUB_OUTPUT
+          fi
+        
+      - name: Download digests
+        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # ratchet:actions/download-artifact@v4
+        with:
+          path: /tmp/digests
+          pattern: web-digests-*-${{ github.run_id }}
+          merge-multiple: true
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # ratchet:docker/setup-buildx-action@v3
+
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@318604b99e75e41977312d83839a89be02ca4893 # ratchet:docker/metadata-action@v5
+        with:
+          images: ${{ env.REGISTRY_IMAGE }}
+          flavor: |
+            latest=false
+          tags: |
+            type=raw,value=${{ github.ref_name }}
+            type=raw,value=${{ steps.check_version.outputs.is_stable == 'true' && 'latest' || '' }}
+            type=raw,value=${{ env.EDGE_TAG == 'true' && 'edge' || '' }}
+            type=raw,value=${{ steps.check_version.outputs.is_beta == 'true' && 'beta' || '' }}
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # ratchet:docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_TOKEN }}
+
+      - name: Create manifest list and push
+        working-directory: /tmp/digests
+        run: |
+          docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
+            $(printf '${{ env.REGISTRY_IMAGE }}@sha256:%s ' *)
+
+      - name: Inspect image
+        run: |
+          docker buildx imagetools inspect ${{ env.REGISTRY_IMAGE }}:${{ steps.meta.outputs.version }}
+
+      # trivy has their own rate limiting issues causing this action to flake
+      # we worked around it by hardcoding to different db repos in env
+      # can re-enable when they figure it out
+      # https://github.com/aquasecurity/trivy/discussions/7538
+      # https://github.com/aquasecurity/trivy-action/issues/389
+      - name: Run Trivy vulnerability scanner
+        uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # ratchet:nick-fields/retry@v3
+        with:
+          timeout_minutes: 30
+          max_attempts: 3
+          retry_wait_seconds: 10
+          command: |
+            docker run --rm -v $HOME/.cache/trivy:/root/.cache/trivy \
+              -e TRIVY_DB_REPOSITORY="public.ecr.aws/aquasecurity/trivy-db:2" \
+              -e TRIVY_JAVA_DB_REPOSITORY="public.ecr.aws/aquasecurity/trivy-java-db:1" \
+              -e TRIVY_USERNAME="${{ secrets.DOCKER_USERNAME }}" \
+              -e TRIVY_PASSWORD="${{ secrets.DOCKER_TOKEN }}" \
+              aquasec/trivy@sha256:a22415a38938a56c379387a8163fcb0ce38b10ace73e593475d3658d578b2436 \
+              image \
+              --skip-version-check \
+              --timeout 20m \
+              --severity CRITICAL,HIGH \
+              docker.io/${{ env.REGISTRY_IMAGE }}:${{ github.ref_name }}

.github/workflows/docker-tag-beta.yml

@@ -10,21 +10,17 @@ on:
         description: "The version (ie v1.0.0-beta.0) to tag as beta"
         required: true
 
-permissions:
-  contents: read
-
 jobs:
   tag:
     # See https://runs-on.com/runners/linux/
     # use a lower powered instance since this just does i/o to docker hub
-    runs-on: [runs-on, runner=2cpu-linux-x64, "run-id=${{ github.run_id }}-tag"]
-    timeout-minutes: 45
+    runs-on: [runs-on, runner=2cpu-linux-x64, "run-id=${{ github.run_id }}"]
     steps:
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # ratchet:docker/setup-buildx-action@v3
+        uses: docker/setup-buildx-action@f211e3e9ded2d9377c8cadc4489a4e38014bc4c9 # ratchet:docker/setup-buildx-action@v1
 
       - name: Login to Docker Hub
-        uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # ratchet:docker/login-action@v3
+        uses: docker/login-action@dd4fa0671be5250ee6f50aedf4cb05514abda2c7 # ratchet:docker/login-action@v1
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_TOKEN }}
@@ -33,19 +29,13 @@ jobs:
         run: echo "DOCKER_CLI_EXPERIMENTAL=enabled" >> $GITHUB_ENV
 
       - name: Pull, Tag and Push Web Server Image
-        env:
-          VERSION: ${{ github.event.inputs.version }}
         run: |
-          docker buildx imagetools create -t onyxdotapp/onyx-web-server:beta onyxdotapp/onyx-web-server:${VERSION}
+          docker buildx imagetools create -t onyxdotapp/onyx-web-server:beta onyxdotapp/onyx-web-server:${{ github.event.inputs.version }}
 
       - name: Pull, Tag and Push API Server Image
-        env:
-          VERSION: ${{ github.event.inputs.version }}
         run: |
-          docker buildx imagetools create -t onyxdotapp/onyx-backend:beta onyxdotapp/onyx-backend:${VERSION}
+          docker buildx imagetools create -t onyxdotapp/onyx-backend:beta onyxdotapp/onyx-backend:${{ github.event.inputs.version }}
 
       - name: Pull, Tag and Push Model Server Image
-        env:
-          VERSION: ${{ github.event.inputs.version }}
         run: |
-          docker buildx imagetools create -t onyxdotapp/onyx-model-server:beta onyxdotapp/onyx-model-server:${VERSION}
+          docker buildx imagetools create -t onyxdotapp/onyx-model-server:beta onyxdotapp/onyx-model-server:${{ github.event.inputs.version }}

.github/workflows/docker-tag-latest.yml

@@ -10,21 +10,17 @@ on:
         description: "The version (ie v0.0.1) to tag as latest"
         required: true
 
-permissions:
-  contents: read
-
 jobs:
   tag:
     # See https://runs-on.com/runners/linux/
     # use a lower powered instance since this just does i/o to docker hub
-    runs-on: [runs-on, runner=2cpu-linux-x64, "run-id=${{ github.run_id }}-tag"]
-    timeout-minutes: 45
+    runs-on: [runs-on, runner=2cpu-linux-x64, "run-id=${{ github.run_id }}"]
     steps:
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # ratchet:docker/setup-buildx-action@v3
+        uses: docker/setup-buildx-action@f211e3e9ded2d9377c8cadc4489a4e38014bc4c9 # ratchet:docker/setup-buildx-action@v1
 
       - name: Login to Docker Hub
-        uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # ratchet:docker/login-action@v3
+        uses: docker/login-action@dd4fa0671be5250ee6f50aedf4cb05514abda2c7 # ratchet:docker/login-action@v1
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_TOKEN }}
@@ -33,19 +29,13 @@ jobs:
         run: echo "DOCKER_CLI_EXPERIMENTAL=enabled" >> $GITHUB_ENV
 
       - name: Pull, Tag and Push Web Server Image
-        env:
-          VERSION: ${{ github.event.inputs.version }}
         run: |
-          docker buildx imagetools create -t onyxdotapp/onyx-web-server:latest onyxdotapp/onyx-web-server:${VERSION}
+          docker buildx imagetools create -t onyxdotapp/onyx-web-server:latest onyxdotapp/onyx-web-server:${{ github.event.inputs.version }}
 
       - name: Pull, Tag and Push API Server Image
-        env:
-          VERSION: ${{ github.event.inputs.version }}
         run: |
-          docker buildx imagetools create -t onyxdotapp/onyx-backend:latest onyxdotapp/onyx-backend:${VERSION}
+          docker buildx imagetools create -t onyxdotapp/onyx-backend:latest onyxdotapp/onyx-backend:${{ github.event.inputs.version }}
 
       - name: Pull, Tag and Push Model Server Image
-        env:
-          VERSION: ${{ github.event.inputs.version }}
         run: |
-          docker buildx imagetools create -t onyxdotapp/onyx-model-server:latest onyxdotapp/onyx-model-server:${VERSION}
+          docker buildx imagetools create -t onyxdotapp/onyx-model-server:latest onyxdotapp/onyx-model-server:${{ github.event.inputs.version }}

.github/workflows/helm-chart-releases.yml

@@ -12,13 +12,11 @@ jobs:
     permissions:
       contents: write
     runs-on: ubuntu-latest
-    timeout-minutes: 45
     steps:
       - name: Checkout
-        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # ratchet:actions/checkout@v6
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # ratchet:actions/checkout@v4
         with:
           fetch-depth: 0
-          persist-credentials: false
 
       - name: Install Helm CLI
         uses: azure/setup-helm@1a275c3b69536ee54be43f2070a358922e12c8d4 # ratchet:azure/setup-helm@v4
@@ -32,7 +30,6 @@ jobs:
           helm repo add cloudnative-pg https://cloudnative-pg.github.io/charts
           helm repo add ot-container-kit https://ot-container-kit.github.io/helm-charts
           helm repo add minio https://charts.min.io/
-          helm repo add code-interpreter https://onyx-dot-app.github.io/code-interpreter/
           helm repo update
 
       - name: Build chart dependencies

.github/workflows/nightly-close-stale-issues.yml

@@ -7,13 +7,12 @@ permissions:
   # contents: write # only for delete-branch option
   issues: write
   pull-requests: write
-
+  
 jobs:
   stale:
     runs-on: ubuntu-latest
-    timeout-minutes: 45
     steps:
-      - uses: actions/stale@5f858e3efba33a5ca4407a664cc011ad407f2008 # ratchet:actions/stale@v10
+      - uses: actions/stale@5bef64f19d7facfb25b37b414482c7164d639639 # ratchet:actions/stale@v9
         with:
           stale-issue-message: 'This issue is stale because it has been open 75 days with no activity. Remove stale label or comment or this will be closed in 15 days.'
           stale-pr-message: 'This PR is stale because it has been open 75 days with no activity. Remove stale label or comment or this will be closed in 15 days.'
@@ -21,3 +20,4 @@ jobs:
           close-pr-message: 'This PR was closed because it has been stalled for 90 days with no activity.'
           days-before-stale: 75
 #           days-before-close: 90  # uncomment after we test stale behavior
+          

.github/workflows/nightly-scan-licenses.yml

@@ -15,25 +15,19 @@ on:
 permissions:
   actions: read
   contents: read
-
+  security-events: write
+  
 jobs:
   scan-licenses:
     # See https://runs-on.com/runners/linux/
-    runs-on: [runs-on,runner=2cpu-linux-x64,"run-id=${{ github.run_id }}-scan-licenses"]
-    timeout-minutes: 45
-    permissions:
-      actions: read
-      contents: read
-      security-events: write
+    runs-on: [runs-on,runner=2cpu-linux-x64,"run-id=${{ github.run_id }}"]
 
     steps:
       - name: Checkout code
-        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # ratchet:actions/checkout@v6
-        with:
-          persist-credentials: false
-
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # ratchet:actions/checkout@v4
+        
       - name: Set up Python
-        uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # ratchet:actions/setup-python@v6
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # ratchet:actions/setup-python@v5
         with:
           python-version: '3.11'
           cache: 'pip'
@@ -41,7 +35,7 @@ jobs:
             backend/requirements/default.txt
             backend/requirements/dev.txt
             backend/requirements/model_server.txt
-
+      
       - name: Get explicit and transitive dependencies
         run: |
           python -m pip install --upgrade pip
@@ -49,30 +43,28 @@ jobs:
           pip install --retries 5 --timeout 30 -r backend/requirements/dev.txt
           pip install --retries 5 --timeout 30 -r backend/requirements/model_server.txt
           pip freeze > requirements-all.txt
-
+                    
       - name: Check python
         id: license_check_report
-        uses: pilosus/action-pip-license-checker@e909b0226ff49d3235c99c4585bc617f49fff16a # ratchet:pilosus/action-pip-license-checker@v3
+        uses: pilosus/action-pip-license-checker@cc7a461bfa27b44ad187b8578c881ef5138c13fd # ratchet:pilosus/action-pip-license-checker@v2
         with:
           requirements: 'requirements-all.txt'
           fail: 'Copyleft'
           exclude: '(?i)^(pylint|aio[-_]*).*'
-
+          
       - name: Print report
         if: always()
-        env:
-          REPORT: ${{ steps.license_check_report.outputs.report }}
-        run: echo "$REPORT"
-
+        run: echo "${{ steps.license_check_report.outputs.report }}"
+      
       - name: Install npm dependencies
         working-directory: ./web
         run: npm ci
 
         # be careful enabling the sarif and upload as it may spam the security tab
-        # with a huge amount of items. Work out the issues before enabling upload.
+        # with a huge amount of items. Work out the issues before enabling upload.       
 #       - name: Run Trivy vulnerability scanner in repo mode
 #         if: always()
-#         uses: aquasecurity/trivy-action@b6643a29fecd7f34b3597bc6acb0a98b03d33ff8 # ratchet:aquasecurity/trivy-action@0.33.1
+#         uses: aquasecurity/trivy-action@18f2510ee396bbf400402947b394f2dd8c87dbb0 # ratchet:aquasecurity/trivy-action@0.29.0
 #         with:
 #           scan-type: fs
 #           scan-ref: .
@@ -81,7 +73,7 @@ jobs:
 #           severity: HIGH,CRITICAL
 # #           format: sarif
 # #           output: trivy-results.sarif
-#
+# 
 # #       - name: Upload Trivy scan results to GitHub Security tab
 # #         uses: github/codeql-action/upload-sarif@v3
 # #         with:
@@ -89,9 +81,8 @@ jobs:
 
   scan-trivy:
     # See https://runs-on.com/runners/linux/
-    runs-on: [runs-on,runner=2cpu-linux-x64,"run-id=${{ github.run_id }}-scan-trivy"]
-    timeout-minutes: 45
-
+    runs-on: [runs-on,runner=2cpu-linux-x64,"run-id=${{ github.run_id }}"]
+      
     steps:
     - name: Set up Docker Buildx
       uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # ratchet:docker/setup-buildx-action@v3
@@ -107,7 +98,7 @@ jobs:
       run: docker pull onyxdotapp/onyx-backend:latest
 
     - name: Run Trivy vulnerability scanner on backend
-      uses: aquasecurity/trivy-action@b6643a29fecd7f34b3597bc6acb0a98b03d33ff8 # ratchet:aquasecurity/trivy-action@0.33.1
+      uses: aquasecurity/trivy-action@18f2510ee396bbf400402947b394f2dd8c87dbb0 # ratchet:aquasecurity/trivy-action@0.29.0
       env:
         TRIVY_DB_REPOSITORY: 'public.ecr.aws/aquasecurity/trivy-db:2'
         TRIVY_JAVA_DB_REPOSITORY: 'public.ecr.aws/aquasecurity/trivy-java-db:1'
@@ -121,9 +112,9 @@ jobs:
     # Web server
     - name: Pull web server docker image
       run: docker pull onyxdotapp/onyx-web-server:latest
-
+          
     - name: Run Trivy vulnerability scanner on web server
-      uses: aquasecurity/trivy-action@b6643a29fecd7f34b3597bc6acb0a98b03d33ff8 # ratchet:aquasecurity/trivy-action@0.33.1
+      uses: aquasecurity/trivy-action@18f2510ee396bbf400402947b394f2dd8c87dbb0 # ratchet:aquasecurity/trivy-action@0.29.0
       env:
         TRIVY_DB_REPOSITORY: 'public.ecr.aws/aquasecurity/trivy-db:2'
         TRIVY_JAVA_DB_REPOSITORY: 'public.ecr.aws/aquasecurity/trivy-java-db:1'
@@ -139,7 +130,7 @@ jobs:
       run: docker pull onyxdotapp/onyx-model-server:latest
 
     - name: Run Trivy vulnerability scanner
-      uses: aquasecurity/trivy-action@b6643a29fecd7f34b3597bc6acb0a98b03d33ff8 # ratchet:aquasecurity/trivy-action@0.33.1
+      uses: aquasecurity/trivy-action@18f2510ee396bbf400402947b394f2dd8c87dbb0 # ratchet:aquasecurity/trivy-action@0.29.0
       env:
         TRIVY_DB_REPOSITORY: 'public.ecr.aws/aquasecurity/trivy-db:2'
         TRIVY_JAVA_DB_REPOSITORY: 'public.ecr.aws/aquasecurity/trivy-java-db:1'
@@ -148,4 +139,4 @@ jobs:
         scanners: license
         severity: HIGH,CRITICAL
         vuln-type: library
-        exit-code: 0
+        exit-code: 0

.github/workflows/pr-external-dependency-unit-tests.yml

@@ -1,28 +1,17 @@
 name: External Dependency Unit Tests
-concurrency:
-  group: External-Dependency-Unit-Tests-${{ github.workflow }}-${{ github.head_ref || github.event.workflow_run.head_branch || github.run_id }}
-  cancel-in-progress: true
 
 on:
   merge_group:
   pull_request:
     branches: [main]
-  push:
-    tags:
-      - "v*.*.*"
-
-permissions:
-  contents: read
 
 env:
-  # AWS credentials for S3-specific test
-  S3_AWS_ACCESS_KEY_ID_FOR_TEST: ${{ secrets.S3_AWS_ACCESS_KEY_ID }}
-  S3_AWS_SECRET_ACCESS_KEY_FOR_TEST: ${{ secrets.S3_AWS_SECRET_ACCESS_KEY }}
+  # AWS
+  S3_AWS_ACCESS_KEY_ID: ${{ secrets.S3_AWS_ACCESS_KEY_ID }}
+  S3_AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_AWS_SECRET_ACCESS_KEY }}
 
   # MinIO
   S3_ENDPOINT_URL: "http://localhost:9004"
-  S3_AWS_ACCESS_KEY_ID: "minioadmin"
-  S3_AWS_SECRET_ACCESS_KEY: "minioadmin"
 
   # Confluence
   CONFLUENCE_TEST_SPACE_URL: ${{ vars.CONFLUENCE_TEST_SPACE_URL }}
@@ -32,29 +21,18 @@ env:
   CONFLUENCE_ACCESS_TOKEN: ${{ secrets.CONFLUENCE_ACCESS_TOKEN }}
   CONFLUENCE_ACCESS_TOKEN_SCOPED: ${{ secrets.CONFLUENCE_ACCESS_TOKEN_SCOPED }}
 
-  # Jira
-  JIRA_ADMIN_API_TOKEN: ${{ secrets.JIRA_ADMIN_API_TOKEN }}
-
   # LLMs
   OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
   ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
 
-  # Code Interpreter
-  # TODO: debug why this is failing and enable
-  CODE_INTERPRETER_BASE_URL: http://localhost:8000
-
 jobs:
   discover-test-dirs:
-    # NOTE: Github-hosted runners have about 20s faster queue times and are preferred here.
-    runs-on: ubuntu-slim
-    timeout-minutes: 45
+    runs-on: ubuntu-latest
     outputs:
       test-dirs: ${{ steps.set-matrix.outputs.test-dirs }}
     steps:
       - name: Checkout code
-        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # ratchet:actions/checkout@v6
-        with:
-          persist-credentials: false
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # ratchet:actions/checkout@v4
 
       - name: Discover test directories
         id: set-matrix
@@ -66,12 +44,7 @@ jobs:
   external-dependency-unit-tests:
     needs: discover-test-dirs
     # Use larger runner with more resources for Vespa
-    runs-on:
-      - runs-on
-      - runner=2cpu-linux-arm64
-      - ${{ format('run-id={0}-external-dependency-unit-tests-job-{1}', github.run_id, strategy['job-index']) }}
-      - extras=s3-cache
-    timeout-minutes: 45
+    runs-on: [runs-on, runner=16cpu-linux-x64, "run-id=${{ github.run_id }}", "extras=s3-cache"]
     strategy:
       fail-fast: false
       matrix:
@@ -80,55 +53,43 @@ jobs:
     env:
       PYTHONPATH: ./backend
       MODEL_SERVER_HOST: "disabled"
-      DISABLE_TELEMETRY: "true"
 
     steps:
       - uses: runs-on/action@cd2b598b0515d39d78c38a02d529db87d2196d1e # ratchet:runs-on/action@v2
 
       - name: Checkout code
-        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # ratchet:actions/checkout@v6
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # ratchet:actions/checkout@v4
+
+      - name: Setup uv
+        uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # ratchet:astral-sh/setup-uv@v6
         with:
-          persist-credentials: false
+          enable-cache: true
 
-      - name: Setup Python and Install Dependencies
-        uses: ./.github/actions/setup-python-and-install-dependencies
+      - name: Cache uv cache directory
+        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # ratchet:actions/cache@v4
         with:
-          requirements: |
-            backend/requirements/default.txt
-            backend/requirements/dev.txt
-            backend/requirements/ee.txt
+          path: ~/.cache/uv
+          key: ${{ runner.os }}-${{ github.workflow }}-uv-${{ hashFiles('backend/requirements/*.txt', 'backend/pyproject.toml') }}
+          restore-keys: |
+            ${{ runner.os }}-${{ github.workflow }}-uv-
 
-      - name: Setup Playwright
-        uses: ./.github/actions/setup-playwright
-
-      # needed for pulling Vespa, Redis, Postgres, and Minio images
-      # otherwise, we hit the "Unauthenticated users" limit
-      # https://docs.docker.com/docker-hub/usage/
-      - name: Login to Docker Hub
-        uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # ratchet:docker/login-action@v3
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # ratchet:actions/setup-python@v5
         with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_TOKEN }}
+          python-version: "3.11"
 
-      - name: Create .env file for Docker Compose
+      - name: Install Dependencies
         run: |
-          cat <<EOF > deployment/docker_compose/.env
-          CODE_INTERPRETER_BETA_ENABLED=true
-          DISABLE_TELEMETRY=true
-          EOF
+          uv pip install --system \
+            -r backend/requirements/default.txt \
+            -r backend/requirements/dev.txt
+          playwright install chromium
+          playwright install-deps chromium
 
       - name: Set up Standard Dependencies
         run: |
           cd deployment/docker_compose
-          docker compose \
-            -f docker-compose.yml \
-            -f docker-compose.dev.yml \
-            up -d \
-            minio \
-            relational_db \
-            cache \
-            index \
-            code-interpreter
+          docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d minio relational_db cache index
 
       - name: Run migrations
         run: |
@@ -139,39 +100,10 @@ jobs:
 
       - name: Run Tests for ${{ matrix.test-dir }}
         shell: script -q -e -c "bash --noprofile --norc -eo pipefail {0}"
-        env:
-          TEST_DIR: ${{ matrix.test-dir }}
         run: |
           py.test \
             --durations=8 \
             -o junit_family=xunit2 \
             -xv \
             --ff \
-            backend/tests/external_dependency_unit/${TEST_DIR}
-
-      - name: Collect Docker logs on failure
-        if: failure()
-        run: |
-          mkdir -p docker-logs
-          cd deployment/docker_compose
-
-          # Get list of running containers
-          containers=$(docker compose -f docker-compose.yml -f docker-compose.dev.yml ps -q)
-
-          # Collect logs from each container
-          for container in $containers; do
-            container_name=$(docker inspect --format='{{.Name}}' $container | sed 's/^\///')
-            echo "Collecting logs from $container_name..."
-            docker logs $container > ../../docker-logs/${container_name}.log 2>&1
-          done
-
-          cd ../..
-          echo "Docker logs collected in docker-logs directory"
-
-      - name: Upload Docker logs
-        if: failure()
-        uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # ratchet:actions/upload-artifact@v5
-        with:
-          name: docker-logs-${{ matrix.test-dir }}
-          path: docker-logs/
-          retention-days: 7
+            backend/tests/external_dependency_unit/${{ matrix.test-dir }}

.github/workflows/pr-helm-chart-testing.yml

@@ -1,50 +1,37 @@
 name: Helm - Lint and Test Charts
-concurrency:
-  group: Helm-Lint-and-Test-Charts-${{ github.workflow }}-${{ github.head_ref || github.event.workflow_run.head_branch || github.run_id }}
-  cancel-in-progress: true
 
 on:
   merge_group:
   pull_request:
     branches: [ main ]
-  push:
-    tags:
-      - "v*.*.*"
   workflow_dispatch:  # Allows manual triggering
-
-permissions:
-  contents: read
-
+  
 jobs:
   helm-chart-check:
     # See https://runs-on.com/runners/linux/
-    runs-on: [runs-on,runner=8cpu-linux-x64,hdd=256,"run-id=${{ github.run_id }}-helm-chart-check"]
-    timeout-minutes: 45
+    runs-on: [runs-on,runner=8cpu-linux-x64,hdd=256,"run-id=${{ github.run_id }}"]
 
     # fetch-depth 0 is required for helm/chart-testing-action
     steps:
     - name: Checkout code
-      uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # ratchet:actions/checkout@v6
+      uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # ratchet:actions/checkout@v4
       with:
         fetch-depth: 0
-        persist-credentials: false
-
+        
     - name: Set up Helm
       uses: azure/setup-helm@1a275c3b69536ee54be43f2070a358922e12c8d4 # ratchet:azure/setup-helm@v4.3.1
       with:
         version: v3.19.0
-
+      
     - name: Set up chart-testing
       uses: helm/chart-testing-action@6ec842c01de15ebb84c8627d2744a0c2f2755c9f # ratchet:helm/chart-testing-action@v2.8.0
 
     # even though we specify chart-dirs in ct.yaml, it isn't used by ct for the list-changed command...
     - name: Run chart-testing (list-changed)
       id: list-changed
-      env:
-        DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
       run: |
-        echo "default_branch: ${DEFAULT_BRANCH}"
-        changed=$(ct list-changed --remote origin --target-branch ${DEFAULT_BRANCH} --chart-dirs deployment/helm/charts)
+        echo "default_branch: ${{ github.event.repository.default_branch }}"
+        changed=$(ct list-changed --remote origin --target-branch ${{ github.event.repository.default_branch }} --chart-dirs deployment/helm/charts)
         echo "list-changed output: $changed"
         if [[ -n "$changed" ]]; then
           echo "changed=true" >> "$GITHUB_OUTPUT"
@@ -54,7 +41,7 @@ jobs:
 #     - name: Force run chart-testing (list-changed)
 #       id: list-changed
 #       run: echo "changed=true" >> $GITHUB_OUTPUT
-
+        
     # lint all charts if any changes were detected
     - name: Run chart-testing (lint)
       if: steps.list-changed.outputs.changed == 'true'
@@ -64,7 +51,7 @@ jobs:
 
     - name: Create kind cluster
       if: steps.list-changed.outputs.changed == 'true'
-      uses: helm/kind-action@92086f6be054225fa813e0a4b13787fc9088faab # ratchet:helm/kind-action@v1.13.0
+      uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # ratchet:helm/kind-action@v1.12.0
 
     - name: Pre-install cluster status check
       if: steps.list-changed.outputs.changed == 'true'
@@ -83,7 +70,6 @@ jobs:
         helm repo add cloudnative-pg https://cloudnative-pg.github.io/charts
         helm repo add ot-container-kit https://ot-container-kit.github.io/helm-charts
         helm repo add minio https://charts.min.io/
-        helm repo add code-interpreter https://onyx-dot-app.github.io/code-interpreter/
         helm repo update
 
     - name: Install Redis operator
@@ -132,7 +118,7 @@ jobs:
       if: steps.list-changed.outputs.changed == 'true'
       run: |
         echo "=== Starting chart installation with monitoring ==="
-
+        
         # Function to monitor cluster state
         monitor_cluster() {
           while true; do
@@ -154,11 +140,11 @@ jobs:
             sleep 60
           done
         }
-
+        
         # Start monitoring in background
         monitor_cluster &
         MONITOR_PID=$!
-
+        
         # Set up cleanup
         cleanup() {
           echo "=== Cleaning up monitoring process ==="
@@ -167,10 +153,10 @@ jobs:
           kubectl get pods --all-namespaces
           kubectl get events --all-namespaces --sort-by=.lastTimestamp | tail -20
         }
-
+        
         # Trap cleanup on exit
         trap cleanup EXIT
-
+        
         # Run the actual installation with detailed logging
         echo "=== Starting ct install ==="
         set +e
@@ -228,15 +214,15 @@ jobs:
         echo "=== Final cluster state ==="
         kubectl get pods --all-namespaces
         kubectl get events --all-namespaces --sort-by=.lastTimestamp | tail -10
-
+        
         echo "=== Pod descriptions for debugging ==="
         kubectl describe pods --all-namespaces | grep -A 10 -B 3 "Failed\|Error\|Warning\|Pending" || echo "No problematic pods found"
-
+        
         echo "=== Recent logs for debugging ==="
         kubectl logs --all-namespaces --tail=50 | grep -i "error\|timeout\|failed\|pull" || echo "No error logs found"
 
         echo "=== Helm releases ==="
         helm list --all-namespaces
-      # the following would install only changed charts, but we only have one chart so
+      # the following would install only changed charts, but we only have one chart so 
       # don't worry about that for now
       # run: ct install --target-branch ${{ github.event.repository.default_branch }}

.github/workflows/pr-integration-tests.yml

@@ -9,14 +9,13 @@ on:
     branches:
       - main
       - "release/**"
-  push:
-    tags:
-      - "v*.*.*"
-
-permissions:
-  contents: read
 
 env:
+  # Private Registry Configuration
+  PRIVATE_REGISTRY: experimental-registry.blacksmith.sh:5000
+  PRIVATE_REGISTRY_USERNAME: ${{ secrets.PRIVATE_REGISTRY_USERNAME }}
+  PRIVATE_REGISTRY_PASSWORD: ${{ secrets.PRIVATE_REGISTRY_PASSWORD }}
+
   # Test Environment Variables
   OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
   SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
@@ -36,16 +35,12 @@ env:
 
 jobs:
   discover-test-dirs:
-    # NOTE: Github-hosted runners have about 20s faster queue times and are preferred here.
-    runs-on: ubuntu-slim
-    timeout-minutes: 45
+    runs-on: blacksmith-2vcpu-ubuntu-2404-arm
     outputs:
       test-dirs: ${{ steps.set-matrix.outputs.test-dirs }}
     steps:
       - name: Checkout code
-        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # ratchet:actions/checkout@v6
-        with:
-          persist-credentials: false
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # ratchet:actions/checkout@v4
 
       - name: Discover test directories
         id: set-matrix
@@ -67,174 +62,100 @@ jobs:
           all_dirs="[${all_dirs%,}]"
           echo "test-dirs=$all_dirs" >> $GITHUB_OUTPUT
 
+  prepare-build:
+    runs-on: blacksmith-2vcpu-ubuntu-2404-arm
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # ratchet:actions/checkout@v4
+
+      - name: Prepare build
+        uses: ./.github/actions/prepare-build
 
   build-backend-image:
-    runs-on: [runs-on, runner=1cpu-linux-arm64, "run-id=${{ github.run_id }}-build-backend-image", "extras=ecr-cache"]
-    timeout-minutes: 45
+    runs-on: blacksmith-16vcpu-ubuntu-2404-arm
     steps:
-      - uses: runs-on/action@cd2b598b0515d39d78c38a02d529db87d2196d1e # ratchet:runs-on/action@v2
       - name: Checkout code
-        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # ratchet:actions/checkout@v6
-        with:
-          persist-credentials: false
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # ratchet:actions/checkout@v4
 
-      - name: Format branch name for cache
-        id: format-branch
-        env:
-          PR_NUMBER: ${{ github.event.pull_request.number }}
-          REF_NAME: ${{ github.ref_name }}
-        run: |
-          if [ -n "${PR_NUMBER}" ]; then
-            CACHE_SUFFIX="${PR_NUMBER}"
-          else
-            # shellcheck disable=SC2001
-            CACHE_SUFFIX=$(echo "${REF_NAME}" | sed 's/[^A-Za-z0-9._-]/-/g')
-          fi
-          echo "cache-suffix=${CACHE_SUFFIX}" >> $GITHUB_OUTPUT
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # ratchet:docker/setup-buildx-action@v3
-
-      # needed for pulling Vespa, Redis, Postgres, and Minio images
-      # otherwise, we hit the "Unauthenticated users" limit
-      # https://docs.docker.com/docker-hub/usage/
-      - name: Login to Docker Hub
+      - name: Login to Private Registry
         uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # ratchet:docker/login-action@v3
         with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_TOKEN }}
+          registry: ${{ env.PRIVATE_REGISTRY }}
+          username: ${{ env.PRIVATE_REGISTRY_USERNAME }}
+          password: ${{ env.PRIVATE_REGISTRY_PASSWORD }}
+
+      - name: Set up Docker Buildx
+        uses: useblacksmith/setup-docker-builder@affa10db466676f3dfb3e54caeb228ee0691510f # ratchet:useblacksmith/setup-docker-builder@v1
 
       - name: Build and push Backend Docker image
-        uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # ratchet:docker/build-push-action@v6
+        uses: useblacksmith/build-push-action@30c71162f16ea2c27c3e21523255d209b8b538c1 # ratchet:useblacksmith/build-push-action@v2
         with:
           context: ./backend
           file: ./backend/Dockerfile
+          platforms: linux/arm64
+          tags: ${{ env.PRIVATE_REGISTRY }}/integration-test-onyx-backend:test-${{ github.run_id }}
           push: true
-          tags: ${{ env.RUNS_ON_ECR_CACHE }}:integration-test-backend-test-${{ github.run_id }}
-          cache-from: |
-            type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:backend-cache-${{ github.event.pull_request.head.sha || github.sha }}
-            type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:backend-cache-${{ steps.format-branch.outputs.cache-suffix }}
-            type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:backend-cache
-            type=registry,ref=onyxdotapp/onyx-backend:latest
-          cache-to: |
-            type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:backend-cache-${{ github.event.pull_request.head.sha || github.sha }},mode=max
-            type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:backend-cache-${{ steps.format-branch.outputs.cache-suffix }},mode=max
-            type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:backend-cache,mode=max
-          no-cache: ${{ vars.DOCKER_NO_CACHE == 'true' }}
+          outputs: type=registry
+          no-cache: true
 
 
   build-model-server-image:
-    runs-on: [runs-on, runner=1cpu-linux-arm64, "run-id=${{ github.run_id }}-build-model-server-image", "extras=ecr-cache"]
-    timeout-minutes: 45
+    runs-on: blacksmith-16vcpu-ubuntu-2404-arm
     steps:
-      - uses: runs-on/action@cd2b598b0515d39d78c38a02d529db87d2196d1e # ratchet:runs-on/action@v2
       - name: Checkout code
-        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # ratchet:actions/checkout@v6
-        with:
-          persist-credentials: false
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # ratchet:actions/checkout@v4
 
-      - name: Format branch name for cache
-        id: format-branch
-        env:
-          PR_NUMBER: ${{ github.event.pull_request.number }}
-          REF_NAME: ${{ github.ref_name }}
-        run: |
-          if [ -n "${PR_NUMBER}" ]; then
-            CACHE_SUFFIX="${PR_NUMBER}"
-          else
-            # shellcheck disable=SC2001
-            CACHE_SUFFIX=$(echo "${REF_NAME}" | sed 's/[^A-Za-z0-9._-]/-/g')
-          fi
-          echo "cache-suffix=${CACHE_SUFFIX}" >> $GITHUB_OUTPUT
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # ratchet:docker/setup-buildx-action@v3
-
-      # needed for pulling Vespa, Redis, Postgres, and Minio images
-      # otherwise, we hit the "Unauthenticated users" limit
-      # https://docs.docker.com/docker-hub/usage/
-      - name: Login to Docker Hub
+      - name: Login to Private Registry
         uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # ratchet:docker/login-action@v3
         with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_TOKEN }}
+          registry: ${{ env.PRIVATE_REGISTRY }}
+          username: ${{ env.PRIVATE_REGISTRY_USERNAME }}
+          password: ${{ env.PRIVATE_REGISTRY_PASSWORD }}
+
+      - name: Set up Docker Buildx
+        uses: useblacksmith/setup-docker-builder@affa10db466676f3dfb3e54caeb228ee0691510f # ratchet:useblacksmith/setup-docker-builder@v1
 
       - name: Build and push Model Server Docker image
-        uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # ratchet:docker/build-push-action@v6
+        uses: useblacksmith/build-push-action@30c71162f16ea2c27c3e21523255d209b8b538c1 # ratchet:useblacksmith/build-push-action@v2
         with:
           context: ./backend
           file: ./backend/Dockerfile.model_server
+          platforms: linux/arm64
+          tags: ${{ env.PRIVATE_REGISTRY }}/integration-test-onyx-model-server:test-${{ github.run_id }}
           push: true
-          tags: ${{ env.RUNS_ON_ECR_CACHE }}:integration-test-model-server-test-${{ github.run_id }}
-          cache-from: |
-            type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:model-server-cache-${{ github.event.pull_request.head.sha || github.sha }}
-            type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:model-server-cache-${{ steps.format-branch.outputs.cache-suffix }}
-            type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:model-server-cache
-            type=registry,ref=onyxdotapp/onyx-model-server:latest
-          cache-to: |
-            type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:model-server-cache-${{ github.event.pull_request.head.sha || github.sha }},mode=max
-            type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:model-server-cache-${{ steps.format-branch.outputs.cache-suffix }},mode=max
-            type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:model-server-cache,mode=max
+          outputs: type=registry
+          provenance: false
+          no-cache: ${{ vars.DOCKER_NO_CACHE == 'true' }}
 
 
   build-integration-image:
-    runs-on: [runs-on, runner=2cpu-linux-arm64, "run-id=${{ github.run_id }}-build-integration-image", "extras=ecr-cache"]
-    timeout-minutes: 45
+    needs: prepare-build
+    runs-on: blacksmith-16vcpu-ubuntu-2404-arm
     steps:
-      - uses: runs-on/action@cd2b598b0515d39d78c38a02d529db87d2196d1e # ratchet:runs-on/action@v2
       - name: Checkout code
-        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # ratchet:actions/checkout@v6
-        with:
-          persist-credentials: false
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # ratchet:actions/checkout@v4
 
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # ratchet:docker/setup-buildx-action@v3
-
-      # needed for pulling openapitools/openapi-generator-cli
-      # otherwise, we hit the "Unauthenticated users" limit
-      # https://docs.docker.com/docker-hub/usage/
-      - name: Login to Docker Hub
+      - name: Login to Private Registry
         uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # ratchet:docker/login-action@v3
         with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_TOKEN }}
+          registry: ${{ env.PRIVATE_REGISTRY }}
+          username: ${{ env.PRIVATE_REGISTRY_USERNAME }}
+          password: ${{ env.PRIVATE_REGISTRY_PASSWORD }}
 
-      - name: Format branch name for cache
-        id: format-branch
-        env:
-          PR_NUMBER: ${{ github.event.pull_request.number }}
-          REF_NAME: ${{ github.ref_name }}
-        run: |
-          if [ -n "${PR_NUMBER}" ]; then
-            CACHE_SUFFIX="${PR_NUMBER}"
-          else
-            # shellcheck disable=SC2001
-            CACHE_SUFFIX=$(echo "${REF_NAME}" | sed 's/[^A-Za-z0-9._-]/-/g')
-          fi
-          echo "cache-suffix=${CACHE_SUFFIX}" >> $GITHUB_OUTPUT
+      - name: Download OpenAPI artifacts
+        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # ratchet:actions/download-artifact@v4
+        with:
+          name: openapi-artifacts
+          path: backend/generated/
+
+      - name: Set up Docker Buildx
+        uses: useblacksmith/setup-docker-builder@affa10db466676f3dfb3e54caeb228ee0691510f # ratchet:useblacksmith/setup-docker-builder@v1
 
       - name: Build and push integration test image with Docker Bake
         env:
-          INTEGRATION_REPOSITORY: ${{ env.RUNS_ON_ECR_CACHE }}
-          TAG: integration-test-${{ github.run_id }}
-          CACHE_SUFFIX: ${{ steps.format-branch.outputs.cache-suffix }}
-          HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
-        run: |
-          cd backend && docker buildx bake --push \
-            --set backend.cache-from=type=registry,ref=${RUNS_ON_ECR_CACHE}:backend-cache-${HEAD_SHA} \
-            --set backend.cache-from=type=registry,ref=${RUNS_ON_ECR_CACHE}:backend-cache-${CACHE_SUFFIX} \
-            --set backend.cache-from=type=registry,ref=${RUNS_ON_ECR_CACHE}:backend-cache \
-            --set backend.cache-from=type=registry,ref=onyxdotapp/onyx-backend:latest \
-            --set backend.cache-to=type=registry,ref=${RUNS_ON_ECR_CACHE}:backend-cache-${HEAD_SHA},mode=max \
-            --set backend.cache-to=type=registry,ref=${RUNS_ON_ECR_CACHE}:backend-cache-${CACHE_SUFFIX},mode=max \
-            --set backend.cache-to=type=registry,ref=${RUNS_ON_ECR_CACHE}:backend-cache,mode=max \
-            --set integration.cache-from=type=registry,ref=${RUNS_ON_ECR_CACHE}:integration-cache-${HEAD_SHA} \
-            --set integration.cache-from=type=registry,ref=${RUNS_ON_ECR_CACHE}:integration-cache-${CACHE_SUFFIX} \
-            --set integration.cache-from=type=registry,ref=${RUNS_ON_ECR_CACHE}:integration-cache \
-            --set integration.cache-to=type=registry,ref=${RUNS_ON_ECR_CACHE}:integration-cache-${HEAD_SHA},mode=max \
-            --set integration.cache-to=type=registry,ref=${RUNS_ON_ECR_CACHE}:integration-cache-${CACHE_SUFFIX},mode=max \
-            --set integration.cache-to=type=registry,ref=${RUNS_ON_ECR_CACHE}:integration-cache,mode=max \
-            integration
+          REGISTRY: ${{ env.PRIVATE_REGISTRY }}
+          TAG: test-${{ github.run_id }}
+        run: cd backend && docker buildx bake --no-cache --push integration
 
   integration-tests:
     needs:
@@ -244,12 +165,7 @@ jobs:
         build-model-server-image,
         build-integration-image,
       ]
-    runs-on:
-      - runs-on
-      - runner=4cpu-linux-arm64
-      - ${{ format('run-id={0}-integration-tests-job-{1}', github.run_id, strategy['job-index']) }}
-      - extras=ecr-cache
-    timeout-minutes: 45
+    runs-on: blacksmith-8vcpu-ubuntu-2404-arm
 
     strategy:
       fail-fast: false
@@ -257,11 +173,15 @@ jobs:
         test-dir: ${{ fromJson(needs.discover-test-dirs.outputs.test-dirs) }}
 
     steps:
-      - uses: runs-on/action@cd2b598b0515d39d78c38a02d529db87d2196d1e # ratchet:runs-on/action@v2
       - name: Checkout code
-        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # ratchet:actions/checkout@v6
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # ratchet:actions/checkout@v4
+
+      - name: Login to Private Registry
+        uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # ratchet:docker/login-action@v3
         with:
-          persist-credentials: false
+          registry: ${{ env.PRIVATE_REGISTRY }}
+          username: ${{ env.PRIVATE_REGISTRY_USERNAME }}
+          password: ${{ env.PRIVATE_REGISTRY_PASSWORD }}
 
       # needed for pulling Vespa, Redis, Postgres, and Minio images
       # otherwise, we hit the "Unauthenticated users" limit
@@ -272,30 +192,38 @@ jobs:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_TOKEN }}
 
+      - name: Pull Docker images
+        run: |
+          # Pull all images from registry in parallel
+          echo "Pulling Docker images in parallel..."
+          # Pull images from private registry
+          (docker pull --platform linux/arm64 ${{ env.PRIVATE_REGISTRY }}/integration-test-onyx-backend:test-${{ github.run_id }}) &
+          (docker pull --platform linux/arm64 ${{ env.PRIVATE_REGISTRY }}/integration-test-onyx-model-server:test-${{ github.run_id }}) &
+          (docker pull --platform linux/arm64 ${{ env.PRIVATE_REGISTRY }}/integration-test-onyx-integration:test-${{ github.run_id }}) &
+
+          # Wait for all background jobs to complete
+          wait
+          echo "All Docker images pulled successfully"
+
+          # Re-tag to remove registry prefix for docker-compose
+          docker tag ${{ env.PRIVATE_REGISTRY }}/integration-test-onyx-backend:test-${{ github.run_id }} onyxdotapp/onyx-backend:test
+          docker tag ${{ env.PRIVATE_REGISTRY }}/integration-test-onyx-model-server:test-${{ github.run_id }} onyxdotapp/onyx-model-server:test
+          docker tag ${{ env.PRIVATE_REGISTRY }}/integration-test-onyx-integration:test-${{ github.run_id }} onyxdotapp/onyx-integration:test
+
       # NOTE: Use pre-ping/null pool to reduce flakiness due to dropped connections
       # NOTE: don't need web server for integration tests
-      - name: Create .env file for Docker Compose
-        env:
-          ECR_CACHE: ${{ env.RUNS_ON_ECR_CACHE }}
-          RUN_ID: ${{ github.run_id }}
-        run: |
-          cat <<EOF > deployment/docker_compose/.env
-          ENABLE_PAID_ENTERPRISE_EDITION_FEATURES=true
-          AUTH_TYPE=basic
-          POSTGRES_POOL_PRE_PING=true
-          POSTGRES_USE_NULL_POOL=true
-          REQUIRE_EMAIL_VERIFICATION=false
-          DISABLE_TELEMETRY=true
-          ONYX_BACKEND_IMAGE=${ECR_CACHE}:integration-test-backend-test-${RUN_ID}
-          ONYX_MODEL_SERVER_IMAGE=${ECR_CACHE}:integration-test-model-server-test-${RUN_ID}
-          INTEGRATION_TESTS_MODE=true
-          CHECK_TTL_MANAGEMENT_TASK_FREQUENCY_IN_HOURS=0.001
-          MCP_SERVER_ENABLED=true
-          EOF
-
       - name: Start Docker containers
         run: |
           cd deployment/docker_compose
+          ENABLE_PAID_ENTERPRISE_EDITION_FEATURES=true \
+          AUTH_TYPE=basic \
+          POSTGRES_POOL_PRE_PING=true \
+          POSTGRES_USE_NULL_POOL=true \
+          REQUIRE_EMAIL_VERIFICATION=false \
+          DISABLE_TELEMETRY=true \
+          IMAGE_TAG=test \
+          INTEGRATION_TESTS_MODE=true \
+          CHECK_TTL_MANAGEMENT_TASK_FREQUENCY_IN_HOURS=0.001 \
           docker compose -f docker-compose.yml -f docker-compose.dev.yml up \
             relational_db \
             index \
@@ -304,56 +232,43 @@ jobs:
             api_server \
             inference_model_server \
             indexing_model_server \
-            mcp_server \
             background \
             -d
         id: start_docker
 
-      - name: Wait for services to be ready
+      - name: Wait for service to be ready
         run: |
           echo "Starting wait-for-service script..."
 
-          wait_for_service() {
-            local url=$1
-            local label=$2
-            local timeout=${3:-300}  # default 5 minutes
-            local start_time
-            start_time=$(date +%s)
+          docker logs -f onyx-api_server-1 &
 
-            while true; do
-              local current_time
-              current_time=$(date +%s)
-              local elapsed_time=$((current_time - start_time))
+          start_time=$(date +%s)
+          timeout=300  # 5 minutes in seconds
 
-              if [ $elapsed_time -ge $timeout ]; then
-                echo "Timeout reached. ${label} did not become ready in $timeout seconds."
-                exit 1
-              fi
-
-              local response
-              response=$(curl -s -o /dev/null -w "%{http_code}" "$url" || echo "curl_error")
-
-              if [ "$response" = "200" ]; then
-                echo "${label} is ready!"
-                break
-              elif [ "$response" = "curl_error" ]; then
-                echo "Curl encountered an error while checking ${label}. Retrying in 5 seconds..."
-              else
-                echo "${label} not ready yet (HTTP status $response). Retrying in 5 seconds..."
-              fi
-
-              sleep 5
-            done
-          }
-
-          wait_for_service "http://localhost:8080/health" "API server"
-          test_dir="${{ matrix.test-dir.path }}"
-          if [ "$test_dir" = "tests/mcp" ]; then
-            wait_for_service "http://localhost:8090/health" "MCP server"
-          else
-            echo "Skipping MCP server wait for non-MCP suite: $test_dir"
-          fi
-          echo "Finished waiting for services."
+          while true; do
+            current_time=$(date +%s)
+            elapsed_time=$((current_time - start_time))
+            
+            if [ $elapsed_time -ge $timeout ]; then
+              echo "Timeout reached. Service did not become ready in 5 minutes."
+              exit 1
+            fi
+            
+            # Use curl with error handling to ignore specific exit code 56
+            response=$(curl -s -o /dev/null -w "%{http_code}" http://localhost:8080/health || echo "curl_error")
+            
+            if [ "$response" = "200" ]; then
+              echo "Service is ready!"
+              break
+            elif [ "$response" = "curl_error" ]; then
+              echo "Curl encountered an error, possibly exit code 56. Continuing to retry..."
+            else
+              echo "Service not ready yet (HTTP status $response). Retrying in 5 seconds..."
+            fi
+            
+            sleep 5
+          done
+          echo "Finished waiting for service."
 
       - name: Start Mock Services
         run: |
@@ -382,10 +297,7 @@ jobs:
               -e VESPA_HOST=index \
               -e REDIS_HOST=cache \
               -e API_SERVER_HOST=api_server \
-              -e MCP_SERVER_HOST=mcp_server \
-              -e MCP_SERVER_PORT=8090 \
               -e OPENAI_API_KEY=${OPENAI_API_KEY} \
-              -e EXA_API_KEY=${EXA_API_KEY} \
               -e SLACK_BOT_TOKEN=${SLACK_BOT_TOKEN} \
               -e CONFLUENCE_TEST_SPACE_URL=${CONFLUENCE_TEST_SPACE_URL} \
               -e CONFLUENCE_USER_NAME=${CONFLUENCE_USER_NAME} \
@@ -402,7 +314,7 @@ jobs:
               -e TEST_WEB_HOSTNAME=test-runner \
               -e MOCK_CONNECTOR_SERVER_HOST=mock_connector_server \
               -e MOCK_CONNECTOR_SERVER_PORT=8001 \
-              ${{ env.RUNS_ON_ECR_CACHE }}:integration-test-${{ github.run_id }} \
+              onyxdotapp/onyx-integration:test \
               /app/tests/integration/${{ matrix.test-dir.path }}
 
       # ------------------------------------------------------------
@@ -421,12 +333,18 @@ jobs:
 
       - name: Upload logs
         if: always()
-        uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # ratchet:actions/upload-artifact@v4
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # ratchet:actions/upload-artifact@v4
         with:
           name: docker-all-logs-${{ matrix.test-dir.name }}
           path: ${{ github.workspace }}/docker-compose.log
       # ------------------------------------------------------------
 
+      - name: Stop Docker containers
+        if: always()
+        run: |
+          cd deployment/docker_compose
+          docker compose down -v
+
 
   multitenant-tests:
     needs:
@@ -435,15 +353,18 @@ jobs:
         build-model-server-image,
         build-integration-image,
       ]
-    runs-on: [runs-on, runner=8cpu-linux-arm64, "run-id=${{ github.run_id }}-multitenant-tests", "extras=ecr-cache"]
-    timeout-minutes: 45
+    runs-on: blacksmith-8vcpu-ubuntu-2404-arm
 
     steps:
-      - uses: runs-on/action@cd2b598b0515d39d78c38a02d529db87d2196d1e # ratchet:runs-on/action@v2
       - name: Checkout code
-        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # ratchet:actions/checkout@v6
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # ratchet:actions/checkout@v4
+
+      - name: Login to Private Registry
+        uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # ratchet:docker/login-action@v3
         with:
-          persist-credentials: false
+          registry: ${{ env.PRIVATE_REGISTRY }}
+          username: ${{ env.PRIVATE_REGISTRY_USERNAME }}
+          password: ${{ env.PRIVATE_REGISTRY_PASSWORD }}
 
       - name: Login to Docker Hub
         uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # ratchet:docker/login-action@v3
@@ -451,10 +372,17 @@ jobs:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_TOKEN }}
 
+      - name: Pull Docker images
+        run: |
+          (docker pull --platform linux/arm64 ${{ env.PRIVATE_REGISTRY }}/integration-test-onyx-backend:test-${{ github.run_id }}) &
+          (docker pull --platform linux/arm64 ${{ env.PRIVATE_REGISTRY }}/integration-test-onyx-model-server:test-${{ github.run_id }}) &
+          (docker pull --platform linux/arm64 ${{ env.PRIVATE_REGISTRY }}/integration-test-onyx-integration:test-${{ github.run_id }}) &
+          wait
+          docker tag ${{ env.PRIVATE_REGISTRY }}/integration-test-onyx-backend:test-${{ github.run_id }} onyxdotapp/onyx-backend:test
+          docker tag ${{ env.PRIVATE_REGISTRY }}/integration-test-onyx-model-server:test-${{ github.run_id }} onyxdotapp/onyx-model-server:test
+          docker tag ${{ env.PRIVATE_REGISTRY }}/integration-test-onyx-integration:test-${{ github.run_id }} onyxdotapp/onyx-integration:test
+
       - name: Start Docker containers for multi-tenant tests
-        env:
-          ECR_CACHE: ${{ env.RUNS_ON_ECR_CACHE }}
-          RUN_ID: ${{ github.run_id }}
         run: |
           cd deployment/docker_compose
           ENABLE_PAID_ENTERPRISE_EDITION_FEATURES=true \
@@ -462,10 +390,8 @@ jobs:
           AUTH_TYPE=cloud \
           REQUIRE_EMAIL_VERIFICATION=false \
           DISABLE_TELEMETRY=true \
-          ONYX_BACKEND_IMAGE=${ECR_CACHE}:integration-test-backend-test-${RUN_ID} \
-          ONYX_MODEL_SERVER_IMAGE=${ECR_CACHE}:integration-test-model-server-test-${RUN_ID} \
+          IMAGE_TAG=test \
           DEV_MODE=true \
-          MCP_SERVER_ENABLED=true \
           docker compose -f docker-compose.multitenant-dev.yml up \
             relational_db \
             index \
@@ -474,7 +400,6 @@ jobs:
             api_server \
             inference_model_server \
             indexing_model_server \
-            mcp_server \
             background \
             -d
         id: start_docker_multi_tenant
@@ -506,9 +431,6 @@ jobs:
           echo "Finished waiting for service."
 
       - name: Run Multi-Tenant Integration Tests
-        env:
-          ECR_CACHE: ${{ env.RUNS_ON_ECR_CACHE }}
-          RUN_ID: ${{ github.run_id }}
         run: |
           echo "Running multi-tenant integration tests..."
           docker run --rm --network onyx_default \
@@ -523,10 +445,7 @@ jobs:
             -e VESPA_HOST=index \
             -e REDIS_HOST=cache \
             -e API_SERVER_HOST=api_server \
-            -e MCP_SERVER_HOST=mcp_server \
-            -e MCP_SERVER_PORT=8090 \
             -e OPENAI_API_KEY=${OPENAI_API_KEY} \
-            -e EXA_API_KEY=${EXA_API_KEY} \
             -e SLACK_BOT_TOKEN=${SLACK_BOT_TOKEN} \
             -e TEST_WEB_HOSTNAME=test-runner \
             -e AUTH_TYPE=cloud \
@@ -534,8 +453,9 @@ jobs:
             -e SKIP_RESET=true \
             -e REQUIRE_EMAIL_VERIFICATION=false \
             -e DISABLE_TELEMETRY=true \
+            -e IMAGE_TAG=test \
             -e DEV_MODE=true \
-            ${ECR_CACHE}:integration-test-${RUN_ID} \
+            onyxdotapp/onyx-integration:test \
             /app/tests/integration/multitenant_tests
 
       - name: Dump API server logs (multi-tenant)
@@ -552,7 +472,7 @@ jobs:
 
       - name: Upload logs (multi-tenant)
         if: always()
-        uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # ratchet:actions/upload-artifact@v4
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # ratchet:actions/upload-artifact@v4
         with:
           name: docker-all-logs-multitenant
           path: ${{ github.workspace }}/docker-compose-multitenant.log
@@ -563,13 +483,18 @@ jobs:
           cd deployment/docker_compose
           docker compose -f docker-compose.multitenant-dev.yml down -v
 
-  required:
-    # NOTE: Github-hosted runners have about 20s faster queue times and are preferred here.
-    runs-on: ubuntu-slim
-    timeout-minutes: 45
+  required: 
+    runs-on: blacksmith-2vcpu-ubuntu-2404-arm
     needs: [integration-tests, multitenant-tests]
     if: ${{ always() }}
     steps:
-      - name: Check job status
-        if: ${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') || contains(needs.*.result, 'skipped') }}
-        run: exit 1
+      - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # ratchet:actions/github-script@v8
+        with:
+          script: |
+            const needs = ${{ toJSON(needs) }};
+            const failed = Object.values(needs).some(n => n.result !== 'success');
+            if (failed) {
+              core.setFailed('One or more upstream jobs failed or were cancelled.');
+            } else {
+              core.notice('All required jobs succeeded.');
+            }

.github/workflows/pr-jest-tests.yml

@@ -3,29 +3,20 @@ concurrency:
   group: Run-Jest-Tests-${{ github.workflow }}-${{ github.head_ref || github.event.workflow_run.head_branch || github.run_id }}
   cancel-in-progress: true
 
-on:
-  push:
-
-permissions:
-  contents: read
+on: push
 
 jobs:
   jest-tests:
     name: Jest Tests
     runs-on: ubuntu-latest
-    timeout-minutes: 45
     steps:
       - name: Checkout code
-        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # ratchet:actions/checkout@v6
-        with:
-          persist-credentials: false
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # ratchet:actions/checkout@v4
 
       - name: Setup node
-        uses: actions/setup-node@395ad3262231945c25e8478fd5baf05154b1d79f # ratchet:actions/setup-node@v4
+        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # ratchet:actions/setup-node@v4
         with:
           node-version: 22
-          cache: "npm"
-          cache-dependency-path: ./web/package-lock.json
 
       - name: Install node dependencies
         working-directory: ./web
@@ -37,7 +28,7 @@ jobs:
 
       - name: Upload coverage reports
         if: always()
-        uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # ratchet:actions/upload-artifact@v4
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # ratchet:actions/upload-artifact@v4
         with:
           name: jest-coverage-${{ github.run_id }}
           path: ./web/coverage

.github/workflows/pr-labeler.yml

@@ -1,7 +1,7 @@
 name: PR Labeler
 
 on:
-  pull_request:
+  pull_request_target:
     branches:
       - main
     types:
@@ -12,11 +12,11 @@ on:
 
 permissions:
   contents: read
+  pull-requests: write
 
 jobs:
   validate_pr_title:
     runs-on: ubuntu-latest
-    timeout-minutes: 45
     steps:
       - name: Check PR title for Conventional Commits
         env:
@@ -27,7 +27,7 @@ jobs:
             echo "::error::❌ Your PR title does not follow the Conventional Commits format.
               This check ensures that all pull requests use clear, consistent titles that help automate changelogs and improve project history.
 
-              Please update your PR title to follow the Conventional Commits style.
+              Please update your PR title to follow the Conventional Commits style.  
               Here is a link to a blog explaining the reason why we've included the Conventional Commits style into our PR titles: https://xfuture-blog.com/working-with-conventional-commits
 
               **Here are some examples of valid PR titles:**

.github/workflows/pr-linear-check.yml

@@ -1,19 +1,12 @@
 name: Ensure PR references Linear
-concurrency:
-  group: Ensure-PR-references-Linear-${{ github.workflow }}-${{ github.head_ref || github.event.workflow_run.head_branch || github.run_id }}
-  cancel-in-progress: true
 
 on:
   pull_request:
     types: [opened, edited, reopened, synchronize]
 
-permissions:
-  contents: read
-
 jobs:
   linear-check:
     runs-on: ubuntu-latest
-    timeout-minutes: 45
     steps:
       - name: Check PR body for Linear link or override
         env:

.github/workflows/pr-mit-integration-tests.yml

@@ -6,18 +6,16 @@ concurrency:
 on:
   merge_group:
     types: [checks_requested]
-  push:
-    tags:
-      - "v*.*.*"
-
-permissions:
-  contents: read
 
 env:
+  # Private Registry Configuration
+  PRIVATE_REGISTRY: experimental-registry.blacksmith.sh:5000
+  PRIVATE_REGISTRY_USERNAME: ${{ secrets.PRIVATE_REGISTRY_USERNAME }}
+  PRIVATE_REGISTRY_PASSWORD: ${{ secrets.PRIVATE_REGISTRY_PASSWORD }}
+
   # Test Environment Variables
   OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
   SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
-  EXA_API_KEY: ${{ secrets.EXA_API_KEY }}
   CONFLUENCE_TEST_SPACE_URL: ${{ vars.CONFLUENCE_TEST_SPACE_URL }}
   CONFLUENCE_USER_NAME: ${{ vars.CONFLUENCE_USER_NAME }}
   CONFLUENCE_ACCESS_TOKEN: ${{ secrets.CONFLUENCE_ACCESS_TOKEN }}
@@ -33,16 +31,12 @@ env:
 
 jobs:
   discover-test-dirs:
-    # NOTE: Github-hosted runners have about 20s faster queue times and are preferred here.
-    runs-on: ubuntu-slim
-    timeout-minutes: 45
+    runs-on: blacksmith-2vcpu-ubuntu-2404-arm
     outputs:
       test-dirs: ${{ steps.set-matrix.outputs.test-dirs }}
     steps:
       - name: Checkout code
-        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # ratchet:actions/checkout@v6
-        with:
-          persist-credentials: false
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # ratchet:actions/checkout@v4
 
       - name: Discover test directories
         id: set-matrix
@@ -64,171 +58,100 @@ jobs:
           all_dirs="[${all_dirs%,}]"
           echo "test-dirs=$all_dirs" >> $GITHUB_OUTPUT
 
-  build-backend-image:
-    runs-on: [runs-on, runner=1cpu-linux-arm64, "run-id=${{ github.run_id }}-build-backend-image", "extras=ecr-cache"]
-    timeout-minutes: 45
+  prepare-build:
+    runs-on: blacksmith-2vcpu-ubuntu-2404-arm
     steps:
-      - uses: runs-on/action@cd2b598b0515d39d78c38a02d529db87d2196d1e # ratchet:runs-on/action@v2
       - name: Checkout code
-        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # ratchet:actions/checkout@v6
-        with:
-          persist-credentials: false
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # ratchet:actions/checkout@v4
 
-      - name: Format branch name for cache
-        id: format-branch
-        env:
-          PR_NUMBER: ${{ github.event.pull_request.number }}
-          REF_NAME: ${{ github.ref_name }}
-        run: |
-          if [ -n "${PR_NUMBER}" ]; then
-            CACHE_SUFFIX="${PR_NUMBER}"
-          else
-            # shellcheck disable=SC2001
-            CACHE_SUFFIX=$(echo "${REF_NAME}" | sed 's/[^A-Za-z0-9._-]/-/g')
-          fi
-          echo "cache-suffix=${CACHE_SUFFIX}" >> $GITHUB_OUTPUT
+      - name: Prepare build
+        uses: ./.github/actions/prepare-build
 
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # ratchet:docker/setup-buildx-action@v3
+  build-backend-image:
+    runs-on: blacksmith-16vcpu-ubuntu-2404-arm
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # ratchet:actions/checkout@v4
 
-      # needed for pulling Vespa, Redis, Postgres, and Minio images
-      # otherwise, we hit the "Unauthenticated users" limit
-      # https://docs.docker.com/docker-hub/usage/
-      - name: Login to Docker Hub
+      - name: Login to Private Registry
         uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # ratchet:docker/login-action@v3
         with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_TOKEN }}
+          registry: ${{ env.PRIVATE_REGISTRY }}
+          username: ${{ env.PRIVATE_REGISTRY_USERNAME }}
+          password: ${{ env.PRIVATE_REGISTRY_PASSWORD }}
+
+      - name: Set up Docker Buildx
+        uses: useblacksmith/setup-docker-builder@affa10db466676f3dfb3e54caeb228ee0691510f # ratchet:useblacksmith/setup-docker-builder@v1
 
       - name: Build and push Backend Docker image
-        uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # ratchet:docker/build-push-action@v6
+        uses: useblacksmith/build-push-action@30c71162f16ea2c27c3e21523255d209b8b538c1 # ratchet:useblacksmith/build-push-action@v2
         with:
           context: ./backend
           file: ./backend/Dockerfile
+          platforms: linux/arm64
+          tags: ${{ env.PRIVATE_REGISTRY }}/integration-test-onyx-backend:test-${{ github.run_id }}
           push: true
-          tags: ${{ env.RUNS_ON_ECR_CACHE }}:integration-test-backend-test-${{ github.run_id }}
-          cache-from: |
-            type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:backend-cache-${{ github.event.pull_request.head.sha || github.sha }}
-            type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:backend-cache-${{ steps.format-branch.outputs.cache-suffix }}
-            type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:backend-cache
-            type=registry,ref=onyxdotapp/onyx-backend:latest
-          cache-to: |
-            type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:backend-cache-${{ github.event.pull_request.head.sha || github.sha }},mode=max
-            type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:backend-cache-${{ steps.format-branch.outputs.cache-suffix }},mode=max
-            type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:backend-cache,mode=max
-          no-cache: ${{ vars.DOCKER_NO_CACHE == 'true' }}
+          outputs: type=registry
+          no-cache: true
+
 
   build-model-server-image:
-    runs-on: [runs-on, runner=1cpu-linux-arm64, "run-id=${{ github.run_id }}-build-model-server-image", "extras=ecr-cache"]
-    timeout-minutes: 45
+    runs-on: blacksmith-16vcpu-ubuntu-2404-arm
     steps:
-      - uses: runs-on/action@cd2b598b0515d39d78c38a02d529db87d2196d1e # ratchet:runs-on/action@v2
       - name: Checkout code
-        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # ratchet:actions/checkout@v6
-        with:
-          persist-credentials: false
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # ratchet:actions/checkout@v4
 
-      - name: Format branch name for cache
-        id: format-branch
-        env:
-          PR_NUMBER: ${{ github.event.pull_request.number }}
-          REF_NAME: ${{ github.ref_name }}
-        run: |
-          if [ -n "${PR_NUMBER}" ]; then
-            CACHE_SUFFIX="${PR_NUMBER}"
-          else
-            # shellcheck disable=SC2001
-            CACHE_SUFFIX=$(echo "${REF_NAME}" | sed 's/[^A-Za-z0-9._-]/-/g')
-          fi
-          echo "cache-suffix=${CACHE_SUFFIX}" >> $GITHUB_OUTPUT
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # ratchet:docker/setup-buildx-action@v3
-
-      # needed for pulling Vespa, Redis, Postgres, and Minio images
-      # otherwise, we hit the "Unauthenticated users" limit
-      # https://docs.docker.com/docker-hub/usage/
-      - name: Login to Docker Hub
+      - name: Login to Private Registry
         uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # ratchet:docker/login-action@v3
         with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_TOKEN }}
+          registry: ${{ env.PRIVATE_REGISTRY }}
+          username: ${{ env.PRIVATE_REGISTRY_USERNAME }}
+          password: ${{ env.PRIVATE_REGISTRY_PASSWORD }}
+
+      - name: Set up Docker Buildx
+        uses: useblacksmith/setup-docker-builder@affa10db466676f3dfb3e54caeb228ee0691510f # ratchet:useblacksmith/setup-docker-builder@v1
 
       - name: Build and push Model Server Docker image
-        uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # ratchet:docker/build-push-action@v6
+        uses: useblacksmith/build-push-action@30c71162f16ea2c27c3e21523255d209b8b538c1 # ratchet:useblacksmith/build-push-action@v2
         with:
           context: ./backend
           file: ./backend/Dockerfile.model_server
+          platforms: linux/arm64
+          tags: ${{ env.PRIVATE_REGISTRY }}/integration-test-onyx-model-server:test-${{ github.run_id }}
           push: true
-          tags: ${{ env.RUNS_ON_ECR_CACHE }}:integration-test-model-server-test-${{ github.run_id }}
-          cache-from: |
-            type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:model-server-cache-${{ github.event.pull_request.head.sha || github.sha }}
-            type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:model-server-cache-${{ steps.format-branch.outputs.cache-suffix }}
-            type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:model-server-cache
-            type=registry,ref=onyxdotapp/onyx-model-server:latest
-          cache-to: |
-            type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:model-server-cache-${{ github.event.pull_request.head.sha || github.sha }},mode=max
-            type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:model-server-cache-${{ steps.format-branch.outputs.cache-suffix }},mode=max
-            type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:model-server-cache,mode=max
+          outputs: type=registry
+          provenance: false
+          no-cache: ${{ vars.DOCKER_NO_CACHE == 'true' }}
+
 
   build-integration-image:
-    runs-on: [runs-on, runner=2cpu-linux-arm64, "run-id=${{ github.run_id }}-build-integration-image", "extras=ecr-cache"]
-    timeout-minutes: 45
+    needs: prepare-build
+    runs-on: blacksmith-16vcpu-ubuntu-2404-arm
     steps:
-      - uses: runs-on/action@cd2b598b0515d39d78c38a02d529db87d2196d1e # ratchet:runs-on/action@v2
       - name: Checkout code
-        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # ratchet:actions/checkout@v6
-        with:
-          persist-credentials: false
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # ratchet:actions/checkout@v4
 
-      - name: Format branch name for cache
-        id: format-branch
-        env:
-          PR_NUMBER: ${{ github.event.pull_request.number }}
-          REF_NAME: ${{ github.ref_name }}
-        run: |
-          if [ -n "${PR_NUMBER}" ]; then
-            CACHE_SUFFIX="${PR_NUMBER}"
-          else
-            # shellcheck disable=SC2001
-            CACHE_SUFFIX=$(echo "${REF_NAME}" | sed 's/[^A-Za-z0-9._-]/-/g')
-          fi
-          echo "cache-suffix=${CACHE_SUFFIX}" >> $GITHUB_OUTPUT
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # ratchet:docker/setup-buildx-action@v3
-
-      # needed for pulling openapitools/openapi-generator-cli
-      # otherwise, we hit the "Unauthenticated users" limit
-      # https://docs.docker.com/docker-hub/usage/
-      - name: Login to Docker Hub
+      - name: Login to Private Registry
         uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # ratchet:docker/login-action@v3
         with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_TOKEN }}
+          registry: ${{ env.PRIVATE_REGISTRY }}
+          username: ${{ env.PRIVATE_REGISTRY_USERNAME }}
+          password: ${{ env.PRIVATE_REGISTRY_PASSWORD }}
+
+      - name: Download OpenAPI artifacts
+        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # ratchet:actions/download-artifact@v4
+        with:
+          name: openapi-artifacts
+          path: backend/generated/
+
+      - name: Set up Docker Buildx
+        uses: useblacksmith/setup-docker-builder@affa10db466676f3dfb3e54caeb228ee0691510f # ratchet:useblacksmith/setup-docker-builder@v1
 
       - name: Build and push integration test image with Docker Bake
         env:
-          INTEGRATION_REPOSITORY: ${{ env.RUNS_ON_ECR_CACHE }}
-          TAG: integration-test-${{ github.run_id }}
-          CACHE_SUFFIX: ${{ steps.format-branch.outputs.cache-suffix }}
-          HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
-        run: |
-          cd backend && docker buildx bake --push \
-            --set backend.cache-from=type=registry,ref=${RUNS_ON_ECR_CACHE}:backend-cache-${HEAD_SHA} \
-            --set backend.cache-from=type=registry,ref=${RUNS_ON_ECR_CACHE}:backend-cache-${CACHE_SUFFIX} \
-            --set backend.cache-from=type=registry,ref=${RUNS_ON_ECR_CACHE}:backend-cache \
-            --set backend.cache-from=type=registry,ref=onyxdotapp/onyx-backend:latest \
-            --set backend.cache-to=type=registry,ref=${RUNS_ON_ECR_CACHE}:backend-cache-${HEAD_SHA},mode=max \
-            --set backend.cache-to=type=registry,ref=${RUNS_ON_ECR_CACHE}:backend-cache-${CACHE_SUFFIX},mode=max \
-            --set backend.cache-to=type=registry,ref=${RUNS_ON_ECR_CACHE}:backend-cache,mode=max \
-            --set integration.cache-from=type=registry,ref=${RUNS_ON_ECR_CACHE}:integration-cache-${HEAD_SHA} \
-            --set integration.cache-from=type=registry,ref=${RUNS_ON_ECR_CACHE}:integration-cache-${CACHE_SUFFIX} \
-            --set integration.cache-from=type=registry,ref=${RUNS_ON_ECR_CACHE}:integration-cache \
-            --set integration.cache-to=type=registry,ref=${RUNS_ON_ECR_CACHE}:integration-cache-${HEAD_SHA},mode=max \
-            --set integration.cache-to=type=registry,ref=${RUNS_ON_ECR_CACHE}:integration-cache-${CACHE_SUFFIX},mode=max \
-            --set integration.cache-to=type=registry,ref=${RUNS_ON_ECR_CACHE}:integration-cache,mode=max \
-            integration
+          REGISTRY: ${{ env.PRIVATE_REGISTRY }}
+          TAG: test-${{ github.run_id }}
+        run: cd backend && docker buildx bake --no-cache --push integration
 
   integration-tests-mit:
     needs:
@@ -238,12 +161,8 @@ jobs:
         build-model-server-image,
         build-integration-image,
       ]
-    runs-on:
-      - runs-on
-      - runner=4cpu-linux-arm64
-      - ${{ format('run-id={0}-integration-tests-mit-job-{1}', github.run_id, strategy['job-index']) }}
-      - extras=ecr-cache
-    timeout-minutes: 45
+    # See https://docs.blacksmith.sh/blacksmith-runners/overview
+    runs-on: blacksmith-8vcpu-ubuntu-2404-arm
 
     strategy:
       fail-fast: false
@@ -251,11 +170,15 @@ jobs:
         test-dir: ${{ fromJson(needs.discover-test-dirs.outputs.test-dirs) }}
 
     steps:
-      - uses: runs-on/action@cd2b598b0515d39d78c38a02d529db87d2196d1e # ratchet:runs-on/action@v2
       - name: Checkout code
-        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # ratchet:actions/checkout@v6
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # ratchet:actions/checkout@v4
+
+      - name: Login to Private Registry
+        uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # ratchet:docker/login-action@v3
         with:
-          persist-credentials: false
+          registry: ${{ env.PRIVATE_REGISTRY }}
+          username: ${{ env.PRIVATE_REGISTRY_USERNAME }}
+          password: ${{ env.PRIVATE_REGISTRY_PASSWORD }}
 
       # needed for pulling Vespa, Redis, Postgres, and Minio images
       # otherwise, we hit the "Unauthenticated users" limit
@@ -266,28 +189,36 @@ jobs:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_TOKEN }}
 
+      - name: Pull Docker images
+        run: |
+          # Pull all images from registry in parallel
+          echo "Pulling Docker images in parallel..."
+          # Pull images from private registry
+          (docker pull --platform linux/arm64 ${{ env.PRIVATE_REGISTRY }}/integration-test-onyx-backend:test-${{ github.run_id }}) &
+          (docker pull --platform linux/arm64 ${{ env.PRIVATE_REGISTRY }}/integration-test-onyx-model-server:test-${{ github.run_id }}) &
+          (docker pull --platform linux/arm64 ${{ env.PRIVATE_REGISTRY }}/integration-test-onyx-integration:test-${{ github.run_id }}) &
+
+          # Wait for all background jobs to complete
+          wait
+          echo "All Docker images pulled successfully"
+
+          # Re-tag to remove registry prefix for docker-compose
+          docker tag ${{ env.PRIVATE_REGISTRY }}/integration-test-onyx-backend:test-${{ github.run_id }} onyxdotapp/onyx-backend:test
+          docker tag ${{ env.PRIVATE_REGISTRY }}/integration-test-onyx-model-server:test-${{ github.run_id }} onyxdotapp/onyx-model-server:test
+          docker tag ${{ env.PRIVATE_REGISTRY }}/integration-test-onyx-integration:test-${{ github.run_id }} onyxdotapp/onyx-integration:test
+
       # NOTE: Use pre-ping/null pool to reduce flakiness due to dropped connections
       # NOTE: don't need web server for integration tests
-      - name: Create .env file for Docker Compose
-        env:
-          ECR_CACHE: ${{ env.RUNS_ON_ECR_CACHE }}
-          RUN_ID: ${{ github.run_id }}
-        run: |
-          cat <<EOF > deployment/docker_compose/.env
-          AUTH_TYPE=basic
-          POSTGRES_POOL_PRE_PING=true
-          POSTGRES_USE_NULL_POOL=true
-          REQUIRE_EMAIL_VERIFICATION=false
-          DISABLE_TELEMETRY=true
-          ONYX_BACKEND_IMAGE=${ECR_CACHE}:integration-test-backend-test-${RUN_ID}
-          ONYX_MODEL_SERVER_IMAGE=${ECR_CACHE}:integration-test-model-server-test-${RUN_ID}
-          INTEGRATION_TESTS_MODE=true
-          MCP_SERVER_ENABLED=true
-          EOF
-
       - name: Start Docker containers
         run: |
           cd deployment/docker_compose
+          AUTH_TYPE=basic \
+          POSTGRES_POOL_PRE_PING=true \
+          POSTGRES_USE_NULL_POOL=true \
+          REQUIRE_EMAIL_VERIFICATION=false \
+          DISABLE_TELEMETRY=true \
+          IMAGE_TAG=test \
+          INTEGRATION_TESTS_MODE=true \
           docker compose -f docker-compose.yml -f docker-compose.dev.yml up \
             relational_db \
             index \
@@ -296,56 +227,43 @@ jobs:
             api_server \
             inference_model_server \
             indexing_model_server \
-            mcp_server \
             background \
             -d
         id: start_docker
 
-      - name: Wait for services to be ready
+      - name: Wait for service to be ready
         run: |
           echo "Starting wait-for-service script..."
 
-          wait_for_service() {
-            local url=$1
-            local label=$2
-            local timeout=${3:-300}  # default 5 minutes
-            local start_time
-            start_time=$(date +%s)
+          docker logs -f onyx-api_server-1 &
 
-            while true; do
-              local current_time
-              current_time=$(date +%s)
-              local elapsed_time=$((current_time - start_time))
+          start_time=$(date +%s)
+          timeout=300  # 5 minutes in seconds
 
-              if [ $elapsed_time -ge $timeout ]; then
-                echo "Timeout reached. ${label} did not become ready in $timeout seconds."
-                exit 1
-              fi
-
-              local response
-              response=$(curl -s -o /dev/null -w "%{http_code}" "$url" || echo "curl_error")
-
-              if [ "$response" = "200" ]; then
-                echo "${label} is ready!"
-                break
-              elif [ "$response" = "curl_error" ]; then
-                echo "Curl encountered an error while checking ${label}. Retrying in 5 seconds..."
-              else
-                echo "${label} not ready yet (HTTP status $response). Retrying in 5 seconds..."
-              fi
-
-              sleep 5
-            done
-          }
-
-          wait_for_service "http://localhost:8080/health" "API server"
-          test_dir="${{ matrix.test-dir.path }}"
-          if [ "$test_dir" = "tests/mcp" ]; then
-            wait_for_service "http://localhost:8090/health" "MCP server"
-          else
-            echo "Skipping MCP server wait for non-MCP suite: $test_dir"
-          fi
-          echo "Finished waiting for services."
+          while true; do
+            current_time=$(date +%s)
+            elapsed_time=$((current_time - start_time))
+            
+            if [ $elapsed_time -ge $timeout ]; then
+              echo "Timeout reached. Service did not become ready in 5 minutes."
+              exit 1
+            fi
+            
+            # Use curl with error handling to ignore specific exit code 56
+            response=$(curl -s -o /dev/null -w "%{http_code}" http://localhost:8080/health || echo "curl_error")
+            
+            if [ "$response" = "200" ]; then
+              echo "Service is ready!"
+              break
+            elif [ "$response" = "curl_error" ]; then
+              echo "Curl encountered an error, possibly exit code 56. Continuing to retry..."
+            else
+              echo "Service not ready yet (HTTP status $response). Retrying in 5 seconds..."
+            fi
+            
+            sleep 5
+          done
+          echo "Finished waiting for service."
 
       - name: Start Mock Services
         run: |
@@ -375,10 +293,7 @@ jobs:
               -e VESPA_HOST=index \
               -e REDIS_HOST=cache \
               -e API_SERVER_HOST=api_server \
-              -e MCP_SERVER_HOST=mcp_server \
-              -e MCP_SERVER_PORT=8090 \
               -e OPENAI_API_KEY=${OPENAI_API_KEY} \
-              -e EXA_API_KEY=${EXA_API_KEY} \
               -e SLACK_BOT_TOKEN=${SLACK_BOT_TOKEN} \
               -e CONFLUENCE_TEST_SPACE_URL=${CONFLUENCE_TEST_SPACE_URL} \
               -e CONFLUENCE_USER_NAME=${CONFLUENCE_USER_NAME} \
@@ -395,7 +310,7 @@ jobs:
               -e TEST_WEB_HOSTNAME=test-runner \
               -e MOCK_CONNECTOR_SERVER_HOST=mock_connector_server \
               -e MOCK_CONNECTOR_SERVER_PORT=8001 \
-              ${{ env.RUNS_ON_ECR_CACHE }}:integration-test-${{ github.run_id }} \
+              onyxdotapp/onyx-integration:test \
               /app/tests/integration/${{ matrix.test-dir.path }}
 
       # ------------------------------------------------------------
@@ -414,20 +329,31 @@ jobs:
 
       - name: Upload logs
         if: always()
-        uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # ratchet:actions/upload-artifact@v4
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # ratchet:actions/upload-artifact@v4
         with:
           name: docker-all-logs-${{ matrix.test-dir.name }}
           path: ${{ github.workspace }}/docker-compose.log
       # ------------------------------------------------------------
 
+      - name: Stop Docker containers
+        if: always()
+        run: |
+          cd deployment/docker_compose
+          docker compose down -v
 
-  required:
-    # NOTE: Github-hosted runners have about 20s faster queue times and are preferred here.
-    runs-on: ubuntu-slim
-    timeout-minutes: 45
+  
+  required: 
+    runs-on: blacksmith-2vcpu-ubuntu-2404-arm
     needs: [integration-tests-mit]
     if: ${{ always() }}
     steps:
-      - name: Check job status
-        if: ${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') || contains(needs.*.result, 'skipped') }}
-        run: exit 1
+      - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # ratchet:actions/github-script@v8
+        with:
+          script: |
+            const needs = ${{ toJSON(needs) }};
+            const failed = Object.values(needs).some(n => n.result !== 'success');
+            if (failed) {
+              core.setFailed('One or more upstream jobs failed or were cancelled.');
+            } else {
+              core.notice('All required jobs succeeded.');
+            }

.github/workflows/pr-playwright-tests.yml

@@ -3,13 +3,16 @@ concurrency:
   group: Run-Playwright-Tests-${{ github.workflow }}-${{ github.head_ref || github.event.workflow_run.head_branch || github.run_id }}
   cancel-in-progress: true
 
-on:
-  push:
-
-permissions:
-  contents: read
+on: push
 
 env:
+  # AWS ECR Configuration
+  AWS_REGION: ${{ secrets.AWS_REGION || 'us-west-2' }}
+  ECR_REGISTRY: ${{ secrets.ECR_REGISTRY }}
+  AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID_ECR }}
+  AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY_ECR }}
+  BUILDX_NO_DEFAULT_ATTESTATIONS: 1
+
   # Test Environment Variables
   OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
   SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
@@ -20,264 +23,128 @@ env:
   SLACK_CLIENT_ID: ${{ secrets.SLACK_CLIENT_ID }}
   SLACK_CLIENT_SECRET: ${{ secrets.SLACK_CLIENT_SECRET }}
 
-  # for MCP Oauth tests
-  MCP_OAUTH_CLIENT_ID: ${{ secrets.MCP_OAUTH_CLIENT_ID }}
-  MCP_OAUTH_CLIENT_SECRET: ${{ secrets.MCP_OAUTH_CLIENT_SECRET }}
-  MCP_OAUTH_ISSUER: ${{ secrets.MCP_OAUTH_ISSUER }}
-  MCP_OAUTH_JWKS_URI: ${{ secrets.MCP_OAUTH_JWKS_URI }}
-  MCP_OAUTH_USERNAME: ${{ vars.MCP_OAUTH_USERNAME }}
-  MCP_OAUTH_PASSWORD: ${{ secrets.MCP_OAUTH_PASSWORD }}
-
-  # for MCP API Key tests
-  MCP_API_KEY: test-api-key-12345
-  MCP_API_KEY_TEST_PORT: 8005
-  MCP_API_KEY_TEST_URL: http://host.docker.internal:8005/mcp
-  MCP_API_KEY_SERVER_HOST: 0.0.0.0
-  MCP_API_KEY_SERVER_PUBLIC_HOST: host.docker.internal
-
   MOCK_LLM_RESPONSE: true
-  MCP_TEST_SERVER_PORT: 8004
-  MCP_TEST_SERVER_URL: http://host.docker.internal:8004/mcp
-  MCP_TEST_SERVER_PUBLIC_URL: http://host.docker.internal:8004/mcp
-  MCP_TEST_SERVER_BIND_HOST: 0.0.0.0
-  MCP_TEST_SERVER_PUBLIC_HOST: host.docker.internal
-  MCP_SERVER_HOST: 0.0.0.0
-  MCP_SERVER_PUBLIC_HOST: host.docker.internal
-  MCP_SERVER_PUBLIC_URL: http://host.docker.internal:8004/mcp
 
 jobs:
   build-web-image:
-    runs-on: [runs-on, runner=4cpu-linux-arm64, "run-id=${{ github.run_id }}-build-web-image", "extras=ecr-cache"]
-    timeout-minutes: 45
+    runs-on: blacksmith-8vcpu-ubuntu-2404-arm
     steps:
-      - uses: runs-on/action@cd2b598b0515d39d78c38a02d529db87d2196d1e # ratchet:runs-on/action@v2
-
       - name: Checkout code
-        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # ratchet:actions/checkout@v6
-        with:
-          persist-credentials: false
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # ratchet:actions/checkout@v4
 
-      - name: Format branch name for cache
-        id: format-branch
-        env:
-          PR_NUMBER: ${{ github.event.pull_request.number }}
-          REF_NAME: ${{ github.ref_name }}
-        run: |
-          if [ -n "${PR_NUMBER}" ]; then
-            CACHE_SUFFIX="${PR_NUMBER}"
-          else
-            # shellcheck disable=SC2001
-            CACHE_SUFFIX=$(echo "${REF_NAME}" | sed 's/[^A-Za-z0-9._-]/-/g')
-          fi
-          echo "cache-suffix=${CACHE_SUFFIX}" >> $GITHUB_OUTPUT
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@00943011d9042930efac3dcd3a170e4273319bc8 # ratchet:aws-actions/configure-aws-credentials@v4
+        with:
+          aws-access-key-id: ${{ env.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ env.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ env.AWS_REGION }}
+
+      - name: Login to Amazon ECR
+        id: login-ecr
+        uses: aws-actions/amazon-ecr-login@062b18b96a7aff071d4dc91bc00c4c1a7945b076 # ratchet:aws-actions/amazon-ecr-login@v2
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # ratchet:docker/setup-buildx-action@v3
-
-      # needed for pulling external images otherwise, we hit the "Unauthenticated users" limit
-      # https://docs.docker.com/docker-hub/usage/
-      - name: Login to Docker Hub
-        uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # ratchet:docker/login-action@v3
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_TOKEN }}
+        uses: useblacksmith/setup-docker-builder@affa10db466676f3dfb3e54caeb228ee0691510f # ratchet:useblacksmith/setup-docker-builder@v1
 
       - name: Build and push Web Docker image
-        uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # ratchet:docker/build-push-action@v6
+        uses: useblacksmith/build-push-action@30c71162f16ea2c27c3e21523255d209b8b538c1 # ratchet:useblacksmith/build-push-action@v2
         with:
           context: ./web
           file: ./web/Dockerfile
           platforms: linux/arm64
-          tags: ${{ env.RUNS_ON_ECR_CACHE }}:playwright-test-web-${{ github.run_id }}
+          tags: ${{ env.ECR_REGISTRY }}/integration-test-onyx-web-server:playwright-test-${{ github.run_id }}
+          provenance: false
+          sbom: false
           push: true
-          cache-from: |
-            type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:web-cache-${{ github.event.pull_request.head.sha || github.sha }}
-            type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:web-cache-${{ steps.format-branch.outputs.cache-suffix }}
-            type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:web-cache
-            type=registry,ref=onyxdotapp/onyx-web-server:latest
-          cache-to: |
-            type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:web-cache-${{ github.event.pull_request.head.sha || github.sha }},mode=max
-            type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:web-cache-${{ steps.format-branch.outputs.cache-suffix }},mode=max
-            type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:web-cache,mode=max
+          outputs: type=registry
           no-cache: ${{ vars.DOCKER_NO_CACHE == 'true' }}
 
   build-backend-image:
-    runs-on: [runs-on, runner=1cpu-linux-arm64, "run-id=${{ github.run_id }}-build-backend-image", "extras=ecr-cache"]
-    timeout-minutes: 45
+    runs-on: blacksmith-8vcpu-ubuntu-2404-arm
     steps:
-      - uses: runs-on/action@cd2b598b0515d39d78c38a02d529db87d2196d1e # ratchet:runs-on/action@v2
-
       - name: Checkout code
-        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # ratchet:actions/checkout@v6
-        with:
-          persist-credentials: false
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # ratchet:actions/checkout@v4
 
-      - name: Format branch name for cache
-        id: format-branch
-        env:
-          PR_NUMBER: ${{ github.event.pull_request.number }}
-          REF_NAME: ${{ github.ref_name }}
-        run: |
-          if [ -n "${PR_NUMBER}" ]; then
-            CACHE_SUFFIX="${PR_NUMBER}"
-          else
-            # shellcheck disable=SC2001
-            CACHE_SUFFIX=$(echo "${REF_NAME}" | sed 's/[^A-Za-z0-9._-]/-/g')
-          fi
-          echo "cache-suffix=${CACHE_SUFFIX}" >> $GITHUB_OUTPUT
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@00943011d9042930efac3dcd3a170e4273319bc8 # ratchet:aws-actions/configure-aws-credentials@v4
+        with:
+          aws-access-key-id: ${{ env.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ env.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ env.AWS_REGION }}
+
+      - name: Login to Amazon ECR
+        id: login-ecr
+        uses: aws-actions/amazon-ecr-login@062b18b96a7aff071d4dc91bc00c4c1a7945b076 # ratchet:aws-actions/amazon-ecr-login@v2
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # ratchet:docker/setup-buildx-action@v3
-
-      # needed for pulling external images otherwise, we hit the "Unauthenticated users" limit
-      # https://docs.docker.com/docker-hub/usage/
-      - name: Login to Docker Hub
-        uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # ratchet:docker/login-action@v3
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_TOKEN }}
+        uses: useblacksmith/setup-docker-builder@affa10db466676f3dfb3e54caeb228ee0691510f # ratchet:useblacksmith/setup-docker-builder@v1
 
       - name: Build and push Backend Docker image
-        uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # ratchet:docker/build-push-action@v6
+        uses: useblacksmith/build-push-action@30c71162f16ea2c27c3e21523255d209b8b538c1 # ratchet:useblacksmith/build-push-action@v2
         with:
           context: ./backend
           file: ./backend/Dockerfile
           platforms: linux/arm64
-          tags: ${{ env.RUNS_ON_ECR_CACHE }}:playwright-test-backend-${{ github.run_id }}
+          tags: ${{ env.ECR_REGISTRY }}/integration-test-onyx-backend:playwright-test-${{ github.run_id }}
+          provenance: false
+          sbom: false
           push: true
-          cache-from: |
-            type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:backend-cache-${{ github.event.pull_request.head.sha || github.sha }}
-            type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:backend-cache-${{ steps.format-branch.outputs.cache-suffix }}
-            type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:backend-cache
-            type=registry,ref=onyxdotapp/onyx-backend:latest
-          cache-to: |
-            type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:backend-cache-${{ github.event.pull_request.head.sha || github.sha }},mode=max
-            type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:backend-cache-${{ steps.format-branch.outputs.cache-suffix }},mode=max
-            type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:backend-cache,mode=max
+          outputs: type=registry
           no-cache: ${{ vars.DOCKER_NO_CACHE == 'true' }}
 
   build-model-server-image:
-    runs-on: [runs-on, runner=1cpu-linux-arm64, "run-id=${{ github.run_id }}-build-model-server-image", "extras=ecr-cache"]
-    timeout-minutes: 45
+    runs-on: blacksmith-8vcpu-ubuntu-2404-arm
     steps:
-      - uses: runs-on/action@cd2b598b0515d39d78c38a02d529db87d2196d1e # ratchet:runs-on/action@v2
-
       - name: Checkout code
-        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # ratchet:actions/checkout@v6
-        with:
-          persist-credentials: false
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # ratchet:actions/checkout@v4
 
-      - name: Format branch name for cache
-        id: format-branch
-        env:
-          PR_NUMBER: ${{ github.event.pull_request.number }}
-          REF_NAME: ${{ github.ref_name }}
-        run: |
-          if [ -n "${PR_NUMBER}" ]; then
-            CACHE_SUFFIX="${PR_NUMBER}"
-          else
-            # shellcheck disable=SC2001
-            CACHE_SUFFIX=$(echo "${REF_NAME}" | sed 's/[^A-Za-z0-9._-]/-/g')
-          fi
-          echo "cache-suffix=${CACHE_SUFFIX}" >> $GITHUB_OUTPUT
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@00943011d9042930efac3dcd3a170e4273319bc8 # ratchet:aws-actions/configure-aws-credentials@v4
+        with:
+          aws-access-key-id: ${{ env.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ env.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ env.AWS_REGION }}
+
+      - name: Login to Amazon ECR
+        id: login-ecr
+        uses: aws-actions/amazon-ecr-login@062b18b96a7aff071d4dc91bc00c4c1a7945b076 # ratchet:aws-actions/amazon-ecr-login@v2
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # ratchet:docker/setup-buildx-action@v3
-
-      # needed for pulling external images otherwise, we hit the "Unauthenticated users" limit
-      # https://docs.docker.com/docker-hub/usage/
-      - name: Login to Docker Hub
-        uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # ratchet:docker/login-action@v3
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_TOKEN }}
+        uses: useblacksmith/setup-docker-builder@affa10db466676f3dfb3e54caeb228ee0691510f # ratchet:useblacksmith/setup-docker-builder@v1
 
       - name: Build and push Model Server Docker image
-        uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # ratchet:docker/build-push-action@v6
+        uses: useblacksmith/build-push-action@30c71162f16ea2c27c3e21523255d209b8b538c1 # ratchet:useblacksmith/build-push-action@v2
         with:
           context: ./backend
           file: ./backend/Dockerfile.model_server
           platforms: linux/arm64
-          tags: ${{ env.RUNS_ON_ECR_CACHE }}:playwright-test-model-server-${{ github.run_id }}
+          tags: ${{ env.ECR_REGISTRY }}/integration-test-onyx-model-server:playwright-test-${{ github.run_id }}
+          provenance: false
+          sbom: false
           push: true
-          cache-from: |
-            type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:model-server-cache-${{ github.event.pull_request.head.sha || github.sha }}
-            type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:model-server-cache-${{ steps.format-branch.outputs.cache-suffix }}
-            type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:model-server-cache
-            type=registry,ref=onyxdotapp/onyx-model-server:latest
-          cache-to: |
-            type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:model-server-cache-${{ github.event.pull_request.head.sha || github.sha }},mode=max
-            type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:model-server-cache-${{ steps.format-branch.outputs.cache-suffix }},mode=max
-            type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:model-server-cache,mode=max
+          outputs: type=registry
           no-cache: ${{ vars.DOCKER_NO_CACHE == 'true' }}
 
   playwright-tests:
     needs: [build-web-image, build-backend-image, build-model-server-image]
-    name: Playwright Tests (${{ matrix.project }})
-    runs-on:
-      - runs-on
-      - runner=8cpu-linux-arm64
-      - "run-id=${{ github.run_id }}-playwright-tests-${{ matrix.project }}"
-      - "extras=ecr-cache"
-      - volume=50gb
-    timeout-minutes: 45
-    strategy:
-      fail-fast: false
-      matrix:
-        project: [admin, no-auth, exclusive]
+    name: Playwright Tests
+    runs-on: blacksmith-8vcpu-ubuntu-2404-arm
     steps:
-      - uses: runs-on/action@cd2b598b0515d39d78c38a02d529db87d2196d1e # ratchet:runs-on/action@v2
-
       - name: Checkout code
-        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # ratchet:actions/checkout@v6
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # ratchet:actions/checkout@v4
         with:
           fetch-depth: 0
-          persist-credentials: false
 
-      - name: Setup node
-        uses: actions/setup-node@395ad3262231945c25e8478fd5baf05154b1d79f # ratchet:actions/setup-node@v4
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@00943011d9042930efac3dcd3a170e4273319bc8 # ratchet:aws-actions/configure-aws-credentials@v4
         with:
-          node-version: 22
-          cache: 'npm'
-          cache-dependency-path: ./web/package-lock.json
+          aws-access-key-id: ${{ env.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ env.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ env.AWS_REGION }}
 
-      - name: Install node dependencies
-        working-directory: ./web
-        run: npm ci
-
-      - name: Cache playwright cache
-        uses: runs-on/cache@50350ad4242587b6c8c2baa2e740b1bc11285ff4 # ratchet:runs-on/cache@v4
-        with:
-          path: ~/.cache/ms-playwright
-          key: ${{ runner.os }}-playwright-npm-${{ hashFiles('web/package-lock.json') }}
-          restore-keys: |
-            ${{ runner.os }}-playwright-npm-
-
-      - name: Install playwright browsers
-        working-directory: ./web
-        run: npx playwright install --with-deps
-
-      - name: Create .env file for Docker Compose
-        env:
-          OPENAI_API_KEY_VALUE: ${{ env.OPENAI_API_KEY }}
-          EXA_API_KEY_VALUE: ${{ env.EXA_API_KEY }}
-          ECR_CACHE: ${{ env.RUNS_ON_ECR_CACHE }}
-          RUN_ID: ${{ github.run_id }}
-        run: |
-          cat <<EOF > deployment/docker_compose/.env
-          ENABLE_PAID_ENTERPRISE_EDITION_FEATURES=true
-          AUTH_TYPE=basic
-          GEN_AI_API_KEY=${OPENAI_API_KEY_VALUE}
-          EXA_API_KEY=${EXA_API_KEY_VALUE}
-          REQUIRE_EMAIL_VERIFICATION=false
-          DISABLE_TELEMETRY=true
-          ONYX_BACKEND_IMAGE=${ECR_CACHE}:playwright-test-backend-${RUN_ID}
-          ONYX_MODEL_SERVER_IMAGE=${ECR_CACHE}:playwright-test-model-server-${RUN_ID}
-          ONYX_WEB_SERVER_IMAGE=${ECR_CACHE}:playwright-test-web-${RUN_ID}
-          EOF
-          if [ "${{ matrix.project }}" = "no-auth" ]; then
-            echo "PLAYWRIGHT_FORCE_EMPTY_LLM_PROVIDERS=true" >> deployment/docker_compose/.env
-          fi
+      - name: Login to Amazon ECR
+        id: login-ecr
+        uses: aws-actions/amazon-ecr-login@062b18b96a7aff071d4dc91bc00c4c1a7945b076 # ratchet:aws-actions/amazon-ecr-login@v2
 
       # needed for pulling Vespa, Redis, Postgres, and Minio images
       # otherwise, we hit the "Unauthenticated users" limit
@@ -288,10 +155,52 @@ jobs:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_TOKEN }}
 
+      - name: Pull Docker images
+        run: |
+          # Pull all images from ECR in parallel
+          echo "Pulling Docker images in parallel..."
+          (docker pull ${{ env.ECR_REGISTRY }}/integration-test-onyx-web-server:playwright-test-${{ github.run_id }}) &
+          (docker pull ${{ env.ECR_REGISTRY }}/integration-test-onyx-backend:playwright-test-${{ github.run_id }}) &
+          (docker pull ${{ env.ECR_REGISTRY }}/integration-test-onyx-model-server:playwright-test-${{ github.run_id }}) &
+
+          # Wait for all background jobs to complete
+          wait
+          echo "All Docker images pulled successfully"
+
+          # Re-tag with expected names for docker-compose
+          docker tag ${{ env.ECR_REGISTRY }}/integration-test-onyx-web-server:playwright-test-${{ github.run_id }} onyxdotapp/onyx-web-server:test
+          docker tag ${{ env.ECR_REGISTRY }}/integration-test-onyx-backend:playwright-test-${{ github.run_id }} onyxdotapp/onyx-backend:test
+          docker tag ${{ env.ECR_REGISTRY }}/integration-test-onyx-model-server:playwright-test-${{ github.run_id }} onyxdotapp/onyx-model-server:test
+
+      - name: Setup node
+        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # ratchet:actions/setup-node@v4
+        with:
+          node-version: 22
+
+      - name: Install node dependencies
+        working-directory: ./web
+        run: npm ci
+
+      - name: Install playwright browsers
+        working-directory: ./web
+        run: npx playwright install --with-deps
+
+      - name: Create .env file for Docker Compose
+        run: |
+          cat <<EOF > deployment/docker_compose/.env
+          ENABLE_PAID_ENTERPRISE_EDITION_FEATURES=true
+          AUTH_TYPE=basic
+          GEN_AI_API_KEY=${{ env.OPENAI_API_KEY }}
+          EXA_API_KEY=${{ env.EXA_API_KEY }}
+          REQUIRE_EMAIL_VERIFICATION=false
+          DISABLE_TELEMETRY=true
+          IMAGE_TAG=test
+          EOF
+
       - name: Start Docker containers
         run: |
           cd deployment/docker_compose
-          docker compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.mcp-oauth-test.yml -f docker-compose.mcp-api-key-test.yml up -d
+          docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d
         id: start_docker
 
       - name: Wait for service to be ready
@@ -328,125 +237,40 @@ jobs:
           done
           echo "Finished waiting for service."
 
-      - name: Wait for MCP OAuth mock server
-        run: |
-          echo "Waiting for MCP OAuth mock server on port ${MCP_TEST_SERVER_PORT:-8004}..."
-          start_time=$(date +%s)
-          timeout=120
-
-          while true; do
-            current_time=$(date +%s)
-            elapsed_time=$((current_time - start_time))
-
-            if [ $elapsed_time -ge $timeout ]; then
-              echo "Timeout reached. MCP OAuth mock server did not become ready in ${timeout}s."
-              exit 1
-            fi
-
-            if curl -sf "http://localhost:${MCP_TEST_SERVER_PORT:-8004}/healthz" > /dev/null; then
-              echo "MCP OAuth mock server is ready!"
-              break
-            fi
-
-            sleep 3
-          done
-
-      - name: Wait for MCP API Key mock server
-        run: |
-          echo "Waiting for MCP API Key mock server on port ${MCP_API_KEY_TEST_PORT:-8005}..."
-          start_time=$(date +%s)
-          timeout=120
-
-          while true; do
-            current_time=$(date +%s)
-            elapsed_time=$((current_time - start_time))
-
-            if [ $elapsed_time -ge $timeout ]; then
-              echo "Timeout reached. MCP API Key mock server did not become ready in ${timeout}s."
-              exit 1
-            fi
-
-            if curl -sf "http://localhost:${MCP_API_KEY_TEST_PORT:-8005}/healthz" > /dev/null; then
-              echo "MCP API Key mock server is ready!"
-              break
-            fi
-
-            sleep 3
-          done
-
-      - name: Wait for web server to be ready
-        run: |
-          echo "Waiting for web server on port 3000..."
-          start_time=$(date +%s)
-          timeout=120
-
-          while true; do
-            current_time=$(date +%s)
-            elapsed_time=$((current_time - start_time))
-
-            if [ $elapsed_time -ge $timeout ]; then
-              echo "Timeout reached. Web server did not become ready in ${timeout}s."
-              exit 1
-            fi
-
-            if curl -sf "http://localhost:3000/api/health" > /dev/null 2>&1 || \
-               curl -sf "http://localhost:3000/" > /dev/null 2>&1; then
-              echo "Web server is ready!"
-              break
-            fi
-
-            echo "Web server not ready yet. Retrying in 3 seconds..."
-            sleep 3
-          done
-
       - name: Run Playwright tests
         working-directory: ./web
-        env:
-          PROJECT: ${{ matrix.project }}
         run: |
           # Create test-results directory to ensure it exists for artifact upload
           mkdir -p test-results
-          if [ "${PROJECT}" = "no-auth" ]; then
-            export PLAYWRIGHT_FORCE_EMPTY_LLM_PROVIDERS=true
-          fi
-          npx playwright test --project ${PROJECT}
+          npx playwright test
 
-      - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # ratchet:actions/upload-artifact@v4
+      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # ratchet:actions/upload-artifact@v4
         if: always()
         with:
           # Includes test results and trace.zip files
-          name: playwright-test-results-${{ matrix.project }}-${{ github.run_id }}
+          name: playwright-test-results-${{ github.run_id }}
           path: ./web/test-results/
           retention-days: 30
 
       # save before stopping the containers so the logs can be captured
       - name: Save Docker logs
         if: success() || failure()
-        env:
-          WORKSPACE: ${{ github.workspace }}
         run: |
           cd deployment/docker_compose
           docker compose logs > docker-compose.log
-          mv docker-compose.log ${WORKSPACE}/docker-compose.log
+          mv docker-compose.log ${{ github.workspace }}/docker-compose.log
 
       - name: Upload logs
         if: success() || failure()
-        uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # ratchet:actions/upload-artifact@v4
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # ratchet:actions/upload-artifact@v4
         with:
-          name: docker-logs-${{ matrix.project }}-${{ github.run_id }}
+          name: docker-logs
           path: ${{ github.workspace }}/docker-compose.log
 
-  playwright-required:
-    # NOTE: Github-hosted runners have about 20s faster queue times and are preferred here.
-    runs-on: ubuntu-slim
-    timeout-minutes: 45
-    needs: [playwright-tests]
-    if: ${{ always() }}
-    steps:
-      - name: Check job status
-        if: ${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') || contains(needs.*.result, 'skipped') }}
-        run: exit 1
-
+      - name: Stop Docker containers
+        run: |
+          cd deployment/docker_compose
+          docker compose down -v
 
 # NOTE: Chromatic UI diff testing is currently disabled.
 # We are using Playwright for local and CI testing without visual regression checks.
@@ -465,12 +289,12 @@ jobs:
 #     ]
 #   steps:
 #     - name: Checkout code
-#       uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # ratchet:actions/checkout@v6
+#       uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # ratchet:actions/checkout@v4
 #       with:
 #         fetch-depth: 0
 
 #     - name: Setup node
-#       uses: actions/setup-node@395ad3262231945c25e8478fd5baf05154b1d79f # ratchet:actions/setup-node@v4
+#       uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # ratchet:actions/setup-node@v4
 #       with:
 #         node-version: 22
 

.github/workflows/pr-python-checks.yml

@@ -1,7 +1,4 @@
 name: Python Checks
-concurrency:
-  group: Python-Checks-${{ github.workflow }}-${{ github.head_ref || github.event.workflow_run.head_branch || github.run_id }}
-  cancel-in-progress: true
 
 on:
   merge_group:
@@ -9,41 +6,68 @@ on:
     branches:
       - main
       - 'release/**'
-  push:
-    tags:
-      - "v*.*.*"
-
-permissions:
-  contents: read
 
 jobs:
   mypy-check:
     # See https://runs-on.com/runners/linux/
     # Note: Mypy seems quite optimized for x64 compared to arm64.
     # Similarly, mypy is single-threaded and incremental, so 2cpu is sufficient.
-    runs-on: [runs-on, runner=2cpu-linux-x64, "run-id=${{ github.run_id }}-mypy-check", "extras=s3-cache"]
-    timeout-minutes: 45
+    runs-on: [runs-on, runner=2cpu-linux-x64, "run-id=${{ github.run_id }}", "extras=s3-cache"]
 
     steps:
       - uses: runs-on/action@cd2b598b0515d39d78c38a02d529db87d2196d1e # ratchet:runs-on/action@v2
       - name: Checkout code
-        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # ratchet:actions/checkout@v6
-        with:
-          persist-credentials: false
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # ratchet:actions/checkout@v4
 
-      - name: Setup Python and Install Dependencies
-        uses: ./.github/actions/setup-python-and-install-dependencies
+      - name: Setup uv
+        uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # ratchet:astral-sh/setup-uv@v6
         with:
-          requirements: |
-            backend/requirements/default.txt
-            backend/requirements/dev.txt
-            backend/requirements/model_server.txt
-            backend/requirements/ee.txt
+          enable-cache: true
 
-      - name: Generate OpenAPI schema and Python client
-        shell: bash
+      - name: Cache uv cache directory
+        uses: runs-on/cache@50350ad4242587b6c8c2baa2e740b1bc11285ff4 # ratchet:runs-on/cache@v4
+        with:
+          path: ~/.cache/uv
+          key: ${{ runner.os }}-uv-${{ hashFiles('backend/requirements/*.txt', 'backend/pyproject.toml') }}
+          restore-keys: |
+            ${{ runner.os }}-uv-
+
+      - name: Setup Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # ratchet:actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Create virtual environment
+        working-directory: ./backend
+        run: uv venv .venv
+
+      - name: Install Python dependencies with uv
+        working-directory: ./backend
         run: |
-          ods openapi all
+          uv pip install \
+            -r requirements/default.txt \
+            -r requirements/dev.txt \
+            -r requirements/model_server.txt
+
+      - name: Generate OpenAPI schema
+        working-directory: ./backend
+        env:
+          PYTHONPATH: "."
+        run: |
+          uv run python scripts/onyx_openapi_schema.py --filename generated/openapi.json
+
+      - name: Generate OpenAPI Python client
+        working-directory: ./backend
+        run: |
+          docker run --rm \
+            -v "${{ github.workspace }}/backend/generated:/local" \
+            openapitools/openapi-generator-cli generate \
+            -i /local/openapi.json \
+            -g python \
+            -o /local/onyx_openapi_client \
+            --package-name onyx_openapi_client \
+            --skip-validate-spec \
+            --openapi-normalizer "SIMPLIFY_ONEOF_ANYOF=true,SET_OAS3_NULLABLE=true"
 
       - name: Cache mypy cache
         if: ${{ vars.DISABLE_MYPY_CACHE != 'true' }}
@@ -56,13 +80,13 @@ jobs:
 
       - name: Run MyPy
         working-directory: ./backend
-        env:
-          MYPY_FORCE_COLOR: 1
-          TERM: xterm-256color
-        run: mypy .
+        run: uv run mypy .
 
-      - name: Run MyPy (tools/)
-        env:
-          MYPY_FORCE_COLOR: 1
-          TERM: xterm-256color
-        run: mypy tools/
+      - name: Check import order with reorder-python-imports
+        working-directory: ./backend
+        run: |
+          find ./onyx -name "*.py" | xargs uv run reorder-python-imports --py311-plus
+
+      - name: Check code formatting with Black
+        working-directory: ./backend
+        run: uv run black --check .

.github/workflows/pr-python-connector-tests.yml

@@ -1,22 +1,13 @@
 name: Connector Tests
-concurrency:
-  group: Connector-Tests-${{ github.workflow }}-${{ github.head_ref || github.event.workflow_run.head_branch || github.run_id }}
-  cancel-in-progress: true
 
 on:
   merge_group:
   pull_request:
     branches: [main]
-  push:
-    tags:
-      - "v*.*.*"
   schedule:
     # This cron expression runs the job daily at 16:00 UTC (9am PT)
     - cron: "0 16 * * *"
 
-permissions:
-  contents: read
-
 env:
   # AWS
   AWS_ACCESS_KEY_ID_DAILY_CONNECTOR_TESTS: ${{ secrets.AWS_ACCESS_KEY_ID_DAILY_CONNECTOR_TESTS }}
@@ -128,30 +119,42 @@ env:
 jobs:
   connectors-check:
     # See https://runs-on.com/runners/linux/
-    runs-on: [runs-on, runner=8cpu-linux-x64, "run-id=${{ github.run_id }}-connectors-check", "extras=s3-cache"]
-    timeout-minutes: 45
+    runs-on: [runs-on, runner=8cpu-linux-x64, "run-id=${{ github.run_id }}", "extras=s3-cache"]
 
     env:
       PYTHONPATH: ./backend
-      DISABLE_TELEMETRY: "true"
 
     steps:
       - uses: runs-on/action@cd2b598b0515d39d78c38a02d529db87d2196d1e # ratchet:runs-on/action@v2
 
       - name: Checkout code
-        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # ratchet:actions/checkout@v6
-        with:
-          persist-credentials: false
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # ratchet:actions/checkout@v4
 
-      - name: Setup Python and Install Dependencies
-        uses: ./.github/actions/setup-python-and-install-dependencies
+      - name: Setup uv
+        uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # ratchet:astral-sh/setup-uv@v6
         with:
-          requirements: |
-            backend/requirements/default.txt
-            backend/requirements/dev.txt
+          enable-cache: true
 
-      - name: Setup Playwright
-        uses: ./.github/actions/setup-playwright
+      - name: Cache uv cache directory
+        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # ratchet:actions/cache@v4
+        with:
+          path: ~/.cache/uv
+          key: ${{ runner.os }}-${{ github.workflow }}-uv-${{ hashFiles('backend/requirements/*.txt', 'backend/pyproject.toml') }}
+          restore-keys: |
+            ${{ runner.os }}-${{ github.workflow }}-uv-
+
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # ratchet:actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install Dependencies
+        run: |
+          uv pip install --system \
+            -r backend/requirements/default.txt \
+            -r backend/requirements/dev.txt
+          playwright install chromium
+          playwright install-deps chromium
 
       - name: Detect Connector changes
         id: changes
@@ -161,20 +164,16 @@ jobs:
             hubspot:
               - 'backend/onyx/connectors/hubspot/**'
               - 'backend/tests/daily/connectors/hubspot/**'
-              - 'uv.lock'
             salesforce:
               - 'backend/onyx/connectors/salesforce/**'
               - 'backend/tests/daily/connectors/salesforce/**'
-              - 'uv.lock'
             github:
               - 'backend/onyx/connectors/github/**'
               - 'backend/tests/daily/connectors/github/**'
-              - 'uv.lock'
             file_processing:
               - 'backend/onyx/file_processing/**'
-              - 'uv.lock'
 
-      - name: Run Tests (excluding HubSpot, Salesforce, GitHub, and Coda)
+      - name: Run Tests (excluding HubSpot, Salesforce, and GitHub)
         shell: script -q -e -c "bash --noprofile --norc -eo pipefail {0}"
         run: |
           py.test \
@@ -187,8 +186,7 @@ jobs:
             backend/tests/daily/connectors \
             --ignore backend/tests/daily/connectors/hubspot \
             --ignore backend/tests/daily/connectors/salesforce \
-            --ignore backend/tests/daily/connectors/github \
-            --ignore backend/tests/daily/connectors/coda
+            --ignore backend/tests/daily/connectors/github
 
       - name: Run HubSpot Connector Tests
         if: ${{ github.event_name == 'schedule' || steps.changes.outputs.hubspot == 'true' || steps.changes.outputs.file_processing == 'true' }}
@@ -233,10 +231,8 @@ jobs:
         if: failure() && github.event_name == 'schedule'
         env:
           SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
-          REPO: ${{ github.repository }}
-          RUN_ID: ${{ github.run_id }}
         run: |
           curl -X POST \
             -H 'Content-type: application/json' \
-            --data "{\"text\":\"Scheduled Connector Tests failed! Check the run at: https://github.com/${REPO}/actions/runs/${RUN_ID}\"}" \
+            --data '{"text":"Scheduled Connector Tests failed! Check the run at: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"}' \
             $SLACK_WEBHOOK

.github/workflows/pr-python-model-tests.yml

@@ -10,10 +10,7 @@ on:
         description: 'Branch to run the workflow on'
         required: false
         default: 'main'
-
-permissions:
-  contents: read
-
+        
 env:
   # Bedrock
   AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
@@ -31,17 +28,14 @@ env:
 jobs:
   model-check:
     # See https://runs-on.com/runners/linux/
-    runs-on: [runs-on,runner=8cpu-linux-x64,"run-id=${{ github.run_id }}-model-check"]
-    timeout-minutes: 45
+    runs-on: [runs-on,runner=8cpu-linux-x64,"run-id=${{ github.run_id }}"]
 
     env:
       PYTHONPATH: ./backend
 
     steps:
       - name: Checkout code
-        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # ratchet:actions/checkout@v6
-        with:
-          persist-credentials: false
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # ratchet:actions/checkout@v4
 
       - name: Login to Docker Hub
         uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # ratchet:docker/login-action@v3
@@ -59,9 +53,9 @@ jobs:
         run: |
           docker pull onyxdotapp/onyx-model-server:latest
           docker tag onyxdotapp/onyx-model-server:latest onyxdotapp/onyx-model-server:test
-
+          
       - name: Set up Python
-        uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # ratchet:actions/setup-python@v6
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # ratchet:actions/setup-python@v5
         with:
           python-version: "3.11"
           cache: "pip"
@@ -96,15 +90,15 @@ jobs:
           while true; do
             current_time=$(date +%s)
             elapsed_time=$((current_time - start_time))
-
+            
             if [ $elapsed_time -ge $timeout ]; then
               echo "Timeout reached. Service did not become ready in 5 minutes."
               exit 1
             fi
-
+            
             # Use curl with error handling to ignore specific exit code 56
             response=$(curl -s -o /dev/null -w "%{http_code}" http://localhost:9000/api/health || echo "curl_error")
-
+            
             if [ "$response" = "200" ]; then
               echo "Service is ready!"
               break
@@ -113,11 +107,11 @@ jobs:
             else
               echo "Service not ready yet (HTTP status $response). Retrying in 5 seconds..."
             fi
-
+            
             sleep 5
           done
           echo "Finished waiting for service."
-
+          
       - name: Run Tests
         shell: script -q -e -c "bash --noprofile --norc -eo pipefail {0}"
         run: |
@@ -128,14 +122,12 @@ jobs:
         if: failure() && github.event_name == 'schedule'
         env:
           SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
-          REPO: ${{ github.repository }}
-          RUN_ID: ${{ github.run_id }}
         run: |
           curl -X POST \
             -H 'Content-type: application/json' \
-            --data "{\"text\":\"Scheduled Model Tests failed! Check the run at: https://github.com/${REPO}/actions/runs/${RUN_ID}\"}" \
+            --data '{"text":"Scheduled Model Tests failed! Check the run at: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"}' \
             $SLACK_WEBHOOK
-
+            
       - name: Dump all-container logs (optional)
         if: always()
         run: |
@@ -144,7 +136,14 @@ jobs:
 
       - name: Upload logs
         if: always()
-        uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # ratchet:actions/upload-artifact@v4
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # ratchet:actions/upload-artifact@v4
         with:
           name: docker-all-logs
           path: ${{ github.workspace }}/docker-compose.log
+          
+      - name: Stop Docker containers
+        if: always()
+        run: |
+          cd deployment/docker_compose
+          docker compose -f docker-compose.model-server-test.yml down -v
+          

.github/workflows/pr-python-tests.yml

@@ -1,7 +1,4 @@
 name: Python Unit Tests
-concurrency:
-  group: Python-Unit-Tests-${{ github.workflow }}-${{ github.head_ref || github.event.workflow_run.head_branch || github.run_id }}
-  cancel-in-progress: true
 
 on:
   merge_group:
@@ -9,41 +6,39 @@ on:
     branches:
       - main
       - 'release/**'
-  push:
-    tags:
-      - "v*.*.*"
-
-permissions:
-  contents: read
 
 jobs:
   backend-check:
     # See https://runs-on.com/runners/linux/
-    runs-on: [runs-on, runner=2cpu-linux-arm64, "run-id=${{ github.run_id }}-backend-check"]
-    timeout-minutes: 45
-
+    runs-on: [runs-on,runner=8cpu-linux-x64,"run-id=${{ github.run_id }}"]
 
     env:
       PYTHONPATH: ./backend
       REDIS_CLOUD_PYTEST_PASSWORD: ${{ secrets.REDIS_CLOUD_PYTEST_PASSWORD }}
-      DISABLE_TELEMETRY: "true"
-
+      SF_USERNAME: ${{ secrets.SF_USERNAME }}
+      SF_PASSWORD: ${{ secrets.SF_PASSWORD }}
+      SF_SECURITY_TOKEN: ${{ secrets.SF_SECURITY_TOKEN }}
+      
     steps:
-    - uses: runs-on/action@cd2b598b0515d39d78c38a02d529db87d2196d1e # ratchet:runs-on/action@v2
-
     - name: Checkout code
-      uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # ratchet:actions/checkout@v6
-      with:
-        persist-credentials: false
+      uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # ratchet:actions/checkout@v4
 
-    - name: Setup Python and Install Dependencies
-      uses: ./.github/actions/setup-python-and-install-dependencies
+    - name: Set up Python
+      uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # ratchet:actions/setup-python@v5
       with:
-        requirements: |
+        python-version: '3.11'
+        cache: 'pip'
+        cache-dependency-path: |
           backend/requirements/default.txt
           backend/requirements/dev.txt
           backend/requirements/model_server.txt
-          backend/requirements/ee.txt
+
+    - name: Install Dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install --retries 5 --timeout 30 -r backend/requirements/default.txt
+        pip install --retries 5 --timeout 30 -r backend/requirements/dev.txt
+        pip install --retries 5 --timeout 30 -r backend/requirements/model_server.txt
 
     - name: Run Tests
       shell: script -q -e -c "bash --noprofile --norc -eo pipefail {0}"

.github/workflows/pr-quality-checks.yml

@@ -6,44 +6,21 @@ concurrency:
 on:
   merge_group:
   pull_request: null
-  push:
-    branches:
-      - main
-    tags:
-      - "v*.*.*"
-
-permissions:
-  contents: read
 
 jobs:
   quality-checks:
-    runs-on: ubuntu-latest
-    timeout-minutes: 45
+    # See https://runs-on.com/runners/linux/
+    runs-on: [runs-on, runner=2cpu-linux-arm64, "run-id=${{ github.run_id }}"]
     steps:
-      - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # ratchet:actions/checkout@v6
+      - uses: runs-on/action@cd2b598b0515d39d78c38a02d529db87d2196d1e # ratchet:runs-on/action@v2
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # ratchet:actions/checkout@v4
         with:
           fetch-depth: 0
-          persist-credentials: false
-      - uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # ratchet:actions/setup-python@v6
+      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # ratchet:actions/setup-python@v5
         with:
           python-version: "3.11"
       - name: Setup Terraform
         uses: hashicorp/setup-terraform@b9cd54a3c349d3f38e8881555d616ced269862dd # ratchet:hashicorp/setup-terraform@v3
-      - name: Setup node
-        uses: actions/setup-node@395ad3262231945c25e8478fd5baf05154b1d79f # ratchet:actions/setup-node@v6
-        with: # zizmor: ignore[cache-poisoning]
-          node-version: 22
-          cache: "npm"
-          cache-dependency-path: ./web/package-lock.json
-      - name: Install node dependencies
-        working-directory: ./web
-        run: npm ci
-      - uses: j178/prek-action@91fd7d7cf70ae1dee9f4f44e7dfa5d1073fe6623 # ratchet:j178/prek-action@v1
+      - uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # ratchet:pre-commit/action@v3.0.1
         with:
-          prek-version: '0.2.21'
-          extra-args: ${{ github.event_name == 'pull_request' && format('--from-ref {0} --to-ref {1}', github.event.pull_request.base.sha, github.event.pull_request.head.sha) || github.event_name == 'merge_group' && format('--from-ref {0} --to-ref {1}', github.event.merge_group.base_sha, github.event.merge_group.head_sha) || github.ref_name == 'main' && '--all-files' || '' }}
-      - name: Check Actions
-        uses: giner/check-actions@28d366c7cbbe235f9624a88aa31a628167eee28c # ratchet:giner/check-actions@v1.0.1
-        with:
-          check_permissions: false
-          check_versions: false
+          extra_args: ${{ github.event_name == 'pull_request' && format('--from-ref {0} --to-ref {1}', github.event.pull_request.base.sha, github.event.pull_request.head.sha) || '' }}

.github/workflows/release-devtools.yml

@@ -1,40 +0,0 @@
-name: Release Devtools
-
-on:
-  push:
-    tags:
-      - "ods/v*.*.*"
-
-jobs:
-  pypi:
-    runs-on: ubuntu-latest
-    environment:
-      name: release-devtools
-    permissions:
-      id-token: write
-    timeout-minutes: 10
-    strategy:
-      matrix:
-        os-arch:
-          - {goos: "linux", goarch: "amd64"}
-          - {goos: "linux", goarch: "arm64"}
-          - {goos: "windows", goarch: "amd64"}
-          - {goos: "windows", goarch: "arm64"}
-          - {goos: "darwin", goarch: "amd64"}
-          - {goos: "darwin", goarch: "arm64"}
-          - {goos: "", goarch: ""}
-    steps:
-      - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # ratchet:actions/checkout@v6
-        with:
-          persist-credentials: false
-          fetch-depth: 0
-      - uses: astral-sh/setup-uv@1e862dfacbd1d6d858c55d9b792c756523627244 # ratchet:astral-sh/setup-uv@v7
-        with:
-          enable-cache: false
-      - run: |
-          GOOS="${{ matrix.os-arch.goos }}" \
-          GOARCH="${{ matrix.os-arch.goarch }}" \
-          uv build --wheel
-        working-directory: tools/ods
-      - run: uv publish
-        working-directory: tools/ods

.github/workflows/sync_foss.yml

@@ -9,15 +9,13 @@ on:
 jobs:
   sync-foss:
     runs-on: ubuntu-latest
-    timeout-minutes: 45
     permissions:
       contents: read
     steps:
       - name: Checkout main Onyx repo
-        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # ratchet:actions/checkout@v6
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # ratchet:actions/checkout@v4
         with:
           fetch-depth: 0
-          persist-credentials: false
 
       - name: Install git-filter-repo
         run: |

.github/workflows/tag-nightly.yml

@@ -3,30 +3,30 @@ name: Nightly Tag Push
 on:
   schedule:
     - cron: "0 10 * * *" # Runs every day at 2 AM PST / 3 AM PDT / 10 AM UTC
-  workflow_dispatch:
 
 permissions:
   contents: write # Allows pushing tags to the repository
 
 jobs:
   create-and-push-tag:
-    runs-on: ubuntu-slim
-    timeout-minutes: 45
+    runs-on: [runs-on, runner=2cpu-linux-x64, "run-id=${{ github.run_id }}"]
 
     steps:
       # actions using GITHUB_TOKEN cannot trigger another workflow, but we do want this to trigger docker pushes
       # see https://github.com/orgs/community/discussions/27028#discussioncomment-3254367 for the workaround we
       # implement here which needs an actual user's deploy key
+
+      # Additional NOTE: even though this is named "rkuo", the actual key is tied to the onyx repo
+      # and not rkuo's personal account. It is fine to leave this key as is!
       - name: Checkout code
-        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # ratchet:actions/checkout@v6
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # ratchet:actions/checkout@v4
         with:
-          ssh-key: "${{ secrets.DEPLOY_KEY }}"
-          persist-credentials: true
+          ssh-key: "${{ secrets.RKUO_DEPLOY_KEY }}"
 
       - name: Set up Git user
         run: |
-          git config user.name "Onyx Bot [bot]"
-          git config user.email "onyx-bot[bot]@onyx.app"
+          git config user.name "Richard Kuo [bot]"
+          git config user.email "rkuo[bot]@onyx.app"
 
       - name: Check for existing nightly tag
         id: check_tag
@@ -54,12 +54,3 @@ jobs:
         run: |
           TAG_NAME="nightly-latest-$(date +'%Y%m%d')"
           git push origin $TAG_NAME
-
-      - name: Send Slack notification
-        if: failure()
-        uses: ./.github/actions/slack-notify
-        with:
-          webhook-url: ${{ secrets.MONITOR_DEPLOYMENTS_WEBHOOK }}
-          title: "🚨 Nightly Tag Push Failed"
-          ref-name: ${{ github.ref_name }}
-          failed-jobs: "create-and-push-tag"

.github/workflows/zizmor.yml

@@ -1,38 +0,0 @@
-name: Run Zizmor
-
-on:
-  push:
-    branches: ["main"]
-  pull_request:
-    branches: ["**"]
-
-permissions: {}
-
-jobs:
-  zizmor:
-    name: zizmor
-    runs-on: ubuntu-slim
-    timeout-minutes: 45
-    permissions:
-      security-events: write # needed for SARIF uploads
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # ratchet:actions/checkout@v6.0.1
-        with:
-          persist-credentials: false
-
-      - name: Install the latest version of uv
-        uses: astral-sh/setup-uv@1e862dfacbd1d6d858c55d9b792c756523627244 # ratchet:astral-sh/setup-uv@v7.1.4
-        with:
-          enable-cache: false
-
-      - name: Run zizmor
-        run: uv run --no-sync --with zizmor zizmor --format=sarif . > results.sarif
-        env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Upload SARIF file
-        uses: github/codeql-action/upload-sarif@ba454b8ab46733eb6145342877cd148270bb77ab # ratchet:github/codeql-action/upload-sarif@codeql-bundle-v2.23.5
-        with:
-          sarif_file: results.sarif
-          category: zizmor

.gitignore

@@ -1,7 +1,6 @@
 # editors
 .vscode
 .zed
-.cursor
 
 # macos
 .DS_store
@@ -19,7 +18,6 @@ backend/tests/regression/search_quality/eval-*
 backend/tests/regression/search_quality/search_eval_config.yaml
 backend/tests/regression/search_quality/*.json
 backend/onyx/evals/data/
-backend/onyx/evals/one_off/*.json
 *.log
 
 # secret files
@@ -29,8 +27,6 @@ settings.json
 
 # others
 /deployment/data/nginx/app.conf
-/deployment/data/nginx/mcp.conf.inc
-/deployment/data/nginx/mcp_upstream.conf.inc
 *.sw?
 /backend/tests/regression/answer_quality/search_test_config.yaml
 *.egg-info
@@ -49,10 +45,5 @@ CLAUDE.md
 # Local .terraform.lock.hcl file
 .terraform.lock.hcl
 
-node_modules
-
 # MCP configs
 .playwright-mcp
-
-# plans
-plans/

.mcp.json.template

@@ -0,0 +1,8 @@
+{
+  "mcpServers": {
+    "onyx-mcp": {
+      "type": "http",
+      "url": "http://localhost:8000/mcp"
+    }
+  }
+}

.pre-commit-config.yaml

@@ -1,64 +1,13 @@
-default_install_hook_types:
-  - pre-commit
-  - post-checkout
-  - post-merge
-  - post-rewrite
 repos:
-  - repo: https://github.com/astral-sh/uv-pre-commit
-    # From: https://github.com/astral-sh/uv-pre-commit/pull/53/commits/d30b4298e4fb63ce8609e29acdbcf4c9018a483c
-    rev: d30b4298e4fb63ce8609e29acdbcf4c9018a483c
-    hooks:
-      - id: uv-run
-        name: Check lazy imports
-        args: ["--with=onyx-devtools", "ods", "check-lazy-imports"]
-        files: ^backend/(?!\.venv/).*\.py$
-      - id: uv-sync
-        args: ["--locked", "--all-extras"]
-      - id: uv-lock
-        files: ^pyproject\.toml$
-      - id: uv-export
-        name: uv-export default.txt
-        args: ["--no-emit-project", "--no-default-groups", "--no-hashes", "--extra", "backend", "-o", "backend/requirements/default.txt"]
-        files: ^(pyproject\.toml|uv\.lock|backend/requirements/.*\.txt)$
-      - id: uv-export
-        name: uv-export dev.txt
-        args: ["--no-emit-project", "--no-default-groups", "--no-hashes", "--extra", "dev", "-o", "backend/requirements/dev.txt"]
-        files: ^(pyproject\.toml|uv\.lock|backend/requirements/.*\.txt)$
-      - id: uv-export
-        name: uv-export ee.txt
-        args: ["--no-emit-project", "--no-default-groups", "--no-hashes", "--extra", "ee", "-o", "backend/requirements/ee.txt"]
-        files: ^(pyproject\.toml|uv\.lock|backend/requirements/.*\.txt)$
-      - id: uv-export
-        name: uv-export model_server.txt
-        args: ["--no-emit-project", "--no-default-groups", "--no-hashes", "--extra", "model_server", "-o", "backend/requirements/model_server.txt"]
-        files: ^(pyproject\.toml|uv\.lock|backend/requirements/.*\.txt)$
-      # NOTE: This takes ~6s on a single, large module which is prohibitively slow.
-      # - id: uv-run
-      #   name: mypy
-      #   args: ["--all-extras", "mypy"]
-      #   pass_filenames: true
-      #   files: ^backend/.*\.py$
-
-  - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: 3e8a8703264a2f4a69428a0aa4dcb512790b2c8c  # frozen: v6.0.0
-    hooks:
-      - id: check-yaml
-        files: ^.github/
-
-  - repo: https://github.com/rhysd/actionlint
-    rev: a443f344ff32813837fa49f7aa6cbc478d770e62  # frozen: v1.7.9
-    hooks:
-      - id: actionlint
-
   - repo: https://github.com/psf/black
-    rev: 8a737e727ac5ab2f1d4cf5876720ed276dc8dc4b # frozen: 25.1.0
+    rev: 25.1.0
     hooks:
     - id: black
       language_version: python3.11
 
   # this is a fork which keeps compatibility with black
   - repo: https://github.com/wimglenn/reorder-python-imports-black
-    rev: f55cd27f90f0cf0ee775002c2383ce1c7820013d  # frozen: v3.14.0
+    rev: v3.14.0
     hooks:
     - id: reorder-python-imports
       args: ['--py311-plus', '--application-directories=backend/']
@@ -70,32 +19,26 @@ repos:
   # These settings will remove unused imports with side effects
   # Note: The repo currently does not and should not have imports with side effects
   - repo: https://github.com/PyCQA/autoflake
-    rev: 0544741e2b4a22b472d9d93e37d4ea9153820bb1  # frozen: v2.3.1
+    rev: v2.3.1
     hooks:
       - id: autoflake
         args: [ '--remove-all-unused-imports', '--remove-unused-variables', '--in-place' , '--recursive']
 
-  - repo: https://github.com/golangci/golangci-lint
-    rev: 9f61b0f53f80672872fced07b6874397c3ed197b  # frozen: v2.7.2
-    hooks:
-      - id: golangci-lint
-        entry: bash -c "find tools/ -name go.mod -print0 | xargs -0 -I{} bash -c 'cd \"$(dirname {})\" && golangci-lint run ./...'"
-
   - repo: https://github.com/astral-sh/ruff-pre-commit
     # Ruff version.
-    rev: 971923581912ef60a6b70dbf0c3e9a39563c9d47  # frozen: v0.11.4
+    rev: v0.11.4
     hooks:
       - id: ruff
 
   - repo: https://github.com/pre-commit/mirrors-prettier
-    rev: ffb6a759a979008c0e6dff86e39f4745a2d9eac4  # frozen: v3.1.0
+    rev: v3.1.0
     hooks:
     - id: prettier
       types_or: [html, css, javascript, ts, tsx]
       language_version: system
 
   - repo: https://github.com/sirwart/ripsecrets
-    rev: 7d94620933e79b8acaa0cd9e60e9864b07673d86  # frozen: v0.1.11
+    rev: v0.1.11
     hooks:
       - id: ripsecrets
         args:
@@ -111,10 +54,40 @@ repos:
         language: system
         pass_filenames: false
         files: \.tf$
-
-      - id: typescript-check
-        name: TypeScript type check
-        entry: bash -c 'cd web && npm run types:check'
+      - id: check-lazy-imports
+        name: Check lazy imports are not directly imported
+        entry: python3 backend/scripts/check_lazy_imports.py
         language: system
+        files: ^backend/(?!\.venv/).*\.py$
         pass_filenames: false
-        files: ^web/.*\.(ts|tsx)$
+        # Note: pass_filenames is false because tsc must check the entire
+        # project, but the files filter ensures this only runs when relevant
+        # files change. Using --incremental for faster subsequent checks.
+
+  # We would like to have a mypy pre-commit hook, but due to the fact that
+  # pre-commit runs in it's own isolated environment, we would need to install
+  # and keep in sync all dependencies so mypy has access to the appropriate type
+  # stubs. This does not seem worth it at the moment, so for now we will stick to
+  # having mypy run via Github Actions / manually by contributors
+  # - repo: https://github.com/pre-commit/mirrors-mypy
+  #   rev: v1.1.1
+  #   hooks:
+  #     - id: mypy
+  #       exclude: ^tests/
+  #       # below are needed for type stubs since pre-commit runs in it's own
+  #       # isolated environment. Unfortunately, this needs to be kept in sync
+  #       # with requirements/dev.txt + requirements/default.txt
+  #       additional_dependencies: [
+  #         alembic==1.10.4,
+  #         types-beautifulsoup4==4.12.0.3,
+  #         types-html5lib==1.1.11.13,
+  #         types-oauthlib==3.2.0.9,
+  #         types-psycopg2==2.9.21.10,
+  #         types-python-dateutil==2.8.19.13,
+  #         types-regex==2023.3.23.1,
+  #         types-requests==2.28.11.17,
+  #         types-retry==0.9.9.3,
+  #         types-urllib3==1.26.25.11
+  #       ]
+  #       # TODO: add back once errors are addressed
+  #       # args: [--strict]

.vscode/env_template.txt

@@ -5,8 +5,11 @@
 # For local dev, often user Authentication is not needed
 AUTH_TYPE=disabled
 
+# Skip warm up for dev
+SKIP_WARM_UP=True
+
 # Always keep these on for Dev
-# Logs model prompts, reasoning, and answer to stdout
+# Logs all model prompts to stdout
 LOG_ONYX_MODEL_INTERACTIONS=True
 # More verbose logging
 LOG_LEVEL=debug
@@ -34,16 +37,31 @@ OPENAI_API_KEY=<REPLACE THIS>
 GEN_AI_MODEL_VERSION=gpt-4o
 FAST_GEN_AI_MODEL_VERSION=gpt-4o
 
+# For Onyx Slack Bot, overrides the UI values so no need to set this up via UI every time
+# Only needed if using OnyxBot
+#ONYX_BOT_SLACK_APP_TOKEN=<REPLACE THIS>
+#ONYX_BOT_SLACK_BOT_TOKEN=<REPLACE THIS>
+
 
 # Python stuff
 PYTHONPATH=../backend
 PYTHONUNBUFFERED=1
 
 
+# Internet Search
+EXA_API_KEY=<REPLACE THIS>
+
+
 # Enable the full set of Danswer Enterprise Edition features
 # NOTE: DO NOT ENABLE THIS UNLESS YOU HAVE A PAID ENTERPRISE LICENSE (or if you are using this for local testing/development)
 ENABLE_PAID_ENTERPRISE_EDITION_FEATURES=False
 
+# Agent Search configs  # TODO: Remove give proper namings
+AGENT_RETRIEVAL_STATS=False   # Note: This setting will incur substantial re-ranking effort
+AGENT_RERANKING_STATS=True
+AGENT_MAX_QUERY_RETRIEVAL_RESULTS=20
+AGENT_RERANKING_MAX_QUERY_RETRIEVAL_RESULTS=20
+
 # S3 File Store Configuration (MinIO for local development)
 S3_ENDPOINT_URL=http://localhost:9004
 S3_FILE_STORE_BUCKET_NAME=onyx-file-store-bucket

.vscode/launch.template.jsonc

@@ -20,7 +20,6 @@
         "Web Server",
         "Model Server",
         "API Server",
-        "MCP Server",
         "Slack Bot",
         "Celery primary",
         "Celery light",
@@ -133,6 +132,8 @@
       },
       "consoleTitle": "API Server Console"
     },
+    // For the listener to access the Slack API,
+    // ONYX_BOT_SLACK_APP_TOKEN & ONYX_BOT_SLACK_BOT_TOKEN need to be set in .env file located in the root of the project
     {
       "name": "Slack Bot",
       "consoleName": "Slack Bot",
@@ -151,34 +152,6 @@
       },
       "consoleTitle": "Slack Bot Console"
     },
-    {
-      "name": "MCP Server",
-      "consoleName": "MCP Server",
-      "type": "debugpy",
-      "request": "launch",
-      "module": "uvicorn",
-      "cwd": "${workspaceFolder}/backend",
-      "envFile": "${workspaceFolder}/.vscode/.env",
-      "env": {
-        "MCP_SERVER_ENABLED": "true",
-        "MCP_SERVER_PORT": "8090",
-        "MCP_SERVER_CORS_ORIGINS": "http://localhost:*",
-        "LOG_LEVEL": "DEBUG",
-        "PYTHONUNBUFFERED": "1"
-      },
-      "args": [
-        "onyx.mcp_server.api:mcp_app",
-        "--reload",
-        "--port",
-        "8090",
-        "--timeout-graceful-shutdown",
-        "0"
-      ],
-      "presentation": {
-        "group": "2"
-      },
-      "consoleTitle": "MCP Server Console"
-    },
     {
       "name": "Celery primary",
       "type": "debugpy",
@@ -508,6 +481,7 @@
       ],
       "cwd": "${workspaceFolder}",
       "console": "integratedTerminal",
+      "stopOnEntry": true,
       "presentation": {
         "group": "3"
       }
@@ -553,10 +527,10 @@
       "name": "Install Python Requirements",
       "type": "node",
       "request": "launch",
-      "runtimeExecutable": "uv",
+      "runtimeExecutable": "bash",
       "runtimeArgs": [
-        "sync",
-        "--all-extras"
+        "-c",
+        "pip install -r backend/requirements/default.txt && pip install -r backend/requirements/dev.txt && pip install -r backend/requirements/ee.txt && pip install -r backend/requirements/model_server.txt"
       ],
       "cwd": "${workspaceFolder}",
       "console": "integratedTerminal",
@@ -569,14 +543,14 @@
       "name": "Onyx OpenAPI Schema Generator",
       "type": "debugpy",
       "request": "launch",
-      "program": "backend/scripts/onyx_openapi_schema.py",
-      "cwd": "${workspaceFolder}",
+      "program": "scripts/onyx_openapi_schema.py",
+      "cwd": "${workspaceFolder}/backend",
       "envFile": "${workspaceFolder}/.env",
       "env": {
         "PYTHONUNBUFFERED": "1",
-        "PYTHONPATH": "backend"
+        "PYTHONPATH": "."
       },
-      "args": ["--filename", "backend/generated/openapi.json", "--generate-python-client"]
+      "args": ["--filename", "generated/openapi.json"]
     },
     {
       // script to debug multi tenant db issues

CLAUDE.md.template

@@ -4,7 +4,7 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co
 
 ## KEY NOTES
 
-- If you run into any missing python dependency errors, try running your command with `source .venv/bin/activate` \
+- If you run into any missing python dependency errors, try running your command with `source backend/.venv/bin/activate` \
 to assume the python venv.
 - To make tests work, check the `.env` file at the root of the project to find an OpenAI key.
 - If using `playwright` to explore the frontend, you can usually log in with username `a@test.com` and password

CONTRIBUTING.md

@@ -71,12 +71,12 @@ If using a higher version, sometimes some libraries will not be available (i.e.
 
 #### Backend: Python requirements
 
-Currently, we use [uv](https://docs.astral.sh/uv/) and recommend creating a [virtual environment](https://docs.astral.sh/uv/pip/environments/#using-a-virtual-environment).
+Currently, we use pip and recommend creating a virtual environment.
 
 For convenience here's a command for it:
 
 ```bash
-uv venv .venv --python 3.11
+python -m venv .venv
 source .venv/bin/activate
 ```
 
@@ -95,15 +95,33 @@ If using PowerShell, the command slightly differs:
 Install the required python dependencies:
 
 ```bash
-uv sync --all-extras
+pip install -r backend/requirements/combined.txt
 ```
 
-Install Playwright for Python (headless browser required by the Web Connector):
+or
 
 ```bash
-uv run playwright install
+pip install -r backend/requirements/default.txt
+pip install -r backend/requirements/dev.txt
+pip install -r backend/requirements/ee.txt
+pip install -r backend/requirements/model_server.txt
 ```
 
+Fix vscode/cursor auto-imports:
+```bash
+pip install -e .
+```
+
+Install Playwright for Python (headless browser required by the Web Connector)
+
+In the activated Python virtualenv, install Playwright for Python by running:
+
+```bash
+playwright install
+```
+
+You may have to deactivate and reactivate your virtualenv for `playwright` to appear on your path.
+
 #### Frontend: Node dependencies
 
 Onyx uses Node v22.20.0. We highly recommend you use [Node Version Manager (nvm)](https://github.com/nvm-sh/nvm)
@@ -112,7 +130,7 @@ to manage your Node installations. Once installed, you can run
 ```bash
 nvm install 22 && nvm use 22
 node -v # verify your active version
-```
+``` 
 
 Navigate to `onyx/web` and run:
 
@@ -126,15 +144,21 @@ npm i
 
 For the backend, you'll need to setup pre-commit hooks (black / reorder-python-imports).
 
-Then run:
+With the virtual environment active, install the pre-commit library with:
 
 ```bash
-uv run pre-commit install
+pip install pre-commit
+```
+
+Then, from the `onyx/backend` directory, run:
+
+```bash
+pre-commit install
 ```
 
 Additionally, we use `mypy` for static type checking.
 Onyx is fully type-annotated, and we want to keep it that way!
-To run the mypy checks manually, run `uv run mypy .` from the `onyx/backend` directory.
+To run the mypy checks manually, run `python -m mypy .` from the `onyx/backend` directory.
 
 ### Web
 

README.md

@@ -1,34 +1,29 @@
 <a name="readme-top"></a>
 
 <h2 align="center">
-    <a href="https://www.onyx.app/?utm_source=onyx_repo&utm_medium=github&utm_campaign=readme"> <img width="50%" src="https://github.com/onyx-dot-app/onyx/blob/logo/OnyxLogoCropped.jpg?raw=true" /></a>
+    <a href="https://www.onyx.app/"> <img width="50%" src="https://github.com/onyx-dot-app/onyx/blob/logo/OnyxLogoCropped.jpg?raw=true)" /></a>
 </h2>
 
 <p align="center">Open Source AI Platform</p>
 
 <p align="center">
     <a href="https://discord.gg/TDJ59cGV2X" target="_blank">
-        <img src="https://img.shields.io/badge/discord-join-blue.svg?logo=discord&logoColor=white" alt="Discord" />
+        <img src="https://img.shields.io/badge/discord-join-blue.svg?logo=discord&logoColor=white" alt="Discord">
     </a>
-    <a href="https://docs.onyx.app/?utm_source=onyx_repo&utm_medium=github&utm_campaign=readme" target="_blank">
-        <img src="https://img.shields.io/badge/docs-view-blue" alt="Documentation" />
+    <a href="https://docs.onyx.app/" target="_blank">
+        <img src="https://img.shields.io/badge/docs-view-blue" alt="Documentation">
     </a>
-    <a href="https://www.onyx.app/?utm_source=onyx_repo&utm_medium=github&utm_campaign=readme" target="_blank">
-        <img src="https://img.shields.io/website?url=https://www.onyx.app&up_message=visit&up_color=blue" alt="Documentation" />
+    <a href="https://docs.onyx.app/" target="_blank">
+        <img src="https://img.shields.io/website?url=https://www.onyx.app&up_message=visit&up_color=blue" alt="Documentation">
     </a>
     <a href="https://github.com/onyx-dot-app/onyx/blob/main/LICENSE" target="_blank">
-        <img src="https://img.shields.io/static/v1?label=license&message=MIT&color=blue" alt="License" />
+        <img src="https://img.shields.io/static/v1?label=license&message=MIT&color=blue" alt="License">
     </a>
 </p>
 
-<p align="center">
-  <a href="https://trendshift.io/repositories/12516" target="_blank">
-    <img src="https://trendshift.io/api/badge/repositories/12516" alt="onyx-dot-app/onyx | Trendshift" style="width: 250px; height: 55px;" />
-  </a>
-</p>
 
 
-**[Onyx](https://www.onyx.app/?utm_source=onyx_repo&utm_medium=github&utm_campaign=readme)** is a feature-rich, self-hostable Chat UI that works with any LLM. It is easy to deploy and can run in a completely airgapped environment.
+**[Onyx](https://www.onyx.app/)** is a feature-rich, self-hostable Chat UI that works with any LLM. It is easy to deploy and can run in a completely airgapped environment.
 
 Onyx comes loaded with advanced features like Agents, Web Search, RAG, MCP, Deep Research, Connectors to 40+ knowledge sources, and more.
 
@@ -57,7 +52,7 @@ Onyx comes loaded with advanced features like Agents, Web Search, RAG, MCP, Deep
 
 Onyx works with all LLMs (like OpenAI, Anthropic, Gemini, etc.) and self-hosted LLMs (like Ollama, vLLM, etc.)
 
-To learn more about the features, check out our [documentation](https://docs.onyx.app/welcome?utm_source=onyx_repo&utm_medium=github&utm_campaign=readme)!
+To learn more about the features, check out our [documentation](https://docs.onyx.app/welcome)!
 
 
 
@@ -65,13 +60,13 @@ To learn more about the features, check out our [documentation](https://docs.ony
 Onyx supports deployments in Docker, Kubernetes, Terraform, along with guides for major cloud providers.
 
 See guides below:
-- [Docker](https://docs.onyx.app/deployment/local/docker?utm_source=onyx_repo&utm_medium=github&utm_campaign=readme) or [Quickstart](https://docs.onyx.app/deployment/getting_started/quickstart?utm_source=onyx_repo&utm_medium=github&utm_campaign=readme) (best for most users)
-- [Kubernetes](https://docs.onyx.app/deployment/local/kubernetes?utm_source=onyx_repo&utm_medium=github&utm_campaign=readme) (best for large teams)
-- [Terraform](https://docs.onyx.app/deployment/local/terraform?utm_source=onyx_repo&utm_medium=github&utm_campaign=readme) (best for teams already using Terraform)
-- Cloud specific guides (best if specifically using [AWS EKS](https://docs.onyx.app/deployment/cloud/aws/eks?utm_source=onyx_repo&utm_medium=github&utm_campaign=readme), [Azure VMs](https://docs.onyx.app/deployment/cloud/azure?utm_source=onyx_repo&utm_medium=github&utm_campaign=readme), etc.)
+- [Docker](https://docs.onyx.app/deployment/local/docker) or [Quickstart](https://docs.onyx.app/deployment/getting_started/quickstart) (best for most users)
+- [Kubernetes](https://docs.onyx.app/deployment/local/kubernetes) (best for large teams)
+- [Terraform](https://docs.onyx.app/deployment/local/terraform) (best for teams already using Terraform)
+- Cloud specific guides (best if specifically using [AWS EKS](https://docs.onyx.app/deployment/cloud/aws/eks), [Azure VMs](https://docs.onyx.app/deployment/cloud/azure), etc.)
 
 > [!TIP]  
-> **To try Onyx for free without deploying, check out [Onyx Cloud](https://cloud.onyx.app/signup?utm_source=onyx_repo&utm_medium=github&utm_campaign=readme)**.
+> **To try Onyx for free without deploying, check out [Onyx Cloud](https://cloud.onyx.app/signup)**.
 
 
 
@@ -95,7 +90,7 @@ There are two editions of Onyx:
 
 - Onyx Community Edition (CE) is available freely under the MIT license.
 - Onyx Enterprise Edition (EE) includes extra features that are primarily useful for larger organizations.
-For feature details, check out [our website](https://www.onyx.app/pricing?utm_source=onyx_repo&utm_medium=github&utm_campaign=readme).
+For feature details, check out [our website](https://www.onyx.app/pricing).
 
 
 

backend/Dockerfile

@@ -7,19 +7,15 @@ have a contract or agreement with DanswerAI, you are not permitted to use the En
 Edition features outside of personal development or testing purposes. Please reach out to \
 founders@onyx.app for more information. Please visit https://github.com/onyx-dot-app/onyx"
 
+# Default ONYX_VERSION, typically overriden during builds by GitHub Actions.
+ARG ONYX_VERSION=0.0.0-dev
 # DO_NOT_TRACK is used to disable telemetry for Unstructured
-ENV DANSWER_RUNNING_IN_DOCKER="true" \
+ENV ONYX_VERSION=${ONYX_VERSION} \
+    DANSWER_RUNNING_IN_DOCKER="true" \
     DO_NOT_TRACK="true" \
     PLAYWRIGHT_BROWSERS_PATH="/app/.cache/ms-playwright"
 
-# Create non-root user for security best practices
-RUN groupadd -g 1001 onyx && \
-    useradd -u 1001 -g onyx -m -s /bin/bash onyx && \
-    mkdir -p /var/log/onyx && \
-    chmod 755 /var/log/onyx && \
-    chown onyx:onyx /var/log/onyx
-
-COPY --from=ghcr.io/astral-sh/uv:0.9.9 /uv /uvx /bin/
+COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/
 
 # Install system dependencies
 # cmake needed for psycopg (postgres)
@@ -58,7 +54,6 @@ RUN uv pip install --system --no-cache-dir --upgrade \
     pip uninstall -y py && \
     playwright install chromium && \
     playwright install-deps chromium && \
-    chown -R onyx:onyx /app && \
     ln -s /usr/local/bin/supervisord /usr/bin/supervisord && \
     # Cleanup for CVEs and size reduction
     # https://github.com/tornadoweb/tornado/issues/3107
@@ -102,6 +97,13 @@ tiktoken.get_encoding('cl100k_base')"
 # Set up application files
 WORKDIR /app
 
+# Create non-root user for security best practices
+RUN groupadd -g 1001 onyx && \
+    useradd -u 1001 -g onyx -m -s /bin/bash onyx && \
+    mkdir -p /var/log/onyx && \
+    chmod 755 /var/log/onyx && \
+    chown onyx:onyx /var/log/onyx
+
 # Enterprise Version Files
 COPY --chown=onyx:onyx ./ee /app/ee
 COPY supervisord.conf /etc/supervisor/conf.d/supervisord.conf
@@ -126,10 +128,6 @@ COPY --chown=onyx:onyx ./assets /app/assets
 
 ENV PYTHONPATH=/app
 
-# Default ONYX_VERSION, typically overriden during builds by GitHub Actions.
-ARG ONYX_VERSION=0.0.0-dev
-ENV ONYX_VERSION=${ONYX_VERSION}
-
 # Default command which does nothing
 # This container is used by api server and background which specify their own CMD
 CMD ["tail", "-f", "/dev/null"]

backend/Dockerfile.model_server

@@ -1,42 +1,4 @@
-# Base stage with dependencies
-FROM python:3.11.7-slim-bookworm AS base
-
-ENV DANSWER_RUNNING_IN_DOCKER="true" \
-    HF_HOME=/app/.cache/huggingface
-
-COPY --from=ghcr.io/astral-sh/uv:0.9.9 /uv /uvx /bin/
-
-RUN mkdir -p /app/.cache/huggingface
-
-COPY ./requirements/model_server.txt /tmp/requirements.txt
-RUN uv pip install --system --no-cache-dir --upgrade \
-        -r /tmp/requirements.txt && \
-    rm -rf ~/.cache/uv /tmp/*.txt
-
-# Stage for downloading tokenizers
-FROM base AS tokenizers
-RUN python -c "from transformers import AutoTokenizer; \
-AutoTokenizer.from_pretrained('distilbert-base-uncased'); \
-AutoTokenizer.from_pretrained('mixedbread-ai/mxbai-rerank-xsmall-v1');"
-
-# Stage for downloading Onyx models
-FROM base AS onyx-models
-RUN python -c "from huggingface_hub import snapshot_download; \
-snapshot_download(repo_id='onyx-dot-app/hybrid-intent-token-classifier'); \
-snapshot_download(repo_id='onyx-dot-app/information-content-model');"
-
-# Stage for downloading embedding and reranking models
-FROM base AS embedding-models
-RUN python -c "from huggingface_hub import snapshot_download; \
-snapshot_download('nomic-ai/nomic-embed-text-v1'); \
-snapshot_download('mixedbread-ai/mxbai-rerank-xsmall-v1');"
-
-# Initialize SentenceTransformer to cache the custom architecture
-RUN python -c "from sentence_transformers import SentenceTransformer; \
-SentenceTransformer(model_name_or_path='nomic-ai/nomic-embed-text-v1', trust_remote_code=True);"
-
-# Final stage - combine all downloads
-FROM base AS final
+FROM python:3.11.7-slim-bookworm
 
 LABEL com.danswer.maintainer="founders@onyx.app"
 LABEL com.danswer.description="This image is for the Onyx model server which runs all of the \
@@ -44,19 +6,65 @@ AI models for Onyx. This container and all the code is MIT Licensed and free for
 You can find it at https://hub.docker.com/r/onyx/onyx-model-server. For more details, \
 visit https://github.com/onyx-dot-app/onyx."
 
+# Default ONYX_VERSION, typically overriden during builds by GitHub Actions.
+ARG ONYX_VERSION=0.0.0-dev
+ENV ONYX_VERSION=${ONYX_VERSION} \
+    DANSWER_RUNNING_IN_DOCKER="true" \
+    HF_HOME=/app/.cache/huggingface
+
+COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/
+
 # Create non-root user for security best practices
-RUN groupadd -g 1001 onyx && \
-    useradd -u 1001 -g onyx -m -s /bin/bash onyx && \
+RUN mkdir -p /app && \
+    groupadd -g 1001 onyx && \
+    useradd -u 1001 -g onyx -m -s /bin/bash onyx  && \
+    chown -R onyx:onyx /app && \
     mkdir -p /var/log/onyx && \
     chmod 755 /var/log/onyx && \
     chown onyx:onyx /var/log/onyx
 
-# In case the user has volumes mounted to /app/.cache/huggingface that they've downloaded while
-# running Onyx, move the current contents of the cache folder to a temporary location to ensure
-# it's preserved in order to combine with the user's cache contents
-COPY --chown=onyx:onyx --from=tokenizers /app/.cache/huggingface /app/.cache/temp_huggingface
-COPY --chown=onyx:onyx --from=onyx-models /app/.cache/huggingface /app/.cache/temp_huggingface
-COPY --chown=onyx:onyx --from=embedding-models /app/.cache/huggingface /app/.cache/temp_huggingface
+# --- add toolchain needed for Rust/Python builds (fastuuid) ---
+ENV RUSTUP_HOME=/usr/local/rustup \
+    CARGO_HOME=/usr/local/cargo \
+    PATH=/usr/local/cargo/bin:$PATH
+
+RUN set -eux; \
+    apt-get update && apt-get install -y --no-install-recommends \
+        build-essential \
+        pkg-config \
+        curl \
+        ca-certificates \
+    # Install latest stable Rust (supports Cargo.lock v4)
+    && curl -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain stable \
+    && rustc --version && cargo --version \
+    && apt-get remove -y --allow-remove-essential perl-base \
+    && apt-get autoremove -y \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY ./requirements/model_server.txt /tmp/requirements.txt
+RUN uv pip install --system --no-cache-dir --upgrade \
+        -r /tmp/requirements.txt && \
+    rm -rf ~/.cache/uv /tmp/*.txt
+
+# Pre-downloading models for setups with limited egress
+# Download tokenizers, distilbert for the Onyx model
+# Download model weights
+# Run Nomic to pull in the custom architecture and have it cached locally
+RUN python -c "from transformers import AutoTokenizer; \
+AutoTokenizer.from_pretrained('distilbert-base-uncased'); \
+AutoTokenizer.from_pretrained('mixedbread-ai/mxbai-rerank-xsmall-v1'); \
+from huggingface_hub import snapshot_download; \
+snapshot_download(repo_id='onyx-dot-app/hybrid-intent-token-classifier'); \
+snapshot_download(repo_id='onyx-dot-app/information-content-model'); \
+snapshot_download('nomic-ai/nomic-embed-text-v1'); \
+snapshot_download('mixedbread-ai/mxbai-rerank-xsmall-v1'); \
+from sentence_transformers import SentenceTransformer; \
+SentenceTransformer(model_name_or_path='nomic-ai/nomic-embed-text-v1', trust_remote_code=True);" && \
+    # In case the user has volumes mounted to /app/.cache/huggingface that they've downloaded while
+    # running Onyx, move the current contents of the cache folder to a temporary location to ensure
+    # it's preserved in order to combine with the user's cache contents
+    mv /app/.cache/huggingface /app/.cache/temp_huggingface && \
+    chown -R onyx:onyx /app
 
 WORKDIR /app
 
@@ -75,8 +83,4 @@ COPY ./model_server /app/model_server
 
 ENV PYTHONPATH=/app
 
-# Default ONYX_VERSION, typically overriden during builds by GitHub Actions.
-ARG ONYX_VERSION=0.0.0-dev
-ENV ONYX_VERSION=${ONYX_VERSION}
-
 CMD ["uvicorn", "model_server.main:app", "--host", "0.0.0.0", "--port", "9000"]

backend/alembic/README.md

@@ -7,12 +7,8 @@ Onyx migrations use a generic single-database configuration with an async dbapi.
 
 ## To generate new migrations:
 
-From onyx/backend, run:
-`alembic revision -m <DESCRIPTION_OF_MIGRATION>`
-
-Note: you cannot use the `--autogenerate` flag as the automatic schema parsing does not work.
-
-Manually populate the upgrade and downgrade in your new migration.
+run from onyx/backend:
+`alembic revision --autogenerate -m <DESCRIPTION_OF_MIGRATION>`
 
 More info can be found here: https://alembic.sqlalchemy.org/en/latest/autogenerate.html
 

backend/alembic/versions/18b5b2524446_add_is_clarification_to_chat_message.py

@@ -1,29 +0,0 @@
-"""add is_clarification to chat_message
-
-Revision ID: 18b5b2524446
-Revises: 87c52ec39f84
-Create Date: 2025-01-16
-
-"""
-
-from alembic import op
-import sqlalchemy as sa
-
-# revision identifiers, used by Alembic.
-revision = "18b5b2524446"
-down_revision = "87c52ec39f84"
-branch_labels = None
-depends_on = None
-
-
-def upgrade() -> None:
-    op.add_column(
-        "chat_message",
-        sa.Column(
-            "is_clarification", sa.Boolean(), nullable=False, server_default="false"
-        ),
-    )
-
-
-def downgrade() -> None:
-    op.drop_column("chat_message", "is_clarification")

backend/alembic/versions/1f2a3b4c5d6e_add_internet_search_and_content_providers.py

@@ -1,89 +0,0 @@
-"""add internet search and content provider tables
-
-Revision ID: 1f2a3b4c5d6e
-Revises: 9drpiiw74ljy
-Create Date: 2025-11-10 19:45:00.000000
-
-"""
-
-from alembic import op
-import sqlalchemy as sa
-from sqlalchemy.dialects import postgresql
-
-
-# revision identifiers, used by Alembic.
-revision = "1f2a3b4c5d6e"
-down_revision = "9drpiiw74ljy"
-branch_labels = None
-depends_on = None
-
-
-def upgrade() -> None:
-    op.create_table(
-        "internet_search_provider",
-        sa.Column("id", sa.Integer(), primary_key=True),
-        sa.Column("name", sa.String(), nullable=False, unique=True),
-        sa.Column("provider_type", sa.String(), nullable=False),
-        sa.Column("api_key", sa.LargeBinary(), nullable=True),
-        sa.Column("config", postgresql.JSONB(astext_type=sa.Text()), nullable=True),
-        sa.Column(
-            "is_active", sa.Boolean(), nullable=False, server_default=sa.text("false")
-        ),
-        sa.Column(
-            "time_created",
-            sa.DateTime(timezone=True),
-            nullable=False,
-            server_default=sa.text("now()"),
-        ),
-        sa.Column(
-            "time_updated",
-            sa.DateTime(timezone=True),
-            nullable=False,
-            server_default=sa.text("now()"),
-        ),
-    )
-    op.create_index(
-        "ix_internet_search_provider_is_active",
-        "internet_search_provider",
-        ["is_active"],
-    )
-
-    op.create_table(
-        "internet_content_provider",
-        sa.Column("id", sa.Integer(), primary_key=True),
-        sa.Column("name", sa.String(), nullable=False, unique=True),
-        sa.Column("provider_type", sa.String(), nullable=False),
-        sa.Column("api_key", sa.LargeBinary(), nullable=True),
-        sa.Column("config", postgresql.JSONB(astext_type=sa.Text()), nullable=True),
-        sa.Column(
-            "is_active", sa.Boolean(), nullable=False, server_default=sa.text("false")
-        ),
-        sa.Column(
-            "time_created",
-            sa.DateTime(timezone=True),
-            nullable=False,
-            server_default=sa.text("now()"),
-        ),
-        sa.Column(
-            "time_updated",
-            sa.DateTime(timezone=True),
-            nullable=False,
-            server_default=sa.text("now()"),
-        ),
-    )
-    op.create_index(
-        "ix_internet_content_provider_is_active",
-        "internet_content_provider",
-        ["is_active"],
-    )
-
-
-def downgrade() -> None:
-    op.drop_index(
-        "ix_internet_content_provider_is_active", table_name="internet_content_provider"
-    )
-    op.drop_table("internet_content_provider")
-    op.drop_index(
-        "ix_internet_search_provider_is_active", table_name="internet_search_provider"
-    )
-    op.drop_table("internet_search_provider")

backend/alembic/versions/2acdef638fc2_add_switchover_type_field.py

@@ -1,72 +0,0 @@
-"""add switchover_type field and remove background_reindex_enabled
-
-Revision ID: 2acdef638fc2
-Revises: a4f23d6b71c8
-Create Date: 2025-01-XX XX:XX:XX.XXXXXX
-
-"""
-
-from alembic import op
-import sqlalchemy as sa
-
-from onyx.db.enums import SwitchoverType
-
-
-# revision identifiers, used by Alembic.
-revision = "2acdef638fc2"
-down_revision = "a4f23d6b71c8"
-branch_labels = None
-depends_on = None
-
-
-def upgrade() -> None:
-    # Add switchover_type column with default value of REINDEX
-    op.add_column(
-        "search_settings",
-        sa.Column(
-            "switchover_type",
-            sa.Enum(SwitchoverType, native_enum=False),
-            nullable=False,
-            server_default=SwitchoverType.REINDEX.value,
-        ),
-    )
-
-    # Migrate existing data: set switchover_type based on background_reindex_enabled
-    # REINDEX where background_reindex_enabled=True, INSTANT where False
-    op.execute(
-        """
-        UPDATE search_settings
-        SET switchover_type = CASE
-            WHEN background_reindex_enabled = true THEN 'REINDEX'
-            ELSE 'INSTANT'
-        END
-        """
-    )
-
-    # Remove the background_reindex_enabled column (replaced by switchover_type)
-    op.drop_column("search_settings", "background_reindex_enabled")
-
-
-def downgrade() -> None:
-    # Re-add the background_reindex_enabled column with default value of True
-    op.add_column(
-        "search_settings",
-        sa.Column(
-            "background_reindex_enabled",
-            sa.Boolean(),
-            nullable=False,
-            server_default="true",
-        ),
-    )
-    # Set background_reindex_enabled based on switchover_type
-    op.execute(
-        """
-        UPDATE search_settings
-        SET background_reindex_enabled = CASE
-            WHEN switchover_type = 'INSTANT' THEN false
-            ELSE true
-        END
-        """
-    )
-    # Remove the switchover_type column
-    op.drop_column("search_settings", "switchover_type")

backend/alembic/versions/3c9a65f1207f_seed_exa_provider_from_env.py

@@ -1,89 +0,0 @@
-"""seed_exa_provider_from_env
-
-Revision ID: 3c9a65f1207f
-Revises: 1f2a3b4c5d6e
-Create Date: 2025-11-20 19:18:00.000000
-
-"""
-
-from __future__ import annotations
-
-import os
-
-from alembic import op
-import sqlalchemy as sa
-from sqlalchemy.dialects import postgresql
-from dotenv import load_dotenv, find_dotenv
-
-from onyx.utils.encryption import encrypt_string_to_bytes
-
-revision = "3c9a65f1207f"
-down_revision = "1f2a3b4c5d6e"
-branch_labels = None
-depends_on = None
-
-
-EXA_PROVIDER_NAME = "Exa"
-
-
-def _get_internet_search_table(metadata: sa.MetaData) -> sa.Table:
-    return sa.Table(
-        "internet_search_provider",
-        metadata,
-        sa.Column("id", sa.Integer, primary_key=True),
-        sa.Column("name", sa.String),
-        sa.Column("provider_type", sa.String),
-        sa.Column("api_key", sa.LargeBinary),
-        sa.Column("config", postgresql.JSONB),
-        sa.Column("is_active", sa.Boolean),
-        sa.Column(
-            "time_created",
-            sa.DateTime(timezone=True),
-            nullable=False,
-            server_default=sa.text("now()"),
-        ),
-        sa.Column(
-            "time_updated",
-            sa.DateTime(timezone=True),
-            nullable=False,
-            server_default=sa.text("now()"),
-        ),
-    )
-
-
-def upgrade() -> None:
-    load_dotenv(find_dotenv())
-
-    exa_api_key = os.environ.get("EXA_API_KEY")
-    if not exa_api_key:
-        return
-
-    bind = op.get_bind()
-    metadata = sa.MetaData()
-    table = _get_internet_search_table(metadata)
-
-    existing = bind.execute(
-        sa.select(table.c.id).where(table.c.name == EXA_PROVIDER_NAME)
-    ).first()
-    if existing:
-        return
-
-    encrypted_key = encrypt_string_to_bytes(exa_api_key)
-
-    has_active_provider = bind.execute(
-        sa.select(table.c.id).where(table.c.is_active.is_(True))
-    ).first()
-
-    bind.execute(
-        table.insert().values(
-            name=EXA_PROVIDER_NAME,
-            provider_type="exa",
-            api_key=encrypted_key,
-            config=None,
-            is_active=not bool(has_active_provider),
-        )
-    )
-
-
-def downgrade() -> None:
-    return

backend/alembic/versions/4f8a2b3c1d9e_add_open_url_tool.py

@@ -1,104 +0,0 @@
-"""add_open_url_tool
-
-Revision ID: 4f8a2b3c1d9e
-Revises: a852cbe15577
-Create Date: 2025-11-24 12:00:00.000000
-
-"""
-
-from alembic import op
-import sqlalchemy as sa
-
-
-# revision identifiers, used by Alembic.
-revision = "4f8a2b3c1d9e"
-down_revision = "a852cbe15577"
-branch_labels = None
-depends_on = None
-
-
-OPEN_URL_TOOL = {
-    "name": "OpenURLTool",
-    "display_name": "Open URL",
-    "description": (
-        "The Open URL Action allows the agent to fetch and read contents of web pages."
-    ),
-    "in_code_tool_id": "OpenURLTool",
-    "enabled": True,
-}
-
-
-def upgrade() -> None:
-    conn = op.get_bind()
-
-    # Check if tool already exists
-    existing = conn.execute(
-        sa.text("SELECT id FROM tool WHERE in_code_tool_id = :in_code_tool_id"),
-        {"in_code_tool_id": OPEN_URL_TOOL["in_code_tool_id"]},
-    ).fetchone()
-
-    if existing:
-        tool_id = existing[0]
-        # Update existing tool
-        conn.execute(
-            sa.text(
-                """
-                UPDATE tool
-                SET name = :name,
-                    display_name = :display_name,
-                    description = :description
-                WHERE in_code_tool_id = :in_code_tool_id
-                """
-            ),
-            OPEN_URL_TOOL,
-        )
-    else:
-        # Insert new tool
-        conn.execute(
-            sa.text(
-                """
-                INSERT INTO tool (name, display_name, description, in_code_tool_id, enabled)
-                VALUES (:name, :display_name, :description, :in_code_tool_id, :enabled)
-                """
-            ),
-            OPEN_URL_TOOL,
-        )
-        # Get the newly inserted tool's id
-        result = conn.execute(
-            sa.text("SELECT id FROM tool WHERE in_code_tool_id = :in_code_tool_id"),
-            {"in_code_tool_id": OPEN_URL_TOOL["in_code_tool_id"]},
-        ).fetchone()
-        tool_id = result[0]  # type: ignore
-
-    # Associate the tool with all existing personas
-    # Get all persona IDs
-    persona_ids = conn.execute(sa.text("SELECT id FROM persona")).fetchall()
-
-    for (persona_id,) in persona_ids:
-        # Check if association already exists
-        exists = conn.execute(
-            sa.text(
-                """
-                SELECT 1 FROM persona__tool
-                WHERE persona_id = :persona_id AND tool_id = :tool_id
-                """
-            ),
-            {"persona_id": persona_id, "tool_id": tool_id},
-        ).fetchone()
-
-        if not exists:
-            conn.execute(
-                sa.text(
-                    """
-                    INSERT INTO persona__tool (persona_id, tool_id)
-                    VALUES (:persona_id, :tool_id)
-                    """
-                ),
-                {"persona_id": persona_id, "tool_id": tool_id},
-            )
-
-
-def downgrade() -> None:
-    # We don't remove the tool on downgrade since it's fine to have it around.
-    # If we upgrade again, it will be a no-op.
-    pass

backend/alembic/versions/5e6f7a8b9c0d_update_default_persona_prompt.py

@@ -1,55 +0,0 @@
-"""update_default_persona_prompt
-
-Revision ID: 5e6f7a8b9c0d
-Revises: 4f8a2b3c1d9e
-Create Date: 2025-11-30 12:00:00.000000
-
-"""
-
-from alembic import op
-import sqlalchemy as sa
-
-
-# revision identifiers, used by Alembic.
-revision = "5e6f7a8b9c0d"
-down_revision = "4f8a2b3c1d9e"
-branch_labels = None
-depends_on = None
-
-
-DEFAULT_PERSONA_ID = 0
-
-# ruff: noqa: E501, W605 start
-DEFAULT_SYSTEM_PROMPT = """
-You are a highly capable, thoughtful, and precise assistant. Your goal is to deeply understand the user's intent, ask clarifying questions when needed, think step-by-step through complex problems, provide clear and accurate answers, and proactively anticipate helpful follow-up information. Always prioritize being truthful, nuanced, insightful, and efficient.
-
-The current date is [[CURRENT_DATETIME]].{citation_reminder_or_empty}
-
-# Response Style
-You use different text styles, bolding, emojis (sparingly), block quotes, and other formatting to make your responses more readable and engaging.
-You use proper Markdown and LaTeX to format your responses for math, scientific, and chemical formulas, symbols, etc.: '$$\\n[expression]\\n$$' for standalone cases and '\\( [expression] \\)' when inline.
-For code you prefer to use Markdown and specify the language.
-You can use horizontal rules (---) to separate sections of your responses.
-You can use Markdown tables to format your responses for data, lists, and other structured information.
-""".lstrip()
-# ruff: noqa: E501, W605 end
-
-
-def upgrade() -> None:
-    conn = op.get_bind()
-    conn.execute(
-        sa.text(
-            """
-            UPDATE persona
-            SET system_prompt = :system_prompt
-            WHERE id = :persona_id
-            """
-        ),
-        {"system_prompt": DEFAULT_SYSTEM_PROMPT, "persona_id": DEFAULT_PERSONA_ID},
-    )
-
-
-def downgrade() -> None:
-    # We don't revert the system prompt on downgrade since we don't know
-    # what the previous value was. The new prompt is a reasonable default.
-    pass

backend/alembic/versions/6436661d5b65_add_created_at_in_project_userfile.py

@@ -1,44 +0,0 @@
-"""add_created_at_in_project_userfile
-
-Revision ID: 6436661d5b65
-Revises: c7e9f4a3b2d1
-Create Date: 2025-11-24 11:50:24.536052
-
-"""
-
-from alembic import op
-import sqlalchemy as sa
-
-# revision identifiers, used by Alembic.
-revision = "6436661d5b65"
-down_revision = "c7e9f4a3b2d1"
-branch_labels = None
-depends_on = None
-
-
-def upgrade() -> None:
-    # Add created_at column to project__user_file table
-    op.add_column(
-        "project__user_file",
-        sa.Column(
-            "created_at",
-            sa.DateTime(timezone=True),
-            server_default=sa.text("now()"),
-            nullable=False,
-        ),
-    )
-    # Add composite index on (project_id, created_at DESC)
-    op.create_index(
-        "ix_project__user_file_project_id_created_at",
-        "project__user_file",
-        ["project_id", sa.text("created_at DESC")],
-    )
-
-
-def downgrade() -> None:
-    # Remove composite index on (project_id, created_at)
-    op.drop_index(
-        "ix_project__user_file_project_id_created_at", table_name="project__user_file"
-    )
-    # Remove created_at column from project__user_file table
-    op.drop_column("project__user_file", "created_at")

backend/alembic/versions/7bd55f264e1b_add_display_name_to_model_configuration.py

@@ -1,27 +0,0 @@
-"""Add display_name to model_configuration
-
-Revision ID: 7bd55f264e1b
-Revises: e8f0d2a38171
-Create Date: 2025-12-04
-
-"""
-
-from alembic import op
-import sqlalchemy as sa
-
-# revision identifiers, used by Alembic.
-revision = "7bd55f264e1b"
-down_revision = "e8f0d2a38171"
-branch_labels = None
-depends_on = None
-
-
-def upgrade() -> None:
-    op.add_column(
-        "model_configuration",
-        sa.Column("display_name", sa.String(), nullable=True),
-    )
-
-
-def downgrade() -> None:
-    op.drop_column("model_configuration", "display_name")

backend/alembic/versions/87c52ec39f84_update_default_system_prompt.py

@@ -1,55 +0,0 @@
-"""update_default_system_prompt
-
-Revision ID: 87c52ec39f84
-Revises: 7bd55f264e1b
-Create Date: 2025-12-05 15:54:06.002452
-
-"""
-
-from alembic import op
-import sqlalchemy as sa
-
-
-# revision identifiers, used by Alembic.
-revision = "87c52ec39f84"
-down_revision = "7bd55f264e1b"
-branch_labels = None
-depends_on = None
-
-
-DEFAULT_PERSONA_ID = 0
-
-# ruff: noqa: E501, W605 start
-DEFAULT_SYSTEM_PROMPT = """
-You are a highly capable, thoughtful, and precise assistant. Your goal is to deeply understand the user's intent, ask clarifying questions when needed, think step-by-step through complex problems, provide clear and accurate answers, and proactively anticipate helpful follow-up information. Always prioritize being truthful, nuanced, insightful, and efficient.
-
-The current date is [[CURRENT_DATETIME]].[[CITATION_GUIDANCE]]
-
-# Response Style
-You use different text styles, bolding, emojis (sparingly), block quotes, and other formatting to make your responses more readable and engaging.
-You use proper Markdown and LaTeX to format your responses for math, scientific, and chemical formulas, symbols, etc.: '$$\\n[expression]\\n$$' for standalone cases and '\\( [expression] \\)' when inline.
-For code you prefer to use Markdown and specify the language.
-You can use horizontal rules (---) to separate sections of your responses.
-You can use Markdown tables to format your responses for data, lists, and other structured information.
-""".lstrip()
-# ruff: noqa: E501, W605 end
-
-
-def upgrade() -> None:
-    conn = op.get_bind()
-    conn.execute(
-        sa.text(
-            """
-            UPDATE persona
-            SET system_prompt = :system_prompt
-            WHERE id = :persona_id
-            """
-        ),
-        {"system_prompt": DEFAULT_SYSTEM_PROMPT, "persona_id": DEFAULT_PERSONA_ID},
-    )
-
-
-def downgrade() -> None:
-    # We don't revert the system prompt on downgrade since we don't know
-    # what the previous value was. The new prompt is a reasonable default.
-    pass

backend/alembic/versions/9drpiiw74ljy_add_config_to_federated_connector.py

@@ -1,97 +0,0 @@
-"""add config to federated_connector
-
-Revision ID: 9drpiiw74ljy
-Revises: 2acdef638fc2
-Create Date: 2025-11-03 12:00:00.000000
-
-"""
-
-from alembic import op
-import sqlalchemy as sa
-from sqlalchemy.dialects import postgresql
-
-# revision identifiers, used by Alembic.
-revision = "9drpiiw74ljy"
-down_revision = "2acdef638fc2"
-branch_labels = None
-depends_on = None
-
-
-def upgrade() -> None:
-    connection = op.get_bind()
-
-    # Check if column already exists in current schema
-    result = connection.execute(
-        sa.text(
-            """
-            SELECT column_name
-            FROM information_schema.columns
-            WHERE table_schema = current_schema()
-            AND table_name = 'federated_connector'
-            AND column_name = 'config'
-            """
-        )
-    )
-    column_exists = result.fetchone() is not None
-
-    # Add config column with default empty object (only if it doesn't exist)
-    if not column_exists:
-        op.add_column(
-            "federated_connector",
-            sa.Column(
-                "config", postgresql.JSONB(), nullable=False, server_default="{}"
-            ),
-        )
-
-    # Data migration: Single bulk update for all Slack connectors
-    connection.execute(
-        sa.text(
-            """
-            WITH connector_configs AS (
-                SELECT
-                    fc.id as connector_id,
-                    CASE
-                        WHEN fcds.entities->'channels' IS NOT NULL
-                            AND jsonb_typeof(fcds.entities->'channels') = 'array'
-                            AND jsonb_array_length(fcds.entities->'channels') > 0
-                        THEN
-                            jsonb_build_object(
-                                'channels', fcds.entities->'channels',
-                                'search_all_channels', false
-                            ) ||
-                            CASE
-                                WHEN fcds.entities->'include_dm' IS NOT NULL
-                                THEN jsonb_build_object('include_dm', fcds.entities->'include_dm')
-                                ELSE '{}'::jsonb
-                            END
-                        ELSE
-                            jsonb_build_object('search_all_channels', true) ||
-                            CASE
-                                WHEN fcds.entities->'include_dm' IS NOT NULL
-                                THEN jsonb_build_object('include_dm', fcds.entities->'include_dm')
-                                ELSE '{}'::jsonb
-                            END
-                    END as config
-                FROM federated_connector fc
-                LEFT JOIN LATERAL (
-                    SELECT entities
-                    FROM federated_connector__document_set
-                    WHERE federated_connector_id = fc.id
-                    AND entities IS NOT NULL
-                    ORDER BY id
-                    LIMIT 1
-                ) fcds ON true
-                WHERE fc.source = 'FEDERATED_SLACK'
-                AND fcds.entities IS NOT NULL
-            )
-            UPDATE federated_connector fc
-            SET config = cc.config
-            FROM connector_configs cc
-            WHERE fc.id = cc.connector_id
-            """
-        )
-    )
-
-
-def downgrade() -> None:
-    op.drop_column("federated_connector", "config")

backend/alembic/versions/a01bf2971c5d_update_default_tool_descriptions.py

@@ -1,62 +0,0 @@
-"""update_default_tool_descriptions
-
-Revision ID: a01bf2971c5d
-Revises: 87c52ec39f84
-Create Date: 2025-12-16 15:21:25.656375
-
-"""
-
-from alembic import op
-import sqlalchemy as sa
-
-
-# revision identifiers, used by Alembic.
-revision = "a01bf2971c5d"
-down_revision = "18b5b2524446"
-branch_labels = None
-depends_on = None
-
-# new tool descriptions (12/2025)
-TOOL_DESCRIPTIONS = {
-    "SearchTool": "The Search Action allows the agent to search through connected knowledge to help build an answer.",
-    "ImageGenerationTool": (
-        "The Image Generation Action allows the agent to use DALL-E 3 or GPT-IMAGE-1 to generate images. "
-        "The action will be used when the user asks the agent to generate an image."
-    ),
-    "WebSearchTool": (
-        "The Web Search Action allows the agent "
-        "to perform internet searches for up-to-date information."
-    ),
-    "KnowledgeGraphTool": (
-        "The Knowledge Graph Search Action allows the agent to search the "
-        "Knowledge Graph for information. This tool can (for now) only be active in the KG Beta Agent, "
-        "and it requires the Knowledge Graph to be enabled."
-    ),
-    "OktaProfileTool": (
-        "The Okta Profile Action allows the agent to fetch the current user's information from Okta. "
-        "This may include the user's name, email, phone number, address, and other details such as their "
-        "manager and direct reports."
-    ),
-}
-
-
-def upgrade() -> None:
-    conn = op.get_bind()
-    conn.execute(sa.text("BEGIN"))
-
-    try:
-        for tool_id, description in TOOL_DESCRIPTIONS.items():
-            conn.execute(
-                sa.text(
-                    "UPDATE tool SET description = :description WHERE in_code_tool_id = :tool_id"
-                ),
-                {"description": description, "tool_id": tool_id},
-            )
-        conn.execute(sa.text("COMMIT"))
-    except Exception as e:
-        conn.execute(sa.text("ROLLBACK"))
-        raise e
-
-
-def downgrade() -> None:
-    pass

backend/alembic/versions/a4f23d6b71c8_add_llm_provider_persona_restrictions.py

@@ -1,61 +0,0 @@
-"""add llm provider persona restrictions
-
-Revision ID: a4f23d6b71c8
-Revises: 5e1c073d48a3
-Create Date: 2025-10-21 00:00:00.000000
-
-"""
-
-from alembic import op
-import sqlalchemy as sa
-
-
-# revision identifiers, used by Alembic.
-revision = "a4f23d6b71c8"
-down_revision = "5e1c073d48a3"
-branch_labels = None
-depends_on = None
-
-
-def upgrade() -> None:
-    op.create_table(
-        "llm_provider__persona",
-        sa.Column("llm_provider_id", sa.Integer(), nullable=False),
-        sa.Column("persona_id", sa.Integer(), nullable=False),
-        sa.ForeignKeyConstraint(
-            ["llm_provider_id"], ["llm_provider.id"], ondelete="CASCADE"
-        ),
-        sa.ForeignKeyConstraint(["persona_id"], ["persona.id"], ondelete="CASCADE"),
-        sa.PrimaryKeyConstraint("llm_provider_id", "persona_id"),
-    )
-    op.create_index(
-        "ix_llm_provider__persona_llm_provider_id",
-        "llm_provider__persona",
-        ["llm_provider_id"],
-    )
-    op.create_index(
-        "ix_llm_provider__persona_persona_id",
-        "llm_provider__persona",
-        ["persona_id"],
-    )
-    op.create_index(
-        "ix_llm_provider__persona_composite",
-        "llm_provider__persona",
-        ["persona_id", "llm_provider_id"],
-    )
-
-
-def downgrade() -> None:
-    op.drop_index(
-        "ix_llm_provider__persona_composite",
-        table_name="llm_provider__persona",
-    )
-    op.drop_index(
-        "ix_llm_provider__persona_persona_id",
-        table_name="llm_provider__persona",
-    )
-    op.drop_index(
-        "ix_llm_provider__persona_llm_provider_id",
-        table_name="llm_provider__persona",
-    )
-    op.drop_table("llm_provider__persona")

backend/alembic/versions/a852cbe15577_new_chat_history.py

@@ -1,417 +0,0 @@
-"""New Chat History
-
-Revision ID: a852cbe15577
-Revises: 6436661d5b65
-Create Date: 2025-11-08 15:16:37.781308
-
-"""
-
-from alembic import op
-import sqlalchemy as sa
-from sqlalchemy.dialects import postgresql
-
-# revision identifiers, used by Alembic.
-revision = "a852cbe15577"
-down_revision = "6436661d5b65"
-branch_labels = None
-depends_on = None
-
-
-def upgrade() -> None:
-    # 1. Drop old research/agent tables (CASCADE handles dependencies)
-    op.execute("DROP TABLE IF EXISTS research_agent_iteration_sub_step CASCADE")
-    op.execute("DROP TABLE IF EXISTS research_agent_iteration CASCADE")
-    op.execute("DROP TABLE IF EXISTS agent__sub_query__search_doc CASCADE")
-    op.execute("DROP TABLE IF EXISTS agent__sub_query CASCADE")
-    op.execute("DROP TABLE IF EXISTS agent__sub_question CASCADE")
-
-    # 2. ChatMessage table changes
-    # Rename columns and add FKs
-    op.alter_column(
-        "chat_message", "parent_message", new_column_name="parent_message_id"
-    )
-    op.create_foreign_key(
-        "fk_chat_message_parent_message_id",
-        "chat_message",
-        "chat_message",
-        ["parent_message_id"],
-        ["id"],
-    )
-    op.alter_column(
-        "chat_message",
-        "latest_child_message",
-        new_column_name="latest_child_message_id",
-    )
-    op.create_foreign_key(
-        "fk_chat_message_latest_child_message_id",
-        "chat_message",
-        "chat_message",
-        ["latest_child_message_id"],
-        ["id"],
-    )
-
-    # Add new column
-    op.add_column(
-        "chat_message", sa.Column("reasoning_tokens", sa.Text(), nullable=True)
-    )
-
-    # Drop old columns
-    op.drop_column("chat_message", "rephrased_query")
-    op.drop_column("chat_message", "alternate_assistant_id")
-    op.drop_column("chat_message", "overridden_model")
-    op.drop_column("chat_message", "is_agentic")
-    op.drop_column("chat_message", "refined_answer_improvement")
-    op.drop_column("chat_message", "research_type")
-    op.drop_column("chat_message", "research_plan")
-    op.drop_column("chat_message", "research_answer_purpose")
-
-    # 3. ToolCall table changes
-    # Drop the unique constraint first
-    op.drop_constraint("uq_tool_call_message_id", "tool_call", type_="unique")
-
-    # Delete orphaned tool_call rows (those without valid chat_message)
-    op.execute(
-        "DELETE FROM tool_call WHERE message_id NOT IN (SELECT id FROM chat_message)"
-    )
-
-    # Add chat_session_id as nullable first, populate, then make NOT NULL
-    op.add_column(
-        "tool_call",
-        sa.Column("chat_session_id", postgresql.UUID(as_uuid=True), nullable=True),
-    )
-
-    # Populate chat_session_id from the related chat_message
-    op.execute(
-        """
-        UPDATE tool_call
-        SET chat_session_id = chat_message.chat_session_id
-        FROM chat_message
-        WHERE tool_call.message_id = chat_message.id
-    """
-    )
-
-    # Now make it NOT NULL and add FK
-    op.alter_column("tool_call", "chat_session_id", nullable=False)
-    op.create_foreign_key(
-        "fk_tool_call_chat_session_id",
-        "tool_call",
-        "chat_session",
-        ["chat_session_id"],
-        ["id"],
-        ondelete="CASCADE",
-    )
-
-    # Rename message_id and make nullable, recreate FK with CASCADE
-    op.drop_constraint("tool_call_message_id_fkey", "tool_call", type_="foreignkey")
-    op.alter_column(
-        "tool_call",
-        "message_id",
-        new_column_name="parent_chat_message_id",
-        nullable=True,
-    )
-    op.create_foreign_key(
-        "fk_tool_call_parent_chat_message_id",
-        "tool_call",
-        "chat_message",
-        ["parent_chat_message_id"],
-        ["id"],
-        ondelete="CASCADE",
-    )
-
-    # Add parent_tool_call_id with FK
-    op.add_column(
-        "tool_call", sa.Column("parent_tool_call_id", sa.Integer(), nullable=True)
-    )
-    op.create_foreign_key(
-        "fk_tool_call_parent_tool_call_id",
-        "tool_call",
-        "tool_call",
-        ["parent_tool_call_id"],
-        ["id"],
-        ondelete="CASCADE",
-    )
-
-    # Add other new columns
-    op.add_column(
-        "tool_call",
-        sa.Column("turn_number", sa.Integer(), nullable=False, server_default="0"),
-    )
-    op.add_column(
-        "tool_call",
-        sa.Column("tool_call_id", sa.String(), nullable=False, server_default=""),
-    )
-    op.add_column("tool_call", sa.Column("reasoning_tokens", sa.Text(), nullable=True))
-    op.add_column(
-        "tool_call",
-        sa.Column("tool_call_tokens", sa.Integer(), nullable=False, server_default="0"),
-    )
-    op.add_column(
-        "tool_call",
-        sa.Column("generated_images", postgresql.JSONB(), nullable=True),
-    )
-
-    # Rename columns
-    op.alter_column(
-        "tool_call", "tool_arguments", new_column_name="tool_call_arguments"
-    )
-    op.alter_column("tool_call", "tool_result", new_column_name="tool_call_response")
-
-    # Change tool_call_response type from JSONB to Text
-    op.execute(
-        """
-        ALTER TABLE tool_call
-        ALTER COLUMN tool_call_response TYPE TEXT
-        USING tool_call_response::text
-    """
-    )
-
-    # Drop old columns
-    op.drop_column("tool_call", "tool_name")
-
-    # 4. Create new association table
-    op.create_table(
-        "tool_call__search_doc",
-        sa.Column("tool_call_id", sa.Integer(), nullable=False),
-        sa.Column("search_doc_id", sa.Integer(), nullable=False),
-        sa.ForeignKeyConstraint(["tool_call_id"], ["tool_call.id"], ondelete="CASCADE"),
-        sa.ForeignKeyConstraint(
-            ["search_doc_id"], ["search_doc.id"], ondelete="CASCADE"
-        ),
-        sa.PrimaryKeyConstraint("tool_call_id", "search_doc_id"),
-    )
-
-    # 5. Persona table change
-    op.add_column(
-        "persona",
-        sa.Column(
-            "replace_base_system_prompt",
-            sa.Boolean(),
-            nullable=False,
-            server_default="false",
-        ),
-    )
-
-
-def downgrade() -> None:
-    # Reverse persona changes
-    op.drop_column("persona", "replace_base_system_prompt")
-
-    # Drop new association table
-    op.drop_table("tool_call__search_doc")
-
-    # Reverse ToolCall changes
-    op.add_column(
-        "tool_call",
-        sa.Column("tool_name", sa.String(), nullable=False, server_default=""),
-    )
-
-    # Change tool_call_response back to JSONB
-    op.execute(
-        """
-        ALTER TABLE tool_call
-        ALTER COLUMN tool_call_response TYPE JSONB
-        USING tool_call_response::jsonb
-    """
-    )
-
-    op.alter_column("tool_call", "tool_call_response", new_column_name="tool_result")
-    op.alter_column(
-        "tool_call", "tool_call_arguments", new_column_name="tool_arguments"
-    )
-
-    op.drop_column("tool_call", "generated_images")
-    op.drop_column("tool_call", "tool_call_tokens")
-    op.drop_column("tool_call", "reasoning_tokens")
-    op.drop_column("tool_call", "tool_call_id")
-    op.drop_column("tool_call", "turn_number")
-
-    op.drop_constraint(
-        "fk_tool_call_parent_tool_call_id", "tool_call", type_="foreignkey"
-    )
-    op.drop_column("tool_call", "parent_tool_call_id")
-
-    op.drop_constraint(
-        "fk_tool_call_parent_chat_message_id", "tool_call", type_="foreignkey"
-    )
-    op.alter_column(
-        "tool_call",
-        "parent_chat_message_id",
-        new_column_name="message_id",
-        nullable=False,
-    )
-    op.create_foreign_key(
-        "tool_call_message_id_fkey",
-        "tool_call",
-        "chat_message",
-        ["message_id"],
-        ["id"],
-    )
-
-    op.drop_constraint("fk_tool_call_chat_session_id", "tool_call", type_="foreignkey")
-    op.drop_column("tool_call", "chat_session_id")
-
-    op.create_unique_constraint("uq_tool_call_message_id", "tool_call", ["message_id"])
-
-    # Reverse ChatMessage changes
-    # Note: research_answer_purpose and research_type were originally String columns,
-    # not Enum types (see migrations 5ae8240accb3 and f8a9b2c3d4e5)
-    op.add_column(
-        "chat_message",
-        sa.Column("research_answer_purpose", sa.String(), nullable=True),
-    )
-    op.add_column(
-        "chat_message", sa.Column("research_plan", postgresql.JSONB(), nullable=True)
-    )
-    op.add_column(
-        "chat_message",
-        sa.Column("research_type", sa.String(), nullable=True),
-    )
-    op.add_column(
-        "chat_message",
-        sa.Column("refined_answer_improvement", sa.Boolean(), nullable=True),
-    )
-    op.add_column(
-        "chat_message",
-        sa.Column("is_agentic", sa.Boolean(), nullable=False, server_default="false"),
-    )
-    op.add_column(
-        "chat_message", sa.Column("overridden_model", sa.String(), nullable=True)
-    )
-    op.add_column(
-        "chat_message", sa.Column("alternate_assistant_id", sa.Integer(), nullable=True)
-    )
-    op.add_column(
-        "chat_message", sa.Column("rephrased_query", sa.Text(), nullable=True)
-    )
-
-    op.drop_column("chat_message", "reasoning_tokens")
-
-    op.drop_constraint(
-        "fk_chat_message_latest_child_message_id", "chat_message", type_="foreignkey"
-    )
-    op.alter_column(
-        "chat_message",
-        "latest_child_message_id",
-        new_column_name="latest_child_message",
-    )
-
-    op.drop_constraint(
-        "fk_chat_message_parent_message_id", "chat_message", type_="foreignkey"
-    )
-    op.alter_column(
-        "chat_message", "parent_message_id", new_column_name="parent_message"
-    )
-
-    # Recreate agent sub question and sub query tables
-    op.create_table(
-        "agent__sub_question",
-        sa.Column("id", sa.Integer(), primary_key=True),
-        sa.Column("primary_question_id", sa.Integer(), nullable=False),
-        sa.Column("chat_session_id", postgresql.UUID(as_uuid=True), nullable=False),
-        sa.Column("sub_question", sa.Text(), nullable=False),
-        sa.Column("level", sa.Integer(), nullable=False),
-        sa.Column("level_question_num", sa.Integer(), nullable=False),
-        sa.Column(
-            "time_created",
-            sa.DateTime(timezone=True),
-            server_default=sa.text("now()"),
-            nullable=False,
-        ),
-        sa.Column("sub_answer", sa.Text(), nullable=False),
-        sa.Column("sub_question_doc_results", postgresql.JSONB(), nullable=False),
-        sa.ForeignKeyConstraint(
-            ["primary_question_id"], ["chat_message.id"], ondelete="CASCADE"
-        ),
-        sa.ForeignKeyConstraint(["chat_session_id"], ["chat_session.id"]),
-        sa.PrimaryKeyConstraint("id"),
-    )
-
-    op.create_table(
-        "agent__sub_query",
-        sa.Column("id", sa.Integer(), primary_key=True),
-        sa.Column("parent_question_id", sa.Integer(), nullable=False),
-        sa.Column("chat_session_id", postgresql.UUID(as_uuid=True), nullable=False),
-        sa.Column("sub_query", sa.Text(), nullable=False),
-        sa.Column(
-            "time_created",
-            sa.DateTime(timezone=True),
-            server_default=sa.text("now()"),
-            nullable=False,
-        ),
-        sa.ForeignKeyConstraint(
-            ["parent_question_id"], ["agent__sub_question.id"], ondelete="CASCADE"
-        ),
-        sa.ForeignKeyConstraint(["chat_session_id"], ["chat_session.id"]),
-        sa.PrimaryKeyConstraint("id"),
-    )
-
-    op.create_table(
-        "agent__sub_query__search_doc",
-        sa.Column("sub_query_id", sa.Integer(), nullable=False),
-        sa.Column("search_doc_id", sa.Integer(), nullable=False),
-        sa.ForeignKeyConstraint(
-            ["sub_query_id"], ["agent__sub_query.id"], ondelete="CASCADE"
-        ),
-        sa.ForeignKeyConstraint(["search_doc_id"], ["search_doc.id"]),
-        sa.PrimaryKeyConstraint("sub_query_id", "search_doc_id"),
-    )
-
-    # Recreate research agent tables
-    op.create_table(
-        "research_agent_iteration",
-        sa.Column("id", sa.Integer(), autoincrement=True, nullable=False),
-        sa.Column("primary_question_id", sa.Integer(), nullable=False),
-        sa.Column("iteration_nr", sa.Integer(), nullable=False),
-        sa.Column(
-            "created_at",
-            sa.DateTime(timezone=True),
-            server_default=sa.text("now()"),
-            nullable=False,
-        ),
-        sa.Column("purpose", sa.String(), nullable=True),
-        sa.Column("reasoning", sa.String(), nullable=True),
-        sa.ForeignKeyConstraint(
-            ["primary_question_id"], ["chat_message.id"], ondelete="CASCADE"
-        ),
-        sa.PrimaryKeyConstraint("id"),
-        sa.UniqueConstraint(
-            "primary_question_id",
-            "iteration_nr",
-            name="_research_agent_iteration_unique_constraint",
-        ),
-    )
-
-    op.create_table(
-        "research_agent_iteration_sub_step",
-        sa.Column("id", sa.Integer(), autoincrement=True, nullable=False),
-        sa.Column("primary_question_id", sa.Integer(), nullable=False),
-        sa.Column("iteration_nr", sa.Integer(), nullable=False),
-        sa.Column("iteration_sub_step_nr", sa.Integer(), nullable=False),
-        sa.Column(
-            "created_at",
-            sa.DateTime(timezone=True),
-            server_default=sa.text("now()"),
-            nullable=False,
-        ),
-        sa.Column("sub_step_instructions", sa.String(), nullable=True),
-        sa.Column("sub_step_tool_id", sa.Integer(), nullable=True),
-        sa.Column("reasoning", sa.String(), nullable=True),
-        sa.Column("sub_answer", sa.String(), nullable=True),
-        sa.Column("cited_doc_results", postgresql.JSONB(), nullable=False),
-        sa.Column("claims", postgresql.JSONB(), nullable=True),
-        sa.Column("is_web_fetch", sa.Boolean(), nullable=True),
-        sa.Column("queries", postgresql.JSONB(), nullable=True),
-        sa.Column("generated_images", postgresql.JSONB(), nullable=True),
-        sa.Column("additional_data", postgresql.JSONB(), nullable=True),
-        sa.Column("file_ids", postgresql.JSONB(), nullable=True),
-        sa.ForeignKeyConstraint(
-            ["primary_question_id", "iteration_nr"],
-            [
-                "research_agent_iteration.primary_question_id",
-                "research_agent_iteration.iteration_nr",
-            ],
-            ondelete="CASCADE",
-        ),
-        sa.ForeignKeyConstraint(["sub_step_tool_id"], ["tool.id"], ondelete="SET NULL"),
-        sa.PrimaryKeyConstraint("id"),
-    )

backend/alembic/versions/c7e9f4a3b2d1_add_python_tool.py

@@ -1,73 +0,0 @@
-"""add_python_tool
-
-Revision ID: c7e9f4a3b2d1
-Revises: 3c9a65f1207f
-Create Date: 2025-11-08 00:00:00.000000
-
-"""
-
-from alembic import op
-import sqlalchemy as sa
-from sqlalchemy.dialects import postgresql
-
-
-# revision identifiers, used by Alembic.
-revision = "c7e9f4a3b2d1"
-down_revision = "3c9a65f1207f"
-branch_labels = None
-depends_on = None
-
-
-def upgrade() -> None:
-    """Add PythonTool to built-in tools"""
-    conn = op.get_bind()
-
-    conn.execute(
-        sa.text(
-            """
-            INSERT INTO tool (name, display_name, description, in_code_tool_id, enabled)
-            VALUES (:name, :display_name, :description, :in_code_tool_id, :enabled)
-            """
-        ),
-        {
-            "name": "PythonTool",
-            # in the UI, call it `Code Interpreter` since this is a well known term for this tool
-            "display_name": "Code Interpreter",
-            "description": (
-                "The Code Interpreter Action allows the assistant to execute "
-                "Python code in a secure, isolated environment for data analysis, "
-                "computation, visualization, and file processing."
-            ),
-            "in_code_tool_id": "PythonTool",
-            "enabled": True,
-        },
-    )
-
-    # needed to store files generated by the python tool
-    op.add_column(
-        "research_agent_iteration_sub_step",
-        sa.Column(
-            "file_ids",
-            postgresql.JSONB(astext_type=sa.Text()),
-            nullable=True,
-        ),
-    )
-
-
-def downgrade() -> None:
-    """Remove PythonTool from built-in tools"""
-    conn = op.get_bind()
-
-    conn.execute(
-        sa.text(
-            """
-            DELETE FROM tool
-            WHERE in_code_tool_id = :in_code_tool_id
-            """
-        ),
-        {
-            "in_code_tool_id": "PythonTool",
-        },
-    )
-
-    op.drop_column("research_agent_iteration_sub_step", "file_ids")

backend/alembic/versions/e8f0d2a38171_add_status_to_mcp_server_and_make_auth_.py

@@ -1,115 +0,0 @@
-"""add status to mcp server and make auth fields nullable
-
-Revision ID: e8f0d2a38171
-Revises: ed9e44312505
-Create Date: 2025-11-28 11:15:37.667340
-
-"""
-
-from alembic import op
-import sqlalchemy as sa
-from onyx.db.enums import (  # type: ignore[import-untyped]
-    MCPTransport,
-    MCPAuthenticationType,
-    MCPAuthenticationPerformer,
-    MCPServerStatus,
-)
-
-# revision identifiers, used by Alembic.
-revision = "e8f0d2a38171"
-down_revision = "ed9e44312505"
-branch_labels = None
-depends_on = None
-
-
-def upgrade() -> None:
-    # Make auth fields nullable
-    op.alter_column(
-        "mcp_server",
-        "transport",
-        existing_type=sa.Enum(MCPTransport, name="mcp_transport", native_enum=False),
-        nullable=True,
-    )
-
-    op.alter_column(
-        "mcp_server",
-        "auth_type",
-        existing_type=sa.Enum(
-            MCPAuthenticationType, name="mcp_authentication_type", native_enum=False
-        ),
-        nullable=True,
-    )
-
-    op.alter_column(
-        "mcp_server",
-        "auth_performer",
-        existing_type=sa.Enum(
-            MCPAuthenticationPerformer,
-            name="mcp_authentication_performer",
-            native_enum=False,
-        ),
-        nullable=True,
-    )
-
-    # Add status column with default
-    op.add_column(
-        "mcp_server",
-        sa.Column(
-            "status",
-            sa.Enum(MCPServerStatus, name="mcp_server_status", native_enum=False),
-            nullable=False,
-            server_default="CREATED",
-        ),
-    )
-
-    # For existing records, mark status as CONNECTED
-    bind = op.get_bind()
-    bind.execute(
-        sa.text(
-            """
-        UPDATE mcp_server
-        SET status = 'CONNECTED'
-        WHERE status != 'CONNECTED'
-        and admin_connection_config_id IS NOT NULL
-        """
-        )
-    )
-
-
-def downgrade() -> None:
-    # Remove status column
-    op.drop_column("mcp_server", "status")
-
-    # Make auth fields non-nullable (set defaults first)
-    op.execute(
-        "UPDATE mcp_server SET transport = 'STREAMABLE_HTTP' WHERE transport IS NULL"
-    )
-    op.execute("UPDATE mcp_server SET auth_type = 'NONE' WHERE auth_type IS NULL")
-    op.execute(
-        "UPDATE mcp_server SET auth_performer = 'ADMIN' WHERE auth_performer IS NULL"
-    )
-
-    op.alter_column(
-        "mcp_server",
-        "transport",
-        existing_type=sa.Enum(MCPTransport, name="mcp_transport", native_enum=False),
-        nullable=False,
-    )
-    op.alter_column(
-        "mcp_server",
-        "auth_type",
-        existing_type=sa.Enum(
-            MCPAuthenticationType, name="mcp_authentication_type", native_enum=False
-        ),
-        nullable=False,
-    )
-    op.alter_column(
-        "mcp_server",
-        "auth_performer",
-        existing_type=sa.Enum(
-            MCPAuthenticationPerformer,
-            name="mcp_authentication_performer",
-            native_enum=False,
-        ),
-        nullable=False,
-    )

backend/alembic/versions/ed9e44312505_add_icon_name_field.py

@@ -1,34 +0,0 @@
-"""Add icon_name field
-
-Revision ID: ed9e44312505
-Revises: 5e6f7a8b9c0d
-Create Date: 2025-12-03 16:35:07.828393
-
-"""
-
-from alembic import op
-import sqlalchemy as sa
-
-# revision identifiers, used by Alembic.
-revision = "ed9e44312505"
-down_revision = "5e6f7a8b9c0d"
-branch_labels = None
-depends_on = None
-
-
-def upgrade() -> None:
-    # Add icon_name column
-    op.add_column("persona", sa.Column("icon_name", sa.String(), nullable=True))
-
-    # Remove old icon columns
-    op.drop_column("persona", "icon_shape")
-    op.drop_column("persona", "icon_color")
-
-
-def downgrade() -> None:
-    # Re-add old icon columns
-    op.add_column("persona", sa.Column("icon_color", sa.String(), nullable=True))
-    op.add_column("persona", sa.Column("icon_shape", sa.Integer(), nullable=True))
-
-    # Remove icon_name column
-    op.drop_column("persona", "icon_name")

backend/docker-bake.hcl

@@ -1,17 +1,5 @@
-group "default" {
-  targets = ["backend", "model-server"]
-}
-
-variable "BACKEND_REPOSITORY" {
-  default = "onyxdotapp/onyx-backend"
-}
-
-variable "MODEL_SERVER_REPOSITORY" {
-  default = "onyxdotapp/onyx-model-server"
-}
-
-variable "INTEGRATION_REPOSITORY" {
-  default = "onyxdotapp/onyx-integration"
+variable "REGISTRY" {
+  default = "onyxdotapp"
 }
 
 variable "TAG" {
@@ -21,22 +9,6 @@ variable "TAG" {
 target "backend" {
   context    = "."
   dockerfile = "Dockerfile"
-
-  cache-from = ["type=registry,ref=${BACKEND_REPOSITORY}:latest"]
-  cache-to   = ["type=inline"]
-
-  tags      = ["${BACKEND_REPOSITORY}:${TAG}"]
-}
-
-target "model-server" {
-  context = "."
-
-  dockerfile = "Dockerfile.model_server"
-
-  cache-from = ["type=registry,ref=${MODEL_SERVER_REPOSITORY}:latest"]
-  cache-to   = ["type=inline"]
-
-  tags      = ["${MODEL_SERVER_REPOSITORY}:${TAG}"]
 }
 
 target "integration" {
@@ -48,5 +20,5 @@ target "integration" {
     base = "target:backend"
   }
 
-  tags      = ["${INTEGRATION_REPOSITORY}:${TAG}"]
+  tags      = ["${REGISTRY}/integration-test-onyx-integration:${TAG}"]
 }

backend/ee/onyx/configs/app_configs.py

@@ -41,10 +41,6 @@ CONFLUENCE_ANONYMOUS_ACCESS_IS_PUBLIC = (
 JIRA_PERMISSION_DOC_SYNC_FREQUENCY = int(
     os.environ.get("JIRA_PERMISSION_DOC_SYNC_FREQUENCY") or 30 * 60
 )
-# In seconds, default is 30 minutes
-JIRA_PERMISSION_GROUP_SYNC_FREQUENCY = int(
-    os.environ.get("JIRA_PERMISSION_GROUP_SYNC_FREQUENCY") or 30 * 60
-)
 
 
 #####
@@ -128,8 +124,6 @@ SUPER_CLOUD_API_KEY = os.environ.get("SUPER_CLOUD_API_KEY", "api_key")
 POSTHOG_API_KEY = os.environ.get("POSTHOG_API_KEY") or "FooBar"
 POSTHOG_HOST = os.environ.get("POSTHOG_HOST") or "https://us.i.posthog.com"
 
-MARKETING_POSTHOG_API_KEY = os.environ.get("MARKETING_POSTHOG_API_KEY")
-
 HUBSPOT_TRACKING_URL = os.environ.get("HUBSPOT_TRACKING_URL")
 
 GATED_TENANTS_KEY = "gated_tenants"

backend/ee/onyx/db/analytics.py

@@ -199,7 +199,10 @@ def fetch_persona_message_analytics(
             ChatMessage.chat_session_id == ChatSession.id,
         )
         .where(
-            ChatSession.persona_id == persona_id,
+            or_(
+                ChatMessage.alternate_assistant_id == persona_id,
+                ChatSession.persona_id == persona_id,
+            ),
             ChatMessage.time_sent >= start,
             ChatMessage.time_sent <= end,
             ChatMessage.message_type == MessageType.ASSISTANT,
@@ -228,7 +231,10 @@ def fetch_persona_unique_users(
             ChatMessage.chat_session_id == ChatSession.id,
         )
         .where(
-            ChatSession.persona_id == persona_id,
+            or_(
+                ChatMessage.alternate_assistant_id == persona_id,
+                ChatSession.persona_id == persona_id,
+            ),
             ChatMessage.time_sent >= start,
             ChatMessage.time_sent <= end,
             ChatMessage.message_type == MessageType.ASSISTANT,
@@ -259,7 +265,10 @@ def fetch_assistant_message_analytics(
             ChatMessage.chat_session_id == ChatSession.id,
         )
         .where(
-            ChatSession.persona_id == assistant_id,
+            or_(
+                ChatMessage.alternate_assistant_id == assistant_id,
+                ChatSession.persona_id == assistant_id,
+            ),
             ChatMessage.time_sent >= start,
             ChatMessage.time_sent <= end,
             ChatMessage.message_type == MessageType.ASSISTANT,
@@ -290,7 +299,10 @@ def fetch_assistant_unique_users(
             ChatMessage.chat_session_id == ChatSession.id,
         )
         .where(
-            ChatSession.persona_id == assistant_id,
+            or_(
+                ChatMessage.alternate_assistant_id == assistant_id,
+                ChatSession.persona_id == assistant_id,
+            ),
             ChatMessage.time_sent >= start,
             ChatMessage.time_sent <= end,
             ChatMessage.message_type == MessageType.ASSISTANT,
@@ -320,7 +332,10 @@ def fetch_assistant_unique_users_total(
             ChatMessage.chat_session_id == ChatSession.id,
         )
         .where(
-            ChatSession.persona_id == assistant_id,
+            or_(
+                ChatMessage.alternate_assistant_id == assistant_id,
+                ChatSession.persona_id == assistant_id,
+            ),
             ChatMessage.time_sent >= start,
             ChatMessage.time_sent <= end,
             ChatMessage.message_type == MessageType.ASSISTANT,

backend/ee/onyx/db/usage_export.py

@@ -55,7 +55,18 @@ def get_empty_chat_messages_entries__paginated(
 
             # Get assistant name (from session persona, or alternate if specified)
             assistant_name = None
-            if chat_session.persona:
+            if message.alternate_assistant_id:
+                # If there's an alternate assistant, we need to fetch it
+                from onyx.db.models import Persona
+
+                alternate_persona = (
+                    db_session.query(Persona)
+                    .filter(Persona.id == message.alternate_assistant_id)
+                    .first()
+                )
+                if alternate_persona:
+                    assistant_name = alternate_persona.name
+            elif chat_session.persona:
                 assistant_name = chat_session.persona.name
 
             message_skeletons.append(

backend/ee/onyx/db/user_group.py

@@ -8,7 +8,6 @@ from sqlalchemy import func
 from sqlalchemy import Select
 from sqlalchemy import select
 from sqlalchemy import update
-from sqlalchemy.dialects.postgresql import insert
 from sqlalchemy.orm import Session
 
 from ee.onyx.server.user_group.models import SetCuratorRequest
@@ -363,29 +362,14 @@ def _check_user_group_is_modifiable(user_group: UserGroup) -> None:
 
 def _add_user__user_group_relationships__no_commit(
     db_session: Session, user_group_id: int, user_ids: list[UUID]
-) -> None:
-    """NOTE: does not commit the transaction.
-
-    This function is idempotent - it will skip users who are already in the group
-    to avoid duplicate key violations during concurrent operations or re-syncs.
-    Uses ON CONFLICT DO NOTHING to keep inserts atomic under concurrency.
-    """
-    if not user_ids:
-        return
-
-    insert_stmt = (
-        insert(User__UserGroup)
-        .values(
-            [
-                {"user_id": user_id, "user_group_id": user_group_id}
-                for user_id in user_ids
-            ]
-        )
-        .on_conflict_do_nothing(
-            index_elements=[User__UserGroup.user_group_id, User__UserGroup.user_id]
-        )
-    )
-    db_session.execute(insert_stmt)
+) -> list[User__UserGroup]:
+    """NOTE: does not commit the transaction."""
+    relationships = [
+        User__UserGroup(user_id=user_id, user_group_id=user_group_id)
+        for user_id in user_ids
+    ]
+    db_session.add_all(relationships)
+    return relationships
 
 
 def _add_user_group__cc_pair_relationships__no_commit(
@@ -597,48 +581,6 @@ def update_user_curator_relationship(
     db_session.commit()
 
 
-def add_users_to_user_group(
-    db_session: Session,
-    user: User | None,
-    user_group_id: int,
-    user_ids: list[UUID],
-) -> UserGroup:
-    db_user_group = fetch_user_group(db_session=db_session, user_group_id=user_group_id)
-    if db_user_group is None:
-        raise ValueError(f"UserGroup with id '{user_group_id}' not found")
-
-    missing_users = [
-        user_id for user_id in user_ids if fetch_user_by_id(db_session, user_id) is None
-    ]
-    if missing_users:
-        raise ValueError(
-            f"User(s) not found: {', '.join(str(user_id) for user_id in missing_users)}"
-        )
-
-    _check_user_group_is_modifiable(db_user_group)
-
-    current_user_ids = [user.id for user in db_user_group.users]
-    current_user_ids_set = set(current_user_ids)
-    new_user_ids = [
-        user_id for user_id in user_ids if user_id not in current_user_ids_set
-    ]
-
-    if not new_user_ids:
-        return db_user_group
-
-    user_group_update = UserGroupUpdate(
-        user_ids=current_user_ids + new_user_ids,
-        cc_pair_ids=[cc_pair.id for cc_pair in db_user_group.cc_pairs],
-    )
-
-    return update_user_group(
-        db_session=db_session,
-        user=user,
-        user_group_id=user_group_id,
-        user_group_update=user_group_update,
-    )
-
-
 def update_user_group(
     db_session: Session,
     user: User | None,
@@ -661,17 +603,6 @@ def update_user_group(
     added_user_ids = list(updated_user_ids - current_user_ids)
     removed_user_ids = list(current_user_ids - updated_user_ids)
 
-    if added_user_ids:
-        missing_users = [
-            user_id
-            for user_id in added_user_ids
-            if fetch_user_by_id(db_session, user_id) is None
-        ]
-        if missing_users:
-            raise ValueError(
-                f"User(s) not found: {', '.join(str(user_id) for user_id in missing_users)}"
-            )
-
     # LEAVING THIS HERE FOR NOW FOR GIVING DIFFERENT ROLES
     # ACCESS TO DIFFERENT PERMISSIONS
     # if (removed_user_ids or added_user_ids) and (

backend/ee/onyx/external_permissions/confluence/group_sync.py

@@ -3,15 +3,12 @@ from collections.abc import Generator
 from ee.onyx.db.external_perm import ExternalUserGroup
 from ee.onyx.external_permissions.confluence.constants import ALL_CONF_EMAILS_GROUP_NAME
 from onyx.background.error_logging import emit_background_error
-from onyx.configs.app_configs import CONFLUENCE_USE_ONYX_USERS_FOR_GROUP_SYNC
 from onyx.connectors.confluence.onyx_confluence import (
     get_user_email_from_username__server,
 )
 from onyx.connectors.confluence.onyx_confluence import OnyxConfluence
 from onyx.connectors.credentials_provider import OnyxDBCredentialsProvider
-from onyx.db.engine.sql_engine import get_session_with_current_tenant
 from onyx.db.models import ConnectorCredentialPair
-from onyx.db.users import get_all_users
 from onyx.utils.logger import setup_logger
 
 logger = setup_logger()
@@ -22,7 +19,7 @@ def _build_group_member_email_map(
 ) -> dict[str, set[str]]:
     group_member_emails: dict[str, set[str]] = {}
     for user in confluence_client.paginated_cql_user_retrieval():
-        logger.info(f"Processing groups for user: {user}")
+        logger.debug(f"Processing groups for user: {user}")
 
         email = user.email
         if not email:
@@ -34,8 +31,6 @@ def _build_group_member_email_map(
                     confluence_client=confluence_client,
                     user_name=user_name,
                 )
-            else:
-                logger.error(f"user result missing username field: {user}")
 
         if not email:
             # If we still don't have an email, skip this user
@@ -69,92 +64,6 @@ def _build_group_member_email_map(
     return group_member_emails
 
 
-def _build_group_member_email_map_from_onyx_users(
-    confluence_client: OnyxConfluence,
-) -> dict[str, set[str]]:
-    """Hacky, but it's the only way to do this as long as the
-    Confluence APIs are broken.
-
-    This is fixed in Confluence Data Center 10.1.0, so first choice
-    is to tell users to upgrade to 10.1.0.
-    https://jira.atlassian.com/browse/CONFSERVER-95999
-    """
-    with get_session_with_current_tenant() as db_session:
-        # don't include external since they are handled by the "through confluence"
-        # user fetching mechanism
-        user_emails = [
-            user.email for user in get_all_users(db_session, include_external=False)
-        ]
-
-    def _infer_username_from_email(email: str) -> str:
-        return email.split("@")[0]
-
-    group_member_emails: dict[str, set[str]] = {}
-    for email in user_emails:
-        logger.info(f"Processing groups for user with email: {email}")
-        try:
-            user_name = _infer_username_from_email(email)
-            response = confluence_client.get_user_details_by_username(user_name)
-            user_key = response.get("userKey")
-            if not user_key:
-                logger.error(f"User key not found for user with email {email}")
-                continue
-
-            all_users_groups: set[str] = set()
-            for group in confluence_client.paginated_groups_by_user_retrieval(user_key):
-                # group name uniqueness is enforced by Confluence, so we can use it as a group ID
-                group_id = group["name"]
-                group_member_emails.setdefault(group_id, set()).add(email)
-                all_users_groups.add(group_id)
-
-            if not all_users_groups:
-                msg = f"No groups found for user with email: {email}"
-                logger.error(msg)
-            else:
-                logger.info(
-                    f"Found groups {all_users_groups} for user with email {email}"
-                )
-        except Exception:
-            logger.exception(f"Error getting user details for user with email {email}")
-
-    return group_member_emails
-
-
-def _build_final_group_to_member_email_map(
-    confluence_client: OnyxConfluence,
-    cc_pair_id: int,
-    # if set, will infer confluence usernames from onyx users in addition to using the
-    # confluence users API. This is a hacky workaround for the fact that the Confluence
-    # users API is broken before Confluence Data Center 10.1.0.
-    use_onyx_users: bool = CONFLUENCE_USE_ONYX_USERS_FOR_GROUP_SYNC,
-) -> dict[str, set[str]]:
-    group_to_member_email_map = _build_group_member_email_map(
-        confluence_client=confluence_client,
-        cc_pair_id=cc_pair_id,
-    )
-    group_to_member_email_map_from_onyx_users = (
-        (
-            _build_group_member_email_map_from_onyx_users(
-                confluence_client=confluence_client,
-            )
-        )
-        if use_onyx_users
-        else {}
-    )
-
-    all_group_ids = set(group_to_member_email_map.keys()) | set(
-        group_to_member_email_map_from_onyx_users.keys()
-    )
-    final_group_to_member_email_map = {}
-    for group_id in all_group_ids:
-        group_member_emails = group_to_member_email_map.get(
-            group_id, set()
-        ) | group_to_member_email_map_from_onyx_users.get(group_id, set())
-        final_group_to_member_email_map[group_id] = group_member_emails
-
-    return final_group_to_member_email_map
-
-
 def confluence_group_sync(
     tenant_id: str,
     cc_pair: ConnectorCredentialPair,
@@ -178,12 +87,13 @@ def confluence_group_sync(
     confluence_client._probe_connection(**probe_kwargs)
     confluence_client._initialize_connection(**final_kwargs)
 
-    group_to_member_email_map = _build_final_group_to_member_email_map(
-        confluence_client, cc_pair.id
+    group_member_email_map = _build_group_member_email_map(
+        confluence_client=confluence_client,
+        cc_pair_id=cc_pair.id,
     )
 
     all_found_emails = set()
-    for group_id, group_member_emails in group_to_member_email_map.items():
+    for group_id, group_member_emails in group_member_email_map.items():
         yield (
             ExternalUserGroup(
                 id=group_id,

backend/ee/onyx/external_permissions/jira/group_sync.py

@@ -1,136 +0,0 @@
-from collections.abc import Generator
-
-from jira import JIRA
-
-from ee.onyx.db.external_perm import ExternalUserGroup
-from onyx.connectors.jira.utils import build_jira_client
-from onyx.db.models import ConnectorCredentialPair
-from onyx.utils.logger import setup_logger
-
-logger = setup_logger()
-
-
-def _get_jira_group_members_email(
-    jira_client: JIRA,
-    group_name: str,
-) -> list[str]:
-    """Get all member emails for a Jira group.
-
-    Filters out app accounts (bots, integrations) and only returns real user emails.
-    """
-    emails: list[str] = []
-
-    try:
-        # group_members returns an OrderedDict of account_id -> member_info
-        members = jira_client.group_members(group=group_name)
-
-        if not members:
-            logger.warning(f"No members found for group {group_name}")
-            return emails
-
-        for account_id, member_info in members.items():
-            # member_info is a dict with keys like 'fullname', 'email', 'active'
-            email = member_info.get("email")
-
-            # Skip "hidden" emails - these are typically app accounts
-            if email and email != "hidden":
-                emails.append(email)
-            else:
-                # For cloud, we might need to fetch user details separately
-                try:
-                    user = jira_client.user(id=account_id)
-
-                    # Skip app accounts (bots, integrations, etc.)
-                    if hasattr(user, "accountType") and user.accountType == "app":
-                        logger.info(
-                            f"Skipping app account {account_id} for group {group_name}"
-                        )
-                        continue
-
-                    if hasattr(user, "emailAddress") and user.emailAddress:
-                        emails.append(user.emailAddress)
-                    else:
-                        logger.warning(f"User {account_id} has no email address")
-                except Exception as e:
-                    logger.warning(
-                        f"Could not fetch email for user {account_id} in group {group_name}: {e}"
-                    )
-
-    except Exception as e:
-        logger.error(f"Error fetching members for group {group_name}: {e}")
-
-    return emails
-
-
-def _build_group_member_email_map(
-    jira_client: JIRA,
-) -> dict[str, set[str]]:
-    """Build a map of group names to member emails."""
-    group_member_emails: dict[str, set[str]] = {}
-
-    try:
-        # Get all groups from Jira - returns a list of group name strings
-        group_names = jira_client.groups()
-
-        if not group_names:
-            logger.warning("No groups found in Jira")
-            return group_member_emails
-
-        logger.info(f"Found {len(group_names)} groups in Jira")
-
-        for group_name in group_names:
-            if not group_name:
-                continue
-
-            member_emails = _get_jira_group_members_email(
-                jira_client=jira_client,
-                group_name=group_name,
-            )
-
-            if member_emails:
-                group_member_emails[group_name] = set(member_emails)
-                logger.debug(
-                    f"Found {len(member_emails)} members for group {group_name}"
-                )
-            else:
-                logger.debug(f"No members found for group {group_name}")
-
-    except Exception as e:
-        logger.error(f"Error building group member email map: {e}")
-
-    return group_member_emails
-
-
-def jira_group_sync(
-    tenant_id: str,
-    cc_pair: ConnectorCredentialPair,
-) -> Generator[ExternalUserGroup, None, None]:
-    """
-    Sync Jira groups and their members.
-
-    This function fetches all groups from Jira and yields ExternalUserGroup
-    objects containing the group ID and member emails.
-    """
-    jira_base_url = cc_pair.connector.connector_specific_config.get("jira_base_url", "")
-    scoped_token = cc_pair.connector.connector_specific_config.get(
-        "scoped_token", False
-    )
-
-    if not jira_base_url:
-        raise ValueError("No jira_base_url found in connector config")
-
-    jira_client = build_jira_client(
-        credentials=cc_pair.credential.credential_json,
-        jira_base=jira_base_url,
-        scoped_token=scoped_token,
-    )
-
-    group_member_email_map = _build_group_member_email_map(jira_client=jira_client)
-    if not group_member_email_map:
-        raise ValueError(f"No groups with members found for cc_pair_id={cc_pair.id}")
-
-    for group_id, group_member_emails in group_member_email_map.items():
-        yield ExternalUserGroup(
-            id=group_id,
-            user_emails=list(group_member_emails),
-        )

backend/ee/onyx/external_permissions/jira/page_access.py

@@ -16,10 +16,6 @@ HolderMap = dict[str, list[Holder]]
 logger = setup_logger()
 
 
-def _get_role_id(holder: Holder) -> str | None:
-    return holder.get("value") or holder.get("parameter")
-
-
 def _build_holder_map(permissions: list[dict]) -> dict[str, list[Holder]]:
     """
     A "Holder" in JIRA is a person / entity who "holds" the corresponding permission.
@@ -114,137 +110,80 @@ def _get_user_emails(user_holders: list[Holder]) -> list[str]:
     return emails
 
 
-def _get_user_emails_and_groups_from_project_roles(
+def _get_user_emails_from_project_roles(
     jira_client: JIRA,
     jira_project: str,
     project_role_holders: list[Holder],
-) -> tuple[list[str], list[str]]:
-    """
-    Get user emails and group names from project roles.
-    Returns a tuple of (emails, group_names).
-    """
-    # Get role IDs - Cloud uses "value", Data Center uses "parameter"
-    role_ids = []
-    for holder in project_role_holders:
-        role_id = _get_role_id(holder)
-        if role_id:
-            role_ids.append(role_id)
-        else:
-            logger.warning(f"No value or parameter in projectRole holder: {holder}")
-
+) -> list[str]:
+    # NOTE (@raunakab) a `parallel_yield` may be helpful here...?
     roles = [
-        jira_client.project_role(project=jira_project, id=role_id)
-        for role_id in role_ids
+        jira_client.project_role(project=jira_project, id=project_role_holder["value"])
+        for project_role_holder in project_role_holders
+        if "value" in project_role_holder
     ]
 
     emails = []
-    groups = []
 
     for role in roles:
         if not hasattr(role, "actors"):
-            logger.warning(f"Project role {role} has no actors attribute")
             continue
 
         for actor in role.actors:
-            # Handle group actors
-            if hasattr(actor, "actorGroup"):
-                group_name = getattr(actor.actorGroup, "name", None) or getattr(
-                    actor.actorGroup, "displayName", None
-                )
-                if group_name:
-                    groups.append(group_name)
+            if not hasattr(actor, "actorUser") or not hasattr(
+                actor.actorUser, "accountId"
+            ):
                 continue
 
-            # Handle user actors
-            if hasattr(actor, "actorUser"):
-                account_id = getattr(actor.actorUser, "accountId", None)
-                if not account_id:
-                    logger.error(f"No accountId in actorUser: {actor.actorUser}")
-                    continue
-
-                user = jira_client.user(id=account_id)
-                if not hasattr(user, "accountType") or user.accountType != "atlassian":
-                    logger.info(
-                        f"Skipping user {account_id} because it is not an atlassian user"
-                    )
-                    continue
-
-                if not hasattr(user, "emailAddress"):
-                    msg = f"User's email address was not able to be retrieved;  {actor.actorUser.accountId=}"
-                    if hasattr(user, "displayName"):
-                        msg += f" {actor.displayName=}"
-                    logger.warning(msg)
-                    continue
-
-                emails.append(user.emailAddress)
+            user = jira_client.user(id=actor.actorUser.accountId)
+            if not hasattr(user, "accountType") or user.accountType != "atlassian":
                 continue
 
-            logger.debug(f"Skipping actor type: {actor}")
+            if not hasattr(user, "emailAddress"):
+                msg = f"User's email address was not able to be retrieved;  {actor.actorUser.accountId=}"
+                if hasattr(user, "displayName"):
+                    msg += f" {actor.displayName=}"
+                logger.warn(msg)
+                continue
 
-    return emails, groups
+            emails.append(user.emailAddress)
+
+    return emails
 
 
 def _build_external_access_from_holder_map(
     jira_client: JIRA, jira_project: str, holder_map: HolderMap
 ) -> ExternalAccess:
     """
-    Build ExternalAccess from the holder map.
-
-    Holder types handled:
-        - "anyone": Public project, anyone can access
-        - "applicationRole": All users with a Jira license can access (treated as public)
-        - "user": Specific users with access
-        - "projectRole": Project roles containing users and/or groups
-        - "group": Groups directly assigned in the permission scheme
+    # Note:
+        If the `holder_map` contains an instance of "anyone", then this is a public JIRA project.
+        Otherwise, we fetch the "projectRole"s (i.e., the user-groups in JIRA speak), and the user emails.
     """
-    # Public access - anyone can view
+
     if "anyone" in holder_map:
         return ExternalAccess(
             external_user_emails=set(), external_user_group_ids=set(), is_public=True
         )
 
-    # applicationRole means all users with a Jira license can access - treat as public
-    if "applicationRole" in holder_map:
-        return ExternalAccess(
-            external_user_emails=set(), external_user_group_ids=set(), is_public=True
-        )
-
-    # Get emails from explicit user holders
     user_emails = (
         _get_user_emails(user_holders=holder_map["user"])
         if "user" in holder_map
         else []
     )
-
-    # Get emails and groups from project roles
-    project_role_user_emails: list[str] = []
-    project_role_groups: list[str] = []
-    if "projectRole" in holder_map:
-        project_role_user_emails, project_role_groups = (
-            _get_user_emails_and_groups_from_project_roles(
-                jira_client=jira_client,
-                jira_project=jira_project,
-                project_role_holders=holder_map["projectRole"],
-            )
+    project_role_user_emails = (
+        _get_user_emails_from_project_roles(
+            jira_client=jira_client,
+            jira_project=jira_project,
+            project_role_holders=holder_map["projectRole"],
         )
-
-    # Get groups directly assigned in permission scheme (common in Data Center)
-    # Format: {'type': 'group', 'parameter': 'group-name', 'expand': 'group'}
-    direct_groups: list[str] = []
-    if "group" in holder_map:
-        for group_holder in holder_map["group"]:
-            group_name = _get_role_id(group_holder)
-            if group_name:
-                direct_groups.append(group_name)
-            else:
-                logger.error(f"No parameter/value in group holder: {group_holder}")
+        if "projectRole" in holder_map
+        else []
+    )
 
     external_user_emails = set(user_emails + project_role_user_emails)
-    external_user_group_ids = set(project_role_groups + direct_groups)
 
     return ExternalAccess(
         external_user_emails=external_user_emails,
-        external_user_group_ids=external_user_group_ids,
+        external_user_group_ids=set(),
         is_public=False,
     )
 
@@ -258,11 +197,9 @@ def get_project_permissions(
     )
 
     if not hasattr(project_permissions, "permissions"):
-        logger.error(f"Project {jira_project} has no permissions attribute")
         return None
 
     if not isinstance(project_permissions.permissions, list):
-        logger.error(f"Project {jira_project} permissions is not a list")
         return None
 
     holder_map = _build_holder_map(permissions=project_permissions.permissions)

backend/ee/onyx/external_permissions/sharepoint/permission_utils.py

@@ -15,7 +15,6 @@ from ee.onyx.db.external_perm import ExternalUserGroup
 from onyx.access.models import ExternalAccess
 from onyx.access.utils import build_ext_group_name_for_onyx
 from onyx.configs.constants import DocumentSource
-from onyx.connectors.sharepoint.connector import SHARED_DOCUMENTS_MAP_REVERSE
 from onyx.connectors.sharepoint.connector import sleep_and_retry
 from onyx.utils.logger import setup_logger
 
@@ -512,8 +511,8 @@ def get_external_access_from_sharepoint(
                 f"Failed to get SharePoint list item ID for item {drive_item.id}"
             )
 
-        if drive_name in SHARED_DOCUMENTS_MAP_REVERSE:
-            drive_name = SHARED_DOCUMENTS_MAP_REVERSE[drive_name]
+        if drive_name == "Shared Documents":
+            drive_name = "Documents"
 
         item = client_context.web.lists.get_by_title(drive_name).items.get_by_id(
             item_id

backend/ee/onyx/external_permissions/sync_params.py

@@ -11,7 +11,6 @@ from ee.onyx.configs.app_configs import GITHUB_PERMISSION_DOC_SYNC_FREQUENCY
 from ee.onyx.configs.app_configs import GITHUB_PERMISSION_GROUP_SYNC_FREQUENCY
 from ee.onyx.configs.app_configs import GOOGLE_DRIVE_PERMISSION_GROUP_SYNC_FREQUENCY
 from ee.onyx.configs.app_configs import JIRA_PERMISSION_DOC_SYNC_FREQUENCY
-from ee.onyx.configs.app_configs import JIRA_PERMISSION_GROUP_SYNC_FREQUENCY
 from ee.onyx.configs.app_configs import SHAREPOINT_PERMISSION_DOC_SYNC_FREQUENCY
 from ee.onyx.configs.app_configs import SHAREPOINT_PERMISSION_GROUP_SYNC_FREQUENCY
 from ee.onyx.configs.app_configs import SLACK_PERMISSION_DOC_SYNC_FREQUENCY
@@ -24,7 +23,6 @@ from ee.onyx.external_permissions.gmail.doc_sync import gmail_doc_sync
 from ee.onyx.external_permissions.google_drive.doc_sync import gdrive_doc_sync
 from ee.onyx.external_permissions.google_drive.group_sync import gdrive_group_sync
 from ee.onyx.external_permissions.jira.doc_sync import jira_doc_sync
-from ee.onyx.external_permissions.jira.group_sync import jira_group_sync
 from ee.onyx.external_permissions.perm_sync_types import CensoringFuncType
 from ee.onyx.external_permissions.perm_sync_types import DocSyncFuncType
 from ee.onyx.external_permissions.perm_sync_types import FetchAllDocumentsFunction
@@ -112,11 +110,6 @@ _SOURCE_TO_SYNC_CONFIG: dict[DocumentSource, SyncConfig] = {
             doc_sync_func=jira_doc_sync,
             initial_index_should_sync=True,
         ),
-        group_sync_config=GroupSyncConfig(
-            group_sync_frequency=JIRA_PERMISSION_GROUP_SYNC_FREQUENCY,
-            group_sync_func=jira_group_sync,
-            group_sync_is_cc_pair_agnostic=True,
-        ),
     ),
     # Groups are not needed for Slack.
     # All channel access is done at the individual user level.

backend/ee/onyx/main.py

@@ -23,7 +23,7 @@ from ee.onyx.server.query_and_chat.chat_backend import (
     router as chat_router,
 )
 from ee.onyx.server.query_and_chat.query_backend import (
-    basic_router as ee_query_router,
+    basic_router as query_router,
 )
 from ee.onyx.server.query_history.api import router as query_history_router
 from ee.onyx.server.reporting.usage_export_api import router as usage_export_router
@@ -48,9 +48,6 @@ from onyx.main import include_auth_router_with_prefix
 from onyx.main import include_router_with_global_prefix_prepended
 from onyx.main import lifespan as lifespan_base
 from onyx.main import use_route_function_names_as_operation_ids
-from onyx.server.query_and_chat.query_backend import (
-    basic_router as query_router,
-)
 from onyx.utils.logger import setup_logger
 from onyx.utils.variable_functionality import global_version
 from shared_configs.configs import MULTI_TENANT
@@ -122,7 +119,6 @@ def get_application() -> FastAPI:
     include_router_with_global_prefix_prepended(application, query_history_router)
     # EE only backend APIs
     include_router_with_global_prefix_prepended(application, query_router)
-    include_router_with_global_prefix_prepended(application, ee_query_router)
     include_router_with_global_prefix_prepended(application, chat_router)
     include_router_with_global_prefix_prepended(application, standard_answer_router)
     include_router_with_global_prefix_prepended(application, ee_oauth_router)

backend/ee/onyx/server/query_and_chat/chat_backend.py

@@ -8,10 +8,12 @@ from ee.onyx.server.query_and_chat.models import (
     BasicCreateChatMessageWithHistoryRequest,
 )
 from onyx.auth.users import current_user
-from onyx.chat.chat_utils import create_chat_history_chain
+from onyx.chat.chat_utils import combine_message_thread
+from onyx.chat.chat_utils import create_chat_chain
 from onyx.chat.models import ChatBasicResponse
 from onyx.chat.process_message import gather_stream
 from onyx.chat.process_message import stream_chat_message_objects
+from onyx.configs.chat_configs import CHAT_TARGET_CHUNK_PERCENTAGE
 from onyx.configs.constants import MessageType
 from onyx.context.search.models import OptionalSearchSetting
 from onyx.context.search.models import RetrievalDetails
@@ -22,6 +24,7 @@ from onyx.db.engine.sql_engine import get_session
 from onyx.db.models import User
 from onyx.llm.factory import get_llms_for_persona
 from onyx.natural_language_processing.utils import get_tokenizer
+from onyx.secondary_llm_flows.query_expansion import thread_based_query_rephrase
 from onyx.server.query_and_chat.models import CreateChatMessageRequest
 from onyx.utils.logger import setup_logger
 
@@ -66,9 +69,9 @@ def handle_simplified_chat_message(
         chat_session_id = chat_message_req.chat_session_id
 
     try:
-        parent_message = create_chat_history_chain(
+        parent_message, _ = create_chat_chain(
             chat_session_id=chat_session_id, db_session=db_session
-        )[-1]
+        )
     except Exception:
         parent_message = get_or_create_root_message(
             chat_session_id=chat_session_id, db_session=db_session
@@ -158,13 +161,15 @@ def handle_send_message_simple_with_history(
         persona_id=req.persona_id,
     )
 
-    llm, _ = get_llms_for_persona(persona=chat_session.persona, user=user)
+    llm, _ = get_llms_for_persona(persona=chat_session.persona)
 
     llm_tokenizer = get_tokenizer(
         model_name=llm.config.model_name,
         provider_type=llm.config.model_provider,
     )
 
+    max_history_tokens = int(llm.config.max_input_tokens * CHAT_TARGET_CHUNK_PERCENTAGE)
+
     # Every chat Session begins with an empty root message
     root_message = get_or_create_root_message(
         chat_session_id=chat_session.id, db_session=db_session
@@ -183,6 +188,17 @@ def handle_send_message_simple_with_history(
         )
     db_session.commit()
 
+    history_str = combine_message_thread(
+        messages=msg_history,
+        max_tokens=max_history_tokens,
+        llm_tokenizer=llm_tokenizer,
+    )
+
+    rephrased_query = req.query_override or thread_based_query_rephrase(
+        user_query=query,
+        history_str=history_str,
+    )
+
     if req.retrieval_options is None and req.search_doc_ids is None:
         retrieval_options: RetrievalDetails | None = RetrievalDetails(
             run_search=OptionalSearchSetting.ALWAYS,
@@ -200,7 +216,7 @@ def handle_send_message_simple_with_history(
         retrieval_options=retrieval_options,
         # Simple API does not support reranking, hide complexity from user
         rerank_settings=None,
-        query_override=None,
+        query_override=rephrased_query,
         chunks_above=0,
         chunks_below=0,
         full_doc=req.full_doc,

backend/ee/onyx/server/query_and_chat/models.py

@@ -6,14 +6,18 @@ from pydantic import BaseModel
 from pydantic import Field
 from pydantic import model_validator
 
+from onyx.chat.models import PersonaOverrideConfig
+from onyx.chat.models import QADocsResponse
 from onyx.chat.models import ThreadMessage
 from onyx.configs.constants import DocumentSource
-from onyx.context.search.models import BaseFilters
-from onyx.context.search.models import BasicChunkRequest
+from onyx.context.search.enums import LLMEvaluationType
+from onyx.context.search.enums import SearchType
 from onyx.context.search.models import ChunkContext
-from onyx.context.search.models import InferenceChunk
+from onyx.context.search.models import RerankingDetails
 from onyx.context.search.models import RetrievalDetails
 from onyx.server.manage.models import StandardAnswer
+from onyx.server.query_and_chat.streaming_models import CitationInfo
+from onyx.server.query_and_chat.streaming_models import SubQuestionIdentifier
 
 
 class StandardAnswerRequest(BaseModel):
@@ -25,12 +29,14 @@ class StandardAnswerResponse(BaseModel):
     standard_answers: list[StandardAnswer] = Field(default_factory=list)
 
 
-class DocumentSearchRequest(BasicChunkRequest):
-    user_selected_filters: BaseFilters | None = None
-
-
-class DocumentSearchResponse(BaseModel):
-    top_documents: list[InferenceChunk]
+class DocumentSearchRequest(ChunkContext):
+    message: str
+    search_type: SearchType
+    retrieval_options: RetrievalDetails
+    recency_bias_multiplier: float = 1.0
+    evaluation_type: LLMEvaluationType
+    # None to use system defaults for reranking
+    rerank_settings: RerankingDetails | None = None
 
 
 class BasicCreateChatMessageRequest(ChunkContext):
@@ -90,17 +96,17 @@ class SimpleDoc(BaseModel):
     metadata: dict | None
 
 
-class AgentSubQuestion(BaseModel):
+class AgentSubQuestion(SubQuestionIdentifier):
     sub_question: str
     document_ids: list[str]
 
 
-class AgentAnswer(BaseModel):
+class AgentAnswer(SubQuestionIdentifier):
     answer: str
     answer_type: Literal["agent_sub_answer", "agent_level_answer"]
 
 
-class AgentSubQuery(BaseModel):
+class AgentSubQuery(SubQuestionIdentifier):
     sub_query: str
     query_id: int
 
@@ -146,3 +152,45 @@ class AgentSubQuery(BaseModel):
             sorted(level_question_dict.items(), key=lambda x: (x is None, x))
         )
         return sorted_dict
+
+
+class OneShotQARequest(ChunkContext):
+    # Supports simplier APIs that don't deal with chat histories or message edits
+    # Easier APIs to work with for developers
+    persona_override_config: PersonaOverrideConfig | None = None
+    persona_id: int | None = None
+
+    messages: list[ThreadMessage]
+    retrieval_options: RetrievalDetails = Field(default_factory=RetrievalDetails)
+    rerank_settings: RerankingDetails | None = None
+
+    # allows the caller to specify the exact search query they want to use
+    # can be used if the message sent to the LLM / query should not be the same
+    # will also disable Thread-based Rewording if specified
+    query_override: str | None = None
+
+    # If True, skips generating an AI response to the search query
+    skip_gen_ai_answer_generation: bool = False
+
+    # If True, uses agentic search instead of basic search
+    use_agentic_search: bool = False
+
+    @model_validator(mode="after")
+    def check_persona_fields(self) -> "OneShotQARequest":
+        if self.persona_override_config is None and self.persona_id is None:
+            raise ValueError("Exactly one of persona_config or persona_id must be set")
+        elif self.persona_override_config is not None and (self.persona_id is not None):
+            raise ValueError(
+                "If persona_override_config is set, persona_id cannot be set"
+            )
+        return self
+
+
+class OneShotQAResponse(BaseModel):
+    # This is built piece by piece, any of these can be None as the flow could break
+    answer: str | None = None
+    rephrase: str | None = None
+    citations: list[CitationInfo] | None = None
+    docs: QADocsResponse | None = None
+    error_msg: str | None = None
+    chat_message_id: int | None = None

backend/ee/onyx/server/query_and_chat/query_backend.py

@@ -1,23 +1,260 @@
+import json
+from collections.abc import Generator
+
 from fastapi import APIRouter
 from fastapi import Depends
 from fastapi import HTTPException
+from fastapi.responses import StreamingResponse
+from pydantic import BaseModel
 from sqlalchemy.orm import Session
 
 from ee.onyx.onyxbot.slack.handlers.handle_standard_answers import (
     oneoff_standard_answers,
 )
+from ee.onyx.server.query_and_chat.models import DocumentSearchRequest
+from ee.onyx.server.query_and_chat.models import OneShotQARequest
+from ee.onyx.server.query_and_chat.models import OneShotQAResponse
 from ee.onyx.server.query_and_chat.models import StandardAnswerRequest
 from ee.onyx.server.query_and_chat.models import StandardAnswerResponse
 from onyx.auth.users import current_user
+from onyx.chat.chat_utils import combine_message_thread
+from onyx.chat.chat_utils import prepare_chat_message_request
+from onyx.chat.models import AnswerStream
+from onyx.chat.models import PersonaOverrideConfig
+from onyx.chat.models import QADocsResponse
+from onyx.chat.process_message import gather_stream
+from onyx.chat.process_message import stream_chat_message_objects
+from onyx.configs.onyxbot_configs import MAX_THREAD_CONTEXT_PERCENTAGE
+from onyx.context.search.models import SavedSearchDocWithContent
+from onyx.context.search.models import SearchRequest
+from onyx.context.search.pipeline import SearchPipeline
+from onyx.context.search.utils import dedupe_documents
+from onyx.context.search.utils import drop_llm_indices
+from onyx.context.search.utils import relevant_sections_to_indices
 from onyx.db.engine.sql_engine import get_session
+from onyx.db.models import Persona
 from onyx.db.models import User
+from onyx.db.persona import get_persona_by_id
+from onyx.llm.factory import get_default_llms
+from onyx.llm.factory import get_llms_for_persona
+from onyx.llm.factory import get_main_llm_from_tuple
+from onyx.natural_language_processing.utils import get_tokenizer
+from onyx.server.query_and_chat.streaming_models import CitationInfo
+from onyx.server.utils import get_json_line
 from onyx.utils.logger import setup_logger
 
-logger = setup_logger()
 
+logger = setup_logger()
 basic_router = APIRouter(prefix="/query")
 
 
+class DocumentSearchResponse(BaseModel):
+    top_documents: list[SavedSearchDocWithContent]
+    llm_indices: list[int]
+
+
+@basic_router.post("/document-search")
+def handle_search_request(
+    search_request: DocumentSearchRequest,
+    user: User | None = Depends(current_user),
+    db_session: Session = Depends(get_session),
+) -> DocumentSearchResponse:
+    """Simple search endpoint, does not create a new message or records in the DB"""
+    query = search_request.message
+    logger.notice(f"Received document search query: {query}")
+
+    llm, fast_llm = get_default_llms()
+
+    search_pipeline = SearchPipeline(
+        search_request=SearchRequest(
+            query=query,
+            search_type=search_request.search_type,
+            human_selected_filters=search_request.retrieval_options.filters,
+            enable_auto_detect_filters=search_request.retrieval_options.enable_auto_detect_filters,
+            persona=None,  # For simplicity, default settings should be good for this search
+            offset=search_request.retrieval_options.offset,
+            limit=search_request.retrieval_options.limit,
+            rerank_settings=search_request.rerank_settings,
+            evaluation_type=search_request.evaluation_type,
+            chunks_above=search_request.chunks_above,
+            chunks_below=search_request.chunks_below,
+            full_doc=search_request.full_doc,
+        ),
+        user=user,
+        llm=llm,
+        fast_llm=fast_llm,
+        skip_query_analysis=False,
+        db_session=db_session,
+        bypass_acl=False,
+    )
+    top_sections = search_pipeline.reranked_sections
+    relevance_sections = search_pipeline.section_relevance
+    top_docs = [
+        SavedSearchDocWithContent(
+            document_id=section.center_chunk.document_id,
+            chunk_ind=section.center_chunk.chunk_id,
+            content=section.center_chunk.content,
+            semantic_identifier=section.center_chunk.semantic_identifier or "Unknown",
+            link=(
+                section.center_chunk.source_links.get(0)
+                if section.center_chunk.source_links
+                else None
+            ),
+            blurb=section.center_chunk.blurb,
+            source_type=section.center_chunk.source_type,
+            boost=section.center_chunk.boost,
+            hidden=section.center_chunk.hidden,
+            metadata=section.center_chunk.metadata,
+            score=section.center_chunk.score or 0.0,
+            match_highlights=section.center_chunk.match_highlights,
+            updated_at=section.center_chunk.updated_at,
+            primary_owners=section.center_chunk.primary_owners,
+            secondary_owners=section.center_chunk.secondary_owners,
+            is_internet=False,
+            db_doc_id=0,
+        )
+        for section in top_sections
+    ]
+
+    # Deduping happens at the last step to avoid harming quality by dropping content early on
+    deduped_docs = top_docs
+    dropped_inds = None
+
+    if search_request.retrieval_options.dedupe_docs:
+        deduped_docs, dropped_inds = dedupe_documents(top_docs)
+
+    llm_indices = relevant_sections_to_indices(
+        relevance_sections=relevance_sections, items=deduped_docs
+    )
+
+    if dropped_inds:
+        llm_indices = drop_llm_indices(
+            llm_indices=llm_indices,
+            search_docs=deduped_docs,
+            dropped_indices=dropped_inds,
+        )
+
+    return DocumentSearchResponse(top_documents=deduped_docs, llm_indices=llm_indices)
+
+
+def get_answer_stream(
+    query_request: OneShotQARequest,
+    user: User | None = Depends(current_user),
+    db_session: Session = Depends(get_session),
+) -> AnswerStream:
+    query = query_request.messages[0].message
+    logger.notice(f"Received query for Answer API: {query}")
+
+    if (
+        query_request.persona_override_config is None
+        and query_request.persona_id is None
+    ):
+        raise KeyError("Must provide persona ID or Persona Config")
+
+    persona_info: Persona | PersonaOverrideConfig | None = None
+    if query_request.persona_override_config is not None:
+        persona_info = query_request.persona_override_config
+    elif query_request.persona_id is not None:
+        persona_info = get_persona_by_id(
+            persona_id=query_request.persona_id,
+            user=user,
+            db_session=db_session,
+            is_for_edit=False,
+        )
+
+    llm = get_main_llm_from_tuple(get_llms_for_persona(persona_info))
+
+    llm_tokenizer = get_tokenizer(
+        model_name=llm.config.model_name,
+        provider_type=llm.config.model_provider,
+    )
+
+    max_history_tokens = int(
+        llm.config.max_input_tokens * MAX_THREAD_CONTEXT_PERCENTAGE
+    )
+
+    combined_message = combine_message_thread(
+        messages=query_request.messages,
+        max_tokens=max_history_tokens,
+        llm_tokenizer=llm_tokenizer,
+    )
+
+    # Also creates a new chat session
+    request = prepare_chat_message_request(
+        message_text=combined_message,
+        user=user,
+        persona_id=query_request.persona_id,
+        persona_override_config=query_request.persona_override_config,
+        message_ts_to_respond_to=None,
+        retrieval_details=query_request.retrieval_options,
+        rerank_settings=query_request.rerank_settings,
+        db_session=db_session,
+        use_agentic_search=query_request.use_agentic_search,
+        skip_gen_ai_answer_generation=query_request.skip_gen_ai_answer_generation,
+    )
+
+    packets = stream_chat_message_objects(
+        new_msg_req=request,
+        user=user,
+        db_session=db_session,
+    )
+
+    return packets
+
+
+@basic_router.post("/answer-with-citation")
+def get_answer_with_citation(
+    request: OneShotQARequest,
+    db_session: Session = Depends(get_session),
+    user: User | None = Depends(current_user),
+) -> OneShotQAResponse:
+    try:
+        packets = get_answer_stream(request, user, db_session)
+        answer = gather_stream(packets)
+
+        if answer.error_msg:
+            raise RuntimeError(answer.error_msg)
+
+        return OneShotQAResponse(
+            answer=answer.answer,
+            chat_message_id=answer.message_id,
+            error_msg=answer.error_msg,
+            citations=[
+                CitationInfo(citation_num=i, document_id=doc_id)
+                for i, doc_id in answer.cited_documents.items()
+            ],
+            docs=QADocsResponse(
+                top_documents=answer.top_documents,
+                predicted_flow=None,
+                predicted_search=None,
+                applied_source_filters=None,
+                applied_time_cutoff=None,
+                recency_bias_multiplier=0.0,
+            ),
+        )
+    except Exception as e:
+        logger.error(f"Error in get_answer_with_citation: {str(e)}", exc_info=True)
+        raise HTTPException(status_code=500, detail="An internal server error occurred")
+
+
+@basic_router.post("/stream-answer-with-citation")
+def stream_answer_with_citation(
+    request: OneShotQARequest,
+    db_session: Session = Depends(get_session),
+    user: User | None = Depends(current_user),
+) -> StreamingResponse:
+    def stream_generator() -> Generator[str, None, None]:
+        try:
+            for packet in get_answer_stream(request, user, db_session):
+                serialized = get_json_line(packet.model_dump())
+                yield serialized
+        except Exception as e:
+            logger.exception("Error in answer streaming")
+            yield json.dumps({"error": str(e)})
+
+    return StreamingResponse(stream_generator(), media_type="application/json")
+
+
 @basic_router.get("/standard-answer")
 def get_standard_answer(
     request: StandardAnswerRequest,

backend/ee/onyx/server/query_history/api.py

@@ -24,7 +24,7 @@ from onyx.auth.users import current_admin_user
 from onyx.auth.users import get_display_email
 from onyx.background.celery.versioned_apps.client import app as client_app
 from onyx.background.task_utils import construct_query_history_report_name
-from onyx.chat.chat_utils import create_chat_history_chain
+from onyx.chat.chat_utils import create_chat_chain
 from onyx.configs.app_configs import ONYX_QUERY_HISTORY_TYPE
 from onyx.configs.constants import FileOrigin
 from onyx.configs.constants import FileType
@@ -123,9 +123,10 @@ def snapshot_from_chat_session(
 ) -> ChatSessionSnapshot | None:
     try:
         # Older chats may not have the right structure
-        messages = create_chat_history_chain(
+        last_message, messages = create_chat_chain(
             chat_session_id=chat_session.id, db_session=db_session
         )
+        messages.append(last_message)
     except RuntimeError:
         return None
 

backend/ee/onyx/server/tenants/provisioning.py

@@ -38,8 +38,10 @@ from onyx.db.models import IndexModelStatus
 from onyx.db.models import SearchSettings
 from onyx.db.models import UserTenantMapping
 from onyx.llm.llm_provider_options import ANTHROPIC_PROVIDER_NAME
+from onyx.llm.llm_provider_options import ANTHROPIC_VISIBLE_MODEL_NAMES
 from onyx.llm.llm_provider_options import get_anthropic_model_names
-from onyx.llm.llm_provider_options import get_openai_model_names
+from onyx.llm.llm_provider_options import OPEN_AI_MODEL_NAMES
+from onyx.llm.llm_provider_options import OPEN_AI_VISIBLE_MODEL_NAMES
 from onyx.llm.llm_provider_options import OPENAI_PROVIDER_NAME
 from onyx.server.manage.embedding.models import CloudEmbeddingProviderCreationRequest
 from onyx.server.manage.llm.models import LLMProviderUpsertRequest
@@ -273,7 +275,7 @@ def configure_default_api_keys(db_session: Session) -> None:
             model_configurations=[
                 ModelConfigurationUpsertRequest(
                     name=name,
-                    is_visible=False,
+                    is_visible=name in ANTHROPIC_VISIBLE_MODEL_NAMES,
                     max_input_tokens=None,
                 )
                 for name in get_anthropic_model_names()
@@ -300,10 +302,10 @@ def configure_default_api_keys(db_session: Session) -> None:
             model_configurations=[
                 ModelConfigurationUpsertRequest(
                     name=model_name,
-                    is_visible=False,
+                    is_visible=model_name in OPEN_AI_VISIBLE_MODEL_NAMES,
                     max_input_tokens=None,
                 )
-                for model_name in get_openai_model_names()
+                for model_name in OPEN_AI_MODEL_NAMES
             ],
             api_key_changed=True,
         )

backend/ee/onyx/server/user_group/api.py

@@ -4,14 +4,12 @@ from fastapi import HTTPException
 from sqlalchemy.exc import IntegrityError
 from sqlalchemy.orm import Session
 
-from ee.onyx.db.user_group import add_users_to_user_group
 from ee.onyx.db.user_group import fetch_user_groups
 from ee.onyx.db.user_group import fetch_user_groups_for_user
 from ee.onyx.db.user_group import insert_user_group
 from ee.onyx.db.user_group import prepare_user_group_for_deletion
 from ee.onyx.db.user_group import update_user_curator_relationship
 from ee.onyx.db.user_group import update_user_group
-from ee.onyx.server.user_group.models import AddUsersToUserGroupRequest
 from ee.onyx.server.user_group.models import SetCuratorRequest
 from ee.onyx.server.user_group.models import UserGroup
 from ee.onyx.server.user_group.models import UserGroupCreate
@@ -81,26 +79,6 @@ def patch_user_group(
         raise HTTPException(status_code=404, detail=str(e))
 
 
-@router.post("/admin/user-group/{user_group_id}/add-users")
-def add_users(
-    user_group_id: int,
-    add_users_request: AddUsersToUserGroupRequest,
-    user: User | None = Depends(current_curator_or_admin_user),
-    db_session: Session = Depends(get_session),
-) -> UserGroup:
-    try:
-        return UserGroup.from_model(
-            add_users_to_user_group(
-                db_session=db_session,
-                user=user,
-                user_group_id=user_group_id,
-                user_ids=add_users_request.user_ids,
-            )
-        )
-    except ValueError as e:
-        raise HTTPException(status_code=404, detail=str(e))
-
-
 @router.post("/admin/user-group/{user_group_id}/set-curator")
 def set_user_curator(
     user_group_id: int,

backend/ee/onyx/server/user_group/models.py

@@ -87,10 +87,6 @@ class UserGroupUpdate(BaseModel):
     cc_pair_ids: list[int]
 
 
-class AddUsersToUserGroupRequest(BaseModel):
-    user_ids: list[UUID]
-
-
 class SetCuratorRequest(BaseModel):
     user_id: UUID
     is_curator: bool

backend/ee/onyx/utils/posthog_client.py

@@ -1,10 +1,7 @@
-import json
 from typing import Any
-from urllib.parse import unquote
 
 from posthog import Posthog
 
-from ee.onyx.configs.app_configs import MARKETING_POSTHOG_API_KEY
 from ee.onyx.configs.app_configs import POSTHOG_API_KEY
 from ee.onyx.configs.app_configs import POSTHOG_HOST
 from onyx.utils.logger import setup_logger
@@ -23,80 +20,3 @@ posthog = Posthog(
     debug=True,
     on_error=posthog_on_error,
 )
-
-# For cross referencing between cloud and www Onyx sites
-# NOTE: These clients are separate because they are separate posthog projects.
-# We should eventually unify them into a single posthog project,
-# which would no longer require this workaround
-marketing_posthog = None
-if MARKETING_POSTHOG_API_KEY:
-    marketing_posthog = Posthog(
-        project_api_key=MARKETING_POSTHOG_API_KEY,
-        host=POSTHOG_HOST,
-        debug=True,
-        on_error=posthog_on_error,
-    )
-
-
-def capture_and_sync_with_alternate_posthog(
-    alternate_distinct_id: str, event: str, properties: dict[str, Any]
-) -> None:
-    """
-    Identify in both PostHog projects and capture the event in marketing.
-    - Marketing keeps the marketing distinct_id (for feature flags).
-    - Cloud identify uses the cloud distinct_id
-    """
-    if not marketing_posthog:
-        return
-
-    props = properties.copy()
-
-    try:
-        marketing_posthog.identify(distinct_id=alternate_distinct_id, properties=props)
-        marketing_posthog.capture(alternate_distinct_id, event, props)
-        marketing_posthog.flush()
-    except Exception as e:
-        logger.error(f"Error capturing marketing posthog event: {e}")
-
-    try:
-        if cloud_user_id := props.get("onyx_cloud_user_id"):
-            cloud_props = props.copy()
-            cloud_props.pop("onyx_cloud_user_id", None)
-
-            posthog.identify(
-                distinct_id=cloud_user_id,
-                properties=cloud_props,
-            )
-    except Exception as e:
-        logger.error(f"Error identifying cloud posthog user: {e}")
-
-
-def get_marketing_posthog_cookie_name() -> str | None:
-    if not MARKETING_POSTHOG_API_KEY:
-        return None
-    return f"onyx_custom_ph_{MARKETING_POSTHOG_API_KEY}_posthog"
-
-
-def parse_marketing_cookie(cookie_value: str) -> dict[str, Any] | None:
-    """
-    Parse the URL-encoded JSON marketing cookie.
-
-    Expected format (URL-encoded):
-    {"distinct_id":"...", "featureFlags":{"landing_page_variant":"..."}, ...}
-
-    Returns:
-        Dict with 'distinct_id' explicitly required and all other cookie values
-        passed through as-is, or None if parsing fails or distinct_id is missing.
-    """
-    try:
-        decoded_cookie = unquote(cookie_value)
-        cookie_data = json.loads(decoded_cookie)
-
-        distinct_id = cookie_data.get("distinct_id")
-        if not distinct_id:
-            return None
-
-        return cookie_data
-    except (json.JSONDecodeError, KeyError, TypeError, AttributeError) as e:
-        logger.warning(f"Failed to parse cookie: {e}")
-        return None

backend/model_server/custom_models.py

@@ -517,7 +517,7 @@ def run_analysis(intent_req: IntentRequest) -> tuple[bool, list[str]]:
     try:
         keywords = map_keywords(model_input.input_ids[0], tokenizer, keyword_preds)
     except Exception as e:
-        logger.warning(
+        logger.error(
             f"Failed to extract keywords for query: {intent_req.query} due to {e}"
         )
         # Fallback to keeping all words

backend/model_server/encoders.py

@@ -116,7 +116,7 @@ def _concurrent_embedding(
             # the model to fail to encode texts. It's pretty rare and we want to allow
             # concurrent embedding, hence we retry (the specific error is
             # "RuntimeError: Already borrowed" and occurs in the transformers library)
-            logger.warning(f"Error encoding texts, retrying: {e}")
+            logger.error(f"Error encoding texts, retrying: {e}")
             time.sleep(ENCODING_RETRY_DELAY)
     return model.encode(texts, normalize_embeddings=normalize_embeddings)
 

backend/onyx/agents/agent_sdk/message_format.py

@@ -0,0 +1,167 @@
+from collections.abc import Sequence
+
+from langchain.schema.messages import BaseMessage
+
+from onyx.agents.agent_sdk.message_types import AgentSDKMessage
+from onyx.agents.agent_sdk.message_types import AssistantMessageWithContent
+from onyx.agents.agent_sdk.message_types import ImageContent
+from onyx.agents.agent_sdk.message_types import InputTextContent
+from onyx.agents.agent_sdk.message_types import SystemMessage
+from onyx.agents.agent_sdk.message_types import UserMessage
+
+
+# TODO: Currently, we only support native API input for images. For other
+# files, we process the content and share it as text in the message. In
+# the future, we might support native file uploads for other types of files.
+def base_messages_to_agent_sdk_msgs(
+    msgs: Sequence[BaseMessage],
+    is_responses_api: bool,
+) -> list[AgentSDKMessage]:
+    return [_base_message_to_agent_sdk_msg(msg, is_responses_api) for msg in msgs]
+
+
+def _base_message_to_agent_sdk_msg(
+    msg: BaseMessage, is_responses_api: bool
+) -> AgentSDKMessage:
+    message_type_to_agent_sdk_role = {
+        "human": "user",
+        "system": "system",
+        "ai": "assistant",
+    }
+    role = message_type_to_agent_sdk_role[msg.type]
+
+    # Convert content to Agent SDK format
+    content = msg.content
+
+    if isinstance(content, str):
+        # For system/user/assistant messages, use InputTextContent
+        if role in ("system", "user"):
+            input_text_content: list[InputTextContent | ImageContent] = [
+                InputTextContent(type="input_text", text=content)
+            ]
+            if role == "system":
+                # SystemMessage only accepts InputTextContent
+                system_msg: SystemMessage = {
+                    "role": "system",
+                    "content": [InputTextContent(type="input_text", text=content)],
+                }
+                return system_msg
+            else:  # user
+                user_msg: UserMessage = {
+                    "role": "user",
+                    "content": input_text_content,
+                }
+                return user_msg
+        else:  # assistant
+            assistant_msg: AssistantMessageWithContent
+            if is_responses_api:
+                from onyx.agents.agent_sdk.message_types import OutputTextContent
+
+                assistant_msg = {
+                    "role": "assistant",
+                    "content": [OutputTextContent(type="output_text", text=content)],
+                }
+            else:
+                assistant_msg = {
+                    "role": "assistant",
+                    "content": [InputTextContent(type="input_text", text=content)],
+                }
+            return assistant_msg
+    elif isinstance(content, list):
+        # For lists, we need to process based on the role
+        if role == "assistant":
+            # For responses API, use OutputTextContent; otherwise use InputTextContent
+            assistant_content: list[InputTextContent | OutputTextContent] = []
+
+            if is_responses_api:
+                from onyx.agents.agent_sdk.message_types import OutputTextContent
+
+                for item in content:
+                    if isinstance(item, str):
+                        assistant_content.append(
+                            OutputTextContent(type="output_text", text=item)
+                        )
+                    elif isinstance(item, dict) and item.get("type") == "text":
+                        assistant_content.append(
+                            OutputTextContent(
+                                type="output_text", text=item.get("text", "")
+                            )
+                        )
+                    else:
+                        raise ValueError(
+                            f"Unexpected item type for assistant message: {type(item)}. Item: {item}"
+                        )
+            else:
+                for item in content:
+                    if isinstance(item, str):
+                        assistant_content.append(
+                            InputTextContent(type="input_text", text=item)
+                        )
+                    elif isinstance(item, dict) and item.get("type") == "text":
+                        assistant_content.append(
+                            InputTextContent(
+                                type="input_text", text=item.get("text", "")
+                            )
+                        )
+                    else:
+                        raise ValueError(
+                            f"Unexpected item type for assistant message: {type(item)}. Item: {item}"
+                        )
+
+            assistant_msg_list: AssistantMessageWithContent = {
+                "role": "assistant",
+                "content": assistant_content,
+            }
+            return assistant_msg_list
+        else:  # system or user - use InputTextContent
+            input_content: list[InputTextContent | ImageContent] = []
+            for item in content:
+                if isinstance(item, str):
+                    input_content.append(InputTextContent(type="input_text", text=item))
+                elif isinstance(item, dict):
+                    item_type = item.get("type")
+                    if item_type == "text":
+                        input_content.append(
+                            InputTextContent(
+                                type="input_text", text=item.get("text", "")
+                            )
+                        )
+                    elif item_type == "image_url":
+                        # Convert image_url to input_image format
+                        image_url = item.get("image_url", {})
+                        if isinstance(image_url, dict):
+                            url = image_url.get("url", "")
+                        else:
+                            url = image_url
+                        input_content.append(
+                            ImageContent(
+                                type="input_image", image_url=url, detail="auto"
+                            )
+                        )
+                    else:
+                        raise ValueError(f"Unexpected item type: {item_type}")
+                else:
+                    raise ValueError(
+                        f"Unexpected item type: {type(item)}. Item: {item}"
+                    )
+
+            if role == "system":
+                # SystemMessage only accepts InputTextContent (no images)
+                text_only_content = [
+                    c for c in input_content if c["type"] == "input_text"
+                ]
+                system_msg_list: SystemMessage = {
+                    "role": "system",
+                    "content": text_only_content,  # type: ignore[typeddict-item]
+                }
+                return system_msg_list
+            else:  # user
+                user_msg_list: UserMessage = {
+                    "role": "user",
+                    "content": input_content,
+                }
+                return user_msg_list
+    else:
+        raise ValueError(
+            f"Unexpected content type: {type(content)}. Content: {content}"
+        )

backend/onyx/agents/agent_sdk/message_types.py

@@ -0,0 +1,125 @@
+"""Strongly typed message structures for Agent SDK messages."""
+
+from typing import Literal
+from typing import NotRequired
+
+from typing_extensions import TypedDict
+
+
+class InputTextContent(TypedDict):
+    type: Literal["input_text"]
+    text: str
+
+
+class OutputTextContent(TypedDict):
+    type: Literal["output_text"]
+    text: str
+
+
+TextContent = InputTextContent | OutputTextContent
+
+
+class ImageContent(TypedDict):
+    type: Literal["input_image"]
+    image_url: str
+    detail: str
+
+
+# Tool call structures
+class ToolCallFunction(TypedDict):
+    name: str
+    arguments: str
+
+
+class ToolCall(TypedDict):
+    id: str
+    type: Literal["function"]
+    function: ToolCallFunction
+
+
+# Message types
+class SystemMessage(TypedDict):
+    role: Literal["system"]
+    content: list[InputTextContent]  # System messages use input text
+
+
+class UserMessage(TypedDict):
+    role: Literal["user"]
+    content: list[
+        InputTextContent | ImageContent
+    ]  # User messages use input text or images
+
+
+class AssistantMessageWithContent(TypedDict):
+    role: Literal["assistant"]
+    content: list[
+        InputTextContent | OutputTextContent
+    ]  # Assistant messages use output_text for responses API compatibility
+
+
+class AssistantMessageWithToolCalls(TypedDict):
+    role: Literal["assistant"]
+    tool_calls: list[ToolCall]
+
+
+class AssistantMessageDuringAgentRun(TypedDict):
+    role: Literal["assistant"]
+    id: str
+    content: (
+        list[InputTextContent | OutputTextContent] | list[ToolCall]
+    )  # Assistant runtime messages receive output_text from agents SDK for responses API compatibility
+    status: Literal["completed", "failed", "in_progress"]
+    type: Literal["message"]
+
+
+class ToolMessage(TypedDict):
+    role: Literal["tool"]
+    content: str
+    tool_call_id: str
+
+
+class FunctionCallMessage(TypedDict):
+    """Agent SDK function call message format."""
+
+    type: Literal["function_call"]
+    id: NotRequired[str]
+    call_id: str
+    name: str
+    arguments: str
+
+
+class FunctionCallOutputMessage(TypedDict):
+    """Agent SDK function call output message format."""
+
+    type: Literal["function_call_output"]
+    call_id: str
+    output: str
+
+
+class SummaryText(TypedDict):
+    """Summary text item in reasoning messages."""
+
+    text: str
+    type: Literal["summary_text"]
+
+
+class ReasoningMessage(TypedDict):
+    """Agent SDK reasoning message format."""
+
+    id: str
+    type: Literal["reasoning"]
+    summary: list[SummaryText]
+
+
+# Union type for all Agent SDK messages
+AgentSDKMessage = (
+    SystemMessage
+    | UserMessage
+    | AssistantMessageWithContent
+    | AssistantMessageWithToolCalls
+    | AssistantMessageDuringAgentRun
+    | ToolMessage
+    | FunctionCallMessage
+    | FunctionCallOutputMessage
+    | ReasoningMessage
+)

backend/onyx/agents/agent_sdk/monkey_patches.py

@@ -0,0 +1,36 @@
+from typing import Any
+
+from agents.models.openai_responses import Converter as OpenAIResponsesConverter
+
+
+# TODO: I am very sad that I have to monkey patch this :(
+# Basically, OpenAI agents sdk doesn't convert the tool choice correctly
+# when they have a built-in tool in their framework, like they do for web_search
+# and image_generation.
+# Going to open up a thread with OpenAI agents team to see what they recommend
+# or what we can fix.
+# A discussion is warranted, but we likely want to just write our own LitellmModel for
+# the OpenAI agents SDK since they probably don't really care about Litellm and will
+# prioritize functionality for their own models.
+def monkey_patch_convert_tool_choice_to_ignore_openai_hosted_web_search() -> None:
+    if (
+        getattr(OpenAIResponsesConverter.convert_tool_choice, "__name__", "")
+        == "_patched_convert_tool_choice"
+    ):
+        return
+
+    orig_func = OpenAIResponsesConverter.convert_tool_choice.__func__  # type: ignore[attr-defined]
+
+    def _patched_convert_tool_choice(cls: type, tool_choice: Any) -> Any:
+        # Handle OpenAI hosted tools that we have custom implementations for
+        # Without this patch, the library uses special formatting that breaks our custom tools
+        # See: https://platform.openai.com/docs/api-reference/responses/create#responses_create-tool_choice-hosted_tool-type
+        if tool_choice == "web_search":
+            return {"type": "function", "name": "web_search"}
+        if tool_choice == "image_generation":
+            return {"type": "function", "name": "image_generation"}
+        return orig_func(cls, tool_choice)
+
+    OpenAIResponsesConverter.convert_tool_choice = classmethod(  # type: ignore[method-assign, assignment]
+        _patched_convert_tool_choice
+    )

backend/onyx/agents/agent_sdk/sync_agent_stream_adapter.py

@@ -0,0 +1,178 @@
+import asyncio
+import queue
+import threading
+from collections.abc import Iterator
+from collections.abc import Sequence
+from typing import Generic
+from typing import Optional
+from typing import TypeVar
+
+from agents import Agent
+from agents import RunResultStreaming
+from agents import TContext
+from agents.run import Runner
+
+from onyx.agents.agent_sdk.message_types import AgentSDKMessage
+from onyx.utils.threadpool_concurrency import run_in_background
+
+T = TypeVar("T")
+
+
+class SyncAgentStream(Generic[T]):
+    """
+    Convert an async streamed run into a sync iterator with cooperative cancellation.
+    Runs the Agent in a background thread.
+
+    Usage:
+        adapter = SyncStreamAdapter(
+            agent=agent,
+            input=input,
+            context=context,
+            max_turns=100,
+            queue_maxsize=0,         # optional backpressure
+        )
+        for ev in adapter:          # sync iteration
+            ...
+        # or cancel from elsewhere:
+        adapter.cancel()
+    """
+
+    _SENTINEL = object()
+
+    def __init__(
+        self,
+        *,
+        agent: Agent,
+        input: Sequence[AgentSDKMessage],
+        context: TContext | None = None,
+        max_turns: int = 100,
+        queue_maxsize: int = 0,
+    ) -> None:
+        self._agent = agent
+        self._input = input
+        self._context = context
+        self._max_turns = max_turns
+
+        self._q: "queue.Queue[object]" = queue.Queue(maxsize=queue_maxsize)
+        self._loop: Optional[asyncio.AbstractEventLoop] = None
+        self._thread: Optional[threading.Thread] = None
+        self.streamed: RunResultStreaming | None = None
+        self._exc: Optional[BaseException] = None
+        self._cancel_requested = threading.Event()
+        self._started = threading.Event()
+        self._done = threading.Event()
+
+        self._start_thread()
+
+    # ---------- public sync API ----------
+
+    def __iter__(self) -> Iterator[T]:
+        try:
+            while True:
+                item = self._q.get()
+                if item is self._SENTINEL:
+                    # If the consumer thread raised, surface it now
+                    if self._exc is not None:
+                        raise self._exc
+                    # Normal completion
+                    return
+                yield item  # type: ignore[misc,return-value]
+        finally:
+            # Ensure we fully clean up whether we exited due to exception,
+            # StopIteration, or external cancel.
+            self.close()
+
+    def cancel(self) -> bool:
+        """
+        Cooperatively cancel the underlying streamed run and shut down.
+        Safe to call multiple times and from any thread.
+        """
+        self._cancel_requested.set()
+        loop = self._loop
+        streamed = self.streamed
+        if loop is not None and streamed is not None and not self._done.is_set():
+            loop.call_soon_threadsafe(streamed.cancel)
+            return True
+        return False
+
+    def close(self, *, wait: bool = True) -> None:
+        """Idempotent shutdown."""
+        self.cancel()
+        # ask the loop to stop if it's still running
+        loop = self._loop
+        if loop is not None and loop.is_running():
+            try:
+                loop.call_soon_threadsafe(loop.stop)
+            except Exception:
+                pass
+        # join the thread
+        if wait and self._thread is not None and self._thread.is_alive():
+            self._thread.join(timeout=5.0)
+
+    # ---------- internals ----------
+
+    def _start_thread(self) -> None:
+        t = run_in_background(self._thread_main)
+        self._thread = t
+        # Optionally wait until the loop/worker is started so .cancel() is safe soon after init
+        self._started.wait(timeout=1.0)
+
+    def _thread_main(self) -> None:
+        loop = asyncio.new_event_loop()
+        self._loop = loop
+        asyncio.set_event_loop(loop)
+
+        async def worker() -> None:
+            try:
+                # Start the streamed run inside the loop thread
+                self.streamed = Runner.run_streamed(
+                    self._agent,
+                    self._input,  # type: ignore[arg-type]
+                    context=self._context,
+                    max_turns=self._max_turns,
+                )
+
+                # If cancel was requested before we created _streamed, honor it now
+                if self._cancel_requested.is_set():
+                    await self.streamed.cancel()  # type: ignore[func-returns-value]
+
+                # Consume async events and forward into the thread-safe queue
+                async for ev in self.streamed.stream_events():
+                    # Early exit if a late cancel arrives
+                    if self._cancel_requested.is_set():
+                        # Try to cancel gracefully; don't break until cancel takes effect
+                        try:
+                            await self.streamed.cancel()  # type: ignore[func-returns-value]
+                        except Exception:
+                            pass
+                        break
+                    # This put() may block if queue_maxsize > 0 (backpressure)
+                    self._q.put(ev)
+
+            except BaseException as e:
+                # Save exception to surface on the sync iterator side
+                self._exc = e
+            finally:
+                # Signal end-of-stream
+                self._q.put(self._SENTINEL)
+                self._done.set()
+
+        # Mark started and run the worker to completion
+        self._started.set()
+        try:
+            loop.run_until_complete(worker())
+        finally:
+            try:
+                # Drain pending tasks/callbacks safely
+                pending = asyncio.all_tasks(loop=loop)
+                for task in pending:
+                    task.cancel()
+                if pending:
+                    loop.run_until_complete(
+                        asyncio.gather(*pending, return_exceptions=True)
+                    )
+            except Exception:
+                pass
+            finally:
+                loop.close()
+                self._loop = None

backend/onyx/agents/agent_search/core_state.py

@@ -0,0 +1,21 @@
+from operator import add
+from typing import Annotated
+
+from pydantic import BaseModel
+
+
+class CoreState(BaseModel):
+    """
+    This is the core state that is shared across all subgraphs.
+    """
+
+    log_messages: Annotated[list[str], add] = []
+    current_step_nr: int = 1
+
+
+class SubgraphCoreState(BaseModel):
+    """
+    This is the core state that is shared across all subgraphs.
+    """
+
+    log_messages: Annotated[list[str], add] = []

backend/onyx/agents/agent_search/dc_search_analysis/edges.py

@@ -0,0 +1,62 @@
+from collections.abc import Hashable
+from typing import cast
+
+from langchain_core.runnables.config import RunnableConfig
+from langgraph.types import Send
+
+from onyx.agents.agent_search.dc_search_analysis.states import ObjectInformationInput
+from onyx.agents.agent_search.dc_search_analysis.states import (
+    ObjectResearchInformationUpdate,
+)
+from onyx.agents.agent_search.dc_search_analysis.states import ObjectSourceInput
+from onyx.agents.agent_search.dc_search_analysis.states import (
+    SearchSourcesObjectsUpdate,
+)
+from onyx.agents.agent_search.models import GraphConfig
+
+
+def parallel_object_source_research_edge(
+    state: SearchSourcesObjectsUpdate, config: RunnableConfig
+) -> list[Send | Hashable]:
+    """
+    LangGraph edge to parallelize the research for an individual object and source
+    """
+
+    search_objects = state.analysis_objects
+    search_sources = state.analysis_sources
+
+    object_source_combinations = [
+        (object, source) for object in search_objects for source in search_sources
+    ]
+
+    return [
+        Send(
+            "research_object_source",
+            ObjectSourceInput(
+                object_source_combination=object_source_combination,
+                log_messages=[],
+            ),
+        )
+        for object_source_combination in object_source_combinations
+    ]
+
+
+def parallel_object_research_consolidation_edge(
+    state: ObjectResearchInformationUpdate, config: RunnableConfig
+) -> list[Send | Hashable]:
+    """
+    LangGraph edge to parallelize the research for an individual object and source
+    """
+    cast(GraphConfig, config["metadata"]["config"])
+    object_research_information_results = state.object_research_information_results
+
+    return [
+        Send(
+            "consolidate_object_research",
+            ObjectInformationInput(
+                object_information=object_information,
+                log_messages=[],
+            ),
+        )
+        for object_information in object_research_information_results
+    ]

backend/onyx/agents/agent_search/dc_search_analysis/graph_builder.py

@@ -0,0 +1,103 @@
+from langgraph.graph import END
+from langgraph.graph import START
+from langgraph.graph import StateGraph
+
+from onyx.agents.agent_search.dc_search_analysis.edges import (
+    parallel_object_research_consolidation_edge,
+)
+from onyx.agents.agent_search.dc_search_analysis.edges import (
+    parallel_object_source_research_edge,
+)
+from onyx.agents.agent_search.dc_search_analysis.nodes.a1_search_objects import (
+    search_objects,
+)
+from onyx.agents.agent_search.dc_search_analysis.nodes.a2_research_object_source import (
+    research_object_source,
+)
+from onyx.agents.agent_search.dc_search_analysis.nodes.a3_structure_research_by_object import (
+    structure_research_by_object,
+)
+from onyx.agents.agent_search.dc_search_analysis.nodes.a4_consolidate_object_research import (
+    consolidate_object_research,
+)
+from onyx.agents.agent_search.dc_search_analysis.nodes.a5_consolidate_research import (
+    consolidate_research,
+)
+from onyx.agents.agent_search.dc_search_analysis.states import MainInput
+from onyx.agents.agent_search.dc_search_analysis.states import MainState
+from onyx.utils.logger import setup_logger
+
+logger = setup_logger()
+
+test_mode = False
+
+
+def divide_and_conquer_graph_builder(test_mode: bool = False) -> StateGraph:
+    """
+    LangGraph graph builder for the knowledge graph  search process.
+    """
+
+    graph = StateGraph(
+        state_schema=MainState,
+        input=MainInput,
+    )
+
+    ### Add nodes ###
+
+    graph.add_node(
+        "search_objects",
+        search_objects,
+    )
+
+    graph.add_node(
+        "structure_research_by_source",
+        structure_research_by_object,
+    )
+
+    graph.add_node(
+        "research_object_source",
+        research_object_source,
+    )
+
+    graph.add_node(
+        "consolidate_object_research",
+        consolidate_object_research,
+    )
+
+    graph.add_node(
+        "consolidate_research",
+        consolidate_research,
+    )
+
+    ### Add edges ###
+
+    graph.add_edge(start_key=START, end_key="search_objects")
+
+    graph.add_conditional_edges(
+        source="search_objects",
+        path=parallel_object_source_research_edge,
+        path_map=["research_object_source"],
+    )
+
+    graph.add_edge(
+        start_key="research_object_source",
+        end_key="structure_research_by_source",
+    )
+
+    graph.add_conditional_edges(
+        source="structure_research_by_source",
+        path=parallel_object_research_consolidation_edge,
+        path_map=["consolidate_object_research"],
+    )
+
+    graph.add_edge(
+        start_key="consolidate_object_research",
+        end_key="consolidate_research",
+    )
+
+    graph.add_edge(
+        start_key="consolidate_research",
+        end_key=END,
+    )
+
+    return graph

backend/onyx/agents/agent_search/dc_search_analysis/nodes/a1_search_objects.py

@@ -0,0 +1,146 @@
+from typing import cast
+
+from langchain_core.messages import HumanMessage
+from langchain_core.runnables import RunnableConfig
+from langgraph.types import StreamWriter
+
+from onyx.agents.agent_search.dc_search_analysis.ops import extract_section
+from onyx.agents.agent_search.dc_search_analysis.ops import research
+from onyx.agents.agent_search.dc_search_analysis.states import MainState
+from onyx.agents.agent_search.dc_search_analysis.states import (
+    SearchSourcesObjectsUpdate,
+)
+from onyx.agents.agent_search.models import GraphConfig
+from onyx.agents.agent_search.shared_graph_utils.agent_prompt_ops import (
+    trim_prompt_piece,
+)
+from onyx.prompts.agents.dc_prompts import DC_OBJECT_NO_BASE_DATA_EXTRACTION_PROMPT
+from onyx.prompts.agents.dc_prompts import DC_OBJECT_SEPARATOR
+from onyx.prompts.agents.dc_prompts import DC_OBJECT_WITH_BASE_DATA_EXTRACTION_PROMPT
+from onyx.secondary_llm_flows.source_filter import strings_to_document_sources
+from onyx.utils.logger import setup_logger
+from onyx.utils.threadpool_concurrency import run_with_timeout
+
+logger = setup_logger()
+
+
+def search_objects(
+    state: MainState, config: RunnableConfig, writer: StreamWriter = lambda _: None
+) -> SearchSourcesObjectsUpdate:
+    """
+    LangGraph node to start the agentic search process.
+    """
+
+    graph_config = cast(GraphConfig, config["metadata"]["config"])
+    question = graph_config.inputs.prompt_builder.raw_user_query
+    search_tool = graph_config.tooling.search_tool
+
+    if search_tool is None or graph_config.inputs.persona is None:
+        raise ValueError("Search tool and persona must be provided for DivCon search")
+
+    try:
+        instructions = graph_config.inputs.persona.system_prompt or ""
+
+        agent_1_instructions = extract_section(
+            instructions, "Agent Step 1:", "Agent Step 2:"
+        )
+        if agent_1_instructions is None:
+            raise ValueError("Agent 1 instructions not found")
+
+        agent_1_base_data = extract_section(instructions, "|Start Data|", "|End Data|")
+
+        agent_1_task = extract_section(
+            agent_1_instructions, "Task:", "Independent Research Sources:"
+        )
+        if agent_1_task is None:
+            raise ValueError("Agent 1 task not found")
+
+        agent_1_independent_sources_str = extract_section(
+            agent_1_instructions, "Independent Research Sources:", "Output Objective:"
+        )
+        if agent_1_independent_sources_str is None:
+            raise ValueError("Agent 1 Independent Research Sources not found")
+
+        document_sources = strings_to_document_sources(
+            [
+                x.strip().lower()
+                for x in agent_1_independent_sources_str.split(DC_OBJECT_SEPARATOR)
+            ]
+        )
+
+        agent_1_output_objective = extract_section(
+            agent_1_instructions, "Output Objective:"
+        )
+        if agent_1_output_objective is None:
+            raise ValueError("Agent 1 output objective not found")
+
+    except Exception as e:
+        raise ValueError(
+            f"Agent 1 instructions not found or not formatted correctly: {e}"
+        )
+
+    # Extract objects
+
+    if agent_1_base_data is None:
+        # Retrieve chunks for objects
+
+        retrieved_docs = research(question, search_tool)[:10]
+
+        document_texts_list = []
+        for doc_num, doc in enumerate(retrieved_docs):
+            chunk_text = "Document " + str(doc_num) + ":\n" + doc.content
+            document_texts_list.append(chunk_text)
+
+        document_texts = "\n\n".join(document_texts_list)
+
+        dc_object_extraction_prompt = DC_OBJECT_NO_BASE_DATA_EXTRACTION_PROMPT.format(
+            question=question,
+            task=agent_1_task,
+            document_text=document_texts,
+            objects_of_interest=agent_1_output_objective,
+        )
+    else:
+        dc_object_extraction_prompt = DC_OBJECT_WITH_BASE_DATA_EXTRACTION_PROMPT.format(
+            question=question,
+            task=agent_1_task,
+            base_data=agent_1_base_data,
+            objects_of_interest=agent_1_output_objective,
+        )
+
+    msg = [
+        HumanMessage(
+            content=trim_prompt_piece(
+                config=graph_config.tooling.primary_llm.config,
+                prompt_piece=dc_object_extraction_prompt,
+                reserved_str="",
+            ),
+        )
+    ]
+    primary_llm = graph_config.tooling.primary_llm
+    # Grader
+    try:
+        llm_response = run_with_timeout(
+            30,
+            primary_llm.invoke,
+            prompt=msg,
+            timeout_override=30,
+            max_tokens=300,
+        )
+
+        cleaned_response = (
+            str(llm_response.content)
+            .replace("```json\n", "")
+            .replace("\n```", "")
+            .replace("\n", "")
+        )
+        cleaned_response = cleaned_response.split("OBJECTS:")[1]
+        object_list = [x.strip() for x in cleaned_response.split(";")]
+
+    except Exception as e:
+        raise ValueError(f"Error in search_objects: {e}")
+
+    return SearchSourcesObjectsUpdate(
+        analysis_objects=object_list,
+        analysis_sources=document_sources,
+        log_messages=["Agent 1 Task done"],
+    )

backend/onyx/agents/agent_search/dc_search_analysis/nodes/a2_research_object_source.py

@@ -0,0 +1,180 @@
+from datetime import datetime
+from datetime import timedelta
+from datetime import timezone
+from typing import cast
+
+from langchain_core.messages import HumanMessage
+from langchain_core.runnables import RunnableConfig
+from langgraph.types import StreamWriter
+
+from onyx.agents.agent_search.dc_search_analysis.ops import extract_section
+from onyx.agents.agent_search.dc_search_analysis.ops import research
+from onyx.agents.agent_search.dc_search_analysis.states import ObjectSourceInput
+from onyx.agents.agent_search.dc_search_analysis.states import (
+    ObjectSourceResearchUpdate,
+)
+from onyx.agents.agent_search.models import GraphConfig
+from onyx.agents.agent_search.shared_graph_utils.agent_prompt_ops import (
+    trim_prompt_piece,
+)
+from onyx.prompts.agents.dc_prompts import DC_OBJECT_SOURCE_RESEARCH_PROMPT
+from onyx.utils.logger import setup_logger
+from onyx.utils.threadpool_concurrency import run_with_timeout
+
+logger = setup_logger()
+
+
+def research_object_source(
+    state: ObjectSourceInput,
+    config: RunnableConfig,
+    writer: StreamWriter = lambda _: None,
+) -> ObjectSourceResearchUpdate:
+    """
+    LangGraph node to start the agentic search process.
+    """
+    datetime.now()
+
+    graph_config = cast(GraphConfig, config["metadata"]["config"])
+    search_tool = graph_config.tooling.search_tool
+    question = graph_config.inputs.prompt_builder.raw_user_query
+    object, document_source = state.object_source_combination
+
+    if search_tool is None or graph_config.inputs.persona is None:
+        raise ValueError("Search tool and persona must be provided for DivCon search")
+
+    try:
+        instructions = graph_config.inputs.persona.system_prompt or ""
+
+        agent_2_instructions = extract_section(
+            instructions, "Agent Step 2:", "Agent Step 3:"
+        )
+        if agent_2_instructions is None:
+            raise ValueError("Agent 2 instructions not found")
+
+        agent_2_task = extract_section(
+            agent_2_instructions, "Task:", "Independent Research Sources:"
+        )
+        if agent_2_task is None:
+            raise ValueError("Agent 2 task not found")
+
+        agent_2_time_cutoff = extract_section(
+            agent_2_instructions, "Time Cutoff:", "Research Topics:"
+        )
+
+        agent_2_research_topics = extract_section(
+            agent_2_instructions, "Research Topics:", "Output Objective"
+        )
+
+        agent_2_output_objective = extract_section(
+            agent_2_instructions, "Output Objective:"
+        )
+        if agent_2_output_objective is None:
+            raise ValueError("Agent 2 output objective not found")
+
+    except Exception:
+        raise ValueError(
+            "Agent 1 instructions not found or not formatted correctly: {e}"
+        )
+
+    # Populate prompt
+
+    # Retrieve chunks for objects
+
+    if agent_2_time_cutoff is not None and agent_2_time_cutoff.strip() != "":
+        if agent_2_time_cutoff.strip().endswith("d"):
+            try:
+                days = int(agent_2_time_cutoff.strip()[:-1])
+                agent_2_source_start_time = datetime.now(timezone.utc) - timedelta(
+                    days=days
+                )
+            except ValueError:
+                raise ValueError(
+                    f"Invalid time cutoff format: {agent_2_time_cutoff}. Expected format: '<number>d'"
+                )
+        else:
+            raise ValueError(
+                f"Invalid time cutoff format: {agent_2_time_cutoff}. Expected format: '<number>d'"
+            )
+    else:
+        agent_2_source_start_time = None
+
+    document_sources = [document_source] if document_source else None
+
+    if len(question.strip()) > 0:
+        research_area = f"{question} for {object}"
+    elif agent_2_research_topics and len(agent_2_research_topics.strip()) > 0:
+        research_area = f"{agent_2_research_topics} for {object}"
+    else:
+        research_area = object
+
+    retrieved_docs = research(
+        question=research_area,
+        search_tool=search_tool,
+        document_sources=document_sources,
+        time_cutoff=agent_2_source_start_time,
+    )
+
+    # Generate document text
+
+    document_texts_list = []
+    for doc_num, doc in enumerate(retrieved_docs):
+        chunk_text = "Document " + str(doc_num) + ":\n" + doc.content
+        document_texts_list.append(chunk_text)
+
+    document_texts = "\n\n".join(document_texts_list)
+
+    # Built prompt
+
+    today = datetime.now().strftime("%A, %Y-%m-%d")
+
+    dc_object_source_research_prompt = (
+        DC_OBJECT_SOURCE_RESEARCH_PROMPT.format(
+            today=today,
+            question=question,
+            task=agent_2_task,
+            document_text=document_texts,
+            format=agent_2_output_objective,
+        )
+        .replace("---object---", object)
+        .replace("---source---", document_source.value)
+    )
+
+    # Run LLM
+
+    msg = [
+        HumanMessage(
+            content=trim_prompt_piece(
+                config=graph_config.tooling.primary_llm.config,
+                prompt_piece=dc_object_source_research_prompt,
+                reserved_str="",
+            ),
+        )
+    ]
+    primary_llm = graph_config.tooling.primary_llm
+    # Grader
+    try:
+        llm_response = run_with_timeout(
+            30,
+            primary_llm.invoke,
+            prompt=msg,
+            timeout_override=30,
+            max_tokens=300,
+        )
+
+        cleaned_response = str(llm_response.content).replace("```json\n", "")
+        cleaned_response = cleaned_response.split("RESEARCH RESULTS:")[1]
+        object_research_results = {
+            "object": object,
+            "source": document_source.value,
+            "research_result": cleaned_response,
+        }
+
+    except Exception as e:
+        raise ValueError(f"Error in research_object_source: {e}")
+
+    logger.debug("DivCon Step A2 - Object Source Research - completed for an object")
+
+    return ObjectSourceResearchUpdate(
+        object_source_research_results=[object_research_results],
+        log_messages=["Agent Step 2 done for one object"],
+    )

backend/onyx/agents/agent_search/dc_search_analysis/nodes/a3_structure_research_by_object.py

@@ -0,0 +1,48 @@
+from collections import defaultdict
+from typing import Dict
+from typing import List
+
+from langchain_core.runnables import RunnableConfig
+from langgraph.types import StreamWriter
+
+from onyx.agents.agent_search.dc_search_analysis.states import MainState
+from onyx.agents.agent_search.dc_search_analysis.states import (
+    ObjectResearchInformationUpdate,
+)
+from onyx.utils.logger import setup_logger
+
+logger = setup_logger()
+
+
+def structure_research_by_object(
+    state: MainState, config: RunnableConfig, writer: StreamWriter = lambda _: None
+) -> ObjectResearchInformationUpdate:
+    """
+    LangGraph node to start the agentic search process.
+    """
+
+    object_source_research_results = state.object_source_research_results
+
+    object_research_information_results: List[Dict[str, str]] = []
+    object_research_information_results_list: Dict[str, List[str]] = defaultdict(list)
+
+    for object_source_research in object_source_research_results:
+        object = object_source_research["object"]
+        source = object_source_research["source"]
+        research_result = object_source_research["research_result"]
+
+        object_research_information_results_list[object].append(
+            f"Source: {source}\n{research_result}"
+        )
+
+    for object, information in object_research_information_results_list.items():
+        object_research_information_results.append(
+            {"object": object, "information": "\n".join(information)}
+        )
+
+    logger.debug("DivCon Step A3 - Object Research Information Structuring - completed")
+
+    return ObjectResearchInformationUpdate(
+        object_research_information_results=object_research_information_results,
+        log_messages=["A3 - Object Research Information structured"],
+    )

backend/onyx/agents/agent_search/dc_search_analysis/nodes/a4_consolidate_object_research.py

@@ -0,0 +1,103 @@
+from typing import cast
+
+from langchain_core.messages import HumanMessage
+from langchain_core.runnables import RunnableConfig
+from langgraph.types import StreamWriter
+
+from onyx.agents.agent_search.dc_search_analysis.ops import extract_section
+from onyx.agents.agent_search.dc_search_analysis.states import ObjectInformationInput
+from onyx.agents.agent_search.dc_search_analysis.states import ObjectResearchUpdate
+from onyx.agents.agent_search.models import GraphConfig
+from onyx.agents.agent_search.shared_graph_utils.agent_prompt_ops import (
+    trim_prompt_piece,
+)
+from onyx.prompts.agents.dc_prompts import DC_OBJECT_CONSOLIDATION_PROMPT
+from onyx.utils.logger import setup_logger
+from onyx.utils.threadpool_concurrency import run_with_timeout
+
+logger = setup_logger()
+
+
+def consolidate_object_research(
+    state: ObjectInformationInput,
+    config: RunnableConfig,
+    writer: StreamWriter = lambda _: None,
+) -> ObjectResearchUpdate:
+    """
+    LangGraph node to start the agentic search process.
+    """
+    graph_config = cast(GraphConfig, config["metadata"]["config"])
+    search_tool = graph_config.tooling.search_tool
+    question = graph_config.inputs.prompt_builder.raw_user_query
+
+    if search_tool is None or graph_config.inputs.persona is None:
+        raise ValueError("Search tool and persona must be provided for DivCon search")
+
+    instructions = graph_config.inputs.persona.system_prompt or ""
+
+    agent_4_instructions = extract_section(
+        instructions, "Agent Step 4:", "Agent Step 5:"
+    )
+    if agent_4_instructions is None:
+        raise ValueError("Agent 4 instructions not found")
+    agent_4_output_objective = extract_section(
+        agent_4_instructions, "Output Objective:"
+    )
+    if agent_4_output_objective is None:
+        raise ValueError("Agent 4 output objective not found")
+
+    object_information = state.object_information
+
+    object = object_information["object"]
+    information = object_information["information"]
+
+    # Create a prompt for the object consolidation
+
+    dc_object_consolidation_prompt = DC_OBJECT_CONSOLIDATION_PROMPT.format(
+        question=question,
+        object=object,
+        information=information,
+        format=agent_4_output_objective,
+    )
+
+    # Run LLM
+
+    msg = [
+        HumanMessage(
+            content=trim_prompt_piece(
+                config=graph_config.tooling.primary_llm.config,
+                prompt_piece=dc_object_consolidation_prompt,
+                reserved_str="",
+            ),
+        )
+    ]
+    primary_llm = graph_config.tooling.primary_llm
+    # Grader
+    try:
+        llm_response = run_with_timeout(
+            30,
+            primary_llm.invoke,
+            prompt=msg,
+            timeout_override=30,
+            max_tokens=300,
+        )
+
+        cleaned_response = str(llm_response.content).replace("```json\n", "")
+        consolidated_information = cleaned_response.split("INFORMATION:")[1]
+
+    except Exception as e:
+        raise ValueError(f"Error in consolidate_object_research: {e}")
+
+    object_research_results = {
+        "object": object,
+        "research_result": consolidated_information,
+    }
+
+    logger.debug(
+        "DivCon Step A4 - Object Research Consolidation - completed for an object"
+    )
+
+    return ObjectResearchUpdate(
+        object_research_results=[object_research_results],
+        log_messages=["Agent Source Consilidation done"],
+    )

backend/onyx/agents/agent_search/dc_search_analysis/nodes/a5_consolidate_research.py

@@ -0,0 +1,127 @@
+from typing import cast
+
+from langchain_core.messages import HumanMessage
+from langchain_core.runnables import RunnableConfig
+from langgraph.types import StreamWriter
+
+from onyx.agents.agent_search.dc_search_analysis.ops import extract_section
+from onyx.agents.agent_search.dc_search_analysis.states import MainState
+from onyx.agents.agent_search.dc_search_analysis.states import ResearchUpdate
+from onyx.agents.agent_search.models import GraphConfig
+from onyx.agents.agent_search.shared_graph_utils.agent_prompt_ops import (
+    trim_prompt_piece,
+)
+from onyx.agents.agent_search.shared_graph_utils.llm import stream_llm_answer
+from onyx.prompts.agents.dc_prompts import DC_FORMATTING_NO_BASE_DATA_PROMPT
+from onyx.prompts.agents.dc_prompts import DC_FORMATTING_WITH_BASE_DATA_PROMPT
+from onyx.utils.logger import setup_logger
+from onyx.utils.threadpool_concurrency import run_with_timeout
+
+logger = setup_logger()
+
+
+def consolidate_research(
+    state: MainState, config: RunnableConfig, writer: StreamWriter = lambda _: None
+) -> ResearchUpdate:
+    """
+    LangGraph node to start the agentic search process.
+    """
+
+    graph_config = cast(GraphConfig, config["metadata"]["config"])
+
+    search_tool = graph_config.tooling.search_tool
+
+    if search_tool is None or graph_config.inputs.persona is None:
+        raise ValueError("Search tool and persona must be provided for DivCon search")
+
+    # Populate prompt
+    instructions = graph_config.inputs.persona.system_prompt or ""
+
+    try:
+        agent_5_instructions = extract_section(
+            instructions, "Agent Step 5:", "Agent End"
+        )
+        if agent_5_instructions is None:
+            raise ValueError("Agent 5 instructions not found")
+        agent_5_base_data = extract_section(instructions, "|Start Data|", "|End Data|")
+        agent_5_task = extract_section(
+            agent_5_instructions, "Task:", "Independent Research Sources:"
+        )
+        if agent_5_task is None:
+            raise ValueError("Agent 5 task not found")
+        agent_5_output_objective = extract_section(
+            agent_5_instructions, "Output Objective:"
+        )
+        if agent_5_output_objective is None:
+            raise ValueError("Agent 5 output objective not found")
+    except ValueError as e:
+        raise ValueError(
+            f"Instructions for Agent Step 5 were not properly formatted: {e}"
+        )
+
+    research_result_list = []
+
+    if agent_5_task.strip() == "*concatenate*":
+        object_research_results = state.object_research_results
+
+        for object_research_result in object_research_results:
+            object = object_research_result["object"]
+            research_result = object_research_result["research_result"]
+            research_result_list.append(f"Object: {object}\n\n{research_result}")
+
+        research_results = "\n\n".join(research_result_list)
+
+    else:
+        raise NotImplementedError("Only '*concatenate*' is currently supported")
+
+    # Create a prompt for the object consolidation
+
+    if agent_5_base_data is None:
+        dc_formatting_prompt = DC_FORMATTING_NO_BASE_DATA_PROMPT.format(
+            text=research_results,
+            format=agent_5_output_objective,
+        )
+    else:
+        dc_formatting_prompt = DC_FORMATTING_WITH_BASE_DATA_PROMPT.format(
+            base_data=agent_5_base_data,
+            text=research_results,
+            format=agent_5_output_objective,
+        )
+
+    # Run LLM
+
+    msg = [
+        HumanMessage(
+            content=trim_prompt_piece(
+                config=graph_config.tooling.primary_llm.config,
+                prompt_piece=dc_formatting_prompt,
+                reserved_str="",
+            ),
+        )
+    ]
+
+    try:
+        _ = run_with_timeout(
+            60,
+            lambda: stream_llm_answer(
+                llm=graph_config.tooling.primary_llm,
+                prompt=msg,
+                event_name="initial_agent_answer",
+                writer=writer,
+                agent_answer_level=0,
+                agent_answer_question_num=0,
+                agent_answer_type="agent_level_answer",
+                timeout_override=30,
+                max_tokens=None,
+            ),
+        )
+
+    except Exception as e:
+        raise ValueError(f"Error in consolidate_research: {e}")
+
+    logger.debug("DivCon Step A5 - Final Generation - completed")
+
+    return ResearchUpdate(
+        research_results=research_results,
+        log_messages=["Agent Source Consilidation done"],
+    )

backend/onyx/agents/agent_search/dc_search_analysis/ops.py

@@ -0,0 +1,61 @@
+from datetime import datetime
+from typing import cast
+
+from onyx.chat.models import LlmDoc
+from onyx.configs.constants import DocumentSource
+from onyx.context.search.models import InferenceSection
+from onyx.db.engine.sql_engine import get_session_with_current_tenant
+from onyx.tools.models import SearchToolOverrideKwargs
+from onyx.tools.tool_implementations.search.search_tool import (
+    FINAL_CONTEXT_DOCUMENTS_ID,
+)
+from onyx.tools.tool_implementations.search.search_tool import SearchTool
+
+
+def research(
+    question: str,
+    search_tool: SearchTool,
+    document_sources: list[DocumentSource] | None = None,
+    time_cutoff: datetime | None = None,
+) -> list[LlmDoc]:
+    # new db session to avoid concurrency issues
+
+    callback_container: list[list[InferenceSection]] = []
+    retrieved_docs: list[LlmDoc] = []
+
+    with get_session_with_current_tenant() as db_session:
+        for tool_response in search_tool.run(
+            query=question,
+            override_kwargs=SearchToolOverrideKwargs(
+                force_no_rerank=False,
+                alternate_db_session=db_session,
+                retrieved_sections_callback=callback_container.append,
+                skip_query_analysis=True,
+                document_sources=document_sources,
+                time_cutoff=time_cutoff,
+            ),
+        ):
+            # get retrieved docs to send to the rest of the graph
+            if tool_response.id == FINAL_CONTEXT_DOCUMENTS_ID:
+                retrieved_docs = cast(list[LlmDoc], tool_response.response)[:10]
+                break
+    return retrieved_docs
+
+
+def extract_section(
+    text: str, start_marker: str, end_marker: str | None = None
+) -> str | None:
+    """Extract text between markers, returning None if markers not found"""
+    parts = text.split(start_marker)
+
+    if len(parts) == 1:
+        return None
+
+    after_start = parts[1].strip()
+
+    if not end_marker:
+        return after_start
+
+    extract = after_start.split(end_marker)[0]
+
+    return extract.strip()

backend/onyx/agents/agent_search/dc_search_analysis/states.py

@@ -0,0 +1,72 @@
+from operator import add
+from typing import Annotated
+from typing import Dict
+from typing import TypedDict
+
+from pydantic import BaseModel
+
+from onyx.agents.agent_search.core_state import CoreState
+from onyx.agents.agent_search.orchestration.states import ToolCallUpdate
+from onyx.agents.agent_search.orchestration.states import ToolChoiceInput
+from onyx.agents.agent_search.orchestration.states import ToolChoiceUpdate
+from onyx.configs.constants import DocumentSource
+
+
+### States ###
+class LoggerUpdate(BaseModel):
+    log_messages: Annotated[list[str], add] = []
+
+
+class SearchSourcesObjectsUpdate(LoggerUpdate):
+    analysis_objects: list[str] = []
+    analysis_sources: list[DocumentSource] = []
+
+
+class ObjectSourceInput(LoggerUpdate):
+    object_source_combination: tuple[str, DocumentSource]
+
+
+class ObjectSourceResearchUpdate(LoggerUpdate):
+    object_source_research_results: Annotated[list[Dict[str, str]], add] = []
+
+
+class ObjectInformationInput(LoggerUpdate):
+    object_information: Dict[str, str]
+
+
+class ObjectResearchInformationUpdate(LoggerUpdate):
+    object_research_information_results: Annotated[list[Dict[str, str]], add] = []
+
+
+class ObjectResearchUpdate(LoggerUpdate):
+    object_research_results: Annotated[list[Dict[str, str]], add] = []
+
+
+class ResearchUpdate(LoggerUpdate):
+    research_results: str | None = None
+
+
+## Graph Input State
+class MainInput(CoreState):
+    pass
+
+
+## Graph State
+class MainState(
+    # This includes the core state
+    MainInput,
+    ToolChoiceInput,
+    ToolCallUpdate,
+    ToolChoiceUpdate,
+    SearchSourcesObjectsUpdate,
+    ObjectSourceResearchUpdate,
+    ObjectResearchInformationUpdate,
+    ObjectResearchUpdate,
+    ResearchUpdate,
+):
+    pass
+
+
+## Graph Output State - presently not used
+class MainOutput(TypedDict):
+    log_messages: list[str]

backend/onyx/agents/agent_search/deep_search/main/models.py

@@ -0,0 +1,36 @@
+from pydantic import BaseModel
+
+
+class RefinementSubQuestion(BaseModel):
+    sub_question: str
+    sub_question_id: str
+    verified: bool
+    answered: bool
+    answer: str
+
+
+class AgentTimings(BaseModel):
+    base_duration_s: float | None
+    refined_duration_s: float | None
+    full_duration_s: float | None
+
+
+class AgentBaseMetrics(BaseModel):
+    num_verified_documents_total: int | None
+    num_verified_documents_core: int | None
+    verified_avg_score_core: float | None
+    num_verified_documents_base: int | float | None
+    verified_avg_score_base: float | None = None
+    base_doc_boost_factor: float | None = None
+    support_boost_factor: float | None = None
+    duration_s: float | None = None
+
+
+class AgentRefinedMetrics(BaseModel):
+    refined_doc_boost_factor: float | None = None
+    refined_question_boost_factor: float | None = None
+    duration_s: float | None = None
+
+
+class AgentAdditionalMetrics(BaseModel):
+    pass

backend/onyx/agents/agent_search/dr/conditional_edges.py

@@ -0,0 +1,61 @@
+from collections.abc import Hashable
+
+from langgraph.graph import END
+from langgraph.types import Send
+
+from onyx.agents.agent_search.dr.enums import DRPath
+from onyx.agents.agent_search.dr.states import MainState
+
+
+def decision_router(state: MainState) -> list[Send | Hashable] | DRPath | str:
+    if not state.tools_used:
+        raise IndexError("state.tools_used cannot be empty")
+
+    # next_tool is either a generic tool name or a DRPath string
+    next_tool_name = state.tools_used[-1]
+
+    available_tools = state.available_tools
+    if not available_tools:
+        raise ValueError("No tool is available. This should not happen.")
+
+    if next_tool_name in available_tools:
+        next_tool_path = available_tools[next_tool_name].path
+    elif next_tool_name == DRPath.END.value:
+        return END
+    elif next_tool_name == DRPath.LOGGER.value:
+        return DRPath.LOGGER
+    elif next_tool_name == DRPath.CLOSER.value:
+        return DRPath.CLOSER
+    else:
+        return DRPath.ORCHESTRATOR
+
+    # handle invalid paths
+    if next_tool_path == DRPath.CLARIFIER:
+        raise ValueError("CLARIFIER is not a valid path during iteration")
+
+    # handle tool calls without a query
+    if (
+        next_tool_path
+        in (
+            DRPath.INTERNAL_SEARCH,
+            DRPath.WEB_SEARCH,
+            DRPath.KNOWLEDGE_GRAPH,
+            DRPath.IMAGE_GENERATION,
+        )
+        and len(state.query_list) == 0
+    ):
+        return DRPath.CLOSER
+
+    return next_tool_path
+
+
+def completeness_router(state: MainState) -> DRPath | str:
+    if not state.tools_used:
+        raise IndexError("tools_used cannot be empty")
+
+    # go to closer if path is CLOSER or no queries
+    next_path = state.tools_used[-1]
+
+    if next_path == DRPath.ORCHESTRATOR.value:
+        return DRPath.ORCHESTRATOR
+    return DRPath.LOGGER