mirror of
https://github.com/onyx-dot-app/onyx.git
synced 2026-03-29 11:32:42 +00:00
Compare commits
15 Commits
v2.9.9
...
tool_name_
| Author | SHA1 | Date | |
|---|---|---|---|
|
1c5d6b88e3 | ||
|
|
2c881c8b61 | ||
|
|
3725e3485c | ||
|
|
d72fd84bcd | ||
|
|
6b7d560b0c | ||
|
|
2af27ba1f5 | ||
|
|
75ac128836 | ||
|
|
a688f4ee81 | ||
|
|
ca54a85929 | ||
|
|
3571a8d39a | ||
|
|
08ac2f2e20 | ||
|
|
9b47267d48 | ||
|
|
c57ee34a33 | ||
|
|
318a66a189 | ||
|
|
edf9e68abf |
387
.github/workflows/deployment.yml
vendored
387
.github/workflows/deployment.yml
vendored
@@ -8,9 +8,7 @@ on:
|
||||
|
||||
# Set restrictive default permissions for all jobs. Jobs that need more permissions
|
||||
# should explicitly declare them.
|
||||
permissions:
|
||||
# Required for OIDC authentication with AWS
|
||||
id-token: write # zizmor: ignore[excessive-permissions]
|
||||
permissions: {}
|
||||
|
||||
env:
|
||||
EDGE_TAG: ${{ startsWith(github.ref_name, 'nightly-latest') }}
|
||||
@@ -152,30 +150,16 @@ jobs:
|
||||
if: always() && needs.check-version-tag.result == 'failure' && github.event_name != 'workflow_dispatch'
|
||||
runs-on: ubuntu-slim
|
||||
timeout-minutes: 10
|
||||
environment: release
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # ratchet:actions/checkout@v6
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Configure AWS credentials
|
||||
uses: aws-actions/configure-aws-credentials@61815dcd50bd041e203e49132bacad1fd04d2708
|
||||
with:
|
||||
role-to-assume: ${{ secrets.AWS_OIDC_ROLE_ARN }}
|
||||
aws-region: us-east-2
|
||||
|
||||
- name: Get AWS Secrets
|
||||
uses: aws-actions/aws-secretsmanager-get-secrets@a9a7eb4e2f2871d30dc5b892576fde60a2ecc802
|
||||
with:
|
||||
secret-ids: |
|
||||
MONITOR_DEPLOYMENTS_WEBHOOK, deploy/monitor-deployments-webhook
|
||||
parse-json-secrets: true
|
||||
|
||||
- name: Send Slack notification
|
||||
uses: ./.github/actions/slack-notify
|
||||
with:
|
||||
webhook-url: ${{ env.MONITOR_DEPLOYMENTS_WEBHOOK }}
|
||||
webhook-url: ${{ secrets.MONITOR_DEPLOYMENTS_WEBHOOK }}
|
||||
failed-jobs: "• check-version-tag"
|
||||
title: "🚨 Version Tag Check Failed"
|
||||
ref-name: ${{ github.ref_name }}
|
||||
@@ -184,7 +168,6 @@ jobs:
|
||||
needs: determine-builds
|
||||
if: needs.determine-builds.outputs.build-desktop == 'true'
|
||||
permissions:
|
||||
id-token: write
|
||||
contents: write
|
||||
actions: read
|
||||
strategy:
|
||||
@@ -202,33 +185,12 @@ jobs:
|
||||
|
||||
runs-on: ${{ matrix.platform }}
|
||||
timeout-minutes: 90
|
||||
environment: release
|
||||
steps:
|
||||
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # ratchet:actions/checkout@v6.0.1
|
||||
with:
|
||||
# NOTE: persist-credentials is needed for tauri-action to create GitHub releases.
|
||||
persist-credentials: true # zizmor: ignore[artipacked]
|
||||
|
||||
- name: Configure AWS credentials
|
||||
if: startsWith(matrix.platform, 'macos-')
|
||||
uses: aws-actions/configure-aws-credentials@61815dcd50bd041e203e49132bacad1fd04d2708
|
||||
with:
|
||||
role-to-assume: ${{ secrets.AWS_OIDC_ROLE_ARN }}
|
||||
aws-region: us-east-2
|
||||
|
||||
- name: Get AWS Secrets
|
||||
if: startsWith(matrix.platform, 'macos-')
|
||||
uses: aws-actions/aws-secretsmanager-get-secrets@a9a7eb4e2f2871d30dc5b892576fde60a2ecc802
|
||||
with:
|
||||
secret-ids: |
|
||||
APPLE_ID, deploy/apple-id
|
||||
APPLE_PASSWORD, deploy/apple-password
|
||||
APPLE_CERTIFICATE, deploy/apple-certificate
|
||||
APPLE_CERTIFICATE_PASSWORD, deploy/apple-certificate-password
|
||||
KEYCHAIN_PASSWORD, deploy/keychain-password
|
||||
APPLE_TEAM_ID, deploy/apple-team-id
|
||||
parse-json-secrets: true
|
||||
|
||||
- name: install dependencies (ubuntu only)
|
||||
if: startsWith(matrix.platform, 'ubuntu-')
|
||||
run: |
|
||||
@@ -323,40 +285,15 @@ jobs:
|
||||
|
||||
Write-Host "Versions set to: $VERSION"
|
||||
|
||||
- name: Import Apple Developer Certificate
|
||||
if: startsWith(matrix.platform, 'macos-')
|
||||
run: |
|
||||
echo $APPLE_CERTIFICATE | base64 --decode > certificate.p12
|
||||
security create-keychain -p "$KEYCHAIN_PASSWORD" build.keychain
|
||||
security default-keychain -s build.keychain
|
||||
security unlock-keychain -p "$KEYCHAIN_PASSWORD" build.keychain
|
||||
security set-keychain-settings -t 3600 -u build.keychain
|
||||
security import certificate.p12 -k build.keychain -P "$APPLE_CERTIFICATE_PASSWORD" -T /usr/bin/codesign
|
||||
security set-key-partition-list -S apple-tool:,apple:,codesign: -s -k "$KEYCHAIN_PASSWORD" build.keychain
|
||||
security find-identity -v -p codesigning build.keychain
|
||||
|
||||
- name: Verify Certificate
|
||||
if: startsWith(matrix.platform, 'macos-')
|
||||
run: |
|
||||
CERT_INFO=$(security find-identity -v -p codesigning build.keychain | grep -E "(Developer ID Application|Apple Distribution|Apple Development)" | head -n 1)
|
||||
CERT_ID=$(echo "$CERT_INFO" | awk -F'"' '{print $2}')
|
||||
echo "CERT_ID=$CERT_ID" >> $GITHUB_ENV
|
||||
echo "Certificate imported."
|
||||
|
||||
- uses: tauri-apps/tauri-action@73fb865345c54760d875b94642314f8c0c894afa # ratchet:tauri-apps/tauri-action@action-v0.6.1
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
APPLE_ID: ${{ env.APPLE_ID }}
|
||||
APPLE_PASSWORD: ${{ env.APPLE_PASSWORD }}
|
||||
APPLE_SIGNING_IDENTITY: ${{ env.CERT_ID }}
|
||||
APPLE_TEAM_ID: ${{ env.APPLE_TEAM_ID }}
|
||||
with:
|
||||
tagName: ${{ needs.determine-builds.outputs.is-test-run != 'true' && 'v__VERSION__' || format('v0.0.0-dev+{0}', needs.determine-builds.outputs.short-sha) }}
|
||||
releaseName: ${{ needs.determine-builds.outputs.is-test-run != 'true' && 'v__VERSION__' || format('v0.0.0-dev+{0}', needs.determine-builds.outputs.short-sha) }}
|
||||
releaseBody: "See the assets to download this version and install."
|
||||
releaseDraft: true
|
||||
prerelease: false
|
||||
assetNamePattern: "[name]_[arch][ext]"
|
||||
args: ${{ matrix.args }}
|
||||
|
||||
build-web-amd64:
|
||||
@@ -368,7 +305,6 @@ jobs:
|
||||
- run-id=${{ github.run_id }}-web-amd64
|
||||
- extras=ecr-cache
|
||||
timeout-minutes: 90
|
||||
environment: release
|
||||
outputs:
|
||||
digest: ${{ steps.build.outputs.digest }}
|
||||
env:
|
||||
@@ -381,20 +317,6 @@ jobs:
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Configure AWS credentials
|
||||
uses: aws-actions/configure-aws-credentials@61815dcd50bd041e203e49132bacad1fd04d2708
|
||||
with:
|
||||
role-to-assume: ${{ secrets.AWS_OIDC_ROLE_ARN }}
|
||||
aws-region: us-east-2
|
||||
|
||||
- name: Get AWS Secrets
|
||||
uses: aws-actions/aws-secretsmanager-get-secrets@a9a7eb4e2f2871d30dc5b892576fde60a2ecc802
|
||||
with:
|
||||
secret-ids: |
|
||||
DOCKER_USERNAME, deploy/docker-username
|
||||
DOCKER_TOKEN, deploy/docker-token
|
||||
parse-json-secrets: true
|
||||
|
||||
- name: Docker meta
|
||||
id: meta
|
||||
uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # ratchet:docker/metadata-action@v5
|
||||
@@ -409,8 +331,8 @@ jobs:
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # ratchet:docker/login-action@v3
|
||||
with:
|
||||
username: ${{ env.DOCKER_USERNAME }}
|
||||
password: ${{ env.DOCKER_TOKEN }}
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_TOKEN }}
|
||||
|
||||
- name: Build and push AMD64
|
||||
id: build
|
||||
@@ -441,7 +363,6 @@ jobs:
|
||||
- run-id=${{ github.run_id }}-web-arm64
|
||||
- extras=ecr-cache
|
||||
timeout-minutes: 90
|
||||
environment: release
|
||||
outputs:
|
||||
digest: ${{ steps.build.outputs.digest }}
|
||||
env:
|
||||
@@ -454,20 +375,6 @@ jobs:
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Configure AWS credentials
|
||||
uses: aws-actions/configure-aws-credentials@61815dcd50bd041e203e49132bacad1fd04d2708
|
||||
with:
|
||||
role-to-assume: ${{ secrets.AWS_OIDC_ROLE_ARN }}
|
||||
aws-region: us-east-2
|
||||
|
||||
- name: Get AWS Secrets
|
||||
uses: aws-actions/aws-secretsmanager-get-secrets@a9a7eb4e2f2871d30dc5b892576fde60a2ecc802
|
||||
with:
|
||||
secret-ids: |
|
||||
DOCKER_USERNAME, deploy/docker-username
|
||||
DOCKER_TOKEN, deploy/docker-token
|
||||
parse-json-secrets: true
|
||||
|
||||
- name: Docker meta
|
||||
id: meta
|
||||
uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # ratchet:docker/metadata-action@v5
|
||||
@@ -482,8 +389,8 @@ jobs:
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # ratchet:docker/login-action@v3
|
||||
with:
|
||||
username: ${{ env.DOCKER_USERNAME }}
|
||||
password: ${{ env.DOCKER_TOKEN }}
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_TOKEN }}
|
||||
|
||||
- name: Build and push ARM64
|
||||
id: build
|
||||
@@ -516,34 +423,19 @@ jobs:
|
||||
- run-id=${{ github.run_id }}-merge-web
|
||||
- extras=ecr-cache
|
||||
timeout-minutes: 90
|
||||
environment: release
|
||||
env:
|
||||
REGISTRY_IMAGE: onyxdotapp/onyx-web-server
|
||||
steps:
|
||||
- uses: runs-on/action@cd2b598b0515d39d78c38a02d529db87d2196d1e # ratchet:runs-on/action@v2
|
||||
|
||||
- name: Configure AWS credentials
|
||||
uses: aws-actions/configure-aws-credentials@61815dcd50bd041e203e49132bacad1fd04d2708
|
||||
with:
|
||||
role-to-assume: ${{ secrets.AWS_OIDC_ROLE_ARN }}
|
||||
aws-region: us-east-2
|
||||
|
||||
- name: Get AWS Secrets
|
||||
uses: aws-actions/aws-secretsmanager-get-secrets@a9a7eb4e2f2871d30dc5b892576fde60a2ecc802
|
||||
with:
|
||||
secret-ids: |
|
||||
DOCKER_USERNAME, deploy/docker-username
|
||||
DOCKER_TOKEN, deploy/docker-token
|
||||
parse-json-secrets: true
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # ratchet:docker/setup-buildx-action@v3
|
||||
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # ratchet:docker/login-action@v3
|
||||
with:
|
||||
username: ${{ env.DOCKER_USERNAME }}
|
||||
password: ${{ env.DOCKER_TOKEN }}
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_TOKEN }}
|
||||
|
||||
- name: Docker meta
|
||||
id: meta
|
||||
@@ -579,7 +471,6 @@ jobs:
|
||||
- run-id=${{ github.run_id }}-web-cloud-amd64
|
||||
- extras=ecr-cache
|
||||
timeout-minutes: 90
|
||||
environment: release
|
||||
outputs:
|
||||
digest: ${{ steps.build.outputs.digest }}
|
||||
env:
|
||||
@@ -592,20 +483,6 @@ jobs:
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Configure AWS credentials
|
||||
uses: aws-actions/configure-aws-credentials@61815dcd50bd041e203e49132bacad1fd04d2708
|
||||
with:
|
||||
role-to-assume: ${{ secrets.AWS_OIDC_ROLE_ARN }}
|
||||
aws-region: us-east-2
|
||||
|
||||
- name: Get AWS Secrets
|
||||
uses: aws-actions/aws-secretsmanager-get-secrets@a9a7eb4e2f2871d30dc5b892576fde60a2ecc802
|
||||
with:
|
||||
secret-ids: |
|
||||
DOCKER_USERNAME, deploy/docker-username
|
||||
DOCKER_TOKEN, deploy/docker-token
|
||||
parse-json-secrets: true
|
||||
|
||||
- name: Docker meta
|
||||
id: meta
|
||||
uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # ratchet:docker/metadata-action@v5
|
||||
@@ -620,8 +497,8 @@ jobs:
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # ratchet:docker/login-action@v3
|
||||
with:
|
||||
username: ${{ env.DOCKER_USERNAME }}
|
||||
password: ${{ env.DOCKER_TOKEN }}
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_TOKEN }}
|
||||
|
||||
- name: Build and push AMD64
|
||||
id: build
|
||||
@@ -660,7 +537,6 @@ jobs:
|
||||
- run-id=${{ github.run_id }}-web-cloud-arm64
|
||||
- extras=ecr-cache
|
||||
timeout-minutes: 90
|
||||
environment: release
|
||||
outputs:
|
||||
digest: ${{ steps.build.outputs.digest }}
|
||||
env:
|
||||
@@ -673,20 +549,6 @@ jobs:
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Configure AWS credentials
|
||||
uses: aws-actions/configure-aws-credentials@61815dcd50bd041e203e49132bacad1fd04d2708
|
||||
with:
|
||||
role-to-assume: ${{ secrets.AWS_OIDC_ROLE_ARN }}
|
||||
aws-region: us-east-2
|
||||
|
||||
- name: Get AWS Secrets
|
||||
uses: aws-actions/aws-secretsmanager-get-secrets@a9a7eb4e2f2871d30dc5b892576fde60a2ecc802
|
||||
with:
|
||||
secret-ids: |
|
||||
DOCKER_USERNAME, deploy/docker-username
|
||||
DOCKER_TOKEN, deploy/docker-token
|
||||
parse-json-secrets: true
|
||||
|
||||
- name: Docker meta
|
||||
id: meta
|
||||
uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # ratchet:docker/metadata-action@v5
|
||||
@@ -701,8 +563,8 @@ jobs:
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # ratchet:docker/login-action@v3
|
||||
with:
|
||||
username: ${{ env.DOCKER_USERNAME }}
|
||||
password: ${{ env.DOCKER_TOKEN }}
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_TOKEN }}
|
||||
|
||||
- name: Build and push ARM64
|
||||
id: build
|
||||
@@ -743,34 +605,19 @@ jobs:
|
||||
- run-id=${{ github.run_id }}-merge-web-cloud
|
||||
- extras=ecr-cache
|
||||
timeout-minutes: 90
|
||||
environment: release
|
||||
env:
|
||||
REGISTRY_IMAGE: onyxdotapp/onyx-web-server-cloud
|
||||
steps:
|
||||
- uses: runs-on/action@cd2b598b0515d39d78c38a02d529db87d2196d1e # ratchet:runs-on/action@v2
|
||||
|
||||
- name: Configure AWS credentials
|
||||
uses: aws-actions/configure-aws-credentials@61815dcd50bd041e203e49132bacad1fd04d2708
|
||||
with:
|
||||
role-to-assume: ${{ secrets.AWS_OIDC_ROLE_ARN }}
|
||||
aws-region: us-east-2
|
||||
|
||||
- name: Get AWS Secrets
|
||||
uses: aws-actions/aws-secretsmanager-get-secrets@a9a7eb4e2f2871d30dc5b892576fde60a2ecc802
|
||||
with:
|
||||
secret-ids: |
|
||||
DOCKER_USERNAME, deploy/docker-username
|
||||
DOCKER_TOKEN, deploy/docker-token
|
||||
parse-json-secrets: true
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # ratchet:docker/setup-buildx-action@v3
|
||||
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # ratchet:docker/login-action@v3
|
||||
with:
|
||||
username: ${{ env.DOCKER_USERNAME }}
|
||||
password: ${{ env.DOCKER_TOKEN }}
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_TOKEN }}
|
||||
|
||||
- name: Docker meta
|
||||
id: meta
|
||||
@@ -803,7 +650,6 @@ jobs:
|
||||
- run-id=${{ github.run_id }}-backend-amd64
|
||||
- extras=ecr-cache
|
||||
timeout-minutes: 90
|
||||
environment: release
|
||||
outputs:
|
||||
digest: ${{ steps.build.outputs.digest }}
|
||||
env:
|
||||
@@ -816,20 +662,6 @@ jobs:
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Configure AWS credentials
|
||||
uses: aws-actions/configure-aws-credentials@61815dcd50bd041e203e49132bacad1fd04d2708
|
||||
with:
|
||||
role-to-assume: ${{ secrets.AWS_OIDC_ROLE_ARN }}
|
||||
aws-region: us-east-2
|
||||
|
||||
- name: Get AWS Secrets
|
||||
uses: aws-actions/aws-secretsmanager-get-secrets@a9a7eb4e2f2871d30dc5b892576fde60a2ecc802
|
||||
with:
|
||||
secret-ids: |
|
||||
DOCKER_USERNAME, deploy/docker-username
|
||||
DOCKER_TOKEN, deploy/docker-token
|
||||
parse-json-secrets: true
|
||||
|
||||
- name: Docker meta
|
||||
id: meta
|
||||
uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # ratchet:docker/metadata-action@v5
|
||||
@@ -844,8 +676,8 @@ jobs:
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # ratchet:docker/login-action@v3
|
||||
with:
|
||||
username: ${{ env.DOCKER_USERNAME }}
|
||||
password: ${{ env.DOCKER_TOKEN }}
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_TOKEN }}
|
||||
|
||||
- name: Build and push AMD64
|
||||
id: build
|
||||
@@ -875,7 +707,6 @@ jobs:
|
||||
- run-id=${{ github.run_id }}-backend-arm64
|
||||
- extras=ecr-cache
|
||||
timeout-minutes: 90
|
||||
environment: release
|
||||
outputs:
|
||||
digest: ${{ steps.build.outputs.digest }}
|
||||
env:
|
||||
@@ -888,20 +719,6 @@ jobs:
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Configure AWS credentials
|
||||
uses: aws-actions/configure-aws-credentials@61815dcd50bd041e203e49132bacad1fd04d2708
|
||||
with:
|
||||
role-to-assume: ${{ secrets.AWS_OIDC_ROLE_ARN }}
|
||||
aws-region: us-east-2
|
||||
|
||||
- name: Get AWS Secrets
|
||||
uses: aws-actions/aws-secretsmanager-get-secrets@a9a7eb4e2f2871d30dc5b892576fde60a2ecc802
|
||||
with:
|
||||
secret-ids: |
|
||||
DOCKER_USERNAME, deploy/docker-username
|
||||
DOCKER_TOKEN, deploy/docker-token
|
||||
parse-json-secrets: true
|
||||
|
||||
- name: Docker meta
|
||||
id: meta
|
||||
uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # ratchet:docker/metadata-action@v5
|
||||
@@ -916,8 +733,8 @@ jobs:
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # ratchet:docker/login-action@v3
|
||||
with:
|
||||
username: ${{ env.DOCKER_USERNAME }}
|
||||
password: ${{ env.DOCKER_TOKEN }}
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_TOKEN }}
|
||||
|
||||
- name: Build and push ARM64
|
||||
id: build
|
||||
@@ -949,34 +766,19 @@ jobs:
|
||||
- run-id=${{ github.run_id }}-merge-backend
|
||||
- extras=ecr-cache
|
||||
timeout-minutes: 90
|
||||
environment: release
|
||||
env:
|
||||
REGISTRY_IMAGE: ${{ contains(github.ref_name, 'cloud') && 'onyxdotapp/onyx-backend-cloud' || 'onyxdotapp/onyx-backend' }}
|
||||
steps:
|
||||
- uses: runs-on/action@cd2b598b0515d39d78c38a02d529db87d2196d1e # ratchet:runs-on/action@v2
|
||||
|
||||
- name: Configure AWS credentials
|
||||
uses: aws-actions/configure-aws-credentials@61815dcd50bd041e203e49132bacad1fd04d2708
|
||||
with:
|
||||
role-to-assume: ${{ secrets.AWS_OIDC_ROLE_ARN }}
|
||||
aws-region: us-east-2
|
||||
|
||||
- name: Get AWS Secrets
|
||||
uses: aws-actions/aws-secretsmanager-get-secrets@a9a7eb4e2f2871d30dc5b892576fde60a2ecc802
|
||||
with:
|
||||
secret-ids: |
|
||||
DOCKER_USERNAME, deploy/docker-username
|
||||
DOCKER_TOKEN, deploy/docker-token
|
||||
parse-json-secrets: true
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # ratchet:docker/setup-buildx-action@v3
|
||||
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # ratchet:docker/login-action@v3
|
||||
with:
|
||||
username: ${{ env.DOCKER_USERNAME }}
|
||||
password: ${{ env.DOCKER_TOKEN }}
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_TOKEN }}
|
||||
|
||||
- name: Docker meta
|
||||
id: meta
|
||||
@@ -1013,7 +815,6 @@ jobs:
|
||||
- volume=40gb
|
||||
- extras=ecr-cache
|
||||
timeout-minutes: 90
|
||||
environment: release
|
||||
outputs:
|
||||
digest: ${{ steps.build.outputs.digest }}
|
||||
env:
|
||||
@@ -1026,20 +827,6 @@ jobs:
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Configure AWS credentials
|
||||
uses: aws-actions/configure-aws-credentials@61815dcd50bd041e203e49132bacad1fd04d2708
|
||||
with:
|
||||
role-to-assume: ${{ secrets.AWS_OIDC_ROLE_ARN }}
|
||||
aws-region: us-east-2
|
||||
|
||||
- name: Get AWS Secrets
|
||||
uses: aws-actions/aws-secretsmanager-get-secrets@a9a7eb4e2f2871d30dc5b892576fde60a2ecc802
|
||||
with:
|
||||
secret-ids: |
|
||||
DOCKER_USERNAME, deploy/docker-username
|
||||
DOCKER_TOKEN, deploy/docker-token
|
||||
parse-json-secrets: true
|
||||
|
||||
- name: Docker meta
|
||||
id: meta
|
||||
uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # ratchet:docker/metadata-action@v5
|
||||
@@ -1056,8 +843,8 @@ jobs:
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # ratchet:docker/login-action@v3
|
||||
with:
|
||||
username: ${{ env.DOCKER_USERNAME }}
|
||||
password: ${{ env.DOCKER_TOKEN }}
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_TOKEN }}
|
||||
|
||||
- name: Build and push AMD64
|
||||
id: build
|
||||
@@ -1092,7 +879,6 @@ jobs:
|
||||
- volume=40gb
|
||||
- extras=ecr-cache
|
||||
timeout-minutes: 90
|
||||
environment: release
|
||||
outputs:
|
||||
digest: ${{ steps.build.outputs.digest }}
|
||||
env:
|
||||
@@ -1105,20 +891,6 @@ jobs:
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Configure AWS credentials
|
||||
uses: aws-actions/configure-aws-credentials@61815dcd50bd041e203e49132bacad1fd04d2708
|
||||
with:
|
||||
role-to-assume: ${{ secrets.AWS_OIDC_ROLE_ARN }}
|
||||
aws-region: us-east-2
|
||||
|
||||
- name: Get AWS Secrets
|
||||
uses: aws-actions/aws-secretsmanager-get-secrets@a9a7eb4e2f2871d30dc5b892576fde60a2ecc802
|
||||
with:
|
||||
secret-ids: |
|
||||
DOCKER_USERNAME, deploy/docker-username
|
||||
DOCKER_TOKEN, deploy/docker-token
|
||||
parse-json-secrets: true
|
||||
|
||||
- name: Docker meta
|
||||
id: meta
|
||||
uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # ratchet:docker/metadata-action@v5
|
||||
@@ -1135,8 +907,8 @@ jobs:
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # ratchet:docker/login-action@v3
|
||||
with:
|
||||
username: ${{ env.DOCKER_USERNAME }}
|
||||
password: ${{ env.DOCKER_TOKEN }}
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_TOKEN }}
|
||||
|
||||
- name: Build and push ARM64
|
||||
id: build
|
||||
@@ -1172,34 +944,19 @@ jobs:
|
||||
- run-id=${{ github.run_id }}-merge-model-server
|
||||
- extras=ecr-cache
|
||||
timeout-minutes: 90
|
||||
environment: release
|
||||
env:
|
||||
REGISTRY_IMAGE: ${{ contains(github.ref_name, 'cloud') && 'onyxdotapp/onyx-model-server-cloud' || 'onyxdotapp/onyx-model-server' }}
|
||||
steps:
|
||||
- uses: runs-on/action@cd2b598b0515d39d78c38a02d529db87d2196d1e # ratchet:runs-on/action@v2
|
||||
|
||||
- name: Configure AWS credentials
|
||||
uses: aws-actions/configure-aws-credentials@61815dcd50bd041e203e49132bacad1fd04d2708
|
||||
with:
|
||||
role-to-assume: ${{ secrets.AWS_OIDC_ROLE_ARN }}
|
||||
aws-region: us-east-2
|
||||
|
||||
- name: Get AWS Secrets
|
||||
uses: aws-actions/aws-secretsmanager-get-secrets@a9a7eb4e2f2871d30dc5b892576fde60a2ecc802
|
||||
with:
|
||||
secret-ids: |
|
||||
DOCKER_USERNAME, deploy/docker-username
|
||||
DOCKER_TOKEN, deploy/docker-token
|
||||
parse-json-secrets: true
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # ratchet:docker/setup-buildx-action@v3
|
||||
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # ratchet:docker/login-action@v3
|
||||
with:
|
||||
username: ${{ env.DOCKER_USERNAME }}
|
||||
password: ${{ env.DOCKER_TOKEN }}
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_TOKEN }}
|
||||
|
||||
- name: Docker meta
|
||||
id: meta
|
||||
@@ -1237,26 +994,11 @@ jobs:
|
||||
- run-id=${{ github.run_id }}-trivy-scan-web
|
||||
- extras=ecr-cache
|
||||
timeout-minutes: 90
|
||||
environment: release
|
||||
env:
|
||||
REGISTRY_IMAGE: onyxdotapp/onyx-web-server
|
||||
steps:
|
||||
- uses: runs-on/action@cd2b598b0515d39d78c38a02d529db87d2196d1e # ratchet:runs-on/action@v2
|
||||
|
||||
- name: Configure AWS credentials
|
||||
uses: aws-actions/configure-aws-credentials@61815dcd50bd041e203e49132bacad1fd04d2708
|
||||
with:
|
||||
role-to-assume: ${{ secrets.AWS_OIDC_ROLE_ARN }}
|
||||
aws-region: us-east-2
|
||||
|
||||
- name: Get AWS Secrets
|
||||
uses: aws-actions/aws-secretsmanager-get-secrets@a9a7eb4e2f2871d30dc5b892576fde60a2ecc802
|
||||
with:
|
||||
secret-ids: |
|
||||
DOCKER_USERNAME, deploy/docker-username
|
||||
DOCKER_TOKEN, deploy/docker-token
|
||||
parse-json-secrets: true
|
||||
|
||||
- name: Run Trivy vulnerability scanner
|
||||
uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # ratchet:nick-fields/retry@v3
|
||||
with:
|
||||
@@ -1272,8 +1014,8 @@ jobs:
|
||||
docker run --rm -v $HOME/.cache/trivy:/root/.cache/trivy \
|
||||
-e TRIVY_DB_REPOSITORY="public.ecr.aws/aquasecurity/trivy-db:2" \
|
||||
-e TRIVY_JAVA_DB_REPOSITORY="public.ecr.aws/aquasecurity/trivy-java-db:1" \
|
||||
-e TRIVY_USERNAME="${{ env.DOCKER_USERNAME }}" \
|
||||
-e TRIVY_PASSWORD="${{ env.DOCKER_TOKEN }}" \
|
||||
-e TRIVY_USERNAME="${{ secrets.DOCKER_USERNAME }}" \
|
||||
-e TRIVY_PASSWORD="${{ secrets.DOCKER_TOKEN }}" \
|
||||
aquasec/trivy@sha256:a22415a38938a56c379387a8163fcb0ce38b10ace73e593475d3658d578b2436 \
|
||||
image \
|
||||
--skip-version-check \
|
||||
@@ -1292,26 +1034,11 @@ jobs:
|
||||
- run-id=${{ github.run_id }}-trivy-scan-web-cloud
|
||||
- extras=ecr-cache
|
||||
timeout-minutes: 90
|
||||
environment: release
|
||||
env:
|
||||
REGISTRY_IMAGE: onyxdotapp/onyx-web-server-cloud
|
||||
steps:
|
||||
- uses: runs-on/action@cd2b598b0515d39d78c38a02d529db87d2196d1e # ratchet:runs-on/action@v2
|
||||
|
||||
- name: Configure AWS credentials
|
||||
uses: aws-actions/configure-aws-credentials@61815dcd50bd041e203e49132bacad1fd04d2708
|
||||
with:
|
||||
role-to-assume: ${{ secrets.AWS_OIDC_ROLE_ARN }}
|
||||
aws-region: us-east-2
|
||||
|
||||
- name: Get AWS Secrets
|
||||
uses: aws-actions/aws-secretsmanager-get-secrets@a9a7eb4e2f2871d30dc5b892576fde60a2ecc802
|
||||
with:
|
||||
secret-ids: |
|
||||
DOCKER_USERNAME, deploy/docker-username
|
||||
DOCKER_TOKEN, deploy/docker-token
|
||||
parse-json-secrets: true
|
||||
|
||||
- name: Run Trivy vulnerability scanner
|
||||
uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # ratchet:nick-fields/retry@v3
|
||||
with:
|
||||
@@ -1327,8 +1054,8 @@ jobs:
|
||||
docker run --rm -v $HOME/.cache/trivy:/root/.cache/trivy \
|
||||
-e TRIVY_DB_REPOSITORY="public.ecr.aws/aquasecurity/trivy-db:2" \
|
||||
-e TRIVY_JAVA_DB_REPOSITORY="public.ecr.aws/aquasecurity/trivy-java-db:1" \
|
||||
-e TRIVY_USERNAME="${{ env.DOCKER_USERNAME }}" \
|
||||
-e TRIVY_PASSWORD="${{ env.DOCKER_TOKEN }}" \
|
||||
-e TRIVY_USERNAME="${{ secrets.DOCKER_USERNAME }}" \
|
||||
-e TRIVY_PASSWORD="${{ secrets.DOCKER_TOKEN }}" \
|
||||
aquasec/trivy@sha256:a22415a38938a56c379387a8163fcb0ce38b10ace73e593475d3658d578b2436 \
|
||||
image \
|
||||
--skip-version-check \
|
||||
@@ -1347,7 +1074,6 @@ jobs:
|
||||
- run-id=${{ github.run_id }}-trivy-scan-backend
|
||||
- extras=ecr-cache
|
||||
timeout-minutes: 90
|
||||
environment: release
|
||||
env:
|
||||
REGISTRY_IMAGE: ${{ contains(github.ref_name, 'cloud') && 'onyxdotapp/onyx-backend-cloud' || 'onyxdotapp/onyx-backend' }}
|
||||
steps:
|
||||
@@ -1358,20 +1084,6 @@ jobs:
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Configure AWS credentials
|
||||
uses: aws-actions/configure-aws-credentials@61815dcd50bd041e203e49132bacad1fd04d2708
|
||||
with:
|
||||
role-to-assume: ${{ secrets.AWS_OIDC_ROLE_ARN }}
|
||||
aws-region: us-east-2
|
||||
|
||||
- name: Get AWS Secrets
|
||||
uses: aws-actions/aws-secretsmanager-get-secrets@a9a7eb4e2f2871d30dc5b892576fde60a2ecc802
|
||||
with:
|
||||
secret-ids: |
|
||||
DOCKER_USERNAME, deploy/docker-username
|
||||
DOCKER_TOKEN, deploy/docker-token
|
||||
parse-json-secrets: true
|
||||
|
||||
- name: Run Trivy vulnerability scanner
|
||||
uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # ratchet:nick-fields/retry@v3
|
||||
with:
|
||||
@@ -1388,8 +1100,8 @@ jobs:
|
||||
-v ${{ github.workspace }}/backend/.trivyignore:/tmp/.trivyignore:ro \
|
||||
-e TRIVY_DB_REPOSITORY="public.ecr.aws/aquasecurity/trivy-db:2" \
|
||||
-e TRIVY_JAVA_DB_REPOSITORY="public.ecr.aws/aquasecurity/trivy-java-db:1" \
|
||||
-e TRIVY_USERNAME="${{ env.DOCKER_USERNAME }}" \
|
||||
-e TRIVY_PASSWORD="${{ env.DOCKER_TOKEN }}" \
|
||||
-e TRIVY_USERNAME="${{ secrets.DOCKER_USERNAME }}" \
|
||||
-e TRIVY_PASSWORD="${{ secrets.DOCKER_TOKEN }}" \
|
||||
aquasec/trivy@sha256:a22415a38938a56c379387a8163fcb0ce38b10ace73e593475d3658d578b2436 \
|
||||
image \
|
||||
--skip-version-check \
|
||||
@@ -1409,26 +1121,11 @@ jobs:
|
||||
- run-id=${{ github.run_id }}-trivy-scan-model-server
|
||||
- extras=ecr-cache
|
||||
timeout-minutes: 90
|
||||
environment: release
|
||||
env:
|
||||
REGISTRY_IMAGE: ${{ contains(github.ref_name, 'cloud') && 'onyxdotapp/onyx-model-server-cloud' || 'onyxdotapp/onyx-model-server' }}
|
||||
steps:
|
||||
- uses: runs-on/action@cd2b598b0515d39d78c38a02d529db87d2196d1e # ratchet:runs-on/action@v2
|
||||
|
||||
- name: Configure AWS credentials
|
||||
uses: aws-actions/configure-aws-credentials@61815dcd50bd041e203e49132bacad1fd04d2708
|
||||
with:
|
||||
role-to-assume: ${{ secrets.AWS_OIDC_ROLE_ARN }}
|
||||
aws-region: us-east-2
|
||||
|
||||
- name: Get AWS Secrets
|
||||
uses: aws-actions/aws-secretsmanager-get-secrets@a9a7eb4e2f2871d30dc5b892576fde60a2ecc802
|
||||
with:
|
||||
secret-ids: |
|
||||
DOCKER_USERNAME, deploy/docker-username
|
||||
DOCKER_TOKEN, deploy/docker-token
|
||||
parse-json-secrets: true
|
||||
|
||||
- name: Run Trivy vulnerability scanner
|
||||
uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # ratchet:nick-fields/retry@v3
|
||||
with:
|
||||
@@ -1444,8 +1141,8 @@ jobs:
|
||||
docker run --rm -v $HOME/.cache/trivy:/root/.cache/trivy \
|
||||
-e TRIVY_DB_REPOSITORY="public.ecr.aws/aquasecurity/trivy-db:2" \
|
||||
-e TRIVY_JAVA_DB_REPOSITORY="public.ecr.aws/aquasecurity/trivy-java-db:1" \
|
||||
-e TRIVY_USERNAME="${{ env.DOCKER_USERNAME }}" \
|
||||
-e TRIVY_PASSWORD="${{ env.DOCKER_TOKEN }}" \
|
||||
-e TRIVY_USERNAME="${{ secrets.DOCKER_USERNAME }}" \
|
||||
-e TRIVY_PASSWORD="${{ secrets.DOCKER_TOKEN }}" \
|
||||
aquasec/trivy@sha256:a22415a38938a56c379387a8163fcb0ce38b10ace73e593475d3658d578b2436 \
|
||||
image \
|
||||
--skip-version-check \
|
||||
@@ -1473,26 +1170,12 @@ jobs:
|
||||
# NOTE: Github-hosted runners have about 20s faster queue times and are preferred here.
|
||||
runs-on: ubuntu-slim
|
||||
timeout-minutes: 90
|
||||
environment: release
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # ratchet:actions/checkout@v6
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Configure AWS credentials
|
||||
uses: aws-actions/configure-aws-credentials@61815dcd50bd041e203e49132bacad1fd04d2708
|
||||
with:
|
||||
role-to-assume: ${{ secrets.AWS_OIDC_ROLE_ARN }}
|
||||
aws-region: us-east-2
|
||||
|
||||
- name: Get AWS Secrets
|
||||
uses: aws-actions/aws-secretsmanager-get-secrets@a9a7eb4e2f2871d30dc5b892576fde60a2ecc802
|
||||
with:
|
||||
secret-ids: |
|
||||
MONITOR_DEPLOYMENTS_WEBHOOK, deploy/monitor-deployments-webhook
|
||||
parse-json-secrets: true
|
||||
|
||||
- name: Determine failed jobs
|
||||
id: failed-jobs
|
||||
shell: bash
|
||||
@@ -1558,7 +1241,7 @@ jobs:
|
||||
- name: Send Slack notification
|
||||
uses: ./.github/actions/slack-notify
|
||||
with:
|
||||
webhook-url: ${{ env.MONITOR_DEPLOYMENTS_WEBHOOK }}
|
||||
webhook-url: ${{ secrets.MONITOR_DEPLOYMENTS_WEBHOOK }}
|
||||
failed-jobs: ${{ steps.failed-jobs.outputs.jobs }}
|
||||
title: "🚨 Deployment Workflow Failed"
|
||||
ref-name: ${{ github.ref_name }}
|
||||
|
||||
3
.github/workflows/pr-python-checks.yml
vendored
3
.github/workflows/pr-python-checks.yml
vendored
@@ -50,9 +50,8 @@ jobs:
|
||||
uses: runs-on/cache@50350ad4242587b6c8c2baa2e740b1bc11285ff4 # ratchet:runs-on/cache@v4
|
||||
with:
|
||||
path: backend/.mypy_cache
|
||||
key: mypy-${{ runner.os }}-${{ github.base_ref || github.event.merge_group.base_ref || 'main' }}-${{ hashFiles('**/*.py', '**/*.pyi', 'backend/pyproject.toml') }}
|
||||
key: mypy-${{ runner.os }}-${{ hashFiles('**/*.py', '**/*.pyi', 'backend/pyproject.toml') }}
|
||||
restore-keys: |
|
||||
mypy-${{ runner.os }}-${{ github.base_ref || github.event.merge_group.base_ref || 'main' }}-
|
||||
mypy-${{ runner.os }}-
|
||||
|
||||
- name: Run MyPy
|
||||
|
||||
@@ -42,9 +42,7 @@ RUN apt-get update && \
|
||||
pkg-config \
|
||||
gcc \
|
||||
nano \
|
||||
vim \
|
||||
libjemalloc2 \
|
||||
&& \
|
||||
vim && \
|
||||
rm -rf /var/lib/apt/lists/* && \
|
||||
apt-get clean
|
||||
|
||||
@@ -132,13 +130,6 @@ ENV PYTHONPATH=/app
|
||||
ARG ONYX_VERSION=0.0.0-dev
|
||||
ENV ONYX_VERSION=${ONYX_VERSION}
|
||||
|
||||
# Use jemalloc instead of glibc malloc to reduce memory fragmentation
|
||||
# in long-running Python processes (API server, Celery workers).
|
||||
# The soname is architecture-independent; the dynamic linker resolves
|
||||
# the correct path from standard library directories.
|
||||
# Placed after all RUN steps so build-time processes are unaffected.
|
||||
ENV LD_PRELOAD=libjemalloc.so.2
|
||||
|
||||
# Default command which does nothing
|
||||
# This container is used by api server and background which specify their own CMD
|
||||
CMD ["tail", "-f", "/dev/null"]
|
||||
|
||||
@@ -225,6 +225,7 @@ def do_run_migrations(
|
||||
) -> None:
|
||||
if create_schema:
|
||||
connection.execute(text(f'CREATE SCHEMA IF NOT EXISTS "{schema_name}"'))
|
||||
connection.execute(text("COMMIT"))
|
||||
|
||||
connection.execute(text(f'SET search_path TO "{schema_name}"'))
|
||||
|
||||
@@ -308,7 +309,6 @@ async def run_async_migrations() -> None:
|
||||
schema_name=schema,
|
||||
create_schema=create_schema,
|
||||
)
|
||||
await connection.commit()
|
||||
except Exception as e:
|
||||
logger.error(f"Error migrating schema {schema}: {e}")
|
||||
if not continue_on_error:
|
||||
@@ -346,7 +346,6 @@ async def run_async_migrations() -> None:
|
||||
schema_name=schema,
|
||||
create_schema=create_schema,
|
||||
)
|
||||
await connection.commit()
|
||||
except Exception as e:
|
||||
logger.error(f"Error migrating schema {schema}: {e}")
|
||||
if not continue_on_error:
|
||||
|
||||
@@ -85,122 +85,103 @@ class UserRow(NamedTuple):
|
||||
def upgrade() -> None:
|
||||
conn = op.get_bind()
|
||||
|
||||
# Step 1: Create or update the unified assistant (ID 0)
|
||||
search_assistant = conn.execute(
|
||||
sa.text("SELECT * FROM persona WHERE id = 0")
|
||||
).fetchone()
|
||||
# Start transaction
|
||||
conn.execute(sa.text("BEGIN"))
|
||||
|
||||
if search_assistant:
|
||||
# Update existing Search assistant to be the unified assistant
|
||||
try:
|
||||
# Step 1: Create or update the unified assistant (ID 0)
|
||||
search_assistant = conn.execute(
|
||||
sa.text("SELECT * FROM persona WHERE id = 0")
|
||||
).fetchone()
|
||||
|
||||
if search_assistant:
|
||||
# Update existing Search assistant to be the unified assistant
|
||||
conn.execute(
|
||||
sa.text(
|
||||
"""
|
||||
UPDATE persona
|
||||
SET name = :name,
|
||||
description = :description,
|
||||
system_prompt = :system_prompt,
|
||||
num_chunks = :num_chunks,
|
||||
is_default_persona = true,
|
||||
is_visible = true,
|
||||
deleted = false,
|
||||
display_priority = :display_priority,
|
||||
llm_filter_extraction = :llm_filter_extraction,
|
||||
llm_relevance_filter = :llm_relevance_filter,
|
||||
recency_bias = :recency_bias,
|
||||
chunks_above = :chunks_above,
|
||||
chunks_below = :chunks_below,
|
||||
datetime_aware = :datetime_aware,
|
||||
starter_messages = null
|
||||
WHERE id = 0
|
||||
"""
|
||||
),
|
||||
INSERT_DICT,
|
||||
)
|
||||
else:
|
||||
# Create new unified assistant with ID 0
|
||||
conn.execute(
|
||||
sa.text(
|
||||
"""
|
||||
INSERT INTO persona (
|
||||
id, name, description, system_prompt, num_chunks,
|
||||
is_default_persona, is_visible, deleted, display_priority,
|
||||
llm_filter_extraction, llm_relevance_filter, recency_bias,
|
||||
chunks_above, chunks_below, datetime_aware, starter_messages,
|
||||
builtin_persona
|
||||
) VALUES (
|
||||
0, :name, :description, :system_prompt, :num_chunks,
|
||||
true, true, false, :display_priority, :llm_filter_extraction,
|
||||
:llm_relevance_filter, :recency_bias, :chunks_above, :chunks_below,
|
||||
:datetime_aware, null, true
|
||||
)
|
||||
"""
|
||||
),
|
||||
INSERT_DICT,
|
||||
)
|
||||
|
||||
# Step 2: Mark ALL builtin assistants as deleted (except the unified assistant ID 0)
|
||||
conn.execute(
|
||||
sa.text(
|
||||
"""
|
||||
UPDATE persona
|
||||
SET name = :name,
|
||||
description = :description,
|
||||
system_prompt = :system_prompt,
|
||||
num_chunks = :num_chunks,
|
||||
is_default_persona = true,
|
||||
is_visible = true,
|
||||
deleted = false,
|
||||
display_priority = :display_priority,
|
||||
llm_filter_extraction = :llm_filter_extraction,
|
||||
llm_relevance_filter = :llm_relevance_filter,
|
||||
recency_bias = :recency_bias,
|
||||
chunks_above = :chunks_above,
|
||||
chunks_below = :chunks_below,
|
||||
datetime_aware = :datetime_aware,
|
||||
starter_messages = null
|
||||
WHERE id = 0
|
||||
SET deleted = true, is_visible = false, is_default_persona = false
|
||||
WHERE builtin_persona = true AND id != 0
|
||||
"""
|
||||
),
|
||||
INSERT_DICT,
|
||||
)
|
||||
else:
|
||||
# Create new unified assistant with ID 0
|
||||
conn.execute(
|
||||
sa.text(
|
||||
"""
|
||||
INSERT INTO persona (
|
||||
id, name, description, system_prompt, num_chunks,
|
||||
is_default_persona, is_visible, deleted, display_priority,
|
||||
llm_filter_extraction, llm_relevance_filter, recency_bias,
|
||||
chunks_above, chunks_below, datetime_aware, starter_messages,
|
||||
builtin_persona
|
||||
) VALUES (
|
||||
0, :name, :description, :system_prompt, :num_chunks,
|
||||
true, true, false, :display_priority, :llm_filter_extraction,
|
||||
:llm_relevance_filter, :recency_bias, :chunks_above, :chunks_below,
|
||||
:datetime_aware, null, true
|
||||
)
|
||||
"""
|
||||
),
|
||||
INSERT_DICT,
|
||||
)
|
||||
)
|
||||
|
||||
# Step 2: Mark ALL builtin assistants as deleted (except the unified assistant ID 0)
|
||||
conn.execute(
|
||||
sa.text(
|
||||
"""
|
||||
UPDATE persona
|
||||
SET deleted = true, is_visible = false, is_default_persona = false
|
||||
WHERE builtin_persona = true AND id != 0
|
||||
"""
|
||||
)
|
||||
)
|
||||
# Step 3: Add all built-in tools to the unified assistant
|
||||
# First, get the tool IDs for SearchTool, ImageGenerationTool, and WebSearchTool
|
||||
search_tool = conn.execute(
|
||||
sa.text("SELECT id FROM tool WHERE in_code_tool_id = 'SearchTool'")
|
||||
).fetchone()
|
||||
|
||||
# Step 3: Add all built-in tools to the unified assistant
|
||||
# First, get the tool IDs for SearchTool, ImageGenerationTool, and WebSearchTool
|
||||
search_tool = conn.execute(
|
||||
sa.text("SELECT id FROM tool WHERE in_code_tool_id = 'SearchTool'")
|
||||
).fetchone()
|
||||
if not search_tool:
|
||||
raise ValueError(
|
||||
"SearchTool not found in database. Ensure tools migration has run first."
|
||||
)
|
||||
|
||||
if not search_tool:
|
||||
raise ValueError(
|
||||
"SearchTool not found in database. Ensure tools migration has run first."
|
||||
)
|
||||
image_gen_tool = conn.execute(
|
||||
sa.text("SELECT id FROM tool WHERE in_code_tool_id = 'ImageGenerationTool'")
|
||||
).fetchone()
|
||||
|
||||
image_gen_tool = conn.execute(
|
||||
sa.text("SELECT id FROM tool WHERE in_code_tool_id = 'ImageGenerationTool'")
|
||||
).fetchone()
|
||||
if not image_gen_tool:
|
||||
raise ValueError(
|
||||
"ImageGenerationTool not found in database. Ensure tools migration has run first."
|
||||
)
|
||||
|
||||
if not image_gen_tool:
|
||||
raise ValueError(
|
||||
"ImageGenerationTool not found in database. Ensure tools migration has run first."
|
||||
)
|
||||
# WebSearchTool is optional - may not be configured
|
||||
web_search_tool = conn.execute(
|
||||
sa.text("SELECT id FROM tool WHERE in_code_tool_id = 'WebSearchTool'")
|
||||
).fetchone()
|
||||
|
||||
# WebSearchTool is optional - may not be configured
|
||||
web_search_tool = conn.execute(
|
||||
sa.text("SELECT id FROM tool WHERE in_code_tool_id = 'WebSearchTool'")
|
||||
).fetchone()
|
||||
# Clear existing tool associations for persona 0
|
||||
conn.execute(sa.text("DELETE FROM persona__tool WHERE persona_id = 0"))
|
||||
|
||||
# Clear existing tool associations for persona 0
|
||||
conn.execute(sa.text("DELETE FROM persona__tool WHERE persona_id = 0"))
|
||||
|
||||
# Add tools to the unified assistant
|
||||
conn.execute(
|
||||
sa.text(
|
||||
"""
|
||||
INSERT INTO persona__tool (persona_id, tool_id)
|
||||
VALUES (0, :tool_id)
|
||||
ON CONFLICT DO NOTHING
|
||||
"""
|
||||
),
|
||||
{"tool_id": search_tool[0]},
|
||||
)
|
||||
|
||||
conn.execute(
|
||||
sa.text(
|
||||
"""
|
||||
INSERT INTO persona__tool (persona_id, tool_id)
|
||||
VALUES (0, :tool_id)
|
||||
ON CONFLICT DO NOTHING
|
||||
"""
|
||||
),
|
||||
{"tool_id": image_gen_tool[0]},
|
||||
)
|
||||
|
||||
if web_search_tool:
|
||||
# Add tools to the unified assistant
|
||||
conn.execute(
|
||||
sa.text(
|
||||
"""
|
||||
@@ -209,148 +190,191 @@ def upgrade() -> None:
|
||||
ON CONFLICT DO NOTHING
|
||||
"""
|
||||
),
|
||||
{"tool_id": web_search_tool[0]},
|
||||
{"tool_id": search_tool[0]},
|
||||
)
|
||||
|
||||
# Step 4: Migrate existing chat sessions from all builtin assistants to unified assistant
|
||||
conn.execute(
|
||||
sa.text(
|
||||
conn.execute(
|
||||
sa.text(
|
||||
"""
|
||||
INSERT INTO persona__tool (persona_id, tool_id)
|
||||
VALUES (0, :tool_id)
|
||||
ON CONFLICT DO NOTHING
|
||||
"""
|
||||
UPDATE chat_session
|
||||
SET persona_id = 0
|
||||
WHERE persona_id IN (
|
||||
SELECT id FROM persona WHERE builtin_persona = true AND id != 0
|
||||
)
|
||||
"""
|
||||
),
|
||||
{"tool_id": image_gen_tool[0]},
|
||||
)
|
||||
)
|
||||
|
||||
# Step 5: Migrate user preferences - remove references to all builtin assistants
|
||||
# First, get all builtin assistant IDs (except 0)
|
||||
builtin_assistants_result = conn.execute(
|
||||
sa.text(
|
||||
"""
|
||||
SELECT id FROM persona
|
||||
WHERE builtin_persona = true AND id != 0
|
||||
"""
|
||||
)
|
||||
).fetchall()
|
||||
builtin_assistant_ids = [row[0] for row in builtin_assistants_result]
|
||||
|
||||
# Get all users with preferences
|
||||
users_result = conn.execute(
|
||||
sa.text(
|
||||
"""
|
||||
SELECT id, chosen_assistants, visible_assistants,
|
||||
hidden_assistants, pinned_assistants
|
||||
FROM "user"
|
||||
"""
|
||||
)
|
||||
).fetchall()
|
||||
|
||||
for user_row in users_result:
|
||||
user = UserRow(*user_row)
|
||||
user_id: UUID = user.id
|
||||
updates: dict[str, Any] = {}
|
||||
|
||||
# Remove all builtin assistants from chosen_assistants
|
||||
if user.chosen_assistants:
|
||||
new_chosen: list[int] = [
|
||||
assistant_id
|
||||
for assistant_id in user.chosen_assistants
|
||||
if assistant_id not in builtin_assistant_ids
|
||||
]
|
||||
if new_chosen != user.chosen_assistants:
|
||||
updates["chosen_assistants"] = json.dumps(new_chosen)
|
||||
|
||||
# Remove all builtin assistants from visible_assistants
|
||||
if user.visible_assistants:
|
||||
new_visible: list[int] = [
|
||||
assistant_id
|
||||
for assistant_id in user.visible_assistants
|
||||
if assistant_id not in builtin_assistant_ids
|
||||
]
|
||||
if new_visible != user.visible_assistants:
|
||||
updates["visible_assistants"] = json.dumps(new_visible)
|
||||
|
||||
# Add all builtin assistants to hidden_assistants
|
||||
if user.hidden_assistants:
|
||||
new_hidden: list[int] = list(user.hidden_assistants)
|
||||
for old_id in builtin_assistant_ids:
|
||||
if old_id not in new_hidden:
|
||||
new_hidden.append(old_id)
|
||||
if new_hidden != user.hidden_assistants:
|
||||
updates["hidden_assistants"] = json.dumps(new_hidden)
|
||||
else:
|
||||
updates["hidden_assistants"] = json.dumps(builtin_assistant_ids)
|
||||
|
||||
# Remove all builtin assistants from pinned_assistants
|
||||
if user.pinned_assistants:
|
||||
new_pinned: list[int] = [
|
||||
assistant_id
|
||||
for assistant_id in user.pinned_assistants
|
||||
if assistant_id not in builtin_assistant_ids
|
||||
]
|
||||
if new_pinned != user.pinned_assistants:
|
||||
updates["pinned_assistants"] = json.dumps(new_pinned)
|
||||
|
||||
# Apply updates if any
|
||||
if updates:
|
||||
set_clause = ", ".join([f"{k} = :{k}" for k in updates.keys()])
|
||||
updates["user_id"] = str(user_id) # Convert UUID to string for SQL
|
||||
if web_search_tool:
|
||||
conn.execute(
|
||||
sa.text(f'UPDATE "user" SET {set_clause} WHERE id = :user_id'),
|
||||
updates,
|
||||
sa.text(
|
||||
"""
|
||||
INSERT INTO persona__tool (persona_id, tool_id)
|
||||
VALUES (0, :tool_id)
|
||||
ON CONFLICT DO NOTHING
|
||||
"""
|
||||
),
|
||||
{"tool_id": web_search_tool[0]},
|
||||
)
|
||||
|
||||
# Step 4: Migrate existing chat sessions from all builtin assistants to unified assistant
|
||||
conn.execute(
|
||||
sa.text(
|
||||
"""
|
||||
UPDATE chat_session
|
||||
SET persona_id = 0
|
||||
WHERE persona_id IN (
|
||||
SELECT id FROM persona WHERE builtin_persona = true AND id != 0
|
||||
)
|
||||
"""
|
||||
)
|
||||
)
|
||||
|
||||
# Step 5: Migrate user preferences - remove references to all builtin assistants
|
||||
# First, get all builtin assistant IDs (except 0)
|
||||
builtin_assistants_result = conn.execute(
|
||||
sa.text(
|
||||
"""
|
||||
SELECT id FROM persona
|
||||
WHERE builtin_persona = true AND id != 0
|
||||
"""
|
||||
)
|
||||
).fetchall()
|
||||
builtin_assistant_ids = [row[0] for row in builtin_assistants_result]
|
||||
|
||||
# Get all users with preferences
|
||||
users_result = conn.execute(
|
||||
sa.text(
|
||||
"""
|
||||
SELECT id, chosen_assistants, visible_assistants,
|
||||
hidden_assistants, pinned_assistants
|
||||
FROM "user"
|
||||
"""
|
||||
)
|
||||
).fetchall()
|
||||
|
||||
for user_row in users_result:
|
||||
user = UserRow(*user_row)
|
||||
user_id: UUID = user.id
|
||||
updates: dict[str, Any] = {}
|
||||
|
||||
# Remove all builtin assistants from chosen_assistants
|
||||
if user.chosen_assistants:
|
||||
new_chosen: list[int] = [
|
||||
assistant_id
|
||||
for assistant_id in user.chosen_assistants
|
||||
if assistant_id not in builtin_assistant_ids
|
||||
]
|
||||
if new_chosen != user.chosen_assistants:
|
||||
updates["chosen_assistants"] = json.dumps(new_chosen)
|
||||
|
||||
# Remove all builtin assistants from visible_assistants
|
||||
if user.visible_assistants:
|
||||
new_visible: list[int] = [
|
||||
assistant_id
|
||||
for assistant_id in user.visible_assistants
|
||||
if assistant_id not in builtin_assistant_ids
|
||||
]
|
||||
if new_visible != user.visible_assistants:
|
||||
updates["visible_assistants"] = json.dumps(new_visible)
|
||||
|
||||
# Add all builtin assistants to hidden_assistants
|
||||
if user.hidden_assistants:
|
||||
new_hidden: list[int] = list(user.hidden_assistants)
|
||||
for old_id in builtin_assistant_ids:
|
||||
if old_id not in new_hidden:
|
||||
new_hidden.append(old_id)
|
||||
if new_hidden != user.hidden_assistants:
|
||||
updates["hidden_assistants"] = json.dumps(new_hidden)
|
||||
else:
|
||||
updates["hidden_assistants"] = json.dumps(builtin_assistant_ids)
|
||||
|
||||
# Remove all builtin assistants from pinned_assistants
|
||||
if user.pinned_assistants:
|
||||
new_pinned: list[int] = [
|
||||
assistant_id
|
||||
for assistant_id in user.pinned_assistants
|
||||
if assistant_id not in builtin_assistant_ids
|
||||
]
|
||||
if new_pinned != user.pinned_assistants:
|
||||
updates["pinned_assistants"] = json.dumps(new_pinned)
|
||||
|
||||
# Apply updates if any
|
||||
if updates:
|
||||
set_clause = ", ".join([f"{k} = :{k}" for k in updates.keys()])
|
||||
updates["user_id"] = str(user_id) # Convert UUID to string for SQL
|
||||
conn.execute(
|
||||
sa.text(f'UPDATE "user" SET {set_clause} WHERE id = :user_id'),
|
||||
updates,
|
||||
)
|
||||
|
||||
# Commit transaction
|
||||
conn.execute(sa.text("COMMIT"))
|
||||
|
||||
except Exception as e:
|
||||
# Rollback on error
|
||||
conn.execute(sa.text("ROLLBACK"))
|
||||
raise e
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
conn = op.get_bind()
|
||||
|
||||
# Only restore General (ID -1) and Art (ID -3) assistants
|
||||
# Step 1: Keep Search assistant (ID 0) as default but restore original state
|
||||
conn.execute(
|
||||
sa.text(
|
||||
# Start transaction
|
||||
conn.execute(sa.text("BEGIN"))
|
||||
|
||||
try:
|
||||
# Only restore General (ID -1) and Art (ID -3) assistants
|
||||
# Step 1: Keep Search assistant (ID 0) as default but restore original state
|
||||
conn.execute(
|
||||
sa.text(
|
||||
"""
|
||||
UPDATE persona
|
||||
SET is_default_persona = true,
|
||||
is_visible = true,
|
||||
deleted = false
|
||||
WHERE id = 0
|
||||
"""
|
||||
UPDATE persona
|
||||
SET is_default_persona = true,
|
||||
is_visible = true,
|
||||
deleted = false
|
||||
WHERE id = 0
|
||||
"""
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
# Step 2: Restore General assistant (ID -1)
|
||||
conn.execute(
|
||||
sa.text(
|
||||
# Step 2: Restore General assistant (ID -1)
|
||||
conn.execute(
|
||||
sa.text(
|
||||
"""
|
||||
UPDATE persona
|
||||
SET deleted = false,
|
||||
is_visible = true,
|
||||
is_default_persona = true
|
||||
WHERE id = :general_assistant_id
|
||||
"""
|
||||
UPDATE persona
|
||||
SET deleted = false,
|
||||
is_visible = true,
|
||||
is_default_persona = true
|
||||
WHERE id = :general_assistant_id
|
||||
"""
|
||||
),
|
||||
{"general_assistant_id": GENERAL_ASSISTANT_ID},
|
||||
)
|
||||
),
|
||||
{"general_assistant_id": GENERAL_ASSISTANT_ID},
|
||||
)
|
||||
|
||||
# Step 3: Restore Art assistant (ID -3)
|
||||
conn.execute(
|
||||
sa.text(
|
||||
# Step 3: Restore Art assistant (ID -3)
|
||||
conn.execute(
|
||||
sa.text(
|
||||
"""
|
||||
UPDATE persona
|
||||
SET deleted = false,
|
||||
is_visible = true,
|
||||
is_default_persona = true
|
||||
WHERE id = :art_assistant_id
|
||||
"""
|
||||
UPDATE persona
|
||||
SET deleted = false,
|
||||
is_visible = true,
|
||||
is_default_persona = true
|
||||
WHERE id = :art_assistant_id
|
||||
"""
|
||||
),
|
||||
{"art_assistant_id": ART_ASSISTANT_ID},
|
||||
)
|
||||
),
|
||||
{"art_assistant_id": ART_ASSISTANT_ID},
|
||||
)
|
||||
|
||||
# Note: We don't restore the original tool associations, names, or descriptions
|
||||
# as those would require more complex logic to determine original state.
|
||||
# We also cannot restore original chat session persona_ids as we don't
|
||||
# have the original mappings.
|
||||
# Other builtin assistants remain deleted as per the requirement.
|
||||
# Note: We don't restore the original tool associations, names, or descriptions
|
||||
# as those would require more complex logic to determine original state.
|
||||
# We also cannot restore original chat session persona_ids as we don't
|
||||
# have the original mappings.
|
||||
# Other builtin assistants remain deleted as per the requirement.
|
||||
|
||||
# Commit transaction
|
||||
conn.execute(sa.text("COMMIT"))
|
||||
|
||||
except Exception as e:
|
||||
# Rollback on error
|
||||
conn.execute(sa.text("ROLLBACK"))
|
||||
raise e
|
||||
|
||||
@@ -24,9 +24,6 @@ def upgrade() -> None:
|
||||
# in unique constraints, but we want NULL == NULL for deduplication).
|
||||
# The '{}' represents an empty JSONB object as the NULL replacement.
|
||||
|
||||
# Clean up legacy notifications first
|
||||
op.execute("DELETE FROM notification WHERE title = 'New Notification'")
|
||||
|
||||
op.execute(
|
||||
"""
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS ix_notification_user_type_data
|
||||
@@ -43,6 +40,9 @@ def upgrade() -> None:
|
||||
"""
|
||||
)
|
||||
|
||||
# Clean up legacy 'reindex' notifications that are no longer needed
|
||||
op.execute("DELETE FROM notification WHERE title = 'New Notification'")
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.execute("DROP INDEX IF EXISTS ix_notification_user_type_data")
|
||||
|
||||
@@ -42,13 +42,20 @@ TOOL_DESCRIPTIONS = {
|
||||
|
||||
def upgrade() -> None:
|
||||
conn = op.get_bind()
|
||||
for tool_id, description in TOOL_DESCRIPTIONS.items():
|
||||
conn.execute(
|
||||
sa.text(
|
||||
"UPDATE tool SET description = :description WHERE in_code_tool_id = :tool_id"
|
||||
),
|
||||
{"description": description, "tool_id": tool_id},
|
||||
)
|
||||
conn.execute(sa.text("BEGIN"))
|
||||
|
||||
try:
|
||||
for tool_id, description in TOOL_DESCRIPTIONS.items():
|
||||
conn.execute(
|
||||
sa.text(
|
||||
"UPDATE tool SET description = :description WHERE in_code_tool_id = :tool_id"
|
||||
),
|
||||
{"description": description, "tool_id": tool_id},
|
||||
)
|
||||
conn.execute(sa.text("COMMIT"))
|
||||
except Exception as e:
|
||||
conn.execute(sa.text("ROLLBACK"))
|
||||
raise e
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
|
||||
@@ -70,66 +70,80 @@ BUILT_IN_TOOLS = [
|
||||
def upgrade() -> None:
|
||||
conn = op.get_bind()
|
||||
|
||||
# Get existing tools to check what already exists
|
||||
existing_tools = conn.execute(
|
||||
sa.text("SELECT in_code_tool_id FROM tool WHERE in_code_tool_id IS NOT NULL")
|
||||
).fetchall()
|
||||
existing_tool_ids = {row[0] for row in existing_tools}
|
||||
# Start transaction
|
||||
conn.execute(sa.text("BEGIN"))
|
||||
|
||||
# Insert or update built-in tools
|
||||
for tool in BUILT_IN_TOOLS:
|
||||
in_code_id = tool["in_code_tool_id"]
|
||||
try:
|
||||
# Get existing tools to check what already exists
|
||||
existing_tools = conn.execute(
|
||||
sa.text(
|
||||
"SELECT in_code_tool_id FROM tool WHERE in_code_tool_id IS NOT NULL"
|
||||
)
|
||||
).fetchall()
|
||||
existing_tool_ids = {row[0] for row in existing_tools}
|
||||
|
||||
# Handle historical rename: InternetSearchTool -> WebSearchTool
|
||||
if (
|
||||
in_code_id == "WebSearchTool"
|
||||
and "WebSearchTool" not in existing_tool_ids
|
||||
and "InternetSearchTool" in existing_tool_ids
|
||||
):
|
||||
# Rename the existing InternetSearchTool row in place and update fields
|
||||
conn.execute(
|
||||
sa.text(
|
||||
"""
|
||||
UPDATE tool
|
||||
SET name = :name,
|
||||
display_name = :display_name,
|
||||
description = :description,
|
||||
in_code_tool_id = :in_code_tool_id
|
||||
WHERE in_code_tool_id = 'InternetSearchTool'
|
||||
"""
|
||||
),
|
||||
tool,
|
||||
)
|
||||
# Keep the local view of existing ids in sync to avoid duplicate insert
|
||||
existing_tool_ids.discard("InternetSearchTool")
|
||||
existing_tool_ids.add("WebSearchTool")
|
||||
continue
|
||||
# Insert or update built-in tools
|
||||
for tool in BUILT_IN_TOOLS:
|
||||
in_code_id = tool["in_code_tool_id"]
|
||||
|
||||
if in_code_id in existing_tool_ids:
|
||||
# Update existing tool
|
||||
conn.execute(
|
||||
sa.text(
|
||||
"""
|
||||
UPDATE tool
|
||||
SET name = :name,
|
||||
display_name = :display_name,
|
||||
description = :description
|
||||
WHERE in_code_tool_id = :in_code_tool_id
|
||||
"""
|
||||
),
|
||||
tool,
|
||||
)
|
||||
else:
|
||||
# Insert new tool
|
||||
conn.execute(
|
||||
sa.text(
|
||||
"""
|
||||
INSERT INTO tool (name, display_name, description, in_code_tool_id)
|
||||
VALUES (:name, :display_name, :description, :in_code_tool_id)
|
||||
"""
|
||||
),
|
||||
tool,
|
||||
)
|
||||
# Handle historical rename: InternetSearchTool -> WebSearchTool
|
||||
if (
|
||||
in_code_id == "WebSearchTool"
|
||||
and "WebSearchTool" not in existing_tool_ids
|
||||
and "InternetSearchTool" in existing_tool_ids
|
||||
):
|
||||
# Rename the existing InternetSearchTool row in place and update fields
|
||||
conn.execute(
|
||||
sa.text(
|
||||
"""
|
||||
UPDATE tool
|
||||
SET name = :name,
|
||||
display_name = :display_name,
|
||||
description = :description,
|
||||
in_code_tool_id = :in_code_tool_id
|
||||
WHERE in_code_tool_id = 'InternetSearchTool'
|
||||
"""
|
||||
),
|
||||
tool,
|
||||
)
|
||||
# Keep the local view of existing ids in sync to avoid duplicate insert
|
||||
existing_tool_ids.discard("InternetSearchTool")
|
||||
existing_tool_ids.add("WebSearchTool")
|
||||
continue
|
||||
|
||||
if in_code_id in existing_tool_ids:
|
||||
# Update existing tool
|
||||
conn.execute(
|
||||
sa.text(
|
||||
"""
|
||||
UPDATE tool
|
||||
SET name = :name,
|
||||
display_name = :display_name,
|
||||
description = :description
|
||||
WHERE in_code_tool_id = :in_code_tool_id
|
||||
"""
|
||||
),
|
||||
tool,
|
||||
)
|
||||
else:
|
||||
# Insert new tool
|
||||
conn.execute(
|
||||
sa.text(
|
||||
"""
|
||||
INSERT INTO tool (name, display_name, description, in_code_tool_id)
|
||||
VALUES (:name, :display_name, :description, :in_code_tool_id)
|
||||
"""
|
||||
),
|
||||
tool,
|
||||
)
|
||||
|
||||
# Commit transaction
|
||||
conn.execute(sa.text("COMMIT"))
|
||||
|
||||
except Exception as e:
|
||||
# Rollback on error
|
||||
conn.execute(sa.text("ROLLBACK"))
|
||||
raise e
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
|
||||
@@ -1,64 +0,0 @@
|
||||
"""sync_exa_api_key_to_content_provider
|
||||
|
||||
Revision ID: d1b637d7050a
|
||||
Revises: d25168c2beee
|
||||
Create Date: 2026-01-09 15:54:15.646249
|
||||
|
||||
"""
|
||||
|
||||
from alembic import op
|
||||
from sqlalchemy import text
|
||||
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = "d1b637d7050a"
|
||||
down_revision = "d25168c2beee"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
# Exa uses a shared API key between search and content providers.
|
||||
# For existing Exa search providers with API keys, create the corresponding
|
||||
# content provider if it doesn't exist yet.
|
||||
connection = op.get_bind()
|
||||
|
||||
# Check if Exa search provider exists with an API key
|
||||
result = connection.execute(
|
||||
text(
|
||||
"""
|
||||
SELECT api_key FROM internet_search_provider
|
||||
WHERE provider_type = 'exa' AND api_key IS NOT NULL
|
||||
LIMIT 1
|
||||
"""
|
||||
)
|
||||
)
|
||||
row = result.fetchone()
|
||||
|
||||
if row:
|
||||
api_key = row[0]
|
||||
# Create Exa content provider with the shared key
|
||||
connection.execute(
|
||||
text(
|
||||
"""
|
||||
INSERT INTO internet_content_provider
|
||||
(name, provider_type, api_key, is_active)
|
||||
VALUES ('Exa', 'exa', :api_key, false)
|
||||
ON CONFLICT (name) DO NOTHING
|
||||
"""
|
||||
),
|
||||
{"api_key": api_key},
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
# Remove the Exa content provider that was created by this migration
|
||||
connection = op.get_bind()
|
||||
connection.execute(
|
||||
text(
|
||||
"""
|
||||
DELETE FROM internet_content_provider
|
||||
WHERE provider_type = 'exa'
|
||||
"""
|
||||
)
|
||||
)
|
||||
@@ -109,6 +109,7 @@ CHECK_TTL_MANAGEMENT_TASK_FREQUENCY_IN_HOURS = float(
|
||||
|
||||
|
||||
STRIPE_SECRET_KEY = os.environ.get("STRIPE_SECRET_KEY")
|
||||
STRIPE_PRICE_ID = os.environ.get("STRIPE_PRICE")
|
||||
|
||||
# JWT Public Key URL
|
||||
JWT_PUBLIC_KEY_URL: str | None = os.getenv("JWT_PUBLIC_KEY_URL", None)
|
||||
|
||||
@@ -3,42 +3,30 @@ from uuid import UUID
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from onyx.configs.constants import NotificationType
|
||||
from onyx.db.models import Persona
|
||||
from onyx.db.models import Persona__User
|
||||
from onyx.db.models import Persona__UserGroup
|
||||
from onyx.db.notification import create_notification
|
||||
from onyx.server.features.persona.models import PersonaSharedNotificationData
|
||||
|
||||
|
||||
def update_persona_access(
|
||||
def make_persona_private(
|
||||
persona_id: int,
|
||||
creator_user_id: UUID | None,
|
||||
user_ids: list[UUID] | None,
|
||||
group_ids: list[int] | None,
|
||||
db_session: Session,
|
||||
is_public: bool | None = None,
|
||||
user_ids: list[UUID] | None = None,
|
||||
group_ids: list[int] | None = None,
|
||||
) -> None:
|
||||
"""Updates the access settings for a persona including public status, user shares,
|
||||
and group shares.
|
||||
"""NOTE(rkuo): This function batches all updates into a single commit. If we don't
|
||||
dedupe the inputs, the commit will exception."""
|
||||
|
||||
NOTE: This function batches all updates. If we don't dedupe the inputs,
|
||||
the commit will exception.
|
||||
|
||||
NOTE: Callers are responsible for committing."""
|
||||
|
||||
if is_public is not None:
|
||||
persona = db_session.query(Persona).filter(Persona.id == persona_id).first()
|
||||
if persona:
|
||||
persona.is_public = is_public
|
||||
|
||||
# NOTE: For user-ids and group-ids, `None` means "leave unchanged", `[]` means "clear all shares",
|
||||
# and a non-empty list means "replace with these shares".
|
||||
|
||||
if user_ids is not None:
|
||||
db_session.query(Persona__User).filter(
|
||||
Persona__User.persona_id == persona_id
|
||||
).delete(synchronize_session="fetch")
|
||||
db_session.query(Persona__User).filter(
|
||||
Persona__User.persona_id == persona_id
|
||||
).delete(synchronize_session="fetch")
|
||||
db_session.query(Persona__UserGroup).filter(
|
||||
Persona__UserGroup.persona_id == persona_id
|
||||
).delete(synchronize_session="fetch")
|
||||
|
||||
if user_ids:
|
||||
user_ids_set = set(user_ids)
|
||||
for user_id in user_ids_set:
|
||||
db_session.add(Persona__User(persona_id=persona_id, user_id=user_id))
|
||||
@@ -53,13 +41,11 @@ def update_persona_access(
|
||||
).model_dump(),
|
||||
)
|
||||
|
||||
if group_ids is not None:
|
||||
db_session.query(Persona__UserGroup).filter(
|
||||
Persona__UserGroup.persona_id == persona_id
|
||||
).delete(synchronize_session="fetch")
|
||||
|
||||
if group_ids:
|
||||
group_ids_set = set(group_ids)
|
||||
for group_id in group_ids_set:
|
||||
db_session.add(
|
||||
Persona__UserGroup(persona_id=persona_id, user_group_id=group_id)
|
||||
)
|
||||
|
||||
db_session.commit()
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
from typing import cast
|
||||
from typing import Literal
|
||||
|
||||
import requests
|
||||
import stripe
|
||||
|
||||
from ee.onyx.configs.app_configs import STRIPE_PRICE_ID
|
||||
from ee.onyx.configs.app_configs import STRIPE_SECRET_KEY
|
||||
from ee.onyx.server.tenants.access import generate_data_plane_token
|
||||
from ee.onyx.server.tenants.models import BillingInformation
|
||||
@@ -16,21 +16,15 @@ stripe.api_key = STRIPE_SECRET_KEY
|
||||
logger = setup_logger()
|
||||
|
||||
|
||||
def fetch_stripe_checkout_session(
|
||||
tenant_id: str,
|
||||
billing_period: Literal["monthly", "annual"] = "monthly",
|
||||
) -> str:
|
||||
def fetch_stripe_checkout_session(tenant_id: str) -> str:
|
||||
token = generate_data_plane_token()
|
||||
headers = {
|
||||
"Authorization": f"Bearer {token}",
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
url = f"{CONTROL_PLANE_API_BASE_URL}/create-checkout-session"
|
||||
payload = {
|
||||
"tenant_id": tenant_id,
|
||||
"billing_period": billing_period,
|
||||
}
|
||||
response = requests.post(url, headers=headers, json=payload)
|
||||
params = {"tenant_id": tenant_id}
|
||||
response = requests.post(url, headers=headers, params=params)
|
||||
response.raise_for_status()
|
||||
return response.json()["sessionId"]
|
||||
|
||||
@@ -78,24 +72,22 @@ def fetch_billing_information(
|
||||
|
||||
def register_tenant_users(tenant_id: str, number_of_users: int) -> stripe.Subscription:
|
||||
"""
|
||||
Update the number of seats for a tenant's subscription.
|
||||
Preserves the existing price (monthly, annual, or grandfathered).
|
||||
Send a request to the control service to register the number of users for a tenant.
|
||||
"""
|
||||
|
||||
if not STRIPE_PRICE_ID:
|
||||
raise Exception("STRIPE_PRICE_ID is not set")
|
||||
|
||||
response = fetch_tenant_stripe_information(tenant_id)
|
||||
stripe_subscription_id = cast(str, response.get("stripe_subscription_id"))
|
||||
|
||||
subscription = stripe.Subscription.retrieve(stripe_subscription_id)
|
||||
subscription_item = subscription["items"]["data"][0]
|
||||
|
||||
# Use existing price to preserve the customer's current plan
|
||||
current_price_id = subscription_item.price.id
|
||||
|
||||
updated_subscription = stripe.Subscription.modify(
|
||||
stripe_subscription_id,
|
||||
items=[
|
||||
{
|
||||
"id": subscription_item.id,
|
||||
"price": current_price_id,
|
||||
"id": subscription["items"]["data"][0].id,
|
||||
"price": STRIPE_PRICE_ID,
|
||||
"quantity": number_of_users,
|
||||
}
|
||||
],
|
||||
|
||||
@@ -10,7 +10,6 @@ from ee.onyx.server.tenants.billing import fetch_billing_information
|
||||
from ee.onyx.server.tenants.billing import fetch_stripe_checkout_session
|
||||
from ee.onyx.server.tenants.billing import fetch_tenant_stripe_information
|
||||
from ee.onyx.server.tenants.models import BillingInformation
|
||||
from ee.onyx.server.tenants.models import CreateSubscriptionSessionRequest
|
||||
from ee.onyx.server.tenants.models import ProductGatingFullSyncRequest
|
||||
from ee.onyx.server.tenants.models import ProductGatingRequest
|
||||
from ee.onyx.server.tenants.models import ProductGatingResponse
|
||||
@@ -105,18 +104,15 @@ async def create_customer_portal_session(
|
||||
|
||||
@router.post("/create-subscription-session")
|
||||
async def create_subscription_session(
|
||||
request: CreateSubscriptionSessionRequest | None = None,
|
||||
_: User = Depends(current_admin_user),
|
||||
) -> SubscriptionSessionResponse:
|
||||
try:
|
||||
tenant_id = CURRENT_TENANT_ID_CONTEXTVAR.get()
|
||||
if not tenant_id:
|
||||
raise HTTPException(status_code=400, detail="Tenant ID not found")
|
||||
|
||||
billing_period = request.billing_period if request else "monthly"
|
||||
session_id = fetch_stripe_checkout_session(tenant_id, billing_period)
|
||||
session_id = fetch_stripe_checkout_session(tenant_id)
|
||||
return SubscriptionSessionResponse(sessionId=session_id)
|
||||
|
||||
except Exception as e:
|
||||
logger.exception("Failed to create subscription session")
|
||||
logger.exception("Failed to create resubscription session")
|
||||
raise HTTPException(status_code=500, detail=str(e))
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
from datetime import datetime
|
||||
from typing import Literal
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
@@ -74,12 +73,6 @@ class SubscriptionSessionResponse(BaseModel):
|
||||
sessionId: str
|
||||
|
||||
|
||||
class CreateSubscriptionSessionRequest(BaseModel):
|
||||
"""Request to create a subscription checkout session."""
|
||||
|
||||
billing_period: Literal["monthly", "annual"] = "monthly"
|
||||
|
||||
|
||||
class TenantByDomainResponse(BaseModel):
|
||||
tenant_id: str
|
||||
number_of_users: int
|
||||
|
||||
@@ -12,7 +12,6 @@ from retry import retry
|
||||
from sqlalchemy import select
|
||||
|
||||
from onyx.background.celery.apps.app_base import task_logger
|
||||
from onyx.background.celery.celery_redis import celery_get_queue_length
|
||||
from onyx.background.celery.celery_utils import httpx_init_vespa_pool
|
||||
from onyx.background.celery.tasks.shared.RetryDocumentIndex import RetryDocumentIndex
|
||||
from onyx.configs.app_configs import MANAGED_VESPA
|
||||
@@ -20,14 +19,12 @@ from onyx.configs.app_configs import VESPA_CLOUD_CERT_PATH
|
||||
from onyx.configs.app_configs import VESPA_CLOUD_KEY_PATH
|
||||
from onyx.configs.constants import CELERY_GENERIC_BEAT_LOCK_TIMEOUT
|
||||
from onyx.configs.constants import CELERY_USER_FILE_PROCESSING_LOCK_TIMEOUT
|
||||
from onyx.configs.constants import CELERY_USER_FILE_PROCESSING_TASK_EXPIRES
|
||||
from onyx.configs.constants import CELERY_USER_FILE_PROJECT_SYNC_LOCK_TIMEOUT
|
||||
from onyx.configs.constants import DocumentSource
|
||||
from onyx.configs.constants import OnyxCeleryPriority
|
||||
from onyx.configs.constants import OnyxCeleryQueues
|
||||
from onyx.configs.constants import OnyxCeleryTask
|
||||
from onyx.configs.constants import OnyxRedisLocks
|
||||
from onyx.configs.constants import USER_FILE_PROCESSING_MAX_QUEUE_DEPTH
|
||||
from onyx.connectors.file.connector import LocalFileConnector
|
||||
from onyx.connectors.models import Document
|
||||
from onyx.db.engine.sql_engine import get_session_with_current_tenant
|
||||
@@ -56,17 +53,6 @@ def _user_file_lock_key(user_file_id: str | UUID) -> str:
|
||||
return f"{OnyxRedisLocks.USER_FILE_PROCESSING_LOCK_PREFIX}:{user_file_id}"
|
||||
|
||||
|
||||
def _user_file_queued_key(user_file_id: str | UUID) -> str:
|
||||
"""Key that exists while a process_single_user_file task is sitting in the queue.
|
||||
|
||||
The beat generator sets this with a TTL equal to CELERY_USER_FILE_PROCESSING_TASK_EXPIRES
|
||||
before enqueuing and the worker deletes it as its first action. This prevents
|
||||
the beat from adding duplicate tasks for files that already have a live task
|
||||
in flight.
|
||||
"""
|
||||
return f"{OnyxRedisLocks.USER_FILE_QUEUED_PREFIX}:{user_file_id}"
|
||||
|
||||
|
||||
def _user_file_project_sync_lock_key(user_file_id: str | UUID) -> str:
|
||||
return f"{OnyxRedisLocks.USER_FILE_PROJECT_SYNC_LOCK_PREFIX}:{user_file_id}"
|
||||
|
||||
@@ -130,24 +116,7 @@ def _get_document_chunk_count(
|
||||
def check_user_file_processing(self: Task, *, tenant_id: str) -> None:
|
||||
"""Scan for user files with PROCESSING status and enqueue per-file tasks.
|
||||
|
||||
Three mechanisms prevent queue runaway:
|
||||
|
||||
1. **Queue depth backpressure** – if the broker queue already has more than
|
||||
USER_FILE_PROCESSING_MAX_QUEUE_DEPTH items we skip this beat cycle
|
||||
entirely. Workers are clearly behind; adding more tasks would only make
|
||||
the backlog worse.
|
||||
|
||||
2. **Per-file queued guard** – before enqueuing a task we set a short-lived
|
||||
Redis key (TTL = CELERY_USER_FILE_PROCESSING_TASK_EXPIRES). If that key
|
||||
already exists the file already has a live task in the queue, so we skip
|
||||
it. The worker deletes the key the moment it picks up the task so the
|
||||
next beat cycle can re-enqueue if the file is still PROCESSING.
|
||||
|
||||
3. **Task expiry** – every enqueued task carries an `expires` value equal to
|
||||
CELERY_USER_FILE_PROCESSING_TASK_EXPIRES. If a task is still sitting in
|
||||
the queue after that deadline, Celery discards it without touching the DB.
|
||||
This is a belt-and-suspenders defence: even if the guard key is lost (e.g.
|
||||
Redis restart), stale tasks evict themselves rather than piling up forever.
|
||||
Uses direct Redis locks to avoid overlapping runs.
|
||||
"""
|
||||
task_logger.info("check_user_file_processing - Starting")
|
||||
|
||||
@@ -162,21 +131,7 @@ def check_user_file_processing(self: Task, *, tenant_id: str) -> None:
|
||||
return None
|
||||
|
||||
enqueued = 0
|
||||
skipped_guard = 0
|
||||
try:
|
||||
# --- Protection 1: queue depth backpressure ---
|
||||
r_celery = self.app.broker_connection().channel().client # type: ignore
|
||||
queue_len = celery_get_queue_length(
|
||||
OnyxCeleryQueues.USER_FILE_PROCESSING, r_celery
|
||||
)
|
||||
if queue_len > USER_FILE_PROCESSING_MAX_QUEUE_DEPTH:
|
||||
task_logger.warning(
|
||||
f"check_user_file_processing - Queue depth {queue_len} exceeds "
|
||||
f"{USER_FILE_PROCESSING_MAX_QUEUE_DEPTH}, skipping enqueue for "
|
||||
f"tenant={tenant_id}"
|
||||
)
|
||||
return None
|
||||
|
||||
with get_session_with_current_tenant() as db_session:
|
||||
user_file_ids = (
|
||||
db_session.execute(
|
||||
@@ -189,35 +144,12 @@ def check_user_file_processing(self: Task, *, tenant_id: str) -> None:
|
||||
)
|
||||
|
||||
for user_file_id in user_file_ids:
|
||||
# --- Protection 2: per-file queued guard ---
|
||||
queued_key = _user_file_queued_key(user_file_id)
|
||||
guard_set = redis_client.set(
|
||||
queued_key,
|
||||
1,
|
||||
ex=CELERY_USER_FILE_PROCESSING_TASK_EXPIRES,
|
||||
nx=True,
|
||||
self.app.send_task(
|
||||
OnyxCeleryTask.PROCESS_SINGLE_USER_FILE,
|
||||
kwargs={"user_file_id": str(user_file_id), "tenant_id": tenant_id},
|
||||
queue=OnyxCeleryQueues.USER_FILE_PROCESSING,
|
||||
priority=OnyxCeleryPriority.HIGH,
|
||||
)
|
||||
if not guard_set:
|
||||
skipped_guard += 1
|
||||
continue
|
||||
|
||||
# --- Protection 3: task expiry ---
|
||||
# If task submission fails, clear the guard immediately so the
|
||||
# next beat cycle can retry enqueuing this file.
|
||||
try:
|
||||
self.app.send_task(
|
||||
OnyxCeleryTask.PROCESS_SINGLE_USER_FILE,
|
||||
kwargs={
|
||||
"user_file_id": str(user_file_id),
|
||||
"tenant_id": tenant_id,
|
||||
},
|
||||
queue=OnyxCeleryQueues.USER_FILE_PROCESSING,
|
||||
priority=OnyxCeleryPriority.HIGH,
|
||||
expires=CELERY_USER_FILE_PROCESSING_TASK_EXPIRES,
|
||||
)
|
||||
except Exception:
|
||||
redis_client.delete(queued_key)
|
||||
raise
|
||||
enqueued += 1
|
||||
|
||||
finally:
|
||||
@@ -225,8 +157,7 @@ def check_user_file_processing(self: Task, *, tenant_id: str) -> None:
|
||||
lock.release()
|
||||
|
||||
task_logger.info(
|
||||
f"check_user_file_processing - Enqueued {enqueued} skipped_guard={skipped_guard} "
|
||||
f"tasks for tenant={tenant_id}"
|
||||
f"check_user_file_processing - Enqueued {enqueued} tasks for tenant={tenant_id}"
|
||||
)
|
||||
return None
|
||||
|
||||
@@ -241,12 +172,6 @@ def process_single_user_file(self: Task, *, user_file_id: str, tenant_id: str) -
|
||||
start = time.monotonic()
|
||||
|
||||
redis_client = get_redis_client(tenant_id=tenant_id)
|
||||
|
||||
# Clear the "queued" guard set by the beat generator so that the next beat
|
||||
# cycle can re-enqueue this file if it is still in PROCESSING state after
|
||||
# this task completes or fails.
|
||||
redis_client.delete(_user_file_queued_key(user_file_id))
|
||||
|
||||
file_lock: RedisLock = redis_client.lock(
|
||||
_user_file_lock_key(user_file_id),
|
||||
timeout=CELERY_USER_FILE_PROCESSING_LOCK_TIMEOUT,
|
||||
|
||||
@@ -149,17 +149,6 @@ CELERY_EXTERNAL_GROUP_SYNC_LOCK_TIMEOUT = 300 # 5 min
|
||||
|
||||
CELERY_USER_FILE_PROCESSING_LOCK_TIMEOUT = 30 * 60 # 30 minutes (in seconds)
|
||||
|
||||
# How long a queued user-file task is valid before workers discard it.
|
||||
# Should be longer than the beat interval (20 s) but short enough to prevent
|
||||
# indefinite queue growth. Workers drop tasks older than this without touching
|
||||
# the DB, so a shorter value = faster drain of stale duplicates.
|
||||
CELERY_USER_FILE_PROCESSING_TASK_EXPIRES = 60 # 1 minute (in seconds)
|
||||
|
||||
# Maximum number of tasks allowed in the user-file-processing queue before the
|
||||
# beat generator stops adding more. Prevents unbounded queue growth when workers
|
||||
# fall behind.
|
||||
USER_FILE_PROCESSING_MAX_QUEUE_DEPTH = 500
|
||||
|
||||
CELERY_USER_FILE_PROJECT_SYNC_LOCK_TIMEOUT = 5 * 60 # 5 minutes (in seconds)
|
||||
|
||||
DANSWER_REDIS_FUNCTION_LOCK_PREFIX = "da_function_lock:"
|
||||
@@ -430,9 +419,6 @@ class OnyxRedisLocks:
|
||||
# User file processing
|
||||
USER_FILE_PROCESSING_BEAT_LOCK = "da_lock:check_user_file_processing_beat"
|
||||
USER_FILE_PROCESSING_LOCK_PREFIX = "da_lock:user_file_processing"
|
||||
# Short-lived key set when a task is enqueued; cleared when the worker picks it up.
|
||||
# Prevents the beat from re-enqueuing the same file while a task is already queued.
|
||||
USER_FILE_QUEUED_PREFIX = "da_lock:user_file_queued"
|
||||
USER_FILE_PROJECT_SYNC_BEAT_LOCK = "da_lock:check_user_file_project_sync_beat"
|
||||
USER_FILE_PROJECT_SYNC_LOCK_PREFIX = "da_lock:user_file_project_sync"
|
||||
USER_FILE_DELETE_BEAT_LOCK = "da_lock:check_user_file_delete_beat"
|
||||
|
||||
@@ -97,17 +97,10 @@ def basic_expert_info_representation(info: BasicExpertInfo) -> str | None:
|
||||
def get_experts_stores_representations(
|
||||
experts: list[BasicExpertInfo] | None,
|
||||
) -> list[str] | None:
|
||||
"""Gets string representations of experts supplied.
|
||||
|
||||
If an expert cannot be represented as a string, it is omitted from the
|
||||
result.
|
||||
"""
|
||||
if not experts:
|
||||
return None
|
||||
|
||||
reps: list[str | None] = [
|
||||
basic_expert_info_representation(owner) for owner in experts
|
||||
]
|
||||
reps = [basic_expert_info_representation(owner) for owner in experts]
|
||||
return [owner for owner in reps if owner is not None]
|
||||
|
||||
|
||||
|
||||
@@ -566,23 +566,6 @@ def extract_content_words_from_recency_query(
|
||||
return content_words_filtered[:MAX_CONTENT_WORDS]
|
||||
|
||||
|
||||
def _is_valid_keyword_query(line: str) -> bool:
|
||||
"""Check if a line looks like a valid keyword query vs explanatory text.
|
||||
|
||||
Returns False for lines that appear to be LLM explanations rather than keywords.
|
||||
"""
|
||||
# Reject lines that start with parentheses (explanatory notes)
|
||||
if line.startswith("("):
|
||||
return False
|
||||
|
||||
# Reject lines that are too long (likely sentences, not keywords)
|
||||
# Keywords should be short - reject if > 50 chars or > 6 words
|
||||
if len(line) > 50 or len(line.split()) > 6:
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def expand_query_with_llm(query_text: str, llm: LLM) -> list[str]:
|
||||
"""Use LLM to expand query into multiple search variations.
|
||||
|
||||
@@ -603,18 +586,10 @@ def expand_query_with_llm(query_text: str, llm: LLM) -> list[str]:
|
||||
response_clean = _parse_llm_code_block_response(response)
|
||||
|
||||
# Split into lines and filter out empty lines
|
||||
raw_queries = [
|
||||
rephrased_queries = [
|
||||
line.strip() for line in response_clean.split("\n") if line.strip()
|
||||
]
|
||||
|
||||
# Filter out lines that look like explanatory text rather than keywords
|
||||
rephrased_queries = [q for q in raw_queries if _is_valid_keyword_query(q)]
|
||||
|
||||
# Log if we filtered out garbage
|
||||
if len(raw_queries) != len(rephrased_queries):
|
||||
filtered_out = set(raw_queries) - set(rephrased_queries)
|
||||
logger.warning(f"Filtered out non-keyword LLM responses: {filtered_out}")
|
||||
|
||||
# If no queries generated, use empty query
|
||||
if not rephrased_queries:
|
||||
logger.debug("No content keywords extracted from query expansion")
|
||||
|
||||
@@ -444,8 +444,6 @@ def upsert_documents(
|
||||
logger.info("No documents to upsert. Skipping.")
|
||||
return
|
||||
|
||||
includes_permissions = any(doc.external_access for doc in seen_documents.values())
|
||||
|
||||
insert_stmt = insert(DbDocument).values(
|
||||
[
|
||||
model_to_dict(
|
||||
@@ -481,38 +479,21 @@ def upsert_documents(
|
||||
]
|
||||
)
|
||||
|
||||
update_set = {
|
||||
"from_ingestion_api": insert_stmt.excluded.from_ingestion_api,
|
||||
"boost": insert_stmt.excluded.boost,
|
||||
"hidden": insert_stmt.excluded.hidden,
|
||||
"semantic_id": insert_stmt.excluded.semantic_id,
|
||||
"link": insert_stmt.excluded.link,
|
||||
"primary_owners": insert_stmt.excluded.primary_owners,
|
||||
"secondary_owners": insert_stmt.excluded.secondary_owners,
|
||||
"doc_metadata": insert_stmt.excluded.doc_metadata,
|
||||
}
|
||||
if includes_permissions:
|
||||
# Use COALESCE to preserve existing permissions when new values are NULL.
|
||||
# This prevents subsequent indexing runs (which don't fetch permissions)
|
||||
# from overwriting permissions set by permission sync jobs.
|
||||
update_set.update(
|
||||
{
|
||||
"external_user_emails": func.coalesce(
|
||||
insert_stmt.excluded.external_user_emails,
|
||||
DbDocument.external_user_emails,
|
||||
),
|
||||
"external_user_group_ids": func.coalesce(
|
||||
insert_stmt.excluded.external_user_group_ids,
|
||||
DbDocument.external_user_group_ids,
|
||||
),
|
||||
"is_public": func.coalesce(
|
||||
insert_stmt.excluded.is_public,
|
||||
DbDocument.is_public,
|
||||
),
|
||||
}
|
||||
)
|
||||
on_conflict_stmt = insert_stmt.on_conflict_do_update(
|
||||
index_elements=["id"], set_=update_set # Conflict target
|
||||
index_elements=["id"], # Conflict target
|
||||
set_={
|
||||
"from_ingestion_api": insert_stmt.excluded.from_ingestion_api,
|
||||
"boost": insert_stmt.excluded.boost,
|
||||
"hidden": insert_stmt.excluded.hidden,
|
||||
"semantic_id": insert_stmt.excluded.semantic_id,
|
||||
"link": insert_stmt.excluded.link,
|
||||
"primary_owners": insert_stmt.excluded.primary_owners,
|
||||
"secondary_owners": insert_stmt.excluded.secondary_owners,
|
||||
"external_user_emails": insert_stmt.excluded.external_user_emails,
|
||||
"external_user_group_ids": insert_stmt.excluded.external_user_group_ids,
|
||||
"is_public": insert_stmt.excluded.is_public,
|
||||
"doc_metadata": insert_stmt.excluded.doc_metadata,
|
||||
},
|
||||
)
|
||||
db_session.execute(on_conflict_stmt)
|
||||
db_session.commit()
|
||||
|
||||
@@ -187,25 +187,13 @@ def _get_persona_by_name(
|
||||
return result
|
||||
|
||||
|
||||
def update_persona_access(
|
||||
def make_persona_private(
|
||||
persona_id: int,
|
||||
creator_user_id: UUID | None,
|
||||
user_ids: list[UUID] | None,
|
||||
group_ids: list[int] | None,
|
||||
db_session: Session,
|
||||
is_public: bool | None = None,
|
||||
user_ids: list[UUID] | None = None,
|
||||
group_ids: list[int] | None = None,
|
||||
) -> None:
|
||||
"""Updates the access settings for a persona including public status and user shares.
|
||||
|
||||
NOTE: Callers are responsible for committing."""
|
||||
|
||||
if is_public is not None:
|
||||
persona = db_session.query(Persona).filter(Persona.id == persona_id).first()
|
||||
if persona:
|
||||
persona.is_public = is_public
|
||||
|
||||
# NOTE: For user-ids and group-ids, `None` means "leave unchanged", `[]` means "clear all shares",
|
||||
# and a non-empty list means "replace with these shares".
|
||||
if user_ids is not None:
|
||||
db_session.query(Persona__User).filter(
|
||||
Persona__User.persona_id == persona_id
|
||||
@@ -224,15 +212,11 @@ def update_persona_access(
|
||||
).model_dump(),
|
||||
)
|
||||
|
||||
# MIT doesn't support group-based sharing, so we allow clearing (no-op since
|
||||
# there shouldn't be any) but raise an error if trying to add actual groups.
|
||||
if group_ids is not None:
|
||||
db_session.query(Persona__UserGroup).filter(
|
||||
Persona__UserGroup.persona_id == persona_id
|
||||
).delete(synchronize_session="fetch")
|
||||
db_session.commit()
|
||||
|
||||
if group_ids:
|
||||
raise NotImplementedError("Onyx MIT does not support group-based sharing")
|
||||
# May cause error if someone switches down to MIT from EE
|
||||
if group_ids:
|
||||
raise NotImplementedError("Onyx MIT does not support private Personas")
|
||||
|
||||
|
||||
def create_update_persona(
|
||||
@@ -298,21 +282,20 @@ def create_update_persona(
|
||||
llm_filter_extraction=create_persona_request.llm_filter_extraction,
|
||||
is_default_persona=create_persona_request.is_default_persona,
|
||||
user_file_ids=converted_user_file_ids,
|
||||
commit=False,
|
||||
)
|
||||
|
||||
versioned_update_persona_access = fetch_versioned_implementation(
|
||||
"onyx.db.persona", "update_persona_access"
|
||||
versioned_make_persona_private = fetch_versioned_implementation(
|
||||
"onyx.db.persona", "make_persona_private"
|
||||
)
|
||||
|
||||
versioned_update_persona_access(
|
||||
# Privatize Persona
|
||||
versioned_make_persona_private(
|
||||
persona_id=persona.id,
|
||||
creator_user_id=user.id if user else None,
|
||||
db_session=db_session,
|
||||
user_ids=create_persona_request.users,
|
||||
group_ids=create_persona_request.groups,
|
||||
db_session=db_session,
|
||||
)
|
||||
db_session.commit()
|
||||
|
||||
except ValueError as e:
|
||||
logger.exception("Failed to create persona")
|
||||
@@ -321,13 +304,11 @@ def create_update_persona(
|
||||
return FullPersonaSnapshot.from_model(persona)
|
||||
|
||||
|
||||
def update_persona_shared(
|
||||
def update_persona_shared_users(
|
||||
persona_id: int,
|
||||
user_ids: list[UUID],
|
||||
user: User | None,
|
||||
db_session: Session,
|
||||
user_ids: list[UUID] | None = None,
|
||||
group_ids: list[int] | None = None,
|
||||
is_public: bool | None = None,
|
||||
) -> None:
|
||||
"""Simplified version of `create_update_persona` which only touches the
|
||||
accessibility rather than any of the logic (e.g. prompt, connected data sources,
|
||||
@@ -336,25 +317,22 @@ def update_persona_shared(
|
||||
db_session=db_session, persona_id=persona_id, user=user, get_editable=True
|
||||
)
|
||||
|
||||
if user and user.role != UserRole.ADMIN and persona.user_id != user.id:
|
||||
raise HTTPException(
|
||||
status_code=403, detail="You don't have permission to modify this persona"
|
||||
)
|
||||
if persona.is_public:
|
||||
raise HTTPException(status_code=400, detail="Cannot share public persona")
|
||||
|
||||
versioned_update_persona_access = fetch_versioned_implementation(
|
||||
"onyx.db.persona", "update_persona_access"
|
||||
versioned_make_persona_private = fetch_versioned_implementation(
|
||||
"onyx.db.persona", "make_persona_private"
|
||||
)
|
||||
versioned_update_persona_access(
|
||||
|
||||
# Privatize Persona
|
||||
versioned_make_persona_private(
|
||||
persona_id=persona_id,
|
||||
creator_user_id=user.id if user else None,
|
||||
db_session=db_session,
|
||||
is_public=is_public,
|
||||
user_ids=user_ids,
|
||||
group_ids=group_ids,
|
||||
group_ids=None,
|
||||
db_session=db_session,
|
||||
)
|
||||
|
||||
db_session.commit()
|
||||
|
||||
|
||||
def update_persona_public_status(
|
||||
persona_id: int,
|
||||
|
||||
@@ -113,6 +113,7 @@ def upsert_web_search_provider(
|
||||
if activate:
|
||||
set_active_web_search_provider(provider_id=provider.id, db_session=db_session)
|
||||
|
||||
db_session.commit()
|
||||
db_session.refresh(provider)
|
||||
return provider
|
||||
|
||||
@@ -268,6 +269,7 @@ def upsert_web_content_provider(
|
||||
if activate:
|
||||
set_active_web_content_provider(provider_id=provider.id, db_session=db_session)
|
||||
|
||||
db_session.commit()
|
||||
db_session.refresh(provider)
|
||||
return provider
|
||||
|
||||
|
||||
@@ -3,9 +3,6 @@ import json
|
||||
import httpx
|
||||
|
||||
from onyx.configs.chat_configs import TITLE_CONTENT_RATIO
|
||||
from onyx.connectors.cross_connector_utils.miscellaneous_utils import (
|
||||
get_experts_stores_representations,
|
||||
)
|
||||
from onyx.context.search.enums import QueryType
|
||||
from onyx.context.search.models import IndexFilters
|
||||
from onyx.context.search.models import InferenceChunk
|
||||
@@ -47,7 +44,6 @@ from onyx.document_index.opensearch.search import (
|
||||
from onyx.indexing.models import DocMetadataAwareIndexChunk
|
||||
from onyx.indexing.models import Document
|
||||
from onyx.utils.logger import setup_logger
|
||||
from shared_configs.configs import MULTI_TENANT
|
||||
from shared_configs.model_server_models import Embedding
|
||||
|
||||
|
||||
@@ -63,7 +59,10 @@ def _convert_opensearch_chunk_to_inference_chunk_uncleaned(
|
||||
content=chunk.content,
|
||||
source_links=json.loads(chunk.source_links) if chunk.source_links else None,
|
||||
image_file_id=chunk.image_file_id,
|
||||
# Deprecated. Fill in some reasonable default.
|
||||
# TODO(andrei) Yuhong says he doesn't think we need that anymore. Used
|
||||
# if a section needed to be split into diff chunks. A section is a part
|
||||
# of a doc that a link will take you to. But don't chunks have their own
|
||||
# links? Look at this in a followup.
|
||||
section_continuation=False,
|
||||
document_id=chunk.document_id,
|
||||
source_type=DocumentSource(chunk.source_type),
|
||||
@@ -78,20 +77,27 @@ def _convert_opensearch_chunk_to_inference_chunk_uncleaned(
|
||||
# a search result, do that in a followup.
|
||||
score=None,
|
||||
hidden=chunk.hidden,
|
||||
# TODO(andrei): Don't worry about these for now.
|
||||
# is_relevant
|
||||
# relevance_explanation
|
||||
# TODO(andrei): Same comment as in
|
||||
# _convert_onyx_chunk_to_opensearch_document.
|
||||
metadata=json.loads(chunk.metadata),
|
||||
# TODO(andrei): The vector DB needs to supply this. I vaguely know
|
||||
# OpenSearch can from the documentation I've seen till now, look at this
|
||||
# in a followup.
|
||||
match_highlights=[],
|
||||
# TODO(andrei) Consider storing a chunk content index instead of a full
|
||||
# string when working on chunk content augmentation.
|
||||
doc_summary=chunk.doc_summary,
|
||||
# TODO(andrei) This content is not queried on, it is only used to clean
|
||||
# appended content to chunks. Consider storing a chunk content index
|
||||
# instead of a full string when working on chunk content augmentation.
|
||||
doc_summary="",
|
||||
# TODO(andrei) Same thing as contx ret above, LLM gens context for each
|
||||
# chunk.
|
||||
chunk_context=chunk.chunk_context,
|
||||
chunk_context="",
|
||||
updated_at=chunk.last_updated,
|
||||
primary_owners=chunk.primary_owners,
|
||||
secondary_owners=chunk.secondary_owners,
|
||||
# primary_owners TODO(andrei)
|
||||
# secondary_owners TODO(andrei)
|
||||
# large_chunk_reference_ids TODO(andrei): Don't worry about this one.
|
||||
# TODO(andrei): This is the suffix appended to the end of the chunk
|
||||
# content to assist querying. There are better ways we can do this, for
|
||||
# ex. keeping an index of where to string split from.
|
||||
@@ -116,31 +122,34 @@ def _convert_onyx_chunk_to_opensearch_document(
|
||||
title_vector=chunk.title_embedding,
|
||||
content=chunk.content,
|
||||
content_vector=chunk.embeddings.full_embedding,
|
||||
# TODO(andrei): We should know this. Reason to have this is convenience,
|
||||
# but it could also change when you change your embedding model, maybe
|
||||
# we can remove it, Yuhong to look at this. Hardcoded to some nonsense
|
||||
# value for now.
|
||||
num_tokens=0,
|
||||
source_type=chunk.source_document.source.value,
|
||||
metadata=json.dumps(chunk.source_document.metadata),
|
||||
last_updated=chunk.source_document.doc_updated_at,
|
||||
# TODO(andrei): Don't currently see an easy way of porting this, and
|
||||
# besides some connectors genuinely don't have this data. Look at this
|
||||
# closer in a followup. Always defaults to None for now.
|
||||
# created_at=None,
|
||||
public=chunk.access.is_public,
|
||||
access_control_list=list(chunk.access.to_acl()),
|
||||
global_boost=chunk.boost,
|
||||
semantic_identifier=chunk.source_document.semantic_identifier,
|
||||
image_file_id=chunk.image_file_id,
|
||||
# TODO(andrei): Ask Chris more about this later. Always defaults to None
|
||||
# for now.
|
||||
image_file_id=None,
|
||||
source_links=json.dumps(chunk.source_links) if chunk.source_links else None,
|
||||
blurb=chunk.blurb,
|
||||
doc_summary=chunk.doc_summary,
|
||||
chunk_context=chunk.chunk_context,
|
||||
document_sets=list(chunk.document_sets) if chunk.document_sets else None,
|
||||
project_ids=list(chunk.user_project) if chunk.user_project else None,
|
||||
primary_owners=get_experts_stores_representations(
|
||||
chunk.source_document.primary_owners
|
||||
),
|
||||
secondary_owners=get_experts_stores_representations(
|
||||
chunk.source_document.secondary_owners
|
||||
),
|
||||
# TODO(andrei): Consider not even getting this from
|
||||
# DocMetadataAwareIndexChunk and instead using OpenSearchDocumentIndex's
|
||||
# instance variable. One source of truth -> less chance of a very bad
|
||||
# bug in prod.
|
||||
tenant_id=TenantState(tenant_id=chunk.tenant_id, multitenant=MULTI_TENANT),
|
||||
tenant_id=chunk.tenant_id,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -4,29 +4,24 @@ from typing import Any
|
||||
from typing import Self
|
||||
|
||||
from pydantic import BaseModel
|
||||
from pydantic import Field
|
||||
from pydantic import field_serializer
|
||||
from pydantic import field_validator
|
||||
from pydantic import model_serializer
|
||||
from pydantic import model_validator
|
||||
from pydantic import SerializerFunctionWrapHandler
|
||||
|
||||
from onyx.document_index.interfaces_new import TenantState
|
||||
from onyx.document_index.opensearch.constants import DEFAULT_MAX_CHUNK_SIZE
|
||||
from onyx.document_index.opensearch.constants import EF_CONSTRUCTION
|
||||
from onyx.document_index.opensearch.constants import EF_SEARCH
|
||||
from onyx.document_index.opensearch.constants import M
|
||||
from shared_configs.configs import MULTI_TENANT
|
||||
from shared_configs.contextvars import get_current_tenant_id
|
||||
|
||||
|
||||
TITLE_FIELD_NAME = "title"
|
||||
TITLE_VECTOR_FIELD_NAME = "title_vector"
|
||||
CONTENT_FIELD_NAME = "content"
|
||||
CONTENT_VECTOR_FIELD_NAME = "content_vector"
|
||||
NUM_TOKENS_FIELD_NAME = "num_tokens"
|
||||
SOURCE_TYPE_FIELD_NAME = "source_type"
|
||||
METADATA_FIELD_NAME = "metadata"
|
||||
LAST_UPDATED_FIELD_NAME = "last_updated"
|
||||
CREATED_AT_FIELD_NAME = "created_at"
|
||||
PUBLIC_FIELD_NAME = "public"
|
||||
ACCESS_CONTROL_LIST_FIELD_NAME = "access_control_list"
|
||||
HIDDEN_FIELD_NAME = "hidden"
|
||||
@@ -41,10 +36,6 @@ CHUNK_INDEX_FIELD_NAME = "chunk_index"
|
||||
MAX_CHUNK_SIZE_FIELD_NAME = "max_chunk_size"
|
||||
TENANT_ID_FIELD_NAME = "tenant_id"
|
||||
BLURB_FIELD_NAME = "blurb"
|
||||
DOC_SUMMARY_FIELD_NAME = "doc_summary"
|
||||
CHUNK_CONTEXT_FIELD_NAME = "chunk_context"
|
||||
PRIMARY_OWNERS_FIELD_NAME = "primary_owners"
|
||||
SECONDARY_OWNERS_FIELD_NAME = "secondary_owners"
|
||||
|
||||
|
||||
def get_opensearch_doc_chunk_id(
|
||||
@@ -61,27 +52,12 @@ def get_opensearch_doc_chunk_id(
|
||||
return f"{document_id}__{max_chunk_size}__{chunk_index}"
|
||||
|
||||
|
||||
def set_or_convert_timezone_to_utc(value: datetime) -> datetime:
|
||||
if value.tzinfo is None:
|
||||
# astimezone will raise if value does not have a timezone set.
|
||||
value = value.replace(tzinfo=timezone.utc)
|
||||
else:
|
||||
# Does appropriate time conversion if value was set in a different
|
||||
# timezone.
|
||||
value = value.astimezone(timezone.utc)
|
||||
return value
|
||||
|
||||
|
||||
class DocumentChunk(BaseModel):
|
||||
"""
|
||||
Represents a chunk of a document in the OpenSearch index.
|
||||
|
||||
The names of these fields are based on the OpenSearch schema. Changes to the
|
||||
schema require changes here. See get_document_schema.
|
||||
|
||||
WARNING: Relies on MULTI_TENANT which is global state. Also uses
|
||||
get_current_tenant_id. Generally relying on global state is bad, in this
|
||||
case we accept it because of the importance of validating tenant logic.
|
||||
"""
|
||||
|
||||
model_config = {"frozen": True}
|
||||
@@ -95,10 +71,12 @@ class DocumentChunk(BaseModel):
|
||||
max_chunk_size: int = DEFAULT_MAX_CHUNK_SIZE
|
||||
|
||||
# Either both should be None or both should be non-None.
|
||||
title: str | None = None
|
||||
title_vector: list[float] | None = None
|
||||
title: str | None
|
||||
title_vector: list[float] | None
|
||||
content: str
|
||||
content_vector: list[float]
|
||||
# The actual number of tokens in the chunk.
|
||||
num_tokens: int
|
||||
|
||||
source_type: str
|
||||
# Contains a string representation of a dict which maps string key to either
|
||||
@@ -107,8 +85,8 @@ class DocumentChunk(BaseModel):
|
||||
# index pointer, and when we support metadata list that will just be a list
|
||||
# of strings.
|
||||
metadata: str
|
||||
# If it exists, time zone should always be UTC.
|
||||
last_updated: datetime | None = None
|
||||
last_updated: datetime | None
|
||||
created_at: datetime | None = None
|
||||
|
||||
public: bool
|
||||
access_control_list: list[str]
|
||||
@@ -118,25 +96,24 @@ class DocumentChunk(BaseModel):
|
||||
global_boost: int
|
||||
|
||||
semantic_identifier: str
|
||||
image_file_id: str | None = None
|
||||
image_file_id: str | None
|
||||
# Contains a string representation of a dict which maps offset into the raw
|
||||
# chunk text to the link corresponding to that point.
|
||||
source_links: str | None = None
|
||||
source_links: str | None
|
||||
blurb: str
|
||||
doc_summary: str
|
||||
chunk_context: str
|
||||
|
||||
document_sets: list[str] | None = None
|
||||
# User projects.
|
||||
project_ids: list[int] | None = None
|
||||
primary_owners: list[str] | None = None
|
||||
secondary_owners: list[str] | None = None
|
||||
document_sets: list[str] | None
|
||||
project_ids: list[int] | None
|
||||
|
||||
tenant_id: TenantState = Field(
|
||||
default_factory=lambda: TenantState(
|
||||
tenant_id=get_current_tenant_id(), multitenant=MULTI_TENANT
|
||||
)
|
||||
)
|
||||
tenant_id: str | None
|
||||
|
||||
@model_validator(mode="after")
|
||||
def check_num_tokens_fits_within_max_chunk_size(self) -> Self:
|
||||
if self.num_tokens > self.max_chunk_size:
|
||||
raise ValueError(
|
||||
"Bug: Num tokens must be less than or equal to max chunk size."
|
||||
)
|
||||
return self
|
||||
|
||||
@model_validator(mode="after")
|
||||
def check_title_and_title_vector_are_consistent(self) -> Self:
|
||||
@@ -147,116 +124,25 @@ class DocumentChunk(BaseModel):
|
||||
raise ValueError("Bug: Title must not be None if title vector is not None.")
|
||||
return self
|
||||
|
||||
@model_serializer(mode="wrap")
|
||||
def serialize_model(
|
||||
self, handler: SerializerFunctionWrapHandler
|
||||
) -> dict[str, object]:
|
||||
"""Invokes pydantic's serialization logic, then excludes Nones.
|
||||
|
||||
We do this because .model_dump(exclude_none=True) does not work after
|
||||
@field_serializer logic, so for some field serializers which return None
|
||||
and which we would like to exclude from the final dump, they would be
|
||||
included without this.
|
||||
|
||||
Args:
|
||||
handler: Callable from pydantic which takes the instance of the
|
||||
model as an argument and performs standard serialization.
|
||||
|
||||
Returns:
|
||||
The return of handler but with None items excluded.
|
||||
"""
|
||||
serialized: dict[str, object] = handler(self)
|
||||
serialized_exclude_none = {k: v for k, v in serialized.items() if v is not None}
|
||||
return serialized_exclude_none
|
||||
|
||||
@field_serializer("last_updated", mode="wrap")
|
||||
@field_serializer("last_updated", "created_at", mode="plain")
|
||||
def serialize_datetime_fields_to_epoch_millis(
|
||||
self, value: datetime | None, handler: SerializerFunctionWrapHandler
|
||||
self, value: datetime | None
|
||||
) -> int | None:
|
||||
"""
|
||||
Serializes datetime fields to milliseconds since the Unix epoch.
|
||||
|
||||
If there is no datetime, returns None.
|
||||
"""
|
||||
if value is None:
|
||||
return None
|
||||
value = set_or_convert_timezone_to_utc(value)
|
||||
if value.tzinfo is None:
|
||||
# astimezone will raise if value does not have a timezone set.
|
||||
value = value.replace(tzinfo=timezone.utc)
|
||||
else:
|
||||
# Does appropriate time conversion if value was set in a different
|
||||
# timezone.
|
||||
value = value.astimezone(timezone.utc)
|
||||
# timestamp returns a float in seconds so convert to millis.
|
||||
return int(value.timestamp() * 1000)
|
||||
|
||||
@field_validator("last_updated", mode="before")
|
||||
@classmethod
|
||||
def parse_epoch_millis_to_datetime(cls, value: Any) -> datetime | None:
|
||||
"""Parses milliseconds since the Unix epoch to a datetime object.
|
||||
|
||||
If the input is None, returns None.
|
||||
|
||||
The datetime returned will be in UTC.
|
||||
"""
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, datetime):
|
||||
value = set_or_convert_timezone_to_utc(value)
|
||||
return value
|
||||
if not isinstance(value, int):
|
||||
raise ValueError(
|
||||
f"Bug: Expected an int for the last_updated property from OpenSearch, got {type(value)} instead."
|
||||
)
|
||||
return datetime.fromtimestamp(value / 1000, tz=timezone.utc)
|
||||
|
||||
@field_serializer("tenant_id", mode="wrap")
|
||||
def serialize_tenant_state(
|
||||
self, value: TenantState, handler: SerializerFunctionWrapHandler
|
||||
) -> str | None:
|
||||
"""
|
||||
Serializes tenant_state to the tenant str if multitenant, or None if
|
||||
not.
|
||||
|
||||
The idea is that in single tenant mode, the schema does not have a
|
||||
tenant_id field, so we don't want to supply it in our serialized
|
||||
DocumentChunk. This assumes the final serialized model excludes None
|
||||
fields, which serialize_model should enforce.
|
||||
"""
|
||||
if not value.multitenant:
|
||||
return None
|
||||
else:
|
||||
return value.tenant_id
|
||||
|
||||
@field_validator("tenant_id", mode="before")
|
||||
@classmethod
|
||||
def parse_tenant_id(cls, value: Any) -> TenantState:
|
||||
"""
|
||||
Generates a TenantState from OpenSearch's tenant_id if it exists, or
|
||||
generates a default state if it does not (implies we are in single
|
||||
tenant mode).
|
||||
"""
|
||||
if value is None:
|
||||
if MULTI_TENANT:
|
||||
raise ValueError(
|
||||
"Bug: No tenant_id was supplied but multi-tenant mode is enabled."
|
||||
)
|
||||
return TenantState(
|
||||
tenant_id=get_current_tenant_id(), multitenant=MULTI_TENANT
|
||||
)
|
||||
elif isinstance(value, TenantState):
|
||||
if MULTI_TENANT != value.multitenant:
|
||||
raise ValueError(
|
||||
f"Bug: An existing TenantState object was supplied to the DocumentChunk model but its multi-tenant mode "
|
||||
f"({value.multitenant}) does not match the program's current global tenancy state."
|
||||
)
|
||||
return value
|
||||
elif not isinstance(value, str):
|
||||
raise ValueError(
|
||||
f"Bug: Expected a str for the tenant_id property from OpenSearch, got {type(value)} instead."
|
||||
)
|
||||
else:
|
||||
if not MULTI_TENANT:
|
||||
raise ValueError(
|
||||
"Bug: Got a non-null str for the tenant_id property from OpenSearch but multi-tenant mode is not enabled. "
|
||||
"This is unexpected because in single-tenant mode we don't expect to see a tenant_id."
|
||||
)
|
||||
return TenantState(tenant_id=value, multitenant=MULTI_TENANT)
|
||||
|
||||
|
||||
class DocumentSchema:
|
||||
"""
|
||||
@@ -294,12 +180,7 @@ class DocumentSchema:
|
||||
OpenSearch client. The structure of this dictionary is
|
||||
determined by OpenSearch documentation.
|
||||
"""
|
||||
schema: dict[str, Any] = {
|
||||
# By default OpenSearch allows dynamically adding new properties
|
||||
# based on indexed documents. This is awful and we disable it here.
|
||||
# An exception will be raised if you try to index a new doc which
|
||||
# contains unexpected fields.
|
||||
"dynamic": "strict",
|
||||
schema = {
|
||||
"properties": {
|
||||
TITLE_FIELD_NAME: {
|
||||
"type": "text",
|
||||
@@ -336,6 +217,11 @@ class DocumentSchema:
|
||||
"parameters": {"ef_construction": EF_CONSTRUCTION, "m": M},
|
||||
},
|
||||
},
|
||||
# See TODO in _convert_onyx_chunk_to_opensearch_document. I
|
||||
# don't want to actually add this to the schema until we know
|
||||
# for sure we need it. If we decide we don't I will remove this.
|
||||
# # Number of tokens in the chunk's content.
|
||||
# NUM_TOKENS_FIELD_NAME: {"type": "integer", "store": True},
|
||||
SOURCE_TYPE_FIELD_NAME: {"type": "keyword"},
|
||||
METADATA_FIELD_NAME: {"type": "keyword"},
|
||||
# TODO(andrei): Check if Vespa stores seconds, we may wanna do
|
||||
@@ -347,6 +233,16 @@ class DocumentSchema:
|
||||
# would make sense to sort by date.
|
||||
"doc_values": True,
|
||||
},
|
||||
# See TODO in _convert_onyx_chunk_to_opensearch_document. I
|
||||
# don't want to actually add this to the schema until we know
|
||||
# for sure we need it. If we decide we don't I will remove this.
|
||||
# CREATED_AT_FIELD_NAME: {
|
||||
# "type": "date",
|
||||
# "format": "epoch_millis",
|
||||
# # For some reason date defaults to False, even though it
|
||||
# # would make sense to sort by date.
|
||||
# "doc_values": True,
|
||||
# },
|
||||
# Access control fields.
|
||||
# Whether the doc is public. Could have fallen under access
|
||||
# control list but is such a broad and critical filter that it
|
||||
@@ -390,36 +286,16 @@ class DocumentSchema:
|
||||
"doc_values": False,
|
||||
"store": False,
|
||||
},
|
||||
# Same as above.
|
||||
# TODO(andrei): If we want to search on this this needs to be
|
||||
# changed.
|
||||
DOC_SUMMARY_FIELD_NAME: {
|
||||
"type": "keyword",
|
||||
"index": False,
|
||||
"doc_values": False,
|
||||
"store": False,
|
||||
},
|
||||
# Same as above.
|
||||
# TODO(andrei): If we want to search on this this needs to be
|
||||
# changed.
|
||||
CHUNK_CONTEXT_FIELD_NAME: {
|
||||
"type": "keyword",
|
||||
"index": False,
|
||||
"doc_values": False,
|
||||
"store": False,
|
||||
},
|
||||
# Product-specific fields.
|
||||
DOCUMENT_SETS_FIELD_NAME: {"type": "keyword"},
|
||||
PROJECT_IDS_FIELD_NAME: {"type": "integer"},
|
||||
PRIMARY_OWNERS_FIELD_NAME: {"type": "keyword"},
|
||||
SECONDARY_OWNERS_FIELD_NAME: {"type": "keyword"},
|
||||
# OpenSearch metadata fields.
|
||||
DOCUMENT_ID_FIELD_NAME: {"type": "keyword"},
|
||||
CHUNK_INDEX_FIELD_NAME: {"type": "integer"},
|
||||
# The maximum number of tokens this chunk's content can hold.
|
||||
# TODO(andrei): Can we generalize this to embedding type?
|
||||
MAX_CHUNK_SIZE_FIELD_NAME: {"type": "integer"},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
if multitenant:
|
||||
|
||||
@@ -24,7 +24,7 @@ from onyx.document_index.opensearch.schema import TITLE_VECTOR_FIELD_NAME
|
||||
# TODO(andrei): Turn all magic dictionaries to pydantic models.
|
||||
|
||||
MIN_MAX_NORMALIZATION_PIPELINE_NAME = "normalization_pipeline_min_max"
|
||||
MIN_MAX_NORMALIZATION_PIPELINE_CONFIG: dict[str, Any] = {
|
||||
MIN_MAX_NORMALIZATION_PIPELINE_CONFIG = {
|
||||
"description": "Normalization for keyword and vector scores using min-max",
|
||||
"phase_results_processors": [
|
||||
{
|
||||
@@ -49,7 +49,7 @@ MIN_MAX_NORMALIZATION_PIPELINE_CONFIG: dict[str, Any] = {
|
||||
}
|
||||
|
||||
ZSCORE_NORMALIZATION_PIPELINE_NAME = "normalization_pipeline_zscore"
|
||||
ZSCORE_NORMALIZATION_PIPELINE_CONFIG: dict[str, Any] = {
|
||||
ZSCORE_NORMALIZATION_PIPELINE_CONFIG = {
|
||||
"description": "Normalization for keyword and vector scores using z-score",
|
||||
"phase_results_processors": [
|
||||
{
|
||||
@@ -140,7 +140,7 @@ class DocumentQuery:
|
||||
{"term": {DOCUMENT_ID_FIELD_NAME: {"value": document_id}}}
|
||||
]
|
||||
|
||||
if tenant_state.multitenant:
|
||||
if tenant_state.tenant_id is not None:
|
||||
# TODO(andrei): Fix tenant stuff.
|
||||
filter_clauses.append(
|
||||
{"term": {TENANT_ID_FIELD_NAME: {"value": tenant_state.tenant_id}}}
|
||||
@@ -199,7 +199,7 @@ class DocumentQuery:
|
||||
{"term": {DOCUMENT_ID_FIELD_NAME: {"value": document_id}}}
|
||||
]
|
||||
|
||||
if tenant_state.multitenant:
|
||||
if tenant_state.tenant_id is not None:
|
||||
filter_clauses.append(
|
||||
{"term": {TENANT_ID_FIELD_NAME: {"value": tenant_state.tenant_id}}}
|
||||
)
|
||||
@@ -341,7 +341,7 @@ class DocumentQuery:
|
||||
{"term": {PUBLIC_FIELD_NAME: {"value": True}}},
|
||||
{"term": {HIDDEN_FIELD_NAME: {"value": False}}},
|
||||
]
|
||||
if tenant_state.multitenant:
|
||||
if tenant_state.tenant_id is not None:
|
||||
hybrid_search_filters.append(
|
||||
{"term": {TENANT_ID_FIELD_NAME: {"value": tenant_state.tenant_id}}}
|
||||
)
|
||||
|
||||
@@ -40,7 +40,6 @@ class BaseChunk(BaseModel):
|
||||
source_links: dict[int, str] | None
|
||||
image_file_id: str | None
|
||||
# True if this Chunk's start is not at the start of a Section
|
||||
# TODO(andrei): This is deprecated as of the OpenSearch migration. Remove.
|
||||
section_continuation: bool
|
||||
|
||||
|
||||
|
||||
@@ -369,8 +369,6 @@ def _patch_openai_responses_chunk_parser() -> None:
|
||||
# New output item added
|
||||
output_item = parsed_chunk.get("item", {})
|
||||
if output_item.get("type") == "function_call":
|
||||
# Track that we've received tool calls via streaming
|
||||
self._has_streamed_tool_calls = True
|
||||
return GenericStreamingChunk(
|
||||
text="",
|
||||
tool_use=ChatCompletionToolCallChunk(
|
||||
@@ -396,8 +394,6 @@ def _patch_openai_responses_chunk_parser() -> None:
|
||||
elif event_type == "response.function_call_arguments.delta":
|
||||
content_part: Optional[str] = parsed_chunk.get("delta", None)
|
||||
if content_part:
|
||||
# Track that we've received tool calls via streaming
|
||||
self._has_streamed_tool_calls = True
|
||||
return GenericStreamingChunk(
|
||||
text="",
|
||||
tool_use=ChatCompletionToolCallChunk(
|
||||
@@ -495,71 +491,21 @@ def _patch_openai_responses_chunk_parser() -> None:
|
||||
|
||||
elif event_type == "response.completed":
|
||||
# Final event signaling all output items (including parallel tool calls) are done
|
||||
# Check if we already received tool calls via streaming events
|
||||
# There is an issue where OpenAI (not via Azure) will give back the tool calls streamed out as tokens
|
||||
# But on Azure, it's only given out all at once. OpenAI also happens to give back the tool calls in the
|
||||
# response.completed event so we need to throw it out here or there are duplicate tool calls.
|
||||
has_streamed_tool_calls = getattr(self, "_has_streamed_tool_calls", False)
|
||||
|
||||
response_data = parsed_chunk.get("response", {})
|
||||
output_items = response_data.get("output", [])
|
||||
|
||||
# Check if there are function_call items in the output
|
||||
has_function_calls = any(
|
||||
isinstance(item, dict) and item.get("type") == "function_call"
|
||||
for item in output_items
|
||||
)
|
||||
|
||||
if has_function_calls and not has_streamed_tool_calls:
|
||||
# Azure's Responses API returns all tool calls in response.completed
|
||||
# without streaming them incrementally. Extract them here.
|
||||
from litellm.types.utils import (
|
||||
Delta,
|
||||
ModelResponseStream,
|
||||
StreamingChoices,
|
||||
)
|
||||
|
||||
tool_calls = []
|
||||
for idx, item in enumerate(output_items):
|
||||
# Determine finish reason based on response content
|
||||
finish_reason = "stop"
|
||||
if response_data.get("output"):
|
||||
for item in response_data["output"]:
|
||||
if isinstance(item, dict) and item.get("type") == "function_call":
|
||||
tool_calls.append(
|
||||
ChatCompletionToolCallChunk(
|
||||
id=item.get("call_id"),
|
||||
index=idx,
|
||||
type="function",
|
||||
function=ChatCompletionToolCallFunctionChunk(
|
||||
name=item.get("name"),
|
||||
arguments=item.get("arguments", ""),
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
return ModelResponseStream(
|
||||
choices=[
|
||||
StreamingChoices(
|
||||
index=0,
|
||||
delta=Delta(tool_calls=tool_calls),
|
||||
finish_reason="tool_calls",
|
||||
)
|
||||
]
|
||||
)
|
||||
elif has_function_calls:
|
||||
# Tool calls were already streamed, just signal completion
|
||||
return GenericStreamingChunk(
|
||||
text="",
|
||||
tool_use=None,
|
||||
is_finished=True,
|
||||
finish_reason="tool_calls",
|
||||
usage=None,
|
||||
)
|
||||
else:
|
||||
return GenericStreamingChunk(
|
||||
text="",
|
||||
tool_use=None,
|
||||
is_finished=True,
|
||||
finish_reason="stop",
|
||||
usage=None,
|
||||
)
|
||||
finish_reason = "tool_calls"
|
||||
break
|
||||
return GenericStreamingChunk(
|
||||
text="",
|
||||
tool_use=None,
|
||||
is_finished=True,
|
||||
finish_reason=finish_reason,
|
||||
usage=None,
|
||||
)
|
||||
|
||||
else:
|
||||
pass
|
||||
@@ -685,40 +631,6 @@ def _patch_openai_responses_transform_response() -> None:
|
||||
LiteLLMResponsesTransformationHandler.transform_response = _patched_transform_response # type: ignore[method-assign]
|
||||
|
||||
|
||||
def _patch_azure_responses_should_fake_stream() -> None:
|
||||
"""
|
||||
Patches AzureOpenAIResponsesAPIConfig.should_fake_stream to always return False.
|
||||
|
||||
By default, LiteLLM uses "fake streaming" (MockResponsesAPIStreamingIterator) for models
|
||||
not in its database. This causes Azure custom model deployments to buffer the entire
|
||||
response before yielding, resulting in poor time-to-first-token.
|
||||
|
||||
Azure's Responses API supports native streaming, so we override this to always use
|
||||
real streaming (SyncResponsesAPIStreamingIterator).
|
||||
"""
|
||||
from litellm.llms.azure.responses.transformation import (
|
||||
AzureOpenAIResponsesAPIConfig,
|
||||
)
|
||||
|
||||
if (
|
||||
getattr(AzureOpenAIResponsesAPIConfig.should_fake_stream, "__name__", "")
|
||||
== "_patched_should_fake_stream"
|
||||
):
|
||||
return
|
||||
|
||||
def _patched_should_fake_stream(
|
||||
self: Any,
|
||||
model: Optional[str],
|
||||
stream: Optional[bool],
|
||||
custom_llm_provider: Optional[str] = None,
|
||||
) -> bool:
|
||||
# Azure Responses API supports native streaming - never fake it
|
||||
return False
|
||||
|
||||
_patched_should_fake_stream.__name__ = "_patched_should_fake_stream"
|
||||
AzureOpenAIResponsesAPIConfig.should_fake_stream = _patched_should_fake_stream # type: ignore[method-assign]
|
||||
|
||||
|
||||
def apply_monkey_patches() -> None:
|
||||
"""
|
||||
Apply all necessary monkey patches to LiteLLM for compatibility.
|
||||
@@ -728,13 +640,12 @@ def apply_monkey_patches() -> None:
|
||||
- Patching OllamaChatCompletionResponseIterator.chunk_parser for streaming content
|
||||
- Patching OpenAiResponsesToChatCompletionStreamIterator.chunk_parser for OpenAI Responses API
|
||||
- Patching LiteLLMResponsesTransformationHandler.transform_response for non-streaming responses
|
||||
- Patching AzureOpenAIResponsesAPIConfig.should_fake_stream to enable native streaming
|
||||
- Patching LiteLLMResponsesTransformationHandler._convert_content_str_to_input_text for tool content types
|
||||
"""
|
||||
_patch_ollama_transform_request()
|
||||
_patch_ollama_chunk_parser()
|
||||
_patch_openai_responses_chunk_parser()
|
||||
_patch_openai_responses_transform_response()
|
||||
_patch_azure_responses_should_fake_stream()
|
||||
|
||||
|
||||
def _extract_reasoning_content(message: dict) -> Tuple[Optional[str], Optional[str]]:
|
||||
|
||||
@@ -63,7 +63,7 @@ def process_with_prompt_cache(
|
||||
return suffix, None
|
||||
|
||||
# Get provider adapter
|
||||
provider_adapter = get_provider_adapter(llm_config)
|
||||
provider_adapter = get_provider_adapter(llm_config.model_provider)
|
||||
|
||||
# If provider doesn't support caching, combine and return unchanged
|
||||
if not provider_adapter.supports_caching():
|
||||
|
||||
@@ -1,17 +1,14 @@
|
||||
"""Factory for creating provider-specific prompt cache adapters."""
|
||||
|
||||
from onyx.llm.constants import LlmProviderNames
|
||||
from onyx.llm.interfaces import LLMConfig
|
||||
from onyx.llm.prompt_cache.providers.anthropic import AnthropicPromptCacheProvider
|
||||
from onyx.llm.prompt_cache.providers.base import PromptCacheProvider
|
||||
from onyx.llm.prompt_cache.providers.noop import NoOpPromptCacheProvider
|
||||
from onyx.llm.prompt_cache.providers.openai import OpenAIPromptCacheProvider
|
||||
from onyx.llm.prompt_cache.providers.vertex import VertexAIPromptCacheProvider
|
||||
|
||||
ANTHROPIC_BEDROCK_TAG = "anthropic."
|
||||
|
||||
|
||||
def get_provider_adapter(llm_config: LLMConfig) -> PromptCacheProvider:
|
||||
def get_provider_adapter(provider: str) -> PromptCacheProvider:
|
||||
"""Get the appropriate prompt cache provider adapter for a given provider.
|
||||
|
||||
Args:
|
||||
@@ -20,14 +17,11 @@ def get_provider_adapter(llm_config: LLMConfig) -> PromptCacheProvider:
|
||||
Returns:
|
||||
PromptCacheProvider instance for the given provider
|
||||
"""
|
||||
if llm_config.model_provider == LlmProviderNames.OPENAI:
|
||||
if provider == LlmProviderNames.OPENAI:
|
||||
return OpenAIPromptCacheProvider()
|
||||
elif llm_config.model_provider == LlmProviderNames.ANTHROPIC or (
|
||||
llm_config.model_provider == LlmProviderNames.BEDROCK
|
||||
and ANTHROPIC_BEDROCK_TAG in llm_config.model_name
|
||||
):
|
||||
elif provider in [LlmProviderNames.ANTHROPIC, LlmProviderNames.BEDROCK]:
|
||||
return AnthropicPromptCacheProvider()
|
||||
elif llm_config.model_provider == LlmProviderNames.VERTEX_AI:
|
||||
elif provider == LlmProviderNames.VERTEX_AI:
|
||||
return VertexAIPromptCacheProvider()
|
||||
else:
|
||||
# Default to no-op for providers without caching support
|
||||
|
||||
@@ -1,39 +1,30 @@
|
||||
from onyx.configs.app_configs import MAX_SLACK_QUERY_EXPANSIONS
|
||||
|
||||
SLACK_QUERY_EXPANSION_PROMPT = f"""
|
||||
Rewrite the user's query into at most {MAX_SLACK_QUERY_EXPANSIONS} keyword-only queries for Slack's keyword search.
|
||||
Rewrite the user's query and, if helpful, split it into at most {MAX_SLACK_QUERY_EXPANSIONS} \
|
||||
keyword-only queries, so that Slack's keyword search yields the best matches.
|
||||
|
||||
Slack search behavior:
|
||||
- Pure keyword AND search (no semantics)
|
||||
- More words = fewer matches, so keep queries concise (1-3 words)
|
||||
Keep in mind the Slack's search behavior:
|
||||
- Pure keyword AND search (no semantics).
|
||||
- Word order matters.
|
||||
- More words = fewer matches, so keep each query concise.
|
||||
- IMPORTANT: Prefer simple 1-2 word queries over longer multi-word queries.
|
||||
|
||||
ALWAYS include:
|
||||
- Person names (e.g., "Sarah Chen", "Mike Johnson") - people search for messages from/about specific people
|
||||
- Project/product names, technical terms, proper nouns
|
||||
- Actual content words: "performance", "bug", "deployment", "API", "error"
|
||||
Critical: Extract ONLY keywords that would actually appear in Slack message content.
|
||||
|
||||
DO NOT include:
|
||||
- Meta-words: "topics", "conversations", "discussed", "summary", "messages"
|
||||
- Temporal: "today", "yesterday", "week", "month", "recent", "last"
|
||||
- Channel names: "general", "eng-general", "random"
|
||||
- Meta-words: "topics", "conversations", "discussed", "summary", "messages", "big", "main", "talking"
|
||||
- Temporal: "today", "yesterday", "week", "month", "recent", "past", "last"
|
||||
- Channels/Users: "general", "eng-general", "engineering", "@username"
|
||||
|
||||
DO include:
|
||||
- Actual content: "performance", "bug", "deployment", "API", "database", "error", "feature"
|
||||
|
||||
Examples:
|
||||
|
||||
Query: "what are the big topics in eng-general this week?"
|
||||
Output:
|
||||
|
||||
Query: "messages with Sarah about the deployment"
|
||||
Output:
|
||||
Sarah deployment
|
||||
Sarah
|
||||
deployment
|
||||
|
||||
Query: "what did Mike say about the budget?"
|
||||
Output:
|
||||
Mike budget
|
||||
Mike
|
||||
budget
|
||||
|
||||
Query: "performance issues in eng-general"
|
||||
Output:
|
||||
performance issues
|
||||
@@ -50,7 +41,7 @@ Now process this query:
|
||||
|
||||
{{query}}
|
||||
|
||||
Output (keywords only, one per line, NO explanations or commentary):
|
||||
Output:
|
||||
"""
|
||||
|
||||
SLACK_DATE_EXTRACTION_PROMPT = """
|
||||
|
||||
@@ -109,7 +109,6 @@ class TenantRedis(redis.Redis):
|
||||
"unlock",
|
||||
"get",
|
||||
"set",
|
||||
"setex",
|
||||
"delete",
|
||||
"exists",
|
||||
"incrby",
|
||||
|
||||
@@ -697,7 +697,7 @@ def save_user_credentials(
|
||||
# TODO: fix and/or type correctly w/base model
|
||||
config_data = MCPConnectionData(
|
||||
headers=auth_template.config.get("headers", {}),
|
||||
header_substitutions=request.credentials,
|
||||
header_substitutions=auth_template.config.get(HEADER_SUBSTITUTIONS, {}),
|
||||
)
|
||||
for oauth_field_key in MCPOAuthKeys:
|
||||
field_key: Literal["client_info", "tokens", "metadata"] = (
|
||||
|
||||
@@ -34,7 +34,7 @@ from onyx.db.persona import mark_persona_as_not_deleted
|
||||
from onyx.db.persona import update_persona_is_default
|
||||
from onyx.db.persona import update_persona_label
|
||||
from onyx.db.persona import update_persona_public_status
|
||||
from onyx.db.persona import update_persona_shared
|
||||
from onyx.db.persona import update_persona_shared_users
|
||||
from onyx.db.persona import update_persona_visibility
|
||||
from onyx.db.persona import update_personas_display_priority
|
||||
from onyx.file_store.file_store import get_default_file_store
|
||||
@@ -366,9 +366,7 @@ def delete_label(
|
||||
|
||||
|
||||
class PersonaShareRequest(BaseModel):
|
||||
user_ids: list[UUID] | None = None
|
||||
group_ids: list[int] | None = None
|
||||
is_public: bool | None = None
|
||||
user_ids: list[UUID]
|
||||
|
||||
|
||||
# We notify each user when a user is shared with them
|
||||
@@ -379,13 +377,11 @@ def share_persona(
|
||||
user: User = Depends(current_user),
|
||||
db_session: Session = Depends(get_session),
|
||||
) -> None:
|
||||
update_persona_shared(
|
||||
update_persona_shared_users(
|
||||
persona_id=persona_id,
|
||||
user_ids=persona_share_request.user_ids,
|
||||
user=user,
|
||||
db_session=db_session,
|
||||
user_ids=persona_share_request.user_ids,
|
||||
group_ids=persona_share_request.group_ids,
|
||||
is_public=persona_share_request.is_public,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -410,20 +410,26 @@ def list_llm_provider_basics(
|
||||
|
||||
all_providers = fetch_existing_llm_providers(db_session)
|
||||
user_group_ids = fetch_user_group_ids(db_session, user) if user else set()
|
||||
is_admin = user is not None and user.role == UserRole.ADMIN
|
||||
is_admin = user and user.role == UserRole.ADMIN
|
||||
|
||||
accessible_providers = []
|
||||
|
||||
for provider in all_providers:
|
||||
# Use centralized access control logic with persona=None since we're
|
||||
# listing providers without a specific persona context. This correctly:
|
||||
# - Includes all public providers
|
||||
# - Includes providers user can access via group membership
|
||||
# - Excludes persona-only restricted providers (requires specific persona)
|
||||
# - Excludes non-public providers with no restrictions (admin-only)
|
||||
if can_user_access_llm_provider(
|
||||
provider, user_group_ids, persona=None, is_admin=is_admin
|
||||
):
|
||||
# Include all public providers
|
||||
if provider.is_public:
|
||||
accessible_providers.append(LLMProviderDescriptor.from_model(provider))
|
||||
continue
|
||||
|
||||
# Include restricted providers user has access to via groups
|
||||
if is_admin:
|
||||
# Admins see all providers
|
||||
accessible_providers.append(LLMProviderDescriptor.from_model(provider))
|
||||
elif provider.groups:
|
||||
# User must be in at least one of the provider's groups
|
||||
if user_group_ids.intersection({g.id for g in provider.groups}):
|
||||
accessible_providers.append(LLMProviderDescriptor.from_model(provider))
|
||||
elif not provider.personas:
|
||||
# No restrictions = accessible
|
||||
accessible_providers.append(LLMProviderDescriptor.from_model(provider))
|
||||
|
||||
end_time = datetime.now(timezone.utc)
|
||||
|
||||
@@ -4,13 +4,10 @@ from fastapi import APIRouter
|
||||
from fastapi import Depends
|
||||
from fastapi import HTTPException
|
||||
from fastapi import Response
|
||||
from sqlalchemy.dialects.postgresql import insert
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from onyx.auth.users import current_admin_user
|
||||
from onyx.db.engine.sql_engine import get_session
|
||||
from onyx.db.models import InternetContentProvider
|
||||
from onyx.db.models import InternetSearchProvider
|
||||
from onyx.db.models import User
|
||||
from onyx.db.web_search import deactivate_web_content_provider
|
||||
from onyx.db.web_search import deactivate_web_search_provider
|
||||
@@ -97,28 +94,6 @@ def upsert_search_provider_endpoint(
|
||||
db_session=db_session,
|
||||
)
|
||||
|
||||
# Sync Exa key of search engine to content provider
|
||||
if (
|
||||
request.provider_type == WebSearchProviderType.EXA
|
||||
and request.api_key_changed
|
||||
and request.api_key
|
||||
):
|
||||
stmt = (
|
||||
insert(InternetContentProvider)
|
||||
.values(
|
||||
name="Exa",
|
||||
provider_type=WebContentProviderType.EXA.value,
|
||||
api_key=request.api_key,
|
||||
is_active=False,
|
||||
)
|
||||
.on_conflict_do_update(
|
||||
index_elements=["name"],
|
||||
set_={"api_key": request.api_key},
|
||||
)
|
||||
)
|
||||
db_session.execute(stmt)
|
||||
db_session.flush()
|
||||
|
||||
db_session.commit()
|
||||
return WebSearchProviderView(
|
||||
id=provider.id,
|
||||
@@ -270,28 +245,6 @@ def upsert_content_provider_endpoint(
|
||||
db_session=db_session,
|
||||
)
|
||||
|
||||
# Sync Exa key of content provider to search provider
|
||||
if (
|
||||
request.provider_type == WebContentProviderType.EXA
|
||||
and request.api_key_changed
|
||||
and request.api_key
|
||||
):
|
||||
stmt = (
|
||||
insert(InternetSearchProvider)
|
||||
.values(
|
||||
name="Exa",
|
||||
provider_type=WebSearchProviderType.EXA.value,
|
||||
api_key=request.api_key,
|
||||
is_active=False,
|
||||
)
|
||||
.on_conflict_do_update(
|
||||
index_elements=["name"],
|
||||
set_={"api_key": request.api_key},
|
||||
)
|
||||
)
|
||||
db_session.execute(stmt)
|
||||
db_session.flush()
|
||||
|
||||
db_session.commit()
|
||||
return WebContentProviderView(
|
||||
id=provider.id,
|
||||
|
||||
@@ -146,7 +146,6 @@ class ExaClient(WebSearchProvider, WebContentProvider):
|
||||
if result.published_date
|
||||
else None
|
||||
),
|
||||
scrape_successful=bool(full_content),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@@ -98,9 +98,6 @@ def build_content_provider_from_config(
|
||||
timeout_seconds=config.timeout_seconds,
|
||||
)
|
||||
|
||||
if provider_type == WebContentProviderType.EXA:
|
||||
return ExaClient(api_key=api_key)
|
||||
|
||||
|
||||
def get_default_provider() -> WebSearchProvider | None:
|
||||
with get_session_with_current_tenant() as db_session:
|
||||
|
||||
@@ -295,7 +295,7 @@ numpy==2.4.1
|
||||
# pandas-stubs
|
||||
# shapely
|
||||
# voyageai
|
||||
onyx-devtools==0.6.2
|
||||
onyx-devtools==0.2.0
|
||||
# via onyx
|
||||
openai==2.14.0
|
||||
# via
|
||||
|
||||
@@ -31,4 +31,3 @@ class WebSearchProviderType(str, Enum):
|
||||
class WebContentProviderType(str, Enum):
|
||||
ONYX_WEB_CRAWLER = "onyx_web_crawler"
|
||||
FIRECRAWL = "firecrawl"
|
||||
EXA = "exa"
|
||||
|
||||
@@ -1,281 +0,0 @@
|
||||
"""
|
||||
External dependency unit tests for user file processing queue protections.
|
||||
|
||||
Verifies that the three mechanisms added to check_user_file_processing work
|
||||
correctly:
|
||||
|
||||
1. Queue depth backpressure – when the broker queue exceeds
|
||||
USER_FILE_PROCESSING_MAX_QUEUE_DEPTH, no new tasks are enqueued.
|
||||
|
||||
2. Per-file Redis guard key – if the guard key for a file already exists in
|
||||
Redis, that file is skipped even though it is still in PROCESSING status.
|
||||
|
||||
3. Task expiry – every send_task call carries expires=
|
||||
CELERY_USER_FILE_PROCESSING_TASK_EXPIRES so that stale queued tasks are
|
||||
discarded by workers automatically.
|
||||
|
||||
Also verifies that process_single_user_file clears the guard key the moment
|
||||
it is picked up by a worker.
|
||||
|
||||
Uses real Redis (DB 0 via get_redis_client) and real PostgreSQL for UserFile
|
||||
rows. The Celery app is provided as a MagicMock injected via a PropertyMock
|
||||
on the task class so no real broker is needed.
|
||||
"""
|
||||
|
||||
from collections.abc import Generator
|
||||
from contextlib import contextmanager
|
||||
from typing import Any
|
||||
from unittest.mock import MagicMock
|
||||
from unittest.mock import patch
|
||||
from unittest.mock import PropertyMock
|
||||
from uuid import uuid4
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from onyx.background.celery.tasks.user_file_processing.tasks import (
|
||||
_user_file_lock_key,
|
||||
)
|
||||
from onyx.background.celery.tasks.user_file_processing.tasks import (
|
||||
_user_file_queued_key,
|
||||
)
|
||||
from onyx.background.celery.tasks.user_file_processing.tasks import (
|
||||
check_user_file_processing,
|
||||
)
|
||||
from onyx.background.celery.tasks.user_file_processing.tasks import (
|
||||
process_single_user_file,
|
||||
)
|
||||
from onyx.configs.constants import CELERY_USER_FILE_PROCESSING_TASK_EXPIRES
|
||||
from onyx.configs.constants import OnyxCeleryQueues
|
||||
from onyx.configs.constants import OnyxCeleryTask
|
||||
from onyx.configs.constants import USER_FILE_PROCESSING_MAX_QUEUE_DEPTH
|
||||
from onyx.db.enums import UserFileStatus
|
||||
from onyx.db.models import UserFile
|
||||
from onyx.redis.redis_pool import get_redis_client
|
||||
from tests.external_dependency_unit.conftest import create_test_user
|
||||
from tests.external_dependency_unit.constants import TEST_TENANT_ID
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_PATCH_QUEUE_LEN = (
|
||||
"onyx.background.celery.tasks.user_file_processing.tasks.celery_get_queue_length"
|
||||
)
|
||||
|
||||
|
||||
def _create_processing_user_file(db_session: Session, user_id: object) -> UserFile:
|
||||
"""Insert a UserFile in PROCESSING status and return it."""
|
||||
uf = UserFile(
|
||||
id=uuid4(),
|
||||
user_id=user_id,
|
||||
file_id=f"test_file_{uuid4().hex[:8]}",
|
||||
name=f"test_{uuid4().hex[:8]}.txt",
|
||||
file_type="text/plain",
|
||||
status=UserFileStatus.PROCESSING,
|
||||
)
|
||||
db_session.add(uf)
|
||||
db_session.commit()
|
||||
db_session.refresh(uf)
|
||||
return uf
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _patch_task_app(task: Any, mock_app: MagicMock) -> Generator[None, None, None]:
|
||||
"""Patch the ``app`` property on *task*'s class so that ``self.app``
|
||||
inside the task function returns *mock_app*.
|
||||
|
||||
With ``bind=True``, ``task.run`` is a bound method whose ``__self__`` is
|
||||
the actual task instance. We patch ``app`` on that instance's class
|
||||
(a unique Celery-generated Task subclass) so the mock is scoped to this
|
||||
task only.
|
||||
"""
|
||||
task_instance = task.run.__self__
|
||||
with patch.object(
|
||||
type(task_instance), "app", new_callable=PropertyMock, return_value=mock_app
|
||||
):
|
||||
yield
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Test classes
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestQueueDepthBackpressure:
|
||||
"""Protection 1: skip all enqueuing when the broker queue is too deep."""
|
||||
|
||||
def test_no_tasks_enqueued_when_queue_over_limit(
|
||||
self,
|
||||
db_session: Session,
|
||||
tenant_context: None, # noqa: ARG002
|
||||
) -> None:
|
||||
"""When the queue depth exceeds the limit the beat cycle is skipped."""
|
||||
user = create_test_user(db_session, "bp_user")
|
||||
_create_processing_user_file(db_session, user.id)
|
||||
|
||||
mock_app = MagicMock()
|
||||
|
||||
with (
|
||||
_patch_task_app(check_user_file_processing, mock_app),
|
||||
patch(
|
||||
_PATCH_QUEUE_LEN, return_value=USER_FILE_PROCESSING_MAX_QUEUE_DEPTH + 1
|
||||
),
|
||||
):
|
||||
check_user_file_processing.run(tenant_id=TEST_TENANT_ID)
|
||||
|
||||
mock_app.send_task.assert_not_called()
|
||||
|
||||
|
||||
class TestPerFileGuardKey:
|
||||
"""Protection 2: per-file Redis guard key prevents duplicate enqueue."""
|
||||
|
||||
def test_guarded_file_not_re_enqueued(
|
||||
self,
|
||||
db_session: Session,
|
||||
tenant_context: None, # noqa: ARG002
|
||||
) -> None:
|
||||
"""A file whose guard key is already set in Redis is skipped."""
|
||||
user = create_test_user(db_session, "guard_user")
|
||||
uf = _create_processing_user_file(db_session, user.id)
|
||||
|
||||
redis_client = get_redis_client(tenant_id=TEST_TENANT_ID)
|
||||
guard_key = _user_file_queued_key(uf.id)
|
||||
redis_client.setex(guard_key, CELERY_USER_FILE_PROCESSING_TASK_EXPIRES, 1)
|
||||
|
||||
mock_app = MagicMock()
|
||||
|
||||
try:
|
||||
with (
|
||||
_patch_task_app(check_user_file_processing, mock_app),
|
||||
patch(_PATCH_QUEUE_LEN, return_value=0),
|
||||
):
|
||||
check_user_file_processing.run(tenant_id=TEST_TENANT_ID)
|
||||
|
||||
# send_task must not have been called with this specific file's ID
|
||||
for call in mock_app.send_task.call_args_list:
|
||||
kwargs = call.kwargs.get("kwargs", {})
|
||||
assert kwargs.get("user_file_id") != str(
|
||||
uf.id
|
||||
), f"File {uf.id} should have been skipped because its guard key exists"
|
||||
finally:
|
||||
redis_client.delete(guard_key)
|
||||
|
||||
def test_guard_key_exists_in_redis_after_enqueue(
|
||||
self,
|
||||
db_session: Session,
|
||||
tenant_context: None, # noqa: ARG002
|
||||
) -> None:
|
||||
"""After a file is enqueued its guard key is present in Redis with a TTL."""
|
||||
user = create_test_user(db_session, "guard_set_user")
|
||||
uf = _create_processing_user_file(db_session, user.id)
|
||||
|
||||
redis_client = get_redis_client(tenant_id=TEST_TENANT_ID)
|
||||
guard_key = _user_file_queued_key(uf.id)
|
||||
redis_client.delete(guard_key) # clean slate
|
||||
|
||||
mock_app = MagicMock()
|
||||
|
||||
try:
|
||||
with (
|
||||
_patch_task_app(check_user_file_processing, mock_app),
|
||||
patch(_PATCH_QUEUE_LEN, return_value=0),
|
||||
):
|
||||
check_user_file_processing.run(tenant_id=TEST_TENANT_ID)
|
||||
|
||||
assert redis_client.exists(
|
||||
guard_key
|
||||
), "Guard key should be set in Redis after enqueue"
|
||||
ttl = int(redis_client.ttl(guard_key)) # type: ignore[arg-type]
|
||||
assert 0 < ttl <= CELERY_USER_FILE_PROCESSING_TASK_EXPIRES, (
|
||||
f"Guard key TTL {ttl}s is outside the expected range "
|
||||
f"(0, {CELERY_USER_FILE_PROCESSING_TASK_EXPIRES}]"
|
||||
)
|
||||
finally:
|
||||
redis_client.delete(guard_key)
|
||||
|
||||
|
||||
class TestTaskExpiry:
|
||||
"""Protection 3: every send_task call includes an expires value."""
|
||||
|
||||
def test_send_task_called_with_expires(
|
||||
self,
|
||||
db_session: Session,
|
||||
tenant_context: None, # noqa: ARG002
|
||||
) -> None:
|
||||
"""send_task is called with the correct queue, task name, and expires."""
|
||||
user = create_test_user(db_session, "expires_user")
|
||||
uf = _create_processing_user_file(db_session, user.id)
|
||||
|
||||
redis_client = get_redis_client(tenant_id=TEST_TENANT_ID)
|
||||
guard_key = _user_file_queued_key(uf.id)
|
||||
redis_client.delete(guard_key)
|
||||
|
||||
mock_app = MagicMock()
|
||||
|
||||
try:
|
||||
with (
|
||||
_patch_task_app(check_user_file_processing, mock_app),
|
||||
patch(_PATCH_QUEUE_LEN, return_value=0),
|
||||
):
|
||||
check_user_file_processing.run(tenant_id=TEST_TENANT_ID)
|
||||
|
||||
# At least one task should have been submitted (for our file)
|
||||
assert (
|
||||
mock_app.send_task.call_count >= 1
|
||||
), "Expected at least one task to be submitted"
|
||||
|
||||
# Every submitted task must carry expires
|
||||
for call in mock_app.send_task.call_args_list:
|
||||
assert call.args[0] == OnyxCeleryTask.PROCESS_SINGLE_USER_FILE
|
||||
assert call.kwargs.get("queue") == OnyxCeleryQueues.USER_FILE_PROCESSING
|
||||
assert (
|
||||
call.kwargs.get("expires")
|
||||
== CELERY_USER_FILE_PROCESSING_TASK_EXPIRES
|
||||
), (
|
||||
"Task must be submitted with the correct expires value to prevent "
|
||||
"stale task accumulation"
|
||||
)
|
||||
finally:
|
||||
redis_client.delete(guard_key)
|
||||
|
||||
|
||||
class TestWorkerClearsGuardKey:
|
||||
"""process_single_user_file removes the guard key when it picks up a task."""
|
||||
|
||||
def test_guard_key_deleted_on_pickup(
|
||||
self,
|
||||
tenant_context: None, # noqa: ARG002
|
||||
) -> None:
|
||||
"""The guard key is deleted before the worker does any real work.
|
||||
|
||||
We simulate an already-locked file so process_single_user_file returns
|
||||
early – but crucially, after the guard key deletion.
|
||||
"""
|
||||
user_file_id = str(uuid4())
|
||||
|
||||
redis_client = get_redis_client(tenant_id=TEST_TENANT_ID)
|
||||
guard_key = _user_file_queued_key(user_file_id)
|
||||
|
||||
# Simulate the guard key set when the beat enqueued the task
|
||||
redis_client.setex(guard_key, CELERY_USER_FILE_PROCESSING_TASK_EXPIRES, 1)
|
||||
assert redis_client.exists(guard_key), "Guard key must exist before pickup"
|
||||
|
||||
# Hold the per-file processing lock so the worker exits early without
|
||||
# touching the database or file store.
|
||||
lock_key = _user_file_lock_key(user_file_id)
|
||||
processing_lock = redis_client.lock(lock_key, timeout=10)
|
||||
acquired = processing_lock.acquire(blocking=False)
|
||||
assert acquired, "Should be able to acquire the processing lock for this test"
|
||||
|
||||
try:
|
||||
process_single_user_file.run(
|
||||
user_file_id=user_file_id,
|
||||
tenant_id=TEST_TENANT_ID,
|
||||
)
|
||||
finally:
|
||||
if processing_lock.owned():
|
||||
processing_lock.release()
|
||||
|
||||
assert not redis_client.exists(
|
||||
guard_key
|
||||
), "Guard key should be deleted when the worker picks up the task"
|
||||
@@ -4,11 +4,9 @@ These tests assume OpenSearch is running and test all implemented methods
|
||||
using real schemas, pipelines, and search queries from the codebase.
|
||||
"""
|
||||
|
||||
import json
|
||||
import uuid
|
||||
from collections.abc import Generator
|
||||
from datetime import datetime
|
||||
from datetime import timezone
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
@@ -23,31 +21,18 @@ from onyx.document_index.opensearch.search import (
|
||||
MIN_MAX_NORMALIZATION_PIPELINE_CONFIG,
|
||||
)
|
||||
from onyx.document_index.opensearch.search import MIN_MAX_NORMALIZATION_PIPELINE_NAME
|
||||
from shared_configs.configs import POSTGRES_DEFAULT_SCHEMA
|
||||
|
||||
|
||||
def _patch_global_tenant_state(monkeypatch: pytest.MonkeyPatch, state: bool) -> None:
|
||||
"""Patches MULTI_TENANT wherever necessary for this test file.
|
||||
|
||||
Args:
|
||||
monkeypatch: The test instance's monkeypatch instance, used for
|
||||
patching.
|
||||
state: The intended state of MULTI_TENANT.
|
||||
"""
|
||||
monkeypatch.setattr("shared_configs.configs.MULTI_TENANT", state)
|
||||
monkeypatch.setattr("onyx.document_index.opensearch.schema.MULTI_TENANT", state)
|
||||
|
||||
|
||||
def _create_test_document_chunk(
|
||||
document_id: str,
|
||||
chunk_index: int,
|
||||
content: str,
|
||||
tenant_state: TenantState,
|
||||
document_id: str = "test-doc-1",
|
||||
chunk_index: int = 0,
|
||||
content: str = "Test content",
|
||||
content_vector: list[float] | None = None,
|
||||
title: str | None = None,
|
||||
title_vector: list[float] | None = None,
|
||||
public: bool = True,
|
||||
hidden: bool = False,
|
||||
**kwargs: Any,
|
||||
) -> DocumentChunk:
|
||||
if content_vector is None:
|
||||
# Generate dummy vector - 128 dimensions for fast testing.
|
||||
@@ -57,51 +42,31 @@ def _create_test_document_chunk(
|
||||
if title is not None and title_vector is None:
|
||||
title_vector = [0.2] * 128
|
||||
|
||||
now = datetime.now(timezone.utc)
|
||||
# We only store millisecond precision, so to make sure asserts work in this
|
||||
# test file manually lose some precision from datetime.now().
|
||||
now = now.replace(microsecond=(now.microsecond // 1000) * 1000)
|
||||
|
||||
return DocumentChunk(
|
||||
document_id=document_id,
|
||||
chunk_index=chunk_index,
|
||||
title=title,
|
||||
title_vector=title_vector,
|
||||
content=content,
|
||||
content_vector=content_vector,
|
||||
title=title,
|
||||
title_vector=title_vector,
|
||||
# This is not how tokenization necessarily works, this is just for quick
|
||||
# testing.
|
||||
num_tokens=len(content.split()),
|
||||
source_type="test_source",
|
||||
metadata=json.dumps({}),
|
||||
last_updated=now,
|
||||
public=public,
|
||||
access_control_list=[],
|
||||
hidden=hidden,
|
||||
global_boost=0,
|
||||
semantic_identifier="Test semantic identifier",
|
||||
image_file_id=None,
|
||||
source_links=None,
|
||||
blurb="Test blurb",
|
||||
doc_summary="Test doc summary",
|
||||
chunk_context="Test chunk context",
|
||||
document_sets=None,
|
||||
project_ids=None,
|
||||
primary_owners=None,
|
||||
secondary_owners=None,
|
||||
tenant_id=tenant_state,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
|
||||
def _generate_test_vector(base_value: float = 0.1, dimension: int = 128) -> list[float]:
|
||||
"""Generates a test vector with slight variations.
|
||||
|
||||
We round to eliminate floating point precision errors when comparing chunks
|
||||
for equality.
|
||||
"""
|
||||
return [round(base_value + (i * 0.001), 5) for i in range(dimension)]
|
||||
"""Generate a test vector with slight variations."""
|
||||
return [base_value + (i * 0.001) for i in range(dimension)]
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def opensearch_available() -> None:
|
||||
"""Verifies OpenSearch is running, skips all tests if not."""
|
||||
"""Verify OpenSearch is running, skip all tests if not."""
|
||||
client = OpenSearchClient(index_name="test_ping")
|
||||
try:
|
||||
if not client.ping():
|
||||
@@ -263,15 +228,11 @@ class TestOpenSearchClient:
|
||||
pipeline_id=MIN_MAX_NORMALIZATION_PIPELINE_NAME
|
||||
)
|
||||
|
||||
def test_index_document(
|
||||
self, test_client: OpenSearchClient, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
def test_index_document(self, test_client: OpenSearchClient) -> None:
|
||||
"""Tests indexing a document."""
|
||||
# Precondition.
|
||||
_patch_global_tenant_state(monkeypatch, False)
|
||||
tenant_state = TenantState(tenant_id=POSTGRES_DEFAULT_SCHEMA, multitenant=False)
|
||||
mappings = DocumentSchema.get_document_schema(
|
||||
vector_dimension=128, multitenant=tenant_state.multitenant
|
||||
vector_dimension=128, multitenant=True
|
||||
)
|
||||
settings = DocumentSchema.get_index_settings()
|
||||
test_client.create_index(mappings=mappings, settings=settings)
|
||||
@@ -280,22 +241,17 @@ class TestOpenSearchClient:
|
||||
document_id="test-doc-1",
|
||||
chunk_index=0,
|
||||
content="Test content for indexing",
|
||||
tenant_state=tenant_state,
|
||||
)
|
||||
|
||||
# Under test and postcondition.
|
||||
# Should not raise.
|
||||
test_client.index_document(document=doc)
|
||||
|
||||
def test_index_duplicate_document(
|
||||
self, test_client: OpenSearchClient, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
def test_index_duplicate_document(self, test_client: OpenSearchClient) -> None:
|
||||
"""Tests indexing a duplicate document raises an error."""
|
||||
# Precondition.
|
||||
_patch_global_tenant_state(monkeypatch, False)
|
||||
tenant_state = TenantState(tenant_id=POSTGRES_DEFAULT_SCHEMA, multitenant=False)
|
||||
mappings = DocumentSchema.get_document_schema(
|
||||
vector_dimension=128, multitenant=tenant_state.multitenant
|
||||
vector_dimension=128, multitenant=True
|
||||
)
|
||||
settings = DocumentSchema.get_index_settings()
|
||||
test_client.create_index(mappings=mappings, settings=settings)
|
||||
@@ -304,7 +260,6 @@ class TestOpenSearchClient:
|
||||
document_id="test-doc-duplicate",
|
||||
chunk_index=0,
|
||||
content="Duplicate test",
|
||||
tenant_state=tenant_state,
|
||||
)
|
||||
|
||||
# Index once - should succeed.
|
||||
@@ -315,15 +270,11 @@ class TestOpenSearchClient:
|
||||
with pytest.raises(Exception, match="already exists"):
|
||||
test_client.index_document(document=doc)
|
||||
|
||||
def test_get_document(
|
||||
self, test_client: OpenSearchClient, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
def test_get_document(self, test_client: OpenSearchClient) -> None:
|
||||
"""Tests getting a document."""
|
||||
# Precondition.
|
||||
_patch_global_tenant_state(monkeypatch, False)
|
||||
tenant_state = TenantState(tenant_id=POSTGRES_DEFAULT_SCHEMA, multitenant=False)
|
||||
mappings = DocumentSchema.get_document_schema(
|
||||
vector_dimension=128, multitenant=tenant_state.multitenant
|
||||
vector_dimension=128, multitenant=True
|
||||
)
|
||||
settings = DocumentSchema.get_index_settings()
|
||||
test_client.create_index(mappings=mappings, settings=settings)
|
||||
@@ -332,7 +283,6 @@ class TestOpenSearchClient:
|
||||
document_id="test-doc-get",
|
||||
chunk_index=0,
|
||||
content="Content to retrieve",
|
||||
tenant_state=tenant_state,
|
||||
)
|
||||
test_client.index_document(document=original_doc)
|
||||
|
||||
@@ -347,14 +297,11 @@ class TestOpenSearchClient:
|
||||
# Postcondition.
|
||||
assert retrieved_doc == original_doc
|
||||
|
||||
def test_get_nonexistent_document(
|
||||
self, test_client: OpenSearchClient, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
def test_get_nonexistent_document(self, test_client: OpenSearchClient) -> None:
|
||||
"""Tests getting a nonexistent document raises an error."""
|
||||
# Precondition.
|
||||
_patch_global_tenant_state(monkeypatch, False)
|
||||
mappings = DocumentSchema.get_document_schema(
|
||||
vector_dimension=128, multitenant=False
|
||||
vector_dimension=128, multitenant=True
|
||||
)
|
||||
settings = DocumentSchema.get_index_settings()
|
||||
test_client.create_index(mappings=mappings, settings=settings)
|
||||
@@ -365,15 +312,11 @@ class TestOpenSearchClient:
|
||||
document_chunk_id="test_source__nonexistent__512__0"
|
||||
)
|
||||
|
||||
def test_delete_existing_document(
|
||||
self, test_client: OpenSearchClient, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
def test_delete_existing_document(self, test_client: OpenSearchClient) -> None:
|
||||
"""Tests deleting an existing document returns True."""
|
||||
# Precondition.
|
||||
_patch_global_tenant_state(monkeypatch, False)
|
||||
tenant_state = TenantState(tenant_id=POSTGRES_DEFAULT_SCHEMA, multitenant=False)
|
||||
mappings = DocumentSchema.get_document_schema(
|
||||
vector_dimension=128, multitenant=tenant_state.multitenant
|
||||
vector_dimension=128, multitenant=True
|
||||
)
|
||||
settings = DocumentSchema.get_index_settings()
|
||||
test_client.create_index(mappings=mappings, settings=settings)
|
||||
@@ -382,7 +325,6 @@ class TestOpenSearchClient:
|
||||
document_id="test-doc-delete",
|
||||
chunk_index=0,
|
||||
content="Content to delete",
|
||||
tenant_state=tenant_state,
|
||||
)
|
||||
test_client.index_document(document=doc)
|
||||
|
||||
@@ -400,15 +342,11 @@ class TestOpenSearchClient:
|
||||
with pytest.raises(Exception, match="404"):
|
||||
test_client.get_document(document_chunk_id=doc_chunk_id)
|
||||
|
||||
def test_delete_nonexistent_document(
|
||||
self, test_client: OpenSearchClient, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
def test_delete_nonexistent_document(self, test_client: OpenSearchClient) -> None:
|
||||
"""Tests deleting a nonexistent document returns False."""
|
||||
# Precondition.
|
||||
_patch_global_tenant_state(monkeypatch, False)
|
||||
tenant_state = TenantState(tenant_id=POSTGRES_DEFAULT_SCHEMA, multitenant=False)
|
||||
mappings = DocumentSchema.get_document_schema(
|
||||
vector_dimension=128, multitenant=tenant_state.multitenant
|
||||
vector_dimension=128, multitenant=True
|
||||
)
|
||||
settings = DocumentSchema.get_index_settings()
|
||||
test_client.create_index(mappings=mappings, settings=settings)
|
||||
@@ -421,15 +359,11 @@ class TestOpenSearchClient:
|
||||
# Postcondition.
|
||||
assert result is False
|
||||
|
||||
def test_delete_by_query(
|
||||
self, test_client: OpenSearchClient, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
def test_delete_by_query(self, test_client: OpenSearchClient) -> None:
|
||||
"""Tests deleting documents by query."""
|
||||
# Precondition.
|
||||
_patch_global_tenant_state(monkeypatch, False)
|
||||
tenant_state = TenantState(tenant_id=POSTGRES_DEFAULT_SCHEMA, multitenant=False)
|
||||
mappings = DocumentSchema.get_document_schema(
|
||||
vector_dimension=128, multitenant=tenant_state.multitenant
|
||||
vector_dimension=128, multitenant=True
|
||||
)
|
||||
settings = DocumentSchema.get_index_settings()
|
||||
test_client.create_index(mappings=mappings, settings=settings)
|
||||
@@ -440,7 +374,7 @@ class TestOpenSearchClient:
|
||||
document_id="delete-me",
|
||||
chunk_index=i,
|
||||
content=f"Delete this {i}",
|
||||
tenant_state=tenant_state,
|
||||
tenant_id="tenant-x",
|
||||
)
|
||||
for i in range(3)
|
||||
]
|
||||
@@ -449,7 +383,7 @@ class TestOpenSearchClient:
|
||||
document_id="keep-me",
|
||||
chunk_index=0,
|
||||
content="Keep this",
|
||||
tenant_state=tenant_state,
|
||||
tenant_id="tenant-x",
|
||||
)
|
||||
]
|
||||
|
||||
@@ -459,7 +393,7 @@ class TestOpenSearchClient:
|
||||
|
||||
query_body = DocumentQuery.delete_from_document_id_query(
|
||||
document_id="delete-me",
|
||||
tenant_state=tenant_state,
|
||||
tenant_state=TenantState(tenant_id="tenant-x", multitenant=True),
|
||||
)
|
||||
|
||||
# Under test.
|
||||
@@ -472,7 +406,7 @@ class TestOpenSearchClient:
|
||||
test_client.refresh_index()
|
||||
search_query = DocumentQuery.get_from_document_id_query(
|
||||
document_id="delete-me",
|
||||
tenant_state=tenant_state,
|
||||
tenant_state=TenantState(tenant_id="tenant-x", multitenant=True),
|
||||
max_chunk_size=DEFAULT_MAX_CHUNK_SIZE,
|
||||
min_chunk_index=None,
|
||||
max_chunk_index=None,
|
||||
@@ -484,7 +418,7 @@ class TestOpenSearchClient:
|
||||
# Verify other documents still exist.
|
||||
keep_query = DocumentQuery.get_from_document_id_query(
|
||||
document_id="keep-me",
|
||||
tenant_state=tenant_state,
|
||||
tenant_state=TenantState(tenant_id="tenant-x", multitenant=True),
|
||||
max_chunk_size=DEFAULT_MAX_CHUNK_SIZE,
|
||||
min_chunk_index=None,
|
||||
max_chunk_index=None,
|
||||
@@ -498,44 +432,37 @@ class TestOpenSearchClient:
|
||||
with pytest.raises(NotImplementedError):
|
||||
test_client.update_document()
|
||||
|
||||
def test_search_basic(
|
||||
self, test_client: OpenSearchClient, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
def test_search_basic(self, test_client: OpenSearchClient) -> None:
|
||||
"""Tests basic search functionality."""
|
||||
# Precondition.
|
||||
_patch_global_tenant_state(monkeypatch, False)
|
||||
tenant_state = TenantState(tenant_id=POSTGRES_DEFAULT_SCHEMA, multitenant=False)
|
||||
mappings = DocumentSchema.get_document_schema(
|
||||
vector_dimension=128, multitenant=tenant_state.multitenant
|
||||
vector_dimension=128, multitenant=False
|
||||
)
|
||||
settings = DocumentSchema.get_index_settings()
|
||||
test_client.create_index(mappings=mappings, settings=settings)
|
||||
|
||||
# Index multiple documents with different content and vectors.
|
||||
docs = {
|
||||
"search-doc-1": _create_test_document_chunk(
|
||||
docs = [
|
||||
_create_test_document_chunk(
|
||||
document_id="search-doc-1",
|
||||
chunk_index=0,
|
||||
content="Python programming language tutorial",
|
||||
content_vector=_generate_test_vector(0.1),
|
||||
tenant_state=tenant_state,
|
||||
),
|
||||
"search-doc-2": _create_test_document_chunk(
|
||||
_create_test_document_chunk(
|
||||
document_id="search-doc-2",
|
||||
chunk_index=0,
|
||||
content="How to make cheese",
|
||||
content_vector=_generate_test_vector(0.2),
|
||||
tenant_state=tenant_state,
|
||||
),
|
||||
"search-doc-3": _create_test_document_chunk(
|
||||
_create_test_document_chunk(
|
||||
document_id="search-doc-3",
|
||||
chunk_index=0,
|
||||
content="C++ for newborns",
|
||||
content_vector=_generate_test_vector(0.15),
|
||||
tenant_state=tenant_state,
|
||||
),
|
||||
}
|
||||
for doc in docs.values():
|
||||
]
|
||||
for doc in docs:
|
||||
test_client.index_document(document=doc)
|
||||
|
||||
# Refresh index to make documents searchable.
|
||||
@@ -549,57 +476,47 @@ class TestOpenSearchClient:
|
||||
query_vector=query_vector,
|
||||
num_candidates=10,
|
||||
num_hits=5,
|
||||
tenant_state=tenant_state,
|
||||
tenant_state=TenantState(tenant_id="", multitenant=False),
|
||||
)
|
||||
|
||||
# Under test.
|
||||
results = test_client.search(body=search_body, search_pipeline_id=None)
|
||||
|
||||
# Postcondition.
|
||||
assert len(results) == 3
|
||||
assert len(results) > 0
|
||||
# Assert that all the chunks above are present.
|
||||
assert all(
|
||||
chunk.document_id in ["search-doc-1", "search-doc-2", "search-doc-3"]
|
||||
for chunk in results
|
||||
)
|
||||
# Make sure the chunk contents are preserved.
|
||||
for chunk in results:
|
||||
assert chunk == docs[chunk.document_id]
|
||||
|
||||
def test_search_with_pipeline(
|
||||
self,
|
||||
test_client: OpenSearchClient,
|
||||
search_pipeline: None,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
self, test_client: OpenSearchClient, search_pipeline: None
|
||||
) -> None:
|
||||
"""Tests search with a normalization pipeline."""
|
||||
# Precondition.
|
||||
_patch_global_tenant_state(monkeypatch, False)
|
||||
tenant_state = TenantState(tenant_id=POSTGRES_DEFAULT_SCHEMA, multitenant=False)
|
||||
mappings = DocumentSchema.get_document_schema(
|
||||
vector_dimension=128, multitenant=tenant_state.multitenant
|
||||
vector_dimension=128, multitenant=False
|
||||
)
|
||||
settings = DocumentSchema.get_index_settings()
|
||||
test_client.create_index(mappings=mappings, settings=settings)
|
||||
|
||||
# Index documents.
|
||||
docs = {
|
||||
"pipeline-doc-1": _create_test_document_chunk(
|
||||
docs = [
|
||||
_create_test_document_chunk(
|
||||
document_id="pipeline-doc-1",
|
||||
chunk_index=0,
|
||||
content="Machine learning algorithms for single-celled organisms",
|
||||
content_vector=_generate_test_vector(0.3),
|
||||
tenant_state=tenant_state,
|
||||
),
|
||||
"pipeline-doc-2": _create_test_document_chunk(
|
||||
_create_test_document_chunk(
|
||||
document_id="pipeline-doc-2",
|
||||
chunk_index=0,
|
||||
content="Deep learning shallow neural networks",
|
||||
content_vector=_generate_test_vector(0.35),
|
||||
tenant_state=tenant_state,
|
||||
),
|
||||
}
|
||||
for doc in docs.values():
|
||||
]
|
||||
for doc in docs:
|
||||
test_client.index_document(document=doc)
|
||||
|
||||
# Refresh index to make documents searchable
|
||||
@@ -613,7 +530,7 @@ class TestOpenSearchClient:
|
||||
query_vector=query_vector,
|
||||
num_candidates=10,
|
||||
num_hits=5,
|
||||
tenant_state=tenant_state,
|
||||
tenant_state=TenantState(tenant_id="", multitenant=False),
|
||||
)
|
||||
|
||||
# Under test.
|
||||
@@ -622,25 +539,18 @@ class TestOpenSearchClient:
|
||||
)
|
||||
|
||||
# Postcondition.
|
||||
assert len(results) == 2
|
||||
assert len(results) > 0
|
||||
# Assert that all the chunks above are present.
|
||||
assert all(
|
||||
chunk.document_id in ["pipeline-doc-1", "pipeline-doc-2"]
|
||||
for chunk in results
|
||||
)
|
||||
# Make sure the chunk contents are preserved.
|
||||
for chunk in results:
|
||||
assert chunk == docs[chunk.document_id]
|
||||
|
||||
def test_search_empty_index(
|
||||
self, test_client: OpenSearchClient, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
def test_search_empty_index(self, test_client: OpenSearchClient) -> None:
|
||||
"""Tests search on an empty index returns an empty list."""
|
||||
# Precondition.
|
||||
_patch_global_tenant_state(monkeypatch, False)
|
||||
tenant_state = TenantState(tenant_id=POSTGRES_DEFAULT_SCHEMA, multitenant=False)
|
||||
mappings = DocumentSchema.get_document_schema(
|
||||
vector_dimension=128, multitenant=tenant_state.multitenant
|
||||
vector_dimension=128, multitenant=False
|
||||
)
|
||||
settings = DocumentSchema.get_index_settings()
|
||||
test_client.create_index(mappings=mappings, settings=settings)
|
||||
@@ -654,7 +564,7 @@ class TestOpenSearchClient:
|
||||
query_vector=query_vector,
|
||||
num_candidates=10,
|
||||
num_hits=5,
|
||||
tenant_state=tenant_state,
|
||||
tenant_state=TenantState(tenant_id="", multitenant=False),
|
||||
)
|
||||
|
||||
# Under test.
|
||||
@@ -663,60 +573,43 @@ class TestOpenSearchClient:
|
||||
# Postcondition.
|
||||
assert len(results) == 0
|
||||
|
||||
def test_search_filters(
|
||||
self, test_client: OpenSearchClient, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""
|
||||
Tests search filters for public/hidden documents and tenant isolation.
|
||||
"""
|
||||
def test_search_filters(self, test_client: OpenSearchClient) -> None:
|
||||
"""Tests search filters for public/hidden documents."""
|
||||
# Precondition.
|
||||
_patch_global_tenant_state(monkeypatch, True)
|
||||
tenant_x = TenantState(tenant_id="tenant-x", multitenant=True)
|
||||
tenant_y = TenantState(tenant_id="tenant-y", multitenant=True)
|
||||
mappings = DocumentSchema.get_document_schema(
|
||||
vector_dimension=128, multitenant=tenant_x.multitenant
|
||||
vector_dimension=128, multitenant=True
|
||||
)
|
||||
settings = DocumentSchema.get_index_settings()
|
||||
test_client.create_index(mappings=mappings, settings=settings)
|
||||
|
||||
# Index documents with different public/hidden and tenant states.
|
||||
docs = {
|
||||
"public-doc-1": _create_test_document_chunk(
|
||||
# Index documents with different public/hidden states.
|
||||
docs = [
|
||||
_create_test_document_chunk(
|
||||
document_id="public-doc-1",
|
||||
chunk_index=0,
|
||||
content="Public document content",
|
||||
public=True,
|
||||
hidden=False,
|
||||
tenant_state=tenant_x,
|
||||
tenant_id="tenant-x",
|
||||
),
|
||||
"hidden-doc-1": _create_test_document_chunk(
|
||||
_create_test_document_chunk(
|
||||
document_id="hidden-doc-1",
|
||||
chunk_index=0,
|
||||
content="Hidden document content, spooky",
|
||||
public=True,
|
||||
hidden=True,
|
||||
tenant_state=tenant_x,
|
||||
tenant_id="tenant-x",
|
||||
),
|
||||
"private-doc-1": _create_test_document_chunk(
|
||||
_create_test_document_chunk(
|
||||
document_id="private-doc-1",
|
||||
chunk_index=0,
|
||||
content="Private document content, btw my SSN is 123-45-6789",
|
||||
public=False,
|
||||
hidden=False,
|
||||
tenant_state=tenant_x,
|
||||
tenant_id="tenant-x",
|
||||
),
|
||||
"should-not-exist-from-tenant-x-pov": _create_test_document_chunk(
|
||||
document_id="should-not-exist-from-tenant-x-pov",
|
||||
chunk_index=0,
|
||||
content="This is an entirely different tenant, x should never see this",
|
||||
# Make this as permissive as possible to exercise tenant
|
||||
# isolation.
|
||||
public=True,
|
||||
hidden=False,
|
||||
tenant_state=tenant_y,
|
||||
),
|
||||
}
|
||||
for doc in docs.values():
|
||||
]
|
||||
for doc in docs:
|
||||
test_client.index_document(document=doc)
|
||||
|
||||
# Refresh index to make documents searchable.
|
||||
@@ -732,7 +625,7 @@ class TestOpenSearchClient:
|
||||
query_vector=query_vector,
|
||||
num_candidates=10,
|
||||
num_hits=5,
|
||||
tenant_state=tenant_x,
|
||||
tenant_state=TenantState(tenant_id="tenant-x", multitenant=True),
|
||||
)
|
||||
|
||||
# Under test.
|
||||
@@ -742,24 +635,19 @@ class TestOpenSearchClient:
|
||||
# Should only get the public, non-hidden document.
|
||||
assert len(results) == 1
|
||||
assert results[0].document_id == "public-doc-1"
|
||||
# Make sure the chunk contents are preserved.
|
||||
assert results[0] == docs["public-doc-1"]
|
||||
assert results[0].public is True
|
||||
assert results[0].hidden is False
|
||||
|
||||
def test_search_with_pipeline_and_filters_returns_chunks_with_related_content_first(
|
||||
self,
|
||||
test_client: OpenSearchClient,
|
||||
search_pipeline: None,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
self, test_client: OpenSearchClient, search_pipeline: None
|
||||
) -> None:
|
||||
"""
|
||||
Tests search with a normalization pipeline and filters returns chunks
|
||||
with related content first.
|
||||
"""
|
||||
# Precondition.
|
||||
_patch_global_tenant_state(monkeypatch, True)
|
||||
tenant_x = TenantState(tenant_id="tenant-x", multitenant=True)
|
||||
mappings = DocumentSchema.get_document_schema(
|
||||
vector_dimension=128, multitenant=tenant_x.multitenant
|
||||
vector_dimension=128, multitenant=True
|
||||
)
|
||||
settings = DocumentSchema.get_index_settings()
|
||||
test_client.create_index(mappings=mappings, settings=settings)
|
||||
@@ -776,7 +664,7 @@ class TestOpenSearchClient:
|
||||
), # Very close to query vector.
|
||||
public=True,
|
||||
hidden=False,
|
||||
tenant_state=tenant_x,
|
||||
tenant_id="tenant-x",
|
||||
),
|
||||
_create_test_document_chunk(
|
||||
document_id="somewhat-relevant-1",
|
||||
@@ -785,7 +673,7 @@ class TestOpenSearchClient:
|
||||
content_vector=_generate_test_vector(0.5), # Far from query vector.
|
||||
public=True,
|
||||
hidden=False,
|
||||
tenant_state=tenant_x,
|
||||
tenant_id="tenant-x",
|
||||
),
|
||||
_create_test_document_chunk(
|
||||
document_id="not-very-relevant-1",
|
||||
@@ -796,7 +684,7 @@ class TestOpenSearchClient:
|
||||
), # Very far from query vector.
|
||||
public=True,
|
||||
hidden=False,
|
||||
tenant_state=tenant_x,
|
||||
tenant_id="tenant-x",
|
||||
),
|
||||
# These should be filtered out by public/hidden filters.
|
||||
_create_test_document_chunk(
|
||||
@@ -806,7 +694,7 @@ class TestOpenSearchClient:
|
||||
content_vector=_generate_test_vector(0.05), # Very close but hidden.
|
||||
public=True,
|
||||
hidden=True,
|
||||
tenant_state=tenant_x,
|
||||
tenant_id="tenant-x",
|
||||
),
|
||||
_create_test_document_chunk(
|
||||
document_id="private-but-relevant-1",
|
||||
@@ -815,7 +703,7 @@ class TestOpenSearchClient:
|
||||
content_vector=_generate_test_vector(0.08), # Very close but private.
|
||||
public=False,
|
||||
hidden=False,
|
||||
tenant_state=tenant_x,
|
||||
tenant_id="tenant-x",
|
||||
),
|
||||
]
|
||||
for doc in docs:
|
||||
@@ -832,7 +720,7 @@ class TestOpenSearchClient:
|
||||
query_vector=query_vector,
|
||||
num_candidates=10,
|
||||
num_hits=5,
|
||||
tenant_state=tenant_x,
|
||||
tenant_state=TenantState(tenant_id="tenant-x", multitenant=True),
|
||||
)
|
||||
|
||||
# Under test.
|
||||
@@ -854,134 +742,11 @@ class TestOpenSearchClient:
|
||||
# Most relevant document should be first due to normalization pipeline.
|
||||
assert results[0].document_id == "highly-relevant-1"
|
||||
|
||||
def test_delete_by_query_multitenant_isolation(
|
||||
self, test_client: OpenSearchClient, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""
|
||||
Tests delete_by_query respects tenant boundaries in multi-tenant mode.
|
||||
"""
|
||||
def test_search_for_ids_basic(self, test_client: OpenSearchClient) -> None:
|
||||
"""Tests search_for_ids method returns correct chunk IDs."""
|
||||
# Precondition.
|
||||
_patch_global_tenant_state(monkeypatch, True)
|
||||
tenant_x = TenantState(tenant_id="tenant-x", multitenant=True)
|
||||
tenant_y = TenantState(tenant_id="tenant-y", multitenant=True)
|
||||
mappings = DocumentSchema.get_document_schema(
|
||||
vector_dimension=128, multitenant=tenant_x.multitenant
|
||||
)
|
||||
settings = DocumentSchema.get_index_settings()
|
||||
test_client.create_index(mappings=mappings, settings=settings)
|
||||
|
||||
# Index chunks for different doc IDs for different tenants.
|
||||
# NOTE: Since get_opensearch_doc_chunk_id doesn't include tenant_id yet,
|
||||
# we use different document IDs to avoid ID conflicts.
|
||||
tenant_x_chunks = [
|
||||
_create_test_document_chunk(
|
||||
document_id="doc-tenant-x",
|
||||
chunk_index=i,
|
||||
content=f"Tenant A Chunk {i}",
|
||||
tenant_state=tenant_x,
|
||||
)
|
||||
for i in range(3)
|
||||
]
|
||||
|
||||
tenant_y_chunks = [
|
||||
_create_test_document_chunk(
|
||||
document_id="doc-tenant-y",
|
||||
chunk_index=i,
|
||||
content=f"Tenant B Chunk {i}",
|
||||
tenant_state=tenant_y,
|
||||
)
|
||||
for i in range(2)
|
||||
]
|
||||
|
||||
for chunk in tenant_x_chunks + tenant_y_chunks:
|
||||
test_client.index_document(document=chunk)
|
||||
test_client.refresh_index()
|
||||
|
||||
# Build deletion query for tenant-x only.
|
||||
query_body = DocumentQuery.delete_from_document_id_query(
|
||||
document_id="doc-tenant-x",
|
||||
tenant_state=tenant_x,
|
||||
)
|
||||
|
||||
# Under test.
|
||||
# Delete tenant-x chunks using delete_by_query.
|
||||
num_deleted = test_client.delete_by_query(query_body=query_body)
|
||||
|
||||
# Postcondition.
|
||||
assert num_deleted == 3
|
||||
|
||||
# Verify tenant-x chunks are deleted.
|
||||
test_client.refresh_index()
|
||||
verify_query_x = DocumentQuery.get_from_document_id_query(
|
||||
document_id="doc-tenant-x",
|
||||
tenant_state=tenant_x,
|
||||
max_chunk_size=DEFAULT_MAX_CHUNK_SIZE,
|
||||
min_chunk_index=None,
|
||||
max_chunk_index=None,
|
||||
get_full_document=False,
|
||||
)
|
||||
remaining_a_ids = test_client.search_for_document_ids(body=verify_query_x)
|
||||
assert len(remaining_a_ids) == 0
|
||||
|
||||
# Verify tenant-y chunks still exist.
|
||||
verify_query_y = DocumentQuery.get_from_document_id_query(
|
||||
document_id="doc-tenant-y",
|
||||
tenant_state=tenant_y,
|
||||
max_chunk_size=DEFAULT_MAX_CHUNK_SIZE,
|
||||
min_chunk_index=None,
|
||||
max_chunk_index=None,
|
||||
get_full_document=False,
|
||||
)
|
||||
remaining_y_ids = test_client.search_for_document_ids(body=verify_query_y)
|
||||
assert len(remaining_y_ids) == 2
|
||||
expected_y_ids = {
|
||||
get_opensearch_doc_chunk_id(
|
||||
document_id=chunk.document_id,
|
||||
chunk_index=chunk.chunk_index,
|
||||
max_chunk_size=chunk.max_chunk_size,
|
||||
)
|
||||
for chunk in tenant_y_chunks
|
||||
}
|
||||
assert set(remaining_y_ids) == expected_y_ids
|
||||
|
||||
def test_delete_by_query_nonexistent_document(
|
||||
self, test_client: OpenSearchClient, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""
|
||||
Tests delete_by_query for non-existent document returns 0 deleted.
|
||||
"""
|
||||
# Precondition.
|
||||
_patch_global_tenant_state(monkeypatch, False)
|
||||
tenant_state = TenantState(tenant_id=POSTGRES_DEFAULT_SCHEMA, multitenant=False)
|
||||
mappings = DocumentSchema.get_document_schema(
|
||||
vector_dimension=128, multitenant=tenant_state.multitenant
|
||||
)
|
||||
settings = DocumentSchema.get_index_settings()
|
||||
test_client.create_index(mappings=mappings, settings=settings)
|
||||
|
||||
# Don't index any documents.
|
||||
|
||||
# Build deletion query.
|
||||
query_body = DocumentQuery.delete_from_document_id_query(
|
||||
document_id="nonexistent-doc",
|
||||
tenant_state=tenant_state,
|
||||
)
|
||||
|
||||
# Under test.
|
||||
num_deleted = test_client.delete_by_query(query_body=query_body)
|
||||
|
||||
# Postcondition.
|
||||
assert num_deleted == 0
|
||||
|
||||
def test_search_for_document_ids(
|
||||
self, test_client: OpenSearchClient, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""Tests search_for_document_ids method returns correct chunk IDs."""
|
||||
# Precondition.
|
||||
_patch_global_tenant_state(monkeypatch, False)
|
||||
tenant_state = TenantState(tenant_id=POSTGRES_DEFAULT_SCHEMA, multitenant=False)
|
||||
mappings = DocumentSchema.get_document_schema(
|
||||
vector_dimension=128, multitenant=tenant_state.multitenant
|
||||
vector_dimension=128, multitenant=False
|
||||
)
|
||||
settings = DocumentSchema.get_index_settings()
|
||||
test_client.create_index(mappings=mappings, settings=settings)
|
||||
@@ -989,19 +754,205 @@ class TestOpenSearchClient:
|
||||
# Index chunks for two different documents.
|
||||
doc1_chunks = [
|
||||
_create_test_document_chunk(
|
||||
document_id="doc-1",
|
||||
document_id="doc-1", chunk_index=i, content=f"Doc 1 Chunk {i}"
|
||||
)
|
||||
for i in range(3)
|
||||
]
|
||||
|
||||
doc2_chunks = [
|
||||
_create_test_document_chunk(
|
||||
document_id="doc-2", chunk_index=i, content=f"Doc 2 Chunk {i}"
|
||||
)
|
||||
for i in range(2)
|
||||
]
|
||||
|
||||
for chunk in doc1_chunks + doc2_chunks:
|
||||
test_client.index_document(document=chunk)
|
||||
|
||||
test_client.refresh_index()
|
||||
|
||||
# Build query for doc-1.
|
||||
query_body = DocumentQuery.get_from_document_id_query(
|
||||
document_id="doc-1",
|
||||
tenant_state=TenantState(tenant_id="", multitenant=False),
|
||||
max_chunk_size=DEFAULT_MAX_CHUNK_SIZE,
|
||||
min_chunk_index=None,
|
||||
max_chunk_index=None,
|
||||
get_full_document=False,
|
||||
)
|
||||
|
||||
# Under test.
|
||||
chunk_ids = test_client.search_for_document_ids(body=query_body)
|
||||
|
||||
# Postcondition.
|
||||
# Should get 3 IDs for doc-1.
|
||||
assert len(chunk_ids) == 3
|
||||
|
||||
# Verify IDs match expected chunk IDs.
|
||||
expected_ids = {
|
||||
get_opensearch_doc_chunk_id(
|
||||
document_id=chunk.document_id,
|
||||
chunk_index=chunk.chunk_index,
|
||||
max_chunk_size=chunk.max_chunk_size,
|
||||
)
|
||||
for chunk in doc1_chunks
|
||||
}
|
||||
assert set(chunk_ids) == expected_ids
|
||||
|
||||
def test_delete_by_query_multitenant_isolation(
|
||||
self, test_client: OpenSearchClient
|
||||
) -> None:
|
||||
"""
|
||||
Tests delete_by_query respects tenant boundaries in multi-tenant mode.
|
||||
"""
|
||||
# Precondition.
|
||||
mappings = DocumentSchema.get_document_schema(
|
||||
vector_dimension=128, multitenant=True
|
||||
)
|
||||
settings = DocumentSchema.get_index_settings()
|
||||
test_client.create_index(mappings=mappings, settings=settings)
|
||||
|
||||
# Index chunks for different doc IDs for different tenants.
|
||||
# NOTE: Since get_opensearch_doc_chunk_id doesn't include tenant_id yet,
|
||||
# we use different document IDs to avoid ID conflicts.
|
||||
tenant_a_chunks = [
|
||||
_create_test_document_chunk(
|
||||
document_id="doc-tenant-a",
|
||||
chunk_index=i,
|
||||
content=f"Doc 1 Chunk {i}",
|
||||
tenant_state=tenant_state,
|
||||
content=f"Tenant A Chunk {i}",
|
||||
tenant_id="tenant-a",
|
||||
)
|
||||
for i in range(3)
|
||||
]
|
||||
|
||||
tenant_b_chunks = [
|
||||
_create_test_document_chunk(
|
||||
document_id="doc-tenant-b",
|
||||
chunk_index=i,
|
||||
content=f"Tenant B Chunk {i}",
|
||||
tenant_id="tenant-b",
|
||||
)
|
||||
for i in range(2)
|
||||
]
|
||||
|
||||
for chunk in tenant_a_chunks + tenant_b_chunks:
|
||||
test_client.index_document(document=chunk)
|
||||
test_client.refresh_index()
|
||||
|
||||
# Build deletion query for tenant-a only.
|
||||
query_body = DocumentQuery.get_from_document_id_query(
|
||||
document_id="doc-tenant-a",
|
||||
tenant_state=TenantState(tenant_id="tenant-a", multitenant=True),
|
||||
max_chunk_size=DEFAULT_MAX_CHUNK_SIZE,
|
||||
min_chunk_index=None,
|
||||
max_chunk_index=None,
|
||||
get_full_document=False,
|
||||
)
|
||||
|
||||
chunk_ids = test_client.search_for_document_ids(body=query_body)
|
||||
|
||||
assert len(chunk_ids) == 3
|
||||
expected_ids = {
|
||||
get_opensearch_doc_chunk_id(
|
||||
document_id=chunk.document_id,
|
||||
chunk_index=chunk.chunk_index,
|
||||
max_chunk_size=chunk.max_chunk_size,
|
||||
)
|
||||
for chunk in tenant_a_chunks
|
||||
}
|
||||
assert set(chunk_ids) == expected_ids
|
||||
|
||||
# Under test.
|
||||
# Delete tenant-a chunks.
|
||||
for chunk_id in chunk_ids:
|
||||
result = test_client.delete_document(chunk_id)
|
||||
assert result is True
|
||||
|
||||
# Postcondition.
|
||||
# Verify tenant-a chunks are deleted.
|
||||
test_client.refresh_index()
|
||||
verify_query_a = DocumentQuery.get_from_document_id_query(
|
||||
document_id="doc-tenant-a",
|
||||
tenant_state=TenantState(tenant_id="tenant-a", multitenant=True),
|
||||
max_chunk_size=DEFAULT_MAX_CHUNK_SIZE,
|
||||
min_chunk_index=None,
|
||||
max_chunk_index=None,
|
||||
get_full_document=False,
|
||||
)
|
||||
remaining_a_ids = test_client.search_for_document_ids(body=verify_query_a)
|
||||
assert len(remaining_a_ids) == 0
|
||||
|
||||
# Verify tenant-b chunks still exist.
|
||||
verify_query_b = DocumentQuery.get_from_document_id_query(
|
||||
document_id="doc-tenant-b",
|
||||
tenant_state=TenantState(tenant_id="tenant-b", multitenant=True),
|
||||
max_chunk_size=DEFAULT_MAX_CHUNK_SIZE,
|
||||
min_chunk_index=None,
|
||||
max_chunk_index=None,
|
||||
get_full_document=False,
|
||||
)
|
||||
remaining_b_ids = test_client.search_for_document_ids(body=verify_query_b)
|
||||
assert len(remaining_b_ids) == 2
|
||||
expected_b_ids = {
|
||||
get_opensearch_doc_chunk_id(
|
||||
document_id=chunk.document_id,
|
||||
chunk_index=chunk.chunk_index,
|
||||
max_chunk_size=chunk.max_chunk_size,
|
||||
)
|
||||
for chunk in tenant_b_chunks
|
||||
}
|
||||
assert set(remaining_b_ids) == expected_b_ids
|
||||
|
||||
def test_delete_by_query_nonexistent_document(
|
||||
self, test_client: OpenSearchClient
|
||||
) -> None:
|
||||
"""
|
||||
Tests delete_by_query for non-existent document returns 0 deleted.
|
||||
"""
|
||||
# Precondition.
|
||||
mappings = DocumentSchema.get_document_schema(
|
||||
vector_dimension=128, multitenant=False
|
||||
)
|
||||
settings = DocumentSchema.get_index_settings()
|
||||
test_client.create_index(mappings=mappings, settings=settings)
|
||||
|
||||
# Don't index any documents.
|
||||
|
||||
# Build deletion query.
|
||||
query_body = DocumentQuery.get_from_document_id_query(
|
||||
document_id="nonexistent-doc",
|
||||
tenant_state=TenantState(tenant_id="", multitenant=False),
|
||||
max_chunk_size=DEFAULT_MAX_CHUNK_SIZE,
|
||||
min_chunk_index=None,
|
||||
max_chunk_index=None,
|
||||
get_full_document=False,
|
||||
)
|
||||
|
||||
# Under test.
|
||||
chunk_ids = test_client.search_for_document_ids(body=query_body)
|
||||
|
||||
# Postcondition.
|
||||
assert len(chunk_ids) == 0
|
||||
|
||||
def test_search_for_document_ids(self, test_client: OpenSearchClient) -> None:
|
||||
"""Tests search_for_document_ids method returns correct chunk IDs."""
|
||||
# Precondition.
|
||||
mappings = DocumentSchema.get_document_schema(
|
||||
vector_dimension=128, multitenant=False
|
||||
)
|
||||
settings = DocumentSchema.get_index_settings()
|
||||
test_client.create_index(mappings=mappings, settings=settings)
|
||||
|
||||
# Index chunks for two different documents.
|
||||
doc1_chunks = [
|
||||
_create_test_document_chunk(
|
||||
document_id="doc-1", chunk_index=i, content=f"Doc 1 Chunk {i}"
|
||||
)
|
||||
for i in range(3)
|
||||
]
|
||||
doc2_chunks = [
|
||||
_create_test_document_chunk(
|
||||
document_id="doc-2",
|
||||
chunk_index=i,
|
||||
content=f"Doc 2 Chunk {i}",
|
||||
tenant_state=tenant_state,
|
||||
document_id="doc-2", chunk_index=i, content=f"Doc 2 Chunk {i}"
|
||||
)
|
||||
for i in range(2)
|
||||
]
|
||||
@@ -1013,7 +964,7 @@ class TestOpenSearchClient:
|
||||
# Build query for doc-1.
|
||||
query_body = DocumentQuery.get_from_document_id_query(
|
||||
document_id="doc-1",
|
||||
tenant_state=tenant_state,
|
||||
tenant_state=TenantState(tenant_id="", multitenant=False),
|
||||
max_chunk_size=DEFAULT_MAX_CHUNK_SIZE,
|
||||
min_chunk_index=None,
|
||||
max_chunk_index=None,
|
||||
|
||||
@@ -41,12 +41,6 @@ API_KEY_RECORDS: Dict[str, Dict[str, Any]] = {
|
||||
},
|
||||
}
|
||||
|
||||
# These are inferrable from the file anyways, no need to obfuscate.
|
||||
# use them to test your auth with this server
|
||||
#
|
||||
# mcp_live-kid_alice_001-S3cr3tAlice
|
||||
# mcp_live-kid_bob_001-S3cr3tBob
|
||||
|
||||
|
||||
# ---- verifier ---------------------------------------------------------------
|
||||
class ApiKeyVerifier(TokenVerifier):
|
||||
|
||||
@@ -309,63 +309,6 @@ def test_get_llm_for_persona_falls_back_when_access_denied(
|
||||
assert fallback_llm.config.model_name == default_provider.default_model_name
|
||||
|
||||
|
||||
def test_list_llm_provider_basics_excludes_non_public_unrestricted(
|
||||
users: tuple[DATestUser, DATestUser],
|
||||
) -> None:
|
||||
"""Test that the /llm/provider endpoint correctly excludes non-public providers
|
||||
with no group/persona restrictions.
|
||||
|
||||
This tests the fix for the bug where non-public providers with no restrictions
|
||||
were incorrectly shown to all users instead of being admin-only.
|
||||
"""
|
||||
admin_user, basic_user = users
|
||||
|
||||
# Create a public provider (should be visible to all)
|
||||
public_provider = LLMProviderManager.create(
|
||||
name="public-provider",
|
||||
is_public=True,
|
||||
set_as_default=True,
|
||||
user_performing_action=admin_user,
|
||||
)
|
||||
|
||||
# Create a non-public provider with no restrictions (should be admin-only)
|
||||
non_public_provider = LLMProviderManager.create(
|
||||
name="non-public-unrestricted",
|
||||
is_public=False,
|
||||
groups=[],
|
||||
personas=[],
|
||||
set_as_default=False,
|
||||
user_performing_action=admin_user,
|
||||
)
|
||||
|
||||
# Non-admin user calls the /llm/provider endpoint
|
||||
response = requests.get(
|
||||
f"{API_SERVER_URL}/llm/provider",
|
||||
headers=basic_user.headers,
|
||||
)
|
||||
assert response.status_code == 200
|
||||
providers = response.json()
|
||||
provider_names = [p["name"] for p in providers]
|
||||
|
||||
# Public provider should be visible
|
||||
assert public_provider.name in provider_names
|
||||
|
||||
# Non-public provider with no restrictions should NOT be visible to non-admin
|
||||
assert non_public_provider.name not in provider_names
|
||||
|
||||
# Admin user should see both providers
|
||||
admin_response = requests.get(
|
||||
f"{API_SERVER_URL}/llm/provider",
|
||||
headers=admin_user.headers,
|
||||
)
|
||||
assert admin_response.status_code == 200
|
||||
admin_providers = admin_response.json()
|
||||
admin_provider_names = [p["name"] for p in admin_providers]
|
||||
|
||||
assert public_provider.name in admin_provider_names
|
||||
assert non_public_provider.name in admin_provider_names
|
||||
|
||||
|
||||
def test_provider_delete_clears_persona_references(reset: None) -> None:
|
||||
"""Test that deleting a provider automatically clears persona references."""
|
||||
admin_user = UserManager.create(name="admin_user")
|
||||
|
||||
@@ -270,7 +270,7 @@ def test_web_search_endpoints_with_exa(
|
||||
provider_id = _activate_exa_provider(admin_user)
|
||||
assert isinstance(provider_id, int)
|
||||
|
||||
search_request = {"queries": ["wikipedia python programming"], "max_results": 3}
|
||||
search_request = {"queries": ["latest ai research news"], "max_results": 3}
|
||||
|
||||
lite_response = requests.post(
|
||||
f"{API_SERVER_URL}/web-search/search-lite",
|
||||
|
||||
@@ -21,9 +21,9 @@ use tauri::{
|
||||
webview::PageLoadPayload, AppHandle, Manager, Webview, WebviewUrl, WebviewWindowBuilder,
|
||||
};
|
||||
use tauri_plugin_global_shortcut::{Code, GlobalShortcutExt, Modifiers, Shortcut};
|
||||
use url::Url;
|
||||
#[cfg(target_os = "macos")]
|
||||
use tokio::time::sleep;
|
||||
use url::Url;
|
||||
#[cfg(target_os = "macos")]
|
||||
use window_vibrancy::{apply_vibrancy, NSVisualEffectMaterial};
|
||||
|
||||
@@ -76,25 +76,39 @@ fn get_config_path() -> Option<PathBuf> {
|
||||
}
|
||||
|
||||
/// Load config from file, or create default if it doesn't exist
|
||||
fn load_config() -> (AppConfig, bool) {
|
||||
fn load_config() -> AppConfig {
|
||||
let config_path = match get_config_path() {
|
||||
Some(path) => path,
|
||||
None => {
|
||||
return (AppConfig::default(), false);
|
||||
eprintln!("Could not determine config directory, using defaults");
|
||||
return AppConfig::default();
|
||||
}
|
||||
};
|
||||
|
||||
if !config_path.exists() {
|
||||
return (AppConfig::default(), false);
|
||||
if config_path.exists() {
|
||||
match fs::read_to_string(&config_path) {
|
||||
Ok(contents) => match serde_json::from_str(&contents) {
|
||||
Ok(config) => {
|
||||
return config;
|
||||
}
|
||||
Err(e) => {
|
||||
eprintln!("Failed to parse config: {}, using defaults", e);
|
||||
}
|
||||
},
|
||||
Err(e) => {
|
||||
eprintln!("Failed to read config: {}, using defaults", e);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Create default config file
|
||||
if let Err(e) = save_config(&AppConfig::default()) {
|
||||
eprintln!("Failed to create default config: {}", e);
|
||||
} else {
|
||||
println!("Created default config at {:?}", config_path);
|
||||
}
|
||||
}
|
||||
|
||||
match fs::read_to_string(&config_path) {
|
||||
Ok(contents) => match serde_json::from_str(&contents) {
|
||||
Ok(config) => (config, true),
|
||||
Err(_) => (AppConfig::default(), false),
|
||||
},
|
||||
Err(_) => (AppConfig::default(), false),
|
||||
}
|
||||
AppConfig::default()
|
||||
}
|
||||
|
||||
/// Save config to file
|
||||
@@ -114,11 +128,7 @@ fn save_config(config: &AppConfig) -> Result<(), String> {
|
||||
}
|
||||
|
||||
// Global config state
|
||||
struct ConfigState {
|
||||
config: RwLock<AppConfig>,
|
||||
config_initialized: RwLock<bool>,
|
||||
app_base_url: RwLock<Option<Url>>,
|
||||
}
|
||||
struct ConfigState(RwLock<AppConfig>);
|
||||
|
||||
fn focus_main_window(app: &AppHandle) {
|
||||
if let Some(window) = app.get_webview_window("main") {
|
||||
@@ -132,7 +142,7 @@ fn focus_main_window(app: &AppHandle) {
|
||||
|
||||
fn trigger_new_chat(app: &AppHandle) {
|
||||
let state = app.state::<ConfigState>();
|
||||
let server_url = state.config.read().unwrap().server_url.clone();
|
||||
let server_url = state.0.read().unwrap().server_url.clone();
|
||||
|
||||
if let Some(window) = app.get_webview_window("main") {
|
||||
let url = format!("{}/chat", server_url);
|
||||
@@ -142,7 +152,7 @@ fn trigger_new_chat(app: &AppHandle) {
|
||||
|
||||
fn trigger_new_window(app: &AppHandle) {
|
||||
let state = app.state::<ConfigState>();
|
||||
let server_url = state.config.read().unwrap().server_url.clone();
|
||||
let server_url = state.0.read().unwrap().server_url.clone();
|
||||
let handle = app.clone();
|
||||
|
||||
tauri::async_runtime::spawn(async move {
|
||||
@@ -196,30 +206,6 @@ fn open_docs() {
|
||||
}
|
||||
}
|
||||
|
||||
fn open_settings(app: &AppHandle) {
|
||||
// Navigate main window to the settings page (index.html) with settings flag
|
||||
let state = app.state::<ConfigState>();
|
||||
let settings_url = state
|
||||
.app_base_url
|
||||
.read()
|
||||
.unwrap()
|
||||
.as_ref()
|
||||
.cloned()
|
||||
.and_then(|mut url| {
|
||||
url.set_query(None);
|
||||
url.set_fragment(Some("settings"));
|
||||
url.set_path("/");
|
||||
Some(url)
|
||||
})
|
||||
.or_else(|| Url::parse("tauri://localhost/#settings").ok());
|
||||
|
||||
if let Some(window) = app.get_webview_window("main") {
|
||||
if let Some(url) = settings_url {
|
||||
let _ = window.navigate(url);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Tauri Commands
|
||||
// ============================================================================
|
||||
@@ -227,27 +213,7 @@ fn open_settings(app: &AppHandle) {
|
||||
/// Get the current server URL
|
||||
#[tauri::command]
|
||||
fn get_server_url(state: tauri::State<ConfigState>) -> String {
|
||||
state.config.read().unwrap().server_url.clone()
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct BootstrapState {
|
||||
server_url: String,
|
||||
config_exists: bool,
|
||||
}
|
||||
|
||||
/// Get the server URL plus whether a config file exists
|
||||
#[tauri::command]
|
||||
fn get_bootstrap_state(state: tauri::State<ConfigState>) -> BootstrapState {
|
||||
let server_url = state.config.read().unwrap().server_url.clone();
|
||||
let config_initialized = *state.config_initialized.read().unwrap();
|
||||
let config_exists = config_initialized
|
||||
&& get_config_path().map(|path| path.exists()).unwrap_or(false);
|
||||
|
||||
BootstrapState {
|
||||
server_url,
|
||||
config_exists,
|
||||
}
|
||||
state.0.read().unwrap().server_url.clone()
|
||||
}
|
||||
|
||||
/// Set a new server URL and save to config
|
||||
@@ -258,10 +224,9 @@ fn set_server_url(state: tauri::State<ConfigState>, url: String) -> Result<Strin
|
||||
return Err("URL must start with http:// or https://".to_string());
|
||||
}
|
||||
|
||||
let mut config = state.config.write().unwrap();
|
||||
let mut config = state.0.write().unwrap();
|
||||
config.server_url = url.trim_end_matches('/').to_string();
|
||||
save_config(&config)?;
|
||||
*state.config_initialized.write().unwrap() = true;
|
||||
|
||||
Ok(config.server_url.clone())
|
||||
}
|
||||
@@ -350,7 +315,7 @@ fn open_config_directory() -> Result<(), String> {
|
||||
/// Navigate to a specific path on the configured server
|
||||
#[tauri::command]
|
||||
fn navigate_to(window: tauri::WebviewWindow, state: tauri::State<ConfigState>, path: &str) {
|
||||
let base_url = state.config.read().unwrap().server_url.clone();
|
||||
let base_url = state.0.read().unwrap().server_url.clone();
|
||||
let url = format!("{}{}", base_url, path);
|
||||
let _ = window.eval(&format!("window.location.href = '{}'", url));
|
||||
}
|
||||
@@ -376,7 +341,7 @@ fn go_forward(window: tauri::WebviewWindow) {
|
||||
/// Open a new window
|
||||
#[tauri::command]
|
||||
async fn new_window(app: AppHandle, state: tauri::State<'_, ConfigState>) -> Result<(), String> {
|
||||
let server_url = state.config.read().unwrap().server_url.clone();
|
||||
let server_url = state.0.read().unwrap().server_url.clone();
|
||||
let window_label = format!("onyx-{}", uuid::Uuid::new_v4());
|
||||
|
||||
let builder = WebviewWindowBuilder::new(
|
||||
@@ -420,10 +385,9 @@ async fn new_window(app: AppHandle, state: tauri::State<'_, ConfigState>) -> Res
|
||||
/// Reset config to defaults
|
||||
#[tauri::command]
|
||||
fn reset_config(state: tauri::State<ConfigState>) -> Result<(), String> {
|
||||
let mut config = state.config.write().unwrap();
|
||||
let mut config = state.0.write().unwrap();
|
||||
*config = AppConfig::default();
|
||||
save_config(&config)?;
|
||||
*state.config_initialized.write().unwrap() = true;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -459,7 +423,7 @@ fn setup_shortcuts(app: &AppHandle) -> Result<(), Box<dyn std::error::Error>> {
|
||||
let forward = Shortcut::new(Some(Modifiers::SUPER), Code::BracketRight);
|
||||
let new_window_shortcut = Shortcut::new(Some(Modifiers::SUPER | Modifiers::SHIFT), Code::KeyN);
|
||||
let show_app = Shortcut::new(Some(Modifiers::SUPER | Modifiers::SHIFT), Code::Space);
|
||||
let open_settings_shortcut = Shortcut::new(Some(Modifiers::SUPER), Code::Comma);
|
||||
let open_settings = Shortcut::new(Some(Modifiers::SUPER), Code::Comma);
|
||||
|
||||
let app_handle = app.clone();
|
||||
|
||||
@@ -471,7 +435,7 @@ fn setup_shortcuts(app: &AppHandle) -> Result<(), Box<dyn std::error::Error>> {
|
||||
forward,
|
||||
new_window_shortcut,
|
||||
show_app,
|
||||
open_settings_shortcut,
|
||||
open_settings,
|
||||
];
|
||||
|
||||
#[cfg(not(target_os = "macos"))]
|
||||
@@ -482,7 +446,7 @@ fn setup_shortcuts(app: &AppHandle) -> Result<(), Box<dyn std::error::Error>> {
|
||||
forward,
|
||||
new_window_shortcut,
|
||||
show_app,
|
||||
open_settings_shortcut,
|
||||
open_settings,
|
||||
];
|
||||
|
||||
app.global_shortcut().on_shortcuts(
|
||||
@@ -499,8 +463,9 @@ fn setup_shortcuts(app: &AppHandle) -> Result<(), Box<dyn std::error::Error>> {
|
||||
let _ = window.eval("window.history.back()");
|
||||
} else if shortcut == &forward {
|
||||
let _ = window.eval("window.history.forward()");
|
||||
} else if shortcut == &open_settings_shortcut {
|
||||
open_settings(&app_handle);
|
||||
} else if shortcut == &open_settings {
|
||||
// Open config file for editing
|
||||
let _ = open_config_file();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -530,7 +495,6 @@ fn setup_app_menu(app: &AppHandle) -> tauri::Result<()> {
|
||||
true,
|
||||
Some("CmdOrCtrl+Shift+N"),
|
||||
)?;
|
||||
let settings_item = MenuItem::with_id(app, "open_settings", "Settings...", true, Some("CmdOrCtrl+Comma"))?;
|
||||
let docs_item = MenuItem::with_id(app, "open_docs", "Onyx Documentation", true, None::<&str>)?;
|
||||
|
||||
if let Some(file_menu) = menu
|
||||
@@ -539,13 +503,12 @@ fn setup_app_menu(app: &AppHandle) -> tauri::Result<()> {
|
||||
.filter_map(|item| item.as_submenu().cloned())
|
||||
.find(|submenu| submenu.text().ok().as_deref() == Some("File"))
|
||||
{
|
||||
file_menu.insert_items(&[&new_chat_item, &new_window_item, &settings_item], 0)?;
|
||||
file_menu.insert_items(&[&new_chat_item, &new_window_item], 0)?;
|
||||
} else {
|
||||
let file_menu = SubmenuBuilder::new(app, "File")
|
||||
.items(&[
|
||||
&new_chat_item,
|
||||
&new_window_item,
|
||||
&settings_item,
|
||||
&PredefinedMenuItem::close_window(app, None)?,
|
||||
])
|
||||
.build()?;
|
||||
@@ -662,20 +625,22 @@ fn setup_tray_icon(app: &AppHandle) -> tauri::Result<()> {
|
||||
|
||||
fn main() {
|
||||
// Load config at startup
|
||||
let (config, config_initialized) = load_config();
|
||||
let config = load_config();
|
||||
let server_url = config.server_url.clone();
|
||||
|
||||
println!("Starting Onyx Desktop");
|
||||
println!("Server URL: {}", server_url);
|
||||
if let Some(path) = get_config_path() {
|
||||
println!("Config file: {:?}", path);
|
||||
}
|
||||
|
||||
tauri::Builder::default()
|
||||
.plugin(tauri_plugin_shell::init())
|
||||
.plugin(tauri_plugin_global_shortcut::Builder::new().build())
|
||||
.plugin(tauri_plugin_window_state::Builder::default().build())
|
||||
.manage(ConfigState {
|
||||
config: RwLock::new(config),
|
||||
config_initialized: RwLock::new(config_initialized),
|
||||
app_base_url: RwLock::new(None),
|
||||
})
|
||||
.manage(ConfigState(RwLock::new(config)))
|
||||
.invoke_handler(tauri::generate_handler![
|
||||
get_server_url,
|
||||
get_bootstrap_state,
|
||||
set_server_url,
|
||||
get_config_path_cmd,
|
||||
open_config_file,
|
||||
@@ -692,7 +657,6 @@ fn main() {
|
||||
"open_docs" => open_docs(),
|
||||
"new_chat" => trigger_new_chat(app),
|
||||
"new_window" => trigger_new_window(app),
|
||||
"open_settings" => open_settings(app),
|
||||
_ => {}
|
||||
})
|
||||
.setup(move |app| {
|
||||
@@ -711,7 +675,7 @@ fn main() {
|
||||
eprintln!("Failed to setup tray icon: {}", e);
|
||||
}
|
||||
|
||||
// Setup main window with vibrancy effect
|
||||
// Update main window URL to configured server and inject title bar
|
||||
if let Some(window) = app.get_webview_window("main") {
|
||||
// Apply vibrancy effect for translucent glass look
|
||||
#[cfg(target_os = "macos")]
|
||||
@@ -719,12 +683,14 @@ fn main() {
|
||||
let _ = apply_vibrancy(&window, NSVisualEffectMaterial::Sidebar, None, None);
|
||||
}
|
||||
|
||||
if let Ok(url) = window.url() {
|
||||
let mut base_url = url;
|
||||
base_url.set_query(None);
|
||||
base_url.set_fragment(None);
|
||||
base_url.set_path("/");
|
||||
*app.state::<ConfigState>().app_base_url.write().unwrap() = Some(base_url);
|
||||
if let Ok(target) = Url::parse(&server_url) {
|
||||
if let Ok(current) = window.url() {
|
||||
if current != target {
|
||||
let _ = window.navigate(target);
|
||||
}
|
||||
} else {
|
||||
let _ = window.navigate(target);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
{
|
||||
"title": "Onyx",
|
||||
"label": "main",
|
||||
"url": "index.html",
|
||||
"url": "https://cloud.onyx.app",
|
||||
"width": 1200,
|
||||
"height": 800,
|
||||
"minWidth": 800,
|
||||
@@ -52,7 +52,7 @@
|
||||
"entitlements": null,
|
||||
"exceptionDomain": "cloud.onyx.app",
|
||||
"minimumSystemVersion": "10.15",
|
||||
"signingIdentity": null,
|
||||
"signingIdentity": "-",
|
||||
"dmg": {
|
||||
"windowSize": {
|
||||
"width": 660,
|
||||
|
||||
@@ -4,43 +4,28 @@
|
||||
<meta charset="UTF-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>Onyx</title>
|
||||
<link
|
||||
href="https://fonts.googleapis.com/css2?family=Hanken+Grotesk:wght@400;500;600;700&display=swap"
|
||||
rel="stylesheet"
|
||||
/>
|
||||
<style>
|
||||
:root {
|
||||
--background-900: #f5f5f5;
|
||||
--background-800: #ffffff;
|
||||
--text-light-05: rgba(0, 0, 0, 0.95);
|
||||
--text-light-03: rgba(0, 0, 0, 0.6);
|
||||
--white-10: rgba(0, 0, 0, 0.1);
|
||||
--white-15: rgba(0, 0, 0, 0.15);
|
||||
--white-20: rgba(0, 0, 0, 0.2);
|
||||
--white-30: rgba(0, 0, 0, 0.3);
|
||||
--font-hanken-grotesk: "Hanken Grotesk", -apple-system,
|
||||
BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;
|
||||
}
|
||||
|
||||
* {
|
||||
box-sizing: border-box;
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
box-sizing: border-box;
|
||||
}
|
||||
|
||||
body {
|
||||
font-family: var(--font-hanken-grotesk);
|
||||
background: linear-gradient(135deg, #f5f5f5 0%, #ffffff 100%);
|
||||
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto,
|
||||
Oxygen, Ubuntu, sans-serif;
|
||||
background: linear-gradient(135deg, #1a1a2e 0%, #16213e 100%);
|
||||
color: #fff;
|
||||
min-height: 100vh;
|
||||
color: var(--text-light-05);
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
padding: 20px;
|
||||
-webkit-user-select: none;
|
||||
user-select: none;
|
||||
}
|
||||
|
||||
/* Draggable titlebar area for macOS */
|
||||
.titlebar {
|
||||
position: fixed;
|
||||
top: 0;
|
||||
@@ -48,451 +33,198 @@
|
||||
right: 0;
|
||||
height: 28px;
|
||||
-webkit-app-region: drag;
|
||||
z-index: 10000;
|
||||
}
|
||||
|
||||
.settings-container {
|
||||
max-width: 500px;
|
||||
width: 100%;
|
||||
opacity: 0;
|
||||
transform: translateY(8px);
|
||||
pointer-events: none;
|
||||
transition:
|
||||
opacity 0.18s ease,
|
||||
transform 0.18s ease;
|
||||
.container {
|
||||
text-align: center;
|
||||
padding: 2rem;
|
||||
}
|
||||
|
||||
body.show-settings .settings-container {
|
||||
opacity: 1;
|
||||
transform: translateY(0);
|
||||
pointer-events: auto;
|
||||
}
|
||||
|
||||
.settings-panel {
|
||||
background: linear-gradient(
|
||||
to bottom,
|
||||
rgba(255, 255, 255, 0.95),
|
||||
rgba(245, 245, 245, 0.95)
|
||||
);
|
||||
backdrop-filter: blur(24px);
|
||||
border-radius: 16px;
|
||||
border: 1px solid var(--white-10);
|
||||
overflow: hidden;
|
||||
box-shadow: 0 8px 32px rgba(0, 0, 0, 0.1);
|
||||
}
|
||||
|
||||
.settings-header {
|
||||
padding: 24px;
|
||||
border-bottom: 1px solid var(--white-10);
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 12px;
|
||||
}
|
||||
|
||||
.settings-icon {
|
||||
width: 40px;
|
||||
height: 40px;
|
||||
border-radius: 12px;
|
||||
background: white;
|
||||
.logo {
|
||||
width: 80px;
|
||||
height: 80px;
|
||||
background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
|
||||
border-radius: 20px;
|
||||
margin: 0 auto 1.5rem;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
overflow: hidden;
|
||||
font-size: 2.5rem;
|
||||
font-weight: bold;
|
||||
}
|
||||
|
||||
.settings-icon svg {
|
||||
width: 24px;
|
||||
height: 24px;
|
||||
color: #000;
|
||||
}
|
||||
|
||||
.settings-title {
|
||||
font-size: 20px;
|
||||
h1 {
|
||||
font-size: 2rem;
|
||||
margin-bottom: 0.5rem;
|
||||
font-weight: 600;
|
||||
color: var(--text-light-05);
|
||||
}
|
||||
|
||||
.settings-content {
|
||||
padding: 24px;
|
||||
p {
|
||||
color: #a0a0a0;
|
||||
margin-bottom: 2rem;
|
||||
}
|
||||
|
||||
.settings-section {
|
||||
margin-bottom: 32px;
|
||||
.loading {
|
||||
display: flex;
|
||||
gap: 0.5rem;
|
||||
justify-content: center;
|
||||
margin-bottom: 2rem;
|
||||
}
|
||||
|
||||
.settings-section:last-child {
|
||||
margin-bottom: 0;
|
||||
.loading span {
|
||||
width: 10px;
|
||||
height: 10px;
|
||||
background: #667eea;
|
||||
border-radius: 50%;
|
||||
animation: bounce 1.4s ease-in-out infinite;
|
||||
}
|
||||
|
||||
.section-title {
|
||||
font-size: 11px;
|
||||
font-weight: 600;
|
||||
.loading span:nth-child(1) {
|
||||
animation-delay: 0s;
|
||||
}
|
||||
.loading span:nth-child(2) {
|
||||
animation-delay: 0.2s;
|
||||
}
|
||||
.loading span:nth-child(3) {
|
||||
animation-delay: 0.4s;
|
||||
}
|
||||
|
||||
@keyframes bounce {
|
||||
0%,
|
||||
80%,
|
||||
100% {
|
||||
transform: scale(0.8);
|
||||
opacity: 0.5;
|
||||
}
|
||||
40% {
|
||||
transform: scale(1.2);
|
||||
opacity: 1;
|
||||
}
|
||||
}
|
||||
|
||||
.btn {
|
||||
background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
|
||||
color: white;
|
||||
border: none;
|
||||
padding: 0.75rem 2rem;
|
||||
border-radius: 8px;
|
||||
font-size: 1rem;
|
||||
cursor: pointer;
|
||||
transition:
|
||||
transform 0.2s,
|
||||
box-shadow 0.2s;
|
||||
-webkit-app-region: no-drag;
|
||||
}
|
||||
|
||||
.btn:hover {
|
||||
transform: translateY(-2px);
|
||||
box-shadow: 0 4px 20px rgba(102, 126, 234, 0.4);
|
||||
}
|
||||
|
||||
.shortcuts {
|
||||
margin-top: 3rem;
|
||||
padding: 1.5rem;
|
||||
background: rgba(255, 255, 255, 0.05);
|
||||
border-radius: 12px;
|
||||
text-align: left;
|
||||
}
|
||||
|
||||
.shortcuts h3 {
|
||||
font-size: 0.875rem;
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 0.05em;
|
||||
color: var(--text-light-03);
|
||||
margin-bottom: 12px;
|
||||
color: #a0a0a0;
|
||||
margin-bottom: 1rem;
|
||||
}
|
||||
|
||||
.settings-group {
|
||||
background: rgba(0, 0, 0, 0.03);
|
||||
border-radius: 16px;
|
||||
padding: 4px;
|
||||
}
|
||||
|
||||
.setting-row {
|
||||
.shortcut {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
align-items: center;
|
||||
padding: 12px;
|
||||
padding: 0.5rem 0;
|
||||
border-bottom: 1px solid rgba(255, 255, 255, 0.1);
|
||||
}
|
||||
|
||||
.setting-row-content {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 4px;
|
||||
flex: 1;
|
||||
.shortcut:last-child {
|
||||
border-bottom: none;
|
||||
}
|
||||
|
||||
.setting-label {
|
||||
font-size: 14px;
|
||||
font-weight: 400;
|
||||
color: var(--text-light-05);
|
||||
}
|
||||
|
||||
.setting-description {
|
||||
font-size: 12px;
|
||||
color: var(--text-light-03);
|
||||
}
|
||||
|
||||
.setting-divider {
|
||||
height: 1px;
|
||||
background: var(--white-10);
|
||||
margin: 0 4px;
|
||||
}
|
||||
|
||||
.input-field {
|
||||
width: 100%;
|
||||
padding: 10px 12px;
|
||||
border: 1px solid var(--white-10);
|
||||
border-radius: 8px;
|
||||
font-size: 14px;
|
||||
background: rgba(0, 0, 0, 0.05);
|
||||
color: var(--text-light-05);
|
||||
font-family: var(--font-hanken-grotesk);
|
||||
transition: all 0.2s;
|
||||
-webkit-app-region: no-drag;
|
||||
}
|
||||
|
||||
.input-field:focus {
|
||||
outline: none;
|
||||
border-color: var(--white-30);
|
||||
background: rgba(0, 0, 0, 0.08);
|
||||
box-shadow: 0 0 0 2px rgba(0, 0, 0, 0.05);
|
||||
}
|
||||
|
||||
.input-field::placeholder {
|
||||
color: var(--text-light-03);
|
||||
}
|
||||
|
||||
.input-field.error {
|
||||
border-color: #ef4444;
|
||||
}
|
||||
|
||||
.error-message {
|
||||
color: #ef4444;
|
||||
font-size: 12px;
|
||||
margin-top: 4px;
|
||||
padding-left: 12px;
|
||||
display: none;
|
||||
}
|
||||
|
||||
.error-message.visible {
|
||||
display: block;
|
||||
}
|
||||
|
||||
.toggle-switch {
|
||||
position: relative;
|
||||
display: inline-block;
|
||||
width: 44px;
|
||||
height: 24px;
|
||||
flex-shrink: 0;
|
||||
}
|
||||
|
||||
.toggle-switch input {
|
||||
opacity: 0;
|
||||
width: 0;
|
||||
height: 0;
|
||||
}
|
||||
|
||||
.toggle-slider {
|
||||
position: absolute;
|
||||
cursor: pointer;
|
||||
top: 0;
|
||||
left: 0;
|
||||
right: 0;
|
||||
bottom: 0;
|
||||
background-color: rgba(0, 0, 0, 0.15);
|
||||
transition: 0.3s;
|
||||
border-radius: 24px;
|
||||
}
|
||||
|
||||
.toggle-slider:before {
|
||||
position: absolute;
|
||||
content: "";
|
||||
height: 18px;
|
||||
width: 18px;
|
||||
left: 3px;
|
||||
bottom: 3px;
|
||||
background-color: white;
|
||||
box-shadow: 0 1px 3px rgba(0, 0, 0, 0.2);
|
||||
transition: 0.3s;
|
||||
border-radius: 50%;
|
||||
}
|
||||
|
||||
input:checked + .toggle-slider {
|
||||
background-color: rgba(0, 0, 0, 0.3);
|
||||
}
|
||||
|
||||
input:checked + .toggle-slider:before {
|
||||
transform: translateX(20px);
|
||||
}
|
||||
|
||||
.button {
|
||||
padding: 12px 24px;
|
||||
border-radius: 8px;
|
||||
border: none;
|
||||
cursor: pointer;
|
||||
font-size: 14px;
|
||||
font-weight: 600;
|
||||
transition: all 0.2s;
|
||||
font-family: var(--font-hanken-grotesk);
|
||||
width: 100%;
|
||||
margin-top: 24px;
|
||||
-webkit-app-region: no-drag;
|
||||
}
|
||||
|
||||
.button.primary {
|
||||
background: #286df8;
|
||||
color: white;
|
||||
}
|
||||
|
||||
.button.primary:hover {
|
||||
background: #1e5cd6;
|
||||
box-shadow: 0 4px 12px rgba(40, 109, 248, 0.3);
|
||||
}
|
||||
|
||||
.button.primary:disabled {
|
||||
opacity: 0.5;
|
||||
cursor: not-allowed;
|
||||
box-shadow: none;
|
||||
}
|
||||
|
||||
kbd {
|
||||
background: rgba(0, 0, 0, 0.1);
|
||||
border: 1px solid var(--white-10);
|
||||
.shortcut-key {
|
||||
font-family:
|
||||
SF Mono,
|
||||
Monaco,
|
||||
monospace;
|
||||
background: rgba(255, 255, 255, 0.1);
|
||||
padding: 0.25rem 0.5rem;
|
||||
border-radius: 4px;
|
||||
padding: 2px 6px;
|
||||
font-family: monospace;
|
||||
font-weight: 500;
|
||||
color: var(--text-light-05);
|
||||
font-size: 11px;
|
||||
font-size: 0.75rem;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="titlebar"></div>
|
||||
|
||||
<div class="settings-container">
|
||||
<div class="settings-panel">
|
||||
<div class="settings-header">
|
||||
<div class="settings-icon">
|
||||
<svg
|
||||
viewBox="0 0 56 56"
|
||||
xmlns="http://www.w3.org/2000/svg"
|
||||
fill="currentColor"
|
||||
>
|
||||
<path
|
||||
fill-rule="evenodd"
|
||||
clip-rule="evenodd"
|
||||
d="M28 0 10.869 7.77 28 15.539l17.131-7.77L28 0Zm0 40.461-17.131 7.77L28 56l17.131-7.77L28 40.461Zm20.231-29.592L56 28.001l-7.769 17.131L40.462 28l7.769-17.131ZM15.538 28 7.77 10.869 0 28l7.769 17.131L15.538 28Z"
|
||||
/>
|
||||
</svg>
|
||||
</div>
|
||||
<h1 class="settings-title">Settings</h1>
|
||||
<div class="container">
|
||||
<div class="logo">O</div>
|
||||
<h1>Onyx</h1>
|
||||
<p>Connecting to Onyx Cloud...</p>
|
||||
|
||||
<div class="loading">
|
||||
<span></span>
|
||||
<span></span>
|
||||
<span></span>
|
||||
</div>
|
||||
|
||||
<button
|
||||
class="btn"
|
||||
onclick="window.location.href='https://cloud.onyx.app'"
|
||||
>
|
||||
Open Onyx Cloud
|
||||
</button>
|
||||
|
||||
<p style="margin-top: 1.5rem; font-size: 0.875rem; color: #666">
|
||||
Self-hosted? Press
|
||||
<span
|
||||
class="shortcut-key"
|
||||
style="display: inline; padding: 0.15rem 0.4rem"
|
||||
>⌘ ,</span
|
||||
>
|
||||
to configure your server URL.
|
||||
</p>
|
||||
|
||||
<div class="shortcuts">
|
||||
<h3>Keyboard Shortcuts</h3>
|
||||
<div class="shortcut">
|
||||
<span>New Chat</span>
|
||||
<span class="shortcut-key">⌘ N</span>
|
||||
</div>
|
||||
|
||||
<div class="settings-content">
|
||||
<section class="settings-section">
|
||||
<div class="section-title">GENERAL</div>
|
||||
<div class="settings-group">
|
||||
<div class="setting-row">
|
||||
<div class="setting-row-content">
|
||||
<label class="setting-label" for="onyxDomain"
|
||||
>Root Domain</label
|
||||
>
|
||||
<div class="setting-description">
|
||||
The root URL for your Onyx instance
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="setting-divider"></div>
|
||||
<div class="setting-row" style="padding: 12px">
|
||||
<input
|
||||
type="text"
|
||||
id="onyxDomain"
|
||||
class="input-field"
|
||||
placeholder="https://cloud.onyx.app"
|
||||
autocomplete="off"
|
||||
autocorrect="off"
|
||||
autocapitalize="off"
|
||||
spellcheck="false"
|
||||
/>
|
||||
</div>
|
||||
<div class="error-message" id="errorMessage">
|
||||
Please enter a valid URL starting with http:// or https://
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<button class="button primary" id="saveBtn">Save & Connect</button>
|
||||
<div class="shortcut">
|
||||
<span>New Window</span>
|
||||
<span class="shortcut-key">⌘ ⇧ N</span>
|
||||
</div>
|
||||
<div class="shortcut">
|
||||
<span>Reload</span>
|
||||
<span class="shortcut-key">⌘ R</span>
|
||||
</div>
|
||||
<div class="shortcut">
|
||||
<span>Back</span>
|
||||
<span class="shortcut-key">⌘ [</span>
|
||||
</div>
|
||||
<div class="shortcut">
|
||||
<span>Forward</span>
|
||||
<span class="shortcut-key">⌘ ]</span>
|
||||
</div>
|
||||
<div class="shortcut">
|
||||
<span>Settings / Config</span>
|
||||
<span class="shortcut-key">⌘ ,</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
// Import Tauri API
|
||||
const { invoke } = window.__TAURI__.core;
|
||||
|
||||
// Configuration
|
||||
const DEFAULT_DOMAIN = "https://cloud.onyx.app";
|
||||
let currentServerUrl = "";
|
||||
|
||||
// DOM elements
|
||||
const domainInput = document.getElementById("onyxDomain");
|
||||
const errorMessage = document.getElementById("errorMessage");
|
||||
const saveBtn = document.getElementById("saveBtn");
|
||||
|
||||
function showSettings() {
|
||||
document.body.classList.add("show-settings");
|
||||
}
|
||||
|
||||
// Initialize the app
|
||||
async function init() {
|
||||
try {
|
||||
const bootstrap = await invoke("get_bootstrap_state");
|
||||
currentServerUrl = bootstrap.server_url;
|
||||
|
||||
// Set the input value
|
||||
domainInput.value = currentServerUrl || DEFAULT_DOMAIN;
|
||||
|
||||
// Check if user came here explicitly (via Settings menu/shortcut)
|
||||
const urlParams = new URLSearchParams(window.location.search);
|
||||
const isExplicitSettings =
|
||||
window.location.hash === "#settings" ||
|
||||
urlParams.get("settings") === "true";
|
||||
|
||||
// If user explicitly opened settings, show modal
|
||||
if (isExplicitSettings) {
|
||||
// Modal is already visible, user can edit and save
|
||||
showSettings();
|
||||
return;
|
||||
}
|
||||
|
||||
// Otherwise, check if this is first launch
|
||||
// First launch = config doesn't exist
|
||||
if (!bootstrap.config_exists || !currentServerUrl) {
|
||||
// First launch - show modal, require user to configure
|
||||
showSettings();
|
||||
return;
|
||||
}
|
||||
|
||||
// Not first launch and not explicit settings
|
||||
// Auto-redirect to configured domain
|
||||
window.location.href = currentServerUrl;
|
||||
} catch (error) {
|
||||
// On error, default to cloud
|
||||
domainInput.value = DEFAULT_DOMAIN;
|
||||
showSettings();
|
||||
}
|
||||
}
|
||||
|
||||
// Validate URL
|
||||
function validateUrl(url) {
|
||||
const trimmedUrl = url.trim();
|
||||
if (!trimmedUrl) {
|
||||
return { valid: false, error: "URL cannot be empty" };
|
||||
}
|
||||
if (
|
||||
!trimmedUrl.startsWith("http://") &&
|
||||
!trimmedUrl.startsWith("https://")
|
||||
) {
|
||||
return {
|
||||
valid: false,
|
||||
error: "URL must start with http:// or https://",
|
||||
};
|
||||
}
|
||||
try {
|
||||
new URL(trimmedUrl);
|
||||
return { valid: true, url: trimmedUrl };
|
||||
} catch {
|
||||
return { valid: false, error: "Please enter a valid URL" };
|
||||
}
|
||||
}
|
||||
|
||||
// Show error
|
||||
function showError(message) {
|
||||
domainInput.classList.add("error");
|
||||
errorMessage.textContent = message;
|
||||
errorMessage.classList.add("visible");
|
||||
}
|
||||
|
||||
// Clear error
|
||||
function clearError() {
|
||||
domainInput.classList.remove("error");
|
||||
errorMessage.classList.remove("visible");
|
||||
}
|
||||
|
||||
// Save configuration
|
||||
async function saveConfiguration() {
|
||||
clearError();
|
||||
|
||||
const validation = validateUrl(domainInput.value);
|
||||
if (!validation.valid) {
|
||||
showError(validation.error);
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
saveBtn.disabled = true;
|
||||
saveBtn.textContent = "Saving...";
|
||||
|
||||
// Call Tauri command to save the URL
|
||||
await invoke("set_server_url", { url: validation.url });
|
||||
|
||||
// Success - redirect to the new URL (login page)
|
||||
window.location.href = validation.url;
|
||||
} catch (error) {
|
||||
showError(error || "Failed to save configuration");
|
||||
saveBtn.disabled = false;
|
||||
saveBtn.textContent = "Save & Connect";
|
||||
}
|
||||
}
|
||||
|
||||
// Event listeners
|
||||
domainInput.addEventListener("input", clearError);
|
||||
domainInput.addEventListener("keypress", (e) => {
|
||||
if (e.key === "Enter") {
|
||||
saveConfiguration();
|
||||
}
|
||||
});
|
||||
saveBtn.addEventListener("click", saveConfiguration);
|
||||
|
||||
// Initialize when DOM is ready
|
||||
if (document.readyState === "loading") {
|
||||
document.addEventListener("DOMContentLoaded", init);
|
||||
} else {
|
||||
init();
|
||||
}
|
||||
// Auto-redirect to Onyx Cloud after a short delay
|
||||
setTimeout(() => {
|
||||
window.location.href = "https://cloud.onyx.app";
|
||||
}, 1500);
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
|
||||
@@ -2,6 +2,8 @@
|
||||
// This script injects a draggable title bar that matches Onyx design system
|
||||
|
||||
(function () {
|
||||
console.log("[Onyx Desktop] Title bar script loaded");
|
||||
|
||||
const TITLEBAR_ID = "onyx-desktop-titlebar";
|
||||
const TITLEBAR_HEIGHT = 36;
|
||||
const STYLE_ID = "onyx-desktop-titlebar-style";
|
||||
@@ -29,7 +31,12 @@
|
||||
try {
|
||||
await invoke("start_drag_window");
|
||||
return;
|
||||
} catch (err) {}
|
||||
} catch (err) {
|
||||
console.error(
|
||||
"[Onyx Desktop] Failed to start dragging via invoke:",
|
||||
err,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const appWindow =
|
||||
@@ -39,7 +46,14 @@
|
||||
if (appWindow?.startDragging) {
|
||||
try {
|
||||
await appWindow.startDragging();
|
||||
} catch (err) {}
|
||||
} catch (err) {
|
||||
console.error(
|
||||
"[Onyx Desktop] Failed to start dragging via appWindow:",
|
||||
err,
|
||||
);
|
||||
}
|
||||
} else {
|
||||
console.error("[Onyx Desktop] No Tauri drag API available.");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -163,6 +177,7 @@
|
||||
|
||||
function mountTitleBar() {
|
||||
if (!document.body) {
|
||||
console.error("[Onyx Desktop] document.body not found");
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -178,6 +193,7 @@
|
||||
const titleBar = buildTitleBar();
|
||||
document.body.insertBefore(titleBar, document.body.firstChild);
|
||||
injectStyles();
|
||||
console.log("[Onyx Desktop] Title bar injected");
|
||||
}
|
||||
|
||||
function syncViewportHeight() {
|
||||
|
||||
@@ -142,7 +142,7 @@ dev = [
|
||||
"matplotlib==3.10.8",
|
||||
"mypy-extensions==1.0.0",
|
||||
"mypy==1.13.0",
|
||||
"onyx-devtools==0.6.2",
|
||||
"onyx-devtools==0.2.0",
|
||||
"openapi-generator-cli==7.17.0",
|
||||
"pandas-stubs==2.2.3.241009",
|
||||
"pre-commit==3.2.2",
|
||||
|
||||
18
uv.lock
generated
18
uv.lock
generated
@@ -4759,7 +4759,7 @@ requires-dist = [
|
||||
{ name = "numpy", marker = "extra == 'model-server'", specifier = "==2.4.1" },
|
||||
{ name = "oauthlib", marker = "extra == 'backend'", specifier = "==3.2.2" },
|
||||
{ name = "office365-rest-python-client", marker = "extra == 'backend'", specifier = "==2.5.9" },
|
||||
{ name = "onyx-devtools", marker = "extra == 'dev'", specifier = "==0.6.2" },
|
||||
{ name = "onyx-devtools", marker = "extra == 'dev'", specifier = "==0.2.0" },
|
||||
{ name = "openai", specifier = "==2.14.0" },
|
||||
{ name = "openapi-generator-cli", marker = "extra == 'dev'", specifier = "==7.17.0" },
|
||||
{ name = "openinference-instrumentation", marker = "extra == 'backend'", specifier = "==0.1.42" },
|
||||
@@ -4862,20 +4862,20 @@ requires-dist = [{ name = "onyx", extras = ["backend", "dev", "ee"], editable =
|
||||
|
||||
[[package]]
|
||||
name = "onyx-devtools"
|
||||
version = "0.6.2"
|
||||
version = "0.2.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "fastapi" },
|
||||
{ name = "openapi-generator-cli" },
|
||||
]
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/cc/20/d9f6089616044b0fb6e097cbae82122de24f3acd97820be4868d5c28ee3f/onyx_devtools-0.6.2-py3-none-any.whl", hash = "sha256:e48d14695d39d62ec3247a4c76ea56604bc5fb635af84c4ff3e9628bcc67b4fb", size = 3785941, upload-time = "2026-02-25T22:33:43.585Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d6/f5/f754a717f6b011050eb52ef09895cfa2f048f567f4aa3d5e0f773657dea4/onyx_devtools-0.6.2-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:505f9910a04868ab62d99bb483dc37c9f4ad94fa80e6ac0e6a10b86351c31420", size = 3832182, upload-time = "2026-02-25T22:33:43.283Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/6a/35/6e653398c62078e87ebb0d03dc944df6691d92ca427c92867309d2d803b7/onyx_devtools-0.6.2-py3-none-macosx_11_0_arm64.whl", hash = "sha256:edec98e3acc0fa22cf9102c2070409ea7bcf99d7ded72bd8cb184ece8171c36a", size = 3576948, upload-time = "2026-02-25T22:33:42.962Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/3c/97/cff707c5c3d2acd714365b1023f0100676abc99816a29558319e8ef01d5f/onyx_devtools-0.6.2-py3-none-manylinux_2_17_aarch64.whl", hash = "sha256:97abab61216866cdccd8c0a7e27af328776083756ce4fb57c4bd723030449e3b", size = 3439359, upload-time = "2026-02-25T22:33:44.684Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/fc/98/3b768d18e5599178834b966b447075626d224e048d6eb264d89d19abacb4/onyx_devtools-0.6.2-py3-none-manylinux_2_17_x86_64.whl", hash = "sha256:681b038ab6f1457409d14b2490782c7a8014fc0f0f1b9cd69bb2b7199f99aef1", size = 3785959, upload-time = "2026-02-25T22:33:44.342Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d6/38/9b047f9e61c14ccf22b8f386c7a57da3965f90737453f3a577a97da45cdf/onyx_devtools-0.6.2-py3-none-win_amd64.whl", hash = "sha256:a2063be6be104b50a7538cf0d26c7f7ab9159d53327dd6f3e91db05d793c95f3", size = 3878776, upload-time = "2026-02-25T22:33:45.229Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/9d/0f/742f644bae84f5f8f7b500094a2f58da3ff8027fc739944622577e2e2850/onyx_devtools-0.6.2-py3-none-win_arm64.whl", hash = "sha256:00fb90a49a15c932b5cacf818b1b4918e5b5c574bde243dc1828b57690dd5046", size = 3501112, upload-time = "2026-02-25T22:33:41.512Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/27/ab/ba2a77c9b6663573b28a14fb1928ce882ef820a818fb9a7631cc0deccbcf/onyx_devtools-0.2.0-py3-none-any.whl", hash = "sha256:f68d5e8c5f606d8e178d5ea25b952c1ebcc887fceb44804f1a7792f754367a58", size = 1195396, upload-time = "2025-12-16T03:35:22.989Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/03/54/89aa299a4fd4ef4124ef699183c4d5bfbb7877485475d8affc439476f021/onyx_devtools-0.2.0-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:a97cc7b093c86f691cd94668c09ea9f857e9e551f0003dededf6fbaf598d0b40", size = 1194762, upload-time = "2025-12-16T03:35:19.23Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/a1/63/ffbebcacbaaa81859be894cf628f7e73328d1123d482e7a53393cd681520/onyx_devtools-0.2.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:034908351c27f8df5beffdd47ce2ddd4f524099c18ca23893a484ff69feca9d7", size = 1135431, upload-time = "2025-12-16T03:35:18.215Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/9e/79/ad1dba4f9c90057cfcafc143641b4f706410b95d5b0be884ec5d97901b25/onyx_devtools-0.2.0-py3-none-manylinux_2_17_aarch64.whl", hash = "sha256:2b23d21019c8b83905a2fa7c31f781c172e599bbc527bb274a56998981e11661", size = 1103558, upload-time = "2025-12-16T03:35:18.17Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/3c/b0/1da3f0e6e59dedbaeb9775d43976c83dd0068def3b2b71d8545e53f3c932/onyx_devtools-0.2.0-py3-none-manylinux_2_17_x86_64.whl", hash = "sha256:31b51b48ec848f2c4f7c766051d9794141e8759f1d7ce32bb5a0224ac103a66c", size = 1195411, upload-time = "2025-12-16T03:35:18.819Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/b7/6c/523d14b43104b12fc41d38a60d9179cb5ed5c8295b4ec97427ae883f315b/onyx_devtools-0.2.0-py3-none-win_amd64.whl", hash = "sha256:2457c1a7d819b1456eac34b400b8a979af1d41ba91ec4d6a4c4c11051ed23997", size = 1291910, upload-time = "2025-12-16T03:35:19.642Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ec/a9/7b6e632c66d454ba2cac7e797e8dd88ec5cb1bd558df3231931947c0d42d/onyx_devtools-0.2.0-py3-none-win_arm64.whl", hash = "sha256:a9c748e17a1b0925631b9683ad0baef5389f5596e394d86cb55a16f5918487e5", size = 1183025, upload-time = "2025-12-16T03:35:16.586Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 581 B After Width: | Height: | Size: 548 B |
@@ -25,7 +25,7 @@ export default function OnyxApiKeyForm({
|
||||
|
||||
return (
|
||||
<Modal open onOpenChange={onClose}>
|
||||
<Modal.Content width="sm" height="lg">
|
||||
<Modal.Content tall>
|
||||
<Modal.Header
|
||||
icon={SvgKey}
|
||||
title={isUpdate ? "Update API Key" : "Create a new API Key"}
|
||||
|
||||
@@ -105,7 +105,7 @@ function Main() {
|
||||
{popup}
|
||||
|
||||
<Modal open={!!fullApiKey}>
|
||||
<Modal.Content width="sm" height="sm">
|
||||
<Modal.Content small>
|
||||
<Modal.Header
|
||||
title="New API Key"
|
||||
icon={SvgKey}
|
||||
|
||||
@@ -10,7 +10,10 @@ import {
|
||||
} from "@/lib/types";
|
||||
import BackButton from "@/refresh-components/buttons/BackButton";
|
||||
import { InstantSSRAutoRefresh } from "@/components/SSRAutoRefresh";
|
||||
import { FetchAssistantsResponse, fetchAssistantsSS } from "@/lib/agentsSS";
|
||||
import {
|
||||
FetchAssistantsResponse,
|
||||
fetchAssistantsSS,
|
||||
} from "@/lib/assistants/fetchAssistantsSS";
|
||||
import { getStandardAnswerCategoriesIfEE } from "@/components/standardAnswers/getStandardAnswerCategoriesIfEE";
|
||||
|
||||
async function EditslackChannelConfigPage(props: {
|
||||
|
||||
@@ -4,7 +4,7 @@ import { fetchSS } from "@/lib/utilsSS";
|
||||
import { ErrorCallout } from "@/components/ErrorCallout";
|
||||
import { DocumentSetSummary, ValidSources } from "@/lib/types";
|
||||
import BackButton from "@/refresh-components/buttons/BackButton";
|
||||
import { fetchAssistantsSS } from "@/lib/agentsSS";
|
||||
import { fetchAssistantsSS } from "@/lib/assistants/fetchAssistantsSS";
|
||||
import { getStandardAnswerCategoriesIfEE } from "@/components/standardAnswers/getStandardAnswerCategoriesIfEE";
|
||||
import { redirect } from "next/navigation";
|
||||
import { SourceIcon } from "@/components/SourceIcon";
|
||||
|
||||
@@ -1,5 +1,3 @@
|
||||
"use client";
|
||||
|
||||
import { useState, useEffect } from "react";
|
||||
import { Form, Formik, FormikProps } from "formik";
|
||||
import { SelectorFormField, TextFormField } from "@/components/Field";
|
||||
@@ -30,7 +28,13 @@ import { DisplayModels } from "./components/DisplayModels";
|
||||
import { fetchBedrockModels } from "../utils";
|
||||
import Separator from "@/refresh-components/Separator";
|
||||
import Text from "@/refresh-components/texts/Text";
|
||||
import Tabs from "@/refresh-components/Tabs";
|
||||
import {
|
||||
Tabs,
|
||||
TabsList,
|
||||
TabsTrigger,
|
||||
TabsContent,
|
||||
} from "@/refresh-components/tabs/tabs";
|
||||
import { cn } from "@/lib/utils";
|
||||
|
||||
export const BEDROCK_PROVIDER_NAME = "bedrock";
|
||||
const BEDROCK_DISPLAY_NAME = "AWS Bedrock";
|
||||
@@ -157,25 +161,33 @@ function BedrockFormInternals({
|
||||
onValueChange={(value) =>
|
||||
formikProps.setFieldValue(FIELD_BEDROCK_AUTH_METHOD, value)
|
||||
}
|
||||
className="mt-2"
|
||||
>
|
||||
<Tabs.List>
|
||||
<Tabs.Trigger value={AUTH_METHOD_IAM}>IAM Role</Tabs.Trigger>
|
||||
<Tabs.Trigger value={AUTH_METHOD_ACCESS_KEY}>
|
||||
Access Key
|
||||
</Tabs.Trigger>
|
||||
<Tabs.Trigger value={AUTH_METHOD_LONG_TERM_API_KEY}>
|
||||
<TabsList>
|
||||
<TabsTrigger value={AUTH_METHOD_IAM}>IAM Role</TabsTrigger>
|
||||
<TabsTrigger value={AUTH_METHOD_ACCESS_KEY}>Access Key</TabsTrigger>
|
||||
<TabsTrigger value={AUTH_METHOD_LONG_TERM_API_KEY}>
|
||||
Long-term API Key
|
||||
</Tabs.Trigger>
|
||||
</Tabs.List>
|
||||
</TabsTrigger>
|
||||
</TabsList>
|
||||
|
||||
<Tabs.Content value={AUTH_METHOD_IAM}>
|
||||
<TabsContent
|
||||
value={AUTH_METHOD_IAM}
|
||||
className="data-[state=active]:animate-fade-in-scale"
|
||||
>
|
||||
<Text as="p" text03>
|
||||
Uses the IAM role attached to your AWS environment. Recommended
|
||||
for EC2, ECS, Lambda, or other AWS services.
|
||||
</Text>
|
||||
</Tabs.Content>
|
||||
</TabsContent>
|
||||
|
||||
<Tabs.Content value={AUTH_METHOD_ACCESS_KEY}>
|
||||
<TabsContent
|
||||
value={AUTH_METHOD_ACCESS_KEY}
|
||||
className={cn(
|
||||
"data-[state=active]:animate-fade-in-scale",
|
||||
"mt-4 ml-2"
|
||||
)}
|
||||
>
|
||||
<div className="flex flex-col gap-4">
|
||||
<TextFormField
|
||||
name={FIELD_AWS_ACCESS_KEY_ID}
|
||||
@@ -188,9 +200,15 @@ function BedrockFormInternals({
|
||||
placeholder="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
|
||||
/>
|
||||
</div>
|
||||
</Tabs.Content>
|
||||
</TabsContent>
|
||||
|
||||
<Tabs.Content value={AUTH_METHOD_LONG_TERM_API_KEY}>
|
||||
<TabsContent
|
||||
value={AUTH_METHOD_LONG_TERM_API_KEY}
|
||||
className={cn(
|
||||
"data-[state=active]:animate-fade-in-scale",
|
||||
"mt-4 ml-2"
|
||||
)}
|
||||
>
|
||||
<div className="flex flex-col gap-4">
|
||||
<PasswordInputTypeInField
|
||||
name={FIELD_AWS_BEARER_TOKEN_BEDROCK}
|
||||
@@ -198,7 +216,7 @@ function BedrockFormInternals({
|
||||
placeholder="Your long-term API key"
|
||||
/>
|
||||
</div>
|
||||
</Tabs.Content>
|
||||
</TabsContent>
|
||||
</Tabs>
|
||||
</div>
|
||||
|
||||
|
||||
@@ -1,10 +1,9 @@
|
||||
"use client";
|
||||
|
||||
import { useState, ReactNode } from "react";
|
||||
import useSWR, { useSWRConfig, KeyedMutator } from "swr";
|
||||
import { PopupSpec, usePopup } from "@/components/admin/connectors/Popup";
|
||||
import {
|
||||
LLMProviderView,
|
||||
ModelConfiguration,
|
||||
WellKnownLLMProviderDescriptor,
|
||||
} from "../../interfaces";
|
||||
import { errorHandlingFetcher } from "@/lib/fetcher";
|
||||
@@ -115,7 +114,7 @@ export function ProviderFormEntrypointWrapper({
|
||||
|
||||
{formIsVisible && (
|
||||
<Modal open onOpenChange={onClose}>
|
||||
<Modal.Content>
|
||||
<Modal.Content medium>
|
||||
<Modal.Header
|
||||
icon={SvgSettings}
|
||||
title={`Setup ${providerName}`}
|
||||
@@ -197,7 +196,7 @@ export function ProviderFormEntrypointWrapper({
|
||||
|
||||
{formIsVisible && (
|
||||
<Modal open onOpenChange={onClose}>
|
||||
<Modal.Content>
|
||||
<Modal.Content medium>
|
||||
<Modal.Header
|
||||
icon={SvgSettings}
|
||||
title={`${existingLlmProvider ? "Configure" : "Setup"} ${
|
||||
|
||||
@@ -130,7 +130,7 @@ export default function UpgradingPage({
|
||||
{popup}
|
||||
{isCancelling && (
|
||||
<Modal open onOpenChange={() => setIsCancelling(false)}>
|
||||
<Modal.Content width="sm" height="sm">
|
||||
<Modal.Content small>
|
||||
<Modal.Header
|
||||
icon={SvgX}
|
||||
title="Cancel Embedding Model Switch"
|
||||
|
||||
@@ -81,7 +81,7 @@ export const WebProviderSetupModal = memo(
|
||||
|
||||
return (
|
||||
<Modal open={isOpen} onOpenChange={(open) => !open && onClose()}>
|
||||
<Modal.Content width="sm" preventAccidentalClose>
|
||||
<Modal.Content mini preventAccidentalClose>
|
||||
<Modal.Header
|
||||
icon={LogoArrangement}
|
||||
title={`Set up ${providerLabel}`}
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
export type WebContentProviderType =
|
||||
| "firecrawl"
|
||||
| "onyx_web_crawler"
|
||||
| "exa"
|
||||
| (string & {});
|
||||
|
||||
export const CONTENT_PROVIDERS_URL = "/api/admin/web-search/content-providers";
|
||||
@@ -24,13 +23,6 @@ export const CONTENT_PROVIDER_DETAILS: Record<
|
||||
"Connect Firecrawl to fetch and summarize page content from search results.",
|
||||
logoSrc: "/firecrawl.svg",
|
||||
},
|
||||
exa: {
|
||||
label: "Exa",
|
||||
subtitle: "Exa.ai",
|
||||
description:
|
||||
"Use Exa to fetch and summarize page content from search results.",
|
||||
logoSrc: "/Exa.svg",
|
||||
},
|
||||
};
|
||||
|
||||
/**
|
||||
@@ -72,7 +64,6 @@ const CONTENT_PROVIDER_CAPABILITIES: Record<
|
||||
base_url: ["base_url", "api_base_url"],
|
||||
},
|
||||
},
|
||||
// exa uses default capabilities
|
||||
};
|
||||
|
||||
const DEFAULT_CONTENT_PROVIDER_CAPABILITIES: ContentProviderCapabilities = {
|
||||
|
||||
@@ -136,17 +136,6 @@ export default function Page() {
|
||||
|
||||
const isLoading = isLoadingSearchProviders || isLoadingContentProviders;
|
||||
|
||||
// Exa shares API key between search and content providers
|
||||
const exaSearchProvider = searchProviders.find(
|
||||
(p) => p.provider_type === "exa"
|
||||
);
|
||||
const exaContentProvider = contentProviders.find(
|
||||
(p) => p.provider_type === "exa"
|
||||
);
|
||||
const hasSharedExaKey =
|
||||
(exaSearchProvider?.has_api_key || exaContentProvider?.has_api_key) ??
|
||||
false;
|
||||
|
||||
// Modal form state is owned by reducers
|
||||
|
||||
const openSearchModal = (
|
||||
@@ -156,18 +145,12 @@ export default function Page() {
|
||||
const requiresApiKey = searchProviderRequiresApiKey(providerType);
|
||||
const hasStoredKey = provider?.has_api_key ?? false;
|
||||
|
||||
// For Exa search provider, check if we can use the shared Exa key
|
||||
const isExa = providerType === "exa";
|
||||
const canUseSharedExaKey = isExa && hasSharedExaKey && !hasStoredKey;
|
||||
|
||||
dispatchSearchModal({
|
||||
type: "OPEN",
|
||||
providerType,
|
||||
existingProviderId: provider?.id ?? null,
|
||||
initialApiKeyValue:
|
||||
requiresApiKey && (hasStoredKey || canUseSharedExaKey)
|
||||
? MASKED_API_KEY_PLACEHOLDER
|
||||
: "",
|
||||
requiresApiKey && hasStoredKey ? MASKED_API_KEY_PLACEHOLDER : "",
|
||||
initialConfigValue: getSingleConfigFieldValueForForm(
|
||||
providerType,
|
||||
provider
|
||||
@@ -182,16 +165,11 @@ export default function Page() {
|
||||
const hasStoredKey = provider?.has_api_key ?? false;
|
||||
const defaultFirecrawlBaseUrl = "https://api.firecrawl.dev/v1/scrape";
|
||||
|
||||
// For Exa content provider, check if we can use the shared Exa key
|
||||
const isExa = providerType === "exa";
|
||||
const canUseSharedExaKey = isExa && hasSharedExaKey && !hasStoredKey;
|
||||
|
||||
dispatchContentModal({
|
||||
type: "OPEN",
|
||||
providerType,
|
||||
existingProviderId: provider?.id ?? null,
|
||||
initialApiKeyValue:
|
||||
hasStoredKey || canUseSharedExaKey ? MASKED_API_KEY_PLACEHOLDER : "",
|
||||
initialApiKeyValue: hasStoredKey ? MASKED_API_KEY_PLACEHOLDER : "",
|
||||
initialConfigValue:
|
||||
providerType === "firecrawl"
|
||||
? getSingleContentConfigFieldValueForForm(
|
||||
@@ -361,17 +339,6 @@ export default function Page() {
|
||||
} satisfies WebContentProviderView;
|
||||
}
|
||||
|
||||
if (providerType === "exa") {
|
||||
return {
|
||||
id: -3,
|
||||
name: "Exa",
|
||||
provider_type: "exa",
|
||||
is_active: false,
|
||||
config: null,
|
||||
has_api_key: hasSharedExaKey,
|
||||
} satisfies WebContentProviderView;
|
||||
}
|
||||
|
||||
return null;
|
||||
}).filter(Boolean) as WebContentProviderView[];
|
||||
|
||||
@@ -380,7 +347,7 @@ export default function Page() {
|
||||
);
|
||||
|
||||
return [...ordered, ...additional];
|
||||
}, [contentProviders, hasSharedExaKey]);
|
||||
}, [contentProviders]);
|
||||
|
||||
const currentContentProviderType =
|
||||
getCurrentContentProviderType(contentProviders);
|
||||
@@ -501,12 +468,7 @@ export default function Page() {
|
||||
onClose: () => {
|
||||
dispatchSearchModal({ type: "CLOSE" });
|
||||
},
|
||||
mutate: async () => {
|
||||
await mutateSearchProviders();
|
||||
if (selectedProviderType === "exa") {
|
||||
await mutateContentProviders();
|
||||
}
|
||||
},
|
||||
mutate: mutateSearchProviders,
|
||||
});
|
||||
};
|
||||
|
||||
@@ -716,23 +678,6 @@ export default function Page() {
|
||||
selectedContentProviderType
|
||||
: "";
|
||||
|
||||
if (selectedContentProviderType === "exa") {
|
||||
return (
|
||||
<>
|
||||
Paste your{" "}
|
||||
<a
|
||||
href="https://dashboard.exa.ai/api-keys"
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
className="underline"
|
||||
>
|
||||
API key
|
||||
</a>{" "}
|
||||
from Exa to enable crawling.
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
return selectedContentProviderType === "firecrawl" ? (
|
||||
<>
|
||||
Paste your <span className="underline">API key</span> from Firecrawl to
|
||||
@@ -785,10 +730,6 @@ export default function Page() {
|
||||
dispatchContentModal({ type: "SET_PHASE", phase: "saving" });
|
||||
dispatchContentModal({ type: "CLEAR_MESSAGE" });
|
||||
|
||||
const apiKeyChangedForContentProvider =
|
||||
contentModal.apiKeyValue !== MASKED_API_KEY_PLACEHOLDER &&
|
||||
contentProviderValues.apiKey.length > 0;
|
||||
|
||||
await connectProviderFlow({
|
||||
category: "content",
|
||||
providerType: selectedContentProviderType,
|
||||
@@ -799,7 +740,9 @@ export default function Page() {
|
||||
CONTENT_PROVIDER_DETAILS[selectedContentProviderType]?.label ??
|
||||
selectedContentProviderType,
|
||||
providerRequiresApiKey: true,
|
||||
apiKeyChangedForProvider: apiKeyChangedForContentProvider,
|
||||
apiKeyChangedForProvider:
|
||||
contentModal.apiKeyValue !== MASKED_API_KEY_PLACEHOLDER &&
|
||||
contentProviderValues.apiKey.length > 0,
|
||||
apiKey: contentProviderValues.apiKey,
|
||||
config,
|
||||
configChanged,
|
||||
@@ -816,12 +759,7 @@ export default function Page() {
|
||||
onClose: () => {
|
||||
dispatchContentModal({ type: "CLOSE" });
|
||||
},
|
||||
mutate: async () => {
|
||||
await mutateContentProviders();
|
||||
if (selectedContentProviderType === "exa") {
|
||||
await mutateSearchProviders();
|
||||
}
|
||||
},
|
||||
mutate: mutateContentProviders,
|
||||
});
|
||||
};
|
||||
|
||||
@@ -1114,8 +1052,7 @@ export default function Page() {
|
||||
|
||||
const canActivate =
|
||||
providerId > 0 ||
|
||||
provider.provider_type === "onyx_web_crawler" ||
|
||||
isConfigured;
|
||||
provider.provider_type === "onyx_web_crawler";
|
||||
|
||||
return {
|
||||
label: "Set as Default",
|
||||
|
||||
@@ -125,7 +125,7 @@ export default function IndexAttemptErrorsModal({
|
||||
|
||||
return (
|
||||
<Modal open onOpenChange={onClose}>
|
||||
<Modal.Content width="lg" height="full">
|
||||
<Modal.Content large>
|
||||
<Modal.Header
|
||||
icon={SvgAlertTriangle}
|
||||
title="Indexing Errors"
|
||||
|
||||
@@ -353,7 +353,7 @@ export default function InlineFileManagement({
|
||||
|
||||
{/* Confirmation Modal */}
|
||||
<Modal open={showSaveConfirm} onOpenChange={setShowSaveConfirm}>
|
||||
<Modal.Content width="sm">
|
||||
<Modal.Content mini>
|
||||
<Modal.Header
|
||||
icon={SvgFolderPlus}
|
||||
title="Confirm File Changes"
|
||||
|
||||
@@ -128,7 +128,7 @@ export default function ReIndexModal({
|
||||
|
||||
return (
|
||||
<Modal open onOpenChange={hide}>
|
||||
<Modal.Content width="sm" height="sm">
|
||||
<Modal.Content small>
|
||||
<Modal.Header icon={SvgRefreshCw} title="Run Indexing" onClose={hide} />
|
||||
<Modal.Body>
|
||||
<Text as="p">
|
||||
|
||||
@@ -584,7 +584,7 @@ export default function AddConnector({
|
||||
open
|
||||
onOpenChange={() => setCreateCredentialFormToggle(false)}
|
||||
>
|
||||
<Modal.Content>
|
||||
<Modal.Content medium>
|
||||
<Modal.Header
|
||||
icon={SvgKey}
|
||||
title={`Create a ${getSourceDisplayName(
|
||||
|
||||
@@ -9,7 +9,12 @@ import FileInput from "./ConnectorInput/FileInput";
|
||||
import { ConfigurableSources } from "@/lib/types";
|
||||
import { Credential } from "@/lib/connectors/credentials";
|
||||
import CollapsibleSection from "@/app/admin/assistants/CollapsibleSection";
|
||||
import Tabs from "@/refresh-components/Tabs";
|
||||
import {
|
||||
Tabs,
|
||||
TabsContent,
|
||||
TabsList,
|
||||
TabsTrigger,
|
||||
} from "@/components/ui/fully_wrapped_tabs";
|
||||
import { useFormikContext } from "formik";
|
||||
|
||||
// Define a general type for form values
|
||||
@@ -55,6 +60,7 @@ const TabsField: FC<TabsFieldProps> = ({
|
||||
) : (
|
||||
<Tabs
|
||||
defaultValue={tabField.defaultTab || tabField.tabs[0]?.value}
|
||||
className="w-full"
|
||||
onValueChange={(newTab) => {
|
||||
// Clear values from other tabs but preserve defaults
|
||||
tabField.tabs.forEach((tab) => {
|
||||
@@ -69,15 +75,15 @@ const TabsField: FC<TabsFieldProps> = ({
|
||||
});
|
||||
}}
|
||||
>
|
||||
<Tabs.List>
|
||||
<TabsList>
|
||||
{tabField.tabs.map((tab) => (
|
||||
<Tabs.Trigger key={tab.value} value={tab.value}>
|
||||
<TabsTrigger key={tab.value} value={tab.value}>
|
||||
{tab.label}
|
||||
</Tabs.Trigger>
|
||||
</TabsTrigger>
|
||||
))}
|
||||
</Tabs.List>
|
||||
</TabsList>
|
||||
{tabField.tabs.map((tab) => (
|
||||
<Tabs.Content key={tab.value} value={tab.value}>
|
||||
<TabsContent key={tab.value} value={tab.value} className="">
|
||||
{tab.fields.map((subField, index, array) => {
|
||||
// Check visibility condition first
|
||||
if (
|
||||
@@ -106,7 +112,7 @@ const TabsField: FC<TabsFieldProps> = ({
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
</Tabs.Content>
|
||||
</TabsContent>
|
||||
))}
|
||||
</Tabs>
|
||||
)}
|
||||
|
||||
@@ -323,7 +323,7 @@ const RerankingDetailsForm = forwardRef<
|
||||
open
|
||||
onOpenChange={() => setShowGpuWarningModalModel(null)}
|
||||
>
|
||||
<Modal.Content width="sm" height="sm">
|
||||
<Modal.Content small>
|
||||
<Modal.Header
|
||||
icon={SvgAlertTriangle}
|
||||
title="GPU Not Enabled"
|
||||
@@ -358,7 +358,7 @@ const RerankingDetailsForm = forwardRef<
|
||||
setShowLiteLLMConfigurationModal(false);
|
||||
}}
|
||||
>
|
||||
<Modal.Content>
|
||||
<Modal.Content medium>
|
||||
<Modal.Header
|
||||
icon={SvgKey}
|
||||
title="API Key Configuration"
|
||||
@@ -462,7 +462,7 @@ const RerankingDetailsForm = forwardRef<
|
||||
setIsApiKeyModalOpen(false);
|
||||
}}
|
||||
>
|
||||
<Modal.Content>
|
||||
<Modal.Content medium>
|
||||
<Modal.Header
|
||||
icon={SvgKey}
|
||||
title="API Key Configuration"
|
||||
|
||||
@@ -14,7 +14,7 @@ export default function AlreadyPickedModal({
|
||||
}: AlreadyPickedModalProps) {
|
||||
return (
|
||||
<Modal open onOpenChange={onClose}>
|
||||
<Modal.Content width="sm" height="sm">
|
||||
<Modal.Content small>
|
||||
<Modal.Header
|
||||
icon={SvgCheck}
|
||||
title={`${model.model_name} already chosen`}
|
||||
|
||||
@@ -21,7 +21,7 @@ export default function DeleteCredentialsModal({
|
||||
}: DeleteCredentialsModalProps) {
|
||||
return (
|
||||
<Modal open onOpenChange={onCancel}>
|
||||
<Modal.Content width="sm" height="sm">
|
||||
<Modal.Content small>
|
||||
<Modal.Header
|
||||
icon={SvgTrash}
|
||||
title={`Delete ${getFormattedProviderName(
|
||||
|
||||
@@ -13,7 +13,7 @@ export default function InstantSwitchConfirmModal({
|
||||
}: InstantSwitchConfirmModalProps) {
|
||||
return (
|
||||
<Modal open onOpenChange={onClose}>
|
||||
<Modal.Content width="sm" height="sm">
|
||||
<Modal.Content small>
|
||||
<Modal.Header
|
||||
icon={SvgAlertTriangle}
|
||||
title="Are you sure you want to do an instant switch?"
|
||||
|
||||
@@ -20,7 +20,7 @@ export default function ModelSelectionConfirmationModal({
|
||||
}: ModelSelectionConfirmationModalProps) {
|
||||
return (
|
||||
<Modal open onOpenChange={onCancel}>
|
||||
<Modal.Content width="sm" height="lg">
|
||||
<Modal.Content tall>
|
||||
<Modal.Header
|
||||
icon={SvgServer}
|
||||
title="Update Embedding Model"
|
||||
|
||||
@@ -186,7 +186,7 @@ export default function ProviderCreationModal({
|
||||
|
||||
return (
|
||||
<Modal open onOpenChange={onCancel}>
|
||||
<Modal.Content width="sm" height="sm">
|
||||
<Modal.Content small>
|
||||
<Modal.Header
|
||||
icon={SvgSettings}
|
||||
title={`Configure ${getFormattedProviderName(
|
||||
|
||||
@@ -17,7 +17,7 @@ export default function SelectModelModal({
|
||||
}: SelectModelModalProps) {
|
||||
return (
|
||||
<Modal open onOpenChange={onCancel}>
|
||||
<Modal.Content width="sm" height="sm">
|
||||
<Modal.Content small>
|
||||
<Modal.Header
|
||||
icon={SvgServer}
|
||||
title={`Select ${model.model_name}`}
|
||||
|
||||
@@ -539,7 +539,7 @@ export default function EmbeddingForm() {
|
||||
)}
|
||||
{showPoorModel && (
|
||||
<Modal open onOpenChange={() => setShowPoorModel(false)}>
|
||||
<Modal.Content>
|
||||
<Modal.Content medium>
|
||||
<Modal.Header
|
||||
icon={SvgAlertTriangle}
|
||||
title={`Are you sure you want to select ${selectedProvider.model_name}?`}
|
||||
|
||||
@@ -299,7 +299,7 @@ function Main() {
|
||||
)}
|
||||
{configureModalShown && (
|
||||
<Modal open onOpenChange={() => setConfigureModalShown(false)}>
|
||||
<Modal.Content>
|
||||
<Modal.Content medium>
|
||||
<Modal.Header
|
||||
icon={SvgSettings}
|
||||
title="Configure Knowledge Graph"
|
||||
|
||||
@@ -308,7 +308,7 @@ export function SettingsForm() {
|
||||
)}
|
||||
{showConfirmModal && (
|
||||
<Modal open onOpenChange={() => setShowConfirmModal(false)}>
|
||||
<Modal.Content>
|
||||
<Modal.Content medium>
|
||||
<Modal.Header
|
||||
icon={SvgAlertTriangle}
|
||||
title="Enable Anonymous Users"
|
||||
|
||||
@@ -63,7 +63,7 @@ export default function CreateRateLimitModal({
|
||||
|
||||
return (
|
||||
<Modal open={isOpen} onOpenChange={() => setIsOpen(false)}>
|
||||
<Modal.Content width="sm" height="sm">
|
||||
<Modal.Content small>
|
||||
<Modal.Header
|
||||
icon={SvgSettings}
|
||||
title="Create a Token Rate Limit"
|
||||
|
||||
@@ -65,7 +65,7 @@ export const TokenRateLimitTable = ({
|
||||
|
||||
if (tokenRateLimits.length === 0) {
|
||||
return (
|
||||
<div className="w-full">
|
||||
<div>
|
||||
{!hideHeading && title && <Title>{title}</Title>}
|
||||
{!hideHeading && description && (
|
||||
<Text className="my-2">{description}</Text>
|
||||
@@ -78,7 +78,7 @@ export const TokenRateLimitTable = ({
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="w-full">
|
||||
<div>
|
||||
{!hideHeading && title && <Title>{title}</Title>}
|
||||
{!hideHeading && description && (
|
||||
<Text className="my-2">{description}</Text>
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
"use client";
|
||||
|
||||
import { AdminPageTitle } from "@/components/admin/Title";
|
||||
import SimpleTabs from "@/refresh-components/SimpleTabs";
|
||||
import { Tabs, TabsList, TabsTrigger, TabsContent } from "@/components/ui/tabs";
|
||||
import Text from "@/components/ui/text";
|
||||
import { useState } from "react";
|
||||
import {
|
||||
@@ -17,7 +17,6 @@ import CreateRateLimitModal from "./CreateRateLimitModal";
|
||||
import { usePaidEnterpriseFeaturesEnabled } from "@/components/settings/usePaidEnterpriseFeaturesEnabled";
|
||||
import CreateButton from "@/refresh-components/buttons/CreateButton";
|
||||
import { SvgGlobe, SvgShield, SvgUser, SvgUsers } from "@opal/icons";
|
||||
import { Section } from "@/layouts/general-layouts";
|
||||
const BASE_URL = "/api/admin/token-rate-limits";
|
||||
const GLOBAL_TOKEN_FETCH_URL = `${BASE_URL}/global`;
|
||||
const USER_TOKEN_FETCH_URL = `${BASE_URL}/users`;
|
||||
@@ -101,15 +100,15 @@ function Main() {
|
||||
};
|
||||
|
||||
return (
|
||||
<Section alignItems="stretch">
|
||||
<div>
|
||||
{popup}
|
||||
|
||||
<Text>
|
||||
<Text className="mb-2">
|
||||
Token rate limits enable you control how many tokens can be spent in a
|
||||
given time period. With token rate limits, you can:
|
||||
</Text>
|
||||
|
||||
<ul className="list-disc ml-4">
|
||||
<ul className="list-disc mt-2 ml-4 mb-2">
|
||||
<li>
|
||||
<Text>
|
||||
Set a global rate limit to control your team's overall token
|
||||
@@ -140,61 +139,75 @@ function Main() {
|
||||
<CreateButton onClick={() => setModalIsOpen(true)}>
|
||||
Create a Token Rate Limit
|
||||
</CreateButton>
|
||||
|
||||
{isPaidEnterpriseFeaturesEnabled ? (
|
||||
<SimpleTabs
|
||||
tabs={{
|
||||
"0": {
|
||||
name: "Global",
|
||||
icon: SvgGlobe,
|
||||
content: (
|
||||
<GenericTokenRateLimitTable
|
||||
fetchUrl={GLOBAL_TOKEN_FETCH_URL}
|
||||
title={"Global Token Rate Limits"}
|
||||
description={GLOBAL_DESCRIPTION}
|
||||
/>
|
||||
),
|
||||
},
|
||||
"1": {
|
||||
name: "User",
|
||||
icon: SvgUser,
|
||||
content: (
|
||||
<GenericTokenRateLimitTable
|
||||
fetchUrl={USER_TOKEN_FETCH_URL}
|
||||
title={"User Token Rate Limits"}
|
||||
description={USER_DESCRIPTION}
|
||||
/>
|
||||
),
|
||||
},
|
||||
"2": {
|
||||
name: "User Groups",
|
||||
icon: SvgUsers,
|
||||
content: (
|
||||
<GenericTokenRateLimitTable
|
||||
fetchUrl={USER_GROUP_FETCH_URL}
|
||||
title={"User Group Token Rate Limits"}
|
||||
description={USER_GROUP_DESCRIPTION}
|
||||
responseMapper={(data: Record<string, TokenRateLimit[]>) =>
|
||||
Object.entries(data).flatMap(([group_name, elements]) =>
|
||||
elements.map((element) => ({
|
||||
...element,
|
||||
group_name,
|
||||
}))
|
||||
)
|
||||
}
|
||||
/>
|
||||
),
|
||||
},
|
||||
}}
|
||||
{isPaidEnterpriseFeaturesEnabled && (
|
||||
<Tabs
|
||||
className="mt-2"
|
||||
value={tabIndex.toString()}
|
||||
onValueChange={(val) => setTabIndex(parseInt(val))}
|
||||
/>
|
||||
) : (
|
||||
<GenericTokenRateLimitTable
|
||||
fetchUrl={GLOBAL_TOKEN_FETCH_URL}
|
||||
title={"Global Token Rate Limits"}
|
||||
description={GLOBAL_DESCRIPTION}
|
||||
/>
|
||||
>
|
||||
<TabsList>
|
||||
<TabsTrigger value="0" className="flex items-center gap-2">
|
||||
<SvgGlobe
|
||||
aria-hidden="true"
|
||||
className="h-3.5 w-3.5 stroke-text-03 group-data-[state=active]:stroke-text-04 shrink-0"
|
||||
/>
|
||||
Global
|
||||
</TabsTrigger>
|
||||
<TabsTrigger value="1" className="flex items-center gap-2">
|
||||
<SvgUser
|
||||
aria-hidden="true"
|
||||
className="h-3.5 w-3.5 stroke-text-03 group-data-[state=active]:stroke-text-04 shrink-0"
|
||||
/>
|
||||
User
|
||||
</TabsTrigger>
|
||||
<TabsTrigger value="2" className="flex items-center gap-2">
|
||||
<SvgUsers
|
||||
aria-hidden="true"
|
||||
className="h-3.5 w-3.5 stroke-text-03 group-data-[state=active]:stroke-text-04 shrink-0"
|
||||
/>
|
||||
User Groups
|
||||
</TabsTrigger>
|
||||
</TabsList>
|
||||
<TabsContent value="0">
|
||||
<GenericTokenRateLimitTable
|
||||
fetchUrl={GLOBAL_TOKEN_FETCH_URL}
|
||||
title={"Global Token Rate Limits"}
|
||||
description={GLOBAL_DESCRIPTION}
|
||||
/>
|
||||
</TabsContent>
|
||||
<TabsContent value="1">
|
||||
<GenericTokenRateLimitTable
|
||||
fetchUrl={USER_TOKEN_FETCH_URL}
|
||||
title={"User Token Rate Limits"}
|
||||
description={USER_DESCRIPTION}
|
||||
/>
|
||||
</TabsContent>
|
||||
<TabsContent value="2">
|
||||
<GenericTokenRateLimitTable
|
||||
fetchUrl={USER_GROUP_FETCH_URL}
|
||||
title={"User Group Token Rate Limits"}
|
||||
description={USER_GROUP_DESCRIPTION}
|
||||
responseMapper={(data: Record<string, TokenRateLimit[]>) =>
|
||||
Object.entries(data).flatMap(([group_name, elements]) =>
|
||||
elements.map((element) => ({
|
||||
...element,
|
||||
group_name,
|
||||
}))
|
||||
)
|
||||
}
|
||||
/>
|
||||
</TabsContent>
|
||||
</Tabs>
|
||||
)}
|
||||
|
||||
{!isPaidEnterpriseFeaturesEnabled && (
|
||||
<div className="mt-6">
|
||||
<GenericTokenRateLimitTable
|
||||
fetchUrl={GLOBAL_TOKEN_FETCH_URL}
|
||||
title={"Global Token Rate Limits"}
|
||||
description={GLOBAL_DESCRIPTION}
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
|
||||
<CreateRateLimitModal
|
||||
@@ -206,7 +219,7 @@ function Main() {
|
||||
isPaidEnterpriseFeaturesEnabled ? undefined : Scope.GLOBAL
|
||||
}
|
||||
/>
|
||||
</Section>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
"use client";
|
||||
|
||||
import { useState } from "react";
|
||||
import SimpleTabs from "@/refresh-components/SimpleTabs";
|
||||
import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
|
||||
import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card";
|
||||
import InvitedUserTable from "@/components/admin/users/InvitedUserTable";
|
||||
import SignedUpUserTable from "@/components/admin/users/SignedUpUserTable";
|
||||
@@ -145,11 +145,18 @@ const UsersTables = ({
|
||||
);
|
||||
}
|
||||
|
||||
const tabs = SimpleTabs.generateTabs({
|
||||
current: {
|
||||
name: "Current Users",
|
||||
content: (
|
||||
<Card className="w-full">
|
||||
return (
|
||||
<Tabs defaultValue="current">
|
||||
<TabsList>
|
||||
<TabsTrigger value="current">Current Users</TabsTrigger>
|
||||
<TabsTrigger value="invited">Invited Users</TabsTrigger>
|
||||
{NEXT_PUBLIC_CLOUD_ENABLED && (
|
||||
<TabsTrigger value="pending">Pending Users</TabsTrigger>
|
||||
)}
|
||||
</TabsList>
|
||||
|
||||
<TabsContent value="current">
|
||||
<Card>
|
||||
<CardHeader>
|
||||
<div className="flex justify-between items-center gap-1">
|
||||
<CardTitle>Current Users</CardTitle>
|
||||
@@ -185,12 +192,9 @@ const UsersTables = ({
|
||||
/>
|
||||
</CardContent>
|
||||
</Card>
|
||||
),
|
||||
},
|
||||
invited: {
|
||||
name: "Invited Users",
|
||||
content: (
|
||||
<Card className="w-full">
|
||||
</TabsContent>
|
||||
<TabsContent value="invited">
|
||||
<Card>
|
||||
<CardHeader>
|
||||
<div className="flex justify-between items-center gap-1">
|
||||
<CardTitle>Invited Users</CardTitle>
|
||||
@@ -212,12 +216,9 @@ const UsersTables = ({
|
||||
/>
|
||||
</CardContent>
|
||||
</Card>
|
||||
),
|
||||
},
|
||||
...(NEXT_PUBLIC_CLOUD_ENABLED && {
|
||||
pending: {
|
||||
name: "Pending Users",
|
||||
content: (
|
||||
</TabsContent>
|
||||
{NEXT_PUBLIC_CLOUD_ENABLED && (
|
||||
<TabsContent value="pending">
|
||||
<Card>
|
||||
<CardHeader>
|
||||
<div className="flex justify-between items-center gap-1">
|
||||
@@ -240,12 +241,10 @@ const UsersTables = ({
|
||||
/>
|
||||
</CardContent>
|
||||
</Card>
|
||||
),
|
||||
},
|
||||
}),
|
||||
});
|
||||
|
||||
return <SimpleTabs tabs={tabs} defaultValue="current" />;
|
||||
</TabsContent>
|
||||
)}
|
||||
</Tabs>
|
||||
);
|
||||
};
|
||||
|
||||
const SearchableTables = () => {
|
||||
@@ -351,7 +350,7 @@ const AddUserButton = ({
|
||||
|
||||
{bulkAddUsersModal && (
|
||||
<Modal open onOpenChange={() => setBulkAddUsersModal(false)}>
|
||||
<Modal.Content>
|
||||
<Modal.Content medium>
|
||||
<Modal.Header
|
||||
icon={SvgUserPlus}
|
||||
title="Bulk Add Users"
|
||||
|
||||
@@ -1,323 +0,0 @@
|
||||
"use client";
|
||||
|
||||
import { cn, noProp } from "@/lib/utils";
|
||||
import Text from "@/refresh-components/texts/Text";
|
||||
import Button from "@/refresh-components/buttons/Button";
|
||||
import { useCallback, useMemo, useState, useEffect } from "react";
|
||||
import ShareChatSessionModal from "@/app/chat/components/modal/ShareChatSessionModal";
|
||||
import IconButton from "@/refresh-components/buttons/IconButton";
|
||||
import LineItem from "@/refresh-components/buttons/LineItem";
|
||||
import { useProjectsContext } from "@/app/chat/projects/ProjectsContext";
|
||||
import useChatSessions from "@/hooks/useChatSessions";
|
||||
import { usePopup } from "@/components/admin/connectors/Popup";
|
||||
import {
|
||||
handleMoveOperation,
|
||||
shouldShowMoveModal,
|
||||
showErrorNotification,
|
||||
} from "@/sections/sidebar/sidebarUtils";
|
||||
import { LOCAL_STORAGE_KEYS } from "@/sections/sidebar/constants";
|
||||
import { deleteChatSession } from "@/app/chat/services/lib";
|
||||
import { useRouter } from "next/navigation";
|
||||
import MoveCustomAgentChatModal from "@/components/modals/MoveCustomAgentChatModal";
|
||||
import ConfirmationModalLayout from "@/refresh-components/layouts/ConfirmationModalLayout";
|
||||
import { PopoverMenu } from "@/refresh-components/Popover";
|
||||
import { PopoverSearchInput } from "@/sections/sidebar/ChatButton";
|
||||
import SimplePopover from "@/refresh-components/SimplePopover";
|
||||
import { useAppSidebarContext } from "@/refresh-components/contexts/AppSidebarContext";
|
||||
import useScreenSize from "@/hooks/useScreenSize";
|
||||
import {
|
||||
SvgFolderIn,
|
||||
SvgMoreHorizontal,
|
||||
SvgShare,
|
||||
SvgSidebar,
|
||||
SvgTrash,
|
||||
} from "@opal/icons";
|
||||
import { useSettingsContext } from "@/components/settings/SettingsProvider";
|
||||
|
||||
/**
|
||||
* Chat Header Component
|
||||
*
|
||||
* Renders the header for chat sessions with share, move, and delete actions.
|
||||
* Designed to be rendered inside ChatScrollContainer with sticky positioning.
|
||||
*
|
||||
* Features:
|
||||
* - Share chat functionality
|
||||
* - Move chat to project (with confirmation for custom agents)
|
||||
* - Delete chat with confirmation
|
||||
* - Mobile-responsive sidebar toggle
|
||||
* - Custom header content from enterprise settings
|
||||
*/
|
||||
export default function ChatHeader() {
|
||||
const settings = useSettingsContext();
|
||||
const { isMobile } = useScreenSize();
|
||||
const { setFolded } = useAppSidebarContext();
|
||||
const [showShareModal, setShowShareModal] = useState(false);
|
||||
const [deleteModalOpen, setDeleteModalOpen] = useState(false);
|
||||
const [showMoveCustomAgentModal, setShowMoveCustomAgentModal] =
|
||||
useState(false);
|
||||
const [pendingMoveProjectId, setPendingMoveProjectId] = useState<
|
||||
number | null
|
||||
>(null);
|
||||
const [showMoveOptions, setShowMoveOptions] = useState(false);
|
||||
const [searchTerm, setSearchTerm] = useState("");
|
||||
const [popoverOpen, setPopoverOpen] = useState(false);
|
||||
const [popoverItems, setPopoverItems] = useState<React.ReactNode[]>([]);
|
||||
const {
|
||||
projects,
|
||||
fetchProjects,
|
||||
refreshCurrentProjectDetails,
|
||||
currentProjectId,
|
||||
} = useProjectsContext();
|
||||
const { currentChatSession, refreshChatSessions, currentChatSessionId } =
|
||||
useChatSessions();
|
||||
const { popup, setPopup } = usePopup();
|
||||
const router = useRouter();
|
||||
|
||||
const customHeaderContent =
|
||||
settings?.enterpriseSettings?.custom_header_content;
|
||||
|
||||
const availableProjects = useMemo(() => {
|
||||
if (!projects) return [];
|
||||
return projects.filter((project) => project.id !== currentProjectId);
|
||||
}, [projects, currentProjectId]);
|
||||
|
||||
const filteredProjects = useMemo(() => {
|
||||
if (!searchTerm) return availableProjects;
|
||||
const term = searchTerm.toLowerCase();
|
||||
return availableProjects.filter((project) =>
|
||||
project.name.toLowerCase().includes(term)
|
||||
);
|
||||
}, [availableProjects, searchTerm]);
|
||||
|
||||
const resetMoveState = useCallback(() => {
|
||||
setShowMoveOptions(false);
|
||||
setSearchTerm("");
|
||||
setPendingMoveProjectId(null);
|
||||
setShowMoveCustomAgentModal(false);
|
||||
}, []);
|
||||
|
||||
const performMove = useCallback(
|
||||
async (targetProjectId: number) => {
|
||||
if (!currentChatSession) return;
|
||||
try {
|
||||
await handleMoveOperation(
|
||||
{
|
||||
chatSession: currentChatSession,
|
||||
targetProjectId,
|
||||
refreshChatSessions,
|
||||
refreshCurrentProjectDetails,
|
||||
fetchProjects,
|
||||
currentProjectId,
|
||||
},
|
||||
setPopup
|
||||
);
|
||||
resetMoveState();
|
||||
setPopoverOpen(false);
|
||||
} catch (error) {
|
||||
console.error("Failed to move chat session:", error);
|
||||
}
|
||||
},
|
||||
[
|
||||
currentChatSession,
|
||||
refreshChatSessions,
|
||||
refreshCurrentProjectDetails,
|
||||
fetchProjects,
|
||||
currentProjectId,
|
||||
setPopup,
|
||||
resetMoveState,
|
||||
]
|
||||
);
|
||||
|
||||
const handleMoveClick = useCallback(
|
||||
(projectId: number) => {
|
||||
if (!currentChatSession) return;
|
||||
if (shouldShowMoveModal(currentChatSession)) {
|
||||
setPendingMoveProjectId(projectId);
|
||||
setShowMoveCustomAgentModal(true);
|
||||
return;
|
||||
}
|
||||
void performMove(projectId);
|
||||
},
|
||||
[currentChatSession, performMove]
|
||||
);
|
||||
|
||||
const handleDeleteChat = useCallback(async () => {
|
||||
if (!currentChatSession) return;
|
||||
try {
|
||||
const response = await deleteChatSession(currentChatSession.id);
|
||||
if (!response.ok) {
|
||||
throw new Error("Failed to delete chat session");
|
||||
}
|
||||
await Promise.all([refreshChatSessions(), fetchProjects()]);
|
||||
router.replace("/chat");
|
||||
setDeleteModalOpen(false);
|
||||
} catch (error) {
|
||||
console.error("Failed to delete chat:", error);
|
||||
showErrorNotification(
|
||||
setPopup,
|
||||
"Failed to delete chat. Please try again."
|
||||
);
|
||||
}
|
||||
}, [
|
||||
currentChatSession,
|
||||
refreshChatSessions,
|
||||
fetchProjects,
|
||||
router,
|
||||
setPopup,
|
||||
]);
|
||||
|
||||
const setDeleteConfirmationModalOpen = useCallback((open: boolean) => {
|
||||
setDeleteModalOpen(open);
|
||||
if (open) {
|
||||
setPopoverOpen(false);
|
||||
}
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
const items = showMoveOptions
|
||||
? [
|
||||
<PopoverSearchInput
|
||||
key="search"
|
||||
setShowMoveOptions={setShowMoveOptions}
|
||||
onSearch={setSearchTerm}
|
||||
/>,
|
||||
...filteredProjects.map((project) => (
|
||||
<LineItem
|
||||
key={project.id}
|
||||
icon={SvgFolderIn}
|
||||
onClick={noProp(() => handleMoveClick(project.id))}
|
||||
>
|
||||
{project.name}
|
||||
</LineItem>
|
||||
)),
|
||||
]
|
||||
: [
|
||||
<LineItem
|
||||
key="move"
|
||||
icon={SvgFolderIn}
|
||||
onClick={noProp(() => setShowMoveOptions(true))}
|
||||
>
|
||||
Move to Project
|
||||
</LineItem>,
|
||||
<LineItem
|
||||
key="delete"
|
||||
icon={SvgTrash}
|
||||
onClick={noProp(() => setDeleteConfirmationModalOpen(true))}
|
||||
danger
|
||||
>
|
||||
Delete
|
||||
</LineItem>,
|
||||
];
|
||||
|
||||
setPopoverItems(items);
|
||||
}, [
|
||||
showMoveOptions,
|
||||
filteredProjects,
|
||||
currentChatSession,
|
||||
setDeleteConfirmationModalOpen,
|
||||
handleMoveClick,
|
||||
]);
|
||||
|
||||
// Don't render if no chat session
|
||||
if (!currentChatSessionId) return null;
|
||||
|
||||
return (
|
||||
<>
|
||||
{popup}
|
||||
|
||||
{showShareModal && currentChatSession && (
|
||||
<ShareChatSessionModal
|
||||
chatSession={currentChatSession}
|
||||
onClose={() => setShowShareModal(false)}
|
||||
/>
|
||||
)}
|
||||
|
||||
{showMoveCustomAgentModal && (
|
||||
<MoveCustomAgentChatModal
|
||||
onCancel={resetMoveState}
|
||||
onConfirm={async (doNotShowAgain: boolean) => {
|
||||
if (doNotShowAgain && typeof window !== "undefined") {
|
||||
window.localStorage.setItem(
|
||||
LOCAL_STORAGE_KEYS.HIDE_MOVE_CUSTOM_AGENT_MODAL,
|
||||
"true"
|
||||
);
|
||||
}
|
||||
if (pendingMoveProjectId != null) {
|
||||
await performMove(pendingMoveProjectId);
|
||||
}
|
||||
}}
|
||||
/>
|
||||
)}
|
||||
|
||||
{deleteModalOpen && (
|
||||
<ConfirmationModalLayout
|
||||
title="Delete Chat"
|
||||
icon={SvgTrash}
|
||||
onClose={() => setDeleteModalOpen(false)}
|
||||
submit={
|
||||
<Button danger onClick={handleDeleteChat}>
|
||||
Delete
|
||||
</Button>
|
||||
}
|
||||
>
|
||||
Are you sure you want to delete this chat? This action cannot be
|
||||
undone.
|
||||
</ConfirmationModalLayout>
|
||||
)}
|
||||
|
||||
<div className="w-full flex flex-row justify-center items-center py-3 px-4 h-16 bg-background-tint-01 xl:bg-transparent">
|
||||
{/* Left - contains the icon-button to fold the AppSidebar on mobile */}
|
||||
<div className="flex-1">
|
||||
<IconButton
|
||||
icon={SvgSidebar}
|
||||
onClick={() => setFolded(false)}
|
||||
className={cn(!isMobile && "invisible")}
|
||||
internal
|
||||
/>
|
||||
</div>
|
||||
|
||||
{/* Center - contains the custom-header-content */}
|
||||
<div className="flex-1 flex flex-col items-center overflow-hidden">
|
||||
<Text
|
||||
as="p"
|
||||
text03
|
||||
mainUiBody
|
||||
className="text-center break-words w-full"
|
||||
>
|
||||
{customHeaderContent}
|
||||
</Text>
|
||||
</div>
|
||||
|
||||
{/* Right - contains the share and more-options buttons */}
|
||||
<div className="flex-1 flex flex-row items-center justify-end px-1">
|
||||
<Button
|
||||
leftIcon={SvgShare}
|
||||
transient={showShareModal}
|
||||
tertiary
|
||||
onClick={() => setShowShareModal(true)}
|
||||
>
|
||||
Share Chat
|
||||
</Button>
|
||||
<SimplePopover
|
||||
trigger={
|
||||
<IconButton
|
||||
icon={SvgMoreHorizontal}
|
||||
className="ml-2"
|
||||
transient={popoverOpen}
|
||||
tertiary
|
||||
/>
|
||||
}
|
||||
onOpenChange={(state) => {
|
||||
setPopoverOpen(state);
|
||||
if (!state) setShowMoveOptions(false);
|
||||
}}
|
||||
side="bottom"
|
||||
align="end"
|
||||
>
|
||||
<PopoverMenu>{popoverItems}</PopoverMenu>
|
||||
</SimplePopover>
|
||||
</div>
|
||||
</div>
|
||||
</>
|
||||
);
|
||||
}
|
||||
@@ -25,6 +25,7 @@ import { useDocumentSets } from "@/lib/hooks/useDocumentSets";
|
||||
import { useAgents } from "@/hooks/useAgents";
|
||||
import { ChatPopup } from "@/app/chat/components/ChatPopup";
|
||||
import ExceptionTraceModal from "@/components/modals/ExceptionTraceModal";
|
||||
import { SEARCH_TOOL_ID } from "@/app/chat/components/tools/constants";
|
||||
import { useUser } from "@/components/user/UserProvider";
|
||||
import NoAssistantModal from "@/components/modals/NoAssistantModal";
|
||||
import TextView from "@/components/chat/TextView";
|
||||
@@ -50,10 +51,7 @@ import {
|
||||
useDocumentSidebarVisible,
|
||||
} from "@/app/chat/stores/useChatSessionStore";
|
||||
import FederatedOAuthModal from "@/components/chat/FederatedOAuthModal";
|
||||
import ChatScrollContainer, {
|
||||
ChatScrollContainerHandle,
|
||||
} from "@/components/chat/ChatScrollContainer";
|
||||
import MessageList from "@/components/chat/MessageList";
|
||||
import ChatUI, { ChatUIHandle } from "@/sections/ChatUI";
|
||||
import WelcomeMessage from "@/app/chat/components/WelcomeMessage";
|
||||
import ProjectContextPanel from "@/app/chat/components/projects/ProjectContextPanel";
|
||||
import { useProjectsContext } from "@/app/chat/projects/ProjectsContext";
|
||||
@@ -68,9 +66,7 @@ import OnboardingFlow from "@/refresh-components/onboarding/OnboardingFlow";
|
||||
import { OnboardingStep } from "@/refresh-components/onboarding/types";
|
||||
import { useShowOnboarding } from "@/hooks/useShowOnboarding";
|
||||
import * as AppLayouts from "@/layouts/app-layouts";
|
||||
import { SvgChevronDown, SvgFileText } from "@opal/icons";
|
||||
import ChatHeader from "@/app/chat/components/ChatHeader";
|
||||
import IconButton from "@/refresh-components/buttons/IconButton";
|
||||
import { SvgFileText } from "@opal/icons";
|
||||
import Spacer from "@/refresh-components/Spacer";
|
||||
import { DEFAULT_CONTEXT_TOKENS } from "@/lib/constants";
|
||||
|
||||
@@ -271,17 +267,18 @@ export default function ChatPage({ firstMessage }: ChatPageProps) {
|
||||
settings,
|
||||
});
|
||||
|
||||
const scrollContainerRef = useRef<ChatScrollContainerHandle>(null);
|
||||
const [showScrollButton, setShowScrollButton] = useState(false);
|
||||
const chatUiRef = useRef<ChatUIHandle>(null);
|
||||
const autoScrollEnabled = user?.preferences?.auto_scroll ?? false;
|
||||
|
||||
// Reset scroll button when session changes
|
||||
useEffect(() => {
|
||||
setShowScrollButton(false);
|
||||
}, [currentChatSessionId]);
|
||||
|
||||
const handleScrollToBottom = useCallback(() => {
|
||||
scrollContainerRef.current?.scrollToBottom();
|
||||
}, []);
|
||||
// Handle input bar height changes for scroll adjustment
|
||||
const handleInputHeightChange = useCallback(
|
||||
(delta: number) => {
|
||||
if (autoScrollEnabled && delta > 0) {
|
||||
chatUiRef.current?.scrollBy(delta);
|
||||
}
|
||||
},
|
||||
[autoScrollEnabled]
|
||||
);
|
||||
|
||||
const resetInputBar = useCallback(() => {
|
||||
chatInputBarRef.current?.reset();
|
||||
@@ -332,15 +329,6 @@ export default function ChatPage({ firstMessage }: ChatPageProps) {
|
||||
);
|
||||
const messageHistory = useCurrentMessageHistory();
|
||||
|
||||
// Determine anchor: second-to-last message (last user message before current response)
|
||||
const anchorMessage = messageHistory.at(-2) ?? messageHistory[0];
|
||||
const anchorNodeId = anchorMessage?.nodeId;
|
||||
const anchorSelector = anchorNodeId ? `#message-${anchorNodeId}` : undefined;
|
||||
|
||||
// Auto-scroll preference from user settings
|
||||
const autoScrollEnabled = user?.preferences?.auto_scroll !== false;
|
||||
const isStreaming = currentChatState === "streaming";
|
||||
|
||||
const { onSubmit, stopGenerating, handleMessageSpecificFileUpload } =
|
||||
useChatController({
|
||||
filterManager,
|
||||
@@ -377,7 +365,9 @@ export default function ChatPage({ firstMessage }: ChatPageProps) {
|
||||
|
||||
const retrievalEnabled = useMemo(() => {
|
||||
if (liveAssistant) {
|
||||
return personaIncludesRetrieval(liveAssistant);
|
||||
return liveAssistant.tools.some(
|
||||
(tool) => tool.in_code_tool_id === SEARCH_TOOL_ID
|
||||
);
|
||||
}
|
||||
return false;
|
||||
}, [liveAssistant]);
|
||||
@@ -590,7 +580,7 @@ export default function ChatPage({ firstMessage }: ChatPageProps) {
|
||||
open
|
||||
onOpenChange={() => updateCurrentDocumentSidebarVisible(false)}
|
||||
>
|
||||
<Modal.Content>
|
||||
<Modal.Content medium>
|
||||
<Modal.Header
|
||||
icon={SvgFileText}
|
||||
title="Sources"
|
||||
@@ -637,7 +627,7 @@ export default function ChatPage({ firstMessage }: ChatPageProps) {
|
||||
>
|
||||
{({ getRootProps }) => (
|
||||
<div
|
||||
className="h-full w-full flex flex-col items-center outline-none relative"
|
||||
className="h-full w-full flex flex-col items-center outline-none"
|
||||
{...getRootProps({ tabIndex: -1 })}
|
||||
>
|
||||
{/* ProjectUI */}
|
||||
@@ -650,31 +640,19 @@ export default function ChatPage({ firstMessage }: ChatPageProps) {
|
||||
)}
|
||||
|
||||
{/* ChatUI */}
|
||||
{!!currentChatSessionId && liveAssistant && (
|
||||
<ChatScrollContainer
|
||||
ref={scrollContainerRef}
|
||||
sessionId={currentChatSessionId}
|
||||
anchorSelector={anchorSelector}
|
||||
autoScroll={autoScrollEnabled}
|
||||
isStreaming={isStreaming}
|
||||
onScrollButtonVisibilityChange={setShowScrollButton}
|
||||
>
|
||||
<AppLayouts.StickyHeader>
|
||||
<ChatHeader />
|
||||
</AppLayouts.StickyHeader>
|
||||
<MessageList
|
||||
liveAssistant={liveAssistant}
|
||||
llmManager={llmManager}
|
||||
deepResearchEnabled={deepResearchEnabled}
|
||||
currentMessageFiles={currentMessageFiles}
|
||||
setPresentingDocument={setPresentingDocument}
|
||||
onSubmit={onSubmit}
|
||||
onMessageSelection={onMessageSelection}
|
||||
stopGenerating={stopGenerating}
|
||||
onResubmit={handleResubmitLastMessage}
|
||||
anchorNodeId={anchorNodeId}
|
||||
/>
|
||||
</ChatScrollContainer>
|
||||
{!!currentChatSessionId && (
|
||||
<ChatUI
|
||||
ref={chatUiRef}
|
||||
liveAssistant={liveAssistant}
|
||||
llmManager={llmManager}
|
||||
deepResearchEnabled={deepResearchEnabled}
|
||||
currentMessageFiles={currentMessageFiles}
|
||||
setPresentingDocument={setPresentingDocument}
|
||||
onSubmit={onSubmit}
|
||||
onMessageSelection={onMessageSelection}
|
||||
stopGenerating={stopGenerating}
|
||||
handleResubmitLastMessage={handleResubmitLastMessage}
|
||||
/>
|
||||
)}
|
||||
|
||||
{!currentChatSessionId && !currentProjectId && (
|
||||
@@ -687,82 +665,58 @@ export default function ChatPage({ firstMessage }: ChatPageProps) {
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* ChatInputBar container - absolutely positioned when in chat, centered when no session */}
|
||||
<div
|
||||
className={cn(
|
||||
"flex justify-center",
|
||||
currentChatSessionId
|
||||
? "absolute bottom-0 left-0 right-0 pointer-events-none"
|
||||
: "w-full"
|
||||
)}
|
||||
>
|
||||
<div
|
||||
className={cn(
|
||||
"w-[min(50rem,100%)] z-sticky flex flex-col px-4",
|
||||
currentChatSessionId && "pointer-events-auto"
|
||||
)}
|
||||
>
|
||||
{/* Scroll to bottom button - positioned above ChatInputBar */}
|
||||
{showScrollButton && (
|
||||
<div className="mb-2 self-center">
|
||||
<IconButton
|
||||
icon={SvgChevronDown}
|
||||
onClick={handleScrollToBottom}
|
||||
aria-label="Scroll to bottom"
|
||||
/>
|
||||
</div>
|
||||
{/* ChatInputBar container */}
|
||||
<div className="w-[min(50rem,100%)] pointer-events-auto z-sticky flex flex-col px-4 justify-center items-center">
|
||||
{(showOnboarding ||
|
||||
(user?.role !== UserRole.ADMIN &&
|
||||
!user?.personalization?.name)) &&
|
||||
currentProjectId === null && (
|
||||
<OnboardingFlow
|
||||
handleHideOnboarding={hideOnboarding}
|
||||
handleFinishOnboarding={finishOnboarding}
|
||||
state={onboardingState}
|
||||
actions={onboardingActions}
|
||||
llmDescriptors={llmDescriptors}
|
||||
/>
|
||||
)}
|
||||
|
||||
{(showOnboarding ||
|
||||
(user?.role !== UserRole.ADMIN &&
|
||||
!user?.personalization?.name)) &&
|
||||
currentProjectId === null && (
|
||||
<OnboardingFlow
|
||||
handleHideOnboarding={hideOnboarding}
|
||||
handleFinishOnboarding={finishOnboarding}
|
||||
state={onboardingState}
|
||||
actions={onboardingActions}
|
||||
llmDescriptors={llmDescriptors}
|
||||
/>
|
||||
)}
|
||||
<ChatInputBar
|
||||
ref={chatInputBarRef}
|
||||
deepResearchEnabled={deepResearchEnabled}
|
||||
toggleDeepResearch={toggleDeepResearch}
|
||||
toggleDocumentSidebar={toggleDocumentSidebar}
|
||||
filterManager={filterManager}
|
||||
llmManager={llmManager}
|
||||
removeDocs={() => setSelectedDocuments([])}
|
||||
retrievalEnabled={retrievalEnabled}
|
||||
selectedDocuments={selectedDocuments}
|
||||
initialMessage={
|
||||
searchParams?.get(SEARCH_PARAM_NAMES.USER_PROMPT) || ""
|
||||
}
|
||||
stopGenerating={stopGenerating}
|
||||
onSubmit={handleChatInputSubmit}
|
||||
onHeightChange={handleInputHeightChange}
|
||||
chatState={currentChatState}
|
||||
currentSessionFileTokenCount={
|
||||
currentChatSessionId
|
||||
? currentSessionFileTokenCount
|
||||
: projectContextTokenCount
|
||||
}
|
||||
availableContextTokens={availableContextTokens}
|
||||
selectedAssistant={selectedAssistant || liveAssistant}
|
||||
handleFileUpload={handleMessageSpecificFileUpload}
|
||||
setPresentingDocument={setPresentingDocument}
|
||||
disabled={
|
||||
(!llmManager.isLoadingProviders &&
|
||||
llmManager.hasAnyProvider === false) ||
|
||||
(!isLoadingOnboarding &&
|
||||
onboardingState.currentStep !== OnboardingStep.Complete)
|
||||
}
|
||||
/>
|
||||
|
||||
<ChatInputBar
|
||||
ref={chatInputBarRef}
|
||||
deepResearchEnabled={deepResearchEnabled}
|
||||
toggleDeepResearch={toggleDeepResearch}
|
||||
toggleDocumentSidebar={toggleDocumentSidebar}
|
||||
filterManager={filterManager}
|
||||
llmManager={llmManager}
|
||||
removeDocs={() => setSelectedDocuments([])}
|
||||
retrievalEnabled={retrievalEnabled}
|
||||
selectedDocuments={selectedDocuments}
|
||||
initialMessage={
|
||||
searchParams?.get(SEARCH_PARAM_NAMES.USER_PROMPT) || ""
|
||||
}
|
||||
stopGenerating={stopGenerating}
|
||||
onSubmit={handleChatInputSubmit}
|
||||
chatState={currentChatState}
|
||||
currentSessionFileTokenCount={
|
||||
currentChatSessionId
|
||||
? currentSessionFileTokenCount
|
||||
: projectContextTokenCount
|
||||
}
|
||||
availableContextTokens={availableContextTokens}
|
||||
selectedAssistant={selectedAssistant || liveAssistant}
|
||||
handleFileUpload={handleMessageSpecificFileUpload}
|
||||
setPresentingDocument={setPresentingDocument}
|
||||
disabled={
|
||||
(!llmManager.isLoadingProviders &&
|
||||
llmManager.hasAnyProvider === false) ||
|
||||
(!isLoadingOnboarding &&
|
||||
onboardingState.currentStep !== OnboardingStep.Complete)
|
||||
}
|
||||
/>
|
||||
<Spacer rem={0.5} />
|
||||
|
||||
<Spacer rem={0.5} />
|
||||
|
||||
{!!currentProjectId && <ProjectChatSessionList />}
|
||||
</div>
|
||||
{!!currentProjectId && <ProjectChatSessionList />}
|
||||
</div>
|
||||
|
||||
{/* SearchUI */}
|
||||
|
||||
@@ -73,7 +73,7 @@ export function ChatPopup() {
|
||||
|
||||
return (
|
||||
<Modal open onOpenChange={() => {}}>
|
||||
<Modal.Content width="sm" height="lg">
|
||||
<Modal.Content tall>
|
||||
<Modal.Header
|
||||
titleClassName="text-text-04"
|
||||
icon={headerIcon}
|
||||
|
||||
@@ -38,8 +38,6 @@ import {
|
||||
} from "@/app/chat/services/actionUtils";
|
||||
import { SvgArrowUp, SvgHourglass, SvgPlusCircle, SvgStop } from "@opal/icons";
|
||||
|
||||
const LINE_HEIGHT = 24;
|
||||
const MIN_INPUT_HEIGHT = 44;
|
||||
const MAX_INPUT_HEIGHT = 200;
|
||||
|
||||
export interface SourceChipProps {
|
||||
@@ -92,6 +90,7 @@ export interface ChatInputBarProps {
|
||||
initialMessage?: string;
|
||||
stopGenerating: () => void;
|
||||
onSubmit: (message: string) => void;
|
||||
onHeightChange?: (delta: number) => void;
|
||||
llmManager: LlmManager;
|
||||
chatState: ChatState;
|
||||
currentSessionFileTokenCount: number;
|
||||
@@ -122,6 +121,7 @@ const ChatInputBar = React.memo(
|
||||
initialMessage = "",
|
||||
stopGenerating,
|
||||
onSubmit,
|
||||
onHeightChange,
|
||||
chatState,
|
||||
currentSessionFileTokenCount,
|
||||
availableContextTokens,
|
||||
@@ -141,6 +141,9 @@ const ChatInputBar = React.memo(
|
||||
const [message, setMessage] = useState(initialMessage);
|
||||
const textAreaRef = useRef<HTMLTextAreaElement>(null);
|
||||
const containerRef = useRef<HTMLDivElement>(null);
|
||||
const previousHeightRef = useRef<number | null>(null);
|
||||
const onHeightChangeRef = useRef(onHeightChange);
|
||||
onHeightChangeRef.current = onHeightChange;
|
||||
|
||||
// Expose reset and focus methods to parent via ref
|
||||
React.useImperativeHandle(ref, () => ({
|
||||
@@ -195,37 +198,15 @@ const ChatInputBar = React.memo(
|
||||
|
||||
const combinedSettings = useContext(SettingsContext);
|
||||
|
||||
// Track previous message to detect when lines might decrease
|
||||
const prevMessageRef = useRef("");
|
||||
|
||||
// Auto-resize textarea based on content
|
||||
useEffect(() => {
|
||||
const textarea = textAreaRef.current;
|
||||
if (textarea) {
|
||||
const prevLineCount = (prevMessageRef.current.match(/\n/g) || [])
|
||||
.length;
|
||||
const currLineCount = (message.match(/\n/g) || []).length;
|
||||
const lineRemoved = currLineCount < prevLineCount;
|
||||
prevMessageRef.current = message;
|
||||
|
||||
if (message.length === 0) {
|
||||
textarea.style.height = `${MIN_INPUT_HEIGHT}px`;
|
||||
return;
|
||||
} else if (lineRemoved) {
|
||||
const linesRemoved = prevLineCount - currLineCount;
|
||||
textarea.style.height = `${Math.max(
|
||||
MIN_INPUT_HEIGHT,
|
||||
Math.min(
|
||||
textarea.scrollHeight - LINE_HEIGHT * linesRemoved,
|
||||
MAX_INPUT_HEIGHT
|
||||
)
|
||||
)}px`;
|
||||
} else {
|
||||
textarea.style.height = `${Math.min(
|
||||
textarea.scrollHeight,
|
||||
MAX_INPUT_HEIGHT
|
||||
)}px`;
|
||||
}
|
||||
textarea.style.height = "0px"; // this is necessary in order to "reset" the scrollHeight
|
||||
textarea.style.height = `${Math.min(
|
||||
textarea.scrollHeight,
|
||||
MAX_INPUT_HEIGHT
|
||||
)}px`;
|
||||
}
|
||||
}, [message]);
|
||||
|
||||
@@ -235,6 +216,27 @@ const ChatInputBar = React.memo(
|
||||
}
|
||||
}, [initialMessage]);
|
||||
|
||||
// Detect height changes and notify parent for scroll adjustment
|
||||
useEffect(() => {
|
||||
if (!containerRef.current) return;
|
||||
|
||||
const observer = new ResizeObserver((entries) => {
|
||||
for (const entry of entries) {
|
||||
const newHeight = entry.contentRect.height;
|
||||
if (previousHeightRef.current !== null) {
|
||||
const delta = newHeight - previousHeightRef.current;
|
||||
if (delta !== 0) {
|
||||
onHeightChangeRef.current?.(delta);
|
||||
}
|
||||
}
|
||||
previousHeightRef.current = newHeight;
|
||||
}
|
||||
});
|
||||
|
||||
observer.observe(containerRef.current);
|
||||
return () => observer.disconnect();
|
||||
}, []);
|
||||
|
||||
const handlePaste = (event: React.ClipboardEvent) => {
|
||||
const items = event.clipboardData?.items;
|
||||
if (items) {
|
||||
|
||||
@@ -93,7 +93,7 @@ export default function FeedbackModal({
|
||||
{popup}
|
||||
|
||||
<Modal open={modal.isOpen} onOpenChange={modal.toggle}>
|
||||
<Modal.Content width="sm" height="sm">
|
||||
<Modal.Content mini>
|
||||
<Modal.Header
|
||||
icon={icon}
|
||||
title="Provide Additional Feedback"
|
||||
|
||||
@@ -643,7 +643,6 @@ export function useChatController({
|
||||
let toolCall: ToolCallMetadata | null = null;
|
||||
let files = projectFilesToFileDescriptors(currentMessageFiles);
|
||||
let packets: Packet[] = [];
|
||||
let packetsVersion = 0;
|
||||
|
||||
let newUserMessageId: number | null = null;
|
||||
let newAssistantMessageId: number | null = null;
|
||||
@@ -730,6 +729,7 @@ export function useChatController({
|
||||
if (!packet) {
|
||||
continue;
|
||||
}
|
||||
console.debug("Packet:", JSON.stringify(packet));
|
||||
|
||||
// We've processed initial packets and are starting to stream content.
|
||||
// Transition from 'loading' to 'streaming'.
|
||||
@@ -800,8 +800,8 @@ export function useChatController({
|
||||
updateCanContinue(true, frozenSessionId);
|
||||
}
|
||||
} else if (Object.hasOwn(packet, "obj")) {
|
||||
console.debug("Object packet:", JSON.stringify(packet));
|
||||
packets.push(packet as Packet);
|
||||
packetsVersion++;
|
||||
|
||||
// Check if the packet contains document information
|
||||
const packetObj = (packet as Packet).obj;
|
||||
@@ -859,7 +859,6 @@ export function useChatController({
|
||||
overridden_model: finalMessage?.overridden_model,
|
||||
stopReason: stopReason,
|
||||
packets: packets,
|
||||
packetsVersion: packetsVersion,
|
||||
},
|
||||
],
|
||||
// Pass the latest map state
|
||||
|
||||
@@ -139,8 +139,6 @@ export interface Message {
|
||||
|
||||
// new gen
|
||||
packets: Packet[];
|
||||
// Version counter for efficient memo comparison (increments with each packet)
|
||||
packetsVersion?: number;
|
||||
|
||||
// cached values for easy access
|
||||
documents?: OnyxDocument[] | null;
|
||||
|
||||
@@ -68,7 +68,7 @@ export const CodeBlock = memo(function CodeBlock({
|
||||
"bg-background-tint-00",
|
||||
"rounded",
|
||||
"text-xs",
|
||||
"inline",
|
||||
"inline-block",
|
||||
"whitespace-pre-wrap",
|
||||
"break-words",
|
||||
"py-0.5",
|
||||
|
||||
@@ -10,7 +10,8 @@ import IconButton from "@/refresh-components/buttons/IconButton";
|
||||
import CopyIconButton from "@/refresh-components/buttons/CopyIconButton";
|
||||
import Button from "@/refresh-components/buttons/Button";
|
||||
import { SvgEdit } from "@opal/icons";
|
||||
import FileDisplay from "./FileDisplay";
|
||||
import FileDisplay from "@/app/chat/message/FileDisplay";
|
||||
import { useTripleClickSelect } from "@/hooks/useTripleClickSelect";
|
||||
|
||||
interface MessageEditingProps {
|
||||
content: string;
|
||||
@@ -139,6 +140,10 @@ const HumanMessage = React.memo(function HumanMessage({
|
||||
|
||||
const [isEditing, setIsEditing] = useState(false);
|
||||
|
||||
// Ref for the text content element (for triple-click selection)
|
||||
const textContentRef = useRef<HTMLDivElement>(null);
|
||||
const handleTripleClick = useTripleClickSelect(textContentRef);
|
||||
|
||||
// Use nodeId for switching (finding position in siblings)
|
||||
const indexInSiblings = otherMessagesCanSwitchTo?.indexOf(nodeId);
|
||||
// indexOf returns -1 if not found, treat that as undefined
|
||||
@@ -195,18 +200,34 @@ const HumanMessage = React.memo(function HumanMessage({
|
||||
<>
|
||||
<div className="md:max-w-[25rem] flex basis-[100%] md:basis-auto justify-end md:order-1">
|
||||
<div
|
||||
ref={textContentRef}
|
||||
className={
|
||||
"max-w-[25rem] whitespace-break-spaces rounded-t-16 rounded-bl-16 bg-background-tint-02 py-2 px-3"
|
||||
"max-w-[25rem] whitespace-break-spaces rounded-t-16 rounded-bl-16 bg-background-tint-02 py-2 px-3 cursor-text"
|
||||
}
|
||||
onMouseDown={handleTripleClick}
|
||||
onCopy={(e) => {
|
||||
e.preventDefault();
|
||||
const selection = window.getSelection();
|
||||
if (selection) {
|
||||
e.preventDefault();
|
||||
const text = selection
|
||||
.toString()
|
||||
.replace(/\n{2,}/g, "\n")
|
||||
.trim();
|
||||
e.clipboardData.setData("text/plain", text);
|
||||
if (!selection || !selection.rangeCount) {
|
||||
e.clipboardData.setData("text/plain", content);
|
||||
return;
|
||||
}
|
||||
|
||||
const range = selection.getRangeAt(0);
|
||||
const selectedText = selection
|
||||
.toString()
|
||||
.replace(/\n{2,}/g, "\n")
|
||||
.trim();
|
||||
|
||||
// Check if selection is within this element using DOM containment
|
||||
if (
|
||||
textContentRef.current?.contains(
|
||||
range.commonAncestorContainer
|
||||
)
|
||||
) {
|
||||
e.clipboardData.setData("text/plain", selectedText);
|
||||
} else {
|
||||
e.clipboardData.setData("text/plain", content);
|
||||
}
|
||||
}}
|
||||
>
|
||||
|
||||
@@ -62,6 +62,7 @@ import { usePopup } from "@/components/admin/connectors/Popup";
|
||||
import { useFeedbackController } from "../../hooks/useFeedbackController";
|
||||
import { SvgThumbsDown, SvgThumbsUp } from "@opal/icons";
|
||||
import Text from "@/refresh-components/texts/Text";
|
||||
import { useTripleClickSelect } from "@/hooks/useTripleClickSelect";
|
||||
|
||||
// Type for the regeneration factory function passed from ChatUI
|
||||
export type RegenerationFactory = (regenerationRequest: {
|
||||
@@ -72,8 +73,6 @@ export type RegenerationFactory = (regenerationRequest: {
|
||||
|
||||
export interface AIMessageProps {
|
||||
rawPackets: Packet[];
|
||||
// Version counter for efficient memo comparison (avoids array copying)
|
||||
packetsVersion?: number;
|
||||
chatState: FullChatState;
|
||||
nodeId: number;
|
||||
messageId?: number;
|
||||
@@ -88,6 +87,8 @@ export interface AIMessageProps {
|
||||
}
|
||||
|
||||
// TODO: Consider more robust comparisons:
|
||||
// - `rawPackets.length` assumes packets are append-only. Could compare the last
|
||||
// packet or use a shallow comparison if packets can be modified in place.
|
||||
// - `chatState.docs`, `chatState.citations`, and `otherMessagesCanSwitchTo` use
|
||||
// reference equality. Shallow array/object comparison would be more robust if
|
||||
// these are recreated with the same values.
|
||||
@@ -96,7 +97,7 @@ function arePropsEqual(prev: AIMessageProps, next: AIMessageProps): boolean {
|
||||
prev.nodeId === next.nodeId &&
|
||||
prev.messageId === next.messageId &&
|
||||
prev.currentFeedback === next.currentFeedback &&
|
||||
prev.packetsVersion === next.packetsVersion &&
|
||||
prev.rawPackets.length === next.rawPackets.length &&
|
||||
prev.chatState.assistant?.id === next.chatState.assistant?.id &&
|
||||
prev.chatState.docs === next.chatState.docs &&
|
||||
prev.chatState.citations === next.chatState.citations &&
|
||||
@@ -125,6 +126,7 @@ const AIMessage = React.memo(function AIMessage({
|
||||
}: AIMessageProps) {
|
||||
const markdownRef = useRef<HTMLDivElement>(null);
|
||||
const finalAnswerRef = useRef<HTMLDivElement>(null);
|
||||
const handleTripleClick = useTripleClickSelect(markdownRef);
|
||||
const { popup, setPopup } = usePopup();
|
||||
const { handleFeedbackChange } = useFeedbackController({ setPopup });
|
||||
|
||||
@@ -227,14 +229,6 @@ const AIMessage = React.memo(function AIMessage({
|
||||
);
|
||||
const stopReasonRef = useRef<StopReason | undefined>(undefined);
|
||||
|
||||
// Track specifically when MESSAGE_START arrives (for collapsing the tools header).
|
||||
// This is separate from finalAnswerComing which can be set by onAllToolsDisplayed
|
||||
// or by PYTHON/IMAGE tool packets.
|
||||
const [hasTextMessageStarted, setHasTextMessageStarted] = useState(
|
||||
rawPackets.some((p) => p.obj.type === PacketType.MESSAGE_START)
|
||||
);
|
||||
const hasTextMessageStartedRef = useRef(hasTextMessageStarted);
|
||||
|
||||
// Incremental packet processing state
|
||||
const lastProcessedIndexRef = useRef<number>(0);
|
||||
const citationsRef = useRef<StreamingCitation[]>([]);
|
||||
@@ -273,10 +267,6 @@ const AIMessage = React.memo(function AIMessage({
|
||||
seenGroupKeysRef.current = new Set();
|
||||
groupKeysWithSectionEndRef.current = new Set();
|
||||
expectedBranchesRef.current = new Map();
|
||||
hasTextMessageStartedRef.current = rawPackets.some(
|
||||
(p) => p.obj.type === PacketType.MESSAGE_START
|
||||
);
|
||||
setHasTextMessageStarted(hasTextMessageStartedRef.current);
|
||||
};
|
||||
useEffect(() => {
|
||||
resetState();
|
||||
@@ -435,15 +425,6 @@ const AIMessage = React.memo(function AIMessage({
|
||||
finalAnswerComingRef.current = true;
|
||||
}
|
||||
|
||||
// Track specifically when MESSAGE_START arrives (for collapsing tools header)
|
||||
if (
|
||||
packet.obj.type === PacketType.MESSAGE_START &&
|
||||
!hasTextMessageStartedRef.current
|
||||
) {
|
||||
setHasTextMessageStarted(true);
|
||||
hasTextMessageStartedRef.current = true;
|
||||
}
|
||||
|
||||
if (packet.obj.type === PacketType.STOP && !stopPacketSeenRef.current) {
|
||||
setStopPacketSeen(true);
|
||||
// Extract and store the stop reason
|
||||
@@ -557,7 +538,8 @@ const AIMessage = React.memo(function AIMessage({
|
||||
<div className="max-w-message-max break-words pl-4 w-full">
|
||||
<div
|
||||
ref={markdownRef}
|
||||
className="overflow-x-visible max-w-content-max focus:outline-none select-text"
|
||||
className="overflow-x-visible max-w-content-max focus:outline-none select-text cursor-text"
|
||||
onMouseDown={handleTripleClick}
|
||||
onCopy={(e) => {
|
||||
if (markdownRef.current) {
|
||||
handleCopy(e, markdownRef as RefObject<HTMLDivElement>);
|
||||
@@ -604,7 +586,6 @@ const AIMessage = React.memo(function AIMessage({
|
||||
isStreaming={globalChatState === "streaming"}
|
||||
onAllToolsDisplayed={() => setFinalAnswerComing(true)}
|
||||
expectedBranchesPerTurn={expectedBranchesRef.current}
|
||||
hasTextMessageStarted={hasTextMessageStarted}
|
||||
/>
|
||||
)}
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { useState, useMemo, useEffect, useRef, JSX } from "react";
|
||||
import { useState, useMemo, useEffect, JSX } from "react";
|
||||
import {
|
||||
FiCheckCircle,
|
||||
FiChevronRight,
|
||||
@@ -558,7 +558,6 @@ export default function MultiToolRenderer({
|
||||
onAllToolsDisplayed,
|
||||
isStreaming,
|
||||
expectedBranchesPerTurn,
|
||||
hasTextMessageStarted,
|
||||
}: {
|
||||
packetGroups: { turn_index: number; tab_index: number; packets: Packet[] }[];
|
||||
chatState: FullChatState;
|
||||
@@ -570,24 +569,10 @@ export default function MultiToolRenderer({
|
||||
isStreaming?: boolean;
|
||||
// Map of turn_index -> expected number of parallel branches (from TopLevelBranching packet)
|
||||
expectedBranchesPerTurn?: Map<number, number>;
|
||||
// True when MESSAGE_START packet has arrived (specifically for text messages,
|
||||
// not PYTHON/IMAGE tools). Used to determine when to collapse the tools header.
|
||||
hasTextMessageStarted?: boolean;
|
||||
}) {
|
||||
const [isExpanded, setIsExpanded] = useState(false);
|
||||
const [isStreamingExpanded, setIsStreamingExpanded] = useState(false);
|
||||
|
||||
// Track if hasTextMessageStarted has ever been true.
|
||||
// Once true, we always show the collapsed view and never revert to streaming view.
|
||||
// This is latched because we want the user to control expansion after the initial collapse.
|
||||
const hasEverSeenTextMessageRef = useRef(hasTextMessageStarted ?? false);
|
||||
|
||||
useEffect(() => {
|
||||
if (hasTextMessageStarted) {
|
||||
hasEverSeenTextMessageRef.current = true;
|
||||
}
|
||||
}, [hasTextMessageStarted]);
|
||||
|
||||
const toolGroups = useMemo(() => {
|
||||
return packetGroups.filter(
|
||||
(group) => group.packets[0] && isToolPacket(group.packets[0], false)
|
||||
@@ -764,13 +749,8 @@ export default function MultiToolRenderer({
|
||||
return uniqueTabIndices.size > 1;
|
||||
};
|
||||
|
||||
// If the text message hasn't started yet, show tools progressively with timing.
|
||||
// We use hasEverSeenTextMessageRef (latched) instead of isComplete directly to prevent
|
||||
// flickering. isComplete (finalAnswerComing) can be set by onAllToolsDisplayed or by
|
||||
// PYTHON/IMAGE tools before the actual text message starts. hasTextMessageStarted is
|
||||
// only set when MESSAGE_START arrives. Once true, we switch to the collapsed view
|
||||
// permanently and let the user control expansion.
|
||||
if (!hasEverSeenTextMessageRef.current) {
|
||||
// If still processing, show tools progressively with timing
|
||||
if (!isComplete) {
|
||||
// Filter display items to only show those whose (turn_index, tab_index) is visible
|
||||
const itemsToDisplay = displayItems.filter((item) =>
|
||||
visibleTools.has(`${item.turn_index}-${item.tab_index}`)
|
||||
|
||||
@@ -29,7 +29,7 @@ import {
|
||||
useCurrentChatState,
|
||||
useCurrentMessageHistory,
|
||||
} from "@/app/chat/stores/useChatSessionStore";
|
||||
import MessageList from "@/components/chat/MessageList";
|
||||
import ChatUI from "@/sections/ChatUI";
|
||||
import useChatSessions from "@/hooks/useChatSessions";
|
||||
import { cn } from "@/lib/utils";
|
||||
import Logo from "@/refresh-components/Logo";
|
||||
@@ -350,19 +350,17 @@ export default function NRFPage({ isSidePanel = false }: NRFPageProps) {
|
||||
{/* Scrollable messages area */}
|
||||
<div className="nrf-messages-scroll">
|
||||
<div className="nrf-messages-content">
|
||||
{resolvedAssistant && (
|
||||
<MessageList
|
||||
liveAssistant={resolvedAssistant}
|
||||
llmManager={llmManager}
|
||||
currentMessageFiles={currentMessageFiles}
|
||||
setPresentingDocument={() => {}}
|
||||
onSubmit={onSubmit}
|
||||
onMessageSelection={() => {}}
|
||||
stopGenerating={stopGenerating}
|
||||
onResubmit={handleResubmitLastMessage}
|
||||
deepResearchEnabled={deepResearchEnabled}
|
||||
/>
|
||||
)}
|
||||
<ChatUI
|
||||
liveAssistant={resolvedAssistant}
|
||||
llmManager={llmManager}
|
||||
currentMessageFiles={currentMessageFiles}
|
||||
setPresentingDocument={() => {}}
|
||||
onSubmit={onSubmit}
|
||||
onMessageSelection={() => {}}
|
||||
stopGenerating={stopGenerating}
|
||||
handleResubmitLastMessage={handleResubmitLastMessage}
|
||||
deepResearchEnabled={deepResearchEnabled}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -463,7 +461,7 @@ export default function NRFPage({ isSidePanel = false }: NRFPageProps) {
|
||||
/>
|
||||
|
||||
<Modal open={showTurnOffModal} onOpenChange={setShowTurnOffModal}>
|
||||
<Modal.Content width="sm">
|
||||
<Modal.Content mini>
|
||||
<Modal.Header
|
||||
icon={SvgAlertTriangle}
|
||||
title="Turn off Onyx new tab page?"
|
||||
@@ -485,7 +483,7 @@ export default function NRFPage({ isSidePanel = false }: NRFPageProps) {
|
||||
|
||||
{!user && authTypeMetadata.authType !== AuthType.DISABLED && (
|
||||
<Modal open onOpenChange={() => {}}>
|
||||
<Modal.Content width="sm" height="sm">
|
||||
<Modal.Content small>
|
||||
<Modal.Header icon={SvgUser} title="Welcome to Onyx" />
|
||||
<Modal.Body>
|
||||
{authTypeMetadata.authType === AuthType.BASIC ? (
|
||||
|
||||
@@ -4,7 +4,8 @@ import {
|
||||
StreamStopInfo,
|
||||
} from "@/lib/search/interfaces";
|
||||
import { handleSSEStream } from "@/lib/search/streamingUtils";
|
||||
import { FeedbackType } from "@/app/chat/interfaces";
|
||||
import { ChatState, FeedbackType } from "@/app/chat/interfaces";
|
||||
import { MutableRefObject, RefObject, useEffect, useRef } from "react";
|
||||
import {
|
||||
BackendMessage,
|
||||
DocumentsResponse,
|
||||
@@ -456,3 +457,104 @@ export async function uploadFilesForChat(
|
||||
|
||||
return [responseJson.files as FileDescriptor[], null];
|
||||
}
|
||||
|
||||
export function useScrollonStream({
|
||||
chatState,
|
||||
scrollableDivRef,
|
||||
scrollDist,
|
||||
endDivRef,
|
||||
debounceNumber,
|
||||
mobile,
|
||||
enableAutoScroll,
|
||||
}: {
|
||||
chatState: ChatState;
|
||||
scrollableDivRef: RefObject<HTMLDivElement | null>;
|
||||
scrollDist: MutableRefObject<number>;
|
||||
endDivRef: RefObject<HTMLDivElement | null>;
|
||||
debounceNumber: number;
|
||||
mobile?: boolean;
|
||||
enableAutoScroll?: boolean;
|
||||
}) {
|
||||
const mobileDistance = 900; // distance that should "engage" the scroll
|
||||
const desktopDistance = 500; // distance that should "engage" the scroll
|
||||
|
||||
const distance = mobile ? mobileDistance : desktopDistance;
|
||||
|
||||
const preventScrollInterference = useRef<boolean>(false);
|
||||
const preventScroll = useRef<boolean>(false);
|
||||
const blockActionRef = useRef<boolean>(false);
|
||||
const previousScroll = useRef<number>(0);
|
||||
|
||||
useEffect(() => {
|
||||
if (!enableAutoScroll) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (chatState != "input" && scrollableDivRef && scrollableDivRef.current) {
|
||||
const newHeight: number = scrollableDivRef.current?.scrollTop!;
|
||||
const heightDifference = newHeight - previousScroll.current;
|
||||
previousScroll.current = newHeight;
|
||||
|
||||
// Prevent streaming scroll
|
||||
if (heightDifference < 0 && !preventScroll.current) {
|
||||
scrollableDivRef.current.style.scrollBehavior = "auto";
|
||||
scrollableDivRef.current.scrollTop = scrollableDivRef.current.scrollTop;
|
||||
scrollableDivRef.current.style.scrollBehavior = "smooth";
|
||||
preventScrollInterference.current = true;
|
||||
preventScroll.current = true;
|
||||
|
||||
setTimeout(() => {
|
||||
preventScrollInterference.current = false;
|
||||
}, 2000);
|
||||
setTimeout(() => {
|
||||
preventScroll.current = false;
|
||||
}, 10000);
|
||||
}
|
||||
|
||||
// Ensure can scroll if scroll down
|
||||
else if (!preventScrollInterference.current) {
|
||||
preventScroll.current = false;
|
||||
}
|
||||
if (
|
||||
scrollDist.current < distance &&
|
||||
!blockActionRef.current &&
|
||||
!blockActionRef.current &&
|
||||
!preventScroll.current &&
|
||||
endDivRef &&
|
||||
endDivRef.current
|
||||
) {
|
||||
// catch up if necessary!
|
||||
const scrollAmount = scrollDist.current + (mobile ? 1000 : 10000);
|
||||
if (scrollDist.current > 300) {
|
||||
// if (scrollDist.current > 140) {
|
||||
endDivRef.current.scrollIntoView();
|
||||
} else {
|
||||
blockActionRef.current = true;
|
||||
|
||||
scrollableDivRef?.current?.scrollBy({
|
||||
left: 0,
|
||||
top: Math.max(0, scrollAmount),
|
||||
behavior: "smooth",
|
||||
});
|
||||
|
||||
setTimeout(() => {
|
||||
blockActionRef.current = false;
|
||||
}, debounceNumber);
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// scroll on end of stream if within distance
|
||||
useEffect(() => {
|
||||
if (scrollableDivRef?.current && chatState == "input" && enableAutoScroll) {
|
||||
if (scrollDist.current < distance - 50) {
|
||||
scrollableDivRef?.current?.scrollBy({
|
||||
left: 0,
|
||||
top: Math.max(scrollDist.current + 600, 0),
|
||||
behavior: "smooth",
|
||||
});
|
||||
}
|
||||
}
|
||||
}, [chatState, distance, scrollDist, scrollableDivRef, enableAutoScroll]);
|
||||
}
|
||||
|
||||
@@ -35,7 +35,7 @@ export default function UserGroupCreationForm({
|
||||
|
||||
return (
|
||||
<Modal open onOpenChange={onClose}>
|
||||
<Modal.Content>
|
||||
<Modal.Content medium>
|
||||
<Modal.Header
|
||||
icon={SvgUsers}
|
||||
title={isUpdate ? "Update a User Group" : "Create a new User Group"}
|
||||
|
||||
@@ -33,7 +33,7 @@ export default function AddConnectorForm({
|
||||
|
||||
return (
|
||||
<Modal open onOpenChange={onClose}>
|
||||
<Modal.Content width="sm" height="sm">
|
||||
<Modal.Content small>
|
||||
<Modal.Header
|
||||
icon={SvgPlus}
|
||||
title="Add New Connector"
|
||||
|
||||
@@ -22,7 +22,7 @@ export default function AddMemberForm({
|
||||
|
||||
return (
|
||||
<Modal open onOpenChange={onClose}>
|
||||
<Modal.Content width="sm" height="sm">
|
||||
<Modal.Content small>
|
||||
<Modal.Header
|
||||
icon={SvgUserPlus}
|
||||
title="Add New User"
|
||||
|
||||
@@ -4,9 +4,8 @@ import { use } from "react";
|
||||
import { GroupDisplay } from "./GroupDisplay";
|
||||
import { useSpecificUserGroup } from "./hook";
|
||||
import { ThreeDotsLoader } from "@/components/Loading";
|
||||
import { useConnectorStatus } from "@/lib/hooks";
|
||||
import { useConnectorStatus, useUsers } from "@/lib/hooks";
|
||||
import { useRouter } from "next/navigation";
|
||||
import useUsers from "@/hooks/useUsers";
|
||||
import BackButton from "@/refresh-components/buttons/BackButton";
|
||||
import { AdminPageTitle } from "@/components/admin/Title";
|
||||
import { SvgUsers } from "@opal/icons";
|
||||
|
||||
@@ -5,9 +5,8 @@ import UserGroupCreationForm from "./UserGroupCreationForm";
|
||||
import { usePopup } from "@/components/admin/connectors/Popup";
|
||||
import { useState } from "react";
|
||||
import { ThreeDotsLoader } from "@/components/Loading";
|
||||
import { useConnectorStatus, useUserGroups } from "@/lib/hooks";
|
||||
import { useConnectorStatus, useUserGroups, useUsers } from "@/lib/hooks";
|
||||
import { AdminPageTitle } from "@/components/admin/Title";
|
||||
import useUsers from "@/hooks/useUsers";
|
||||
|
||||
import { useUser } from "@/components/user/UserProvider";
|
||||
import CreateButton from "@/refresh-components/buttons/CreateButton";
|
||||
|
||||
@@ -178,7 +178,7 @@ function PreviousQueryHistoryExportsModal({
|
||||
|
||||
return (
|
||||
<Modal open onOpenChange={() => setShowModal(false)}>
|
||||
<Modal.Content width="lg" height="full">
|
||||
<Modal.Content large>
|
||||
<Modal.Header
|
||||
icon={SvgFileText}
|
||||
title="Previous Query History Exports"
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
import { FormField } from "@/refresh-components/form/FormField";
|
||||
import InputTypeIn from "@/refresh-components/inputs/InputTypeIn";
|
||||
import Tabs from "@/refresh-components/Tabs";
|
||||
import { Tabs, TabsList, TabsTrigger } from "@/refresh-components/tabs/tabs";
|
||||
import Separator from "@/refresh-components/Separator";
|
||||
import { Preview } from "./Preview";
|
||||
import InputTextArea from "@/refresh-components/inputs/InputTextArea";
|
||||
@@ -252,16 +252,16 @@ export const AppearanceThemeSettings = forwardRef<
|
||||
setFieldValue("logo_display_style", value)
|
||||
}
|
||||
>
|
||||
<Tabs.List>
|
||||
<Tabs.Trigger
|
||||
<TabsList className="w-full grid grid-cols-3">
|
||||
<TabsTrigger
|
||||
value="logo_and_name"
|
||||
tooltip="Show both your application logo and name."
|
||||
tooltipSide="top"
|
||||
{...getPreviewHandlers("sidebar")}
|
||||
>
|
||||
Logo & Name
|
||||
</Tabs.Trigger>
|
||||
<Tabs.Trigger
|
||||
</TabsTrigger>
|
||||
<TabsTrigger
|
||||
value="logo_only"
|
||||
disabled={!hasLogo}
|
||||
tooltip={
|
||||
@@ -273,8 +273,8 @@ export const AppearanceThemeSettings = forwardRef<
|
||||
{...getPreviewHandlers("sidebar")}
|
||||
>
|
||||
Logo Only
|
||||
</Tabs.Trigger>
|
||||
<Tabs.Trigger
|
||||
</TabsTrigger>
|
||||
<TabsTrigger
|
||||
value="name_only"
|
||||
disabled={!hasApplicationName}
|
||||
tooltip={
|
||||
@@ -286,8 +286,8 @@ export const AppearanceThemeSettings = forwardRef<
|
||||
{...getPreviewHandlers("sidebar")}
|
||||
>
|
||||
Name Only
|
||||
</Tabs.Trigger>
|
||||
</Tabs.List>
|
||||
</TabsTrigger>
|
||||
</TabsList>
|
||||
</Tabs>
|
||||
</FormField.Control>
|
||||
<FormField.Description>
|
||||
|
||||
@@ -30,10 +30,13 @@ export default function SourceTile({
|
||||
w-40
|
||||
cursor-pointer
|
||||
shadow-md
|
||||
bg-background-tint-00
|
||||
hover:bg-background-tint-02
|
||||
relative
|
||||
${preSelect ? "subtle-pulse" : ""}
|
||||
${
|
||||
preSelect
|
||||
? "bg-background-tint-01 subtle-pulse"
|
||||
: "bg-background-tint-00"
|
||||
}
|
||||
`}
|
||||
href={navigationUrl as Route}
|
||||
>
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user