mirror of
https://github.com/onyx-dot-app/onyx.git
synced 2026-03-18 14:12:45 +00:00
Compare commits
3 Commits
bo/hook-ce
...
jamison/vo
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
0f46e1e084 | ||
|
|
f4d379ceed | ||
|
|
8f1076e69d |
25
.github/actions/slack-notify/action.yml
vendored
25
.github/actions/slack-notify/action.yml
vendored
@@ -10,9 +10,6 @@ inputs:
|
||||
failed-jobs:
|
||||
description: "Deprecated alias for details"
|
||||
required: false
|
||||
mention:
|
||||
description: "GitHub username to resolve to a Slack @-mention. Replaces {mention} in details."
|
||||
required: false
|
||||
title:
|
||||
description: "Title for the notification"
|
||||
required: false
|
||||
@@ -29,7 +26,6 @@ runs:
|
||||
SLACK_WEBHOOK_URL: ${{ inputs.webhook-url }}
|
||||
DETAILS: ${{ inputs.details }}
|
||||
FAILED_JOBS: ${{ inputs.failed-jobs }}
|
||||
MENTION_USER: ${{ inputs.mention }}
|
||||
TITLE: ${{ inputs.title }}
|
||||
REF_NAME: ${{ inputs.ref-name }}
|
||||
REPO: ${{ github.repository }}
|
||||
@@ -56,27 +52,6 @@ runs:
|
||||
DETAILS="$FAILED_JOBS"
|
||||
fi
|
||||
|
||||
# Resolve {mention} placeholder if a GitHub username was provided.
|
||||
# Looks up the username in user-mappings.json (co-located with this action)
|
||||
# and replaces {mention} with <@SLACK_ID> for a Slack @-mention.
|
||||
# Falls back to the plain GitHub username if not found in the mapping.
|
||||
if [ -n "$MENTION_USER" ]; then
|
||||
MAPPINGS_FILE="${GITHUB_ACTION_PATH}/user-mappings.json"
|
||||
slack_id="$(jq -r --arg gh "$MENTION_USER" 'to_entries[] | select(.value | ascii_downcase == ($gh | ascii_downcase)) | .key' "$MAPPINGS_FILE" 2>/dev/null | head -1)"
|
||||
|
||||
if [ -n "$slack_id" ]; then
|
||||
mention_text="<@${slack_id}>"
|
||||
else
|
||||
mention_text="${MENTION_USER}"
|
||||
fi
|
||||
|
||||
DETAILS="${DETAILS//\{mention\}/$mention_text}"
|
||||
TITLE="${TITLE//\{mention\}/}"
|
||||
else
|
||||
DETAILS="${DETAILS//\{mention\}/}"
|
||||
TITLE="${TITLE//\{mention\}/}"
|
||||
fi
|
||||
|
||||
normalize_multiline() {
|
||||
printf '%s' "$1" | awk 'BEGIN { ORS=""; first=1 } { if (!first) printf "\\n"; printf "%s", $0; first=0 }'
|
||||
}
|
||||
|
||||
18
.github/actions/slack-notify/user-mappings.json
vendored
18
.github/actions/slack-notify/user-mappings.json
vendored
@@ -1,18 +0,0 @@
|
||||
{
|
||||
"U05SAGZPEA1": "yuhongsun96",
|
||||
"U05SAH6UGUD": "Weves",
|
||||
"U07PWEQB7A5": "evan-onyx",
|
||||
"U07V1SM68KF": "joachim-danswer",
|
||||
"U08JZ9N3QNN": "raunakab",
|
||||
"U08L24NCLJE": "Subash-Mohan",
|
||||
"U090B9M07B2": "wenxi-onyx",
|
||||
"U094RASDP0Q": "duo-onyx",
|
||||
"U096L8ZQ85B": "justin-tahara",
|
||||
"U09AHV8UBQX": "jessicasingh7",
|
||||
"U09KAL5T3C2": "nmgarza5",
|
||||
"U09KPGVQ70R": "acaprau",
|
||||
"U09QR8KTSJH": "rohoswagger",
|
||||
"U09RB4NTXA4": "jmelahman",
|
||||
"U0A6K9VCY6A": "Danelegend",
|
||||
"U0AGC4KH71A": "Bo-Onyx"
|
||||
}
|
||||
30
.github/workflows/deployment.yml
vendored
30
.github/workflows/deployment.yml
vendored
@@ -455,7 +455,7 @@ jobs:
|
||||
|
||||
- name: Docker meta
|
||||
id: meta
|
||||
uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # ratchet:docker/metadata-action@v6.0.0
|
||||
uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # ratchet:docker/metadata-action@v5
|
||||
with:
|
||||
images: ${{ needs.determine-builds.outputs.is-test-run == 'true' && env.RUNS_ON_ECR_CACHE || env.REGISTRY_IMAGE }}
|
||||
flavor: |
|
||||
@@ -529,7 +529,7 @@ jobs:
|
||||
|
||||
- name: Docker meta
|
||||
id: meta
|
||||
uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # ratchet:docker/metadata-action@v6.0.0
|
||||
uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # ratchet:docker/metadata-action@v5
|
||||
with:
|
||||
images: ${{ needs.determine-builds.outputs.is-test-run == 'true' && env.RUNS_ON_ECR_CACHE || env.REGISTRY_IMAGE }}
|
||||
flavor: |
|
||||
@@ -607,7 +607,7 @@ jobs:
|
||||
|
||||
- name: Docker meta
|
||||
id: meta
|
||||
uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # ratchet:docker/metadata-action@v6.0.0
|
||||
uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # ratchet:docker/metadata-action@v5
|
||||
with:
|
||||
images: ${{ needs.determine-builds.outputs.is-test-run == 'true' && env.RUNS_ON_ECR_CACHE || env.REGISTRY_IMAGE }}
|
||||
flavor: |
|
||||
@@ -668,7 +668,7 @@ jobs:
|
||||
|
||||
- name: Docker meta
|
||||
id: meta
|
||||
uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # ratchet:docker/metadata-action@v6.0.0
|
||||
uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # ratchet:docker/metadata-action@v5
|
||||
with:
|
||||
images: ${{ needs.determine-builds.outputs.is-test-run == 'true' && env.RUNS_ON_ECR_CACHE || env.REGISTRY_IMAGE }}
|
||||
flavor: |
|
||||
@@ -750,7 +750,7 @@ jobs:
|
||||
|
||||
- name: Docker meta
|
||||
id: meta
|
||||
uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # ratchet:docker/metadata-action@v6.0.0
|
||||
uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # ratchet:docker/metadata-action@v5
|
||||
with:
|
||||
images: ${{ needs.determine-builds.outputs.is-test-run == 'true' && env.RUNS_ON_ECR_CACHE || env.REGISTRY_IMAGE }}
|
||||
flavor: |
|
||||
@@ -836,7 +836,7 @@ jobs:
|
||||
|
||||
- name: Docker meta
|
||||
id: meta
|
||||
uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # ratchet:docker/metadata-action@v6.0.0
|
||||
uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # ratchet:docker/metadata-action@v5
|
||||
with:
|
||||
images: ${{ needs.determine-builds.outputs.is-test-run == 'true' && env.RUNS_ON_ECR_CACHE || env.REGISTRY_IMAGE }}
|
||||
flavor: |
|
||||
@@ -894,7 +894,7 @@ jobs:
|
||||
|
||||
- name: Docker meta
|
||||
id: meta
|
||||
uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # ratchet:docker/metadata-action@v6.0.0
|
||||
uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # ratchet:docker/metadata-action@v5
|
||||
with:
|
||||
images: ${{ needs.determine-builds.outputs.is-test-run == 'true' && env.RUNS_ON_ECR_CACHE || env.REGISTRY_IMAGE }}
|
||||
flavor: |
|
||||
@@ -967,7 +967,7 @@ jobs:
|
||||
|
||||
- name: Docker meta
|
||||
id: meta
|
||||
uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # ratchet:docker/metadata-action@v6.0.0
|
||||
uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # ratchet:docker/metadata-action@v5
|
||||
with:
|
||||
images: ${{ needs.determine-builds.outputs.is-test-run == 'true' && env.RUNS_ON_ECR_CACHE || env.REGISTRY_IMAGE }}
|
||||
flavor: |
|
||||
@@ -1044,7 +1044,7 @@ jobs:
|
||||
|
||||
- name: Docker meta
|
||||
id: meta
|
||||
uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # ratchet:docker/metadata-action@v6.0.0
|
||||
uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # ratchet:docker/metadata-action@v5
|
||||
with:
|
||||
images: ${{ needs.determine-builds.outputs.is-test-run == 'true' && env.RUNS_ON_ECR_CACHE || env.REGISTRY_IMAGE }}
|
||||
flavor: |
|
||||
@@ -1105,7 +1105,7 @@ jobs:
|
||||
|
||||
- name: Docker meta
|
||||
id: meta
|
||||
uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # ratchet:docker/metadata-action@v6.0.0
|
||||
uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # ratchet:docker/metadata-action@v5
|
||||
with:
|
||||
images: ${{ env.REGISTRY_IMAGE }}
|
||||
flavor: |
|
||||
@@ -1178,7 +1178,7 @@ jobs:
|
||||
|
||||
- name: Docker meta
|
||||
id: meta
|
||||
uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # ratchet:docker/metadata-action@v6.0.0
|
||||
uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # ratchet:docker/metadata-action@v5
|
||||
with:
|
||||
images: ${{ env.REGISTRY_IMAGE }}
|
||||
flavor: |
|
||||
@@ -1256,7 +1256,7 @@ jobs:
|
||||
|
||||
- name: Docker meta
|
||||
id: meta
|
||||
uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # ratchet:docker/metadata-action@v6.0.0
|
||||
uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # ratchet:docker/metadata-action@v5
|
||||
with:
|
||||
images: ${{ env.REGISTRY_IMAGE }}
|
||||
flavor: |
|
||||
@@ -1317,7 +1317,7 @@ jobs:
|
||||
|
||||
- name: Docker meta
|
||||
id: meta
|
||||
uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # ratchet:docker/metadata-action@v6.0.0
|
||||
uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # ratchet:docker/metadata-action@v5
|
||||
with:
|
||||
images: ${{ needs.determine-builds.outputs.is-test-run == 'true' && env.RUNS_ON_ECR_CACHE || env.REGISTRY_IMAGE }}
|
||||
flavor: |
|
||||
@@ -1397,7 +1397,7 @@ jobs:
|
||||
|
||||
- name: Docker meta
|
||||
id: meta
|
||||
uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # ratchet:docker/metadata-action@v6.0.0
|
||||
uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # ratchet:docker/metadata-action@v5
|
||||
with:
|
||||
images: ${{ needs.determine-builds.outputs.is-test-run == 'true' && env.RUNS_ON_ECR_CACHE || env.REGISTRY_IMAGE }}
|
||||
flavor: |
|
||||
@@ -1480,7 +1480,7 @@ jobs:
|
||||
|
||||
- name: Docker meta
|
||||
id: meta
|
||||
uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # ratchet:docker/metadata-action@v6.0.0
|
||||
uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # ratchet:docker/metadata-action@v5
|
||||
with:
|
||||
images: ${{ needs.determine-builds.outputs.is-test-run == 'true' && env.RUNS_ON_ECR_CACHE || env.REGISTRY_IMAGE }}
|
||||
flavor: |
|
||||
|
||||
@@ -207,7 +207,7 @@ jobs:
|
||||
CHERRY_PICK_PR_URL: ${{ needs.cherry-pick-to-latest-release.outputs.cherry_pick_pr_url }}
|
||||
run: |
|
||||
source_pr_url="https://github.com/${GITHUB_REPOSITORY}/pull/${SOURCE_PR_NUMBER}"
|
||||
details="*Cherry-pick PR opened successfully.*\\n• author: {mention}\\n• source PR: ${source_pr_url}"
|
||||
details="*Cherry-pick PR opened successfully.*\\n• source PR: ${source_pr_url}"
|
||||
if [ -n "${CHERRY_PICK_PR_URL}" ]; then
|
||||
details="${details}\\n• cherry-pick PR: ${CHERRY_PICK_PR_URL}"
|
||||
fi
|
||||
@@ -221,7 +221,6 @@ jobs:
|
||||
uses: ./.github/actions/slack-notify
|
||||
with:
|
||||
webhook-url: ${{ secrets.CHERRY_PICK_PRS_WEBHOOK }}
|
||||
mention: ${{ needs.resolve-cherry-pick-request.outputs.merged_by }}
|
||||
details: ${{ steps.success-summary.outputs.details }}
|
||||
title: "✅ Automated Cherry-Pick PR Opened"
|
||||
ref-name: ${{ github.event.pull_request.base.ref }}
|
||||
@@ -276,21 +275,20 @@ jobs:
|
||||
else
|
||||
failed_job_label="cherry-pick-to-latest-release"
|
||||
fi
|
||||
details="• author: {mention}\\n• ${failed_job_label}\\n• source PR: ${source_pr_url}\\n• reason: ${reason_text}"
|
||||
failed_jobs="• ${failed_job_label}\\n• source PR: ${source_pr_url}\\n• reason: ${reason_text}"
|
||||
if [ -n "${MERGE_COMMIT_SHA}" ]; then
|
||||
details="${details}\\n• merge SHA: ${MERGE_COMMIT_SHA}"
|
||||
failed_jobs="${failed_jobs}\\n• merge SHA: ${MERGE_COMMIT_SHA}"
|
||||
fi
|
||||
if [ -n "${details_excerpt}" ]; then
|
||||
details="${details}\\n• excerpt: ${details_excerpt}"
|
||||
failed_jobs="${failed_jobs}\\n• excerpt: ${details_excerpt}"
|
||||
fi
|
||||
|
||||
echo "details=${details}" >> "$GITHUB_OUTPUT"
|
||||
echo "jobs=${failed_jobs}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Notify #cherry-pick-prs about cherry-pick failure
|
||||
uses: ./.github/actions/slack-notify
|
||||
with:
|
||||
webhook-url: ${{ secrets.CHERRY_PICK_PRS_WEBHOOK }}
|
||||
mention: ${{ needs.resolve-cherry-pick-request.outputs.merged_by }}
|
||||
details: ${{ steps.failure-summary.outputs.details }}
|
||||
details: ${{ steps.failure-summary.outputs.jobs }}
|
||||
title: "🚨 Automated Cherry-Pick Failed"
|
||||
ref-name: ${{ github.event.pull_request.base.ref }}
|
||||
|
||||
2
.github/workflows/pr-desktop-build.yml
vendored
2
.github/workflows/pr-desktop-build.yml
vendored
@@ -105,7 +105,7 @@ jobs:
|
||||
|
||||
- name: Upload build artifacts
|
||||
if: always()
|
||||
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f
|
||||
with:
|
||||
name: desktop-build-${{ matrix.platform }}-${{ github.run_id }}
|
||||
path: |
|
||||
|
||||
@@ -174,7 +174,7 @@ jobs:
|
||||
|
||||
- name: Upload Docker logs
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f
|
||||
with:
|
||||
name: docker-logs-${{ matrix.test-dir }}
|
||||
path: docker-logs/
|
||||
|
||||
4
.github/workflows/pr-golang-tests.yml
vendored
4
.github/workflows/pr-golang-tests.yml
vendored
@@ -25,7 +25,7 @@ jobs:
|
||||
outputs:
|
||||
modules: ${{ steps.set-modules.outputs.modules }}
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
|
||||
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8
|
||||
with:
|
||||
persist-credentials: false
|
||||
- id: set-modules
|
||||
@@ -39,7 +39,7 @@ jobs:
|
||||
matrix:
|
||||
modules: ${{ fromJSON(needs.detect-modules.outputs.modules) }}
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # ratchet:actions/checkout@v6
|
||||
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # ratchet:actions/checkout@v6
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: actions/setup-go@4dc6199c7b1a012772edbd06daecab0f50c9053c # zizmor: ignore[cache-poisoning]
|
||||
|
||||
6
.github/workflows/pr-integration-tests.yml
vendored
6
.github/workflows/pr-integration-tests.yml
vendored
@@ -466,7 +466,7 @@ jobs:
|
||||
|
||||
- name: Upload logs
|
||||
if: always()
|
||||
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f
|
||||
with:
|
||||
name: docker-all-logs-${{ matrix.edition }}-${{ matrix.test-dir.name }}
|
||||
path: ${{ github.workspace }}/docker-compose.log
|
||||
@@ -587,7 +587,7 @@ jobs:
|
||||
|
||||
- name: Upload logs (onyx-lite)
|
||||
if: always()
|
||||
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f
|
||||
with:
|
||||
name: docker-all-logs-onyx-lite
|
||||
path: ${{ github.workspace }}/docker-compose-onyx-lite.log
|
||||
@@ -725,7 +725,7 @@ jobs:
|
||||
|
||||
- name: Upload logs (multi-tenant)
|
||||
if: always()
|
||||
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f
|
||||
with:
|
||||
name: docker-all-logs-multitenant
|
||||
path: ${{ github.workspace }}/docker-compose-multitenant.log
|
||||
|
||||
2
.github/workflows/pr-jest-tests.yml
vendored
2
.github/workflows/pr-jest-tests.yml
vendored
@@ -44,7 +44,7 @@ jobs:
|
||||
|
||||
- name: Upload coverage reports
|
||||
if: always()
|
||||
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f
|
||||
with:
|
||||
name: jest-coverage-${{ github.run_id }}
|
||||
path: ./web/coverage
|
||||
|
||||
14
.github/workflows/pr-playwright-tests.yml
vendored
14
.github/workflows/pr-playwright-tests.yml
vendored
@@ -445,7 +445,7 @@ jobs:
|
||||
run: |
|
||||
npx playwright test --project ${PROJECT}
|
||||
|
||||
- uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f
|
||||
- uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f
|
||||
if: always()
|
||||
with:
|
||||
# Includes test results and trace.zip files
|
||||
@@ -454,7 +454,7 @@ jobs:
|
||||
retention-days: 30
|
||||
|
||||
- name: Upload screenshots
|
||||
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f
|
||||
if: always()
|
||||
with:
|
||||
name: playwright-screenshots-${{ matrix.project }}-${{ github.run_id }}
|
||||
@@ -534,7 +534,7 @@ jobs:
|
||||
"s3://${PLAYWRIGHT_S3_BUCKET}/reports/pr-${PR_NUMBER}/${RUN_ID}/${PROJECT}/"
|
||||
|
||||
- name: Upload visual diff summary
|
||||
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f
|
||||
if: always()
|
||||
with:
|
||||
name: screenshot-diff-summary-${{ matrix.project }}
|
||||
@@ -543,7 +543,7 @@ jobs:
|
||||
retention-days: 5
|
||||
|
||||
- name: Upload visual diff report artifact
|
||||
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f
|
||||
if: always()
|
||||
with:
|
||||
name: screenshot-diff-report-${{ matrix.project }}-${{ github.run_id }}
|
||||
@@ -590,7 +590,7 @@ jobs:
|
||||
|
||||
- name: Upload logs
|
||||
if: success() || failure()
|
||||
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f
|
||||
with:
|
||||
name: docker-logs-${{ matrix.project }}-${{ github.run_id }}
|
||||
path: ${{ github.workspace }}/docker-compose.log
|
||||
@@ -674,7 +674,7 @@ jobs:
|
||||
working-directory: ./web
|
||||
run: npx playwright test --project lite
|
||||
|
||||
- uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f
|
||||
- uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f
|
||||
if: always()
|
||||
with:
|
||||
name: playwright-test-results-lite-${{ github.run_id }}
|
||||
@@ -692,7 +692,7 @@ jobs:
|
||||
|
||||
- name: Upload logs
|
||||
if: success() || failure()
|
||||
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f
|
||||
with:
|
||||
name: docker-logs-lite-${{ github.run_id }}
|
||||
path: ${{ github.workspace }}/docker-compose.log
|
||||
|
||||
2
.github/workflows/pr-python-model-tests.yml
vendored
2
.github/workflows/pr-python-model-tests.yml
vendored
@@ -122,7 +122,7 @@ jobs:
|
||||
|
||||
- name: Upload logs
|
||||
if: always()
|
||||
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f
|
||||
with:
|
||||
name: docker-all-logs
|
||||
path: ${{ github.workspace }}/docker-compose.log
|
||||
|
||||
@@ -319,7 +319,7 @@ jobs:
|
||||
|
||||
- name: Upload logs
|
||||
if: always()
|
||||
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f
|
||||
with:
|
||||
name: docker-all-logs-nightly-${{ matrix.provider }}-llm-provider
|
||||
path: |
|
||||
|
||||
6
.github/workflows/sandbox-deployment.yml
vendored
6
.github/workflows/sandbox-deployment.yml
vendored
@@ -125,7 +125,7 @@ jobs:
|
||||
|
||||
- name: Docker meta
|
||||
id: meta
|
||||
uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # ratchet:docker/metadata-action@v6.0.0
|
||||
uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # ratchet:docker/metadata-action@v5
|
||||
with:
|
||||
images: ${{ env.REGISTRY_IMAGE }}
|
||||
flavor: |
|
||||
@@ -195,7 +195,7 @@ jobs:
|
||||
|
||||
- name: Docker meta
|
||||
id: meta
|
||||
uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # ratchet:docker/metadata-action@v6.0.0
|
||||
uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # ratchet:docker/metadata-action@v5
|
||||
with:
|
||||
images: ${{ env.REGISTRY_IMAGE }}
|
||||
flavor: |
|
||||
@@ -268,7 +268,7 @@ jobs:
|
||||
|
||||
- name: Docker meta
|
||||
id: meta
|
||||
uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # ratchet:docker/metadata-action@v6.0.0
|
||||
uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # ratchet:docker/metadata-action@v5
|
||||
with:
|
||||
images: ${{ env.REGISTRY_IMAGE }}
|
||||
flavor: |
|
||||
|
||||
@@ -1,103 +0,0 @@
|
||||
"""add_hook_and_hook_execution_log_tables
|
||||
|
||||
Revision ID: 689433b0d8de
|
||||
Revises: 93a2e195e25c
|
||||
Create Date: 2026-03-13 11:25:06.547474
|
||||
|
||||
"""
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
from sqlalchemy.dialects.postgresql import UUID as PGUUID
|
||||
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = "689433b0d8de"
|
||||
down_revision = "93a2e195e25c"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"hook",
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("name", sa.String(), nullable=False),
|
||||
sa.Column(
|
||||
"hook_point",
|
||||
sa.Enum("document_ingestion", "query_processing", native_enum=False),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column("endpoint_url", sa.Text(), nullable=True),
|
||||
sa.Column("api_key", sa.LargeBinary(), nullable=True),
|
||||
sa.Column("is_reachable", sa.Boolean(), nullable=True),
|
||||
sa.Column(
|
||||
"fail_strategy",
|
||||
sa.Enum("hard", "soft", native_enum=False),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column("timeout_seconds", sa.Float(), nullable=False),
|
||||
sa.Column(
|
||||
"is_active", sa.Boolean(), nullable=False, server_default=sa.text("false")
|
||||
),
|
||||
sa.Column(
|
||||
"deleted", sa.Boolean(), nullable=False, server_default=sa.text("false")
|
||||
),
|
||||
sa.Column("creator_id", PGUUID(as_uuid=True), nullable=True),
|
||||
sa.Column(
|
||||
"created_at",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"updated_at",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.ForeignKeyConstraint(["creator_id"], ["user.id"], ondelete="SET NULL"),
|
||||
sa.PrimaryKeyConstraint("id"),
|
||||
)
|
||||
op.create_index(
|
||||
"ix_hook_one_non_deleted_per_point",
|
||||
"hook",
|
||||
["hook_point"],
|
||||
unique=True,
|
||||
postgresql_where=sa.text("deleted = false"),
|
||||
)
|
||||
|
||||
op.create_table(
|
||||
"hook_execution_log",
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("hook_id", sa.Integer(), nullable=False),
|
||||
sa.Column(
|
||||
"is_success",
|
||||
sa.Boolean(),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column("error_message", sa.Text(), nullable=True),
|
||||
sa.Column("status_code", sa.Integer(), nullable=True),
|
||||
sa.Column("duration_ms", sa.Integer(), nullable=True),
|
||||
sa.Column(
|
||||
"created_at",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.ForeignKeyConstraint(["hook_id"], ["hook.id"], ondelete="CASCADE"),
|
||||
sa.PrimaryKeyConstraint("id"),
|
||||
)
|
||||
op.create_index("ix_hook_execution_log_hook_id", "hook_execution_log", ["hook_id"])
|
||||
op.create_index(
|
||||
"ix_hook_execution_log_created_at", "hook_execution_log", ["created_at"]
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index("ix_hook_execution_log_created_at", table_name="hook_execution_log")
|
||||
op.drop_index("ix_hook_execution_log_hook_id", table_name="hook_execution_log")
|
||||
op.drop_table("hook_execution_log")
|
||||
|
||||
op.drop_index("ix_hook_one_non_deleted_per_point", table_name="hook")
|
||||
op.drop_table("hook")
|
||||
@@ -118,7 +118,9 @@ JWT_PUBLIC_KEY_URL: str | None = os.getenv("JWT_PUBLIC_KEY_URL", None)
|
||||
SUPER_USERS = json.loads(os.environ.get("SUPER_USERS", "[]"))
|
||||
SUPER_CLOUD_API_KEY = os.environ.get("SUPER_CLOUD_API_KEY", "api_key")
|
||||
|
||||
POSTHOG_API_KEY = os.environ.get("POSTHOG_API_KEY")
|
||||
# The posthog client does not accept empty API keys or hosts however it fails silently
|
||||
# when the capture is called. These defaults prevent Posthog issues from breaking the Onyx app
|
||||
POSTHOG_API_KEY = os.environ.get("POSTHOG_API_KEY") or "FooBar"
|
||||
POSTHOG_HOST = os.environ.get("POSTHOG_HOST") or "https://us.i.posthog.com"
|
||||
POSTHOG_DEBUG_LOGS_ENABLED = (
|
||||
os.environ.get("POSTHOG_DEBUG_LOGS_ENABLED", "").lower() == "true"
|
||||
|
||||
@@ -34,9 +34,6 @@ class PostHogFeatureFlagProvider(FeatureFlagProvider):
|
||||
Returns:
|
||||
True if the feature is enabled for the user, False otherwise.
|
||||
"""
|
||||
if not posthog:
|
||||
return False
|
||||
|
||||
try:
|
||||
posthog.set(
|
||||
distinct_id=user_id,
|
||||
|
||||
@@ -29,6 +29,7 @@ from onyx.configs.app_configs import OPENAI_DEFAULT_API_KEY
|
||||
from onyx.configs.app_configs import OPENROUTER_DEFAULT_API_KEY
|
||||
from onyx.configs.app_configs import VERTEXAI_DEFAULT_CREDENTIALS
|
||||
from onyx.configs.app_configs import VERTEXAI_DEFAULT_LOCATION
|
||||
from onyx.configs.constants import MilestoneRecordType
|
||||
from onyx.db.engine.sql_engine import get_session_with_shared_schema
|
||||
from onyx.db.engine.sql_engine import get_session_with_tenant
|
||||
from onyx.db.image_generation import create_default_image_gen_config_from_api_key
|
||||
@@ -58,6 +59,7 @@ from onyx.server.manage.llm.models import LLMProviderUpsertRequest
|
||||
from onyx.server.manage.llm.models import ModelConfigurationUpsertRequest
|
||||
from onyx.setup import setup_onyx
|
||||
from onyx.utils.logger import setup_logger
|
||||
from onyx.utils.telemetry import mt_cloud_telemetry
|
||||
from shared_configs.configs import MULTI_TENANT
|
||||
from shared_configs.configs import POSTGRES_DEFAULT_SCHEMA
|
||||
from shared_configs.configs import TENANT_ID_PREFIX
|
||||
@@ -69,9 +71,7 @@ logger = setup_logger()
|
||||
|
||||
|
||||
async def get_or_provision_tenant(
|
||||
email: str,
|
||||
referral_source: str | None = None,
|
||||
request: Request | None = None,
|
||||
email: str, referral_source: str | None = None, request: Request | None = None
|
||||
) -> str:
|
||||
"""
|
||||
Get existing tenant ID for an email or create a new tenant if none exists.
|
||||
@@ -693,6 +693,12 @@ async def assign_tenant_to_user(
|
||||
|
||||
try:
|
||||
add_users_to_tenant([email], tenant_id)
|
||||
|
||||
mt_cloud_telemetry(
|
||||
tenant_id=tenant_id,
|
||||
distinct_id=email,
|
||||
event=MilestoneRecordType.TENANT_CREATED,
|
||||
)
|
||||
except Exception:
|
||||
logger.exception(f"Failed to assign tenant {tenant_id} to user {email}")
|
||||
raise Exception("Failed to assign tenant to user")
|
||||
|
||||
@@ -9,7 +9,6 @@ from ee.onyx.configs.app_configs import POSTHOG_API_KEY
|
||||
from ee.onyx.configs.app_configs import POSTHOG_DEBUG_LOGS_ENABLED
|
||||
from ee.onyx.configs.app_configs import POSTHOG_HOST
|
||||
from onyx.utils.logger import setup_logger
|
||||
from shared_configs.configs import MULTI_TENANT
|
||||
|
||||
logger = setup_logger()
|
||||
|
||||
@@ -19,19 +18,12 @@ def posthog_on_error(error: Any, items: Any) -> None:
|
||||
logger.error(f"PostHog error: {error}, items: {items}")
|
||||
|
||||
|
||||
posthog: Posthog | None = None
|
||||
if POSTHOG_API_KEY:
|
||||
posthog = Posthog(
|
||||
project_api_key=POSTHOG_API_KEY,
|
||||
host=POSTHOG_HOST,
|
||||
debug=POSTHOG_DEBUG_LOGS_ENABLED,
|
||||
on_error=posthog_on_error,
|
||||
)
|
||||
elif MULTI_TENANT:
|
||||
logger.warning(
|
||||
"POSTHOG_API_KEY is not set but MULTI_TENANT is enabled — "
|
||||
"PostHog telemetry and feature flags will be disabled"
|
||||
)
|
||||
posthog = Posthog(
|
||||
project_api_key=POSTHOG_API_KEY,
|
||||
host=POSTHOG_HOST,
|
||||
debug=POSTHOG_DEBUG_LOGS_ENABLED,
|
||||
on_error=posthog_on_error,
|
||||
)
|
||||
|
||||
# For cross referencing between cloud and www Onyx sites
|
||||
# NOTE: These clients are separate because they are separate posthog projects.
|
||||
@@ -68,7 +60,7 @@ def capture_and_sync_with_alternate_posthog(
|
||||
logger.error(f"Error capturing marketing posthog event: {e}")
|
||||
|
||||
try:
|
||||
if posthog and (cloud_user_id := props.get("onyx_cloud_user_id")):
|
||||
if cloud_user_id := props.get("onyx_cloud_user_id"):
|
||||
cloud_props = props.copy()
|
||||
cloud_props.pop("onyx_cloud_user_id", None)
|
||||
|
||||
|
||||
@@ -1,5 +1,3 @@
|
||||
from typing import Any
|
||||
|
||||
from ee.onyx.utils.posthog_client import posthog
|
||||
from onyx.utils.logger import setup_logger
|
||||
|
||||
@@ -7,27 +5,12 @@ logger = setup_logger()
|
||||
|
||||
|
||||
def event_telemetry(
|
||||
distinct_id: str, event: str, properties: dict[str, Any] | None = None
|
||||
distinct_id: str, event: str, properties: dict | None = None
|
||||
) -> None:
|
||||
"""Capture and send an event to PostHog, flushing immediately."""
|
||||
if not posthog:
|
||||
return
|
||||
|
||||
logger.info(f"Capturing PostHog event: {distinct_id} {event} {properties}")
|
||||
try:
|
||||
posthog.capture(distinct_id, event, properties)
|
||||
posthog.flush()
|
||||
except Exception as e:
|
||||
logger.error(f"Error capturing PostHog event: {e}")
|
||||
|
||||
|
||||
def identify_user(distinct_id: str, properties: dict[str, Any] | None = None) -> None:
|
||||
"""Create/update a PostHog person profile, flushing immediately."""
|
||||
if not posthog:
|
||||
return
|
||||
|
||||
try:
|
||||
posthog.identify(distinct_id, properties)
|
||||
posthog.flush()
|
||||
except Exception as e:
|
||||
logger.error(f"Error identifying PostHog user: {e}")
|
||||
|
||||
@@ -19,7 +19,6 @@ from typing import Optional
|
||||
from typing import Protocol
|
||||
from typing import Tuple
|
||||
from typing import TypeVar
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import jwt
|
||||
from email_validator import EmailNotValidError
|
||||
@@ -135,7 +134,6 @@ from onyx.redis.redis_pool import retrieve_ws_token_data
|
||||
from onyx.server.settings.store import load_settings
|
||||
from onyx.server.utils import BasicAuthenticationError
|
||||
from onyx.utils.logger import setup_logger
|
||||
from onyx.utils.telemetry import mt_cloud_identify
|
||||
from onyx.utils.telemetry import mt_cloud_telemetry
|
||||
from onyx.utils.telemetry import optional_telemetry
|
||||
from onyx.utils.telemetry import RecordType
|
||||
@@ -794,12 +792,6 @@ class UserManager(UUIDIDMixin, BaseUserManager[User, uuid.UUID]):
|
||||
except Exception:
|
||||
logger.exception("Error deleting anonymous user cookie")
|
||||
|
||||
tenant_id = CURRENT_TENANT_ID_CONTEXTVAR.get()
|
||||
mt_cloud_identify(
|
||||
distinct_id=str(user.id),
|
||||
properties={"email": user.email, "tenant_id": tenant_id},
|
||||
)
|
||||
|
||||
async def on_after_register(
|
||||
self, user: User, request: Optional[Request] = None
|
||||
) -> None:
|
||||
@@ -818,25 +810,12 @@ class UserManager(UUIDIDMixin, BaseUserManager[User, uuid.UUID]):
|
||||
user_count = await get_user_count()
|
||||
logger.debug(f"Current tenant user count: {user_count}")
|
||||
|
||||
# Ensure a PostHog person profile exists for this user.
|
||||
mt_cloud_identify(
|
||||
distinct_id=str(user.id),
|
||||
properties={"email": user.email, "tenant_id": tenant_id},
|
||||
)
|
||||
|
||||
mt_cloud_telemetry(
|
||||
tenant_id=tenant_id,
|
||||
distinct_id=str(user.id),
|
||||
distinct_id=user.email,
|
||||
event=MilestoneRecordType.USER_SIGNED_UP,
|
||||
)
|
||||
|
||||
if user_count == 1:
|
||||
mt_cloud_telemetry(
|
||||
tenant_id=tenant_id,
|
||||
distinct_id=str(user.id),
|
||||
event=MilestoneRecordType.TENANT_CREATED,
|
||||
)
|
||||
|
||||
finally:
|
||||
CURRENT_TENANT_ID_CONTEXTVAR.reset(token)
|
||||
|
||||
@@ -1673,33 +1652,6 @@ async def _get_user_from_token_data(token_data: dict) -> User | None:
|
||||
return user
|
||||
|
||||
|
||||
_LOOPBACK_HOSTNAMES = frozenset({"localhost", "127.0.0.1", "::1"})
|
||||
|
||||
|
||||
def _is_same_origin(actual: str, expected: str) -> bool:
|
||||
"""Compare two origins for the WebSocket CSWSH check.
|
||||
|
||||
Scheme and hostname must match exactly. Port must also match, except
|
||||
when the hostname is a loopback address (localhost / 127.0.0.1 / ::1),
|
||||
where port is ignored. On loopback, all ports belong to the same
|
||||
operator, so port differences carry no security significance — the
|
||||
CSWSH threat is remote origins, not local ones.
|
||||
"""
|
||||
a = urlparse(actual.rstrip("/"))
|
||||
e = urlparse(expected.rstrip("/"))
|
||||
|
||||
if a.scheme != e.scheme or a.hostname != e.hostname:
|
||||
return False
|
||||
|
||||
if a.hostname in _LOOPBACK_HOSTNAMES:
|
||||
return True
|
||||
|
||||
actual_port = a.port or (443 if a.scheme == "https" else 80)
|
||||
expected_port = e.port or (443 if e.scheme == "https" else 80)
|
||||
|
||||
return actual_port == expected_port
|
||||
|
||||
|
||||
async def current_user_from_websocket(
|
||||
websocket: WebSocket,
|
||||
token: str = Query(..., description="WebSocket authentication token"),
|
||||
@@ -1719,15 +1671,19 @@ async def current_user_from_websocket(
|
||||
|
||||
This applies the same auth checks as current_user() for HTTP endpoints.
|
||||
"""
|
||||
# Check Origin header to prevent Cross-Site WebSocket Hijacking (CSWSH).
|
||||
# Browsers always send Origin on WebSocket connections.
|
||||
# Check Origin header to prevent Cross-Site WebSocket Hijacking (CSWSH)
|
||||
# Browsers always send Origin on WebSocket connections
|
||||
origin = websocket.headers.get("origin")
|
||||
expected_origin = WEB_DOMAIN.rstrip("/")
|
||||
if not origin:
|
||||
logger.warning("WS auth: missing Origin header")
|
||||
raise BasicAuthenticationError(detail="Access denied. Missing origin.")
|
||||
|
||||
if not _is_same_origin(origin, WEB_DOMAIN):
|
||||
logger.warning(f"WS auth: origin mismatch. Expected {WEB_DOMAIN}, got {origin}")
|
||||
actual_origin = origin.rstrip("/")
|
||||
if actual_origin != expected_origin:
|
||||
logger.warning(
|
||||
f"WS auth: origin mismatch. Expected {expected_origin}, got {actual_origin}"
|
||||
)
|
||||
raise BasicAuthenticationError(detail="Access denied. Invalid origin.")
|
||||
|
||||
# Validate WS token in Redis (single-use, deleted after retrieval)
|
||||
|
||||
@@ -317,7 +317,6 @@ celery_app.autodiscover_tasks(
|
||||
"onyx.background.celery.tasks.docprocessing",
|
||||
"onyx.background.celery.tasks.evals",
|
||||
"onyx.background.celery.tasks.hierarchyfetching",
|
||||
"onyx.background.celery.tasks.hooks",
|
||||
"onyx.background.celery.tasks.periodic",
|
||||
"onyx.background.celery.tasks.pruning",
|
||||
"onyx.background.celery.tasks.shared",
|
||||
|
||||
@@ -9,7 +9,6 @@ from onyx.configs.app_configs import AUTO_LLM_UPDATE_INTERVAL_SECONDS
|
||||
from onyx.configs.app_configs import DISABLE_VECTOR_DB
|
||||
from onyx.configs.app_configs import ENABLE_OPENSEARCH_INDEXING_FOR_ONYX
|
||||
from onyx.configs.app_configs import ENTERPRISE_EDITION_ENABLED
|
||||
from onyx.configs.app_configs import HOOK_ENABLED
|
||||
from onyx.configs.app_configs import SCHEDULED_EVAL_DATASET_NAMES
|
||||
from onyx.configs.constants import ONYX_CLOUD_CELERY_TASK_PREFIX
|
||||
from onyx.configs.constants import OnyxCeleryPriority
|
||||
@@ -362,19 +361,6 @@ if not MULTI_TENANT:
|
||||
|
||||
tasks_to_schedule.extend(beat_task_templates)
|
||||
|
||||
if not MULTI_TENANT and HOOK_ENABLED:
|
||||
tasks_to_schedule.append(
|
||||
{
|
||||
"name": "hook-execution-log-cleanup",
|
||||
"task": OnyxCeleryTask.HOOK_EXECUTION_LOG_CLEANUP_TASK,
|
||||
"schedule": timedelta(days=1),
|
||||
"options": {
|
||||
"priority": OnyxCeleryPriority.LOW,
|
||||
"expires": BEAT_EXPIRES_DEFAULT,
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def generate_cloud_tasks(
|
||||
beat_tasks: list[dict], beat_templates: list[dict], beat_multiplier: float
|
||||
|
||||
@@ -29,8 +29,6 @@ from onyx.configs.constants import OnyxCeleryPriority
|
||||
from onyx.configs.constants import OnyxCeleryQueues
|
||||
from onyx.configs.constants import OnyxCeleryTask
|
||||
from onyx.configs.constants import OnyxRedisLocks
|
||||
from onyx.connectors.factory import ConnectorMissingException
|
||||
from onyx.connectors.factory import identify_connector_class
|
||||
from onyx.connectors.factory import instantiate_connector
|
||||
from onyx.connectors.interfaces import HierarchyConnector
|
||||
from onyx.connectors.models import HierarchyNode as PydanticHierarchyNode
|
||||
@@ -57,26 +55,6 @@ logger = setup_logger()
|
||||
HIERARCHY_FETCH_INTERVAL_SECONDS = 24 * 60 * 60
|
||||
|
||||
|
||||
def _connector_supports_hierarchy_fetching(
|
||||
cc_pair: ConnectorCredentialPair,
|
||||
) -> bool:
|
||||
"""Return True only for connectors whose class implements HierarchyConnector."""
|
||||
try:
|
||||
connector_class = identify_connector_class(
|
||||
cc_pair.connector.source,
|
||||
)
|
||||
except ConnectorMissingException as e:
|
||||
task_logger.warning(
|
||||
"Skipping hierarchy fetching enqueue for source=%s input_type=%s: %s",
|
||||
cc_pair.connector.source,
|
||||
cc_pair.connector.input_type,
|
||||
str(e),
|
||||
)
|
||||
return False
|
||||
|
||||
return issubclass(connector_class, HierarchyConnector)
|
||||
|
||||
|
||||
def _is_hierarchy_fetching_due(cc_pair: ConnectorCredentialPair) -> bool:
|
||||
"""Returns boolean indicating if hierarchy fetching is due for this connector.
|
||||
|
||||
@@ -208,10 +186,7 @@ def check_for_hierarchy_fetching(self: Task, *, tenant_id: str) -> int | None:
|
||||
cc_pair_id=cc_pair_id,
|
||||
)
|
||||
|
||||
if not cc_pair or not _connector_supports_hierarchy_fetching(cc_pair):
|
||||
continue
|
||||
|
||||
if not _is_hierarchy_fetching_due(cc_pair):
|
||||
if not cc_pair or not _is_hierarchy_fetching_due(cc_pair):
|
||||
continue
|
||||
|
||||
task_id = _try_creating_hierarchy_fetching_task(
|
||||
|
||||
@@ -1,35 +0,0 @@
|
||||
from celery import shared_task
|
||||
|
||||
from onyx.configs.app_configs import JOB_TIMEOUT
|
||||
from onyx.configs.constants import OnyxCeleryTask
|
||||
from onyx.db.engine.sql_engine import get_session_with_current_tenant
|
||||
from onyx.db.hook import cleanup_old_execution_logs__no_commit
|
||||
from onyx.utils.logger import setup_logger
|
||||
|
||||
logger = setup_logger()
|
||||
|
||||
_HOOK_EXECUTION_LOG_RETENTION_DAYS: int = 30
|
||||
|
||||
|
||||
@shared_task(
|
||||
name=OnyxCeleryTask.HOOK_EXECUTION_LOG_CLEANUP_TASK,
|
||||
ignore_result=True,
|
||||
soft_time_limit=JOB_TIMEOUT,
|
||||
trail=False,
|
||||
)
|
||||
def hook_execution_log_cleanup_task(*, tenant_id: str) -> None: # noqa: ARG001
|
||||
try:
|
||||
with get_session_with_current_tenant() as db_session:
|
||||
deleted: int = cleanup_old_execution_logs__no_commit(
|
||||
db_session=db_session,
|
||||
max_age_days=_HOOK_EXECUTION_LOG_RETENTION_DAYS,
|
||||
)
|
||||
db_session.commit()
|
||||
if deleted:
|
||||
logger.info(
|
||||
f"Deleted {deleted} hook execution log(s) older than "
|
||||
f"{_HOOK_EXECUTION_LOG_RETENTION_DAYS} days."
|
||||
)
|
||||
except Exception:
|
||||
logger.exception("Failed to clean up hook execution logs")
|
||||
raise
|
||||
@@ -36,11 +36,9 @@ from onyx.db.memory import add_memory
|
||||
from onyx.db.memory import update_memory_at_index
|
||||
from onyx.db.memory import UserMemoryContext
|
||||
from onyx.db.models import Persona
|
||||
from onyx.llm.constants import LlmProviderNames
|
||||
from onyx.llm.interfaces import LLM
|
||||
from onyx.llm.interfaces import LLMUserIdentity
|
||||
from onyx.llm.interfaces import ToolChoiceOptions
|
||||
from onyx.llm.utils import is_true_openai_model
|
||||
from onyx.prompts.chat_prompts import IMAGE_GEN_REMINDER
|
||||
from onyx.prompts.chat_prompts import OPEN_URL_REMINDER
|
||||
from onyx.server.query_and_chat.placement import Placement
|
||||
@@ -74,70 +72,6 @@ from shared_configs.contextvars import get_current_tenant_id
|
||||
logger = setup_logger()
|
||||
|
||||
|
||||
class EmptyLLMResponseError(RuntimeError):
|
||||
"""Raised when the streamed LLM response completes without a usable answer."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
provider: str,
|
||||
model: str,
|
||||
tool_choice: ToolChoiceOptions,
|
||||
client_error_msg: str,
|
||||
error_code: str = "EMPTY_LLM_RESPONSE",
|
||||
is_retryable: bool = True,
|
||||
) -> None:
|
||||
super().__init__(client_error_msg)
|
||||
self.provider = provider
|
||||
self.model = model
|
||||
self.tool_choice = tool_choice
|
||||
self.client_error_msg = client_error_msg
|
||||
self.error_code = error_code
|
||||
self.is_retryable = is_retryable
|
||||
|
||||
|
||||
def _build_empty_llm_response_error(
|
||||
llm: LLM,
|
||||
llm_step_result: LlmStepResult,
|
||||
tool_choice: ToolChoiceOptions,
|
||||
) -> EmptyLLMResponseError:
|
||||
provider = llm.config.model_provider
|
||||
model = llm.config.model_name
|
||||
|
||||
# OpenAI quota exhaustion has reached us as a streamed "stop" with zero content.
|
||||
# When the stream is completely empty and there is no reasoning/tool output, surface
|
||||
# the likely account-level cause instead of a generic tool-calling error.
|
||||
if (
|
||||
not llm_step_result.reasoning
|
||||
and provider == LlmProviderNames.OPENAI
|
||||
and is_true_openai_model(provider, model)
|
||||
):
|
||||
return EmptyLLMResponseError(
|
||||
provider=provider,
|
||||
model=model,
|
||||
tool_choice=tool_choice,
|
||||
client_error_msg=(
|
||||
"The selected OpenAI model returned an empty streamed response "
|
||||
"before producing any tokens. This commonly happens when the API "
|
||||
"key or project has no remaining quota or billing is not enabled. "
|
||||
"Verify quota and billing for this key and try again."
|
||||
),
|
||||
error_code="BUDGET_EXCEEDED",
|
||||
is_retryable=False,
|
||||
)
|
||||
|
||||
return EmptyLLMResponseError(
|
||||
provider=provider,
|
||||
model=model,
|
||||
tool_choice=tool_choice,
|
||||
client_error_msg=(
|
||||
"The selected model returned no final answer before the stream "
|
||||
"completed. No text or tool calls were received from the upstream "
|
||||
"provider."
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _looks_like_xml_tool_call_payload(text: str | None) -> bool:
|
||||
"""Detect XML-style marshaled tool calls emitted as plain text."""
|
||||
if not text:
|
||||
@@ -679,12 +613,7 @@ def run_llm_loop(
|
||||
)
|
||||
citation_processor.update_citation_mapping(project_citation_mapping)
|
||||
|
||||
llm_step_result = LlmStepResult(
|
||||
reasoning=None,
|
||||
answer=None,
|
||||
tool_calls=None,
|
||||
raw_answer=None,
|
||||
)
|
||||
llm_step_result: LlmStepResult | None = None
|
||||
|
||||
# Pass the total budget to construct_message_history, which will handle token allocation
|
||||
available_tokens = llm.config.max_input_tokens
|
||||
@@ -1155,18 +1084,12 @@ def run_llm_loop(
|
||||
# As long as 1 tool with citeable documents is called at any point, we ask the LLM to try to cite
|
||||
should_cite_documents = True
|
||||
|
||||
if not llm_step_result.answer and not llm_step_result.tool_calls:
|
||||
raise _build_empty_llm_response_error(
|
||||
llm=llm,
|
||||
llm_step_result=llm_step_result,
|
||||
tool_choice=tool_choice,
|
||||
)
|
||||
|
||||
if not llm_step_result.answer:
|
||||
if not llm_step_result or not llm_step_result.answer:
|
||||
raise RuntimeError(
|
||||
"The LLM did not return a final answer after tool execution. "
|
||||
"Typically this indicates invalid tool-call output, a model/provider mismatch, "
|
||||
"or serving API misconfiguration."
|
||||
"The LLM did not return an answer. "
|
||||
"Typically this is an issue with LLMs that do not support tool calling natively, "
|
||||
"or the model serving API is not configured correctly. "
|
||||
"This may also happen with models that are lower quality outputting invalid tool calls."
|
||||
)
|
||||
|
||||
emitter.emit(
|
||||
|
||||
@@ -1013,10 +1013,6 @@ def run_llm_step_pkt_generator(
|
||||
accumulated_reasoning = ""
|
||||
accumulated_answer = ""
|
||||
accumulated_raw_answer = ""
|
||||
stream_chunk_count = 0
|
||||
actionable_chunk_count = 0
|
||||
empty_chunk_count = 0
|
||||
finish_reasons: set[str] = set()
|
||||
xml_tool_call_content_filter = _XmlToolCallContentFilter()
|
||||
|
||||
processor_state: Any = None
|
||||
@@ -1149,7 +1145,6 @@ def run_llm_step_pkt_generator(
|
||||
user_identity=user_identity,
|
||||
timeout_override=timeout_override,
|
||||
):
|
||||
stream_chunk_count += 1
|
||||
if packet.usage:
|
||||
usage = packet.usage
|
||||
span_generation.span_data.usage = {
|
||||
@@ -1159,21 +1154,16 @@ def run_llm_step_pkt_generator(
|
||||
"cache_creation_input_tokens": usage.cache_creation_input_tokens,
|
||||
}
|
||||
# Note: LLM cost tracking is now handled in multi_llm.py
|
||||
finish_reason = packet.choice.finish_reason
|
||||
if finish_reason:
|
||||
finish_reasons.add(str(finish_reason))
|
||||
delta = packet.choice.delta
|
||||
|
||||
# Weird behavior from some model providers, just log and ignore for now
|
||||
if (
|
||||
not delta.content
|
||||
delta.content is None
|
||||
and delta.reasoning_content is None
|
||||
and not delta.tool_calls
|
||||
and delta.tool_calls is None
|
||||
):
|
||||
empty_chunk_count += 1
|
||||
logger.warning(
|
||||
"LLM packet is empty (no content, reasoning, or tool calls). "
|
||||
f"finish_reason={finish_reason}. Skipping: {packet}"
|
||||
f"LLM packet is empty (no contents, reasoning or tool calls). Skipping: {packet}"
|
||||
)
|
||||
continue
|
||||
|
||||
@@ -1182,8 +1172,6 @@ def run_llm_step_pkt_generator(
|
||||
time.monotonic() - stream_start_time
|
||||
)
|
||||
first_action_recorded = True
|
||||
if _delta_has_action(delta):
|
||||
actionable_chunk_count += 1
|
||||
|
||||
if custom_token_processor:
|
||||
# The custom token processor can modify the deltas for specific custom logic
|
||||
@@ -1319,15 +1307,6 @@ def run_llm_step_pkt_generator(
|
||||
else:
|
||||
logger.debug("Tool calls: []")
|
||||
|
||||
if actionable_chunk_count == 0:
|
||||
logger.warning(
|
||||
"LLM stream completed with no actionable deltas. "
|
||||
f"chunks={stream_chunk_count}, empty_chunks={empty_chunk_count}, "
|
||||
f"finish_reasons={sorted(finish_reasons)}, "
|
||||
f"provider={llm.config.model_provider}, model={llm.config.model_name}, "
|
||||
f"tool_choice={tool_choice}, tools_sent={len(tool_definitions)}"
|
||||
)
|
||||
|
||||
return (
|
||||
LlmStepResult(
|
||||
reasoning=accumulated_reasoning if accumulated_reasoning else None,
|
||||
|
||||
@@ -29,7 +29,6 @@ from onyx.chat.compression import compress_chat_history
|
||||
from onyx.chat.compression import find_summary_for_branch
|
||||
from onyx.chat.compression import get_compression_params
|
||||
from onyx.chat.emitter import get_default_emitter
|
||||
from onyx.chat.llm_loop import EmptyLLMResponseError
|
||||
from onyx.chat.llm_loop import run_llm_loop
|
||||
from onyx.chat.models import AnswerStream
|
||||
from onyx.chat.models import ChatBasicResponse
|
||||
@@ -491,13 +490,13 @@ def handle_stream_message_objects(
|
||||
# Milestone tracking, most devs using the API don't need to understand this
|
||||
mt_cloud_telemetry(
|
||||
tenant_id=tenant_id,
|
||||
distinct_id=str(user.id) if not user.is_anonymous else tenant_id,
|
||||
distinct_id=user.email if not user.is_anonymous else tenant_id,
|
||||
event=MilestoneRecordType.MULTIPLE_ASSISTANTS,
|
||||
)
|
||||
|
||||
mt_cloud_telemetry(
|
||||
tenant_id=tenant_id,
|
||||
distinct_id=str(user.id) if not user.is_anonymous else tenant_id,
|
||||
distinct_id=user.email if not user.is_anonymous else tenant_id,
|
||||
event=MilestoneRecordType.USER_MESSAGE_SENT,
|
||||
properties={
|
||||
"origin": new_msg_req.origin.value,
|
||||
@@ -926,28 +925,9 @@ def handle_stream_message_objects(
|
||||
db_session.rollback()
|
||||
return
|
||||
|
||||
except EmptyLLMResponseError as e:
|
||||
stack_trace = traceback.format_exc()
|
||||
|
||||
logger.warning(
|
||||
"LLM returned an empty response "
|
||||
f"(provider={e.provider}, model={e.model}, tool_choice={e.tool_choice})"
|
||||
)
|
||||
|
||||
yield StreamingError(
|
||||
error=e.client_error_msg,
|
||||
stack_trace=stack_trace,
|
||||
error_code=e.error_code,
|
||||
is_retryable=e.is_retryable,
|
||||
details={
|
||||
"model": e.model,
|
||||
"provider": e.provider,
|
||||
"tool_choice": e.tool_choice.value,
|
||||
},
|
||||
)
|
||||
db_session.rollback()
|
||||
except Exception as e:
|
||||
logger.exception(f"Failed to process chat message due to {e}")
|
||||
error_msg = str(e)
|
||||
stack_trace = traceback.format_exc()
|
||||
|
||||
if llm:
|
||||
@@ -1066,46 +1046,10 @@ def llm_loop_completion_handle(
|
||||
)
|
||||
|
||||
|
||||
_CITATION_LINK_START_PATTERN = re.compile(r"\s*\[\[\d+\]\]\(")
|
||||
|
||||
|
||||
def _find_markdown_link_end(text: str, destination_start: int) -> int | None:
|
||||
depth = 0
|
||||
i = destination_start
|
||||
|
||||
while i < len(text):
|
||||
curr = text[i]
|
||||
if curr == "\\":
|
||||
i += 2
|
||||
continue
|
||||
|
||||
if curr == "(":
|
||||
depth += 1
|
||||
elif curr == ")":
|
||||
if depth == 0:
|
||||
return i
|
||||
depth -= 1
|
||||
|
||||
i += 1
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def remove_answer_citations(answer: str) -> str:
|
||||
stripped_parts: list[str] = []
|
||||
cursor = 0
|
||||
pattern = r"\s*\[\[\d+\]\]\(http[s]?://[^\s]+\)"
|
||||
|
||||
while match := _CITATION_LINK_START_PATTERN.search(answer, cursor):
|
||||
stripped_parts.append(answer[cursor : match.start()])
|
||||
link_end = _find_markdown_link_end(answer, match.end())
|
||||
if link_end is None:
|
||||
stripped_parts.append(answer[match.start() :])
|
||||
return "".join(stripped_parts)
|
||||
|
||||
cursor = link_end + 1
|
||||
|
||||
stripped_parts.append(answer[cursor:])
|
||||
return "".join(stripped_parts)
|
||||
return re.sub(pattern, "", answer)
|
||||
|
||||
|
||||
@log_function_time()
|
||||
@@ -1143,11 +1087,8 @@ def gather_stream(
|
||||
raise ValueError("Message ID is required")
|
||||
|
||||
if answer is None:
|
||||
if error_msg is not None:
|
||||
answer = ""
|
||||
else:
|
||||
# This should never be the case as these non-streamed flows do not have a stop-generation signal
|
||||
raise RuntimeError("Answer was not generated")
|
||||
# This should never be the case as these non-streamed flows do not have a stop-generation signal
|
||||
raise RuntimeError("Answer was not generated")
|
||||
|
||||
return ChatBasicResponse(
|
||||
answer=answer,
|
||||
|
||||
@@ -1046,8 +1046,6 @@ POD_NAMESPACE = os.environ.get("POD_NAMESPACE")
|
||||
|
||||
DEV_MODE = os.environ.get("DEV_MODE", "").lower() == "true"
|
||||
|
||||
HOOK_ENABLED = os.environ.get("HOOK_ENABLED", "").lower() == "true"
|
||||
|
||||
INTEGRATION_TESTS_MODE = os.environ.get("INTEGRATION_TESTS_MODE", "").lower() == "true"
|
||||
|
||||
#####
|
||||
|
||||
@@ -597,9 +597,6 @@ class OnyxCeleryTask:
|
||||
EXPORT_QUERY_HISTORY_TASK = "export_query_history_task"
|
||||
EXPORT_QUERY_HISTORY_CLEANUP_TASK = "export_query_history_cleanup_task"
|
||||
|
||||
# Hook execution log retention
|
||||
HOOK_EXECUTION_LOG_CLEANUP_TASK = "hook_execution_log_cleanup_task"
|
||||
|
||||
# Sandbox cleanup
|
||||
CLEANUP_IDLE_SANDBOXES = "cleanup_idle_sandboxes"
|
||||
CLEANUP_OLD_SNAPSHOTS = "cleanup_old_snapshots"
|
||||
|
||||
@@ -511,7 +511,7 @@ def add_credential_to_connector(
|
||||
user: User,
|
||||
connector_id: int,
|
||||
credential_id: int,
|
||||
cc_pair_name: str,
|
||||
cc_pair_name: str | None,
|
||||
access_type: AccessType,
|
||||
groups: list[int] | None,
|
||||
auto_sync_options: dict | None = None,
|
||||
|
||||
@@ -304,13 +304,3 @@ class LLMModelFlowType(str, PyEnum):
|
||||
CHAT = "chat"
|
||||
VISION = "vision"
|
||||
CONTEXTUAL_RAG = "contextual_rag"
|
||||
|
||||
|
||||
class HookPoint(str, PyEnum):
|
||||
DOCUMENT_INGESTION = "document_ingestion"
|
||||
QUERY_PROCESSING = "query_processing"
|
||||
|
||||
|
||||
class HookFailStrategy(str, PyEnum):
|
||||
HARD = "hard" # exception propagates, pipeline aborts
|
||||
SOFT = "soft" # log error, return original input, pipeline continues
|
||||
|
||||
@@ -1,233 +0,0 @@
|
||||
import datetime
|
||||
from uuid import UUID
|
||||
|
||||
from sqlalchemy import delete
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.engine import CursorResult
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.orm import selectinload
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from onyx.db.constants import UNSET
|
||||
from onyx.db.constants import UnsetType
|
||||
from onyx.db.enums import HookFailStrategy
|
||||
from onyx.db.enums import HookPoint
|
||||
from onyx.db.models import Hook
|
||||
from onyx.db.models import HookExecutionLog
|
||||
from onyx.error_handling.error_codes import OnyxErrorCode
|
||||
from onyx.error_handling.exceptions import OnyxError
|
||||
|
||||
|
||||
# ── Hook CRUD ────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def get_hook_by_id(
|
||||
*,
|
||||
db_session: Session,
|
||||
hook_id: int,
|
||||
include_deleted: bool = False,
|
||||
include_creator: bool = False,
|
||||
) -> Hook | None:
|
||||
stmt = select(Hook).where(Hook.id == hook_id)
|
||||
if not include_deleted:
|
||||
stmt = stmt.where(Hook.deleted.is_(False))
|
||||
if include_creator:
|
||||
stmt = stmt.options(selectinload(Hook.creator))
|
||||
return db_session.scalar(stmt)
|
||||
|
||||
|
||||
def get_non_deleted_hook_by_hook_point(
|
||||
*,
|
||||
db_session: Session,
|
||||
hook_point: HookPoint,
|
||||
include_creator: bool = False,
|
||||
) -> Hook | None:
|
||||
stmt = (
|
||||
select(Hook).where(Hook.hook_point == hook_point).where(Hook.deleted.is_(False))
|
||||
)
|
||||
if include_creator:
|
||||
stmt = stmt.options(selectinload(Hook.creator))
|
||||
return db_session.scalar(stmt)
|
||||
|
||||
|
||||
def get_hooks(
|
||||
*,
|
||||
db_session: Session,
|
||||
include_deleted: bool = False,
|
||||
include_creator: bool = False,
|
||||
) -> list[Hook]:
|
||||
stmt = select(Hook)
|
||||
if not include_deleted:
|
||||
stmt = stmt.where(Hook.deleted.is_(False))
|
||||
if include_creator:
|
||||
stmt = stmt.options(selectinload(Hook.creator))
|
||||
stmt = stmt.order_by(Hook.hook_point, Hook.created_at.desc())
|
||||
return list(db_session.scalars(stmt).all())
|
||||
|
||||
|
||||
def create_hook__no_commit(
|
||||
*,
|
||||
db_session: Session,
|
||||
name: str,
|
||||
hook_point: HookPoint,
|
||||
endpoint_url: str | None = None,
|
||||
api_key: str | None = None,
|
||||
fail_strategy: HookFailStrategy,
|
||||
timeout_seconds: float,
|
||||
is_active: bool = False,
|
||||
creator_id: UUID | None = None,
|
||||
) -> Hook:
|
||||
"""Create a new hook for the given hook point.
|
||||
|
||||
At most one non-deleted hook per hook point is allowed. Raises
|
||||
OnyxError(CONFLICT) if a hook already exists, including under concurrent
|
||||
duplicate creates where the partial unique index fires an IntegrityError.
|
||||
"""
|
||||
existing = get_non_deleted_hook_by_hook_point(
|
||||
db_session=db_session, hook_point=hook_point
|
||||
)
|
||||
if existing:
|
||||
raise OnyxError(
|
||||
OnyxErrorCode.CONFLICT,
|
||||
f"A hook for '{hook_point.value}' already exists (id={existing.id}).",
|
||||
)
|
||||
|
||||
hook = Hook(
|
||||
name=name,
|
||||
hook_point=hook_point,
|
||||
endpoint_url=endpoint_url,
|
||||
api_key=api_key,
|
||||
fail_strategy=fail_strategy,
|
||||
timeout_seconds=timeout_seconds,
|
||||
is_active=is_active,
|
||||
creator_id=creator_id,
|
||||
)
|
||||
# Use a savepoint so that a failed insert only rolls back this operation,
|
||||
# not the entire outer transaction.
|
||||
savepoint = db_session.begin_nested()
|
||||
try:
|
||||
db_session.add(hook)
|
||||
savepoint.commit()
|
||||
except IntegrityError as exc:
|
||||
savepoint.rollback()
|
||||
if "ix_hook_one_non_deleted_per_point" in str(exc.orig):
|
||||
raise OnyxError(
|
||||
OnyxErrorCode.CONFLICT,
|
||||
f"A hook for '{hook_point.value}' already exists.",
|
||||
)
|
||||
raise # re-raise unrelated integrity errors (FK violations, etc.)
|
||||
return hook
|
||||
|
||||
|
||||
def update_hook__no_commit(
|
||||
*,
|
||||
db_session: Session,
|
||||
hook_id: int,
|
||||
name: str | None = None,
|
||||
endpoint_url: str | None | UnsetType = UNSET,
|
||||
api_key: str | None | UnsetType = UNSET,
|
||||
fail_strategy: HookFailStrategy | None = None,
|
||||
timeout_seconds: float | None = None,
|
||||
is_active: bool | None = None,
|
||||
is_reachable: bool | None = None,
|
||||
include_creator: bool = False,
|
||||
) -> Hook:
|
||||
"""Update hook fields.
|
||||
|
||||
Sentinel conventions:
|
||||
- endpoint_url, api_key: pass UNSET to leave unchanged; pass None to clear.
|
||||
- name, fail_strategy, timeout_seconds, is_active, is_reachable: pass None to leave unchanged.
|
||||
"""
|
||||
hook = get_hook_by_id(
|
||||
db_session=db_session, hook_id=hook_id, include_creator=include_creator
|
||||
)
|
||||
if hook is None:
|
||||
raise OnyxError(OnyxErrorCode.NOT_FOUND, f"Hook with id {hook_id} not found.")
|
||||
|
||||
if name is not None:
|
||||
hook.name = name
|
||||
if not isinstance(endpoint_url, UnsetType):
|
||||
hook.endpoint_url = endpoint_url
|
||||
if not isinstance(api_key, UnsetType):
|
||||
hook.api_key = api_key # type: ignore[assignment] # EncryptedString coerces str → SensitiveValue at the ORM level
|
||||
if fail_strategy is not None:
|
||||
hook.fail_strategy = fail_strategy
|
||||
if timeout_seconds is not None:
|
||||
hook.timeout_seconds = timeout_seconds
|
||||
if is_active is not None:
|
||||
hook.is_active = is_active
|
||||
if is_reachable is not None:
|
||||
hook.is_reachable = is_reachable
|
||||
|
||||
db_session.flush()
|
||||
return hook
|
||||
|
||||
|
||||
def delete_hook__no_commit(
|
||||
*,
|
||||
db_session: Session,
|
||||
hook_id: int,
|
||||
) -> None:
|
||||
hook = get_hook_by_id(db_session=db_session, hook_id=hook_id)
|
||||
if hook is None:
|
||||
raise OnyxError(OnyxErrorCode.NOT_FOUND, f"Hook with id {hook_id} not found.")
|
||||
|
||||
hook.deleted = True
|
||||
hook.is_active = False
|
||||
db_session.flush()
|
||||
|
||||
|
||||
# ── HookExecutionLog CRUD ────────────────────────────────────────────────
|
||||
|
||||
|
||||
def create_hook_execution_log__no_commit(
|
||||
*,
|
||||
db_session: Session,
|
||||
hook_id: int,
|
||||
is_success: bool,
|
||||
error_message: str | None = None,
|
||||
status_code: int | None = None,
|
||||
duration_ms: int | None = None,
|
||||
) -> HookExecutionLog:
|
||||
log = HookExecutionLog(
|
||||
hook_id=hook_id,
|
||||
is_success=is_success,
|
||||
error_message=error_message,
|
||||
status_code=status_code,
|
||||
duration_ms=duration_ms,
|
||||
)
|
||||
db_session.add(log)
|
||||
db_session.flush()
|
||||
return log
|
||||
|
||||
|
||||
def get_hook_execution_logs(
|
||||
*,
|
||||
db_session: Session,
|
||||
hook_id: int,
|
||||
limit: int,
|
||||
) -> list[HookExecutionLog]:
|
||||
stmt = (
|
||||
select(HookExecutionLog)
|
||||
.where(HookExecutionLog.hook_id == hook_id)
|
||||
.order_by(HookExecutionLog.created_at.desc())
|
||||
.limit(limit)
|
||||
)
|
||||
return list(db_session.scalars(stmt).all())
|
||||
|
||||
|
||||
def cleanup_old_execution_logs__no_commit(
|
||||
*,
|
||||
db_session: Session,
|
||||
max_age_days: int,
|
||||
) -> int:
|
||||
"""Delete execution logs older than max_age_days. Returns the number of rows deleted."""
|
||||
cutoff = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(
|
||||
days=max_age_days
|
||||
)
|
||||
result: CursorResult = db_session.execute( # type: ignore[assignment]
|
||||
delete(HookExecutionLog)
|
||||
.where(HookExecutionLog.created_at < cutoff)
|
||||
.execution_options(synchronize_session=False)
|
||||
)
|
||||
return result.rowcount
|
||||
@@ -64,8 +64,6 @@ from onyx.db.enums import (
|
||||
BuildSessionStatus,
|
||||
EmbeddingPrecision,
|
||||
HierarchyNodeType,
|
||||
HookFailStrategy,
|
||||
HookPoint,
|
||||
IndexingMode,
|
||||
OpenSearchDocumentMigrationStatus,
|
||||
OpenSearchTenantMigrationStatus,
|
||||
@@ -5180,90 +5178,3 @@ class CacheStore(Base):
|
||||
expires_at: Mapped[datetime.datetime | None] = mapped_column(
|
||||
DateTime(timezone=True), nullable=True
|
||||
)
|
||||
|
||||
|
||||
class Hook(Base):
|
||||
"""Pairs a HookPoint with a customer-provided API endpoint.
|
||||
|
||||
At most one non-deleted Hook per HookPoint is allowed, enforced by a
|
||||
partial unique index on (hook_point) where deleted=false.
|
||||
"""
|
||||
|
||||
__tablename__ = "hook"
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
name: Mapped[str] = mapped_column(String, nullable=False)
|
||||
hook_point: Mapped[HookPoint] = mapped_column(
|
||||
Enum(HookPoint, native_enum=False), nullable=False
|
||||
)
|
||||
endpoint_url: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
api_key: Mapped[SensitiveValue[str] | None] = mapped_column(
|
||||
EncryptedString(), nullable=True
|
||||
)
|
||||
is_reachable: Mapped[bool | None] = mapped_column(
|
||||
Boolean, nullable=True, default=None
|
||||
) # null = never validated, true = last check passed, false = last check failed
|
||||
fail_strategy: Mapped[HookFailStrategy] = mapped_column(
|
||||
Enum(HookFailStrategy, native_enum=False),
|
||||
nullable=False,
|
||||
default=HookFailStrategy.HARD,
|
||||
)
|
||||
timeout_seconds: Mapped[float] = mapped_column(Float, nullable=False, default=30.0)
|
||||
is_active: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
|
||||
deleted: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
|
||||
creator_id: Mapped[UUID | None] = mapped_column(
|
||||
PGUUID(as_uuid=True),
|
||||
ForeignKey("user.id", ondelete="SET NULL"),
|
||||
nullable=True,
|
||||
)
|
||||
created_at: Mapped[datetime.datetime] = mapped_column(
|
||||
DateTime(timezone=True), server_default=func.now(), nullable=False
|
||||
)
|
||||
updated_at: Mapped[datetime.datetime] = mapped_column(
|
||||
DateTime(timezone=True),
|
||||
server_default=func.now(),
|
||||
onupdate=func.now(),
|
||||
nullable=False,
|
||||
)
|
||||
|
||||
creator: Mapped["User | None"] = relationship("User", foreign_keys=[creator_id])
|
||||
execution_logs: Mapped[list["HookExecutionLog"]] = relationship(
|
||||
"HookExecutionLog", back_populates="hook", cascade="all, delete-orphan"
|
||||
)
|
||||
|
||||
__table_args__ = (
|
||||
Index(
|
||||
"ix_hook_one_non_deleted_per_point",
|
||||
"hook_point",
|
||||
unique=True,
|
||||
postgresql_where=(deleted == False), # noqa: E712
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class HookExecutionLog(Base):
|
||||
"""Records hook executions for health monitoring and debugging.
|
||||
|
||||
Currently only failures are logged; the is_success column exists so
|
||||
success logging can be added later without a schema change.
|
||||
Retention: rows older than 30 days are deleted by a nightly Celery task.
|
||||
"""
|
||||
|
||||
__tablename__ = "hook_execution_log"
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
hook_id: Mapped[int] = mapped_column(
|
||||
Integer,
|
||||
ForeignKey("hook.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
index=True,
|
||||
)
|
||||
is_success: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
|
||||
error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
status_code: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||
duration_ms: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||
created_at: Mapped[datetime.datetime] = mapped_column(
|
||||
DateTime(timezone=True), server_default=func.now(), nullable=False, index=True
|
||||
)
|
||||
|
||||
hook: Mapped["Hook"] = relationship("Hook", back_populates="execution_logs")
|
||||
|
||||
@@ -1,8 +1,5 @@
|
||||
# Default value for the maximum number of tokens a chunk can hold, if none is
|
||||
# specified when creating an index.
|
||||
import os
|
||||
from enum import Enum
|
||||
|
||||
from onyx.configs.app_configs import (
|
||||
OPENSEARCH_OVERRIDE_DEFAULT_NUM_HYBRID_SEARCH_CANDIDATES,
|
||||
)
|
||||
@@ -10,18 +7,6 @@ from onyx.configs.app_configs import (
|
||||
|
||||
DEFAULT_MAX_CHUNK_SIZE = 512
|
||||
|
||||
|
||||
# By default OpenSearch will only return a maximum of this many results in a
|
||||
# given search. This value is configurable in the index settings.
|
||||
DEFAULT_OPENSEARCH_MAX_RESULT_WINDOW = 10_000
|
||||
|
||||
|
||||
# For documents which do not have a value for LAST_UPDATED_FIELD_NAME, we assume
|
||||
# that the document was last updated this many days ago for the purpose of time
|
||||
# cutoff filtering during retrieval.
|
||||
ASSUMED_DOCUMENT_AGE_DAYS = 90
|
||||
|
||||
|
||||
# Size of the dynamic list used to consider elements during kNN graph creation.
|
||||
# Higher values improve search quality but increase indexing time. Values
|
||||
# typically range between 100 - 512.
|
||||
@@ -56,41 +41,21 @@ DEFAULT_NUM_HYBRID_SEARCH_CANDIDATES = (
|
||||
# https://docs.opensearch.org/latest/query-dsl/specialized/k-nn/index/#ef_search
|
||||
EF_SEARCH = DEFAULT_NUM_HYBRID_SEARCH_CANDIDATES
|
||||
|
||||
# Since the titles are included in the contents, the embedding matches are
|
||||
# heavily downweighted as they act as a boost rather than an independent scoring
|
||||
# component.
|
||||
SEARCH_TITLE_VECTOR_WEIGHT = 0.1
|
||||
SEARCH_CONTENT_VECTOR_WEIGHT = 0.45
|
||||
# Single keyword weight for both title and content (merged from former title
|
||||
# keyword + content keyword).
|
||||
SEARCH_KEYWORD_WEIGHT = 0.45
|
||||
|
||||
class HybridSearchSubqueryConfiguration(Enum):
|
||||
TITLE_VECTOR_CONTENT_VECTOR_TITLE_CONTENT_COMBINED_KEYWORD = 1
|
||||
# Current default.
|
||||
CONTENT_VECTOR_TITLE_CONTENT_COMBINED_KEYWORD = 2
|
||||
# NOTE: It is critical that the order of these weights matches the order of the
|
||||
# sub-queries in the hybrid search.
|
||||
HYBRID_SEARCH_NORMALIZATION_WEIGHTS = [
|
||||
SEARCH_TITLE_VECTOR_WEIGHT,
|
||||
SEARCH_CONTENT_VECTOR_WEIGHT,
|
||||
SEARCH_KEYWORD_WEIGHT,
|
||||
]
|
||||
|
||||
|
||||
# Will raise and block application start if HYBRID_SEARCH_SUBQUERY_CONFIGURATION
|
||||
# is set but not a valid value. If not set, defaults to
|
||||
# CONTENT_VECTOR_TITLE_CONTENT_COMBINED_KEYWORD.
|
||||
HYBRID_SEARCH_SUBQUERY_CONFIGURATION: HybridSearchSubqueryConfiguration = (
|
||||
HybridSearchSubqueryConfiguration(
|
||||
int(os.environ["HYBRID_SEARCH_SUBQUERY_CONFIGURATION"])
|
||||
)
|
||||
if os.environ.get("HYBRID_SEARCH_SUBQUERY_CONFIGURATION", None) is not None
|
||||
else HybridSearchSubqueryConfiguration.CONTENT_VECTOR_TITLE_CONTENT_COMBINED_KEYWORD
|
||||
)
|
||||
|
||||
|
||||
class HybridSearchNormalizationPipeline(Enum):
|
||||
# Current default.
|
||||
MIN_MAX = 1
|
||||
# NOTE: Using z-score normalization is better for hybrid search from a
|
||||
# theoretical standpoint. Empirically on a small dataset of up to 10K docs,
|
||||
# it's not very different. Likely more impactful at scale.
|
||||
# https://opensearch.org/blog/introducing-the-z-score-normalization-technique-for-hybrid-search/
|
||||
ZSCORE = 2
|
||||
|
||||
|
||||
# Will raise and block application start if HYBRID_SEARCH_NORMALIZATION_PIPELINE
|
||||
# is set but not a valid value. If not set, defaults to MIN_MAX.
|
||||
HYBRID_SEARCH_NORMALIZATION_PIPELINE: HybridSearchNormalizationPipeline = (
|
||||
HybridSearchNormalizationPipeline(
|
||||
int(os.environ["HYBRID_SEARCH_NORMALIZATION_PIPELINE"])
|
||||
)
|
||||
if os.environ.get("HYBRID_SEARCH_NORMALIZATION_PIPELINE", None) is not None
|
||||
else HybridSearchNormalizationPipeline.MIN_MAX
|
||||
)
|
||||
assert sum(HYBRID_SEARCH_NORMALIZATION_WEIGHTS) == 1.0
|
||||
|
||||
@@ -55,13 +55,16 @@ from onyx.document_index.opensearch.schema import PERSONAS_FIELD_NAME
|
||||
from onyx.document_index.opensearch.schema import USER_PROJECTS_FIELD_NAME
|
||||
from onyx.document_index.opensearch.search import DocumentQuery
|
||||
from onyx.document_index.opensearch.search import (
|
||||
get_min_max_normalization_pipeline_name_and_config,
|
||||
MIN_MAX_NORMALIZATION_PIPELINE_CONFIG,
|
||||
)
|
||||
from onyx.document_index.opensearch.search import (
|
||||
get_normalization_pipeline_name_and_config,
|
||||
MIN_MAX_NORMALIZATION_PIPELINE_NAME,
|
||||
)
|
||||
from onyx.document_index.opensearch.search import (
|
||||
get_zscore_normalization_pipeline_name_and_config,
|
||||
ZSCORE_NORMALIZATION_PIPELINE_CONFIG,
|
||||
)
|
||||
from onyx.document_index.opensearch.search import (
|
||||
ZSCORE_NORMALIZATION_PIPELINE_NAME,
|
||||
)
|
||||
from onyx.indexing.models import DocMetadataAwareIndexChunk
|
||||
from onyx.indexing.models import Document
|
||||
@@ -100,19 +103,13 @@ def set_cluster_state(client: OpenSearchClient) -> None:
|
||||
"is not the first time running Onyx against this instance of OpenSearch, these "
|
||||
"settings have likely already been set. Not taking any further action..."
|
||||
)
|
||||
min_max_normalization_pipeline_name, min_max_normalization_pipeline_config = (
|
||||
get_min_max_normalization_pipeline_name_and_config()
|
||||
)
|
||||
zscore_normalization_pipeline_name, zscore_normalization_pipeline_config = (
|
||||
get_zscore_normalization_pipeline_name_and_config()
|
||||
client.create_search_pipeline(
|
||||
pipeline_id=MIN_MAX_NORMALIZATION_PIPELINE_NAME,
|
||||
pipeline_body=MIN_MAX_NORMALIZATION_PIPELINE_CONFIG,
|
||||
)
|
||||
client.create_search_pipeline(
|
||||
pipeline_id=min_max_normalization_pipeline_name,
|
||||
pipeline_body=min_max_normalization_pipeline_config,
|
||||
)
|
||||
client.create_search_pipeline(
|
||||
pipeline_id=zscore_normalization_pipeline_name,
|
||||
pipeline_body=zscore_normalization_pipeline_config,
|
||||
pipeline_id=ZSCORE_NORMALIZATION_PIPELINE_NAME,
|
||||
pipeline_body=ZSCORE_NORMALIZATION_PIPELINE_CONFIG,
|
||||
)
|
||||
|
||||
|
||||
@@ -943,10 +940,14 @@ class OpenSearchDocumentIndex(DocumentIndex):
|
||||
index_filters=filters,
|
||||
include_hidden=False,
|
||||
)
|
||||
normalization_pipeline_name, _ = get_normalization_pipeline_name_and_config()
|
||||
# NOTE: Using z-score normalization here because it's better for hybrid
|
||||
# search from a theoretical standpoint. Empirically on a small dataset
|
||||
# of up to 10K docs, it's not very different. Likely more impactful at
|
||||
# scale.
|
||||
# https://opensearch.org/blog/introducing-the-z-score-normalization-technique-for-hybrid-search/
|
||||
search_hits: list[SearchHit[DocumentChunk]] = self._client.search(
|
||||
body=query_body,
|
||||
search_pipeline_id=normalization_pipeline_name,
|
||||
search_pipeline_id=ZSCORE_NORMALIZATION_PIPELINE_NAME,
|
||||
)
|
||||
|
||||
# Good place for a breakpoint to inspect the search hits if you have "explain" enabled.
|
||||
|
||||
@@ -13,21 +13,10 @@ from onyx.configs.constants import INDEX_SEPARATOR
|
||||
from onyx.context.search.models import IndexFilters
|
||||
from onyx.context.search.models import Tag
|
||||
from onyx.document_index.interfaces_new import TenantState
|
||||
from onyx.document_index.opensearch.constants import ASSUMED_DOCUMENT_AGE_DAYS
|
||||
from onyx.document_index.opensearch.constants import (
|
||||
DEFAULT_NUM_HYBRID_SEARCH_CANDIDATES,
|
||||
)
|
||||
from onyx.document_index.opensearch.constants import (
|
||||
DEFAULT_OPENSEARCH_MAX_RESULT_WINDOW,
|
||||
)
|
||||
from onyx.document_index.opensearch.constants import (
|
||||
HYBRID_SEARCH_NORMALIZATION_PIPELINE,
|
||||
)
|
||||
from onyx.document_index.opensearch.constants import (
|
||||
HYBRID_SEARCH_SUBQUERY_CONFIGURATION,
|
||||
)
|
||||
from onyx.document_index.opensearch.constants import HybridSearchNormalizationPipeline
|
||||
from onyx.document_index.opensearch.constants import HybridSearchSubqueryConfiguration
|
||||
from onyx.document_index.opensearch.constants import HYBRID_SEARCH_NORMALIZATION_WEIGHTS
|
||||
from onyx.document_index.opensearch.schema import ACCESS_CONTROL_LIST_FIELD_NAME
|
||||
from onyx.document_index.opensearch.schema import ANCESTOR_HIERARCHY_NODE_IDS_FIELD_NAME
|
||||
from onyx.document_index.opensearch.schema import CHUNK_INDEX_FIELD_NAME
|
||||
@@ -54,113 +43,49 @@ from onyx.document_index.opensearch.schema import USER_PROJECTS_FIELD_NAME
|
||||
|
||||
# TODO(andrei): Turn all magic dictionaries to pydantic models.
|
||||
|
||||
|
||||
def _get_hybrid_search_normalization_weights() -> list[float]:
|
||||
if (
|
||||
HYBRID_SEARCH_SUBQUERY_CONFIGURATION
|
||||
is HybridSearchSubqueryConfiguration.TITLE_VECTOR_CONTENT_VECTOR_TITLE_CONTENT_COMBINED_KEYWORD
|
||||
):
|
||||
# Since the titles are included in the contents, the embedding matches
|
||||
# are heavily downweighted as they act as a boost rather than an
|
||||
# independent scoring component.
|
||||
search_title_vector_weight = 0.1
|
||||
search_content_vector_weight = 0.45
|
||||
# Single keyword weight for both title and content (merged from former
|
||||
# title keyword + content keyword).
|
||||
search_keyword_weight = 0.45
|
||||
|
||||
# NOTE: It is critical that the order of these weights matches the order
|
||||
# of the sub-queries in the hybrid search.
|
||||
hybrid_search_normalization_weights = [
|
||||
search_title_vector_weight,
|
||||
search_content_vector_weight,
|
||||
search_keyword_weight,
|
||||
]
|
||||
elif (
|
||||
HYBRID_SEARCH_SUBQUERY_CONFIGURATION
|
||||
is HybridSearchSubqueryConfiguration.CONTENT_VECTOR_TITLE_CONTENT_COMBINED_KEYWORD
|
||||
):
|
||||
search_content_vector_weight = 0.5
|
||||
# Single keyword weight for both title and content (merged from former
|
||||
# title keyword + content keyword).
|
||||
search_keyword_weight = 0.5
|
||||
|
||||
# NOTE: It is critical that the order of these weights matches the order
|
||||
# of the sub-queries in the hybrid search.
|
||||
hybrid_search_normalization_weights = [
|
||||
search_content_vector_weight,
|
||||
search_keyword_weight,
|
||||
]
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Bug: Unhandled hybrid search subquery configuration: {HYBRID_SEARCH_SUBQUERY_CONFIGURATION}."
|
||||
)
|
||||
|
||||
assert (
|
||||
sum(hybrid_search_normalization_weights) == 1.0
|
||||
), "Bug: Hybrid search normalization weights do not sum to 1.0."
|
||||
|
||||
return hybrid_search_normalization_weights
|
||||
|
||||
|
||||
def get_min_max_normalization_pipeline_name_and_config() -> tuple[str, dict[str, Any]]:
|
||||
min_max_normalization_pipeline_name = "normalization_pipeline_min_max"
|
||||
min_max_normalization_pipeline_config: dict[str, Any] = {
|
||||
"description": "Normalization for keyword and vector scores using min-max",
|
||||
"phase_results_processors": [
|
||||
{
|
||||
# https://docs.opensearch.org/latest/search-plugins/search-pipelines/normalization-processor/
|
||||
"normalization-processor": {
|
||||
"normalization": {"technique": "min_max"},
|
||||
"combination": {
|
||||
"technique": "arithmetic_mean",
|
||||
"parameters": {
|
||||
"weights": _get_hybrid_search_normalization_weights()
|
||||
},
|
||||
},
|
||||
}
|
||||
MIN_MAX_NORMALIZATION_PIPELINE_NAME = "normalization_pipeline_min_max"
|
||||
MIN_MAX_NORMALIZATION_PIPELINE_CONFIG: dict[str, Any] = {
|
||||
"description": "Normalization for keyword and vector scores using min-max",
|
||||
"phase_results_processors": [
|
||||
{
|
||||
# https://docs.opensearch.org/latest/search-plugins/search-pipelines/normalization-processor/
|
||||
"normalization-processor": {
|
||||
"normalization": {"technique": "min_max"},
|
||||
"combination": {
|
||||
"technique": "arithmetic_mean",
|
||||
"parameters": {"weights": HYBRID_SEARCH_NORMALIZATION_WEIGHTS},
|
||||
},
|
||||
}
|
||||
],
|
||||
}
|
||||
return min_max_normalization_pipeline_name, min_max_normalization_pipeline_config
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def get_zscore_normalization_pipeline_name_and_config() -> tuple[str, dict[str, Any]]:
|
||||
zscore_normalization_pipeline_name = "normalization_pipeline_zscore"
|
||||
zscore_normalization_pipeline_config: dict[str, Any] = {
|
||||
"description": "Normalization for keyword and vector scores using z-score",
|
||||
"phase_results_processors": [
|
||||
{
|
||||
# https://docs.opensearch.org/latest/search-plugins/search-pipelines/normalization-processor/
|
||||
"normalization-processor": {
|
||||
"normalization": {"technique": "z_score"},
|
||||
"combination": {
|
||||
"technique": "arithmetic_mean",
|
||||
"parameters": {
|
||||
"weights": _get_hybrid_search_normalization_weights()
|
||||
},
|
||||
},
|
||||
}
|
||||
ZSCORE_NORMALIZATION_PIPELINE_NAME = "normalization_pipeline_zscore"
|
||||
ZSCORE_NORMALIZATION_PIPELINE_CONFIG: dict[str, Any] = {
|
||||
"description": "Normalization for keyword and vector scores using z-score",
|
||||
"phase_results_processors": [
|
||||
{
|
||||
# https://docs.opensearch.org/latest/search-plugins/search-pipelines/normalization-processor/
|
||||
"normalization-processor": {
|
||||
"normalization": {"technique": "z_score"},
|
||||
"combination": {
|
||||
"technique": "arithmetic_mean",
|
||||
"parameters": {"weights": HYBRID_SEARCH_NORMALIZATION_WEIGHTS},
|
||||
},
|
||||
}
|
||||
],
|
||||
}
|
||||
return zscore_normalization_pipeline_name, zscore_normalization_pipeline_config
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def get_normalization_pipeline_name_and_config() -> tuple[str, dict[str, Any]]:
|
||||
if (
|
||||
HYBRID_SEARCH_NORMALIZATION_PIPELINE
|
||||
is HybridSearchNormalizationPipeline.MIN_MAX
|
||||
):
|
||||
return get_min_max_normalization_pipeline_name_and_config()
|
||||
elif (
|
||||
HYBRID_SEARCH_NORMALIZATION_PIPELINE is HybridSearchNormalizationPipeline.ZSCORE
|
||||
):
|
||||
return get_zscore_normalization_pipeline_name_and_config()
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Bug: Unhandled hybrid search normalization pipeline: {HYBRID_SEARCH_NORMALIZATION_PIPELINE}."
|
||||
)
|
||||
# By default OpenSearch will only return a maximum of this many results in a
|
||||
# given search. This value is configurable in the index settings.
|
||||
DEFAULT_OPENSEARCH_MAX_RESULT_WINDOW = 10_000
|
||||
|
||||
# For documents which do not have a value for LAST_UPDATED_FIELD_NAME, we assume
|
||||
# that the document was last updated this many days ago for the purpose of time
|
||||
# cutoff filtering during retrieval.
|
||||
ASSUMED_DOCUMENT_AGE_DAYS = 90
|
||||
|
||||
|
||||
class DocumentQuery:
|
||||
@@ -470,18 +395,20 @@ class DocumentQuery:
|
||||
The return of this function is not sufficient to be directly supplied to
|
||||
the OpenSearch client. See get_hybrid_search_query.
|
||||
|
||||
Matches:
|
||||
- Title vector
|
||||
- Content vector
|
||||
- Keyword (title + content, match and phrase)
|
||||
|
||||
Normalization is not performed here.
|
||||
The weights of each of these subqueries should be configured in a search
|
||||
pipeline.
|
||||
|
||||
The exact subqueries executed depend on the
|
||||
HYBRID_SEARCH_SUBQUERY_CONFIGURATION setting.
|
||||
|
||||
NOTE: For OpenSearch, 5 is the maximum number of query clauses allowed
|
||||
in a single hybrid query. Source:
|
||||
https://docs.opensearch.org/latest/query-dsl/compound/hybrid/
|
||||
|
||||
NOTE: Each query is independent during the search phase; there is no
|
||||
NOTE: Each query is independent during the search phase, there is no
|
||||
backfilling of scores for missing query components. What this means is
|
||||
that if a document was a good vector match but did not show up for
|
||||
keyword, it gets a score of 0 for the keyword component of the hybrid
|
||||
@@ -510,115 +437,74 @@ class DocumentQuery:
|
||||
similarity search.
|
||||
"""
|
||||
# Build sub-queries for hybrid search. Order must match normalization
|
||||
# pipeline weights.
|
||||
if (
|
||||
HYBRID_SEARCH_SUBQUERY_CONFIGURATION
|
||||
is HybridSearchSubqueryConfiguration.TITLE_VECTOR_CONTENT_VECTOR_TITLE_CONTENT_COMBINED_KEYWORD
|
||||
):
|
||||
return [
|
||||
DocumentQuery._get_title_vector_similarity_search_query(
|
||||
query_vector, vector_candidates
|
||||
),
|
||||
DocumentQuery._get_content_vector_similarity_search_query(
|
||||
query_vector, vector_candidates
|
||||
),
|
||||
DocumentQuery._get_title_content_combined_keyword_search_query(
|
||||
query_text
|
||||
),
|
||||
]
|
||||
elif (
|
||||
HYBRID_SEARCH_SUBQUERY_CONFIGURATION
|
||||
is HybridSearchSubqueryConfiguration.CONTENT_VECTOR_TITLE_CONTENT_COMBINED_KEYWORD
|
||||
):
|
||||
return [
|
||||
DocumentQuery._get_content_vector_similarity_search_query(
|
||||
query_vector, vector_candidates
|
||||
),
|
||||
DocumentQuery._get_title_content_combined_keyword_search_query(
|
||||
query_text
|
||||
),
|
||||
]
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Bug: Unhandled hybrid search subquery configuration: {HYBRID_SEARCH_SUBQUERY_CONFIGURATION}"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _get_title_vector_similarity_search_query(
|
||||
query_vector: list[float],
|
||||
vector_candidates: int = DEFAULT_NUM_HYBRID_SEARCH_CANDIDATES,
|
||||
) -> dict[str, Any]:
|
||||
return {
|
||||
"knn": {
|
||||
TITLE_VECTOR_FIELD_NAME: {
|
||||
"vector": query_vector,
|
||||
"k": vector_candidates,
|
||||
# pipeline weights: title vector, content vector, keyword (title + content).
|
||||
hybrid_search_queries: list[dict[str, Any]] = [
|
||||
# 1. Title vector search
|
||||
{
|
||||
"knn": {
|
||||
TITLE_VECTOR_FIELD_NAME: {
|
||||
"vector": query_vector,
|
||||
"k": vector_candidates,
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _get_content_vector_similarity_search_query(
|
||||
query_vector: list[float],
|
||||
vector_candidates: int = DEFAULT_NUM_HYBRID_SEARCH_CANDIDATES,
|
||||
) -> dict[str, Any]:
|
||||
return {
|
||||
"knn": {
|
||||
CONTENT_VECTOR_FIELD_NAME: {
|
||||
"vector": query_vector,
|
||||
"k": vector_candidates,
|
||||
},
|
||||
# 2. Content vector search
|
||||
{
|
||||
"knn": {
|
||||
CONTENT_VECTOR_FIELD_NAME: {
|
||||
"vector": query_vector,
|
||||
"k": vector_candidates,
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
# 3. Keyword (title + content) match and phrase search.
|
||||
{
|
||||
"bool": {
|
||||
"should": [
|
||||
{
|
||||
"match": {
|
||||
TITLE_FIELD_NAME: {
|
||||
"query": query_text,
|
||||
"operator": "or",
|
||||
# The title fields are strongly discounted as they are included in the content.
|
||||
# It just acts as a minor boost
|
||||
"boost": 0.1,
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"match_phrase": {
|
||||
TITLE_FIELD_NAME: {
|
||||
"query": query_text,
|
||||
"slop": 1,
|
||||
"boost": 0.2,
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"match": {
|
||||
CONTENT_FIELD_NAME: {
|
||||
"query": query_text,
|
||||
"operator": "or",
|
||||
"boost": 1.0,
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"match_phrase": {
|
||||
CONTENT_FIELD_NAME: {
|
||||
"query": query_text,
|
||||
"slop": 1,
|
||||
"boost": 1.5,
|
||||
}
|
||||
}
|
||||
},
|
||||
]
|
||||
}
|
||||
},
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def _get_title_content_combined_keyword_search_query(
|
||||
query_text: str,
|
||||
) -> dict[str, Any]:
|
||||
return {
|
||||
"bool": {
|
||||
"should": [
|
||||
{
|
||||
"match": {
|
||||
TITLE_FIELD_NAME: {
|
||||
"query": query_text,
|
||||
"operator": "or",
|
||||
# The title fields are strongly discounted as they are included in the content.
|
||||
# It just acts as a minor boost
|
||||
"boost": 0.1,
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"match_phrase": {
|
||||
TITLE_FIELD_NAME: {
|
||||
"query": query_text,
|
||||
"slop": 1,
|
||||
"boost": 0.2,
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"match": {
|
||||
CONTENT_FIELD_NAME: {
|
||||
"query": query_text,
|
||||
"operator": "or",
|
||||
"boost": 1.0,
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"match_phrase": {
|
||||
CONTENT_FIELD_NAME: {
|
||||
"query": query_text,
|
||||
"slop": 1,
|
||||
"boost": 1.5,
|
||||
}
|
||||
}
|
||||
},
|
||||
]
|
||||
}
|
||||
}
|
||||
return hybrid_search_queries
|
||||
|
||||
@staticmethod
|
||||
def _get_search_filters(
|
||||
|
||||
@@ -501,31 +501,20 @@ def query_vespa(
|
||||
response = http_client.post(SEARCH_ENDPOINT, json=params)
|
||||
response.raise_for_status()
|
||||
except httpx.HTTPError as e:
|
||||
response_text = (
|
||||
e.response.text if isinstance(e, httpx.HTTPStatusError) else None
|
||||
)
|
||||
status_code = (
|
||||
e.response.status_code if isinstance(e, httpx.HTTPStatusError) else None
|
||||
)
|
||||
yql_value = params.get("yql", "")
|
||||
yql_length = len(str(yql_value))
|
||||
|
||||
# Log each detail on its own line so log collectors capture them
|
||||
# as separate entries rather than truncating a single multiline msg
|
||||
error_base = "Failed to query Vespa"
|
||||
logger.error(
|
||||
f"Failed to query Vespa | "
|
||||
f"status={status_code} | "
|
||||
f"yql_length={yql_length} | "
|
||||
f"exception={str(e)}"
|
||||
f"{error_base}:\n"
|
||||
f"Request URL: {e.request.url}\n"
|
||||
f"Request Headers: {e.request.headers}\n"
|
||||
f"Request Payload: {params}\n"
|
||||
f"Exception: {str(e)}"
|
||||
+ (
|
||||
f"\nResponse: {e.response.text}"
|
||||
if isinstance(e, httpx.HTTPStatusError)
|
||||
else ""
|
||||
)
|
||||
)
|
||||
if response_text:
|
||||
logger.error(f"Vespa error response: {response_text[:1000]}")
|
||||
logger.error(f"Vespa request URL: {e.request.url}")
|
||||
|
||||
# Re-raise with diagnostics so callers see what actually went wrong
|
||||
raise httpx.HTTPError(
|
||||
f"Failed to query Vespa (status={status_code}, " f"yql_length={yql_length})"
|
||||
) from e
|
||||
raise httpx.HTTPError(error_base) from e
|
||||
|
||||
response_json: dict[str, Any] = response.json()
|
||||
|
||||
|
||||
@@ -43,22 +43,6 @@ def build_vespa_filters(
|
||||
return ""
|
||||
return f"({' or '.join(eq_elems)})"
|
||||
|
||||
def _build_weighted_set_filter(key: str, vals: list[str] | None) -> str:
|
||||
"""Build a Vespa weightedSet filter for large value lists.
|
||||
|
||||
Uses Vespa's native weightedSet() operator instead of OR-chained
|
||||
'contains' clauses. This is critical for fields like
|
||||
access_control_list where a single user may have tens of thousands
|
||||
of ACL entries — OR clauses at that scale cause Vespa to reject
|
||||
the query with HTTP 400."""
|
||||
if not key or not vals:
|
||||
return ""
|
||||
filtered = [val for val in vals if val]
|
||||
if not filtered:
|
||||
return ""
|
||||
items = ", ".join(f'"{val}":1' for val in filtered)
|
||||
return f"weightedSet({key}, {{{items}}})"
|
||||
|
||||
def _build_int_or_filters(key: str, vals: list[int] | None) -> str:
|
||||
"""For an integer field filter.
|
||||
Returns a bare clause or ""."""
|
||||
@@ -173,16 +157,11 @@ def build_vespa_filters(
|
||||
if filters.tenant_id and MULTI_TENANT:
|
||||
filter_parts.append(build_tenant_id_filter(filters.tenant_id))
|
||||
|
||||
# ACL filters — use weightedSet for efficient matching against the
|
||||
# access_control_list weightedset<string> field. OR-chaining thousands
|
||||
# of 'contains' clauses causes Vespa to reject the query (HTTP 400)
|
||||
# for users with large numbers of external permission groups.
|
||||
# ACL filters
|
||||
if filters.access_control_list is not None:
|
||||
_append(
|
||||
filter_parts,
|
||||
_build_weighted_set_filter(
|
||||
ACCESS_CONTROL_LIST, filters.access_control_list
|
||||
),
|
||||
_build_or_filters(ACCESS_CONTROL_LIST, filters.access_control_list),
|
||||
)
|
||||
|
||||
# Source type filters
|
||||
|
||||
@@ -35,8 +35,6 @@ class OnyxErrorCode(Enum):
|
||||
INSUFFICIENT_PERMISSIONS = ("INSUFFICIENT_PERMISSIONS", 403)
|
||||
ADMIN_ONLY = ("ADMIN_ONLY", 403)
|
||||
EE_REQUIRED = ("EE_REQUIRED", 403)
|
||||
SINGLE_TENANT_ONLY = ("SINGLE_TENANT_ONLY", 403)
|
||||
ENV_VAR_GATED = ("ENV_VAR_GATED", 403)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Validation / Bad Request (400)
|
||||
|
||||
@@ -1,26 +0,0 @@
|
||||
from onyx.configs.app_configs import HOOK_ENABLED
|
||||
from onyx.error_handling.error_codes import OnyxErrorCode
|
||||
from onyx.error_handling.exceptions import OnyxError
|
||||
from shared_configs.configs import MULTI_TENANT
|
||||
|
||||
|
||||
def require_hook_enabled() -> None:
|
||||
"""FastAPI dependency that gates all hook management endpoints.
|
||||
|
||||
Hooks are only available in single-tenant / self-hosted deployments with
|
||||
HOOK_ENABLED=true explicitly set. Two layers of protection:
|
||||
1. MULTI_TENANT check — rejects even if HOOK_ENABLED is accidentally set true
|
||||
2. HOOK_ENABLED flag — explicit opt-in by the operator
|
||||
|
||||
Use as: Depends(require_hook_enabled)
|
||||
"""
|
||||
if MULTI_TENANT:
|
||||
raise OnyxError(
|
||||
OnyxErrorCode.SINGLE_TENANT_ONLY,
|
||||
"Hooks are not available in multi-tenant deployments",
|
||||
)
|
||||
if not HOOK_ENABLED:
|
||||
raise OnyxError(
|
||||
OnyxErrorCode.ENV_VAR_GATED,
|
||||
"Hooks are not enabled. Set HOOK_ENABLED=true to enable.",
|
||||
)
|
||||
@@ -219,26 +219,13 @@ def litellm_exception_to_error_msg(
|
||||
"ratelimiterror"
|
||||
):
|
||||
upstream_detail = upstream_detail.split(":", 1)[1].strip()
|
||||
upstream_detail_lower = upstream_detail.lower()
|
||||
if (
|
||||
"insufficient_quota" in upstream_detail_lower
|
||||
or "exceeded your current quota" in upstream_detail_lower
|
||||
):
|
||||
error_msg = (
|
||||
f"{provider_name} quota exceeded: {upstream_detail}"
|
||||
if upstream_detail
|
||||
else f"{provider_name} quota exceeded: Verify billing and quota for this API key."
|
||||
)
|
||||
error_code = "BUDGET_EXCEEDED"
|
||||
is_retryable = False
|
||||
else:
|
||||
error_msg = (
|
||||
f"{provider_name} rate limit: {upstream_detail}"
|
||||
if upstream_detail
|
||||
else f"{provider_name} rate limit exceeded: Please slow down your requests and try again later."
|
||||
)
|
||||
error_code = "RATE_LIMIT"
|
||||
is_retryable = True
|
||||
error_msg = (
|
||||
f"{provider_name} rate limit: {upstream_detail}"
|
||||
if upstream_detail
|
||||
else f"{provider_name} rate limit exceeded: Please slow down your requests and try again later."
|
||||
)
|
||||
error_code = "RATE_LIMIT"
|
||||
is_retryable = True
|
||||
elif isinstance(core_exception, ServiceUnavailableError):
|
||||
provider_name = (
|
||||
llm.config.model_provider
|
||||
|
||||
@@ -1319,7 +1319,7 @@ def get_connector_indexing_status(
|
||||
# Track admin page visit for analytics
|
||||
mt_cloud_telemetry(
|
||||
tenant_id=tenant_id,
|
||||
distinct_id=str(user.id),
|
||||
distinct_id=user.email,
|
||||
event=MilestoneRecordType.VISITED_ADMIN_PAGE,
|
||||
)
|
||||
|
||||
@@ -1533,7 +1533,7 @@ def create_connector_from_model(
|
||||
|
||||
mt_cloud_telemetry(
|
||||
tenant_id=tenant_id,
|
||||
distinct_id=str(user.id),
|
||||
distinct_id=user.email,
|
||||
event=MilestoneRecordType.CREATED_CONNECTOR,
|
||||
)
|
||||
|
||||
@@ -1611,7 +1611,7 @@ def create_connector_with_mock_credential(
|
||||
|
||||
mt_cloud_telemetry(
|
||||
tenant_id=tenant_id,
|
||||
distinct_id=str(user.id),
|
||||
distinct_id=user.email,
|
||||
event=MilestoneRecordType.CREATED_CONNECTOR,
|
||||
)
|
||||
return response
|
||||
@@ -1915,7 +1915,9 @@ def submit_connector_request(
|
||||
if not connector_name:
|
||||
raise HTTPException(status_code=400, detail="Connector name cannot be empty")
|
||||
|
||||
# Get user identifier for telemetry
|
||||
user_email = user.email
|
||||
distinct_id = user_email or tenant_id
|
||||
|
||||
# Track connector request via PostHog telemetry (Cloud only)
|
||||
from shared_configs.configs import MULTI_TENANT
|
||||
@@ -1923,11 +1925,11 @@ def submit_connector_request(
|
||||
if MULTI_TENANT:
|
||||
mt_cloud_telemetry(
|
||||
tenant_id=tenant_id,
|
||||
distinct_id=str(user.id),
|
||||
distinct_id=distinct_id,
|
||||
event=MilestoneRecordType.REQUESTED_CONNECTOR,
|
||||
properties={
|
||||
"connector_name": connector_name,
|
||||
"user_email": user.email,
|
||||
"user_email": user_email,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@@ -408,7 +408,7 @@ class FailedConnectorIndexingStatus(BaseModel):
|
||||
"""Simplified version of ConnectorIndexingStatus for failed indexing attempts"""
|
||||
|
||||
cc_pair_id: int
|
||||
name: str
|
||||
name: str | None
|
||||
error_msg: str | None
|
||||
is_deletable: bool
|
||||
connector_id: int
|
||||
@@ -422,7 +422,7 @@ class ConnectorStatus(BaseModel):
|
||||
"""
|
||||
|
||||
cc_pair_id: int
|
||||
name: str
|
||||
name: str | None
|
||||
connector: ConnectorSnapshot
|
||||
credential: CredentialSnapshot
|
||||
access_type: AccessType
|
||||
@@ -453,7 +453,7 @@ class DocsCountOperator(str, Enum):
|
||||
|
||||
class ConnectorIndexingStatusLite(BaseModel):
|
||||
cc_pair_id: int
|
||||
name: str
|
||||
name: str | None
|
||||
source: DocumentSource
|
||||
access_type: AccessType
|
||||
cc_pair_status: ConnectorCredentialPairStatus
|
||||
@@ -488,7 +488,7 @@ class ConnectorCredentialPairIdentifier(BaseModel):
|
||||
|
||||
|
||||
class ConnectorCredentialPairMetadata(BaseModel):
|
||||
name: str
|
||||
name: str | None = None
|
||||
access_type: AccessType
|
||||
auto_sync_options: dict[str, Any] | None = None
|
||||
groups: list[int] = Field(default_factory=list)
|
||||
@@ -501,7 +501,7 @@ class CCStatusUpdateRequest(BaseModel):
|
||||
|
||||
class ConnectorCredentialPairDescriptor(BaseModel):
|
||||
id: int
|
||||
name: str
|
||||
name: str | None = None
|
||||
connector: ConnectorSnapshot
|
||||
credential: CredentialSnapshot
|
||||
access_type: AccessType
|
||||
@@ -511,7 +511,7 @@ class CCPairSummary(BaseModel):
|
||||
"""Simplified connector-credential pair information with just essential data"""
|
||||
|
||||
id: int
|
||||
name: str
|
||||
name: str | None
|
||||
source: DocumentSource
|
||||
access_type: AccessType
|
||||
|
||||
|
||||
@@ -15,7 +15,7 @@
|
||||
"date-fns": "^4.1.0",
|
||||
"embla-carousel-react": "^8.6.0",
|
||||
"lucide-react": "^0.562.0",
|
||||
"next": "16.1.7",
|
||||
"next": "16.1.5",
|
||||
"next-themes": "^0.4.6",
|
||||
"radix-ui": "^1.4.3",
|
||||
"react": "19.2.3",
|
||||
@@ -1711,9 +1711,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@next/env": {
|
||||
"version": "16.1.7",
|
||||
"resolved": "https://registry.npmjs.org/@next/env/-/env-16.1.7.tgz",
|
||||
"integrity": "sha512-rJJbIdJB/RQr2F1nylZr/PJzamvNNhfr3brdKP6s/GW850jbtR70QlSfFselvIBbcPUOlQwBakexjFzqLzF6pg==",
|
||||
"version": "16.1.5",
|
||||
"resolved": "https://registry.npmjs.org/@next/env/-/env-16.1.5.tgz",
|
||||
"integrity": "sha512-CRSCPJiSZoi4Pn69RYBDI9R7YK2g59vLexPQFXY0eyw+ILevIenCywzg+DqmlBik9zszEnw2HLFOUlLAcJbL7g==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@next/eslint-plugin-next": {
|
||||
@@ -1727,9 +1727,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@next/swc-darwin-arm64": {
|
||||
"version": "16.1.7",
|
||||
"resolved": "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-16.1.7.tgz",
|
||||
"integrity": "sha512-b2wWIE8sABdyafc4IM8r5Y/dS6kD80JRtOGrUiKTsACFQfWWgUQ2NwoUX1yjFMXVsAwcQeNpnucF2ZrujsBBPg==",
|
||||
"version": "16.1.5",
|
||||
"resolved": "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-16.1.5.tgz",
|
||||
"integrity": "sha512-eK7Wdm3Hjy/SCL7TevlH0C9chrpeOYWx2iR7guJDaz4zEQKWcS1IMVfMb9UKBFMg1XgzcPTYPIp1Vcpukkjg6Q==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
@@ -1743,9 +1743,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@next/swc-darwin-x64": {
|
||||
"version": "16.1.7",
|
||||
"resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-16.1.7.tgz",
|
||||
"integrity": "sha512-zcnVaaZulS1WL0Ss38R5Q6D2gz7MtBu8GZLPfK+73D/hp4GFMrC2sudLky1QibfV7h6RJBJs/gOFvYP0X7UVlQ==",
|
||||
"version": "16.1.5",
|
||||
"resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-16.1.5.tgz",
|
||||
"integrity": "sha512-foQscSHD1dCuxBmGkbIr6ScAUF6pRoDZP6czajyvmXPAOFNnQUJu2Os1SGELODjKp/ULa4fulnBWoHV3XdPLfA==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
@@ -1759,9 +1759,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@next/swc-linux-arm64-gnu": {
|
||||
"version": "16.1.7",
|
||||
"resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-16.1.7.tgz",
|
||||
"integrity": "sha512-2ant89Lux/Q3VyC8vNVg7uBaFVP9SwoK2jJOOR0L8TQnX8CAYnh4uctAScy2Hwj2dgjVHqHLORQZJ2wH6VxhSQ==",
|
||||
"version": "16.1.5",
|
||||
"resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-16.1.5.tgz",
|
||||
"integrity": "sha512-qNIb42o3C02ccIeSeKjacF3HXotGsxh/FMk/rSRmCzOVMtoWH88odn2uZqF8RLsSUWHcAqTgYmPD3pZ03L9ZAA==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
@@ -1775,9 +1775,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@next/swc-linux-arm64-musl": {
|
||||
"version": "16.1.7",
|
||||
"resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-16.1.7.tgz",
|
||||
"integrity": "sha512-uufcze7LYv0FQg9GnNeZ3/whYfo+1Q3HnQpm16o6Uyi0OVzLlk2ZWoY7j07KADZFY8qwDbsmFnMQP3p3+Ftprw==",
|
||||
"version": "16.1.5",
|
||||
"resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-16.1.5.tgz",
|
||||
"integrity": "sha512-U+kBxGUY1xMAzDTXmuVMfhaWUZQAwzRaHJ/I6ihtR5SbTVUEaDRiEU9YMjy1obBWpdOBuk1bcm+tsmifYSygfw==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
@@ -1791,9 +1791,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@next/swc-linux-x64-gnu": {
|
||||
"version": "16.1.7",
|
||||
"resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-16.1.7.tgz",
|
||||
"integrity": "sha512-KWVf2gxYvHtvuT+c4MBOGxuse5TD7DsMFYSxVxRBnOzok/xryNeQSjXgxSv9QpIVlaGzEn/pIuI6Koosx8CGWA==",
|
||||
"version": "16.1.5",
|
||||
"resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-16.1.5.tgz",
|
||||
"integrity": "sha512-gq2UtoCpN7Ke/7tKaU7i/1L7eFLfhMbXjNghSv0MVGF1dmuoaPeEVDvkDuO/9LVa44h5gqpWeJ4mRRznjDv7LA==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
@@ -1807,9 +1807,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@next/swc-linux-x64-musl": {
|
||||
"version": "16.1.7",
|
||||
"resolved": "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-16.1.7.tgz",
|
||||
"integrity": "sha512-HguhaGwsGr1YAGs68uRKc4aGWxLET+NevJskOcCAwXbwj0fYX0RgZW2gsOCzr9S11CSQPIkxmoSbuVaBp4Z3dA==",
|
||||
"version": "16.1.5",
|
||||
"resolved": "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-16.1.5.tgz",
|
||||
"integrity": "sha512-bQWSE729PbXT6mMklWLf8dotislPle2L70E9q6iwETYEOt092GDn0c+TTNj26AjmeceSsC4ndyGsK5nKqHYXjQ==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
@@ -1823,9 +1823,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@next/swc-win32-arm64-msvc": {
|
||||
"version": "16.1.7",
|
||||
"resolved": "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-16.1.7.tgz",
|
||||
"integrity": "sha512-S0n3KrDJokKTeFyM/vGGGR8+pCmXYrjNTk2ZozOL1C/JFdfUIL9O1ATaJOl5r2POe56iRChbsszrjMAdWSv7kQ==",
|
||||
"version": "16.1.5",
|
||||
"resolved": "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-16.1.5.tgz",
|
||||
"integrity": "sha512-LZli0anutkIllMtTAWZlDqdfvjWX/ch8AFK5WgkNTvaqwlouiD1oHM+WW8RXMiL0+vAkAJyAGEzPPjO+hnrSNQ==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
@@ -1839,9 +1839,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@next/swc-win32-x64-msvc": {
|
||||
"version": "16.1.7",
|
||||
"resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-16.1.7.tgz",
|
||||
"integrity": "sha512-mwgtg8CNZGYm06LeEd+bNnOUfwOyNem/rOiP14Lsz+AnUY92Zq/LXwtebtUiaeVkhbroRCQ0c8GlR4UT1U+0yg==",
|
||||
"version": "16.1.5",
|
||||
"resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-16.1.5.tgz",
|
||||
"integrity": "sha512-7is37HJTNQGhjPpQbkKjKEboHYQnCgpVt/4rBrrln0D9nderNxZ8ZWs8w1fAtzUx7wEyYjQ+/13myFgFj6K2Ng==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
@@ -4971,15 +4971,12 @@
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/baseline-browser-mapping": {
|
||||
"version": "2.10.8",
|
||||
"resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.10.8.tgz",
|
||||
"integrity": "sha512-PCLz/LXGBsNTErbtB6i5u4eLpHeMfi93aUv5duMmj6caNu6IphS4q6UevDnL36sZQv9lrP11dbPKGMaXPwMKfQ==",
|
||||
"version": "2.9.17",
|
||||
"resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.9.17.tgz",
|
||||
"integrity": "sha512-agD0MgJFUP/4nvjqzIB29zRPUuCF7Ge6mEv9s8dHrtYD7QWXRcx75rOADE/d5ah1NI+0vkDl0yorDd5U852IQQ==",
|
||||
"license": "Apache-2.0",
|
||||
"bin": {
|
||||
"baseline-browser-mapping": "dist/cli.cjs"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=6.0.0"
|
||||
"baseline-browser-mapping": "dist/cli.js"
|
||||
}
|
||||
},
|
||||
"node_modules/body-parser": {
|
||||
@@ -8978,14 +8975,14 @@
|
||||
}
|
||||
},
|
||||
"node_modules/next": {
|
||||
"version": "16.1.7",
|
||||
"resolved": "https://registry.npmjs.org/next/-/next-16.1.7.tgz",
|
||||
"integrity": "sha512-WM0L7WrSvKwoLegLYr6V+mz+RIofqQgVAfHhMp9a88ms0cFX8iX9ew+snpWlSBwpkURJOUdvCEt3uLl3NNzvWg==",
|
||||
"version": "16.1.5",
|
||||
"resolved": "https://registry.npmjs.org/next/-/next-16.1.5.tgz",
|
||||
"integrity": "sha512-f+wE+NSbiQgh3DSAlTaw2FwY5yGdVViAtp8TotNQj4kk4Q8Bh1sC/aL9aH+Rg1YAVn18OYXsRDT7U/079jgP7w==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@next/env": "16.1.7",
|
||||
"@next/env": "16.1.5",
|
||||
"@swc/helpers": "0.5.15",
|
||||
"baseline-browser-mapping": "^2.9.19",
|
||||
"baseline-browser-mapping": "^2.8.3",
|
||||
"caniuse-lite": "^1.0.30001579",
|
||||
"postcss": "8.4.31",
|
||||
"styled-jsx": "5.1.6"
|
||||
@@ -8997,14 +8994,14 @@
|
||||
"node": ">=20.9.0"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"@next/swc-darwin-arm64": "16.1.7",
|
||||
"@next/swc-darwin-x64": "16.1.7",
|
||||
"@next/swc-linux-arm64-gnu": "16.1.7",
|
||||
"@next/swc-linux-arm64-musl": "16.1.7",
|
||||
"@next/swc-linux-x64-gnu": "16.1.7",
|
||||
"@next/swc-linux-x64-musl": "16.1.7",
|
||||
"@next/swc-win32-arm64-msvc": "16.1.7",
|
||||
"@next/swc-win32-x64-msvc": "16.1.7",
|
||||
"@next/swc-darwin-arm64": "16.1.5",
|
||||
"@next/swc-darwin-x64": "16.1.5",
|
||||
"@next/swc-linux-arm64-gnu": "16.1.5",
|
||||
"@next/swc-linux-arm64-musl": "16.1.5",
|
||||
"@next/swc-linux-x64-gnu": "16.1.5",
|
||||
"@next/swc-linux-x64-musl": "16.1.5",
|
||||
"@next/swc-win32-arm64-msvc": "16.1.5",
|
||||
"@next/swc-win32-x64-msvc": "16.1.5",
|
||||
"sharp": "^0.34.4"
|
||||
},
|
||||
"peerDependencies": {
|
||||
|
||||
@@ -16,7 +16,7 @@
|
||||
"date-fns": "^4.1.0",
|
||||
"embla-carousel-react": "^8.6.0",
|
||||
"lucide-react": "^0.562.0",
|
||||
"next": "16.1.7",
|
||||
"next": "16.1.5",
|
||||
"next-themes": "^0.4.6",
|
||||
"radix-ui": "^1.4.3",
|
||||
"react": "19.2.3",
|
||||
|
||||
@@ -314,7 +314,7 @@ def create_persona(
|
||||
)
|
||||
mt_cloud_telemetry(
|
||||
tenant_id=tenant_id,
|
||||
distinct_id=str(user.id),
|
||||
distinct_id=user.email,
|
||||
event=MilestoneRecordType.CREATED_ASSISTANT,
|
||||
)
|
||||
|
||||
|
||||
@@ -81,7 +81,6 @@ from onyx.server.manage.llm.models import VisionProviderResponse
|
||||
from onyx.server.manage.llm.utils import generate_bedrock_display_name
|
||||
from onyx.server.manage.llm.utils import generate_ollama_display_name
|
||||
from onyx.server.manage.llm.utils import infer_vision_support
|
||||
from onyx.server.manage.llm.utils import is_embedding_model
|
||||
from onyx.server.manage.llm.utils import is_reasoning_model
|
||||
from onyx.server.manage.llm.utils import is_valid_bedrock_model
|
||||
from onyx.server.manage.llm.utils import ModelMetadata
|
||||
@@ -1375,10 +1374,6 @@ def get_litellm_available_models(
|
||||
try:
|
||||
model_details = LitellmModelDetails.model_validate(model)
|
||||
|
||||
# Skip embedding models
|
||||
if is_embedding_model(model_details.id):
|
||||
continue
|
||||
|
||||
results.append(
|
||||
LitellmFinalModelResponse(
|
||||
provider_name=model_details.owned_by,
|
||||
|
||||
@@ -366,18 +366,3 @@ def extract_vendor_from_model_name(model_name: str, provider: str) -> str | None
|
||||
return None
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def is_embedding_model(model_name: str) -> bool:
|
||||
"""Checks for if a model is an embedding model"""
|
||||
from litellm import get_model_info
|
||||
|
||||
try:
|
||||
# get_model_info raises on unknown models
|
||||
# default to False
|
||||
model_info = get_model_info(model_name)
|
||||
except Exception:
|
||||
return False
|
||||
is_embedding_mode = model_info.get("mode") == "embedding"
|
||||
|
||||
return is_embedding_mode
|
||||
|
||||
@@ -118,6 +118,12 @@ async def handle_streaming_transcription(
|
||||
if result is None: # End of stream
|
||||
logger.info("Streaming transcription: transcript stream ended")
|
||||
break
|
||||
if result.error:
|
||||
logger.warning(
|
||||
f"Streaming transcription: provider error: {result.error}"
|
||||
)
|
||||
await websocket.send_json({"type": "error", "message": result.error})
|
||||
continue
|
||||
# Send if text changed OR if VAD detected end of speech (for auto-send trigger)
|
||||
if result.text and (result.text != last_transcript or result.is_vad_end):
|
||||
last_transcript = result.text
|
||||
|
||||
@@ -561,7 +561,7 @@ def handle_send_chat_message(
|
||||
tenant_id = get_current_tenant_id()
|
||||
mt_cloud_telemetry(
|
||||
tenant_id=tenant_id,
|
||||
distinct_id=tenant_id if user.is_anonymous else str(user.id),
|
||||
distinct_id=tenant_id if user.is_anonymous else user.email,
|
||||
event=MilestoneRecordType.RAN_QUERY,
|
||||
)
|
||||
|
||||
|
||||
239
backend/onyx/server/query_and_chat/streaming_utils.py
Normal file
239
backend/onyx/server/query_and_chat/streaming_utils.py
Normal file
@@ -0,0 +1,239 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from onyx.context.search.models import SavedSearchDoc
|
||||
from onyx.context.search.models import SearchDoc
|
||||
from onyx.server.query_and_chat.placement import Placement
|
||||
from onyx.server.query_and_chat.streaming_models import AgentResponseDelta
|
||||
from onyx.server.query_and_chat.streaming_models import AgentResponseStart
|
||||
from onyx.server.query_and_chat.streaming_models import CitationInfo
|
||||
from onyx.server.query_and_chat.streaming_models import GeneratedImage
|
||||
from onyx.server.query_and_chat.streaming_models import ImageGenerationFinal
|
||||
from onyx.server.query_and_chat.streaming_models import ImageGenerationToolStart
|
||||
from onyx.server.query_and_chat.streaming_models import OpenUrlDocuments
|
||||
from onyx.server.query_and_chat.streaming_models import OpenUrlStart
|
||||
from onyx.server.query_and_chat.streaming_models import OpenUrlUrls
|
||||
from onyx.server.query_and_chat.streaming_models import Packet
|
||||
from onyx.server.query_and_chat.streaming_models import ReasoningDelta
|
||||
from onyx.server.query_and_chat.streaming_models import ReasoningStart
|
||||
from onyx.server.query_and_chat.streaming_models import SearchToolDocumentsDelta
|
||||
from onyx.server.query_and_chat.streaming_models import SearchToolQueriesDelta
|
||||
from onyx.server.query_and_chat.streaming_models import SearchToolStart
|
||||
from onyx.server.query_and_chat.streaming_models import SectionEnd
|
||||
|
||||
|
||||
_CANNOT_SHOW_STEP_RESULTS_STR = "[Cannot display step results]"
|
||||
|
||||
|
||||
def _adjust_message_text_for_agent_search_results(
|
||||
adjusted_message_text: str,
|
||||
final_documents: list[SavedSearchDoc], # noqa: ARG001
|
||||
) -> str:
|
||||
# Remove all [Q<integer>] patterns (sub-question citations)
|
||||
return re.sub(r"\[Q\d+\]", "", adjusted_message_text)
|
||||
|
||||
|
||||
def _replace_d_citations_with_links(
|
||||
message_text: str, final_documents: list[SavedSearchDoc]
|
||||
) -> str:
|
||||
def replace_citation(match: re.Match[str]) -> str:
|
||||
d_number = match.group(1)
|
||||
try:
|
||||
doc_index = int(d_number) - 1
|
||||
if 0 <= doc_index < len(final_documents):
|
||||
doc = final_documents[doc_index]
|
||||
link = doc.link if doc.link else ""
|
||||
return f"[[{d_number}]]({link})"
|
||||
return match.group(0)
|
||||
except (ValueError, IndexError):
|
||||
return match.group(0)
|
||||
|
||||
return re.sub(r"\[D(\d+)\]", replace_citation, message_text)
|
||||
|
||||
|
||||
def create_message_packets(
|
||||
message_text: str,
|
||||
final_documents: list[SavedSearchDoc] | None,
|
||||
turn_index: int,
|
||||
is_legacy_agentic: bool = False,
|
||||
) -> list[Packet]:
|
||||
packets: list[Packet] = []
|
||||
|
||||
packets.append(
|
||||
Packet(
|
||||
placement=Placement(turn_index=turn_index),
|
||||
obj=AgentResponseStart(
|
||||
final_documents=SearchDoc.from_saved_search_docs(final_documents or []),
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
adjusted_message_text = message_text
|
||||
if is_legacy_agentic:
|
||||
if final_documents is not None:
|
||||
adjusted_message_text = _adjust_message_text_for_agent_search_results(
|
||||
message_text, final_documents
|
||||
)
|
||||
adjusted_message_text = _replace_d_citations_with_links(
|
||||
adjusted_message_text, final_documents
|
||||
)
|
||||
else:
|
||||
adjusted_message_text = re.sub(r"\[Q\d+\]", "", message_text)
|
||||
|
||||
packets.append(
|
||||
Packet(
|
||||
placement=Placement(turn_index=turn_index),
|
||||
obj=AgentResponseDelta(
|
||||
content=adjusted_message_text,
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
packets.append(
|
||||
Packet(
|
||||
placement=Placement(turn_index=turn_index),
|
||||
obj=SectionEnd(),
|
||||
)
|
||||
)
|
||||
|
||||
return packets
|
||||
|
||||
|
||||
def create_citation_packets(
|
||||
citation_info_list: list[CitationInfo], turn_index: int
|
||||
) -> list[Packet]:
|
||||
packets: list[Packet] = []
|
||||
|
||||
# Emit each citation as a separate CitationInfo packet
|
||||
for citation_info in citation_info_list:
|
||||
packets.append(
|
||||
Packet(
|
||||
placement=Placement(turn_index=turn_index),
|
||||
obj=citation_info,
|
||||
)
|
||||
)
|
||||
|
||||
packets.append(Packet(placement=Placement(turn_index=turn_index), obj=SectionEnd()))
|
||||
|
||||
return packets
|
||||
|
||||
|
||||
def create_reasoning_packets(reasoning_text: str, turn_index: int) -> list[Packet]:
|
||||
packets: list[Packet] = []
|
||||
|
||||
packets.append(
|
||||
Packet(placement=Placement(turn_index=turn_index), obj=ReasoningStart())
|
||||
)
|
||||
|
||||
packets.append(
|
||||
Packet(
|
||||
placement=Placement(turn_index=turn_index),
|
||||
obj=ReasoningDelta(
|
||||
reasoning=reasoning_text,
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
packets.append(Packet(placement=Placement(turn_index=turn_index), obj=SectionEnd()))
|
||||
|
||||
return packets
|
||||
|
||||
|
||||
def create_image_generation_packets(
|
||||
images: list[GeneratedImage], turn_index: int
|
||||
) -> list[Packet]:
|
||||
packets: list[Packet] = []
|
||||
|
||||
packets.append(
|
||||
Packet(
|
||||
placement=Placement(turn_index=turn_index),
|
||||
obj=ImageGenerationToolStart(),
|
||||
)
|
||||
)
|
||||
|
||||
packets.append(
|
||||
Packet(
|
||||
placement=Placement(turn_index=turn_index),
|
||||
obj=ImageGenerationFinal(images=images),
|
||||
),
|
||||
)
|
||||
|
||||
packets.append(Packet(placement=Placement(turn_index=turn_index), obj=SectionEnd()))
|
||||
|
||||
return packets
|
||||
|
||||
|
||||
def create_fetch_packets(
|
||||
fetch_docs: list[SavedSearchDoc],
|
||||
urls: list[str],
|
||||
turn_index: int,
|
||||
) -> list[Packet]:
|
||||
packets: list[Packet] = []
|
||||
# Emit start packet
|
||||
packets.append(
|
||||
Packet(
|
||||
placement=Placement(turn_index=turn_index),
|
||||
obj=OpenUrlStart(),
|
||||
)
|
||||
)
|
||||
# Emit URLs packet
|
||||
packets.append(
|
||||
Packet(
|
||||
placement=Placement(turn_index=turn_index),
|
||||
obj=OpenUrlUrls(urls=urls),
|
||||
)
|
||||
)
|
||||
# Emit documents packet
|
||||
packets.append(
|
||||
Packet(
|
||||
placement=Placement(turn_index=turn_index),
|
||||
obj=OpenUrlDocuments(
|
||||
documents=SearchDoc.from_saved_search_docs(fetch_docs)
|
||||
),
|
||||
)
|
||||
)
|
||||
packets.append(Packet(placement=Placement(turn_index=turn_index), obj=SectionEnd()))
|
||||
return packets
|
||||
|
||||
|
||||
def create_search_packets(
|
||||
search_queries: list[str],
|
||||
saved_search_docs: list[SavedSearchDoc],
|
||||
is_internet_search: bool,
|
||||
turn_index: int,
|
||||
) -> list[Packet]:
|
||||
packets: list[Packet] = []
|
||||
|
||||
packets.append(
|
||||
Packet(
|
||||
placement=Placement(turn_index=turn_index),
|
||||
obj=SearchToolStart(
|
||||
is_internet_search=is_internet_search,
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
# Emit queries if present
|
||||
if search_queries:
|
||||
packets.append(
|
||||
Packet(
|
||||
placement=Placement(turn_index=turn_index),
|
||||
obj=SearchToolQueriesDelta(queries=search_queries),
|
||||
),
|
||||
)
|
||||
|
||||
# Emit documents if present
|
||||
if saved_search_docs:
|
||||
packets.append(
|
||||
Packet(
|
||||
placement=Placement(turn_index=turn_index),
|
||||
obj=SearchToolDocumentsDelta(
|
||||
documents=SearchDoc.from_saved_search_docs(saved_search_docs)
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
packets.append(Packet(placement=Placement(turn_index=turn_index), obj=SectionEnd()))
|
||||
|
||||
return packets
|
||||
@@ -2,7 +2,6 @@ import contextvars
|
||||
import threading
|
||||
import uuid
|
||||
from enum import Enum
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
|
||||
@@ -153,7 +152,7 @@ def mt_cloud_telemetry(
|
||||
tenant_id: str,
|
||||
distinct_id: str,
|
||||
event: MilestoneRecordType,
|
||||
properties: dict[str, Any] | None = None,
|
||||
properties: dict | None = None,
|
||||
) -> None:
|
||||
if not MULTI_TENANT:
|
||||
return
|
||||
@@ -174,18 +173,3 @@ def mt_cloud_telemetry(
|
||||
attribute="event_telemetry",
|
||||
fallback=noop_fallback,
|
||||
)(distinct_id, event, all_properties)
|
||||
|
||||
|
||||
def mt_cloud_identify(
|
||||
distinct_id: str,
|
||||
properties: dict[str, Any] | None = None,
|
||||
) -> None:
|
||||
"""Create/update a PostHog person profile (Cloud only)."""
|
||||
if not MULTI_TENANT:
|
||||
return
|
||||
|
||||
fetch_versioned_implementation_with_fallback(
|
||||
module="onyx.utils.telemetry",
|
||||
attribute="identify_user",
|
||||
fallback=noop_fallback,
|
||||
)(distinct_id, properties)
|
||||
|
||||
@@ -15,6 +15,9 @@ class TranscriptResult(BaseModel):
|
||||
is_vad_end: bool = False
|
||||
"""True if VAD detected end of speech (silence). Use for auto-send."""
|
||||
|
||||
error: str | None = None
|
||||
"""Provider error message to forward to the client, if any."""
|
||||
|
||||
|
||||
class StreamingTranscriberProtocol(Protocol):
|
||||
"""Protocol for streaming transcription sessions."""
|
||||
|
||||
@@ -56,6 +56,17 @@ def _http_to_ws_url(http_url: str) -> str:
|
||||
return http_url
|
||||
|
||||
|
||||
_USER_FACING_ERROR_MESSAGES: dict[str, str] = {
|
||||
"input_audio_buffer_commit_empty": (
|
||||
"No audio was recorded. Please check your microphone and try again."
|
||||
),
|
||||
"invalid_api_key": "Voice service authentication failed. Please contact support.",
|
||||
"rate_limit_exceeded": "Voice service is temporarily busy. Please try again shortly.",
|
||||
}
|
||||
|
||||
_DEFAULT_USER_ERROR = "A voice transcription error occurred. Please try again."
|
||||
|
||||
|
||||
class OpenAIStreamingTranscriber(StreamingTranscriberProtocol):
|
||||
"""Streaming transcription using OpenAI Realtime API."""
|
||||
|
||||
@@ -142,6 +153,17 @@ class OpenAIStreamingTranscriber(StreamingTranscriberProtocol):
|
||||
if msg_type == OpenAIRealtimeMessageType.ERROR:
|
||||
error = data.get("error", {})
|
||||
self._logger.error(f"OpenAI error: {error}")
|
||||
error_code = error.get("code", "")
|
||||
user_message = _USER_FACING_ERROR_MESSAGES.get(
|
||||
error_code, _DEFAULT_USER_ERROR
|
||||
)
|
||||
await self._transcript_queue.put(
|
||||
TranscriptResult(
|
||||
text="",
|
||||
is_vad_end=False,
|
||||
error=user_message,
|
||||
)
|
||||
)
|
||||
continue
|
||||
|
||||
# Handle VAD events
|
||||
|
||||
@@ -65,7 +65,7 @@ attrs==25.4.0
|
||||
# jsonschema
|
||||
# referencing
|
||||
# zeep
|
||||
authlib==1.6.9
|
||||
authlib==1.6.7
|
||||
# via fastmcp
|
||||
azure-cognitiveservices-speech==1.38.0
|
||||
# via onyx
|
||||
@@ -698,7 +698,7 @@ py-key-value-aio==0.4.4
|
||||
# via fastmcp
|
||||
pyairtable==3.0.1
|
||||
# via onyx
|
||||
pyasn1==0.6.3
|
||||
pyasn1==0.6.2
|
||||
# via
|
||||
# pyasn1-modules
|
||||
# rsa
|
||||
@@ -737,7 +737,7 @@ pygithub==2.5.0
|
||||
# via onyx
|
||||
pygments==2.19.2
|
||||
# via rich
|
||||
pyjwt==2.12.0
|
||||
pyjwt==2.11.0
|
||||
# via
|
||||
# fastapi-users
|
||||
# mcp
|
||||
|
||||
@@ -263,7 +263,7 @@ oauthlib==3.2.2
|
||||
# via
|
||||
# kubernetes
|
||||
# requests-oauthlib
|
||||
onyx-devtools==0.7.1
|
||||
onyx-devtools==0.7.0
|
||||
# via onyx
|
||||
openai==2.14.0
|
||||
# via
|
||||
@@ -326,7 +326,7 @@ pure-eval==0.2.3
|
||||
# via stack-data
|
||||
py==1.11.0
|
||||
# via retry
|
||||
pyasn1==0.6.3
|
||||
pyasn1==0.6.2
|
||||
# via
|
||||
# pyasn1-modules
|
||||
# rsa
|
||||
@@ -353,7 +353,7 @@ pygments==2.19.2
|
||||
# via
|
||||
# ipython
|
||||
# ipython-pygments-lexers
|
||||
pyjwt==2.12.0
|
||||
pyjwt==2.11.0
|
||||
# via mcp
|
||||
pyparsing==3.2.5
|
||||
# via matplotlib
|
||||
|
||||
@@ -195,7 +195,7 @@ propcache==0.4.1
|
||||
# yarl
|
||||
py==1.11.0
|
||||
# via retry
|
||||
pyasn1==0.6.3
|
||||
pyasn1==0.6.2
|
||||
# via
|
||||
# pyasn1-modules
|
||||
# rsa
|
||||
@@ -218,7 +218,7 @@ pydantic-core==2.33.2
|
||||
# via pydantic
|
||||
pydantic-settings==2.12.0
|
||||
# via mcp
|
||||
pyjwt==2.12.0
|
||||
pyjwt==2.11.0
|
||||
# via mcp
|
||||
python-dateutil==2.8.2
|
||||
# via
|
||||
|
||||
@@ -285,7 +285,7 @@ psutil==7.1.3
|
||||
# via accelerate
|
||||
py==1.11.0
|
||||
# via retry
|
||||
pyasn1==0.6.3
|
||||
pyasn1==0.6.2
|
||||
# via
|
||||
# pyasn1-modules
|
||||
# rsa
|
||||
@@ -308,7 +308,7 @@ pydantic-core==2.33.2
|
||||
# via pydantic
|
||||
pydantic-settings==2.12.0
|
||||
# via mcp
|
||||
pyjwt==2.12.0
|
||||
pyjwt==2.11.0
|
||||
# via mcp
|
||||
python-dateutil==2.8.2
|
||||
# via
|
||||
|
||||
@@ -1,471 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import sys
|
||||
from dataclasses import asdict
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from typing import TypedDict
|
||||
from typing import TypeGuard
|
||||
|
||||
import aiohttp
|
||||
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s %(levelname)s %(message)s",
|
||||
)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
DEFAULT_API_BASE = "http://localhost:3000"
|
||||
INTERNAL_SEARCH_TOOL_NAME = "internal_search"
|
||||
INTERNAL_SEARCH_IN_CODE_TOOL_ID = "SearchTool"
|
||||
MAX_REQUEST_ATTEMPTS = 5
|
||||
RETRIABLE_STATUS_CODES = {429, 500, 502, 503, 504}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class QuestionRecord:
|
||||
question_id: str
|
||||
question: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class AnswerRecord:
|
||||
question_id: str
|
||||
answer: str
|
||||
document_ids: list[str]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FailedQuestionRecord:
|
||||
question_id: str
|
||||
error: str
|
||||
|
||||
|
||||
class Citation(TypedDict, total=False):
|
||||
citation_number: int
|
||||
document_id: str
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(
|
||||
description=(
|
||||
"Submit questions to Onyx chat with internal search forced and write "
|
||||
"answers to a JSONL file."
|
||||
)
|
||||
)
|
||||
parser.add_argument(
|
||||
"--questions-file",
|
||||
type=Path,
|
||||
required=True,
|
||||
help="Path to the input questions JSONL file.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output-file",
|
||||
type=Path,
|
||||
required=True,
|
||||
help="Path to the output answers JSONL file.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--api-key",
|
||||
type=str,
|
||||
required=True,
|
||||
help="API key used to authenticate against Onyx.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--api-base",
|
||||
type=str,
|
||||
default=DEFAULT_API_BASE,
|
||||
help=(
|
||||
"Frontend base URL for Onyx. If `/api` is omitted, it will be added "
|
||||
f"automatically. Default: {DEFAULT_API_BASE}"
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--parallelism",
|
||||
type=int,
|
||||
default=1,
|
||||
help="Number of questions to process in parallel. Default: 1.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--max-questions",
|
||||
type=int,
|
||||
default=None,
|
||||
help="Optional cap on how many questions to process. Defaults to all.",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def normalize_api_base(api_base: str) -> str:
|
||||
normalized = api_base.rstrip("/")
|
||||
if normalized.endswith("/api"):
|
||||
return normalized
|
||||
return f"{normalized}/api"
|
||||
|
||||
|
||||
def load_questions(questions_file: Path) -> list[QuestionRecord]:
|
||||
if not questions_file.exists():
|
||||
raise FileNotFoundError(f"Questions file not found: {questions_file}")
|
||||
|
||||
questions: list[QuestionRecord] = []
|
||||
with questions_file.open("r", encoding="utf-8") as file:
|
||||
for line_number, line in enumerate(file, start=1):
|
||||
stripped_line = line.strip()
|
||||
if not stripped_line:
|
||||
continue
|
||||
|
||||
try:
|
||||
payload = json.loads(stripped_line)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise ValueError(
|
||||
f"Invalid JSON on line {line_number} of {questions_file}"
|
||||
) from exc
|
||||
|
||||
question_id = payload.get("question_id")
|
||||
question = payload.get("question")
|
||||
|
||||
if not isinstance(question_id, str) or not question_id:
|
||||
raise ValueError(
|
||||
f"Line {line_number} is missing a non-empty `question_id`."
|
||||
)
|
||||
if not isinstance(question, str) or not question:
|
||||
raise ValueError(
|
||||
f"Line {line_number} is missing a non-empty `question`."
|
||||
)
|
||||
|
||||
questions.append(QuestionRecord(question_id=question_id, question=question))
|
||||
|
||||
return questions
|
||||
|
||||
|
||||
async def read_json_response(
|
||||
response: aiohttp.ClientResponse,
|
||||
) -> dict[str, Any] | list[dict[str, Any]]:
|
||||
response_text = await response.text()
|
||||
if response.status >= 400:
|
||||
raise RuntimeError(
|
||||
f"Request to {response.url} failed with {response.status}: {response_text}"
|
||||
)
|
||||
|
||||
try:
|
||||
payload = json.loads(response_text)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise RuntimeError(
|
||||
f"Request to {response.url} returned non-JSON content: {response_text}"
|
||||
) from exc
|
||||
|
||||
if not isinstance(payload, (dict, list)):
|
||||
raise RuntimeError(
|
||||
f"Unexpected response payload type from {response.url}: {type(payload)}"
|
||||
)
|
||||
|
||||
return payload
|
||||
|
||||
|
||||
async def request_json_with_retries(
|
||||
session: aiohttp.ClientSession,
|
||||
method: str,
|
||||
url: str,
|
||||
headers: dict[str, str],
|
||||
json_payload: dict[str, Any] | None = None,
|
||||
) -> dict[str, Any] | list[dict[str, Any]]:
|
||||
backoff_seconds = 1.0
|
||||
|
||||
for attempt in range(1, MAX_REQUEST_ATTEMPTS + 1):
|
||||
try:
|
||||
async with session.request(
|
||||
method=method,
|
||||
url=url,
|
||||
headers=headers,
|
||||
json=json_payload,
|
||||
) as response:
|
||||
if (
|
||||
response.status in RETRIABLE_STATUS_CODES
|
||||
and attempt < MAX_REQUEST_ATTEMPTS
|
||||
):
|
||||
response_text = await response.text()
|
||||
logger.warning(
|
||||
"Retryable response from %s on attempt %s/%s: %s %s",
|
||||
url,
|
||||
attempt,
|
||||
MAX_REQUEST_ATTEMPTS,
|
||||
response.status,
|
||||
response_text,
|
||||
)
|
||||
await asyncio.sleep(backoff_seconds)
|
||||
backoff_seconds *= 2
|
||||
continue
|
||||
|
||||
return await read_json_response(response)
|
||||
except (aiohttp.ClientError, asyncio.TimeoutError) as exc:
|
||||
if attempt == MAX_REQUEST_ATTEMPTS:
|
||||
raise RuntimeError(
|
||||
f"Request to {url} failed after {MAX_REQUEST_ATTEMPTS} attempts."
|
||||
) from exc
|
||||
|
||||
logger.warning(
|
||||
"Request to %s failed on attempt %s/%s: %s",
|
||||
url,
|
||||
attempt,
|
||||
MAX_REQUEST_ATTEMPTS,
|
||||
exc,
|
||||
)
|
||||
await asyncio.sleep(backoff_seconds)
|
||||
backoff_seconds *= 2
|
||||
|
||||
raise RuntimeError(f"Request to {url} failed unexpectedly.")
|
||||
|
||||
|
||||
def extract_document_ids(citation_info: object) -> list[str]:
|
||||
if not isinstance(citation_info, list):
|
||||
return []
|
||||
|
||||
sorted_citations = sorted(
|
||||
(citation for citation in citation_info if _is_valid_citation(citation)),
|
||||
key=_citation_sort_key,
|
||||
)
|
||||
|
||||
document_ids: list[str] = []
|
||||
seen_document_ids: set[str] = set()
|
||||
for citation in sorted_citations:
|
||||
document_id = citation["document_id"]
|
||||
if document_id not in seen_document_ids:
|
||||
seen_document_ids.add(document_id)
|
||||
document_ids.append(document_id)
|
||||
|
||||
return document_ids
|
||||
|
||||
|
||||
def _is_valid_citation(citation: object) -> TypeGuard[Citation]:
|
||||
return (
|
||||
isinstance(citation, dict)
|
||||
and isinstance(citation.get("document_id"), str)
|
||||
and bool(citation["document_id"])
|
||||
)
|
||||
|
||||
|
||||
def _citation_sort_key(citation: Citation) -> int:
|
||||
citation_number = citation.get("citation_number")
|
||||
if isinstance(citation_number, int):
|
||||
return citation_number
|
||||
return sys.maxsize
|
||||
|
||||
|
||||
async def fetch_internal_search_tool_id(
|
||||
session: aiohttp.ClientSession,
|
||||
api_base: str,
|
||||
headers: dict[str, str],
|
||||
) -> int:
|
||||
payload = await request_json_with_retries(
|
||||
session=session,
|
||||
method="GET",
|
||||
url=f"{api_base}/tool",
|
||||
headers=headers,
|
||||
)
|
||||
|
||||
if not isinstance(payload, list):
|
||||
raise RuntimeError("Expected `/tool` to return a list.")
|
||||
|
||||
for tool in payload:
|
||||
if not isinstance(tool, dict):
|
||||
continue
|
||||
|
||||
if tool.get("in_code_tool_id") == INTERNAL_SEARCH_IN_CODE_TOOL_ID:
|
||||
tool_id = tool.get("id")
|
||||
if isinstance(tool_id, int):
|
||||
return tool_id
|
||||
|
||||
for tool in payload:
|
||||
if not isinstance(tool, dict):
|
||||
continue
|
||||
|
||||
if tool.get("name") == INTERNAL_SEARCH_TOOL_NAME:
|
||||
tool_id = tool.get("id")
|
||||
if isinstance(tool_id, int):
|
||||
return tool_id
|
||||
|
||||
raise RuntimeError(
|
||||
"Could not find the internal search tool in `/tool`. "
|
||||
"Make sure SearchTool is available for this environment."
|
||||
)
|
||||
|
||||
|
||||
async def submit_question(
|
||||
session: aiohttp.ClientSession,
|
||||
api_base: str,
|
||||
headers: dict[str, str],
|
||||
internal_search_tool_id: int,
|
||||
question_record: QuestionRecord,
|
||||
) -> AnswerRecord:
|
||||
payload = {
|
||||
"message": question_record.question,
|
||||
"chat_session_info": {"persona_id": 0},
|
||||
"parent_message_id": None,
|
||||
"file_descriptors": [],
|
||||
"allowed_tool_ids": [internal_search_tool_id],
|
||||
"forced_tool_id": internal_search_tool_id,
|
||||
"stream": False,
|
||||
}
|
||||
|
||||
response_payload = await request_json_with_retries(
|
||||
session=session,
|
||||
method="POST",
|
||||
url=f"{api_base}/chat/send-chat-message",
|
||||
headers=headers,
|
||||
json_payload=payload,
|
||||
)
|
||||
|
||||
if not isinstance(response_payload, dict):
|
||||
raise RuntimeError(
|
||||
"Expected `/chat/send-chat-message` to return an object when `stream=false`."
|
||||
)
|
||||
|
||||
answer = response_payload.get("answer_citationless")
|
||||
if not isinstance(answer, str):
|
||||
answer = response_payload.get("answer")
|
||||
|
||||
if not isinstance(answer, str):
|
||||
raise RuntimeError(
|
||||
f"Response for question {question_record.question_id} is missing `answer`."
|
||||
)
|
||||
|
||||
return AnswerRecord(
|
||||
question_id=question_record.question_id,
|
||||
answer=answer,
|
||||
document_ids=extract_document_ids(response_payload.get("citation_info")),
|
||||
)
|
||||
|
||||
|
||||
async def generate_answers(
|
||||
questions: list[QuestionRecord],
|
||||
output_file: Path,
|
||||
api_base: str,
|
||||
api_key: str,
|
||||
parallelism: int,
|
||||
) -> None:
|
||||
if parallelism < 1:
|
||||
raise ValueError("`--parallelism` must be at least 1.")
|
||||
|
||||
headers = {
|
||||
"Authorization": f"Bearer {api_key}",
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
|
||||
timeout = aiohttp.ClientTimeout(
|
||||
total=None,
|
||||
connect=30,
|
||||
sock_connect=30,
|
||||
sock_read=600,
|
||||
)
|
||||
connector = aiohttp.TCPConnector(limit=parallelism)
|
||||
|
||||
output_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
with output_file.open("a", encoding="utf-8") as file:
|
||||
async with aiohttp.ClientSession(
|
||||
timeout=timeout, connector=connector
|
||||
) as session:
|
||||
internal_search_tool_id = await fetch_internal_search_tool_id(
|
||||
session=session,
|
||||
api_base=api_base,
|
||||
headers=headers,
|
||||
)
|
||||
logger.info("Using internal search tool id %s", internal_search_tool_id)
|
||||
|
||||
semaphore = asyncio.Semaphore(parallelism)
|
||||
progress_lock = asyncio.Lock()
|
||||
write_lock = asyncio.Lock()
|
||||
completed = 0
|
||||
successful = 0
|
||||
failed_questions: list[FailedQuestionRecord] = []
|
||||
total = len(questions)
|
||||
|
||||
async def process_question(question_record: QuestionRecord) -> None:
|
||||
nonlocal completed
|
||||
nonlocal successful
|
||||
|
||||
try:
|
||||
async with semaphore:
|
||||
result = await submit_question(
|
||||
session=session,
|
||||
api_base=api_base,
|
||||
headers=headers,
|
||||
internal_search_tool_id=internal_search_tool_id,
|
||||
question_record=question_record,
|
||||
)
|
||||
except Exception as exc:
|
||||
async with progress_lock:
|
||||
completed += 1
|
||||
failed_questions.append(
|
||||
FailedQuestionRecord(
|
||||
question_id=question_record.question_id,
|
||||
error=str(exc),
|
||||
)
|
||||
)
|
||||
logger.exception(
|
||||
"Failed question %s (%s/%s)",
|
||||
question_record.question_id,
|
||||
completed,
|
||||
total,
|
||||
)
|
||||
return
|
||||
|
||||
async with write_lock:
|
||||
file.write(json.dumps(asdict(result), ensure_ascii=False))
|
||||
file.write("\n")
|
||||
file.flush()
|
||||
|
||||
async with progress_lock:
|
||||
completed += 1
|
||||
successful += 1
|
||||
logger.info("Processed %s/%s questions", completed, total)
|
||||
|
||||
await asyncio.gather(
|
||||
*(process_question(question_record) for question_record in questions)
|
||||
)
|
||||
|
||||
if failed_questions:
|
||||
logger.warning(
|
||||
"Completed with %s failed questions and %s successful questions.",
|
||||
len(failed_questions),
|
||||
successful,
|
||||
)
|
||||
for failed_question in failed_questions:
|
||||
logger.warning(
|
||||
"Failed question %s: %s",
|
||||
failed_question.question_id,
|
||||
failed_question.error,
|
||||
)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
args = parse_args()
|
||||
questions = load_questions(args.questions_file)
|
||||
api_base = normalize_api_base(args.api_base)
|
||||
|
||||
if args.max_questions is not None:
|
||||
if args.max_questions < 1:
|
||||
raise ValueError("`--max-questions` must be at least 1 when provided.")
|
||||
questions = questions[: args.max_questions]
|
||||
|
||||
logger.info("Loaded %s questions from %s", len(questions), args.questions_file)
|
||||
logger.info("Writing answers to %s", args.output_file)
|
||||
|
||||
asyncio.run(
|
||||
generate_answers(
|
||||
questions=questions,
|
||||
output_file=args.output_file,
|
||||
api_base=api_base,
|
||||
api_key=args.api_key,
|
||||
parallelism=args.parallelism,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,291 +0,0 @@
|
||||
"""
|
||||
Script to upload files from a directory as individual file connectors in Onyx.
|
||||
Each file gets its own connector named after the file.
|
||||
|
||||
Usage:
|
||||
python upload_files_as_connectors.py --data-dir /path/to/files --api-key YOUR_KEY
|
||||
python upload_files_as_connectors.py --data-dir /path/to/files --api-key YOUR_KEY --api-base http://onyxserver:3000
|
||||
python upload_files_as_connectors.py --data-dir /path/to/files --api-key YOUR_KEY --file-glob '*.zip'
|
||||
|
||||
Requires:
|
||||
pip install requests
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import fnmatch
|
||||
import os
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
|
||||
import requests
|
||||
|
||||
REQUEST_TIMEOUT = 900 # 15 minutes
|
||||
|
||||
|
||||
def _elapsed_printer(label: str, stop_event: threading.Event) -> None:
|
||||
"""Print a live elapsed-time counter until stop_event is set."""
|
||||
start = time.monotonic()
|
||||
while not stop_event.wait(timeout=1):
|
||||
elapsed = int(time.monotonic() - start)
|
||||
m, s = divmod(elapsed, 60)
|
||||
print(f"\r {label} ... {m:02d}:{s:02d}", end="", flush=True)
|
||||
elapsed = int(time.monotonic() - start)
|
||||
m, s = divmod(elapsed, 60)
|
||||
print(f"\r {label} ... {m:02d}:{s:02d} done")
|
||||
|
||||
|
||||
def _timed_request(label: str, fn: object) -> requests.Response:
|
||||
"""Run a request function while displaying a live elapsed timer."""
|
||||
stop = threading.Event()
|
||||
t = threading.Thread(target=_elapsed_printer, args=(label, stop), daemon=True)
|
||||
t.start()
|
||||
try:
|
||||
resp = fn() # type: ignore[operator]
|
||||
finally:
|
||||
stop.set()
|
||||
t.join()
|
||||
return resp
|
||||
|
||||
|
||||
def upload_file(
|
||||
session: requests.Session, base_url: str, file_path: str
|
||||
) -> dict | None:
|
||||
"""Upload a single file and return the response with file_paths and file_names."""
|
||||
with open(file_path, "rb") as f:
|
||||
resp = _timed_request(
|
||||
"Uploading",
|
||||
lambda: session.post(
|
||||
f"{base_url}/api/manage/admin/connector/file/upload",
|
||||
files={"files": (os.path.basename(file_path), f)},
|
||||
timeout=REQUEST_TIMEOUT,
|
||||
),
|
||||
)
|
||||
if not resp.ok:
|
||||
print(f" ERROR uploading: {resp.text}")
|
||||
return None
|
||||
return resp.json()
|
||||
|
||||
|
||||
def create_connector(
|
||||
session: requests.Session,
|
||||
base_url: str,
|
||||
name: str,
|
||||
file_paths: list[str],
|
||||
file_names: list[str],
|
||||
zip_metadata_file_id: str | None,
|
||||
) -> int | None:
|
||||
"""Create a file connector and return its ID."""
|
||||
resp = _timed_request(
|
||||
"Creating connector",
|
||||
lambda: session.post(
|
||||
f"{base_url}/api/manage/admin/connector",
|
||||
json={
|
||||
"name": name,
|
||||
"source": "file",
|
||||
"input_type": "load_state",
|
||||
"connector_specific_config": {
|
||||
"file_locations": file_paths,
|
||||
"file_names": file_names,
|
||||
"zip_metadata_file_id": zip_metadata_file_id,
|
||||
},
|
||||
"refresh_freq": None,
|
||||
"prune_freq": None,
|
||||
"indexing_start": None,
|
||||
"access_type": "public",
|
||||
"groups": [],
|
||||
},
|
||||
timeout=REQUEST_TIMEOUT,
|
||||
),
|
||||
)
|
||||
if not resp.ok:
|
||||
print(f" ERROR creating connector: {resp.text}")
|
||||
return None
|
||||
return resp.json()["id"]
|
||||
|
||||
|
||||
def create_credential(
|
||||
session: requests.Session, base_url: str, name: str
|
||||
) -> int | None:
|
||||
"""Create a dummy credential for the file connector."""
|
||||
resp = session.post(
|
||||
f"{base_url}/api/manage/credential",
|
||||
json={
|
||||
"credential_json": {},
|
||||
"admin_public": True,
|
||||
"source": "file",
|
||||
"curator_public": True,
|
||||
"groups": [],
|
||||
"name": name,
|
||||
},
|
||||
timeout=REQUEST_TIMEOUT,
|
||||
)
|
||||
if not resp.ok:
|
||||
print(f" ERROR creating credential: {resp.text}")
|
||||
return None
|
||||
return resp.json()["id"]
|
||||
|
||||
|
||||
def link_credential(
|
||||
session: requests.Session,
|
||||
base_url: str,
|
||||
connector_id: int,
|
||||
credential_id: int,
|
||||
name: str,
|
||||
) -> bool:
|
||||
"""Link the connector to the credential (create CC pair)."""
|
||||
resp = session.put(
|
||||
f"{base_url}/api/manage/connector/{connector_id}/credential/{credential_id}",
|
||||
json={
|
||||
"name": name,
|
||||
"access_type": "public",
|
||||
"groups": [],
|
||||
"auto_sync_options": None,
|
||||
"processing_mode": "REGULAR",
|
||||
},
|
||||
timeout=REQUEST_TIMEOUT,
|
||||
)
|
||||
if not resp.ok:
|
||||
print(f" ERROR linking credential: {resp.text}")
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def run_connector(
|
||||
session: requests.Session,
|
||||
base_url: str,
|
||||
connector_id: int,
|
||||
credential_id: int,
|
||||
) -> bool:
|
||||
"""Trigger the connector to start indexing."""
|
||||
resp = session.post(
|
||||
f"{base_url}/api/manage/admin/connector/run-once",
|
||||
json={
|
||||
"connector_id": connector_id,
|
||||
"credentialIds": [credential_id],
|
||||
"from_beginning": False,
|
||||
},
|
||||
timeout=REQUEST_TIMEOUT,
|
||||
)
|
||||
if not resp.ok:
|
||||
print(f" ERROR running connector: {resp.text}")
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def process_file(session: requests.Session, base_url: str, file_path: str) -> bool:
|
||||
"""Process a single file through the full connector creation flow."""
|
||||
file_name = os.path.basename(file_path)
|
||||
connector_name = file_name
|
||||
print(f"Processing: {file_name}")
|
||||
|
||||
# Step 1: Upload
|
||||
upload_resp = upload_file(session, base_url, file_path)
|
||||
if not upload_resp:
|
||||
return False
|
||||
|
||||
# Step 2: Create connector
|
||||
connector_id = create_connector(
|
||||
session,
|
||||
base_url,
|
||||
name=f"FileConnector-{connector_name}",
|
||||
file_paths=upload_resp["file_paths"],
|
||||
file_names=upload_resp["file_names"],
|
||||
zip_metadata_file_id=upload_resp.get("zip_metadata_file_id"),
|
||||
)
|
||||
if connector_id is None:
|
||||
return False
|
||||
|
||||
# Step 3: Create credential
|
||||
credential_id = create_credential(session, base_url, name=connector_name)
|
||||
if credential_id is None:
|
||||
return False
|
||||
|
||||
# Step 4: Link connector to credential
|
||||
if not link_credential(
|
||||
session, base_url, connector_id, credential_id, connector_name
|
||||
):
|
||||
return False
|
||||
|
||||
# Step 5: Trigger indexing
|
||||
if not run_connector(session, base_url, connector_id, credential_id):
|
||||
return False
|
||||
|
||||
print(f" OK (connector_id={connector_id})")
|
||||
return True
|
||||
|
||||
|
||||
def get_authenticated_session(api_key: str) -> requests.Session:
|
||||
"""Create a session authenticated with an API key."""
|
||||
session = requests.Session()
|
||||
session.headers.update({"Authorization": f"Bearer {api_key}"})
|
||||
return session
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Upload files as individual Onyx file connectors."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--data-dir",
|
||||
required=True,
|
||||
help="Directory containing files to upload.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--api-base",
|
||||
default="http://localhost:3000",
|
||||
help="Base URL for the Onyx API (default: http://localhost:3000).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--api-key",
|
||||
required=True,
|
||||
help="API key for authentication.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--file-glob",
|
||||
default=None,
|
||||
help="Glob pattern to filter files (e.g. '*.json', '*.zip').",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
data_dir = args.data_dir
|
||||
base_url = args.api_base.rstrip("/")
|
||||
api_key = args.api_key
|
||||
file_glob = args.file_glob
|
||||
|
||||
if not os.path.isdir(data_dir):
|
||||
print(f"Error: {data_dir} is not a directory")
|
||||
sys.exit(1)
|
||||
|
||||
script_path = os.path.realpath(__file__)
|
||||
files = sorted(
|
||||
os.path.join(data_dir, f)
|
||||
for f in os.listdir(data_dir)
|
||||
if os.path.isfile(os.path.join(data_dir, f))
|
||||
and os.path.realpath(os.path.join(data_dir, f)) != script_path
|
||||
and (file_glob is None or fnmatch.fnmatch(f, file_glob))
|
||||
)
|
||||
|
||||
if not files:
|
||||
print(f"No files found in {data_dir}")
|
||||
sys.exit(1)
|
||||
|
||||
print(f"Found {len(files)} file(s) in {data_dir}\n")
|
||||
|
||||
session = get_authenticated_session(api_key)
|
||||
|
||||
success = 0
|
||||
failed = 0
|
||||
for file_path in files:
|
||||
if process_file(session, base_url, file_path):
|
||||
success += 1
|
||||
else:
|
||||
failed += 1
|
||||
# Small delay to avoid overwhelming the server
|
||||
time.sleep(0.5)
|
||||
|
||||
print(f"\nDone: {success} succeeded, {failed} failed out of {len(files)} files.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -45,21 +45,6 @@ npx playwright test <TEST_NAME>
|
||||
Shared fixtures live in `backend/tests/conftest.py`. Test subdirectories can define
|
||||
their own `conftest.py` for directory-scoped fixtures.
|
||||
|
||||
## Running Tests Repeatedly (`pytest-repeat`)
|
||||
|
||||
Use `pytest-repeat` to catch flaky tests by running them multiple times:
|
||||
|
||||
```bash
|
||||
# Run a specific test 50 times
|
||||
pytest --count=50 backend/tests/unit/path/to/test.py::test_name
|
||||
|
||||
# Stop on first failure with -x
|
||||
pytest --count=50 -x backend/tests/unit/path/to/test.py::test_name
|
||||
|
||||
# Repeat an entire test file
|
||||
pytest --count=10 backend/tests/unit/path/to/test_file.py
|
||||
```
|
||||
|
||||
## Best Practices
|
||||
|
||||
### Use `enable_ee` fixture instead of inlining
|
||||
|
||||
@@ -22,8 +22,6 @@ from onyx.document_index.interfaces_new import TenantState
|
||||
from onyx.document_index.opensearch.client import OpenSearchIndexClient
|
||||
from onyx.document_index.opensearch.client import wait_for_opensearch_with_timeout
|
||||
from onyx.document_index.opensearch.constants import DEFAULT_MAX_CHUNK_SIZE
|
||||
from onyx.document_index.opensearch.constants import HybridSearchNormalizationPipeline
|
||||
from onyx.document_index.opensearch.constants import HybridSearchSubqueryConfiguration
|
||||
from onyx.document_index.opensearch.opensearch_document_index import (
|
||||
generate_opensearch_filtered_access_control_list,
|
||||
)
|
||||
@@ -33,14 +31,9 @@ from onyx.document_index.opensearch.schema import DocumentSchema
|
||||
from onyx.document_index.opensearch.schema import get_opensearch_doc_chunk_id
|
||||
from onyx.document_index.opensearch.search import DocumentQuery
|
||||
from onyx.document_index.opensearch.search import (
|
||||
get_min_max_normalization_pipeline_name_and_config,
|
||||
)
|
||||
from onyx.document_index.opensearch.search import (
|
||||
get_normalization_pipeline_name_and_config,
|
||||
)
|
||||
from onyx.document_index.opensearch.search import (
|
||||
get_zscore_normalization_pipeline_name_and_config,
|
||||
MIN_MAX_NORMALIZATION_PIPELINE_CONFIG,
|
||||
)
|
||||
from onyx.document_index.opensearch.search import MIN_MAX_NORMALIZATION_PIPELINE_NAME
|
||||
from shared_configs.configs import POSTGRES_DEFAULT_SCHEMA
|
||||
|
||||
|
||||
@@ -56,46 +49,6 @@ def _patch_global_tenant_state(monkeypatch: pytest.MonkeyPatch, state: bool) ->
|
||||
monkeypatch.setattr("onyx.document_index.opensearch.schema.MULTI_TENANT", state)
|
||||
|
||||
|
||||
def _patch_hybrid_search_subquery_configuration(
|
||||
monkeypatch: pytest.MonkeyPatch, configuration: HybridSearchSubqueryConfiguration
|
||||
) -> None:
|
||||
"""
|
||||
Patches HYBRID_SEARCH_SUBQUERY_CONFIGURATION wherever necessary for this
|
||||
test file.
|
||||
|
||||
Args:
|
||||
monkeypatch: The test instance's monkeypatch instance, used for
|
||||
patching.
|
||||
configuration: The intended state of
|
||||
HYBRID_SEARCH_SUBQUERY_CONFIGURATION.
|
||||
"""
|
||||
monkeypatch.setattr(
|
||||
"onyx.document_index.opensearch.constants.HYBRID_SEARCH_SUBQUERY_CONFIGURATION",
|
||||
configuration,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"onyx.document_index.opensearch.search.HYBRID_SEARCH_SUBQUERY_CONFIGURATION",
|
||||
configuration,
|
||||
)
|
||||
|
||||
|
||||
def _patch_hybrid_search_normalization_pipeline(
|
||||
monkeypatch: pytest.MonkeyPatch, pipeline: HybridSearchNormalizationPipeline
|
||||
) -> None:
|
||||
"""
|
||||
Patches HYBRID_SEARCH_NORMALIZATION_PIPELINE wherever necessary for this
|
||||
test file.
|
||||
"""
|
||||
monkeypatch.setattr(
|
||||
"onyx.document_index.opensearch.constants.HYBRID_SEARCH_NORMALIZATION_PIPELINE",
|
||||
pipeline,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"onyx.document_index.opensearch.search.HYBRID_SEARCH_NORMALIZATION_PIPELINE",
|
||||
pipeline,
|
||||
)
|
||||
|
||||
|
||||
def _create_test_document_chunk(
|
||||
document_id: str,
|
||||
content: str,
|
||||
@@ -191,27 +144,14 @@ def test_client(
|
||||
@pytest.fixture(scope="function")
|
||||
def search_pipeline(test_client: OpenSearchIndexClient) -> Generator[None, None, None]:
|
||||
"""Creates a search pipeline for testing with automatic cleanup."""
|
||||
min_max_normalization_pipeline_name, min_max_normalization_pipeline_config = (
|
||||
get_min_max_normalization_pipeline_name_and_config()
|
||||
)
|
||||
zscore_normalization_pipeline_name, zscore_normalization_pipeline_config = (
|
||||
get_zscore_normalization_pipeline_name_and_config()
|
||||
)
|
||||
test_client.create_search_pipeline(
|
||||
pipeline_id=min_max_normalization_pipeline_name,
|
||||
pipeline_body=min_max_normalization_pipeline_config,
|
||||
)
|
||||
test_client.create_search_pipeline(
|
||||
pipeline_id=zscore_normalization_pipeline_name,
|
||||
pipeline_body=zscore_normalization_pipeline_config,
|
||||
pipeline_id=MIN_MAX_NORMALIZATION_PIPELINE_NAME,
|
||||
pipeline_body=MIN_MAX_NORMALIZATION_PIPELINE_CONFIG,
|
||||
)
|
||||
yield # Test runs here.
|
||||
try:
|
||||
test_client.delete_search_pipeline(
|
||||
pipeline_id=min_max_normalization_pipeline_name,
|
||||
)
|
||||
test_client.delete_search_pipeline(
|
||||
pipeline_id=zscore_normalization_pipeline_name,
|
||||
pipeline_id=MIN_MAX_NORMALIZATION_PIPELINE_NAME,
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
@@ -437,19 +377,18 @@ class TestOpenSearchClient:
|
||||
self, test_client: OpenSearchIndexClient
|
||||
) -> None:
|
||||
"""Tests creating and deleting a search pipeline."""
|
||||
# Precondition.
|
||||
pipeline_name, pipeline_config = get_normalization_pipeline_name_and_config()
|
||||
|
||||
# Under test and postcondition.
|
||||
# Should not raise.
|
||||
test_client.create_search_pipeline(
|
||||
pipeline_id=pipeline_name,
|
||||
pipeline_body=pipeline_config,
|
||||
pipeline_id=MIN_MAX_NORMALIZATION_PIPELINE_NAME,
|
||||
pipeline_body=MIN_MAX_NORMALIZATION_PIPELINE_CONFIG,
|
||||
)
|
||||
|
||||
# Under test and postcondition.
|
||||
# Should not raise.
|
||||
test_client.delete_search_pipeline(pipeline_id=pipeline_name)
|
||||
test_client.delete_search_pipeline(
|
||||
pipeline_id=MIN_MAX_NORMALIZATION_PIPELINE_NAME
|
||||
)
|
||||
|
||||
def test_index_document(
|
||||
self, test_client: OpenSearchIndexClient, monkeypatch: pytest.MonkeyPatch
|
||||
@@ -795,13 +734,13 @@ class TestOpenSearchClient:
|
||||
properties_to_update={"hidden": True},
|
||||
)
|
||||
|
||||
def test_hybrid_search_configurations_and_pipelines(
|
||||
def test_hybrid_search_with_pipeline(
|
||||
self,
|
||||
test_client: OpenSearchIndexClient,
|
||||
search_pipeline: None, # noqa: ARG002
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""Tests all hybrid search configurations and pipelines."""
|
||||
"""Tests hybrid search with a normalization pipeline."""
|
||||
# Precondition.
|
||||
_patch_global_tenant_state(monkeypatch, False)
|
||||
tenant_state = TenantState(tenant_id=POSTGRES_DEFAULT_SCHEMA, multitenant=False)
|
||||
@@ -810,6 +749,7 @@ class TestOpenSearchClient:
|
||||
)
|
||||
settings = DocumentSchema.get_index_settings()
|
||||
test_client.create_index(mappings=mappings, settings=settings)
|
||||
|
||||
# Index documents.
|
||||
docs = {
|
||||
"doc-1": _create_test_document_chunk(
|
||||
@@ -840,58 +780,40 @@ class TestOpenSearchClient:
|
||||
# Refresh index to make documents searchable.
|
||||
test_client.refresh_index()
|
||||
|
||||
for configuration in HybridSearchSubqueryConfiguration:
|
||||
_patch_hybrid_search_subquery_configuration(monkeypatch, configuration)
|
||||
for pipeline in HybridSearchNormalizationPipeline:
|
||||
_patch_hybrid_search_normalization_pipeline(monkeypatch, pipeline)
|
||||
pipeline_name, pipeline_config = (
|
||||
get_normalization_pipeline_name_and_config()
|
||||
)
|
||||
test_client.create_search_pipeline(
|
||||
pipeline_id=pipeline_name,
|
||||
pipeline_body=pipeline_config,
|
||||
)
|
||||
# Search query.
|
||||
query_text = "Python programming"
|
||||
query_vector = _generate_test_vector(0.12)
|
||||
search_body = DocumentQuery.get_hybrid_search_query(
|
||||
query_text=query_text,
|
||||
query_vector=query_vector,
|
||||
num_hits=5,
|
||||
tenant_state=tenant_state,
|
||||
# We're not worried about filtering here. tenant_id in this object
|
||||
# is not relevant.
|
||||
index_filters=IndexFilters(access_control_list=None, tenant_id=None),
|
||||
include_hidden=False,
|
||||
)
|
||||
|
||||
# Search query.
|
||||
query_text = "Python programming"
|
||||
query_vector = _generate_test_vector(0.12)
|
||||
search_body = DocumentQuery.get_hybrid_search_query(
|
||||
query_text=query_text,
|
||||
query_vector=query_vector,
|
||||
num_hits=5,
|
||||
tenant_state=tenant_state,
|
||||
# We're not worried about filtering here. tenant_id in this object
|
||||
# is not relevant.
|
||||
index_filters=IndexFilters(
|
||||
access_control_list=None, tenant_id=None
|
||||
),
|
||||
include_hidden=False,
|
||||
)
|
||||
# Under test.
|
||||
results = test_client.search(
|
||||
body=search_body, search_pipeline_id=MIN_MAX_NORMALIZATION_PIPELINE_NAME
|
||||
)
|
||||
|
||||
# Under test.
|
||||
results = test_client.search(
|
||||
body=search_body, search_pipeline_id=pipeline_name
|
||||
)
|
||||
|
||||
# Postcondition.
|
||||
assert len(results) == len(docs)
|
||||
# Assert that all the chunks above are present.
|
||||
assert all(
|
||||
chunk.document_chunk.document_id in docs.keys() for chunk in results
|
||||
)
|
||||
# Make sure the chunk contents are preserved.
|
||||
for i, chunk in enumerate(results):
|
||||
assert (
|
||||
chunk.document_chunk == docs[chunk.document_chunk.document_id]
|
||||
)
|
||||
# Make sure score reporting seems reasonable (it should not be None
|
||||
# or 0).
|
||||
assert chunk.score
|
||||
# Make sure there is some kind of match highlight only for the first
|
||||
# result. The other results are so bad they're not expected to have
|
||||
# match highlights.
|
||||
if i == 0:
|
||||
assert chunk.match_highlights.get(CONTENT_FIELD_NAME, [])
|
||||
# Postcondition.
|
||||
assert len(results) == len(docs)
|
||||
# Assert that all the chunks above are present.
|
||||
assert all(chunk.document_chunk.document_id in docs.keys() for chunk in results)
|
||||
# Make sure the chunk contents are preserved.
|
||||
for i, chunk in enumerate(results):
|
||||
assert chunk.document_chunk == docs[chunk.document_chunk.document_id]
|
||||
# Make sure score reporting seems reasonable (it should not be None
|
||||
# or 0).
|
||||
assert chunk.score
|
||||
# Make sure there is some kind of match highlight only for the first
|
||||
# result. The other results are so bad they're not expected to have
|
||||
# match highlights.
|
||||
if i == 0:
|
||||
assert chunk.match_highlights.get(CONTENT_FIELD_NAME, [])
|
||||
|
||||
def test_search_empty_index(
|
||||
self,
|
||||
@@ -923,10 +845,11 @@ class TestOpenSearchClient:
|
||||
index_filters=IndexFilters(access_control_list=None, tenant_id=None),
|
||||
include_hidden=False,
|
||||
)
|
||||
pipeline_name, _ = get_normalization_pipeline_name_and_config()
|
||||
|
||||
# Under test.
|
||||
results = test_client.search(body=search_body, search_pipeline_id=pipeline_name)
|
||||
results = test_client.search(
|
||||
body=search_body, search_pipeline_id=MIN_MAX_NORMALIZATION_PIPELINE_NAME
|
||||
)
|
||||
|
||||
# Postcondition.
|
||||
assert len(results) == 0
|
||||
@@ -1025,10 +948,11 @@ class TestOpenSearchClient:
|
||||
),
|
||||
include_hidden=False,
|
||||
)
|
||||
pipeline_name, _ = get_normalization_pipeline_name_and_config()
|
||||
|
||||
# Under test.
|
||||
results = test_client.search(body=search_body, search_pipeline_id=pipeline_name)
|
||||
results = test_client.search(
|
||||
body=search_body, search_pipeline_id=MIN_MAX_NORMALIZATION_PIPELINE_NAME
|
||||
)
|
||||
|
||||
# Postcondition.
|
||||
# Should only get the public, non-hidden document, and the private
|
||||
@@ -1143,10 +1067,11 @@ class TestOpenSearchClient:
|
||||
index_filters=IndexFilters(access_control_list=[], tenant_id=None),
|
||||
include_hidden=False,
|
||||
)
|
||||
pipeline_name, _ = get_normalization_pipeline_name_and_config()
|
||||
|
||||
# Under test.
|
||||
results = test_client.search(body=search_body, search_pipeline_id=pipeline_name)
|
||||
results = test_client.search(
|
||||
body=search_body, search_pipeline_id=MIN_MAX_NORMALIZATION_PIPELINE_NAME
|
||||
)
|
||||
|
||||
# Postcondition.
|
||||
# Should only get public, non-hidden documents (3 out of 5).
|
||||
@@ -1516,16 +1441,15 @@ class TestOpenSearchClient:
|
||||
),
|
||||
include_hidden=False,
|
||||
)
|
||||
pipeline_name, _ = get_normalization_pipeline_name_and_config()
|
||||
|
||||
# Under test.
|
||||
last_week_results = test_client.search(
|
||||
body=last_week_search_body,
|
||||
search_pipeline_id=pipeline_name,
|
||||
search_pipeline_id=MIN_MAX_NORMALIZATION_PIPELINE_NAME,
|
||||
)
|
||||
last_six_months_results = test_client.search(
|
||||
body=last_six_months_search_body,
|
||||
search_pipeline_id=pipeline_name,
|
||||
search_pipeline_id=MIN_MAX_NORMALIZATION_PIPELINE_NAME,
|
||||
)
|
||||
|
||||
# Postcondition.
|
||||
|
||||
@@ -1,120 +0,0 @@
|
||||
import pytest
|
||||
|
||||
from onyx.auth.users import _is_same_origin
|
||||
|
||||
|
||||
class TestExactMatch:
|
||||
"""Origins that are textually identical should always match."""
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"origin",
|
||||
[
|
||||
"http://localhost:3000",
|
||||
"https://app.example.com",
|
||||
"https://app.example.com:8443",
|
||||
"http://127.0.0.1:8080",
|
||||
],
|
||||
)
|
||||
def test_identical_origins(self, origin: str) -> None:
|
||||
assert _is_same_origin(origin, origin)
|
||||
|
||||
|
||||
class TestLoopbackPortRelaxation:
|
||||
"""On loopback addresses, port differences should be ignored."""
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"actual,expected",
|
||||
[
|
||||
("http://localhost:3001", "http://localhost:3000"),
|
||||
("http://localhost:8080", "http://localhost:3000"),
|
||||
("http://localhost", "http://localhost:3000"),
|
||||
("http://127.0.0.1:3001", "http://127.0.0.1:3000"),
|
||||
("http://[::1]:3001", "http://[::1]:3000"),
|
||||
],
|
||||
)
|
||||
def test_loopback_different_ports_accepted(
|
||||
self, actual: str, expected: str
|
||||
) -> None:
|
||||
assert _is_same_origin(actual, expected)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"actual,expected",
|
||||
[
|
||||
("https://localhost:3001", "http://localhost:3000"),
|
||||
("http://localhost:3001", "https://localhost:3000"),
|
||||
],
|
||||
)
|
||||
def test_loopback_different_scheme_rejected(
|
||||
self, actual: str, expected: str
|
||||
) -> None:
|
||||
assert not _is_same_origin(actual, expected)
|
||||
|
||||
def test_loopback_hostname_mismatch_rejected(self) -> None:
|
||||
assert not _is_same_origin("http://localhost:3001", "http://127.0.0.1:3000")
|
||||
|
||||
|
||||
class TestNonLoopbackStrictPort:
|
||||
"""Non-loopback origins must match scheme, hostname, AND port."""
|
||||
|
||||
def test_different_port_rejected(self) -> None:
|
||||
assert not _is_same_origin(
|
||||
"https://app.example.com:8443", "https://app.example.com"
|
||||
)
|
||||
|
||||
def test_different_hostname_rejected(self) -> None:
|
||||
assert not _is_same_origin("https://evil.com", "https://app.example.com")
|
||||
|
||||
def test_different_scheme_rejected(self) -> None:
|
||||
assert not _is_same_origin("http://app.example.com", "https://app.example.com")
|
||||
|
||||
def test_same_port_explicit(self) -> None:
|
||||
assert _is_same_origin(
|
||||
"https://app.example.com:443", "https://app.example.com:443"
|
||||
)
|
||||
|
||||
|
||||
class TestDefaultPortNormalization:
|
||||
"""Port should be normalized so that omitted default port == explicit default port."""
|
||||
|
||||
def test_http_implicit_vs_explicit_80(self) -> None:
|
||||
assert _is_same_origin("http://example.com", "http://example.com:80")
|
||||
|
||||
def test_http_explicit_80_vs_implicit(self) -> None:
|
||||
assert _is_same_origin("http://example.com:80", "http://example.com")
|
||||
|
||||
def test_https_implicit_vs_explicit_443(self) -> None:
|
||||
assert _is_same_origin("https://example.com", "https://example.com:443")
|
||||
|
||||
def test_https_explicit_443_vs_implicit(self) -> None:
|
||||
assert _is_same_origin("https://example.com:443", "https://example.com")
|
||||
|
||||
def test_http_non_default_port_vs_implicit_rejected(self) -> None:
|
||||
assert not _is_same_origin("http://example.com:8080", "http://example.com")
|
||||
|
||||
|
||||
class TestTrailingSlash:
|
||||
"""Trailing slashes should not affect comparison."""
|
||||
|
||||
def test_trailing_slash_on_actual(self) -> None:
|
||||
assert _is_same_origin("https://app.example.com/", "https://app.example.com")
|
||||
|
||||
def test_trailing_slash_on_expected(self) -> None:
|
||||
assert _is_same_origin("https://app.example.com", "https://app.example.com/")
|
||||
|
||||
def test_trailing_slash_on_both(self) -> None:
|
||||
assert _is_same_origin("https://app.example.com/", "https://app.example.com/")
|
||||
|
||||
|
||||
class TestCSWSHScenarios:
|
||||
"""Realistic attack scenarios that must be rejected."""
|
||||
|
||||
def test_remote_attacker_rejected(self) -> None:
|
||||
assert not _is_same_origin("https://evil.com", "http://localhost:3000")
|
||||
|
||||
def test_remote_attacker_same_port_rejected(self) -> None:
|
||||
assert not _is_same_origin("http://evil.com:3000", "http://localhost:3000")
|
||||
|
||||
def test_remote_attacker_matching_hostname_different_port(self) -> None:
|
||||
assert not _is_same_origin(
|
||||
"https://app.example.com:9999", "https://app.example.com"
|
||||
)
|
||||
@@ -1,194 +0,0 @@
|
||||
from unittest.mock import MagicMock
|
||||
from unittest.mock import patch
|
||||
|
||||
from onyx.background.celery.tasks.hierarchyfetching.tasks import (
|
||||
_connector_supports_hierarchy_fetching,
|
||||
)
|
||||
from onyx.background.celery.tasks.hierarchyfetching.tasks import (
|
||||
check_for_hierarchy_fetching,
|
||||
)
|
||||
from onyx.connectors.factory import ConnectorMissingException
|
||||
from onyx.connectors.interfaces import BaseConnector
|
||||
from onyx.connectors.interfaces import HierarchyConnector
|
||||
from onyx.connectors.interfaces import HierarchyOutput
|
||||
from onyx.connectors.interfaces import SecondsSinceUnixEpoch
|
||||
|
||||
TASKS_MODULE = "onyx.background.celery.tasks.hierarchyfetching.tasks"
|
||||
|
||||
|
||||
class _NonHierarchyConnector(BaseConnector):
|
||||
def load_credentials(self, credentials: dict) -> dict | None: # noqa: ARG002
|
||||
return None
|
||||
|
||||
|
||||
class _HierarchyCapableConnector(HierarchyConnector):
|
||||
def load_credentials(self, credentials: dict) -> dict | None: # noqa: ARG002
|
||||
return None
|
||||
|
||||
def load_hierarchy(
|
||||
self,
|
||||
start: SecondsSinceUnixEpoch, # noqa: ARG002
|
||||
end: SecondsSinceUnixEpoch, # noqa: ARG002
|
||||
) -> HierarchyOutput:
|
||||
return
|
||||
yield
|
||||
|
||||
|
||||
def _build_cc_pair_mock() -> MagicMock:
|
||||
cc_pair = MagicMock()
|
||||
cc_pair.connector.source = "mock-source"
|
||||
cc_pair.connector.input_type = "mock-input-type"
|
||||
return cc_pair
|
||||
|
||||
|
||||
def _build_redis_mock_with_lock() -> tuple[MagicMock, MagicMock]:
|
||||
redis_client = MagicMock()
|
||||
lock = MagicMock()
|
||||
lock.acquire.return_value = True
|
||||
lock.owned.return_value = True
|
||||
redis_client.lock.return_value = lock
|
||||
return redis_client, lock
|
||||
|
||||
|
||||
@patch(f"{TASKS_MODULE}.identify_connector_class")
|
||||
def test_connector_supports_hierarchy_fetching_false_for_non_hierarchy_connector(
|
||||
mock_identify_connector_class: MagicMock,
|
||||
) -> None:
|
||||
mock_identify_connector_class.return_value = _NonHierarchyConnector
|
||||
|
||||
assert _connector_supports_hierarchy_fetching(_build_cc_pair_mock()) is False
|
||||
mock_identify_connector_class.assert_called_once_with("mock-source")
|
||||
|
||||
|
||||
@patch(f"{TASKS_MODULE}.task_logger.warning")
|
||||
@patch(f"{TASKS_MODULE}.identify_connector_class")
|
||||
def test_connector_supports_hierarchy_fetching_false_when_class_missing(
|
||||
mock_identify_connector_class: MagicMock,
|
||||
mock_warning: MagicMock,
|
||||
) -> None:
|
||||
mock_identify_connector_class.side_effect = ConnectorMissingException("missing")
|
||||
|
||||
assert _connector_supports_hierarchy_fetching(_build_cc_pair_mock()) is False
|
||||
mock_warning.assert_called_once()
|
||||
|
||||
|
||||
@patch(f"{TASKS_MODULE}.identify_connector_class")
|
||||
def test_connector_supports_hierarchy_fetching_true_for_supported_connector(
|
||||
mock_identify_connector_class: MagicMock,
|
||||
) -> None:
|
||||
mock_identify_connector_class.return_value = _HierarchyCapableConnector
|
||||
|
||||
assert _connector_supports_hierarchy_fetching(_build_cc_pair_mock()) is True
|
||||
mock_identify_connector_class.assert_called_once_with("mock-source")
|
||||
|
||||
|
||||
@patch(f"{TASKS_MODULE}._try_creating_hierarchy_fetching_task")
|
||||
@patch(f"{TASKS_MODULE}._is_hierarchy_fetching_due")
|
||||
@patch(f"{TASKS_MODULE}.get_connector_credential_pair_from_id")
|
||||
@patch(f"{TASKS_MODULE}.fetch_indexable_standard_connector_credential_pair_ids")
|
||||
@patch(f"{TASKS_MODULE}.get_session_with_current_tenant")
|
||||
@patch(f"{TASKS_MODULE}.get_redis_client")
|
||||
@patch(f"{TASKS_MODULE}._connector_supports_hierarchy_fetching")
|
||||
def test_check_for_hierarchy_fetching_skips_unsupported_connectors(
|
||||
mock_supports_hierarchy_fetching: MagicMock,
|
||||
mock_get_redis_client: MagicMock,
|
||||
mock_get_session: MagicMock,
|
||||
mock_fetch_cc_pair_ids: MagicMock,
|
||||
mock_get_cc_pair: MagicMock,
|
||||
mock_is_due: MagicMock,
|
||||
mock_try_create_task: MagicMock,
|
||||
) -> None:
|
||||
redis_client, lock = _build_redis_mock_with_lock()
|
||||
mock_get_redis_client.return_value = redis_client
|
||||
mock_get_session.return_value.__enter__.return_value = MagicMock()
|
||||
mock_fetch_cc_pair_ids.return_value = [123]
|
||||
mock_get_cc_pair.return_value = _build_cc_pair_mock()
|
||||
mock_supports_hierarchy_fetching.return_value = False
|
||||
mock_is_due.return_value = True
|
||||
|
||||
task_app = MagicMock()
|
||||
with patch.object(check_for_hierarchy_fetching, "app", task_app):
|
||||
result = check_for_hierarchy_fetching.run(tenant_id="test-tenant")
|
||||
|
||||
assert result == 0
|
||||
mock_is_due.assert_not_called()
|
||||
mock_try_create_task.assert_not_called()
|
||||
lock.release.assert_called_once()
|
||||
|
||||
|
||||
@patch(f"{TASKS_MODULE}._try_creating_hierarchy_fetching_task")
|
||||
@patch(f"{TASKS_MODULE}._is_hierarchy_fetching_due")
|
||||
@patch(f"{TASKS_MODULE}.get_connector_credential_pair_from_id")
|
||||
@patch(f"{TASKS_MODULE}.fetch_indexable_standard_connector_credential_pair_ids")
|
||||
@patch(f"{TASKS_MODULE}.get_session_with_current_tenant")
|
||||
@patch(f"{TASKS_MODULE}.get_redis_client")
|
||||
@patch(f"{TASKS_MODULE}._connector_supports_hierarchy_fetching")
|
||||
def test_check_for_hierarchy_fetching_creates_task_for_supported_due_connector(
|
||||
mock_supports_hierarchy_fetching: MagicMock,
|
||||
mock_get_redis_client: MagicMock,
|
||||
mock_get_session: MagicMock,
|
||||
mock_fetch_cc_pair_ids: MagicMock,
|
||||
mock_get_cc_pair: MagicMock,
|
||||
mock_is_due: MagicMock,
|
||||
mock_try_create_task: MagicMock,
|
||||
) -> None:
|
||||
redis_client, lock = _build_redis_mock_with_lock()
|
||||
cc_pair = _build_cc_pair_mock()
|
||||
db_session = MagicMock()
|
||||
mock_get_redis_client.return_value = redis_client
|
||||
mock_get_session.return_value.__enter__.return_value = db_session
|
||||
mock_fetch_cc_pair_ids.return_value = [123]
|
||||
mock_get_cc_pair.return_value = cc_pair
|
||||
mock_supports_hierarchy_fetching.return_value = True
|
||||
mock_is_due.return_value = True
|
||||
mock_try_create_task.return_value = "task-id"
|
||||
|
||||
task_app = MagicMock()
|
||||
with patch.object(check_for_hierarchy_fetching, "app", task_app):
|
||||
result = check_for_hierarchy_fetching.run(tenant_id="test-tenant")
|
||||
|
||||
assert result == 1
|
||||
mock_is_due.assert_called_once_with(cc_pair)
|
||||
mock_try_create_task.assert_called_once_with(
|
||||
celery_app=task_app,
|
||||
cc_pair=cc_pair,
|
||||
db_session=db_session,
|
||||
r=redis_client,
|
||||
tenant_id="test-tenant",
|
||||
)
|
||||
lock.release.assert_called_once()
|
||||
|
||||
|
||||
@patch(f"{TASKS_MODULE}._try_creating_hierarchy_fetching_task")
|
||||
@patch(f"{TASKS_MODULE}._is_hierarchy_fetching_due")
|
||||
@patch(f"{TASKS_MODULE}.get_connector_credential_pair_from_id")
|
||||
@patch(f"{TASKS_MODULE}.fetch_indexable_standard_connector_credential_pair_ids")
|
||||
@patch(f"{TASKS_MODULE}.get_session_with_current_tenant")
|
||||
@patch(f"{TASKS_MODULE}.get_redis_client")
|
||||
@patch(f"{TASKS_MODULE}._connector_supports_hierarchy_fetching")
|
||||
def test_check_for_hierarchy_fetching_skips_supported_connector_when_not_due(
|
||||
mock_supports_hierarchy_fetching: MagicMock,
|
||||
mock_get_redis_client: MagicMock,
|
||||
mock_get_session: MagicMock,
|
||||
mock_fetch_cc_pair_ids: MagicMock,
|
||||
mock_get_cc_pair: MagicMock,
|
||||
mock_is_due: MagicMock,
|
||||
mock_try_create_task: MagicMock,
|
||||
) -> None:
|
||||
redis_client, lock = _build_redis_mock_with_lock()
|
||||
cc_pair = _build_cc_pair_mock()
|
||||
mock_get_redis_client.return_value = redis_client
|
||||
mock_get_session.return_value.__enter__.return_value = MagicMock()
|
||||
mock_fetch_cc_pair_ids.return_value = [123]
|
||||
mock_get_cc_pair.return_value = cc_pair
|
||||
mock_supports_hierarchy_fetching.return_value = True
|
||||
mock_is_due.return_value = False
|
||||
|
||||
task_app = MagicMock()
|
||||
with patch.object(check_for_hierarchy_fetching, "app", task_app):
|
||||
result = check_for_hierarchy_fetching.run(tenant_id="test-tenant")
|
||||
|
||||
assert result == 0
|
||||
mock_is_due.assert_called_once_with(cc_pair)
|
||||
mock_try_create_task.assert_not_called()
|
||||
lock.release.assert_called_once()
|
||||
@@ -1,13 +1,9 @@
|
||||
"""Tests for llm_loop.py, including history construction and empty-response paths."""
|
||||
|
||||
from unittest.mock import Mock
|
||||
"""Tests for llm_loop.py, specifically the construct_message_history function."""
|
||||
|
||||
import pytest
|
||||
|
||||
from onyx.chat.llm_loop import _build_empty_llm_response_error
|
||||
from onyx.chat.llm_loop import _try_fallback_tool_extraction
|
||||
from onyx.chat.llm_loop import construct_message_history
|
||||
from onyx.chat.llm_loop import EmptyLLMResponseError
|
||||
from onyx.chat.models import ChatLoadedFile
|
||||
from onyx.chat.models import ChatMessageSimple
|
||||
from onyx.chat.models import ContextFileMetadata
|
||||
@@ -17,7 +13,6 @@ from onyx.chat.models import LlmStepResult
|
||||
from onyx.chat.models import ToolCallSimple
|
||||
from onyx.configs.constants import MessageType
|
||||
from onyx.file_store.models import ChatFileType
|
||||
from onyx.llm.interfaces import LLMConfig
|
||||
from onyx.llm.interfaces import ToolChoiceOptions
|
||||
from onyx.server.query_and_chat.placement import Placement
|
||||
from onyx.tools.models import ToolCallKickoff
|
||||
@@ -1172,57 +1167,3 @@ class TestFallbackToolExtraction:
|
||||
|
||||
assert result is llm_step_result
|
||||
assert attempted is False
|
||||
|
||||
|
||||
class TestEmptyLlmResponseClassification:
|
||||
def _make_llm(self, provider: str = "openai", model: str = "gpt-5.2") -> Mock:
|
||||
llm = Mock()
|
||||
llm.config = LLMConfig(
|
||||
model_provider=provider,
|
||||
model_name=model,
|
||||
temperature=0.0,
|
||||
max_input_tokens=4096,
|
||||
)
|
||||
return llm
|
||||
|
||||
def test_openai_empty_stream_is_classified_as_budget_exceeded(
|
||||
self, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr("onyx.chat.llm_loop.is_true_openai_model", lambda *_: True)
|
||||
|
||||
err = _build_empty_llm_response_error(
|
||||
llm=self._make_llm(),
|
||||
llm_step_result=LlmStepResult(
|
||||
reasoning=None,
|
||||
answer=None,
|
||||
tool_calls=None,
|
||||
raw_answer=None,
|
||||
),
|
||||
tool_choice=ToolChoiceOptions.AUTO,
|
||||
)
|
||||
|
||||
assert isinstance(err, EmptyLLMResponseError)
|
||||
assert err.error_code == "BUDGET_EXCEEDED"
|
||||
assert err.is_retryable is False
|
||||
assert "quota" in err.client_error_msg.lower()
|
||||
|
||||
def test_reasoning_only_response_uses_generic_empty_response_error(
|
||||
self, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr("onyx.chat.llm_loop.is_true_openai_model", lambda *_: True)
|
||||
|
||||
err = _build_empty_llm_response_error(
|
||||
llm=self._make_llm(),
|
||||
llm_step_result=LlmStepResult(
|
||||
reasoning="scratchpad only",
|
||||
answer=None,
|
||||
tool_calls=None,
|
||||
raw_answer=None,
|
||||
),
|
||||
tool_choice=ToolChoiceOptions.AUTO,
|
||||
)
|
||||
|
||||
assert isinstance(err, EmptyLLMResponseError)
|
||||
assert err.error_code == "EMPTY_LLM_RESPONSE"
|
||||
assert err.is_retryable is True
|
||||
assert "quota" not in err.client_error_msg.lower()
|
||||
|
||||
@@ -1,34 +0,0 @@
|
||||
from onyx.chat.process_message import remove_answer_citations
|
||||
|
||||
|
||||
def test_remove_answer_citations_strips_http_markdown_citation() -> None:
|
||||
answer = "The answer is Paris [[1]](https://example.com/doc)."
|
||||
|
||||
assert remove_answer_citations(answer) == "The answer is Paris."
|
||||
|
||||
|
||||
def test_remove_answer_citations_strips_empty_markdown_citation() -> None:
|
||||
answer = "The answer is Paris [[1]]()."
|
||||
|
||||
assert remove_answer_citations(answer) == "The answer is Paris."
|
||||
|
||||
|
||||
def test_remove_answer_citations_strips_citation_with_parentheses_in_url() -> None:
|
||||
answer = (
|
||||
"The answer is Paris "
|
||||
"[[1]](https://en.wikipedia.org/wiki/Function_(mathematics))."
|
||||
)
|
||||
|
||||
assert remove_answer_citations(answer) == "The answer is Paris."
|
||||
|
||||
|
||||
def test_remove_answer_citations_preserves_non_citation_markdown_links() -> None:
|
||||
answer = (
|
||||
"See [reference](https://example.com/Function_(mathematics)) "
|
||||
"for context [[1]](https://en.wikipedia.org/wiki/Function_(mathematics))."
|
||||
)
|
||||
|
||||
assert (
|
||||
remove_answer_citations(answer)
|
||||
== "See [reference](https://example.com/Function_(mathematics)) for context."
|
||||
)
|
||||
@@ -3,10 +3,7 @@ from unittest.mock import Mock
|
||||
import pytest
|
||||
|
||||
from onyx.chat import process_message
|
||||
from onyx.chat.models import AnswerStream
|
||||
from onyx.chat.models import StreamingError
|
||||
from onyx.configs import app_configs
|
||||
from onyx.server.query_and_chat.models import MessageResponseIDInfo
|
||||
from onyx.server.query_and_chat.models import SendMessageRequest
|
||||
|
||||
|
||||
@@ -38,26 +35,3 @@ def test_mock_llm_response_requires_integration_mode() -> None:
|
||||
db_session=Mock(),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def test_gather_stream_returns_empty_answer_when_streaming_error_only() -> None:
|
||||
packets: AnswerStream = iter(
|
||||
[
|
||||
MessageResponseIDInfo(
|
||||
user_message_id=None,
|
||||
reserved_assistant_message_id=42,
|
||||
),
|
||||
StreamingError(
|
||||
error="OpenAI quota exceeded",
|
||||
error_code="BUDGET_EXCEEDED",
|
||||
is_retryable=False,
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
result = process_message.gather_stream(packets)
|
||||
|
||||
assert result.answer == ""
|
||||
assert result.answer_citationless == ""
|
||||
assert result.error_msg == "OpenAI quota exceeded"
|
||||
assert result.message_id == 42
|
||||
|
||||
@@ -1,40 +0,0 @@
|
||||
"""Unit tests for the hooks feature gate."""
|
||||
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
from onyx.error_handling.error_codes import OnyxErrorCode
|
||||
from onyx.error_handling.exceptions import OnyxError
|
||||
from onyx.hooks.api_dependencies import require_hook_enabled
|
||||
|
||||
|
||||
class TestRequireHookEnabled:
|
||||
def test_raises_when_multi_tenant(self) -> None:
|
||||
with (
|
||||
patch("onyx.hooks.api_dependencies.MULTI_TENANT", True),
|
||||
patch("onyx.hooks.api_dependencies.HOOK_ENABLED", True),
|
||||
):
|
||||
with pytest.raises(OnyxError) as exc_info:
|
||||
require_hook_enabled()
|
||||
assert exc_info.value.error_code is OnyxErrorCode.SINGLE_TENANT_ONLY
|
||||
assert exc_info.value.status_code == 403
|
||||
assert "multi-tenant" in exc_info.value.detail
|
||||
|
||||
def test_raises_when_flag_disabled(self) -> None:
|
||||
with (
|
||||
patch("onyx.hooks.api_dependencies.MULTI_TENANT", False),
|
||||
patch("onyx.hooks.api_dependencies.HOOK_ENABLED", False),
|
||||
):
|
||||
with pytest.raises(OnyxError) as exc_info:
|
||||
require_hook_enabled()
|
||||
assert exc_info.value.error_code is OnyxErrorCode.ENV_VAR_GATED
|
||||
assert exc_info.value.status_code == 403
|
||||
assert "HOOK_ENABLED" in exc_info.value.detail
|
||||
|
||||
def test_passes_when_enabled_single_tenant(self) -> None:
|
||||
with (
|
||||
patch("onyx.hooks.api_dependencies.MULTI_TENANT", False),
|
||||
patch("onyx.hooks.api_dependencies.HOOK_ENABLED", True),
|
||||
):
|
||||
require_hook_enabled() # must not raise
|
||||
@@ -3,7 +3,6 @@
|
||||
from onyx.server.manage.llm.utils import generate_bedrock_display_name
|
||||
from onyx.server.manage.llm.utils import generate_ollama_display_name
|
||||
from onyx.server.manage.llm.utils import infer_vision_support
|
||||
from onyx.server.manage.llm.utils import is_embedding_model
|
||||
from onyx.server.manage.llm.utils import is_reasoning_model
|
||||
from onyx.server.manage.llm.utils import is_valid_bedrock_model
|
||||
from onyx.server.manage.llm.utils import strip_openrouter_vendor_prefix
|
||||
@@ -210,35 +209,3 @@ class TestIsReasoningModel:
|
||||
is_reasoning_model("anthropic/claude-3-5-sonnet", "Claude 3.5 Sonnet")
|
||||
is False
|
||||
)
|
||||
|
||||
|
||||
class TestIsEmbeddingModel:
|
||||
"""Tests for embedding model detection."""
|
||||
|
||||
def test_openai_embedding_ada(self) -> None:
|
||||
assert is_embedding_model("text-embedding-ada-002") is True
|
||||
|
||||
def test_openai_embedding_3_small(self) -> None:
|
||||
assert is_embedding_model("text-embedding-3-small") is True
|
||||
|
||||
def test_openai_embedding_3_large(self) -> None:
|
||||
assert is_embedding_model("text-embedding-3-large") is True
|
||||
|
||||
def test_cohere_embed_model(self) -> None:
|
||||
assert is_embedding_model("embed-english-v3.0") is True
|
||||
|
||||
def test_bedrock_titan_embed(self) -> None:
|
||||
assert is_embedding_model("amazon.titan-embed-text-v1") is True
|
||||
|
||||
def test_gpt4o_not_embedding(self) -> None:
|
||||
assert is_embedding_model("gpt-4o") is False
|
||||
|
||||
def test_gpt4_not_embedding(self) -> None:
|
||||
assert is_embedding_model("gpt-4") is False
|
||||
|
||||
def test_dall_e_not_embedding(self) -> None:
|
||||
assert is_embedding_model("dall-e-3") is False
|
||||
|
||||
def test_unknown_custom_model_not_embedding(self) -> None:
|
||||
"""Custom/local models not in litellm's model DB should default to False."""
|
||||
assert is_embedding_model("my-custom-local-model-v1") is False
|
||||
|
||||
@@ -17,7 +17,7 @@ def test_mt_cloud_telemetry_noop_when_not_multi_tenant(monkeypatch: Any) -> None
|
||||
|
||||
telemetry_utils.mt_cloud_telemetry(
|
||||
tenant_id="tenant-1",
|
||||
distinct_id="12345678-1234-1234-1234-123456789abc",
|
||||
distinct_id="user@example.com",
|
||||
event=MilestoneRecordType.USER_MESSAGE_SENT,
|
||||
properties={"origin": "web"},
|
||||
)
|
||||
@@ -40,7 +40,7 @@ def test_mt_cloud_telemetry_calls_event_telemetry_when_multi_tenant(
|
||||
|
||||
telemetry_utils.mt_cloud_telemetry(
|
||||
tenant_id="tenant-1",
|
||||
distinct_id="12345678-1234-1234-1234-123456789abc",
|
||||
distinct_id="user@example.com",
|
||||
event=MilestoneRecordType.USER_MESSAGE_SENT,
|
||||
properties={"origin": "web"},
|
||||
)
|
||||
@@ -51,52 +51,7 @@ def test_mt_cloud_telemetry_calls_event_telemetry_when_multi_tenant(
|
||||
fallback=telemetry_utils.noop_fallback,
|
||||
)
|
||||
event_telemetry.assert_called_once_with(
|
||||
"12345678-1234-1234-1234-123456789abc",
|
||||
"user@example.com",
|
||||
MilestoneRecordType.USER_MESSAGE_SENT,
|
||||
{"origin": "web", "tenant_id": "tenant-1"},
|
||||
)
|
||||
|
||||
|
||||
def test_mt_cloud_identify_noop_when_not_multi_tenant(monkeypatch: Any) -> None:
|
||||
fetch_impl = Mock()
|
||||
monkeypatch.setattr(
|
||||
telemetry_utils,
|
||||
"fetch_versioned_implementation_with_fallback",
|
||||
fetch_impl,
|
||||
)
|
||||
monkeypatch.setattr("onyx.utils.telemetry.MULTI_TENANT", False)
|
||||
|
||||
telemetry_utils.mt_cloud_identify(
|
||||
distinct_id="12345678-1234-1234-1234-123456789abc",
|
||||
properties={"email": "user@example.com"},
|
||||
)
|
||||
|
||||
fetch_impl.assert_not_called()
|
||||
|
||||
|
||||
def test_mt_cloud_identify_calls_identify_user_when_multi_tenant(
|
||||
monkeypatch: Any,
|
||||
) -> None:
|
||||
identify_user = Mock()
|
||||
fetch_impl = Mock(return_value=identify_user)
|
||||
monkeypatch.setattr(
|
||||
telemetry_utils,
|
||||
"fetch_versioned_implementation_with_fallback",
|
||||
fetch_impl,
|
||||
)
|
||||
monkeypatch.setattr("onyx.utils.telemetry.MULTI_TENANT", True)
|
||||
|
||||
telemetry_utils.mt_cloud_identify(
|
||||
distinct_id="12345678-1234-1234-1234-123456789abc",
|
||||
properties={"email": "user@example.com"},
|
||||
)
|
||||
|
||||
fetch_impl.assert_called_once_with(
|
||||
module="onyx.utils.telemetry",
|
||||
attribute="identify_user",
|
||||
fallback=telemetry_utils.noop_fallback,
|
||||
)
|
||||
identify_user.assert_called_once_with(
|
||||
"12345678-1234-1234-1234-123456789abc",
|
||||
{"email": "user@example.com"},
|
||||
)
|
||||
|
||||
@@ -32,17 +32,15 @@ def test_run_with_timeout_raises_on_timeout(slow: float, timeout: float) -> None
|
||||
"""Test that a function that exceeds timeout raises TimeoutError"""
|
||||
|
||||
def slow_function() -> None:
|
||||
time.sleep(slow)
|
||||
time.sleep(slow) # Sleep for 2 seconds
|
||||
|
||||
start = time.monotonic()
|
||||
with pytest.raises(TimeoutError) as exc_info:
|
||||
run_with_timeout(timeout, slow_function)
|
||||
elapsed = time.monotonic() - start
|
||||
|
||||
start = time.time()
|
||||
run_with_timeout(timeout, slow_function) # Set timeout to 0.1 seconds
|
||||
end = time.time()
|
||||
assert end - start >= timeout
|
||||
assert end - start < (slow + timeout) / 2
|
||||
assert f"timed out after {timeout} seconds" in str(exc_info.value)
|
||||
assert elapsed >= timeout
|
||||
# Should return around the timeout duration, not the full sleep duration
|
||||
assert elapsed == pytest.approx(timeout, abs=0.8)
|
||||
|
||||
|
||||
@pytest.mark.filterwarnings("ignore::pytest.PytestUnhandledThreadExceptionWarning")
|
||||
|
||||
@@ -44,13 +44,13 @@ class TestBuildVespaFilters:
|
||||
assert result == f'({SOURCE_TYPE} contains "web") and '
|
||||
|
||||
def test_acl(self) -> None:
|
||||
"""Test with acls — uses weightedSet operator for efficient matching."""
|
||||
"""Test with acls."""
|
||||
# Single ACL
|
||||
filters = IndexFilters(access_control_list=["user1"])
|
||||
result = build_vespa_filters(filters)
|
||||
assert (
|
||||
result
|
||||
== f'!({HIDDEN}=true) and weightedSet(access_control_list, {{"user1":1}}) and '
|
||||
== f'!({HIDDEN}=true) and (access_control_list contains "user1") and '
|
||||
)
|
||||
|
||||
# Multiple ACL's
|
||||
@@ -58,7 +58,7 @@ class TestBuildVespaFilters:
|
||||
result = build_vespa_filters(filters)
|
||||
assert (
|
||||
result
|
||||
== f'!({HIDDEN}=true) and weightedSet(access_control_list, {{"user2":1, "group2":1}}) and '
|
||||
== f'!({HIDDEN}=true) and (access_control_list contains "user2" or access_control_list contains "group2") and '
|
||||
)
|
||||
|
||||
def test_tenant_filter(self) -> None:
|
||||
@@ -250,7 +250,7 @@ class TestBuildVespaFilters:
|
||||
result = build_vespa_filters(filters)
|
||||
|
||||
expected = f"!({HIDDEN}=true) and "
|
||||
expected += 'weightedSet(access_control_list, {"user1":1, "group1":1}) and '
|
||||
expected += '(access_control_list contains "user1" or access_control_list contains "group1") and '
|
||||
expected += f'({SOURCE_TYPE} contains "web") and '
|
||||
expected += f'({METADATA_LIST} contains "color{INDEX_SEPARATOR}red") and '
|
||||
# Knowledge scope filters are OR'd together
|
||||
@@ -290,38 +290,6 @@ class TestBuildVespaFilters:
|
||||
expected = f'!({HIDDEN}=true) and (({DOCUMENT_SETS} contains "engineering") or ({DOCUMENT_ID} contains "{str(id1)}")) and '
|
||||
assert expected == result
|
||||
|
||||
def test_acl_large_list_uses_weighted_set(self) -> None:
|
||||
"""Verify that large ACL lists produce a weightedSet clause
|
||||
instead of OR-chained contains — this is what prevents Vespa
|
||||
HTTP 400 errors for users with thousands of permission groups."""
|
||||
acl = [f"external_group:google_drive_{i}" for i in range(10_000)]
|
||||
acl += ["user_email:user@example.com", "__PUBLIC__"]
|
||||
filters = IndexFilters(access_control_list=acl)
|
||||
result = build_vespa_filters(filters)
|
||||
|
||||
assert "weightedSet(access_control_list, {" in result
|
||||
# Must NOT contain OR-chained contains clauses
|
||||
assert "access_control_list contains" not in result
|
||||
# All entries should be present
|
||||
assert '"external_group:google_drive_0":1' in result
|
||||
assert '"external_group:google_drive_9999":1' in result
|
||||
assert '"user_email:user@example.com":1' in result
|
||||
assert '"__PUBLIC__":1' in result
|
||||
|
||||
def test_acl_empty_strings_filtered(self) -> None:
|
||||
"""Empty strings in the ACL list should be filtered out."""
|
||||
filters = IndexFilters(access_control_list=["user1", "", "group1"])
|
||||
result = build_vespa_filters(filters)
|
||||
assert (
|
||||
result
|
||||
== f'!({HIDDEN}=true) and weightedSet(access_control_list, {{"user1":1, "group1":1}}) and '
|
||||
)
|
||||
|
||||
# All empty
|
||||
filters = IndexFilters(access_control_list=["", ""])
|
||||
result = build_vespa_filters(filters)
|
||||
assert result == f"!({HIDDEN}=true) and "
|
||||
|
||||
def test_empty_or_none_values(self) -> None:
|
||||
"""Test with empty or None values in filter lists."""
|
||||
# Empty strings in document set
|
||||
|
||||
@@ -12,11 +12,6 @@ services:
|
||||
api_server:
|
||||
ports:
|
||||
- "8080:8080"
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "${API_SERVER_CPU_LIMIT:-0}"
|
||||
memory: "${API_SERVER_MEM_LIMIT:-0}"
|
||||
|
||||
# Uncomment the block below to enable the MCP server for Onyx.
|
||||
# mcp_server:
|
||||
|
||||
@@ -15,9 +15,8 @@
|
||||
# -f docker-compose.dev.yml up -d --wait
|
||||
#
|
||||
# This overlay:
|
||||
# - Moves Vespa (index), both model servers, OpenSearch, MinIO,
|
||||
# Redis (cache), and the background worker to profiles so they do
|
||||
# not start by default
|
||||
# - Moves Vespa (index), both model servers, code-interpreter, Redis (cache),
|
||||
# and the background worker to profiles so they do not start by default
|
||||
# - Makes depends_on references to removed services optional
|
||||
# - Sets DISABLE_VECTOR_DB=true on the api_server
|
||||
# - Uses PostgreSQL for caching and auth instead of Redis
|
||||
@@ -28,8 +27,7 @@
|
||||
# --profile inference Inference model server
|
||||
# --profile background Background worker (Celery) — also needs redis
|
||||
# --profile redis Redis cache
|
||||
# --profile opensearch OpenSearch
|
||||
# --profile s3-filestore MinIO (S3-compatible file store)
|
||||
# --profile code-interpreter Code interpreter
|
||||
# =============================================================================
|
||||
|
||||
name: onyx
|
||||
@@ -40,9 +38,6 @@ services:
|
||||
index:
|
||||
condition: service_started
|
||||
required: false
|
||||
opensearch:
|
||||
condition: service_started
|
||||
required: false
|
||||
cache:
|
||||
condition: service_started
|
||||
required: false
|
||||
@@ -89,10 +84,4 @@ services:
|
||||
inference_model_server:
|
||||
profiles: ["inference"]
|
||||
|
||||
# OpenSearch is not needed in lite mode (no indexing).
|
||||
opensearch:
|
||||
profiles: ["opensearch"]
|
||||
|
||||
# MinIO is not needed in lite mode (Postgres handles file storage).
|
||||
minio:
|
||||
profiles: ["s3-filestore"]
|
||||
code-interpreter: {}
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -euo pipefail
|
||||
set -e
|
||||
|
||||
# Expected resource requirements (overridden below if --lite)
|
||||
# Expected resource requirements
|
||||
EXPECTED_DOCKER_RAM_GB=10
|
||||
EXPECTED_DISK_GB=32
|
||||
|
||||
@@ -10,11 +10,6 @@ EXPECTED_DISK_GB=32
|
||||
SHUTDOWN_MODE=false
|
||||
DELETE_DATA_MODE=false
|
||||
INCLUDE_CRAFT=false # Disabled by default, use --include-craft to enable
|
||||
LITE_MODE=false # Disabled by default, use --lite to enable
|
||||
USE_LOCAL_FILES=false # Disabled by default, use --local to skip downloading config files
|
||||
NO_PROMPT=false
|
||||
DRY_RUN=false
|
||||
VERBOSE=false
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
@@ -30,26 +25,6 @@ while [[ $# -gt 0 ]]; do
|
||||
INCLUDE_CRAFT=true
|
||||
shift
|
||||
;;
|
||||
--lite)
|
||||
LITE_MODE=true
|
||||
shift
|
||||
;;
|
||||
--local)
|
||||
USE_LOCAL_FILES=true
|
||||
shift
|
||||
;;
|
||||
--no-prompt)
|
||||
NO_PROMPT=true
|
||||
shift
|
||||
;;
|
||||
--dry-run)
|
||||
DRY_RUN=true
|
||||
shift
|
||||
;;
|
||||
--verbose)
|
||||
VERBOSE=true
|
||||
shift
|
||||
;;
|
||||
--help|-h)
|
||||
echo "Onyx Installation Script"
|
||||
echo ""
|
||||
@@ -57,23 +32,15 @@ while [[ $# -gt 0 ]]; do
|
||||
echo ""
|
||||
echo "Options:"
|
||||
echo " --include-craft Enable Onyx Craft (AI-powered web app building)"
|
||||
echo " --lite Deploy Onyx Lite (no Vespa, Redis, or model servers)"
|
||||
echo " --local Use existing config files instead of downloading from GitHub"
|
||||
echo " --shutdown Stop (pause) Onyx containers"
|
||||
echo " --delete-data Remove all Onyx data (containers, volumes, and files)"
|
||||
echo " --no-prompt Run non-interactively with defaults (for CI/automation)"
|
||||
echo " --dry-run Show what would be done without making changes"
|
||||
echo " --verbose Show detailed output for debugging"
|
||||
echo " --help, -h Show this help message"
|
||||
echo ""
|
||||
echo "Examples:"
|
||||
echo " $0 # Install Onyx"
|
||||
echo " $0 --lite # Install Onyx Lite (minimal deployment)"
|
||||
echo " $0 --include-craft # Install Onyx with Craft enabled"
|
||||
echo " $0 --shutdown # Pause Onyx services"
|
||||
echo " $0 --delete-data # Completely remove Onyx and all data"
|
||||
echo " $0 --local # Re-run using existing config files on disk"
|
||||
echo " $0 --no-prompt # Non-interactive install with defaults"
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
@@ -84,129 +51,8 @@ while [[ $# -gt 0 ]]; do
|
||||
esac
|
||||
done
|
||||
|
||||
if [[ "$VERBOSE" = true ]]; then
|
||||
set -x
|
||||
fi
|
||||
|
||||
if [[ "$LITE_MODE" = true ]] && [[ "$INCLUDE_CRAFT" = true ]]; then
|
||||
echo "ERROR: --lite and --include-craft cannot be used together."
|
||||
echo "Craft requires services (Vespa, Redis, background workers) that lite mode disables."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# When --lite is passed as a flag, lower resource thresholds early (before the
|
||||
# resource check). When lite is chosen interactively, the thresholds are adjusted
|
||||
# inside the new-deployment flow, after the resource check has already passed
|
||||
# with the standard thresholds — which is the safer direction.
|
||||
if [[ "$LITE_MODE" = true ]]; then
|
||||
EXPECTED_DOCKER_RAM_GB=4
|
||||
EXPECTED_DISK_GB=16
|
||||
fi
|
||||
|
||||
INSTALL_ROOT="${INSTALL_PREFIX:-onyx_data}"
|
||||
|
||||
LITE_COMPOSE_FILE="docker-compose.onyx-lite.yml"
|
||||
|
||||
# Build the -f flags for docker compose.
|
||||
# Pass "true" as $1 to auto-detect a previously-downloaded lite overlay
|
||||
# (used by shutdown/delete-data so users don't need to remember --lite).
|
||||
# Without the argument, the lite overlay is only included when --lite was
|
||||
# explicitly passed — preventing install/start from silently staying in
|
||||
# lite mode just because the file exists on disk from a prior run.
|
||||
compose_file_args() {
|
||||
local auto_detect="${1:-false}"
|
||||
local args="-f docker-compose.yml"
|
||||
if [[ "$LITE_MODE" = true ]] || { [[ "$auto_detect" = true ]] && [[ -f "${INSTALL_ROOT}/deployment/${LITE_COMPOSE_FILE}" ]]; }; then
|
||||
args="$args -f ${LITE_COMPOSE_FILE}"
|
||||
fi
|
||||
echo "$args"
|
||||
}
|
||||
|
||||
# --- Downloader detection (curl with wget fallback) ---
|
||||
DOWNLOADER=""
|
||||
detect_downloader() {
|
||||
if command -v curl &> /dev/null; then
|
||||
DOWNLOADER="curl"
|
||||
return 0
|
||||
fi
|
||||
if command -v wget &> /dev/null; then
|
||||
DOWNLOADER="wget"
|
||||
return 0
|
||||
fi
|
||||
echo "ERROR: Neither curl nor wget found. Please install one and retry."
|
||||
exit 1
|
||||
}
|
||||
detect_downloader
|
||||
|
||||
download_file() {
|
||||
local url="$1"
|
||||
local output="$2"
|
||||
if [[ "$DOWNLOADER" == "curl" ]]; then
|
||||
curl -fsSL --retry 3 --retry-delay 2 --retry-connrefused -o "$output" "$url"
|
||||
else
|
||||
wget -q --tries=3 --timeout=20 -O "$output" "$url"
|
||||
fi
|
||||
}
|
||||
|
||||
# Ensures a required file is present. With --local, verifies the file exists on
|
||||
# disk. Otherwise, downloads it from the given URL. Returns 0 on success, 1 on
|
||||
# failure (caller should handle the exit).
|
||||
ensure_file() {
|
||||
local path="$1"
|
||||
local url="$2"
|
||||
local desc="$3"
|
||||
|
||||
if [[ "$USE_LOCAL_FILES" = true ]]; then
|
||||
if [[ -f "$path" ]]; then
|
||||
print_success "Using existing ${desc}"
|
||||
return 0
|
||||
fi
|
||||
print_error "Required file missing: ${desc} (${path})"
|
||||
return 1
|
||||
fi
|
||||
|
||||
print_info "Downloading ${desc}..."
|
||||
if download_file "$url" "$path" 2>/dev/null; then
|
||||
print_success "${desc} downloaded"
|
||||
return 0
|
||||
fi
|
||||
print_error "Failed to download ${desc}"
|
||||
print_info "Please ensure you have internet connection and try again"
|
||||
return 1
|
||||
}
|
||||
|
||||
# --- Interactive prompt helpers ---
|
||||
is_interactive() {
|
||||
[[ "$NO_PROMPT" = false ]] && [[ -t 0 ]]
|
||||
}
|
||||
|
||||
prompt_or_default() {
|
||||
local prompt_text="$1"
|
||||
local default_value="$2"
|
||||
if is_interactive; then
|
||||
read -p "$prompt_text" -r REPLY
|
||||
if [[ -z "$REPLY" ]]; then
|
||||
REPLY="$default_value"
|
||||
fi
|
||||
else
|
||||
REPLY="$default_value"
|
||||
fi
|
||||
}
|
||||
|
||||
prompt_yn_or_default() {
|
||||
local prompt_text="$1"
|
||||
local default_value="$2"
|
||||
if is_interactive; then
|
||||
read -p "$prompt_text" -n 1 -r
|
||||
echo ""
|
||||
if [[ -z "$REPLY" ]]; then
|
||||
REPLY="$default_value"
|
||||
fi
|
||||
else
|
||||
REPLY="$default_value"
|
||||
fi
|
||||
}
|
||||
|
||||
# Colors for output
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
@@ -265,7 +111,7 @@ if [ "$SHUTDOWN_MODE" = true ]; then
|
||||
fi
|
||||
|
||||
# Stop containers (without removing them)
|
||||
(cd "${INSTALL_ROOT}/deployment" && $COMPOSE_CMD $(compose_file_args true) stop)
|
||||
(cd "${INSTALL_ROOT}/deployment" && $COMPOSE_CMD -f docker-compose.yml stop)
|
||||
if [ $? -eq 0 ]; then
|
||||
print_success "Onyx containers stopped (paused)"
|
||||
else
|
||||
@@ -294,17 +140,12 @@ if [ "$DELETE_DATA_MODE" = true ]; then
|
||||
echo " • All downloaded files and configurations"
|
||||
echo " • All user data and documents"
|
||||
echo ""
|
||||
if is_interactive; then
|
||||
read -p "Are you sure you want to continue? Type 'DELETE' to confirm: " -r
|
||||
echo ""
|
||||
if [ "$REPLY" != "DELETE" ]; then
|
||||
print_info "Operation cancelled."
|
||||
exit 0
|
||||
fi
|
||||
else
|
||||
print_error "Cannot confirm destructive operation in non-interactive mode."
|
||||
print_info "Run interactively or remove the ${INSTALL_ROOT} directory manually."
|
||||
exit 1
|
||||
read -p "Are you sure you want to continue? Type 'DELETE' to confirm: " -r
|
||||
echo ""
|
||||
|
||||
if [ "$REPLY" != "DELETE" ]; then
|
||||
print_info "Operation cancelled."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
print_info "Removing Onyx containers and volumes..."
|
||||
@@ -323,7 +164,7 @@ if [ "$DELETE_DATA_MODE" = true ]; then
|
||||
fi
|
||||
|
||||
# Stop and remove containers with volumes
|
||||
(cd "${INSTALL_ROOT}/deployment" && $COMPOSE_CMD $(compose_file_args true) down -v)
|
||||
(cd "${INSTALL_ROOT}/deployment" && $COMPOSE_CMD -f docker-compose.yml down -v)
|
||||
if [ $? -eq 0 ]; then
|
||||
print_success "Onyx containers and volumes removed"
|
||||
else
|
||||
@@ -345,117 +186,6 @@ if [ "$DELETE_DATA_MODE" = true ]; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# --- Auto-install Docker (Linux only) ---
|
||||
# Runs before the banner so a group-based re-exec doesn't repeat it.
|
||||
install_docker_linux() {
|
||||
local distro_id=""
|
||||
if [[ -f /etc/os-release ]]; then
|
||||
distro_id="$(. /etc/os-release && echo "${ID:-}")"
|
||||
fi
|
||||
|
||||
case "$distro_id" in
|
||||
amzn)
|
||||
print_info "Detected Amazon Linux — installing Docker via package manager..."
|
||||
if command -v dnf &> /dev/null; then
|
||||
sudo dnf install -y docker
|
||||
else
|
||||
sudo yum install -y docker
|
||||
fi
|
||||
;;
|
||||
*)
|
||||
print_info "Installing Docker via get.docker.com..."
|
||||
download_file "https://get.docker.com" /tmp/get-docker.sh
|
||||
sudo sh /tmp/get-docker.sh
|
||||
rm -f /tmp/get-docker.sh
|
||||
;;
|
||||
esac
|
||||
|
||||
sudo systemctl start docker 2>/dev/null || sudo service docker start 2>/dev/null || true
|
||||
sudo systemctl enable docker 2>/dev/null || true
|
||||
}
|
||||
|
||||
# Detect OS (including WSL)
|
||||
IS_WSL=false
|
||||
if [[ -n "${WSL_DISTRO_NAME:-}" ]] || grep -qi microsoft /proc/version 2>/dev/null; then
|
||||
IS_WSL=true
|
||||
fi
|
||||
|
||||
# Dry-run: show plan and exit
|
||||
if [[ "$DRY_RUN" = true ]]; then
|
||||
print_info "Dry run mode — showing what would happen:"
|
||||
echo " • Install root: ${INSTALL_ROOT}"
|
||||
echo " • Lite mode: ${LITE_MODE}"
|
||||
echo " • Include Craft: ${INCLUDE_CRAFT}"
|
||||
echo " • OS type: ${OSTYPE:-unknown} (WSL: ${IS_WSL})"
|
||||
echo " • Downloader: ${DOWNLOADER}"
|
||||
echo ""
|
||||
print_success "Dry run complete (no changes made)"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if ! command -v docker &> /dev/null; then
|
||||
if [[ "$OSTYPE" == "linux-gnu"* ]] || [[ -n "${WSL_DISTRO_NAME:-}" ]]; then
|
||||
install_docker_linux
|
||||
if ! command -v docker &> /dev/null; then
|
||||
print_error "Docker installation failed."
|
||||
echo " Visit: https://docs.docker.com/get-docker/"
|
||||
exit 1
|
||||
fi
|
||||
print_success "Docker installed successfully"
|
||||
fi
|
||||
fi
|
||||
|
||||
# --- Auto-install Docker Compose plugin (Linux only) ---
|
||||
if command -v docker &> /dev/null \
|
||||
&& ! docker compose version &> /dev/null \
|
||||
&& ! command -v docker-compose &> /dev/null \
|
||||
&& { [[ "$OSTYPE" == "linux-gnu"* ]] || [[ -n "${WSL_DISTRO_NAME:-}" ]]; }; then
|
||||
|
||||
print_info "Docker Compose not found — installing plugin..."
|
||||
COMPOSE_ARCH="$(uname -m)"
|
||||
COMPOSE_URL="https://github.com/docker/compose/releases/latest/download/docker-compose-linux-${COMPOSE_ARCH}"
|
||||
COMPOSE_DIR="/usr/local/lib/docker/cli-plugins"
|
||||
COMPOSE_TMP="$(mktemp)"
|
||||
sudo mkdir -p "$COMPOSE_DIR"
|
||||
if download_file "$COMPOSE_URL" "$COMPOSE_TMP"; then
|
||||
sudo mv "$COMPOSE_TMP" "$COMPOSE_DIR/docker-compose"
|
||||
sudo chmod +x "$COMPOSE_DIR/docker-compose"
|
||||
if docker compose version &> /dev/null; then
|
||||
print_success "Docker Compose plugin installed"
|
||||
else
|
||||
print_error "Docker Compose plugin installed but not detected."
|
||||
echo " Visit: https://docs.docker.com/compose/install/"
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
rm -f "$COMPOSE_TMP"
|
||||
print_error "Failed to download Docker Compose plugin."
|
||||
echo " Visit: https://docs.docker.com/compose/install/"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# On Linux, ensure the current user can talk to the Docker daemon without
|
||||
# sudo. If necessary, add them to the "docker" group and re-exec the
|
||||
# script under that group so the rest of the install proceeds normally.
|
||||
if command -v docker &> /dev/null \
|
||||
&& { [[ "$OSTYPE" == "linux-gnu"* ]] || [[ -n "${WSL_DISTRO_NAME:-}" ]]; } \
|
||||
&& [[ "$(id -u)" -ne 0 ]] \
|
||||
&& ! docker info &> /dev/null; then
|
||||
if [[ "${_ONYX_REEXEC:-}" = "1" ]]; then
|
||||
print_error "Cannot connect to Docker after group re-exec."
|
||||
print_info "Log out and back in, then run the script again."
|
||||
exit 1
|
||||
fi
|
||||
if ! getent group docker &> /dev/null; then
|
||||
sudo groupadd docker
|
||||
fi
|
||||
print_info "Adding $USER to the docker group..."
|
||||
sudo usermod -aG docker "$USER"
|
||||
print_info "Re-launching with docker group active..."
|
||||
exec sg docker -c "_ONYX_REEXEC=1 bash $(printf '%q ' "$0" "$@")"
|
||||
fi
|
||||
|
||||
# ASCII Art Banner
|
||||
echo ""
|
||||
echo -e "${BLUE}${BOLD}"
|
||||
@@ -479,7 +209,8 @@ echo "2. Check your system resources (Docker, memory, disk space)"
|
||||
echo "3. Guide you through deployment options (version, authentication)"
|
||||
echo ""
|
||||
|
||||
if is_interactive; then
|
||||
# Only prompt for acknowledgment if running interactively
|
||||
if [ -t 0 ]; then
|
||||
echo -e "${YELLOW}${BOLD}Please acknowledge and press Enter to continue...${NC}"
|
||||
read -r
|
||||
echo ""
|
||||
@@ -529,35 +260,41 @@ else
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Returns 0 if $1 <= $2, 1 if $1 > $2
|
||||
# Handles missing or non-numeric parts gracefully (treats them as 0)
|
||||
# Function to compare version numbers
|
||||
version_compare() {
|
||||
local version1="${1:-0.0.0}"
|
||||
local version2="${2:-0.0.0}"
|
||||
# Returns 0 if $1 <= $2, 1 if $1 > $2
|
||||
local version1=$1
|
||||
local version2=$2
|
||||
|
||||
local v1_major v1_minor v1_patch v2_major v2_minor v2_patch
|
||||
v1_major=$(echo "$version1" | cut -d. -f1)
|
||||
v1_minor=$(echo "$version1" | cut -d. -f2)
|
||||
v1_patch=$(echo "$version1" | cut -d. -f3)
|
||||
v2_major=$(echo "$version2" | cut -d. -f1)
|
||||
v2_minor=$(echo "$version2" | cut -d. -f2)
|
||||
v2_patch=$(echo "$version2" | cut -d. -f3)
|
||||
# Split versions into components
|
||||
local v1_major=$(echo $version1 | cut -d. -f1)
|
||||
local v1_minor=$(echo $version1 | cut -d. -f2)
|
||||
local v1_patch=$(echo $version1 | cut -d. -f3)
|
||||
|
||||
# Default non-numeric or empty parts to 0
|
||||
[[ "$v1_major" =~ ^[0-9]+$ ]] || v1_major=0
|
||||
[[ "$v1_minor" =~ ^[0-9]+$ ]] || v1_minor=0
|
||||
[[ "$v1_patch" =~ ^[0-9]+$ ]] || v1_patch=0
|
||||
[[ "$v2_major" =~ ^[0-9]+$ ]] || v2_major=0
|
||||
[[ "$v2_minor" =~ ^[0-9]+$ ]] || v2_minor=0
|
||||
[[ "$v2_patch" =~ ^[0-9]+$ ]] || v2_patch=0
|
||||
local v2_major=$(echo $version2 | cut -d. -f1)
|
||||
local v2_minor=$(echo $version2 | cut -d. -f2)
|
||||
local v2_patch=$(echo $version2 | cut -d. -f3)
|
||||
|
||||
if [ "$v1_major" -lt "$v2_major" ]; then return 0
|
||||
elif [ "$v1_major" -gt "$v2_major" ]; then return 1; fi
|
||||
# Compare major version
|
||||
if [ "$v1_major" -lt "$v2_major" ]; then
|
||||
return 0
|
||||
elif [ "$v1_major" -gt "$v2_major" ]; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
if [ "$v1_minor" -lt "$v2_minor" ]; then return 0
|
||||
elif [ "$v1_minor" -gt "$v2_minor" ]; then return 1; fi
|
||||
# Compare minor version
|
||||
if [ "$v1_minor" -lt "$v2_minor" ]; then
|
||||
return 0
|
||||
elif [ "$v1_minor" -gt "$v2_minor" ]; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
[ "$v1_patch" -le "$v2_patch" ]
|
||||
# Compare patch version
|
||||
if [ "$v1_patch" -le "$v2_patch" ]; then
|
||||
return 0
|
||||
else
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Check Docker daemon
|
||||
@@ -599,20 +336,10 @@ fi
|
||||
|
||||
# Convert to GB for display
|
||||
if [ "$MEMORY_MB" -gt 0 ]; then
|
||||
MEMORY_GB=$(awk "BEGIN {printf \"%.1f\", $MEMORY_MB / 1024}")
|
||||
if [ "$(awk "BEGIN {print ($MEMORY_MB >= 1024)}")" = "1" ]; then
|
||||
MEMORY_DISPLAY="~${MEMORY_GB}GB"
|
||||
else
|
||||
MEMORY_DISPLAY="${MEMORY_MB}MB"
|
||||
fi
|
||||
if [[ "$OSTYPE" == "darwin"* ]]; then
|
||||
print_info "Docker memory allocation: ${MEMORY_DISPLAY}"
|
||||
else
|
||||
print_info "System memory: ${MEMORY_DISPLAY} (Docker uses host memory directly)"
|
||||
fi
|
||||
MEMORY_GB=$((MEMORY_MB / 1024))
|
||||
print_info "Docker memory allocation: ~${MEMORY_GB}GB"
|
||||
else
|
||||
print_warning "Could not determine memory allocation"
|
||||
MEMORY_DISPLAY="unknown"
|
||||
print_warning "Could not determine Docker memory allocation"
|
||||
MEMORY_MB=0
|
||||
fi
|
||||
|
||||
@@ -631,7 +358,7 @@ RESOURCE_WARNING=false
|
||||
EXPECTED_RAM_MB=$((EXPECTED_DOCKER_RAM_GB * 1024))
|
||||
|
||||
if [ "$MEMORY_MB" -gt 0 ] && [ "$MEMORY_MB" -lt "$EXPECTED_RAM_MB" ]; then
|
||||
print_warning "Less than ${EXPECTED_DOCKER_RAM_GB}GB RAM available (found: ${MEMORY_DISPLAY})"
|
||||
print_warning "Docker has less than ${EXPECTED_DOCKER_RAM_GB}GB RAM allocated (found: ~${MEMORY_GB}GB)"
|
||||
RESOURCE_WARNING=true
|
||||
fi
|
||||
|
||||
@@ -642,10 +369,10 @@ fi
|
||||
|
||||
if [ "$RESOURCE_WARNING" = true ]; then
|
||||
echo ""
|
||||
print_warning "Onyx recommends at least ${EXPECTED_DOCKER_RAM_GB}GB RAM and ${EXPECTED_DISK_GB}GB disk space for optimal performance in standard mode."
|
||||
print_warning "Lite mode requires less resources (1-4GB RAM, 8-16GB disk depending on usage), but does not include a vector database."
|
||||
print_warning "Onyx recommends at least ${EXPECTED_DOCKER_RAM_GB}GB RAM and ${EXPECTED_DISK_GB}GB disk space for optimal performance."
|
||||
echo ""
|
||||
read -p "Do you want to continue anyway? (y/N): " -n 1 -r
|
||||
echo ""
|
||||
prompt_yn_or_default "Do you want to continue anyway? (Y/n): " "y"
|
||||
if [[ ! $REPLY =~ ^[Yy]$ ]]; then
|
||||
print_info "Installation cancelled. Please allocate more resources and try again."
|
||||
exit 1
|
||||
@@ -658,89 +385,117 @@ print_step "Creating directory structure"
|
||||
if [ -d "${INSTALL_ROOT}" ]; then
|
||||
print_info "Directory structure already exists"
|
||||
print_success "Using existing ${INSTALL_ROOT} directory"
|
||||
else
|
||||
mkdir -p "${INSTALL_ROOT}/deployment"
|
||||
mkdir -p "${INSTALL_ROOT}/data/nginx/local"
|
||||
print_success "Directory structure created"
|
||||
fi
|
||||
mkdir -p "${INSTALL_ROOT}/deployment"
|
||||
mkdir -p "${INSTALL_ROOT}/data/nginx/local"
|
||||
print_success "Directory structure created"
|
||||
|
||||
# Ensure all required configuration files are present
|
||||
# Download all required files
|
||||
print_step "Downloading Onyx configuration files"
|
||||
print_info "This step downloads all necessary configuration files from GitHub..."
|
||||
echo ""
|
||||
print_info "Downloading the following files:"
|
||||
echo " • docker-compose.yml - Main Docker Compose configuration"
|
||||
echo " • env.template - Environment variables template"
|
||||
echo " • nginx/app.conf.template - Nginx web server configuration"
|
||||
echo " • nginx/run-nginx.sh - Nginx startup script"
|
||||
echo " • README.md - Documentation and setup instructions"
|
||||
echo ""
|
||||
|
||||
# Download Docker Compose file
|
||||
COMPOSE_FILE="${INSTALL_ROOT}/deployment/docker-compose.yml"
|
||||
print_info "Downloading docker-compose.yml..."
|
||||
if curl -fsSL -o "$COMPOSE_FILE" "${GITHUB_RAW_URL}/docker-compose.yml" 2>/dev/null; then
|
||||
print_success "Docker Compose file downloaded successfully"
|
||||
|
||||
# Check if Docker Compose version is older than 2.24.0 and show warning
|
||||
# Skip check for dev builds (assume they're recent enough)
|
||||
if [ "$COMPOSE_VERSION" != "dev" ] && version_compare "$COMPOSE_VERSION" "2.24.0"; then
|
||||
print_warning "Docker Compose version $COMPOSE_VERSION is older than 2.24.0"
|
||||
echo ""
|
||||
print_warning "The docker-compose.yml file uses the newer env_file format that requires Docker Compose 2.24.0 or later."
|
||||
echo ""
|
||||
print_info "To use this configuration with your current Docker Compose version, you have two options:"
|
||||
echo ""
|
||||
echo "1. Upgrade Docker Compose to version 2.24.0 or later (recommended)"
|
||||
echo " Visit: https://docs.docker.com/compose/install/"
|
||||
echo ""
|
||||
echo "2. Manually replace all env_file sections in docker-compose.yml"
|
||||
echo " Change from:"
|
||||
echo " env_file:"
|
||||
echo " - path: .env"
|
||||
echo " required: false"
|
||||
echo " To:"
|
||||
echo " env_file: .env"
|
||||
echo ""
|
||||
print_warning "The installation will continue, but may fail if Docker Compose cannot parse the file."
|
||||
echo ""
|
||||
read -p "Do you want to continue anyway? (y/N): " -n 1 -r
|
||||
echo ""
|
||||
if [[ ! $REPLY =~ ^[Yy]$ ]]; then
|
||||
print_info "Installation cancelled. Please upgrade Docker Compose or manually edit the docker-compose.yml file."
|
||||
exit 1
|
||||
fi
|
||||
print_info "Proceeding with installation despite Docker Compose version compatibility issues..."
|
||||
fi
|
||||
else
|
||||
print_error "Failed to download Docker Compose file"
|
||||
print_info "Please ensure you have internet connection and try again"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Download env.template file
|
||||
ENV_TEMPLATE="${INSTALL_ROOT}/deployment/env.template"
|
||||
print_info "Downloading env.template..."
|
||||
if curl -fsSL -o "$ENV_TEMPLATE" "${GITHUB_RAW_URL}/env.template" 2>/dev/null; then
|
||||
print_success "Environment template downloaded successfully"
|
||||
else
|
||||
print_error "Failed to download env.template"
|
||||
print_info "Please ensure you have internet connection and try again"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Download nginx config files
|
||||
NGINX_BASE_URL="https://raw.githubusercontent.com/onyx-dot-app/onyx/main/deployment/data/nginx"
|
||||
|
||||
if [[ "$USE_LOCAL_FILES" = true ]]; then
|
||||
print_step "Verifying existing configuration files"
|
||||
# Download app.conf.template
|
||||
NGINX_CONFIG="${INSTALL_ROOT}/data/nginx/app.conf.template"
|
||||
print_info "Downloading nginx configuration template..."
|
||||
if curl -fsSL -o "$NGINX_CONFIG" "$NGINX_BASE_URL/app.conf.template" 2>/dev/null; then
|
||||
print_success "Nginx configuration template downloaded"
|
||||
else
|
||||
print_step "Downloading Onyx configuration files"
|
||||
print_info "This step downloads all necessary configuration files from GitHub..."
|
||||
print_error "Failed to download nginx configuration template"
|
||||
print_info "Please ensure you have internet connection and try again"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
ensure_file "${INSTALL_ROOT}/deployment/docker-compose.yml" \
|
||||
"${GITHUB_RAW_URL}/docker-compose.yml" "docker-compose.yml" || exit 1
|
||||
|
||||
# Check Docker Compose version compatibility after obtaining docker-compose.yml
|
||||
if [ "$COMPOSE_VERSION" != "dev" ] && version_compare "$COMPOSE_VERSION" "2.24.0"; then
|
||||
print_warning "Docker Compose version $COMPOSE_VERSION is older than 2.24.0"
|
||||
echo ""
|
||||
print_warning "The docker-compose.yml file uses the newer env_file format that requires Docker Compose 2.24.0 or later."
|
||||
echo ""
|
||||
print_info "To use this configuration with your current Docker Compose version, you have two options:"
|
||||
echo ""
|
||||
echo "1. Upgrade Docker Compose to version 2.24.0 or later (recommended)"
|
||||
echo " Visit: https://docs.docker.com/compose/install/"
|
||||
echo ""
|
||||
echo "2. Manually replace all env_file sections in docker-compose.yml"
|
||||
echo " Change from:"
|
||||
echo " env_file:"
|
||||
echo " - path: .env"
|
||||
echo " required: false"
|
||||
echo " To:"
|
||||
echo " env_file: .env"
|
||||
echo ""
|
||||
print_warning "The installation will continue, but may fail if Docker Compose cannot parse the file."
|
||||
echo ""
|
||||
prompt_yn_or_default "Do you want to continue anyway? (Y/n): " "y"
|
||||
if [[ ! $REPLY =~ ^[Yy]$ ]]; then
|
||||
print_info "Installation cancelled. Please upgrade Docker Compose or manually edit the docker-compose.yml file."
|
||||
exit 1
|
||||
fi
|
||||
print_info "Proceeding with installation despite Docker Compose version compatibility issues..."
|
||||
# Download run-nginx.sh script
|
||||
NGINX_RUN_SCRIPT="${INSTALL_ROOT}/data/nginx/run-nginx.sh"
|
||||
print_info "Downloading nginx startup script..."
|
||||
if curl -fsSL -o "$NGINX_RUN_SCRIPT" "$NGINX_BASE_URL/run-nginx.sh" 2>/dev/null; then
|
||||
chmod +x "$NGINX_RUN_SCRIPT"
|
||||
print_success "Nginx startup script downloaded and made executable"
|
||||
else
|
||||
print_error "Failed to download nginx startup script"
|
||||
print_info "Please ensure you have internet connection and try again"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Handle lite overlay: ensure it if --lite, clean up stale copies otherwise
|
||||
if [[ "$LITE_MODE" = true ]]; then
|
||||
ensure_file "${INSTALL_ROOT}/deployment/${LITE_COMPOSE_FILE}" \
|
||||
"${GITHUB_RAW_URL}/${LITE_COMPOSE_FILE}" "${LITE_COMPOSE_FILE}" || exit 1
|
||||
elif [[ -f "${INSTALL_ROOT}/deployment/${LITE_COMPOSE_FILE}" ]]; then
|
||||
if [[ -f "${INSTALL_ROOT}/deployment/.env" ]]; then
|
||||
print_warning "Existing lite overlay found but --lite was not passed."
|
||||
prompt_yn_or_default "Remove lite overlay and switch to standard mode? (y/N): " "n"
|
||||
if [[ ! $REPLY =~ ^[Yy]$ ]]; then
|
||||
print_info "Keeping existing lite overlay. Pass --lite to keep using lite mode."
|
||||
LITE_MODE=true
|
||||
else
|
||||
rm -f "${INSTALL_ROOT}/deployment/${LITE_COMPOSE_FILE}"
|
||||
print_info "Removed lite overlay (switching to standard mode)"
|
||||
fi
|
||||
else
|
||||
rm -f "${INSTALL_ROOT}/deployment/${LITE_COMPOSE_FILE}"
|
||||
print_info "Removed previous lite overlay (switching to standard mode)"
|
||||
fi
|
||||
# Download README file
|
||||
README_FILE="${INSTALL_ROOT}/README.md"
|
||||
print_info "Downloading README.md..."
|
||||
if curl -fsSL -o "$README_FILE" "${GITHUB_RAW_URL}/README.md" 2>/dev/null; then
|
||||
print_success "README.md downloaded successfully"
|
||||
else
|
||||
print_error "Failed to download README.md"
|
||||
print_info "Please ensure you have internet connection and try again"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
ensure_file "${INSTALL_ROOT}/deployment/env.template" \
|
||||
"${GITHUB_RAW_URL}/env.template" "env.template" || exit 1
|
||||
|
||||
ensure_file "${INSTALL_ROOT}/data/nginx/app.conf.template" \
|
||||
"$NGINX_BASE_URL/app.conf.template" "nginx/app.conf.template" || exit 1
|
||||
|
||||
ensure_file "${INSTALL_ROOT}/data/nginx/run-nginx.sh" \
|
||||
"$NGINX_BASE_URL/run-nginx.sh" "nginx/run-nginx.sh" || exit 1
|
||||
chmod +x "${INSTALL_ROOT}/data/nginx/run-nginx.sh"
|
||||
|
||||
ensure_file "${INSTALL_ROOT}/README.md" \
|
||||
"${GITHUB_RAW_URL}/README.md" "README.md" || exit 1
|
||||
|
||||
# Create empty local directory marker (if needed)
|
||||
touch "${INSTALL_ROOT}/data/nginx/local/.gitkeep"
|
||||
print_success "All configuration files ready"
|
||||
print_success "All configuration files downloaded successfully"
|
||||
|
||||
# Set up deployment configuration
|
||||
print_step "Setting up deployment configs"
|
||||
@@ -758,7 +513,7 @@ if [ -d "${INSTALL_ROOT}/deployment" ] && [ -f "${INSTALL_ROOT}/deployment/docke
|
||||
|
||||
if [ -n "$COMPOSE_CMD" ]; then
|
||||
# Check if any containers are running
|
||||
RUNNING_CONTAINERS=$(cd "${INSTALL_ROOT}/deployment" && $COMPOSE_CMD $(compose_file_args true) ps -q 2>/dev/null | wc -l)
|
||||
RUNNING_CONTAINERS=$(cd "${INSTALL_ROOT}/deployment" && $COMPOSE_CMD -f docker-compose.yml ps -q 2>/dev/null | wc -l)
|
||||
if [ "$RUNNING_CONTAINERS" -gt 0 ]; then
|
||||
print_error "Onyx services are currently running!"
|
||||
echo ""
|
||||
@@ -779,7 +534,7 @@ if [ -f "$ENV_FILE" ]; then
|
||||
echo "• Press Enter to restart with current configuration"
|
||||
echo "• Type 'update' to update to a newer version"
|
||||
echo ""
|
||||
prompt_or_default "Choose an option [default: restart]: " ""
|
||||
read -p "Choose an option [default: restart]: " -r
|
||||
echo ""
|
||||
|
||||
if [ "$REPLY" = "update" ]; then
|
||||
@@ -788,30 +543,26 @@ if [ -f "$ENV_FILE" ]; then
|
||||
echo "• Press Enter for latest (recommended)"
|
||||
echo "• Type a specific tag (e.g., v0.1.0)"
|
||||
echo ""
|
||||
# If --include-craft was passed, default to craft-latest
|
||||
if [ "$INCLUDE_CRAFT" = true ]; then
|
||||
prompt_or_default "Enter tag [default: craft-latest]: " "craft-latest"
|
||||
VERSION="$REPLY"
|
||||
read -p "Enter tag [default: craft-latest]: " -r VERSION
|
||||
else
|
||||
prompt_or_default "Enter tag [default: latest]: " "latest"
|
||||
VERSION="$REPLY"
|
||||
read -p "Enter tag [default: latest]: " -r VERSION
|
||||
fi
|
||||
echo ""
|
||||
|
||||
if [ "$INCLUDE_CRAFT" = true ] && [ "$VERSION" = "craft-latest" ]; then
|
||||
print_info "Selected: craft-latest (Craft enabled)"
|
||||
elif [ "$VERSION" = "latest" ]; then
|
||||
print_info "Selected: Latest version"
|
||||
if [ -z "$VERSION" ]; then
|
||||
if [ "$INCLUDE_CRAFT" = true ]; then
|
||||
VERSION="craft-latest"
|
||||
print_info "Selected: craft-latest (Craft enabled)"
|
||||
else
|
||||
VERSION="latest"
|
||||
print_info "Selected: Latest version"
|
||||
fi
|
||||
else
|
||||
print_info "Selected: $VERSION"
|
||||
fi
|
||||
|
||||
# Reject craft image tags when running in lite mode
|
||||
if [[ "$LITE_MODE" = true ]] && [[ "${VERSION:-}" == craft-* ]]; then
|
||||
print_error "Cannot use a craft image tag (${VERSION}) with --lite."
|
||||
print_info "Craft requires services (Vespa, Redis, background workers) that lite mode disables."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Update .env file with new version
|
||||
print_info "Updating configuration for version $VERSION..."
|
||||
if grep -q "^IMAGE_TAG=" "$ENV_FILE"; then
|
||||
@@ -830,67 +581,13 @@ if [ -f "$ENV_FILE" ]; then
|
||||
fi
|
||||
print_success "Configuration updated for upgrade"
|
||||
else
|
||||
# Reject restarting a craft deployment in lite mode
|
||||
EXISTING_TAG=$(grep "^IMAGE_TAG=" "$ENV_FILE" | head -1 | cut -d'=' -f2 | tr -d ' "'"'"'')
|
||||
if [[ "$LITE_MODE" = true ]] && [[ "${EXISTING_TAG:-}" == craft-* ]]; then
|
||||
print_error "Cannot restart a craft deployment (${EXISTING_TAG}) with --lite."
|
||||
print_info "Craft requires services (Vespa, Redis, background workers) that lite mode disables."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
print_info "Keeping existing configuration..."
|
||||
print_success "Will restart with current settings"
|
||||
fi
|
||||
|
||||
# Ensure COMPOSE_PROFILES is cleared when running in lite mode on an
|
||||
# existing .env (the template ships with s3-filestore enabled).
|
||||
if [[ "$LITE_MODE" = true ]] && grep -q "^COMPOSE_PROFILES=.*s3-filestore" "$ENV_FILE" 2>/dev/null; then
|
||||
sed -i.bak 's/^COMPOSE_PROFILES=.*/COMPOSE_PROFILES=/' "$ENV_FILE" 2>/dev/null || true
|
||||
print_success "Cleared COMPOSE_PROFILES for lite mode"
|
||||
fi
|
||||
else
|
||||
print_info "No existing .env file found. Setting up new deployment..."
|
||||
echo ""
|
||||
|
||||
# Ask for deployment mode (standard vs lite) unless already set via --lite flag
|
||||
if [[ "$LITE_MODE" = false ]]; then
|
||||
print_info "Which deployment mode would you like?"
|
||||
echo ""
|
||||
echo " 1) Standard - Full deployment with search, connectors, and RAG"
|
||||
echo " 2) Lite - Minimal deployment (no Vespa, Redis, or model servers)"
|
||||
echo " LLM chat, tools, file uploads, and Projects still work"
|
||||
echo ""
|
||||
prompt_or_default "Choose a mode (1 or 2) [default: 1]: " "1"
|
||||
echo ""
|
||||
|
||||
case "$REPLY" in
|
||||
2)
|
||||
LITE_MODE=true
|
||||
print_info "Selected: Lite mode"
|
||||
ensure_file "${INSTALL_ROOT}/deployment/${LITE_COMPOSE_FILE}" \
|
||||
"${GITHUB_RAW_URL}/${LITE_COMPOSE_FILE}" "${LITE_COMPOSE_FILE}" || exit 1
|
||||
;;
|
||||
*)
|
||||
print_info "Selected: Standard mode"
|
||||
;;
|
||||
esac
|
||||
else
|
||||
print_info "Deployment mode: Lite (set via --lite flag)"
|
||||
fi
|
||||
|
||||
# Validate lite + craft combination (could now be set interactively)
|
||||
if [[ "$LITE_MODE" = true ]] && [[ "$INCLUDE_CRAFT" = true ]]; then
|
||||
print_error "--include-craft cannot be used with Lite mode."
|
||||
print_info "Craft requires services (Vespa, Redis, background workers) that lite mode disables."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Adjust resource expectations for lite mode
|
||||
if [[ "$LITE_MODE" = true ]]; then
|
||||
EXPECTED_DOCKER_RAM_GB=4
|
||||
EXPECTED_DISK_GB=16
|
||||
fi
|
||||
|
||||
# Ask for version
|
||||
print_info "Which tag would you like to deploy?"
|
||||
echo ""
|
||||
@@ -898,21 +595,23 @@ else
|
||||
echo "• Press Enter for craft-latest (recommended for Craft)"
|
||||
echo "• Type a specific tag (e.g., craft-v1.0.0)"
|
||||
echo ""
|
||||
prompt_or_default "Enter tag [default: craft-latest]: " "craft-latest"
|
||||
VERSION="$REPLY"
|
||||
read -p "Enter tag [default: craft-latest]: " -r VERSION
|
||||
else
|
||||
echo "• Press Enter for latest (recommended)"
|
||||
echo "• Type a specific tag (e.g., v0.1.0)"
|
||||
echo ""
|
||||
prompt_or_default "Enter tag [default: latest]: " "latest"
|
||||
VERSION="$REPLY"
|
||||
read -p "Enter tag [default: latest]: " -r VERSION
|
||||
fi
|
||||
echo ""
|
||||
|
||||
if [ "$INCLUDE_CRAFT" = true ] && [ "$VERSION" = "craft-latest" ]; then
|
||||
print_info "Selected: craft-latest (Craft enabled)"
|
||||
elif [ "$VERSION" = "latest" ]; then
|
||||
print_info "Selected: Latest tag"
|
||||
if [ -z "$VERSION" ]; then
|
||||
if [ "$INCLUDE_CRAFT" = true ]; then
|
||||
VERSION="craft-latest"
|
||||
print_info "Selected: craft-latest (Craft enabled)"
|
||||
else
|
||||
VERSION="latest"
|
||||
print_info "Selected: Latest tag"
|
||||
fi
|
||||
else
|
||||
print_info "Selected: $VERSION"
|
||||
fi
|
||||
@@ -946,13 +645,6 @@ else
|
||||
# Use basic auth by default
|
||||
AUTH_SCHEMA="basic"
|
||||
|
||||
# Reject craft image tags when running in lite mode (must check before writing .env)
|
||||
if [[ "$LITE_MODE" = true ]] && [[ "${VERSION:-}" == craft-* ]]; then
|
||||
print_error "Cannot use a craft image tag (${VERSION}) with --lite."
|
||||
print_info "Craft requires services (Vespa, Redis, background workers) that lite mode disables."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Create .env file from template
|
||||
print_info "Creating .env file with your selections..."
|
||||
cp "$ENV_TEMPLATE" "$ENV_FILE"
|
||||
@@ -962,13 +654,6 @@ else
|
||||
sed -i.bak "s/^IMAGE_TAG=.*/IMAGE_TAG=$VERSION/" "$ENV_FILE"
|
||||
print_success "IMAGE_TAG set to $VERSION"
|
||||
|
||||
# In lite mode, clear COMPOSE_PROFILES so profiled services (MinIO, etc.)
|
||||
# stay disabled — the template ships with s3-filestore enabled by default.
|
||||
if [[ "$LITE_MODE" = true ]]; then
|
||||
sed -i.bak 's/^COMPOSE_PROFILES=.*/COMPOSE_PROFILES=/' "$ENV_FILE" 2>/dev/null || true
|
||||
print_success "Cleared COMPOSE_PROFILES for lite mode"
|
||||
fi
|
||||
|
||||
# Configure basic authentication (default)
|
||||
sed -i.bak 's/^AUTH_TYPE=.*/AUTH_TYPE=basic/' "$ENV_FILE" 2>/dev/null || true
|
||||
print_success "Basic authentication enabled in configuration"
|
||||
@@ -1089,7 +774,7 @@ print_step "Pulling Docker images"
|
||||
print_info "This may take several minutes depending on your internet connection..."
|
||||
echo ""
|
||||
print_info "Downloading Docker images (this may take a while)..."
|
||||
(cd "${INSTALL_ROOT}/deployment" && $COMPOSE_CMD $(compose_file_args) pull --quiet)
|
||||
(cd "${INSTALL_ROOT}/deployment" && $COMPOSE_CMD -f docker-compose.yml pull --quiet)
|
||||
if [ $? -eq 0 ]; then
|
||||
print_success "Docker images downloaded successfully"
|
||||
else
|
||||
@@ -1103,9 +788,9 @@ print_info "Launching containers..."
|
||||
echo ""
|
||||
if [ "$USE_LATEST" = true ]; then
|
||||
print_info "Force pulling latest images and recreating containers..."
|
||||
(cd "${INSTALL_ROOT}/deployment" && $COMPOSE_CMD $(compose_file_args) up -d --pull always --force-recreate)
|
||||
(cd "${INSTALL_ROOT}/deployment" && $COMPOSE_CMD -f docker-compose.yml up -d --pull always --force-recreate)
|
||||
else
|
||||
(cd "${INSTALL_ROOT}/deployment" && $COMPOSE_CMD $(compose_file_args) up -d)
|
||||
(cd "${INSTALL_ROOT}/deployment" && $COMPOSE_CMD -f docker-compose.yml up -d)
|
||||
fi
|
||||
if [ $? -ne 0 ]; then
|
||||
print_error "Failed to start Onyx services"
|
||||
@@ -1127,7 +812,7 @@ echo ""
|
||||
# Check for restart loops
|
||||
print_info "Checking container health status..."
|
||||
RESTART_ISSUES=false
|
||||
CONTAINERS=$(cd "${INSTALL_ROOT}/deployment" && $COMPOSE_CMD $(compose_file_args) ps -q 2>/dev/null)
|
||||
CONTAINERS=$(cd "${INSTALL_ROOT}/deployment" && $COMPOSE_CMD -f docker-compose.yml ps -q 2>/dev/null)
|
||||
|
||||
for CONTAINER in $CONTAINERS; do
|
||||
PROJECT_NAME="$(basename "$INSTALL_ROOT")_deployment_"
|
||||
@@ -1156,7 +841,7 @@ if [ "$RESTART_ISSUES" = true ]; then
|
||||
print_error "Some containers are experiencing issues!"
|
||||
echo ""
|
||||
print_info "Please check the logs for more information:"
|
||||
echo " (cd \"${INSTALL_ROOT}/deployment\" && $COMPOSE_CMD $(compose_file_args) logs)"
|
||||
echo " (cd \"${INSTALL_ROOT}/deployment\" && $COMPOSE_CMD -f docker-compose.yml logs)"
|
||||
|
||||
echo ""
|
||||
print_info "If the issue persists, please contact: founders@onyx.app"
|
||||
@@ -1175,12 +860,8 @@ check_onyx_health() {
|
||||
echo ""
|
||||
|
||||
while [ $attempt -le $max_attempts ]; do
|
||||
local http_code=""
|
||||
if [[ "$DOWNLOADER" == "curl" ]]; then
|
||||
http_code=$(curl -s -o /dev/null -w "%{http_code}" "http://localhost:$port" 2>/dev/null || echo "000")
|
||||
else
|
||||
http_code=$(wget -q --spider -S "http://localhost:$port" 2>&1 | grep "HTTP/" | tail -1 | awk '{print $2}' || echo "000")
|
||||
fi
|
||||
# Check for successful HTTP responses (200, 301, 302, etc.)
|
||||
local http_code=$(curl -s -o /dev/null -w "%{http_code}" "http://localhost:$port")
|
||||
if echo "$http_code" | grep -qE "^(200|301|302|303|307|308)$"; then
|
||||
return 0
|
||||
fi
|
||||
@@ -1236,18 +917,6 @@ print_info "If authentication is enabled, you can create your admin account here
|
||||
echo " • Visit http://localhost:${HOST_PORT}/auth/signup to create your admin account"
|
||||
echo " • The first user created will automatically have admin privileges"
|
||||
echo ""
|
||||
if [[ "$LITE_MODE" = true ]]; then
|
||||
echo ""
|
||||
print_info "Running in Lite mode — the following services are NOT started:"
|
||||
echo " • Vespa (vector database)"
|
||||
echo " • Redis (cache)"
|
||||
echo " • Model servers (embedding/inference)"
|
||||
echo " • Background workers (Celery)"
|
||||
echo ""
|
||||
print_info "Connectors and RAG search are disabled. LLM chat, tools, user file"
|
||||
print_info "uploads, Projects, Agent knowledge, and code interpreter still work."
|
||||
fi
|
||||
echo ""
|
||||
print_info "Refer to the README in the ${INSTALL_ROOT} directory for more information."
|
||||
echo ""
|
||||
print_info "For help or issues, contact: founders@onyx.app"
|
||||
|
||||
@@ -19,13 +19,12 @@
|
||||
|
||||
## Output template to file and inspect
|
||||
* cd charts/onyx
|
||||
* helm template test-output . --set auth.opensearch.values.opensearch_admin_password='StrongPassword123!' > test-output.yaml
|
||||
* helm template test-output . > test-output.yaml
|
||||
|
||||
## Test the entire cluster manually
|
||||
* cd charts/onyx
|
||||
* helm install onyx . -n onyx --set postgresql.primary.persistence.enabled=false --set auth.opensearch.values.opensearch_admin_password='StrongPassword123!'
|
||||
* helm install onyx . -n onyx --set postgresql.primary.persistence.enabled=false
|
||||
* the postgres flag is to keep the storage ephemeral for testing. You probably don't want to set that in prod.
|
||||
* the OpenSearch admin password must be set on first install unless you are supplying `auth.opensearch.existingSecret`.
|
||||
* no flag for ephemeral vespa storage yet, might be good for testing
|
||||
* kubectl -n onyx port-forward service/onyx-nginx 8080:80
|
||||
* this will forward the local port 8080 to the installed chart for you to run tests, etc.
|
||||
|
||||
@@ -5,7 +5,7 @@ home: https://www.onyx.app/
|
||||
sources:
|
||||
- "https://github.com/onyx-dot-app/onyx"
|
||||
type: application
|
||||
version: 0.4.36
|
||||
version: 0.4.35
|
||||
appVersion: latest
|
||||
annotations:
|
||||
category: Productivity
|
||||
|
||||
@@ -1,9 +1,6 @@
|
||||
# Values for chart-testing (ct lint/install)
|
||||
# This file is automatically used by ct when running lint and install commands
|
||||
auth:
|
||||
opensearch:
|
||||
values:
|
||||
opensearch_admin_password: "placeholder-OpenSearch1!"
|
||||
userauth:
|
||||
values:
|
||||
user_auth_secret: "placeholder-for-ci-testing"
|
||||
|
||||
@@ -1177,17 +1177,12 @@ auth:
|
||||
# Secrets values IF existingSecret is empty. Key here must match the value
|
||||
# in secretKeys to be used. Values will be base64 encoded in the k8s
|
||||
# cluster.
|
||||
# For the bundled OpenSearch chart, the admin password is consumed during
|
||||
# initial cluster setup. Changing this value later will update Onyx's
|
||||
# client credentials, but will not rotate the OpenSearch admin password.
|
||||
# Set this before first install or use existingSecret to preserve the
|
||||
# current secret on upgrade.
|
||||
# Password must meet OpenSearch complexity requirements:
|
||||
# min 8 chars, uppercase, lowercase, digit, and special character.
|
||||
# Required when auth.opensearch.enabled=true and no existing secret exists.
|
||||
# CHANGE THIS FOR PRODUCTION.
|
||||
values:
|
||||
opensearch_admin_username: "admin"
|
||||
opensearch_admin_password: ""
|
||||
opensearch_admin_password: "OnyxDev1!"
|
||||
userauth:
|
||||
# -- Used for password reset / verification tokens and OAuth/OIDC state signing.
|
||||
# Disabled by default to preserve upgrade compatibility for existing Helm customers.
|
||||
|
||||
@@ -144,7 +144,7 @@ dev = [
|
||||
"matplotlib==3.10.8",
|
||||
"mypy-extensions==1.0.0",
|
||||
"mypy==1.13.0",
|
||||
"onyx-devtools==0.7.1",
|
||||
"onyx-devtools==0.7.0",
|
||||
"openapi-generator-cli==7.17.0",
|
||||
"pandas-stubs~=2.3.3",
|
||||
"pre-commit==3.2.2",
|
||||
|
||||
@@ -4,11 +4,9 @@ import (
|
||||
"bufio"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
@@ -94,57 +92,12 @@ Examples:
|
||||
return cmd
|
||||
}
|
||||
|
||||
func isPortAvailable(port int) bool {
|
||||
ln, err := net.Listen("tcp", fmt.Sprintf(":%d", port))
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
_ = ln.Close()
|
||||
return true
|
||||
}
|
||||
|
||||
func getProcessOnPort(port int) string {
|
||||
out, err := exec.Command("lsof", "-i", fmt.Sprintf(":%d", port), "-t").Output()
|
||||
if err != nil || len(strings.TrimSpace(string(out))) == 0 {
|
||||
return "an unknown process"
|
||||
}
|
||||
pid := strings.Split(strings.TrimSpace(string(out)), "\n")[0]
|
||||
nameOut, err := exec.Command("ps", "-p", pid, "-o", "comm=").Output()
|
||||
if err != nil || len(strings.TrimSpace(string(nameOut))) == 0 {
|
||||
return fmt.Sprintf("process (PID %s)", pid)
|
||||
}
|
||||
return fmt.Sprintf("%s (PID %s)", strings.TrimSpace(string(nameOut)), pid)
|
||||
}
|
||||
|
||||
func resolvePort(port string) string {
|
||||
portNum, err := strconv.Atoi(port)
|
||||
if err != nil {
|
||||
log.Fatalf("Invalid port %q: %v", port, err)
|
||||
}
|
||||
if isPortAvailable(portNum) {
|
||||
return port
|
||||
}
|
||||
proc := getProcessOnPort(portNum)
|
||||
candidate := portNum + 1
|
||||
for candidate <= 65535 {
|
||||
if isPortAvailable(candidate) {
|
||||
log.Warnf("⚠ Port %d is in use by %s, using available port %d instead.", portNum, proc, candidate)
|
||||
return strconv.Itoa(candidate)
|
||||
}
|
||||
candidate++
|
||||
}
|
||||
log.Fatalf("No available ports found starting from %d", portNum)
|
||||
return port
|
||||
}
|
||||
|
||||
func runBackendService(name, module, port string, opts *BackendOptions) {
|
||||
root, err := paths.GitRoot()
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to find git root: %v", err)
|
||||
}
|
||||
|
||||
port = resolvePort(port)
|
||||
|
||||
envFile := ensureBackendEnvFile(root)
|
||||
fileVars := loadBackendEnvFile(envFile)
|
||||
|
||||
|
||||
34
uv.lock
generated
34
uv.lock
generated
@@ -453,14 +453,14 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "authlib"
|
||||
version = "1.6.9"
|
||||
version = "1.6.7"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "cryptography" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/af/98/00d3dd826d46959ad8e32af2dbb2398868fd9fd0683c26e56d0789bd0e68/authlib-1.6.9.tar.gz", hash = "sha256:d8f2421e7e5980cc1ddb4e32d3f5fa659cfaf60d8eaf3281ebed192e4ab74f04", size = 165134, upload-time = "2026-03-02T07:44:01.998Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/49/dc/ed1681bf1339dd6ea1ce56136bad4baabc6f7ad466e375810702b0237047/authlib-1.6.7.tar.gz", hash = "sha256:dbf10100011d1e1b34048c9d120e83f13b35d69a826ae762b93d2fb5aafc337b", size = 164950, upload-time = "2026-02-06T14:04:14.171Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/53/23/b65f568ed0c22f1efacb744d2db1a33c8068f384b8c9b482b52ebdbc3ef6/authlib-1.6.9-py2.py3-none-any.whl", hash = "sha256:f08b4c14e08f0861dc18a32357b33fbcfd2ea86cfe3fe149484b4d764c4a0ac3", size = 244197, upload-time = "2026-03-02T07:44:00.307Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f8/00/3ed12264094ec91f534fae429945efbaa9f8c666f3aa7061cc3b2a26a0cd/authlib-1.6.7-py2.py3-none-any.whl", hash = "sha256:c637340d9a02789d2efa1d003a7437d10d3e565237bcb5fcbc6c134c7b95bab0", size = 244115, upload-time = "2026-02-06T14:04:12.141Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -4458,7 +4458,7 @@ requires-dist = [
|
||||
{ name = "numpy", marker = "extra == 'model-server'", specifier = "==2.4.1" },
|
||||
{ name = "oauthlib", marker = "extra == 'backend'", specifier = "==3.2.2" },
|
||||
{ name = "office365-rest-python-client", marker = "extra == 'backend'", specifier = "==2.6.2" },
|
||||
{ name = "onyx-devtools", marker = "extra == 'dev'", specifier = "==0.7.1" },
|
||||
{ name = "onyx-devtools", marker = "extra == 'dev'", specifier = "==0.7.0" },
|
||||
{ name = "openai", specifier = "==2.14.0" },
|
||||
{ name = "openapi-generator-cli", marker = "extra == 'dev'", specifier = "==7.17.0" },
|
||||
{ name = "openinference-instrumentation", marker = "extra == 'backend'", specifier = "==0.1.42" },
|
||||
@@ -4563,19 +4563,19 @@ requires-dist = [{ name = "onyx", extras = ["backend", "dev", "ee"], editable =
|
||||
|
||||
[[package]]
|
||||
name = "onyx-devtools"
|
||||
version = "0.7.1"
|
||||
version = "0.7.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "fastapi" },
|
||||
{ name = "openapi-generator-cli" },
|
||||
]
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/65/9d/74bcd02583706bdf90c8ac9084eb60bd71d0671392152410ab21b7b68ea1/onyx_devtools-0.7.1-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:178385dce0b413fd2a1f761055a99f556ec536ef5c32963fc273e751813621eb", size = 4007974, upload-time = "2026-03-17T21:10:39.267Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f0/f8/d8ddb32120428c083c60eb07244479da6e07eaebd31847658a049ab33815/onyx_devtools-0.7.1-py3-none-macosx_11_0_arm64.whl", hash = "sha256:7960ae6e440ebf1584e02d9e1d0c9ef543b1d54c2584cdcace15695aec3121b2", size = 3696924, upload-time = "2026-03-17T21:10:50.716Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/87/21/1e427280066db42ff9dd5f34c70b9dca5d9781f96d0d9a88aaa454fdb432/onyx_devtools-0.7.1-py3-none-manylinux_2_17_aarch64.whl", hash = "sha256:6785dda88ca0a3d8464a9bfab76a253ed90da89d53a9c4a67227980f37df1ccf", size = 3568300, upload-time = "2026-03-17T21:10:41.997Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/0e/0e/afbbe1164b3d016ddb5352353cb2541eef5a8b2c04e8f02d5d1319cb8b8c/onyx_devtools-0.7.1-py3-none-manylinux_2_17_x86_64.whl", hash = "sha256:9e77f2b725c0c00061a3dda5eba199404b51638cec0bf54fc7611fee1f26db34", size = 3974668, upload-time = "2026-03-17T21:10:43.879Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/8a/a5/22840643289ef4ca83931b7a79fba8f1db7e626b4b870d4b4f8206c4ff5f/onyx_devtools-0.7.1-py3-none-win_amd64.whl", hash = "sha256:de37daa0e4db9b5dccf94408a3422be4f821e380ab70081bd1032cec1e3c91e6", size = 4078640, upload-time = "2026-03-17T21:10:40.275Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/1e/c1/a0295506a521d9942b0f55523781a113e4555420d800a386d5a2eb46a7ad/onyx_devtools-0.7.1-py3-none-win_arm64.whl", hash = "sha256:ab88c53ebda6dff27350316b4ac9bd5f258cd586c2109971a9d976411e1e22ea", size = 3636787, upload-time = "2026-03-17T21:10:37.492Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/22/9e/6957b11555da57d9e97092f4cd8ac09a86666264b0c9491838f4b27db5dc/onyx_devtools-0.7.0-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:ad962a168d46ea11dcde9fa3b37e4f12ec520b4a4cb4d49d8732de110d46c4b6", size = 3998057, upload-time = "2026-03-12T03:09:11.585Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/cd/90/c72f3d06ba677012d77c77de36195b6a32a15c755c79ba0282be74e3c366/onyx_devtools-0.7.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:e46d252e2b048ff053b03519c3a875998780738d7c334eaa1c9a32ff445e3e1a", size = 3687753, upload-time = "2026-03-12T03:09:11.742Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/10/42/4e9fe36eccf9f76d67ba8f4ff6539196a09cd60351fb63f5865e1544cbfa/onyx_devtools-0.7.0-py3-none-manylinux_2_17_aarch64.whl", hash = "sha256:f280bc9320e1cc310e7d753a371009bfaab02cc0e0cfd78559663b15655b5a50", size = 3560144, upload-time = "2026-03-12T03:12:24.02Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/76/40/36dc12d99760b358c7f39b27361cb18fa9681ffe194107f982d0e1a74016/onyx_devtools-0.7.0-py3-none-manylinux_2_17_x86_64.whl", hash = "sha256:e31df751c7540ae7e70a7fe8e1153c79c31c2254af6aa4c72c0dd54fa381d2ab", size = 3964387, upload-time = "2026-03-12T03:09:11.356Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/34/18/74744230c3820a5a7687335507ca5f1dbebab2c5325805041c1cd5703e6a/onyx_devtools-0.7.0-py3-none-win_amd64.whl", hash = "sha256:541bfd347c2d5b11e7f63ab5001d2594df91d215ad9d07b1562f5e715700f7e6", size = 4068030, upload-time = "2026-03-12T03:09:12.98Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/8c/78/1320436607d3ffcb321ba7b064556c020ea15843a7e7d903fbb7529a71f5/onyx_devtools-0.7.0-py3-none-win_arm64.whl", hash = "sha256:83016330a9d39712431916cc25b2fb2cfcaa0112a55cc4f919d545da3a8974f9", size = 3626409, upload-time = "2026-03-12T03:09:10.222Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -5458,11 +5458,11 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "pyasn1"
|
||||
version = "0.6.3"
|
||||
version = "0.6.2"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/5c/5f/6583902b6f79b399c9c40674ac384fd9cd77805f9e6205075f828ef11fb2/pyasn1-0.6.3.tar.gz", hash = "sha256:697a8ecd6d98891189184ca1fa05d1bb00e2f84b5977c481452050549c8a72cf", size = 148685, upload-time = "2026-03-17T01:06:53.382Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/fe/b6/6e630dff89739fcd427e3f72b3d905ce0acb85a45d4ec3e2678718a3487f/pyasn1-0.6.2.tar.gz", hash = "sha256:9b59a2b25ba7e4f8197db7686c09fb33e658b98339fadb826e9512629017833b", size = 146586, upload-time = "2026-01-16T18:04:18.534Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/5d/a0/7d793dce3fa811fe047d6ae2431c672364b462850c6235ae306c0efd025f/pyasn1-0.6.3-py3-none-any.whl", hash = "sha256:a80184d120f0864a52a073acc6fc642847d0be408e7c7252f31390c0f4eadcde", size = 83997, upload-time = "2026-03-17T01:06:52.036Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/44/b5/a96872e5184f354da9c84ae119971a0a4c221fe9b27a4d94bd43f2596727/pyasn1-0.6.2-py3-none-any.whl", hash = "sha256:1eb26d860996a18e9b6ed05e7aae0e9fc21619fcee6af91cca9bad4fbea224bf", size = 83371, upload-time = "2026-01-16T18:04:17.174Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -5643,11 +5643,11 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "pyjwt"
|
||||
version = "2.12.0"
|
||||
version = "2.11.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/a8/10/e8192be5f38f3e8e7e046716de4cae33d56fd5ae08927a823bb916be36c1/pyjwt-2.12.0.tar.gz", hash = "sha256:2f62390b667cd8257de560b850bb5a883102a388829274147f1d724453f8fb02", size = 102511, upload-time = "2026-03-12T17:15:30.831Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/5c/5a/b46fa56bf322901eee5b0454a34343cdbdae202cd421775a8ee4e42fd519/pyjwt-2.11.0.tar.gz", hash = "sha256:35f95c1f0fbe5d5ba6e43f00271c275f7a1a4db1dab27bf708073b75318ea623", size = 98019, upload-time = "2026-01-30T19:59:55.694Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/15/70/70f895f404d363d291dcf62c12c85fdd47619ad9674ac0f53364d035925a/pyjwt-2.12.0-py3-none-any.whl", hash = "sha256:9bb459d1bdd0387967d287f5656bf7ec2b9a26645d1961628cda1764e087fd6e", size = 29700, upload-time = "2026-03-12T17:15:29.257Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/6f/01/c26ce75ba460d5cd503da9e13b21a33804d38c2165dec7b716d06b13010c/pyjwt-2.11.0-py3-none-any.whl", hash = "sha256:94a6bde30eb5c8e04fee991062b534071fd1439ef58d2adc9ccb823e7bcd0469", size = 28224, upload-time = "2026-01-30T19:59:54.539Z" },
|
||||
]
|
||||
|
||||
[package.optional-dependencies]
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import "@opal/components/buttons/button/styles.css";
|
||||
import "@opal/components/tooltip.css";
|
||||
import { Interactive, type InteractiveStatelessProps } from "@opal/core";
|
||||
import type { ContainerSizeVariants, ExtremaSizeVariants } from "@opal/types";
|
||||
import type { SizeVariant, WidthVariant } from "@opal/shared";
|
||||
import type { TooltipSide } from "@opal/components";
|
||||
import type { IconFunctionComponent } from "@opal/types";
|
||||
import * as TooltipPrimitive from "@radix-ui/react-tooltip";
|
||||
@@ -31,7 +31,7 @@ type ButtonProps = InteractiveStatelessProps &
|
||||
/**
|
||||
* Size preset — controls gap, text size, and Container height/rounding.
|
||||
*/
|
||||
size?: ContainerSizeVariants;
|
||||
size?: SizeVariant;
|
||||
|
||||
/** HTML button type. When provided, Container renders a `<button>` element. */
|
||||
type?: "submit" | "button" | "reset";
|
||||
@@ -40,7 +40,7 @@ type ButtonProps = InteractiveStatelessProps &
|
||||
tooltip?: string;
|
||||
|
||||
/** Width preset. `"fit"` shrink-wraps, `"full"` stretches to parent width. */
|
||||
width?: ExtremaSizeVariants;
|
||||
width?: WidthVariant;
|
||||
|
||||
/** Which side the tooltip appears on. */
|
||||
tooltipSide?: TooltipSide;
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import type { ContainerSizeVariants } from "@opal/types";
|
||||
import type { SizeVariant } from "@opal/shared";
|
||||
import type { IconFunctionComponent } from "@opal/types";
|
||||
import { cn } from "@opal/utils";
|
||||
|
||||
@@ -13,7 +13,7 @@ const iconVariants = {
|
||||
|
||||
function iconWrapper(
|
||||
Icon: IconFunctionComponent | undefined,
|
||||
size: ContainerSizeVariants,
|
||||
size: SizeVariant,
|
||||
includeSpacer: boolean
|
||||
) {
|
||||
const { padding: p, size: s } = iconVariants[size];
|
||||
|
||||
@@ -6,7 +6,7 @@ import {
|
||||
type InteractiveStatefulProps,
|
||||
InteractiveContainerRoundingVariant,
|
||||
} from "@opal/core";
|
||||
import type { ExtremaSizeVariants } from "@opal/types";
|
||||
import { type WidthVariant } from "@opal/shared";
|
||||
import type { TooltipSide } from "@opal/components";
|
||||
import type { DistributiveOmit } from "@opal/types";
|
||||
import type { ContentActionProps } from "@opal/layouts/content-action/components";
|
||||
@@ -51,7 +51,7 @@ type LineItemButtonOwnProps = {
|
||||
roundingVariant?: InteractiveContainerRoundingVariant;
|
||||
|
||||
/** Container width. @default "full" */
|
||||
width?: ExtremaSizeVariants;
|
||||
width?: WidthVariant;
|
||||
|
||||
/** HTML button type. @default "button" */
|
||||
type?: "submit" | "button" | "reset";
|
||||
|
||||
@@ -6,8 +6,7 @@ import {
|
||||
type InteractiveStatefulProps,
|
||||
type InteractiveStatefulInteraction,
|
||||
} from "@opal/core";
|
||||
import type { ContainerSizeVariants, ExtremaSizeVariants } from "@opal/types";
|
||||
import type { InteractiveContainerRoundingVariant } from "@opal/core";
|
||||
import type { SizeVariant, WidthVariant } from "@opal/shared";
|
||||
import type { TooltipSide } from "@opal/components";
|
||||
import type { IconFunctionComponent, IconProps } from "@opal/types";
|
||||
import { SvgChevronDownSmall } from "@opal/icons";
|
||||
@@ -64,10 +63,10 @@ type OpenButtonProps = Omit<InteractiveStatefulProps, "variant"> & {
|
||||
/**
|
||||
* Size preset — controls gap, text size, and Container height/rounding.
|
||||
*/
|
||||
size?: ContainerSizeVariants;
|
||||
size?: SizeVariant;
|
||||
|
||||
/** Width preset. */
|
||||
width?: ExtremaSizeVariants;
|
||||
width?: WidthVariant;
|
||||
|
||||
/**
|
||||
* Content justify mode. When `"between"`, icon+label group left and
|
||||
@@ -81,9 +80,6 @@ type OpenButtonProps = Omit<InteractiveStatefulProps, "variant"> & {
|
||||
|
||||
/** Which side the tooltip appears on. */
|
||||
tooltipSide?: TooltipSide;
|
||||
|
||||
/** Override the default rounding derived from `size`. */
|
||||
roundingVariant?: InteractiveContainerRoundingVariant;
|
||||
};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -99,7 +95,6 @@ function OpenButton({
|
||||
justifyContent,
|
||||
tooltip,
|
||||
tooltipSide = "top",
|
||||
roundingVariant: roundingVariantOverride,
|
||||
interaction,
|
||||
variant = "select-heavy",
|
||||
...statefulProps
|
||||
@@ -137,8 +132,7 @@ function OpenButton({
|
||||
heightVariant={size}
|
||||
widthVariant={width}
|
||||
roundingVariant={
|
||||
roundingVariantOverride ??
|
||||
(isLarge ? "default" : size === "2xs" ? "mini" : "compact")
|
||||
isLarge ? "default" : size === "2xs" ? "mini" : "compact"
|
||||
}
|
||||
>
|
||||
<div
|
||||
|
||||
@@ -5,7 +5,7 @@ import {
|
||||
useDisabled,
|
||||
type InteractiveStatefulProps,
|
||||
} from "@opal/core";
|
||||
import type { ContainerSizeVariants, ExtremaSizeVariants } from "@opal/types";
|
||||
import type { SizeVariant, WidthVariant } from "@opal/shared";
|
||||
import type { TooltipSide } from "@opal/components";
|
||||
import type { IconFunctionComponent } from "@opal/types";
|
||||
import * as TooltipPrimitive from "@radix-ui/react-tooltip";
|
||||
@@ -48,7 +48,7 @@ type SelectButtonProps = InteractiveStatefulProps &
|
||||
/**
|
||||
* Size preset — controls gap, text size, and Container height/rounding.
|
||||
*/
|
||||
size?: ContainerSizeVariants;
|
||||
size?: SizeVariant;
|
||||
|
||||
/** HTML button type. Container renders a `<button>` element. */
|
||||
type?: "submit" | "button" | "reset";
|
||||
@@ -57,7 +57,7 @@ type SelectButtonProps = InteractiveStatefulProps &
|
||||
tooltip?: string;
|
||||
|
||||
/** Width preset. `"fit"` shrink-wraps, `"full"` stretches to parent width. */
|
||||
width?: ExtremaSizeVariants;
|
||||
width?: WidthVariant;
|
||||
|
||||
/** Which side the tooltip appears on. */
|
||||
tooltipSide?: TooltipSide;
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import "@opal/components/cards/card/styles.css";
|
||||
import type { ContainerSizeVariants } from "@opal/types";
|
||||
import { containerSizeVariants } from "@opal/shared";
|
||||
import type { SizeVariant } from "@opal/shared";
|
||||
import { sizeVariants } from "@opal/shared";
|
||||
import { cn } from "@opal/utils";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -26,7 +26,7 @@ type CardProps = {
|
||||
*
|
||||
* @default "lg"
|
||||
*/
|
||||
sizeVariant?: ContainerSizeVariants;
|
||||
sizeVariant?: SizeVariant;
|
||||
|
||||
/**
|
||||
* Background fill intensity.
|
||||
@@ -59,7 +59,7 @@ type CardProps = {
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** Maps a size variant to a rounding class, mirroring the Button pattern. */
|
||||
const roundingForSize: Record<ContainerSizeVariants, string> = {
|
||||
const roundingForSize: Record<SizeVariant, string> = {
|
||||
lg: "rounded-12",
|
||||
md: "rounded-08",
|
||||
sm: "rounded-08",
|
||||
@@ -79,7 +79,7 @@ function Card({
|
||||
ref,
|
||||
children,
|
||||
}: CardProps) {
|
||||
const { padding } = containerSizeVariants[sizeVariant];
|
||||
const { padding } = sizeVariants[sizeVariant];
|
||||
const rounding = roundingForSize[sizeVariant];
|
||||
|
||||
return (
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { Card } from "@opal/components/cards/card/components";
|
||||
import { Content } from "@opal/layouts";
|
||||
import { SvgEmpty } from "@opal/icons";
|
||||
import type { ContainerSizeVariants } from "@opal/types";
|
||||
import type { SizeVariant } from "@opal/shared";
|
||||
import type { IconFunctionComponent } from "@opal/types";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -16,7 +16,7 @@ type EmptyMessageCardProps = {
|
||||
title: string;
|
||||
|
||||
/** Size preset controlling padding and rounding of the card. */
|
||||
sizeVariant?: ContainerSizeVariants;
|
||||
sizeVariant?: SizeVariant;
|
||||
|
||||
/** Ref forwarded to the root Card div. */
|
||||
ref?: React.Ref<HTMLDivElement>;
|
||||
|
||||
@@ -45,10 +45,3 @@ export {
|
||||
EmptyMessageCard,
|
||||
type EmptyMessageCardProps,
|
||||
} from "@opal/components/cards/empty-message-card/components";
|
||||
|
||||
/* Pagination */
|
||||
export {
|
||||
Pagination,
|
||||
type PaginationProps,
|
||||
type PaginationSize,
|
||||
} from "@opal/components/pagination/components";
|
||||
|
||||
@@ -1,239 +0,0 @@
|
||||
import type { Meta, StoryObj } from "@storybook/react";
|
||||
import { Pagination } from "@opal/components";
|
||||
import { useState } from "react";
|
||||
|
||||
const meta: Meta<typeof Pagination> = {
|
||||
title: "opal/components/Pagination",
|
||||
component: Pagination,
|
||||
tags: ["autodocs"],
|
||||
};
|
||||
|
||||
export default meta;
|
||||
type Story = StoryObj<typeof Pagination>;
|
||||
|
||||
// ===========================================================================
|
||||
// variant="simple"
|
||||
// ===========================================================================
|
||||
|
||||
export const Simple: Story = {
|
||||
args: {
|
||||
variant: "simple",
|
||||
currentPage: 3,
|
||||
totalPages: 10,
|
||||
},
|
||||
};
|
||||
|
||||
export const SimpleSmall: Story = {
|
||||
args: {
|
||||
variant: "simple",
|
||||
currentPage: 2,
|
||||
totalPages: 8,
|
||||
size: "sm",
|
||||
},
|
||||
};
|
||||
|
||||
export const SimpleWithUnits: Story = {
|
||||
args: {
|
||||
variant: "simple",
|
||||
currentPage: 1,
|
||||
totalPages: 5,
|
||||
units: "pages",
|
||||
},
|
||||
};
|
||||
|
||||
export const SimpleArrowsOnly: Story = {
|
||||
args: {
|
||||
variant: "simple",
|
||||
currentPage: 2,
|
||||
totalPages: 8,
|
||||
hidePages: true,
|
||||
},
|
||||
};
|
||||
|
||||
export const SimpleAllSizes: Story = {
|
||||
render: () => (
|
||||
<div className="flex flex-col gap-4 items-start">
|
||||
{(["lg", "md", "sm"] as const).map((size) => (
|
||||
<div key={size} className="flex flex-col gap-1">
|
||||
<span className="font-secondary-body text-text-03">
|
||||
size="{size}"
|
||||
</span>
|
||||
<Pagination
|
||||
variant="simple"
|
||||
currentPage={3}
|
||||
totalPages={10}
|
||||
size={size}
|
||||
/>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
),
|
||||
};
|
||||
|
||||
// ===========================================================================
|
||||
// variant="count"
|
||||
// ===========================================================================
|
||||
|
||||
export const Count: Story = {
|
||||
args: {
|
||||
variant: "count",
|
||||
pageSize: 10,
|
||||
totalItems: 95,
|
||||
currentPage: 2,
|
||||
totalPages: 10,
|
||||
},
|
||||
};
|
||||
|
||||
export const CountWithUnits: Story = {
|
||||
args: {
|
||||
variant: "count",
|
||||
pageSize: 25,
|
||||
totalItems: 203,
|
||||
currentPage: 1,
|
||||
totalPages: 9,
|
||||
units: "items",
|
||||
},
|
||||
};
|
||||
|
||||
export const CountArrowsOnly: Story = {
|
||||
args: {
|
||||
variant: "count",
|
||||
pageSize: 10,
|
||||
totalItems: 50,
|
||||
currentPage: 2,
|
||||
totalPages: 5,
|
||||
hidePages: true,
|
||||
},
|
||||
};
|
||||
|
||||
export const CountAllSizes: Story = {
|
||||
render: () => (
|
||||
<div className="flex flex-col gap-4 items-start">
|
||||
{(["lg", "md", "sm"] as const).map((size) => (
|
||||
<div key={size} className="flex flex-col gap-1">
|
||||
<span className="font-secondary-body text-text-03">
|
||||
size="{size}"
|
||||
</span>
|
||||
<Pagination
|
||||
variant="count"
|
||||
pageSize={10}
|
||||
totalItems={95}
|
||||
currentPage={3}
|
||||
totalPages={10}
|
||||
size={size}
|
||||
units="items"
|
||||
/>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
),
|
||||
};
|
||||
|
||||
// ===========================================================================
|
||||
// variant="list" (default)
|
||||
// ===========================================================================
|
||||
|
||||
export const List: Story = {
|
||||
args: {
|
||||
currentPage: 5,
|
||||
totalPages: 20,
|
||||
onChange: () => {},
|
||||
},
|
||||
};
|
||||
|
||||
export const ListFewPages: Story = {
|
||||
args: {
|
||||
currentPage: 2,
|
||||
totalPages: 4,
|
||||
onChange: () => {},
|
||||
},
|
||||
};
|
||||
|
||||
export const ListAllSizes: Story = {
|
||||
render: () => (
|
||||
<div className="flex flex-col gap-4 items-start">
|
||||
{(["lg", "md", "sm"] as const).map((size) => (
|
||||
<div key={size} className="flex flex-col gap-1">
|
||||
<span className="font-secondary-body text-text-03">
|
||||
size="{size}"
|
||||
</span>
|
||||
<Pagination
|
||||
currentPage={3}
|
||||
totalPages={10}
|
||||
onChange={() => {}}
|
||||
size={size}
|
||||
/>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
),
|
||||
};
|
||||
|
||||
// ===========================================================================
|
||||
// Interactive
|
||||
// ===========================================================================
|
||||
|
||||
function InteractiveSimpleDemo() {
|
||||
const [page, setPage] = useState(1);
|
||||
return (
|
||||
<div className="flex flex-col gap-4 items-start">
|
||||
<Pagination
|
||||
variant="simple"
|
||||
currentPage={page}
|
||||
totalPages={15}
|
||||
onChange={setPage}
|
||||
units="pages"
|
||||
/>
|
||||
<span className="font-secondary-body text-text-03">
|
||||
Current page: {page}
|
||||
</span>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
export const InteractiveSimple: Story = {
|
||||
render: () => <InteractiveSimpleDemo />,
|
||||
};
|
||||
|
||||
function InteractiveListDemo() {
|
||||
const [page, setPage] = useState(1);
|
||||
return (
|
||||
<div className="flex flex-col gap-4 items-start">
|
||||
<Pagination currentPage={page} totalPages={15} onChange={setPage} />
|
||||
<span className="font-secondary-body text-text-03">
|
||||
Current page: {page}
|
||||
</span>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
export const InteractiveList: Story = {
|
||||
render: () => <InteractiveListDemo />,
|
||||
};
|
||||
|
||||
function InteractiveCountDemo() {
|
||||
const [page, setPage] = useState(1);
|
||||
const pageSize = 10;
|
||||
const totalItems = 95;
|
||||
const totalPages = Math.ceil(totalItems / pageSize);
|
||||
return (
|
||||
<div className="flex flex-col gap-4 items-start">
|
||||
<Pagination
|
||||
variant="count"
|
||||
currentPage={page}
|
||||
totalPages={totalPages}
|
||||
pageSize={pageSize}
|
||||
totalItems={totalItems}
|
||||
onChange={setPage}
|
||||
units="items"
|
||||
/>
|
||||
<span className="font-secondary-body text-text-03">
|
||||
Current page: {page}
|
||||
</span>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
export const InteractiveCount: Story = {
|
||||
render: () => <InteractiveCountDemo />,
|
||||
};
|
||||
@@ -1,89 +0,0 @@
|
||||
# Pagination
|
||||
|
||||
**Import:** `import { Pagination, type PaginationProps } from "@opal/components";`
|
||||
|
||||
Page navigation with three display variants and prev/next arrow controls.
|
||||
|
||||
## Variants
|
||||
|
||||
### `"list"` (default)
|
||||
|
||||
Numbered page buttons with ellipsis truncation for large page counts.
|
||||
|
||||
```tsx
|
||||
<Pagination currentPage={3} totalPages={10} onChange={setPage} />
|
||||
```
|
||||
|
||||
### `"simple"`
|
||||
|
||||
Compact `currentPage/totalPages` display with prev/next arrows. Can be reduced to just arrows via `hidePages`.
|
||||
|
||||
```tsx
|
||||
// With summary (default)
|
||||
<Pagination variant="simple" currentPage={1} totalPages={5} onChange={setPage} />
|
||||
|
||||
// Arrows only
|
||||
<Pagination variant="simple" currentPage={1} totalPages={5} onChange={setPage} hidePages />
|
||||
|
||||
// With units
|
||||
<Pagination variant="simple" currentPage={1} totalPages={5} onChange={setPage} units="pages" />
|
||||
```
|
||||
|
||||
### `"count"`
|
||||
|
||||
Item-count display (`X~Y of Z`) with prev/next arrows. Designed for table footers.
|
||||
|
||||
```tsx
|
||||
// Basic
|
||||
<Pagination
|
||||
variant="count"
|
||||
pageSize={10}
|
||||
totalItems={95}
|
||||
currentPage={2}
|
||||
totalPages={10}
|
||||
onChange={setPage}
|
||||
/>
|
||||
|
||||
// With units
|
||||
<Pagination
|
||||
variant="count"
|
||||
pageSize={10}
|
||||
totalItems={95}
|
||||
currentPage={2}
|
||||
totalPages={10}
|
||||
onChange={setPage}
|
||||
units="items"
|
||||
/>
|
||||
```
|
||||
|
||||
## Props (shared)
|
||||
|
||||
| Prop | Type | Default | Description |
|
||||
|---|---|---|---|
|
||||
| `variant` | `"list" \| "simple" \| "count"` | `"list"` | Display variant |
|
||||
| `currentPage` | `number` | **(required)** | 1-based current page number |
|
||||
| `totalPages` | `number` | **(required)** | Total number of pages |
|
||||
| `onChange` | `(page: number) => void` | — | Called when the page changes |
|
||||
| `size` | `PaginationSize` | `"lg"` | Button and text sizing |
|
||||
|
||||
## Props (variant-specific)
|
||||
|
||||
### `"simple"`
|
||||
|
||||
| Prop | Type | Default | Description |
|
||||
|---|---|---|---|
|
||||
| `hidePages` | `boolean` | `false` | Hides the `currentPage/totalPages` text between arrows |
|
||||
| `units` | `string` | — | Label after the summary (e.g. `"pages"`), always 4px spacing |
|
||||
|
||||
### `"count"`
|
||||
|
||||
| Prop | Type | Default | Description |
|
||||
|---|---|---|---|
|
||||
| `pageSize` | `number` | **(required)** | Items per page (for range calculation) |
|
||||
| `totalItems` | `number` | **(required)** | Total item count |
|
||||
| `hidePages` | `boolean` | `false` | Hides the current page number between arrows |
|
||||
| `units` | `string` | — | Label after the total (e.g. `"items"`), always 4px spacing |
|
||||
|
||||
### `PaginationSize`
|
||||
|
||||
`"lg" | "md" | "sm"`
|
||||
@@ -1,537 +0,0 @@
|
||||
"use client";
|
||||
|
||||
import { Button } from "@opal/components";
|
||||
import { Disabled } from "@opal/core";
|
||||
import { SvgArrowRight, SvgChevronLeft, SvgChevronRight } from "@opal/icons";
|
||||
import { containerSizeVariants } from "@opal/shared";
|
||||
import type { WithoutStyles } from "@opal/types";
|
||||
import { cn } from "@opal/utils";
|
||||
import * as PopoverPrimitive from "@radix-ui/react-popover";
|
||||
import {
|
||||
useState,
|
||||
type ChangeEvent,
|
||||
type HTMLAttributes,
|
||||
type KeyboardEvent,
|
||||
type ReactNode,
|
||||
} from "react";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Types
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
type PaginationSize = "lg" | "md" | "sm";
|
||||
|
||||
/**
|
||||
* Compact `currentPage / totalPages` display with prev/next arrows.
|
||||
*/
|
||||
interface SimplePaginationProps
|
||||
extends Omit<WithoutStyles<HTMLAttributes<HTMLDivElement>>, "onChange"> {
|
||||
variant: "simple";
|
||||
/** The 1-based current page number. */
|
||||
currentPage: number;
|
||||
/** Total number of pages. */
|
||||
totalPages: number;
|
||||
/** Called when the page changes. */
|
||||
onChange?: (page: number) => void;
|
||||
/** Controls button and text sizing. Default: `"lg"`. */
|
||||
size?: PaginationSize;
|
||||
/** Hides the `currentPage/totalPages` summary text between arrows. Default: `false`. */
|
||||
hidePages?: boolean;
|
||||
/** Unit label shown after the summary (e.g. `"pages"`). Always has 4px spacing. */
|
||||
units?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Item-count display (`X~Y of Z`) with prev/next arrows.
|
||||
* Designed for table footers.
|
||||
*/
|
||||
interface CountPaginationProps
|
||||
extends Omit<WithoutStyles<HTMLAttributes<HTMLDivElement>>, "onChange"> {
|
||||
variant: "count";
|
||||
/** The 1-based current page number. */
|
||||
currentPage: number;
|
||||
/** Total number of pages. */
|
||||
totalPages: number;
|
||||
/** Number of items displayed per page. Used to compute the visible range. */
|
||||
pageSize: number;
|
||||
/** Total number of items across all pages. */
|
||||
totalItems: number;
|
||||
/** Called when the page changes. */
|
||||
onChange?: (page: number) => void;
|
||||
/** Controls button and text sizing. Default: `"lg"`. */
|
||||
size?: PaginationSize;
|
||||
/** Hides the current page number between the arrows. Default: `false`. */
|
||||
hidePages?: boolean;
|
||||
/** Unit label shown after the total count (e.g. `"items"`). Always has 4px spacing. */
|
||||
units?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Numbered page buttons with ellipsis truncation for large page counts.
|
||||
* This is the default variant.
|
||||
*/
|
||||
interface ListPaginationProps
|
||||
extends Omit<WithoutStyles<HTMLAttributes<HTMLDivElement>>, "onChange"> {
|
||||
variant?: "list";
|
||||
/** The 1-based current page number. */
|
||||
currentPage: number;
|
||||
/** Total number of pages. */
|
||||
totalPages: number;
|
||||
/** Called when the page changes. */
|
||||
onChange: (page: number) => void;
|
||||
/** Controls button and text sizing. Default: `"lg"`. */
|
||||
size?: PaginationSize;
|
||||
}
|
||||
|
||||
/**
|
||||
* Discriminated union of all pagination variants.
|
||||
* Use `variant` to select between `"simple"`, `"count"`, and `"list"` (default).
|
||||
*/
|
||||
type PaginationProps =
|
||||
| SimplePaginationProps
|
||||
| CountPaginationProps
|
||||
| ListPaginationProps;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Computes the page numbers to display.
|
||||
*
|
||||
* - <=7 pages: render all pages individually (no ellipsis).
|
||||
* - >7 pages: always render exactly 7 slots (numbers or ellipsis).
|
||||
* First and last page are always shown. Ellipsis takes one slot.
|
||||
*
|
||||
* Examples for totalPages=20:
|
||||
* - page 1: `1 2 3 4 5 ... 20`
|
||||
* - page 4: `1 2 3 4 5 ... 20`
|
||||
* - page 5: `1 ... 4 5 6 ... 20`
|
||||
* - page 16: `1 ... 15 16 17 ... 20`
|
||||
* - page 17: `1 ... 16 17 18 19 20`
|
||||
* - page 20: `1 ... 16 17 18 19 20`
|
||||
*/
|
||||
function getPageNumbers(
|
||||
currentPage: number,
|
||||
totalPages: number
|
||||
): (number | string)[] {
|
||||
if (totalPages <= 7) {
|
||||
const pages: number[] = [];
|
||||
for (let i = 1; i <= totalPages; i++) pages.push(i);
|
||||
return pages;
|
||||
}
|
||||
|
||||
// Always 7 slots. First and last are always page 1 and totalPages.
|
||||
// That leaves 5 inner slots.
|
||||
|
||||
// Near the start: no start-ellipsis needed
|
||||
// Slots: 1, 2, 3, 4, 5, ..., totalPages
|
||||
if (currentPage <= 4) {
|
||||
return [1, 2, 3, 4, 5, "end-ellipsis", totalPages];
|
||||
}
|
||||
|
||||
// Near the end: no end-ellipsis needed
|
||||
// Slots: 1, ..., tp-4, tp-3, tp-2, tp-1, tp
|
||||
if (currentPage >= totalPages - 3) {
|
||||
return [
|
||||
1,
|
||||
"start-ellipsis",
|
||||
totalPages - 4,
|
||||
totalPages - 3,
|
||||
totalPages - 2,
|
||||
totalPages - 1,
|
||||
totalPages,
|
||||
];
|
||||
}
|
||||
|
||||
// Middle: both ellipses
|
||||
// Slots: 1, ..., cur-1, cur, cur+1, ..., totalPages
|
||||
return [
|
||||
1,
|
||||
"start-ellipsis",
|
||||
currentPage - 1,
|
||||
currentPage,
|
||||
currentPage + 1,
|
||||
"end-ellipsis",
|
||||
totalPages,
|
||||
];
|
||||
}
|
||||
|
||||
function monoClass(size: PaginationSize): string {
|
||||
return size === "sm" ? "font-secondary-mono" : "font-main-ui-mono";
|
||||
}
|
||||
|
||||
function textClasses(size: PaginationSize, style: "mono" | "muted"): string {
|
||||
if (style === "mono") return monoClass(size);
|
||||
return size === "sm" ? "font-secondary-body" : "font-main-ui-muted";
|
||||
}
|
||||
|
||||
const PAGE_NUMBER_FONT: Record<
|
||||
PaginationSize,
|
||||
{ active: string; inactive: string }
|
||||
> = {
|
||||
lg: {
|
||||
active: "font-main-ui-body text-text-04",
|
||||
inactive: "font-main-ui-muted text-text-02",
|
||||
},
|
||||
md: {
|
||||
active: "font-secondary-action text-text-04",
|
||||
inactive: "font-secondary-body text-text-02",
|
||||
},
|
||||
sm: {
|
||||
active: "font-secondary-action text-text-04",
|
||||
inactive: "font-secondary-body text-text-02",
|
||||
},
|
||||
};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// GoToPagePopup
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
interface GoToPagePopupProps {
|
||||
totalPages: number;
|
||||
onSubmit: (page: number) => void;
|
||||
children: ReactNode;
|
||||
}
|
||||
|
||||
function GoToPagePopup({ totalPages, onSubmit, children }: GoToPagePopupProps) {
|
||||
const [open, setOpen] = useState(false);
|
||||
const [value, setValue] = useState("");
|
||||
|
||||
const parsed = parseInt(value, 10);
|
||||
const isValid = !isNaN(parsed) && parsed >= 1 && parsed <= totalPages;
|
||||
|
||||
function handleChange(e: ChangeEvent<HTMLInputElement>) {
|
||||
const raw = e.target.value;
|
||||
if (raw === "" || /^\d+$/.test(raw)) {
|
||||
setValue(raw);
|
||||
}
|
||||
}
|
||||
|
||||
function handleSubmit() {
|
||||
if (!isValid) return;
|
||||
onSubmit(parsed);
|
||||
setOpen(false);
|
||||
setValue("");
|
||||
}
|
||||
|
||||
function handleKeyDown(e: KeyboardEvent<HTMLInputElement>) {
|
||||
if (e.key === "Enter") {
|
||||
handleSubmit();
|
||||
}
|
||||
}
|
||||
|
||||
return (
|
||||
<PopoverPrimitive.Root
|
||||
open={open}
|
||||
onOpenChange={(next) => {
|
||||
setOpen(next);
|
||||
if (!next) setValue("");
|
||||
}}
|
||||
>
|
||||
<PopoverPrimitive.Trigger asChild>{children}</PopoverPrimitive.Trigger>
|
||||
<PopoverPrimitive.Portal>
|
||||
<PopoverPrimitive.Content
|
||||
className={cn(
|
||||
"flex items-center gap-1 p-1",
|
||||
"bg-background-neutral-00 rounded-12 border border-border-01 shadow-md z-popover",
|
||||
"data-[state=open]:animate-in data-[state=closed]:animate-out",
|
||||
"data-[state=closed]:fade-out-0 data-[state=open]:fade-in-0",
|
||||
"data-[state=closed]:zoom-out-95 data-[state=open]:zoom-in-95"
|
||||
)}
|
||||
sideOffset={4}
|
||||
>
|
||||
{/* TODO(@raunakab): migrate this input to the opal Input component once inputs have been migrated into Opal */}
|
||||
<input
|
||||
type="text"
|
||||
inputMode="numeric"
|
||||
value={value}
|
||||
onChange={handleChange}
|
||||
onKeyDown={handleKeyDown}
|
||||
placeholder="Go to page"
|
||||
autoFocus
|
||||
className={cn(
|
||||
"w-[7rem] bg-transparent px-1.5 py-1 rounded-08",
|
||||
containerSizeVariants.lg.height,
|
||||
"border border-border-02 focus:outline-none focus:border-border-04",
|
||||
"font-main-ui-body",
|
||||
"text-text-04 placeholder:text-text-02"
|
||||
)}
|
||||
/>
|
||||
<Disabled disabled={!isValid}>
|
||||
<Button
|
||||
icon={SvgArrowRight}
|
||||
size="lg"
|
||||
onClick={handleSubmit}
|
||||
tooltip="Go to page"
|
||||
/>
|
||||
</Disabled>
|
||||
</PopoverPrimitive.Content>
|
||||
</PopoverPrimitive.Portal>
|
||||
</PopoverPrimitive.Root>
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Nav buttons (shared across all variants)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
interface NavButtonsProps {
|
||||
currentPage: number;
|
||||
totalPages: number;
|
||||
onChange: (page: number) => void;
|
||||
size: PaginationSize;
|
||||
children?: ReactNode;
|
||||
}
|
||||
|
||||
function NavButtons({
|
||||
currentPage,
|
||||
totalPages,
|
||||
onChange,
|
||||
size,
|
||||
children,
|
||||
}: NavButtonsProps) {
|
||||
return (
|
||||
<>
|
||||
<Disabled disabled={currentPage <= 1}>
|
||||
<Button
|
||||
icon={SvgChevronLeft}
|
||||
onClick={() => onChange(Math.max(1, currentPage - 1))}
|
||||
size={size}
|
||||
prominence="tertiary"
|
||||
tooltip="Previous page"
|
||||
/>
|
||||
</Disabled>
|
||||
{children}
|
||||
<Disabled disabled={currentPage >= totalPages}>
|
||||
<Button
|
||||
icon={SvgChevronRight}
|
||||
onClick={() => onChange(Math.min(totalPages, currentPage + 1))}
|
||||
size={size}
|
||||
prominence="tertiary"
|
||||
tooltip="Next page"
|
||||
/>
|
||||
</Disabled>
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// PaginationSimple
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function PaginationSimple({
|
||||
currentPage,
|
||||
totalPages,
|
||||
onChange,
|
||||
size = "lg",
|
||||
hidePages = false,
|
||||
units,
|
||||
...props
|
||||
}: SimplePaginationProps) {
|
||||
const handleChange = (page: number) => onChange?.(page);
|
||||
|
||||
const label = `${currentPage}/${totalPages}${units ? ` ${units}` : ""}`;
|
||||
|
||||
return (
|
||||
<div {...props} className="flex items-center">
|
||||
<NavButtons
|
||||
currentPage={currentPage}
|
||||
totalPages={totalPages}
|
||||
onChange={handleChange}
|
||||
size={size}
|
||||
>
|
||||
{!hidePages && (
|
||||
<GoToPagePopup totalPages={totalPages} onSubmit={handleChange}>
|
||||
<Button size={size} prominence="tertiary">
|
||||
{label}
|
||||
</Button>
|
||||
</GoToPagePopup>
|
||||
)}
|
||||
</NavButtons>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// PaginationCount
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function PaginationCount({
|
||||
pageSize,
|
||||
totalItems,
|
||||
currentPage,
|
||||
totalPages,
|
||||
onChange,
|
||||
size = "lg",
|
||||
hidePages = false,
|
||||
units,
|
||||
...props
|
||||
}: CountPaginationProps) {
|
||||
const handleChange = (page: number) => onChange?.(page);
|
||||
const rangeStart = totalItems === 0 ? 0 : (currentPage - 1) * pageSize + 1;
|
||||
const rangeEnd = Math.min(currentPage * pageSize, totalItems);
|
||||
|
||||
return (
|
||||
<div {...props} className="flex items-center gap-1">
|
||||
{/* Summary: range of total [units] */}
|
||||
<span
|
||||
className={cn(
|
||||
"inline-flex items-center gap-1",
|
||||
monoClass(size),
|
||||
"text-text-03"
|
||||
)}
|
||||
>
|
||||
{rangeStart}~{rangeEnd}
|
||||
<span className={textClasses(size, "muted")}>of</span>
|
||||
{totalItems}
|
||||
{units && <span className="ml-1">{units}</span>}
|
||||
</span>
|
||||
|
||||
{/* Buttons: < [page] > */}
|
||||
<div className="flex items-center">
|
||||
<NavButtons
|
||||
currentPage={currentPage}
|
||||
totalPages={totalPages}
|
||||
onChange={handleChange}
|
||||
size={size}
|
||||
>
|
||||
{!hidePages && (
|
||||
<GoToPagePopup totalPages={totalPages} onSubmit={handleChange}>
|
||||
<Button size={size} prominence="tertiary">
|
||||
{String(currentPage)}
|
||||
</Button>
|
||||
</GoToPagePopup>
|
||||
)}
|
||||
</NavButtons>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// PaginationList (default)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function PaginationList({
|
||||
currentPage,
|
||||
totalPages,
|
||||
onChange,
|
||||
size = "lg",
|
||||
...props
|
||||
}: ListPaginationProps) {
|
||||
const pageNumbers = getPageNumbers(currentPage, totalPages);
|
||||
const fonts = PAGE_NUMBER_FONT[size];
|
||||
|
||||
return (
|
||||
<div {...props} className="flex items-center gap-1">
|
||||
<NavButtons
|
||||
currentPage={currentPage}
|
||||
totalPages={totalPages}
|
||||
onChange={onChange}
|
||||
size={size}
|
||||
>
|
||||
<div className="flex items-center">
|
||||
{pageNumbers.map((page) => {
|
||||
if (typeof page === "string") {
|
||||
return (
|
||||
<GoToPagePopup
|
||||
key={page}
|
||||
totalPages={totalPages}
|
||||
onSubmit={onChange}
|
||||
>
|
||||
<Button
|
||||
size={size}
|
||||
prominence="tertiary"
|
||||
icon={({ className: iconClassName }) => (
|
||||
<div
|
||||
className={cn(
|
||||
iconClassName,
|
||||
"flex flex-col justify-center",
|
||||
fonts.inactive
|
||||
)}
|
||||
>
|
||||
...
|
||||
</div>
|
||||
)}
|
||||
/>
|
||||
</GoToPagePopup>
|
||||
);
|
||||
}
|
||||
|
||||
const isActive = page === currentPage;
|
||||
|
||||
return (
|
||||
<Button
|
||||
key={page}
|
||||
onClick={() => onChange(page)}
|
||||
size={size}
|
||||
prominence="tertiary"
|
||||
interaction={isActive ? "hover" : "rest"}
|
||||
icon={({ className: iconClassName }) => (
|
||||
<div
|
||||
className={cn(
|
||||
iconClassName,
|
||||
"flex flex-col justify-center",
|
||||
isActive ? fonts.active : fonts.inactive
|
||||
)}
|
||||
>
|
||||
{page}
|
||||
</div>
|
||||
)}
|
||||
/>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
</NavButtons>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Pagination (entry point)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Page navigation component with three variants:
|
||||
*
|
||||
* - `"list"` (default) — Numbered page buttons with ellipsis truncation.
|
||||
* - `"simple"` — Compact `currentPage / totalPages` with prev/next arrows.
|
||||
* - `"count"` — Item-count display (`X~Y of Z`) with prev/next arrows.
|
||||
*
|
||||
* All variants include a "go to page" popup activated by clicking on the
|
||||
* page indicator (simple/count) or the ellipsis (list).
|
||||
*
|
||||
* @example
|
||||
* ```tsx
|
||||
* // List (default)
|
||||
* <Pagination currentPage={3} totalPages={10} onChange={setPage} />
|
||||
*
|
||||
* // Simple
|
||||
* <Pagination variant="simple" currentPage={1} totalPages={5} onChange={setPage} />
|
||||
*
|
||||
* // Count
|
||||
* <Pagination variant="count" pageSize={10} totalItems={95} currentPage={2} totalPages={10} onChange={setPage} />
|
||||
* ```
|
||||
*/
|
||||
function Pagination(props: PaginationProps) {
|
||||
const normalized = {
|
||||
...props,
|
||||
totalPages: Math.max(1, props.totalPages),
|
||||
currentPage: Math.max(
|
||||
1,
|
||||
Math.min(props.currentPage, Math.max(1, props.totalPages))
|
||||
),
|
||||
};
|
||||
const variant = normalized.variant ?? "list";
|
||||
switch (variant) {
|
||||
case "simple":
|
||||
return <PaginationSimple {...(normalized as SimplePaginationProps)} />;
|
||||
case "count":
|
||||
return <PaginationCount {...(normalized as CountPaginationProps)} />;
|
||||
case "list":
|
||||
return <PaginationList {...(normalized as ListPaginationProps)} />;
|
||||
}
|
||||
}
|
||||
|
||||
export { Pagination, type PaginationProps, type PaginationSize };
|
||||
@@ -1,8 +1,8 @@
|
||||
import "@opal/core/animations/styles.css";
|
||||
import React, { createContext, useContext, useState, useCallback } from "react";
|
||||
import { cn } from "@opal/utils";
|
||||
import type { WithoutStyles, ExtremaSizeVariants } from "@opal/types";
|
||||
import { widthVariants } from "@opal/shared";
|
||||
import type { WithoutStyles } from "@opal/types";
|
||||
import { widthVariants, type WidthVariant } from "@opal/shared";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Context-per-group registry
|
||||
@@ -40,7 +40,7 @@ interface HoverableRootProps
|
||||
children: React.ReactNode;
|
||||
group: string;
|
||||
/** Width preset. @default "auto" */
|
||||
widthVariant?: ExtremaSizeVariants;
|
||||
widthVariant?: WidthVariant;
|
||||
/** Ref forwarded to the root `<div>`. */
|
||||
ref?: React.Ref<HTMLDivElement>;
|
||||
}
|
||||
@@ -84,7 +84,7 @@ interface HoverableItemProps
|
||||
function HoverableRoot({
|
||||
group,
|
||||
children,
|
||||
widthVariant = "full",
|
||||
widthVariant = "auto",
|
||||
ref,
|
||||
onMouseEnter: consumerMouseEnter,
|
||||
onMouseLeave: consumerMouseLeave,
|
||||
|
||||
@@ -5,10 +5,10 @@ import React from "react";
|
||||
import { cn } from "@opal/utils";
|
||||
import type { WithoutStyles } from "@opal/types";
|
||||
import {
|
||||
containerSizeVariants,
|
||||
type ContainerSizeVariants,
|
||||
sizeVariants,
|
||||
type SizeVariant,
|
||||
widthVariants,
|
||||
type ExtremaSizeVariants,
|
||||
type WidthVariant,
|
||||
} from "@opal/shared";
|
||||
import { useDisabled } from "@opal/core/disabled/components";
|
||||
|
||||
@@ -73,14 +73,14 @@ interface InteractiveContainerProps
|
||||
*
|
||||
* @default "lg"
|
||||
*/
|
||||
heightVariant?: ContainerSizeVariants;
|
||||
heightVariant?: SizeVariant;
|
||||
|
||||
/**
|
||||
* Width preset controlling the container's horizontal size.
|
||||
*
|
||||
* @default "fit"
|
||||
*/
|
||||
widthVariant?: ExtremaSizeVariants;
|
||||
widthVariant?: WidthVariant;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -119,7 +119,7 @@ function InteractiveContainer({
|
||||
target?: string;
|
||||
rel?: string;
|
||||
};
|
||||
const { height, minWidth, padding } = containerSizeVariants[heightVariant];
|
||||
const { height, minWidth, padding } = sizeVariants[heightVariant];
|
||||
const sharedProps = {
|
||||
...rest,
|
||||
className: cn(
|
||||
|
||||
@@ -1,20 +0,0 @@
|
||||
import type { IconProps } from "@opal/types";
|
||||
const SvgCurate = ({ size, ...props }: IconProps) => (
|
||||
<svg
|
||||
width={size}
|
||||
height={size}
|
||||
viewBox="0 0 16 16"
|
||||
fill="none"
|
||||
xmlns="http://www.w3.org/2000/svg"
|
||||
stroke="currentColor"
|
||||
{...props}
|
||||
>
|
||||
<path
|
||||
d="M8 9L8 14.5M8 9C7.35971 8.35971 6.9055 8 6 8H2.5L2.5 13.5H6C6.9055 13.5 7.35971 13.8597 8 14.5M8 9C8.64029 8.35971 9.09449 8 10 8H13.5L13.5 13.5H10C9.09449 13.5 8.64029 13.8597 8 14.5M10.25 3.75C10.25 4.99264 9.24264 6 8 6C6.75736 6 5.75 4.99264 5.75 3.75C5.75 2.50736 6.75736 1.5 8 1.5C9.24264 1.5 10.25 2.50736 10.25 3.75Z"
|
||||
strokeWidth={1.5}
|
||||
strokeLinecap="round"
|
||||
strokeLinejoin="round"
|
||||
/>
|
||||
</svg>
|
||||
);
|
||||
export default SvgCurate;
|
||||
@@ -54,7 +54,6 @@ export { default as SvgColumn } from "@opal/icons/column";
|
||||
export { default as SvgCopy } from "@opal/icons/copy";
|
||||
export { default as SvgCornerRightUpDot } from "@opal/icons/corner-right-up-dot";
|
||||
export { default as SvgCpu } from "@opal/icons/cpu";
|
||||
export { default as SvgCurate } from "@opal/icons/curate";
|
||||
export { default as SvgCreditCard } from "@opal/icons/credit-card";
|
||||
export { default as SvgDashboard } from "@opal/icons/dashboard";
|
||||
export { default as SvgDevKit } from "@opal/icons/dev-kit";
|
||||
@@ -136,7 +135,6 @@ export { default as SvgPlayCircle } from "@opal/icons/play-circle";
|
||||
export { default as SvgPlug } from "@opal/icons/plug";
|
||||
export { default as SvgPlus } from "@opal/icons/plus";
|
||||
export { default as SvgPlusCircle } from "@opal/icons/plus-circle";
|
||||
export { default as SvgProgressBars } from "@opal/icons/progress-bars";
|
||||
export { default as SvgProgressCircle } from "@opal/icons/progress-circle";
|
||||
export { default as SvgQuestionMarkSmall } from "@opal/icons/question-mark-small";
|
||||
export { default as SvgQuoteEnd } from "@opal/icons/quote-end";
|
||||
@@ -178,16 +176,9 @@ export { default as SvgTwoLineSmall } from "@opal/icons/two-line-small";
|
||||
export { default as SvgUnplug } from "@opal/icons/unplug";
|
||||
export { default as SvgUploadCloud } from "@opal/icons/upload-cloud";
|
||||
export { default as SvgUser } from "@opal/icons/user";
|
||||
export { default as SvgUserCheck } from "@opal/icons/user-check";
|
||||
export { default as SvgUserEdit } from "@opal/icons/user-edit";
|
||||
export { default as SvgUserKey } from "@opal/icons/user-key";
|
||||
export { default as SvgUserManage } from "@opal/icons/user-manage";
|
||||
export { default as SvgUserMinus } from "@opal/icons/user-minus";
|
||||
export { default as SvgUserPlus } from "@opal/icons/user-plus";
|
||||
export { default as SvgUserShield } from "@opal/icons/user-shield";
|
||||
export { default as SvgUserSpeaker } from "@opal/icons/user-speaker";
|
||||
export { default as SvgUserSync } from "@opal/icons/user-sync";
|
||||
export { default as SvgUserX } from "@opal/icons/user-x";
|
||||
export { default as SvgUsers } from "@opal/icons/users";
|
||||
export { default as SvgVolume } from "@opal/icons/volume";
|
||||
export { default as SvgVolumeOff } from "@opal/icons/volume-off";
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user