mirror of
https://github.com/onyx-dot-app/onyx.git
synced 2026-04-04 06:22:44 +00:00
Compare commits
28 Commits
cli/v0.2.0
...
temp/pr-98
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
569fef22b4 | ||
|
|
517d1035e0 | ||
|
|
ad108dd573 | ||
|
|
adf5691b5f | ||
|
|
c1a8a5bd83 | ||
|
|
8fd486da99 | ||
|
|
4bda4d3637 | ||
|
|
13c25eadad | ||
|
|
1f244e6388 | ||
|
|
18b0416d30 | ||
|
|
4bc0bc1efb | ||
|
|
1555217061 | ||
|
|
d177a833f0 | ||
|
|
086997d3c5 | ||
|
|
dccec78397 | ||
|
|
0123133621 | ||
|
|
0b9d154a73 | ||
|
|
6e65e55bf5 | ||
|
|
3f9e208759 | ||
|
|
fb8edda14a | ||
|
|
2a5810a44a | ||
|
|
58decd8a6b | ||
|
|
e97204d9cc | ||
|
|
44ab02c94f | ||
|
|
a98cc30f25 | ||
|
|
a709dcb8fa | ||
|
|
a3dfe6aa1b | ||
|
|
23e4d55fb1 |
6
.github/workflows/deployment.yml
vendored
6
.github/workflows/deployment.yml
vendored
@@ -704,6 +704,9 @@ jobs:
|
||||
NEXT_PUBLIC_FORGOT_PASSWORD_ENABLED=true
|
||||
NEXT_PUBLIC_INCLUDE_ERROR_POPUP_SUPPORT_LINK=true
|
||||
NODE_OPTIONS=--max-old-space-size=8192
|
||||
SENTRY_RELEASE=${{ github.sha }}
|
||||
secrets: |
|
||||
sentry_auth_token=${{ secrets.SENTRY_AUTH_TOKEN }}
|
||||
cache-from: |
|
||||
type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:cloudweb-cache-amd64
|
||||
type=registry,ref=${{ env.REGISTRY_IMAGE }}:latest
|
||||
@@ -786,6 +789,9 @@ jobs:
|
||||
NEXT_PUBLIC_FORGOT_PASSWORD_ENABLED=true
|
||||
NEXT_PUBLIC_INCLUDE_ERROR_POPUP_SUPPORT_LINK=true
|
||||
NODE_OPTIONS=--max-old-space-size=8192
|
||||
SENTRY_RELEASE=${{ github.sha }}
|
||||
secrets: |
|
||||
sentry_auth_token=${{ secrets.SENTRY_AUTH_TOKEN }}
|
||||
cache-from: |
|
||||
type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:cloudweb-cache-arm64
|
||||
type=registry,ref=${{ env.REGISTRY_IMAGE }}:latest
|
||||
|
||||
@@ -35,6 +35,7 @@ jobs:
|
||||
needs: [provider-chat-test]
|
||||
if: failure() && github.event_name == 'schedule'
|
||||
runs-on: ubuntu-slim
|
||||
environment: ci-protected
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
- name: Checkout
|
||||
|
||||
@@ -183,6 +183,7 @@ jobs:
|
||||
- cherry-pick-to-latest-release
|
||||
if: needs.resolve-cherry-pick-request.outputs.should_cherrypick == 'true' && needs.resolve-cherry-pick-request.result == 'success' && needs.cherry-pick-to-latest-release.result == 'success'
|
||||
runs-on: ubuntu-slim
|
||||
environment: ci-protected
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- name: Checkout
|
||||
@@ -232,6 +233,7 @@ jobs:
|
||||
- cherry-pick-to-latest-release
|
||||
if: always() && needs.resolve-cherry-pick-request.outputs.should_cherrypick == 'true' && (needs.resolve-cherry-pick-request.result == 'failure' || needs.cherry-pick-to-latest-release.result == 'failure')
|
||||
runs-on: ubuntu-slim
|
||||
environment: ci-protected
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- name: Checkout
|
||||
|
||||
2
.github/workflows/pr-desktop-build.yml
vendored
2
.github/workflows/pr-desktop-build.yml
vendored
@@ -63,7 +63,7 @@ jobs:
|
||||
targets: ${{ matrix.target }}
|
||||
|
||||
- name: Cache Cargo registry and build
|
||||
uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # zizmor: ignore[cache-poisoning]
|
||||
uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # zizmor: ignore[cache-poisoning]
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/bin/
|
||||
|
||||
2
.github/workflows/pr-helm-chart-testing.yml
vendored
2
.github/workflows/pr-helm-chart-testing.yml
vendored
@@ -41,7 +41,7 @@ jobs:
|
||||
version: v3.19.0
|
||||
|
||||
- name: Set up chart-testing
|
||||
uses: helm/chart-testing-action@b5eebdd9998021f29756c53432f48dab66394810
|
||||
uses: helm/chart-testing-action@2e2940618cb426dce2999631d543b53cdcfc8527
|
||||
with:
|
||||
uv_version: "0.9.9"
|
||||
|
||||
|
||||
4
.github/workflows/pr-playwright-tests.yml
vendored
4
.github/workflows/pr-playwright-tests.yml
vendored
@@ -284,7 +284,7 @@ jobs:
|
||||
|
||||
- name: Cache playwright cache
|
||||
# zizmor: ignore[cache-poisoning] ephemeral runners; no release artifacts
|
||||
uses: runs-on/cache@50350ad4242587b6c8c2baa2e740b1bc11285ff4 # ratchet:runs-on/cache@v4
|
||||
uses: runs-on/cache@a5f51d6f3fece787d03b7b4e981c82538a0654ed # ratchet:runs-on/cache@v4
|
||||
with:
|
||||
path: ~/.cache/ms-playwright
|
||||
key: ${{ runner.os }}-playwright-npm-${{ hashFiles('web/package-lock.json') }}
|
||||
@@ -626,7 +626,7 @@ jobs:
|
||||
|
||||
- name: Cache playwright cache
|
||||
# zizmor: ignore[cache-poisoning] ephemeral runners; no release artifacts
|
||||
uses: runs-on/cache@50350ad4242587b6c8c2baa2e740b1bc11285ff4 # ratchet:runs-on/cache@v4
|
||||
uses: runs-on/cache@a5f51d6f3fece787d03b7b4e981c82538a0654ed # ratchet:runs-on/cache@v4
|
||||
with:
|
||||
path: ~/.cache/ms-playwright
|
||||
key: ${{ runner.os }}-playwright-npm-${{ hashFiles('web/package-lock.json') }}
|
||||
|
||||
2
.github/workflows/pr-python-checks.yml
vendored
2
.github/workflows/pr-python-checks.yml
vendored
@@ -56,7 +56,7 @@ jobs:
|
||||
|
||||
- name: Cache mypy cache
|
||||
if: ${{ vars.DISABLE_MYPY_CACHE != 'true' }}
|
||||
uses: runs-on/cache@50350ad4242587b6c8c2baa2e740b1bc11285ff4 # ratchet:runs-on/cache@v4
|
||||
uses: runs-on/cache@a5f51d6f3fece787d03b7b4e981c82538a0654ed # ratchet:runs-on/cache@v4
|
||||
with:
|
||||
path: .mypy_cache
|
||||
key: mypy-${{ runner.os }}-${{ github.base_ref || github.event.merge_group.base_ref || 'main' }}-${{ hashFiles('**/*.py', '**/*.pyi', 'pyproject.toml') }}
|
||||
|
||||
1
.github/workflows/pr-python-model-tests.yml
vendored
1
.github/workflows/pr-python-model-tests.yml
vendored
@@ -31,6 +31,7 @@ jobs:
|
||||
- runner=4cpu-linux-arm64
|
||||
- "run-id=${{ github.run_id }}-model-check"
|
||||
- "extras=ecr-cache"
|
||||
environment: ci-protected
|
||||
timeout-minutes: 45
|
||||
|
||||
env:
|
||||
|
||||
1
.github/workflows/preview.yml
vendored
1
.github/workflows/preview.yml
vendored
@@ -15,6 +15,7 @@ permissions:
|
||||
jobs:
|
||||
Deploy-Preview:
|
||||
runs-on: ubuntu-latest
|
||||
environment: ci-protected
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
|
||||
|
||||
2
.github/workflows/storybook-deploy.yml
vendored
2
.github/workflows/storybook-deploy.yml
vendored
@@ -25,6 +25,7 @@ permissions:
|
||||
jobs:
|
||||
Deploy-Storybook:
|
||||
runs-on: ubuntu-latest
|
||||
environment: ci-protected
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # ratchet:actions/checkout@v4
|
||||
@@ -54,6 +55,7 @@ jobs:
|
||||
needs: Deploy-Storybook
|
||||
if: always() && needs.Deploy-Storybook.result == 'failure'
|
||||
runs-on: ubuntu-latest
|
||||
environment: ci-protected
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # ratchet:actions/checkout@v4
|
||||
|
||||
1
.github/workflows/sync_foss.yml
vendored
1
.github/workflows/sync_foss.yml
vendored
@@ -9,6 +9,7 @@ on:
|
||||
jobs:
|
||||
sync-foss:
|
||||
runs-on: ubuntu-latest
|
||||
environment: ci-protected
|
||||
timeout-minutes: 45
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
1
.github/workflows/tag-nightly.yml
vendored
1
.github/workflows/tag-nightly.yml
vendored
@@ -11,6 +11,7 @@ permissions:
|
||||
jobs:
|
||||
create-and-push-tag:
|
||||
runs-on: ubuntu-slim
|
||||
environment: ci-protected
|
||||
timeout-minutes: 45
|
||||
|
||||
steps:
|
||||
|
||||
@@ -13,6 +13,7 @@ from redis.lock import Lock as RedisLock
|
||||
from ee.onyx.server.tenants.provisioning import setup_tenant
|
||||
from ee.onyx.server.tenants.schema_management import create_schema_if_not_exists
|
||||
from ee.onyx.server.tenants.schema_management import get_current_alembic_version
|
||||
from ee.onyx.server.tenants.schema_management import run_alembic_migrations
|
||||
from onyx.background.celery.apps.app_base import task_logger
|
||||
from onyx.configs.app_configs import TARGET_AVAILABLE_TENANTS
|
||||
from onyx.configs.constants import ONYX_CLOUD_TENANT_ID
|
||||
@@ -29,9 +30,10 @@ from shared_configs.configs import TENANT_ID_PREFIX
|
||||
# Each tenant takes ~80s (alembic migrations), so 5 tenants ≈ 7 minutes.
|
||||
_MAX_TENANTS_PER_RUN = 5
|
||||
|
||||
# Time limits sized for worst-case batch: _MAX_TENANTS_PER_RUN × ~90s + buffer.
|
||||
_TENANT_PROVISIONING_SOFT_TIME_LIMIT = 60 * 10 # 10 minutes
|
||||
_TENANT_PROVISIONING_TIME_LIMIT = 60 * 15 # 15 minutes
|
||||
# Time limits sized for worst-case: provisioning up to _MAX_TENANTS_PER_RUN new tenants
|
||||
# (~90s each) plus migrating up to TARGET_AVAILABLE_TENANTS pool tenants (~90s each).
|
||||
_TENANT_PROVISIONING_SOFT_TIME_LIMIT = 60 * 20 # 20 minutes
|
||||
_TENANT_PROVISIONING_TIME_LIMIT = 60 * 25 # 25 minutes
|
||||
|
||||
|
||||
@shared_task(
|
||||
@@ -91,8 +93,7 @@ def check_available_tenants(self: Task) -> None: # noqa: ARG001
|
||||
batch_size = min(tenants_to_provision, _MAX_TENANTS_PER_RUN)
|
||||
if batch_size < tenants_to_provision:
|
||||
task_logger.info(
|
||||
f"Capping batch to {batch_size} "
|
||||
f"(need {tenants_to_provision}, will catch up next cycle)"
|
||||
f"Capping batch to {batch_size} (need {tenants_to_provision}, will catch up next cycle)"
|
||||
)
|
||||
|
||||
provisioned = 0
|
||||
@@ -103,12 +104,14 @@ def check_available_tenants(self: Task) -> None: # noqa: ARG001
|
||||
provisioned += 1
|
||||
except Exception:
|
||||
task_logger.exception(
|
||||
f"Failed to provision tenant {i + 1}/{batch_size}, "
|
||||
"continuing with remaining tenants"
|
||||
f"Failed to provision tenant {i + 1}/{batch_size}, continuing with remaining tenants"
|
||||
)
|
||||
|
||||
task_logger.info(f"Provisioning complete: {provisioned}/{batch_size} succeeded")
|
||||
|
||||
# Migrate any pool tenants that were provisioned before a new migration was deployed
|
||||
_migrate_stale_pool_tenants()
|
||||
|
||||
except Exception:
|
||||
task_logger.exception("Error in check_available_tenants task")
|
||||
|
||||
@@ -121,6 +124,46 @@ def check_available_tenants(self: Task) -> None: # noqa: ARG001
|
||||
)
|
||||
|
||||
|
||||
def _migrate_stale_pool_tenants() -> None:
|
||||
"""
|
||||
Run alembic upgrade head on all pool tenants. Since alembic upgrade head is
|
||||
idempotent, tenants already at head are a fast no-op. This ensures pool
|
||||
tenants are always current so that signup doesn't hit schema mismatches
|
||||
(e.g. missing columns added after the tenant was pre-provisioned).
|
||||
"""
|
||||
with get_session_with_shared_schema() as db_session:
|
||||
pool_tenants = db_session.query(AvailableTenant).all()
|
||||
tenant_ids = [t.tenant_id for t in pool_tenants]
|
||||
|
||||
if not tenant_ids:
|
||||
return
|
||||
|
||||
task_logger.info(
|
||||
f"Checking {len(tenant_ids)} pool tenant(s) for pending migrations"
|
||||
)
|
||||
|
||||
for tenant_id in tenant_ids:
|
||||
try:
|
||||
run_alembic_migrations(tenant_id)
|
||||
new_version = get_current_alembic_version(tenant_id)
|
||||
with get_session_with_shared_schema() as db_session:
|
||||
tenant = (
|
||||
db_session.query(AvailableTenant)
|
||||
.filter_by(tenant_id=tenant_id)
|
||||
.first()
|
||||
)
|
||||
if tenant and tenant.alembic_version != new_version:
|
||||
task_logger.info(
|
||||
f"Migrated pool tenant {tenant_id}: {tenant.alembic_version} -> {new_version}"
|
||||
)
|
||||
tenant.alembic_version = new_version
|
||||
db_session.commit()
|
||||
except Exception:
|
||||
task_logger.exception(
|
||||
f"Failed to migrate pool tenant {tenant_id}, skipping"
|
||||
)
|
||||
|
||||
|
||||
def pre_provision_tenant() -> bool:
|
||||
"""
|
||||
Pre-provision a new tenant and store it in the NewAvailableTenant table.
|
||||
|
||||
@@ -54,27 +54,35 @@ def perform_ttl_management_task(
|
||||
retention_limit_days, db_session
|
||||
)
|
||||
|
||||
failures = 0
|
||||
for user_id, session_id in old_chat_sessions:
|
||||
# one session per delete so that we don't blow up if a deletion fails.
|
||||
with get_session_with_current_tenant() as db_session:
|
||||
delete_chat_session(
|
||||
user_id,
|
||||
session_id,
|
||||
db_session,
|
||||
include_deleted=True,
|
||||
hard_delete=True,
|
||||
try:
|
||||
with get_session_with_current_tenant() as db_session:
|
||||
delete_chat_session(
|
||||
user_id,
|
||||
session_id,
|
||||
db_session,
|
||||
include_deleted=True,
|
||||
hard_delete=True,
|
||||
)
|
||||
except Exception:
|
||||
failures += 1
|
||||
logger.exception(
|
||||
"Failed to delete chat session "
|
||||
f"user_id={user_id} session_id={session_id}, "
|
||||
"continuing with remaining sessions"
|
||||
)
|
||||
|
||||
with get_session_with_current_tenant() as db_session:
|
||||
mark_task_as_finished_with_id(
|
||||
db_session=db_session,
|
||||
task_id=task_id,
|
||||
success=True,
|
||||
success=failures == 0,
|
||||
)
|
||||
|
||||
except Exception:
|
||||
logger.exception(
|
||||
f"delete_chat_session exceptioned. user_id={user_id} session_id={session_id}"
|
||||
f"TTL management task failed. user_id={user_id} session_id={session_id}"
|
||||
)
|
||||
with get_session_with_current_tenant() as db_session:
|
||||
mark_task_as_finished_with_id(
|
||||
|
||||
@@ -99,6 +99,26 @@ async def get_or_provision_tenant(
|
||||
tenant_id = await get_available_tenant()
|
||||
|
||||
if tenant_id:
|
||||
# Run migrations to ensure the pre-provisioned tenant schema is current.
|
||||
# Pool tenants may have been created before a new migration was deployed.
|
||||
# Capture as a non-optional local so mypy can type the lambda correctly.
|
||||
_tenant_id: str = tenant_id
|
||||
loop = asyncio.get_running_loop()
|
||||
try:
|
||||
await loop.run_in_executor(
|
||||
None, lambda: run_alembic_migrations(_tenant_id)
|
||||
)
|
||||
except Exception:
|
||||
# The tenant was already dequeued from the pool — roll it back so
|
||||
# it doesn't end up orphaned (schema exists, but not assigned to anyone).
|
||||
logger.exception(
|
||||
f"Migration failed for pre-provisioned tenant {_tenant_id}; rolling back"
|
||||
)
|
||||
try:
|
||||
await rollback_tenant_provisioning(_tenant_id)
|
||||
except Exception:
|
||||
logger.exception(f"Failed to rollback orphaned tenant {_tenant_id}")
|
||||
raise
|
||||
# If we have a pre-provisioned tenant, assign it to the user
|
||||
await assign_tenant_to_user(tenant_id, email, referral_source)
|
||||
logger.info(f"Assigned pre-provisioned tenant {tenant_id} to user {email}")
|
||||
|
||||
@@ -100,6 +100,7 @@ def get_model_app() -> FastAPI:
|
||||
dsn=SENTRY_DSN,
|
||||
integrations=[StarletteIntegration(), FastApiIntegration()],
|
||||
traces_sample_rate=0.1,
|
||||
release=__version__,
|
||||
)
|
||||
logger.info("Sentry initialized")
|
||||
else:
|
||||
|
||||
@@ -20,6 +20,7 @@ from sentry_sdk.integrations.celery import CeleryIntegration
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from onyx import __version__
|
||||
from onyx.background.celery.apps.task_formatters import CeleryTaskColoredFormatter
|
||||
from onyx.background.celery.apps.task_formatters import CeleryTaskPlainFormatter
|
||||
from onyx.background.celery.celery_utils import celery_is_worker_primary
|
||||
@@ -65,6 +66,7 @@ if SENTRY_DSN:
|
||||
dsn=SENTRY_DSN,
|
||||
integrations=[CeleryIntegration()],
|
||||
traces_sample_rate=0.1,
|
||||
release=__version__,
|
||||
)
|
||||
logger.info("Sentry initialized")
|
||||
else:
|
||||
@@ -515,7 +517,8 @@ def reset_tenant_id(
|
||||
|
||||
|
||||
def wait_for_vespa_or_shutdown(
|
||||
sender: Any, **kwargs: Any # noqa: ARG001
|
||||
sender: Any, # noqa: ARG001
|
||||
**kwargs: Any, # noqa: ARG001
|
||||
) -> None: # noqa: ARG001
|
||||
"""Waits for Vespa to become ready subject to a timeout.
|
||||
Raises WorkerShutdown if the timeout is reached."""
|
||||
|
||||
@@ -9,6 +9,7 @@ from celery import Celery
|
||||
from celery import shared_task
|
||||
from celery import Task
|
||||
|
||||
from onyx import __version__
|
||||
from onyx.background.celery.apps.app_base import task_logger
|
||||
from onyx.background.celery.memory_monitoring import emit_process_memory
|
||||
from onyx.background.celery.tasks.docprocessing.heartbeat import start_heartbeat
|
||||
@@ -137,6 +138,7 @@ def _docfetching_task(
|
||||
sentry_sdk.init(
|
||||
dsn=SENTRY_DSN,
|
||||
traces_sample_rate=0.1,
|
||||
release=__version__,
|
||||
)
|
||||
logger.info("Sentry initialized")
|
||||
else:
|
||||
|
||||
@@ -805,6 +805,10 @@ MINI_CHUNK_SIZE = 150
|
||||
# This is the number of regular chunks per large chunk
|
||||
LARGE_CHUNK_RATIO = 4
|
||||
|
||||
# The maximum number of chunks that can be held for 1 document processing batch
|
||||
# The purpose of this is to set an upper bound on memory usage
|
||||
MAX_CHUNKS_PER_DOC_BATCH = int(os.environ.get("MAX_CHUNKS_PER_DOC_BATCH") or 1000)
|
||||
|
||||
# Include the document level metadata in each chunk. If the metadata is too long, then it is thrown out
|
||||
# We don't want the metadata to overwhelm the actual contents of the chunk
|
||||
SKIP_METADATA_IN_CHUNK = os.environ.get("SKIP_METADATA_IN_CHUNK", "").lower() == "true"
|
||||
|
||||
@@ -212,6 +212,7 @@ class DocumentSource(str, Enum):
|
||||
PRODUCTBOARD = "productboard"
|
||||
FILE = "file"
|
||||
CODA = "coda"
|
||||
CANVAS = "canvas"
|
||||
NOTION = "notion"
|
||||
ZULIP = "zulip"
|
||||
LINEAR = "linear"
|
||||
@@ -672,6 +673,7 @@ DocumentSourceDescription: dict[DocumentSource, str] = {
|
||||
DocumentSource.SLAB: "slab data",
|
||||
DocumentSource.PRODUCTBOARD: "productboard data (boards, etc.)",
|
||||
DocumentSource.FILE: "files",
|
||||
DocumentSource.CANVAS: "canvas lms - courses, pages, assignments, and announcements",
|
||||
DocumentSource.CODA: "coda - team workspace with docs, tables, and pages",
|
||||
DocumentSource.NOTION: "notion data - a workspace that combines note-taking, \
|
||||
project management, and collaboration tools into a single, customizable platform",
|
||||
|
||||
32
backend/onyx/connectors/canvas/access.py
Normal file
32
backend/onyx/connectors/canvas/access.py
Normal file
@@ -0,0 +1,32 @@
|
||||
"""
|
||||
Permissioning / AccessControl logic for Canvas courses.
|
||||
|
||||
CE stub — returns None (no permissions). The EE implementation is loaded
|
||||
at runtime via ``fetch_versioned_implementation``.
|
||||
"""
|
||||
|
||||
from collections.abc import Callable
|
||||
from typing import cast
|
||||
|
||||
from onyx.access.models import ExternalAccess
|
||||
from onyx.connectors.canvas.client import CanvasApiClient
|
||||
from onyx.utils.variable_functionality import fetch_versioned_implementation
|
||||
from onyx.utils.variable_functionality import global_version
|
||||
|
||||
|
||||
def get_course_permissions(
|
||||
canvas_client: CanvasApiClient,
|
||||
course_id: int,
|
||||
) -> ExternalAccess | None:
|
||||
if not global_version.is_ee_version():
|
||||
return None
|
||||
|
||||
ee_get_course_permissions = cast(
|
||||
Callable[[CanvasApiClient, int], ExternalAccess | None],
|
||||
fetch_versioned_implementation(
|
||||
"onyx.external_permissions.canvas.access",
|
||||
"get_course_permissions",
|
||||
),
|
||||
)
|
||||
|
||||
return ee_get_course_permissions(canvas_client, course_id)
|
||||
@@ -2,6 +2,7 @@ from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
from collections.abc import Iterator
|
||||
from typing import Any
|
||||
from urllib.parse import urlparse
|
||||
|
||||
@@ -190,3 +191,22 @@ class CanvasApiClient:
|
||||
if clean_endpoint:
|
||||
final_url += "/" + clean_endpoint
|
||||
return final_url
|
||||
|
||||
def paginate(
|
||||
self,
|
||||
endpoint: str,
|
||||
params: dict[str, Any] | None = None,
|
||||
) -> Iterator[list[Any]]:
|
||||
"""Yield each page of results, following Link-header pagination.
|
||||
|
||||
Makes the first request with endpoint + params, then follows
|
||||
next_url from Link headers for subsequent pages.
|
||||
"""
|
||||
response, next_url = self.get(endpoint, params=params)
|
||||
while True:
|
||||
if not response:
|
||||
break
|
||||
yield response
|
||||
if not next_url:
|
||||
break
|
||||
response, next_url = self.get(full_url=next_url)
|
||||
|
||||
@@ -1,17 +1,82 @@
|
||||
from datetime import datetime
|
||||
from datetime import timezone
|
||||
from typing import Any
|
||||
from typing import cast
|
||||
from typing import Literal
|
||||
from typing import NoReturn
|
||||
from typing import TypeAlias
|
||||
|
||||
from pydantic import BaseModel
|
||||
from retry import retry
|
||||
from typing_extensions import override
|
||||
|
||||
from onyx.access.models import ExternalAccess
|
||||
from onyx.configs.app_configs import INDEX_BATCH_SIZE
|
||||
from onyx.configs.constants import DocumentSource
|
||||
from onyx.connectors.canvas.access import get_course_permissions
|
||||
from onyx.connectors.canvas.client import CanvasApiClient
|
||||
from onyx.connectors.exceptions import ConnectorValidationError
|
||||
from onyx.connectors.exceptions import CredentialExpiredError
|
||||
from onyx.connectors.exceptions import InsufficientPermissionsError
|
||||
from onyx.connectors.exceptions import UnexpectedValidationError
|
||||
from onyx.connectors.interfaces import CheckpointedConnectorWithPermSync
|
||||
from onyx.connectors.interfaces import CheckpointOutput
|
||||
from onyx.connectors.interfaces import GenerateSlimDocumentOutput
|
||||
from onyx.connectors.interfaces import SecondsSinceUnixEpoch
|
||||
from onyx.connectors.interfaces import SlimConnectorWithPermSync
|
||||
from onyx.connectors.models import ConnectorCheckpoint
|
||||
from onyx.connectors.models import ConnectorMissingCredentialError
|
||||
from onyx.connectors.models import Document
|
||||
from onyx.connectors.models import ImageSection
|
||||
from onyx.connectors.models import TextSection
|
||||
from onyx.error_handling.exceptions import OnyxError
|
||||
from onyx.file_processing.html_utils import parse_html_page_basic
|
||||
from onyx.indexing.indexing_heartbeat import IndexingHeartbeatInterface
|
||||
from onyx.utils.logger import setup_logger
|
||||
|
||||
logger = setup_logger()
|
||||
|
||||
|
||||
def _handle_canvas_api_error(e: OnyxError) -> NoReturn:
|
||||
"""Map Canvas API errors to connector framework exceptions."""
|
||||
if e.status_code == 401:
|
||||
raise CredentialExpiredError(
|
||||
"Canvas API token is invalid or expired (HTTP 401)."
|
||||
)
|
||||
elif e.status_code == 403:
|
||||
raise InsufficientPermissionsError(
|
||||
"Canvas API token does not have sufficient permissions (HTTP 403)."
|
||||
)
|
||||
elif e.status_code == 429:
|
||||
raise ConnectorValidationError(
|
||||
"Canvas rate-limit exceeded (HTTP 429). Please try again later."
|
||||
)
|
||||
elif e.status_code >= 500:
|
||||
raise UnexpectedValidationError(
|
||||
f"Unexpected Canvas HTTP error (status={e.status_code}): {e}"
|
||||
)
|
||||
else:
|
||||
raise ConnectorValidationError(
|
||||
f"Canvas API error (status={e.status_code}): {e}"
|
||||
)
|
||||
|
||||
|
||||
class CanvasCourse(BaseModel):
|
||||
id: int
|
||||
name: str
|
||||
course_code: str
|
||||
created_at: str
|
||||
workflow_state: str
|
||||
name: str | None = None
|
||||
course_code: str | None = None
|
||||
created_at: str | None = None
|
||||
workflow_state: str | None = None
|
||||
|
||||
@classmethod
|
||||
def from_api(cls, payload: dict[str, Any]) -> "CanvasCourse":
|
||||
return cls(
|
||||
id=payload["id"],
|
||||
name=payload.get("name"),
|
||||
course_code=payload.get("course_code"),
|
||||
created_at=payload.get("created_at"),
|
||||
workflow_state=payload.get("workflow_state"),
|
||||
)
|
||||
|
||||
|
||||
class CanvasPage(BaseModel):
|
||||
@@ -19,10 +84,22 @@ class CanvasPage(BaseModel):
|
||||
url: str
|
||||
title: str
|
||||
body: str | None = None
|
||||
created_at: str
|
||||
updated_at: str
|
||||
created_at: str | None = None
|
||||
updated_at: str | None = None
|
||||
course_id: int
|
||||
|
||||
@classmethod
|
||||
def from_api(cls, payload: dict[str, Any], course_id: int) -> "CanvasPage":
|
||||
return cls(
|
||||
page_id=payload["page_id"],
|
||||
url=payload["url"],
|
||||
title=payload["title"],
|
||||
body=payload.get("body"),
|
||||
created_at=payload.get("created_at"),
|
||||
updated_at=payload.get("updated_at"),
|
||||
course_id=course_id,
|
||||
)
|
||||
|
||||
|
||||
class CanvasAssignment(BaseModel):
|
||||
id: int
|
||||
@@ -30,10 +107,23 @@ class CanvasAssignment(BaseModel):
|
||||
description: str | None = None
|
||||
html_url: str
|
||||
course_id: int
|
||||
created_at: str
|
||||
updated_at: str
|
||||
created_at: str | None = None
|
||||
updated_at: str | None = None
|
||||
due_at: str | None = None
|
||||
|
||||
@classmethod
|
||||
def from_api(cls, payload: dict[str, Any], course_id: int) -> "CanvasAssignment":
|
||||
return cls(
|
||||
id=payload["id"],
|
||||
name=payload["name"],
|
||||
description=payload.get("description"),
|
||||
html_url=payload["html_url"],
|
||||
course_id=course_id,
|
||||
created_at=payload.get("created_at"),
|
||||
updated_at=payload.get("updated_at"),
|
||||
due_at=payload.get("due_at"),
|
||||
)
|
||||
|
||||
|
||||
class CanvasAnnouncement(BaseModel):
|
||||
id: int
|
||||
@@ -43,6 +133,17 @@ class CanvasAnnouncement(BaseModel):
|
||||
posted_at: str | None = None
|
||||
course_id: int
|
||||
|
||||
@classmethod
|
||||
def from_api(cls, payload: dict[str, Any], course_id: int) -> "CanvasAnnouncement":
|
||||
return cls(
|
||||
id=payload["id"],
|
||||
title=payload["title"],
|
||||
message=payload.get("message"),
|
||||
html_url=payload["html_url"],
|
||||
posted_at=payload.get("posted_at"),
|
||||
course_id=course_id,
|
||||
)
|
||||
|
||||
|
||||
CanvasStage: TypeAlias = Literal["pages", "assignments", "announcements"]
|
||||
|
||||
@@ -72,3 +173,286 @@ class CanvasConnectorCheckpoint(ConnectorCheckpoint):
|
||||
self.current_course_index += 1
|
||||
self.stage = "pages"
|
||||
self.next_url = None
|
||||
|
||||
|
||||
class CanvasConnector(
|
||||
CheckpointedConnectorWithPermSync[CanvasConnectorCheckpoint],
|
||||
SlimConnectorWithPermSync,
|
||||
):
|
||||
def __init__(
|
||||
self,
|
||||
canvas_base_url: str,
|
||||
batch_size: int = INDEX_BATCH_SIZE,
|
||||
) -> None:
|
||||
self.canvas_base_url = canvas_base_url.rstrip("/").removesuffix("/api/v1")
|
||||
self.batch_size = batch_size
|
||||
self._canvas_client: CanvasApiClient | None = None
|
||||
self._course_permissions_cache: dict[int, ExternalAccess | None] = {}
|
||||
|
||||
@property
|
||||
def canvas_client(self) -> CanvasApiClient:
|
||||
if self._canvas_client is None:
|
||||
raise ConnectorMissingCredentialError("Canvas")
|
||||
return self._canvas_client
|
||||
|
||||
def _get_course_permissions(self, course_id: int) -> ExternalAccess | None:
|
||||
"""Get course permissions with caching."""
|
||||
if course_id not in self._course_permissions_cache:
|
||||
self._course_permissions_cache[course_id] = get_course_permissions(
|
||||
canvas_client=self.canvas_client,
|
||||
course_id=course_id,
|
||||
)
|
||||
return self._course_permissions_cache[course_id]
|
||||
|
||||
@retry(tries=3, delay=1, backoff=2)
|
||||
def _list_courses(self) -> list[CanvasCourse]:
|
||||
"""Fetch all courses accessible to the authenticated user."""
|
||||
logger.debug("Fetching Canvas courses")
|
||||
|
||||
courses: list[CanvasCourse] = []
|
||||
for page in self.canvas_client.paginate(
|
||||
"courses", params={"per_page": "100", "state[]": "available"}
|
||||
):
|
||||
courses.extend(CanvasCourse.from_api(c) for c in page)
|
||||
return courses
|
||||
|
||||
@retry(tries=3, delay=1, backoff=2)
|
||||
def _list_pages(self, course_id: int) -> list[CanvasPage]:
|
||||
"""Fetch all pages for a given course."""
|
||||
logger.debug(f"Fetching pages for course {course_id}")
|
||||
|
||||
pages: list[CanvasPage] = []
|
||||
for page in self.canvas_client.paginate(
|
||||
f"courses/{course_id}/pages",
|
||||
params={"per_page": "100", "include[]": "body", "published": "true"},
|
||||
):
|
||||
pages.extend(CanvasPage.from_api(p, course_id=course_id) for p in page)
|
||||
return pages
|
||||
|
||||
@retry(tries=3, delay=1, backoff=2)
|
||||
def _list_assignments(self, course_id: int) -> list[CanvasAssignment]:
|
||||
"""Fetch all assignments for a given course."""
|
||||
logger.debug(f"Fetching assignments for course {course_id}")
|
||||
|
||||
assignments: list[CanvasAssignment] = []
|
||||
for page in self.canvas_client.paginate(
|
||||
f"courses/{course_id}/assignments",
|
||||
params={"per_page": "100", "published": "true"},
|
||||
):
|
||||
assignments.extend(
|
||||
CanvasAssignment.from_api(a, course_id=course_id) for a in page
|
||||
)
|
||||
return assignments
|
||||
|
||||
@retry(tries=3, delay=1, backoff=2)
|
||||
def _list_announcements(self, course_id: int) -> list[CanvasAnnouncement]:
|
||||
"""Fetch all announcements for a given course."""
|
||||
logger.debug(f"Fetching announcements for course {course_id}")
|
||||
|
||||
announcements: list[CanvasAnnouncement] = []
|
||||
for page in self.canvas_client.paginate(
|
||||
"announcements",
|
||||
params={
|
||||
"per_page": "100",
|
||||
"context_codes[]": f"course_{course_id}",
|
||||
"active_only": "true",
|
||||
},
|
||||
):
|
||||
announcements.extend(
|
||||
CanvasAnnouncement.from_api(a, course_id=course_id) for a in page
|
||||
)
|
||||
return announcements
|
||||
|
||||
def _build_document(
|
||||
self,
|
||||
doc_id: str,
|
||||
link: str,
|
||||
text: str,
|
||||
semantic_identifier: str,
|
||||
doc_updated_at: datetime | None,
|
||||
course_id: int,
|
||||
doc_type: str,
|
||||
) -> Document:
|
||||
"""Build a Document with standard Canvas fields."""
|
||||
return Document(
|
||||
id=doc_id,
|
||||
sections=cast(
|
||||
list[TextSection | ImageSection],
|
||||
[TextSection(link=link, text=text)],
|
||||
),
|
||||
source=DocumentSource.CANVAS,
|
||||
semantic_identifier=semantic_identifier,
|
||||
doc_updated_at=doc_updated_at,
|
||||
metadata={"course_id": str(course_id), "type": doc_type},
|
||||
)
|
||||
|
||||
def _convert_page_to_document(self, page: CanvasPage) -> Document:
|
||||
"""Convert a Canvas page to a Document."""
|
||||
link = f"{self.canvas_base_url}/courses/{page.course_id}/pages/{page.url}"
|
||||
|
||||
text_parts = [page.title]
|
||||
body_text = parse_html_page_basic(page.body) if page.body else ""
|
||||
if body_text:
|
||||
text_parts.append(body_text)
|
||||
|
||||
doc_updated_at = (
|
||||
datetime.fromisoformat(page.updated_at.replace("Z", "+00:00")).astimezone(
|
||||
timezone.utc
|
||||
)
|
||||
if page.updated_at
|
||||
else None
|
||||
)
|
||||
|
||||
document = self._build_document(
|
||||
doc_id=f"canvas-page-{page.course_id}-{page.page_id}",
|
||||
link=link,
|
||||
text="\n\n".join(text_parts),
|
||||
semantic_identifier=page.title or f"Page {page.page_id}",
|
||||
doc_updated_at=doc_updated_at,
|
||||
course_id=page.course_id,
|
||||
doc_type="page",
|
||||
)
|
||||
return document
|
||||
|
||||
def _convert_assignment_to_document(self, assignment: CanvasAssignment) -> Document:
|
||||
"""Convert a Canvas assignment to a Document."""
|
||||
text_parts = [assignment.name]
|
||||
desc_text = (
|
||||
parse_html_page_basic(assignment.description)
|
||||
if assignment.description
|
||||
else ""
|
||||
)
|
||||
if desc_text:
|
||||
text_parts.append(desc_text)
|
||||
if assignment.due_at:
|
||||
due_dt = datetime.fromisoformat(
|
||||
assignment.due_at.replace("Z", "+00:00")
|
||||
).astimezone(timezone.utc)
|
||||
text_parts.append(f"Due: {due_dt.strftime('%B %d, %Y %H:%M UTC')}")
|
||||
|
||||
doc_updated_at = (
|
||||
datetime.fromisoformat(
|
||||
assignment.updated_at.replace("Z", "+00:00")
|
||||
).astimezone(timezone.utc)
|
||||
if assignment.updated_at
|
||||
else None
|
||||
)
|
||||
|
||||
document = self._build_document(
|
||||
doc_id=f"canvas-assignment-{assignment.course_id}-{assignment.id}",
|
||||
link=assignment.html_url,
|
||||
text="\n\n".join(text_parts),
|
||||
semantic_identifier=assignment.name or f"Assignment {assignment.id}",
|
||||
doc_updated_at=doc_updated_at,
|
||||
course_id=assignment.course_id,
|
||||
doc_type="assignment",
|
||||
)
|
||||
return document
|
||||
|
||||
def _convert_announcement_to_document(
|
||||
self, announcement: CanvasAnnouncement
|
||||
) -> Document:
|
||||
"""Convert a Canvas announcement to a Document."""
|
||||
text_parts = [announcement.title]
|
||||
msg_text = (
|
||||
parse_html_page_basic(announcement.message) if announcement.message else ""
|
||||
)
|
||||
if msg_text:
|
||||
text_parts.append(msg_text)
|
||||
|
||||
doc_updated_at = (
|
||||
datetime.fromisoformat(
|
||||
announcement.posted_at.replace("Z", "+00:00")
|
||||
).astimezone(timezone.utc)
|
||||
if announcement.posted_at
|
||||
else None
|
||||
)
|
||||
|
||||
document = self._build_document(
|
||||
doc_id=f"canvas-announcement-{announcement.course_id}-{announcement.id}",
|
||||
link=announcement.html_url,
|
||||
text="\n\n".join(text_parts),
|
||||
semantic_identifier=announcement.title or f"Announcement {announcement.id}",
|
||||
doc_updated_at=doc_updated_at,
|
||||
course_id=announcement.course_id,
|
||||
doc_type="announcement",
|
||||
)
|
||||
return document
|
||||
|
||||
@override
|
||||
def load_credentials(self, credentials: dict[str, Any]) -> dict[str, Any] | None:
|
||||
"""Load and validate Canvas credentials."""
|
||||
access_token = credentials.get("canvas_access_token")
|
||||
if not access_token:
|
||||
raise ConnectorMissingCredentialError("Canvas")
|
||||
|
||||
try:
|
||||
client = CanvasApiClient(
|
||||
bearer_token=access_token,
|
||||
canvas_base_url=self.canvas_base_url,
|
||||
)
|
||||
client.get("courses", params={"per_page": "1"})
|
||||
except ValueError as e:
|
||||
raise ConnectorValidationError(f"Invalid Canvas base URL: {e}")
|
||||
except OnyxError as e:
|
||||
_handle_canvas_api_error(e)
|
||||
|
||||
self._canvas_client = client
|
||||
return None
|
||||
|
||||
@override
|
||||
def validate_connector_settings(self) -> None:
|
||||
"""Validate Canvas connector settings by testing API access."""
|
||||
try:
|
||||
self.canvas_client.get("courses", params={"per_page": "1"})
|
||||
logger.info("Canvas connector settings validated successfully")
|
||||
except OnyxError as e:
|
||||
_handle_canvas_api_error(e)
|
||||
except ConnectorMissingCredentialError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
raise UnexpectedValidationError(
|
||||
f"Unexpected error during Canvas settings validation: {exc}"
|
||||
)
|
||||
|
||||
@override
|
||||
def load_from_checkpoint(
|
||||
self,
|
||||
start: SecondsSinceUnixEpoch,
|
||||
end: SecondsSinceUnixEpoch,
|
||||
checkpoint: CanvasConnectorCheckpoint,
|
||||
) -> CheckpointOutput[CanvasConnectorCheckpoint]:
|
||||
# TODO(benwu408): implemented in PR3 (checkpoint)
|
||||
raise NotImplementedError
|
||||
|
||||
@override
|
||||
def load_from_checkpoint_with_perm_sync(
|
||||
self,
|
||||
start: SecondsSinceUnixEpoch,
|
||||
end: SecondsSinceUnixEpoch,
|
||||
checkpoint: CanvasConnectorCheckpoint,
|
||||
) -> CheckpointOutput[CanvasConnectorCheckpoint]:
|
||||
# TODO(benwu408): implemented in PR3 (checkpoint)
|
||||
raise NotImplementedError
|
||||
|
||||
@override
|
||||
def build_dummy_checkpoint(self) -> CanvasConnectorCheckpoint:
|
||||
# TODO(benwu408): implemented in PR3 (checkpoint)
|
||||
raise NotImplementedError
|
||||
|
||||
@override
|
||||
def validate_checkpoint_json(
|
||||
self, checkpoint_json: str
|
||||
) -> CanvasConnectorCheckpoint:
|
||||
# TODO(benwu408): implemented in PR3 (checkpoint)
|
||||
raise NotImplementedError
|
||||
|
||||
@override
|
||||
def retrieve_all_slim_docs_perm_sync(
|
||||
self,
|
||||
start: SecondsSinceUnixEpoch | None = None,
|
||||
end: SecondsSinceUnixEpoch | None = None,
|
||||
callback: IndexingHeartbeatInterface | None = None,
|
||||
) -> GenerateSlimDocumentOutput:
|
||||
# TODO(benwu408): implemented in PR4 (perm sync)
|
||||
raise NotImplementedError
|
||||
|
||||
@@ -72,6 +72,10 @@ CONNECTOR_CLASS_MAP = {
|
||||
module_path="onyx.connectors.coda.connector",
|
||||
class_name="CodaConnector",
|
||||
),
|
||||
DocumentSource.CANVAS: ConnectorMapping(
|
||||
module_path="onyx.connectors.canvas.connector",
|
||||
class_name="CanvasConnector",
|
||||
),
|
||||
DocumentSource.NOTION: ConnectorMapping(
|
||||
module_path="onyx.connectors.notion.connector",
|
||||
class_name="NotionConnector",
|
||||
|
||||
@@ -185,7 +185,15 @@ def delete_messages_and_files_from_chat_session(
|
||||
for _, files in messages_with_files:
|
||||
file_store = get_default_file_store()
|
||||
for file_info in files or []:
|
||||
file_store.delete_file(file_id=file_info.get("id"))
|
||||
try:
|
||||
file_store.delete_file(file_id=file_info.get("id"))
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"Failed to delete file %s from file store during "
|
||||
"chat session cleanup, skipping.",
|
||||
file_info.get("id"),
|
||||
)
|
||||
continue
|
||||
|
||||
# Delete ChatMessage records - CASCADE constraints will automatically handle:
|
||||
# - ChatMessage__StandardAnswer relationship records
|
||||
|
||||
@@ -6,6 +6,7 @@ import httpx
|
||||
from opensearchpy import NotFoundError
|
||||
|
||||
from onyx.access.models import DocumentAccess
|
||||
from onyx.configs.app_configs import MAX_CHUNKS_PER_DOC_BATCH
|
||||
from onyx.configs.app_configs import VERIFY_CREATE_OPENSEARCH_INDEX_ON_INIT_MT
|
||||
from onyx.configs.chat_configs import NUM_RETURNED_HITS
|
||||
from onyx.configs.chat_configs import TITLE_CONTENT_RATIO
|
||||
@@ -738,6 +739,9 @@ class OpenSearchDocumentIndex(DocumentIndex):
|
||||
_flush_chunks(current_chunks)
|
||||
current_doc_id = doc_id
|
||||
current_chunks = [chunk]
|
||||
elif len(current_chunks) >= MAX_CHUNKS_PER_DOC_BATCH:
|
||||
_flush_chunks(current_chunks)
|
||||
current_chunks = [chunk]
|
||||
else:
|
||||
current_chunks.append(chunk)
|
||||
|
||||
|
||||
@@ -10,6 +10,7 @@ import httpx
|
||||
from pydantic import BaseModel
|
||||
from retry import retry
|
||||
|
||||
from onyx.configs.app_configs import MAX_CHUNKS_PER_DOC_BATCH
|
||||
from onyx.configs.app_configs import RECENCY_BIAS_MULTIPLIER
|
||||
from onyx.configs.app_configs import RERANK_COUNT
|
||||
from onyx.configs.chat_configs import DOC_TIME_DECAY
|
||||
@@ -427,7 +428,9 @@ class VespaDocumentIndex(DocumentIndex):
|
||||
new_document_id_to_original_document_id,
|
||||
all_cleaned_doc_ids,
|
||||
)
|
||||
for chunk_batch in batch_generator(cleaned_chunks, BATCH_SIZE):
|
||||
for chunk_batch in batch_generator(
|
||||
cleaned_chunks, min(BATCH_SIZE, MAX_CHUNKS_PER_DOC_BATCH)
|
||||
):
|
||||
batch_index_vespa_chunks(
|
||||
chunks=chunk_batch,
|
||||
index_name=self._index_name,
|
||||
|
||||
@@ -19,7 +19,8 @@ from onyx.db.document import update_docs_updated_at__no_commit
|
||||
from onyx.db.document_set import fetch_document_sets_for_documents
|
||||
from onyx.indexing.indexing_pipeline import DocumentBatchPrepareContext
|
||||
from onyx.indexing.indexing_pipeline import index_doc_batch_prepare
|
||||
from onyx.indexing.models import BuildMetadataAwareChunksResult
|
||||
from onyx.indexing.models import ChunkEnrichmentContext
|
||||
from onyx.indexing.models import DocAwareChunk
|
||||
from onyx.indexing.models import DocMetadataAwareIndexChunk
|
||||
from onyx.indexing.models import IndexChunk
|
||||
from onyx.indexing.models import UpdatableChunkData
|
||||
@@ -85,14 +86,21 @@ class DocumentIndexingBatchAdapter:
|
||||
) as transaction:
|
||||
yield transaction
|
||||
|
||||
def build_metadata_aware_chunks(
|
||||
def prepare_enrichment(
|
||||
self,
|
||||
chunks_with_embeddings: list[IndexChunk],
|
||||
chunk_content_scores: list[float],
|
||||
tenant_id: str,
|
||||
context: DocumentBatchPrepareContext,
|
||||
) -> BuildMetadataAwareChunksResult:
|
||||
"""Enrich chunks with access, document sets, boosts, token counts, and hierarchy."""
|
||||
tenant_id: str,
|
||||
chunks: list[DocAwareChunk],
|
||||
) -> "DocumentChunkEnricher":
|
||||
"""Do all DB lookups once and return a per-chunk enricher."""
|
||||
updatable_ids = [doc.id for doc in context.updatable_docs]
|
||||
|
||||
doc_id_to_new_chunk_cnt: dict[str, int] = {
|
||||
doc_id: 0 for doc_id in updatable_ids
|
||||
}
|
||||
for chunk in chunks:
|
||||
if chunk.source_document.id in doc_id_to_new_chunk_cnt:
|
||||
doc_id_to_new_chunk_cnt[chunk.source_document.id] += 1
|
||||
|
||||
no_access = DocumentAccess.build(
|
||||
user_emails=[],
|
||||
@@ -102,67 +110,30 @@ class DocumentIndexingBatchAdapter:
|
||||
is_public=False,
|
||||
)
|
||||
|
||||
updatable_ids = [doc.id for doc in context.updatable_docs]
|
||||
|
||||
doc_id_to_access_info = get_access_for_documents(
|
||||
document_ids=updatable_ids, db_session=self.db_session
|
||||
)
|
||||
doc_id_to_document_set = {
|
||||
document_id: document_sets
|
||||
for document_id, document_sets in fetch_document_sets_for_documents(
|
||||
return DocumentChunkEnricher(
|
||||
doc_id_to_access_info=get_access_for_documents(
|
||||
document_ids=updatable_ids, db_session=self.db_session
|
||||
)
|
||||
}
|
||||
|
||||
doc_id_to_previous_chunk_cnt: dict[str, int] = {
|
||||
document_id: chunk_count
|
||||
for document_id, chunk_count in fetch_chunk_counts_for_documents(
|
||||
document_ids=updatable_ids,
|
||||
db_session=self.db_session,
|
||||
)
|
||||
}
|
||||
|
||||
doc_id_to_new_chunk_cnt: dict[str, int] = {
|
||||
doc_id: 0 for doc_id in updatable_ids
|
||||
}
|
||||
for chunk in chunks_with_embeddings:
|
||||
if chunk.source_document.id in doc_id_to_new_chunk_cnt:
|
||||
doc_id_to_new_chunk_cnt[chunk.source_document.id] += 1
|
||||
|
||||
# Get ancestor hierarchy node IDs for each document
|
||||
doc_id_to_ancestor_ids = self._get_ancestor_ids_for_documents(
|
||||
context.updatable_docs, tenant_id
|
||||
)
|
||||
|
||||
access_aware_chunks = [
|
||||
DocMetadataAwareIndexChunk.from_index_chunk(
|
||||
index_chunk=chunk,
|
||||
access=doc_id_to_access_info.get(chunk.source_document.id, no_access),
|
||||
document_sets=set(
|
||||
doc_id_to_document_set.get(chunk.source_document.id, [])
|
||||
),
|
||||
user_project=[],
|
||||
personas=[],
|
||||
boost=(
|
||||
context.id_to_boost_map[chunk.source_document.id]
|
||||
if chunk.source_document.id in context.id_to_boost_map
|
||||
else DEFAULT_BOOST
|
||||
),
|
||||
tenant_id=tenant_id,
|
||||
aggregated_chunk_boost_factor=chunk_content_scores[chunk_num],
|
||||
ancestor_hierarchy_node_ids=doc_id_to_ancestor_ids[
|
||||
chunk.source_document.id
|
||||
],
|
||||
)
|
||||
for chunk_num, chunk in enumerate(chunks_with_embeddings)
|
||||
]
|
||||
|
||||
return BuildMetadataAwareChunksResult(
|
||||
chunks=access_aware_chunks,
|
||||
doc_id_to_previous_chunk_cnt=doc_id_to_previous_chunk_cnt,
|
||||
doc_id_to_new_chunk_cnt=doc_id_to_new_chunk_cnt,
|
||||
user_file_id_to_raw_text={},
|
||||
user_file_id_to_token_count={},
|
||||
),
|
||||
doc_id_to_document_set={
|
||||
document_id: document_sets
|
||||
for document_id, document_sets in fetch_document_sets_for_documents(
|
||||
document_ids=updatable_ids, db_session=self.db_session
|
||||
)
|
||||
},
|
||||
doc_id_to_ancestor_ids=self._get_ancestor_ids_for_documents(
|
||||
context.updatable_docs, tenant_id
|
||||
),
|
||||
id_to_boost_map=context.id_to_boost_map,
|
||||
doc_id_to_previous_chunk_cnt={
|
||||
document_id: chunk_count
|
||||
for document_id, chunk_count in fetch_chunk_counts_for_documents(
|
||||
document_ids=updatable_ids,
|
||||
db_session=self.db_session,
|
||||
)
|
||||
},
|
||||
doc_id_to_new_chunk_cnt=dict(doc_id_to_new_chunk_cnt),
|
||||
no_access=no_access,
|
||||
tenant_id=tenant_id,
|
||||
)
|
||||
|
||||
def _get_ancestor_ids_for_documents(
|
||||
@@ -203,7 +174,7 @@ class DocumentIndexingBatchAdapter:
|
||||
context: DocumentBatchPrepareContext,
|
||||
updatable_chunk_data: list[UpdatableChunkData],
|
||||
filtered_documents: list[Document],
|
||||
result: BuildMetadataAwareChunksResult,
|
||||
enrichment: ChunkEnrichmentContext,
|
||||
) -> None:
|
||||
"""Finalize DB updates, store plaintext, and mark docs as indexed."""
|
||||
updatable_ids = [doc.id for doc in context.updatable_docs]
|
||||
@@ -227,7 +198,7 @@ class DocumentIndexingBatchAdapter:
|
||||
|
||||
update_docs_chunk_count__no_commit(
|
||||
document_ids=updatable_ids,
|
||||
doc_id_to_chunk_count=result.doc_id_to_new_chunk_cnt,
|
||||
doc_id_to_chunk_count=enrichment.doc_id_to_new_chunk_cnt,
|
||||
db_session=self.db_session,
|
||||
)
|
||||
|
||||
@@ -249,3 +220,52 @@ class DocumentIndexingBatchAdapter:
|
||||
)
|
||||
|
||||
self.db_session.commit()
|
||||
|
||||
|
||||
class DocumentChunkEnricher:
|
||||
"""Pre-computed metadata for per-chunk enrichment of connector documents."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
doc_id_to_access_info: dict[str, DocumentAccess],
|
||||
doc_id_to_document_set: dict[str, list[str]],
|
||||
doc_id_to_ancestor_ids: dict[str, list[int]],
|
||||
id_to_boost_map: dict[str, int],
|
||||
doc_id_to_previous_chunk_cnt: dict[str, int],
|
||||
doc_id_to_new_chunk_cnt: dict[str, int],
|
||||
no_access: DocumentAccess,
|
||||
tenant_id: str,
|
||||
) -> None:
|
||||
self._doc_id_to_access_info = doc_id_to_access_info
|
||||
self._doc_id_to_document_set = doc_id_to_document_set
|
||||
self._doc_id_to_ancestor_ids = doc_id_to_ancestor_ids
|
||||
self._id_to_boost_map = id_to_boost_map
|
||||
self._no_access = no_access
|
||||
self._tenant_id = tenant_id
|
||||
self.doc_id_to_previous_chunk_cnt = doc_id_to_previous_chunk_cnt
|
||||
self.doc_id_to_new_chunk_cnt = doc_id_to_new_chunk_cnt
|
||||
|
||||
def enrich_chunk(
|
||||
self, chunk: IndexChunk, score: float
|
||||
) -> DocMetadataAwareIndexChunk:
|
||||
return DocMetadataAwareIndexChunk.from_index_chunk(
|
||||
index_chunk=chunk,
|
||||
access=self._doc_id_to_access_info.get(
|
||||
chunk.source_document.id, self._no_access
|
||||
),
|
||||
document_sets=set(
|
||||
self._doc_id_to_document_set.get(chunk.source_document.id, [])
|
||||
),
|
||||
user_project=[],
|
||||
personas=[],
|
||||
boost=(
|
||||
self._id_to_boost_map[chunk.source_document.id]
|
||||
if chunk.source_document.id in self._id_to_boost_map
|
||||
else DEFAULT_BOOST
|
||||
),
|
||||
tenant_id=self._tenant_id,
|
||||
aggregated_chunk_boost_factor=score,
|
||||
ancestor_hierarchy_node_ids=self._doc_id_to_ancestor_ids[
|
||||
chunk.source_document.id
|
||||
],
|
||||
)
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import datetime
|
||||
import time
|
||||
from collections import defaultdict
|
||||
from collections.abc import Generator
|
||||
from uuid import UUID
|
||||
|
||||
@@ -24,7 +27,8 @@ from onyx.db.user_file import fetch_persona_ids_for_user_files
|
||||
from onyx.db.user_file import fetch_user_project_ids_for_user_files
|
||||
from onyx.file_store.utils import store_user_file_plaintext
|
||||
from onyx.indexing.indexing_pipeline import DocumentBatchPrepareContext
|
||||
from onyx.indexing.models import BuildMetadataAwareChunksResult
|
||||
from onyx.indexing.models import ChunkEnrichmentContext
|
||||
from onyx.indexing.models import DocAwareChunk
|
||||
from onyx.indexing.models import DocMetadataAwareIndexChunk
|
||||
from onyx.indexing.models import IndexChunk
|
||||
from onyx.indexing.models import UpdatableChunkData
|
||||
@@ -102,13 +106,20 @@ class UserFileIndexingAdapter:
|
||||
f"Failed to acquire locks after {_NUM_LOCK_ATTEMPTS} attempts for user files: {[doc.id for doc in documents]}"
|
||||
)
|
||||
|
||||
def build_metadata_aware_chunks(
|
||||
def prepare_enrichment(
|
||||
self,
|
||||
chunks_with_embeddings: list[IndexChunk],
|
||||
chunk_content_scores: list[float],
|
||||
tenant_id: str,
|
||||
context: DocumentBatchPrepareContext,
|
||||
) -> BuildMetadataAwareChunksResult:
|
||||
tenant_id: str,
|
||||
chunks: list[DocAwareChunk],
|
||||
) -> UserFileChunkEnricher:
|
||||
"""Do all DB lookups and pre-compute file metadata from chunks."""
|
||||
updatable_ids = [doc.id for doc in context.updatable_docs]
|
||||
|
||||
doc_id_to_new_chunk_cnt: dict[str, int] = defaultdict(int)
|
||||
content_by_file: dict[str, list[str]] = defaultdict(list)
|
||||
for chunk in chunks:
|
||||
doc_id_to_new_chunk_cnt[chunk.source_document.id] += 1
|
||||
content_by_file[chunk.source_document.id].append(chunk.content)
|
||||
|
||||
no_access = DocumentAccess.build(
|
||||
user_emails=[],
|
||||
@@ -118,7 +129,6 @@ class UserFileIndexingAdapter:
|
||||
is_public=False,
|
||||
)
|
||||
|
||||
updatable_ids = [doc.id for doc in context.updatable_docs]
|
||||
user_file_id_to_project_ids = fetch_user_project_ids_for_user_files(
|
||||
user_file_ids=updatable_ids,
|
||||
db_session=self.db_session,
|
||||
@@ -139,17 +149,6 @@ class UserFileIndexingAdapter:
|
||||
)
|
||||
}
|
||||
|
||||
user_file_id_to_new_chunk_cnt: dict[str, int] = {
|
||||
user_file_id: len(
|
||||
[
|
||||
chunk
|
||||
for chunk in chunks_with_embeddings
|
||||
if chunk.source_document.id == user_file_id
|
||||
]
|
||||
)
|
||||
for user_file_id in updatable_ids
|
||||
}
|
||||
|
||||
# Initialize tokenizer used for token count calculation
|
||||
try:
|
||||
llm = get_default_llm()
|
||||
@@ -164,15 +163,9 @@ class UserFileIndexingAdapter:
|
||||
user_file_id_to_raw_text: dict[str, str] = {}
|
||||
user_file_id_to_token_count: dict[str, int | None] = {}
|
||||
for user_file_id in updatable_ids:
|
||||
user_file_chunks = [
|
||||
chunk
|
||||
for chunk in chunks_with_embeddings
|
||||
if chunk.source_document.id == user_file_id
|
||||
]
|
||||
if user_file_chunks:
|
||||
combined_content = " ".join(
|
||||
[chunk.content for chunk in user_file_chunks]
|
||||
)
|
||||
contents = content_by_file.get(user_file_id)
|
||||
if contents:
|
||||
combined_content = " ".join(contents)
|
||||
user_file_id_to_raw_text[str(user_file_id)] = combined_content
|
||||
token_count: int = (
|
||||
count_tokens(combined_content, llm_tokenizer)
|
||||
@@ -184,28 +177,16 @@ class UserFileIndexingAdapter:
|
||||
user_file_id_to_raw_text[str(user_file_id)] = ""
|
||||
user_file_id_to_token_count[str(user_file_id)] = None
|
||||
|
||||
access_aware_chunks = [
|
||||
DocMetadataAwareIndexChunk.from_index_chunk(
|
||||
index_chunk=chunk,
|
||||
access=user_file_id_to_access.get(chunk.source_document.id, no_access),
|
||||
document_sets=set(),
|
||||
user_project=user_file_id_to_project_ids.get(
|
||||
chunk.source_document.id, []
|
||||
),
|
||||
personas=user_file_id_to_persona_ids.get(chunk.source_document.id, []),
|
||||
boost=DEFAULT_BOOST,
|
||||
tenant_id=tenant_id,
|
||||
aggregated_chunk_boost_factor=chunk_content_scores[chunk_num],
|
||||
)
|
||||
for chunk_num, chunk in enumerate(chunks_with_embeddings)
|
||||
]
|
||||
|
||||
return BuildMetadataAwareChunksResult(
|
||||
chunks=access_aware_chunks,
|
||||
return UserFileChunkEnricher(
|
||||
user_file_id_to_access=user_file_id_to_access,
|
||||
user_file_id_to_project_ids=user_file_id_to_project_ids,
|
||||
user_file_id_to_persona_ids=user_file_id_to_persona_ids,
|
||||
doc_id_to_previous_chunk_cnt=user_file_id_to_previous_chunk_cnt,
|
||||
doc_id_to_new_chunk_cnt=user_file_id_to_new_chunk_cnt,
|
||||
doc_id_to_new_chunk_cnt=dict(doc_id_to_new_chunk_cnt),
|
||||
user_file_id_to_raw_text=user_file_id_to_raw_text,
|
||||
user_file_id_to_token_count=user_file_id_to_token_count,
|
||||
no_access=no_access,
|
||||
tenant_id=tenant_id,
|
||||
)
|
||||
|
||||
def _notify_assistant_owners_if_files_ready(
|
||||
@@ -249,8 +230,9 @@ class UserFileIndexingAdapter:
|
||||
context: DocumentBatchPrepareContext,
|
||||
updatable_chunk_data: list[UpdatableChunkData], # noqa: ARG002
|
||||
filtered_documents: list[Document], # noqa: ARG002
|
||||
result: BuildMetadataAwareChunksResult,
|
||||
enrichment: ChunkEnrichmentContext,
|
||||
) -> None:
|
||||
assert isinstance(enrichment, UserFileChunkEnricher)
|
||||
user_file_ids = [doc.id for doc in context.updatable_docs]
|
||||
|
||||
user_files = (
|
||||
@@ -266,8 +248,10 @@ class UserFileIndexingAdapter:
|
||||
user_file.last_project_sync_at = datetime.datetime.now(
|
||||
datetime.timezone.utc
|
||||
)
|
||||
user_file.chunk_count = result.doc_id_to_new_chunk_cnt[str(user_file.id)]
|
||||
user_file.token_count = result.user_file_id_to_token_count[
|
||||
user_file.chunk_count = enrichment.doc_id_to_new_chunk_cnt.get(
|
||||
str(user_file.id), 0
|
||||
)
|
||||
user_file.token_count = enrichment.user_file_id_to_token_count[
|
||||
str(user_file.id)
|
||||
]
|
||||
|
||||
@@ -279,8 +263,54 @@ class UserFileIndexingAdapter:
|
||||
# Store the plaintext in the file store for faster retrieval
|
||||
# NOTE: this creates its own session to avoid committing the overall
|
||||
# transaction.
|
||||
for user_file_id, raw_text in result.user_file_id_to_raw_text.items():
|
||||
for user_file_id, raw_text in enrichment.user_file_id_to_raw_text.items():
|
||||
store_user_file_plaintext(
|
||||
user_file_id=UUID(user_file_id),
|
||||
plaintext_content=raw_text,
|
||||
)
|
||||
|
||||
|
||||
class UserFileChunkEnricher:
|
||||
"""Pre-computed metadata for per-chunk enrichment of user-uploaded files."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
user_file_id_to_access: dict[str, DocumentAccess],
|
||||
user_file_id_to_project_ids: dict[str, list[int]],
|
||||
user_file_id_to_persona_ids: dict[str, list[int]],
|
||||
doc_id_to_previous_chunk_cnt: dict[str, int],
|
||||
doc_id_to_new_chunk_cnt: dict[str, int],
|
||||
user_file_id_to_raw_text: dict[str, str],
|
||||
user_file_id_to_token_count: dict[str, int | None],
|
||||
no_access: DocumentAccess,
|
||||
tenant_id: str,
|
||||
) -> None:
|
||||
self._user_file_id_to_access = user_file_id_to_access
|
||||
self._user_file_id_to_project_ids = user_file_id_to_project_ids
|
||||
self._user_file_id_to_persona_ids = user_file_id_to_persona_ids
|
||||
self._no_access = no_access
|
||||
self._tenant_id = tenant_id
|
||||
self.doc_id_to_previous_chunk_cnt = doc_id_to_previous_chunk_cnt
|
||||
self.doc_id_to_new_chunk_cnt = doc_id_to_new_chunk_cnt
|
||||
self.user_file_id_to_raw_text = user_file_id_to_raw_text
|
||||
self.user_file_id_to_token_count = user_file_id_to_token_count
|
||||
|
||||
def enrich_chunk(
|
||||
self, chunk: IndexChunk, score: float
|
||||
) -> DocMetadataAwareIndexChunk:
|
||||
return DocMetadataAwareIndexChunk.from_index_chunk(
|
||||
index_chunk=chunk,
|
||||
access=self._user_file_id_to_access.get(
|
||||
chunk.source_document.id, self._no_access
|
||||
),
|
||||
document_sets=set(),
|
||||
user_project=self._user_file_id_to_project_ids.get(
|
||||
chunk.source_document.id, []
|
||||
),
|
||||
personas=self._user_file_id_to_persona_ids.get(
|
||||
chunk.source_document.id, []
|
||||
),
|
||||
boost=DEFAULT_BOOST,
|
||||
tenant_id=self._tenant_id,
|
||||
aggregated_chunk_boost_factor=score,
|
||||
)
|
||||
|
||||
89
backend/onyx/indexing/chunk_batch_store.py
Normal file
89
backend/onyx/indexing/chunk_batch_store.py
Normal file
@@ -0,0 +1,89 @@
|
||||
import pickle
|
||||
import shutil
|
||||
import tempfile
|
||||
from collections.abc import Iterator
|
||||
from pathlib import Path
|
||||
|
||||
from onyx.indexing.models import IndexChunk
|
||||
|
||||
|
||||
class ChunkBatchStore:
|
||||
"""Manages serialization of embedded chunks to a temporary directory.
|
||||
|
||||
Owns the temp directory lifetime and provides save/load/stream/scrub
|
||||
operations.
|
||||
|
||||
Use as a context manager to ensure cleanup::
|
||||
|
||||
with ChunkBatchStore() as store:
|
||||
store.save(chunks, batch_idx=0)
|
||||
for chunk in store.stream():
|
||||
...
|
||||
"""
|
||||
|
||||
_EXT = ".pkl"
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._tmpdir: Path | None = None
|
||||
|
||||
# -- context manager -----------------------------------------------------
|
||||
|
||||
def __enter__(self) -> "ChunkBatchStore":
|
||||
self._tmpdir = Path(tempfile.mkdtemp(prefix="onyx_embeddings_"))
|
||||
return self
|
||||
|
||||
def __exit__(self, *_exc: object) -> None:
|
||||
if self._tmpdir is not None:
|
||||
shutil.rmtree(self._tmpdir, ignore_errors=True)
|
||||
self._tmpdir = None
|
||||
|
||||
@property
|
||||
def _dir(self) -> Path:
|
||||
assert self._tmpdir is not None, "ChunkBatchStore used outside context manager"
|
||||
return self._tmpdir
|
||||
|
||||
# -- storage primitives --------------------------------------------------
|
||||
|
||||
def save(self, chunks: list[IndexChunk], batch_idx: int) -> None:
|
||||
"""Serialize a batch of embedded chunks to disk."""
|
||||
with open(self._dir / f"batch_{batch_idx}{self._EXT}", "wb") as f:
|
||||
pickle.dump(chunks, f)
|
||||
|
||||
def _load(self, batch_file: Path) -> list[IndexChunk]:
|
||||
"""Deserialize a batch of embedded chunks from a file."""
|
||||
with open(batch_file, "rb") as f:
|
||||
return pickle.load(f)
|
||||
|
||||
def _batch_files(self) -> list[Path]:
|
||||
"""Return batch files sorted by numeric index."""
|
||||
return sorted(
|
||||
self._dir.glob(f"batch_*{self._EXT}"),
|
||||
key=lambda p: int(p.stem.removeprefix("batch_")),
|
||||
)
|
||||
|
||||
# -- higher-level operations ---------------------------------------------
|
||||
|
||||
def stream(self) -> Iterator[IndexChunk]:
|
||||
"""Yield all chunks across all batch files.
|
||||
|
||||
Each call returns a fresh generator, so the data can be iterated
|
||||
multiple times (e.g. once per document index).
|
||||
"""
|
||||
for batch_file in self._batch_files():
|
||||
yield from self._load(batch_file)
|
||||
|
||||
def scrub_failed_docs(self, failed_doc_ids: set[str]) -> None:
|
||||
"""Remove chunks belonging to *failed_doc_ids* from all batch files.
|
||||
|
||||
When a document fails embedding in batch N, earlier batches may
|
||||
already contain successfully embedded chunks for that document.
|
||||
This ensures the output is all-or-nothing per document.
|
||||
"""
|
||||
for batch_file in self._batch_files():
|
||||
batch_chunks = self._load(batch_file)
|
||||
cleaned = [
|
||||
c for c in batch_chunks if c.source_document.id not in failed_doc_ids
|
||||
]
|
||||
if len(cleaned) != len(batch_chunks):
|
||||
with open(batch_file, "wb") as f:
|
||||
pickle.dump(cleaned, f)
|
||||
@@ -1,5 +1,8 @@
|
||||
from collections import defaultdict
|
||||
from collections.abc import Callable
|
||||
from collections.abc import Generator
|
||||
from collections.abc import Iterator
|
||||
from contextlib import contextmanager
|
||||
from typing import Protocol
|
||||
|
||||
from pydantic import BaseModel
|
||||
@@ -9,6 +12,7 @@ from sqlalchemy.orm import Session
|
||||
from onyx.configs.app_configs import DEFAULT_CONTEXTUAL_RAG_LLM_NAME
|
||||
from onyx.configs.app_configs import DEFAULT_CONTEXTUAL_RAG_LLM_PROVIDER
|
||||
from onyx.configs.app_configs import ENABLE_CONTEXTUAL_RAG
|
||||
from onyx.configs.app_configs import MAX_CHUNKS_PER_DOC_BATCH
|
||||
from onyx.configs.app_configs import MAX_DOCUMENT_CHARS
|
||||
from onyx.configs.app_configs import MAX_TOKENS_FOR_FULL_INCLUSION
|
||||
from onyx.configs.app_configs import USE_CHUNK_SUMMARY
|
||||
@@ -43,10 +47,12 @@ from onyx.document_index.interfaces import DocumentMetadata
|
||||
from onyx.document_index.interfaces import IndexBatchParams
|
||||
from onyx.file_processing.image_summarization import summarize_image_with_error_handling
|
||||
from onyx.file_store.file_store import get_default_file_store
|
||||
from onyx.indexing.chunk_batch_store import ChunkBatchStore
|
||||
from onyx.indexing.chunker import Chunker
|
||||
from onyx.indexing.embedder import embed_chunks_with_failure_handling
|
||||
from onyx.indexing.embedder import IndexingEmbedder
|
||||
from onyx.indexing.models import DocAwareChunk
|
||||
from onyx.indexing.models import DocMetadataAwareIndexChunk
|
||||
from onyx.indexing.models import IndexingBatchAdapter
|
||||
from onyx.indexing.models import UpdatableChunkData
|
||||
from onyx.indexing.vector_db_insertion import write_chunks_to_vector_db_with_backoff
|
||||
@@ -63,6 +69,7 @@ from onyx.natural_language_processing.utils import tokenizer_trim_middle
|
||||
from onyx.prompts.contextual_retrieval import CONTEXTUAL_RAG_PROMPT1
|
||||
from onyx.prompts.contextual_retrieval import CONTEXTUAL_RAG_PROMPT2
|
||||
from onyx.prompts.contextual_retrieval import DOCUMENT_SUMMARY_PROMPT
|
||||
from onyx.utils.batching import batch_generator
|
||||
from onyx.utils.logger import setup_logger
|
||||
from onyx.utils.postgres_sanitization import sanitize_documents_for_postgres
|
||||
from onyx.utils.threadpool_concurrency import run_functions_tuples_in_parallel
|
||||
@@ -91,6 +98,20 @@ class IndexingPipelineResult(BaseModel):
|
||||
|
||||
failures: list[ConnectorFailure]
|
||||
|
||||
@classmethod
|
||||
def empty(cls, total_docs: int) -> "IndexingPipelineResult":
|
||||
return cls(
|
||||
new_docs=0,
|
||||
total_docs=total_docs,
|
||||
total_chunks=0,
|
||||
failures=[],
|
||||
)
|
||||
|
||||
|
||||
class ChunkEmbeddingResult(BaseModel):
|
||||
successful_chunk_ids: list[tuple[int, str]] # (chunk_id, document_id)
|
||||
connector_failures: list[ConnectorFailure]
|
||||
|
||||
|
||||
class IndexingPipelineProtocol(Protocol):
|
||||
def __call__(
|
||||
@@ -139,6 +160,110 @@ def _upsert_documents_in_db(
|
||||
)
|
||||
|
||||
|
||||
def _get_failed_doc_ids(failures: list[ConnectorFailure]) -> set[str]:
|
||||
"""Extract document IDs from a list of connector failures."""
|
||||
return {f.failed_document.document_id for f in failures if f.failed_document}
|
||||
|
||||
|
||||
def _embed_chunks_to_store(
|
||||
chunks: list[DocAwareChunk],
|
||||
embedder: IndexingEmbedder,
|
||||
tenant_id: str,
|
||||
request_id: str | None,
|
||||
store: ChunkBatchStore,
|
||||
) -> ChunkEmbeddingResult:
|
||||
"""Embed chunks in batches, spilling each batch to *store*.
|
||||
|
||||
If a document fails embedding in any batch, its chunks are excluded from
|
||||
all batches (including earlier ones already written) so that the output
|
||||
is all-or-nothing per document.
|
||||
"""
|
||||
successful_chunk_ids: list[tuple[int, str]] = []
|
||||
all_embedding_failures: list[ConnectorFailure] = []
|
||||
# Track failed doc IDs across all batches so that a failure in batch N
|
||||
# causes chunks for that doc to be skipped in batch N+1 and stripped
|
||||
# from earlier batches.
|
||||
all_failed_doc_ids: set[str] = set()
|
||||
|
||||
for batch_idx, chunk_batch in enumerate(
|
||||
batch_generator(chunks, MAX_CHUNKS_PER_DOC_BATCH)
|
||||
):
|
||||
# Skip chunks belonging to documents that failed in earlier batches.
|
||||
chunk_batch = [
|
||||
c for c in chunk_batch if c.source_document.id not in all_failed_doc_ids
|
||||
]
|
||||
if not chunk_batch:
|
||||
continue
|
||||
|
||||
logger.debug(f"Embedding batch {batch_idx}: {len(chunk_batch)} chunks")
|
||||
|
||||
chunks_with_embeddings, embedding_failures = embed_chunks_with_failure_handling(
|
||||
chunks=chunk_batch,
|
||||
embedder=embedder,
|
||||
tenant_id=tenant_id,
|
||||
request_id=request_id,
|
||||
)
|
||||
all_embedding_failures.extend(embedding_failures)
|
||||
all_failed_doc_ids.update(_get_failed_doc_ids(embedding_failures))
|
||||
|
||||
# Only keep successfully embedded chunks for non-failed docs.
|
||||
chunks_with_embeddings = [
|
||||
c
|
||||
for c in chunks_with_embeddings
|
||||
if c.source_document.id not in all_failed_doc_ids
|
||||
]
|
||||
|
||||
successful_chunk_ids.extend(
|
||||
(c.chunk_id, c.source_document.id) for c in chunks_with_embeddings
|
||||
)
|
||||
|
||||
store.save(chunks_with_embeddings, batch_idx)
|
||||
del chunks_with_embeddings
|
||||
|
||||
# Scrub earlier batches for docs that failed in later batches.
|
||||
if all_failed_doc_ids:
|
||||
store.scrub_failed_docs(all_failed_doc_ids)
|
||||
successful_chunk_ids = [
|
||||
(chunk_id, doc_id)
|
||||
for chunk_id, doc_id in successful_chunk_ids
|
||||
if doc_id not in all_failed_doc_ids
|
||||
]
|
||||
|
||||
return ChunkEmbeddingResult(
|
||||
successful_chunk_ids=successful_chunk_ids,
|
||||
connector_failures=all_embedding_failures,
|
||||
)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def embed_and_stream(
|
||||
chunks: list[DocAwareChunk],
|
||||
embedder: IndexingEmbedder,
|
||||
tenant_id: str,
|
||||
request_id: str | None,
|
||||
) -> Generator[tuple[ChunkEmbeddingResult, ChunkBatchStore], None, None]:
|
||||
"""Embed chunks to disk and yield a ``(result, store)`` pair.
|
||||
|
||||
The store owns the temp directory — files are cleaned up when the context
|
||||
manager exits.
|
||||
|
||||
Usage::
|
||||
|
||||
with embed_and_stream(chunks, embedder, tenant_id, req_id) as (result, store):
|
||||
for chunk in store.stream():
|
||||
...
|
||||
"""
|
||||
with ChunkBatchStore() as store:
|
||||
result = _embed_chunks_to_store(
|
||||
chunks=chunks,
|
||||
embedder=embedder,
|
||||
tenant_id=tenant_id,
|
||||
request_id=request_id,
|
||||
store=store,
|
||||
)
|
||||
yield result, store
|
||||
|
||||
|
||||
def get_doc_ids_to_update(
|
||||
documents: list[Document], db_docs: list[DBDocument]
|
||||
) -> list[Document]:
|
||||
@@ -637,6 +762,29 @@ def add_contextual_summaries(
|
||||
return chunks
|
||||
|
||||
|
||||
def _verify_indexing_completeness(
|
||||
insertion_records: list[DocumentInsertionRecord],
|
||||
write_failures: list[ConnectorFailure],
|
||||
embedding_failed_doc_ids: set[str],
|
||||
updatable_ids: list[str],
|
||||
document_index_name: str,
|
||||
) -> None:
|
||||
"""Verify that every updatable document was either indexed or reported as failed."""
|
||||
all_returned_doc_ids = (
|
||||
{r.document_id for r in insertion_records}
|
||||
| {f.failed_document.document_id for f in write_failures if f.failed_document}
|
||||
| embedding_failed_doc_ids
|
||||
)
|
||||
if all_returned_doc_ids != set(updatable_ids):
|
||||
raise RuntimeError(
|
||||
f"Some documents were not successfully indexed. "
|
||||
f"Updatable IDs: {updatable_ids}, "
|
||||
f"Returned IDs: {all_returned_doc_ids}. "
|
||||
f"This should never happen. "
|
||||
f"This occured for document index {document_index_name}"
|
||||
)
|
||||
|
||||
|
||||
@log_function_time(debug_only=True)
|
||||
def index_doc_batch(
|
||||
*,
|
||||
@@ -672,12 +820,7 @@ def index_doc_batch(
|
||||
filtered_documents = filter_fnc(document_batch)
|
||||
context = adapter.prepare(filtered_documents, ignore_time_skip)
|
||||
if not context:
|
||||
return IndexingPipelineResult(
|
||||
new_docs=0,
|
||||
total_docs=len(filtered_documents),
|
||||
total_chunks=0,
|
||||
failures=[],
|
||||
)
|
||||
return IndexingPipelineResult.empty(len(filtered_documents))
|
||||
|
||||
# Convert documents to IndexingDocument objects with processed section
|
||||
# logger.debug("Processing image sections")
|
||||
@@ -716,117 +859,99 @@ def index_doc_batch(
|
||||
)
|
||||
|
||||
logger.debug("Starting embedding")
|
||||
chunks_with_embeddings, embedding_failures = (
|
||||
embed_chunks_with_failure_handling(
|
||||
chunks=chunks,
|
||||
embedder=embedder,
|
||||
tenant_id=tenant_id,
|
||||
request_id=request_id,
|
||||
)
|
||||
if chunks
|
||||
else ([], [])
|
||||
)
|
||||
with embed_and_stream(chunks, embedder, tenant_id, request_id) as (
|
||||
embedding_result,
|
||||
chunk_store,
|
||||
):
|
||||
updatable_ids = [doc.id for doc in context.updatable_docs]
|
||||
updatable_chunk_data = [
|
||||
UpdatableChunkData(
|
||||
chunk_id=chunk_id,
|
||||
document_id=document_id,
|
||||
boost_score=1.0,
|
||||
)
|
||||
for chunk_id, document_id in embedding_result.successful_chunk_ids
|
||||
]
|
||||
|
||||
chunk_content_scores = [1.0] * len(chunks_with_embeddings)
|
||||
|
||||
updatable_ids = [doc.id for doc in context.updatable_docs]
|
||||
updatable_chunk_data = [
|
||||
UpdatableChunkData(
|
||||
chunk_id=chunk.chunk_id,
|
||||
document_id=chunk.source_document.id,
|
||||
boost_score=score,
|
||||
)
|
||||
for chunk, score in zip(chunks_with_embeddings, chunk_content_scores)
|
||||
]
|
||||
|
||||
# Acquires a lock on the documents so that no other process can modify them
|
||||
# NOTE: don't need to acquire till here, since this is when the actual race condition
|
||||
# with Vespa can occur.
|
||||
with adapter.lock_context(context.updatable_docs):
|
||||
# we're concerned about race conditions where multiple simultaneous indexings might result
|
||||
# in one set of metadata overwriting another one in vespa.
|
||||
# we still write data here for the immediate and most likely correct sync, but
|
||||
# to resolve this, an update of the last modified field at the end of this loop
|
||||
# always triggers a final metadata sync via the celery queue
|
||||
result = adapter.build_metadata_aware_chunks(
|
||||
chunks_with_embeddings=chunks_with_embeddings,
|
||||
chunk_content_scores=chunk_content_scores,
|
||||
tenant_id=tenant_id,
|
||||
context=context,
|
||||
embedding_failed_doc_ids = _get_failed_doc_ids(
|
||||
embedding_result.connector_failures
|
||||
)
|
||||
|
||||
short_descriptor_list = [chunk.to_short_descriptor() for chunk in result.chunks]
|
||||
short_descriptor_log = str(short_descriptor_list)[:1024]
|
||||
logger.debug(f"Indexing the following chunks: {short_descriptor_log}")
|
||||
# Filter to only successfully embedded chunks so
|
||||
# doc_id_to_new_chunk_cnt reflects what's actually written to Vespa.
|
||||
embedded_chunks = [
|
||||
c for c in chunks if c.source_document.id not in embedding_failed_doc_ids
|
||||
]
|
||||
|
||||
primary_doc_idx_insertion_records: list[DocumentInsertionRecord] | None = None
|
||||
primary_doc_idx_vector_db_write_failures: list[ConnectorFailure] | None = None
|
||||
for document_index in document_indices:
|
||||
# A document will not be spread across different batches, so all the
|
||||
# documents with chunks in this set, are fully represented by the chunks
|
||||
# in this set
|
||||
(
|
||||
insertion_records,
|
||||
vector_db_write_failures,
|
||||
) = write_chunks_to_vector_db_with_backoff(
|
||||
document_index=document_index,
|
||||
chunks=result.chunks,
|
||||
index_batch_params=IndexBatchParams(
|
||||
doc_id_to_previous_chunk_cnt=result.doc_id_to_previous_chunk_cnt,
|
||||
doc_id_to_new_chunk_cnt=result.doc_id_to_new_chunk_cnt,
|
||||
tenant_id=tenant_id,
|
||||
large_chunks_enabled=chunker.enable_large_chunks,
|
||||
),
|
||||
# Acquires a lock on the documents so that no other process can modify
|
||||
# them. Not needed until here, since this is when the actual race
|
||||
# condition with vector db can occur.
|
||||
with adapter.lock_context(context.updatable_docs):
|
||||
enricher = adapter.prepare_enrichment(
|
||||
context=context,
|
||||
tenant_id=tenant_id,
|
||||
chunks=embedded_chunks,
|
||||
)
|
||||
|
||||
all_returned_doc_ids: set[str] = (
|
||||
{record.document_id for record in insertion_records}
|
||||
.union(
|
||||
{
|
||||
record.failed_document.document_id
|
||||
for record in vector_db_write_failures
|
||||
if record.failed_document
|
||||
}
|
||||
)
|
||||
.union(
|
||||
{
|
||||
record.failed_document.document_id
|
||||
for record in embedding_failures
|
||||
if record.failed_document
|
||||
}
|
||||
)
|
||||
index_batch_params = IndexBatchParams(
|
||||
doc_id_to_previous_chunk_cnt=enricher.doc_id_to_previous_chunk_cnt,
|
||||
doc_id_to_new_chunk_cnt=enricher.doc_id_to_new_chunk_cnt,
|
||||
tenant_id=tenant_id,
|
||||
large_chunks_enabled=chunker.enable_large_chunks,
|
||||
)
|
||||
if all_returned_doc_ids != set(updatable_ids):
|
||||
raise RuntimeError(
|
||||
f"Some documents were not successfully indexed. "
|
||||
f"Updatable IDs: {updatable_ids}, "
|
||||
f"Returned IDs: {all_returned_doc_ids}. "
|
||||
"This should never happen."
|
||||
f"This occured for document index {document_index.__class__.__name__}"
|
||||
)
|
||||
# We treat the first document index we got as the primary one used
|
||||
# for reporting the state of indexing.
|
||||
if primary_doc_idx_insertion_records is None:
|
||||
primary_doc_idx_insertion_records = insertion_records
|
||||
if primary_doc_idx_vector_db_write_failures is None:
|
||||
primary_doc_idx_vector_db_write_failures = vector_db_write_failures
|
||||
|
||||
adapter.post_index(
|
||||
context=context,
|
||||
updatable_chunk_data=updatable_chunk_data,
|
||||
filtered_documents=filtered_documents,
|
||||
result=result,
|
||||
)
|
||||
primary_doc_idx_insertion_records: list[DocumentInsertionRecord] | None = (
|
||||
None
|
||||
)
|
||||
primary_doc_idx_vector_db_write_failures: list[ConnectorFailure] | None = (
|
||||
None
|
||||
)
|
||||
|
||||
for document_index in document_indices:
|
||||
|
||||
def _enriched_stream() -> Iterator[DocMetadataAwareIndexChunk]:
|
||||
for chunk in chunk_store.stream():
|
||||
yield enricher.enrich_chunk(chunk, 1.0)
|
||||
|
||||
insertion_records, write_failures = (
|
||||
write_chunks_to_vector_db_with_backoff(
|
||||
document_index=document_index,
|
||||
make_chunks=_enriched_stream,
|
||||
index_batch_params=index_batch_params,
|
||||
)
|
||||
)
|
||||
|
||||
_verify_indexing_completeness(
|
||||
insertion_records=insertion_records,
|
||||
write_failures=write_failures,
|
||||
embedding_failed_doc_ids=embedding_failed_doc_ids,
|
||||
updatable_ids=updatable_ids,
|
||||
document_index_name=document_index.__class__.__name__,
|
||||
)
|
||||
# We treat the first document index we got as the primary one used
|
||||
# for reporting the state of indexing.
|
||||
if primary_doc_idx_insertion_records is None:
|
||||
primary_doc_idx_insertion_records = insertion_records
|
||||
if primary_doc_idx_vector_db_write_failures is None:
|
||||
primary_doc_idx_vector_db_write_failures = write_failures
|
||||
|
||||
adapter.post_index(
|
||||
context=context,
|
||||
updatable_chunk_data=updatable_chunk_data,
|
||||
filtered_documents=filtered_documents,
|
||||
enrichment=enricher,
|
||||
)
|
||||
|
||||
assert primary_doc_idx_insertion_records is not None
|
||||
assert primary_doc_idx_vector_db_write_failures is not None
|
||||
return IndexingPipelineResult(
|
||||
new_docs=len(
|
||||
[r for r in primary_doc_idx_insertion_records if not r.already_existed]
|
||||
new_docs=sum(
|
||||
1 for r in primary_doc_idx_insertion_records if not r.already_existed
|
||||
),
|
||||
total_docs=len(filtered_documents),
|
||||
total_chunks=len(chunks_with_embeddings),
|
||||
failures=primary_doc_idx_vector_db_write_failures + embedding_failures,
|
||||
total_chunks=len(embedding_result.successful_chunk_ids),
|
||||
failures=primary_doc_idx_vector_db_write_failures
|
||||
+ embedding_result.connector_failures,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -235,12 +235,16 @@ class UpdatableChunkData(BaseModel):
|
||||
boost_score: float
|
||||
|
||||
|
||||
class BuildMetadataAwareChunksResult(BaseModel):
|
||||
chunks: list[DocMetadataAwareIndexChunk]
|
||||
class ChunkEnrichmentContext(Protocol):
|
||||
"""Returned by prepare_enrichment. Holds pre-computed metadata lookups
|
||||
and provides per-chunk enrichment."""
|
||||
|
||||
doc_id_to_previous_chunk_cnt: dict[str, int]
|
||||
doc_id_to_new_chunk_cnt: dict[str, int]
|
||||
user_file_id_to_raw_text: dict[str, str]
|
||||
user_file_id_to_token_count: dict[str, int | None]
|
||||
|
||||
def enrich_chunk(
|
||||
self, chunk: IndexChunk, score: float
|
||||
) -> DocMetadataAwareIndexChunk: ...
|
||||
|
||||
|
||||
class IndexingBatchAdapter(Protocol):
|
||||
@@ -254,18 +258,24 @@ class IndexingBatchAdapter(Protocol):
|
||||
) -> Generator[TransactionalContext, None, None]:
|
||||
"""Provide a transaction/row-lock context for critical updates."""
|
||||
|
||||
def build_metadata_aware_chunks(
|
||||
def prepare_enrichment(
|
||||
self,
|
||||
chunks_with_embeddings: list[IndexChunk],
|
||||
chunk_content_scores: list[float],
|
||||
tenant_id: str,
|
||||
context: "DocumentBatchPrepareContext",
|
||||
) -> BuildMetadataAwareChunksResult: ...
|
||||
tenant_id: str,
|
||||
chunks: list[DocAwareChunk],
|
||||
) -> ChunkEnrichmentContext:
|
||||
"""Prepare per-chunk enrichment data (access, document sets, boost, etc.).
|
||||
|
||||
Precondition: ``chunks`` have already been through the embedding step
|
||||
(i.e. they are ``IndexChunk`` instances with populated embeddings,
|
||||
passed here as the base ``DocAwareChunk`` type).
|
||||
"""
|
||||
...
|
||||
|
||||
def post_index(
|
||||
self,
|
||||
context: "DocumentBatchPrepareContext",
|
||||
updatable_chunk_data: list[UpdatableChunkData],
|
||||
filtered_documents: list[Document],
|
||||
result: BuildMetadataAwareChunksResult,
|
||||
enrichment: ChunkEnrichmentContext,
|
||||
) -> None: ...
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
import time
|
||||
from collections import defaultdict
|
||||
from collections.abc import Callable
|
||||
from collections.abc import Iterable
|
||||
from http import HTTPStatus
|
||||
from itertools import chain
|
||||
from itertools import groupby
|
||||
|
||||
import httpx
|
||||
|
||||
@@ -28,22 +31,22 @@ def _log_insufficient_storage_error(e: Exception) -> None:
|
||||
|
||||
def write_chunks_to_vector_db_with_backoff(
|
||||
document_index: DocumentIndex,
|
||||
chunks: list[DocMetadataAwareIndexChunk],
|
||||
make_chunks: Callable[[], Iterable[DocMetadataAwareIndexChunk]],
|
||||
index_batch_params: IndexBatchParams,
|
||||
) -> tuple[list[DocumentInsertionRecord], list[ConnectorFailure]]:
|
||||
"""Tries to insert all chunks in one large batch. If that batch fails for any reason,
|
||||
goes document by document to isolate the failure(s).
|
||||
|
||||
IMPORTANT: must pass in whole documents at a time not individual chunks, since the
|
||||
vector DB interface assumes that all chunks for a single document are present.
|
||||
vector DB interface assumes that all chunks for a single document are present. The
|
||||
chunks must also be in contiguous batches
|
||||
"""
|
||||
|
||||
# first try to write the chunks to the vector db
|
||||
try:
|
||||
return (
|
||||
list(
|
||||
document_index.index(
|
||||
chunks=chunks,
|
||||
chunks=make_chunks(),
|
||||
index_batch_params=index_batch_params,
|
||||
)
|
||||
),
|
||||
@@ -60,14 +63,23 @@ def write_chunks_to_vector_db_with_backoff(
|
||||
# wait a couple seconds just to give the vector db a chance to recover
|
||||
time.sleep(2)
|
||||
|
||||
# try writing each doc one by one
|
||||
chunks_for_docs: dict[str, list[DocMetadataAwareIndexChunk]] = defaultdict(list)
|
||||
for chunk in chunks:
|
||||
chunks_for_docs[chunk.source_document.id].append(chunk)
|
||||
|
||||
insertion_records: list[DocumentInsertionRecord] = []
|
||||
failures: list[ConnectorFailure] = []
|
||||
for doc_id, chunks_for_doc in chunks_for_docs.items():
|
||||
|
||||
def key(chunk: DocMetadataAwareIndexChunk) -> str:
|
||||
return chunk.source_document.id
|
||||
|
||||
seen_doc_ids: set[str] = set()
|
||||
for doc_id, chunks_for_doc in groupby(make_chunks(), key=key):
|
||||
if doc_id in seen_doc_ids:
|
||||
raise RuntimeError(
|
||||
f"Doc chunks are not arriving in order. Current doc_id={doc_id}, seen_doc_ids={list(seen_doc_ids)}"
|
||||
)
|
||||
seen_doc_ids.add(doc_id)
|
||||
|
||||
first_chunk = next(chunks_for_doc)
|
||||
chunks_for_doc = chain([first_chunk], chunks_for_doc)
|
||||
|
||||
try:
|
||||
insertion_records.extend(
|
||||
document_index.index(
|
||||
@@ -87,9 +99,7 @@ def write_chunks_to_vector_db_with_backoff(
|
||||
ConnectorFailure(
|
||||
failed_document=DocumentFailure(
|
||||
document_id=doc_id,
|
||||
document_link=(
|
||||
chunks_for_doc[0].get_link() if chunks_for_doc else None
|
||||
),
|
||||
document_link=first_chunk.get_link(),
|
||||
),
|
||||
failure_message=str(e),
|
||||
exception=e,
|
||||
|
||||
@@ -439,6 +439,7 @@ def get_application(lifespan_override: Lifespan | None = None) -> FastAPI:
|
||||
dsn=SENTRY_DSN,
|
||||
integrations=[StarletteIntegration(), FastApiIntegration()],
|
||||
traces_sample_rate=0.1,
|
||||
release=__version__,
|
||||
)
|
||||
logger.info("Sentry initialized")
|
||||
else:
|
||||
|
||||
@@ -735,7 +735,7 @@ pyee==13.0.0
|
||||
# via playwright
|
||||
pygithub==2.5.0
|
||||
# via onyx
|
||||
pygments==2.19.2
|
||||
pygments==2.20.0
|
||||
# via rich
|
||||
pyjwt==2.12.0
|
||||
# via
|
||||
|
||||
@@ -349,7 +349,7 @@ pydantic-core==2.33.2
|
||||
# via pydantic
|
||||
pydantic-settings==2.12.0
|
||||
# via mcp
|
||||
pygments==2.19.2
|
||||
pygments==2.20.0
|
||||
# via
|
||||
# ipython
|
||||
# ipython-pygments-lexers
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
"""
|
||||
External dependency unit tests for UserFileIndexingAdapter metadata writing.
|
||||
|
||||
Validates that build_metadata_aware_chunks produces DocMetadataAwareIndexChunk
|
||||
Validates that prepare_enrichment produces DocMetadataAwareIndexChunk
|
||||
objects with both `user_project` and `personas` fields populated correctly
|
||||
based on actual DB associations.
|
||||
|
||||
@@ -127,7 +127,7 @@ def _make_index_chunk(user_file: UserFile) -> IndexChunk:
|
||||
|
||||
|
||||
class TestAdapterWritesBothMetadataFields:
|
||||
"""build_metadata_aware_chunks must populate user_project AND personas."""
|
||||
"""prepare_enrichment must populate user_project AND personas."""
|
||||
|
||||
@patch(
|
||||
"onyx.indexing.adapters.user_file_indexing_adapter.get_default_llm",
|
||||
@@ -153,15 +153,13 @@ class TestAdapterWritesBothMetadataFields:
|
||||
doc = chunk.source_document
|
||||
context = DocumentBatchPrepareContext(updatable_docs=[doc], id_to_boost_map={})
|
||||
|
||||
result = adapter.build_metadata_aware_chunks(
|
||||
chunks_with_embeddings=[chunk],
|
||||
chunk_content_scores=[1.0],
|
||||
tenant_id=TEST_TENANT_ID,
|
||||
enricher = adapter.prepare_enrichment(
|
||||
context=context,
|
||||
tenant_id=TEST_TENANT_ID,
|
||||
chunks=[chunk],
|
||||
)
|
||||
aware_chunk = enricher.enrich_chunk(chunk, 1.0)
|
||||
|
||||
assert len(result.chunks) == 1
|
||||
aware_chunk = result.chunks[0]
|
||||
assert persona.id in aware_chunk.personas
|
||||
assert aware_chunk.user_project == []
|
||||
|
||||
@@ -190,15 +188,13 @@ class TestAdapterWritesBothMetadataFields:
|
||||
updatable_docs=[chunk.source_document], id_to_boost_map={}
|
||||
)
|
||||
|
||||
result = adapter.build_metadata_aware_chunks(
|
||||
chunks_with_embeddings=[chunk],
|
||||
chunk_content_scores=[1.0],
|
||||
tenant_id=TEST_TENANT_ID,
|
||||
enricher = adapter.prepare_enrichment(
|
||||
context=context,
|
||||
tenant_id=TEST_TENANT_ID,
|
||||
chunks=[chunk],
|
||||
)
|
||||
aware_chunk = enricher.enrich_chunk(chunk, 1.0)
|
||||
|
||||
assert len(result.chunks) == 1
|
||||
aware_chunk = result.chunks[0]
|
||||
assert project.id in aware_chunk.user_project
|
||||
assert aware_chunk.personas == []
|
||||
|
||||
@@ -229,14 +225,13 @@ class TestAdapterWritesBothMetadataFields:
|
||||
updatable_docs=[chunk.source_document], id_to_boost_map={}
|
||||
)
|
||||
|
||||
result = adapter.build_metadata_aware_chunks(
|
||||
chunks_with_embeddings=[chunk],
|
||||
chunk_content_scores=[1.0],
|
||||
tenant_id=TEST_TENANT_ID,
|
||||
enricher = adapter.prepare_enrichment(
|
||||
context=context,
|
||||
tenant_id=TEST_TENANT_ID,
|
||||
chunks=[chunk],
|
||||
)
|
||||
aware_chunk = enricher.enrich_chunk(chunk, 1.0)
|
||||
|
||||
aware_chunk = result.chunks[0]
|
||||
assert persona.id in aware_chunk.personas
|
||||
assert project.id in aware_chunk.user_project
|
||||
|
||||
@@ -261,14 +256,13 @@ class TestAdapterWritesBothMetadataFields:
|
||||
updatable_docs=[chunk.source_document], id_to_boost_map={}
|
||||
)
|
||||
|
||||
result = adapter.build_metadata_aware_chunks(
|
||||
chunks_with_embeddings=[chunk],
|
||||
chunk_content_scores=[1.0],
|
||||
tenant_id=TEST_TENANT_ID,
|
||||
enricher = adapter.prepare_enrichment(
|
||||
context=context,
|
||||
tenant_id=TEST_TENANT_ID,
|
||||
chunks=[chunk],
|
||||
)
|
||||
aware_chunk = enricher.enrich_chunk(chunk, 1.0)
|
||||
|
||||
aware_chunk = result.chunks[0]
|
||||
assert aware_chunk.personas == []
|
||||
assert aware_chunk.user_project == []
|
||||
|
||||
@@ -300,12 +294,11 @@ class TestAdapterWritesBothMetadataFields:
|
||||
updatable_docs=[chunk.source_document], id_to_boost_map={}
|
||||
)
|
||||
|
||||
result = adapter.build_metadata_aware_chunks(
|
||||
chunks_with_embeddings=[chunk],
|
||||
chunk_content_scores=[1.0],
|
||||
tenant_id=TEST_TENANT_ID,
|
||||
enricher = adapter.prepare_enrichment(
|
||||
context=context,
|
||||
tenant_id=TEST_TENANT_ID,
|
||||
chunks=[chunk],
|
||||
)
|
||||
aware_chunk = enricher.enrich_chunk(chunk, 1.0)
|
||||
|
||||
aware_chunk = result.chunks[0]
|
||||
assert set(aware_chunk.personas) == {persona_a.id, persona_b.id}
|
||||
|
||||
0
backend/tests/unit/ee/onyx/background/__init__.py
Normal file
0
backend/tests/unit/ee/onyx/background/__init__.py
Normal file
@@ -0,0 +1,109 @@
|
||||
"""Tests for TTL management task resilience."""
|
||||
|
||||
from typing import Any
|
||||
from unittest.mock import MagicMock
|
||||
from unittest.mock import patch
|
||||
from uuid import uuid4
|
||||
|
||||
|
||||
_TASK_MODULE = "ee.onyx.background.celery.tasks.ttl_management.tasks"
|
||||
|
||||
|
||||
def _setup_db_session_mock(mock_get_db_session: MagicMock) -> None:
|
||||
mock_db_session = MagicMock()
|
||||
mock_get_db_session.return_value.__enter__ = MagicMock(
|
||||
return_value=mock_db_session
|
||||
)
|
||||
mock_get_db_session.return_value.__exit__ = MagicMock(return_value=False)
|
||||
|
||||
|
||||
@patch(f"{_TASK_MODULE}.get_session_with_current_tenant")
|
||||
@patch(f"{_TASK_MODULE}.delete_chat_session")
|
||||
@patch(f"{_TASK_MODULE}.get_chat_sessions_older_than")
|
||||
@patch(f"{_TASK_MODULE}.mark_task_as_finished_with_id")
|
||||
@patch(f"{_TASK_MODULE}.register_task")
|
||||
def test_ttl_task_continues_after_session_delete_failure(
|
||||
_mock_register: Any,
|
||||
mock_mark_finished: MagicMock,
|
||||
mock_get_old_sessions: MagicMock,
|
||||
mock_delete_session: MagicMock,
|
||||
mock_get_db_session: MagicMock,
|
||||
) -> None:
|
||||
"""One failing session should not prevent cleanup of remaining sessions."""
|
||||
from ee.onyx.background.celery.tasks.ttl_management.tasks import (
|
||||
perform_ttl_management_task,
|
||||
)
|
||||
|
||||
user1, session1 = uuid4(), uuid4()
|
||||
user2, session2 = uuid4(), uuid4()
|
||||
user3, session3 = uuid4(), uuid4()
|
||||
|
||||
mock_get_old_sessions.return_value = [
|
||||
(user1, session1),
|
||||
(user2, session2),
|
||||
(user3, session3),
|
||||
]
|
||||
|
||||
# Second session fails
|
||||
mock_delete_session.side_effect = [
|
||||
None,
|
||||
RuntimeError("File does not exist"),
|
||||
None,
|
||||
]
|
||||
|
||||
_setup_db_session_mock(mock_get_db_session)
|
||||
|
||||
mock_task = MagicMock()
|
||||
mock_task.request.id = "test-task-id"
|
||||
|
||||
# Call the underlying function directly, bypassing Celery decorator
|
||||
perform_ttl_management_task.__wrapped__(
|
||||
mock_task, retention_limit_days=30, tenant_id="test"
|
||||
)
|
||||
|
||||
# All three sessions should have been attempted
|
||||
assert mock_delete_session.call_count == 3
|
||||
|
||||
# Task marked as finished with success=False (due to the one failure)
|
||||
mock_mark_finished.assert_called()
|
||||
finish_call_kwargs = mock_mark_finished.call_args[1]
|
||||
assert finish_call_kwargs["success"] is False
|
||||
|
||||
|
||||
@patch(f"{_TASK_MODULE}.get_session_with_current_tenant")
|
||||
@patch(f"{_TASK_MODULE}.delete_chat_session")
|
||||
@patch(f"{_TASK_MODULE}.get_chat_sessions_older_than")
|
||||
@patch(f"{_TASK_MODULE}.mark_task_as_finished_with_id")
|
||||
@patch(f"{_TASK_MODULE}.register_task")
|
||||
def test_ttl_task_reports_success_when_all_deletions_pass(
|
||||
_mock_register: Any,
|
||||
mock_mark_finished: MagicMock,
|
||||
mock_get_old_sessions: MagicMock,
|
||||
mock_delete_session: MagicMock,
|
||||
mock_get_db_session: MagicMock,
|
||||
) -> None:
|
||||
"""Task should report success when all sessions are deleted."""
|
||||
from ee.onyx.background.celery.tasks.ttl_management.tasks import (
|
||||
perform_ttl_management_task,
|
||||
)
|
||||
|
||||
mock_get_old_sessions.return_value = [
|
||||
(uuid4(), uuid4()),
|
||||
(uuid4(), uuid4()),
|
||||
]
|
||||
mock_delete_session.side_effect = None
|
||||
|
||||
_setup_db_session_mock(mock_get_db_session)
|
||||
|
||||
mock_task = MagicMock()
|
||||
mock_task.request.id = "test-task-id"
|
||||
|
||||
perform_ttl_management_task.__wrapped__(
|
||||
mock_task, retention_limit_days=30, tenant_id="test"
|
||||
)
|
||||
|
||||
assert mock_delete_session.call_count == 2
|
||||
|
||||
mock_mark_finished.assert_called()
|
||||
finish_call_kwargs = mock_mark_finished.call_args[1]
|
||||
assert finish_call_kwargs["success"] is True
|
||||
@@ -1,15 +1,23 @@
|
||||
"""Tests for Canvas connector — client (PR1)."""
|
||||
"""Tests for Canvas connector — client, credentials, conversion."""
|
||||
|
||||
from datetime import datetime
|
||||
from datetime import timezone
|
||||
from typing import Any
|
||||
from unittest.mock import MagicMock
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
from onyx.configs.constants import DocumentSource
|
||||
from onyx.connectors.canvas.client import CanvasApiClient
|
||||
from onyx.connectors.canvas.connector import CanvasConnector
|
||||
from onyx.connectors.exceptions import ConnectorValidationError
|
||||
from onyx.connectors.exceptions import CredentialExpiredError
|
||||
from onyx.connectors.exceptions import InsufficientPermissionsError
|
||||
from onyx.connectors.exceptions import UnexpectedValidationError
|
||||
from onyx.connectors.models import ConnectorMissingCredentialError
|
||||
from onyx.error_handling.exceptions import OnyxError
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -18,6 +26,77 @@ FAKE_BASE_URL = "https://myschool.instructure.com"
|
||||
FAKE_TOKEN = "fake-canvas-token"
|
||||
|
||||
|
||||
def _mock_course(
|
||||
course_id: int = 1,
|
||||
name: str = "Intro to CS",
|
||||
course_code: str = "CS101",
|
||||
) -> dict[str, Any]:
|
||||
return {
|
||||
"id": course_id,
|
||||
"name": name,
|
||||
"course_code": course_code,
|
||||
"created_at": "2025-01-01T00:00:00Z",
|
||||
"workflow_state": "available",
|
||||
}
|
||||
|
||||
|
||||
def _build_connector(base_url: str = FAKE_BASE_URL) -> CanvasConnector:
|
||||
"""Build a connector with mocked credential validation."""
|
||||
with patch("onyx.connectors.canvas.client.rl_requests") as mock_req:
|
||||
mock_req.get.return_value = _mock_response(json_data=[_mock_course()])
|
||||
connector = CanvasConnector(canvas_base_url=base_url)
|
||||
connector.load_credentials({"canvas_access_token": FAKE_TOKEN})
|
||||
return connector
|
||||
|
||||
|
||||
def _mock_page(
|
||||
page_id: int = 10,
|
||||
title: str = "Syllabus",
|
||||
updated_at: str = "2025-06-01T12:00:00Z",
|
||||
) -> dict[str, Any]:
|
||||
return {
|
||||
"page_id": page_id,
|
||||
"url": "syllabus",
|
||||
"title": title,
|
||||
"body": "<p>Welcome to the course</p>",
|
||||
"created_at": "2025-01-15T00:00:00Z",
|
||||
"updated_at": updated_at,
|
||||
}
|
||||
|
||||
|
||||
def _mock_assignment(
|
||||
assignment_id: int = 20,
|
||||
name: str = "Homework 1",
|
||||
course_id: int = 1,
|
||||
updated_at: str = "2025-06-01T12:00:00Z",
|
||||
) -> dict[str, Any]:
|
||||
return {
|
||||
"id": assignment_id,
|
||||
"name": name,
|
||||
"description": "<p>Solve these problems</p>",
|
||||
"html_url": f"{FAKE_BASE_URL}/courses/{course_id}/assignments/{assignment_id}",
|
||||
"course_id": course_id,
|
||||
"created_at": "2025-01-20T00:00:00Z",
|
||||
"updated_at": updated_at,
|
||||
"due_at": "2025-02-01T23:59:00Z",
|
||||
}
|
||||
|
||||
|
||||
def _mock_announcement(
|
||||
announcement_id: int = 30,
|
||||
title: str = "Class Cancelled",
|
||||
course_id: int = 1,
|
||||
posted_at: str = "2025-06-01T12:00:00Z",
|
||||
) -> dict[str, Any]:
|
||||
return {
|
||||
"id": announcement_id,
|
||||
"title": title,
|
||||
"message": "<p>No class today</p>",
|
||||
"html_url": f"{FAKE_BASE_URL}/courses/{course_id}/discussion_topics/{announcement_id}",
|
||||
"posted_at": posted_at,
|
||||
}
|
||||
|
||||
|
||||
def _mock_response(
|
||||
status_code: int = 200,
|
||||
json_data: Any = None,
|
||||
@@ -325,6 +404,57 @@ class TestGet:
|
||||
assert result == expected
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CanvasApiClient.paginate tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestPaginate:
|
||||
@patch("onyx.connectors.canvas.client.rl_requests")
|
||||
def test_single_page(self, mock_requests: MagicMock) -> None:
|
||||
mock_requests.get.return_value = _mock_response(
|
||||
json_data=[{"id": 1}, {"id": 2}]
|
||||
)
|
||||
client = CanvasApiClient(
|
||||
bearer_token=FAKE_TOKEN,
|
||||
canvas_base_url=FAKE_BASE_URL,
|
||||
)
|
||||
|
||||
pages = list(client.paginate("courses"))
|
||||
|
||||
assert len(pages) == 1
|
||||
assert pages[0] == [{"id": 1}, {"id": 2}]
|
||||
|
||||
@patch("onyx.connectors.canvas.client.rl_requests")
|
||||
def test_two_pages(self, mock_requests: MagicMock) -> None:
|
||||
next_link = f'<{FAKE_BASE_URL}/api/v1/courses?page=2>; rel="next"'
|
||||
page1 = _mock_response(json_data=[{"id": 1}], link_header=next_link)
|
||||
page2 = _mock_response(json_data=[{"id": 2}])
|
||||
mock_requests.get.side_effect = [page1, page2]
|
||||
client = CanvasApiClient(
|
||||
bearer_token=FAKE_TOKEN,
|
||||
canvas_base_url=FAKE_BASE_URL,
|
||||
)
|
||||
|
||||
pages = list(client.paginate("courses"))
|
||||
|
||||
assert len(pages) == 2
|
||||
assert pages[0] == [{"id": 1}]
|
||||
assert pages[1] == [{"id": 2}]
|
||||
|
||||
@patch("onyx.connectors.canvas.client.rl_requests")
|
||||
def test_empty_response(self, mock_requests: MagicMock) -> None:
|
||||
mock_requests.get.return_value = _mock_response(json_data=[])
|
||||
client = CanvasApiClient(
|
||||
bearer_token=FAKE_TOKEN,
|
||||
canvas_base_url=FAKE_BASE_URL,
|
||||
)
|
||||
|
||||
pages = list(client.paginate("courses"))
|
||||
|
||||
assert pages == []
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CanvasApiClient._parse_next_link tests
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -379,3 +509,368 @@ class TestParseNextLink:
|
||||
|
||||
with pytest.raises(OnyxError, match="must use https"):
|
||||
self.client._parse_next_link(header)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CanvasConnector — credential loading
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestLoadCredentials:
|
||||
def _assert_load_credentials_raises(
|
||||
self,
|
||||
status_code: int,
|
||||
expected_error: type[Exception],
|
||||
mock_requests: MagicMock,
|
||||
) -> None:
|
||||
"""Helper: assert load_credentials raises expected_error for a given status."""
|
||||
mock_requests.get.return_value = _mock_response(status_code, {})
|
||||
connector = CanvasConnector(canvas_base_url=FAKE_BASE_URL)
|
||||
with pytest.raises(expected_error):
|
||||
connector.load_credentials({"canvas_access_token": FAKE_TOKEN})
|
||||
|
||||
@patch("onyx.connectors.canvas.client.rl_requests")
|
||||
def test_load_credentials_success(self, mock_requests: MagicMock) -> None:
|
||||
mock_requests.get.return_value = _mock_response(json_data=[_mock_course()])
|
||||
connector = CanvasConnector(canvas_base_url=FAKE_BASE_URL)
|
||||
|
||||
result = connector.load_credentials({"canvas_access_token": FAKE_TOKEN})
|
||||
|
||||
assert result is None
|
||||
assert connector._canvas_client is not None
|
||||
|
||||
def test_canvas_client_raises_without_credentials(self) -> None:
|
||||
connector = CanvasConnector(canvas_base_url=FAKE_BASE_URL)
|
||||
|
||||
with pytest.raises(ConnectorMissingCredentialError):
|
||||
_ = connector.canvas_client
|
||||
|
||||
@patch("onyx.connectors.canvas.client.rl_requests")
|
||||
def test_load_credentials_invalid_token(self, mock_requests: MagicMock) -> None:
|
||||
self._assert_load_credentials_raises(401, CredentialExpiredError, mock_requests)
|
||||
|
||||
@patch("onyx.connectors.canvas.client.rl_requests")
|
||||
def test_load_credentials_insufficient_permissions(
|
||||
self, mock_requests: MagicMock
|
||||
) -> None:
|
||||
self._assert_load_credentials_raises(
|
||||
403, InsufficientPermissionsError, mock_requests
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CanvasConnector — URL normalization
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestConnectorUrlNormalization:
|
||||
def test_strips_api_v1_suffix(self) -> None:
|
||||
connector = _build_connector(base_url=f"{FAKE_BASE_URL}/api/v1")
|
||||
|
||||
result = connector.canvas_base_url
|
||||
expected = FAKE_BASE_URL
|
||||
|
||||
assert result == expected
|
||||
|
||||
def test_strips_trailing_slash(self) -> None:
|
||||
connector = _build_connector(base_url=f"{FAKE_BASE_URL}/")
|
||||
|
||||
result = connector.canvas_base_url
|
||||
expected = FAKE_BASE_URL
|
||||
|
||||
assert result == expected
|
||||
|
||||
def test_no_change_for_clean_url(self) -> None:
|
||||
connector = _build_connector(base_url=FAKE_BASE_URL)
|
||||
|
||||
result = connector.canvas_base_url
|
||||
expected = FAKE_BASE_URL
|
||||
|
||||
assert result == expected
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CanvasConnector — document conversion
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestDocumentConversion:
|
||||
def setup_method(self) -> None:
|
||||
self.connector = _build_connector()
|
||||
|
||||
def test_convert_page_to_document(self) -> None:
|
||||
from onyx.connectors.canvas.connector import CanvasPage
|
||||
|
||||
page = CanvasPage(
|
||||
page_id=10,
|
||||
url="syllabus",
|
||||
title="Syllabus",
|
||||
body="<p>Welcome</p>",
|
||||
created_at="2025-01-15T00:00:00Z",
|
||||
updated_at="2025-06-01T12:00:00Z",
|
||||
course_id=1,
|
||||
)
|
||||
|
||||
doc = self.connector._convert_page_to_document(page)
|
||||
|
||||
expected_id = "canvas-page-1-10"
|
||||
expected_metadata = {"course_id": "1", "type": "page"}
|
||||
expected_updated_at = datetime(2025, 6, 1, 12, 0, tzinfo=timezone.utc)
|
||||
|
||||
assert doc.id == expected_id
|
||||
assert doc.source == DocumentSource.CANVAS
|
||||
assert doc.semantic_identifier == "Syllabus"
|
||||
assert doc.metadata == expected_metadata
|
||||
assert doc.sections[0].link is not None
|
||||
assert f"{FAKE_BASE_URL}/courses/1/pages/syllabus" in doc.sections[0].link
|
||||
assert doc.doc_updated_at == expected_updated_at
|
||||
|
||||
def test_convert_page_without_body(self) -> None:
|
||||
from onyx.connectors.canvas.connector import CanvasPage
|
||||
|
||||
page = CanvasPage(
|
||||
page_id=11,
|
||||
url="empty-page",
|
||||
title="Empty Page",
|
||||
body=None,
|
||||
created_at="2025-01-15T00:00:00Z",
|
||||
updated_at="2025-06-01T12:00:00Z",
|
||||
course_id=1,
|
||||
)
|
||||
|
||||
doc = self.connector._convert_page_to_document(page)
|
||||
section_text = doc.sections[0].text
|
||||
assert section_text is not None
|
||||
|
||||
assert "Empty Page" in section_text
|
||||
assert "<p>" not in section_text
|
||||
|
||||
def test_convert_assignment_to_document(self) -> None:
|
||||
from onyx.connectors.canvas.connector import CanvasAssignment
|
||||
|
||||
assignment = CanvasAssignment(
|
||||
id=20,
|
||||
name="Homework 1",
|
||||
description="<p>Solve these</p>",
|
||||
html_url=f"{FAKE_BASE_URL}/courses/1/assignments/20",
|
||||
course_id=1,
|
||||
created_at="2025-01-20T00:00:00Z",
|
||||
updated_at="2025-06-01T12:00:00Z",
|
||||
due_at="2025-02-01T23:59:00Z",
|
||||
)
|
||||
|
||||
doc = self.connector._convert_assignment_to_document(assignment)
|
||||
|
||||
expected_id = "canvas-assignment-1-20"
|
||||
expected_due_text = "Due: February 01, 2025 23:59 UTC"
|
||||
|
||||
assert doc.id == expected_id
|
||||
assert doc.source == DocumentSource.CANVAS
|
||||
assert doc.semantic_identifier == "Homework 1"
|
||||
assert doc.sections[0].text is not None
|
||||
assert expected_due_text in doc.sections[0].text
|
||||
|
||||
def test_convert_assignment_without_description(self) -> None:
|
||||
from onyx.connectors.canvas.connector import CanvasAssignment
|
||||
|
||||
assignment = CanvasAssignment(
|
||||
id=21,
|
||||
name="Quiz 1",
|
||||
description=None,
|
||||
html_url=f"{FAKE_BASE_URL}/courses/1/assignments/21",
|
||||
course_id=1,
|
||||
created_at="2025-01-20T00:00:00Z",
|
||||
updated_at="2025-06-01T12:00:00Z",
|
||||
due_at=None,
|
||||
)
|
||||
|
||||
doc = self.connector._convert_assignment_to_document(assignment)
|
||||
section_text = doc.sections[0].text
|
||||
assert section_text is not None
|
||||
|
||||
assert "Quiz 1" in section_text
|
||||
assert "Due:" not in section_text
|
||||
|
||||
def test_convert_announcement_to_document(self) -> None:
|
||||
from onyx.connectors.canvas.connector import CanvasAnnouncement
|
||||
|
||||
announcement = CanvasAnnouncement(
|
||||
id=30,
|
||||
title="Class Cancelled",
|
||||
message="<p>No class today</p>",
|
||||
html_url=f"{FAKE_BASE_URL}/courses/1/discussion_topics/30",
|
||||
posted_at="2025-06-01T12:00:00Z",
|
||||
course_id=1,
|
||||
)
|
||||
|
||||
doc = self.connector._convert_announcement_to_document(announcement)
|
||||
|
||||
expected_id = "canvas-announcement-1-30"
|
||||
expected_updated_at = datetime(2025, 6, 1, 12, 0, tzinfo=timezone.utc)
|
||||
|
||||
assert doc.id == expected_id
|
||||
assert doc.source == DocumentSource.CANVAS
|
||||
assert doc.semantic_identifier == "Class Cancelled"
|
||||
assert doc.doc_updated_at == expected_updated_at
|
||||
|
||||
def test_convert_announcement_without_posted_at(self) -> None:
|
||||
from onyx.connectors.canvas.connector import CanvasAnnouncement
|
||||
|
||||
announcement = CanvasAnnouncement(
|
||||
id=31,
|
||||
title="TBD Announcement",
|
||||
message=None,
|
||||
html_url=f"{FAKE_BASE_URL}/courses/1/discussion_topics/31",
|
||||
posted_at=None,
|
||||
course_id=1,
|
||||
)
|
||||
|
||||
doc = self.connector._convert_announcement_to_document(announcement)
|
||||
|
||||
assert doc.doc_updated_at is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CanvasConnector — validate_connector_settings
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestValidateConnectorSettings:
|
||||
def _assert_validate_raises(
|
||||
self,
|
||||
status_code: int,
|
||||
expected_error: type[Exception],
|
||||
mock_requests: MagicMock,
|
||||
) -> None:
|
||||
"""Helper: assert validate_connector_settings raises expected_error."""
|
||||
success_resp = _mock_response(json_data=[_mock_course()])
|
||||
fail_resp = _mock_response(status_code, {})
|
||||
mock_requests.get.side_effect = [success_resp, fail_resp]
|
||||
connector = CanvasConnector(canvas_base_url=FAKE_BASE_URL)
|
||||
connector.load_credentials({"canvas_access_token": FAKE_TOKEN})
|
||||
with pytest.raises(expected_error):
|
||||
connector.validate_connector_settings()
|
||||
|
||||
@patch("onyx.connectors.canvas.client.rl_requests")
|
||||
def test_validate_success(self, mock_requests: MagicMock) -> None:
|
||||
mock_requests.get.return_value = _mock_response(json_data=[_mock_course()])
|
||||
connector = _build_connector()
|
||||
|
||||
connector.validate_connector_settings() # should not raise
|
||||
|
||||
@patch("onyx.connectors.canvas.client.rl_requests")
|
||||
def test_validate_expired_credential(self, mock_requests: MagicMock) -> None:
|
||||
self._assert_validate_raises(401, CredentialExpiredError, mock_requests)
|
||||
|
||||
@patch("onyx.connectors.canvas.client.rl_requests")
|
||||
def test_validate_insufficient_permissions(self, mock_requests: MagicMock) -> None:
|
||||
self._assert_validate_raises(403, InsufficientPermissionsError, mock_requests)
|
||||
|
||||
@patch("onyx.connectors.canvas.client.rl_requests")
|
||||
def test_validate_rate_limited(self, mock_requests: MagicMock) -> None:
|
||||
self._assert_validate_raises(429, ConnectorValidationError, mock_requests)
|
||||
|
||||
@patch("onyx.connectors.canvas.client.rl_requests")
|
||||
def test_validate_unexpected_error(self, mock_requests: MagicMock) -> None:
|
||||
self._assert_validate_raises(500, UnexpectedValidationError, mock_requests)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _list_* pagination tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestListCourses:
|
||||
@patch("onyx.connectors.canvas.client.rl_requests")
|
||||
def test_single_page(self, mock_requests: MagicMock) -> None:
|
||||
mock_requests.get.return_value = _mock_response(
|
||||
json_data=[_mock_course(1), _mock_course(2, "CS201", "Data Structures")]
|
||||
)
|
||||
connector = _build_connector()
|
||||
|
||||
result = connector._list_courses()
|
||||
|
||||
assert len(result) == 2
|
||||
assert result[0].id == 1
|
||||
assert result[1].id == 2
|
||||
|
||||
@patch("onyx.connectors.canvas.client.rl_requests")
|
||||
def test_empty_response(self, mock_requests: MagicMock) -> None:
|
||||
mock_requests.get.return_value = _mock_response(json_data=[])
|
||||
connector = _build_connector()
|
||||
|
||||
result = connector._list_courses()
|
||||
|
||||
assert result == []
|
||||
|
||||
|
||||
class TestListPages:
|
||||
@patch("onyx.connectors.canvas.client.rl_requests")
|
||||
def test_single_page(self, mock_requests: MagicMock) -> None:
|
||||
mock_requests.get.return_value = _mock_response(
|
||||
json_data=[_mock_page(10), _mock_page(11, "Notes")]
|
||||
)
|
||||
connector = _build_connector()
|
||||
|
||||
result = connector._list_pages(course_id=1)
|
||||
|
||||
assert len(result) == 2
|
||||
assert result[0].page_id == 10
|
||||
assert result[1].page_id == 11
|
||||
|
||||
@patch("onyx.connectors.canvas.client.rl_requests")
|
||||
def test_empty_response(self, mock_requests: MagicMock) -> None:
|
||||
mock_requests.get.return_value = _mock_response(json_data=[])
|
||||
connector = _build_connector()
|
||||
|
||||
result = connector._list_pages(course_id=1)
|
||||
|
||||
assert result == []
|
||||
|
||||
|
||||
class TestListAssignments:
|
||||
@patch("onyx.connectors.canvas.client.rl_requests")
|
||||
def test_single_page(self, mock_requests: MagicMock) -> None:
|
||||
mock_requests.get.return_value = _mock_response(
|
||||
json_data=[_mock_assignment(20), _mock_assignment(21, "Quiz 1")]
|
||||
)
|
||||
connector = _build_connector()
|
||||
|
||||
result = connector._list_assignments(course_id=1)
|
||||
|
||||
assert len(result) == 2
|
||||
assert result[0].id == 20
|
||||
assert result[1].id == 21
|
||||
|
||||
@patch("onyx.connectors.canvas.client.rl_requests")
|
||||
def test_empty_response(self, mock_requests: MagicMock) -> None:
|
||||
mock_requests.get.return_value = _mock_response(json_data=[])
|
||||
connector = _build_connector()
|
||||
|
||||
result = connector._list_assignments(course_id=1)
|
||||
|
||||
assert result == []
|
||||
|
||||
|
||||
class TestListAnnouncements:
|
||||
@patch("onyx.connectors.canvas.client.rl_requests")
|
||||
def test_single_page(self, mock_requests: MagicMock) -> None:
|
||||
mock_requests.get.return_value = _mock_response(
|
||||
json_data=[_mock_announcement(30), _mock_announcement(31, "Update")]
|
||||
)
|
||||
connector = _build_connector()
|
||||
|
||||
result = connector._list_announcements(course_id=1)
|
||||
|
||||
assert len(result) == 2
|
||||
assert result[0].id == 30
|
||||
assert result[1].id == 31
|
||||
|
||||
@patch("onyx.connectors.canvas.client.rl_requests")
|
||||
def test_empty_response(self, mock_requests: MagicMock) -> None:
|
||||
mock_requests.get.return_value = _mock_response(json_data=[])
|
||||
connector = _build_connector()
|
||||
|
||||
result = connector._list_announcements(course_id=1)
|
||||
|
||||
assert result == []
|
||||
|
||||
58
backend/tests/unit/onyx/db/test_chat_deletion.py
Normal file
58
backend/tests/unit/onyx/db/test_chat_deletion.py
Normal file
@@ -0,0 +1,58 @@
|
||||
"""Tests for chat session and message deletion resilience."""
|
||||
|
||||
from typing import Any
|
||||
from unittest.mock import MagicMock
|
||||
from unittest.mock import patch
|
||||
from uuid import uuid4
|
||||
|
||||
from onyx.db.chat import delete_messages_and_files_from_chat_session
|
||||
|
||||
|
||||
@patch("onyx.db.chat.delete_orphaned_search_docs")
|
||||
@patch("onyx.db.chat.get_default_file_store")
|
||||
def test_delete_messages_skips_missing_files(
|
||||
mock_get_file_store: MagicMock,
|
||||
_mock_delete_orphaned: Any,
|
||||
) -> None:
|
||||
"""Deletion should continue when a referenced file record no longer exists."""
|
||||
session_id = uuid4()
|
||||
|
||||
file_store = MagicMock()
|
||||
file_store.delete_file.side_effect = [
|
||||
None, # first file deletes fine
|
||||
RuntimeError("File by id abc does not exist or was deleted"),
|
||||
None, # third file deletes fine
|
||||
]
|
||||
mock_get_file_store.return_value = file_store
|
||||
|
||||
mock_db_session = MagicMock()
|
||||
mock_db_session.execute.return_value.fetchall.return_value = [
|
||||
(1, [{"id": "file-ok-1"}, {"id": "file-missing"}, {"id": "file-ok-2"}]),
|
||||
]
|
||||
|
||||
delete_messages_and_files_from_chat_session(session_id, mock_db_session)
|
||||
|
||||
assert file_store.delete_file.call_count == 3
|
||||
mock_db_session.execute.assert_called()
|
||||
mock_db_session.commit.assert_called()
|
||||
|
||||
|
||||
@patch("onyx.db.chat.delete_orphaned_search_docs")
|
||||
@patch("onyx.db.chat.get_default_file_store")
|
||||
def test_delete_messages_succeeds_with_no_files(
|
||||
mock_get_file_store: MagicMock,
|
||||
_mock_delete_orphaned: Any,
|
||||
) -> None:
|
||||
"""Deletion works when messages have no attached files."""
|
||||
session_id = uuid4()
|
||||
|
||||
mock_db_session = MagicMock()
|
||||
mock_db_session.execute.return_value.fetchall.return_value = [
|
||||
(1, None),
|
||||
(2, []),
|
||||
]
|
||||
|
||||
delete_messages_and_files_from_chat_session(session_id, mock_db_session)
|
||||
|
||||
mock_get_file_store.return_value.delete_file.assert_not_called()
|
||||
mock_db_session.commit.assert_called()
|
||||
@@ -0,0 +1,223 @@
|
||||
from unittest.mock import MagicMock
|
||||
from unittest.mock import patch
|
||||
|
||||
from onyx.access.models import DocumentAccess
|
||||
from onyx.configs.constants import DocumentSource
|
||||
from onyx.connectors.models import Document
|
||||
from onyx.connectors.models import TextSection
|
||||
from onyx.document_index.interfaces_new import IndexingMetadata
|
||||
from onyx.document_index.interfaces_new import TenantState
|
||||
from onyx.document_index.opensearch.opensearch_document_index import (
|
||||
OpenSearchDocumentIndex,
|
||||
)
|
||||
from onyx.indexing.models import ChunkEmbedding
|
||||
from onyx.indexing.models import DocMetadataAwareIndexChunk
|
||||
|
||||
|
||||
def _make_chunk(
|
||||
doc_id: str,
|
||||
chunk_id: int,
|
||||
) -> DocMetadataAwareIndexChunk:
|
||||
"""Creates a minimal DocMetadataAwareIndexChunk for testing."""
|
||||
doc = Document(
|
||||
id=doc_id,
|
||||
sections=[TextSection(text="test", link="http://test.com")],
|
||||
source=DocumentSource.FILE,
|
||||
semantic_identifier="test_doc",
|
||||
metadata={},
|
||||
)
|
||||
access = DocumentAccess.build(
|
||||
user_emails=[],
|
||||
user_groups=[],
|
||||
external_user_emails=[],
|
||||
external_user_group_ids=[],
|
||||
is_public=True,
|
||||
)
|
||||
return DocMetadataAwareIndexChunk(
|
||||
chunk_id=chunk_id,
|
||||
blurb="test",
|
||||
content="test content",
|
||||
source_links={0: "http://test.com"},
|
||||
image_file_id=None,
|
||||
section_continuation=False,
|
||||
source_document=doc,
|
||||
title_prefix="",
|
||||
metadata_suffix_semantic="",
|
||||
metadata_suffix_keyword="",
|
||||
mini_chunk_texts=None,
|
||||
large_chunk_id=None,
|
||||
doc_summary="",
|
||||
chunk_context="",
|
||||
contextual_rag_reserved_tokens=0,
|
||||
embeddings=ChunkEmbedding(full_embedding=[0.1] * 10, mini_chunk_embeddings=[]),
|
||||
title_embedding=[0.1] * 10,
|
||||
tenant_id="test_tenant",
|
||||
access=access,
|
||||
document_sets=set(),
|
||||
user_project=[],
|
||||
personas=[],
|
||||
boost=0,
|
||||
aggregated_chunk_boost_factor=1.0,
|
||||
ancestor_hierarchy_node_ids=[],
|
||||
)
|
||||
|
||||
|
||||
def _make_index() -> tuple[OpenSearchDocumentIndex, MagicMock]:
|
||||
"""Creates an OpenSearchDocumentIndex with a mocked client.
|
||||
Returns the index and the mock for bulk_index_documents."""
|
||||
mock_client = MagicMock()
|
||||
mock_bulk = MagicMock()
|
||||
mock_client.bulk_index_documents = mock_bulk
|
||||
|
||||
tenant_state = TenantState(tenant_id="test_tenant", multitenant=False)
|
||||
|
||||
index = OpenSearchDocumentIndex.__new__(OpenSearchDocumentIndex)
|
||||
index._index_name = "test_index"
|
||||
index._client = mock_client
|
||||
index._tenant_state = tenant_state
|
||||
|
||||
return index, mock_bulk
|
||||
|
||||
|
||||
def _make_metadata(doc_id: str, chunk_count: int) -> IndexingMetadata:
|
||||
return IndexingMetadata(
|
||||
doc_id_to_chunk_cnt_diff={
|
||||
doc_id: IndexingMetadata.ChunkCounts(
|
||||
old_chunk_cnt=0,
|
||||
new_chunk_cnt=chunk_count,
|
||||
),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@patch(
|
||||
"onyx.document_index.opensearch.opensearch_document_index.MAX_CHUNKS_PER_DOC_BATCH",
|
||||
100,
|
||||
)
|
||||
def test_single_doc_under_batch_limit_flushes_once() -> None:
|
||||
"""A document with fewer chunks than MAX_CHUNKS_PER_DOC_BATCH should flush once."""
|
||||
index, mock_bulk = _make_index()
|
||||
doc_id = "doc_1"
|
||||
num_chunks = 50
|
||||
chunks = [_make_chunk(doc_id, i) for i in range(num_chunks)]
|
||||
metadata = _make_metadata(doc_id, num_chunks)
|
||||
|
||||
with patch.object(index, "delete", return_value=0):
|
||||
index.index(chunks, metadata)
|
||||
|
||||
assert mock_bulk.call_count == 1
|
||||
batch_arg = mock_bulk.call_args_list[0]
|
||||
assert len(batch_arg.kwargs["documents"]) == num_chunks
|
||||
|
||||
|
||||
@patch(
|
||||
"onyx.document_index.opensearch.opensearch_document_index.MAX_CHUNKS_PER_DOC_BATCH",
|
||||
100,
|
||||
)
|
||||
def test_single_doc_over_batch_limit_flushes_multiple_times() -> None:
|
||||
"""A document with more chunks than MAX_CHUNKS_PER_DOC_BATCH should flush multiple times."""
|
||||
index, mock_bulk = _make_index()
|
||||
doc_id = "doc_1"
|
||||
num_chunks = 250
|
||||
chunks = [_make_chunk(doc_id, i) for i in range(num_chunks)]
|
||||
metadata = _make_metadata(doc_id, num_chunks)
|
||||
|
||||
with patch.object(index, "delete", return_value=0):
|
||||
index.index(chunks, metadata)
|
||||
|
||||
# 250 chunks / 100 per batch = 3 flushes (100 + 100 + 50)
|
||||
assert mock_bulk.call_count == 3
|
||||
batch_sizes = [len(call.kwargs["documents"]) for call in mock_bulk.call_args_list]
|
||||
assert batch_sizes == [100, 100, 50]
|
||||
|
||||
|
||||
@patch(
|
||||
"onyx.document_index.opensearch.opensearch_document_index.MAX_CHUNKS_PER_DOC_BATCH",
|
||||
100,
|
||||
)
|
||||
def test_single_doc_exactly_at_batch_limit() -> None:
|
||||
"""A document with exactly MAX_CHUNKS_PER_DOC_BATCH chunks should flush once
|
||||
(the flush happens on the next chunk, not at the boundary)."""
|
||||
index, mock_bulk = _make_index()
|
||||
doc_id = "doc_1"
|
||||
num_chunks = 100
|
||||
chunks = [_make_chunk(doc_id, i) for i in range(num_chunks)]
|
||||
metadata = _make_metadata(doc_id, num_chunks)
|
||||
|
||||
with patch.object(index, "delete", return_value=0):
|
||||
index.index(chunks, metadata)
|
||||
|
||||
# 100 chunks hit the >= check on chunk 101 which doesn't exist,
|
||||
# so final flush handles all 100
|
||||
# Actually: the elif fires when len(current_chunks) >= 100, which happens
|
||||
# when current_chunks has 100 items and the 101st chunk arrives.
|
||||
# With exactly 100 chunks, the 100th chunk makes len == 99, then appended -> 100.
|
||||
# No 101st chunk arrives, so the final flush handles all 100.
|
||||
assert mock_bulk.call_count == 1
|
||||
|
||||
|
||||
@patch(
|
||||
"onyx.document_index.opensearch.opensearch_document_index.MAX_CHUNKS_PER_DOC_BATCH",
|
||||
100,
|
||||
)
|
||||
def test_single_doc_one_over_batch_limit() -> None:
|
||||
"""101 chunks for one doc: first 100 flushed when the 101st arrives, then
|
||||
the 101st is flushed at the end."""
|
||||
index, mock_bulk = _make_index()
|
||||
doc_id = "doc_1"
|
||||
num_chunks = 101
|
||||
chunks = [_make_chunk(doc_id, i) for i in range(num_chunks)]
|
||||
metadata = _make_metadata(doc_id, num_chunks)
|
||||
|
||||
with patch.object(index, "delete", return_value=0):
|
||||
index.index(chunks, metadata)
|
||||
|
||||
assert mock_bulk.call_count == 2
|
||||
batch_sizes = [len(call.kwargs["documents"]) for call in mock_bulk.call_args_list]
|
||||
assert batch_sizes == [100, 1]
|
||||
|
||||
|
||||
@patch(
|
||||
"onyx.document_index.opensearch.opensearch_document_index.MAX_CHUNKS_PER_DOC_BATCH",
|
||||
100,
|
||||
)
|
||||
def test_multiple_docs_each_under_limit_flush_per_doc() -> None:
|
||||
"""Multiple documents each under the batch limit should flush once per document."""
|
||||
index, mock_bulk = _make_index()
|
||||
chunks = []
|
||||
for doc_idx in range(3):
|
||||
doc_id = f"doc_{doc_idx}"
|
||||
for chunk_idx in range(50):
|
||||
chunks.append(_make_chunk(doc_id, chunk_idx))
|
||||
|
||||
metadata = IndexingMetadata(
|
||||
doc_id_to_chunk_cnt_diff={
|
||||
f"doc_{i}": IndexingMetadata.ChunkCounts(old_chunk_cnt=0, new_chunk_cnt=50)
|
||||
for i in range(3)
|
||||
},
|
||||
)
|
||||
|
||||
with patch.object(index, "delete", return_value=0):
|
||||
index.index(chunks, metadata)
|
||||
|
||||
# 3 documents = 3 flushes (one per doc boundary + final)
|
||||
assert mock_bulk.call_count == 3
|
||||
|
||||
|
||||
@patch(
|
||||
"onyx.document_index.opensearch.opensearch_document_index.MAX_CHUNKS_PER_DOC_BATCH",
|
||||
100,
|
||||
)
|
||||
def test_delete_called_once_per_document() -> None:
|
||||
"""Even with multiple flushes for a single document, delete should only be
|
||||
called once per document."""
|
||||
index, _mock_bulk = _make_index()
|
||||
doc_id = "doc_1"
|
||||
num_chunks = 250
|
||||
chunks = [_make_chunk(doc_id, i) for i in range(num_chunks)]
|
||||
metadata = _make_metadata(doc_id, num_chunks)
|
||||
|
||||
with patch.object(index, "delete", return_value=0) as mock_delete:
|
||||
index.index(chunks, metadata)
|
||||
|
||||
mock_delete.assert_called_once_with(doc_id, None)
|
||||
@@ -0,0 +1,152 @@
|
||||
"""Unit tests for VespaDocumentIndex.index().
|
||||
|
||||
These tests mock all external I/O (HTTP calls, thread pools) and verify
|
||||
the streaming logic, ID cleaning/mapping, and DocumentInsertionRecord
|
||||
construction.
|
||||
"""
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
from unittest.mock import patch
|
||||
|
||||
from onyx.access.models import DocumentAccess
|
||||
from onyx.configs.constants import DocumentSource
|
||||
from onyx.connectors.models import Document
|
||||
from onyx.connectors.models import TextSection
|
||||
from onyx.document_index.interfaces import EnrichedDocumentIndexingInfo
|
||||
from onyx.document_index.interfaces_new import IndexingMetadata
|
||||
from onyx.document_index.interfaces_new import TenantState
|
||||
from onyx.document_index.vespa.vespa_document_index import VespaDocumentIndex
|
||||
from onyx.indexing.models import ChunkEmbedding
|
||||
from onyx.indexing.models import DocMetadataAwareIndexChunk
|
||||
from onyx.indexing.models import IndexChunk
|
||||
|
||||
|
||||
def _make_chunk(
|
||||
doc_id: str,
|
||||
chunk_id: int = 0,
|
||||
content: str = "test content",
|
||||
) -> DocMetadataAwareIndexChunk:
|
||||
doc = Document(
|
||||
id=doc_id,
|
||||
semantic_identifier="test_doc",
|
||||
sections=[TextSection(text=content, link=None)],
|
||||
source=DocumentSource.NOT_APPLICABLE,
|
||||
metadata={},
|
||||
)
|
||||
index_chunk = IndexChunk(
|
||||
chunk_id=chunk_id,
|
||||
blurb=content[:50],
|
||||
content=content,
|
||||
source_links=None,
|
||||
image_file_id=None,
|
||||
section_continuation=False,
|
||||
source_document=doc,
|
||||
title_prefix="",
|
||||
metadata_suffix_semantic="",
|
||||
metadata_suffix_keyword="",
|
||||
contextual_rag_reserved_tokens=0,
|
||||
doc_summary="",
|
||||
chunk_context="",
|
||||
mini_chunk_texts=None,
|
||||
large_chunk_id=None,
|
||||
embeddings=ChunkEmbedding(
|
||||
full_embedding=[0.1] * 10,
|
||||
mini_chunk_embeddings=[],
|
||||
),
|
||||
title_embedding=None,
|
||||
)
|
||||
access = DocumentAccess.build(
|
||||
user_emails=[],
|
||||
user_groups=[],
|
||||
external_user_emails=[],
|
||||
external_user_group_ids=[],
|
||||
is_public=True,
|
||||
)
|
||||
return DocMetadataAwareIndexChunk.from_index_chunk(
|
||||
index_chunk=index_chunk,
|
||||
access=access,
|
||||
document_sets=set(),
|
||||
user_project=[],
|
||||
personas=[],
|
||||
boost=0,
|
||||
aggregated_chunk_boost_factor=1.0,
|
||||
tenant_id="test_tenant",
|
||||
)
|
||||
|
||||
|
||||
def _make_indexing_metadata(
|
||||
doc_ids: list[str],
|
||||
old_counts: list[int],
|
||||
new_counts: list[int],
|
||||
) -> IndexingMetadata:
|
||||
return IndexingMetadata(
|
||||
doc_id_to_chunk_cnt_diff={
|
||||
doc_id: IndexingMetadata.ChunkCounts(
|
||||
old_chunk_cnt=old,
|
||||
new_chunk_cnt=new,
|
||||
)
|
||||
for doc_id, old, new in zip(doc_ids, old_counts, new_counts)
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _stub_enrich(
|
||||
doc_id: str,
|
||||
old_chunk_cnt: int,
|
||||
) -> EnrichedDocumentIndexingInfo:
|
||||
"""Build an EnrichedDocumentIndexingInfo that says 'no chunks to delete'
|
||||
when old_chunk_cnt == 0, or 'has existing chunks' otherwise."""
|
||||
return EnrichedDocumentIndexingInfo(
|
||||
doc_id=doc_id,
|
||||
chunk_start_index=0,
|
||||
old_version=False,
|
||||
chunk_end_index=old_chunk_cnt,
|
||||
)
|
||||
|
||||
|
||||
@patch("onyx.document_index.vespa.vespa_document_index.batch_index_vespa_chunks")
|
||||
@patch("onyx.document_index.vespa.vespa_document_index.delete_vespa_chunks")
|
||||
@patch(
|
||||
"onyx.document_index.vespa.vespa_document_index.get_document_chunk_ids",
|
||||
return_value=[],
|
||||
)
|
||||
@patch("onyx.document_index.vespa.vespa_document_index._enrich_basic_chunk_info")
|
||||
@patch(
|
||||
"onyx.document_index.vespa.vespa_document_index.BATCH_SIZE",
|
||||
3,
|
||||
)
|
||||
def test_index_respects_batch_size(
|
||||
mock_enrich: MagicMock,
|
||||
mock_get_chunk_ids: MagicMock, # noqa: ARG001
|
||||
mock_delete: MagicMock, # noqa: ARG001
|
||||
mock_batch_index: MagicMock,
|
||||
) -> None:
|
||||
"""When chunks exceed BATCH_SIZE, batch_index_vespa_chunks is called
|
||||
multiple times with correctly sized batches."""
|
||||
mock_enrich.return_value = _stub_enrich("doc1", old_chunk_cnt=0)
|
||||
|
||||
index = VespaDocumentIndex(
|
||||
index_name="test_index",
|
||||
tenant_state=TenantState(tenant_id="test_tenant", multitenant=False),
|
||||
large_chunks_enabled=False,
|
||||
httpx_client=MagicMock(),
|
||||
)
|
||||
|
||||
chunks = [_make_chunk("doc1", chunk_id=i) for i in range(7)]
|
||||
metadata = _make_indexing_metadata(["doc1"], old_counts=[0], new_counts=[7])
|
||||
|
||||
results = index.index(chunks=chunks, indexing_metadata=metadata)
|
||||
|
||||
assert len(results) == 1
|
||||
|
||||
# With BATCH_SIZE=3 and 7 chunks: batches of 3, 3, 1
|
||||
assert mock_batch_index.call_count == 3
|
||||
batch_sizes = [len(c.kwargs["chunks"]) for c in mock_batch_index.call_args_list]
|
||||
assert batch_sizes == [3, 3, 1]
|
||||
|
||||
# Verify all chunks are accounted for and in order
|
||||
all_indexed = [
|
||||
chunk for c in mock_batch_index.call_args_list for chunk in c.kwargs["chunks"]
|
||||
]
|
||||
assert len(all_indexed) == 7
|
||||
assert [c.chunk_id for c in all_indexed] == list(range(7))
|
||||
391
backend/tests/unit/onyx/indexing/test_embed_chunks_in_batches.py
Normal file
391
backend/tests/unit/onyx/indexing/test_embed_chunks_in_batches.py
Normal file
@@ -0,0 +1,391 @@
|
||||
"""Unit tests for _embed_chunks_to_store.
|
||||
|
||||
Tests cover:
|
||||
- Single batch, no failures
|
||||
- Multiple batches, no failures
|
||||
- Failure in a single batch
|
||||
- Cross-batch document failure scrubbing
|
||||
- Later batches skip already-failed docs
|
||||
- Empty input
|
||||
- All chunks fail
|
||||
"""
|
||||
|
||||
from collections.abc import Callable
|
||||
from unittest.mock import MagicMock
|
||||
from unittest.mock import patch
|
||||
|
||||
from onyx.connectors.models import ConnectorFailure
|
||||
from onyx.connectors.models import Document
|
||||
from onyx.connectors.models import DocumentFailure
|
||||
from onyx.connectors.models import DocumentSource
|
||||
from onyx.connectors.models import TextSection
|
||||
from onyx.indexing.chunk_batch_store import ChunkBatchStore
|
||||
from onyx.indexing.indexing_pipeline import _embed_chunks_to_store
|
||||
from onyx.indexing.models import ChunkEmbedding
|
||||
from onyx.indexing.models import DocAwareChunk
|
||||
from onyx.indexing.models import IndexChunk
|
||||
|
||||
|
||||
def _make_doc(doc_id: str) -> Document:
|
||||
return Document(
|
||||
id=doc_id,
|
||||
semantic_identifier="test",
|
||||
source=DocumentSource.FILE,
|
||||
sections=[TextSection(text="test", link=None)],
|
||||
metadata={},
|
||||
)
|
||||
|
||||
|
||||
def _make_chunk(doc_id: str, chunk_id: int) -> DocAwareChunk:
|
||||
return DocAwareChunk(
|
||||
chunk_id=chunk_id,
|
||||
blurb="test",
|
||||
content="test content",
|
||||
source_links=None,
|
||||
image_file_id=None,
|
||||
section_continuation=False,
|
||||
source_document=_make_doc(doc_id),
|
||||
title_prefix="",
|
||||
metadata_suffix_semantic="",
|
||||
metadata_suffix_keyword="",
|
||||
mini_chunk_texts=None,
|
||||
large_chunk_id=None,
|
||||
doc_summary="",
|
||||
chunk_context="",
|
||||
contextual_rag_reserved_tokens=0,
|
||||
)
|
||||
|
||||
|
||||
def _make_index_chunk(doc_id: str, chunk_id: int) -> IndexChunk:
|
||||
"""Create an IndexChunk (a DocAwareChunk with embeddings)."""
|
||||
return IndexChunk(
|
||||
chunk_id=chunk_id,
|
||||
blurb="test",
|
||||
content="test content",
|
||||
source_links=None,
|
||||
image_file_id=None,
|
||||
section_continuation=False,
|
||||
source_document=_make_doc(doc_id),
|
||||
title_prefix="",
|
||||
metadata_suffix_semantic="",
|
||||
metadata_suffix_keyword="",
|
||||
mini_chunk_texts=None,
|
||||
large_chunk_id=None,
|
||||
doc_summary="",
|
||||
chunk_context="",
|
||||
contextual_rag_reserved_tokens=0,
|
||||
embeddings=ChunkEmbedding(
|
||||
full_embedding=[0.1] * 10,
|
||||
mini_chunk_embeddings=[],
|
||||
),
|
||||
title_embedding=None,
|
||||
)
|
||||
|
||||
|
||||
def _make_failure(doc_id: str) -> ConnectorFailure:
|
||||
return ConnectorFailure(
|
||||
failed_document=DocumentFailure(document_id=doc_id, document_link=None),
|
||||
failure_message="embedding failed",
|
||||
exception=RuntimeError("embedding failed"),
|
||||
)
|
||||
|
||||
|
||||
def _mock_embed_success(
|
||||
chunks: list[DocAwareChunk], **_kwargs: object
|
||||
) -> tuple[list[IndexChunk], list[ConnectorFailure]]:
|
||||
"""Simulate successful embedding of all chunks."""
|
||||
return (
|
||||
[_make_index_chunk(c.source_document.id, c.chunk_id) for c in chunks],
|
||||
[],
|
||||
)
|
||||
|
||||
|
||||
def _mock_embed_fail_doc(
|
||||
fail_doc_id: str,
|
||||
) -> Callable[..., tuple[list[IndexChunk], list[ConnectorFailure]]]:
|
||||
"""Return an embed mock that fails all chunks for a specific doc."""
|
||||
|
||||
def _embed(
|
||||
chunks: list[DocAwareChunk], **_kwargs: object
|
||||
) -> tuple[list[IndexChunk], list[ConnectorFailure]]:
|
||||
successes = [
|
||||
_make_index_chunk(c.source_document.id, c.chunk_id)
|
||||
for c in chunks
|
||||
if c.source_document.id != fail_doc_id
|
||||
]
|
||||
failures = (
|
||||
[_make_failure(fail_doc_id)]
|
||||
if any(c.source_document.id == fail_doc_id for c in chunks)
|
||||
else []
|
||||
)
|
||||
return successes, failures
|
||||
|
||||
return _embed
|
||||
|
||||
|
||||
class TestEmbedChunksInBatches:
|
||||
@patch(
|
||||
"onyx.indexing.indexing_pipeline.embed_chunks_with_failure_handling",
|
||||
)
|
||||
@patch("onyx.indexing.indexing_pipeline.MAX_CHUNKS_PER_DOC_BATCH", 100)
|
||||
def test_single_batch_no_failures(self, mock_embed: MagicMock) -> None:
|
||||
"""All chunks fit in one batch and embed successfully."""
|
||||
mock_embed.side_effect = _mock_embed_success
|
||||
|
||||
with ChunkBatchStore() as store:
|
||||
chunks = [_make_chunk("doc1", i) for i in range(3)]
|
||||
result = _embed_chunks_to_store(
|
||||
chunks=chunks,
|
||||
embedder=MagicMock(),
|
||||
tenant_id="test",
|
||||
request_id=None,
|
||||
store=store,
|
||||
)
|
||||
|
||||
assert len(result.successful_chunk_ids) == 3
|
||||
assert len(result.connector_failures) == 0
|
||||
|
||||
# Verify stored contents
|
||||
assert len(store._batch_files()) == 1
|
||||
stored = list(store.stream())
|
||||
assert len(stored) == 3
|
||||
|
||||
@patch(
|
||||
"onyx.indexing.indexing_pipeline.embed_chunks_with_failure_handling",
|
||||
)
|
||||
@patch("onyx.indexing.indexing_pipeline.MAX_CHUNKS_PER_DOC_BATCH", 3)
|
||||
def test_multiple_batches_no_failures(self, mock_embed: MagicMock) -> None:
|
||||
"""Chunks are split across multiple batches, all succeed."""
|
||||
mock_embed.side_effect = _mock_embed_success
|
||||
|
||||
with ChunkBatchStore() as store:
|
||||
chunks = [_make_chunk("doc1", i) for i in range(7)]
|
||||
result = _embed_chunks_to_store(
|
||||
chunks=chunks,
|
||||
embedder=MagicMock(),
|
||||
tenant_id="test",
|
||||
request_id=None,
|
||||
store=store,
|
||||
)
|
||||
|
||||
assert len(result.successful_chunk_ids) == 7
|
||||
assert len(result.connector_failures) == 0
|
||||
assert len(store._batch_files()) == 3 # 3 + 3 + 1
|
||||
|
||||
@patch(
|
||||
"onyx.indexing.indexing_pipeline.embed_chunks_with_failure_handling",
|
||||
)
|
||||
@patch("onyx.indexing.indexing_pipeline.MAX_CHUNKS_PER_DOC_BATCH", 100)
|
||||
def test_single_batch_with_failure(self, mock_embed: MagicMock) -> None:
|
||||
"""One doc fails embedding, its chunks are excluded from results."""
|
||||
mock_embed.side_effect = _mock_embed_fail_doc("doc2")
|
||||
|
||||
with ChunkBatchStore() as store:
|
||||
chunks = [
|
||||
_make_chunk("doc1", 0),
|
||||
_make_chunk("doc2", 1),
|
||||
_make_chunk("doc1", 2),
|
||||
]
|
||||
result = _embed_chunks_to_store(
|
||||
chunks=chunks,
|
||||
embedder=MagicMock(),
|
||||
tenant_id="test",
|
||||
request_id=None,
|
||||
store=store,
|
||||
)
|
||||
|
||||
assert len(result.connector_failures) == 1
|
||||
successful_doc_ids = {doc_id for _, doc_id in result.successful_chunk_ids}
|
||||
assert "doc2" not in successful_doc_ids
|
||||
assert "doc1" in successful_doc_ids
|
||||
|
||||
@patch(
|
||||
"onyx.indexing.indexing_pipeline.embed_chunks_with_failure_handling",
|
||||
)
|
||||
@patch("onyx.indexing.indexing_pipeline.MAX_CHUNKS_PER_DOC_BATCH", 3)
|
||||
def test_cross_batch_failure_scrubs_earlier_batch(
|
||||
self, mock_embed: MagicMock
|
||||
) -> None:
|
||||
"""Doc A spans batches 0 and 1. It succeeds in batch 0 but fails in
|
||||
batch 1. Its chunks should be scrubbed from batch 0's batch file."""
|
||||
call_count = 0
|
||||
|
||||
def _embed(
|
||||
chunks: list[DocAwareChunk], **_kwargs: object
|
||||
) -> tuple[list[IndexChunk], list[ConnectorFailure]]:
|
||||
nonlocal call_count
|
||||
call_count += 1
|
||||
if call_count == 1:
|
||||
return _mock_embed_success(chunks)
|
||||
else:
|
||||
return _mock_embed_fail_doc("docA")(chunks)
|
||||
|
||||
mock_embed.side_effect = _embed
|
||||
|
||||
with ChunkBatchStore() as store:
|
||||
chunks = [
|
||||
_make_chunk("docA", 0),
|
||||
_make_chunk("docA", 1),
|
||||
_make_chunk("docA", 2),
|
||||
_make_chunk("docA", 3),
|
||||
_make_chunk("docB", 0),
|
||||
_make_chunk("docB", 1),
|
||||
]
|
||||
result = _embed_chunks_to_store(
|
||||
chunks=chunks,
|
||||
embedder=MagicMock(),
|
||||
tenant_id="test",
|
||||
request_id=None,
|
||||
store=store,
|
||||
)
|
||||
|
||||
# docA should be fully excluded from results
|
||||
successful_doc_ids = {doc_id for _, doc_id in result.successful_chunk_ids}
|
||||
assert "docA" not in successful_doc_ids
|
||||
assert "docB" in successful_doc_ids
|
||||
assert len(result.connector_failures) == 1
|
||||
|
||||
# Verify batch 0 was scrubbed of docA chunks
|
||||
all_stored = list(store.stream())
|
||||
stored_doc_ids = {c.source_document.id for c in all_stored}
|
||||
assert "docA" not in stored_doc_ids
|
||||
assert "docB" in stored_doc_ids
|
||||
|
||||
@patch(
|
||||
"onyx.indexing.indexing_pipeline.embed_chunks_with_failure_handling",
|
||||
)
|
||||
@patch("onyx.indexing.indexing_pipeline.MAX_CHUNKS_PER_DOC_BATCH", 3)
|
||||
def test_later_batch_skips_already_failed_doc(self, mock_embed: MagicMock) -> None:
|
||||
"""If docA fails in batch 0, its chunks in batch 1 are skipped
|
||||
entirely (never sent to the embedder)."""
|
||||
embedded_doc_ids: list[str] = []
|
||||
|
||||
def _embed(
|
||||
chunks: list[DocAwareChunk], **_kwargs: object
|
||||
) -> tuple[list[IndexChunk], list[ConnectorFailure]]:
|
||||
for c in chunks:
|
||||
embedded_doc_ids.append(c.source_document.id)
|
||||
return _mock_embed_fail_doc("docA")(chunks)
|
||||
|
||||
mock_embed.side_effect = _embed
|
||||
|
||||
with ChunkBatchStore() as store:
|
||||
chunks = [
|
||||
_make_chunk("docA", 0),
|
||||
_make_chunk("docA", 1),
|
||||
_make_chunk("docA", 2),
|
||||
_make_chunk("docA", 3),
|
||||
_make_chunk("docB", 0),
|
||||
_make_chunk("docB", 1),
|
||||
]
|
||||
_embed_chunks_to_store(
|
||||
chunks=chunks,
|
||||
embedder=MagicMock(),
|
||||
tenant_id="test",
|
||||
request_id=None,
|
||||
store=store,
|
||||
)
|
||||
|
||||
# docA should only appear in batch 0, not batch 1
|
||||
batch_1_doc_ids = embedded_doc_ids[3:]
|
||||
assert "docA" not in batch_1_doc_ids
|
||||
|
||||
@patch(
|
||||
"onyx.indexing.indexing_pipeline.embed_chunks_with_failure_handling",
|
||||
)
|
||||
@patch("onyx.indexing.indexing_pipeline.MAX_CHUNKS_PER_DOC_BATCH", 3)
|
||||
def test_failed_doc_skipped_in_later_batch_while_other_doc_succeeds(
|
||||
self, mock_embed: MagicMock
|
||||
) -> None:
|
||||
"""doc1 spans batches 0 and 1, doc2 only in batch 1. Batch 0 fails
|
||||
doc1. In batch 1, doc1 chunks should be skipped but doc2 chunks
|
||||
should still be embedded successfully."""
|
||||
embedded_chunks: list[list[str]] = []
|
||||
|
||||
def _embed(
|
||||
chunks: list[DocAwareChunk], **_kwargs: object
|
||||
) -> tuple[list[IndexChunk], list[ConnectorFailure]]:
|
||||
embedded_chunks.append([c.source_document.id for c in chunks])
|
||||
return _mock_embed_fail_doc("doc1")(chunks)
|
||||
|
||||
mock_embed.side_effect = _embed
|
||||
|
||||
with ChunkBatchStore() as store:
|
||||
chunks = [
|
||||
_make_chunk("doc1", 0),
|
||||
_make_chunk("doc1", 1),
|
||||
_make_chunk("doc1", 2),
|
||||
_make_chunk("doc1", 3),
|
||||
_make_chunk("doc2", 0),
|
||||
_make_chunk("doc2", 1),
|
||||
]
|
||||
result = _embed_chunks_to_store(
|
||||
chunks=chunks,
|
||||
embedder=MagicMock(),
|
||||
tenant_id="test",
|
||||
request_id=None,
|
||||
store=store,
|
||||
)
|
||||
|
||||
# doc1 should be fully excluded, doc2 fully included
|
||||
successful_doc_ids = {doc_id for _, doc_id in result.successful_chunk_ids}
|
||||
assert "doc1" not in successful_doc_ids
|
||||
assert "doc2" in successful_doc_ids
|
||||
assert len(result.successful_chunk_ids) == 2 # doc2's 2 chunks
|
||||
|
||||
# Batch 1 should only contain doc2 (doc1 was filtered before embedding)
|
||||
assert len(embedded_chunks) == 2
|
||||
assert "doc1" not in embedded_chunks[1]
|
||||
assert embedded_chunks[1] == ["doc2", "doc2"]
|
||||
|
||||
# Verify on-disk state has no doc1 chunks
|
||||
all_stored = list(store.stream())
|
||||
assert all(c.source_document.id == "doc2" for c in all_stored)
|
||||
|
||||
@patch(
|
||||
"onyx.indexing.indexing_pipeline.embed_chunks_with_failure_handling",
|
||||
)
|
||||
def test_empty_input(self, mock_embed: MagicMock) -> None:
|
||||
"""Empty chunk list produces empty results."""
|
||||
mock_embed.side_effect = _mock_embed_success
|
||||
|
||||
with ChunkBatchStore() as store:
|
||||
result = _embed_chunks_to_store(
|
||||
chunks=[],
|
||||
embedder=MagicMock(),
|
||||
tenant_id="test",
|
||||
request_id=None,
|
||||
store=store,
|
||||
)
|
||||
|
||||
assert len(result.successful_chunk_ids) == 0
|
||||
assert len(result.connector_failures) == 0
|
||||
mock_embed.assert_not_called()
|
||||
|
||||
@patch(
|
||||
"onyx.indexing.indexing_pipeline.embed_chunks_with_failure_handling",
|
||||
)
|
||||
@patch("onyx.indexing.indexing_pipeline.MAX_CHUNKS_PER_DOC_BATCH", 100)
|
||||
def test_all_chunks_fail(self, mock_embed: MagicMock) -> None:
|
||||
"""When all documents fail, results have no successful chunks."""
|
||||
|
||||
def _fail_all(
|
||||
chunks: list[DocAwareChunk], **_kwargs: object
|
||||
) -> tuple[list[IndexChunk], list[ConnectorFailure]]:
|
||||
doc_ids = {c.source_document.id for c in chunks}
|
||||
return [], [_make_failure(doc_id) for doc_id in doc_ids]
|
||||
|
||||
mock_embed.side_effect = _fail_all
|
||||
|
||||
with ChunkBatchStore() as store:
|
||||
chunks = [_make_chunk("doc1", 0), _make_chunk("doc2", 1)]
|
||||
result = _embed_chunks_to_store(
|
||||
chunks=chunks,
|
||||
embedder=MagicMock(),
|
||||
tenant_id="test",
|
||||
request_id=None,
|
||||
store=store,
|
||||
)
|
||||
|
||||
assert len(result.successful_chunk_ids) == 0
|
||||
assert len(result.connector_failures) == 2
|
||||
@@ -116,7 +116,7 @@ def _run_adapter_build(
|
||||
project_ids_map: dict[str, list[int]],
|
||||
persona_ids_map: dict[str, list[int]],
|
||||
) -> list[DocMetadataAwareIndexChunk]:
|
||||
"""Helper that runs UserFileIndexingAdapter.build_metadata_aware_chunks
|
||||
"""Helper that runs UserFileIndexingAdapter.prepare_enrichment + enrich_chunk
|
||||
with all external dependencies mocked."""
|
||||
from onyx.indexing.adapters.user_file_indexing_adapter import (
|
||||
UserFileIndexingAdapter,
|
||||
@@ -155,18 +155,16 @@ def _run_adapter_build(
|
||||
side_effect=Exception("no LLM in tests"),
|
||||
),
|
||||
):
|
||||
result = adapter.build_metadata_aware_chunks(
|
||||
chunks_with_embeddings=[chunk],
|
||||
chunk_content_scores=[1.0],
|
||||
tenant_id="test_tenant",
|
||||
enricher = adapter.prepare_enrichment(
|
||||
context=context,
|
||||
tenant_id="test_tenant",
|
||||
chunks=[chunk],
|
||||
)
|
||||
|
||||
return result.chunks
|
||||
return [enricher.enrich_chunk(chunk, 1.0)]
|
||||
|
||||
|
||||
def test_build_metadata_aware_chunks_includes_persona_ids() -> None:
|
||||
"""UserFileIndexingAdapter.build_metadata_aware_chunks writes persona IDs
|
||||
def test_prepare_enrichment_includes_persona_ids() -> None:
|
||||
"""UserFileIndexingAdapter.prepare_enrichment writes persona IDs
|
||||
fetched from the DB into each chunk's metadata."""
|
||||
file_id = str(uuid4())
|
||||
persona_ids = [5, 12]
|
||||
@@ -183,7 +181,7 @@ def test_build_metadata_aware_chunks_includes_persona_ids() -> None:
|
||||
assert chunks[0].user_project == project_ids
|
||||
|
||||
|
||||
def test_build_metadata_aware_chunks_missing_file_defaults_to_empty() -> None:
|
||||
def test_prepare_enrichment_missing_file_defaults_to_empty() -> None:
|
||||
"""When a file has no persona or project associations in the DB, the
|
||||
adapter should default to empty lists (not KeyError or None)."""
|
||||
file_id = str(uuid4())
|
||||
|
||||
@@ -5,7 +5,7 @@ home: https://www.onyx.app/
|
||||
sources:
|
||||
- "https://github.com/onyx-dot-app/onyx"
|
||||
type: application
|
||||
version: 0.4.38
|
||||
version: 0.4.39
|
||||
appVersion: latest
|
||||
annotations:
|
||||
category: Productivity
|
||||
|
||||
2037
deployment/helm/charts/onyx/dashboards/indexing-pipeline.json
Normal file
2037
deployment/helm/charts/onyx/dashboards/indexing-pipeline.json
Normal file
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,77 @@
|
||||
{{- if and .Values.monitoring.serviceMonitors.enabled .Values.vectorDB.enabled }}
|
||||
{{- if gt (int .Values.celery_worker_monitoring.replicaCount) 0 }}
|
||||
---
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: ServiceMonitor
|
||||
metadata:
|
||||
name: {{ include "onyx.fullname" . }}-celery-worker-monitoring
|
||||
labels:
|
||||
{{- include "onyx.labels" . | nindent 4 }}
|
||||
{{- with .Values.monitoring.serviceMonitors.labels }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
namespaceSelector:
|
||||
matchNames:
|
||||
- {{ .Release.Namespace }}
|
||||
selector:
|
||||
matchLabels:
|
||||
app: {{ .Values.celery_worker_monitoring.deploymentLabels.app }}
|
||||
metrics: "true"
|
||||
endpoints:
|
||||
- port: metrics
|
||||
path: /metrics
|
||||
interval: 30s
|
||||
scrapeTimeout: 10s
|
||||
{{- end }}
|
||||
{{- if gt (int .Values.celery_worker_docfetching.replicaCount) 0 }}
|
||||
---
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: ServiceMonitor
|
||||
metadata:
|
||||
name: {{ include "onyx.fullname" . }}-celery-worker-docfetching
|
||||
labels:
|
||||
{{- include "onyx.labels" . | nindent 4 }}
|
||||
{{- with .Values.monitoring.serviceMonitors.labels }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
namespaceSelector:
|
||||
matchNames:
|
||||
- {{ .Release.Namespace }}
|
||||
selector:
|
||||
matchLabels:
|
||||
app: {{ .Values.celery_worker_docfetching.deploymentLabels.app }}
|
||||
metrics: "true"
|
||||
endpoints:
|
||||
- port: metrics
|
||||
path: /metrics
|
||||
interval: 30s
|
||||
scrapeTimeout: 10s
|
||||
{{- end }}
|
||||
{{- if gt (int .Values.celery_worker_docprocessing.replicaCount) 0 }}
|
||||
---
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: ServiceMonitor
|
||||
metadata:
|
||||
name: {{ include "onyx.fullname" . }}-celery-worker-docprocessing
|
||||
labels:
|
||||
{{- include "onyx.labels" . | nindent 4 }}
|
||||
{{- with .Values.monitoring.serviceMonitors.labels }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
namespaceSelector:
|
||||
matchNames:
|
||||
- {{ .Release.Namespace }}
|
||||
selector:
|
||||
matchLabels:
|
||||
app: {{ .Values.celery_worker_docprocessing.deploymentLabels.app }}
|
||||
metrics: "true"
|
||||
endpoints:
|
||||
- port: metrics
|
||||
path: /metrics
|
||||
interval: 30s
|
||||
scrapeTimeout: 10s
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
@@ -0,0 +1,15 @@
|
||||
{{- if .Values.monitoring.grafana.dashboards.enabled }}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: {{ include "onyx.fullname" . }}-indexing-pipeline-dashboard
|
||||
labels:
|
||||
{{- include "onyx.labels" . | nindent 4 }}
|
||||
grafana_dashboard: "1"
|
||||
annotations:
|
||||
grafana_folder: "Onyx"
|
||||
data:
|
||||
onyx-indexing-pipeline.json: |
|
||||
{{- .Files.Get "dashboards/indexing-pipeline.json" | nindent 4 }}
|
||||
{{- end }}
|
||||
@@ -256,6 +256,20 @@ tooling:
|
||||
# -- Which client binary to call; change if your image uses a non-default path.
|
||||
psqlBinary: psql
|
||||
|
||||
monitoring:
|
||||
grafana:
|
||||
dashboards:
|
||||
# -- Set to true to deploy Grafana dashboard ConfigMaps for the Onyx indexing pipeline.
|
||||
# Requires kube-prometheus-stack (or equivalent) with the Grafana sidecar enabled and watching this namespace.
|
||||
# The sidecar must be configured with label selector: grafana_dashboard=1
|
||||
enabled: false
|
||||
serviceMonitors:
|
||||
# -- Set to true to deploy ServiceMonitor resources for Celery worker metrics endpoints.
|
||||
# Requires the Prometheus Operator CRDs (included in kube-prometheus-stack).
|
||||
# Use `labels` to match your Prometheus CR's serviceMonitorSelector (e.g. release: onyx-monitoring).
|
||||
enabled: false
|
||||
labels: {}
|
||||
|
||||
serviceAccount:
|
||||
# Specifies whether a service account should be created
|
||||
create: false
|
||||
|
||||
@@ -19,6 +19,10 @@ module "eks" {
|
||||
cluster_endpoint_public_access_cidrs = var.cluster_endpoint_public_access_cidrs
|
||||
enable_cluster_creator_admin_permissions = true
|
||||
|
||||
# Control plane logging
|
||||
cluster_enabled_log_types = var.cluster_enabled_log_types
|
||||
cloudwatch_log_group_retention_in_days = var.cloudwatch_log_group_retention_in_days
|
||||
|
||||
eks_managed_node_group_defaults = {
|
||||
ami_type = "AL2023_x86_64_STANDARD"
|
||||
}
|
||||
|
||||
@@ -161,3 +161,25 @@ variable "rds_db_connect_arn" {
|
||||
description = "Full rds-db:connect ARN to allow (required when enable_rds_iam_for_service_account is true)"
|
||||
default = null
|
||||
}
|
||||
|
||||
variable "cluster_enabled_log_types" {
|
||||
type = list(string)
|
||||
description = "EKS control plane log types to enable (valid: api, audit, authenticator, controllerManager, scheduler)"
|
||||
default = ["api", "audit", "authenticator", "controllerManager", "scheduler"]
|
||||
|
||||
validation {
|
||||
condition = alltrue([for t in var.cluster_enabled_log_types : contains(["api", "audit", "authenticator", "controllerManager", "scheduler"], t)])
|
||||
error_message = "Each entry must be one of: api, audit, authenticator, controllerManager, scheduler."
|
||||
}
|
||||
}
|
||||
|
||||
variable "cloudwatch_log_group_retention_in_days" {
|
||||
type = number
|
||||
description = "Number of days to retain EKS control plane logs in CloudWatch (0 = never expire)"
|
||||
default = 30
|
||||
|
||||
validation {
|
||||
condition = contains([0, 1, 3, 5, 7, 14, 30, 60, 90, 120, 150, 180, 365, 400, 545, 731, 1096, 1827, 2192, 2557, 2922, 3288, 3653], var.cloudwatch_log_group_retention_in_days)
|
||||
error_message = "Must be a valid CloudWatch retention value (0, 1, 3, 5, 7, 14, 30, 60, 90, 120, 150, 180, 365, 400, 545, 731, 1096, 1827, 2192, 2557, 2922, 3288, 3653)."
|
||||
}
|
||||
}
|
||||
|
||||
@@ -54,6 +54,9 @@ module "postgres" {
|
||||
password = var.postgres_password
|
||||
tags = local.merged_tags
|
||||
enable_rds_iam_auth = var.enable_iam_auth
|
||||
|
||||
backup_retention_period = var.postgres_backup_retention_period
|
||||
backup_window = var.postgres_backup_window
|
||||
}
|
||||
|
||||
module "s3" {
|
||||
@@ -80,6 +83,10 @@ module "eks" {
|
||||
public_cluster_enabled = var.public_cluster_enabled
|
||||
private_cluster_enabled = var.private_cluster_enabled
|
||||
cluster_endpoint_public_access_cidrs = var.cluster_endpoint_public_access_cidrs
|
||||
|
||||
# Control plane logging
|
||||
cluster_enabled_log_types = var.eks_cluster_enabled_log_types
|
||||
cloudwatch_log_group_retention_in_days = var.eks_cloudwatch_log_group_retention_in_days
|
||||
}
|
||||
|
||||
module "waf" {
|
||||
|
||||
@@ -250,3 +250,34 @@ variable "opensearch_subnet_ids" {
|
||||
description = "Subnet IDs for OpenSearch. If empty, uses first 3 private subnets."
|
||||
default = []
|
||||
}
|
||||
|
||||
# RDS Backup Configuration
|
||||
variable "postgres_backup_retention_period" {
|
||||
type = number
|
||||
description = "Number of days to retain automated RDS backups (0 to disable)"
|
||||
default = 7
|
||||
}
|
||||
|
||||
variable "postgres_backup_window" {
|
||||
type = string
|
||||
description = "Preferred UTC time window for automated RDS backups (hh24:mi-hh24:mi)"
|
||||
default = "03:00-04:00"
|
||||
}
|
||||
|
||||
# EKS Control Plane Logging
|
||||
variable "eks_cluster_enabled_log_types" {
|
||||
type = list(string)
|
||||
description = "EKS control plane log types to enable (valid: api, audit, authenticator, controllerManager, scheduler)"
|
||||
default = ["api", "audit", "authenticator", "controllerManager", "scheduler"]
|
||||
}
|
||||
|
||||
variable "eks_cloudwatch_log_group_retention_in_days" {
|
||||
type = number
|
||||
description = "Number of days to retain EKS control plane logs in CloudWatch (0 = never expire)"
|
||||
default = 30
|
||||
|
||||
validation {
|
||||
condition = contains([0, 1, 3, 5, 7, 14, 30, 60, 90, 120, 150, 180, 365, 400, 545, 731, 1096, 1827, 2192, 2557, 2922, 3288, 3653], var.eks_cloudwatch_log_group_retention_in_days)
|
||||
error_message = "Must be a valid CloudWatch retention value (0, 1, 3, 5, 7, 14, 30, 60, 90, 120, 150, 180, 365, 400, 545, 731, 1096, 1827, 2192, 2557, 2922, 3288, 3653)."
|
||||
}
|
||||
}
|
||||
|
||||
@@ -44,5 +44,56 @@ resource "aws_db_instance" "this" {
|
||||
publicly_accessible = false
|
||||
deletion_protection = true
|
||||
storage_encrypted = true
|
||||
tags = var.tags
|
||||
|
||||
# Automated backups
|
||||
backup_retention_period = var.backup_retention_period
|
||||
backup_window = var.backup_window
|
||||
|
||||
tags = var.tags
|
||||
}
|
||||
|
||||
# CloudWatch alarm for CPU utilization monitoring
|
||||
resource "aws_cloudwatch_metric_alarm" "cpu_utilization" {
|
||||
alarm_name = "${var.identifier}-cpu-utilization"
|
||||
alarm_description = "RDS CPU utilization for ${var.identifier}"
|
||||
comparison_operator = "GreaterThanThreshold"
|
||||
evaluation_periods = var.cpu_alarm_evaluation_periods
|
||||
metric_name = "CPUUtilization"
|
||||
namespace = "AWS/RDS"
|
||||
period = var.cpu_alarm_period
|
||||
statistic = "Average"
|
||||
threshold = var.cpu_alarm_threshold
|
||||
treat_missing_data = "missing"
|
||||
|
||||
alarm_actions = var.alarm_actions
|
||||
ok_actions = var.alarm_actions
|
||||
|
||||
dimensions = {
|
||||
DBInstanceIdentifier = aws_db_instance.this.identifier
|
||||
}
|
||||
|
||||
tags = var.tags
|
||||
}
|
||||
|
||||
# CloudWatch alarm for freeable memory monitoring
|
||||
resource "aws_cloudwatch_metric_alarm" "freeable_memory" {
|
||||
alarm_name = "${var.identifier}-freeable-memory"
|
||||
alarm_description = "RDS freeable memory for ${var.identifier}"
|
||||
comparison_operator = "LessThanThreshold"
|
||||
evaluation_periods = var.memory_alarm_evaluation_periods
|
||||
metric_name = "FreeableMemory"
|
||||
namespace = "AWS/RDS"
|
||||
period = var.memory_alarm_period
|
||||
statistic = "Average"
|
||||
threshold = var.memory_alarm_threshold
|
||||
treat_missing_data = "missing"
|
||||
|
||||
alarm_actions = var.alarm_actions
|
||||
ok_actions = var.alarm_actions
|
||||
|
||||
dimensions = {
|
||||
DBInstanceIdentifier = aws_db_instance.this.identifier
|
||||
}
|
||||
|
||||
tags = var.tags
|
||||
}
|
||||
|
||||
@@ -67,3 +67,98 @@ variable "enable_rds_iam_auth" {
|
||||
description = "Enable AWS IAM database authentication for this RDS instance"
|
||||
default = false
|
||||
}
|
||||
|
||||
variable "backup_retention_period" {
|
||||
type = number
|
||||
description = "Number of days to retain automated backups (0 to disable)"
|
||||
default = 7
|
||||
|
||||
validation {
|
||||
condition = var.backup_retention_period >= 0 && var.backup_retention_period <= 35
|
||||
error_message = "backup_retention_period must be between 0 and 35 (AWS RDS limit)."
|
||||
}
|
||||
}
|
||||
|
||||
variable "backup_window" {
|
||||
type = string
|
||||
description = "Preferred UTC time window for automated backups (hh24:mi-hh24:mi)"
|
||||
default = "03:00-04:00"
|
||||
|
||||
validation {
|
||||
condition = can(regex("^([01]\\d|2[0-3]):[0-5]\\d-([01]\\d|2[0-3]):[0-5]\\d$", var.backup_window))
|
||||
error_message = "backup_window must be in hh24:mi-hh24:mi format (e.g. \"03:00-04:00\")."
|
||||
}
|
||||
}
|
||||
|
||||
# CloudWatch CPU alarm configuration
|
||||
variable "cpu_alarm_threshold" {
|
||||
type = number
|
||||
description = "CPU utilization percentage threshold for the CloudWatch alarm"
|
||||
default = 80
|
||||
|
||||
validation {
|
||||
condition = var.cpu_alarm_threshold >= 0 && var.cpu_alarm_threshold <= 100
|
||||
error_message = "cpu_alarm_threshold must be between 0 and 100 (percentage)."
|
||||
}
|
||||
}
|
||||
|
||||
variable "cpu_alarm_evaluation_periods" {
|
||||
type = number
|
||||
description = "Number of consecutive periods the threshold must be breached before alarming"
|
||||
default = 3
|
||||
|
||||
validation {
|
||||
condition = var.cpu_alarm_evaluation_periods >= 1
|
||||
error_message = "cpu_alarm_evaluation_periods must be at least 1."
|
||||
}
|
||||
}
|
||||
|
||||
variable "cpu_alarm_period" {
|
||||
type = number
|
||||
description = "Period in seconds over which the CPU metric is evaluated"
|
||||
default = 300
|
||||
|
||||
validation {
|
||||
condition = var.cpu_alarm_period >= 60 && var.cpu_alarm_period % 60 == 0
|
||||
error_message = "cpu_alarm_period must be a multiple of 60 seconds and at least 60 (CloudWatch requirement)."
|
||||
}
|
||||
}
|
||||
|
||||
variable "memory_alarm_threshold" {
|
||||
type = number
|
||||
description = "Freeable memory threshold in bytes. Alarm fires when memory drops below this value."
|
||||
default = 256000000 # 256 MB
|
||||
|
||||
validation {
|
||||
condition = var.memory_alarm_threshold > 0
|
||||
error_message = "memory_alarm_threshold must be greater than 0."
|
||||
}
|
||||
}
|
||||
|
||||
variable "memory_alarm_evaluation_periods" {
|
||||
type = number
|
||||
description = "Number of consecutive periods the threshold must be breached before alarming"
|
||||
default = 3
|
||||
|
||||
validation {
|
||||
condition = var.memory_alarm_evaluation_periods >= 1
|
||||
error_message = "memory_alarm_evaluation_periods must be at least 1."
|
||||
}
|
||||
}
|
||||
|
||||
variable "memory_alarm_period" {
|
||||
type = number
|
||||
description = "Period in seconds over which the freeable memory metric is evaluated"
|
||||
default = 300
|
||||
|
||||
validation {
|
||||
condition = var.memory_alarm_period >= 60 && var.memory_alarm_period % 60 == 0
|
||||
error_message = "memory_alarm_period must be a multiple of 60 seconds and at least 60 (CloudWatch requirement)."
|
||||
}
|
||||
}
|
||||
|
||||
variable "alarm_actions" {
|
||||
type = list(string)
|
||||
description = "List of ARNs to notify when the alarm transitions state (e.g. SNS topic ARNs)"
|
||||
default = []
|
||||
}
|
||||
|
||||
6
uv.lock
generated
6
uv.lock
generated
@@ -5634,11 +5634,11 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "pygments"
|
||||
version = "2.19.2"
|
||||
version = "2.20.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/b0/77/a5b8c569bf593b0140bde72ea885a803b82086995367bf2037de0159d924/pygments-2.19.2.tar.gz", hash = "sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887", size = 4968631, upload-time = "2025-06-21T13:39:12.283Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/c3/b2/bc9c9196916376152d655522fdcebac55e66de6603a76a02bca1b6414f6c/pygments-2.20.0.tar.gz", hash = "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f", size = 4955991, upload-time = "2026-03-29T13:29:33.898Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/c7/21/705964c7812476f378728bdf590ca4b771ec72385c533964653c68e86bdc/pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b", size = 1225217, upload-time = "2025-06-21T13:39:07.939Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
||||
@@ -73,11 +73,17 @@ ENV NEXT_PUBLIC_STRIPE_PUBLISHABLE_KEY=${NEXT_PUBLIC_STRIPE_PUBLISHABLE_KEY}
|
||||
ARG NEXT_PUBLIC_RECAPTCHA_SITE_KEY
|
||||
ENV NEXT_PUBLIC_RECAPTCHA_SITE_KEY=${NEXT_PUBLIC_RECAPTCHA_SITE_KEY}
|
||||
|
||||
ARG SENTRY_RELEASE
|
||||
ENV SENTRY_RELEASE=${SENTRY_RELEASE}
|
||||
|
||||
# Add NODE_OPTIONS argument
|
||||
ARG NODE_OPTIONS
|
||||
|
||||
# SENTRY_AUTH_TOKEN is injected via BuildKit secret mount so it is never written
|
||||
# to any image layer, build cache, or registry manifest.
|
||||
# Use NODE_OPTIONS in the build command
|
||||
RUN NODE_OPTIONS="${NODE_OPTIONS}" npx next build
|
||||
RUN --mount=type=secret,id=sentry_auth_token,env=SENTRY_AUTH_TOKEN \
|
||||
NODE_OPTIONS="${NODE_OPTIONS}" npx next build
|
||||
|
||||
# Step 2. Production image, copy all the files and run next
|
||||
FROM base AS runner
|
||||
@@ -150,6 +156,9 @@ ENV NEXT_PUBLIC_STRIPE_PUBLISHABLE_KEY=${NEXT_PUBLIC_STRIPE_PUBLISHABLE_KEY}
|
||||
ARG NEXT_PUBLIC_RECAPTCHA_SITE_KEY
|
||||
ENV NEXT_PUBLIC_RECAPTCHA_SITE_KEY=${NEXT_PUBLIC_RECAPTCHA_SITE_KEY}
|
||||
|
||||
ARG SENTRY_RELEASE
|
||||
ENV SENTRY_RELEASE=${SENTRY_RELEASE}
|
||||
|
||||
# Default ONYX_VERSION, typically overriden during builds by GitHub Actions.
|
||||
ARG ONYX_VERSION=0.0.0-dev
|
||||
ENV ONYX_VERSION=${ONYX_VERSION}
|
||||
|
||||
@@ -104,7 +104,7 @@ function Button({
|
||||
isLarge ? "default" : size === "2xs" ? "mini" : "compact"
|
||||
}
|
||||
>
|
||||
<div className="flex flex-row items-center gap-1 interactive-foreground">
|
||||
<div className="flex flex-row items-center gap-1">
|
||||
{iconWrapper(Icon, size, !!children)}
|
||||
|
||||
{labelEl}
|
||||
|
||||
@@ -67,7 +67,7 @@ function FilterButton({
|
||||
state={active ? "selected" : "empty"}
|
||||
>
|
||||
<Interactive.Container type="button">
|
||||
<div className="interactive-foreground flex flex-row items-center gap-1">
|
||||
<div className="flex flex-row items-center gap-1">
|
||||
{iconWrapper(Icon, "lg", true)}
|
||||
<Text font="main-ui-action" color="inherit" nowrap>
|
||||
{children}
|
||||
|
||||
@@ -16,7 +16,7 @@ import * as TooltipPrimitive from "@radix-ui/react-tooltip";
|
||||
|
||||
type ContentPassthroughProps = DistributiveOmit<
|
||||
ContentActionProps,
|
||||
"paddingVariant" | "widthVariant" | "ref" | "withInteractive"
|
||||
"paddingVariant" | "widthVariant" | "ref"
|
||||
>;
|
||||
|
||||
type LineItemButtonOwnProps = Pick<
|
||||
@@ -92,7 +92,6 @@ function LineItemButton({
|
||||
>
|
||||
<ContentAction
|
||||
{...(contentActionProps as ContentActionProps)}
|
||||
withInteractive
|
||||
paddingVariant="fit"
|
||||
/>
|
||||
</Interactive.Container>
|
||||
|
||||
@@ -132,7 +132,7 @@ function OpenButton({
|
||||
>
|
||||
<div
|
||||
className={cn(
|
||||
"interactive-foreground flex flex-row items-center",
|
||||
"flex flex-row items-center",
|
||||
justifyContent === "between" ? "w-full justify-between" : "gap-1",
|
||||
foldable &&
|
||||
justifyContent !== "between" &&
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
"use client";
|
||||
|
||||
import "@opal/components/buttons/select-button/styles.css";
|
||||
import {
|
||||
Interactive,
|
||||
@@ -84,11 +86,13 @@ function SelectButton({
|
||||
const isLarge = size === "lg";
|
||||
|
||||
const labelEl = children ? (
|
||||
<span className="opal-select-button-label">
|
||||
<Text font={isLarge ? "main-ui-body" : "secondary-body"} color="inherit">
|
||||
{children}
|
||||
</Text>
|
||||
</span>
|
||||
<Text
|
||||
font={isLarge ? "main-ui-body" : "secondary-body"}
|
||||
color="inherit"
|
||||
nowrap
|
||||
>
|
||||
{children}
|
||||
</Text>
|
||||
) : null;
|
||||
|
||||
const button = (
|
||||
@@ -103,7 +107,7 @@ function SelectButton({
|
||||
>
|
||||
<div
|
||||
className={cn(
|
||||
"opal-select-button interactive-foreground",
|
||||
"opal-select-button",
|
||||
foldable && "interactive-foldable-host"
|
||||
)}
|
||||
>
|
||||
|
||||
@@ -3,7 +3,3 @@
|
||||
.opal-select-button {
|
||||
@apply flex flex-row items-center gap-1;
|
||||
}
|
||||
|
||||
.opal-select-button-label {
|
||||
@apply whitespace-nowrap;
|
||||
}
|
||||
|
||||
136
web/lib/opal/src/components/cards/select-card/README.md
Normal file
136
web/lib/opal/src/components/cards/select-card/README.md
Normal file
@@ -0,0 +1,136 @@
|
||||
# SelectCard
|
||||
|
||||
**Import:** `import { SelectCard, type SelectCardProps } from "@opal/components";`
|
||||
|
||||
A stateful interactive card — the card counterpart to [`SelectButton`](../../buttons/select-button/README.md). Built on `Interactive.Stateful` (Slot) with a structural `<div>` that owns padding, rounding, border, and overflow.
|
||||
|
||||
## Relationship to Card
|
||||
|
||||
`Card` is a plain, non-interactive container. `SelectCard` adds stateful interactivity (hover, active, disabled, state-driven colors) by wrapping its root div with `Interactive.Stateful`. The relationship mirrors `Button` (stateless) vs `SelectButton` (stateful).
|
||||
|
||||
## Relationship to SelectButton
|
||||
|
||||
SelectCard and SelectButton share the same call stack:
|
||||
|
||||
```
|
||||
Interactive.Stateful → structural element → content
|
||||
```
|
||||
|
||||
The key differences:
|
||||
|
||||
- SelectCard renders a `<div>` (not `Interactive.Container`) — cards have their own rounding scale (one notch larger than buttons) and don't need Container's height/min-width.
|
||||
- SelectCard has no `foldable` prop — use `Interactive.Foldable` directly inside children.
|
||||
- SelectCard's children are fully composable — use `CardHeaderLayout`, `ContentAction`, `Content`, buttons, etc. inside.
|
||||
|
||||
## Architecture
|
||||
|
||||
```
|
||||
Interactive.Stateful <- variant, state, interaction, disabled, onClick
|
||||
└─ div.opal-select-card <- padding, rounding, border, overflow
|
||||
└─ children (composable)
|
||||
```
|
||||
|
||||
The `Interactive.Stateful` Slot merges onto the div, producing a single DOM element with both `.opal-select-card` and `.interactive` classes plus `data-interactive-*` attributes. This activates the Stateful color matrix for backgrounds and `--interactive-foreground` / `--interactive-foreground-icon` CSS properties for descendants.
|
||||
|
||||
## Props
|
||||
|
||||
Inherits **all** props from `InteractiveStatefulProps` (variant, state, interaction, onClick, href, etc.) plus:
|
||||
|
||||
| Prop | Type | Default | Description |
|
||||
|---|---|---|---|
|
||||
| `sizeVariant` | `ContainerSizeVariants` | `"lg"` | Controls padding and border-radius |
|
||||
| `ref` | `React.Ref<HTMLDivElement>` | — | Ref forwarded to the root div |
|
||||
| `children` | `React.ReactNode` | — | Card content |
|
||||
|
||||
### Rounding scale
|
||||
|
||||
Cards use a bumped-up rounding scale compared to buttons:
|
||||
|
||||
| Size | Rounding | Effective radius |
|
||||
|---|---|---|
|
||||
| `lg` | `rounded-16` | 1rem (16px) |
|
||||
| `md`–`sm` | `rounded-12` | 0.75rem (12px) |
|
||||
| `xs`–`2xs` | `rounded-08` | 0.5rem (8px) |
|
||||
| `fit` | `rounded-16` | 1rem (16px) |
|
||||
|
||||
### Recommended variant: `select-card`
|
||||
|
||||
The `select-card` Interactive.Stateful variant is specifically designed for cards. Unlike `select-heavy` (which only changes foreground color between empty and filled), `select-card` gives the filled state a visible background — important on larger surfaces where background carries more of the visual distinction.
|
||||
|
||||
| State | Rest background | Rest foreground |
|
||||
|---|---|---|
|
||||
| `empty` | transparent | `text-04` / icon `text-03` |
|
||||
| `filled` | `background-tint-00` | `text-04` / icon `text-03` |
|
||||
| `selected` | `action-link-01` | `action-link-05` |
|
||||
|
||||
The selected state also gets a `border-action-link-05` via SelectCard's CSS.
|
||||
|
||||
## CSS
|
||||
|
||||
SelectCard's stylesheet (`styles.css`) provides:
|
||||
|
||||
- `w-full overflow-clip border` on all states
|
||||
- `border-action-link-05` when `data-interactive-state="selected"`
|
||||
|
||||
All background and foreground colors come from the Interactive.Stateful CSS, not from SelectCard.
|
||||
|
||||
## Usage
|
||||
|
||||
### Provider selection card
|
||||
|
||||
```tsx
|
||||
import { SelectCard } from "@opal/components";
|
||||
import { CardHeaderLayout } from "@opal/layouts";
|
||||
|
||||
<SelectCard variant="select-card" state="selected" onClick={handleClick}>
|
||||
<CardHeaderLayout
|
||||
icon={SvgGlobe}
|
||||
title="Google"
|
||||
description="Search engine"
|
||||
sizePreset="main-ui"
|
||||
variant="section"
|
||||
rightChildren={<Button icon={SvgCheckSquare} variant="action" prominence="tertiary">Current Default</Button>}
|
||||
bottomRightChildren={
|
||||
<Button icon={SvgSettings} size="sm" prominence="tertiary" />
|
||||
}
|
||||
/>
|
||||
</SelectCard>
|
||||
```
|
||||
|
||||
### Disconnected state (clickable)
|
||||
|
||||
```tsx
|
||||
<SelectCard variant="select-card" state="empty" onClick={handleConnect}>
|
||||
<CardHeaderLayout
|
||||
icon={SvgCloud}
|
||||
title="OpenAI"
|
||||
description="Not configured"
|
||||
sizePreset="main-ui"
|
||||
variant="section"
|
||||
rightChildren={<Button rightIcon={SvgArrowExchange} prominence="tertiary">Connect</Button>}
|
||||
/>
|
||||
</SelectCard>
|
||||
```
|
||||
|
||||
### With foldable hover-reveal
|
||||
|
||||
```tsx
|
||||
<SelectCard variant="select-card" state="filled">
|
||||
<CardHeaderLayout
|
||||
icon={SvgCloud}
|
||||
title="OpenAI"
|
||||
description="Connected"
|
||||
sizePreset="main-ui"
|
||||
variant="section"
|
||||
rightChildren={
|
||||
<div className="interactive-foldable-host flex items-center">
|
||||
<Interactive.Foldable>
|
||||
<Button rightIcon={SvgArrowRightCircle} prominence="tertiary">
|
||||
Set as Default
|
||||
</Button>
|
||||
</Interactive.Foldable>
|
||||
</div>
|
||||
}
|
||||
/>
|
||||
</SelectCard>
|
||||
```
|
||||
@@ -0,0 +1,247 @@
|
||||
import type { Meta, StoryObj } from "@storybook/react";
|
||||
import { SelectCard } from "@opal/components";
|
||||
import { Button } from "@opal/components";
|
||||
import { Content } from "@opal/layouts";
|
||||
import {
|
||||
SvgArrowExchange,
|
||||
SvgArrowRightCircle,
|
||||
SvgCheckSquare,
|
||||
SvgGlobe,
|
||||
SvgSettings,
|
||||
SvgUnplug,
|
||||
} from "@opal/icons";
|
||||
import { Interactive } from "@opal/core";
|
||||
import * as TooltipPrimitive from "@radix-ui/react-tooltip";
|
||||
import type { Decorator } from "@storybook/react";
|
||||
|
||||
const withTooltipProvider: Decorator = (Story) => (
|
||||
<TooltipPrimitive.Provider>
|
||||
<Story />
|
||||
</TooltipPrimitive.Provider>
|
||||
);
|
||||
|
||||
const STATES = ["empty", "filled", "selected"] as const;
|
||||
const SIZE_VARIANTS = ["lg", "md", "sm", "xs", "2xs", "fit"] as const;
|
||||
|
||||
const meta = {
|
||||
title: "opal/components/SelectCard",
|
||||
component: SelectCard,
|
||||
tags: ["autodocs"],
|
||||
decorators: [withTooltipProvider],
|
||||
parameters: {
|
||||
layout: "centered",
|
||||
},
|
||||
} satisfies Meta<typeof SelectCard>;
|
||||
|
||||
export default meta;
|
||||
|
||||
type Story = StoryObj<typeof meta>;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Stories
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export const Default: Story = {
|
||||
render: () => (
|
||||
<div className="w-96">
|
||||
<SelectCard variant="select-card" state="empty">
|
||||
<div className="p-2">
|
||||
<Content
|
||||
sizePreset="main-ui"
|
||||
variant="section"
|
||||
icon={SvgGlobe}
|
||||
title="Google Search"
|
||||
description="Web search provider"
|
||||
/>
|
||||
</div>
|
||||
</SelectCard>
|
||||
</div>
|
||||
),
|
||||
};
|
||||
|
||||
export const AllStates: Story = {
|
||||
render: () => (
|
||||
<div className="flex flex-col gap-4 w-96">
|
||||
{STATES.map((state) => (
|
||||
<SelectCard key={state} variant="select-card" state={state}>
|
||||
<div className="p-2">
|
||||
<Content
|
||||
sizePreset="main-ui"
|
||||
variant="section"
|
||||
icon={SvgGlobe}
|
||||
title={`State: ${state}`}
|
||||
description="Hover to see interaction states."
|
||||
/>
|
||||
</div>
|
||||
</SelectCard>
|
||||
))}
|
||||
</div>
|
||||
),
|
||||
};
|
||||
|
||||
export const Clickable: Story = {
|
||||
render: () => (
|
||||
<div className="w-96">
|
||||
<SelectCard
|
||||
variant="select-card"
|
||||
state="empty"
|
||||
onClick={() => alert("Card clicked")}
|
||||
>
|
||||
<div className="p-2">
|
||||
<Content
|
||||
sizePreset="main-ui"
|
||||
variant="section"
|
||||
icon={SvgGlobe}
|
||||
title="Clickable Card"
|
||||
description="Click anywhere on this card."
|
||||
/>
|
||||
</div>
|
||||
</SelectCard>
|
||||
</div>
|
||||
),
|
||||
};
|
||||
|
||||
export const WithActions: Story = {
|
||||
render: () => (
|
||||
<div className="flex flex-col gap-4 w-[28rem]">
|
||||
{/* Disconnected */}
|
||||
<SelectCard variant="select-card" state="empty" onClick={() => {}}>
|
||||
<div className="flex flex-row items-stretch w-full">
|
||||
<div className="flex-1 p-2">
|
||||
<Content
|
||||
sizePreset="main-ui"
|
||||
variant="section"
|
||||
icon={SvgGlobe}
|
||||
title="Disconnected"
|
||||
description="Click to connect."
|
||||
/>
|
||||
</div>
|
||||
<div className="flex items-center">
|
||||
<Button prominence="tertiary" rightIcon={SvgArrowExchange}>
|
||||
Connect
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
</SelectCard>
|
||||
|
||||
{/* Connected with foldable */}
|
||||
<SelectCard variant="select-card" state="filled">
|
||||
<div className="flex flex-row items-stretch w-full">
|
||||
<div className="flex-1 p-2">
|
||||
<Content
|
||||
sizePreset="main-ui"
|
||||
variant="section"
|
||||
icon={SvgGlobe}
|
||||
title="Connected"
|
||||
description="Hover to reveal Set as Default."
|
||||
/>
|
||||
</div>
|
||||
<div className="flex flex-col items-end justify-between">
|
||||
<div className="interactive-foldable-host flex items-center">
|
||||
<Interactive.Foldable>
|
||||
<Button prominence="tertiary" rightIcon={SvgArrowRightCircle}>
|
||||
Set as Default
|
||||
</Button>
|
||||
</Interactive.Foldable>
|
||||
</div>
|
||||
<div className="flex flex-row px-1 pb-1">
|
||||
<Button
|
||||
icon={SvgUnplug}
|
||||
tooltip="Disconnect"
|
||||
prominence="tertiary"
|
||||
size="sm"
|
||||
/>
|
||||
<Button
|
||||
icon={SvgSettings}
|
||||
tooltip="Edit"
|
||||
prominence="tertiary"
|
||||
size="sm"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</SelectCard>
|
||||
|
||||
{/* Selected */}
|
||||
<SelectCard variant="select-card" state="selected">
|
||||
<div className="flex flex-row items-stretch w-full">
|
||||
<div className="flex-1 p-2">
|
||||
<Content
|
||||
sizePreset="main-ui"
|
||||
variant="section"
|
||||
icon={SvgGlobe}
|
||||
title="Selected"
|
||||
description="Currently the default provider."
|
||||
/>
|
||||
</div>
|
||||
<div className="flex flex-col items-end justify-between">
|
||||
<Button
|
||||
variant="action"
|
||||
prominence="tertiary"
|
||||
icon={SvgCheckSquare}
|
||||
>
|
||||
Current Default
|
||||
</Button>
|
||||
<div className="flex flex-row px-1 pb-1">
|
||||
<Button
|
||||
icon={SvgUnplug}
|
||||
tooltip="Disconnect"
|
||||
prominence="tertiary"
|
||||
size="sm"
|
||||
/>
|
||||
<Button
|
||||
icon={SvgSettings}
|
||||
tooltip="Edit"
|
||||
prominence="tertiary"
|
||||
size="sm"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</SelectCard>
|
||||
</div>
|
||||
),
|
||||
};
|
||||
|
||||
export const SizeVariants: Story = {
|
||||
render: () => (
|
||||
<div className="flex flex-col gap-4 w-96">
|
||||
{SIZE_VARIANTS.map((size) => (
|
||||
<SelectCard
|
||||
key={size}
|
||||
variant="select-card"
|
||||
state="filled"
|
||||
sizeVariant={size}
|
||||
>
|
||||
<Content
|
||||
sizePreset="main-ui"
|
||||
variant="section"
|
||||
icon={SvgGlobe}
|
||||
title={`sizeVariant: ${size}`}
|
||||
description="Shows padding and rounding differences."
|
||||
/>
|
||||
</SelectCard>
|
||||
))}
|
||||
</div>
|
||||
),
|
||||
};
|
||||
|
||||
export const SelectHeavyVariant: Story = {
|
||||
render: () => (
|
||||
<div className="flex flex-col gap-4 w-96">
|
||||
{STATES.map((state) => (
|
||||
<SelectCard key={state} variant="select-heavy" state={state}>
|
||||
<div className="p-2">
|
||||
<Content
|
||||
sizePreset="main-ui"
|
||||
variant="section"
|
||||
icon={SvgGlobe}
|
||||
title={`select-heavy / ${state}`}
|
||||
description="For comparison with select-card variant."
|
||||
/>
|
||||
</div>
|
||||
</SelectCard>
|
||||
))}
|
||||
</div>
|
||||
),
|
||||
};
|
||||
96
web/lib/opal/src/components/cards/select-card/components.tsx
Normal file
96
web/lib/opal/src/components/cards/select-card/components.tsx
Normal file
@@ -0,0 +1,96 @@
|
||||
import "@opal/components/cards/select-card/styles.css";
|
||||
import type { ContainerSizeVariants } from "@opal/types";
|
||||
import { containerSizeVariants } from "@opal/shared";
|
||||
import { cn } from "@opal/utils";
|
||||
import { Interactive, type InteractiveStatefulProps } from "@opal/core";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Types
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
type SelectCardProps = InteractiveStatefulProps & {
|
||||
/**
|
||||
* Size preset — controls padding and border-radius.
|
||||
*
|
||||
* Padding comes from the shared size scale. Rounding follows the same
|
||||
* mapping as `Card` / `Button` / `Interactive.Container`:
|
||||
*
|
||||
* | Size | Rounding |
|
||||
* |------------|--------------|
|
||||
* | `lg` | `rounded-16` |
|
||||
* | `md`–`sm` | `rounded-12` |
|
||||
* | `xs`–`2xs` | `rounded-08` |
|
||||
* | `fit` | `rounded-16` |
|
||||
*
|
||||
* @default "lg"
|
||||
*/
|
||||
sizeVariant?: ContainerSizeVariants;
|
||||
|
||||
/** Ref forwarded to the root `<div>`. */
|
||||
ref?: React.Ref<HTMLDivElement>;
|
||||
|
||||
children?: React.ReactNode;
|
||||
};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Rounding
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const roundingForSize: Record<ContainerSizeVariants, string> = {
|
||||
lg: "rounded-16",
|
||||
md: "rounded-12",
|
||||
sm: "rounded-12",
|
||||
xs: "rounded-08",
|
||||
"2xs": "rounded-08",
|
||||
fit: "rounded-16",
|
||||
};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// SelectCard
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* A stateful interactive card — the card counterpart to `SelectButton`.
|
||||
*
|
||||
* Built on `Interactive.Stateful` (Slot) → a structural `<div>`. The
|
||||
* Stateful system owns background and foreground colors; the card owns
|
||||
* padding, rounding, border, and overflow.
|
||||
*
|
||||
* Children are fully composable — use `ContentAction`, `Content`, buttons,
|
||||
* `Interactive.Foldable`, etc. inside.
|
||||
*
|
||||
* @example
|
||||
* ```tsx
|
||||
* <SelectCard variant="select-card" state="selected" onClick={handleClick}>
|
||||
* <ContentAction
|
||||
* icon={SvgGlobe}
|
||||
* title="Google"
|
||||
* description="Search engine"
|
||||
* rightChildren={<Button>Set as Default</Button>}
|
||||
* />
|
||||
* </SelectCard>
|
||||
* ```
|
||||
*/
|
||||
function SelectCard({
|
||||
sizeVariant = "lg",
|
||||
ref,
|
||||
children,
|
||||
...statefulProps
|
||||
}: SelectCardProps) {
|
||||
const { padding } = containerSizeVariants[sizeVariant];
|
||||
const rounding = roundingForSize[sizeVariant];
|
||||
|
||||
return (
|
||||
<Interactive.Stateful {...statefulProps}>
|
||||
<div ref={ref} className={cn("opal-select-card", padding, rounding)}>
|
||||
{children}
|
||||
</div>
|
||||
</Interactive.Stateful>
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Exports
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export { SelectCard, type SelectCardProps };
|
||||
9
web/lib/opal/src/components/cards/select-card/styles.css
Normal file
9
web/lib/opal/src/components/cards/select-card/styles.css
Normal file
@@ -0,0 +1,9 @@
|
||||
/* SelectCard — structural styles; colors handled by Interactive.Stateful */
|
||||
|
||||
.opal-select-card {
|
||||
@apply w-full overflow-clip border;
|
||||
}
|
||||
|
||||
.opal-select-card[data-interactive-state="selected"] {
|
||||
@apply border-action-link-05;
|
||||
}
|
||||
@@ -56,6 +56,12 @@ export {
|
||||
type BorderVariant,
|
||||
} from "@opal/components/cards/card/components";
|
||||
|
||||
/* SelectCard */
|
||||
export {
|
||||
SelectCard,
|
||||
type SelectCardProps,
|
||||
} from "@opal/components/cards/select-card/components";
|
||||
|
||||
/* EmptyMessageCard */
|
||||
export {
|
||||
EmptyMessageCard,
|
||||
|
||||
@@ -390,14 +390,12 @@ function PaginationCount({
|
||||
<span className={textClasses(size, "muted")}>of</span>
|
||||
{totalItems}
|
||||
{units && (
|
||||
<span className="ml-1">
|
||||
<Text
|
||||
color="inherit"
|
||||
font={size === "sm" ? "secondary-body" : "main-ui-muted"}
|
||||
>
|
||||
{units}
|
||||
</Text>
|
||||
</span>
|
||||
<Text
|
||||
color="inherit"
|
||||
font={size === "sm" ? "secondary-body" : "main-ui-muted"}
|
||||
>
|
||||
{units}
|
||||
</Text>
|
||||
)}
|
||||
</span>
|
||||
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
import "@opal/components/tag/styles.css";
|
||||
|
||||
import type { IconFunctionComponent, RichStr } from "@opal/types";
|
||||
import { Text } from "@opal/components";
|
||||
import { cn } from "@opal/utils";
|
||||
@@ -46,20 +45,22 @@ function Tag({ icon: Icon, title, color = "gray", size = "sm" }: TagProps) {
|
||||
const config = COLOR_CONFIG[color];
|
||||
|
||||
return (
|
||||
<div className={cn("opal-auxiliary-tag", config.bg)} data-size={size}>
|
||||
<div
|
||||
className={cn("opal-auxiliary-tag", config.bg, config.text)}
|
||||
data-size={size}
|
||||
>
|
||||
{Icon && (
|
||||
<div className="opal-auxiliary-tag-icon-container">
|
||||
<Icon className={cn("opal-auxiliary-tag-icon", config.text)} />
|
||||
</div>
|
||||
)}
|
||||
<span className={cn("opal-auxiliary-tag-title px-[2px]", config.text)}>
|
||||
<Text
|
||||
font={size === "md" ? "secondary-body" : "figure-small-value"}
|
||||
color="inherit"
|
||||
>
|
||||
{title}
|
||||
</Text>
|
||||
</span>
|
||||
<Text
|
||||
font={size === "md" ? "secondary-body" : "figure-small-value"}
|
||||
color="inherit"
|
||||
nowrap
|
||||
>
|
||||
{title}
|
||||
</Text>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -13,7 +13,9 @@ const SAFE_PROTOCOL = /^https?:|^mailto:|^tel:/i;
|
||||
const ALLOWED_ELEMENTS = ["p", "br", "a", "strong", "em", "code", "del"];
|
||||
|
||||
const INLINE_COMPONENTS = {
|
||||
p: ({ children }: { children?: ReactNode }) => <>{children}</>,
|
||||
p: ({ children }: { children?: ReactNode }) => (
|
||||
<span className="block">{children}</span>
|
||||
),
|
||||
a: ({ children, href }: { children?: ReactNode; href?: string }) => {
|
||||
if (!href || !SAFE_PROTOCOL.test(href)) {
|
||||
return <>{children}</>;
|
||||
|
||||
@@ -125,6 +125,7 @@ function Text({
|
||||
...rest
|
||||
}: TextProps) {
|
||||
const resolvedClassName = cn(
|
||||
"px-[2px]",
|
||||
FONT_CONFIG[font],
|
||||
COLOR_CONFIG[color],
|
||||
nowrap && "whitespace-nowrap",
|
||||
|
||||
@@ -18,9 +18,11 @@ export {
|
||||
import { InteractiveStateless } from "@opal/core/interactive/stateless/components";
|
||||
import { InteractiveStateful } from "@opal/core/interactive/stateful/components";
|
||||
import { InteractiveContainer } from "@opal/core/interactive/container/components";
|
||||
import { InteractiveSimple } from "@opal/core/interactive/simple/components";
|
||||
import { Foldable } from "@opal/core/interactive/foldable/components";
|
||||
|
||||
const Interactive = {
|
||||
Simple: InteractiveSimple,
|
||||
Stateless: InteractiveStateless,
|
||||
Stateful: InteractiveStateful,
|
||||
Container: InteractiveContainer,
|
||||
@@ -50,3 +52,5 @@ export type {
|
||||
} from "@opal/core/interactive/container/components";
|
||||
|
||||
export type { FoldableProps } from "@opal/core/interactive/foldable/components";
|
||||
|
||||
export type { InteractiveSimpleProps } from "@opal/core/interactive/simple/components";
|
||||
|
||||
@@ -9,7 +9,6 @@ const VARIANT_PROMINENCE_MAP: Record<string, string[]> = {
|
||||
default: ["primary", "secondary", "tertiary", "internal"],
|
||||
action: ["primary", "secondary", "tertiary", "internal"],
|
||||
danger: ["primary", "secondary", "tertiary", "internal"],
|
||||
none: [],
|
||||
};
|
||||
|
||||
const SIZE_VARIANTS = ["lg", "md", "sm", "xs", "2xs", "fit"] as const;
|
||||
@@ -43,7 +42,7 @@ export const Default: StoryObj = {
|
||||
onClick={() => {}}
|
||||
>
|
||||
<Interactive.Container border>
|
||||
<span className="interactive-foreground">Secondary</span>
|
||||
<span>Secondary</span>
|
||||
</Interactive.Container>
|
||||
</Interactive.Stateless>
|
||||
|
||||
@@ -53,7 +52,7 @@ export const Default: StoryObj = {
|
||||
onClick={() => {}}
|
||||
>
|
||||
<Interactive.Container border>
|
||||
<span className="interactive-foreground">Primary</span>
|
||||
<span>Primary</span>
|
||||
</Interactive.Container>
|
||||
</Interactive.Stateless>
|
||||
|
||||
@@ -63,7 +62,7 @@ export const Default: StoryObj = {
|
||||
onClick={() => {}}
|
||||
>
|
||||
<Interactive.Container border>
|
||||
<span className="interactive-foreground">Tertiary</span>
|
||||
<span>Tertiary</span>
|
||||
</Interactive.Container>
|
||||
</Interactive.Stateless>
|
||||
</div>
|
||||
@@ -115,9 +114,7 @@ export const VariantMatrix: StoryObj = {
|
||||
onClick={() => {}}
|
||||
>
|
||||
<Interactive.Container border>
|
||||
<span className="interactive-foreground">
|
||||
{prominence}
|
||||
</span>
|
||||
<span>{prominence}</span>
|
||||
</Interactive.Container>
|
||||
</Interactive.Stateless>
|
||||
<span
|
||||
@@ -150,7 +147,7 @@ export const Sizes: StoryObj = {
|
||||
onClick={() => {}}
|
||||
>
|
||||
<Interactive.Container border heightVariant={size}>
|
||||
<span className="interactive-foreground">{size}</span>
|
||||
<span>{size}</span>
|
||||
</Interactive.Container>
|
||||
</Interactive.Stateless>
|
||||
))}
|
||||
@@ -168,7 +165,7 @@ export const WidthFull: StoryObj = {
|
||||
onClick={() => {}}
|
||||
>
|
||||
<Interactive.Container border widthVariant="full">
|
||||
<span className="interactive-foreground">Full width container</span>
|
||||
<span>Full width container</span>
|
||||
</Interactive.Container>
|
||||
</Interactive.Stateless>
|
||||
</div>
|
||||
@@ -187,7 +184,7 @@ export const Rounding: StoryObj = {
|
||||
onClick={() => {}}
|
||||
>
|
||||
<Interactive.Container border roundingVariant={rounding}>
|
||||
<span className="interactive-foreground">{rounding}</span>
|
||||
<span>{rounding}</span>
|
||||
</Interactive.Container>
|
||||
</Interactive.Stateless>
|
||||
))}
|
||||
@@ -207,7 +204,7 @@ export const DisabledStory: StoryObj = {
|
||||
onClick={() => {}}
|
||||
>
|
||||
<Interactive.Container border>
|
||||
<span className="interactive-foreground">Disabled</span>
|
||||
<span>Disabled</span>
|
||||
</Interactive.Container>
|
||||
</Interactive.Stateless>
|
||||
</Disabled>
|
||||
@@ -218,7 +215,7 @@ export const DisabledStory: StoryObj = {
|
||||
onClick={() => {}}
|
||||
>
|
||||
<Interactive.Container border>
|
||||
<span className="interactive-foreground">Enabled</span>
|
||||
<span>Enabled</span>
|
||||
</Interactive.Container>
|
||||
</Interactive.Stateless>
|
||||
</div>
|
||||
@@ -236,7 +233,7 @@ export const Interaction: StoryObj = {
|
||||
onClick={() => {}}
|
||||
>
|
||||
<Interactive.Container border>
|
||||
<span className="interactive-foreground">Forced hover</span>
|
||||
<span>Forced hover</span>
|
||||
</Interactive.Container>
|
||||
</Interactive.Stateless>
|
||||
|
||||
@@ -247,7 +244,7 @@ export const Interaction: StoryObj = {
|
||||
onClick={() => {}}
|
||||
>
|
||||
<Interactive.Container border>
|
||||
<span className="interactive-foreground">Forced active</span>
|
||||
<span>Forced active</span>
|
||||
</Interactive.Container>
|
||||
</Interactive.Stateless>
|
||||
|
||||
@@ -257,7 +254,7 @@ export const Interaction: StoryObj = {
|
||||
onClick={() => {}}
|
||||
>
|
||||
<Interactive.Container border>
|
||||
<span className="interactive-foreground">Normal (rest)</span>
|
||||
<span>Normal (rest)</span>
|
||||
</Interactive.Container>
|
||||
</Interactive.Stateless>
|
||||
</div>
|
||||
@@ -274,7 +271,7 @@ export const WithBorder: StoryObj = {
|
||||
onClick={() => {}}
|
||||
>
|
||||
<Interactive.Container border>
|
||||
<span className="interactive-foreground">With border</span>
|
||||
<span>With border</span>
|
||||
</Interactive.Container>
|
||||
</Interactive.Stateless>
|
||||
|
||||
@@ -284,7 +281,7 @@ export const WithBorder: StoryObj = {
|
||||
onClick={() => {}}
|
||||
>
|
||||
<Interactive.Container>
|
||||
<span className="interactive-foreground">Without border</span>
|
||||
<span>Without border</span>
|
||||
</Interactive.Container>
|
||||
</Interactive.Stateless>
|
||||
</div>
|
||||
@@ -296,7 +293,7 @@ export const AsLink: StoryObj = {
|
||||
render: () => (
|
||||
<Interactive.Stateless variant="action" href="/settings">
|
||||
<Interactive.Container border>
|
||||
<span className="interactive-foreground">Go to Settings</span>
|
||||
<span>Go to Settings</span>
|
||||
</Interactive.Container>
|
||||
</Interactive.Stateless>
|
||||
),
|
||||
@@ -312,7 +309,7 @@ export const SelectVariant: StoryObj = {
|
||||
onClick={() => {}}
|
||||
>
|
||||
<Interactive.Container border>
|
||||
<span className="interactive-foreground">Selected (light)</span>
|
||||
<span>Selected (light)</span>
|
||||
</Interactive.Container>
|
||||
</Interactive.Stateful>
|
||||
|
||||
@@ -322,7 +319,7 @@ export const SelectVariant: StoryObj = {
|
||||
onClick={() => {}}
|
||||
>
|
||||
<Interactive.Container border>
|
||||
<span className="interactive-foreground">Unselected (light)</span>
|
||||
<span>Unselected (light)</span>
|
||||
</Interactive.Container>
|
||||
</Interactive.Stateful>
|
||||
|
||||
@@ -332,7 +329,7 @@ export const SelectVariant: StoryObj = {
|
||||
onClick={() => {}}
|
||||
>
|
||||
<Interactive.Container border>
|
||||
<span className="interactive-foreground">Selected (heavy)</span>
|
||||
<span>Selected (heavy)</span>
|
||||
</Interactive.Container>
|
||||
</Interactive.Stateful>
|
||||
|
||||
@@ -342,7 +339,7 @@ export const SelectVariant: StoryObj = {
|
||||
onClick={() => {}}
|
||||
>
|
||||
<Interactive.Container border>
|
||||
<span className="interactive-foreground">Unselected (heavy)</span>
|
||||
<span>Unselected (heavy)</span>
|
||||
</Interactive.Container>
|
||||
</Interactive.Stateful>
|
||||
</div>
|
||||
|
||||
@@ -21,9 +21,10 @@
|
||||
--interactive-easing: ease-in-out;
|
||||
}
|
||||
|
||||
/* Base interactive surface */
|
||||
/* Base interactive surface — sets color directly so all descendants inherit. */
|
||||
.interactive {
|
||||
@apply cursor-pointer select-none;
|
||||
color: var(--interactive-foreground);
|
||||
transition:
|
||||
background-color var(--interactive-duration) var(--interactive-easing),
|
||||
--interactive-foreground var(--interactive-duration)
|
||||
@@ -43,14 +44,8 @@
|
||||
@apply border;
|
||||
}
|
||||
|
||||
/* Utility classes — descendants opt in to parent-controlled foreground color.
|
||||
No transition here — the parent interpolates the variables directly. */
|
||||
.interactive-foreground {
|
||||
color: var(--interactive-foreground);
|
||||
}
|
||||
|
||||
/* Icon foreground — reads from --interactive-foreground-icon, which defaults
|
||||
to --interactive-foreground via the variant CSS rules. */
|
||||
/* Icon foreground — reads from --interactive-foreground-icon, which may differ
|
||||
from --interactive-foreground (e.g. muted icons beside normal text). */
|
||||
.interactive-foreground-icon {
|
||||
color: var(--interactive-foreground-icon);
|
||||
}
|
||||
|
||||
100
web/lib/opal/src/core/interactive/simple/components.tsx
Normal file
100
web/lib/opal/src/core/interactive/simple/components.tsx
Normal file
@@ -0,0 +1,100 @@
|
||||
import React from "react";
|
||||
import { Slot } from "@radix-ui/react-slot";
|
||||
import { cn } from "@opal/utils";
|
||||
import { useDisabled } from "@opal/core/disabled/components";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Types
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
interface InteractiveSimpleProps
|
||||
extends Omit<
|
||||
React.HTMLAttributes<HTMLElement>,
|
||||
"className" | "style" | "color"
|
||||
> {
|
||||
ref?: React.Ref<HTMLElement>;
|
||||
|
||||
/**
|
||||
* Tailwind group class (e.g. `"group/Card"`) for `group-hover:*` utilities.
|
||||
*/
|
||||
group?: string;
|
||||
|
||||
/**
|
||||
* URL to navigate to when clicked. Passed through Slot to the child.
|
||||
*/
|
||||
href?: string;
|
||||
|
||||
/**
|
||||
* Link target (e.g. `"_blank"`). Only used when `href` is provided.
|
||||
*/
|
||||
target?: string;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// InteractiveSimple
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Minimal interactive surface primitive.
|
||||
*
|
||||
* Provides cursor styling, click handling, disabled integration, and
|
||||
* optional link/group support — but **no color or background styling**.
|
||||
*
|
||||
* Use this for elements that need interactivity (click, cursor, disabled)
|
||||
* without participating in the Interactive color system.
|
||||
*
|
||||
* Uses Radix `Slot` — merges props onto a single child element without
|
||||
* adding any DOM node.
|
||||
*
|
||||
* @example
|
||||
* ```tsx
|
||||
* <Interactive.Simple onClick={handleClick} group="group/Card">
|
||||
* <Card>...</Card>
|
||||
* </Interactive.Simple>
|
||||
* ```
|
||||
*/
|
||||
function InteractiveSimple({
|
||||
ref,
|
||||
group,
|
||||
href,
|
||||
target,
|
||||
...props
|
||||
}: InteractiveSimpleProps) {
|
||||
const { isDisabled, allowClick } = useDisabled();
|
||||
|
||||
const classes = cn(
|
||||
"cursor-pointer select-none",
|
||||
isDisabled && "cursor-not-allowed",
|
||||
!props.onClick && !href && "!cursor-default !select-auto",
|
||||
group
|
||||
);
|
||||
|
||||
const { onClick, ...slotProps } = props;
|
||||
|
||||
const linkAttrs = href
|
||||
? {
|
||||
href: isDisabled ? undefined : href,
|
||||
target,
|
||||
rel: target === "_blank" ? "noopener noreferrer" : undefined,
|
||||
}
|
||||
: {};
|
||||
|
||||
return (
|
||||
<Slot
|
||||
ref={ref}
|
||||
className={classes}
|
||||
aria-disabled={isDisabled || undefined}
|
||||
{...linkAttrs}
|
||||
{...slotProps}
|
||||
onClick={
|
||||
isDisabled && !allowClick
|
||||
? href
|
||||
? (e: React.MouseEvent) => e.preventDefault()
|
||||
: undefined
|
||||
: onClick
|
||||
}
|
||||
/>
|
||||
);
|
||||
}
|
||||
|
||||
export { InteractiveSimple, type InteractiveSimpleProps };
|
||||
@@ -8,7 +8,7 @@ Stateful interactive surface primitive for elements that maintain a value state
|
||||
|
||||
| Prop | Type | Default | Description |
|
||||
|------|------|---------|-------------|
|
||||
| `variant` | `"select-light" \| "select-heavy" \| "select-tinted" \| "select-filter" \| "sidebar"` | `"select-heavy"` | Color variant |
|
||||
| `variant` | `"select-light" \| "select-heavy" \| "select-card" \| "select-tinted" \| "select-filter" \| "sidebar-heavy" \| "sidebar-light"` | `"select-heavy"` | Color variant |
|
||||
| `state` | `"empty" \| "filled" \| "selected"` | `"empty"` | Current value state |
|
||||
| `interaction` | `"rest" \| "hover" \| "active"` | `"rest"` | JS-controlled interaction override |
|
||||
| `group` | `string` | — | Tailwind group class for `group-hover:*` |
|
||||
@@ -16,6 +16,16 @@ Stateful interactive surface primitive for elements that maintain a value state
|
||||
| `href` | `string` | — | URL for link behavior |
|
||||
| `target` | `string` | — | Link target (e.g. `"_blank"`) |
|
||||
|
||||
## Variants
|
||||
|
||||
- **`select-light`** — Transparent selected background. For inline toggles.
|
||||
- **`select-heavy`** — Tinted selected background (`action-link-01`). For list rows, model pickers, buttons.
|
||||
- **`select-card`** — Like `select-heavy`, but the filled state gets a visible background (`background-tint-00`) with neutral foreground. Designed for larger surfaces (cards) where background carries more of the visual distinction than foreground color alone.
|
||||
- **`select-tinted`** — Like `select-heavy` but with a tinted rest background (`background-tint-01`).
|
||||
- **`select-filter`** — Like `select-tinted` for empty/filled; selected state uses inverted backgrounds and inverted text.
|
||||
- **`sidebar-heavy`** — Sidebar navigation: muted when unselected, bold when selected.
|
||||
- **`sidebar-light`** — Sidebar navigation: uniformly muted across all states.
|
||||
|
||||
## State attribute
|
||||
|
||||
Uses `data-interactive-state` (not `data-state`) to avoid conflicts with Radix UI, which injects its own `data-state` on trigger elements.
|
||||
|
||||
@@ -13,9 +13,11 @@ import type { ButtonType, WithoutStyles } from "@opal/types";
|
||||
type InteractiveStatefulVariant =
|
||||
| "select-light"
|
||||
| "select-heavy"
|
||||
| "select-card"
|
||||
| "select-tinted"
|
||||
| "select-filter"
|
||||
| "sidebar";
|
||||
| "sidebar-heavy"
|
||||
| "sidebar-light";
|
||||
type InteractiveStatefulState = "empty" | "filled" | "selected";
|
||||
type InteractiveStatefulInteraction = "rest" | "hover" | "active";
|
||||
|
||||
@@ -31,9 +33,11 @@ interface InteractiveStatefulProps
|
||||
*
|
||||
* - `"select-light"` — transparent selected background (for inline toggles)
|
||||
* - `"select-heavy"` — tinted selected background (for list rows, model pickers)
|
||||
* - `"select-card"` — like select-heavy but filled state has a visible background (for cards/larger surfaces)
|
||||
* - `"select-tinted"` — like select-heavy but with a tinted rest background
|
||||
* - `"select-filter"` — like select-tinted for empty/filled; selected state uses inverted tint backgrounds and inverted text (for filter buttons)
|
||||
* - `"sidebar"` — for sidebar navigation items
|
||||
* - `"sidebar-heavy"` — sidebar navigation items: muted when unselected (text-03/text-02), bold when selected (text-04/text-03)
|
||||
* - `"sidebar-light"` — sidebar navigation items: uniformly muted across all states (text-02/text-02)
|
||||
*
|
||||
* @default "select-heavy"
|
||||
*/
|
||||
|
||||
@@ -11,7 +11,7 @@
|
||||
Children read the variables with no independent transitions.
|
||||
|
||||
State dimension: `data-interactive-state` = "empty" | "filled" | "selected"
|
||||
Variant dimension: `data-interactive-variant` = "select-light" | "select-heavy" | "select-tinted" | "sidebar"
|
||||
Variant dimension: `data-interactive-variant` = "select-light" | "select-heavy" | "select-card" | "select-tinted" | "select-filter" | "sidebar-heavy" | "sidebar-light"
|
||||
|
||||
Interaction override: `data-interaction="hover"` and `data-interaction="active"`
|
||||
allow JS-controlled visual state overrides.
|
||||
@@ -114,6 +114,108 @@
|
||||
--interactive-foreground-icon: var(--action-link-03);
|
||||
}
|
||||
|
||||
/* ===========================================================================
|
||||
Select-Card — like Select-Heavy but filled has a visible background.
|
||||
Designed for larger surfaces (cards) where background carries more of
|
||||
the visual distinction than foreground color alone.
|
||||
=========================================================================== */
|
||||
|
||||
/* ---------------------------------------------------------------------------
|
||||
Select-Card — Empty
|
||||
--------------------------------------------------------------------------- */
|
||||
.interactive[data-interactive-variant="select-card"][data-interactive-state="empty"] {
|
||||
@apply bg-transparent;
|
||||
--interactive-foreground: var(--text-04);
|
||||
--interactive-foreground-icon: var(--text-03);
|
||||
}
|
||||
.interactive[data-interactive-variant="select-card"][data-interactive-state="empty"]:hover:not(
|
||||
[data-disabled]
|
||||
),
|
||||
.interactive[data-interactive-variant="select-card"][data-interactive-state="empty"][data-interaction="hover"]:not(
|
||||
[data-disabled]
|
||||
) {
|
||||
@apply bg-background-tint-02;
|
||||
--interactive-foreground-icon: var(--text-04);
|
||||
}
|
||||
.interactive[data-interactive-variant="select-card"][data-interactive-state="empty"]:active:not(
|
||||
[data-disabled]
|
||||
),
|
||||
.interactive[data-interactive-variant="select-card"][data-interactive-state="empty"][data-interaction="active"]:not(
|
||||
[data-disabled]
|
||||
) {
|
||||
@apply bg-background-neutral-00;
|
||||
--interactive-foreground: var(--text-05);
|
||||
--interactive-foreground-icon: var(--text-05);
|
||||
}
|
||||
.interactive[data-interactive-variant="select-card"][data-interactive-state="empty"][data-disabled] {
|
||||
@apply bg-transparent;
|
||||
--interactive-foreground: var(--text-01);
|
||||
--interactive-foreground-icon: var(--text-01);
|
||||
}
|
||||
|
||||
/* ---------------------------------------------------------------------------
|
||||
Select-Card — Filled (visible background, neutral foreground)
|
||||
--------------------------------------------------------------------------- */
|
||||
.interactive[data-interactive-variant="select-card"][data-interactive-state="filled"] {
|
||||
@apply bg-background-tint-00;
|
||||
--interactive-foreground: var(--text-04);
|
||||
--interactive-foreground-icon: var(--text-03);
|
||||
}
|
||||
.interactive[data-interactive-variant="select-card"][data-interactive-state="filled"]:hover:not(
|
||||
[data-disabled]
|
||||
),
|
||||
.interactive[data-interactive-variant="select-card"][data-interactive-state="filled"][data-interaction="hover"]:not(
|
||||
[data-disabled]
|
||||
) {
|
||||
@apply bg-background-tint-02;
|
||||
--interactive-foreground-icon: var(--text-04);
|
||||
}
|
||||
.interactive[data-interactive-variant="select-card"][data-interactive-state="filled"]:active:not(
|
||||
[data-disabled]
|
||||
),
|
||||
.interactive[data-interactive-variant="select-card"][data-interactive-state="filled"][data-interaction="active"]:not(
|
||||
[data-disabled]
|
||||
) {
|
||||
@apply bg-background-tint-00;
|
||||
--interactive-foreground: var(--text-05);
|
||||
--interactive-foreground-icon: var(--text-05);
|
||||
}
|
||||
.interactive[data-interactive-variant="select-card"][data-interactive-state="filled"][data-disabled] {
|
||||
@apply bg-transparent;
|
||||
--interactive-foreground: var(--text-01);
|
||||
--interactive-foreground-icon: var(--text-01);
|
||||
}
|
||||
|
||||
/* ---------------------------------------------------------------------------
|
||||
Select-Card — Selected
|
||||
--------------------------------------------------------------------------- */
|
||||
.interactive[data-interactive-variant="select-card"][data-interactive-state="selected"] {
|
||||
@apply bg-[var(--action-link-01)];
|
||||
--interactive-foreground: var(--action-link-05);
|
||||
--interactive-foreground-icon: var(--action-link-05);
|
||||
}
|
||||
.interactive[data-interactive-variant="select-card"][data-interactive-state="selected"]:hover:not(
|
||||
[data-disabled]
|
||||
),
|
||||
.interactive[data-interactive-variant="select-card"][data-interactive-state="selected"][data-interaction="hover"]:not(
|
||||
[data-disabled]
|
||||
) {
|
||||
@apply bg-background-tint-02;
|
||||
}
|
||||
.interactive[data-interactive-variant="select-card"][data-interactive-state="selected"]:active:not(
|
||||
[data-disabled]
|
||||
),
|
||||
.interactive[data-interactive-variant="select-card"][data-interactive-state="selected"][data-interaction="active"]:not(
|
||||
[data-disabled]
|
||||
) {
|
||||
@apply bg-background-tint-00;
|
||||
}
|
||||
.interactive[data-interactive-variant="select-card"][data-interactive-state="selected"][data-disabled] {
|
||||
@apply bg-transparent;
|
||||
--interactive-foreground: var(--action-link-03);
|
||||
--interactive-foreground-icon: var(--action-link-03);
|
||||
}
|
||||
|
||||
/* ===========================================================================
|
||||
Select-Light — identical to Select-Heavy except selected bg is transparent
|
||||
=========================================================================== */
|
||||
@@ -392,49 +494,115 @@
|
||||
}
|
||||
|
||||
/* ===========================================================================
|
||||
Sidebar
|
||||
Sidebar-Heavy
|
||||
|
||||
Not selected: muted (text-03 / icon text-02)
|
||||
Selected: default (text-04 / icon text-03)
|
||||
=========================================================================== */
|
||||
|
||||
/* ---------------------------------------------------------------------------
|
||||
Sidebar — Empty
|
||||
Sidebar-Heavy — Empty
|
||||
--------------------------------------------------------------------------- */
|
||||
.interactive[data-interactive-variant="sidebar"][data-interactive-state="empty"] {
|
||||
.interactive[data-interactive-variant="sidebar-heavy"][data-interactive-state="empty"] {
|
||||
@apply bg-transparent;
|
||||
--interactive-foreground: var(--text-03);
|
||||
--interactive-foreground-icon: var(--text-02);
|
||||
}
|
||||
.interactive[data-interactive-variant="sidebar"][data-interactive-state="empty"]:hover:not(
|
||||
.interactive[data-interactive-variant="sidebar-heavy"][data-interactive-state="empty"]:hover:not(
|
||||
[data-disabled]
|
||||
),
|
||||
.interactive[data-interactive-variant="sidebar"][data-interactive-state="empty"][data-interaction="hover"]:not(
|
||||
.interactive[data-interactive-variant="sidebar-heavy"][data-interactive-state="empty"][data-interaction="hover"]:not(
|
||||
[data-disabled]
|
||||
) {
|
||||
@apply bg-background-tint-03;
|
||||
}
|
||||
|
||||
/* ---------------------------------------------------------------------------
|
||||
Sidebar — Filled
|
||||
Sidebar-Heavy — Filled
|
||||
--------------------------------------------------------------------------- */
|
||||
.interactive[data-interactive-variant="sidebar"][data-interactive-state="filled"] {
|
||||
.interactive[data-interactive-variant="sidebar-heavy"][data-interactive-state="filled"] {
|
||||
@apply bg-transparent;
|
||||
--interactive-foreground: var(--text-03);
|
||||
--interactive-foreground-icon: var(--text-02);
|
||||
}
|
||||
.interactive[data-interactive-variant="sidebar"][data-interactive-state="filled"]:hover:not(
|
||||
.interactive[data-interactive-variant="sidebar-heavy"][data-interactive-state="filled"]:hover:not(
|
||||
[data-disabled]
|
||||
),
|
||||
.interactive[data-interactive-variant="sidebar"][data-interactive-state="filled"][data-interaction="hover"]:not(
|
||||
.interactive[data-interactive-variant="sidebar-heavy"][data-interactive-state="filled"][data-interaction="hover"]:not(
|
||||
[data-disabled]
|
||||
) {
|
||||
@apply bg-background-tint-03;
|
||||
}
|
||||
|
||||
/* ---------------------------------------------------------------------------
|
||||
Sidebar — Selected
|
||||
Sidebar-Heavy — Selected
|
||||
--------------------------------------------------------------------------- */
|
||||
.interactive[data-interactive-variant="sidebar"][data-interactive-state="selected"] {
|
||||
.interactive[data-interactive-variant="sidebar-heavy"][data-interactive-state="selected"] {
|
||||
@apply bg-background-tint-00;
|
||||
--interactive-foreground: var(--text-04);
|
||||
--interactive-foreground-icon: var(--text-03);
|
||||
}
|
||||
.interactive[data-interactive-variant="sidebar"][data-interactive-state="selected"]:hover:not(
|
||||
.interactive[data-interactive-variant="sidebar-heavy"][data-interactive-state="selected"]:hover:not(
|
||||
[data-disabled]
|
||||
),
|
||||
.interactive[data-interactive-variant="sidebar"][data-interactive-state="selected"][data-interaction="hover"]:not(
|
||||
.interactive[data-interactive-variant="sidebar-heavy"][data-interactive-state="selected"][data-interaction="hover"]:not(
|
||||
[data-disabled]
|
||||
) {
|
||||
@apply bg-background-tint-03;
|
||||
}
|
||||
|
||||
/* ===========================================================================
|
||||
Sidebar-Light
|
||||
|
||||
All states: prominence="muted-2x" colors (text-02 / icon text-02)
|
||||
=========================================================================== */
|
||||
|
||||
/* ---------------------------------------------------------------------------
|
||||
Sidebar-Light — Empty
|
||||
--------------------------------------------------------------------------- */
|
||||
.interactive[data-interactive-variant="sidebar-light"][data-interactive-state="empty"] {
|
||||
@apply bg-transparent;
|
||||
--interactive-foreground: var(--text-02);
|
||||
--interactive-foreground-icon: var(--text-02);
|
||||
}
|
||||
.interactive[data-interactive-variant="sidebar-light"][data-interactive-state="empty"]:hover:not(
|
||||
[data-disabled]
|
||||
),
|
||||
.interactive[data-interactive-variant="sidebar-light"][data-interactive-state="empty"][data-interaction="hover"]:not(
|
||||
[data-disabled]
|
||||
) {
|
||||
@apply bg-background-tint-03;
|
||||
}
|
||||
|
||||
/* ---------------------------------------------------------------------------
|
||||
Sidebar-Light — Filled
|
||||
--------------------------------------------------------------------------- */
|
||||
.interactive[data-interactive-variant="sidebar-light"][data-interactive-state="filled"] {
|
||||
@apply bg-transparent;
|
||||
--interactive-foreground: var(--text-02);
|
||||
--interactive-foreground-icon: var(--text-02);
|
||||
}
|
||||
.interactive[data-interactive-variant="sidebar-light"][data-interactive-state="filled"]:hover:not(
|
||||
[data-disabled]
|
||||
),
|
||||
.interactive[data-interactive-variant="sidebar-light"][data-interactive-state="filled"][data-interaction="hover"]:not(
|
||||
[data-disabled]
|
||||
) {
|
||||
@apply bg-background-tint-03;
|
||||
}
|
||||
|
||||
/* ---------------------------------------------------------------------------
|
||||
Sidebar-Light — Selected
|
||||
--------------------------------------------------------------------------- */
|
||||
.interactive[data-interactive-variant="sidebar-light"][data-interactive-state="selected"] {
|
||||
@apply bg-background-tint-00;
|
||||
--interactive-foreground: var(--text-02);
|
||||
--interactive-foreground-icon: var(--text-02);
|
||||
}
|
||||
.interactive[data-interactive-variant="sidebar-light"][data-interactive-state="selected"]:hover:not(
|
||||
[data-disabled]
|
||||
),
|
||||
.interactive[data-interactive-variant="sidebar-light"][data-interactive-state="selected"][data-interaction="hover"]:not(
|
||||
[data-disabled]
|
||||
) {
|
||||
@apply bg-background-tint-03;
|
||||
|
||||
@@ -10,7 +10,7 @@ import type { ButtonType, WithoutStyles } from "@opal/types";
|
||||
// Types
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
type InteractiveStatelessVariant = "none" | "default" | "action" | "danger";
|
||||
type InteractiveStatelessVariant = "default" | "action" | "danger";
|
||||
type InteractiveStatelessProminence =
|
||||
| "primary"
|
||||
| "secondary"
|
||||
@@ -108,8 +108,8 @@ function InteractiveStateless({
|
||||
);
|
||||
|
||||
const dataAttrs = {
|
||||
"data-interactive-variant": variant !== "none" ? variant : undefined,
|
||||
"data-interactive-prominence": variant !== "none" ? prominence : undefined,
|
||||
"data-interactive-variant": variant,
|
||||
"data-interactive-prominence": prominence,
|
||||
"data-interaction": interaction !== "rest" ? interaction : undefined,
|
||||
"data-disabled": isDisabled ? "true" : undefined,
|
||||
"aria-disabled": isDisabled || undefined,
|
||||
|
||||
@@ -0,0 +1,158 @@
|
||||
import type { Meta, StoryObj } from "@storybook/react";
|
||||
import { CardHeaderLayout } from "@opal/layouts";
|
||||
import { Button } from "@opal/components";
|
||||
import {
|
||||
SvgArrowExchange,
|
||||
SvgCheckSquare,
|
||||
SvgGlobe,
|
||||
SvgSettings,
|
||||
SvgUnplug,
|
||||
} from "@opal/icons";
|
||||
import * as TooltipPrimitive from "@radix-ui/react-tooltip";
|
||||
import type { Decorator } from "@storybook/react";
|
||||
|
||||
const withTooltipProvider: Decorator = (Story) => (
|
||||
<TooltipPrimitive.Provider>
|
||||
<Story />
|
||||
</TooltipPrimitive.Provider>
|
||||
);
|
||||
|
||||
const meta = {
|
||||
title: "Layouts/CardHeaderLayout",
|
||||
component: CardHeaderLayout,
|
||||
tags: ["autodocs"],
|
||||
decorators: [withTooltipProvider],
|
||||
parameters: {
|
||||
layout: "centered",
|
||||
},
|
||||
} satisfies Meta<typeof CardHeaderLayout>;
|
||||
|
||||
export default meta;
|
||||
|
||||
type Story = StoryObj<typeof meta>;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Stories
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export const Default: Story = {
|
||||
render: () => (
|
||||
<div className="w-[28rem] border rounded-16">
|
||||
<CardHeaderLayout
|
||||
sizePreset="main-ui"
|
||||
variant="section"
|
||||
icon={SvgGlobe}
|
||||
title="Google Search"
|
||||
description="Web search provider"
|
||||
rightChildren={
|
||||
<Button prominence="tertiary" rightIcon={SvgArrowExchange}>
|
||||
Connect
|
||||
</Button>
|
||||
}
|
||||
/>
|
||||
</div>
|
||||
),
|
||||
};
|
||||
|
||||
export const WithBothSlots: Story = {
|
||||
render: () => (
|
||||
<div className="w-[28rem] border rounded-16">
|
||||
<CardHeaderLayout
|
||||
sizePreset="main-ui"
|
||||
variant="section"
|
||||
icon={SvgGlobe}
|
||||
title="Google Search"
|
||||
description="Currently the default provider."
|
||||
rightChildren={
|
||||
<Button variant="action" prominence="tertiary" icon={SvgCheckSquare}>
|
||||
Current Default
|
||||
</Button>
|
||||
}
|
||||
bottomRightChildren={
|
||||
<>
|
||||
<Button
|
||||
icon={SvgUnplug}
|
||||
tooltip="Disconnect"
|
||||
prominence="tertiary"
|
||||
size="sm"
|
||||
/>
|
||||
<Button
|
||||
icon={SvgSettings}
|
||||
tooltip="Edit"
|
||||
prominence="tertiary"
|
||||
size="sm"
|
||||
/>
|
||||
</>
|
||||
}
|
||||
/>
|
||||
</div>
|
||||
),
|
||||
};
|
||||
|
||||
export const RightChildrenOnly: Story = {
|
||||
render: () => (
|
||||
<div className="w-[28rem] border rounded-16">
|
||||
<CardHeaderLayout
|
||||
sizePreset="main-ui"
|
||||
variant="section"
|
||||
icon={SvgGlobe}
|
||||
title="OpenAI"
|
||||
description="Not configured"
|
||||
rightChildren={
|
||||
<Button prominence="tertiary" rightIcon={SvgArrowExchange}>
|
||||
Connect
|
||||
</Button>
|
||||
}
|
||||
/>
|
||||
</div>
|
||||
),
|
||||
};
|
||||
|
||||
export const NoRightChildren: Story = {
|
||||
render: () => (
|
||||
<div className="w-[28rem] border rounded-16">
|
||||
<CardHeaderLayout
|
||||
sizePreset="main-ui"
|
||||
variant="section"
|
||||
icon={SvgGlobe}
|
||||
title="Section Header"
|
||||
description="No actions on the right."
|
||||
/>
|
||||
</div>
|
||||
),
|
||||
};
|
||||
|
||||
export const LongContent: Story = {
|
||||
render: () => (
|
||||
<div className="w-[28rem] border rounded-16">
|
||||
<CardHeaderLayout
|
||||
sizePreset="main-ui"
|
||||
variant="section"
|
||||
icon={SvgGlobe}
|
||||
title="Very Long Provider Name That Should Truncate"
|
||||
description="This is a much longer description that tests how the layout handles overflow when the content area needs to shrink."
|
||||
rightChildren={
|
||||
<Button variant="action" prominence="tertiary" icon={SvgCheckSquare}>
|
||||
Current Default
|
||||
</Button>
|
||||
}
|
||||
bottomRightChildren={
|
||||
<>
|
||||
<Button
|
||||
icon={SvgUnplug}
|
||||
prominence="tertiary"
|
||||
size="sm"
|
||||
tooltip="Disconnect"
|
||||
/>
|
||||
<Button
|
||||
icon={SvgSettings}
|
||||
prominence="tertiary"
|
||||
size="sm"
|
||||
tooltip="Edit"
|
||||
/>
|
||||
</>
|
||||
}
|
||||
/>
|
||||
</div>
|
||||
),
|
||||
};
|
||||
94
web/lib/opal/src/layouts/cards/header-layout/README.md
Normal file
94
web/lib/opal/src/layouts/cards/header-layout/README.md
Normal file
@@ -0,0 +1,94 @@
|
||||
# CardHeaderLayout
|
||||
|
||||
**Import:** `import { CardHeaderLayout, type CardHeaderLayoutProps } from "@opal/layouts";`
|
||||
|
||||
A card header layout that pairs a [`Content`](../../content/README.md) block with a right-side column of vertically stacked children.
|
||||
|
||||
## Why CardHeaderLayout?
|
||||
|
||||
[`ContentAction`](../../content-action/README.md) provides a single `rightChildren` slot. Card headers typically need two distinct right-side regions — a primary action on top and secondary actions on the bottom. `CardHeaderLayout` provides this with `rightChildren` and `bottomRightChildren` slots, with no padding or gap between them so the caller has full control over spacing.
|
||||
|
||||
## Props
|
||||
|
||||
Inherits **all** props from [`Content`](../../content/README.md) (icon, title, description, sizePreset, variant, etc.) plus:
|
||||
|
||||
| Prop | Type | Default | Description |
|
||||
|---|---|---|---|
|
||||
| `rightChildren` | `ReactNode` | `undefined` | Content rendered to the right of the Content block (top of right column). |
|
||||
| `bottomRightChildren` | `ReactNode` | `undefined` | Content rendered below `rightChildren` in the same column. Laid out as `flex flex-row`. |
|
||||
|
||||
## Layout Structure
|
||||
|
||||
```
|
||||
┌──────────────────────────────────────────────────────┐
|
||||
│ [Content (p-2, self-start)] [rightChildren] │
|
||||
│ icon + title + description [bottomRightChildren] │
|
||||
└──────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
- Outer wrapper: `flex flex-row items-stretch w-full`
|
||||
- Content area: `flex-1 min-w-0 self-start p-2` — top-aligned with fixed padding
|
||||
- Right column: `flex flex-col items-end justify-between shrink-0` — no padding, no gap
|
||||
- `bottomRightChildren` wrapper: `flex flex-row` — lays children out horizontally
|
||||
|
||||
The right column uses `justify-between` so when both slots are present, `rightChildren` sits at the top and `bottomRightChildren` at the bottom.
|
||||
|
||||
## Usage
|
||||
|
||||
### Card with primary and secondary actions
|
||||
|
||||
```tsx
|
||||
import { CardHeaderLayout } from "@opal/layouts";
|
||||
import { Button } from "@opal/components";
|
||||
import { SvgGlobe, SvgSettings, SvgUnplug, SvgCheckSquare } from "@opal/icons";
|
||||
|
||||
<CardHeaderLayout
|
||||
icon={SvgGlobe}
|
||||
title="Google Search"
|
||||
description="Web search provider"
|
||||
sizePreset="main-ui"
|
||||
variant="section"
|
||||
rightChildren={
|
||||
<Button icon={SvgCheckSquare} variant="action" prominence="tertiary">
|
||||
Current Default
|
||||
</Button>
|
||||
}
|
||||
bottomRightChildren={
|
||||
<>
|
||||
<Button icon={SvgUnplug} size="sm" prominence="tertiary" tooltip="Disconnect" />
|
||||
<Button icon={SvgSettings} size="sm" prominence="tertiary" tooltip="Edit" />
|
||||
</>
|
||||
}
|
||||
/>
|
||||
```
|
||||
|
||||
### Card with only a connect action
|
||||
|
||||
```tsx
|
||||
<CardHeaderLayout
|
||||
icon={SvgCloud}
|
||||
title="OpenAI"
|
||||
description="Not configured"
|
||||
sizePreset="main-ui"
|
||||
variant="section"
|
||||
rightChildren={
|
||||
<Button rightIcon={SvgArrowExchange} prominence="tertiary">
|
||||
Connect
|
||||
</Button>
|
||||
}
|
||||
/>
|
||||
```
|
||||
|
||||
### No right children
|
||||
|
||||
```tsx
|
||||
<CardHeaderLayout
|
||||
icon={SvgInfo}
|
||||
title="Section Header"
|
||||
description="Description text"
|
||||
sizePreset="main-content"
|
||||
variant="section"
|
||||
/>
|
||||
```
|
||||
|
||||
When both `rightChildren` and `bottomRightChildren` are omitted, the component renders only the padded `Content`.
|
||||
73
web/lib/opal/src/layouts/cards/header-layout/components.tsx
Normal file
73
web/lib/opal/src/layouts/cards/header-layout/components.tsx
Normal file
@@ -0,0 +1,73 @@
|
||||
import { Content, type ContentProps } from "@opal/layouts/content/components";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Types
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
type CardHeaderLayoutProps = ContentProps & {
|
||||
/** Content rendered to the right of the Content block. */
|
||||
rightChildren?: React.ReactNode;
|
||||
|
||||
/** Content rendered below `rightChildren` in the same column. */
|
||||
bottomRightChildren?: React.ReactNode;
|
||||
};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// CardHeaderLayout
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* A card header layout that pairs a {@link Content} block (with `p-2`)
|
||||
* with a right-side column.
|
||||
*
|
||||
* The right column contains two vertically stacked slots —
|
||||
* `rightChildren` on top, `bottomRightChildren` below — with no
|
||||
* padding or gap between them.
|
||||
*
|
||||
* @example
|
||||
* ```tsx
|
||||
* <CardHeaderLayout
|
||||
* icon={SvgGlobe}
|
||||
* title="Google"
|
||||
* description="Search engine"
|
||||
* sizePreset="main-ui"
|
||||
* variant="section"
|
||||
* rightChildren={<Button>Connect</Button>}
|
||||
* bottomRightChildren={
|
||||
* <>
|
||||
* <Button icon={SvgUnplug} size="sm" prominence="tertiary" />
|
||||
* <Button icon={SvgSettings} size="sm" prominence="tertiary" />
|
||||
* </>
|
||||
* }
|
||||
* />
|
||||
* ```
|
||||
*/
|
||||
function CardHeaderLayout({
|
||||
rightChildren,
|
||||
bottomRightChildren,
|
||||
...contentProps
|
||||
}: CardHeaderLayoutProps) {
|
||||
const hasRight = rightChildren || bottomRightChildren;
|
||||
|
||||
return (
|
||||
<div className="flex flex-row items-stretch w-full">
|
||||
<div className="flex-1 min-w-0 self-start p-2">
|
||||
<Content {...contentProps} />
|
||||
</div>
|
||||
{hasRight && (
|
||||
<div className="flex flex-col items-end shrink-0">
|
||||
{rightChildren && <div className="flex-1">{rightChildren}</div>}
|
||||
{bottomRightChildren && (
|
||||
<div className="flex flex-row">{bottomRightChildren}</div>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Exports
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export { CardHeaderLayout, type CardHeaderLayoutProps };
|
||||
@@ -4,10 +4,8 @@ import { Button } from "@opal/components/buttons/button/components";
|
||||
import type { ContainerSizeVariants } from "@opal/types";
|
||||
import SvgEdit from "@opal/icons/edit";
|
||||
import type { IconFunctionComponent, RichStr } from "@opal/types";
|
||||
import {
|
||||
resolveStr,
|
||||
toPlainString,
|
||||
} from "@opal/components/text/InlineMarkdown";
|
||||
import { Text, type TextFont } from "@opal/components/text/components";
|
||||
import { toPlainString } from "@opal/components/text/InlineMarkdown";
|
||||
import { cn } from "@opal/utils";
|
||||
import { useState } from "react";
|
||||
|
||||
@@ -24,8 +22,8 @@ interface ContentLgPresetConfig {
|
||||
iconContainerPadding: string;
|
||||
/** Gap between icon container and content (CSS value). */
|
||||
gap: string;
|
||||
/** Tailwind font class for the title. */
|
||||
titleFont: string;
|
||||
/** Opal font name for the title (without `font-` prefix). */
|
||||
titleFont: TextFont;
|
||||
/** Title line-height — also used as icon container min-height (CSS value). */
|
||||
lineHeight: string;
|
||||
/** Button `size` prop for the edit button. Uses the shared `SizeVariant` scale. */
|
||||
@@ -53,9 +51,6 @@ interface ContentLgProps {
|
||||
/** Size preset. Default: `"headline"`. */
|
||||
sizePreset?: ContentLgSizePreset;
|
||||
|
||||
/** When `true`, the title color hooks into `Interactive`'s `--interactive-foreground` variable. */
|
||||
withInteractive?: boolean;
|
||||
|
||||
/** Ref forwarded to the root `<div>`. */
|
||||
ref?: React.Ref<HTMLDivElement>;
|
||||
}
|
||||
@@ -69,7 +64,7 @@ const CONTENT_LG_PRESETS: Record<ContentLgSizePreset, ContentLgPresetConfig> = {
|
||||
iconSize: "2rem",
|
||||
iconContainerPadding: "p-0.5",
|
||||
gap: "0.25rem",
|
||||
titleFont: "font-heading-h2",
|
||||
titleFont: "heading-h2",
|
||||
lineHeight: "2.25rem",
|
||||
editButtonSize: "md",
|
||||
editButtonPadding: "p-1",
|
||||
@@ -78,7 +73,7 @@ const CONTENT_LG_PRESETS: Record<ContentLgSizePreset, ContentLgPresetConfig> = {
|
||||
iconSize: "1.25rem",
|
||||
iconContainerPadding: "p-1",
|
||||
gap: "0rem",
|
||||
titleFont: "font-heading-h3-muted",
|
||||
titleFont: "heading-h3-muted",
|
||||
lineHeight: "1.75rem",
|
||||
editButtonSize: "sm",
|
||||
editButtonPadding: "p-0.5",
|
||||
@@ -96,7 +91,6 @@ function ContentLg({
|
||||
description,
|
||||
editable,
|
||||
onTitleChange,
|
||||
withInteractive,
|
||||
ref,
|
||||
}: ContentLgProps) {
|
||||
const [editing, setEditing] = useState(false);
|
||||
@@ -116,12 +110,7 @@ function ContentLg({
|
||||
}
|
||||
|
||||
return (
|
||||
<div
|
||||
ref={ref}
|
||||
className="opal-content-lg"
|
||||
data-interactive={withInteractive || undefined}
|
||||
style={{ gap: config.gap }}
|
||||
>
|
||||
<div ref={ref} className="opal-content-lg" style={{ gap: config.gap }}>
|
||||
{Icon && (
|
||||
<div
|
||||
className={cn(
|
||||
@@ -142,14 +131,17 @@ function ContentLg({
|
||||
{editing ? (
|
||||
<div className="opal-content-lg-input-sizer">
|
||||
<span
|
||||
className={cn("opal-content-lg-input-mirror", config.titleFont)}
|
||||
className={cn(
|
||||
"opal-content-lg-input-mirror",
|
||||
`font-${config.titleFont}`
|
||||
)}
|
||||
>
|
||||
{editValue || "\u00A0"}
|
||||
</span>
|
||||
<input
|
||||
className={cn(
|
||||
"opal-content-lg-input",
|
||||
config.titleFont,
|
||||
`font-${config.titleFont}`,
|
||||
"text-text-04"
|
||||
)}
|
||||
value={editValue}
|
||||
@@ -169,19 +161,15 @@ function ContentLg({
|
||||
/>
|
||||
</div>
|
||||
) : (
|
||||
<span
|
||||
className={cn(
|
||||
"opal-content-lg-title",
|
||||
config.titleFont,
|
||||
"text-text-04",
|
||||
editable && "cursor-pointer"
|
||||
)}
|
||||
onClick={editable ? startEditing : undefined}
|
||||
style={{ height: config.lineHeight }}
|
||||
<Text
|
||||
font={config.titleFont}
|
||||
color="inherit"
|
||||
maxLines={1}
|
||||
title={toPlainString(title)}
|
||||
onClick={editable ? startEditing : undefined}
|
||||
>
|
||||
{resolveStr(title)}
|
||||
</span>
|
||||
{title}
|
||||
</Text>
|
||||
)}
|
||||
|
||||
{editable && !editing && (
|
||||
@@ -204,8 +192,10 @@ function ContentLg({
|
||||
</div>
|
||||
|
||||
{description && toPlainString(description) && (
|
||||
<div className="opal-content-lg-description font-secondary-body text-text-03">
|
||||
{resolveStr(description)}
|
||||
<div className="opal-content-lg-description">
|
||||
<Text font="secondary-body" color="text-03" as="p">
|
||||
{description}
|
||||
</Text>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
@@ -8,10 +8,8 @@ import SvgAlertTriangle from "@opal/icons/alert-triangle";
|
||||
import SvgEdit from "@opal/icons/edit";
|
||||
import SvgXOctagon from "@opal/icons/x-octagon";
|
||||
import type { IconFunctionComponent, RichStr } from "@opal/types";
|
||||
import {
|
||||
resolveStr,
|
||||
toPlainString,
|
||||
} from "@opal/components/text/InlineMarkdown";
|
||||
import { Text, type TextFont } from "@opal/components/text/components";
|
||||
import { toPlainString } from "@opal/components/text/InlineMarkdown";
|
||||
import { cn } from "@opal/utils";
|
||||
import { useRef, useState } from "react";
|
||||
|
||||
@@ -23,16 +21,18 @@ type ContentMdSizePreset = "main-content" | "main-ui" | "secondary";
|
||||
|
||||
type ContentMdAuxIcon = "info-gray" | "info-blue" | "warning" | "error";
|
||||
|
||||
type ContentMdSuffix = "optional" | (string & {});
|
||||
|
||||
interface ContentMdPresetConfig {
|
||||
iconSize: string;
|
||||
iconContainerPadding: string;
|
||||
iconColorClass: string;
|
||||
titleFont: string;
|
||||
titleFont: TextFont;
|
||||
lineHeight: string;
|
||||
/** Button `size` prop for the edit button. Uses the shared `SizeVariant` scale. */
|
||||
editButtonSize: ContainerSizeVariants;
|
||||
editButtonPadding: string;
|
||||
optionalFont: string;
|
||||
optionalFont: TextFont;
|
||||
/** Aux icon size = lineHeight − 2 × p-0.5. */
|
||||
auxIconSize: string;
|
||||
/** Left indent for the description so it aligns with the title (past the icon). */
|
||||
@@ -55,11 +55,11 @@ interface ContentMdProps {
|
||||
/** Called when the user commits an edit. */
|
||||
onTitleChange?: (newTitle: string) => void;
|
||||
|
||||
/** When `true`, renders "(Optional)" beside the title. */
|
||||
optional?: boolean;
|
||||
|
||||
/** Custom muted suffix rendered beside the title. */
|
||||
titleSuffix?: string;
|
||||
/**
|
||||
* Muted suffix rendered beside the title.
|
||||
* Use `"optional"` for the standard "(Optional)" label, or pass any string.
|
||||
*/
|
||||
suffix?: ContentMdSuffix;
|
||||
|
||||
/** Auxiliary status icon rendered beside the title. */
|
||||
auxIcon?: ContentMdAuxIcon;
|
||||
@@ -70,18 +70,6 @@ interface ContentMdProps {
|
||||
/** Size preset. Default: `"main-ui"`. */
|
||||
sizePreset?: ContentMdSizePreset;
|
||||
|
||||
/** When `true`, the title color hooks into `Interactive`'s `--interactive-foreground` variable. */
|
||||
withInteractive?: boolean;
|
||||
|
||||
/** Optional class name applied to the title element. */
|
||||
titleClassName?: string;
|
||||
|
||||
/** Optional class name applied to the icon element. */
|
||||
iconClassName?: string;
|
||||
|
||||
/** Content rendered below the description, indented to align with it. */
|
||||
bottomChildren?: React.ReactNode;
|
||||
|
||||
/** Ref forwarded to the root `<div>`. */
|
||||
ref?: React.Ref<HTMLDivElement>;
|
||||
}
|
||||
@@ -95,11 +83,11 @@ const CONTENT_MD_PRESETS: Record<ContentMdSizePreset, ContentMdPresetConfig> = {
|
||||
iconSize: "1rem",
|
||||
iconContainerPadding: "p-1",
|
||||
iconColorClass: "text-text-04",
|
||||
titleFont: "font-main-content-emphasis",
|
||||
titleFont: "main-content-emphasis",
|
||||
lineHeight: "1.5rem",
|
||||
editButtonSize: "sm",
|
||||
editButtonPadding: "p-0",
|
||||
optionalFont: "font-main-content-muted",
|
||||
optionalFont: "main-content-muted",
|
||||
auxIconSize: "1.25rem",
|
||||
descriptionIndent: "1.625rem",
|
||||
},
|
||||
@@ -107,11 +95,11 @@ const CONTENT_MD_PRESETS: Record<ContentMdSizePreset, ContentMdPresetConfig> = {
|
||||
iconSize: "1rem",
|
||||
iconContainerPadding: "p-0.5",
|
||||
iconColorClass: "text-text-03",
|
||||
titleFont: "font-main-ui-action",
|
||||
titleFont: "main-ui-action",
|
||||
lineHeight: "1.25rem",
|
||||
editButtonSize: "xs",
|
||||
editButtonPadding: "p-0",
|
||||
optionalFont: "font-main-ui-muted",
|
||||
optionalFont: "main-ui-muted",
|
||||
auxIconSize: "1rem",
|
||||
descriptionIndent: "1.375rem",
|
||||
},
|
||||
@@ -119,11 +107,11 @@ const CONTENT_MD_PRESETS: Record<ContentMdSizePreset, ContentMdPresetConfig> = {
|
||||
iconSize: "0.75rem",
|
||||
iconContainerPadding: "p-0.5",
|
||||
iconColorClass: "text-text-04",
|
||||
titleFont: "font-secondary-action",
|
||||
titleFont: "secondary-action",
|
||||
lineHeight: "1rem",
|
||||
editButtonSize: "2xs",
|
||||
editButtonPadding: "p-0",
|
||||
optionalFont: "font-secondary-action",
|
||||
optionalFont: "secondary-action",
|
||||
auxIconSize: "0.75rem",
|
||||
descriptionIndent: "1.125rem",
|
||||
},
|
||||
@@ -149,15 +137,10 @@ function ContentMd({
|
||||
description,
|
||||
editable,
|
||||
onTitleChange,
|
||||
optional,
|
||||
titleSuffix,
|
||||
suffix,
|
||||
auxIcon,
|
||||
tag,
|
||||
sizePreset = "main-ui",
|
||||
withInteractive,
|
||||
titleClassName,
|
||||
iconClassName,
|
||||
bottomChildren,
|
||||
ref,
|
||||
}: ContentMdProps) {
|
||||
const [editing, setEditing] = useState(false);
|
||||
@@ -178,11 +161,7 @@ function ContentMd({
|
||||
}
|
||||
|
||||
return (
|
||||
<div
|
||||
ref={ref}
|
||||
className="opal-content-md"
|
||||
data-interactive={withInteractive || undefined}
|
||||
>
|
||||
<div ref={ref} className="opal-content-md">
|
||||
<div
|
||||
className="opal-content-md-header"
|
||||
data-editing={editing || undefined}
|
||||
@@ -196,11 +175,7 @@ function ContentMd({
|
||||
style={{ minHeight: config.lineHeight }}
|
||||
>
|
||||
<Icon
|
||||
className={cn(
|
||||
"opal-content-md-icon",
|
||||
config.iconColorClass,
|
||||
iconClassName
|
||||
)}
|
||||
className={cn("opal-content-md-icon", config.iconColorClass)}
|
||||
style={{ width: config.iconSize, height: config.iconSize }}
|
||||
/>
|
||||
</div>
|
||||
@@ -210,7 +185,10 @@ function ContentMd({
|
||||
{editing ? (
|
||||
<div className="opal-content-md-input-sizer">
|
||||
<span
|
||||
className={cn("opal-content-md-input-mirror", config.titleFont)}
|
||||
className={cn(
|
||||
"opal-content-md-input-mirror",
|
||||
`font-${config.titleFont}`
|
||||
)}
|
||||
>
|
||||
{editValue || "\u00A0"}
|
||||
</span>
|
||||
@@ -218,7 +196,7 @@ function ContentMd({
|
||||
ref={inputRef}
|
||||
className={cn(
|
||||
"opal-content-md-input",
|
||||
config.titleFont,
|
||||
`font-${config.titleFont}`,
|
||||
"text-text-04"
|
||||
)}
|
||||
value={editValue}
|
||||
@@ -238,29 +216,21 @@ function ContentMd({
|
||||
/>
|
||||
</div>
|
||||
) : (
|
||||
<span
|
||||
className={cn(
|
||||
"opal-content-md-title",
|
||||
config.titleFont,
|
||||
"text-text-04",
|
||||
editable && "cursor-pointer",
|
||||
titleClassName
|
||||
)}
|
||||
<Text
|
||||
font={config.titleFont}
|
||||
color="inherit"
|
||||
maxLines={1}
|
||||
title={toPlainString(title)}
|
||||
onClick={editable ? startEditing : undefined}
|
||||
style={{ height: config.lineHeight }}
|
||||
>
|
||||
{resolveStr(title)}
|
||||
</span>
|
||||
{title}
|
||||
</Text>
|
||||
)}
|
||||
|
||||
{(optional || titleSuffix) && (
|
||||
<span
|
||||
className={cn(config.optionalFont, "text-text-03 shrink-0")}
|
||||
style={{ height: config.lineHeight }}
|
||||
>
|
||||
{titleSuffix ?? "(Optional)"}
|
||||
</span>
|
||||
{suffix && (
|
||||
<Text font={config.optionalFont} color="text-03">
|
||||
{suffix === "optional" ? "(Optional)" : suffix}
|
||||
</Text>
|
||||
)}
|
||||
|
||||
{auxIcon &&
|
||||
@@ -306,17 +276,12 @@ function ContentMd({
|
||||
|
||||
{description && toPlainString(description) && (
|
||||
<div
|
||||
className="opal-content-md-description font-secondary-body text-text-03"
|
||||
className="opal-content-md-description"
|
||||
style={Icon ? { paddingLeft: config.descriptionIndent } : undefined}
|
||||
>
|
||||
{resolveStr(description)}
|
||||
</div>
|
||||
)}
|
||||
{bottomChildren && (
|
||||
<div
|
||||
style={Icon ? { paddingLeft: config.descriptionIndent } : undefined}
|
||||
>
|
||||
{bottomChildren}
|
||||
<Text font="secondary-body" color="text-03" as="p">
|
||||
{description}
|
||||
</Text>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
@@ -327,5 +292,6 @@ export {
|
||||
ContentMd,
|
||||
type ContentMdProps,
|
||||
type ContentMdSizePreset,
|
||||
type ContentMdSuffix,
|
||||
type ContentMdAuxIcon,
|
||||
};
|
||||
|
||||
@@ -1,10 +1,8 @@
|
||||
"use client";
|
||||
|
||||
import type { IconFunctionComponent, RichStr } from "@opal/types";
|
||||
import {
|
||||
resolveStr,
|
||||
toPlainString,
|
||||
} from "@opal/components/text/InlineMarkdown";
|
||||
import { Text, type TextFont } from "@opal/components/text/components";
|
||||
import { toPlainString } from "@opal/components/text/InlineMarkdown";
|
||||
import { cn } from "@opal/utils";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -13,15 +11,15 @@ import { cn } from "@opal/utils";
|
||||
|
||||
type ContentSmSizePreset = "main-content" | "main-ui" | "secondary";
|
||||
type ContentSmOrientation = "vertical" | "inline" | "reverse";
|
||||
type ContentSmProminence = "default" | "muted" | "muted-2x";
|
||||
type ContentSmProminence = "default" | "muted";
|
||||
|
||||
interface ContentSmPresetConfig {
|
||||
/** Icon width/height (CSS value). */
|
||||
iconSize: string;
|
||||
/** Tailwind padding class for the icon container. */
|
||||
iconContainerPadding: string;
|
||||
/** Tailwind font class for the title. */
|
||||
titleFont: string;
|
||||
/** Font preset for the title. */
|
||||
titleFont: TextFont;
|
||||
/** Title line-height — also used as icon container min-height (CSS value). */
|
||||
lineHeight: string;
|
||||
/** Gap between icon container and title (CSS value). */
|
||||
@@ -45,9 +43,6 @@ interface ContentSmProps {
|
||||
/** Title prominence. Default: `"default"`. */
|
||||
prominence?: ContentSmProminence;
|
||||
|
||||
/** When `true`, the title color hooks into `Interactive`'s `--interactive-foreground` variable. */
|
||||
withInteractive?: boolean;
|
||||
|
||||
/** Ref forwarded to the root `<div>`. */
|
||||
ref?: React.Ref<HTMLDivElement>;
|
||||
}
|
||||
@@ -60,21 +55,21 @@ const CONTENT_SM_PRESETS: Record<ContentSmSizePreset, ContentSmPresetConfig> = {
|
||||
"main-content": {
|
||||
iconSize: "1rem",
|
||||
iconContainerPadding: "p-1",
|
||||
titleFont: "font-main-content-body",
|
||||
titleFont: "main-content-body",
|
||||
lineHeight: "1.5rem",
|
||||
gap: "0.125rem",
|
||||
},
|
||||
"main-ui": {
|
||||
iconSize: "1rem",
|
||||
iconContainerPadding: "p-0.5",
|
||||
titleFont: "font-main-ui-action",
|
||||
titleFont: "main-ui-action",
|
||||
lineHeight: "1.25rem",
|
||||
gap: "0.25rem",
|
||||
},
|
||||
secondary: {
|
||||
iconSize: "0.75rem",
|
||||
iconContainerPadding: "p-0.5",
|
||||
titleFont: "font-secondary-action",
|
||||
titleFont: "secondary-action",
|
||||
lineHeight: "1rem",
|
||||
gap: "0.125rem",
|
||||
},
|
||||
@@ -90,7 +85,6 @@ function ContentSm({
|
||||
sizePreset = "main-ui",
|
||||
orientation = "inline",
|
||||
prominence = "default",
|
||||
withInteractive,
|
||||
ref,
|
||||
}: ContentSmProps) {
|
||||
const config = CONTENT_SM_PRESETS[sizePreset];
|
||||
@@ -101,7 +95,6 @@ function ContentSm({
|
||||
className="opal-content-sm"
|
||||
data-orientation={orientation}
|
||||
data-prominence={prominence}
|
||||
data-interactive={withInteractive || undefined}
|
||||
style={{ gap: config.gap }}
|
||||
>
|
||||
{Icon && (
|
||||
@@ -119,13 +112,14 @@ function ContentSm({
|
||||
</div>
|
||||
)}
|
||||
|
||||
<span
|
||||
className={cn("opal-content-sm-title", config.titleFont)}
|
||||
style={{ height: config.lineHeight }}
|
||||
<Text
|
||||
font={config.titleFont}
|
||||
color="inherit"
|
||||
maxLines={1}
|
||||
title={toPlainString(title)}
|
||||
>
|
||||
{resolveStr(title)}
|
||||
</span>
|
||||
{title}
|
||||
</Text>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -4,10 +4,8 @@ import { Button } from "@opal/components/buttons/button/components";
|
||||
import type { ContainerSizeVariants } from "@opal/types";
|
||||
import SvgEdit from "@opal/icons/edit";
|
||||
import type { IconFunctionComponent, RichStr } from "@opal/types";
|
||||
import {
|
||||
resolveStr,
|
||||
toPlainString,
|
||||
} from "@opal/components/text/InlineMarkdown";
|
||||
import { Text, type TextFont } from "@opal/components/text/components";
|
||||
import { toPlainString } from "@opal/components/text/InlineMarkdown";
|
||||
import { cn } from "@opal/utils";
|
||||
import { useState } from "react";
|
||||
|
||||
@@ -30,8 +28,8 @@ interface ContentXlPresetConfig {
|
||||
moreIcon2Size: string;
|
||||
/** Tailwind padding class for the more-icon-2 container. */
|
||||
moreIcon2ContainerPadding: string;
|
||||
/** Tailwind font class for the title. */
|
||||
titleFont: string;
|
||||
/** Opal font name for the title (without `font-` prefix). */
|
||||
titleFont: TextFont;
|
||||
/** Title line-height — also used as icon container min-height (CSS value). */
|
||||
lineHeight: string;
|
||||
/** Button `size` prop for the edit button. Uses the shared `SizeVariant` scale. */
|
||||
@@ -65,9 +63,6 @@ interface ContentXlProps {
|
||||
/** Optional tertiary icon rendered in the icon row. */
|
||||
moreIcon2?: IconFunctionComponent;
|
||||
|
||||
/** When `true`, the title color hooks into `Interactive`'s `--interactive-foreground` variable. */
|
||||
withInteractive?: boolean;
|
||||
|
||||
/** Ref forwarded to the root `<div>`. */
|
||||
ref?: React.Ref<HTMLDivElement>;
|
||||
}
|
||||
@@ -84,7 +79,7 @@ const CONTENT_XL_PRESETS: Record<ContentXlSizePreset, ContentXlPresetConfig> = {
|
||||
moreIcon1ContainerPadding: "p-0.5",
|
||||
moreIcon2Size: "2rem",
|
||||
moreIcon2ContainerPadding: "p-0.5",
|
||||
titleFont: "font-heading-h2",
|
||||
titleFont: "heading-h2",
|
||||
lineHeight: "2.25rem",
|
||||
editButtonSize: "md",
|
||||
editButtonPadding: "p-1",
|
||||
@@ -96,7 +91,7 @@ const CONTENT_XL_PRESETS: Record<ContentXlSizePreset, ContentXlPresetConfig> = {
|
||||
moreIcon1ContainerPadding: "p-0.5",
|
||||
moreIcon2Size: "1.5rem",
|
||||
moreIcon2ContainerPadding: "p-0.5",
|
||||
titleFont: "font-heading-h3",
|
||||
titleFont: "heading-h3",
|
||||
lineHeight: "1.75rem",
|
||||
editButtonSize: "sm",
|
||||
editButtonPadding: "p-0.5",
|
||||
@@ -116,7 +111,6 @@ function ContentXl({
|
||||
onTitleChange,
|
||||
moreIcon1: MoreIcon1,
|
||||
moreIcon2: MoreIcon2,
|
||||
withInteractive,
|
||||
ref,
|
||||
}: ContentXlProps) {
|
||||
const [editing, setEditing] = useState(false);
|
||||
@@ -136,11 +130,7 @@ function ContentXl({
|
||||
}
|
||||
|
||||
return (
|
||||
<div
|
||||
ref={ref}
|
||||
className="opal-content-xl"
|
||||
data-interactive={withInteractive || undefined}
|
||||
>
|
||||
<div ref={ref} className="opal-content-xl">
|
||||
{(Icon || MoreIcon1 || MoreIcon2) && (
|
||||
<div className="opal-content-xl-icon-row">
|
||||
{Icon && (
|
||||
@@ -199,14 +189,17 @@ function ContentXl({
|
||||
{editing ? (
|
||||
<div className="opal-content-xl-input-sizer">
|
||||
<span
|
||||
className={cn("opal-content-xl-input-mirror", config.titleFont)}
|
||||
className={cn(
|
||||
"opal-content-xl-input-mirror",
|
||||
`font-${config.titleFont}`
|
||||
)}
|
||||
>
|
||||
{editValue || "\u00A0"}
|
||||
</span>
|
||||
<input
|
||||
className={cn(
|
||||
"opal-content-xl-input",
|
||||
config.titleFont,
|
||||
`font-${config.titleFont}`,
|
||||
"text-text-04"
|
||||
)}
|
||||
value={editValue}
|
||||
@@ -226,19 +219,15 @@ function ContentXl({
|
||||
/>
|
||||
</div>
|
||||
) : (
|
||||
<span
|
||||
className={cn(
|
||||
"opal-content-xl-title",
|
||||
config.titleFont,
|
||||
"text-text-04",
|
||||
editable && "cursor-pointer"
|
||||
)}
|
||||
onClick={editable ? startEditing : undefined}
|
||||
style={{ height: config.lineHeight }}
|
||||
<Text
|
||||
font={config.titleFont}
|
||||
color="inherit"
|
||||
maxLines={1}
|
||||
title={toPlainString(title)}
|
||||
onClick={editable ? startEditing : undefined}
|
||||
>
|
||||
{resolveStr(title)}
|
||||
</span>
|
||||
{title}
|
||||
</Text>
|
||||
)}
|
||||
|
||||
{editable && !editing && (
|
||||
@@ -261,8 +250,10 @@ function ContentXl({
|
||||
</div>
|
||||
|
||||
{description && toPlainString(description) && (
|
||||
<div className="opal-content-xl-description font-secondary-body text-text-03">
|
||||
{resolveStr(description)}
|
||||
<div className="opal-content-xl-description">
|
||||
<Text font="secondary-body" color="text-03" as="p">
|
||||
{description}
|
||||
</Text>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
@@ -1,39 +1,3 @@
|
||||
// ---------------------------------------------------------------------------
|
||||
// NOTE (@raunakab): Why Content uses resolveStr() instead of <Text>
|
||||
//
|
||||
// Content sub-components (ContentXl, ContentLg, ContentMd, ContentSm) render
|
||||
// titles and descriptions inside styled <span> elements that carry CSS classes
|
||||
// (e.g., `.opal-content-md-title`) for:
|
||||
//
|
||||
// 1. Truncation — `-webkit-box` + `-webkit-line-clamp` for single-line
|
||||
// clamping with ellipsis. This requires the text to be a DIRECT child
|
||||
// of the `-webkit-box` element. Wrapping it in a child <span> (which
|
||||
// is what <Text> renders) breaks the clamping behavior.
|
||||
//
|
||||
// 2. Pixel-exact sizing — the wrapper <span> has an explicit `height`
|
||||
// matching the font's `line-height`. Adding a child <Text> <span>
|
||||
// inside creates a double-span where the inner element's line-height
|
||||
// conflicts with the outer element's height, causing a ~4px vertical
|
||||
// offset.
|
||||
//
|
||||
// 3. Interactive color overrides — CSS selectors like
|
||||
// `.opal-content-md[data-interactive] .opal-content-md-title` set
|
||||
// `color: var(--interactive-foreground)`. <Text> with `color="inherit"`
|
||||
// can inherit this, but <Text> with any explicit color prop overrides
|
||||
// it. And the wrapper <span> needs the CSS class for the selector to
|
||||
// match — removing it breaks the cascade.
|
||||
//
|
||||
// 4. Horizontal padding — the title CSS class applies `padding: 0 0.125rem`
|
||||
// (2px). Since <Text> uses WithoutStyles (no className/style), this
|
||||
// padding cannot be applied to <Text> directly. A wrapper <div> was
|
||||
// attempted but introduced additional layout conflicts.
|
||||
//
|
||||
// For these reasons, Content uses `resolveStr()` from InlineMarkdown.tsx to
|
||||
// handle `string | RichStr` rendering. `resolveStr()` returns a ReactNode
|
||||
// that slots directly into the existing single <span> — no extra wrapper,
|
||||
// no layout conflicts, pixel-exact match with main.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
import "@opal/layouts/content/styles.css";
|
||||
import {
|
||||
ContentSm,
|
||||
@@ -98,9 +62,6 @@ interface ContentBaseProps {
|
||||
*/
|
||||
widthVariant?: ExtremaSizeVariants;
|
||||
|
||||
/** When `true`, the title color hooks into `Interactive.Stateful`/`Interactive.Stateless`'s `--interactive-foreground` variable. */
|
||||
withInteractive?: boolean;
|
||||
|
||||
/** Ref forwarded to the root `<div>` of the resolved layout. */
|
||||
ref?: React.Ref<HTMLDivElement>;
|
||||
}
|
||||
@@ -130,20 +91,12 @@ type LgContentProps = ContentBaseProps & {
|
||||
type MdContentProps = ContentBaseProps & {
|
||||
sizePreset: "main-content" | "main-ui" | "secondary";
|
||||
variant?: "section";
|
||||
/** When `true`, renders "(Optional)" beside the title in the muted font variant. */
|
||||
optional?: boolean;
|
||||
/** Custom muted suffix rendered beside the title. */
|
||||
titleSuffix?: string;
|
||||
/** Muted suffix rendered beside the title. Use `"optional"` for "(Optional)". */
|
||||
suffix?: "optional" | (string & {});
|
||||
/** Auxiliary status icon rendered beside the title. */
|
||||
auxIcon?: "info-gray" | "info-blue" | "warning" | "error";
|
||||
/** Tag rendered beside the title. */
|
||||
tag?: TagProps;
|
||||
/** Optional class name applied to the title element. */
|
||||
titleClassName?: string;
|
||||
/** Optional class name applied to the icon element. */
|
||||
iconClassName?: string;
|
||||
/** Content rendered below the description, indented to align with it. */
|
||||
bottomChildren?: React.ReactNode;
|
||||
};
|
||||
|
||||
/** ContentSm does not support descriptions or inline editing. */
|
||||
@@ -174,7 +127,6 @@ function Content(props: ContentProps) {
|
||||
sizePreset = "headline",
|
||||
variant = "heading",
|
||||
widthVariant = "full",
|
||||
withInteractive,
|
||||
ref,
|
||||
...rest
|
||||
} = props;
|
||||
@@ -187,7 +139,6 @@ function Content(props: ContentProps) {
|
||||
layout = (
|
||||
<ContentXl
|
||||
sizePreset={sizePreset}
|
||||
withInteractive={withInteractive}
|
||||
ref={ref}
|
||||
{...(rest as Omit<ContentXlProps, "sizePreset">)}
|
||||
/>
|
||||
@@ -196,7 +147,6 @@ function Content(props: ContentProps) {
|
||||
layout = (
|
||||
<ContentLg
|
||||
sizePreset={sizePreset}
|
||||
withInteractive={withInteractive}
|
||||
ref={ref}
|
||||
{...(rest as Omit<ContentLgProps, "sizePreset">)}
|
||||
/>
|
||||
@@ -210,7 +160,6 @@ function Content(props: ContentProps) {
|
||||
layout = (
|
||||
<ContentMd
|
||||
sizePreset={sizePreset}
|
||||
withInteractive={withInteractive}
|
||||
ref={ref}
|
||||
{...(rest as Omit<ContentMdProps, "sizePreset">)}
|
||||
/>
|
||||
@@ -222,7 +171,6 @@ function Content(props: ContentProps) {
|
||||
layout = (
|
||||
<ContentSm
|
||||
sizePreset={sizePreset}
|
||||
withInteractive={withInteractive}
|
||||
ref={ref}
|
||||
{...(rest as Omit<
|
||||
React.ComponentProps<typeof ContentSm>,
|
||||
|
||||
@@ -13,7 +13,7 @@
|
||||
--------------------------------------------------------------------------- */
|
||||
|
||||
.opal-content-xl {
|
||||
@apply flex flex-col items-start;
|
||||
@apply flex flex-col items-start text-text-04;
|
||||
}
|
||||
|
||||
/* ---------------------------------------------------------------------------
|
||||
@@ -63,15 +63,6 @@
|
||||
gap: 0.25rem;
|
||||
}
|
||||
|
||||
.opal-content-xl-title {
|
||||
@apply text-left overflow-hidden text-text-04;
|
||||
display: -webkit-box;
|
||||
-webkit-box-orient: vertical;
|
||||
-webkit-line-clamp: 1;
|
||||
padding: 0 0.125rem;
|
||||
min-width: 0.0625rem;
|
||||
}
|
||||
|
||||
.opal-content-xl-input-sizer {
|
||||
display: inline-grid;
|
||||
align-items: stretch;
|
||||
@@ -110,7 +101,6 @@
|
||||
|
||||
.opal-content-xl-description {
|
||||
@apply text-left w-full;
|
||||
padding: 0 0.125rem;
|
||||
}
|
||||
|
||||
/* ===========================================================================
|
||||
@@ -127,7 +117,7 @@
|
||||
--------------------------------------------------------------------------- */
|
||||
|
||||
.opal-content-lg {
|
||||
@apply flex flex-row items-start;
|
||||
@apply flex flex-row items-start text-text-04;
|
||||
}
|
||||
|
||||
/* ---------------------------------------------------------------------------
|
||||
@@ -162,15 +152,6 @@
|
||||
gap: 0.25rem;
|
||||
}
|
||||
|
||||
.opal-content-lg-title {
|
||||
@apply text-left overflow-hidden text-text-04;
|
||||
display: -webkit-box;
|
||||
-webkit-box-orient: vertical;
|
||||
-webkit-line-clamp: 1;
|
||||
padding: 0 0.125rem;
|
||||
min-width: 0.0625rem;
|
||||
}
|
||||
|
||||
.opal-content-lg-input-sizer {
|
||||
display: inline-grid;
|
||||
align-items: stretch;
|
||||
@@ -209,7 +190,6 @@
|
||||
|
||||
.opal-content-lg-description {
|
||||
@apply text-left w-full;
|
||||
padding: 0 0.125rem;
|
||||
}
|
||||
|
||||
/* ===========================================================================
|
||||
@@ -224,7 +204,7 @@
|
||||
--------------------------------------------------------------------------- */
|
||||
|
||||
.opal-content-md {
|
||||
@apply flex flex-col items-start;
|
||||
@apply flex flex-col items-start text-text-04;
|
||||
}
|
||||
|
||||
.opal-content-md-header {
|
||||
@@ -255,15 +235,6 @@
|
||||
gap: 0.25rem;
|
||||
}
|
||||
|
||||
.opal-content-md-title {
|
||||
@apply text-left overflow-hidden text-text-04;
|
||||
display: -webkit-box;
|
||||
-webkit-box-orient: vertical;
|
||||
-webkit-line-clamp: 1;
|
||||
padding: 0 0.125rem;
|
||||
min-width: 0.0625rem;
|
||||
}
|
||||
|
||||
.opal-content-md-input-sizer {
|
||||
display: inline-grid;
|
||||
align-items: stretch;
|
||||
@@ -313,7 +284,6 @@
|
||||
|
||||
.opal-content-md-description {
|
||||
@apply text-left w-full;
|
||||
padding: 0 0.125rem;
|
||||
}
|
||||
|
||||
/* ===========================================================================
|
||||
@@ -325,7 +295,7 @@
|
||||
reverse : flex-row-reverse — title left, icon right
|
||||
|
||||
Icon color is always text-03. Title color varies by prominence
|
||||
(text-04 default, text-03 muted, text-02 muted-2x) via data-prominence.
|
||||
(text-04 default, text-03 muted) via data-prominence.
|
||||
=========================================================================== */
|
||||
|
||||
/* ---------------------------------------------------------------------------
|
||||
@@ -334,7 +304,7 @@
|
||||
|
||||
.opal-content-sm {
|
||||
/* since `ContentSm` doesn't have a description, it's possible to center-align the icon and text */
|
||||
@apply flex items-center;
|
||||
@apply flex items-center text-text-04;
|
||||
}
|
||||
|
||||
.opal-content-sm[data-orientation="inline"] {
|
||||
@@ -367,68 +337,51 @@
|
||||
@apply text-text-02;
|
||||
}
|
||||
|
||||
.opal-content-sm[data-prominence="muted-2x"] .opal-content-sm-icon {
|
||||
@apply text-text-02;
|
||||
}
|
||||
|
||||
/* ---------------------------------------------------------------------------
|
||||
Title
|
||||
--------------------------------------------------------------------------- */
|
||||
|
||||
.opal-content-sm-title {
|
||||
@apply text-left overflow-hidden text-text-04 truncate;
|
||||
display: -webkit-box;
|
||||
-webkit-box-orient: vertical;
|
||||
-webkit-line-clamp: 1;
|
||||
padding: 0 0.125rem;
|
||||
min-width: 0.0625rem;
|
||||
}
|
||||
|
||||
.opal-content-sm[data-prominence="muted"] .opal-content-sm-title {
|
||||
.opal-content-sm[data-prominence="muted"] {
|
||||
@apply text-text-03;
|
||||
}
|
||||
|
||||
.opal-content-sm[data-prominence="muted-2x"] .opal-content-sm-title {
|
||||
@apply text-text-02;
|
||||
}
|
||||
|
||||
/* ===========================================================================
|
||||
Interactive-foreground opt-in
|
||||
Interactive override
|
||||
|
||||
When a Content variant is nested inside an Interactive and
|
||||
`withInteractive` is set, the title and icon delegate their color to the
|
||||
`--interactive-foreground` / `--interactive-foreground-icon` CSS variables
|
||||
controlled by the ancestor Interactive variant.
|
||||
When a Content variant is nested inside an `.interactive` element,
|
||||
the title inherits color from the Interactive's `--interactive-foreground`
|
||||
and icons switch to `--interactive-foreground-icon`. This is automatic —
|
||||
no opt-in prop is required.
|
||||
=========================================================================== */
|
||||
|
||||
.opal-content-xl[data-interactive] .opal-content-xl-title {
|
||||
color: var(--interactive-foreground);
|
||||
.interactive .opal-content-xl {
|
||||
color: inherit;
|
||||
}
|
||||
|
||||
.opal-content-xl[data-interactive] .opal-content-xl-icon {
|
||||
.interactive .opal-content-xl .opal-content-xl-icon {
|
||||
color: var(--interactive-foreground-icon);
|
||||
}
|
||||
|
||||
.opal-content-lg[data-interactive] .opal-content-lg-title {
|
||||
color: var(--interactive-foreground);
|
||||
.interactive .opal-content-lg {
|
||||
color: inherit;
|
||||
}
|
||||
|
||||
.opal-content-lg[data-interactive] .opal-content-lg-icon {
|
||||
.interactive .opal-content-lg .opal-content-lg-icon {
|
||||
color: var(--interactive-foreground-icon);
|
||||
}
|
||||
|
||||
.opal-content-md[data-interactive] .opal-content-md-title {
|
||||
color: var(--interactive-foreground);
|
||||
.interactive .opal-content-md {
|
||||
color: inherit;
|
||||
}
|
||||
|
||||
.opal-content-md[data-interactive] .opal-content-md-icon {
|
||||
.interactive .opal-content-md .opal-content-md-icon {
|
||||
color: var(--interactive-foreground-icon);
|
||||
}
|
||||
|
||||
.opal-content-sm[data-interactive] .opal-content-sm-title {
|
||||
color: var(--interactive-foreground);
|
||||
.interactive .opal-content-sm {
|
||||
color: inherit;
|
||||
}
|
||||
|
||||
.opal-content-sm[data-interactive] .opal-content-sm-icon {
|
||||
.interactive .opal-content-sm .opal-content-sm-icon {
|
||||
color: var(--interactive-foreground-icon);
|
||||
}
|
||||
|
||||
@@ -12,6 +12,12 @@ export {
|
||||
type ContentActionProps,
|
||||
} from "@opal/layouts/content-action/components";
|
||||
|
||||
/* CardHeaderLayout */
|
||||
export {
|
||||
CardHeaderLayout,
|
||||
type CardHeaderLayoutProps,
|
||||
} from "@opal/layouts/cards/header-layout/components";
|
||||
|
||||
/* IllustrationContent */
|
||||
export {
|
||||
IllustrationContent,
|
||||
|
||||
@@ -6,7 +6,14 @@ export function cn(...inputs: ClassValue[]) {
|
||||
return twMerge(clsx(inputs));
|
||||
}
|
||||
|
||||
/** Wraps a string for inline markdown parsing by `Text` and other Opal components. */
|
||||
export function markdown(content: string): RichStr {
|
||||
return { __brand: "RichStr", raw: content };
|
||||
/**
|
||||
* Wraps strings for inline markdown parsing by `Text` and other Opal components.
|
||||
*
|
||||
* Multiple arguments are joined with newlines, so each string renders on its own line:
|
||||
* ```tsx
|
||||
* markdown("Line one", "Line two", "Line three")
|
||||
* ```
|
||||
*/
|
||||
export function markdown(...lines: string[]): RichStr {
|
||||
return { __brand: "RichStr", raw: lines.join("\n") };
|
||||
}
|
||||
|
||||
@@ -8,6 +8,7 @@ import * as Sentry from "@sentry/nextjs";
|
||||
if (process.env.NEXT_PUBLIC_SENTRY_DSN) {
|
||||
Sentry.init({
|
||||
dsn: process.env.NEXT_PUBLIC_SENTRY_DSN,
|
||||
release: process.env.SENTRY_RELEASE,
|
||||
// Only capture unhandled exceptions
|
||||
tracesSampleRate: 0,
|
||||
debug: false,
|
||||
|
||||
@@ -7,6 +7,7 @@ import * as Sentry from "@sentry/nextjs";
|
||||
if (process.env.NEXT_PUBLIC_SENTRY_DSN) {
|
||||
Sentry.init({
|
||||
dsn: process.env.NEXT_PUBLIC_SENTRY_DSN,
|
||||
release: process.env.SENTRY_RELEASE,
|
||||
|
||||
// Setting this option to true will print useful information to the console while you're setting up Sentry.
|
||||
debug: false,
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
import { useState, useMemo, useEffect } from "react";
|
||||
import useSWR from "swr";
|
||||
import { Select } from "@/refresh-components/cards";
|
||||
import ProviderCard from "@/sections/cards/ProviderCard";
|
||||
import { useCreateModal } from "@/refresh-components/contexts/ModalContext";
|
||||
import { toast } from "@/hooks/useToast";
|
||||
import { Section } from "@/layouts/general-layouts";
|
||||
@@ -228,11 +228,11 @@ export default function ImageGenerationContent() {
|
||||
</Text>
|
||||
<div className="flex flex-col gap-2">
|
||||
{group.providers.map((provider) => (
|
||||
<Select
|
||||
<ProviderCard
|
||||
key={provider.image_provider_id}
|
||||
aria-label={`image-gen-provider-${provider.image_provider_id}`}
|
||||
icon={() => (
|
||||
<ProviderIcon provider={provider.provider_name} size={18} />
|
||||
<ProviderIcon provider={provider.provider_name} size={16} />
|
||||
)}
|
||||
title={provider.title}
|
||||
description={provider.description}
|
||||
|
||||
@@ -4,7 +4,7 @@ import Image from "next/image";
|
||||
import { useEffect, useMemo, useState, useReducer } from "react";
|
||||
import { InfoIcon } from "@/components/icons/icons";
|
||||
import Text from "@/refresh-components/texts/Text";
|
||||
import { Select } from "@/refresh-components/cards";
|
||||
import ProviderCard from "@/sections/cards/ProviderCard";
|
||||
import { Section } from "@/layouts/general-layouts";
|
||||
import * as SettingsLayouts from "@/layouts/settings-layouts";
|
||||
import { Content } from "@opal/layouts";
|
||||
@@ -1091,7 +1091,7 @@ export default function Page() {
|
||||
: "connected";
|
||||
|
||||
return (
|
||||
<Select
|
||||
<ProviderCard
|
||||
key={`${key}-${providerType}`}
|
||||
icon={() =>
|
||||
logoSrc ? (
|
||||
@@ -1207,7 +1207,7 @@ export default function Page() {
|
||||
CONTENT_PROVIDER_DETAILS[provider.provider_type]?.logoSrc;
|
||||
|
||||
return (
|
||||
<Select
|
||||
<ProviderCard
|
||||
key={`${provider.provider_type}-${provider.id}`}
|
||||
icon={() =>
|
||||
contentLogoSrc ? (
|
||||
|
||||
@@ -229,7 +229,7 @@ export const RenderField: FC<RenderFieldProps> = ({
|
||||
name={field.name}
|
||||
title={label}
|
||||
description={description}
|
||||
optional={field.optional}
|
||||
suffix={field.optional ? "optional" : undefined}
|
||||
>
|
||||
<InputTextAreaField
|
||||
name={field.name}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user