forked from github/onyx
Compare commits
8 Commits
main
...
syncing_st
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
12cbc7642f | ||
|
|
eec25fb51d | ||
|
|
b1af9f7f24 | ||
|
|
96b1b66e09 | ||
|
|
f16ab9b419 | ||
|
|
84f5e69826 | ||
|
|
b493985b30 | ||
|
|
2db0017a95 |
@@ -1475,7 +1475,6 @@ class DocumentRetrievalFeedback(Base):
|
||||
feedback: Mapped[SearchFeedbackType | None] = mapped_column(
|
||||
Enum(SearchFeedbackType, native_enum=False), nullable=True
|
||||
)
|
||||
|
||||
chat_message: Mapped[ChatMessage] = relationship(
|
||||
"ChatMessage",
|
||||
back_populates="document_feedbacks",
|
||||
|
||||
@@ -13,6 +13,37 @@ from onyx.setup import setup_logger
|
||||
logger = setup_logger()
|
||||
|
||||
|
||||
def fetch_paginated_sync_records(
|
||||
db_session: Session,
|
||||
entity_id: int,
|
||||
sync_type: SyncType,
|
||||
page_num: int,
|
||||
page_size: int,
|
||||
) -> tuple[list[SyncRecord], int]:
|
||||
total_count = (
|
||||
db_session.query(SyncRecord)
|
||||
.filter(
|
||||
SyncRecord.entity_id == entity_id,
|
||||
SyncRecord.sync_type == sync_type,
|
||||
)
|
||||
.count()
|
||||
)
|
||||
|
||||
sync_records = (
|
||||
db_session.query(SyncRecord)
|
||||
.filter(
|
||||
SyncRecord.entity_id == entity_id,
|
||||
SyncRecord.sync_type == sync_type,
|
||||
)
|
||||
.order_by(SyncRecord.sync_start_time.desc())
|
||||
.offset(page_num * page_size)
|
||||
.limit(page_size)
|
||||
.all()
|
||||
)
|
||||
|
||||
return sync_records, total_count
|
||||
|
||||
|
||||
def insert_sync_record(
|
||||
db_session: Session,
|
||||
entity_id: int,
|
||||
|
||||
@@ -9,6 +9,7 @@ from fastapi.responses import JSONResponse
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from ee.onyx.db.user_group import fetch_user_group
|
||||
from onyx.auth.users import current_curator_or_admin_user
|
||||
from onyx.auth.users import current_user
|
||||
from onyx.background.celery.celery_utils import get_deletion_attempt_snapshot
|
||||
@@ -41,6 +42,7 @@ from onyx.db.document import get_documents_for_cc_pair
|
||||
from onyx.db.engine import get_session
|
||||
from onyx.db.enums import AccessType
|
||||
from onyx.db.enums import ConnectorCredentialPairStatus
|
||||
from onyx.db.enums import SyncType
|
||||
from onyx.db.index_attempt import count_index_attempt_errors_for_cc_pair
|
||||
from onyx.db.index_attempt import count_index_attempts_for_connector
|
||||
from onyx.db.index_attempt import get_index_attempt_errors_for_cc_pair
|
||||
@@ -50,6 +52,7 @@ from onyx.db.models import SearchSettings
|
||||
from onyx.db.models import User
|
||||
from onyx.db.search_settings import get_active_search_settings_list
|
||||
from onyx.db.search_settings import get_current_search_settings
|
||||
from onyx.db.sync_record import fetch_paginated_sync_records
|
||||
from onyx.redis.redis_connector import RedisConnector
|
||||
from onyx.redis.redis_pool import get_redis_client
|
||||
from onyx.server.documents.models import CCPairFullInfo
|
||||
@@ -60,6 +63,7 @@ from onyx.server.documents.models import ConnectorCredentialPairMetadata
|
||||
from onyx.server.documents.models import DocumentSyncStatus
|
||||
from onyx.server.documents.models import IndexAttemptSnapshot
|
||||
from onyx.server.documents.models import PaginatedReturn
|
||||
from onyx.server.documents.models import SyncRecordSnapshot
|
||||
from onyx.server.models import StatusResponse
|
||||
from onyx.utils.logger import setup_logger
|
||||
from onyx.utils.variable_functionality import fetch_ee_implementation_or_noop
|
||||
@@ -69,6 +73,85 @@ logger = setup_logger()
|
||||
router = APIRouter(prefix="/manage")
|
||||
|
||||
|
||||
@router.get("/admin/user-group/{group_id}/sync-status")
|
||||
def get_user_group_sync_status(
|
||||
group_id: int,
|
||||
page_num: int = Query(0, ge=0),
|
||||
page_size: int = Query(10, ge=1, le=1000),
|
||||
user: User | None = Depends(current_curator_or_admin_user),
|
||||
db_session: Session = Depends(get_session),
|
||||
) -> PaginatedReturn[SyncRecordSnapshot]:
|
||||
user_group = fetch_user_group(db_session, group_id)
|
||||
if not user_group:
|
||||
raise HTTPException(status_code=404, detail="User group not found")
|
||||
|
||||
sync_records, total_count = fetch_paginated_sync_records(
|
||||
db_session, group_id, SyncType.USER_GROUP, page_num, page_size
|
||||
)
|
||||
|
||||
return PaginatedReturn(
|
||||
items=[
|
||||
SyncRecordSnapshot.from_sync_record_db_model(sync_record)
|
||||
for sync_record in sync_records
|
||||
],
|
||||
total_items=total_count,
|
||||
)
|
||||
|
||||
|
||||
@router.post("/admin/user-group/{group_id}/sync")
|
||||
def sync_user_group(
|
||||
group_id: int,
|
||||
user: User | None = Depends(current_curator_or_admin_user),
|
||||
db_session: Session = Depends(get_session),
|
||||
) -> StatusResponse[None]:
|
||||
"""Triggers sync of a user group immediately"""
|
||||
get_current_tenant_id()
|
||||
|
||||
user_group = fetch_user_group(db_session, group_id)
|
||||
if not user_group:
|
||||
raise HTTPException(status_code=404, detail="User group not found")
|
||||
|
||||
# Add logic to actually trigger the sync - this would depend on your implementation
|
||||
# For example:
|
||||
# try_creating_usergroup_sync_task(primary_app, group_id, get_redis_client(), tenant_id)
|
||||
|
||||
logger.info(f"User group sync queued: group_id={group_id}")
|
||||
|
||||
return StatusResponse(
|
||||
success=True,
|
||||
message="Successfully created the user group sync task.",
|
||||
)
|
||||
|
||||
|
||||
@router.get("/admin/cc-pair/{cc_pair_id}/sync-status")
|
||||
def get_cc_pair_sync_status(
|
||||
cc_pair_id: int,
|
||||
page_num: int = Query(0, ge=0),
|
||||
page_size: int = Query(10, ge=1, le=1000),
|
||||
user: User | None = Depends(current_curator_or_admin_user),
|
||||
db_session: Session = Depends(get_session),
|
||||
) -> PaginatedReturn[SyncRecordSnapshot]:
|
||||
cc_pair = get_connector_credential_pair_from_id_for_user(
|
||||
cc_pair_id, db_session, user, get_editable=False
|
||||
)
|
||||
if not cc_pair:
|
||||
raise HTTPException(
|
||||
status_code=400, detail="CC Pair not found for current user permissions"
|
||||
)
|
||||
|
||||
sync_records, total_count = fetch_paginated_sync_records(
|
||||
db_session, cc_pair_id, SyncType.EXTERNAL_PERMISSIONS, page_num, page_size
|
||||
)
|
||||
|
||||
return PaginatedReturn(
|
||||
items=[
|
||||
SyncRecordSnapshot.from_sync_record_db_model(sync_record)
|
||||
for sync_record in sync_records
|
||||
],
|
||||
total_items=total_count,
|
||||
)
|
||||
|
||||
|
||||
@router.get("/admin/cc-pair/{cc_pair_id}/index-attempts")
|
||||
def get_cc_pair_index_attempts(
|
||||
cc_pair_id: int,
|
||||
|
||||
@@ -14,12 +14,15 @@ from onyx.configs.constants import DocumentSource
|
||||
from onyx.connectors.models import InputType
|
||||
from onyx.db.enums import AccessType
|
||||
from onyx.db.enums import ConnectorCredentialPairStatus
|
||||
from onyx.db.enums import SyncStatus
|
||||
from onyx.db.enums import SyncType
|
||||
from onyx.db.models import Connector
|
||||
from onyx.db.models import ConnectorCredentialPair
|
||||
from onyx.db.models import Credential
|
||||
from onyx.db.models import Document as DbDocument
|
||||
from onyx.db.models import IndexAttempt
|
||||
from onyx.db.models import IndexingStatus
|
||||
from onyx.db.models import SyncRecord
|
||||
from onyx.db.models import TaskStatus
|
||||
from onyx.server.models import FullUserSnapshot
|
||||
from onyx.server.models import InvitedUserSnapshot
|
||||
@@ -67,6 +70,26 @@ class ConnectorBase(BaseModel):
|
||||
indexing_start: datetime | None = None
|
||||
|
||||
|
||||
class SyncRecordSnapshot(BaseModel):
|
||||
id: int
|
||||
entity_id: int
|
||||
sync_type: SyncType
|
||||
created_at: datetime
|
||||
num_docs_synced: int
|
||||
sync_status: SyncStatus
|
||||
|
||||
@classmethod
|
||||
def from_sync_record_db_model(cls, sync_record: SyncRecord) -> "SyncRecordSnapshot":
|
||||
return cls(
|
||||
id=sync_record.id,
|
||||
entity_id=sync_record.entity_id,
|
||||
sync_type=sync_record.sync_type,
|
||||
created_at=sync_record.sync_start_time,
|
||||
num_docs_synced=sync_record.num_docs_synced,
|
||||
sync_status=sync_record.sync_status,
|
||||
)
|
||||
|
||||
|
||||
class ConnectorUpdateRequest(ConnectorBase):
|
||||
access_type: AccessType
|
||||
groups: list[int] = Field(default_factory=list)
|
||||
@@ -194,6 +217,7 @@ PaginatedType = TypeVar(
|
||||
InvitedUserSnapshot,
|
||||
ChatSessionMinimal,
|
||||
IndexAttemptErrorPydantic,
|
||||
SyncRecordSnapshot,
|
||||
)
|
||||
|
||||
|
||||
|
||||
12551
web/package-lock.json
generated
12551
web/package-lock.json
generated
File diff suppressed because it is too large
Load Diff
@@ -93,11 +93,12 @@
|
||||
},
|
||||
"devDependencies": {
|
||||
"@chromatic-com/playwright": "^0.10.2",
|
||||
"@playwright/test": "^1.39.0",
|
||||
"@tailwindcss/typography": "^0.5.10",
|
||||
"@types/chrome": "^0.0.287",
|
||||
"@types/jest": "^29.5.14",
|
||||
"chromatic": "^11.25.2",
|
||||
"eslint": "^8.48.0",
|
||||
"eslint": "^8.57.1",
|
||||
"eslint-config-next": "^14.1.0",
|
||||
"jest": "^29.7.0",
|
||||
"prettier": "2.8.8",
|
||||
|
||||
@@ -13,7 +13,7 @@ import Text from "@/components/ui/text";
|
||||
import { Callout } from "@/components/ui/callout";
|
||||
import { CCPairFullInfo } from "./types";
|
||||
import { IndexAttemptSnapshot } from "@/lib/types";
|
||||
import { IndexAttemptStatus } from "@/components/Status";
|
||||
import { AttemptStatus } from "@/components/Status";
|
||||
import { PageSelector } from "@/components/PageSelector";
|
||||
import { ThreeDotsLoader } from "@/components/Loading";
|
||||
import { buildCCPairInfoUrl } from "./lib";
|
||||
@@ -118,7 +118,7 @@ export function IndexingAttemptsTable({
|
||||
: "-"}
|
||||
</TableCell>
|
||||
<TableCell>
|
||||
<IndexAttemptStatus
|
||||
<AttemptStatus
|
||||
status={indexAttempt.status || "not_started"}
|
||||
/>
|
||||
{docsPerMinute ? (
|
||||
|
||||
148
web/src/app/admin/connector/[ccPairId]/SyncStatusTable.tsx
Normal file
148
web/src/app/admin/connector/[ccPairId]/SyncStatusTable.tsx
Normal file
@@ -0,0 +1,148 @@
|
||||
import {
|
||||
Table,
|
||||
TableBody,
|
||||
TableCell,
|
||||
TableHead,
|
||||
TableHeader,
|
||||
TableRow,
|
||||
} from "@/components/ui/table";
|
||||
import { CCPairFullInfo, SyncRecord } from "./types";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import {
|
||||
ChevronDown,
|
||||
ChevronLeft,
|
||||
ChevronRight,
|
||||
ChevronUp,
|
||||
CheckCircle,
|
||||
} from "lucide-react";
|
||||
import { useState } from "react";
|
||||
import { AttemptStatus } from "@/components/Status";
|
||||
import { PageSelector } from "@/components/PageSelector";
|
||||
|
||||
// Helper function to format date
|
||||
const formatDateTime = (date: Date): string => {
|
||||
return new Intl.DateTimeFormat("en-US", {
|
||||
year: "numeric",
|
||||
month: "short",
|
||||
day: "numeric",
|
||||
hour: "2-digit",
|
||||
minute: "2-digit",
|
||||
second: "2-digit",
|
||||
}).format(date);
|
||||
};
|
||||
|
||||
interface SyncStatus {
|
||||
status: string;
|
||||
}
|
||||
|
||||
interface SyncStatusTableProps {
|
||||
ccPair: CCPairFullInfo;
|
||||
syncRecords: SyncRecord[];
|
||||
currentPage: number;
|
||||
totalPages: number;
|
||||
onPageChange: (page: number) => void;
|
||||
}
|
||||
|
||||
export function SyncStatusTable({
|
||||
ccPair,
|
||||
syncRecords,
|
||||
currentPage,
|
||||
totalPages,
|
||||
onPageChange,
|
||||
}: SyncStatusTableProps) {
|
||||
const hasPagination = totalPages > 1;
|
||||
const [isExpanded, setIsExpanded] = useState(false);
|
||||
|
||||
// Filter to only show external permissions
|
||||
const filteredRecords =
|
||||
syncRecords?.filter(
|
||||
(record) => record.sync_type === "external_permissions"
|
||||
) || [];
|
||||
|
||||
// Check if we have any records to show
|
||||
if (filteredRecords.length === 0 && currentPage === 0) {
|
||||
return (
|
||||
<div className="text-sm text-gray-500 italic my-2">
|
||||
No permissions sync records found
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
// The total documents synced is the total number of documents in the cc_pair
|
||||
const totalDocsSynced = ccPair.num_docs_indexed || 0;
|
||||
|
||||
return (
|
||||
<div className="mb-6">
|
||||
<div className="flex items-center mb-2">
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="sm"
|
||||
onClick={() => setIsExpanded(!isExpanded)}
|
||||
className="mr-2 text-muted-foreground"
|
||||
>
|
||||
{isExpanded ? (
|
||||
<ChevronUp className="h-4 w-4" />
|
||||
) : (
|
||||
<ChevronDown className="h-4 w-4" />
|
||||
)}
|
||||
{isExpanded ? "Hide details" : "Show sync history"}
|
||||
</Button>
|
||||
<div className="text-sm">
|
||||
<span className="flex items-center text-green-600 dark:text-green-400">
|
||||
<CheckCircle className="h-4 w-4 mr-1 inline" />
|
||||
Permissions sync enabled • {totalDocsSynced} documents synced
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{isExpanded && (
|
||||
<>
|
||||
<Table>
|
||||
<TableHeader>
|
||||
<TableRow>
|
||||
<TableHead>Time</TableHead>
|
||||
<TableHead>Status</TableHead>
|
||||
<TableHead></TableHead>
|
||||
<TableHead>Docs Processed</TableHead>
|
||||
<TableHead>Error</TableHead>
|
||||
</TableRow>
|
||||
</TableHeader>
|
||||
<TableBody>
|
||||
{filteredRecords.length === 0 ? (
|
||||
<TableRow>
|
||||
<TableCell colSpan={3} className="text-center py-4">
|
||||
No permission sync records found
|
||||
</TableCell>
|
||||
</TableRow>
|
||||
) : (
|
||||
filteredRecords.map((record, index) => (
|
||||
<TableRow key={record.id}>
|
||||
<TableCell>
|
||||
{formatDateTime(new Date(record.created_at))}
|
||||
</TableCell>
|
||||
<TableCell>
|
||||
<AttemptStatus status={record.sync_status as any} />
|
||||
</TableCell>
|
||||
<TableCell></TableCell>
|
||||
|
||||
<TableCell>{record.num_docs_synced}</TableCell>
|
||||
</TableRow>
|
||||
))
|
||||
)}
|
||||
</TableBody>
|
||||
</Table>
|
||||
|
||||
{hasPagination && (
|
||||
<div className="flex justify-end mt-4">
|
||||
<PageSelector
|
||||
currentPage={currentPage}
|
||||
totalPages={totalPages}
|
||||
onPageChange={onPageChange}
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -23,6 +23,7 @@ import { AdvancedConfigDisplay, ConfigDisplay } from "./ConfigDisplay";
|
||||
import { DeletionButton } from "./DeletionButton";
|
||||
import DeletionErrorStatus from "./DeletionErrorStatus";
|
||||
import { IndexingAttemptsTable } from "./IndexingAttemptsTable";
|
||||
import { SyncStatusTable } from "./SyncStatusTable";
|
||||
import { ModifyStatusButtonCluster } from "./ModifyStatusButtonCluster";
|
||||
import { ReIndexButton } from "./ReIndexButton";
|
||||
import { buildCCPairInfoUrl, triggerIndexing } from "./lib";
|
||||
@@ -32,6 +33,7 @@ import {
|
||||
ConnectorCredentialPairStatus,
|
||||
IndexAttemptError,
|
||||
PaginatedIndexAttemptErrors,
|
||||
SyncRecord,
|
||||
} from "./types";
|
||||
import { EditableStringFieldDisplay } from "@/components/EditableStringFieldDisplay";
|
||||
import { Button } from "@/components/ui/button";
|
||||
@@ -90,13 +92,25 @@ function Main({ ccPairId }: { ccPairId: number }) {
|
||||
endpoint: `${buildCCPairInfoUrl(ccPairId)}/index-attempts`,
|
||||
});
|
||||
|
||||
const {
|
||||
currentPageData: syncStatus,
|
||||
isLoading: isLoadingSyncStatus,
|
||||
currentPage: syncStatusCurrentPage,
|
||||
totalPages: syncStatusTotalPages,
|
||||
goToPage: goToSyncStatusPage,
|
||||
} = usePaginatedFetch<SyncRecord>({
|
||||
itemsPerPage: ITEMS_PER_PAGE,
|
||||
pagesPerBatch: PAGES_PER_BATCH,
|
||||
endpoint: `${buildCCPairInfoUrl(ccPairId)}/sync-status`,
|
||||
});
|
||||
|
||||
const {
|
||||
currentPageData: indexAttemptErrorsPage,
|
||||
currentPage: errorsCurrentPage,
|
||||
totalPages: errorsTotalPages,
|
||||
goToPage: goToErrorsPage,
|
||||
} = usePaginatedFetch<IndexAttemptError>({
|
||||
itemsPerPage: 10,
|
||||
itemsPerPage: 300,
|
||||
pagesPerBatch: 1,
|
||||
endpoint: `/api/manage/admin/cc-pair/${ccPairId}/errors`,
|
||||
});
|
||||
@@ -450,6 +464,52 @@ function Main({ ccPairId }: { ccPairId: number }) {
|
||||
/>
|
||||
)}
|
||||
|
||||
<div className="mt-6">
|
||||
<div className="flex justify-between items-center">
|
||||
<Title>Permissions Sync</Title>
|
||||
{ccPair.is_editable_for_current_user && (
|
||||
<Button
|
||||
variant="outline"
|
||||
size="sm"
|
||||
onClick={async () => {
|
||||
try {
|
||||
const response = await fetch(
|
||||
`/api/manage/admin/cc-pair/${ccPair.id}/sync-permissions`,
|
||||
{
|
||||
method: "POST",
|
||||
}
|
||||
);
|
||||
if (!response.ok) {
|
||||
throw new Error(await response.text());
|
||||
}
|
||||
setPopup({
|
||||
message: "Permissions sync started successfully",
|
||||
type: "success",
|
||||
});
|
||||
mutate(buildCCPairInfoUrl(ccPairId));
|
||||
} catch (error) {
|
||||
setPopup({
|
||||
message: "Failed to start permissions sync",
|
||||
type: "error",
|
||||
});
|
||||
}
|
||||
}}
|
||||
disabled={ccPair.status !== ConnectorCredentialPairStatus.ACTIVE}
|
||||
>
|
||||
Sync Now
|
||||
</Button>
|
||||
)}
|
||||
</div>
|
||||
{syncStatus && (
|
||||
<SyncStatusTable
|
||||
ccPair={ccPair}
|
||||
syncRecords={syncStatus}
|
||||
currentPage={syncStatusCurrentPage}
|
||||
totalPages={syncStatusTotalPages}
|
||||
onPageChange={goToSyncStatusPage}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
<div className="mt-6">
|
||||
<div className="flex">
|
||||
<Title>Indexing Attempts</Title>
|
||||
|
||||
@@ -74,3 +74,11 @@ export interface PaginatedIndexAttemptErrors {
|
||||
items: IndexAttemptError[];
|
||||
total_items: number;
|
||||
}
|
||||
export interface SyncRecord {
|
||||
id: number;
|
||||
entity_id: number;
|
||||
sync_type: string;
|
||||
created_at: string;
|
||||
num_docs_synced: number;
|
||||
sync_status: ValidStatuses;
|
||||
}
|
||||
|
||||
@@ -9,7 +9,7 @@ import {
|
||||
} from "@/components/ui/table";
|
||||
import { Badge } from "@/components/ui/badge";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { IndexAttemptStatus } from "@/components/Status";
|
||||
import { AttemptStatus } from "@/components/Status";
|
||||
import { timeAgo } from "@/lib/time";
|
||||
import {
|
||||
ConnectorIndexingStatus,
|
||||
@@ -244,7 +244,7 @@ border border-border dark:border-neutral-700
|
||||
)}
|
||||
<TableCell>{ccPairsIndexingStatus.docs_indexed}</TableCell>
|
||||
<TableCell>
|
||||
<IndexAttemptStatus
|
||||
<AttemptStatus
|
||||
status={ccPairsIndexingStatus.last_finished_status || null}
|
||||
errorMsg={ccPairsIndexingStatus?.latest_index_attempt?.error_msg}
|
||||
/>
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { Modal } from "@/components/Modal";
|
||||
import { updateUserGroup } from "./lib";
|
||||
import { PopupSpec } from "@/components/admin/connectors/Popup";
|
||||
import { User, UserGroup } from "@/lib/types";
|
||||
import { CCPairDescriptor, User, UserGroup } from "@/lib/types";
|
||||
import { UserEditor } from "../UserEditor";
|
||||
import { useState } from "react";
|
||||
|
||||
@@ -42,10 +42,14 @@ export const AddMemberForm: React.FC<AddMemberFormProps> = ({
|
||||
)
|
||||
),
|
||||
];
|
||||
|
||||
const response = await updateUserGroup(userGroup.id, {
|
||||
user_ids: newUserIds,
|
||||
cc_pair_ids: userGroup.cc_pairs.map((ccPair) => ccPair.id),
|
||||
cc_pair_ids: userGroup.cc_pairs.map(
|
||||
(ccPair: CCPairDescriptor<number, number>) => ccPair.id
|
||||
),
|
||||
});
|
||||
|
||||
if (response.ok) {
|
||||
setPopup({
|
||||
message: "Successfully added users to group",
|
||||
|
||||
@@ -46,6 +46,10 @@ import { AddTokenRateLimitForm } from "./AddTokenRateLimitForm";
|
||||
import { GenericTokenRateLimitTable } from "@/app/admin/token-rate-limits/TokenRateLimitTables";
|
||||
import { useUser } from "@/components/user/UserProvider";
|
||||
import { GenericConfirmModal } from "@/components/modals/GenericConfirmModal";
|
||||
import { GroupSyncStatusTable } from "./GroupSyncStatusTable";
|
||||
import usePaginatedFetch from "@/hooks/usePaginatedFetch";
|
||||
import Title from "@/components/ui/title";
|
||||
import { SyncRecord } from "@/app/admin/connector/[ccPairId]/types";
|
||||
|
||||
interface GroupDisplayProps {
|
||||
users: User[];
|
||||
@@ -178,6 +182,18 @@ export const GroupDisplay = ({
|
||||
|
||||
const { isAdmin } = useUser();
|
||||
|
||||
const {
|
||||
currentPageData: syncStatus,
|
||||
isLoading: isLoadingSyncStatus,
|
||||
currentPage: syncStatusCurrentPage,
|
||||
totalPages: syncStatusTotalPages,
|
||||
goToPage: goToSyncStatusPage,
|
||||
} = usePaginatedFetch<SyncRecord>({
|
||||
itemsPerPage: 8,
|
||||
pagesPerBatch: 8,
|
||||
endpoint: `/api/manage/admin/user-group/${userGroup.id}/sync-status`,
|
||||
});
|
||||
|
||||
const handlePopup = (message: string, type: "success" | "error") => {
|
||||
setPopup({ message, type });
|
||||
};
|
||||
@@ -204,6 +220,22 @@ export const GroupDisplay = ({
|
||||
|
||||
<Separator />
|
||||
|
||||
<div className="mt-4 mb-4">
|
||||
<Title>Group Sync Status</Title>
|
||||
{syncStatus && (
|
||||
<GroupSyncStatusTable
|
||||
userGroup={userGroup}
|
||||
syncRecords={syncStatus}
|
||||
currentPage={syncStatusCurrentPage}
|
||||
totalPages={syncStatusTotalPages}
|
||||
onPageChange={goToSyncStatusPage}
|
||||
lastSyncTime={userGroup.last_synced_at}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<Separator />
|
||||
|
||||
<div className="flex w-full">
|
||||
<h2 className="text-xl font-bold">Users</h2>
|
||||
</div>
|
||||
|
||||
145
web/src/app/ee/admin/groups/[groupId]/GroupSyncStatusTable.tsx
Normal file
145
web/src/app/ee/admin/groups/[groupId]/GroupSyncStatusTable.tsx
Normal file
@@ -0,0 +1,145 @@
|
||||
import {
|
||||
Table,
|
||||
TableBody,
|
||||
TableCell,
|
||||
TableHead,
|
||||
TableHeader,
|
||||
TableRow,
|
||||
} from "@/components/ui/table";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { ChevronDown, ChevronUp, CheckCircle, Users } from "lucide-react";
|
||||
import { useState } from "react";
|
||||
import { PageSelector } from "@/components/PageSelector";
|
||||
import { UserGroup } from "@/lib/types";
|
||||
import { SyncRecord } from "@/app/admin/connector/[ccPairId]/types";
|
||||
|
||||
// Helper function to format date
|
||||
const formatDateTime = (date: Date): string => {
|
||||
return new Intl.DateTimeFormat("en-US", {
|
||||
year: "numeric",
|
||||
month: "short",
|
||||
day: "numeric",
|
||||
hour: "2-digit",
|
||||
minute: "2-digit",
|
||||
second: "2-digit",
|
||||
}).format(date);
|
||||
};
|
||||
|
||||
interface GroupSyncStatusTableProps {
|
||||
userGroup: UserGroup;
|
||||
syncRecords: SyncRecord[];
|
||||
currentPage: number;
|
||||
totalPages: number;
|
||||
onPageChange: (page: number) => void;
|
||||
lastSyncTime?: string | null;
|
||||
}
|
||||
|
||||
export function GroupSyncStatusTable({
|
||||
userGroup,
|
||||
syncRecords,
|
||||
currentPage,
|
||||
totalPages,
|
||||
onPageChange,
|
||||
lastSyncTime,
|
||||
}: GroupSyncStatusTableProps) {
|
||||
const hasPagination = totalPages > 1;
|
||||
const [isExpanded, setIsExpanded] = useState(false);
|
||||
|
||||
// Filter to only show user group syncs
|
||||
const filteredRecords =
|
||||
syncRecords?.filter((record) => record.sync_type === "user_group") || [];
|
||||
|
||||
// Check if we have any records to show
|
||||
if (filteredRecords.length === 0 && currentPage === 0) {
|
||||
return (
|
||||
<div className="text-sm text-gray-500 italic my-2">
|
||||
No group sync records found
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
// Estimate the total number of users synced
|
||||
const totalUsersSynced = userGroup.users?.length || 0;
|
||||
|
||||
return (
|
||||
<div className="mb-6">
|
||||
<div className="flex items-center justify-between mb-2">
|
||||
<div className="flex items-center">
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="sm"
|
||||
onClick={() => setIsExpanded(!isExpanded)}
|
||||
className="mr-2 text-muted-foreground"
|
||||
>
|
||||
{isExpanded ? (
|
||||
<ChevronUp className="h-4 w-4" />
|
||||
) : (
|
||||
<ChevronDown className="h-4 w-4" />
|
||||
)}
|
||||
{isExpanded ? "Hide details" : "Show sync history"}
|
||||
</Button>
|
||||
<div className="text-sm">
|
||||
<span className="flex items-center text-green-600 dark:text-green-400">
|
||||
<CheckCircle className="h-4 w-4 mr-1 inline" />
|
||||
Groups synced
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{lastSyncTime && (
|
||||
<div className="text-sm text-muted-foreground mb-2">
|
||||
Last synced: {new Date(lastSyncTime).toLocaleString()}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{isExpanded && (
|
||||
<>
|
||||
<Table>
|
||||
<TableHeader>
|
||||
<TableRow>
|
||||
<TableHead>Time</TableHead>
|
||||
<TableHead>Status</TableHead>
|
||||
<TableHead>Docs Processed</TableHead>
|
||||
<TableHead>Error</TableHead>
|
||||
</TableRow>
|
||||
</TableHeader>
|
||||
<TableBody>
|
||||
{filteredRecords.length === 0 ? (
|
||||
<TableRow>
|
||||
<TableCell colSpan={3} className="text-center py-4">
|
||||
No group sync records found
|
||||
</TableCell>
|
||||
</TableRow>
|
||||
) : (
|
||||
filteredRecords.map((record) => (
|
||||
<TableRow key={record.id}>
|
||||
<TableCell>
|
||||
{formatDateTime(new Date(record.created_at))}
|
||||
</TableCell>
|
||||
<TableCell>
|
||||
<span className="flex items-center text-green-600 dark:text-green-400">
|
||||
<CheckCircle className="h-4 w-4 mr-1" /> Success
|
||||
</span>
|
||||
</TableCell>
|
||||
<TableCell>{record.num_docs_synced}</TableCell>
|
||||
</TableRow>
|
||||
))
|
||||
)}
|
||||
</TableBody>
|
||||
</Table>
|
||||
|
||||
{hasPagination && (
|
||||
<div className="flex justify-end mt-4">
|
||||
<PageSelector
|
||||
currentPage={currentPage}
|
||||
totalPages={totalPages}
|
||||
onPageChange={onPageChange}
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -12,7 +12,7 @@ import {
|
||||
import { HoverPopup } from "./HoverPopup";
|
||||
import { ConnectorCredentialPairStatus } from "@/app/admin/connector/[ccPairId]/types";
|
||||
|
||||
export function IndexAttemptStatus({
|
||||
export function AttemptStatus({
|
||||
status,
|
||||
errorMsg,
|
||||
}: {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { buildCCPairInfoUrl } from "@/app/admin/connector/[ccPairId]/lib";
|
||||
import { PageSelector } from "@/components/PageSelector";
|
||||
import { IndexAttemptStatus } from "@/components/Status";
|
||||
import { AttemptStatus } from "@/components/Status";
|
||||
import { deleteCCPair } from "@/lib/documentDeletion";
|
||||
import { FailedConnectorIndexingStatus } from "@/lib/types";
|
||||
import { Button } from "@/components/ui/button";
|
||||
@@ -76,7 +76,7 @@ export function FailedReIndexAttempts({
|
||||
</Link>
|
||||
</TableCell>
|
||||
<TableCell>
|
||||
<IndexAttemptStatus status="failed" />
|
||||
<AttemptStatus status="failed" />
|
||||
</TableCell>
|
||||
|
||||
<TableCell>
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { PageSelector } from "@/components/PageSelector";
|
||||
import { IndexAttemptStatus } from "@/components/Status";
|
||||
import { AttemptStatus } from "@/components/Status";
|
||||
import { ConnectorIndexingStatus } from "@/lib/types";
|
||||
import {
|
||||
Table,
|
||||
@@ -49,7 +49,7 @@ export function ReindexingProgressTable({
|
||||
</TableCell>
|
||||
<TableCell>
|
||||
{reindexingProgress.latest_index_attempt?.status && (
|
||||
<IndexAttemptStatus
|
||||
<AttemptStatus
|
||||
status={reindexingProgress.latest_index_attempt.status}
|
||||
/>
|
||||
)}
|
||||
|
||||
@@ -27,6 +27,7 @@ interface PaginationConfig {
|
||||
query?: string;
|
||||
filter?: Record<string, string | boolean | number | string[] | Date>;
|
||||
refreshIntervalInMs?: number;
|
||||
zeroIndexed?: boolean; // Flag to indicate if backend uses zero-indexed pages
|
||||
}
|
||||
|
||||
interface PaginatedHookReturnData<T extends PaginatedType> {
|
||||
@@ -46,15 +47,18 @@ function usePaginatedFetch<T extends PaginatedType>({
|
||||
query,
|
||||
filter,
|
||||
refreshIntervalInMs = 5000,
|
||||
zeroIndexed = true, // Default to true for zero-indexed pages
|
||||
}: PaginationConfig): PaginatedHookReturnData<T> {
|
||||
const router = useRouter();
|
||||
const currentPath = usePathname();
|
||||
const searchParams = useSearchParams();
|
||||
|
||||
// State to initialize and hold the current page number
|
||||
const [currentPage, setCurrentPage] = useState(() =>
|
||||
parseInt(searchParams?.get("page") || "1", 10)
|
||||
);
|
||||
const [currentPage, setCurrentPage] = useState(() => {
|
||||
const urlPage = parseInt(searchParams?.get("page") || "1", 10);
|
||||
// For URL display we use 1-indexed, but for internal state we use the appropriate index based on API
|
||||
return zeroIndexed ? urlPage - 1 : urlPage;
|
||||
});
|
||||
const [currentPageData, setCurrentPageData] = useState<T[] | null>(null);
|
||||
const [error, setError] = useState<Error | null>(null);
|
||||
const [isLoading, setIsLoading] = useState<boolean>(false);
|
||||
@@ -73,10 +77,11 @@ function usePaginatedFetch<T extends PaginatedType>({
|
||||
|
||||
// Calculates which batch we're in, and which page within that batch
|
||||
const batchAndPageIndices = useMemo(() => {
|
||||
const batchNum = Math.floor((currentPage - 1) / pagesPerBatch);
|
||||
const batchPageNum = (currentPage - 1) % pagesPerBatch;
|
||||
const pageForCalc = zeroIndexed ? currentPage : currentPage - 1;
|
||||
const batchNum = Math.floor(pageForCalc / pagesPerBatch);
|
||||
const batchPageNum = pageForCalc % pagesPerBatch;
|
||||
return { batchNum, batchPageNum };
|
||||
}, [currentPage, pagesPerBatch]);
|
||||
}, [currentPage, pagesPerBatch, zeroIndexed]);
|
||||
|
||||
// Fetches a batch of data and stores it in the cache
|
||||
const fetchBatchData = useCallback(
|
||||
@@ -88,9 +93,9 @@ function usePaginatedFetch<T extends PaginatedType>({
|
||||
ongoingRequestsRef.current.add(batchNum);
|
||||
|
||||
try {
|
||||
// Build query params
|
||||
// Build query params - use zero-based indexing for backend
|
||||
const params = new URLSearchParams({
|
||||
page_num: batchNum.toString(),
|
||||
page_num: (batchNum * pagesPerBatch).toString(),
|
||||
page_size: (pagesPerBatch * itemsPerPage).toString(),
|
||||
});
|
||||
|
||||
@@ -145,27 +150,31 @@ function usePaginatedFetch<T extends PaginatedType>({
|
||||
[endpoint, pagesPerBatch, itemsPerPage, query, filter]
|
||||
);
|
||||
|
||||
// Updates the URL with the current page number
|
||||
// Updates the URL with the current page number (always 1-indexed for URL)
|
||||
const updatePageUrl = useCallback(
|
||||
(page: number) => {
|
||||
if (currentPath) {
|
||||
const params = new URLSearchParams(searchParams);
|
||||
params.set("page", page.toString());
|
||||
// For URL display we always use 1-indexed
|
||||
const urlPage = zeroIndexed ? page + 1 : page;
|
||||
params.set("page", urlPage.toString());
|
||||
router.replace(`${currentPath}?${params.toString()}`, {
|
||||
scroll: false,
|
||||
});
|
||||
}
|
||||
},
|
||||
[currentPath, router, searchParams]
|
||||
[currentPath, router, searchParams, zeroIndexed]
|
||||
);
|
||||
|
||||
// Updates the current page
|
||||
const goToPage = useCallback(
|
||||
(newPage: number) => {
|
||||
setCurrentPage(newPage);
|
||||
updatePageUrl(newPage);
|
||||
// Ensure page is within bounds
|
||||
const boundedPage = Math.max(0, Math.min(newPage, totalPages - 1));
|
||||
setCurrentPage(boundedPage);
|
||||
updatePageUrl(boundedPage);
|
||||
},
|
||||
[updatePageUrl]
|
||||
[updatePageUrl, totalPages]
|
||||
);
|
||||
|
||||
// Loads the current and adjacent batches
|
||||
@@ -199,7 +208,14 @@ function usePaginatedFetch<T extends PaginatedType>({
|
||||
if (!cachedBatches[0]) {
|
||||
fetchBatchData(0);
|
||||
}
|
||||
}, [currentPage, cachedBatches, totalPages, pagesPerBatch, fetchBatchData]);
|
||||
}, [
|
||||
currentPage,
|
||||
cachedBatches,
|
||||
totalPages,
|
||||
pagesPerBatch,
|
||||
fetchBatchData,
|
||||
batchAndPageIndices,
|
||||
]);
|
||||
|
||||
// Updates current page data from the cache
|
||||
useEffect(() => {
|
||||
@@ -209,7 +225,7 @@ function usePaginatedFetch<T extends PaginatedType>({
|
||||
setCurrentPageData(cachedBatches[batchNum][batchPageNum]);
|
||||
setIsLoading(false);
|
||||
}
|
||||
}, [currentPage, cachedBatches, pagesPerBatch]);
|
||||
}, [currentPage, cachedBatches, pagesPerBatch, batchAndPageIndices]);
|
||||
|
||||
// Implements periodic refresh
|
||||
useEffect(() => {
|
||||
@@ -221,21 +237,28 @@ function usePaginatedFetch<T extends PaginatedType>({
|
||||
}, refreshIntervalInMs);
|
||||
|
||||
return () => clearInterval(interval);
|
||||
}, [currentPage, pagesPerBatch, refreshIntervalInMs, fetchBatchData]);
|
||||
}, [
|
||||
currentPage,
|
||||
pagesPerBatch,
|
||||
refreshIntervalInMs,
|
||||
fetchBatchData,
|
||||
batchAndPageIndices,
|
||||
]);
|
||||
|
||||
// Manually refreshes the current batch
|
||||
const refresh = useCallback(async () => {
|
||||
const { batchNum } = batchAndPageIndices;
|
||||
await fetchBatchData(batchNum);
|
||||
}, [currentPage, pagesPerBatch, fetchBatchData]);
|
||||
}, [batchAndPageIndices, fetchBatchData]);
|
||||
|
||||
// Cache invalidation
|
||||
useEffect(() => {
|
||||
setCachedBatches({});
|
||||
setTotalItems(0);
|
||||
goToPage(1);
|
||||
// Start at page 0 for zero-indexed APIs, page 1 for one-indexed
|
||||
goToPage(zeroIndexed ? 0 : 1);
|
||||
setError(null);
|
||||
}, [currentPath, query, filter]);
|
||||
}, [currentPath, query, filter, zeroIndexed, goToPage]);
|
||||
|
||||
return {
|
||||
currentPage,
|
||||
|
||||
@@ -342,6 +342,7 @@ export interface UserGroup {
|
||||
personas: Persona[];
|
||||
is_up_to_date: boolean;
|
||||
is_up_for_deletion: boolean;
|
||||
last_synced_at: string | null;
|
||||
}
|
||||
|
||||
export enum ValidSources {
|
||||
|
||||
Reference in New Issue
Block a user