1
0
forked from github/onyx

Compare commits

...

8 Commits

Author SHA1 Message Date
pablonyx
12cbc7642f update 2025-04-04 11:03:03 -07:00
pablonyx
eec25fb51d fix build 2025-03-27 11:47:22 -07:00
pablonyx
b1af9f7f24 nit 2025-03-26 18:36:43 -07:00
pablonyx
96b1b66e09 nit 2025-03-26 18:24:31 -07:00
pablonyx
f16ab9b419 k 2025-03-26 18:09:12 -07:00
pablonyx
84f5e69826 nit 2025-03-26 17:25:53 -07:00
pablonyx
b493985b30 update 2025-03-26 16:47:06 -07:00
pablonyx
2db0017a95 update 2025-03-26 16:16:42 -07:00
19 changed files with 4899 additions and 8279 deletions

View File

@@ -1475,7 +1475,6 @@ class DocumentRetrievalFeedback(Base):
feedback: Mapped[SearchFeedbackType | None] = mapped_column(
Enum(SearchFeedbackType, native_enum=False), nullable=True
)
chat_message: Mapped[ChatMessage] = relationship(
"ChatMessage",
back_populates="document_feedbacks",

View File

@@ -13,6 +13,37 @@ from onyx.setup import setup_logger
logger = setup_logger()
def fetch_paginated_sync_records(
db_session: Session,
entity_id: int,
sync_type: SyncType,
page_num: int,
page_size: int,
) -> tuple[list[SyncRecord], int]:
total_count = (
db_session.query(SyncRecord)
.filter(
SyncRecord.entity_id == entity_id,
SyncRecord.sync_type == sync_type,
)
.count()
)
sync_records = (
db_session.query(SyncRecord)
.filter(
SyncRecord.entity_id == entity_id,
SyncRecord.sync_type == sync_type,
)
.order_by(SyncRecord.sync_start_time.desc())
.offset(page_num * page_size)
.limit(page_size)
.all()
)
return sync_records, total_count
def insert_sync_record(
db_session: Session,
entity_id: int,

View File

@@ -9,6 +9,7 @@ from fastapi.responses import JSONResponse
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import Session
from ee.onyx.db.user_group import fetch_user_group
from onyx.auth.users import current_curator_or_admin_user
from onyx.auth.users import current_user
from onyx.background.celery.celery_utils import get_deletion_attempt_snapshot
@@ -41,6 +42,7 @@ from onyx.db.document import get_documents_for_cc_pair
from onyx.db.engine import get_session
from onyx.db.enums import AccessType
from onyx.db.enums import ConnectorCredentialPairStatus
from onyx.db.enums import SyncType
from onyx.db.index_attempt import count_index_attempt_errors_for_cc_pair
from onyx.db.index_attempt import count_index_attempts_for_connector
from onyx.db.index_attempt import get_index_attempt_errors_for_cc_pair
@@ -50,6 +52,7 @@ from onyx.db.models import SearchSettings
from onyx.db.models import User
from onyx.db.search_settings import get_active_search_settings_list
from onyx.db.search_settings import get_current_search_settings
from onyx.db.sync_record import fetch_paginated_sync_records
from onyx.redis.redis_connector import RedisConnector
from onyx.redis.redis_pool import get_redis_client
from onyx.server.documents.models import CCPairFullInfo
@@ -60,6 +63,7 @@ from onyx.server.documents.models import ConnectorCredentialPairMetadata
from onyx.server.documents.models import DocumentSyncStatus
from onyx.server.documents.models import IndexAttemptSnapshot
from onyx.server.documents.models import PaginatedReturn
from onyx.server.documents.models import SyncRecordSnapshot
from onyx.server.models import StatusResponse
from onyx.utils.logger import setup_logger
from onyx.utils.variable_functionality import fetch_ee_implementation_or_noop
@@ -69,6 +73,85 @@ logger = setup_logger()
router = APIRouter(prefix="/manage")
@router.get("/admin/user-group/{group_id}/sync-status")
def get_user_group_sync_status(
group_id: int,
page_num: int = Query(0, ge=0),
page_size: int = Query(10, ge=1, le=1000),
user: User | None = Depends(current_curator_or_admin_user),
db_session: Session = Depends(get_session),
) -> PaginatedReturn[SyncRecordSnapshot]:
user_group = fetch_user_group(db_session, group_id)
if not user_group:
raise HTTPException(status_code=404, detail="User group not found")
sync_records, total_count = fetch_paginated_sync_records(
db_session, group_id, SyncType.USER_GROUP, page_num, page_size
)
return PaginatedReturn(
items=[
SyncRecordSnapshot.from_sync_record_db_model(sync_record)
for sync_record in sync_records
],
total_items=total_count,
)
@router.post("/admin/user-group/{group_id}/sync")
def sync_user_group(
group_id: int,
user: User | None = Depends(current_curator_or_admin_user),
db_session: Session = Depends(get_session),
) -> StatusResponse[None]:
"""Triggers sync of a user group immediately"""
get_current_tenant_id()
user_group = fetch_user_group(db_session, group_id)
if not user_group:
raise HTTPException(status_code=404, detail="User group not found")
# Add logic to actually trigger the sync - this would depend on your implementation
# For example:
# try_creating_usergroup_sync_task(primary_app, group_id, get_redis_client(), tenant_id)
logger.info(f"User group sync queued: group_id={group_id}")
return StatusResponse(
success=True,
message="Successfully created the user group sync task.",
)
@router.get("/admin/cc-pair/{cc_pair_id}/sync-status")
def get_cc_pair_sync_status(
cc_pair_id: int,
page_num: int = Query(0, ge=0),
page_size: int = Query(10, ge=1, le=1000),
user: User | None = Depends(current_curator_or_admin_user),
db_session: Session = Depends(get_session),
) -> PaginatedReturn[SyncRecordSnapshot]:
cc_pair = get_connector_credential_pair_from_id_for_user(
cc_pair_id, db_session, user, get_editable=False
)
if not cc_pair:
raise HTTPException(
status_code=400, detail="CC Pair not found for current user permissions"
)
sync_records, total_count = fetch_paginated_sync_records(
db_session, cc_pair_id, SyncType.EXTERNAL_PERMISSIONS, page_num, page_size
)
return PaginatedReturn(
items=[
SyncRecordSnapshot.from_sync_record_db_model(sync_record)
for sync_record in sync_records
],
total_items=total_count,
)
@router.get("/admin/cc-pair/{cc_pair_id}/index-attempts")
def get_cc_pair_index_attempts(
cc_pair_id: int,

View File

@@ -14,12 +14,15 @@ from onyx.configs.constants import DocumentSource
from onyx.connectors.models import InputType
from onyx.db.enums import AccessType
from onyx.db.enums import ConnectorCredentialPairStatus
from onyx.db.enums import SyncStatus
from onyx.db.enums import SyncType
from onyx.db.models import Connector
from onyx.db.models import ConnectorCredentialPair
from onyx.db.models import Credential
from onyx.db.models import Document as DbDocument
from onyx.db.models import IndexAttempt
from onyx.db.models import IndexingStatus
from onyx.db.models import SyncRecord
from onyx.db.models import TaskStatus
from onyx.server.models import FullUserSnapshot
from onyx.server.models import InvitedUserSnapshot
@@ -67,6 +70,26 @@ class ConnectorBase(BaseModel):
indexing_start: datetime | None = None
class SyncRecordSnapshot(BaseModel):
id: int
entity_id: int
sync_type: SyncType
created_at: datetime
num_docs_synced: int
sync_status: SyncStatus
@classmethod
def from_sync_record_db_model(cls, sync_record: SyncRecord) -> "SyncRecordSnapshot":
return cls(
id=sync_record.id,
entity_id=sync_record.entity_id,
sync_type=sync_record.sync_type,
created_at=sync_record.sync_start_time,
num_docs_synced=sync_record.num_docs_synced,
sync_status=sync_record.sync_status,
)
class ConnectorUpdateRequest(ConnectorBase):
access_type: AccessType
groups: list[int] = Field(default_factory=list)
@@ -194,6 +217,7 @@ PaginatedType = TypeVar(
InvitedUserSnapshot,
ChatSessionMinimal,
IndexAttemptErrorPydantic,
SyncRecordSnapshot,
)

12551
web/package-lock.json generated

File diff suppressed because it is too large Load Diff

View File

@@ -93,11 +93,12 @@
},
"devDependencies": {
"@chromatic-com/playwright": "^0.10.2",
"@playwright/test": "^1.39.0",
"@tailwindcss/typography": "^0.5.10",
"@types/chrome": "^0.0.287",
"@types/jest": "^29.5.14",
"chromatic": "^11.25.2",
"eslint": "^8.48.0",
"eslint": "^8.57.1",
"eslint-config-next": "^14.1.0",
"jest": "^29.7.0",
"prettier": "2.8.8",

View File

@@ -13,7 +13,7 @@ import Text from "@/components/ui/text";
import { Callout } from "@/components/ui/callout";
import { CCPairFullInfo } from "./types";
import { IndexAttemptSnapshot } from "@/lib/types";
import { IndexAttemptStatus } from "@/components/Status";
import { AttemptStatus } from "@/components/Status";
import { PageSelector } from "@/components/PageSelector";
import { ThreeDotsLoader } from "@/components/Loading";
import { buildCCPairInfoUrl } from "./lib";
@@ -118,7 +118,7 @@ export function IndexingAttemptsTable({
: "-"}
</TableCell>
<TableCell>
<IndexAttemptStatus
<AttemptStatus
status={indexAttempt.status || "not_started"}
/>
{docsPerMinute ? (

View File

@@ -0,0 +1,148 @@
import {
Table,
TableBody,
TableCell,
TableHead,
TableHeader,
TableRow,
} from "@/components/ui/table";
import { CCPairFullInfo, SyncRecord } from "./types";
import { Button } from "@/components/ui/button";
import {
ChevronDown,
ChevronLeft,
ChevronRight,
ChevronUp,
CheckCircle,
} from "lucide-react";
import { useState } from "react";
import { AttemptStatus } from "@/components/Status";
import { PageSelector } from "@/components/PageSelector";
// Helper function to format date
const formatDateTime = (date: Date): string => {
return new Intl.DateTimeFormat("en-US", {
year: "numeric",
month: "short",
day: "numeric",
hour: "2-digit",
minute: "2-digit",
second: "2-digit",
}).format(date);
};
interface SyncStatus {
status: string;
}
interface SyncStatusTableProps {
ccPair: CCPairFullInfo;
syncRecords: SyncRecord[];
currentPage: number;
totalPages: number;
onPageChange: (page: number) => void;
}
export function SyncStatusTable({
ccPair,
syncRecords,
currentPage,
totalPages,
onPageChange,
}: SyncStatusTableProps) {
const hasPagination = totalPages > 1;
const [isExpanded, setIsExpanded] = useState(false);
// Filter to only show external permissions
const filteredRecords =
syncRecords?.filter(
(record) => record.sync_type === "external_permissions"
) || [];
// Check if we have any records to show
if (filteredRecords.length === 0 && currentPage === 0) {
return (
<div className="text-sm text-gray-500 italic my-2">
No permissions sync records found
</div>
);
}
// The total documents synced is the total number of documents in the cc_pair
const totalDocsSynced = ccPair.num_docs_indexed || 0;
return (
<div className="mb-6">
<div className="flex items-center mb-2">
<Button
variant="ghost"
size="sm"
onClick={() => setIsExpanded(!isExpanded)}
className="mr-2 text-muted-foreground"
>
{isExpanded ? (
<ChevronUp className="h-4 w-4" />
) : (
<ChevronDown className="h-4 w-4" />
)}
{isExpanded ? "Hide details" : "Show sync history"}
</Button>
<div className="text-sm">
<span className="flex items-center text-green-600 dark:text-green-400">
<CheckCircle className="h-4 w-4 mr-1 inline" />
Permissions sync enabled {totalDocsSynced} documents synced
</span>
</div>
</div>
{isExpanded && (
<>
<Table>
<TableHeader>
<TableRow>
<TableHead>Time</TableHead>
<TableHead>Status</TableHead>
<TableHead></TableHead>
<TableHead>Docs Processed</TableHead>
<TableHead>Error</TableHead>
</TableRow>
</TableHeader>
<TableBody>
{filteredRecords.length === 0 ? (
<TableRow>
<TableCell colSpan={3} className="text-center py-4">
No permission sync records found
</TableCell>
</TableRow>
) : (
filteredRecords.map((record, index) => (
<TableRow key={record.id}>
<TableCell>
{formatDateTime(new Date(record.created_at))}
</TableCell>
<TableCell>
<AttemptStatus status={record.sync_status as any} />
</TableCell>
<TableCell></TableCell>
<TableCell>{record.num_docs_synced}</TableCell>
</TableRow>
))
)}
</TableBody>
</Table>
{hasPagination && (
<div className="flex justify-end mt-4">
<PageSelector
currentPage={currentPage}
totalPages={totalPages}
onPageChange={onPageChange}
/>
</div>
)}
</>
)}
</div>
);
}

View File

@@ -23,6 +23,7 @@ import { AdvancedConfigDisplay, ConfigDisplay } from "./ConfigDisplay";
import { DeletionButton } from "./DeletionButton";
import DeletionErrorStatus from "./DeletionErrorStatus";
import { IndexingAttemptsTable } from "./IndexingAttemptsTable";
import { SyncStatusTable } from "./SyncStatusTable";
import { ModifyStatusButtonCluster } from "./ModifyStatusButtonCluster";
import { ReIndexButton } from "./ReIndexButton";
import { buildCCPairInfoUrl, triggerIndexing } from "./lib";
@@ -32,6 +33,7 @@ import {
ConnectorCredentialPairStatus,
IndexAttemptError,
PaginatedIndexAttemptErrors,
SyncRecord,
} from "./types";
import { EditableStringFieldDisplay } from "@/components/EditableStringFieldDisplay";
import { Button } from "@/components/ui/button";
@@ -90,13 +92,25 @@ function Main({ ccPairId }: { ccPairId: number }) {
endpoint: `${buildCCPairInfoUrl(ccPairId)}/index-attempts`,
});
const {
currentPageData: syncStatus,
isLoading: isLoadingSyncStatus,
currentPage: syncStatusCurrentPage,
totalPages: syncStatusTotalPages,
goToPage: goToSyncStatusPage,
} = usePaginatedFetch<SyncRecord>({
itemsPerPage: ITEMS_PER_PAGE,
pagesPerBatch: PAGES_PER_BATCH,
endpoint: `${buildCCPairInfoUrl(ccPairId)}/sync-status`,
});
const {
currentPageData: indexAttemptErrorsPage,
currentPage: errorsCurrentPage,
totalPages: errorsTotalPages,
goToPage: goToErrorsPage,
} = usePaginatedFetch<IndexAttemptError>({
itemsPerPage: 10,
itemsPerPage: 300,
pagesPerBatch: 1,
endpoint: `/api/manage/admin/cc-pair/${ccPairId}/errors`,
});
@@ -450,6 +464,52 @@ function Main({ ccPairId }: { ccPairId: number }) {
/>
)}
<div className="mt-6">
<div className="flex justify-between items-center">
<Title>Permissions Sync</Title>
{ccPair.is_editable_for_current_user && (
<Button
variant="outline"
size="sm"
onClick={async () => {
try {
const response = await fetch(
`/api/manage/admin/cc-pair/${ccPair.id}/sync-permissions`,
{
method: "POST",
}
);
if (!response.ok) {
throw new Error(await response.text());
}
setPopup({
message: "Permissions sync started successfully",
type: "success",
});
mutate(buildCCPairInfoUrl(ccPairId));
} catch (error) {
setPopup({
message: "Failed to start permissions sync",
type: "error",
});
}
}}
disabled={ccPair.status !== ConnectorCredentialPairStatus.ACTIVE}
>
Sync Now
</Button>
)}
</div>
{syncStatus && (
<SyncStatusTable
ccPair={ccPair}
syncRecords={syncStatus}
currentPage={syncStatusCurrentPage}
totalPages={syncStatusTotalPages}
onPageChange={goToSyncStatusPage}
/>
)}
</div>
<div className="mt-6">
<div className="flex">
<Title>Indexing Attempts</Title>

View File

@@ -74,3 +74,11 @@ export interface PaginatedIndexAttemptErrors {
items: IndexAttemptError[];
total_items: number;
}
export interface SyncRecord {
id: number;
entity_id: number;
sync_type: string;
created_at: string;
num_docs_synced: number;
sync_status: ValidStatuses;
}

View File

@@ -9,7 +9,7 @@ import {
} from "@/components/ui/table";
import { Badge } from "@/components/ui/badge";
import { Button } from "@/components/ui/button";
import { IndexAttemptStatus } from "@/components/Status";
import { AttemptStatus } from "@/components/Status";
import { timeAgo } from "@/lib/time";
import {
ConnectorIndexingStatus,
@@ -244,7 +244,7 @@ border border-border dark:border-neutral-700
)}
<TableCell>{ccPairsIndexingStatus.docs_indexed}</TableCell>
<TableCell>
<IndexAttemptStatus
<AttemptStatus
status={ccPairsIndexingStatus.last_finished_status || null}
errorMsg={ccPairsIndexingStatus?.latest_index_attempt?.error_msg}
/>

View File

@@ -1,7 +1,7 @@
import { Modal } from "@/components/Modal";
import { updateUserGroup } from "./lib";
import { PopupSpec } from "@/components/admin/connectors/Popup";
import { User, UserGroup } from "@/lib/types";
import { CCPairDescriptor, User, UserGroup } from "@/lib/types";
import { UserEditor } from "../UserEditor";
import { useState } from "react";
@@ -42,10 +42,14 @@ export const AddMemberForm: React.FC<AddMemberFormProps> = ({
)
),
];
const response = await updateUserGroup(userGroup.id, {
user_ids: newUserIds,
cc_pair_ids: userGroup.cc_pairs.map((ccPair) => ccPair.id),
cc_pair_ids: userGroup.cc_pairs.map(
(ccPair: CCPairDescriptor<number, number>) => ccPair.id
),
});
if (response.ok) {
setPopup({
message: "Successfully added users to group",

View File

@@ -46,6 +46,10 @@ import { AddTokenRateLimitForm } from "./AddTokenRateLimitForm";
import { GenericTokenRateLimitTable } from "@/app/admin/token-rate-limits/TokenRateLimitTables";
import { useUser } from "@/components/user/UserProvider";
import { GenericConfirmModal } from "@/components/modals/GenericConfirmModal";
import { GroupSyncStatusTable } from "./GroupSyncStatusTable";
import usePaginatedFetch from "@/hooks/usePaginatedFetch";
import Title from "@/components/ui/title";
import { SyncRecord } from "@/app/admin/connector/[ccPairId]/types";
interface GroupDisplayProps {
users: User[];
@@ -178,6 +182,18 @@ export const GroupDisplay = ({
const { isAdmin } = useUser();
const {
currentPageData: syncStatus,
isLoading: isLoadingSyncStatus,
currentPage: syncStatusCurrentPage,
totalPages: syncStatusTotalPages,
goToPage: goToSyncStatusPage,
} = usePaginatedFetch<SyncRecord>({
itemsPerPage: 8,
pagesPerBatch: 8,
endpoint: `/api/manage/admin/user-group/${userGroup.id}/sync-status`,
});
const handlePopup = (message: string, type: "success" | "error") => {
setPopup({ message, type });
};
@@ -204,6 +220,22 @@ export const GroupDisplay = ({
<Separator />
<div className="mt-4 mb-4">
<Title>Group Sync Status</Title>
{syncStatus && (
<GroupSyncStatusTable
userGroup={userGroup}
syncRecords={syncStatus}
currentPage={syncStatusCurrentPage}
totalPages={syncStatusTotalPages}
onPageChange={goToSyncStatusPage}
lastSyncTime={userGroup.last_synced_at}
/>
)}
</div>
<Separator />
<div className="flex w-full">
<h2 className="text-xl font-bold">Users</h2>
</div>

View File

@@ -0,0 +1,145 @@
import {
Table,
TableBody,
TableCell,
TableHead,
TableHeader,
TableRow,
} from "@/components/ui/table";
import { Button } from "@/components/ui/button";
import { ChevronDown, ChevronUp, CheckCircle, Users } from "lucide-react";
import { useState } from "react";
import { PageSelector } from "@/components/PageSelector";
import { UserGroup } from "@/lib/types";
import { SyncRecord } from "@/app/admin/connector/[ccPairId]/types";
// Helper function to format date
const formatDateTime = (date: Date): string => {
return new Intl.DateTimeFormat("en-US", {
year: "numeric",
month: "short",
day: "numeric",
hour: "2-digit",
minute: "2-digit",
second: "2-digit",
}).format(date);
};
interface GroupSyncStatusTableProps {
userGroup: UserGroup;
syncRecords: SyncRecord[];
currentPage: number;
totalPages: number;
onPageChange: (page: number) => void;
lastSyncTime?: string | null;
}
export function GroupSyncStatusTable({
userGroup,
syncRecords,
currentPage,
totalPages,
onPageChange,
lastSyncTime,
}: GroupSyncStatusTableProps) {
const hasPagination = totalPages > 1;
const [isExpanded, setIsExpanded] = useState(false);
// Filter to only show user group syncs
const filteredRecords =
syncRecords?.filter((record) => record.sync_type === "user_group") || [];
// Check if we have any records to show
if (filteredRecords.length === 0 && currentPage === 0) {
return (
<div className="text-sm text-gray-500 italic my-2">
No group sync records found
</div>
);
}
// Estimate the total number of users synced
const totalUsersSynced = userGroup.users?.length || 0;
return (
<div className="mb-6">
<div className="flex items-center justify-between mb-2">
<div className="flex items-center">
<Button
variant="ghost"
size="sm"
onClick={() => setIsExpanded(!isExpanded)}
className="mr-2 text-muted-foreground"
>
{isExpanded ? (
<ChevronUp className="h-4 w-4" />
) : (
<ChevronDown className="h-4 w-4" />
)}
{isExpanded ? "Hide details" : "Show sync history"}
</Button>
<div className="text-sm">
<span className="flex items-center text-green-600 dark:text-green-400">
<CheckCircle className="h-4 w-4 mr-1 inline" />
Groups synced
</span>
</div>
</div>
</div>
{lastSyncTime && (
<div className="text-sm text-muted-foreground mb-2">
Last synced: {new Date(lastSyncTime).toLocaleString()}
</div>
)}
{isExpanded && (
<>
<Table>
<TableHeader>
<TableRow>
<TableHead>Time</TableHead>
<TableHead>Status</TableHead>
<TableHead>Docs Processed</TableHead>
<TableHead>Error</TableHead>
</TableRow>
</TableHeader>
<TableBody>
{filteredRecords.length === 0 ? (
<TableRow>
<TableCell colSpan={3} className="text-center py-4">
No group sync records found
</TableCell>
</TableRow>
) : (
filteredRecords.map((record) => (
<TableRow key={record.id}>
<TableCell>
{formatDateTime(new Date(record.created_at))}
</TableCell>
<TableCell>
<span className="flex items-center text-green-600 dark:text-green-400">
<CheckCircle className="h-4 w-4 mr-1" /> Success
</span>
</TableCell>
<TableCell>{record.num_docs_synced}</TableCell>
</TableRow>
))
)}
</TableBody>
</Table>
{hasPagination && (
<div className="flex justify-end mt-4">
<PageSelector
currentPage={currentPage}
totalPages={totalPages}
onPageChange={onPageChange}
/>
</div>
)}
</>
)}
</div>
);
}

View File

@@ -12,7 +12,7 @@ import {
import { HoverPopup } from "./HoverPopup";
import { ConnectorCredentialPairStatus } from "@/app/admin/connector/[ccPairId]/types";
export function IndexAttemptStatus({
export function AttemptStatus({
status,
errorMsg,
}: {

View File

@@ -1,6 +1,6 @@
import { buildCCPairInfoUrl } from "@/app/admin/connector/[ccPairId]/lib";
import { PageSelector } from "@/components/PageSelector";
import { IndexAttemptStatus } from "@/components/Status";
import { AttemptStatus } from "@/components/Status";
import { deleteCCPair } from "@/lib/documentDeletion";
import { FailedConnectorIndexingStatus } from "@/lib/types";
import { Button } from "@/components/ui/button";
@@ -76,7 +76,7 @@ export function FailedReIndexAttempts({
</Link>
</TableCell>
<TableCell>
<IndexAttemptStatus status="failed" />
<AttemptStatus status="failed" />
</TableCell>
<TableCell>

View File

@@ -1,5 +1,5 @@
import { PageSelector } from "@/components/PageSelector";
import { IndexAttemptStatus } from "@/components/Status";
import { AttemptStatus } from "@/components/Status";
import { ConnectorIndexingStatus } from "@/lib/types";
import {
Table,
@@ -49,7 +49,7 @@ export function ReindexingProgressTable({
</TableCell>
<TableCell>
{reindexingProgress.latest_index_attempt?.status && (
<IndexAttemptStatus
<AttemptStatus
status={reindexingProgress.latest_index_attempt.status}
/>
)}

View File

@@ -27,6 +27,7 @@ interface PaginationConfig {
query?: string;
filter?: Record<string, string | boolean | number | string[] | Date>;
refreshIntervalInMs?: number;
zeroIndexed?: boolean; // Flag to indicate if backend uses zero-indexed pages
}
interface PaginatedHookReturnData<T extends PaginatedType> {
@@ -46,15 +47,18 @@ function usePaginatedFetch<T extends PaginatedType>({
query,
filter,
refreshIntervalInMs = 5000,
zeroIndexed = true, // Default to true for zero-indexed pages
}: PaginationConfig): PaginatedHookReturnData<T> {
const router = useRouter();
const currentPath = usePathname();
const searchParams = useSearchParams();
// State to initialize and hold the current page number
const [currentPage, setCurrentPage] = useState(() =>
parseInt(searchParams?.get("page") || "1", 10)
);
const [currentPage, setCurrentPage] = useState(() => {
const urlPage = parseInt(searchParams?.get("page") || "1", 10);
// For URL display we use 1-indexed, but for internal state we use the appropriate index based on API
return zeroIndexed ? urlPage - 1 : urlPage;
});
const [currentPageData, setCurrentPageData] = useState<T[] | null>(null);
const [error, setError] = useState<Error | null>(null);
const [isLoading, setIsLoading] = useState<boolean>(false);
@@ -73,10 +77,11 @@ function usePaginatedFetch<T extends PaginatedType>({
// Calculates which batch we're in, and which page within that batch
const batchAndPageIndices = useMemo(() => {
const batchNum = Math.floor((currentPage - 1) / pagesPerBatch);
const batchPageNum = (currentPage - 1) % pagesPerBatch;
const pageForCalc = zeroIndexed ? currentPage : currentPage - 1;
const batchNum = Math.floor(pageForCalc / pagesPerBatch);
const batchPageNum = pageForCalc % pagesPerBatch;
return { batchNum, batchPageNum };
}, [currentPage, pagesPerBatch]);
}, [currentPage, pagesPerBatch, zeroIndexed]);
// Fetches a batch of data and stores it in the cache
const fetchBatchData = useCallback(
@@ -88,9 +93,9 @@ function usePaginatedFetch<T extends PaginatedType>({
ongoingRequestsRef.current.add(batchNum);
try {
// Build query params
// Build query params - use zero-based indexing for backend
const params = new URLSearchParams({
page_num: batchNum.toString(),
page_num: (batchNum * pagesPerBatch).toString(),
page_size: (pagesPerBatch * itemsPerPage).toString(),
});
@@ -145,27 +150,31 @@ function usePaginatedFetch<T extends PaginatedType>({
[endpoint, pagesPerBatch, itemsPerPage, query, filter]
);
// Updates the URL with the current page number
// Updates the URL with the current page number (always 1-indexed for URL)
const updatePageUrl = useCallback(
(page: number) => {
if (currentPath) {
const params = new URLSearchParams(searchParams);
params.set("page", page.toString());
// For URL display we always use 1-indexed
const urlPage = zeroIndexed ? page + 1 : page;
params.set("page", urlPage.toString());
router.replace(`${currentPath}?${params.toString()}`, {
scroll: false,
});
}
},
[currentPath, router, searchParams]
[currentPath, router, searchParams, zeroIndexed]
);
// Updates the current page
const goToPage = useCallback(
(newPage: number) => {
setCurrentPage(newPage);
updatePageUrl(newPage);
// Ensure page is within bounds
const boundedPage = Math.max(0, Math.min(newPage, totalPages - 1));
setCurrentPage(boundedPage);
updatePageUrl(boundedPage);
},
[updatePageUrl]
[updatePageUrl, totalPages]
);
// Loads the current and adjacent batches
@@ -199,7 +208,14 @@ function usePaginatedFetch<T extends PaginatedType>({
if (!cachedBatches[0]) {
fetchBatchData(0);
}
}, [currentPage, cachedBatches, totalPages, pagesPerBatch, fetchBatchData]);
}, [
currentPage,
cachedBatches,
totalPages,
pagesPerBatch,
fetchBatchData,
batchAndPageIndices,
]);
// Updates current page data from the cache
useEffect(() => {
@@ -209,7 +225,7 @@ function usePaginatedFetch<T extends PaginatedType>({
setCurrentPageData(cachedBatches[batchNum][batchPageNum]);
setIsLoading(false);
}
}, [currentPage, cachedBatches, pagesPerBatch]);
}, [currentPage, cachedBatches, pagesPerBatch, batchAndPageIndices]);
// Implements periodic refresh
useEffect(() => {
@@ -221,21 +237,28 @@ function usePaginatedFetch<T extends PaginatedType>({
}, refreshIntervalInMs);
return () => clearInterval(interval);
}, [currentPage, pagesPerBatch, refreshIntervalInMs, fetchBatchData]);
}, [
currentPage,
pagesPerBatch,
refreshIntervalInMs,
fetchBatchData,
batchAndPageIndices,
]);
// Manually refreshes the current batch
const refresh = useCallback(async () => {
const { batchNum } = batchAndPageIndices;
await fetchBatchData(batchNum);
}, [currentPage, pagesPerBatch, fetchBatchData]);
}, [batchAndPageIndices, fetchBatchData]);
// Cache invalidation
useEffect(() => {
setCachedBatches({});
setTotalItems(0);
goToPage(1);
// Start at page 0 for zero-indexed APIs, page 1 for one-indexed
goToPage(zeroIndexed ? 0 : 1);
setError(null);
}, [currentPath, query, filter]);
}, [currentPath, query, filter, zeroIndexed, goToPage]);
return {
currentPage,

View File

@@ -342,6 +342,7 @@ export interface UserGroup {
personas: Persona[];
is_up_to_date: boolean;
is_up_for_deletion: boolean;
last_synced_at: string | null;
}
export enum ValidSources {