Compare commits

...

2 Commits

Author SHA1 Message Date
SubashMohan
94ea89c6c5 mypy fix 2025-07-30 20:09:48 +05:30
SubashMohan
9f14212650 Add sync warnings to sync record and implement GitHub sync warning notifications 2025-07-30 19:39:00 +05:30
18 changed files with 1026 additions and 39 deletions

View File

@@ -0,0 +1,30 @@
"""add_sync_warnings_to_sync_record
Revision ID: 9ee7f8d9787d
Revises: 3fc5d75723b3
Create Date: 2025-07-30 19:11:54.452420
"""
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import postgresql
# revision identifiers, used by Alembic.
revision = "9ee7f8d9787d"
down_revision = "3fc5d75723b3"
branch_labels = None
depends_on = None
def upgrade() -> None:
# Add sync_warnings column to store validation warnings from connector perm sync
op.add_column(
"sync_record",
sa.Column("sync_warnings", postgresql.JSONB(), nullable=True),
)
def downgrade() -> None:
# Drop the sync_warnings column
op.drop_column("sync_record", "sync_warnings")

View File

@@ -23,6 +23,7 @@ from ee.onyx.db.external_perm import ExternalUserGroup
from ee.onyx.db.external_perm import mark_old_external_groups_as_stale
from ee.onyx.db.external_perm import remove_stale_external_groups
from ee.onyx.db.external_perm import upsert_external_groups
from ee.onyx.external_permissions.perm_sync_types import SyncWarning
from ee.onyx.external_permissions.sync_params import (
get_all_cc_pair_agnostic_group_sync_sources,
)
@@ -36,6 +37,7 @@ from onyx.configs.app_configs import JOB_TIMEOUT
from onyx.configs.constants import CELERY_EXTERNAL_GROUP_SYNC_LOCK_TIMEOUT
from onyx.configs.constants import CELERY_GENERIC_BEAT_LOCK_TIMEOUT
from onyx.configs.constants import CELERY_TASK_WAIT_FOR_FENCE_TIMEOUT
from onyx.configs.constants import NotificationType
from onyx.configs.constants import OnyxCeleryPriority
from onyx.configs.constants import OnyxCeleryQueues
from onyx.configs.constants import OnyxCeleryTask
@@ -49,6 +51,7 @@ from onyx.db.enums import ConnectorCredentialPairStatus
from onyx.db.enums import SyncStatus
from onyx.db.enums import SyncType
from onyx.db.models import ConnectorCredentialPair
from onyx.db.notification import create_notification
from onyx.db.sync_record import insert_sync_record
from onyx.db.sync_record import update_sync_record_status
from onyx.redis.redis_connector import RedisConnector
@@ -398,7 +401,8 @@ def connector_external_group_sync_generator_task(
payload.started = datetime.now(timezone.utc)
redis_connector.external_group_sync.set_fence(payload)
_perform_external_group_sync(
# sync warnings are stored in the sync record and it will displayed in the connector settings page
sync_warnings: SyncWarning | None = _perform_external_group_sync(
cc_pair_id=cc_pair_id,
tenant_id=tenant_id,
)
@@ -409,7 +413,15 @@ def connector_external_group_sync_generator_task(
entity_id=cc_pair_id,
sync_type=SyncType.EXTERNAL_GROUP,
sync_status=SyncStatus.SUCCESS,
sync_warnings=sync_warnings.warnings if sync_warnings else None,
)
if sync_warnings:
create_notification(
user_id=sync_warnings.cc_pair_owner,
notif_type=NotificationType.GROUP_SYNC_WARNING,
db_session=db_session,
additional_data=sync_warnings.warnings,
)
except Exception as e:
error_msg = format_error_for_logging(e)
task_logger.warning(
@@ -448,7 +460,7 @@ def connector_external_group_sync_generator_task(
def _perform_external_group_sync(
cc_pair_id: int,
tenant_id: str,
) -> None:
) -> SyncWarning | None:
with get_session_with_current_tenant() as db_session:
cc_pair = get_connector_credential_pair_from_id(
db_session=db_session,
@@ -481,10 +493,14 @@ def _perform_external_group_sync(
f"Syncing external groups for {source_type} for cc_pair: {cc_pair_id}"
)
external_user_group_batch: list[ExternalUserGroup] = []
sync_warnings: SyncWarning | None = None
try:
external_user_group_generator = ext_group_sync_func(tenant_id, cc_pair)
for external_user_group in external_user_group_generator:
external_user_group_batch.append(external_user_group)
if isinstance(external_user_group, ExternalUserGroup):
external_user_group_batch.append(external_user_group)
else:
sync_warnings = external_user_group
if len(external_user_group_batch) >= _EXTERNAL_GROUP_BATCH_SIZE:
logger.debug(
f"New external user groups: {external_user_group_batch}"
@@ -519,6 +535,8 @@ def _perform_external_group_sync(
mark_all_relevant_cc_pairs_as_external_group_synced(db_session, cc_pair)
return sync_warnings
def validate_external_group_sync_fences(
tenant_id: str,

View File

@@ -4,6 +4,7 @@ from github import Repository
from ee.onyx.db.external_perm import ExternalUserGroup
from ee.onyx.external_permissions.github.utils import get_external_user_group
from ee.onyx.external_permissions.perm_sync_types import SyncWarning
from onyx.connectors.github.connector import GithubConnector
from onyx.db.models import ConnectorCredentialPair
from onyx.utils.logger import setup_logger
@@ -14,7 +15,7 @@ logger = setup_logger()
def github_group_sync(
tenant_id: str,
cc_pair: ConnectorCredentialPair,
) -> Generator[ExternalUserGroup, None, None]:
) -> Generator[ExternalUserGroup | SyncWarning, None, None]:
github_connector: GithubConnector = GithubConnector(
**cc_pair.connector.connector_specific_config
)
@@ -24,6 +25,7 @@ def github_group_sync(
logger.info("Starting GitHub group sync...")
repos: list[Repository.Repository] = []
users_without_email: set[str] = set()
if github_connector.repositories:
if "," in github_connector.repositories:
# Multiple repositories specified
@@ -37,10 +39,21 @@ def github_group_sync(
for repo in repos:
try:
for external_group in get_external_user_group(
external_groups, user_set_without_email = get_external_user_group(
repo, github_connector.github_client
):
)
for external_group in external_groups:
logger.info(f"External group: {external_group}")
yield external_group
users_without_email.update(user_set_without_email)
except Exception as e:
logger.error(f"Error processing repository {repo.id} ({repo.name}): {e}")
# user names without email
if users_without_email:
yield SyncWarning(
cc_pair_id=cc_pair.id,
connector_name=cc_pair.connector.name,
source=cc_pair.connector.source,
warnings={"users_without_email": list(users_without_email)},
cc_pair_owner=cc_pair.creator_id if cc_pair.creator_id else None,
)

View File

@@ -384,7 +384,7 @@ def get_external_access_permission(
def get_external_user_group(
repo: Repository, github_client: Github
) -> list[ExternalUserGroup]:
) -> tuple[list[ExternalUserGroup], set[str]]:
"""
Get the external user group for a repository.
Creates ExternalUserGroup objects with actual user emails for each permission group.
@@ -394,6 +394,7 @@ def get_external_user_group(
f"Generating ExternalUserGroups for {repo.full_name}: visibility={repo_visibility.value}"
)
user_without_email: set[str] = set()
if repo_visibility == GitHubVisibility.PRIVATE:
logger.info(f"Processing private repository {repo.full_name}")
@@ -408,7 +409,7 @@ def get_external_user_group(
if collab.email:
user_emails.add(collab.email)
else:
logger.error(f"Collaborator {collab.login} has no email")
user_without_email.add(collab.login)
if user_emails:
collaborators_group = ExternalUserGroup(
@@ -424,7 +425,7 @@ def get_external_user_group(
if collab.email:
user_emails.add(collab.email)
else:
logger.error(f"Outside collaborator {collab.login} has no email")
user_without_email.add(collab.login)
if user_emails:
outside_collaborators_group = ExternalUserGroup(
@@ -443,8 +444,7 @@ def get_external_user_group(
if member.email:
user_emails.add(member.email)
else:
logger.error(f"Team member {member.login} has no email")
user_without_email.add(member.login)
if user_emails:
team_group = ExternalUserGroup(
id=team.slug,
@@ -458,7 +458,7 @@ def get_external_user_group(
logger.info(
f"Created {len(external_user_groups)} ExternalUserGroups for private repository {repo.full_name}"
)
return external_user_groups
return external_user_groups, user_without_email
if repo_visibility == GitHubVisibility.INTERNAL:
logger.info(f"Processing internal repository {repo.full_name}")
@@ -473,8 +473,7 @@ def get_external_user_group(
if member.email:
user_emails.add(member.email)
else:
logger.error(f"Org member {member.login} has no email")
user_without_email.add(member.login)
org_group = ExternalUserGroup(
id=org_group_id,
user_emails=list(user_emails),
@@ -482,7 +481,7 @@ def get_external_user_group(
logger.info(
f"Created organization group with {len(user_emails)} emails for internal repository {repo.full_name}"
)
return [org_group]
return [org_group], user_without_email
logger.info(f"Repository {repo.full_name} is public - no user groups needed")
return []
return [], set()

View File

@@ -1,8 +1,12 @@
from collections.abc import Callable
from collections.abc import Generator
from typing import Any
from typing import Optional
from typing import Protocol
from typing import TYPE_CHECKING
from uuid import UUID
from pydantic import BaseModel
from onyx.context.search.models import InferenceChunk
from onyx.db.utils import DocumentRow
@@ -65,8 +69,18 @@ GroupSyncFuncType = Callable[
str, # tenant_id
"ConnectorCredentialPair", # cc_pair
],
Generator["ExternalUserGroup", None, None],
Generator["ExternalUserGroup | SyncWarning", None, None],
]
# list of chunks to be censored and the user email. returns censored chunks
CensoringFuncType = Callable[[list[InferenceChunk], str], list[InferenceChunk]]
class SyncWarning(BaseModel):
"""Represents a sync warning with connector information and warning messages."""
cc_pair_id: int
connector_name: str
cc_pair_owner: UUID | None
source: str
warnings: dict[str, Any]

View File

@@ -1,5 +1,6 @@
from datetime import datetime
from http import HTTPStatus
from typing import Any
from fastapi import APIRouter
from fastapi import Depends
@@ -18,7 +19,10 @@ from onyx.db.connector_credential_pair import (
get_connector_credential_pair_from_id_for_user,
)
from onyx.db.engine.sql_engine import get_session
from onyx.db.enums import SyncStatus
from onyx.db.enums import SyncType
from onyx.db.models import User
from onyx.db.sync_record import fetch_latest_sync_record
from onyx.redis.redis_connector import RedisConnector
from onyx.redis.redis_pool import get_redis_client
from onyx.server.models import StatusResponse
@@ -175,3 +179,33 @@ def sync_cc_pair_groups(
success=True,
message="Successfully created the external group sync task.",
)
@router.get("/admin/cc-pair/{cc_pair_id}/sync-groups/warnings")
def get_cc_pair_latest_group_sync_warnings(
cc_pair_id: int,
user: User = Depends(current_curator_or_admin_user),
db_session: Session = Depends(get_session),
) -> dict[str, Any] | None:
cc_pair = get_connector_credential_pair_from_id_for_user(
cc_pair_id=cc_pair_id,
db_session=db_session,
user=user,
get_editable=False,
)
if not cc_pair:
raise HTTPException(
status_code=400,
detail="cc_pair not found for current user's permissions",
)
sync_record = fetch_latest_sync_record(
db_session=db_session,
entity_id=cc_pair_id,
sync_type=SyncType.EXTERNAL_GROUP,
sync_status=SyncStatus.SUCCESS,
)
if sync_record:
return sync_record.sync_warnings
return None

View File

@@ -211,6 +211,7 @@ class NotificationType(str, Enum):
REINDEX = "reindex"
PERSONA_SHARED = "persona_shared"
TRIAL_ENDS_TWO_DAYS = "two_day_trial_ending" # 2 days left in trial
GROUP_SYNC_WARNING = "group_sync_warning" # Warning about group sync
class BlobType(str, Enum):

View File

@@ -1848,6 +1848,10 @@ class SyncRecord(Base):
sync_end_time: Mapped[datetime.datetime | None] = mapped_column(
DateTime(timezone=True), nullable=True
)
# Store validation warnings from connector permission sync operations
sync_warnings: Mapped[dict[str, Any] | None] = mapped_column(
postgresql.JSONB(), nullable=True
)
__table_args__ = (
Index(

View File

@@ -1,3 +1,5 @@
from typing import Any
from sqlalchemy import and_
from sqlalchemy import desc
from sqlalchemy import func
@@ -117,6 +119,7 @@ def update_sync_record_status(
sync_type: SyncType,
sync_status: SyncStatus,
num_docs_synced: int | None = None,
sync_warnings: dict[str, Any] | None = None,
) -> None:
"""Update the status of a sync record.
@@ -136,6 +139,8 @@ def update_sync_record_status(
sync_record.sync_status = sync_status
if num_docs_synced is not None:
sync_record.num_docs_synced = num_docs_synced
if sync_warnings is not None:
sync_record.sync_warnings = sync_warnings
if sync_status.is_terminal():
sync_record.sync_end_time = func.now() # type: ignore

View File

@@ -0,0 +1,115 @@
import os
from collections.abc import Generator
from datetime import datetime
from datetime import timezone
import pytest
from onyx.configs.constants import DocumentSource
from onyx.connectors.models import InputType
from onyx.db.enums import AccessType
from tests.integration.common_utils.managers.cc_pair import CCPairManager
from tests.integration.common_utils.managers.connector import ConnectorManager
from tests.integration.common_utils.managers.credential import CredentialManager
from tests.integration.common_utils.managers.llm_provider import LLMProviderManager
from tests.integration.common_utils.managers.user import UserManager
from tests.integration.common_utils.reset import reset_all
from tests.integration.common_utils.test_models import DATestCCPair
from tests.integration.common_utils.test_models import DATestConnector
from tests.integration.common_utils.test_models import DATestCredential
from tests.integration.common_utils.test_models import DATestUser
GitHubTestEnvSetupTuple = tuple[
DATestUser, # admin_user
DATestUser, # test_user_1
DATestUser, # test_user_2
DATestCredential, # github_credential
DATestConnector, # github_connector
DATestCCPair, # github_cc_pair
]
@pytest.fixture(scope="module")
def github_test_env_setup() -> Generator[GitHubTestEnvSetupTuple]:
"""
Create a complete GitHub test environment with:
- 3 users with email IDs from environment variables
- GitHub credentials using ACCESS_TOKEN_GITHUB from environment
- GitHub connector configured for testing
- Connector-Credential pair linking them together
Returns:
Tuple containing: (admin_user, test_user_1, test_user_2, github_credential, github_connector, github_cc_pair)
"""
# Reset all resources before setting up the test environment
reset_all()
# Get GitHub credentials from environment
github_access_token = os.environ.get("GITHUB_PERMISSION_SYNC_TEST_ACCESS_TOKEN")
# Get user emails from environment (with fallbacks)
admin_email = os.environ.get("GITHUB_ADMIN_EMAIL")
test_user_1_email = os.environ.get("GITHUB_TEST_USER_1_EMAIL")
test_user_2_email = os.environ.get("GITHUB_TEST_USER_2_EMAIL")
if not admin_email or not test_user_1_email or not test_user_2_email:
pytest.skip(
"Skipping GitHub test environment setup due to missing environment variables"
)
# Create users
admin_user: DATestUser = UserManager.create(email=admin_email)
test_user_1: DATestUser = UserManager.create(email=test_user_1_email)
test_user_2: DATestUser = UserManager.create(email=test_user_2_email)
# Create LLM provider - required for document search to work
LLMProviderManager.create(user_performing_action=admin_user)
# Create GitHub credentials
github_credentials = {
"github_access_token": github_access_token,
}
github_credential: DATestCredential = CredentialManager.create(
source=DocumentSource.GITHUB,
credential_json=github_credentials,
user_performing_action=admin_user,
)
# Create GitHub connector
github_connector: DATestConnector = ConnectorManager.create(
name="GitHub Test Connector",
input_type=InputType.POLL,
source=DocumentSource.GITHUB,
connector_specific_config={
"repo_owner": "permission-sync-test",
"include_prs": True,
"repositories": "perm-sync-test-minimal", # Revert to original repository name
"include_issues": True,
},
access_type=AccessType.SYNC,
user_performing_action=admin_user,
)
# Create CC pair linking connector and credential
github_cc_pair: DATestCCPair = CCPairManager.create(
credential_id=github_credential.id,
connector_id=github_connector.id,
name="GitHub Test CC Pair",
access_type=AccessType.SYNC,
user_performing_action=admin_user,
)
# Wait for initial indexing to complete
# GitHub API operations can be slow due to rate limiting and network latency
# Use a longer timeout for initial indexing to avoid flaky test failures
before = datetime.now(tz=timezone.utc)
CCPairManager.wait_for_indexing_completion(
cc_pair=github_cc_pair,
after=before,
user_performing_action=admin_user,
timeout=float("inf"),
)
yield admin_user, test_user_1, test_user_2, github_credential, github_connector, github_cc_pair

View File

@@ -0,0 +1,289 @@
import os
from datetime import datetime
from datetime import timezone
import pytest
from github import Github
from onyx.utils.logger import setup_logger
from tests.integration.common_utils.managers.cc_pair import CCPairManager
from tests.integration.common_utils.managers.document_search import (
DocumentSearchManager,
)
from tests.integration.connector_job_tests.github.conftest import (
GitHubTestEnvSetupTuple,
)
from tests.integration.connector_job_tests.github.utils import GitHubManager
logger = setup_logger()
@pytest.mark.skipif(
os.environ.get("ENABLE_PAID_ENTERPRISE_EDITION_FEATURES", "").lower() != "true",
reason="Permission tests are enterprise only",
)
def test_github_private_repo_permission_sync(
github_test_env_setup: GitHubTestEnvSetupTuple,
) -> None:
(
admin_user,
test_user_1,
test_user_2,
github_credential,
github_connector,
github_cc_pair,
) = github_test_env_setup
# Create GitHub client from credential
github_access_token = github_credential.credential_json["github_access_token"]
github_client = Github(github_access_token)
github_manager = GitHubManager(github_client)
# Get repository configuration from connector
repo_owner = github_connector.connector_specific_config["repo_owner"]
repo_name = github_connector.connector_specific_config["repositories"]
success = github_manager.change_repository_visibility(
repo_owner=repo_owner, repo_name=repo_name, visibility="private"
)
if not success:
pytest.fail(f"Failed to change repository {repo_owner}/{repo_name} to private")
# Add test-team to repository at the start
logger.info(f"Adding test-team to repository {repo_owner}/{repo_name}")
team_added = github_manager.add_team_to_repository(
repo_owner=repo_owner,
repo_name=repo_name,
team_slug="test-team",
permission="pull",
)
if not team_added:
logger.warning(
f"Failed to add test-team to repository {repo_owner}/{repo_name}"
)
try:
CCPairManager.sync(
cc_pair=github_cc_pair,
user_performing_action=admin_user,
)
# Use a longer timeout for GitHub permission sync operations
# GitHub API operations can be slow, especially with rate limiting
# This accounts for document sync, group sync, and vespa sync operations
CCPairManager.wait_for_sync(
cc_pair=github_cc_pair,
user_performing_action=admin_user,
after=datetime.now(timezone.utc),
should_wait_for_group_sync=True,
timeout=float("inf"),
)
test_user_1_results = DocumentSearchManager.search_documents(
query="Is there any PR about middlewares?",
user_performing_action=test_user_1,
)
test_user_2_results = DocumentSearchManager.search_documents(
query="Is there any PR about middlewares?",
user_performing_action=test_user_2,
)
logger.info(f"test_user_1_results: {test_user_1_results}")
logger.info(f"test_user_2_results: {test_user_2_results}")
assert len(test_user_1_results) > 0
assert len(test_user_2_results) == 0
finally:
# Remove test-team from repository at the end
logger.info(f"Removing test-team from repository {repo_owner}/{repo_name}")
team_removed = github_manager.remove_team_from_repository(
repo_owner=repo_owner, repo_name=repo_name, team_slug="test-team"
)
if not team_removed:
logger.warning(
f"Failed to remove test-team from repository {repo_owner}/{repo_name}"
)
@pytest.mark.skipif(
os.environ.get("ENABLE_PAID_ENTERPRISE_EDITION_FEATURES", "").lower() != "true",
reason="Permission tests are enterprise only",
)
def test_github_public_repo_permission_sync(
github_test_env_setup: GitHubTestEnvSetupTuple,
) -> None:
"""
Test that when a repository is changed to public, both users can access the documents.
"""
(
admin_user,
test_user_1,
test_user_2,
github_credential,
github_connector,
github_cc_pair,
) = github_test_env_setup
# Create GitHub client from credential
github_access_token = github_credential.credential_json["github_access_token"]
github_client = Github(github_access_token)
github_manager = GitHubManager(github_client)
# Get repository configuration from connector
repo_owner = github_connector.connector_specific_config["repo_owner"]
repo_name = github_connector.connector_specific_config["repositories"]
# Change repository to public
logger.info(f"Changing repository {repo_owner}/{repo_name} to public")
success = github_manager.change_repository_visibility(
repo_owner=repo_owner, repo_name=repo_name, visibility="public"
)
if not success:
pytest.fail(f"Failed to change repository {repo_owner}/{repo_name} to public")
# Verify repository is now public
current_visibility = github_manager.get_repository_visibility(
repo_owner=repo_owner, repo_name=repo_name
)
logger.info(f"Repository {repo_owner}/{repo_name} visibility: {current_visibility}")
assert (
current_visibility == "public"
), f"Repository should be public, but is {current_visibility}"
# Trigger sync to update permissions
CCPairManager.sync(
cc_pair=github_cc_pair,
user_performing_action=admin_user,
)
# Wait for sync to complete with group sync
# Public repositories should be accessible to all users
CCPairManager.wait_for_sync(
cc_pair=github_cc_pair,
user_performing_action=admin_user,
after=datetime.now(timezone.utc),
should_wait_for_group_sync=True,
timeout=float("inf"),
)
# Test document search for both users
test_user_1_results = DocumentSearchManager.search_documents(
query="How many PR's are there?",
user_performing_action=test_user_1,
)
test_user_2_results = DocumentSearchManager.search_documents(
query="How many PR's are there?",
user_performing_action=test_user_2,
)
logger.info(f"test_user_1_results: {test_user_1_results}")
logger.info(f"test_user_2_results: {test_user_2_results}")
# Both users should have access to the public repository documents
assert (
len(test_user_1_results) > 0
), "test_user_1 should have access to public repository documents"
assert (
len(test_user_2_results) > 0
), "test_user_2 should have access to public repository documents"
# Verify that both users get the same results (since repo is public)
assert len(test_user_1_results) == len(
test_user_2_results
), "Both users should see the same documents from public repository"
@pytest.mark.skipif(
os.environ.get("ENABLE_PAID_ENTERPRISE_EDITION_FEATURES", "").lower() != "true",
reason="Permission tests are enterprise only",
)
def test_github_internal_repo_permission_sync(
github_test_env_setup: GitHubTestEnvSetupTuple,
) -> None:
"""
Test that when a repository is changed to internal, test_user_1 has access but test_user_2 doesn't.
Internal repositories are accessible only to organization members.
"""
(
admin_user,
test_user_1,
test_user_2,
github_credential,
github_connector,
github_cc_pair,
) = github_test_env_setup
# Create GitHub client from credential
github_access_token = github_credential.credential_json["github_access_token"]
github_client = Github(github_access_token)
github_manager = GitHubManager(github_client)
# Get repository configuration from connector
repo_owner = github_connector.connector_specific_config["repo_owner"]
repo_name = github_connector.connector_specific_config["repositories"]
# Change repository to internal
logger.info(f"Changing repository {repo_owner}/{repo_name} to internal")
success = github_manager.change_repository_visibility(
repo_owner=repo_owner, repo_name=repo_name, visibility="internal"
)
if not success:
pytest.fail(f"Failed to change repository {repo_owner}/{repo_name} to internal")
# Verify repository is now internal
current_visibility = github_manager.get_repository_visibility(
repo_owner=repo_owner, repo_name=repo_name
)
logger.info(f"Repository {repo_owner}/{repo_name} visibility: {current_visibility}")
assert (
current_visibility == "internal"
), f"Repository should be internal, but is {current_visibility}"
# Trigger sync to update permissions
CCPairManager.sync(
cc_pair=github_cc_pair,
user_performing_action=admin_user,
)
# Wait for sync to complete with group sync
# Internal repositories should be accessible only to organization members
CCPairManager.wait_for_sync(
cc_pair=github_cc_pair,
user_performing_action=admin_user,
after=datetime.now(timezone.utc),
should_wait_for_group_sync=True,
timeout=float("inf"),
)
# Test document search for both users
test_user_1_results = DocumentSearchManager.search_documents(
query="How many PR's are there?",
user_performing_action=test_user_1,
)
test_user_2_results = DocumentSearchManager.search_documents(
query="How many PR's are there?",
user_performing_action=test_user_2,
)
logger.info(f"test_user_1_results: {test_user_1_results}")
logger.info(f"test_user_2_results: {test_user_2_results}")
# For internal repositories:
# - test_user_1 should have access (assuming they're part of the organization)
# - test_user_2 should NOT have access (assuming they're not part of the organization)
assert (
len(test_user_1_results) > 0
), "test_user_1 should have access to internal repository documents (organization member)"
assert (
len(test_user_2_results) == 0
), "test_user_2 should NOT have access to internal repository documents (not organization member)"

View File

@@ -0,0 +1,195 @@
from typing import Optional
from github import Github
from github.GithubException import GithubException
from onyx.utils.logger import setup_logger
logger = setup_logger()
class GitHubManager:
"""
Manager class for GitHub operations used in testing.
Provides methods to change repository visibility, check repository visibility, and manage teams.
"""
def __init__(self, github_client: Github):
"""
Initialize the GitHub manager with a GitHub client.
Args:
github_client: Authenticated GitHub client instance
"""
self.github_client = github_client
def change_repository_visibility(
self, repo_owner: str, repo_name: str, visibility: str
) -> bool:
"""
Change the visibility of a repository.
Args:
repo_owner: Repository owner (organization or username)
repo_name: Repository name
visibility: New visibility ('public', 'private', or 'internal')
Returns:
bool: True if successful, False otherwise
Raises:
ValueError: If visibility is not valid
GithubException: If GitHub API call fails
"""
if visibility not in ["public", "private", "internal"]:
raise ValueError(
f"Invalid visibility: {visibility}. Must be 'public', 'private', or 'internal'"
)
try:
repo = self.github_client.get_repo(f"{repo_owner}/{repo_name}")
# Check if we have admin permissions
if not repo.permissions.admin:
logger.error(
f"No admin permissions for repository {repo_owner}/{repo_name}"
)
return False
# Change visibility
repo.edit(private=(visibility == "private"))
# For internal repositories, we need to set the visibility explicitly
if visibility == "internal":
# Note: Internal repositories are only available for GitHub Enterprise
# This might not work in all environments
try:
repo.edit(visibility="internal")
except GithubException as e:
logger.warning(f"Could not set repository to internal: {e}")
return False
elif visibility == "public":
try:
repo.edit(visibility="public")
except GithubException as e:
logger.warning(f"Could not set repository to public: {e}")
return False
elif visibility == "private":
try:
repo.edit(visibility="private")
except GithubException as e:
logger.warning(f"Could not set repository to private: {e}")
return False
logger.info(
f"Successfully changed {repo_owner}/{repo_name} visibility to {visibility}"
)
return True
except GithubException as e:
logger.error(f"Failed to change repository visibility: {e}")
return False
def add_team_to_repository(
self, repo_owner: str, repo_name: str, team_slug: str, permission: str = "push"
) -> bool:
"""
Add a team to a repository with specified permissions.
Args:
repo_owner: Repository owner (organization)
repo_name: Repository name
team_slug: Team slug (not team name)
permission: Permission level ('pull', 'push', 'admin', 'maintain', 'triage')
Returns:
bool: True if successful, False otherwise
Raises:
GithubException: If GitHub API call fails
"""
valid_permissions = ["pull", "push", "admin", "maintain", "triage"]
if permission not in valid_permissions:
raise ValueError(
f"Invalid permission: {permission}. Must be one of {valid_permissions}"
)
try:
repo = self.github_client.get_repo(f"{repo_owner}/{repo_name}")
org = self.github_client.get_organization(repo_owner)
team = org.get_team_by_slug(team_slug)
# Add team to repository
team.add_to_repos(repo)
# Set team permissions on the repository
team.set_repo_permission(repo, permission)
logger.info(
f"Successfully added team {team_slug} to {repo_owner}/{repo_name} with {permission} permissions"
)
return True
except GithubException as e:
logger.error(f"Failed to add team to repository: {e}")
return False
def remove_team_from_repository(
self, repo_owner: str, repo_name: str, team_slug: str
) -> bool:
"""
Remove a team from a repository.
Args:
repo_owner: Repository owner (organization)
repo_name: Repository name
team_slug: Team slug (not team name)
Returns:
bool: True if successful, False otherwise
Raises:
GithubException: If GitHub API call fails
"""
try:
repo = self.github_client.get_repo(f"{repo_owner}/{repo_name}")
org = self.github_client.get_organization(repo_owner)
team = org.get_team_by_slug(team_slug)
# Remove team from repository
team.remove_from_repos(repo)
logger.info(
f"Successfully removed team {team_slug} from {repo_owner}/{repo_name}"
)
return True
except GithubException as e:
logger.error(f"Failed to remove team from repository: {e}")
return False
def get_repository_visibility(
self, repo_owner: str, repo_name: str
) -> Optional[str]:
"""
Get the current visibility of a repository.
Args:
repo_owner: Repository owner
repo_name: Repository name
Returns:
Optional[str]: Repository visibility ('public', 'private', 'internal') or None if failed
"""
try:
repo = self.github_client.get_repo(f"{repo_owner}/{repo_name}")
if hasattr(repo, "visibility"):
return repo.visibility
else:
# Fallback for older GitHub API versions
return "private" if repo.private else "public"
except GithubException as e:
logger.error(f"Failed to get repository visibility: {e}")
return None

View File

@@ -0,0 +1,105 @@
"use client";
import React, { useState } from "react";
import { Badge } from "@/components/ui/badge";
import { Button } from "@/components/ui/button";
import {
ChevronDownIcon,
ChevronUpIcon,
CopyIcon,
CheckIcon,
} from "lucide-react";
interface GithubSyncWarningProps {
usernames: string[];
}
const INITIAL_DISPLAY_COUNT = 8;
export const GithubSyncWarning: React.FC<GithubSyncWarningProps> = ({
usernames,
}) => {
const [isExpanded, setIsExpanded] = useState(false);
const [isCopied, setIsCopied] = useState(false);
if (!usernames || usernames.length === 0) {
return null;
}
const hasMoreUsers = usernames.length > INITIAL_DISPLAY_COUNT;
const displayedUsernames = isExpanded
? usernames
: usernames.slice(0, INITIAL_DISPLAY_COUNT);
const remainingCount = usernames.length - INITIAL_DISPLAY_COUNT;
const handleCopyUsernames = async () => {
try {
await navigator.clipboard.writeText(usernames.join(", "));
setIsCopied(true);
setTimeout(() => setIsCopied(false), 2000);
} catch (err) {
console.error("Failed to copy usernames:", err);
}
};
return (
<div className="relative border-l-4 border-amber-300 bg-amber-50 dark:border-amber-700 dark:bg-amber-900/30 p-4 rounded-r-md">
<Button
variant="outline"
size="sm"
onClick={handleCopyUsernames}
className={`absolute top-2 right-2 h-8 w-8 p-0 border-amber-400 dark:border-amber-600 hover:bg-amber-100 dark:hover:bg-amber-800/50 transition-all duration-200 ${
isCopied
? "scale-110 bg-green-100 dark:bg-green-800/50 border-green-300 dark:border-green-600"
: "hover:scale-105"
}`}
>
{isCopied ? (
<CheckIcon className="w-4 h-4 text-green-700 dark:text-green-200 animate-bounce" />
) : (
<CopyIcon className="w-4 h-4 text-amber-700 dark:text-amber-200 transition-transform duration-200 hover:scale-110" />
)}
</Button>
<div className="text-amber-700 dark:text-amber-200 text-sm mb-3 pr-12">
The following usernames have not made their email addresses public on
their GitHub profiles. Access to documents is granted via emailwithout
it, users will not be able to access non-public repositories. Please ask
them to update their profiles accordingly.
</div>
<div className="flex flex-wrap gap-2 mb-3">
{displayedUsernames.map((username, index) => (
<Badge
key={index}
variant="outline"
className="bg-amber-50 dark:bg-amber-700/20 border-amber-300 dark:border-amber-600 text-amber-800 dark:text-amber-200 text-xs"
>
{username}
</Badge>
))}
</div>
{hasMoreUsers && (
<Button
variant="ghost"
size="sm"
onClick={() => setIsExpanded(!isExpanded)}
className="text-amber-700 dark:text-amber-200 hover:text-amber-800 dark:hover:text-amber-100 hover:bg-amber-100 dark:hover:bg-amber-800/50 px-2 py-1 h-auto text-xs"
>
{isExpanded ? (
<>
<ChevronUpIcon className="w-3 h-3 mr-1" />
View less
</>
) : (
<>
<ChevronDownIcon className="w-3 h-3 mr-1" />
View more ({remainingCount} remaining)
</>
)}
</Button>
)}
</div>
);
};

View File

@@ -32,6 +32,7 @@ import {
ConnectorCredentialPairStatus,
IndexAttemptError,
statusIsNotCurrentlyActive,
SyncWarnings,
} from "./types";
import { EditableStringFieldDisplay } from "@/components/EditableStringFieldDisplay";
import EditPropertyModal from "@/components/modals/EditPropertyModal";
@@ -62,6 +63,8 @@ import { timeAgo } from "@/lib/time";
import { useStatusChange } from "./useStatusChange";
import { useReIndexModal } from "./ReIndexModal";
import { Button } from "@/components/ui/button";
import { usePaidEnterpriseFeaturesEnabled } from "@/components/settings/usePaidEnterpriseFeaturesEnabled";
import { GithubSyncWarning } from "./GithubSyncWarning";
// synchronize these validations with the SQLAlchemy connector class until we have a
// centralized schema for both frontend and backend
@@ -99,6 +102,17 @@ function Main({ ccPairId }: { ccPairId: number }) {
errorHandlingFetcher,
{ refreshInterval: 5000 } // 5 seconds
);
const isPaidEnterpriseFeaturesEnabled = usePaidEnterpriseFeaturesEnabled();
const {
data: syncWarnings,
isLoading: isLoadingSyncWarnings,
error: syncWarningsError,
} = useSWR<SyncWarnings>(
isPaidEnterpriseFeaturesEnabled
? `${buildCCPairInfoUrl(ccPairId)}/sync-groups/warnings`
: null,
errorHandlingFetcher
);
const {
currentPageData: indexAttempts,
@@ -354,7 +368,8 @@ function Main({ ccPairId }: { ccPairId: number }) {
if (
isLoadingCCPair ||
isLoadingIndexAttempts ||
isLoadingMostRecentIndexAttempts
isLoadingMostRecentIndexAttempts ||
isLoadingSyncWarnings
) {
return <ThreeDotsLoader />;
}
@@ -677,6 +692,18 @@ function Main({ ccPairId }: { ccPairId: number }) {
</>
)}
{ccPair.connector.source === "github" &&
syncWarnings &&
syncWarnings.users_without_email &&
syncWarnings.users_without_email.length > 0 && (
<>
<Title size="md" className="mt-10 mb-2">
Permission Sync Warnings
</Title>
<GithubSyncWarning usernames={syncWarnings.users_without_email} />
</>
)}
{ccPair.connector.connector_specific_config &&
Object.keys(ccPair.connector.connector_specific_config).length > 0 && (
<>

View File

@@ -54,6 +54,10 @@ export interface CCPairFullInfo {
latest_checkpoint_description: string | null;
}
export interface SyncWarnings {
users_without_email: string[];
}
export interface PaginatedIndexAttempts {
index_attempts: IndexAttemptSnapshot[];
page: number;

View File

@@ -36,6 +36,7 @@ export enum NotificationType {
PERSONA_SHARED = "persona_shared",
REINDEX_NEEDED = "reindex_needed",
TRIAL_ENDS_TWO_DAYS = "two_day_trial_ending",
GROUP_SYNC_WARNING = "group_sync_warning",
}
export interface Notification {

View File

@@ -0,0 +1,104 @@
"use client";
import React from "react";
import { Notification } from "@/app/admin/settings/interfaces";
import { WarningCircle } from "@phosphor-icons/react";
import { useRouter } from "next/navigation";
import { SourceIcon } from "@/components/SourceIcon";
import { ValidSources } from "@/lib/types";
interface GroupSyncWarningNotificationProps {
notification: Notification;
onDismiss: (notificationId: number) => void;
}
export const GroupSyncWarningNotification: React.FC<
GroupSyncWarningNotificationProps
> = ({ notification, onDismiss }) => {
const router = useRouter();
const handleNavigateToConnector = () => {
const ccPairId = notification.additional_data?.cc_pair_id;
if (ccPairId) {
router.push(`/admin/connector/${ccPairId}`);
onDismiss(notification.id);
}
};
const getWarningMessage = () => {
const warnings = notification.additional_data?.warnings;
const connectorName =
notification.additional_data?.connector_name || "Unknown Connector";
const source = notification.additional_data?.source || "unknown";
if (
warnings?.users_without_email &&
warnings.users_without_email.length > 0
) {
return `${connectorName} (${source}) has ${warnings.users_without_email.length} users without public email addresses.`;
}
return `${connectorName} (${source}) has synchronization warnings.`;
};
const getWarningDetails = () => {
const warnings = notification.additional_data?.warnings;
if (
warnings?.users_without_email &&
warnings.users_without_email.length > 0
) {
const userCount = warnings.users_without_email.length;
const displayUsers = warnings.users_without_email.slice(0, 3);
const hasMore = userCount > 3;
return (
<div className="text-xs text-text-600 mt-1">
Users without email: {displayUsers.join(", ")}
{hasMore && ` and ${userCount - 3} more`}
</div>
);
}
return null;
};
return (
<div className="w-72 px-4 py-3 border-b last:border-b-0 hover:bg-background-50 transition duration-150 ease-in-out">
<div className="flex items-start">
<div className="mt-2 flex-shrink-0 mr-3 relative">
{notification.additional_data?.source ? (
<SourceIcon
sourceType={notification.additional_data.source as ValidSources}
iconSize={24}
/>
) : (
<WarningCircle size={24} className="text-amber-500" weight="fill" />
)}
{/* Warning indicator overlay */}
<div className="absolute -top-1 -right-1 w-3 h-3 bg-amber-500 rounded-full border border-white dark:border-gray-800">
<WarningCircle size={10} className="text-white" weight="fill" />
</div>
</div>
<div className="flex-grow">
<p className="font-semibold text-sm text-text-800">
Group Sync Warning
</p>
<p className="text-xs text-text-600 mt-1">{getWarningMessage()}</p>
{getWarningDetails()}
</div>
</div>
<div className="flex justify-end mt-2 space-x-2">
<button
onClick={handleNavigateToConnector}
className="px-3 py-1 text-sm font-medium text-blue-600 hover:text-blue-800 transition duration-150 ease-in-out"
>
View Connector
</button>
<button
onClick={() => onDismiss(notification.id)}
className="px-3 py-1 text-sm font-medium text-text-600 hover:text-text-800 transition duration-150 ease-in-out"
>
Dismiss
</button>
</div>
</div>
);
};

View File

@@ -10,6 +10,7 @@ import { useUser } from "../user/UserProvider";
import { XIcon } from "../icons/icons";
import { Spinner } from "@phosphor-icons/react";
import { useRouter } from "next/navigation";
import { GroupSyncWarningNotification } from "./GroupSyncWarningNotification";
export const Notifications = ({
notifications,
@@ -97,12 +98,20 @@ export const Notifications = ({
const sortedNotifications = notifications
? notifications
.filter((notification) => {
const personaId = notification.additional_data?.persona_id;
return (
personaId !== undefined &&
personas &&
personas[personaId] !== undefined
);
// Include persona shared notifications that have valid personas
if (notification.notif_type === NotificationType.PERSONA_SHARED) {
const personaId = notification.additional_data?.persona_id;
return (
personaId !== undefined &&
personas &&
personas[personaId] !== undefined
);
}
// Include group sync warning notifications
if (notification.notif_type === NotificationType.GROUP_SYNC_WARNING) {
return true;
}
return false;
})
.sort(
(a, b) =>
@@ -137,28 +146,38 @@ export const Notifications = ({
</button>
{notifications && notifications.length > 0 ? (
sortedNotifications.length > 0 && personas ? (
sortedNotifications
.filter(
(notification) =>
notification.notif_type === NotificationType.PERSONA_SHARED
)
.map((notification) => {
sortedNotifications.length > 0 ? (
sortedNotifications.map((notification) => {
// Render Group Sync Warning Notifications
if (
notification.notif_type === NotificationType.GROUP_SYNC_WARNING
) {
return (
<GroupSyncWarningNotification
key={notification.id}
notification={notification}
onDismiss={dismissNotification}
/>
);
}
// Render Persona Shared Notifications
if (notification.notif_type === NotificationType.PERSONA_SHARED) {
const persona = notification.additional_data?.persona_id
? personas[notification.additional_data.persona_id]
? personas?.[notification.additional_data.persona_id]
: null;
if (!persona) return null;
return (
<div
key={notification.id}
className="w-72 px-4 py-3 border-b last:border-b-0 hover:bg-background-50 transition duration-150 ease-in-out"
>
<div className="flex items-start">
{persona && (
<div className="mt-2 flex-shrink-0 mr-3">
<AssistantIcon assistant={persona} size="small" />
</div>
)}
<div className="mt-2 flex-shrink-0 mr-3">
<AssistantIcon assistant={persona} size="small" />
</div>
<div className="flex-grow">
<p className="font-semibold text-sm text-text-800">
New Assistant Shared: {persona?.name}
@@ -213,11 +232,21 @@ export const Notifications = ({
</div>
</div>
);
})
) : (
}
return null;
})
) : // Show loading spinner only if we have persona notifications but personas aren't loaded yet
notifications.some(
(n) => n.notif_type === NotificationType.PERSONA_SHARED
) && !personas ? (
<div className="flex h-20 justify-center items-center w-72">
<Spinner size={20} />
</div>
) : (
<div className="px-4 py-3 text-center text-text-600">
No new notifications
</div>
)
) : (
<div className="px-4 py-3 text-center text-text-600">