chris/onyx
1
0
Fork 0
forked from github/onyx
Code Pull Requests Activity

merge upstream

Browse Source
This commit is contained in:
chris
2025-08-18 08:40:20 +00:00
parent 6f6d7277f6 fe029eccae
commit 2cedaa1537
4 changed files with 52 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
.github/workflows/pr-integration-tests.yml vendored
View File

@@ -19,6 +19,10 @@ env:
JIRA_BASE_URL: ${{ secrets.JIRA_BASE_URL }}
JIRA_USER_EMAIL: ${{ secrets.JIRA_USER_EMAIL }}
JIRA_API_TOKEN: ${{ secrets.JIRA_API_TOKEN }}
PERM_SYNC_SHAREPOINT_CLIENT_ID: ${{ secrets.PERM_SYNC_SHAREPOINT_CLIENT_ID }}
PERM_SYNC_SHAREPOINT_PRIVATE_KEY: ${{ secrets.PERM_SYNC_SHAREPOINT_PRIVATE_KEY }}
PERM_SYNC_SHAREPOINT_CERTIFICATE_PASSWORD: ${{ secrets.PERM_SYNC_SHAREPOINT_CERTIFICATE_PASSWORD }}
PERM_SYNC_SHAREPOINT_DIRECTORY_ID: ${{ secrets.PERM_SYNC_SHAREPOINT_DIRECTORY_ID }}
PLATFORM_PAIR: linux-amd64
jobs:
@@ -272,6 +276,10 @@ jobs:
-e JIRA_BASE_URL=${JIRA_BASE_URL} \
-e JIRA_USER_EMAIL=${JIRA_USER_EMAIL} \
-e JIRA_API_TOKEN=${JIRA_API_TOKEN} \
-e PERM_SYNC_SHAREPOINT_CLIENT_ID=${PERM_SYNC_SHAREPOINT_CLIENT_ID} \
-e PERM_SYNC_SHAREPOINT_PRIVATE_KEY="${PERM_SYNC_SHAREPOINT_PRIVATE_KEY}" \
-e PERM_SYNC_SHAREPOINT_CERTIFICATE_PASSWORD=${PERM_SYNC_SHAREPOINT_CERTIFICATE_PASSWORD} \
-e PERM_SYNC_SHAREPOINT_DIRECTORY_ID=${PERM_SYNC_SHAREPOINT_DIRECTORY_ID} \
-e TEST_WEB_HOSTNAME=test-runner \
-e MOCK_CONNECTOR_SERVER_HOST=mock_connector_server \
-e MOCK_CONNECTOR_SERVER_PORT=8001 \

8
.github/workflows/pr-mit-integration-tests.yml vendored
View File

@@ -19,6 +19,10 @@ env:
JIRA_BASE_URL: ${{ secrets.JIRA_BASE_URL }}
JIRA_USER_EMAIL: ${{ secrets.JIRA_USER_EMAIL }}
JIRA_API_TOKEN: ${{ secrets.JIRA_API_TOKEN }}
PERM_SYNC_SHAREPOINT_CLIENT_ID: ${{ secrets.PERM_SYNC_SHAREPOINT_CLIENT_ID }}
PERM_SYNC_SHAREPOINT_PRIVATE_KEY: ${{ secrets.PERM_SYNC_SHAREPOINT_PRIVATE_KEY }}
PERM_SYNC_SHAREPOINT_CERTIFICATE_PASSWORD: ${{ secrets.PERM_SYNC_SHAREPOINT_CERTIFICATE_PASSWORD }}
PERM_SYNC_SHAREPOINT_DIRECTORY_ID: ${{ secrets.PERM_SYNC_SHAREPOINT_DIRECTORY_ID }}
PLATFORM_PAIR: linux-amd64
jobs:
integration-tests-mit:
@@ -207,6 +211,10 @@ jobs:
-e JIRA_BASE_URL=${JIRA_BASE_URL} \
-e JIRA_USER_EMAIL=${JIRA_USER_EMAIL} \
-e JIRA_API_TOKEN=${JIRA_API_TOKEN} \
-e PERM_SYNC_SHAREPOINT_CLIENT_ID=${PERM_SYNC_SHAREPOINT_CLIENT_ID} \
-e PERM_SYNC_SHAREPOINT_PRIVATE_KEY="${PERM_SYNC_SHAREPOINT_PRIVATE_KEY}" \
-e PERM_SYNC_SHAREPOINT_CERTIFICATE_PASSWORD=${PERM_SYNC_SHAREPOINT_CERTIFICATE_PASSWORD} \
-e PERM_SYNC_SHAREPOINT_DIRECTORY_ID=${PERM_SYNC_SHAREPOINT_DIRECTORY_ID} \
-e TEST_WEB_HOSTNAME=test-runner \
-e MOCK_CONNECTOR_SERVER_HOST=mock_connector_server \
-e MOCK_CONNECTOR_SERVER_PORT=8001 \

77
backend/tests/daily/connectors/sharepoint/test_sharepoint_connector.py
View File

@@ -14,6 +14,8 @@ from onyx.connectors.models import ImageSection
from onyx.connectors.sharepoint.connector import SharepointConnector
from tests.daily.connectors.utils import load_all_docs_from_checkpoint_connector
# NOTE: Sharepoint site for tests is "sharepoint-tests"
@dataclass
class ExpectedDocument:
@@ -26,7 +28,7 @@ class ExpectedDocument:
EXPECTED_DOCUMENTS = [
ExpectedDocument(
semantic_identifier="test1.docx",
content="Test1 password: 1234",
content="test1",
folder_path="test",
),
ExpectedDocument(
@@ -45,14 +47,22 @@ EXPECTED_DOCUMENTS = [
folder_path=None,
library="Other Library",
),
]
EXPECTED_PAGES = [
ExpectedDocument(
semantic_identifier="Book.xlsx",
content="## Sheet1\n| exel | 9090 |\n| --- | --- |",
semantic_identifier="CollabHome",
content=(
"# Home\n\nDisplay recent news.\n\n## News\n\nShow recent activities from your site\n\n"
"## Site activity\n\n## Quick links\n\nLearn about a team site\n\nLearn how to add a page\n\n"
"Add links to important documents and pages.\n\n## Quick links\n\nDocuments\n\n"
"Add a document library\n\n## Document library"
),
folder_path=None,
),
ExpectedDocument(
semantic_identifier="Presentation.pptx",
content="<!-- Slide number: 1 -->\n# Powerpoint 6565\n\n### Notes:\n6767",
semantic_identifier="Home",
content="# Home",
folder_path=None,
),
]
@@ -75,7 +85,7 @@ def verify_document_content(doc: Document, expected: ExpectedDocument) -> None:
assert doc.semantic_identifier == expected.semantic_identifier
assert len(doc.sections) == 1
assert doc.sections[0].text is not None
assert expected.content in doc.sections[0].text
assert expected.content == doc.sections[0].text
verify_document_metadata(doc)
@@ -294,9 +304,7 @@ def test_sharepoint_connector_poll(
mock_store_image,
):
# Initialize connector with the base site URL
connector = SharepointConnector(
sites=["https://danswerai.sharepoint.com/sites/sharepoint-tests"]
)
connector = SharepointConnector(sites=[os.environ["SHAREPOINT_SITE"]])
# Load credentials
connector.load_credentials(sharepoint_credentials)
@@ -320,10 +328,11 @@ def test_sharepoint_connector_poll(
), "Should only find one document in the time window"
doc = found_documents[0]
assert doc.semantic_identifier == "test1.docx"
verify_document_metadata(doc)
verify_document_content(
doc,
[d for d in EXPECTED_DOCUMENTS if d.semantic_identifier == "test1.docx"][0],
next(
d for d in EXPECTED_DOCUMENTS if d.semantic_identifier == "test1.docx"
),
)
@@ -336,56 +345,24 @@ def test_sharepoint_connector_pages(
"onyx.connectors.sharepoint.connector.store_image_and_create_section",
mock_store_image,
):
# Initialize connector with the base site URL
connector = SharepointConnector(
sites=["https://danswerai.sharepoint.com/sites/sharepoint-tests-pages"],
sites=[os.environ["SHAREPOINT_SITE"]],
include_site_pages=True,
include_site_documents=False,
)
# Load credentials
connector.load_credentials(sharepoint_credentials)
# Get documents within the time window
found_documents = load_all_docs_from_checkpoint_connector(
connector=connector,
start=0,
end=time.time(),
)
# Should only find CollabHome
assert len(found_documents) == 1, "Should only find one page"
doc = found_documents[0]
assert doc.semantic_identifier == "CollabHome"
verify_document_metadata(doc)
assert len(doc.sections) == 1
assert (
doc.sections[0].text
== """
# Home
assert len(found_documents) == len(
EXPECTED_PAGES
), "Should find all pages in test site"
Display recent news.
## News
Show recent activities from your site
## Site activity
## Quick links
Learn about a team site
Learn how to add a page
Add links to important documents and pages.
## Quick links
Documents
Add a document library
## Document library
""".strip()
)
for expected in EXPECTED_PAGES:
doc = find_document(found_documents, expected.semantic_identifier)
verify_document_content(doc, expected)

9
backend/tests/integration/connector_job_tests/sharepoint/test_sharepoint_permissions.py
View File

@@ -1,3 +1,4 @@
import os
from typing import List
from uuid import UUID
@@ -106,6 +107,10 @@ def get_documents_by_permission_type(
return public_docs
@pytest.mark.skipif(
os.environ.get("ENABLE_PAID_ENTERPRISE_EDITION_FEATURES", "").lower() != "true",
reason="Permission tests are enterprise only",
)
def test_public_documents_accessible_by_all_users(
sharepoint_test_env_setup: SharepointTestEnvSetupTuple,
) -> None:
@@ -157,6 +162,10 @@ def test_public_documents_accessible_by_all_users(
)
@pytest.mark.skipif(
os.environ.get("ENABLE_PAID_ENTERPRISE_EDITION_FEATURES", "").lower() != "true",
reason="Permission tests are enterprise only",
)
def test_group_based_permissions(
sharepoint_test_env_setup: SharepointTestEnvSetupTuple,
) -> None: