merge upstream

* chore: add SharePoint sync environment variables to integration test workflows

* fix cubic comments

* test: skip SharePoint permission tests for non-enterprise

* test: update SharePoint permission tests to skip for non-enterprise environments

.github/workflows/pr-integration-tests.yml

@@ -19,6 +19,10 @@ env:
   JIRA_BASE_URL: ${{ secrets.JIRA_BASE_URL }}
   JIRA_USER_EMAIL: ${{ secrets.JIRA_USER_EMAIL }}
   JIRA_API_TOKEN: ${{ secrets.JIRA_API_TOKEN }}
+  PERM_SYNC_SHAREPOINT_CLIENT_ID: ${{ secrets.PERM_SYNC_SHAREPOINT_CLIENT_ID }}
+  PERM_SYNC_SHAREPOINT_PRIVATE_KEY: ${{ secrets.PERM_SYNC_SHAREPOINT_PRIVATE_KEY }}
+  PERM_SYNC_SHAREPOINT_CERTIFICATE_PASSWORD: ${{ secrets.PERM_SYNC_SHAREPOINT_CERTIFICATE_PASSWORD }}
+  PERM_SYNC_SHAREPOINT_DIRECTORY_ID: ${{ secrets.PERM_SYNC_SHAREPOINT_DIRECTORY_ID }}
   PLATFORM_PAIR: linux-amd64
 
 jobs:
@@ -272,6 +276,10 @@ jobs:
             -e JIRA_BASE_URL=${JIRA_BASE_URL} \
             -e JIRA_USER_EMAIL=${JIRA_USER_EMAIL} \
             -e JIRA_API_TOKEN=${JIRA_API_TOKEN} \
+            -e PERM_SYNC_SHAREPOINT_CLIENT_ID=${PERM_SYNC_SHAREPOINT_CLIENT_ID} \
+            -e PERM_SYNC_SHAREPOINT_PRIVATE_KEY="${PERM_SYNC_SHAREPOINT_PRIVATE_KEY}" \
+            -e PERM_SYNC_SHAREPOINT_CERTIFICATE_PASSWORD=${PERM_SYNC_SHAREPOINT_CERTIFICATE_PASSWORD} \
+            -e PERM_SYNC_SHAREPOINT_DIRECTORY_ID=${PERM_SYNC_SHAREPOINT_DIRECTORY_ID} \
             -e TEST_WEB_HOSTNAME=test-runner \
             -e MOCK_CONNECTOR_SERVER_HOST=mock_connector_server \
             -e MOCK_CONNECTOR_SERVER_PORT=8001 \

.github/workflows/pr-mit-integration-tests.yml

@@ -19,6 +19,10 @@ env:
   JIRA_BASE_URL: ${{ secrets.JIRA_BASE_URL }}
   JIRA_USER_EMAIL: ${{ secrets.JIRA_USER_EMAIL }}
   JIRA_API_TOKEN: ${{ secrets.JIRA_API_TOKEN }}
+  PERM_SYNC_SHAREPOINT_CLIENT_ID: ${{ secrets.PERM_SYNC_SHAREPOINT_CLIENT_ID }}
+  PERM_SYNC_SHAREPOINT_PRIVATE_KEY: ${{ secrets.PERM_SYNC_SHAREPOINT_PRIVATE_KEY }}
+  PERM_SYNC_SHAREPOINT_CERTIFICATE_PASSWORD: ${{ secrets.PERM_SYNC_SHAREPOINT_CERTIFICATE_PASSWORD }}
+  PERM_SYNC_SHAREPOINT_DIRECTORY_ID: ${{ secrets.PERM_SYNC_SHAREPOINT_DIRECTORY_ID }}
   PLATFORM_PAIR: linux-amd64
 jobs:
   integration-tests-mit:
@@ -207,6 +211,10 @@ jobs:
             -e JIRA_BASE_URL=${JIRA_BASE_URL} \
             -e JIRA_USER_EMAIL=${JIRA_USER_EMAIL} \
             -e JIRA_API_TOKEN=${JIRA_API_TOKEN} \
+            -e PERM_SYNC_SHAREPOINT_CLIENT_ID=${PERM_SYNC_SHAREPOINT_CLIENT_ID} \
+            -e PERM_SYNC_SHAREPOINT_PRIVATE_KEY="${PERM_SYNC_SHAREPOINT_PRIVATE_KEY}" \
+            -e PERM_SYNC_SHAREPOINT_CERTIFICATE_PASSWORD=${PERM_SYNC_SHAREPOINT_CERTIFICATE_PASSWORD} \
+            -e PERM_SYNC_SHAREPOINT_DIRECTORY_ID=${PERM_SYNC_SHAREPOINT_DIRECTORY_ID} \
             -e TEST_WEB_HOSTNAME=test-runner \
             -e MOCK_CONNECTOR_SERVER_HOST=mock_connector_server \
             -e MOCK_CONNECTOR_SERVER_PORT=8001 \

backend/tests/daily/connectors/sharepoint/test_sharepoint_connector.py

@@ -14,6 +14,8 @@ from onyx.connectors.models import ImageSection
 from onyx.connectors.sharepoint.connector import SharepointConnector
 from tests.daily.connectors.utils import load_all_docs_from_checkpoint_connector
 
+# NOTE: Sharepoint site for tests is "sharepoint-tests"
+
 
 @dataclass
 class ExpectedDocument:
@@ -26,7 +28,7 @@ class ExpectedDocument:
 EXPECTED_DOCUMENTS = [
     ExpectedDocument(
         semantic_identifier="test1.docx",
-        content="Test1 password: 1234",
+        content="test1",
         folder_path="test",
     ),
     ExpectedDocument(
@@ -45,14 +47,22 @@ EXPECTED_DOCUMENTS = [
         folder_path=None,
         library="Other Library",
     ),
+]
+
+EXPECTED_PAGES = [
     ExpectedDocument(
-        semantic_identifier="Book.xlsx",
-        content="## Sheet1\n| exel | 9090 |\n| --- | --- |",
+        semantic_identifier="CollabHome",
+        content=(
+            "# Home\n\nDisplay recent news.\n\n## News\n\nShow recent activities from your site\n\n"
+            "## Site activity\n\n## Quick links\n\nLearn about a team site\n\nLearn how to add a page\n\n"
+            "Add links to important documents and pages.\n\n## Quick links\n\nDocuments\n\n"
+            "Add a document library\n\n## Document library"
+        ),
         folder_path=None,
     ),
     ExpectedDocument(
-        semantic_identifier="Presentation.pptx",
-        content="<!-- Slide number: 1 -->\n# Powerpoint 6565\n\n### Notes:\n6767",
+        semantic_identifier="Home",
+        content="# Home",
         folder_path=None,
     ),
 ]
@@ -75,7 +85,7 @@ def verify_document_content(doc: Document, expected: ExpectedDocument) -> None:
     assert doc.semantic_identifier == expected.semantic_identifier
     assert len(doc.sections) == 1
     assert doc.sections[0].text is not None
-    assert expected.content in doc.sections[0].text
+    assert expected.content == doc.sections[0].text
     verify_document_metadata(doc)
 
 
@@ -294,9 +304,7 @@ def test_sharepoint_connector_poll(
         mock_store_image,
     ):
         # Initialize connector with the base site URL
-        connector = SharepointConnector(
-            sites=["https://danswerai.sharepoint.com/sites/sharepoint-tests"]
-        )
+        connector = SharepointConnector(sites=[os.environ["SHAREPOINT_SITE"]])
 
         # Load credentials
         connector.load_credentials(sharepoint_credentials)
@@ -320,10 +328,11 @@ def test_sharepoint_connector_poll(
         ), "Should only find one document in the time window"
         doc = found_documents[0]
         assert doc.semantic_identifier == "test1.docx"
-        verify_document_metadata(doc)
         verify_document_content(
             doc,
-            [d for d in EXPECTED_DOCUMENTS if d.semantic_identifier == "test1.docx"][0],
+            next(
+                d for d in EXPECTED_DOCUMENTS if d.semantic_identifier == "test1.docx"
+            ),
         )
 
 
@@ -336,56 +345,24 @@ def test_sharepoint_connector_pages(
         "onyx.connectors.sharepoint.connector.store_image_and_create_section",
         mock_store_image,
     ):
-        # Initialize connector with the base site URL
         connector = SharepointConnector(
-            sites=["https://danswerai.sharepoint.com/sites/sharepoint-tests-pages"],
+            sites=[os.environ["SHAREPOINT_SITE"]],
             include_site_pages=True,
             include_site_documents=False,
         )
 
-        # Load credentials
         connector.load_credentials(sharepoint_credentials)
 
-        # Get documents within the time window
         found_documents = load_all_docs_from_checkpoint_connector(
             connector=connector,
             start=0,
             end=time.time(),
         )
 
-        # Should only find CollabHome
-        assert len(found_documents) == 1, "Should only find one page"
-        doc = found_documents[0]
-        assert doc.semantic_identifier == "CollabHome"
-        verify_document_metadata(doc)
-        assert len(doc.sections) == 1
-        assert (
-            doc.sections[0].text
-            == """
-# Home
+        assert len(found_documents) == len(
+            EXPECTED_PAGES
+        ), "Should find all pages in test site"
 
-Display recent news.
-
-## News
-
-Show recent activities from your site
-
-## Site activity
-
-## Quick links
-
-Learn about a team site
-
-Learn how to add a page
-
-Add links to important documents and pages.
-
-## Quick links
-
-Documents
-
-Add a document library
-
-## Document library
-""".strip()
-        )
+        for expected in EXPECTED_PAGES:
+            doc = find_document(found_documents, expected.semantic_identifier)
+            verify_document_content(doc, expected)

backend/tests/integration/connector_job_tests/sharepoint/test_sharepoint_permissions.py

@@ -1,3 +1,4 @@
+import os
 from typing import List
 from uuid import UUID
 
@@ -106,6 +107,10 @@ def get_documents_by_permission_type(
     return public_docs
 
 
+@pytest.mark.skipif(
+    os.environ.get("ENABLE_PAID_ENTERPRISE_EDITION_FEATURES", "").lower() != "true",
+    reason="Permission tests are enterprise only",
+)
 def test_public_documents_accessible_by_all_users(
     sharepoint_test_env_setup: SharepointTestEnvSetupTuple,
 ) -> None:
@@ -157,6 +162,10 @@ def test_public_documents_accessible_by_all_users(
         )
 
 
+@pytest.mark.skipif(
+    os.environ.get("ENABLE_PAID_ENTERPRISE_EDITION_FEATURES", "").lower() != "true",
+    reason="Permission tests are enterprise only",
+)
 def test_group_based_permissions(
     sharepoint_test_env_setup: SharepointTestEnvSetupTuple,
 ) -> None:

deployment/docker_compose/.env

@@ -0,0 +1,67 @@
+# Fill in the values and copy the contents of this file to .env in the deployment directory.
+# Some valid default values are provided where applicable, delete the variables which you don't set values for.
+# This is only necessary when using the docker-compose.prod.yml compose file.
+
+
+# Could be something like onyx.companyname.com
+WEB_DOMAIN=http://localhost:3000
+
+
+# NOTE: Generative AI configurations are done via the UI now
+EXA_API_KEY=
+
+# The following are for configuring User Authentication, supported flows are:
+# disabled
+# basic (standard username / password)
+# google_oauth (login with google/gmail account)
+# oidc (only in Onyx enterprise edition)
+# saml (only in Onyx enterprise edition)
+AUTH_TYPE=google_oauth
+
+# Set the values below to use with Google OAuth
+GOOGLE_OAUTH_CLIENT_ID=
+GOOGLE_OAUTH_CLIENT_SECRET=
+SECRET=
+
+# if using basic auth and you want to require email verification, 
+# then uncomment / set the following
+#REQUIRE_EMAIL_VERIFICATION=true
+#SMTP_USER=your-email@company.com
+#SMTP_PASS=your-gmail-password
+
+# The below are only needed if you aren't using gmail as your SMTP
+#SMTP_SERVER=  
+#SMTP_PORT=
+# When missing SMTP_USER, this is used instead
+#EMAIL_FROM=
+
+# OpenID Connect (OIDC)
+#OPENID_CONFIG_URL=
+
+# SAML config directory for OneLogin compatible setups
+#SAML_CONF_DIR=
+
+
+# How long before user needs to reauthenticate, default to 7 days. (cookie expiration time)
+SESSION_EXPIRE_TIME_SECONDS=604800
+
+
+# Use the below to specify a list of allowed user domains, only checked if user Auth is turned on
+# e.g. `VALID_EMAIL_DOMAINS=example.com,example.org` will only allow users
+# with an @example.com or an @example.org email
+#VALID_EMAIL_DOMAINS=
+
+
+# Default values here are what Postgres uses by default, feel free to change.
+POSTGRES_USER=postgres
+POSTGRES_PASSWORD=password
+
+
+# Default values here for the read-only user for the knowledge graph and other future read-only purposes. 
+# Please change password!
+DB_READONLY_USER=db_readonly_user
+DB_READONLY_PASSWORD=password
+
+# If setting the vespa language is required, set this ('en', 'de', etc.).
+# See: https://docs.vespa.ai/en/linguistics.html 
+#VESPA_LANGUAGE_OVERRIDE=