fix(file upload): Allow zip file upload via query param (#9432)

backend/onyx/background/celery/apps/primary.py

@@ -317,7 +317,6 @@ celery_app.autodiscover_tasks(
             "onyx.background.celery.tasks.docprocessing",
             "onyx.background.celery.tasks.evals",
             "onyx.background.celery.tasks.hierarchyfetching",
-            "onyx.background.celery.tasks.hooks",
             "onyx.background.celery.tasks.periodic",
             "onyx.background.celery.tasks.pruning",
             "onyx.background.celery.tasks.shared",

backend/onyx/background/celery/tasks/beat_schedule.py

@@ -9,7 +9,6 @@ from onyx.configs.app_configs import AUTO_LLM_UPDATE_INTERVAL_SECONDS
 from onyx.configs.app_configs import DISABLE_VECTOR_DB
 from onyx.configs.app_configs import ENABLE_OPENSEARCH_INDEXING_FOR_ONYX
 from onyx.configs.app_configs import ENTERPRISE_EDITION_ENABLED
-from onyx.configs.app_configs import HOOK_ENABLED
 from onyx.configs.app_configs import SCHEDULED_EVAL_DATASET_NAMES
 from onyx.configs.constants import ONYX_CLOUD_CELERY_TASK_PREFIX
 from onyx.configs.constants import OnyxCeleryPriority
@@ -362,19 +361,6 @@ if not MULTI_TENANT:
 
     tasks_to_schedule.extend(beat_task_templates)
 
-if not MULTI_TENANT and HOOK_ENABLED:
-    tasks_to_schedule.append(
-        {
-            "name": "hook-execution-log-cleanup",
-            "task": OnyxCeleryTask.HOOK_EXECUTION_LOG_CLEANUP_TASK,
-            "schedule": timedelta(days=1),
-            "options": {
-                "priority": OnyxCeleryPriority.LOW,
-                "expires": BEAT_EXPIRES_DEFAULT,
-            },
-        }
-    )
-
 
 def generate_cloud_tasks(
     beat_tasks: list[dict], beat_templates: list[dict], beat_multiplier: float

backend/onyx/background/celery/tasks/hooks/tasks.py

@@ -1,35 +0,0 @@
-from celery import shared_task
-
-from onyx.configs.app_configs import JOB_TIMEOUT
-from onyx.configs.constants import OnyxCeleryTask
-from onyx.db.engine.sql_engine import get_session_with_current_tenant
-from onyx.db.hook import cleanup_old_execution_logs__no_commit
-from onyx.utils.logger import setup_logger
-
-logger = setup_logger()
-
-_HOOK_EXECUTION_LOG_RETENTION_DAYS: int = 30
-
-
-@shared_task(
-    name=OnyxCeleryTask.HOOK_EXECUTION_LOG_CLEANUP_TASK,
-    ignore_result=True,
-    soft_time_limit=JOB_TIMEOUT,
-    trail=False,
-)
-def hook_execution_log_cleanup_task(*, tenant_id: str) -> None:  # noqa: ARG001
-    try:
-        with get_session_with_current_tenant() as db_session:
-            deleted: int = cleanup_old_execution_logs__no_commit(
-                db_session=db_session,
-                max_age_days=_HOOK_EXECUTION_LOG_RETENTION_DAYS,
-            )
-            db_session.commit()
-            if deleted:
-                logger.info(
-                    f"Deleted {deleted} hook execution log(s) older than "
-                    f"{_HOOK_EXECUTION_LOG_RETENTION_DAYS} days."
-                )
-    except Exception:
-        logger.exception("Failed to clean up hook execution logs")
-        raise

backend/onyx/configs/app_configs.py

@@ -318,9 +318,6 @@ VERIFY_CREATE_OPENSEARCH_INDEX_ON_INIT_MT = (
 OPENSEARCH_MIGRATION_GET_VESPA_CHUNKS_PAGE_SIZE = int(
     os.environ.get("OPENSEARCH_MIGRATION_GET_VESPA_CHUNKS_PAGE_SIZE") or 500
 )
-OPENSEARCH_OVERRIDE_DEFAULT_NUM_HYBRID_SEARCH_CANDIDATES = int(
-    os.environ.get("OPENSEARCH_DEFAULT_NUM_HYBRID_SEARCH_CANDIDATES") or 0
-)
 
 VESPA_HOST = os.environ.get("VESPA_HOST") or "localhost"
 # NOTE: this is used if and only if the vespa config server is accessible via a

backend/onyx/configs/constants.py

@@ -597,9 +597,6 @@ class OnyxCeleryTask:
     EXPORT_QUERY_HISTORY_TASK = "export_query_history_task"
     EXPORT_QUERY_HISTORY_CLEANUP_TASK = "export_query_history_cleanup_task"
 
-    # Hook execution log retention
-    HOOK_EXECUTION_LOG_CLEANUP_TASK = "hook_execution_log_cleanup_task"
-
     # Sandbox cleanup
     CLEANUP_IDLE_SANDBOXES = "cleanup_idle_sandboxes"
     CLEANUP_OLD_SNAPSHOTS = "cleanup_old_snapshots"

backend/onyx/document_index/opensearch/constants.py

@@ -3,10 +3,6 @@
 import os
 from enum import Enum
 
-from onyx.configs.app_configs import (
-    OPENSEARCH_OVERRIDE_DEFAULT_NUM_HYBRID_SEARCH_CANDIDATES,
-)
-
 
 DEFAULT_MAX_CHUNK_SIZE = 512
 
@@ -41,10 +37,10 @@ M = 32  # Set relatively high for better accuracy.
 # we have a much higher chance of all 10 of the final desired docs showing up
 # and getting scored. In worse situations, the final 10 docs don't even show up
 # as the final 10 (worse than just a miss at the reranking step).
-DEFAULT_NUM_HYBRID_SEARCH_CANDIDATES = (
-    OPENSEARCH_OVERRIDE_DEFAULT_NUM_HYBRID_SEARCH_CANDIDATES
-    if OPENSEARCH_OVERRIDE_DEFAULT_NUM_HYBRID_SEARCH_CANDIDATES > 0
-    else 750
+# Defaults to 100 for now. Initially this defaulted to 750 but we were seeing
+# poor search performance.
+DEFAULT_NUM_HYBRID_SUBQUERY_CANDIDATES = int(
+    os.environ.get("DEFAULT_NUM_HYBRID_SUBQUERY_CANDIDATES", 100)
 )
 
 # Number of vectors to examine to decide the top k neighbors for the HNSW
@@ -54,7 +50,7 @@ DEFAULT_NUM_HYBRID_SEARCH_CANDIDATES = (
 # larger than k, you can provide the size parameter to limit the final number of
 # results to k." from
 # https://docs.opensearch.org/latest/query-dsl/specialized/k-nn/index/#ef_search
-EF_SEARCH = DEFAULT_NUM_HYBRID_SEARCH_CANDIDATES
+EF_SEARCH = DEFAULT_NUM_HYBRID_SUBQUERY_CANDIDATES
 
 
 class HybridSearchSubqueryConfiguration(Enum):

backend/onyx/document_index/opensearch/search.py

@@ -15,7 +15,7 @@ from onyx.context.search.models import Tag
 from onyx.document_index.interfaces_new import TenantState
 from onyx.document_index.opensearch.constants import ASSUMED_DOCUMENT_AGE_DAYS
 from onyx.document_index.opensearch.constants import (
-    DEFAULT_NUM_HYBRID_SEARCH_CANDIDATES,
+    DEFAULT_NUM_HYBRID_SUBQUERY_CANDIDATES,
 )
 from onyx.document_index.opensearch.constants import (
     DEFAULT_OPENSEARCH_MAX_RESULT_WINDOW,
@@ -332,7 +332,7 @@ class DocumentQuery:
 
         # TODO(andrei, yuhong): We can tune this more dynamically based on
         # num_hits.
-        max_results_per_subquery = DEFAULT_NUM_HYBRID_SEARCH_CANDIDATES
+        max_results_per_subquery = DEFAULT_NUM_HYBRID_SUBQUERY_CANDIDATES
 
         hybrid_search_subqueries = DocumentQuery._get_hybrid_search_subqueries(
             query_text, query_vector, vector_candidates=max_results_per_subquery
@@ -460,7 +460,7 @@ class DocumentQuery:
         # search. This is higher than the number of results because the scoring
         # is hybrid. For a detailed breakdown, see where the default value is
         # set.
-        vector_candidates: int = DEFAULT_NUM_HYBRID_SEARCH_CANDIDATES,
+        vector_candidates: int = DEFAULT_NUM_HYBRID_SUBQUERY_CANDIDATES,
     ) -> list[dict[str, Any]]:
         """Returns subqueries for hybrid search.
 
@@ -546,7 +546,7 @@ class DocumentQuery:
     @staticmethod
     def _get_title_vector_similarity_search_query(
         query_vector: list[float],
-        vector_candidates: int = DEFAULT_NUM_HYBRID_SEARCH_CANDIDATES,
+        vector_candidates: int = DEFAULT_NUM_HYBRID_SUBQUERY_CANDIDATES,
     ) -> dict[str, Any]:
         return {
             "knn": {
@@ -560,7 +560,7 @@ class DocumentQuery:
     @staticmethod
     def _get_content_vector_similarity_search_query(
         query_vector: list[float],
-        vector_candidates: int = DEFAULT_NUM_HYBRID_SEARCH_CANDIDATES,
+        vector_candidates: int = DEFAULT_NUM_HYBRID_SUBQUERY_CANDIDATES,
     ) -> dict[str, Any]:
         return {
             "knn": {

backend/onyx/server/documents/connector.py

@@ -479,7 +479,9 @@ def is_zip_file(file: UploadFile) -> bool:
 
 
 def upload_files(
-    files: list[UploadFile], file_origin: FileOrigin = FileOrigin.CONNECTOR
+    files: list[UploadFile],
+    file_origin: FileOrigin = FileOrigin.CONNECTOR,
+    unzip: bool = True,
 ) -> FileUploadResponse:
 
     # Skip directories and known macOS metadata entries
@@ -502,31 +504,46 @@ def upload_files(
                 if seen_zip:
                     raise HTTPException(status_code=400, detail=SEEN_ZIP_DETAIL)
                 seen_zip = True
+
+                # Validate the zip by opening it (catches corrupt/non-zip files)
                 with zipfile.ZipFile(file.file, "r") as zf:
-                    zip_metadata_file_id = save_zip_metadata_to_file_store(
-                        zf, file_store
-                    )
-                    for file_info in zf.namelist():
-                        if zf.getinfo(file_info).is_dir():
-                            continue
-
-                        if not should_process_file(file_info):
-                            continue
-
-                        sub_file_bytes = zf.read(file_info)
-
-                        mime_type, __ = mimetypes.guess_type(file_info)
-                        if mime_type is None:
-                            mime_type = "application/octet-stream"
-
-                        file_id = file_store.save_file(
-                            content=BytesIO(sub_file_bytes),
-                            display_name=os.path.basename(file_info),
-                            file_origin=file_origin,
-                            file_type=mime_type,
+                    if unzip:
+                        zip_metadata_file_id = save_zip_metadata_to_file_store(
+                            zf, file_store
                         )
-                        deduped_file_paths.append(file_id)
-                        deduped_file_names.append(os.path.basename(file_info))
+                        for file_info in zf.namelist():
+                            if zf.getinfo(file_info).is_dir():
+                                continue
+
+                            if not should_process_file(file_info):
+                                continue
+
+                            sub_file_bytes = zf.read(file_info)
+
+                            mime_type, __ = mimetypes.guess_type(file_info)
+                            if mime_type is None:
+                                mime_type = "application/octet-stream"
+
+                            file_id = file_store.save_file(
+                                content=BytesIO(sub_file_bytes),
+                                display_name=os.path.basename(file_info),
+                                file_origin=file_origin,
+                                file_type=mime_type,
+                            )
+                            deduped_file_paths.append(file_id)
+                            deduped_file_names.append(os.path.basename(file_info))
+                        continue
+
+                # Store the zip as-is (unzip=False)
+                file.file.seek(0)
+                file_id = file_store.save_file(
+                    content=file.file,
+                    display_name=file.filename,
+                    file_origin=file_origin,
+                    file_type=file.content_type or "application/zip",
+                )
+                deduped_file_paths.append(file_id)
+                deduped_file_names.append(file.filename)
                 continue
 
             # Since we can't render docx files in the UI,
@@ -613,9 +630,10 @@ def _fetch_and_check_file_connector_cc_pair_permissions(
 @router.post("/admin/connector/file/upload", tags=PUBLIC_API_TAGS)
 def upload_files_api(
     files: list[UploadFile],
+    unzip: bool = True,
     _: User = Depends(current_curator_or_admin_user),
 ) -> FileUploadResponse:
-    return upload_files(files, FileOrigin.OTHER)
+    return upload_files(files, FileOrigin.OTHER, unzip=unzip)
 
 
 @router.get("/admin/connector/{connector_id}/files", tags=PUBLIC_API_TAGS)

backend/tests/external_dependency_unit/document_index/test_document_index_old.py

@@ -297,6 +297,10 @@ def index_batch_params(
 class TestDocumentIndexOld:
     """Tests the old DocumentIndex interface."""
 
+    # TODO(ENG-3864)(andrei): Re-enable this test.
+    @pytest.mark.xfail(
+        reason="Flaky test: Retrieved chunks vary non-deterministically before and after changing user projects and personas. Likely a timing issue with the index being updated."
+    )
     def test_update_single_can_clear_user_projects_and_personas(
         self,
         document_indices: list[DocumentIndex],

backend/tests/unit/onyx/server/test_upload_files.py

@@ -0,0 +1,109 @@
+import io
+import zipfile
+from unittest.mock import MagicMock
+from unittest.mock import patch
+from zipfile import BadZipFile
+
+import pytest
+from fastapi import UploadFile
+from starlette.datastructures import Headers
+
+from onyx.configs.constants import FileOrigin
+from onyx.server.documents.connector import upload_files
+
+
+def _create_test_zip() -> bytes:
+    """Create a simple in-memory zip file containing two text files."""
+    buf = io.BytesIO()
+    with zipfile.ZipFile(buf, "w") as zf:
+        zf.writestr("file1.txt", "hello")
+        zf.writestr("file2.txt", "world")
+    return buf.getvalue()
+
+
+def _make_upload_file(content: bytes, filename: str, content_type: str) -> UploadFile:
+    return UploadFile(
+        file=io.BytesIO(content),
+        filename=filename,
+        headers=Headers({"content-type": content_type}),
+    )
+
+
+@patch("onyx.server.documents.connector.get_default_file_store")
+def test_upload_zip_with_unzip_true_extracts_files(
+    mock_get_store: MagicMock,
+) -> None:
+    """When unzip=True (default), a zip upload is extracted into individual files."""
+    mock_store = MagicMock()
+    mock_store.save_file.side_effect = lambda **kwargs: f"id-{kwargs['display_name']}"
+    mock_get_store.return_value = mock_store
+
+    zip_bytes = _create_test_zip()
+    upload = _make_upload_file(zip_bytes, "test.zip", "application/zip")
+
+    result = upload_files([upload], FileOrigin.CONNECTOR)
+
+    # Should have extracted the two individual files, not stored the zip itself
+    assert len(result.file_paths) == 2
+    assert "id-file1.txt" in result.file_paths
+    assert "id-file2.txt" in result.file_paths
+    assert "file1.txt" in result.file_names
+    assert "file2.txt" in result.file_names
+
+
+@patch("onyx.server.documents.connector.get_default_file_store")
+def test_upload_zip_with_unzip_false_stores_zip_as_is(
+    mock_get_store: MagicMock,
+) -> None:
+    """When unzip=False, the zip file is stored as-is without extraction."""
+    mock_store = MagicMock()
+    mock_store.save_file.return_value = "zip-file-id"
+    mock_get_store.return_value = mock_store
+
+    zip_bytes = _create_test_zip()
+    upload = _make_upload_file(zip_bytes, "site_export.zip", "application/zip")
+
+    result = upload_files([upload], FileOrigin.CONNECTOR, unzip=False)
+
+    # Should store exactly one file (the zip itself)
+    assert len(result.file_paths) == 1
+    assert result.file_paths[0] == "zip-file-id"
+    assert result.file_names == ["site_export.zip"]
+    # No zip metadata should be created
+    assert result.zip_metadata_file_id is None
+
+    # Verify the stored content is a valid zip
+    saved_content: io.BytesIO = mock_store.save_file.call_args[1]["content"]
+    saved_content.seek(0)
+    with zipfile.ZipFile(saved_content, "r") as zf:
+        assert set(zf.namelist()) == {"file1.txt", "file2.txt"}
+
+
+@patch("onyx.server.documents.connector.get_default_file_store")
+def test_upload_invalid_zip_with_unzip_false_raises(
+    mock_get_store: MagicMock,
+) -> None:
+    """An invalid zip is rejected even when unzip=False (validation still runs)."""
+    mock_get_store.return_value = MagicMock()
+
+    bad_zip = _make_upload_file(b"not a zip", "bad.zip", "application/zip")
+
+    with pytest.raises(BadZipFile):
+        upload_files([bad_zip], FileOrigin.CONNECTOR, unzip=False)
+
+
+@patch("onyx.server.documents.connector.get_default_file_store")
+def test_upload_multiple_zips_rejected_when_unzip_false(
+    mock_get_store: MagicMock,
+) -> None:
+    """The seen_zip guard rejects a second zip even when unzip=False."""
+    mock_store = MagicMock()
+    mock_store.save_file.return_value = "zip-id"
+    mock_get_store.return_value = mock_store
+
+    zip_bytes = _create_test_zip()
+    zip1 = _make_upload_file(zip_bytes, "a.zip", "application/zip")
+    zip2 = _make_upload_file(zip_bytes, "b.zip", "application/zip")
+
+    with pytest.raises(Exception, match="Only one zip file"):
+        upload_files([zip1, zip2], FileOrigin.CONNECTOR, unzip=False)

web/src/app/admin/connectors/[connector]/pages/utils/google_site.ts

@@ -21,10 +21,13 @@ export const submitGoogleSite = async (
       formData.append("files", file);
     });
 
-    const response = await fetch("/api/manage/admin/connector/file/upload", {
-      method: "POST",
-      body: formData,
-    });
+    const response = await fetch(
+      "/api/manage/admin/connector/file/upload?unzip=false",
+      {
+        method: "POST",
+        body: formData,
+      }
+    );
     const responseJson = await response.json();
     if (!response.ok) {
       toast.error(`Unable to upload files - ${responseJson.detail}`);

web/src/refresh-pages/admin/UsersPage/UserRowActions.tsx

@@ -12,6 +12,7 @@ import {
   SvgKey,
 } from "@opal/icons";
 import { Disabled } from "@opal/core";
+import LineItem from "@/refresh-components/buttons/LineItem";
 import Popover from "@/refresh-components/Popover";
 import Separator from "@/refresh-components/Separator";
 import { Section } from "@/layouts/general-layouts";
@@ -78,18 +79,17 @@ export default function UserRowActions({
       return (
         <>
           {user.id && (
-            <Button
-              prominence="tertiary"
+            <LineItem
               icon={SvgUsers}
               onClick={() => openModal(Modal.EDIT_GROUPS)}
             >
               Groups &amp; Roles
-            </Button>
+            </LineItem>
           )}
           <Disabled disabled>
-            <Button prominence="tertiary" variant="danger" icon={SvgUserX}>
+            <LineItem danger icon={SvgUserX}>
               Deactivate User
-            </Button>
+            </LineItem>
           </Disabled>
           <Separator paddingXRem={0.5} />
           <Text as="p" secondaryBody text03 className="px-3 py-1">
@@ -102,20 +102,18 @@ export default function UserRowActions({
     switch (user.status) {
       case UserStatus.INVITED:
         return (
-          <Button
-            prominence="tertiary"
-            variant="danger"
+          <LineItem
+            danger
             icon={SvgXCircle}
             onClick={() => openModal(Modal.CANCEL_INVITE)}
           >
             Cancel Invite
-          </Button>
+          </LineItem>
         );
 
       case UserStatus.REQUESTED:
         return (
-          <Button
-            prominence="tertiary"
+          <LineItem
             icon={SvgUserCheck}
             onClick={() => {
               setPopoverOpen(false);
@@ -133,37 +131,34 @@ export default function UserRowActions({
             }}
           >
             Approve
-          </Button>
+          </LineItem>
         );
 
       case UserStatus.ACTIVE:
         return (
           <>
             {user.id && (
-              <Button
-                prominence="tertiary"
+              <LineItem
                 icon={SvgUsers}
                 onClick={() => openModal(Modal.EDIT_GROUPS)}
               >
                 Groups &amp; Roles
-              </Button>
+              </LineItem>
             )}
-            <Button
-              prominence="tertiary"
+            <LineItem
               icon={SvgKey}
               onClick={() => openModal(Modal.RESET_PASSWORD)}
             >
               Reset Password
-            </Button>
+            </LineItem>
             <Separator paddingXRem={0.5} />
-            <Button
-              prominence="tertiary"
-              variant="danger"
+            <LineItem
+              danger
               icon={SvgUserX}
               onClick={() => openModal(Modal.DEACTIVATE)}
             >
               Deactivate User
-            </Button>
+            </LineItem>
           </>
         );
 
@@ -171,38 +166,34 @@ export default function UserRowActions({
         return (
           <>
             {user.id && (
-              <Button
-                prominence="tertiary"
+              <LineItem
                 icon={SvgUsers}
                 onClick={() => openModal(Modal.EDIT_GROUPS)}
               >
                 Groups &amp; Roles
-              </Button>
+              </LineItem>
             )}
-            <Button
-              prominence="tertiary"
+            <LineItem
               icon={SvgKey}
               onClick={() => openModal(Modal.RESET_PASSWORD)}
             >
               Reset Password
-            </Button>
+            </LineItem>
             <Separator paddingXRem={0.5} />
-            <Button
-              prominence="tertiary"
+            <LineItem
               icon={SvgUserPlus}
               onClick={() => openModal(Modal.ACTIVATE)}
             >
               Activate User
-            </Button>
+            </LineItem>
             <Separator paddingXRem={0.5} />
-            <Button
-              prominence="tertiary"
-              variant="danger"
+            <LineItem
+              danger
               icon={SvgUserX}
               onClick={() => openModal(Modal.DELETE)}
             >
               Delete User
-            </Button>
+            </LineItem>
           </>
         );
 

web/tests/e2e/chat/project_files_visual_regression.spec.ts

@@ -169,7 +169,9 @@ test.describe("Project Files visual regression", () => {
       .first();
     await expect(iconWrapper).toBeVisible();
 
-    await expectElementScreenshot(filesSection, {
+    const container = page.locator("[data-main-container]");
+    await expect(container).toBeVisible();
+    await expectElementScreenshot(container, {
       name: "project-files-long-underscore-filename",
     });