fix

.github/workflows/pr-python-connector-tests.yml

@@ -9,10 +9,6 @@ on:
     - cron: "0 16 * * *"
 
 env:
-  # AWS
-  AWS_ACCESS_KEY_ID_DAILY_CONNECTOR_TESTS: ${{ secrets.AWS_ACCESS_KEY_ID_DAILY_CONNECTOR_TESTS }}
-  AWS_SECRET_ACCESS_KEY_DAILY_CONNECTOR_TESTS: ${{ secrets.AWS_SECRET_ACCESS_KEY_DAILY_CONNECTOR_TESTS }}
-  
   # Confluence
   CONFLUENCE_TEST_SPACE_URL: ${{ secrets.CONFLUENCE_TEST_SPACE_URL }}
   CONFLUENCE_TEST_SPACE: ${{ secrets.CONFLUENCE_TEST_SPACE }}

backend/ee/onyx/configs/app_configs.py

@@ -25,10 +25,6 @@ SAML_CONF_DIR = os.environ.get("SAML_CONF_DIR") or "/app/ee/onyx/configs/saml_co
 #####
 # Auto Permission Sync
 #####
-DEFAULT_PERMISSION_DOC_SYNC_FREQUENCY = int(
-    os.environ.get("DEFAULT_PERMISSION_DOC_SYNC_FREQUENCY") or 5 * 60
-)
-
 # In seconds, default is 5 minutes
 CONFLUENCE_PERMISSION_GROUP_SYNC_FREQUENCY = int(
     os.environ.get("CONFLUENCE_PERMISSION_GROUP_SYNC_FREQUENCY") or 5 * 60
@@ -43,7 +39,6 @@ CONFLUENCE_ANONYMOUS_ACCESS_IS_PUBLIC = (
 CONFLUENCE_PERMISSION_DOC_SYNC_FREQUENCY = int(
     os.environ.get("CONFLUENCE_PERMISSION_DOC_SYNC_FREQUENCY") or 5 * 60
 )
-
 NUM_PERMISSION_WORKERS = int(os.environ.get("NUM_PERMISSION_WORKERS") or 2)
 
 
@@ -77,13 +72,6 @@ OAUTH_GOOGLE_DRIVE_CLIENT_SECRET = os.environ.get(
     "OAUTH_GOOGLE_DRIVE_CLIENT_SECRET", ""
 )
 
-GOOGLE_DRIVE_PERMISSION_GROUP_SYNC_FREQUENCY = int(
-    os.environ.get("GOOGLE_DRIVE_PERMISSION_GROUP_SYNC_FREQUENCY") or 5 * 60
-)
-
-SLACK_PERMISSION_DOC_SYNC_FREQUENCY = int(
-    os.environ.get("SLACK_PERMISSION_DOC_SYNC_FREQUENCY") or 5 * 60
-)
 
 # The posthog client does not accept empty API keys or hosts however it fails silently
 # when the capture is called. These defaults prevent Posthog issues from breaking the Onyx app

backend/ee/onyx/external_permissions/sync_params.py

@@ -3,8 +3,6 @@ from collections.abc import Generator
 
 from ee.onyx.configs.app_configs import CONFLUENCE_PERMISSION_DOC_SYNC_FREQUENCY
 from ee.onyx.configs.app_configs import CONFLUENCE_PERMISSION_GROUP_SYNC_FREQUENCY
-from ee.onyx.configs.app_configs import GOOGLE_DRIVE_PERMISSION_GROUP_SYNC_FREQUENCY
-from ee.onyx.configs.app_configs import SLACK_PERMISSION_DOC_SYNC_FREQUENCY
 from ee.onyx.db.external_perm import ExternalUserGroup
 from ee.onyx.external_permissions.confluence.doc_sync import confluence_doc_sync
 from ee.onyx.external_permissions.confluence.group_sync import confluence_group_sync
@@ -68,13 +66,13 @@ GROUP_PERMISSIONS_IS_CC_PAIR_AGNOSTIC: set[DocumentSource] = {
 DOC_PERMISSION_SYNC_PERIODS: dict[DocumentSource, int] = {
     # Polling is not supported so we fetch all doc permissions every 5 minutes
     DocumentSource.CONFLUENCE: CONFLUENCE_PERMISSION_DOC_SYNC_FREQUENCY,
-    DocumentSource.SLACK: SLACK_PERMISSION_DOC_SYNC_FREQUENCY,
+    DocumentSource.SLACK: 5 * 60,
 }
 
 # If nothing is specified here, we run the doc_sync every time the celery beat runs
 EXTERNAL_GROUP_SYNC_PERIODS: dict[DocumentSource, int] = {
     # Polling is not supported so we fetch all group permissions every 30 minutes
-    DocumentSource.GOOGLE_DRIVE: GOOGLE_DRIVE_PERMISSION_GROUP_SYNC_FREQUENCY,
+    DocumentSource.GOOGLE_DRIVE: 5 * 60,
     DocumentSource.CONFLUENCE: CONFLUENCE_PERMISSION_GROUP_SYNC_FREQUENCY,
 }
 

backend/ee/onyx/server/tenants/provisioning.py

@@ -505,8 +505,11 @@ async def setup_tenant(tenant_id: str) -> None:
     try:
         token = CURRENT_TENANT_ID_CONTEXTVAR.set(tenant_id)
 
-        # Run Alembic migrations
-        await asyncio.to_thread(run_alembic_migrations, tenant_id)
+        # Run Alembic migrations in a way that isolates it from the current event loop
+        # Create a new event loop for this synchronous operation
+        loop = asyncio.get_event_loop()
+        # Use run_in_executor which properly isolates the thread execution
+        await loop.run_in_executor(None, lambda: run_alembic_migrations(tenant_id))
 
         # Configure the tenant with default settings
         with get_session_with_tenant(tenant_id=tenant_id) as db_session:

backend/onyx/background/celery/apps/beat.py

@@ -1,5 +1,6 @@
 from datetime import timedelta
 from typing import Any
+from typing import cast
 
 from celery import Celery
 from celery import signals
@@ -9,10 +10,12 @@ from celery.utils.log import get_task_logger
 
 import onyx.background.celery.apps.app_base as app_base
 from onyx.background.celery.tasks.beat_schedule import CLOUD_BEAT_MULTIPLIER_DEFAULT
+from onyx.configs.constants import ONYX_CLOUD_REDIS_RUNTIME
+from onyx.configs.constants import ONYX_CLOUD_TENANT_ID
 from onyx.configs.constants import POSTGRES_CELERY_BEAT_APP_NAME
 from onyx.db.engine import get_all_tenant_ids
 from onyx.db.engine import SqlEngine
-from onyx.server.runtime.onyx_runtime import OnyxRuntime
+from onyx.redis.redis_pool import get_redis_replica_client
 from onyx.utils.variable_functionality import fetch_versioned_implementation
 from shared_configs.configs import IGNORED_SYNCING_TENANT_LIST
 from shared_configs.configs import MULTI_TENANT
@@ -138,6 +141,8 @@ class DynamicTenantScheduler(PersistentScheduler):
         """Only updates the actual beat schedule on the celery app when it changes"""
         do_update = False
 
+        r = get_redis_replica_client(tenant_id=ONYX_CLOUD_TENANT_ID)
+
         task_logger.debug("_try_updating_schedule starting")
 
         tenant_ids = get_all_tenant_ids()
@@ -147,7 +152,16 @@ class DynamicTenantScheduler(PersistentScheduler):
         current_schedule = self.schedule.items()
 
         # get potential new state
-        beat_multiplier = OnyxRuntime.get_beat_multiplier()
+        beat_multiplier = CLOUD_BEAT_MULTIPLIER_DEFAULT
+        beat_multiplier_raw = r.get(f"{ONYX_CLOUD_REDIS_RUNTIME}:beat_multiplier")
+        if beat_multiplier_raw is not None:
+            try:
+                beat_multiplier_bytes = cast(bytes, beat_multiplier_raw)
+                beat_multiplier = float(beat_multiplier_bytes.decode())
+            except ValueError:
+                task_logger.error(
+                    f"Invalid beat_multiplier value: {beat_multiplier_raw}"
+                )
 
         new_schedule = self._generate_schedule(tenant_ids, beat_multiplier)
 

backend/onyx/background/celery/tasks/beat_schedule.py

@@ -21,7 +21,6 @@ BEAT_EXPIRES_DEFAULT = 15 * 60  # 15 minutes (in seconds)
 # we have a better implementation (backpressure, etc)
 # Note that DynamicTenantScheduler can adjust the runtime value for this via Redis
 CLOUD_BEAT_MULTIPLIER_DEFAULT = 8.0
-CLOUD_DOC_PERMISSION_SYNC_MULTIPLIER_DEFAULT = 1.0
 
 # tasks that run in either self-hosted on cloud
 beat_task_templates: list[dict] = []

backend/onyx/background/celery/tasks/doc_permission_syncing/tasks.py

@@ -17,7 +17,6 @@ from redis.exceptions import LockError
 from redis.lock import Lock as RedisLock
 from sqlalchemy.orm import Session
 
-from ee.onyx.configs.app_configs import DEFAULT_PERMISSION_DOC_SYNC_FREQUENCY
 from ee.onyx.db.connector_credential_pair import get_all_auto_sync_cc_pairs
 from ee.onyx.db.document import upsert_document_external_perms
 from ee.onyx.external_permissions.sync_params import DOC_PERMISSION_SYNC_PERIODS
@@ -64,7 +63,6 @@ from onyx.redis.redis_connector_doc_perm_sync import RedisConnectorPermissionSyn
 from onyx.redis.redis_pool import get_redis_client
 from onyx.redis.redis_pool import get_redis_replica_client
 from onyx.redis.redis_pool import redis_lock_dump
-from onyx.server.runtime.onyx_runtime import OnyxRuntime
 from onyx.server.utils import make_short_id
 from onyx.utils.logger import doc_permission_sync_ctx
 from onyx.utils.logger import format_error_for_logging
@@ -108,10 +106,9 @@ def _is_external_doc_permissions_sync_due(cc_pair: ConnectorCredentialPair) -> b
 
     source_sync_period = DOC_PERMISSION_SYNC_PERIODS.get(cc_pair.connector.source)
 
+    # If RESTRICTED_FETCH_PERIOD[source] is None, we always run the sync.
     if not source_sync_period:
-        source_sync_period = DEFAULT_PERMISSION_DOC_SYNC_FREQUENCY
-
-    source_sync_period *= int(OnyxRuntime.get_doc_permission_sync_multiplier())
+        return True
 
     # If the last sync is greater than the full fetch period, we run the sync
     next_sync = last_perm_sync + timedelta(seconds=source_sync_period)

backend/onyx/background/celery/tasks/indexing/tasks.py

@@ -72,7 +72,6 @@ from onyx.redis.redis_pool import get_redis_replica_client
 from onyx.redis.redis_pool import redis_lock_dump
 from onyx.redis.redis_pool import SCAN_ITER_COUNT_DEFAULT
 from onyx.redis.redis_utils import is_fence
-from onyx.server.runtime.onyx_runtime import OnyxRuntime
 from onyx.utils.logger import setup_logger
 from onyx.utils.variable_functionality import global_version
 from shared_configs.configs import INDEXING_MODEL_SERVER_HOST
@@ -402,11 +401,7 @@ def check_for_indexing(self: Task, *, tenant_id: str) -> int | None:
                     logger.warning(f"Adding {key_bytes} to the lookup table.")
                     redis_client.sadd(OnyxRedisConstants.ACTIVE_FENCES, key_bytes)
 
-            redis_client.set(
-                OnyxRedisSignals.BLOCK_BUILD_FENCE_LOOKUP_TABLE,
-                1,
-                ex=OnyxRuntime.get_build_fence_lookup_table_interval(),
-            )
+            redis_client.set(OnyxRedisSignals.BLOCK_BUILD_FENCE_LOOKUP_TABLE, 1, ex=300)
 
         # 1/3: KICKOFF
 

backend/onyx/configs/constants.py

@@ -382,7 +382,6 @@ ONYX_CLOUD_TENANT_ID = "cloud"
 
 # the redis namespace for runtime variables
 ONYX_CLOUD_REDIS_RUNTIME = "runtime"
-CLOUD_BUILD_FENCE_LOOKUP_TABLE_INTERVAL_DEFAULT = 600
 
 
 class OnyxCeleryTask:

backend/onyx/connectors/blob/connector.py

@@ -87,7 +87,7 @@ class BlobStorageConnector(LoadConnector, PollConnector):
                 credentials.get(key)
                 for key in ["aws_access_key_id", "aws_secret_access_key"]
             ):
-                raise ConnectorMissingCredentialError("Amazon S3")
+                raise ConnectorMissingCredentialError("Google Cloud Storage")
 
             session = boto3.Session(
                 aws_access_key_id=credentials["aws_access_key_id"],

backend/onyx/connectors/egnyte/connector.py

@@ -28,9 +28,8 @@ from onyx.connectors.models import TextSection
 from onyx.file_processing.extract_file_text import detect_encoding
 from onyx.file_processing.extract_file_text import extract_file_text
 from onyx.file_processing.extract_file_text import get_file_ext
-from onyx.file_processing.extract_file_text import is_accepted_file_ext
 from onyx.file_processing.extract_file_text import is_text_file_extension
-from onyx.file_processing.extract_file_text import OnyxExtensionType
+from onyx.file_processing.extract_file_text import is_valid_file_ext
 from onyx.file_processing.extract_file_text import read_text_file
 from onyx.utils.logger import setup_logger
 from onyx.utils.retry_wrapper import request_with_retries
@@ -70,9 +69,7 @@ def _process_egnyte_file(
 
     file_name = file_metadata["name"]
     extension = get_file_ext(file_name)
-    if not is_accepted_file_ext(
-        extension, OnyxExtensionType.Plain | OnyxExtensionType.Document
-    ):
+    if not is_valid_file_ext(extension):
         logger.warning(f"Skipping file '{file_name}' with extension '{extension}'")
         return None
 

backend/onyx/connectors/file/connector.py

@@ -22,9 +22,8 @@ from onyx.db.engine import get_session_with_current_tenant
 from onyx.db.pg_file_store import get_pgfilestore_by_file_name
 from onyx.file_processing.extract_file_text import extract_text_and_images
 from onyx.file_processing.extract_file_text import get_file_ext
-from onyx.file_processing.extract_file_text import is_accepted_file_ext
+from onyx.file_processing.extract_file_text import is_valid_file_ext
 from onyx.file_processing.extract_file_text import load_files_from_zip
-from onyx.file_processing.extract_file_text import OnyxExtensionType
 from onyx.file_processing.image_utils import store_image_and_create_section
 from onyx.file_store.file_store import get_default_file_store
 from onyx.utils.logger import setup_logger
@@ -52,7 +51,7 @@ def _read_files_and_metadata(
             file_content, ignore_dirs=True
         ):
             yield os.path.join(directory_path, file_info.filename), subfile, metadata
-    elif is_accepted_file_ext(extension, OnyxExtensionType.All):
+    elif is_valid_file_ext(extension):
         yield file_name, file_content, metadata
     else:
         logger.warning(f"Skipping file '{file_name}' with extension '{extension}'")
@@ -123,7 +122,7 @@ def _process_file(
         logger.warning(f"No file record found for '{file_name}' in PG; skipping.")
         return []
 
-    if not is_accepted_file_ext(extension, OnyxExtensionType.All):
+    if not is_valid_file_ext(extension):
         logger.warning(
             f"Skipping file '{file_name}' with unrecognized extension '{extension}'"
         )

backend/onyx/connectors/highspot/connector.py

@@ -20,8 +20,8 @@ from onyx.connectors.models import ConnectorMissingCredentialError
 from onyx.connectors.models import Document
 from onyx.connectors.models import SlimDocument
 from onyx.connectors.models import TextSection
-from onyx.file_processing.extract_file_text import ALL_ACCEPTED_FILE_EXTENSIONS
 from onyx.file_processing.extract_file_text import extract_file_text
+from onyx.file_processing.extract_file_text import VALID_FILE_EXTENSIONS
 from onyx.indexing.indexing_heartbeat import IndexingHeartbeatInterface
 from onyx.utils.logger import setup_logger
 
@@ -298,7 +298,7 @@ class HighspotConnector(LoadConnector, PollConnector, SlimConnector):
 
             elif (
                 is_valid_format
-                and file_extension in ALL_ACCEPTED_FILE_EXTENSIONS
+                and file_extension in VALID_FILE_EXTENSIONS
                 and can_download
             ):
                 # For documents, try to get the text content

backend/onyx/document_index/vespa/index.py

@@ -821,30 +821,26 @@ class VespaIndex(DocumentIndex):
         num_to_retrieve: int = NUM_RETURNED_HITS,
         offset: int = 0,
     ) -> list[InferenceChunkUncleaned]:
-        vespa_where_clauses = build_vespa_filters(
-            filters, include_hidden=True, remove_trailing_and=True
+        vespa_where_clauses = build_vespa_filters(filters, include_hidden=True)
+        yql = (
+            YQL_BASE.format(index_name=self.index_name)
+            + vespa_where_clauses
+            + '({grammar: "weakAnd"}userInput(@query) '
+            # `({defaultIndex: "content_summary"}userInput(@query))` section is
+            # needed for highlighting while the N-gram highlighting is broken /
+            # not working as desired
+            + f'or ({{defaultIndex: "{CONTENT_SUMMARY}"}}userInput(@query)))'
         )
-        yql = YQL_BASE.format(index_name=self.index_name) + vespa_where_clauses
 
         params: dict[str, str | int] = {
             "yql": yql,
+            "query": query,
             "hits": num_to_retrieve,
             "offset": 0,
             "ranking.profile": "admin_search",
             "timeout": VESPA_TIMEOUT,
         }
 
-        if len(query.strip()) > 0:
-            yql += (
-                ' and ({grammar: "weakAnd"}userInput(@query) '
-                # `({defaultIndex: "content_summary"}userInput(@query))` section is
-                # needed for highlighting while the N-gram highlighting is broken /
-                # not working as desired
-                + f'or ({{defaultIndex: "{CONTENT_SUMMARY}"}}userInput(@query)))'
-            )
-            params["yql"] = yql
-            params["query"] = query
-
         return query_vespa(params)
 
     # Retrieves chunk information for a document:

backend/onyx/file_processing/extract_file_text.py

@@ -7,8 +7,6 @@ from collections.abc import Callable
 from collections.abc import Iterator
 from collections.abc import Sequence
 from email.parser import Parser as EmailParser
-from enum import auto
-from enum import IntFlag
 from io import BytesIO
 from pathlib import Path
 from typing import Any
@@ -37,7 +35,7 @@ logger = setup_logger()
 
 TEXT_SECTION_SEPARATOR = "\n\n"
 
-ACCEPTED_PLAIN_TEXT_FILE_EXTENSIONS = [
+PLAIN_TEXT_FILE_EXTENSIONS = [
     ".txt",
     ".md",
     ".mdx",
@@ -51,7 +49,7 @@ ACCEPTED_PLAIN_TEXT_FILE_EXTENSIONS = [
     ".yaml",
 ]
 
-ACCEPTED_DOCUMENT_FILE_EXTENSIONS = [
+VALID_FILE_EXTENSIONS = PLAIN_TEXT_FILE_EXTENSIONS + [
     ".pdf",
     ".docx",
     ".pptx",
@@ -59,21 +57,12 @@ ACCEPTED_DOCUMENT_FILE_EXTENSIONS = [
     ".eml",
     ".epub",
     ".html",
-]
-
-ACCEPTED_IMAGE_FILE_EXTENSIONS = [
     ".png",
     ".jpg",
     ".jpeg",
     ".webp",
 ]
 
-ALL_ACCEPTED_FILE_EXTENSIONS = (
-    ACCEPTED_PLAIN_TEXT_FILE_EXTENSIONS
-    + ACCEPTED_DOCUMENT_FILE_EXTENSIONS
-    + ACCEPTED_IMAGE_FILE_EXTENSIONS
-)
-
 IMAGE_MEDIA_TYPES = [
     "image/png",
     "image/jpeg",
@@ -81,15 +70,8 @@ IMAGE_MEDIA_TYPES = [
 ]
 
 
-class OnyxExtensionType(IntFlag):
-    Plain = auto()
-    Document = auto()
-    Multimedia = auto()
-    All = Plain | Document | Multimedia
-
-
 def is_text_file_extension(file_name: str) -> bool:
-    return any(file_name.endswith(ext) for ext in ACCEPTED_PLAIN_TEXT_FILE_EXTENSIONS)
+    return any(file_name.endswith(ext) for ext in PLAIN_TEXT_FILE_EXTENSIONS)
 
 
 def get_file_ext(file_path_or_name: str | Path) -> str:
@@ -101,20 +83,8 @@ def is_valid_media_type(media_type: str) -> bool:
     return media_type in IMAGE_MEDIA_TYPES
 
 
-def is_accepted_file_ext(ext: str, ext_type: OnyxExtensionType) -> bool:
-    if ext_type & OnyxExtensionType.Plain:
-        if ext in ACCEPTED_PLAIN_TEXT_FILE_EXTENSIONS:
-            return True
-
-    if ext_type & OnyxExtensionType.Document:
-        if ext in ACCEPTED_DOCUMENT_FILE_EXTENSIONS:
-            return True
-
-    if ext_type & OnyxExtensionType.Multimedia:
-        if ext in ACCEPTED_IMAGE_FILE_EXTENSIONS:
-            return True
-
-    return False
+def is_valid_file_ext(ext: str) -> bool:
+    return ext in VALID_FILE_EXTENSIONS
 
 
 def is_text_file(file: IO[bytes]) -> bool:
@@ -412,9 +382,6 @@ def extract_file_text(
     """
     Legacy function that returns *only text*, ignoring embedded images.
     For backward-compatibility in code that only wants text.
-
-    NOTE: Ignoring seems to be defined as returning an empty string for files it can't
-    handle (such as images).
     """
     extension_to_function: dict[str, Callable[[IO[Any]], str]] = {
         ".pdf": pdf_to_text,
@@ -438,9 +405,7 @@ def extract_file_text(
         if extension is None:
             extension = get_file_ext(file_name)
 
-        if is_accepted_file_ext(
-            extension, OnyxExtensionType.Plain | OnyxExtensionType.Document
-        ):
+        if is_valid_file_ext(extension):
             func = extension_to_function.get(extension, file_io_to_text)
             file.seek(0)
             return func(file)

backend/onyx/server/runtime/onyx_runtime.py

@@ -1,19 +1,10 @@
 import io
-from typing import cast
 
 from PIL import Image
 
-from onyx.background.celery.tasks.beat_schedule import CLOUD_BEAT_MULTIPLIER_DEFAULT
-from onyx.background.celery.tasks.beat_schedule import (
-    CLOUD_DOC_PERMISSION_SYNC_MULTIPLIER_DEFAULT,
-)
-from onyx.configs.constants import CLOUD_BUILD_FENCE_LOOKUP_TABLE_INTERVAL_DEFAULT
-from onyx.configs.constants import ONYX_CLOUD_REDIS_RUNTIME
-from onyx.configs.constants import ONYX_CLOUD_TENANT_ID
 from onyx.configs.constants import ONYX_EMAILABLE_LOGO_MAX_DIM
 from onyx.db.engine import get_session_with_shared_schema
 from onyx.file_store.file_store import PostgresBackedFileStore
-from onyx.redis.redis_pool import get_redis_replica_client
 from onyx.utils.file import FileWithMimeType
 from onyx.utils.file import OnyxStaticFileManager
 from onyx.utils.variable_functionality import (
@@ -96,72 +87,3 @@ class OnyxRuntime:
         )
 
         return OnyxRuntime._get_with_static_fallback(db_filename, STATIC_FILENAME)
-
-    @staticmethod
-    def get_beat_multiplier() -> float:
-        """the beat multiplier is used to scale up or down the frequency of certain beat
-        tasks in the cloud. It has a significant effect on load and is useful to adjust
-        in real time."""
-
-        beat_multiplier: float = CLOUD_BEAT_MULTIPLIER_DEFAULT
-
-        r = get_redis_replica_client(tenant_id=ONYX_CLOUD_TENANT_ID)
-
-        beat_multiplier_raw = r.get(f"{ONYX_CLOUD_REDIS_RUNTIME}:beat_multiplier")
-        if beat_multiplier_raw is not None:
-            try:
-                beat_multiplier_bytes = cast(bytes, beat_multiplier_raw)
-                beat_multiplier = float(beat_multiplier_bytes.decode())
-            except ValueError:
-                pass
-
-        if beat_multiplier <= 0.0:
-            return 1.0
-
-        return beat_multiplier
-
-    @staticmethod
-    def get_doc_permission_sync_multiplier() -> float:
-        """Permission syncs are a significant source of load / queueing in the cloud."""
-
-        value: float = CLOUD_DOC_PERMISSION_SYNC_MULTIPLIER_DEFAULT
-
-        r = get_redis_replica_client(tenant_id=ONYX_CLOUD_TENANT_ID)
-
-        value_raw = r.get(f"{ONYX_CLOUD_REDIS_RUNTIME}:doc_permission_sync_multiplier")
-        if value_raw is not None:
-            try:
-                value_bytes = cast(bytes, value_raw)
-                value = float(value_bytes.decode())
-            except ValueError:
-                pass
-
-        if value <= 0.0:
-            return 1.0
-
-        return value
-
-    @staticmethod
-    def get_build_fence_lookup_table_interval() -> int:
-        """We maintain an active fence table to make lookups of existing fences efficient.
-        However, reconstructing the table is expensive, so adjusting it in realtime is useful.
-        """
-
-        interval: int = CLOUD_BUILD_FENCE_LOOKUP_TABLE_INTERVAL_DEFAULT
-
-        r = get_redis_replica_client(tenant_id=ONYX_CLOUD_TENANT_ID)
-
-        interval_raw = r.get(
-            f"{ONYX_CLOUD_REDIS_RUNTIME}:build_fence_lookup_table_interval"
-        )
-        if interval_raw is not None:
-            try:
-                interval_bytes = cast(bytes, interval_raw)
-                interval = int(interval_bytes.decode())
-            except ValueError:
-                pass
-
-        if interval <= 0.0:
-            return CLOUD_BUILD_FENCE_LOOKUP_TABLE_INTERVAL_DEFAULT
-
-        return interval

backend/tests/daily/connectors/blob/test_blob_connector.py

@@ -1,77 +0,0 @@
-import os
-from unittest.mock import MagicMock
-from unittest.mock import patch
-
-import pytest
-
-from onyx.configs.constants import BlobType
-from onyx.connectors.blob.connector import BlobStorageConnector
-from onyx.connectors.models import Document
-from onyx.connectors.models import TextSection
-from onyx.file_processing.extract_file_text import ACCEPTED_DOCUMENT_FILE_EXTENSIONS
-from onyx.file_processing.extract_file_text import ACCEPTED_IMAGE_FILE_EXTENSIONS
-from onyx.file_processing.extract_file_text import ACCEPTED_PLAIN_TEXT_FILE_EXTENSIONS
-from onyx.file_processing.extract_file_text import get_file_ext
-
-
-@pytest.fixture
-def blob_connector(request: pytest.FixtureRequest) -> BlobStorageConnector:
-    connector = BlobStorageConnector(
-        bucket_type=BlobType.S3, bucket_name="onyx-connector-tests"
-    )
-
-    connector.load_credentials(
-        {
-            "aws_access_key_id": os.environ["AWS_ACCESS_KEY_ID_DAILY_CONNECTOR_TESTS"],
-            "aws_secret_access_key": os.environ[
-                "AWS_SECRET_ACCESS_KEY_DAILY_CONNECTOR_TESTS"
-            ],
-        }
-    )
-
-    return connector
-
-
-@patch(
-    "onyx.file_processing.extract_file_text.get_unstructured_api_key",
-    return_value=None,
-)
-def test_blob_s3_connector(
-    mock_get_api_key: MagicMock, blob_connector: BlobStorageConnector
-) -> None:
-    """
-    Plain and document file types should be fully indexed.
-
-    Multimedia and unknown file types will be indexed by title only with one empty section.
-
-    This is intentional in order to allow searching by just the title even if we can't
-    index the file content.
-    """
-    all_docs: list[Document] = []
-    document_batches = blob_connector.load_from_state()
-    for doc_batch in document_batches:
-        for doc in doc_batch:
-            all_docs.append(doc)
-
-    #
-    assert len(all_docs) == 19
-
-    for doc in all_docs:
-        section = doc.sections[0]
-        assert isinstance(section, TextSection)
-
-        file_extension = get_file_ext(doc.semantic_identifier)
-        if file_extension in ACCEPTED_PLAIN_TEXT_FILE_EXTENSIONS:
-            assert len(section.text) > 0
-            continue
-
-        if file_extension in ACCEPTED_DOCUMENT_FILE_EXTENSIONS:
-            assert len(section.text) > 0
-            continue
-
-        if file_extension in ACCEPTED_IMAGE_FILE_EXTENSIONS:
-            assert len(section.text) == 0
-            continue
-
-        # unknown extension
-        assert len(section.text) == 0

web/src/app/admin/documents/explorer/Explorer.tsx

@@ -148,8 +148,7 @@ export function Explorer({
       clearTimeout(timeoutId);
     }
 
-    let doSearch = true;
-    if (doSearch) {
+    if (query && query.trim() !== "") {
       router.replace(
         `/admin/documents/explorer?query=${encodeURIComponent(query)}`
       );

web/src/app/chat/ChatPage.tsx

@@ -1384,7 +1384,6 @@ export function ChatPage({
           if (!packet) {
             continue;
           }
-          console.log("Packet:", JSON.stringify(packet));
 
           if (!initialFetchDetails) {
             if (!Object.hasOwn(packet, "user_message_id")) {
@@ -1730,7 +1729,6 @@ export function ChatPage({
         }
       }
     } catch (e: any) {
-      console.log("Error:", e);
       const errorMsg = e.message;
       upsertToCompleteMessageMap({
         messages: [
@@ -1758,13 +1756,11 @@ export function ChatPage({
         completeMessageMapOverride: currentMessageMap(completeMessageDetail),
       });
     }
-    console.log("Finished streaming");
     setAgenticGenerating(false);
     resetRegenerationState(currentSessionId());
 
     updateChatState("input");
     if (isNewSession) {
-      console.log("Setting up new session");
       if (finalMessage) {
         setSelectedMessageForDocDisplay(finalMessage.message_id);
       }

web/src/components/user/UserProvider.tsx

@@ -102,7 +102,7 @@ export function UserProvider({
   };
 
   // Use the custom token refresh hook
-  // useTokenRefresh(upToDateUser, fetchUser);
+  useTokenRefresh(upToDateUser, fetchUser);
 
   const updateUserTemperatureOverrideEnabled = async (enabled: boolean) => {
     try {