k

update
2026-02-18 00:05:47 +00:00 · 2025-04-01 12:32:19 -07:00 · 2025-04-01 12:31:40 -07:00 · 2025-04-01 12:30:38 -07:00 · 2025-04-01 12:04:05 -07:00 · 2025-04-01 12:03:52 -07:00
7 changed files with 48 additions and 16 deletions
--- a/backend/onyx/auth/email_utils.py
+++ b/backend/onyx/auth/email_utils.py
@@ -23,7 +23,6 @@ from onyx.utils.url import add_url_params
 from onyx.utils.variable_functionality import fetch_versioned_implementation
 from shared_configs.configs import MULTI_TENANT

-
 HTML_EMAIL_TEMPLATE = """\
 <!DOCTYPE html>
 <html lang="en">
--- a/backend/onyx/background/celery/tasks/doc_permission_syncing/tasks.py
+++ b/backend/onyx/background/celery/tasks/doc_permission_syncing/tasks.py
@@ -886,8 +886,11 @@ def monitor_ccpair_permissions_taskset(
        record_type=RecordType.PERMISSION_SYNC_PROGRESS,
        data={
            "cc_pair_id": cc_pair_id,
-            "total_docs_synced": initial if initial is not None else 0,
-            "remaining_docs_to_sync": remaining,
+            "id": payload.id if payload else None,
+            "total_docs": initial if initial is not None else 0,
+            "remaining_docs": remaining,
+            "synced_docs": (initial - remaining) if initial is not None else 0,
+            "is_complete": remaining == 0,
        },
        tenant_id=tenant_id,
    )
@@ -903,13 +906,6 @@ def monitor_ccpair_permissions_taskset(
        f"num_synced={initial}"
    )

-    # Add telemetry for permission syncing complete
-    optional_telemetry(
-        record_type=RecordType.PERMISSION_SYNC_COMPLETE,
-        data={"cc_pair_id": cc_pair_id},
-        tenant_id=tenant_id,
-    )
-
    update_sync_record_status(
        db_session=db_session,
        entity_id=cc_pair_id,
--- a/backend/onyx/background/indexing/run_indexing.py
+++ b/backend/onyx/background/indexing/run_indexing.py
@@ -56,6 +56,7 @@ from onyx.indexing.indexing_pipeline import build_indexing_pipeline
 from onyx.natural_language_processing.search_nlp_models import (
    InformationContentClassificationModel,
 )
+from onyx.redis.redis_connector import RedisConnector
 from onyx.utils.logger import setup_logger
 from onyx.utils.logger import TaskAttemptSingleton
 from onyx.utils.telemetry import create_milestone_and_report
@@ -577,8 +578,11 @@ def _run_indexing(
                    data={
                        "index_attempt_id": index_attempt_id,
                        "cc_pair_id": ctx.cc_pair_id,
-                        "current_docs_indexed": document_count,
-                        "current_chunks_indexed": chunk_count,
+                        "connector_id": ctx.connector_id,
+                        "credential_id": ctx.credential_id,
+                        "total_docs_indexed": document_count,
+                        "total_chunks": chunk_count,
+                        "batch_num": batch_num,
                        "source": ctx.source.value,
                    },
                    tenant_id=tenant_id,
@@ -599,15 +603,26 @@ def _run_indexing(
                    checkpoint=checkpoint,
                )

+        # Add telemetry for completed indexing
+        redis_connector = RedisConnector(tenant_id, ctx.cc_pair_id)
+        redis_connector_index = redis_connector.new_index(
+            index_attempt_start.search_settings_id
+        )
+        final_progress = redis_connector_index.get_progress() or 0
+
        optional_telemetry(
            record_type=RecordType.INDEXING_COMPLETE,
            data={
                "index_attempt_id": index_attempt_id,
                "cc_pair_id": ctx.cc_pair_id,
+                "connector_id": ctx.connector_id,
+                "credential_id": ctx.credential_id,
                "total_docs_indexed": document_count,
                "total_chunks": chunk_count,
+                "batch_count": batch_num,
                "time_elapsed_seconds": time.monotonic() - start_time,
                "source": ctx.source.value,
+                "redis_progress": final_progress,
            },
            tenant_id=tenant_id,
        )
--- a/backend/onyx/db/index_attempt.py
+++ b/backend/onyx/db/index_attempt.py
@@ -217,6 +217,7 @@ def mark_attempt_in_progress(
                "index_attempt_id": index_attempt.id,
                "status": IndexingStatus.IN_PROGRESS.value,
                "cc_pair_id": index_attempt.connector_credential_pair_id,
+                "search_settings_id": index_attempt.search_settings_id,
            },
        )
    except Exception:
@@ -245,6 +246,9 @@ def mark_attempt_succeeded(
                "index_attempt_id": index_attempt_id,
                "status": IndexingStatus.SUCCESS.value,
                "cc_pair_id": attempt.connector_credential_pair_id,
+                "search_settings_id": attempt.search_settings_id,
+                "total_docs_indexed": attempt.total_docs_indexed,
+                "new_docs_indexed": attempt.new_docs_indexed,
            },
        )
    except Exception:
@@ -273,6 +277,9 @@ def mark_attempt_partially_succeeded(
                "index_attempt_id": index_attempt_id,
                "status": IndexingStatus.COMPLETED_WITH_ERRORS.value,
                "cc_pair_id": attempt.connector_credential_pair_id,
+                "search_settings_id": attempt.search_settings_id,
+                "total_docs_indexed": attempt.total_docs_indexed,
+                "new_docs_indexed": attempt.new_docs_indexed,
            },
        )
    except Exception:
@@ -305,6 +312,10 @@ def mark_attempt_canceled(
                "index_attempt_id": index_attempt_id,
                "status": IndexingStatus.CANCELED.value,
                "cc_pair_id": attempt.connector_credential_pair_id,
+                "search_settings_id": attempt.search_settings_id,
+                "reason": reason,
+                "total_docs_indexed": attempt.total_docs_indexed,
+                "new_docs_indexed": attempt.new_docs_indexed,
            },
        )
    except Exception:
@@ -339,6 +350,10 @@ def mark_attempt_failed(
                "index_attempt_id": index_attempt_id,
                "status": IndexingStatus.FAILED.value,
                "cc_pair_id": attempt.connector_credential_pair_id,
+                "search_settings_id": attempt.search_settings_id,
+                "reason": failure_reason,
+                "total_docs_indexed": attempt.total_docs_indexed,
+                "new_docs_indexed": attempt.new_docs_indexed,
            },
        )
    except Exception:
--- a/backend/onyx/utils/telemetry.py
+++ b/backend/onyx/utils/telemetry.py
@@ -39,7 +39,6 @@ class RecordType(str, Enum):
    INDEXING_PROGRESS = "indexing_progress"
    INDEXING_COMPLETE = "indexing_complete"
    PERMISSION_SYNC_PROGRESS = "permission_sync_progress"
-    PERMISSION_SYNC_COMPLETE = "permission_sync_complete"
    INDEX_ATTEMPT_STATUS = "index_attempt_status"


--- a/web/src/app/admin/connectors/[connector]/AddConnectorPage.tsx
+++ b/web/src/app/admin/connectors/[connector]/AddConnectorPage.tsx
@@ -302,11 +302,17 @@ export default function AddConnector({
          ...connector_specific_config
        } = values;

-        // Apply transforms from connectors.ts configuration
+        // Apply special transforms according to application logic
        const transformedConnectorSpecificConfig = Object.entries(
          connector_specific_config
        ).reduce(
          (acc, [key, value]) => {
+            // Filter out empty strings from arrays
+            if (Array.isArray(value)) {
+              value = (value as any[]).filter(
+                (item) => typeof item !== "string" || item.trim() !== ""
+              );
+            }
            const matchingConfigValue = configuration.values.find(
              (configValue) => configValue.name === key
            );
--- a/web/src/lib/connectors/connectors.tsx
+++ b/web/src/lib/connectors/connectors.tsx
@@ -1333,10 +1333,10 @@ export function createConnectorValidationSchema(
 ): Yup.ObjectSchema<Record<string, any>> {
  const configuration = connectorConfigs[connector];

-  return Yup.object().shape({
+  const object = Yup.object().shape({
    access_type: Yup.string().required("Access Type is required"),
    name: Yup.string().required("Connector Name is required"),
-    ...configuration.values.reduce(
+    ...[...configuration.values, ...configuration.advanced_values].reduce(
      (acc, field) => {
        let schema: any =
          field.type === "select"
@@ -1363,6 +1363,8 @@ export function createConnectorValidationSchema(
    pruneFreq: Yup.number().min(0, "Prune frequency must be non-negative"),
    refreshFreq: Yup.number().min(0, "Refresh frequency must be non-negative"),
  });
+
+  return object;
 }

 export const defaultPruneFreqDays = 30; // 30 days
Author	SHA1	Message	Date
pablonyx	411f912b9f	k	2025-04-01 12:32:19 -07:00
pablonyx	7de576ce55	k	2025-04-01 12:31:40 -07:00
pablonyx	b48ebeacab	update	2025-04-01 12:30:38 -07:00
pablonyx	d7a2329580	k	2025-04-01 12:04:05 -07:00
pablonyx	4db048797c	k	2025-04-01 12:03:52 -07:00
pablonyx	70197e8329	k	2025-04-01 12:03:34 -07:00
pablonyx	ca0738ed65	remove logs	2025-04-01 12:03:34 -07:00
pablonyx	6a0e8a1f8c	validate advanced fields + proper yup assurances for lists	2025-04-01 12:03:34 -07:00