k

validation
update
2026-02-21 01:35:46 +00:00 · 2025-04-01 11:36:52 -07:00 · 2025-04-01 11:34:01 -07:00 · 2025-04-01 11:18:31 -07:00 · 2025-04-01 10:59:12 -07:00 · 2025-04-01 10:55:06 -07:00
5 changed files with 37 additions and 13 deletions
--- a/backend/onyx/auth/email_utils.py
+++ b/backend/onyx/auth/email_utils.py
@@ -23,7 +23,6 @@ from onyx.utils.url import add_url_params
 from onyx.utils.variable_functionality import fetch_versioned_implementation
 from shared_configs.configs import MULTI_TENANT

-
 HTML_EMAIL_TEMPLATE = """\
 <!DOCTYPE html>
 <html lang="en">
--- a/backend/onyx/background/celery/tasks/doc_permission_syncing/tasks.py
+++ b/backend/onyx/background/celery/tasks/doc_permission_syncing/tasks.py
@@ -886,8 +886,11 @@ def monitor_ccpair_permissions_taskset(
        record_type=RecordType.PERMISSION_SYNC_PROGRESS,
        data={
            "cc_pair_id": cc_pair_id,
-            "total_docs_synced": initial if initial is not None else 0,
-            "remaining_docs_to_sync": remaining,
+            "id": payload.id if payload else None,
+            "total_docs": initial if initial is not None else 0,
+            "remaining_docs": remaining,
+            "synced_docs": (initial - remaining) if initial is not None else 0,
+            "is_complete": remaining == 0,
        },
        tenant_id=tenant_id,
    )
@@ -903,13 +906,6 @@ def monitor_ccpair_permissions_taskset(
        f"num_synced={initial}"
    )

-    # Add telemetry for permission syncing complete
-    optional_telemetry(
-        record_type=RecordType.PERMISSION_SYNC_COMPLETE,
-        data={"cc_pair_id": cc_pair_id},
-        tenant_id=tenant_id,
-    )
-
    update_sync_record_status(
        db_session=db_session,
        entity_id=cc_pair_id,
--- a/backend/onyx/background/indexing/run_indexing.py
+++ b/backend/onyx/background/indexing/run_indexing.py
@@ -56,6 +56,7 @@ from onyx.indexing.indexing_pipeline import build_indexing_pipeline
 from onyx.natural_language_processing.search_nlp_models import (
    InformationContentClassificationModel,
 )
+from onyx.redis.redis_connector import RedisConnector
 from onyx.utils.logger import setup_logger
 from onyx.utils.logger import TaskAttemptSingleton
 from onyx.utils.telemetry import create_milestone_and_report
@@ -577,8 +578,11 @@ def _run_indexing(
                    data={
                        "index_attempt_id": index_attempt_id,
                        "cc_pair_id": ctx.cc_pair_id,
-                        "current_docs_indexed": document_count,
-                        "current_chunks_indexed": chunk_count,
+                        "connector_id": ctx.connector_id,
+                        "credential_id": ctx.credential_id,
+                        "total_docs_indexed": document_count,
+                        "total_chunks": chunk_count,
+                        "batch_num": batch_num,
                        "source": ctx.source.value,
                    },
                    tenant_id=tenant_id,
@@ -599,15 +603,26 @@ def _run_indexing(
                    checkpoint=checkpoint,
                )

+        # Add telemetry for completed indexing
+        redis_connector = RedisConnector(tenant_id, ctx.cc_pair_id)
+        redis_connector_index = redis_connector.new_index(
+            index_attempt_start.search_settings_id
+        )
+        final_progress = redis_connector_index.get_progress() or 0
+
        optional_telemetry(
            record_type=RecordType.INDEXING_COMPLETE,
            data={
                "index_attempt_id": index_attempt_id,
                "cc_pair_id": ctx.cc_pair_id,
+                "connector_id": ctx.connector_id,
+                "credential_id": ctx.credential_id,
                "total_docs_indexed": document_count,
                "total_chunks": chunk_count,
+                "batch_count": batch_num,
                "time_elapsed_seconds": time.monotonic() - start_time,
                "source": ctx.source.value,
+                "redis_progress": final_progress,
            },
            tenant_id=tenant_id,
        )
--- a/backend/onyx/db/index_attempt.py
+++ b/backend/onyx/db/index_attempt.py
@@ -217,6 +217,7 @@ def mark_attempt_in_progress(
                "index_attempt_id": index_attempt.id,
                "status": IndexingStatus.IN_PROGRESS.value,
                "cc_pair_id": index_attempt.connector_credential_pair_id,
+                "search_settings_id": index_attempt.search_settings_id,
            },
        )
    except Exception:
@@ -245,6 +246,9 @@ def mark_attempt_succeeded(
                "index_attempt_id": index_attempt_id,
                "status": IndexingStatus.SUCCESS.value,
                "cc_pair_id": attempt.connector_credential_pair_id,
+                "search_settings_id": attempt.search_settings_id,
+                "total_docs_indexed": attempt.total_docs_indexed,
+                "new_docs_indexed": attempt.new_docs_indexed,
            },
        )
    except Exception:
@@ -273,6 +277,9 @@ def mark_attempt_partially_succeeded(
                "index_attempt_id": index_attempt_id,
                "status": IndexingStatus.COMPLETED_WITH_ERRORS.value,
                "cc_pair_id": attempt.connector_credential_pair_id,
+                "search_settings_id": attempt.search_settings_id,
+                "total_docs_indexed": attempt.total_docs_indexed,
+                "new_docs_indexed": attempt.new_docs_indexed,
            },
        )
    except Exception:
@@ -305,6 +312,10 @@ def mark_attempt_canceled(
                "index_attempt_id": index_attempt_id,
                "status": IndexingStatus.CANCELED.value,
                "cc_pair_id": attempt.connector_credential_pair_id,
+                "search_settings_id": attempt.search_settings_id,
+                "reason": reason,
+                "total_docs_indexed": attempt.total_docs_indexed,
+                "new_docs_indexed": attempt.new_docs_indexed,
            },
        )
    except Exception:
@@ -339,6 +350,10 @@ def mark_attempt_failed(
                "index_attempt_id": index_attempt_id,
                "status": IndexingStatus.FAILED.value,
                "cc_pair_id": attempt.connector_credential_pair_id,
+                "search_settings_id": attempt.search_settings_id,
+                "reason": failure_reason,
+                "total_docs_indexed": attempt.total_docs_indexed,
+                "new_docs_indexed": attempt.new_docs_indexed,
            },
        )
    except Exception:
--- a/backend/onyx/utils/telemetry.py
+++ b/backend/onyx/utils/telemetry.py
@@ -39,7 +39,6 @@ class RecordType(str, Enum):
    INDEXING_PROGRESS = "indexing_progress"
    INDEXING_COMPLETE = "indexing_complete"
    PERMISSION_SYNC_PROGRESS = "permission_sync_progress"
-    PERMISSION_SYNC_COMPLETE = "permission_sync_complete"
    INDEX_ATTEMPT_STATUS = "index_attempt_status"
Author	SHA1	Message	Date
pablonyx	730c0d12a8	k	2025-04-01 11:36:52 -07:00
pablonyx	1027fbe845	validation	2025-04-01 11:34:01 -07:00
pablonyx	3ec738caa5	update	2025-04-01 11:18:31 -07:00
pablonyx	4fbdbbbfa0	k	2025-04-01 10:59:12 -07:00
pablonyx	e655b4b248	fix build	2025-04-01 10:55:06 -07:00
pablonyx	7535280e6e	stop using sharp	2025-04-01 10:51:54 -07:00
pablonyx	9aa4037434	nit	2025-04-01 10:35:04 -07:00
pablonyx	fe70cae942	k	2025-04-01 10:30:00 -07:00
pablonyx	e978e4e26f	k	2025-04-01 10:18:11 -07:00
pablonyx	995d4feafd	k	2025-04-01 10:16:05 -07:00
pablonyx	999af49687	k	2025-04-01 10:08:38 -07:00
pablonyx	6df124f593	k# Please enter the commit message for your changes. Lines starting	2025-04-01 10:04:08 -07:00
pablonyx	12f8627799	very minor standardization	2025-04-01 09:41:48 -07:00