Added the config entity , artifact entity everything

Ananddd06 · Ananddd06 · commit 3f016947e5c7 · 2025-07-09T11:52:35.000+05:30
diff --git a/networksecurity/components/data_validation.py b/networksecurity/components/data_validation.py
@@ -1,19 +1,110 @@
-from networksecurity.entity.artifact_entity import DataValidationArtifact , DataIngestionArtifact
+from networksecurity.entity.artifact_entity import DataValidationArtifact, DataIngestionArtifact
 from networksecurity.entity.config_entity import DataValidationConfig
 from networksecurity.exception.exception import CustomException
+from networksecurity.utils.main_utils import read_yaml_file, write_yaml_file
 from networksecurity.logger.customlogger import Custom_Logger
 from networksecurity.constants.training_pipeline import SCHEMA_FILE_PATH
-from scipy.stats import ks_2samp
+
+from evidently.report import Report
+from evidently.metric_preset import DataDriftPreset
+
 import pandas as pd
-import numpy as np
 import os, sys
-import yaml
 
 class DataValidation:
     def __init__(self, data_validation_config: DataValidationConfig, data_ingestion_artifact: DataIngestionArtifact):
-        self.data_validation_config = data_validation_config
-        self.data_ingestion_artifact = data_ingestion_artifact
-        self._schema_config = read_yaml_file(SCHEMA_FILE_PATH)
-        self.logger = Custom_Logger().get_logger()
+        try:
+            self.data_validation_config = data_validation_config
+            self.data_ingestion_artifact = data_ingestion_artifact
+            self._schema_config = read_yaml_file(SCHEMA_FILE_PATH)
+            self.logger = Custom_Logger().get_logger()
+        except Exception as e:
+            raise CustomException(e, sys) from e
+
+    @staticmethod
+    def read_data(file_path: str) -> pd.DataFrame:
+        """Reads data from a file and returns it as a DataFrame."""
+        try:
+            if not os.path.exists(file_path):
+                raise FileNotFoundError(f"The file {file_path} does not exist.")
+            return pd.read_csv(file_path)
+        except Exception as e:
+            raise CustomException(e, sys) from e
+
+    def validate_number_of_columns(self, dataframe: pd.DataFrame) -> bool:
+        """Validates the number of columns in the DataFrame against the schema."""
+        try:
+            expected_columns = self._schema_config['columns']
+            actual_columns = dataframe.columns.tolist()
+
+            self.logger.info(f"Expected columns: {expected_columns}")
+            self.logger.info(f"Actual columns: {actual_columns}")
+
+            if len(actual_columns) != len(expected_columns):
+                self.logger.critical(f"❌ Column mismatch: Expected {len(expected_columns)}, got {len(actual_columns)}")
+                return False
+            return True
+        except Exception as e:
+            raise CustomException(e, sys) from e
+
+    def get_data_drift_report(self, train_df: pd.DataFrame, test_df: pd.DataFrame, html_path: str, yaml_path: str = None):
+        """Generates a data drift report using Evidently (HTML + optional YAML)."""
+        try:
+            self.logger.info("📊 Generating data drift report...")
+
+            report = Report(metrics=[DataDriftPreset()])
+            report.run(reference_data=train_df, current_data=test_df)
+
+            report.save_html(html_path)
+            self.logger.info(f"✅ Drift report saved: {html_path}")
+
+            # If YAML path is given, save the summary
+            if yaml_path:
+                result_dict = report.as_dict()
+                drift_result = result_dict["metrics"][0]["result"]
+
+                summary = {
+                    "drift_detected": drift_result["dataset_drift"],
+                    "drifted_feature_count": drift_result["number_of_drifted_columns"],
+                    "total_feature_count": drift_result["number_of_columns"],
+                    "drift_share": drift_result["share_of_drifted_columns"],
+                    "drifted_features": drift_result.get("drifted_columns", [])
+                }
+
+                write_yaml_file(yaml_path, summary)
+                self.logger.info(f"📄 Drift summary saved to YAML: {yaml_path}")
+
+        except Exception as e:
+            self.logger.error("❌ Drift report generation failed.")
+            raise CustomException(e, sys) from e
+
+    def initiate_data_validation(self) -> DataValidationArtifact:
+        """Runs the complete data validation pipeline."""
+        try:
+            train_file_path = self.data_ingestion_artifact.train_file_path
+            test_file_path = self.data_ingestion_artifact.test_file_path
+
+            train_df = self.read_data(train_file_path)
+            test_df = self.read_data(test_file_path)
+            self.logger.info("✅ Train and test data loaded successfully.")
+
+            # Validate schema for both
+            if not self.validate_number_of_columns(train_df):
+                raise ValueError("Train data schema mismatch.")
+            if not self.validate_number_of_columns(test_df):
+                raise ValueError("Test data schema mismatch.")
+
+            # Drift Report (HTML + YAML)
+            drift_html = self.data_validation_config.drift_report_path
+            drift_yaml = self.data_validation_config.drift_yaml_path
+
+            self.get_data_drift_report(train_df, test_df, drift_html, drift_yaml)
+
+            return DataValidationArtifact(
+                validation_status=True,
+                drift_report_file_path=drift_html,
+                drift_summary_file_path=drift_yaml
+            )
 
-    
+        except Exception as e:
+            raise CustomException(e, sys) from e
diff --git a/networksecurity/entity/artifact_entity.py b/networksecurity/entity/artifact_entity.py
@@ -2,14 +2,15 @@
 
 @dataclass
 class DataIngestionArtifact:
-    trained_file_path:str 
-    test_file_path:str 
+    trained_file_path: str
+    test_file_path: str
 
 @dataclass
 class DataValidationArtifact:
-    vaildation_status:bool
-    valid_train_file_path:str
-    valid_test_file_path:str
-    invalid_train_file_path:str
-    invalid_test_file_path:str
-    drift_report_file_path:str
+    validation_status: bool
+    valid_train_file_path: str
+    valid_test_file_path: str
+    invalid_train_file_path: str
+    invalid_test_file_path: str
+    drift_report_file_path: str
+    drift_summary_file_path: str
diff --git a/networksecurity/entity/config_entity.py b/networksecurity/entity/config_entity.py
@@ -37,18 +37,46 @@ class DataValidationConfigEntity:
 
     def __post_init__(self):
         self.data_validation_dir: str = os.path.join(
-            self.training_pipeline_config.artifact_dir, training_pipeline.DATA_VALIDATION_DIR_NAME
-        )
-        self.valid_data_dir: str = os.path.join(self.data_validation_dir, training_pipeline.DATA_VALIDATION_VALID_DIR)
-        self.invalid_data_dir: str = os.path.join(self.data_validation_dir, training_pipeline.DATA_VALIDATION_INVALID_DIR)
-        self.valid_train_file_path : str = os.path.join(self.valid_data_dir, training_pipeline.TRAIN_FILE_NAME)
-        self.valid_test_file_path : str = os.path.join(self.valid_data_dir, training_pipeline.TEST_FILE_NAME)
-        self.invalid_train_file_path : str = os.path.join(self.invalid_data_dir, training_pipeline.TRAIN_FILE_NAME)
-        self.invalid_test_file_path : str = os.path.join(self.invalid_data_dir, training_pipeline.TEST_FILE_NAME)
-        # New: Drift report paths
-        self.drift_report_file_path : str = os.path.join(
+            self.training_pipeline_config.artifact_dir,
+            training_pipeline.DATA_VALIDATION_DIR_NAME
+        )
+
+        self.valid_data_dir: str = os.path.join(
+            self.data_validation_dir,
+            training_pipeline.DATA_VALIDATION_VALID_DIR
+        )
+        self.invalid_data_dir: str = os.path.join(
+            self.data_validation_dir,
+            training_pipeline.DATA_VALIDATION_INVALID_DIR
+        )
+
+        self.valid_train_file_path: str = os.path.join(
+            self.valid_data_dir,
+            training_pipeline.TRAIN_FILE_NAME
+        )
+        self.valid_test_file_path: str = os.path.join(
+            self.valid_data_dir,
+            training_pipeline.TEST_FILE_NAME
+        )
+        self.invalid_train_file_path: str = os.path.join(
+            self.invalid_data_dir,
+            training_pipeline.TRAIN_FILE_NAME
+        )
+        self.invalid_test_file_path: str = os.path.join(
+            self.invalid_data_dir,
+            training_pipeline.TEST_FILE_NAME
+        )
+
+        # Drift Report HTML path
+        self.drift_report_file_path: str = os.path.join(
             self.data_validation_dir,
-            training_pipeline.DATA_VALIDATION_DRIFT_REPORT_DIR, 
+            training_pipeline.DATA_VALIDATION_DRIFT_REPORT_DIR,
             training_pipeline.DATA_VALIDATION_DRIFT_REPORT_FILE_NAME
         )
-    
+
+        # ✅ Drift Report YAML Summary Path (NEW)
+        self.drift_yaml_path: str = os.path.join(
+            self.data_validation_dir,
+            training_pipeline.DATA_VALIDATION_DRIFT_REPORT_DIR,
+            training_pipeline.DATA_VALIDATION_DRIFT_SUMMARY_FILE_NAME
+        )
diff --git a/requirements.txt b/requirements.txt
@@ -5,6 +5,7 @@ numpy==1.26
 scikit-learn
 matplotlib
 seaborn
+evidently
 setuptools
 scipy
 dvc
