{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":50160,"databundleVersionId":7921029,"sourceType":"competition"}],"dockerImageVersionId":30698,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-05-19T03:33:31.630302Z","iopub.execute_input":"2024-05-19T03:33:31.630810Z","iopub.status.idle":"2024-05-19T03:33:31.671145Z","shell.execute_reply.started":"2024-05-19T03:33:31.630772Z","shell.execute_reply":"2024-05-19T03:33:31.669710Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import polars as pl","metadata":{"execution":{"iopub.status.busy":"2024-05-19T03:33:36.869184Z","iopub.execute_input":"2024-05-19T03:33:36.870464Z","iopub.status.idle":"2024-05-19T03:33:36.876009Z","shell.execute_reply.started":"2024-05-19T03:33:36.870403Z","shell.execute_reply":"2024-05-19T03:33:36.874508Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataPath = \"/kaggle/input/home-credit-credit-risk-model-stability/\"\ntrainPath = \"/kaggle/input/home-credit-credit-risk-model-stability/csv_files/train/\"\ntestPath = \"/kaggle/input/home-credit-credit-risk-model-stability/csv_files/test/\"\nworkingPath = \"/kaggle/working/\"","metadata":{"execution":{"iopub.status.busy":"2024-05-19T03:33:39.753827Z","iopub.execute_input":"2024-05-19T03:33:39.754309Z","iopub.status.idle":"2024-05-19T03:33:39.760880Z","shell.execute_reply.started":"2024-05-19T03:33:39.754271Z","shell.execute_reply":"2024-05-19T03:33:39.759396Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature_definitions_df = pd.read_csv(dataPath + \"feature_definitions.csv\")\nfeature_definitions_df","metadata":{"execution":{"iopub.status.busy":"2024-05-19T03:33:44.588141Z","iopub.execute_input":"2024-05-19T03:33:44.589323Z","iopub.status.idle":"2024-05-19T03:33:44.632777Z","shell.execute_reply.started":"2024-05-19T03:33:44.589270Z","shell.execute_reply":"2024-05-19T03:33:44.631144Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"file_paths = [os.path.join(trainPath, file) for file in os.listdir(trainPath) if file.endswith('.csv')]\nprint(len(file_paths))\nfile_paths","metadata":{"execution":{"iopub.status.busy":"2024-05-19T03:33:48.533192Z","iopub.execute_input":"2024-05-19T03:33:48.533651Z","iopub.status.idle":"2024-05-19T03:33:48.544227Z","shell.execute_reply.started":"2024-05-19T03:33:48.533608Z","shell.execute_reply":"2024-05-19T03:33:48.543234Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create a dictionary to store which file contains which feature\nfeature_file_mapping = {feature: [] for feature in feature_definitions_df['Variable'].tolist()}\nfeature_file_mapping\n\n# Check each file for the presence of each feature using pandas to read headers\nfor file_path in file_paths:\n    try:\n        # Read only the header (first row) of the CSV file with pandas\n        df_header = pd.read_csv(file_path, nrows=0)\n        # Extract the file name from the file path\n        file_name = os.path.basename(file_path)\n        # Check each feature\n        for feature in feature_file_mapping.keys():\n            if feature in df_header.columns:\n                feature_file_mapping[feature].append(file_name)\n    except Exception as e:\n        print(f\"Error reading {file_path}: {e}\")\n        \n# Add the file names to the feature definitions DataFrame\nfeature_definitions_df['source_files'] = feature_definitions_df['Variable'].map(lambda x: ', '.join(feature_file_mapping[x]))\n\n# Display the updated DataFrame\nfeature_definitions_df","metadata":{"execution":{"iopub.status.busy":"2024-05-19T03:33:51.865260Z","iopub.execute_input":"2024-05-19T03:33:51.865751Z","iopub.status.idle":"2024-05-19T03:33:52.364589Z","shell.execute_reply.started":"2024-05-19T03:33:51.865719Z","shell.execute_reply":"2024-05-19T03:33:52.363222Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Save the updated DataFrame back to the CSV and Excel files to the working path\nfeature_definitions_df.to_csv(workingPath + \"feature_definitions.csv\", index=False)\nfeature_definitions_df.to_excel(workingPath + \"feature_definitions.xlsx\", index=False)","metadata":{"execution":{"iopub.status.busy":"2024-05-19T03:34:17.686483Z","iopub.execute_input":"2024-05-19T03:34:17.687824Z","iopub.status.idle":"2024-05-19T03:34:18.090758Z","shell.execute_reply.started":"2024-05-19T03:34:17.687774Z","shell.execute_reply":"2024-05-19T03:34:18.088988Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Ensure the 'source_files' column is of string type and fill any missing values\nfeature_definitions_df['source_files'] = feature_definitions_df['source_files'].astype(str).fillna('')\n\n# Initialize a dictionary to store the file groups\nfile_groups = {}\n\n# Iterate over each row in the DataFrame\nfor index, row in feature_definitions_df.iterrows():\n    feature = row['Variable']\n    files = row['source_files'].split(', ')\n    \n    for file in files:\n        if file not in file_groups:\n            file_groups[file] = []\n        file_groups[file].append(feature)\n\n# Convert the dictionary to a DataFrame\nfile_groups_df = pd.DataFrame(list(file_groups.items()), columns=['File', 'Features'])\n\n# Display the grouped DataFrame\nfile_groups_df","metadata":{"execution":{"iopub.status.busy":"2024-05-19T03:34:32.736193Z","iopub.execute_input":"2024-05-19T03:34:32.736795Z","iopub.status.idle":"2024-05-19T03:34:32.802871Z","shell.execute_reply.started":"2024-05-19T03:34:32.736758Z","shell.execute_reply":"2024-05-19T03:34:32.801546Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Save the grouped DataFrame to a new Excel file\nfile_groups_df.to_csv(workingPath + \"grouped_features_by_file.csv\", index=False)\nfile_groups_df.to_excel(workingPath + \"grouped_features_by_file.xlsx\", index=False)","metadata":{"execution":{"iopub.status.busy":"2024-05-19T03:34:45.838293Z","iopub.execute_input":"2024-05-19T03:34:45.838742Z","iopub.status.idle":"2024-05-19T03:34:45.863370Z","shell.execute_reply.started":"2024-05-19T03:34:45.838705Z","shell.execute_reply":"2024-05-19T03:34:45.862244Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Identify files that do not contain any features\nfiles_without_features = [os.path.basename(file) for file in file_paths if os.path.basename(file) not in file_groups]\nfiles_without_features ","metadata":{"execution":{"iopub.status.busy":"2024-05-19T03:35:06.698710Z","iopub.execute_input":"2024-05-19T03:35:06.699965Z","iopub.status.idle":"2024-05-19T03:35:06.714309Z","shell.execute_reply.started":"2024-05-19T03:35:06.699878Z","shell.execute_reply":"2024-05-19T03:35:06.712630Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Save the grouped DataFrame to a new Excel file\nfile_groups_df.to_excel(workingPath + \"grouped_features_by_file.xlsx\", index=False)\n\n# Save the list of files without features to a new Excel file\nfiles_without_features_df = pd.DataFrame(files_without_features, columns=['File'])\nfiles_without_features_df","metadata":{"execution":{"iopub.status.busy":"2024-05-19T03:35:17.194213Z","iopub.execute_input":"2024-05-19T03:35:17.194671Z","iopub.status.idle":"2024-05-19T03:35:17.221540Z","shell.execute_reply.started":"2024-05-19T03:35:17.194640Z","shell.execute_reply":"2024-05-19T03:35:17.220283Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"file_groups_df.head(), files_without_features_df.head()","metadata":{"execution":{"iopub.status.busy":"2024-05-19T03:35:23.066686Z","iopub.execute_input":"2024-05-19T03:35:23.067344Z","iopub.status.idle":"2024-05-19T03:35:23.082010Z","shell.execute_reply.started":"2024-05-19T03:35:23.067286Z","shell.execute_reply":"2024-05-19T03:35:23.080827Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_base = pd.read_csv(trainPath + \"train_base.csv\")\ntrain_base","metadata":{"execution":{"iopub.status.busy":"2024-05-19T03:35:42.169443Z","iopub.execute_input":"2024-05-19T03:35:42.169870Z","iopub.status.idle":"2024-05-19T03:35:43.363634Z","shell.execute_reply.started":"2024-05-19T03:35:42.169837Z","shell.execute_reply":"2024-05-19T03:35:43.362264Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get unique values for each column\nunique_case_id = train_base['case_id'].unique()\nunique_date_decision = train_base['date_decision'].unique()\nunique_month = train_base['MONTH'].unique()\nunique_week_num = train_base['WEEK_NUM'].unique()\nunique_target = train_base['target'].unique()\n\n# Display the unique values\nprint(\"Unique values in 'case_id':\", len(unique_case_id))\nprint(\"Unique values in 'date_decision':\", len(unique_date_decision))\nprint(\"Unique values in 'MONTH':\", len(unique_month))\nprint(\"Unique values in 'WEEK_NUM':\", len(unique_week_num))\nprint(\"Unique values in 'target':\", len(unique_target))","metadata":{"execution":{"iopub.status.busy":"2024-05-19T03:35:59.997791Z","iopub.execute_input":"2024-05-19T03:35:59.998698Z","iopub.status.idle":"2024-05-19T03:36:00.211602Z","shell.execute_reply.started":"2024-05-19T03:35:59.998658Z","shell.execute_reply":"2024-05-19T03:36:00.210188Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_base['target'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2024-05-19T03:36:11.657632Z","iopub.execute_input":"2024-05-19T03:36:11.658209Z","iopub.status.idle":"2024-05-19T03:36:11.688721Z","shell.execute_reply.started":"2024-05-19T03:36:11.658164Z","shell.execute_reply":"2024-05-19T03:36:11.687252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Initialize a list to store the shape of each file\nshapes = []\n\n# Iterate over each row in the DataFrame\nfor file in file_groups_df['File']:\n    file_path = os.path.join(trainPath, file)\n    try:\n        # Read only the headers with polars to infer shape\n        df = pl.scan_csv(file_path).collect()\n        # Calculate the shape: (number of rows, number of columns)\n        shape = (df.height, df.width)\n        print(f\"Shape of {file}: {shape}\")  # Debug statement to print the shape\n    except Exception as e:\n        print(f\"Error reading {file_path}: {e}\")\n        shape = (None, None)\n    \n    # Append the shape to the list\n    shapes.append(shape)\n\n# Add the shape information as a new column\nfile_groups_df['Shape'] = shapes\nfile_groups_df","metadata":{"execution":{"iopub.status.busy":"2024-05-19T03:36:23.870064Z","iopub.execute_input":"2024-05-19T03:36:23.871092Z","iopub.status.idle":"2024-05-19T03:39:42.074674Z","shell.execute_reply.started":"2024-05-19T03:36:23.871046Z","shell.execute_reply":"2024-05-19T03:39:42.071987Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"file_groups_df.to_excel(workingPath + \"grouped_features_by_file.xlsx\", index=False)","metadata":{"execution":{"iopub.status.busy":"2024-05-19T03:39:42.079440Z","iopub.execute_input":"2024-05-19T03:39:42.080099Z","iopub.status.idle":"2024-05-19T03:39:42.119379Z","shell.execute_reply.started":"2024-05-19T03:39:42.080039Z","shell.execute_reply":"2024-05-19T03:39:42.117765Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}