{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":50160,"databundleVersionId":7921029,"sourceType":"competition"}],"dockerImageVersionId":30699,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import dask.dataframe as dd\nimport pandas as pd\nimport polars as pl\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import roc_auc_score\n\ndataPath = \"/kaggle/input/home-credit-credit-risk-model-stability/\"\n\ndata_type1 = {'datefirstoffer_1144D': 'object',\n               'datelastinstal40dpd_247D': 'object',\n               'datelastunpaid_3546854D': 'object',\n               'dtlastpmtallstes_4499206D': 'object',\n               'firstclxcampaign_1125D': 'object',\n               'firstdatedue_489D': 'object',\n               'lastactivateddate_801D': 'object',\n               'lastapprdate_640D': 'object',\n               'lastdelinqdate_224D': 'object',\n               'lastrepayingdate_696D': 'object',\n               'maxdpdinstldate_3546855D': 'object',\n               'payvacationpostpone_4187118D': 'object',\n               'validfrom_1069D': 'object'}\ndtype_type12={'bankacctype_710L': 'object',\n               'cardtype_51L': 'object',\n               'datefirstoffer_1144D': 'object',\n               'datelastinstal40dpd_247D': 'object',\n               'datelastunpaid_3546854D': 'object',\n               'dtlastpmtallstes_4499206D': 'object',\n               'firstclxcampaign_1125D': 'object',\n               'firstdatedue_489D': 'object',\n               'lastactivateddate_801D': 'object',\n               'lastapprdate_640D': 'object',\n               'lastdelinqdate_224D': 'object',\n               'lastrepayingdate_696D': 'object',\n               'maxdpdinstldate_3546855D': 'object',\n               'payvacationpostpone_4187118D': 'object',\n               'validfrom_1069D': 'object'}","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-05-03T11:17:40.023381Z","iopub.execute_input":"2024-05-03T11:17:40.023647Z","iopub.status.idle":"2024-05-03T11:17:44.922791Z","shell.execute_reply.started":"2024-05-03T11:17:40.023623Z","shell.execute_reply":"2024-05-03T11:17:44.921823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_countplot(df, column):\n    # Create the count plot using Seaborn\n    plt.figure(figsize=(10, 6))  # Set the size of the figure\n    sns.countplot(x=column, data=df.compute())  # Create the count plot\n    plt.xlabel(column)  # Set the label for the x-axis\n    plt.ylabel('Count')  # Set the label for the y-axis\n    plt.title('Count Plot of {}'.format(column))  # Set the title of the plot\n    plt.xticks(rotation=45)  # Rotate x-axis labels for better readability if needed\n    plt.show()  # Display the plot","metadata":{"execution":{"iopub.status.busy":"2024-05-03T05:42:01.728382Z","iopub.execute_input":"2024-05-03T05:42:01.729086Z","iopub.status.idle":"2024-05-03T05:42:01.735595Z","shell.execute_reply.started":"2024-05-03T05:42:01.72905Z","shell.execute_reply":"2024-05-03T05:42:01.734435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def fill_na(df, column_name, value):\n    # Fill missing values in the specified column with the given value\n    df[column_name] = df[column_name].fillna(value)\n    return df","metadata":{"execution":{"iopub.status.busy":"2024-05-03T11:19:01.231994Z","iopub.execute_input":"2024-05-03T11:19:01.232804Z","iopub.status.idle":"2024-05-03T11:19:01.237282Z","shell.execute_reply.started":"2024-05-03T11:19:01.232772Z","shell.execute_reply":"2024-05-03T11:19:01.236274Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def nulls_df(df, column_type=None, percentage_threshold=0):\n    \"\"\"\n    Computes the count and percentage of null values in a DataFrame using Dask for memory optimization.\n    \n    Args:\n        df (dask.dataframe.DataFrame): The input Dask DataFrame.\n        column_type (str, optional): The type of columns to filter ('Categorical' or 'Numerical').\n        percentage_threshold (int, optional): The minimum percentage of null values to include.\n        \n    Returns:\n        tuple: A tuple containing the filtered DataFrame and a list of column names.\n    \"\"\"\n    \n    # Compute null counts and percentages\n    nulls_counts = df.isna().sum().compute()\n    nulls_percentages = (nulls_counts / df.shape[0].compute()) * 100\n    \n    # Identify column types\n    column_types = df.dtypes.apply(lambda x: 'Numerical' if x.kind in 'bifc' else 'Categorical')\n    \n    # Create a DataFrame for null information\n    null_df = pd.DataFrame({\n        'Column': nulls_counts.index,\n        'Nulls_Count': nulls_counts,\n        'Nulls_Percentage': nulls_percentages,\n        'Type': column_types\n    })\n    \n    # Filter based on column type if specified\n    if column_type:\n        null_df = null_df[null_df['Type'] == column_type]\n    \n    # Filter based on null percentage threshold\n    null_df = null_df[null_df['Nulls_Percentage'] >= percentage_threshold]\n    \n    # Reset index to default numeric indices\n    null_df = null_df.reset_index(drop=True)\n    \n    # Return the filtered DataFrame and a list of column names\n    column_list = null_df['Column'].tolist()\n    return null_df, column_list\n\n","metadata":{"execution":{"iopub.status.busy":"2024-05-03T11:18:57.381141Z","iopub.execute_input":"2024-05-03T11:18:57.381796Z","iopub.status.idle":"2024-05-03T11:18:57.390161Z","shell.execute_reply.started":"2024-05-03T11:18:57.381763Z","shell.execute_reply":"2024-05-03T11:18:57.389271Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Find repeated columns\ndef find_repeated_columns(df):\n    # Convert Dask DataFrame to Pandas DataFrame to perform column comparison\n    pandas_df = df.compute()\n\n    # Identify repeated columns\n    repeated_columns = []\n    seen_columns = set()\n    for column in pandas_df.columns:\n        if column in seen_columns:\n            repeated_columns.append(column)\n        else:\n            seen_columns.add(column)\n\n    return repeated_columns","metadata":{"execution":{"iopub.status.busy":"2024-05-03T05:42:12.475212Z","iopub.execute_input":"2024-05-03T05:42:12.475841Z","iopub.status.idle":"2024-05-03T05:42:12.48168Z","shell.execute_reply.started":"2024-05-03T05:42:12.4758Z","shell.execute_reply":"2024-05-03T05:42:12.480562Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# importing all the dipth 0 data sets\ntrain_static_0_0 = dd.read_csv(dataPath + \"csv_files/train/train_static_0_0.csv\", dtype=data_type1)\ntrain_static_0_1 = dd.read_csv(dataPath + \"csv_files/train/train_static_0_1.csv\", dtype={'bankacctype_710L': 'object',\n       'cardtype_51L': 'object',\n       'datefirstoffer_1144D': 'object',\n       'datelastinstal40dpd_247D': 'object',\n       'datelastunpaid_3546854D': 'object',\n       'dtlastpmtallstes_4499206D': 'object',\n       'firstclxcampaign_1125D': 'object',\n       'firstdatedue_489D': 'object',\n       'lastactivateddate_801D': 'object',\n       'lastapprdate_640D': 'object',\n       'lastdelinqdate_224D': 'object',\n       'lastrepayingdate_696D': 'object',\n       'maxdpdinstldate_3546855D': 'object',\n       'payvacationpostpone_4187118D': 'object',\n       'validfrom_1069D': 'object'})\n# train_static_cb_0 = dd.read_csv(dataPath + 'csv_files/train/train_static_cb_0.csv',dtype=data_type2)\ntrain_static_0 = dd.concat([train_static_0_0, train_static_0_1])","metadata":{"execution":{"iopub.status.busy":"2024-05-03T11:18:25.016023Z","iopub.execute_input":"2024-05-03T11:18:25.016628Z","iopub.status.idle":"2024-05-03T11:18:25.305311Z","shell.execute_reply.started":"2024-05-03T11:18:25.016596Z","shell.execute_reply":"2024-05-03T11:18:25.304551Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_static_0_1.head()","metadata":{"execution":{"iopub.status.busy":"2024-05-03T05:42:15.717518Z","iopub.execute_input":"2024-05-03T05:42:15.717849Z","iopub.status.idle":"2024-05-03T05:42:20.839815Z","shell.execute_reply.started":"2024-05-03T05:42:15.717823Z","shell.execute_reply":"2024-05-03T05:42:20.838883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(train_static_0)","metadata":{"execution":{"iopub.status.busy":"2024-05-03T05:42:20.841346Z","iopub.execute_input":"2024-05-03T05:42:20.841663Z","iopub.status.idle":"2024-05-03T05:42:56.10794Z","shell.execute_reply.started":"2024-05-03T05:42:20.841636Z","shell.execute_reply":"2024-05-03T05:42:56.106807Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Checking if there are repeated columns or not \n\nrepeated_columns = find_repeated_columns(train_static_0)\nrepeated_columns","metadata":{"execution":{"iopub.status.busy":"2024-05-03T05:42:56.110047Z","iopub.execute_input":"2024-05-03T05:42:56.110504Z","iopub.status.idle":"2024-05-03T05:43:37.649994Z","shell.execute_reply.started":"2024-05-03T05:42:56.11046Z","shell.execute_reply":"2024-05-03T05:43:37.648877Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#  Getting the columns that contains more than 60% \nnull_df, column_list = nulls_df(train_static_0, percentage_threshold=60)\nnull_df","metadata":{"execution":{"iopub.status.busy":"2024-05-03T05:43:37.651619Z","iopub.execute_input":"2024-05-03T05:43:37.652058Z","iopub.status.idle":"2024-05-03T05:44:48.557194Z","shell.execute_reply.started":"2024-05-03T05:43:37.65202Z","shell.execute_reply":"2024-05-03T05:44:48.555973Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Drop columns that contains null values more than 60%\n\ntrain_static_0 = train_static_0.drop(columns = column_list, axis = 1)","metadata":{"execution":{"iopub.status.busy":"2024-05-03T05:45:54.220768Z","iopub.execute_input":"2024-05-03T05:45:54.221687Z","iopub.status.idle":"2024-05-03T05:45:58.049791Z","shell.execute_reply.started":"2024-05-03T05:45:54.221645Z","shell.execute_reply":"2024-05-03T05:45:58.048099Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create a data frame contains the number of null values and the percentage\n\ndef df(df = train_static_0):\n    Count_null_values = df.isna().sum().compute()\n    percentage = (Count_null_values / len(train_static_0)) * 100\n    df_null = pd.DataFrame({\n        \"Columns\":Count_null_values.index,\n        \"Count\": Count_null_values,\n        \"Percentage\": percentage\n    })\n    df_null = df_null.reset_index(drop=True)\n    return df_null","metadata":{"execution":{"iopub.status.busy":"2024-05-03T05:46:34.233553Z","iopub.execute_input":"2024-05-03T05:46:34.234513Z","iopub.status.idle":"2024-05-03T05:46:34.240718Z","shell.execute_reply.started":"2024-05-03T05:46:34.234476Z","shell.execute_reply":"2024-05-03T05:46:34.239526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Getting columns that contains less than 1%\ndf_null = df()\ndf_null = df_null[df_null['Percentage'] < 1]\nvalues_to_drop = df_null['Columns'].to_list()","metadata":{"execution":{"iopub.status.busy":"2024-05-03T05:46:35.215337Z","iopub.execute_input":"2024-05-03T05:46:35.216391Z","iopub.status.idle":"2024-05-03T05:47:39.264821Z","shell.execute_reply.started":"2024-05-03T05:46:35.216348Z","shell.execute_reply":"2024-05-03T05:47:39.263807Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# drop the null values that less than 1%\n\ntrain_static_0 = train_static_0.dropna(subset=values_to_drop)","metadata":{"execution":{"iopub.status.busy":"2024-05-03T05:47:39.269169Z","iopub.execute_input":"2024-05-03T05:47:39.269479Z","iopub.status.idle":"2024-05-03T05:47:39.276184Z","shell.execute_reply.started":"2024-05-03T05:47:39.269454Z","shell.execute_reply":"2024-05-03T05:47:39.27527Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Now Getting the columns that contains the Null values between 1 and 60\n\nnull_df_f, columns = nulls_df(train_static_0, percentage_threshold = 1)","metadata":{"execution":{"iopub.status.busy":"2024-05-03T11:19:06.736616Z","iopub.execute_input":"2024-05-03T11:19:06.737521Z","iopub.status.idle":"2024-05-03T11:20:16.909178Z","shell.execute_reply.started":"2024-05-03T11:19:06.737484Z","shell.execute_reply":"2024-05-03T11:20:16.908231Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"null_df_f.sort_values(by='Nulls_Percentage').head(30)","metadata":{"execution":{"iopub.status.busy":"2024-05-03T11:20:56.793927Z","iopub.execute_input":"2024-05-03T11:20:56.79427Z","iopub.status.idle":"2024-05-03T11:20:56.810627Z","shell.execute_reply.started":"2024-05-03T11:20:56.794243Z","shell.execute_reply":"2024-05-03T11:20:56.809557Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# a function to showing the numirical values plotting \n\ndef plot_histogram_and_boxplot(df, column_name):\n    # Create the figure and subplots\n    fig, axs = plt.subplots(1, 2, figsize=(12, 5))\n\n    # Plot histogram\n    sns.histplot(df[column_name], kde=False, ax=axs[0])\n    axs[0].set_xlabel('Values')\n    axs[0].set_ylabel('Frequency')\n    axs[0].set_title('Histogram of {}'.format(column_name))\n\n    # Plot box plot\n    sns.boxplot(y=df[column_name], ax=axs[1])\n    axs[1].set_ylabel('Values')\n    axs[1].set_title('Box Plot of {}'.format(column_name))\n\n    # Adjust layout to prevent overlapping\n    plt.tight_layout()\n\n    # Show plot\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-05-03T05:48:30.35048Z","iopub.execute_input":"2024-05-03T05:48:30.350766Z","iopub.status.idle":"2024-05-03T05:48:30.360738Z","shell.execute_reply.started":"2024-05-03T05:48:30.350741Z","shell.execute_reply":"2024-05-03T05:48:30.359709Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_histogram_and_boxplot(train_static_0, 'pmtnum_254L')","metadata":{"execution":{"iopub.status.busy":"2024-05-03T07:59:08.816966Z","iopub.execute_input":"2024-05-03T07:59:08.817335Z","iopub.status.idle":"2024-05-03T08:01:16.995904Z","shell.execute_reply.started":"2024-05-03T07:59:08.817305Z","shell.execute_reply":"2024-05-03T08:01:16.994516Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Because the pmtnum_254L column has outliers then we will compute it with mediain\ntrain_static_0 = fill_na(train_static_0, 'pmtnum_254L',train_static_0['pmtnum_254L'].median_approximate().compute())","metadata":{"execution":{"iopub.status.busy":"2024-05-03T05:50:44.25444Z","iopub.execute_input":"2024-05-03T05:50:44.254794Z","iopub.status.idle":"2024-05-03T05:50:59.090029Z","shell.execute_reply.started":"2024-05-03T05:50:44.254761Z","shell.execute_reply":"2024-05-03T05:50:59.089268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_static_0['pmtnum_254L'].isna().sum().compute()","metadata":{"execution":{"iopub.status.busy":"2024-05-03T06:36:26.984771Z","iopub.execute_input":"2024-05-03T06:36:26.985542Z","iopub.status.idle":"2024-05-03T06:36:33.247943Z","shell.execute_reply.started":"2024-05-03T06:36:26.985498Z","shell.execute_reply":"2024-05-03T06:36:33.24704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_static_0['paytype1st_925L'].value_counts().compute()","metadata":{"execution":{"iopub.status.busy":"2024-05-02T17:34:27.708475Z","iopub.execute_input":"2024-05-02T17:34:27.708753Z","iopub.status.idle":"2024-05-02T17:34:42.625676Z","shell.execute_reply.started":"2024-05-02T17:34:27.708729Z","shell.execute_reply":"2024-05-02T17:34:42.624745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_countplot(train_static_0,'paytype1st_925L')","metadata":{"execution":{"iopub.status.busy":"2024-05-02T17:34:42.626937Z","iopub.execute_input":"2024-05-02T17:34:42.627287Z","iopub.status.idle":"2024-05-02T17:35:10.746429Z","shell.execute_reply.started":"2024-05-02T17:34:42.627249Z","shell.execute_reply":"2024-05-02T17:35:10.745503Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Because it has only one category then we will drop this column\ntrain_static_0 = train_static_0.drop(columns = 'paytype1st_925L', axis = 1)","metadata":{"execution":{"iopub.status.busy":"2024-05-02T17:35:10.747685Z","iopub.execute_input":"2024-05-02T17:35:10.74959Z","iopub.status.idle":"2024-05-02T17:35:10.755583Z","shell.execute_reply.started":"2024-05-02T17:35:10.749561Z","shell.execute_reply":"2024-05-02T17:35:10.75461Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"null_df_f.sort_values(by='Nulls_Percentage').head(10)","metadata":{"execution":{"iopub.status.busy":"2024-05-02T17:35:10.756852Z","iopub.execute_input":"2024-05-02T17:35:10.757626Z","iopub.status.idle":"2024-05-02T17:35:10.770868Z","shell.execute_reply.started":"2024-05-02T17:35:10.757593Z","shell.execute_reply":"2024-05-02T17:35:10.769932Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_static_0['paytype_783L'].value_counts().compute()","metadata":{"execution":{"iopub.status.busy":"2024-05-03T06:39:57.533685Z","iopub.execute_input":"2024-05-03T06:39:57.534711Z","iopub.status.idle":"2024-05-03T06:40:04.456627Z","shell.execute_reply.started":"2024-05-03T06:39:57.534677Z","shell.execute_reply":"2024-05-03T06:40:04.455652Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Also because it has only one category then we're gonna drop this column \ntrain_static_0 = train_static_0.drop(columns = 'paytype_783L', axis = 1)","metadata":{"execution":{"iopub.status.busy":"2024-05-02T17:35:26.312235Z","iopub.execute_input":"2024-05-02T17:35:26.312548Z","iopub.status.idle":"2024-05-02T17:35:26.318934Z","shell.execute_reply.started":"2024-05-02T17:35:26.312522Z","shell.execute_reply":"2024-05-02T17:35:26.31793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_countplot(train_static_0,'posfpd10lastmonth_333P')","metadata":{"execution":{"iopub.status.busy":"2024-05-02T17:35:26.320426Z","iopub.execute_input":"2024-05-02T17:35:26.320794Z","iopub.status.idle":"2024-05-02T17:35:54.57832Z","shell.execute_reply.started":"2024-05-02T17:35:26.320759Z","shell.execute_reply":"2024-05-02T17:35:54.577317Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_static_0 = fill_na(train_static_0, 'posfpd10lastmonth_333P',0.0)","metadata":{"execution":{"iopub.status.busy":"2024-05-02T17:35:54.579695Z","iopub.execute_input":"2024-05-02T17:35:54.580081Z","iopub.status.idle":"2024-05-02T17:35:54.588736Z","shell.execute_reply.started":"2024-05-02T17:35:54.580043Z","shell.execute_reply":"2024-05-02T17:35:54.587658Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_static_0['posfpd30lastmonth_3976960P'].value_counts().compute()","metadata":{"execution":{"iopub.status.busy":"2024-05-02T17:35:54.590721Z","iopub.execute_input":"2024-05-02T17:35:54.591159Z","iopub.status.idle":"2024-05-02T17:36:09.665357Z","shell.execute_reply.started":"2024-05-02T17:35:54.591119Z","shell.execute_reply":"2024-05-02T17:36:09.664436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_static_0 = fill_na(train_static_0, 'posfpd30lastmonth_3976960P',0.0)","metadata":{"execution":{"iopub.status.busy":"2024-05-02T17:36:09.666851Z","iopub.execute_input":"2024-05-02T17:36:09.667147Z","iopub.status.idle":"2024-05-02T17:36:09.674189Z","shell.execute_reply.started":"2024-05-02T17:36:09.667122Z","shell.execute_reply":"2024-05-02T17:36:09.673304Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_static_0['posfstqpd30lastmonth_3976962P'].value_counts().compute()","metadata":{"execution":{"iopub.status.busy":"2024-05-02T17:36:09.675259Z","iopub.execute_input":"2024-05-02T17:36:09.675584Z","iopub.status.idle":"2024-05-02T17:36:25.09951Z","shell.execute_reply.started":"2024-05-02T17:36:09.675551Z","shell.execute_reply":"2024-05-02T17:36:25.0986Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_static_0 = fill_na(train_static_0, 'posfstqpd30lastmonth_3976962P',0.0)","metadata":{"execution":{"iopub.status.busy":"2024-05-02T17:36:25.100939Z","iopub.execute_input":"2024-05-02T17:36:25.101741Z","iopub.status.idle":"2024-05-02T17:36:25.109289Z","shell.execute_reply.started":"2024-05-02T17:36:25.101703Z","shell.execute_reply":"2024-05-02T17:36:25.108327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"null_df_f.sort_values(by='Nulls_Percentage').head(10)","metadata":{"execution":{"iopub.status.busy":"2024-05-02T17:36:25.110548Z","iopub.execute_input":"2024-05-02T17:36:25.110921Z","iopub.status.idle":"2024-05-02T17:36:25.126136Z","shell.execute_reply.started":"2024-05-02T17:36:25.110885Z","shell.execute_reply":"2024-05-02T17:36:25.125339Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_static_0['interestrate_311L'].value_counts().compute()","metadata":{"execution":{"iopub.status.busy":"2024-05-02T17:36:25.127774Z","iopub.execute_input":"2024-05-02T17:36:25.128077Z","iopub.status.idle":"2024-05-02T17:36:40.519724Z","shell.execute_reply.started":"2024-05-02T17:36:25.128051Z","shell.execute_reply":"2024-05-02T17:36:40.518789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_histogram_and_boxplot(train_static_0, 'interestrate_311L')","metadata":{"execution":{"iopub.status.busy":"2024-05-02T17:36:40.525245Z","iopub.execute_input":"2024-05-02T17:36:40.525573Z","iopub.status.idle":"2024-05-02T17:38:44.615056Z","shell.execute_reply.started":"2024-05-02T17:36:40.525549Z","shell.execute_reply":"2024-05-02T17:38:44.614139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Ill impute the missing values with medain because it has outliers.\ntrain_static_0 = fill_na(train_static_0, 'interestrate_311L',train_static_0['interestrate_311L'].median_approximate().compute())","metadata":{"execution":{"iopub.status.busy":"2024-05-02T17:38:44.616514Z","iopub.execute_input":"2024-05-02T17:38:44.617215Z","iopub.status.idle":"2024-05-02T17:38:59.949942Z","shell.execute_reply.started":"2024-05-02T17:38:44.61718Z","shell.execute_reply":"2024-05-02T17:38:59.948872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_static_0['eir_270L'].value_counts().compute()","metadata":{"execution":{"iopub.status.busy":"2024-05-02T17:38:59.951377Z","iopub.execute_input":"2024-05-02T17:38:59.951773Z","iopub.status.idle":"2024-05-02T17:39:16.044498Z","shell.execute_reply.started":"2024-05-02T17:38:59.951737Z","shell.execute_reply":"2024-05-02T17:39:16.043587Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_histogram_and_boxplot(train_static_0, 'eir_270L')","metadata":{"execution":{"iopub.status.busy":"2024-05-02T17:39:16.045685Z","iopub.execute_input":"2024-05-02T17:39:16.045954Z","iopub.status.idle":"2024-05-02T17:41:20.293121Z","shell.execute_reply.started":"2024-05-02T17:39:16.045929Z","shell.execute_reply":"2024-05-02T17:41:20.292135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Ill impute the missing values with medain because it has outliers.\ntrain_static_0 = fill_na(train_static_0, 'eir_270L',train_static_0['eir_270L'].median_approximate().compute())","metadata":{"execution":{"iopub.status.busy":"2024-05-02T17:41:20.294686Z","iopub.execute_input":"2024-05-02T17:41:20.295521Z","iopub.status.idle":"2024-05-02T17:41:35.553051Z","shell.execute_reply.started":"2024-05-02T17:41:20.295481Z","shell.execute_reply":"2024-05-02T17:41:35.552243Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Note:\n*** there is a relation between 'eir_270L' and 'interestrate_311L'**","metadata":{}},{"cell_type":"code","source":"null_df_f.sort_values(by='Nulls_Percentage').iloc[8:20]","metadata":{"execution":{"iopub.status.busy":"2024-05-02T17:41:35.554258Z","iopub.execute_input":"2024-05-02T17:41:35.554615Z","iopub.status.idle":"2024-05-02T17:41:35.567034Z","shell.execute_reply.started":"2024-05-02T17:41:35.554587Z","shell.execute_reply":"2024-05-02T17:41:35.565981Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_static_0['price_1097A'].value_counts().compute()","metadata":{"execution":{"iopub.status.busy":"2024-05-02T17:41:35.568257Z","iopub.execute_input":"2024-05-02T17:41:35.568556Z","iopub.status.idle":"2024-05-02T17:41:51.093084Z","shell.execute_reply.started":"2024-05-02T17:41:35.568532Z","shell.execute_reply":"2024-05-02T17:41:51.092168Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_histogram_and_boxplot(train_static_0, 'price_1097A')","metadata":{"execution":{"iopub.status.busy":"2024-05-02T17:41:51.094328Z","iopub.execute_input":"2024-05-02T17:41:51.094607Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Ill impute the missing values with medain because it has outliers.\ntrain_static_0 = fill_na(train_static_0, 'price_1097A',train_static_0['price_1097A'].median_approximate().compute())","metadata":{"execution":{"iopub.status.idle":"2024-05-02T17:44:13.006919Z","shell.execute_reply.started":"2024-05-02T17:43:57.407206Z","shell.execute_reply":"2024-05-02T17:44:13.006071Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_static_0['lastapplicationdate_877D'].value_counts().compute().reset_index()","metadata":{"execution":{"iopub.status.busy":"2024-05-02T17:44:13.008129Z","iopub.execute_input":"2024-05-02T17:44:13.008437Z","iopub.status.idle":"2024-05-02T17:44:28.875388Z","shell.execute_reply.started":"2024-05-02T17:44:13.008411Z","shell.execute_reply":"2024-05-02T17:44:28.874555Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_time_series_optimized(data, time_column, value_column, sample_size=.1, xlabel=None, ylabel=None, title=None):\n    \"\"\"\n    Plot a time series line plot with downsampling for optimization using Seaborn.\n    \n    Parameters:\n        data (DataFrame): The DataFrame containing the time series data.\n        time_column (str): The name of the column containing the time values.\n        value_column (str): The name of the column containing the values to be plotted.\n        sample_size (int): Number of data points to sample for plotting. Defaults to 1000.\n        xlabel (str, optional): Label for the x-axis.\n        ylabel (str, optional): Label for the y-axis.\n        title (str, optional): Title for the plot.\n    \n    Returns:\n        None\n    \"\"\"\n    # Downsampling the data\n    if len(data) > sample_size:\n        data_downsampled = data.sample(sample_size)\n    else:\n        data_downsampled = data\n    \n    sns.lineplot(x=time_column, y=value_column, data=data_downsampled)\n    plt.xlabel(xlabel)\n    plt.ylabel(ylabel)\n    plt.title(title)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-05-02T17:44:28.876472Z","iopub.execute_input":"2024-05-02T17:44:28.87675Z","iopub.status.idle":"2024-05-02T17:44:28.883393Z","shell.execute_reply.started":"2024-05-02T17:44:28.876727Z","shell.execute_reply":"2024-05-02T17:44:28.882494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_static_0[['case_id','lastapplicationdate_877D']].head(10)","metadata":{"execution":{"iopub.status.busy":"2024-05-02T17:44:28.884492Z","iopub.execute_input":"2024-05-02T17:44:28.884746Z","iopub.status.idle":"2024-05-02T17:44:52.112714Z","shell.execute_reply.started":"2024-05-02T17:44:28.884725Z","shell.execute_reply":"2024-05-02T17:44:52.111826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_static_0['lastapplicationdate_877D']=dd.to_datetime(train_static_0['lastapplicationdate_877D'],unit='ns')","metadata":{"execution":{"iopub.status.busy":"2024-05-02T17:44:52.113787Z","iopub.execute_input":"2024-05-02T17:44:52.114033Z","iopub.status.idle":"2024-05-02T17:44:52.122246Z","shell.execute_reply.started":"2024-05-02T17:44:52.114011Z","shell.execute_reply":"2024-05-02T17:44:52.121425Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_static_0['lastapplicationdate_877D'].dt.year.head()","metadata":{"execution":{"iopub.status.busy":"2024-05-02T17:44:52.123578Z","iopub.execute_input":"2024-05-02T17:44:52.12391Z","iopub.status.idle":"2024-05-02T17:45:15.691687Z","shell.execute_reply.started":"2024-05-02T17:44:52.123885Z","shell.execute_reply":"2024-05-02T17:45:15.690652Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_static_0['lastst_736L'].value_counts().compute().reset_index()","metadata":{"execution":{"iopub.status.busy":"2024-05-02T17:45:15.693111Z","iopub.execute_input":"2024-05-02T17:45:15.693764Z","iopub.status.idle":"2024-05-02T17:45:31.89412Z","shell.execute_reply.started":"2024-05-02T17:45:15.693736Z","shell.execute_reply":"2024-05-02T17:45:31.893226Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_countplot(train_static_0,'lastst_736L')","metadata":{"execution":{"iopub.status.busy":"2024-05-02T17:45:31.895314Z","iopub.execute_input":"2024-05-02T17:45:31.895593Z","iopub.status.idle":"2024-05-02T17:46:02.185459Z","shell.execute_reply.started":"2024-05-02T17:45:31.895567Z","shell.execute_reply":"2024-05-02T17:46:02.184559Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# hello world again","metadata":{}},{"cell_type":"code","source":"my_df = null_df_f.sort_values(by='Nulls_Percentage').iloc[12:20]\nmy_df","metadata":{"execution":{"iopub.status.busy":"2024-05-03T05:56:23.083956Z","iopub.execute_input":"2024-05-03T05:56:23.084998Z","iopub.status.idle":"2024-05-03T05:56:23.104203Z","shell.execute_reply.started":"2024-05-03T05:56:23.084952Z","shell.execute_reply":"2024-05-03T05:56:23.103122Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_static_0['maxdpdlast24m_143P'].value_counts().compute().reset_index().head(50)","metadata":{"execution":{"iopub.status.busy":"2024-05-03T06:07:39.739268Z","iopub.execute_input":"2024-05-03T06:07:39.740135Z","iopub.status.idle":"2024-05-03T06:07:57.926768Z","shell.execute_reply.started":"2024-05-03T06:07:39.740086Z","shell.execute_reply":"2024-05-03T06:07:57.925362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_static_0['maxdpdlast24m_143P'].isna().sum().compute()","metadata":{"execution":{"iopub.status.busy":"2024-05-03T06:09:37.459801Z","iopub.execute_input":"2024-05-03T06:09:37.460173Z","iopub.status.idle":"2024-05-03T06:09:54.246531Z","shell.execute_reply.started":"2024-05-03T06:09:37.460142Z","shell.execute_reply":"2024-05-03T06:09:54.24561Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_static_0 = fill_na(train_static_0, 'maxdpdlast24m_143P',0.0)","metadata":{"execution":{"iopub.status.busy":"2024-05-03T06:40:50.818572Z","iopub.execute_input":"2024-05-03T06:40:50.819245Z","iopub.status.idle":"2024-05-03T06:40:50.827479Z","shell.execute_reply.started":"2024-05-03T06:40:50.819195Z","shell.execute_reply":"2024-05-03T06:40:50.826394Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_histogram_and_boxplot_ver_2(train_static_0, 'maxdpdlast24m_143P')","metadata":{"execution":{"iopub.status.busy":"2024-05-03T06:58:02.795408Z","iopub.execute_input":"2024-05-03T06:58:02.796343Z","iopub.status.idle":"2024-05-03T07:00:13.01252Z","shell.execute_reply.started":"2024-05-03T06:58:02.796299Z","shell.execute_reply":"2024-05-03T07:00:13.011438Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Compute the approximate median for 'maxdpdlast24m_143P'\napprox_median_value = train_static_0['maxdpdlast24m_143P'].median_approximate().compute()\n\n# Impute missing values with the approximate median\ntrain_static_0['maxdpdlast24m_143P'] = train_static_0['maxdpdlast24m_143P'].fillna(approx_median_value)","metadata":{"execution":{"iopub.status.busy":"2024-05-03T07:36:41.344529Z","iopub.execute_input":"2024-05-03T07:36:41.345373Z","iopub.status.idle":"2024-05-03T07:36:48.382658Z","shell.execute_reply.started":"2024-05-03T07:36:41.345333Z","shell.execute_reply":"2024-05-03T07:36:48.381848Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_static_0.head()","metadata":{"execution":{"iopub.status.busy":"2024-05-03T06:37:33.837082Z","iopub.execute_input":"2024-05-03T06:37:33.837551Z","iopub.status.idle":"2024-05-03T06:38:07.256236Z","shell.execute_reply.started":"2024-05-03T06:37:33.837522Z","shell.execute_reply":"2024-05-03T06:38:07.254947Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_static_0.info()","metadata":{"execution":{"iopub.status.busy":"2024-05-03T06:38:31.149366Z","iopub.execute_input":"2024-05-03T06:38:31.149764Z","iopub.status.idle":"2024-05-03T06:38:31.15777Z","shell.execute_reply.started":"2024-05-03T06:38:31.149733Z","shell.execute_reply":"2024-05-03T06:38:31.156565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_static_0['maxdpdlast24m_143P'] = train_static_0['maxdpdlast24m_143P'].replace([np.inf, -np.inf], np.nan)\ntrain_static_0 = fill_na(train_static_0, 'maxdpdlast24m_143P',0.0)\n\n","metadata":{"execution":{"iopub.status.busy":"2024-05-03T06:51:55.63833Z","iopub.execute_input":"2024-05-03T06:51:55.63869Z","iopub.status.idle":"2024-05-03T06:51:55.650861Z","shell.execute_reply.started":"2024-05-03T06:51:55.638662Z","shell.execute_reply":"2024-05-03T06:51:55.649773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import dask.dataframe as dd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport pandas as pd\n\n# Set the option to convert inf values to NaN\npd.set_option('mode.use_inf_as_na', True)\n\ndef plot_histogram_and_boxplot(df, column_name):\n    # Ensure df is a Dask DataFrame\n    if not isinstance(df, dd.DataFrame):\n        raise ValueError(\"Input dataframe must be a Dask DataFrame\")\n\n    # Create the figure and subplots\n    fig, axs = plt.subplots(1, 2, figsize=(12, 5))\n\n    # Plot histogram\n    hist_plot = sns.histplot(df[column_name], kde=False, ax=axs[0], bins=20)\n    axs[0].set_xlabel('Values')\n    axs[0].set_ylabel('Frequency')\n    axs[0].set_title('Histogram of {}'.format(column_name))\n\n    # Plot box plot\n    box_plot = sns.boxplot(y=df[column_name], ax=axs[1])\n    axs[1].set_ylabel('Values')\n    axs[1].set_title('Box Plot of {}'.format(column_name))\n\n    # Adjust layout to prevent overlapping\n    plt.tight_layout()\n\n    # Show plot\n    plt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-05-03T11:22:14.559534Z","iopub.execute_input":"2024-05-03T11:22:14.560662Z","iopub.status.idle":"2024-05-03T11:22:14.570323Z","shell.execute_reply.started":"2024-05-03T11:22:14.560629Z","shell.execute_reply":"2024-05-03T11:22:14.569231Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Analyze a specific column\nresult = analyze_column(train_static_0, 'maxdpdlast24m_143P')\nprint(result)","metadata":{"execution":{"iopub.status.busy":"2024-05-03T07:01:58.58681Z","iopub.execute_input":"2024-05-03T07:01:58.587254Z","iopub.status.idle":"2024-05-03T07:02:12.266787Z","shell.execute_reply.started":"2024-05-03T07:01:58.587209Z","shell.execute_reply":"2024-05-03T07:02:12.265673Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_histogram_and_boxplot_ver_2(train_static_0, 'maxdpdlast24m_143P')","metadata":{"execution":{"iopub.status.busy":"2024-05-03T07:38:44.495059Z","iopub.execute_input":"2024-05-03T07:38:44.495765Z","iopub.status.idle":"2024-05-03T07:40:57.955912Z","shell.execute_reply.started":"2024-05-03T07:38:44.495733Z","shell.execute_reply":"2024-05-03T07:40:57.955107Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# pppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppp\n# pppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppppp\n","metadata":{}},{"cell_type":"code","source":"my_df","metadata":{"execution":{"iopub.status.busy":"2024-05-03T07:03:01.045619Z","iopub.execute_input":"2024-05-03T07:03:01.046866Z","iopub.status.idle":"2024-05-03T07:03:01.059185Z","shell.execute_reply.started":"2024-05-03T07:03:01.046815Z","shell.execute_reply":"2024-05-03T07:03:01.058227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Analyze a specific column\nresult = analyze_column(train_static_0, 'maxdpdlast12m_727P')\nprint(result)","metadata":{"execution":{"iopub.status.busy":"2024-05-03T07:37:13.727345Z","iopub.execute_input":"2024-05-03T07:37:13.727764Z","iopub.status.idle":"2024-05-03T07:37:27.409396Z","shell.execute_reply.started":"2024-05-03T07:37:13.727732Z","shell.execute_reply":"2024-05-03T07:37:27.408435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_1 = train_static_0['maxdpdlast12m_727P'].value_counts().compute().reset_index()","metadata":{"execution":{"iopub.status.busy":"2024-05-03T07:51:38.84332Z","iopub.execute_input":"2024-05-03T07:51:38.844239Z","iopub.status.idle":"2024-05-03T07:51:46.028448Z","shell.execute_reply.started":"2024-05-03T07:51:38.844185Z","shell.execute_reply":"2024-05-03T07:51:46.027681Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_static_0 = fill_na(train_static_0, 'maxdpdlast12m_727P',0.0)","metadata":{"execution":{"iopub.status.busy":"2024-05-03T07:05:01.841543Z","iopub.execute_input":"2024-05-03T07:05:01.842401Z","iopub.status.idle":"2024-05-03T07:05:01.85169Z","shell.execute_reply.started":"2024-05-03T07:05:01.842365Z","shell.execute_reply":"2024-05-03T07:05:01.850291Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_histogram_and_boxplot_ver_2(train_static_0, 'maxdpdlast12m_727P')","metadata":{"execution":{"iopub.status.busy":"2024-05-03T07:05:23.245085Z","iopub.execute_input":"2024-05-03T07:05:23.245859Z","iopub.status.idle":"2024-05-03T07:07:33.706909Z","shell.execute_reply.started":"2024-05-03T07:05:23.245823Z","shell.execute_reply":"2024-05-03T07:07:33.705616Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Assuming you've already loaded the 'train_static_0' dataset\nthreshold = 0.8  # Set the threshold (e.g., 80%)\nrows_with_high_nan = train_static_0.isnull().sum(axis=1) >= threshold * train_static_0.shape[1]\n\n# Get the rows with high NaN values\nhigh_nan_rows = train_static_0[rows_with_high_nan]\n\n# Calculate the number of rows\nnum_high_nan_rows = high_nan_rows.shape[0]\nprint(f\"Number of rows with 80% or more NaN values: {num_high_nan_rows}\")\n","metadata":{"execution":{"iopub.status.busy":"2024-05-03T07:31:36.590446Z","iopub.execute_input":"2024-05-03T07:31:36.59082Z","iopub.status.idle":"2024-05-03T07:31:36.627822Z","shell.execute_reply.started":"2024-05-03T07:31:36.590791Z","shell.execute_reply":"2024-05-03T07:31:36.626834Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Compute the approximate median for 'maxdpdlast24m_143P'\napprox_median_value = train_static_0['maxdpdlast12m_727P'].median_approximate().compute()\n\n# Impute missing values with the approximate median\ntrain_static_0['maxdpdlast12m_727P'] = train_static_0['maxdpdlast12m_727P'].fillna(approx_median_value)","metadata":{"execution":{"iopub.status.busy":"2024-05-03T07:48:18.575077Z","iopub.execute_input":"2024-05-03T07:48:18.57546Z","iopub.status.idle":"2024-05-03T07:48:25.518753Z","shell.execute_reply.started":"2024-05-03T07:48:18.57543Z","shell.execute_reply":"2024-05-03T07:48:25.517941Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Analyze a specific column\nresult = analyze_column(train_static_0, 'maxdpdlast12m_727P')\nprint(result)","metadata":{"execution":{"iopub.status.busy":"2024-05-03T07:49:07.841Z","iopub.execute_input":"2024-05-03T07:49:07.841417Z","iopub.status.idle":"2024-05-03T07:49:21.510603Z","shell.execute_reply.started":"2024-05-03T07:49:07.841384Z","shell.execute_reply":"2024-05-03T07:49:21.509649Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"my_df","metadata":{"execution":{"iopub.status.busy":"2024-05-03T07:49:56.011087Z","iopub.execute_input":"2024-05-03T07:49:56.01185Z","iopub.status.idle":"2024-05-03T07:49:56.025928Z","shell.execute_reply.started":"2024-05-03T07:49:56.011816Z","shell.execute_reply":"2024-05-03T07:49:56.024781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndf_2 = train_static_0['maxdpdlast6m_474P'].value_counts().compute().reset_index()\ndf_2","metadata":{"execution":{"iopub.status.busy":"2024-05-03T07:50:42.336144Z","iopub.execute_input":"2024-05-03T07:50:42.336834Z","iopub.status.idle":"2024-05-03T07:50:49.566157Z","shell.execute_reply.started":"2024-05-03T07:50:42.336802Z","shell.execute_reply":"2024-05-03T07:50:49.565262Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"concatenated_df = pd.concat([df_1, df_2], axis=1)\nconcatenated_df\n","metadata":{"execution":{"iopub.status.busy":"2024-05-03T07:54:56.386636Z","iopub.execute_input":"2024-05-03T07:54:56.38706Z","iopub.status.idle":"2024-05-03T07:54:56.408799Z","shell.execute_reply.started":"2024-05-03T07:54:56.387026Z","shell.execute_reply":"2024-05-03T07:54:56.407641Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_histogram_and_boxplot_ver_2(train_static_0, 'maxdpdlast6m_474P')","metadata":{"execution":{"iopub.status.busy":"2024-05-03T07:56:45.090713Z","iopub.execute_input":"2024-05-03T07:56:45.09148Z","iopub.status.idle":"2024-05-03T07:58:55.328337Z","shell.execute_reply.started":"2024-05-03T07:56:45.091444Z","shell.execute_reply":"2024-05-03T07:58:55.327277Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_histogram_and_boxplot(train_static_0, 'maxdpdlast6m_474P')\n","metadata":{"execution":{"iopub.status.busy":"2024-05-03T08:01:42.830884Z","iopub.execute_input":"2024-05-03T08:01:42.831601Z","iopub.status.idle":"2024-05-03T08:03:53.480638Z","shell.execute_reply.started":"2024-05-03T08:01:42.831566Z","shell.execute_reply":"2024-05-03T08:03:53.479407Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_static_0[train_static_0['maxdpdlast6m_474P']!=0].head(50)['maxdpdlast6m_474P']","metadata":{"execution":{"iopub.status.busy":"2024-05-03T08:14:29.661983Z","iopub.execute_input":"2024-05-03T08:14:29.662702Z","iopub.status.idle":"2024-05-03T08:15:05.087813Z","shell.execute_reply.started":"2024-05-03T08:14:29.662669Z","shell.execute_reply":"2024-05-03T08:15:05.086456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"col_names = my_df.Column.values\ncol_names","metadata":{"execution":{"iopub.status.busy":"2024-05-03T08:25:21.7004Z","iopub.execute_input":"2024-05-03T08:25:21.700825Z","iopub.status.idle":"2024-05-03T08:25:21.708385Z","shell.execute_reply.started":"2024-05-03T08:25:21.700794Z","shell.execute_reply":"2024-05-03T08:25:21.707208Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import dask.dataframe as dd\n\ndef count_zeros_per_row(df, columns):\n    \"\"\"\n    Counts the number of zeros (0) in each row for specified columns in a Dask DataFrame.\n\n    Args:\n        df (dask.dataframe.DataFrame): Input Dask DataFrame.\n        columns (list): List of column names to analyze.\n\n    Returns:\n        dask.dataframe.Series: Series containing the count of zeros per row.\n    \"\"\"\n    # Apply a function to each row (axis=1) for the specified columns\n    def count_zeros(row):\n        return (row[columns] == 0).sum()\n\n    # Compute the result\n    return df.apply(count_zeros, axis=1, meta=('int64', 'i8'))\n\n\n\ncolumns_to_count = ['maxdpdlast24m_143P', 'maxdpdlast12m_727P', 'maxdpdlast6m_474P',\n                    'maxdpdlast9m_1059P', 'maxdpdtolerance_374P', 'maxdebt4_972A',\n                    'mastercontrelectronic_519L', 'maxdpdlast3m_392P']\n\nzero_counts = count_zeros_per_row(train_static_0, columns_to_count)\nprint(zero_counts.compute())\n","metadata":{"execution":{"iopub.status.busy":"2024-05-03T08:29:01.061163Z","iopub.execute_input":"2024-05-03T08:29:01.062144Z","iopub.status.idle":"2024-05-03T08:43:50.219117Z","shell.execute_reply.started":"2024-05-03T08:29:01.062107Z","shell.execute_reply":"2024-05-03T08:43:50.218061Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import dask.dataframe as dd\n\ndef count_zeros_in_column(df, col_name):\n    \"\"\"\n    Counts the number of zeros (0) in a specified column of a Dask DataFrame.\n\n    Args:\n        df (dask.dataframe.DataFrame): Input Dask DataFrame.\n        col_name (str): Name of the column to analyze.\n\n    Returns:\n        dask.dataframe.Series: Series containing the count of zeros per column.\n    \"\"\"\n    return (df[col_name] == 0).sum()\n\n","metadata":{"execution":{"iopub.status.busy":"2024-05-03T11:21:38.502565Z","iopub.execute_input":"2024-05-03T11:21:38.50321Z","iopub.status.idle":"2024-05-03T11:21:38.508977Z","shell.execute_reply.started":"2024-05-03T11:21:38.503176Z","shell.execute_reply":"2024-05-03T11:21:38.507622Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Example usage:\n# Assuming you have a Dask DataFrame named 'my_dask_df'\ncolumn_to_analyze = 'maxdpdlast6m_474P'\nzero_counts_per_column = count_zeros_in_column(train_static_0, column_to_analyze)\n\n# Compute the result\nprint(zero_counts_per_column.compute())\n","metadata":{"execution":{"iopub.status.busy":"2024-05-03T08:47:25.277313Z","iopub.execute_input":"2024-05-03T08:47:25.278037Z","iopub.status.idle":"2024-05-03T08:47:31.966728Z","shell.execute_reply.started":"2024-05-03T08:47:25.278009Z","shell.execute_reply":"2024-05-03T08:47:31.965436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"col_names = list(col_names)\ncol_names","metadata":{"execution":{"iopub.status.busy":"2024-05-03T08:50:48.144876Z","iopub.execute_input":"2024-05-03T08:50:48.145347Z","iopub.status.idle":"2024-05-03T08:50:48.152668Z","shell.execute_reply.started":"2024-05-03T08:50:48.145308Z","shell.execute_reply":"2024-05-03T08:50:48.151493Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Example usage:\n# Assuming you have a Dask DataFrame named 'my_dask_df'\ncolumn_to_analyze = 'maxdpdlast24m_143P'\nzero_counts_per_column = count_zeros_in_column(train_static_0, column_to_analyze)\n\n# Compute the result\nprint(zero_counts_per_column.compute())\n","metadata":{"execution":{"iopub.status.busy":"2024-05-03T08:51:12.324827Z","iopub.execute_input":"2024-05-03T08:51:12.325754Z","iopub.status.idle":"2024-05-03T08:51:18.907149Z","shell.execute_reply.started":"2024-05-03T08:51:12.325712Z","shell.execute_reply":"2024-05-03T08:51:18.905886Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Example usage:\n# Assuming you have a Dask DataFrame named 'my_dask_df'\ncolumn_to_analyze = 'maxdpdlast12m_727P'\nzero_counts_per_column = count_zeros_in_column(train_static_0, column_to_analyze)\n\n# Compute the result\nprint(zero_counts_per_column.compute())\n","metadata":{"execution":{"iopub.status.busy":"2024-05-03T08:52:04.447815Z","iopub.execute_input":"2024-05-03T08:52:04.448624Z","iopub.status.idle":"2024-05-03T08:52:11.003734Z","shell.execute_reply.started":"2024-05-03T08:52:04.448581Z","shell.execute_reply":"2024-05-03T08:52:11.002774Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Example usage:\n# Assuming you have a Dask DataFrame named 'my_dask_df'\ncolumn_to_analyze = 'maxdpdlast6m_474P'\nzero_counts_per_column = count_zeros_in_column(train_static_0, column_to_analyze)\n\n# Compute the result\nprint(zero_counts_per_column.compute())\n","metadata":{"execution":{"iopub.status.busy":"2024-05-03T08:52:11.005983Z","iopub.execute_input":"2024-05-03T08:52:11.006677Z","iopub.status.idle":"2024-05-03T08:52:17.131044Z","shell.execute_reply.started":"2024-05-03T08:52:11.006637Z","shell.execute_reply":"2024-05-03T08:52:17.130133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Example usage:\n# Assuming you have a Dask DataFrame named 'my_dask_df'\ncolumn_to_analyze = 'maxdpdlast9m_1059P'\nzero_counts_per_column = count_zeros_in_column(train_static_0, column_to_analyze)\n\n# Compute the result\nprint(zero_counts_per_column.compute())\n","metadata":{"execution":{"iopub.status.busy":"2024-05-03T08:52:17.132477Z","iopub.execute_input":"2024-05-03T08:52:17.132832Z","iopub.status.idle":"2024-05-03T08:52:23.22603Z","shell.execute_reply.started":"2024-05-03T08:52:17.132799Z","shell.execute_reply":"2024-05-03T08:52:23.224906Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# FILLING NANS WITH 0 FOR ALL col_names","metadata":{}},{"cell_type":"code","source":"for col in col_names:\n    train_static_0 = fill_na(train_static_0, col,0.0)","metadata":{"execution":{"iopub.status.busy":"2024-05-03T09:18:20.363937Z","iopub.execute_input":"2024-05-03T09:18:20.364382Z","iopub.status.idle":"2024-05-03T09:18:20.399771Z","shell.execute_reply.started":"2024-05-03T09:18:20.364351Z","shell.execute_reply":"2024-05-03T09:18:20.398525Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in col_names:\n    num_na = train_static_0[col].isna().sum().compute()\n    print(f'the number of nans in {col} is: {num_na}')\n","metadata":{"execution":{"iopub.status.busy":"2024-05-03T09:20:29.445288Z","iopub.execute_input":"2024-05-03T09:20:29.446142Z","iopub.status.idle":"2024-05-03T09:21:30.180309Z","shell.execute_reply.started":"2024-05-03T09:20:29.44611Z","shell.execute_reply":"2024-05-03T09:21:30.178909Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# secod round :---------------------------------------","metadata":{}},{"cell_type":"code","source":"null_df_f.sort_values(by='Nulls_Percentage').iloc[19:30]","metadata":{"execution":{"iopub.status.busy":"2024-05-03T11:23:49.650923Z","iopub.execute_input":"2024-05-03T11:23:49.651275Z","iopub.status.idle":"2024-05-03T11:23:49.664448Z","shell.execute_reply.started":"2024-05-03T11:23:49.651246Z","shell.execute_reply":"2024-05-03T11:23:49.663452Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"my_df = null_df_f.sort_values(by='Nulls_Percentage').iloc[19:30]\nmy_df","metadata":{"execution":{"iopub.status.busy":"2024-05-03T11:56:30.997117Z","iopub.execute_input":"2024-05-03T11:56:30.997801Z","iopub.status.idle":"2024-05-03T11:56:31.011111Z","shell.execute_reply.started":"2024-05-03T11:56:30.997747Z","shell.execute_reply":"2024-05-03T11:56:31.010062Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"col_to_drop = []","metadata":{"execution":{"iopub.status.busy":"2024-05-03T11:48:03.79906Z","iopub.execute_input":"2024-05-03T11:48:03.799698Z","iopub.status.idle":"2024-05-03T11:48:03.804719Z","shell.execute_reply.started":"2024-05-03T11:48:03.799669Z","shell.execute_reply":"2024-05-03T11:48:03.803765Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_static_0['mastercontrelectronic_519L'].value_counts().compute().reset_index().head()","metadata":{"execution":{"iopub.status.busy":"2024-05-03T11:26:35.49483Z","iopub.execute_input":"2024-05-03T11:26:35.495201Z","iopub.status.idle":"2024-05-03T11:26:41.927973Z","shell.execute_reply.started":"2024-05-03T11:26:35.495173Z","shell.execute_reply":"2024-05-03T11:26:41.927098Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# this col is dropped","metadata":{}},{"cell_type":"code","source":"count_zeros_in_column(train_static_0,columne_to_check[0]).compute()","metadata":{"execution":{"iopub.status.busy":"2024-05-03T11:45:35.975009Z","iopub.execute_input":"2024-05-03T11:45:35.975797Z","iopub.status.idle":"2024-05-03T11:45:42.25639Z","shell.execute_reply.started":"2024-05-03T11:45:35.975738Z","shell.execute_reply":"2024-05-03T11:45:42.255257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"col_to_drop.append(columne_to_check[0])","metadata":{"execution":{"iopub.status.busy":"2024-05-03T11:48:38.413053Z","iopub.execute_input":"2024-05-03T11:48:38.414021Z","iopub.status.idle":"2024-05-03T11:48:38.418345Z","shell.execute_reply.started":"2024-05-03T11:48:38.413988Z","shell.execute_reply":"2024-05-03T11:48:38.41727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def check_num_na(df,columne):\n    y = df[columne].isna().sum().compute()\n    return y\n    ","metadata":{"execution":{"iopub.status.busy":"2024-05-03T12:42:18.950074Z","iopub.execute_input":"2024-05-03T12:42:18.950443Z","iopub.status.idle":"2024-05-03T12:42:18.956166Z","shell.execute_reply.started":"2024-05-03T12:42:18.950416Z","shell.execute_reply":"2024-05-03T12:42:18.955186Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"columne_to_check = list(my_df.Column.values)\ncolumne_to_check","metadata":{"execution":{"iopub.status.busy":"2024-05-03T11:30:57.543077Z","iopub.execute_input":"2024-05-03T11:30:57.543431Z","iopub.status.idle":"2024-05-03T11:30:57.550015Z","shell.execute_reply.started":"2024-05-03T11:30:57.543403Z","shell.execute_reply":"2024-05-03T11:30:57.549007Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in columne_to_check:\n    y = check_num_na(train_static_0,i)\n    print(y)\n    ","metadata":{"execution":{"iopub.status.busy":"2024-05-03T11:36:10.660014Z","iopub.execute_input":"2024-05-03T11:36:10.660385Z","iopub.status.idle":"2024-05-03T11:38:31.020997Z","shell.execute_reply.started":"2024-05-03T11:36:10.660357Z","shell.execute_reply":"2024-05-03T11:38:31.020033Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_static_0[columne_to_check[1]].value_counts().compute().reset_index().head()\n","metadata":{"execution":{"iopub.status.busy":"2024-05-03T11:39:06.109734Z","iopub.execute_input":"2024-05-03T11:39:06.110435Z","iopub.status.idle":"2024-05-03T11:39:12.815002Z","shell.execute_reply.started":"2024-05-03T11:39:06.110406Z","shell.execute_reply":"2024-05-03T11:39:12.813807Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"count_zeros_in_column(train_static_0,columne_to_check[1]).compute()","metadata":{"execution":{"iopub.status.busy":"2024-05-03T11:45:18.676342Z","iopub.execute_input":"2024-05-03T11:45:18.677003Z","iopub.status.idle":"2024-05-03T11:45:25.05825Z","shell.execute_reply.started":"2024-05-03T11:45:18.676971Z","shell.execute_reply":"2024-05-03T11:45:25.057109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_histogram_and_boxplot(train_static_0, columne_to_check[1])\n","metadata":{"execution":{"iopub.status.busy":"2024-05-03T12:06:43.862089Z","iopub.execute_input":"2024-05-03T12:06:43.86274Z","iopub.status.idle":"2024-05-03T12:08:45.022328Z","shell.execute_reply.started":"2024-05-03T12:06:43.862709Z","shell.execute_reply":"2024-05-03T12:08:45.021373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_static_0 = fill_na(train_static_0, columne_to_check[1],0.0)","metadata":{"execution":{"iopub.status.busy":"2024-05-03T12:10:48.13679Z","iopub.execute_input":"2024-05-03T12:10:48.137723Z","iopub.status.idle":"2024-05-03T12:10:48.149386Z","shell.execute_reply.started":"2024-05-03T12:10:48.137692Z","shell.execute_reply":"2024-05-03T12:10:48.148397Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_static_0[columne_to_check[2]].value_counts().compute().reset_index().head()\n","metadata":{"execution":{"iopub.status.busy":"2024-05-03T11:39:16.633414Z","iopub.execute_input":"2024-05-03T11:39:16.633789Z","iopub.status.idle":"2024-05-03T11:39:23.29734Z","shell.execute_reply.started":"2024-05-03T11:39:16.633746Z","shell.execute_reply":"2024-05-03T11:39:23.296504Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"count_zeros_in_column(train_static_0,columne_to_check[2]).compute()","metadata":{"execution":{"iopub.status.busy":"2024-05-03T11:45:11.114431Z","iopub.execute_input":"2024-05-03T11:45:11.114786Z","iopub.status.idle":"2024-05-03T11:45:17.278063Z","shell.execute_reply.started":"2024-05-03T11:45:11.114762Z","shell.execute_reply":"2024-05-03T11:45:17.276728Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"col_to_drop.append(columne_to_check[2])","metadata":{"execution":{"iopub.status.busy":"2024-05-03T11:50:08.360023Z","iopub.execute_input":"2024-05-03T11:50:08.360423Z","iopub.status.idle":"2024-05-03T11:50:08.365334Z","shell.execute_reply.started":"2024-05-03T11:50:08.360392Z","shell.execute_reply":"2024-05-03T11:50:08.364167Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_static_0[columne_to_check[3]].value_counts().compute().reset_index().head()\n","metadata":{"execution":{"iopub.status.busy":"2024-05-03T11:39:23.29906Z","iopub.execute_input":"2024-05-03T11:39:23.299386Z","iopub.status.idle":"2024-05-03T11:39:29.639956Z","shell.execute_reply.started":"2024-05-03T11:39:23.299358Z","shell.execute_reply":"2024-05-03T11:39:29.638941Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"count_zeros_in_column(train_static_0,columne_to_check[3]).compute()","metadata":{"execution":{"iopub.status.busy":"2024-05-03T11:44:50.545806Z","iopub.execute_input":"2024-05-03T11:44:50.546144Z","iopub.status.idle":"2024-05-03T11:44:56.507729Z","shell.execute_reply.started":"2024-05-03T11:44:50.546117Z","shell.execute_reply":"2024-05-03T11:44:56.506724Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"col_to_drop.append(columne_to_check[3])","metadata":{"execution":{"iopub.status.busy":"2024-05-03T11:50:44.619512Z","iopub.execute_input":"2024-05-03T11:50:44.620177Z","iopub.status.idle":"2024-05-03T11:50:44.624488Z","shell.execute_reply.started":"2024-05-03T11:50:44.620144Z","shell.execute_reply":"2024-05-03T11:50:44.623493Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_static_0[columne_to_check[4]].value_counts().compute().reset_index().head()\n","metadata":{"execution":{"iopub.status.busy":"2024-05-03T11:39:29.641952Z","iopub.execute_input":"2024-05-03T11:39:29.642647Z","iopub.status.idle":"2024-05-03T11:39:36.793585Z","shell.execute_reply.started":"2024-05-03T11:39:29.64261Z","shell.execute_reply":"2024-05-03T11:39:36.792424Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"count_zeros_in_column(train_static_0,columne_to_check[4]).compute()","metadata":{"execution":{"iopub.status.busy":"2024-05-03T11:44:43.526819Z","iopub.execute_input":"2024-05-03T11:44:43.527167Z","iopub.status.idle":"2024-05-03T11:44:50.168716Z","shell.execute_reply.started":"2024-05-03T11:44:43.52714Z","shell.execute_reply":"2024-05-03T11:44:50.16776Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_histogram_and_boxplot(train_static_0, columne_to_check[4])\n","metadata":{"execution":{"iopub.status.busy":"2024-05-03T12:14:54.697837Z","iopub.execute_input":"2024-05-03T12:14:54.698594Z","iopub.status.idle":"2024-05-03T12:16:59.687701Z","shell.execute_reply.started":"2024-05-03T12:14:54.698565Z","shell.execute_reply":"2024-05-03T12:16:59.686709Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_static_0 = fill_na(train_static_0, columne_to_check[4],0.0)","metadata":{"execution":{"iopub.status.busy":"2024-05-03T12:14:31.012684Z","iopub.execute_input":"2024-05-03T12:14:31.013617Z","iopub.status.idle":"2024-05-03T12:14:31.023842Z","shell.execute_reply.started":"2024-05-03T12:14:31.013577Z","shell.execute_reply":"2024-05-03T12:14:31.022339Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_static_0[columne_to_check[5]].value_counts().compute().reset_index().head()\n","metadata":{"execution":{"iopub.status.busy":"2024-05-03T11:39:37.204979Z","iopub.execute_input":"2024-05-03T11:39:37.205395Z","iopub.status.idle":"2024-05-03T11:39:43.487251Z","shell.execute_reply.started":"2024-05-03T11:39:37.205363Z","shell.execute_reply":"2024-05-03T11:39:43.486275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"count_zeros_in_column(train_static_0,columne_to_check[5]).compute()","metadata":{"execution":{"iopub.status.busy":"2024-05-03T11:44:13.99172Z","iopub.execute_input":"2024-05-03T11:44:13.992657Z","iopub.status.idle":"2024-05-03T11:44:20.733321Z","shell.execute_reply.started":"2024-05-03T11:44:13.992617Z","shell.execute_reply":"2024-05-03T11:44:20.732133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_histogram_and_boxplot(train_static_0, columne_to_check[5])","metadata":{"execution":{"iopub.status.busy":"2024-05-03T12:17:16.752417Z","iopub.execute_input":"2024-05-03T12:17:16.753175Z","iopub.status.idle":"2024-05-03T12:19:16.269733Z","shell.execute_reply.started":"2024-05-03T12:17:16.753139Z","shell.execute_reply":"2024-05-03T12:19:16.268793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_static_0 = fill_na(train_static_0, columne_to_check[5],0.0)\n","metadata":{"execution":{"iopub.status.busy":"2024-05-03T12:19:21.996886Z","iopub.execute_input":"2024-05-03T12:19:21.997267Z","iopub.status.idle":"2024-05-03T12:19:22.006515Z","shell.execute_reply.started":"2024-05-03T12:19:21.997239Z","shell.execute_reply":"2024-05-03T12:19:22.00544Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_static_0[columne_to_check[6]].value_counts().compute().reset_index().head()\n","metadata":{"execution":{"iopub.status.busy":"2024-05-03T11:39:49.469262Z","iopub.execute_input":"2024-05-03T11:39:49.469632Z","iopub.status.idle":"2024-05-03T11:39:56.42723Z","shell.execute_reply.started":"2024-05-03T11:39:49.469606Z","shell.execute_reply":"2024-05-03T11:39:56.42615Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"count_zeros_in_column(train_static_0,columne_to_check[6]).compute()","metadata":{"execution":{"iopub.status.busy":"2024-05-03T11:43:59.718732Z","iopub.execute_input":"2024-05-03T11:43:59.719097Z","iopub.status.idle":"2024-05-03T11:44:06.024962Z","shell.execute_reply.started":"2024-05-03T11:43:59.719066Z","shell.execute_reply":"2024-05-03T11:44:06.023998Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_static_0[columne_to_check[7]].value_counts().compute().reset_index().head()\n","metadata":{"execution":{"iopub.status.busy":"2024-05-03T11:39:56.429392Z","iopub.execute_input":"2024-05-03T11:39:56.430032Z","iopub.status.idle":"2024-05-03T11:40:03.176933Z","shell.execute_reply.started":"2024-05-03T11:39:56.429996Z","shell.execute_reply":"2024-05-03T11:40:03.175939Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"count_zeros_in_column(train_static_0,columne_to_check[7]).compute()","metadata":{"execution":{"iopub.status.busy":"2024-05-03T11:43:44.611973Z","iopub.execute_input":"2024-05-03T11:43:44.612328Z","iopub.status.idle":"2024-05-03T11:43:50.956259Z","shell.execute_reply.started":"2024-05-03T11:43:44.612301Z","shell.execute_reply":"2024-05-03T11:43:50.95529Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_histogram_and_boxplot(train_static_0, columne_to_check[7])","metadata":{"execution":{"iopub.status.busy":"2024-05-03T12:20:04.279708Z","iopub.execute_input":"2024-05-03T12:20:04.280126Z","iopub.status.idle":"2024-05-03T12:22:10.523043Z","shell.execute_reply.started":"2024-05-03T12:20:04.280097Z","shell.execute_reply":"2024-05-03T12:22:10.521996Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_static_0 = fill_na(train_static_0, columne_to_check[7],train_static_0[columne_to_check[7]].median_approximate().compute())\n","metadata":{"execution":{"iopub.status.busy":"2024-05-03T12:36:16.019533Z","iopub.execute_input":"2024-05-03T12:36:16.020048Z","iopub.status.idle":"2024-05-03T12:36:22.172023Z","shell.execute_reply.started":"2024-05-03T12:36:16.020014Z","shell.execute_reply":"2024-05-03T12:36:22.171183Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_static_0[columne_to_check[8]].value_counts().compute().reset_index().head()\n","metadata":{"execution":{"iopub.status.busy":"2024-05-03T11:40:03.178957Z","iopub.execute_input":"2024-05-03T11:40:03.179953Z","iopub.status.idle":"2024-05-03T11:40:10.457796Z","shell.execute_reply.started":"2024-05-03T11:40:03.179923Z","shell.execute_reply":"2024-05-03T11:40:10.456796Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"count_zeros_in_column(train_static_0,columne_to_check[8]).compute()","metadata":{"execution":{"iopub.status.busy":"2024-05-03T11:43:38.012493Z","iopub.execute_input":"2024-05-03T11:43:38.012887Z","iopub.status.idle":"2024-05-03T11:43:44.235744Z","shell.execute_reply.started":"2024-05-03T11:43:38.012853Z","shell.execute_reply":"2024-05-03T11:43:44.234563Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_histogram_and_boxplot(train_static_0, columne_to_check[8])\n","metadata":{"execution":{"iopub.status.busy":"2024-05-03T12:22:10.525001Z","iopub.execute_input":"2024-05-03T12:22:10.525341Z","iopub.status.idle":"2024-05-03T12:24:10.019093Z","shell.execute_reply.started":"2024-05-03T12:22:10.525312Z","shell.execute_reply":"2024-05-03T12:24:10.018024Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_static_0 = fill_na(train_static_0, columne_to_check[8],train_static_0[columne_to_check[8]].median_approximate().compute())\n","metadata":{"execution":{"iopub.status.busy":"2024-05-03T12:36:58.942239Z","iopub.execute_input":"2024-05-03T12:36:58.942624Z","iopub.status.idle":"2024-05-03T12:37:05.600851Z","shell.execute_reply.started":"2024-05-03T12:36:58.942596Z","shell.execute_reply":"2024-05-03T12:37:05.600045Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_static_0[columne_to_check[9]].value_counts().compute().reset_index().head()\n","metadata":{"execution":{"iopub.status.busy":"2024-05-03T11:40:10.459913Z","iopub.execute_input":"2024-05-03T11:40:10.46038Z","iopub.status.idle":"2024-05-03T11:40:16.94911Z","shell.execute_reply.started":"2024-05-03T11:40:10.460315Z","shell.execute_reply":"2024-05-03T11:40:16.948154Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"count_zeros_in_column(train_static_0,columne_to_check[9]).compute()","metadata":{"execution":{"iopub.status.busy":"2024-05-03T11:43:31.78898Z","iopub.execute_input":"2024-05-03T11:43:31.78933Z","iopub.status.idle":"2024-05-03T11:43:38.010591Z","shell.execute_reply.started":"2024-05-03T11:43:31.789301Z","shell.execute_reply":"2024-05-03T11:43:38.009646Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_histogram_and_boxplot(train_static_0, columne_to_check[9])\n","metadata":{"execution":{"iopub.status.busy":"2024-05-03T12:24:10.020174Z","iopub.execute_input":"2024-05-03T12:24:10.020479Z","iopub.status.idle":"2024-05-03T12:26:09.242724Z","shell.execute_reply.started":"2024-05-03T12:24:10.020453Z","shell.execute_reply":"2024-05-03T12:26:09.241756Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_static_0 = fill_na(train_static_0, columne_to_check[9],train_static_0[columne_to_check[9]].median_approximate().compute())\n","metadata":{"execution":{"iopub.status.busy":"2024-05-03T12:38:43.61145Z","iopub.execute_input":"2024-05-03T12:38:43.611817Z","iopub.status.idle":"2024-05-03T12:38:49.889476Z","shell.execute_reply.started":"2024-05-03T12:38:43.611787Z","shell.execute_reply":"2024-05-03T12:38:49.888663Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_static_0[columne_to_check[10]].value_counts().compute().reset_index().head()\n","metadata":{"execution":{"iopub.status.busy":"2024-05-03T11:40:16.951205Z","iopub.execute_input":"2024-05-03T11:40:16.951864Z","iopub.status.idle":"2024-05-03T11:40:23.508766Z","shell.execute_reply.started":"2024-05-03T11:40:16.951828Z","shell.execute_reply":"2024-05-03T11:40:23.507834Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"count_zeros_in_column(train_static_0,columne_to_check[10]).compute()","metadata":{"execution":{"iopub.status.busy":"2024-05-03T11:43:05.449344Z","iopub.execute_input":"2024-05-03T11:43:05.449828Z","iopub.status.idle":"2024-05-03T11:43:11.902436Z","shell.execute_reply.started":"2024-05-03T11:43:05.44978Z","shell.execute_reply":"2024-05-03T11:43:11.901041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_histogram_and_boxplot(train_static_0, columne_to_check[10])\n","metadata":{"execution":{"iopub.status.busy":"2024-05-03T12:26:09.244691Z","iopub.execute_input":"2024-05-03T12:26:09.245007Z","iopub.status.idle":"2024-05-03T12:28:09.143592Z","shell.execute_reply.started":"2024-05-03T12:26:09.24498Z","shell.execute_reply":"2024-05-03T12:28:09.142542Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_static_0 = fill_na(train_static_0, columne_to_check[10],train_static_0[columne_to_check[10]].median_approximate().compute())\n","metadata":{"execution":{"iopub.status.busy":"2024-05-03T12:38:24.219656Z","iopub.execute_input":"2024-05-03T12:38:24.220037Z","iopub.status.idle":"2024-05-03T12:38:30.292675Z","shell.execute_reply.started":"2024-05-03T12:38:24.220008Z","shell.execute_reply":"2024-05-03T12:38:30.291756Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# checking again :\n","metadata":{}},{"cell_type":"code","source":"for i in columne_to_check:\n    y = check_num_na(train_static_0,i)\n    print(f' column{i} : nans : {y}')\n    ","metadata":{"execution":{"iopub.status.busy":"2024-05-03T12:43:24.911838Z","iopub.execute_input":"2024-05-03T12:43:24.912199Z","iopub.status.idle":"2024-05-03T12:44:40.639556Z","shell.execute_reply.started":"2024-05-03T12:43:24.912171Z","shell.execute_reply":"2024-05-03T12:44:40.638559Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"col_to_drop","metadata":{"execution":{"iopub.status.busy":"2024-05-03T12:44:40.641555Z","iopub.execute_input":"2024-05-03T12:44:40.642009Z","iopub.status.idle":"2024-05-03T12:44:40.648551Z","shell.execute_reply.started":"2024-05-03T12:44:40.641974Z","shell.execute_reply":"2024-05-03T12:44:40.647548Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_static_0.drop(columns=col_to_drop,axis=1).compute()","metadata":{"execution":{"iopub.status.busy":"2024-05-03T11:53:35.211729Z","iopub.execute_input":"2024-05-03T11:53:35.212477Z","iopub.status.idle":"2024-05-03T11:54:14.66371Z","shell.execute_reply.started":"2024-05-03T11:53:35.212447Z","shell.execute_reply":"2024-05-03T11:54:14.662603Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"my_df","metadata":{"execution":{"iopub.status.busy":"2024-05-03T11:56:43.268283Z","iopub.execute_input":"2024-05-03T11:56:43.269113Z","iopub.status.idle":"2024-05-03T11:56:43.28611Z","shell.execute_reply.started":"2024-05-03T11:56:43.26908Z","shell.execute_reply":"2024-05-03T11:56:43.285038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"my_df = my_df[~my_df['Column'].isin(col_to_drop)]\nmy_df\n\n","metadata":{"execution":{"iopub.status.busy":"2024-05-03T12:02:12.517838Z","iopub.execute_input":"2024-05-03T12:02:12.518565Z","iopub.status.idle":"2024-05-03T12:02:12.531149Z","shell.execute_reply.started":"2024-05-03T12:02:12.518531Z","shell.execute_reply":"2024-05-03T12:02:12.530006Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"columne_to_check","metadata":{"execution":{"iopub.status.busy":"2024-05-03T12:45:41.789195Z","iopub.execute_input":"2024-05-03T12:45:41.789558Z","iopub.status.idle":"2024-05-03T12:45:41.797886Z","shell.execute_reply.started":"2024-05-03T12:45:41.789528Z","shell.execute_reply":"2024-05-03T12:45:41.796809Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"my_df","metadata":{"execution":{"iopub.status.busy":"2024-05-03T12:45:50.914621Z","iopub.execute_input":"2024-05-03T12:45:50.915613Z","iopub.status.idle":"2024-05-03T12:45:50.927559Z","shell.execute_reply.started":"2024-05-03T12:45:50.91557Z","shell.execute_reply":"2024-05-03T12:45:50.926602Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# dooooooone","metadata":{}},{"cell_type":"markdown","source":"# columne_to_check[6] still has Nan\n","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}