{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Project: Tabular_Playground_Series_May_22_ML_Competition\n\n## Table of Contents\n<ul>\n<li><a href=\"#wrangling\">Data Wrangling</a></li>\n<li><a href=\"#clean\">Data Cleaning</a></li>\n<li><a href=\"#eda\">Exploratory Data Analysis</a></li>\n<li><a href=\"#ready\">Prepare Data For ML</a></li>\n<li><a href=\"#conclusions\">Conclusions</a></li>\n</ul>","metadata":{}},{"cell_type":"markdown","source":"### importing libraries that will be used to investigate Dataset","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport seaborn as sns\nsns.set_style(\"whitegrid\")\nfrom sklearn.preprocessing import OrdinalEncoder\nfrom sklearn.preprocessing import OneHotEncoder\npd.options.display.max_colwidth = 250\npd.options.display.max_columns = 50","metadata":{"execution":{"iopub.status.busy":"2022-07-29T17:32:04.734295Z","iopub.execute_input":"2022-07-29T17:32:04.735385Z","iopub.status.idle":"2022-07-29T17:32:06.223780Z","shell.execute_reply.started":"2022-07-29T17:32:04.735265Z","shell.execute_reply":"2022-07-29T17:32:06.222314Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id='wrangling'></a>\n## Data Wrangling\n\n **This is a three step process:**\n\n*  Gathering the data from Dataset and investegate it trying to understand more details about it. \n\n\n*  Assessing data to identify any issues with data types, structure, or quality.\n\n\n*  Cleaning data by changing data types, replacing values, removing unnecessary data and modifying Dataset for easy and fast analysis.\n","metadata":{}},{"cell_type":"markdown","source":"### Gathering Data","metadata":{}},{"cell_type":"code","source":"# loading CSV files in to 3 Dataframes  //df, df_test and sub//\n\ndf = pd.read_csv(\"../input/tabular-playground-series-may-2022/train.csv\")\ndf_test = pd.read_csv(\"../input/tabular-playground-series-may-2022/test.csv\")\nsub = pd.read_csv(\"../input/tabular-playground-series-may-2022/sample_submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-07-29T17:32:06.225854Z","iopub.execute_input":"2022-07-29T17:32:06.226758Z","iopub.status.idle":"2022-07-29T17:32:23.252065Z","shell.execute_reply.started":"2022-07-29T17:32:06.226719Z","shell.execute_reply":"2022-07-29T17:32:23.250931Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#checking 5 rows sample from Dataframes\n\ndf.head(5)","metadata":{"execution":{"iopub.status.busy":"2022-07-29T17:32:23.253918Z","iopub.execute_input":"2022-07-29T17:32:23.254380Z","iopub.status.idle":"2022-07-29T17:32:23.296724Z","shell.execute_reply.started":"2022-07-29T17:32:23.254333Z","shell.execute_reply":"2022-07-29T17:32:23.295259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-29T17:32:23.299564Z","iopub.execute_input":"2022-07-29T17:32:23.300063Z","iopub.status.idle":"2022-07-29T17:32:23.331320Z","shell.execute_reply.started":"2022-07-29T17:32:23.300027Z","shell.execute_reply":"2022-07-29T17:32:23.330044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-29T17:32:23.332377Z","iopub.execute_input":"2022-07-29T17:32:23.332688Z","iopub.status.idle":"2022-07-29T17:32:23.346012Z","shell.execute_reply.started":"2022-07-29T17:32:23.332659Z","shell.execute_reply":"2022-07-29T17:32:23.344468Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Assessing Data","metadata":{}},{"cell_type":"code","source":"#checking Dataframe basic informations (columns names, number of values, data types ......)\n\ndf.info(memory_usage=\"deep\")","metadata":{"execution":{"iopub.status.busy":"2022-07-29T17:32:23.347878Z","iopub.execute_input":"2022-07-29T17:32:23.348269Z","iopub.status.idle":"2022-07-29T17:32:23.714325Z","shell.execute_reply.started":"2022-07-29T17:32:23.348234Z","shell.execute_reply":"2022-07-29T17:32:23.712807Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test.info(memory_usage=\"deep\")","metadata":{"execution":{"iopub.status.busy":"2022-07-29T17:32:23.715957Z","iopub.execute_input":"2022-07-29T17:32:23.716372Z","iopub.status.idle":"2022-07-29T17:32:23.991456Z","shell.execute_reply.started":"2022-07-29T17:32:23.716338Z","shell.execute_reply":"2022-07-29T17:32:23.990291Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#checking Dataframe shape (number of rows and columns)\ndf.shape, df_test.shape","metadata":{"execution":{"iopub.status.busy":"2022-07-29T17:32:23.992875Z","iopub.execute_input":"2022-07-29T17:32:23.993734Z","iopub.status.idle":"2022-07-29T17:32:24.001264Z","shell.execute_reply.started":"2022-07-29T17:32:23.993698Z","shell.execute_reply":"2022-07-29T17:32:23.999155Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#checking more information and descriptive statistics\n\ndf.describe()","metadata":{"execution":{"iopub.status.busy":"2022-07-29T17:32:24.002734Z","iopub.execute_input":"2022-07-29T17:32:24.003162Z","iopub.status.idle":"2022-07-29T17:32:25.495296Z","shell.execute_reply.started":"2022-07-29T17:32:24.003015Z","shell.execute_reply":"2022-07-29T17:32:25.494274Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.describe(include=\"O\")","metadata":{"execution":{"iopub.status.busy":"2022-07-29T17:32:25.499442Z","iopub.execute_input":"2022-07-29T17:32:25.499852Z","iopub.status.idle":"2022-07-29T17:32:26.806225Z","shell.execute_reply.started":"2022-07-29T17:32:25.499818Z","shell.execute_reply":"2022-07-29T17:32:26.804685Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test.describe()","metadata":{"execution":{"iopub.status.busy":"2022-07-29T17:32:26.808003Z","iopub.execute_input":"2022-07-29T17:32:26.808553Z","iopub.status.idle":"2022-07-29T17:32:27.893137Z","shell.execute_reply.started":"2022-07-29T17:32:26.808515Z","shell.execute_reply":"2022-07-29T17:32:27.891633Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test.describe(include=\"O\")","metadata":{"execution":{"iopub.status.busy":"2022-07-29T17:32:27.894998Z","iopub.execute_input":"2022-07-29T17:32:27.895527Z","iopub.status.idle":"2022-07-29T17:32:28.883084Z","shell.execute_reply.started":"2022-07-29T17:32:27.895480Z","shell.execute_reply":"2022-07-29T17:32:28.882201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# checking for NaN values patients\n\ndf.isnull().sum().sum(),df_test.isnull().sum().sum() ","metadata":{"execution":{"iopub.status.busy":"2022-07-29T17:32:28.884638Z","iopub.execute_input":"2022-07-29T17:32:28.884916Z","iopub.status.idle":"2022-07-29T17:32:29.160069Z","shell.execute_reply.started":"2022-07-29T17:32:28.884889Z","shell.execute_reply":"2022-07-29T17:32:29.159017Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#checking for duplicated rows \n\ndf.duplicated().sum()","metadata":{"execution":{"iopub.status.busy":"2022-07-29T17:32:29.161805Z","iopub.execute_input":"2022-07-29T17:32:29.162117Z","iopub.status.idle":"2022-07-29T17:32:32.720217Z","shell.execute_reply.started":"2022-07-29T17:32:29.162089Z","shell.execute_reply":"2022-07-29T17:32:32.718980Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id='clean'></a>\n\n## Data Cleaning","metadata":{}},{"cell_type":"markdown","source":"### Reduce memory usage","metadata":{}},{"cell_type":"code","source":"# Function to reduce memory usage\ndef reduce_mem_usage(df):\n    \"\"\" iterate through all the columns of a dataframe and modify the data type\n        to reduce memory usage.        \n    \"\"\"\n    start_mem = df.memory_usage(deep=True).sum() / 1024**2\n    print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n    \n    for col in df.columns:\n        col_type = df[col].dtype\n        \n        if col_type != object:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n        else:\n            df[col] = df[col].astype('category')\n\n    end_mem = df.memory_usage(deep=True).sum() / 1024**2\n    print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n    print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n    \n    return df","metadata":{"execution":{"iopub.status.busy":"2022-07-29T17:32:32.721846Z","iopub.execute_input":"2022-07-29T17:32:32.722352Z","iopub.status.idle":"2022-07-29T17:32:32.739466Z","shell.execute_reply.started":"2022-07-29T17:32:32.722322Z","shell.execute_reply":"2022-07-29T17:32:32.738216Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = reduce_mem_usage(df)","metadata":{"execution":{"iopub.status.busy":"2022-07-29T17:32:32.741453Z","iopub.execute_input":"2022-07-29T17:32:32.742235Z","iopub.status.idle":"2022-07-29T17:32:37.648315Z","shell.execute_reply.started":"2022-07-29T17:32:32.742191Z","shell.execute_reply":"2022-07-29T17:32:37.646920Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test = reduce_mem_usage(df_test)","metadata":{"execution":{"iopub.status.busy":"2022-07-29T17:32:37.649989Z","iopub.execute_input":"2022-07-29T17:32:37.650505Z","iopub.status.idle":"2022-07-29T17:32:41.497630Z","shell.execute_reply.started":"2022-07-29T17:32:37.650469Z","shell.execute_reply":"2022-07-29T17:32:41.496772Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = df.drop(\"target\", axis=1).copy()\ny = df.target.copy()","metadata":{"execution":{"iopub.status.busy":"2022-07-29T17:32:41.498789Z","iopub.execute_input":"2022-07-29T17:32:41.499983Z","iopub.status.idle":"2022-07-29T17:32:41.647969Z","shell.execute_reply.started":"2022-07-29T17:32:41.499944Z","shell.execute_reply":"2022-07-29T17:32:41.646703Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Encode f_27 column","metadata":{}},{"cell_type":"code","source":"# Check len of charachters in f_27 \n\n(df.f_27.str.len().min(), df.f_27.str.len().max(), df_test.f_27.str.len().min(), df_test.f_27.str.len().min())","metadata":{"execution":{"iopub.status.busy":"2022-07-29T17:32:41.649671Z","iopub.execute_input":"2022-07-29T17:32:41.650043Z","iopub.status.idle":"2022-07-29T17:32:44.551578Z","shell.execute_reply.started":"2022-07-29T17:32:41.650012Z","shell.execute_reply":"2022-07-29T17:32:44.550255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Encode f_27 \nfor d_f in [df, df_test]:\n    d_f[\"f_27_uniques\"] = d_f.f_27.apply(lambda x : len(set(x)))\n    for i in range(10):\n        d_f[\"f_27\" + str(i)] = d_f.f_27.apply(lambda x : x.rstrip()[i])\n        \ndf = df.drop([\"f_27\"], axis=1)\ndf_test = df_test.drop([\"f_27\"], axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-07-29T17:32:44.553621Z","iopub.execute_input":"2022-07-29T17:32:44.553955Z","iopub.status.idle":"2022-07-29T17:32:58.385031Z","shell.execute_reply.started":"2022-07-29T17:32:44.553926Z","shell.execute_reply":"2022-07-29T17:32:58.383642Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Split columns into ordinal, discrete and continuous\n\nordinal = [\"f_\"+str(i) for i in range(270,280)]\ndiscrete =  [\"f_0\"+str(i) for i in range(7,10)] +[\"f_\"+str(i) for i in range(10,19)] +[\"f_29\",\"f_30\", \"f_27_uniques\"]\ncontinuous = [\"f_0\"+str(i) for i in range(0,7)] +[\"f_\"+str(i) for i in range(19,27)] + [\"f_28\"]","metadata":{"execution":{"iopub.status.busy":"2022-07-29T17:32:58.386788Z","iopub.execute_input":"2022-07-29T17:32:58.387182Z","iopub.status.idle":"2022-07-29T17:32:58.395554Z","shell.execute_reply.started":"2022-07-29T17:32:58.387135Z","shell.execute_reply":"2022-07-29T17:32:58.394062Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check if all columns in df are same on oredinal, cat , count\n\nsorted(ordinal + discrete + continuous + [\"id\"]) == sorted(df_test.columns.values)","metadata":{"execution":{"iopub.status.busy":"2022-07-29T17:32:58.397864Z","iopub.execute_input":"2022-07-29T17:32:58.398381Z","iopub.status.idle":"2022-07-29T17:32:58.408524Z","shell.execute_reply.started":"2022-07-29T17:32:58.398334Z","shell.execute_reply":"2022-07-29T17:32:58.407529Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id='eda'></a>\n## Exploratory Data Analysis\n\n> Now I'm going to explore this data and try to find patterns in it, compute statistics and visualize the relationships.\n","metadata":{}},{"cell_type":"markdown","source":"### 1. Continuous Features Plots (Train and Test)","metadata":{}},{"cell_type":"code","source":"# continuous Variables Density plot\n\nfig,ax = plt.subplots(4,4,figsize=(25,18))\nk=0\nj=0\nfor col in continuous:\n    sns.kdeplot(df[col], ax=ax[k,j],\n                shade=True,\n                color='#2f5586', edgecolor='black',\n                linewidth=1.5, alpha=0.9,\n                zorder=3\n               )\n    \n    ax[k,j].set_xlabel(col, fontsize=17, color=\"k\")\n    ax[k,j].set_ylabel(\"Density\", fontsize=17, color=\"k\")\n    #ax[k,j].set_xticklabels(fontsize=11, color=\"k\")\n    if j>=3:\n        k+=1\n        j=-1\n    j+=1\nfig.suptitle('Continuous Variables Density (Train Dataset)', fontsize=25, color=\"k\");","metadata":{"execution":{"iopub.status.busy":"2022-07-29T17:32:58.412328Z","iopub.execute_input":"2022-07-29T17:32:58.413503Z","iopub.status.idle":"2022-07-29T17:34:00.013602Z","shell.execute_reply.started":"2022-07-29T17:32:58.413454Z","shell.execute_reply":"2022-07-29T17:34:00.012310Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Quantitative Variables Density plot\n\nfig,ax = plt.subplots(4,4,figsize=(25,18))\nk=0\nj=0\nfor col in continuous:\n    sns.kdeplot(df_test[col], ax=ax[k,j],\n                shade=True,\n                color='orange', edgecolor='black',\n                linewidth=1.5, alpha=0.9,\n                zorder=3\n               )\n    \n    ax[k,j].set_xlabel(col, fontsize=17, color=\"k\")\n    ax[k,j].set_ylabel(\"Density\", fontsize=17, color=\"k\")\n    #ax[k,j].set_xticklabels(fontsize=11, color=\"k\")\n    if j>=3:\n        k+=1\n        j=-1\n    j+=1\nfig.suptitle('Continuous Variables Density (Test Dataset)', fontsize=25, color=\"k\");","metadata":{"execution":{"iopub.status.busy":"2022-07-29T17:34:00.015454Z","iopub.execute_input":"2022-07-29T17:34:00.016104Z","iopub.status.idle":"2022-07-29T17:34:49.023102Z","shell.execute_reply.started":"2022-07-29T17:34:00.016068Z","shell.execute_reply":"2022-07-29T17:34:49.021621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Continuous Variables Distribution\n\nfig,ax = plt.subplots(4,4,figsize=(25,18))\nk=0\nj=0\nfor col in continuous:\n    ax[k,j].hist(df[col], label=\"Train Dataset\", alpha=0.8, color=\"orange\", bins=15)\n    ax[k,j].hist(df_test[col], label=\"Test Dataset\",alpha=0.4, color=\"b\", bins=15)\n    \n    ax[k,j].set_xlabel(col, fontsize=17, color=\"k\")\n    ax[k,j].set_ylabel(\"Frequency\", fontsize=17, color=\"k\")\n    #ax[k,j].set_xticklabels(fontsize=11, color=\"k\")\n    if j>=3:\n        k+=1\n        j=-1\n    j+=1\nfig.suptitle('Continuous Variables Distribution (Train & Test)', fontsize=25, color=\"k\")\nfig.legend([\"Train Dataset\",\"Test Dataset\"], fontsize=20);","metadata":{"execution":{"iopub.status.busy":"2022-07-29T17:34:49.024943Z","iopub.execute_input":"2022-07-29T17:34:49.025381Z","iopub.status.idle":"2022-07-29T17:34:56.691581Z","shell.execute_reply.started":"2022-07-29T17:34:49.025341Z","shell.execute_reply":"2022-07-29T17:34:56.690595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"co_box = [i for i in continuous if i not in \"f_28\"]\nfig = plt.figure(figsize=(18,6))\nsns.boxplot(data=df[co_box],saturation=.5, palette=\"viridis\")\nplt.title(\"Continuous Variables Boxplots(Train Dataset)\", fontsize=25, color=\"k\", pad=20)\nplt.xticks(fontsize= 13);","metadata":{"execution":{"iopub.status.busy":"2022-07-29T17:34:56.692890Z","iopub.execute_input":"2022-07-29T17:34:56.694092Z","iopub.status.idle":"2022-07-29T17:34:58.526272Z","shell.execute_reply.started":"2022-07-29T17:34:56.694051Z","shell.execute_reply":"2022-07-29T17:34:58.524866Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = plt.figure(figsize=(18,6))\nsns.boxplot(data=df_test[co_box],saturation=.5, palette=\"rocket\")\nplt.title(\"Continuous Variables Boxplots(Test Dataset)\", fontsize=25, color=\"k\", pad=20)\nplt.xticks(fontsize= 13);","metadata":{"execution":{"iopub.status.busy":"2022-07-29T17:34:58.528320Z","iopub.execute_input":"2022-07-29T17:34:58.528812Z","iopub.status.idle":"2022-07-29T17:35:00.019160Z","shell.execute_reply.started":"2022-07-29T17:34:58.528766Z","shell.execute_reply":"2022-07-29T17:35:00.017611Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 2. Discrete Features Plots (Train and Test)","metadata":{}},{"cell_type":"code","source":"# Categorical Variables Barplots train dataset\n\nfig,ax = plt.subplots(4,4,figsize=(25,18))\nk=0\nj=0\nfor col in discrete:\n    sns.countplot(data = df[discrete], x=col, label=\"Train Dataset\", ax=ax[k,j], palette=\"mako\")\n    ax[k,j].set_xlabel(col, fontsize=17, color=\"k\")\n    #ax[k,j].set_xticklabels(fontsize=11, color=\"k\")\n    if j>=3:\n        k+=1\n        j=-1\n    j+=1\nfig.suptitle('Discrete Variables Countplots (Train dataset)', fontsize=25, color=\"k\");","metadata":{"execution":{"iopub.status.busy":"2022-07-29T17:35:00.025605Z","iopub.execute_input":"2022-07-29T17:35:00.026022Z","iopub.status.idle":"2022-07-29T17:35:04.609801Z","shell.execute_reply.started":"2022-07-29T17:35:00.025988Z","shell.execute_reply":"2022-07-29T17:35:04.608450Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Categorical Variables Barplots train dataset\n\nfig,ax = plt.subplots(4,4,figsize=(25,18))\nk=0\nj=0\nfor col in discrete:\n    sns.countplot(data = df_test[discrete], x=col, label=\"Train Dataset\", ax=ax[k,j], palette=\"rocket\")\n    ax[k,j].set_xlabel(col, fontsize=17, color=\"k\")\n    #ax[k,j].set_xticklabels(fontsize=11, color=\"k\")\n    if j>=3:\n        k+=1\n        j=-1\n    j+=1\nfig.suptitle('Discrete Variables Countplots (Test dataset)', fontsize=25, color=\"k\");","metadata":{"execution":{"iopub.status.busy":"2022-07-29T17:35:04.612151Z","iopub.execute_input":"2022-07-29T17:35:04.613070Z","iopub.status.idle":"2022-07-29T17:35:09.098062Z","shell.execute_reply.started":"2022-07-29T17:35:04.613032Z","shell.execute_reply":"2022-07-29T17:35:09.096869Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = plt.figure(figsize=(18,6))\nsns.boxplot(data=df[discrete],saturation=.5, palette=\"viridis\")\nplt.title(\"Discrete Variables Boxplots(Train Dataset)\", fontsize=25, color=\"k\", pad=20)\nplt.xticks(fontsize= 13);","metadata":{"execution":{"iopub.status.busy":"2022-07-29T17:35:09.099844Z","iopub.execute_input":"2022-07-29T17:35:09.101215Z","iopub.status.idle":"2022-07-29T17:35:10.471619Z","shell.execute_reply.started":"2022-07-29T17:35:09.101138Z","shell.execute_reply":"2022-07-29T17:35:10.470109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = plt.figure(figsize=(18,6))\nsns.boxplot(data=df_test[discrete],saturation=.5, palette=\"rocket\")\nplt.title(\"Discrete Variables Boxplots(Test Dataset)\", fontsize=25, color=\"k\", pad=20)\nplt.xticks(fontsize= 13);","metadata":{"execution":{"iopub.status.busy":"2022-07-29T17:35:10.473037Z","iopub.execute_input":"2022-07-29T17:35:10.473432Z","iopub.status.idle":"2022-07-29T17:35:11.618387Z","shell.execute_reply.started":"2022-07-29T17:35:10.473399Z","shell.execute_reply":"2022-07-29T17:35:11.617064Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 3. Categorical Features Plots (Train and Test)","metadata":{}},{"cell_type":"code","source":"# Categorical Variables Barplots train dataset\n\nfig,ax = plt.subplots(3,4,figsize=(25,18))\nk=0\nj=0\nfor col in ordinal:\n    sns.countplot(data = df[ordinal], x=col, label=\"Train Dataset\", ax=ax[k,j], palette=\"mako\")\n    ax[k,j].set_xlabel(col, fontsize=17, color=\"k\")\n    #ax[k,j].set_xticklabels(fontsize=11, color=\"k\")\n    if j>=3:\n        k+=1\n        j=-1\n    j+=1\nfig.suptitle('Categorical Variables Countplots (Train dataset)', fontsize=25, color=\"k\");","metadata":{"execution":{"iopub.status.busy":"2022-07-29T17:35:11.620482Z","iopub.execute_input":"2022-07-29T17:35:11.620851Z","iopub.status.idle":"2022-07-29T17:35:25.366402Z","shell.execute_reply.started":"2022-07-29T17:35:11.620814Z","shell.execute_reply":"2022-07-29T17:35:25.365023Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Categorical Variables Barplots train dataset\n\nfig,ax = plt.subplots(3,4,figsize=(25,18))\nk=0\nj=0\nfor col in ordinal:\n    sns.countplot(data = df_test[ordinal], x=col, label=\"Train Dataset\", ax=ax[k,j], palette=\"rocket\")\n    ax[k,j].set_xlabel(col, fontsize=17, color=\"k\")\n    #ax[k,j].set_xticklabels(fontsize=11, color=\"k\")\n    if j>=3:\n        k+=1\n        j=-1\n    j+=1\nfig.suptitle('Categorical Variables Countplots (Test dataset)', fontsize=25, color=\"k\");","metadata":{"execution":{"iopub.status.busy":"2022-07-29T17:35:25.368548Z","iopub.execute_input":"2022-07-29T17:35:25.369223Z","iopub.status.idle":"2022-07-29T17:35:36.871012Z","shell.execute_reply.started":"2022-07-29T17:35:25.369147Z","shell.execute_reply":"2022-07-29T17:35:36.869759Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 4. Heatmap for Train and Test","metadata":{}},{"cell_type":"code","source":"fig,ax = plt.subplots(2,1, figsize=(19,14))\ncorr_matrix = df[continuous+ [\"target\"]].corr()\nmask = np.zeros_like(corr_matrix)\nmask[np.triu_indices_from(mask)] = True\nsns.heatmap(corr_matrix, ax=ax[0], annot=True, mask=mask)\n\ncorr_matrix = df_test[continuous].corr()\nmask = np.zeros_like(corr_matrix)\nmask[np.triu_indices_from(mask)] = True\nsns.heatmap(corr_matrix, ax=ax[1], annot=True , mask=mask)\n\n#sns.heatmap(df_test[count].corr(),ax=ax[1], annot=True)\nax[1].set_yticklabels(continuous, rotation=0)\nax[0].text(-0.1, -1, '                   Features Correlations Heatmap on Train Dataset', fontsize=20, fontweight='bold', fontfamily='serif')\nax[1].text(-0.1, -1, '                   Features Correlations Heatmap on Test Dataset', fontsize=20, fontweight='bold', fontfamily='serif');","metadata":{"execution":{"iopub.status.busy":"2022-07-29T17:35:36.872924Z","iopub.execute_input":"2022-07-29T17:35:36.873503Z","iopub.status.idle":"2022-07-29T17:35:40.499113Z","shell.execute_reply.started":"2022-07-29T17:35:36.873456Z","shell.execute_reply":"2022-07-29T17:35:40.497834Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Observations:\n\n**- There are 16 continuous column.**\n\n**- There are 14 numeric columns with low number of unique values which give indecation that they are categorical features from f_07 to f_18, f_29 and f_30.**\n\n**- There are one Categorical variable f_27 every cell has 10 characters and I split them into 10 separate columns each column has one character from f_270 to f_97**\n\n**- There is no missing values in both train and test datasets**\n\n**- There is no duplicated values**\n\n","metadata":{}},{"cell_type":"markdown","source":"<a id='ready'></a>\n\n#  Preparing Data for ML model","metadata":{}},{"cell_type":"code","source":"# use onehot encoding on categorical variables\n\nordinal_encoder = OrdinalEncoder()\nX_oridinal = df[ordinal].copy()\ntest_oridinal = df_test[ordinal].copy()\n\nX_oridinal[ordinal] = pd.DataFrame(ordinal_encoder.fit_transform(X_oridinal[ordinal]))\ntest_oridinal[ordinal] = pd.DataFrame(ordinal_encoder.fit_transform(test_oridinal[ordinal]))\n\nX = pd.concat([df[continuous], X_oridinal ,df[discrete]], axis=1)\ntest =  pd.concat([df_test[continuous], test_oridinal,df_test[discrete]], axis=1)\ny = df.target\nsorted(list(X.columns)) == sorted(list(test.columns))","metadata":{"execution":{"iopub.status.busy":"2022-07-29T17:35:40.501165Z","iopub.execute_input":"2022-07-29T17:35:40.501547Z","iopub.status.idle":"2022-07-29T17:35:47.930286Z","shell.execute_reply.started":"2022-07-29T17:35:40.501516Z","shell.execute_reply":"2022-07-29T17:35:47.929135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Now we have 3 dataframes (X, y and test)","metadata":{}},{"cell_type":"code","source":"# Import Models for ML \n\nfrom sklearn.model_selection import train_test_split\nimport lightgbm as lgb\nimport optuna\nfrom sklearn.model_selection import cross_val_score, KFold\nfrom sklearn.metrics import confusion_matrix, precision_score, recall_score, accuracy_score, roc_auc_score","metadata":{"execution":{"iopub.status.busy":"2022-07-29T17:35:47.931793Z","iopub.execute_input":"2022-07-29T17:35:47.932538Z","iopub.status.idle":"2022-07-29T17:35:50.008371Z","shell.execute_reply.started":"2022-07-29T17:35:47.932500Z","shell.execute_reply":"2022-07-29T17:35:50.007008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Bulding First ML Model","metadata":{}},{"cell_type":"code","source":"# Set Random_state to 1\n\nrandom_state=1\n\n\n# Split data into train and test datasets\n\ntrain_X, test_X, train_y, test_y = train_test_split(X, y, random_state = random_state, test_size=0.25)","metadata":{"execution":{"iopub.status.busy":"2022-07-29T17:35:50.010322Z","iopub.execute_input":"2022-07-29T17:35:50.010828Z","iopub.status.idle":"2022-07-29T17:35:50.657411Z","shell.execute_reply.started":"2022-07-29T17:35:50.010780Z","shell.execute_reply":"2022-07-29T17:35:50.655923Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\n# Bulid first model\n\n\n\nclf = lgb.LGBMClassifier()\nclf.fit(train_X, train_y)\npreds = clf.predict(test_X)\nroc_score = roc_auc_score(test_y, preds)\nprint(\"roc_score: %.2f%%\" % (roc_score * 100.0))","metadata":{"execution":{"iopub.status.busy":"2022-07-29T17:35:50.659264Z","iopub.execute_input":"2022-07-29T17:35:50.659635Z","iopub.status.idle":"2022-07-29T17:36:07.098008Z","shell.execute_reply.started":"2022-07-29T17:35:50.659605Z","shell.execute_reply":"2022-07-29T17:36:07.095876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n# Bulid model with cross validation and check overfitting and underfitting\n\nclf = lgb.LGBMClassifier()\nclf.fit(train_X, train_y)\npreds = clf.predict(test_X)\n\nkf = KFold(n_splits=5, shuffle=True, random_state=random_state)\ncv_trian_scores = cross_val_score(clf, train_X, train_y,  cv=kf, scoring=\"roc_auc\")\ncv_test_scores = cross_val_score(clf, test_X, test_y,  cv=kf, scoring=\"roc_auc\")\n\nprint(\"test_score: {} train_score: {} \\n_________________________________________________\\n\".format(cv_test_scores.mean() * 100.0, cv_trian_scores.mean() * 100.0))","metadata":{"execution":{"iopub.status.busy":"2022-07-29T17:36:07.100727Z","iopub.execute_input":"2022-07-29T17:36:07.101345Z","iopub.status.idle":"2022-07-29T17:37:23.788292Z","shell.execute_reply.started":"2022-07-29T17:36:07.101291Z","shell.execute_reply":"2022-07-29T17:37:23.787124Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Hyperparameters Tuning using optuna","metadata":{}},{"cell_type":"code","source":"from lightgbm import early_stopping\nfrom lightgbm import log_evaluation\ndef objective(trial,data=X,target=y):\n    \n    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.15,random_state=random_state)\n    param = {\n        #'tree_method':'gpu_hist',  # this parameter means using the GPU when training our model to speedup the training process\n        #'lambda': trial.suggest_loguniform('lambda', 1e-3, 10.0),\n        'alpha': trial.suggest_loguniform('alpha', 1e-3, 10.0),\n        'colsample_bytree': trial.suggest_categorical('colsample_bytree', [0.3,0.4,0.5,0.6,0.7,0.8,0.9, 1.0]),\n        'subsample': trial.suggest_categorical('subsample', [0.4,0.5,0.6,0.7,0.8,1.0]),\n        'learning_rate': trial.suggest_loguniform('learning_rate', 0.001 , 1),\n        'n_estimators':trial.suggest_int('n_estimators',100,10000, 10),\n        'max_depth': trial.suggest_categorical('max_depth', [5,7,9,11,13,15,17]),\n        'min_child_weight': trial.suggest_int('min_child_weight', 1, 300),\n    }\n    \n    model = lgb.LGBMClassifier(**param)  \n    callbacks = [lgb.early_stopping(10, verbose=0)]#, lgb.log_evaluation(period=0)]\n    model.fit(X_train,y_train,eval_set=[(X_test,y_test)], early_stopping_rounds=100,verbose=False)\n    preds = model.predict(X_test)\n    #predictions = [round(value) for value in preds]\n    roc_score = roc_auc_score(y_test, preds)\n    return roc_score","metadata":{"execution":{"iopub.status.busy":"2022-07-29T17:37:23.789773Z","iopub.execute_input":"2022-07-29T17:37:23.790114Z","iopub.status.idle":"2022-07-29T17:37:23.804282Z","shell.execute_reply.started":"2022-07-29T17:37:23.790082Z","shell.execute_reply":"2022-07-29T17:37:23.802421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nimport warnings\nwarnings.filterwarnings('ignore')\n\nstudy = optuna.create_study(direction='maximize')\nstudy.optimize(objective, n_trials=5)\nprint('Number of finished trials:', len(study.trials))\nprint('Best trial:', study.best_trial.params)\nprint('Best Score:', study.best_trial.value)","metadata":{"execution":{"iopub.status.busy":"2022-07-29T17:37:23.806331Z","iopub.execute_input":"2022-07-29T17:37:23.807815Z","iopub.status.idle":"2022-07-29T18:02:49.116344Z","shell.execute_reply.started":"2022-07-29T17:37:23.807758Z","shell.execute_reply":"2022-07-29T18:02:49.114824Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"best_params = study.best_params","metadata":{"execution":{"iopub.status.busy":"2022-07-29T18:02:49.118504Z","iopub.execute_input":"2022-07-29T18:02:49.118890Z","iopub.status.idle":"2022-07-29T18:02:49.125671Z","shell.execute_reply.started":"2022-07-29T18:02:49.118857Z","shell.execute_reply":"2022-07-29T18:02:49.124091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model After Tuning","metadata":{}},{"cell_type":"code","source":"%%time\n\n#b =  {'learning_rate': 0.11094580978935166, 'n_estimators': 9950}\n\nclf = lgb.LGBMClassifier(random_state=random_state, **best_params)\nclf.fit(train_X, train_y)\npreds = clf.predict(test_X)\nroc_score = roc_auc_score(test_y, preds)\n\n\npreds_train = clf.predict(train_X)\nroc_score_train = roc_auc_score(train_y, preds_train)\nprint(\"test_score: {}% train_score: {}% \\n ---------------- ---------------- ---------------- ----------------\\n \".format(roc_score*100.0, roc_score_train*100.0))","metadata":{"execution":{"iopub.status.busy":"2022-07-29T18:02:49.127407Z","iopub.execute_input":"2022-07-29T18:02:49.128515Z","iopub.status.idle":"2022-07-29T18:14:03.476828Z","shell.execute_reply.started":"2022-07-29T18:02:49.128474Z","shell.execute_reply":"2022-07-29T18:14:03.475120Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Final Model","metadata":{}},{"cell_type":"code","source":"%%time\n\nfinal_clf = lgb.LGBMClassifier(**best_params,\n                        random_state=random_state,\n                        n_jobs=-1\n                       )\nfinal_clf.fit(X, y)\nfinal_preds = clf.predict(test)","metadata":{"execution":{"iopub.status.busy":"2022-07-29T18:14:03.478560Z","iopub.execute_input":"2022-07-29T18:14:03.479016Z","iopub.status.idle":"2022-07-29T18:26:15.860267Z","shell.execute_reply.started":"2022-07-29T18:14:03.478971Z","shell.execute_reply":"2022-07-29T18:26:15.858525Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Save prediction to submission file","metadata":{}},{"cell_type":"code","source":"# Save test predictions to file\n\noutput = pd.DataFrame({'Id': df_test.id,\n                       'target': final_preds})\noutput.to_csv('submission_may.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-07-29T18:26:15.862397Z","iopub.execute_input":"2022-07-29T18:26:15.862814Z","iopub.status.idle":"2022-07-29T18:26:17.102844Z","shell.execute_reply.started":"2022-07-29T18:26:15.862779Z","shell.execute_reply":"2022-07-29T18:26:17.101387Z"},"trusted":true},"execution_count":null,"outputs":[]}]}