{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"

Table of Contents

\n\n### [**1. Import Libraries and Load Data**](#title-first)\n\n### [**2. Data Info**](#title-second)\n\n### [**3. Handling Outliers**](#title-ten)\n \n### [**4. Feature Engineering**](#title-third) \n\n### [**5. Exploratory Data Analysis**](#title-four)\n \n### [**6. Objective and Metrics**](#title-five)\n\n### [**7. Preprocessing**](#title-six)\n\n### [**8. Modeling - Scikit-learn Classifiers - Fit and Check Metrics**](#title-seven)\n\n### [**9. Advanced Ensemble ML Classifiers**](#title-eight)\n\n### [**10. Final Model Metrics Check, Prediction and Submission**](#title-nine)\n \n\n

Import Libraries and Load Data

","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n for filename in filenames:\n print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-08-12T13:20:20.642627Z","iopub.execute_input":"2022-08-12T13:20:20.644088Z","iopub.status.idle":"2022-08-12T13:20:20.677886Z","shell.execute_reply.started":"2022-08-12T13:20:20.643952Z","shell.execute_reply":"2022-08-12T13:20:20.676371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import seaborn as sns\nfrom matplotlib import pyplot as plt\nfrom matplotlib.pyplot import figure\n\nfrom sklearn.feature_selection import SelectKBest\nfrom sklearn.feature_selection import chi2\n\n\nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.preprocessing import OneHotEncoder\n\n\nfrom sklearn.model_selection import cross_val_score\nfrom sklearn.model_selection import GridSearchCV\nfrom sklearn.metrics import f1_score\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.model_selection import KFold\n\n\nfrom sklearn.neighbors import KNeighborsClassifier\nfrom sklearn.svm import SVC\nfrom sklearn.svm import LinearSVC\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.ensemble import GradientBoostingClassifier\nfrom sklearn.ensemble import AdaBoostClassifier\nfrom sklearn.naive_bayes import GaussianNB\n\nfrom catboost import CatBoostClassifier\nfrom xgboost import XGBClassifier\n\nfrom sklearn.model_selection import GridSearchCV\n\nfrom sklearn.model_selection import learning_curve \nfrom sklearn.metrics import confusion_matrix\n\nimport copy","metadata":{"execution":{"iopub.status.busy":"2022-08-12T13:20:20.831550Z","iopub.execute_input":"2022-08-12T13:20:20.832337Z","iopub.status.idle":"2022-08-12T13:20:22.799164Z","shell.execute_reply.started":"2022-08-12T13:20:20.832286Z","shell.execute_reply":"2022-08-12T13:20:22.797737Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"original_test_data = pd.read_csv('../input/spaceship-titanic/test.csv', index_col='PassengerId')\noriginal_train_data = pd.read_csv('../input/spaceship-titanic/train.csv', index_col='PassengerId')","metadata":{"execution":{"iopub.status.busy":"2022-08-12T13:20:22.801435Z","iopub.execute_input":"2022-08-12T13:20:22.801867Z","iopub.status.idle":"2022-08-12T13:20:22.899871Z","shell.execute_reply.started":"2022-08-12T13:20:22.801829Z","shell.execute_reply":"2022-08-12T13:20:22.898293Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n

Data Info

","metadata":{}},{"cell_type":"code","source":"original_train_data.info()","metadata":{"execution":{"iopub.status.busy":"2022-08-12T13:20:22.901902Z","iopub.execute_input":"2022-08-12T13:20:22.902574Z","iopub.status.idle":"2022-08-12T13:20:22.929825Z","shell.execute_reply.started":"2022-08-12T13:20:22.902527Z","shell.execute_reply":"2022-08-12T13:20:22.928680Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"original_train_data.head(4000)","metadata":{"execution":{"iopub.status.busy":"2022-08-12T13:20:22.931844Z","iopub.execute_input":"2022-08-12T13:20:22.933016Z","iopub.status.idle":"2022-08-12T13:20:22.985000Z","shell.execute_reply.started":"2022-08-12T13:20:22.932964Z","shell.execute_reply":"2022-08-12T13:20:22.983273Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"original_train_data.describe()","metadata":{"execution":{"iopub.status.busy":"2022-08-12T13:20:22.986464Z","iopub.execute_input":"2022-08-12T13:20:22.987341Z","iopub.status.idle":"2022-08-12T13:20:23.038654Z","shell.execute_reply.started":"2022-08-12T13:20:22.987281Z","shell.execute_reply":"2022-08-12T13:20:23.037302Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"original_test_data.info()","metadata":{"execution":{"iopub.status.busy":"2022-08-12T13:20:23.040410Z","iopub.execute_input":"2022-08-12T13:20:23.040797Z","iopub.status.idle":"2022-08-12T13:20:23.056605Z","shell.execute_reply.started":"2022-08-12T13:20:23.040761Z","shell.execute_reply":"2022-08-12T13:20:23.055320Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"original_test_data.describe()","metadata":{"execution":{"iopub.status.busy":"2022-08-12T13:20:23.057870Z","iopub.execute_input":"2022-08-12T13:20:23.058387Z","iopub.status.idle":"2022-08-12T13:20:23.093372Z","shell.execute_reply.started":"2022-08-12T13:20:23.058355Z","shell.execute_reply":"2022-08-12T13:20:23.092116Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"original_test_data.head(2000)","metadata":{"execution":{"iopub.status.busy":"2022-08-12T13:20:23.095736Z","iopub.execute_input":"2022-08-12T13:20:23.096825Z","iopub.status.idle":"2022-08-12T13:20:23.131047Z","shell.execute_reply.started":"2022-08-12T13:20:23.096737Z","shell.execute_reply":"2022-08-12T13:20:23.129239Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n

Handling Outliers

","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(12,1))\nax = sns.boxplot(x=original_train_data.Age)\nplt.figure(figsize=(12,1))\nax = sns.boxplot(x=original_train_data.RoomService)\nplt.figure(figsize=(12,1))\nax = sns.boxplot(x=original_train_data.FoodCourt)\nplt.figure(figsize=(12,1))\nax = sns.boxplot(x=original_train_data.ShoppingMall)\nplt.figure(figsize=(12,1))\nax = sns.boxplot(x=original_train_data.Spa)\nplt.figure(figsize=(12,1))\nax = sns.boxplot(x=original_train_data.VRDeck)","metadata":{"execution":{"iopub.status.busy":"2022-08-12T13:20:23.132977Z","iopub.execute_input":"2022-08-12T13:20:23.134103Z","iopub.status.idle":"2022-08-12T13:20:23.900207Z","shell.execute_reply.started":"2022-08-12T13:20:23.134060Z","shell.execute_reply":"2022-08-12T13:20:23.898912Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Looks like features GroupMembersCount, Age, RoomService, FoodCourt, ShoppingMall, Spa and VRDeck have outliers. Replace outliers with q1 - 1.5*IRQ and q3 + 1.5*IRQ","metadata":{}},{"cell_type":"code","source":"def handle_multiple_outliers(df: pd.DataFrame, cols: list):\n new_df = df.copy()\n for col in cols:\n new_df = handle_outliers(new_df, col)\n return new_df\n\ndef handle_outliers(df: pd.DataFrame, col: str):\n res_df = df.copy()\n q1 = np.nanpercentile(res_df[col], 25)\n q3 = np.nanpercentile(res_df[col], 75)\n IRQ = q3 - q1\n min_value = q1 - 1.5 * IRQ\n max_value = q3 + 1.5 * IRQ\n median = res_df[col].median()\n \n print(f\"\\ncol = {col}; len={len(res_df[col])}\")\n print(f\"median ={median}, q1={q1}, q1={q3}, IRQ={IRQ}\")\n print(f\"max={max_value}, min={min_value}\")\n print(f\"top outliers count = {len(res_df.loc[res_df[col] > max_value, col])}; \")\n print(f\"bottom outliers count = {len(res_df.loc[res_df[col] < min_value, col])}; \\n\")\n \n res_df.loc[res_df[col] > max_value, col] = max_value\n res_df.loc[res_df[col] < min_value, col] = min_value\n \n #res_df.loc[res_df[col] > max_value, col] = median\n #res_df.loc[res_df[col] < min_value, col] = median\n \n return res_df\n \ntrain_data = original_train_data.copy()\ntest_data = original_test_data.copy()\n#train_data = handle_multiple_outliers(original_train_data, ['Age', 'RoomService', 'FoodCourt', 'ShoppingMall', 'Spa', 'VRDeck'])\n#test_data = handle_multiple_outliers(original_test_data, ['Age', 'RoomService', 'FoodCourt', 'ShoppingMall', 'Spa', 'VRDeck'])","metadata":{"execution":{"iopub.status.busy":"2022-08-12T13:20:23.905785Z","iopub.execute_input":"2022-08-12T13:20:23.907260Z","iopub.status.idle":"2022-08-12T13:20:23.922630Z","shell.execute_reply.started":"2022-08-12T13:20:23.907211Z","shell.execute_reply":"2022-08-12T13:20:23.921487Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(12,1))\nax = sns.boxplot(x=train_data.Age)\n\nplt.figure(figsize=(12,1))\nax = sns.boxplot(x=train_data.RoomService)\nplt.figure(figsize=(12,1))\nax = sns.boxplot(x=train_data.FoodCourt)\nplt.figure(figsize=(12,1))\nax = sns.boxplot(x=train_data.ShoppingMall)\nplt.figure(figsize=(12,1))\nax = sns.boxplot(x=train_data.Spa)\nplt.figure(figsize=(12,1))\nax = sns.boxplot(x=train_data.VRDeck)","metadata":{"execution":{"iopub.status.busy":"2022-08-12T13:20:23.926008Z","iopub.execute_input":"2022-08-12T13:20:23.926790Z","iopub.status.idle":"2022-08-12T13:20:24.912523Z","shell.execute_reply.started":"2022-08-12T13:20:23.926739Z","shell.execute_reply":"2022-08-12T13:20:24.911390Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data.VRDeck.hist()","metadata":{"execution":{"iopub.status.busy":"2022-08-12T13:20:24.913872Z","iopub.execute_input":"2022-08-12T13:20:24.914295Z","iopub.status.idle":"2022-08-12T13:20:25.172718Z","shell.execute_reply.started":"2022-08-12T13:20:24.914263Z","shell.execute_reply":"2022-08-12T13:20:25.171267Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n

\n Feature Engineering

","metadata":{}},{"cell_type":"code","source":"def fill_missed_values(df):\n res_df = df.copy()\n res_df.VIP.fillna(False, inplace=True)\n res_df.CryoSleep.fillna(False, inplace=True)\n \n res_df.HomePlanet.fillna(res_df.HomePlanet.mode()[0], inplace=True)\n res_df.Destination.fillna(res_df.Destination.mode()[0], inplace=True)\n \n res_df.Age.fillna(train_data.Age.median(), inplace=True)\n res_df.RoomService.fillna(res_df.RoomService.median(), inplace=True)\n res_df.FoodCourt.fillna(res_df.FoodCourt.median(), inplace=True)\n res_df.ShoppingMall.fillna(res_df.ShoppingMall.median(), inplace=True)\n res_df.Spa.fillna(res_df.Spa.median(), inplace=True)\n res_df.VRDeck.fillna(res_df.VRDeck.median(), inplace=True)\n \n return res_df\n\ntrain_data = fill_missed_values(train_data)\ntest_data = fill_missed_values(test_data)\n\n\ndef change_bool_columns_to_int(df):\n res_df = df.copy()\n res_df.VIP = res_df.VIP.astype(int)\n res_df.CryoSleep = res_df.CryoSleep.astype(int)\n if 'Transported' in res_df.columns :\n res_df.Transported = res_df.Transported.astype(int)\n return res_df\n\ntrain_data = change_bool_columns_to_int(train_data)\ntest_data = change_bool_columns_to_int(test_data)\n","metadata":{"execution":{"iopub.status.busy":"2022-08-12T13:20:25.174525Z","iopub.execute_input":"2022-08-12T13:20:25.174983Z","iopub.status.idle":"2022-08-12T13:20:25.395835Z","shell.execute_reply.started":"2022-08-12T13:20:25.174937Z","shell.execute_reply":"2022-08-12T13:20:25.394531Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_passenger_group_feature(df):\n res_df = df.copy()\n \n res_df['PassengerGroup'] = res_df.index.str.split('_', 1).str[0]\n return res_df\n\ntrain_data = create_passenger_group_feature(train_data)\ntest_data = create_passenger_group_feature(test_data)\n\n\n# Cabin\n# F/4/S\n# deck/num/side\ndef create_features_from_cabin(df):\n res_df = df.copy()\n res_df.Cabin.fillna('F/0/S', inplace=True)\n \n splits = res_df['Cabin'].str.split('/', 3)\n \n res_df['CabinDeck'] = splits.str[0]\n res_df['CabinNum'] = splits.str[1].astype(int)\n res_df['CabinSide'] = splits.str[2]\n return res_df\n\ntrain_data = create_features_from_cabin(train_data)\ntest_data = create_features_from_cabin(test_data)","metadata":{"execution":{"iopub.status.busy":"2022-08-12T13:20:25.398559Z","iopub.execute_input":"2022-08-12T13:20:25.399581Z","iopub.status.idle":"2022-08-12T13:20:25.482418Z","shell.execute_reply.started":"2022-08-12T13:20:25.399538Z","shell.execute_reply":"2022-08-12T13:20:25.481126Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def generate_new_features(df):\n res_df = df.copy() \n res_df['TotalSpent'] = res_df['RoomService'] + res_df['Spa'] + res_df['VRDeck'] \n res_df['SpentRatio'] = res_df['TotalSpent'] / (res_df['FoodCourt'] + res_df['ShoppingMall'] + 1)\n res_df['TotalSpentLog'] = (res_df['TotalSpent'] + 1).transform(np.log)\n res_df['VIPPlusCryoSleep'] = res_df['VIP'] + res_df['CryoSleep']\n res_df['CabinDeckSide'] = res_df['CabinDeck'] + res_df['CabinSide']\n res_df['HomePlanetDestination'] = res_df['HomePlanet'] + '-' + res_df['Destination']\n res_df['GroupMembersCount'] = res_df.groupby('PassengerGroup')['PassengerGroup'].transform('count')\n return res_df\ntrain_data = generate_new_features(train_data)\ntest_data = generate_new_features(test_data)\n\ndef binning_spent_ratio_feature(df):\n res_df = df.copy() \n bins = [0, .00001, .1, 5, 100000]\n labels = ['1', '2','3', '4']\n res_df['SpentRatioBin'] = pd.cut(res_df['SpentRatio'], bins=bins, labels=labels, include_lowest=True).astype(str) #\n return res_df\ntrain_data = binning_spent_ratio_feature(train_data)\ntest_data = binning_spent_ratio_feature(test_data)\n\ndef binning_age_feature(df):\n res_df = df.copy() \n bins = [0, 5, 18, 40, 55, 100]\n labels = ['1', '2','3', '4', '5']\n res_df['AgeBin'] = pd.cut(res_df['Age'], bins=bins, labels=labels, include_lowest=True).astype(str) #\n return res_df\ntrain_data = binning_age_feature(train_data)\ntest_data = binning_age_feature(test_data)","metadata":{"execution":{"iopub.status.busy":"2022-08-12T13:20:25.484074Z","iopub.execute_input":"2022-08-12T13:20:25.484403Z","iopub.status.idle":"2022-08-12T13:20:25.552022Z","shell.execute_reply.started":"2022-08-12T13:20:25.484373Z","shell.execute_reply":"2022-08-12T13:20:25.550835Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(12,1))\nax = sns.boxplot(x=train_data.TotalSpent)\nplt.figure(figsize=(12,1))\nax = sns.boxplot(x=train_data.GroupMembersCount)\nplt.figure(figsize=(12,1))\nax = sns.boxplot(x=train_data.VIPPlusCryoSleep)","metadata":{"execution":{"iopub.status.busy":"2022-08-12T13:20:25.553577Z","iopub.execute_input":"2022-08-12T13:20:25.553922Z","iopub.status.idle":"2022-08-12T13:20:25.983615Z","shell.execute_reply.started":"2022-08-12T13:20:25.553890Z","shell.execute_reply":"2022-08-12T13:20:25.982665Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train_data = handle_outliers(train_data, 'TotalSpent')\n#test_data = handle_outliers(test_data, 'TotalSpent')\n#train_data = handle_outliers(train_data, 'GroupMembersCount')\n#test_data = handle_outliers(test_data, 'GroupMembersCount')\n\nplt.figure(figsize=(12,1))\nax = sns.boxplot(x=train_data.GroupMembersCount)\nplt.figure(figsize=(12,1))\nax = sns.boxplot(x=train_data.TotalSpent)","metadata":{"execution":{"iopub.status.busy":"2022-08-12T13:20:25.987354Z","iopub.execute_input":"2022-08-12T13:20:25.988126Z","iopub.status.idle":"2022-08-12T13:20:26.229792Z","shell.execute_reply.started":"2022-08-12T13:20:25.988075Z","shell.execute_reply":"2022-08-12T13:20:26.228514Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Select univariate features**","metadata":{}},{"cell_type":"code","source":"\nX_select = train_data.drop('Transported', axis=1)[['GroupMembersCount', 'TotalSpentLog', 'TotalSpent', 'SpentRatio', 'VRDeck', 'Spa', 'ShoppingMall', 'FoodCourt', 'RoomService', 'CryoSleep', 'VIP', 'VIPPlusCryoSleep', 'Age']]\ny_select = train_data.Transported.astype(bool)\nselector = SelectKBest(chi2, k='all').fit(X_select, y_select)\nx_new = selector.transform(X_select) # not needed to get the score\n\npd.DataFrame({'variable': X_select.columns,\n 'score': selector.scores_})","metadata":{"execution":{"iopub.status.busy":"2022-08-12T13:20:26.231540Z","iopub.execute_input":"2022-08-12T13:20:26.232758Z","iopub.status.idle":"2022-08-12T13:20:26.278217Z","shell.execute_reply.started":"2022-08-12T13:20:26.232706Z","shell.execute_reply":"2022-08-12T13:20:26.276887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n

Exploratory Data Analysis

","metadata":{}},{"cell_type":"code","source":"train_data['Age'].hist()","metadata":{"execution":{"iopub.status.busy":"2022-08-12T13:20:26.280017Z","iopub.execute_input":"2022-08-12T13:20:26.280486Z","iopub.status.idle":"2022-08-12T13:20:26.452636Z","shell.execute_reply.started":"2022-08-12T13:20:26.280439Z","shell.execute_reply":"2022-08-12T13:20:26.451103Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(18,8))\nsns.countplot(x=\"Age\", hue=\"Transported\", data=train_data)","metadata":{"execution":{"iopub.status.busy":"2022-08-12T13:20:26.454731Z","iopub.execute_input":"2022-08-12T13:20:26.455482Z","iopub.status.idle":"2022-08-12T13:20:27.779050Z","shell.execute_reply.started":"2022-08-12T13:20:26.455431Z","shell.execute_reply":"2022-08-12T13:20:27.776771Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(18,8))\nsns.countplot(x=\"AgeBin\", hue=\"Transported\", data=train_data)","metadata":{"execution":{"iopub.status.busy":"2022-08-12T13:20:27.781300Z","iopub.execute_input":"2022-08-12T13:20:27.782230Z","iopub.status.idle":"2022-08-12T13:20:28.006338Z","shell.execute_reply.started":"2022-08-12T13:20:27.782178Z","shell.execute_reply":"2022-08-12T13:20:28.005262Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Corellation matrix**","metadata":{}},{"cell_type":"markdown","source":"Seaborn's cmaps: 'terrain', 'coolwarm', 'Greens', 'Greys', 'Blues'","metadata":{}},{"cell_type":"code","source":"figure(figsize=(12, 12), dpi=80)\nsns.heatmap(train_data.corr(), annot = True, square = True, linewidths = 1, cmap = 'coolwarm')","metadata":{"execution":{"iopub.status.busy":"2022-08-12T13:20:28.015064Z","iopub.execute_input":"2022-08-12T13:20:28.015545Z","iopub.status.idle":"2022-08-12T13:20:29.378364Z","shell.execute_reply.started":"2022-08-12T13:20:28.015480Z","shell.execute_reply":"2022-08-12T13:20:29.377086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data.loc[train_data.CabinNum == 150][['CabinNum', 'CabinDeck', 'CabinSide', 'PassengerGroup', 'GroupMembersCount', 'Transported']]","metadata":{"execution":{"iopub.status.busy":"2022-08-12T13:20:29.379768Z","iopub.execute_input":"2022-08-12T13:20:29.381280Z","iopub.status.idle":"2022-08-12T13:20:29.404827Z","shell.execute_reply.started":"2022-08-12T13:20:29.381230Z","shell.execute_reply":"2022-08-12T13:20:29.403605Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Some plots**","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(10,5))\nsns.countplot(x=\"CryoSleep\", hue=\"Transported\", data=train_data)","metadata":{"execution":{"iopub.status.busy":"2022-08-12T13:20:29.406354Z","iopub.execute_input":"2022-08-12T13:20:29.407553Z","iopub.status.idle":"2022-08-12T13:20:29.814454Z","shell.execute_reply.started":"2022-08-12T13:20:29.407484Z","shell.execute_reply":"2022-08-12T13:20:29.813581Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10,5))\nsns.countplot(x=\"VIPPlusCryoSleep\", hue=\"Transported\", data=train_data)","metadata":{"execution":{"iopub.status.busy":"2022-08-12T13:20:29.816040Z","iopub.execute_input":"2022-08-12T13:20:29.817076Z","iopub.status.idle":"2022-08-12T13:20:30.071064Z","shell.execute_reply.started":"2022-08-12T13:20:29.817024Z","shell.execute_reply":"2022-08-12T13:20:30.069446Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(12,8))\nsns.catplot(x=\"VIPPlusCryoSleep\", kind=\"count\", data=train_data)\nplt.figure(figsize=(10,5))\nsns.countplot(x=\"VIPPlusCryoSleep\", hue=\"Transported\", data=train_data)","metadata":{"execution":{"iopub.status.busy":"2022-08-12T13:20:30.072848Z","iopub.execute_input":"2022-08-12T13:20:30.073329Z","iopub.status.idle":"2022-08-12T13:20:30.604924Z","shell.execute_reply.started":"2022-08-12T13:20:30.073282Z","shell.execute_reply":"2022-08-12T13:20:30.603374Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10,5))\nsns.countplot(x=\"HomePlanet\", hue=\"Transported\", data=train_data)","metadata":{"execution":{"iopub.status.busy":"2022-08-12T13:20:30.606634Z","iopub.execute_input":"2022-08-12T13:20:30.606991Z","iopub.status.idle":"2022-08-12T13:20:30.861111Z","shell.execute_reply.started":"2022-08-12T13:20:30.606960Z","shell.execute_reply":"2022-08-12T13:20:30.859735Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10,5))\nsns.countplot(y=\"HomePlanetDestination\", hue=\"Transported\", data=train_data)","metadata":{"execution":{"iopub.status.busy":"2022-08-12T13:20:30.862894Z","iopub.execute_input":"2022-08-12T13:20:30.863231Z","iopub.status.idle":"2022-08-12T13:20:31.198181Z","shell.execute_reply.started":"2022-08-12T13:20:30.863201Z","shell.execute_reply":"2022-08-12T13:20:31.196288Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(12,8))\nsns.catplot(x=\"HomePlanet\", y=\"Transported\", kind=\"bar\", data=train_data)","metadata":{"execution":{"iopub.status.busy":"2022-08-12T13:20:31.200688Z","iopub.execute_input":"2022-08-12T13:20:31.201230Z","iopub.status.idle":"2022-08-12T13:20:31.698946Z","shell.execute_reply.started":"2022-08-12T13:20:31.201180Z","shell.execute_reply":"2022-08-12T13:20:31.697380Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(12,8))\nsns.catplot(y=\"HomePlanetDestination\", x=\"Transported\", kind=\"bar\", data=train_data)","metadata":{"execution":{"iopub.status.busy":"2022-08-12T13:20:31.700008Z","iopub.execute_input":"2022-08-12T13:20:31.701041Z","iopub.status.idle":"2022-08-12T13:20:32.392394Z","shell.execute_reply.started":"2022-08-12T13:20:31.700995Z","shell.execute_reply":"2022-08-12T13:20:32.391070Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data.SpentRatio.describe()","metadata":{"execution":{"iopub.status.busy":"2022-08-12T13:20:32.394471Z","iopub.execute_input":"2022-08-12T13:20:32.395053Z","iopub.status.idle":"2022-08-12T13:20:32.409844Z","shell.execute_reply.started":"2022-08-12T13:20:32.395003Z","shell.execute_reply":"2022-08-12T13:20:32.408549Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(12,8))\nsns.catplot(x=\"Transported\", y=\"SpentRatio\", kind=\"bar\", data=train_data)","metadata":{"execution":{"iopub.status.busy":"2022-08-12T13:20:32.411523Z","iopub.execute_input":"2022-08-12T13:20:32.411947Z","iopub.status.idle":"2022-08-12T13:20:32.836232Z","shell.execute_reply.started":"2022-08-12T13:20:32.411910Z","shell.execute_reply":"2022-08-12T13:20:32.834788Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(12,8))\nsns.countplot(x=\"SpentRatioBin\", hue=\"Transported\", data=train_data)","metadata":{"execution":{"iopub.status.busy":"2022-08-12T13:20:32.838396Z","iopub.execute_input":"2022-08-12T13:20:32.838930Z","iopub.status.idle":"2022-08-12T13:20:33.113416Z","shell.execute_reply.started":"2022-08-12T13:20:32.838881Z","shell.execute_reply":"2022-08-12T13:20:33.112127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n

Objective and Metrics

\n\n**Objective** - classification accuracy\n\n**Metrics** - ROC, AUC, F1-score, confusion matrix","metadata":{}},{"cell_type":"markdown","source":"\n

Preprocessing

","metadata":{}},{"cell_type":"code","source":"# selected features\n#numerical_cols = [ 'VIPPlusCryoSleep', 'GroupMembersCount', 'Age', 'TotalSpentLog', 'TotalSpent'] \n#categorical_cols = [ 'HomePlanetDestination', 'CabinDeckSide', 'SpentRatioBin'] \n\nnumerical_cols = [ 'VIPPlusCryoSleep', 'FoodCourt', 'ShoppingMall', 'RoomService', 'VRDeck', 'Spa', 'TotalSpentLog', 'TotalSpent'] \ncategorical_cols = [ 'HomePlanetDestination', 'CabinDeckSide', 'SpentRatioBin', 'AgeBin'] \n\ny = train_data.Transported\n\nX = pd.DataFrame(train_data[numerical_cols + categorical_cols])\nX_test = pd.DataFrame(test_data[numerical_cols + categorical_cols])\n\n# Preprocessing for numerical data\nnumerical_transformer = Pipeline(steps=[\n ('scaler', StandardScaler())\n])\n\n# Preprocessing for categorical data\ncategorical_transformer = Pipeline(steps=[\n ('onehot', OneHotEncoder(sparse=False, handle_unknown='ignore'))\n])\n\n# Bundle preprocessing for numerical and categorical data\npreprocessor = ColumnTransformer(\n transformers=[\n ('num', numerical_transformer, numerical_cols),\n ('cat', categorical_transformer, categorical_cols)\n ])\n\ndata_full = pd.concat([X, X_test])\n\ndata_processed = preprocessor.fit_transform(data_full)\nX_processed = pd.DataFrame(data_processed[:X.shape[0]])\nX_processed.index = train_data.index\nX_test_processed = pd.DataFrame(data_processed[X.shape[0]:])\nX_test_processed.index = test_data.index\n\nX_processed.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-12T13:20:33.115406Z","iopub.execute_input":"2022-08-12T13:20:33.115860Z","iopub.status.idle":"2022-08-12T13:20:33.205994Z","shell.execute_reply.started":"2022-08-12T13:20:33.115824Z","shell.execute_reply":"2022-08-12T13:20:33.204846Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n

Modeling - Scikit-learn classifiers - Fit and Check Metrics

","metadata":{}},{"cell_type":"markdown","source":"# **Baseline model - .7947** \n.7759 -> .7767 -> .7770 -> .7785 -> .7796 -> .7804 -> .7833 -> .7871 -> .7900 -> .7913 -> .7917 -> .7947","metadata":{}},{"cell_type":"code","source":"def checkModelCVAccuracy(model): \n num_folds = 10 \n seed = 13\n \n kfold = KFold(n_splits = num_folds, random_state = seed, shuffle= True) \n cv_results = cross_val_score(model, X_processed, y, cv = kfold, scoring = 'accuracy') \n print(f\"Model CV accuracy: {cv_results.mean()} {cv_results.std()}\")\n\ncheckModelCVAccuracy(LogisticRegression(penalty=\"l1\", solver=\"liblinear\"))","metadata":{"execution":{"iopub.status.busy":"2022-08-12T13:20:33.207279Z","iopub.execute_input":"2022-08-12T13:20:33.207704Z","iopub.status.idle":"2022-08-12T13:20:36.677755Z","shell.execute_reply.started":"2022-08-12T13:20:33.207670Z","shell.execute_reply":"2022-08-12T13:20:36.672438Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Try different classifiers**","metadata":{}},{"cell_type":"code","source":"classifiers = [\n ('LogR', LogisticRegression(penalty=\"l1\", solver=\"liblinear\")),\n #('KNC', KNeighborsClassifier()),\n ('SVC', SVC()),\n ('LinearSVC', LinearSVC()),\n #('RandFC', RandomForestClassifier()),\n ('GradB', GradientBoostingClassifier()),\n #('AdaB', AdaBoostClassifier()),\n #('GNB', GaussianNB())\n]","metadata":{"execution":{"iopub.status.busy":"2022-08-12T13:20:36.679831Z","iopub.execute_input":"2022-08-12T13:20:36.680746Z","iopub.status.idle":"2022-08-12T13:20:36.691008Z","shell.execute_reply.started":"2022-08-12T13:20:36.680690Z","shell.execute_reply":"2022-08-12T13:20:36.688348Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def checkClassifiers(models):\n num_folds = 10 \n seed = 13\n accuracies = [] \n model_names = [] \n for name, model in models: \n kfold = KFold(n_splits = num_folds, random_state = seed, shuffle= True) \n cv_results = cross_val_score(model, X_processed, y, cv = kfold, scoring = 'accuracy')\n accuracies.append(cv_results.mean()) \n model_names.append(name) \n \n results_df = pd.DataFrame({\"Accuracy\": accuracies, \"Model\": model_names})\n \n plt.figure(figsize=(10,5)) \n ax = sns.barplot(x=\"Accuracy\", y=\"Model\", data=results_df, errwidth=0) \n ax.bar_label(ax.containers[0])\n \ncheckClassifiers(classifiers)","metadata":{"execution":{"iopub.status.busy":"2022-08-12T13:20:36.693018Z","iopub.execute_input":"2022-08-12T13:20:36.694320Z","iopub.status.idle":"2022-08-12T13:21:33.322579Z","shell.execute_reply.started":"2022-08-12T13:20:36.694266Z","shell.execute_reply":"2022-08-12T13:21:33.321268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Parameter tuning for best classifiers**","metadata":{}},{"cell_type":"code","source":"def grid_searching(classificator, params, folds, scoring): \n grid_search = GridSearchCV(classificator, params, verbose=1, cv=folds, scoring=scoring)\n grid_search.fit(X_processed, y)\n print(f\"best score - {grid_search.best_score_}; best params - {grid_search.best_params_}\")","metadata":{"execution":{"iopub.status.busy":"2022-08-12T13:21:33.324371Z","iopub.execute_input":"2022-08-12T13:21:33.324792Z","iopub.status.idle":"2022-08-12T13:21:33.332227Z","shell.execute_reply.started":"2022-08-12T13:21:33.324752Z","shell.execute_reply":"2022-08-12T13:21:33.330609Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"svc_params = [{\n 'kernel': ['rbf', ], # 'linear', 'poly', 'sigmoid'\n 'gamma': [0.1, ], # 1, 10, 100, 1000\n 'C': [1, ], # 0.1, 10, 100, 1000\n 'degree': [0, ], # 1, 2, 3, 4, 6\n}]\n\n#grid_searching(SVC(), svc_params, 5, 'accuracy')","metadata":{"execution":{"iopub.status.busy":"2022-08-12T13:21:33.333848Z","iopub.execute_input":"2022-08-12T13:21:33.334775Z","iopub.status.idle":"2022-08-12T13:21:33.341296Z","shell.execute_reply.started":"2022-08-12T13:21:33.334733Z","shell.execute_reply":"2022-08-12T13:21:33.340352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gradient_boosting_params = [{\n #'learning_rate': [.1, .01, 10],\n #'n_estimators': [100, 500, 1000],\n 'max_depth': [3, 6, 9, 15]\n}]\n\n#grid_searching(GradientBoostingClassifier(), gradient_boosting_params, 5, 'accuracy')\n# best score - 0.7943187194735808; best params - {'learning_rate': 0.1, 'n_estimators': 100}","metadata":{"execution":{"iopub.status.busy":"2022-08-12T13:21:33.342588Z","iopub.execute_input":"2022-08-12T13:21:33.344058Z","iopub.status.idle":"2022-08-12T13:21:33.355592Z","shell.execute_reply.started":"2022-08-12T13:21:33.344001Z","shell.execute_reply":"2022-08-12T13:21:33.354075Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n

Advanced Ensemble ML Classifiers

","metadata":{}},{"cell_type":"code","source":"ensemble_classifiers = [\n ('CatBoost', CatBoostClassifier(verbose=False,)),\n ('XGB', XGBClassifier(verbosity=0)),\n]\ncheckClassifiers(ensemble_classifiers)","metadata":{"execution":{"iopub.status.busy":"2022-08-12T13:21:33.357481Z","iopub.execute_input":"2022-08-12T13:21:33.358242Z","iopub.status.idle":"2022-08-12T13:22:33.783761Z","shell.execute_reply.started":"2022-08-12T13:21:33.358199Z","shell.execute_reply":"2022-08-12T13:22:33.782813Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n

Final Model - Metrics Check, Prediction and Submission

","metadata":{}},{"cell_type":"markdown","source":"# **Final Prediction**","metadata":{}},{"cell_type":"code","source":"#final_model = LogisticRegression(penalty=\"l1\", solver=\"liblinear\")\n#final_model = SVC()\nfinal_model = CatBoostClassifier(verbose=False, eval_metric='Accuracy')\n\n# check model CV accuracy\ncheckModelCVAccuracy(final_model)\n \nfinal_model.fit(X_processed, y)\nfinal_predictions = final_model.predict(X_test_processed)\n\nfinal_predictions_data_frame = pd.DataFrame(index=X_test_processed.index).assign(Transported=final_predictions.astype(bool))\n#final_predictions_data_frame.index = X_test_processed.index","metadata":{"execution":{"iopub.status.busy":"2022-08-12T13:22:33.785193Z","iopub.execute_input":"2022-08-12T13:22:33.785823Z","iopub.status.idle":"2022-08-12T13:23:29.373167Z","shell.execute_reply.started":"2022-08-12T13:22:33.785785Z","shell.execute_reply":"2022-08-12T13:23:29.371693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_predictions_data_frame.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-12T13:23:29.375075Z","iopub.execute_input":"2022-08-12T13:23:29.375586Z","iopub.status.idle":"2022-08-12T13:23:29.389065Z","shell.execute_reply.started":"2022-08-12T13:23:29.375532Z","shell.execute_reply":"2022-08-12T13:23:29.387654Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Check underfitting and overfitting**","metadata":{}},{"cell_type":"code","source":"def plotLearningCurves():\n train_sizes, train_scores, test_scores = learning_curve(LogisticRegression(penalty=\"l1\", solver=\"liblinear\"), X_processed, y, cv=10, scoring='accuracy')\n\n plt.plot(train_sizes, np.mean(train_scores,axis=1), label='Train score') \n plt.plot(train_sizes, np.mean(test_scores,axis=1), label='CV score') \n plt.xlabel(\"Train set size\") \n plt.ylabel(\"Accuracy\") \n plt.legend()\n\nplotLearningCurves()","metadata":{"execution":{"iopub.status.busy":"2022-08-12T13:23:29.391077Z","iopub.execute_input":"2022-08-12T13:23:29.391485Z","iopub.status.idle":"2022-08-12T13:23:40.258225Z","shell.execute_reply.started":"2022-08-12T13:23:29.391449Z","shell.execute_reply":"2022-08-12T13:23:40.256958Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Confusion Matrix**","metadata":{}},{"cell_type":"code","source":"train_pred = final_model.predict(X_processed)\nconfusion_matrix = confusion_matrix(y, train_pred)\ncm_plot = sns.heatmap(confusion_matrix,\n annot=True, \n cmap='Greens',\n fmt='d');\ncm_plot.set_xlabel('Predicted Values')\ncm_plot.set_ylabel('Actual Values')\ncm_plot.set_title('Confusion Matrix', size=20)","metadata":{"execution":{"iopub.status.busy":"2022-08-12T13:23:40.260003Z","iopub.execute_input":"2022-08-12T13:23:40.260480Z","iopub.status.idle":"2022-08-12T13:23:40.545758Z","shell.execute_reply.started":"2022-08-12T13:23:40.260435Z","shell.execute_reply":"2022-08-12T13:23:40.544769Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Prediction bias check - 2.4%**(5%)\naverage of predictions must be equal to average of labels. \nIf more than 5% - something went wrong","metadata":{}},{"cell_type":"code","source":"prediction_average = train_pred.sum() / len(train_pred)\nlabel_average = y.sum() / len(y)\nprediction_bias = prediction_average - label_average\nprint(f'Prediction avg - {prediction_average}; Labels average - {label_average}; \\nPrediction bias = {prediction_bias}')","metadata":{"execution":{"iopub.status.busy":"2022-08-12T13:23:40.547670Z","iopub.execute_input":"2022-08-12T13:23:40.548786Z","iopub.status.idle":"2022-08-12T13:23:40.556941Z","shell.execute_reply.started":"2022-08-12T13:23:40.548731Z","shell.execute_reply":"2022-08-12T13:23:40.555208Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Save results**","metadata":{}},{"cell_type":"code","source":"final_predictions_data_frame.to_csv('./result.csv', index=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-12T13:23:40.558431Z","iopub.execute_input":"2022-08-12T13:23:40.559742Z","iopub.status.idle":"2022-08-12T13:23:40.581372Z","shell.execute_reply.started":"2022-08-12T13:23:40.559696Z","shell.execute_reply":"2022-08-12T13:23:40.579277Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result_df = pd.read_csv('./result.csv', index_col='PassengerId')\nresult_df.head(1000)","metadata":{"execution":{"iopub.status.busy":"2022-08-12T13:23:40.588291Z","iopub.execute_input":"2022-08-12T13:23:40.589654Z","iopub.status.idle":"2022-08-12T13:23:40.609094Z","shell.execute_reply.started":"2022-08-12T13:23:40.589598Z","shell.execute_reply":"2022-08-12T13:23:40.607714Z"},"trusted":true},"execution_count":null,"outputs":[]}]}