{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# **Setup**","metadata":{}},{"cell_type":"code","source":"#import required libraries\nimport numpy as np\nimport pandas as pd\n#from sklearn.svm import SVC","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-01T18:26:58.731397Z","iopub.execute_input":"2022-07-01T18:26:58.731879Z","iopub.status.idle":"2022-07-01T18:26:58.736865Z","shell.execute_reply.started":"2022-07-01T18:26:58.731840Z","shell.execute_reply":"2022-07-01T18:26:58.735969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#import training data\npassengers_known = pd.read_csv('../input/spaceship-titanic/train.csv')\n#import training data\npassengers_unknown = pd.read_csv('../input/spaceship-titanic/test.csv')","metadata":{"execution":{"iopub.status.busy":"2022-07-01T18:26:58.771115Z","iopub.execute_input":"2022-07-01T18:26:58.771591Z","iopub.status.idle":"2022-07-01T18:26:58.821180Z","shell.execute_reply.started":"2022-07-01T18:26:58.771551Z","shell.execute_reply":"2022-07-01T18:26:58.820027Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Exploratory Data Analysis**","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Handling Missing Data**","metadata":{}},{"cell_type":"code","source":"#Credits - BARBAGRANDE007\n\n# Assumption 1: When in cryo sleep, you will not use any facilities like ShoppingMall, RoomService etc.\n# All missing values will be replaced with 0.0.\n# Assumption 2: If no facilities have been used, CrySleep-NaN will be changed to True.\n# Assumption 3: If food is consumed or roomservice has been used, CryoSleep-NaN will be changed to False\n\ndef cryosleep(df):\n    df['ShoppingMall'] = np.where((df['ShoppingMall'].isnull()) & (df['CryoSleep'] == True), 0.0, df['ShoppingMall'])\n    df['RoomService'] = np.where((df['RoomService'].isnull()) & (df['CryoSleep'] == True), 0.0, df['RoomService'])\n    df['FoodCourt'] = np.where((df['FoodCourt'].isnull()) & (df['CryoSleep'] == True), 0.0, df['FoodCourt'])\n    df['Spa'] = np.where((df['Spa'].isnull()) & (df['CryoSleep'] == True), 0.0, df['Spa'])\n    df['VRDeck'] = np.where((df['VRDeck'].isnull()) & (df['CryoSleep'] == True), 0.0, df['VRDeck'])\n    df['CryoSleep'] = np.where((df['CryoSleep'].isnull()) & ((df['RoomService'] == 0.0) & (df['FoodCourt'] == 0.0) & (df['ShoppingMall'] == 0.0) & (df['Spa'] == 0.0) & (df['VRDeck'] == 0.0)), True, df['CryoSleep'])\n    df['CryoSleep'] = np.where((df['CryoSleep'].isnull()) & ((df['RoomService'] > 0.0) | (df['FoodCourt'] > 0.0) | (df['ShoppingMall'] > 0.0) | (df['Spa'] > 0.0) | (df['VRDeck'] > 0.0)), False, df['CryoSleep'])\n    return df\n\ncryosleep(passengers_known)\ncryosleep(passengers_unknown)","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-07-01T18:26:58.862398Z","iopub.execute_input":"2022-07-01T18:26:58.862868Z","iopub.status.idle":"2022-07-01T18:26:58.916151Z","shell.execute_reply.started":"2022-07-01T18:26:58.862830Z","shell.execute_reply":"2022-07-01T18:26:58.915109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#fill home planet based on cabins\n\ndef fill_home_planet(df):\n    df.loc[df['Cabin'].str.startswith('A', na=True), \"HomePlanet\"] = 'Europa'\n    df.loc[df['Cabin'].str.startswith('B', na=True), \"HomePlanet\"] = 'Europa'\n    df.loc[df['Cabin'].str.startswith('C', na=True), \"HomePlanet\"] = 'Europa'\n    df.loc[df['Cabin'].str.startswith('G', na=True), \"HomePlanet\"] = 'Earth'\n    df.loc[df['Cabin'].str.startswith('T', na=True), \"HomePlanet\"] = 'Europa'\n    df[\"HomePlanet\"].fillna(\"unknown\", inplace = True)\n    return df\n\nfill_home_planet(passengers_known)\nfill_home_planet(passengers_unknown)","metadata":{"execution":{"iopub.status.busy":"2022-07-01T18:26:58.917804Z","iopub.execute_input":"2022-07-01T18:26:58.918404Z","iopub.status.idle":"2022-07-01T18:26:58.983448Z","shell.execute_reply.started":"2022-07-01T18:26:58.918368Z","shell.execute_reply":"2022-07-01T18:26:58.982222Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#fill missing features\n\ndef fill_missing_features(df):\n    df[\"Destination\"].fillna(\"unknown\", inplace = True)\n    df[\"Cabin\"].fillna(\"F/0/S\", inplace = True) \n    df[\"Age\"].fillna(passengers_known['Age'].mean(), inplace = True) \n    df[\"VIP\"].fillna(False, inplace = True) \n    df[\"CryoSleep\"].fillna(False, inplace = True) \n    df[\"RoomService\"].fillna(0, inplace = True) \n    df[\"FoodCourt\"].fillna(0, inplace = True) \n    df[\"ShoppingMall\"].fillna(0, inplace = True) \n    df[\"Spa\"].fillna(0, inplace = True) \n    df[\"VRDeck\"].fillna(0, inplace = True)\n    return df\n\nfill_missing_features(passengers_known)\nfill_missing_features(passengers_unknown)","metadata":{"execution":{"iopub.status.busy":"2022-07-01T18:26:58.986055Z","iopub.execute_input":"2022-07-01T18:26:58.987012Z","iopub.status.idle":"2022-07-01T18:26:59.028672Z","shell.execute_reply.started":"2022-07-01T18:26:58.986957Z","shell.execute_reply":"2022-07-01T18:26:59.027545Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Feature Engineering**","metadata":{}},{"cell_type":"code","source":"def add_group_features(df):\n    df['Group'] = df['PassengerId'].apply(lambda x: x.split('_')[0]).astype(int)\n    df['Group_size'] = df['Group'].map(lambda x: df['Group'].value_counts()[x])\n    df['IsAlone'] = np.where((df[\"Group_size\"] == 1),1,0)\n    return df\n\ndef add_billing_features(df):\n    df['Premium'] = df.RoomService +  df.Spa + df.VRDeck\n    df['Essential'] =  df.FoodCourt + df.ShoppingMall\n    return df\n\ndef add_age_features(df):\n    df['IsAdult'] = np.where((df[\"Age\"] > 18),1,0)\n\ndef encode_features(df):\n    df['VIP']=df['VIP'].map({False:0, True:1})\n    df['CryoSleep']=df['CryoSleep'].map({False:0, True:1})\n    df['Destination']=df['Destination'].map({'TRAPPIST-1e':0, 'PSO J318.5-22':1, '55 Cancri e':2, 'unknown':3})\n    df['HomePlanet']=df['HomePlanet'].map({'Europa':0, 'Earth':1, 'Mars':2, 'unknown':3})\n    return df\n\nadd_group_features(passengers_known)\nadd_group_features(passengers_unknown)\n\nadd_billing_features(passengers_known)\nadd_billing_features(passengers_unknown)\n\nadd_age_features(passengers_known)\nadd_age_features(passengers_unknown)\n\nencode_features(passengers_known)\nencode_features(passengers_unknown)","metadata":{"execution":{"iopub.status.busy":"2022-07-01T18:26:59.030244Z","iopub.execute_input":"2022-07-01T18:26:59.030940Z","iopub.status.idle":"2022-07-01T18:27:08.071013Z","shell.execute_reply.started":"2022-07-01T18:26:59.030891Z","shell.execute_reply":"2022-07-01T18:27:08.070001Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#copy transported column for known passengers\npassengers_known_status = passengers_known.Transported\npassengers_known_status\n\n#split cabin column\ncabin_params = passengers_known['Cabin'].str.split('/',expand=True).rename(columns = lambda x: \"cabin_param_\"+str(x+1))\npassengers_known = pd.concat([passengers_known, cabin_params],axis=1)\n\npassengers_known['cabin_param_1']=passengers_known['cabin_param_1'].map({'B':1, 'F':5, 'A':0, 'G':6 ,'E':4, 'D':3, 'C':2, 'T':7, 'Z':8})\npassengers_known['cabin_param_3']=passengers_known['cabin_param_3'].map({'P':0,  'S':1, 'Z':2})\n","metadata":{"execution":{"iopub.status.busy":"2022-07-01T18:27:08.073178Z","iopub.execute_input":"2022-07-01T18:27:08.073561Z","iopub.status.idle":"2022-07-01T18:27:08.311121Z","shell.execute_reply.started":"2022-07-01T18:27:08.073525Z","shell.execute_reply":"2022-07-01T18:27:08.310115Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def find_home_planet(df, deck, cabin):\n    df = df[(df['cabin_param_1'] == deck) & (df['HomePlanet'].notnull()) & (df['cabin_param_2'] == cabin)]\n    if df['HomePlanet'].empty:\n        return \"unknown\"\n    else:\n        return df['HomePlanet'].iloc[0]\n\ndf = passengers_known[passengers_known['HomePlanet'] == 'unknown']\ndf2 = df[['PassengerId', 'cabin_param_1','cabin_param_2']]\nfor index, row in df2.iterrows():\n    home_planet = find_home_planet(passengers_known, row['cabin_param_1'], row['cabin_param_2'])\n    passengers_known.loc[passengers_known['PassengerId'] == row['PassengerId'], 'HomePlanet'] = home_planet","metadata":{"execution":{"iopub.status.busy":"2022-07-01T18:27:08.312635Z","iopub.execute_input":"2022-07-01T18:27:08.313702Z","iopub.status.idle":"2022-07-01T18:27:08.327696Z","shell.execute_reply.started":"2022-07-01T18:27:08.313652Z","shell.execute_reply":"2022-07-01T18:27:08.326584Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#drop transported column\npassengers_known = passengers_known.drop(columns={'Transported'})\n#drop unnecessary columns\npassengers_known = passengers_known.drop(columns={'Name','PassengerId', 'Cabin', 'RoomService', 'Spa', 'VRDeck', 'ShoppingMall', 'FoodCourt', 'cabin_param_2', 'Age'})\n\npassengers_known_v2 = passengers_known","metadata":{"execution":{"iopub.status.busy":"2022-07-01T18:27:08.329491Z","iopub.execute_input":"2022-07-01T18:27:08.330194Z","iopub.status.idle":"2022-07-01T18:27:08.341408Z","shell.execute_reply.started":"2022-07-01T18:27:08.330143Z","shell.execute_reply":"2022-07-01T18:27:08.340383Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Super Vector Machine Approach**","metadata":{}},{"cell_type":"code","source":"#learn from training data\n#m = SVC(gamma='auto') # 0.70773\n#m = SVC(kernel='rbf') # 0.7905\n#m = SVC(kernel='rbf', C=2, gamma=1.2) # 0.50268\n#m = SVC(kernel='rbf', C=0.1, gamma=10) # 0.50689\n#m = SVC(kernel='rbf', C=11, random_state=123) # 0.79448\n#m = SVC(kernel='rbf', C=10, random_state=123) # 0.79471\n#m.fit(passengers_known_v2,passengers_known_status)\n#verify prediction with known data\n#passengers_known_pred = m.predict(passengers_known_v2)","metadata":{"execution":{"iopub.status.busy":"2022-07-01T18:27:08.343218Z","iopub.execute_input":"2022-07-01T18:27:08.343875Z","iopub.status.idle":"2022-07-01T18:27:08.352426Z","shell.execute_reply.started":"2022-07-01T18:27:08.343828Z","shell.execute_reply":"2022-07-01T18:27:08.351704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Cat Boost Classifier Approach**","metadata":{}},{"cell_type":"code","source":"#from catboost import CatBoostClassifier\n#m = CatBoostClassifier(iterations=100000,verbose=5000)\n#m.fit(passengers_known_v2,passengers_known_status)\n#passengers_known_pred = m.predict(passengers_known_v2)","metadata":{"execution":{"iopub.status.busy":"2022-07-01T18:27:08.353840Z","iopub.execute_input":"2022-07-01T18:27:08.354444Z","iopub.status.idle":"2022-07-01T18:27:08.365693Z","shell.execute_reply.started":"2022-07-01T18:27:08.354399Z","shell.execute_reply":"2022-07-01T18:27:08.364551Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Cat Boost Classifier Approach with Grid Search**","metadata":{}},{"cell_type":"code","source":"from pandas.api.types import is_numeric_dtype\nimport catboost as cb\n\ndef get_categorical_indicies(X):\n    cats = []\n    for col in X.columns:\n        if is_numeric_dtype(X[col]):\n            pass\n        else:\n            cats.append(col)\n    cat_indicies = []\n    for col in cats:\n        cat_indicies.append(X.columns.get_loc(col))\n    return cat_indicies\n\ndef convert_cats(X):\n    cats = []\n    for col in X.columns:\n        if is_numeric_dtype(X[col]):\n            pass\n        else:\n            cats.append(col)\n    cat_indicies = []\n    for col in cats:\n        X[col] = X[col].astype('category')\n\nm = cb.CatBoostClassifier(\n    loss_function='Logloss', \n    eval_metric='Accuracy'\n)\n\ngrid = {'learning_rate': [0.03, 0.1],\n'depth': [6, 8, 10],\n'l2_leaf_reg': [1, 3, 5,],\n'iterations': [50, 100, 150]}\n\ncategorical_indicies = get_categorical_indicies(passengers_known_v2)\nconvert_cats(passengers_known_v2)\n\ntrain_dataset = cb.Pool(passengers_known_v2,passengers_known_status, cat_features=categorical_indicies)\n\nm.grid_search(grid,train_dataset)\n\nm.get_params()","metadata":{"execution":{"iopub.status.busy":"2022-07-01T18:27:08.368145Z","iopub.execute_input":"2022-07-01T18:27:08.368779Z","iopub.status.idle":"2022-07-01T18:27:08.380673Z","shell.execute_reply.started":"2022-07-01T18:27:08.368733Z","shell.execute_reply":"2022-07-01T18:27:08.379586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **High Gradient Boosting Classifier**","metadata":{}},{"cell_type":"code","source":"#from sklearn.ensemble import HistGradientBoostingClassifier\n\n#m = HistGradientBoostingClassifier()\n\n#m.fit(passengers_known_v2,passengers_known_status)\n","metadata":{"execution":{"iopub.status.busy":"2022-07-01T18:27:08.382481Z","iopub.execute_input":"2022-07-01T18:27:08.382983Z","iopub.status.idle":"2022-07-01T18:27:08.392395Z","shell.execute_reply.started":"2022-07-01T18:27:08.382935Z","shell.execute_reply":"2022-07-01T18:27:08.391321Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Voting Classifier**","metadata":{}},{"cell_type":"code","source":"#from sklearn.linear_model import LogisticRegression\n#from sklearn.naive_bayes import GaussianNB\n#from sklearn.ensemble import RandomForestClassifier, VotingClassifier\n\n#clf1 = LogisticRegression(multi_class='multinomial', random_state=1)\n#clf2 = RandomForestClassifier(n_estimators=50, random_state=1)\n#clf3 = GaussianNB()\n\n#m = VotingClassifier(estimators=[('lr', clf1), ('rf', clf2), ('gnb', clf3)])\n\n#m.fit(passengers_known_v2,passengers_known_status)","metadata":{"execution":{"iopub.status.busy":"2022-07-01T18:27:23.351563Z","iopub.execute_input":"2022-07-01T18:27:23.352058Z","iopub.status.idle":"2022-07-01T18:27:27.170106Z","shell.execute_reply.started":"2022-07-01T18:27:23.352018Z","shell.execute_reply":"2022-07-01T18:27:27.169023Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#passengers_known_pred = m.predict(passengers_known_v2)","metadata":{"execution":{"iopub.status.busy":"2022-07-01T18:10:19.829098Z","iopub.execute_input":"2022-07-01T18:10:19.829449Z","iopub.status.idle":"2022-07-01T18:10:19.834493Z","shell.execute_reply.started":"2022-07-01T18:10:19.829419Z","shell.execute_reply":"2022-07-01T18:10:19.833496Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Predict Transport Status of Remaining Passengers**","metadata":{}},{"cell_type":"code","source":"passengers_unknown_ids = pd.DataFrame(passengers_unknown.PassengerId)\n\n#split cabin column\ncabin_params = passengers_unknown['Cabin'].str.split('/',expand=True).rename(columns = lambda x: \"cabin_param_\"+str(x+1))\npassengers_unknown = pd.concat([passengers_unknown, cabin_params],axis=1)\n\npassengers_unknown['cabin_param_1']=passengers_unknown['cabin_param_1'].map({'B':1, 'F':5, 'A':0, 'G':6 ,'E':4, 'D':3, 'C':2, 'T':7})\npassengers_unknown['cabin_param_3']=passengers_unknown['cabin_param_3'].map({'P':0,  'S':1, 'Z':2})\n\ndf = passengers_unknown[passengers_unknown['HomePlanet'] == 'unknown']\ndf2 = df[['PassengerId', 'cabin_param_1','cabin_param_2']]\nfor index, row in df2.iterrows():\n    home_planet = find_home_planet(passengers_unknown, row['cabin_param_1'], row['cabin_param_2'])\n    passengers_unknown.loc[passengers_unknown['PassengerId'] == row['PassengerId'], 'HomePlanet'] = home_planet\n\npassengers_unknown = passengers_unknown.drop(columns={'Name','PassengerId', 'Cabin', 'RoomService', 'Spa', 'VRDeck', 'ShoppingMall', 'FoodCourt', 'cabin_param_2', 'Age'})\n\npassengers_unknown_v2 = passengers_unknown\n\n\n","metadata":{"execution":{"iopub.status.busy":"2022-07-01T18:10:32.190914Z","iopub.execute_input":"2022-07-01T18:10:32.192215Z","iopub.status.idle":"2022-07-01T18:10:32.217231Z","shell.execute_reply.started":"2022-07-01T18:10:32.192166Z","shell.execute_reply":"2022-07-01T18:10:32.216352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#predict status\npassengers_unknown_pred = m.predict(passengers_unknown_v2)\n","metadata":{"execution":{"iopub.status.busy":"2022-07-01T18:10:39.609926Z","iopub.execute_input":"2022-07-01T18:10:39.610739Z","iopub.status.idle":"2022-07-01T18:10:39.621302Z","shell.execute_reply.started":"2022-07-01T18:10:39.610692Z","shell.execute_reply":"2022-07-01T18:10:39.620594Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"passengers_unknown_pred = pd.DataFrame(passengers_unknown_pred)\npassengers_unknown_pred = passengers_unknown_pred.set_axis(['Transported'] , axis=1, inplace=False)","metadata":{"execution":{"iopub.status.busy":"2022-07-01T18:10:43.511163Z","iopub.execute_input":"2022-07-01T18:10:43.511859Z","iopub.status.idle":"2022-07-01T18:10:43.517122Z","shell.execute_reply.started":"2022-07-01T18:10:43.511816Z","shell.execute_reply":"2022-07-01T18:10:43.516232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Generate Output**","metadata":{}},{"cell_type":"code","source":"final_prediction = pd.concat([passengers_unknown_ids, passengers_unknown_pred],axis=1)\nfinal_prediction.to_csv(\"./submission.csv\",index=False)","metadata":{"execution":{"iopub.status.busy":"2022-06-13T19:25:40.315135Z","iopub.execute_input":"2022-06-13T19:25:40.315522Z","iopub.status.idle":"2022-06-13T19:25:40.331668Z","shell.execute_reply.started":"2022-06-13T19:25:40.315491Z","shell.execute_reply":"2022-06-13T19:25:40.330677Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Experiments to Improve Score**","metadata":{}},{"cell_type":"markdown","source":"**Super Vector Machine Results**\n* #m = SVC(kernel='rbf', C=10, random_state=123) # 0.79471\n\n**Cat Boost Classifier Results**\n* 0.79798\n* Introduced Group, Group_size, IsAlone features - Improved score to 0.80056\n* Introduced Premium and Essential features - Improved score to 0.80219\n* Removed one hot encoding for cabin deck and sides and VIP and mapped with numeric values - Improved score to 0.80289\n* Used grid search for Catboost - Improved score to 0.80406\n* Removed cabin number - Improved score to 0.80523\n\n\n","metadata":{}}]}