{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport matplotlib\nimport seaborn as sns\n\nfrom sklearn.ensemble import RandomForestRegressor, RandomForestClassifier\nfrom sklearn.preprocessing import OrdinalEncoder, OneHotEncoder, StandardScaler\nfrom sklearn.linear_model import LinearRegression\nfrom sklearn.experimental import enable_iterative_imputer\nfrom sklearn.impute import KNNImputer, IterativeImputer\nfrom sklearn.naive_bayes import MultinomialNB\nfrom sklearn.model_selection import cross_val_score, train_test_split\nfrom sklearn.feature_selection import mutual_info_classif\nfrom sklearn.mixture import GaussianMixture\nfrom sklearn.cluster import KMeans, DBSCAN\nfrom yellowbrick.cluster import KElbowVisualizer\nfrom xgboost import XGBRegressor, XGBClassifier\nfrom sklearn.svm import SVC\nimport optuna","metadata":{"execution":{"iopub.status.busy":"2022-07-29T16:51:34.957815Z","iopub.execute_input":"2022-07-29T16:51:34.958612Z","iopub.status.idle":"2022-07-29T16:51:34.970737Z","shell.execute_reply.started":"2022-07-29T16:51:34.958566Z","shell.execute_reply":"2022-07-29T16:51:34.969330Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv(\"../input/spaceship-titanic/train.csv\", index_col='PassengerId').reset_index(drop=True)\ndf_test = pd.read_csv(\"../input/spaceship-titanic/test.csv\", index_col='PassengerId').reset_index(drop=True)\n\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-29T16:51:35.378672Z","iopub.execute_input":"2022-07-29T16:51:35.379083Z","iopub.status.idle":"2022-07-29T16:51:35.442408Z","shell.execute_reply.started":"2022-07-29T16:51:35.379048Z","shell.execute_reply":"2022-07-29T16:51:35.441025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Shape \")\nprint()\nprint(\"Train: \", df.shape)\nprint(\"Test: \", df_test.shape)","metadata":{"execution":{"iopub.status.busy":"2022-07-29T16:51:35.922882Z","iopub.execute_input":"2022-07-29T16:51:35.923641Z","iopub.status.idle":"2022-07-29T16:51:35.929917Z","shell.execute_reply.started":"2022-07-29T16:51:35.923599Z","shell.execute_reply":"2022-07-29T16:51:35.928632Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"NaN values \")\nprint()\nprint(\"Train: \\n\", df.isna().sum(), \"\\n\")\nprint(\"Test: \\n\", df_test.isna().sum())","metadata":{"execution":{"iopub.status.busy":"2022-07-29T16:51:36.399754Z","iopub.execute_input":"2022-07-29T16:51:36.400473Z","iopub.status.idle":"2022-07-29T16:51:36.414932Z","shell.execute_reply.started":"2022-07-29T16:51:36.400427Z","shell.execute_reply":"2022-07-29T16:51:36.413788Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df[\"Transported\"] = df[\"Transported\"].astype(np.int8)\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-29T16:51:36.832899Z","iopub.execute_input":"2022-07-29T16:51:36.833527Z","iopub.status.idle":"2022-07-29T16:51:36.853766Z","shell.execute_reply.started":"2022-07-29T16:51:36.833479Z","shell.execute_reply":"2022-07-29T16:51:36.852534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.Transported.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-07-29T16:51:37.148482Z","iopub.execute_input":"2022-07-29T16:51:37.149116Z","iopub.status.idle":"2022-07-29T16:51:37.156602Z","shell.execute_reply.started":"2022-07-29T16:51:37.149081Z","shell.execute_reply":"2022-07-29T16:51:37.155476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cabin_splited = df[\"Cabin\"].str.split(\"/\", expand=True)\ncabin_splited.columns = [\"Cabin_deck\", \"Cabin_num\", \"Cabin_side\"]\ncabin_splited.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-29T16:51:37.433318Z","iopub.execute_input":"2022-07-29T16:51:37.433714Z","iopub.status.idle":"2022-07-29T16:51:37.460939Z","shell.execute_reply.started":"2022-07-29T16:51:37.433681Z","shell.execute_reply":"2022-07-29T16:51:37.459814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df1 = pd.concat([df, cabin_splited], axis=1).drop(\"Cabin\", axis=1)\ndf1.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-29T16:51:37.640376Z","iopub.execute_input":"2022-07-29T16:51:37.640900Z","iopub.status.idle":"2022-07-29T16:51:37.666795Z","shell.execute_reply.started":"2022-07-29T16:51:37.640831Z","shell.execute_reply":"2022-07-29T16:51:37.665835Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cabin_splited = df_test[\"Cabin\"].str.split(\"/\", expand=True)\ncabin_splited.columns = [\"Cabin_deck\", \"Cabin_num\", \"Cabin_side\"]\ndf_test_1 = pd.concat([df_test, cabin_splited], axis=1).drop(\"Cabin\", axis=1)\ndf_test_1.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-29T16:51:37.940881Z","iopub.execute_input":"2022-07-29T16:51:37.941824Z","iopub.status.idle":"2022-07-29T16:51:37.975037Z","shell.execute_reply.started":"2022-07-29T16:51:37.941762Z","shell.execute_reply":"2022-07-29T16:51:37.973679Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"object_cols = [i for i in df1.columns if df1[i].dtype == \"O\"]\n\nfor i in object_cols:\n    print(i, \": \", df1[i].nunique())","metadata":{"execution":{"iopub.status.busy":"2022-07-29T16:51:38.127218Z","iopub.execute_input":"2022-07-29T16:51:38.127803Z","iopub.status.idle":"2022-07-29T16:51:38.142303Z","shell.execute_reply.started":"2022-07-29T16:51:38.127770Z","shell.execute_reply":"2022-07-29T16:51:38.141049Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Since Cabin_num hasn't any relation ship is a passenger Transported, we'll remove it. With Name column the same thing","metadata":{}},{"cell_type":"code","source":"df2 = df1.drop([\"Name\", \"Cabin_num\"], axis=1)\ndf_test_2 = df_test_1.drop([\"Name\", \"Cabin_num\"], axis=1)\ndf2.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-29T16:51:38.516471Z","iopub.execute_input":"2022-07-29T16:51:38.517154Z","iopub.status.idle":"2022-07-29T16:51:38.541307Z","shell.execute_reply.started":"2022-07-29T16:51:38.517117Z","shell.execute_reply":"2022-07-29T16:51:38.539947Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Handle NaN values","metadata":{}},{"cell_type":"code","source":"object_cols = [i for i in df2.columns if df2[i].dtype == \"O\"]\n\ndf2[object_cols] = df2[object_cols].fillna(-1)\ndf_test_2[object_cols] = df_test_2[object_cols].fillna(-1)\ndf2.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2022-07-29T16:51:39.030794Z","iopub.execute_input":"2022-07-29T16:51:39.031165Z","iopub.status.idle":"2022-07-29T16:51:39.056036Z","shell.execute_reply.started":"2022-07-29T16:51:39.031135Z","shell.execute_reply":"2022-07-29T16:51:39.054958Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cost_cols = [\"RoomService\", \"FoodCourt\", \"ShoppingMall\", \"Spa\", \"VRDeck\"]\ndf[df[cost_cols].isna().any(axis=1)]","metadata":{"execution":{"iopub.status.busy":"2022-07-29T16:51:39.301807Z","iopub.execute_input":"2022-07-29T16:51:39.302199Z","iopub.status.idle":"2022-07-29T16:51:39.336791Z","shell.execute_reply.started":"2022-07-29T16:51:39.302167Z","shell.execute_reply":"2022-07-29T16:51:39.335534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df3 = df2.copy()\ndf_test_3 = df_test_2.copy()\n\ndf3[cost_cols] = df3[cost_cols].fillna(0.0)\ndf_test_3[cost_cols] = df_test_3[cost_cols].fillna(0.0)\ndf3.head(8)","metadata":{"execution":{"iopub.status.busy":"2022-07-29T16:51:39.532395Z","iopub.execute_input":"2022-07-29T16:51:39.533499Z","iopub.status.idle":"2022-07-29T16:51:39.558088Z","shell.execute_reply.started":"2022-07-29T16:51:39.533459Z","shell.execute_reply":"2022-07-29T16:51:39.556925Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df3.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2022-07-29T16:51:39.724703Z","iopub.execute_input":"2022-07-29T16:51:39.725158Z","iopub.status.idle":"2022-07-29T16:51:39.737833Z","shell.execute_reply.started":"2022-07-29T16:51:39.725122Z","shell.execute_reply":"2022-07-29T16:51:39.736512Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df3[\"Age\"].hist(bins=20, rwidth=0.8)","metadata":{"execution":{"iopub.status.busy":"2022-07-29T16:51:39.943269Z","iopub.execute_input":"2022-07-29T16:51:39.943657Z","iopub.status.idle":"2022-07-29T16:51:40.176256Z","shell.execute_reply.started":"2022-07-29T16:51:39.943624Z","shell.execute_reply":"2022-07-29T16:51:40.175045Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df3.Age.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-07-29T16:51:40.183366Z","iopub.execute_input":"2022-07-29T16:51:40.184061Z","iopub.status.idle":"2022-07-29T16:51:40.194764Z","shell.execute_reply.started":"2022-07-29T16:51:40.184021Z","shell.execute_reply":"2022-07-29T16:51:40.193593Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df4 = df3.copy()\ndf_test_4 = df_test_3.copy()\n\ndf4[\"Age\"] = df4[\"Age\"].fillna(24.0)\ndf_test_4[\"Age\"] = df_test_4[\"Age\"].fillna(24.0)\n\ndf4.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2022-07-29T16:51:40.388382Z","iopub.execute_input":"2022-07-29T16:51:40.388734Z","iopub.status.idle":"2022-07-29T16:51:40.403584Z","shell.execute_reply.started":"2022-07-29T16:51:40.388706Z","shell.execute_reply":"2022-07-29T16:51:40.402698Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in cost_cols:\n    plt.subplots()\n    sns.distplot(df[i])","metadata":{"execution":{"iopub.status.busy":"2022-07-29T16:51:40.599012Z","iopub.execute_input":"2022-07-29T16:51:40.599369Z","iopub.status.idle":"2022-07-29T16:51:42.170323Z","shell.execute_reply.started":"2022-07-29T16:51:40.599340Z","shell.execute_reply":"2022-07-29T16:51:42.169199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df4[cost_cols].describe()","metadata":{"execution":{"iopub.status.busy":"2022-07-29T16:51:42.172555Z","iopub.execute_input":"2022-07-29T16:51:42.173243Z","iopub.status.idle":"2022-07-29T16:51:42.200976Z","shell.execute_reply.started":"2022-07-29T16:51:42.173184Z","shell.execute_reply":"2022-07-29T16:51:42.199845Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Encoding values","metadata":{}},{"cell_type":"code","source":"df4.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-29T16:51:42.202766Z","iopub.execute_input":"2022-07-29T16:51:42.203233Z","iopub.status.idle":"2022-07-29T16:51:42.221805Z","shell.execute_reply.started":"2022-07-29T16:51:42.203191Z","shell.execute_reply":"2022-07-29T16:51:42.220704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"obj_cols = [i for i in df4.columns if df4[i].dtype == \"O\"]\n\nfor i in obj_cols:\n    print(df4[i].unique())","metadata":{"execution":{"iopub.status.busy":"2022-07-29T16:51:42.224711Z","iopub.execute_input":"2022-07-29T16:51:42.225519Z","iopub.status.idle":"2022-07-29T16:51:42.239112Z","shell.execute_reply.started":"2022-07-29T16:51:42.225474Z","shell.execute_reply":"2022-07-29T16:51:42.237915Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in obj_cols:\n    print(df_test_4[i].unique())","metadata":{"execution":{"iopub.status.busy":"2022-07-29T16:51:42.240957Z","iopub.execute_input":"2022-07-29T16:51:42.241752Z","iopub.status.idle":"2022-07-29T16:51:42.253689Z","shell.execute_reply.started":"2022-07-29T16:51:42.241708Z","shell.execute_reply":"2022-07-29T16:51:42.252754Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df5 = pd.concat([df4, pd.get_dummies(df4[obj_cols], drop_first=True)], axis=1).drop(obj_cols, axis=1)\ndf_test_5 = pd.concat([df_test_4, pd.get_dummies(df_test_4[obj_cols], drop_first=True)], axis=1).drop(obj_cols, axis=1)\ndf5.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-29T16:51:42.256348Z","iopub.execute_input":"2022-07-29T16:51:42.257100Z","iopub.status.idle":"2022-07-29T16:51:42.311552Z","shell.execute_reply.started":"2022-07-29T16:51:42.257054Z","shell.execute_reply":"2022-07-29T16:51:42.310233Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df6 = df5.copy()\ndf_test_6 = df_test_5.copy()\n\ndf6[\"SumOfPay\"] = df6[cost_cols].sum(axis=1)\ndf_test_6[\"SumOfPay\"] = df_test_6[cost_cols].sum(axis=1)\ndf6","metadata":{"execution":{"iopub.status.busy":"2022-07-29T16:55:50.214602Z","iopub.execute_input":"2022-07-29T16:55:50.215030Z","iopub.status.idle":"2022-07-29T16:55:50.247878Z","shell.execute_reply.started":"2022-07-29T16:55:50.214996Z","shell.execute_reply":"2022-07-29T16:55:50.246906Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.heatmap(df6[cost_cols].corr(), annot=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-29T16:59:11.073889Z","iopub.execute_input":"2022-07-29T16:59:11.074742Z","iopub.status.idle":"2022-07-29T16:59:11.383036Z","shell.execute_reply.started":"2022-07-29T16:59:11.074703Z","shell.execute_reply":"2022-07-29T16:59:11.381917Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df7 = df6.copy()\ndf_test_7 = df_test_6.copy()","metadata":{"execution":{"iopub.status.busy":"2022-07-29T17:01:59.519740Z","iopub.execute_input":"2022-07-29T17:01:59.520134Z","iopub.status.idle":"2022-07-29T17:01:59.526229Z","shell.execute_reply.started":"2022-07-29T17:01:59.520104Z","shell.execute_reply":"2022-07-29T17:01:59.525303Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df7[\"SumOfPay_per_Age\"] = (df7.SumOfPay/df7.Age).replace(np.inf, 0).fillna(0.0)\ndf7.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-29T17:03:57.452214Z","iopub.execute_input":"2022-07-29T17:03:57.453001Z","iopub.status.idle":"2022-07-29T17:03:57.479601Z","shell.execute_reply.started":"2022-07-29T17:03:57.452955Z","shell.execute_reply":"2022-07-29T17:03:57.478184Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df7 = df7[(df7.SumOfPay_per_Age - df7.SumOfPay_per_Age.mean()) / df7.SumOfPay_per_Age.std() < 3]\ndf7.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-29T17:07:45.543733Z","iopub.execute_input":"2022-07-29T17:07:45.544128Z","iopub.status.idle":"2022-07-29T17:07:45.572466Z","shell.execute_reply.started":"2022-07-29T17:07:45.544097Z","shell.execute_reply":"2022-07-29T17:07:45.571549Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df7 = df7.drop(\"SumOfPay_per_Age\", axis=1)\ndf7","metadata":{"execution":{"iopub.status.busy":"2022-07-29T17:08:21.429241Z","iopub.execute_input":"2022-07-29T17:08:21.429936Z","iopub.status.idle":"2022-07-29T17:08:21.461192Z","shell.execute_reply.started":"2022-07-29T17:08:21.429901Z","shell.execute_reply":"2022-07-29T17:08:21.460174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = df6.copy()\ny = X.pop(\"Transported\")","metadata":{"execution":{"iopub.status.busy":"2022-07-29T17:09:17.409948Z","iopub.execute_input":"2022-07-29T17:09:17.410598Z","iopub.status.idle":"2022-07-29T17:09:17.417506Z","shell.execute_reply.started":"2022-07-29T17:09:17.410551Z","shell.execute_reply":"2022-07-29T17:09:17.416194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mi = mutual_info_classif(X, y)\nmi","metadata":{"execution":{"iopub.status.busy":"2022-07-29T17:09:18.284949Z","iopub.execute_input":"2022-07-29T17:09:18.285297Z","iopub.status.idle":"2022-07-29T17:09:19.283656Z","shell.execute_reply.started":"2022-07-29T17:09:18.285268Z","shell.execute_reply":"2022-07-29T17:09:19.282245Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mi = pd.Series(mi, index=X.columns).sort_values(ascending=False)\nmi.head(10)","metadata":{"execution":{"iopub.status.busy":"2022-07-29T17:09:19.285316Z","iopub.execute_input":"2022-07-29T17:09:19.285756Z","iopub.status.idle":"2022-07-29T17:09:19.295203Z","shell.execute_reply.started":"2022-07-29T17:09:19.285724Z","shell.execute_reply":"2022-07-29T17:09:19.293842Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = XGBClassifier(n_estimators=100, random_state=0)\n\nscores = cross_val_score(model, X, y, cv=10)\nscores","metadata":{"execution":{"iopub.status.busy":"2022-07-29T17:09:20.178010Z","iopub.execute_input":"2022-07-29T17:09:20.178365Z","iopub.status.idle":"2022-07-29T17:09:29.561210Z","shell.execute_reply.started":"2022-07-29T17:09:20.178337Z","shell.execute_reply":"2022-07-29T17:09:29.560040Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.mean(scores)","metadata":{"execution":{"iopub.status.busy":"2022-07-29T17:09:29.562965Z","iopub.execute_input":"2022-07-29T17:09:29.563286Z","iopub.status.idle":"2022-07-29T17:09:29.569553Z","shell.execute_reply.started":"2022-07-29T17:09:29.563258Z","shell.execute_reply":"2022-07-29T17:09:29.568577Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"0.8029502797508036 < 0.801108289353597","metadata":{"execution":{"iopub.status.busy":"2022-07-29T17:09:33.600395Z","iopub.execute_input":"2022-07-29T17:09:33.600791Z","iopub.status.idle":"2022-07-29T17:09:33.606646Z","shell.execute_reply.started":"2022-07-29T17:09:33.600759Z","shell.execute_reply":"2022-07-29T17:09:33.605943Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scores","metadata":{"execution":{"iopub.status.busy":"2022-07-29T17:09:34.453644Z","iopub.execute_input":"2022-07-29T17:09:34.454926Z","iopub.status.idle":"2022-07-29T17:09:34.464685Z","shell.execute_reply.started":"2022-07-29T17:09:34.454825Z","shell.execute_reply":"2022-07-29T17:09:34.463256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = RandomForestClassifier(n_estimators=100, random_state=0)\n\nscores = cross_val_score(model, X, y, cv=10)\nscores","metadata":{"execution":{"iopub.status.busy":"2022-07-29T17:09:35.831675Z","iopub.execute_input":"2022-07-29T17:09:35.832561Z","iopub.status.idle":"2022-07-29T17:09:44.957840Z","shell.execute_reply.started":"2022-07-29T17:09:35.832488Z","shell.execute_reply":"2022-07-29T17:09:44.957006Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.mean(scores)","metadata":{"execution":{"iopub.status.busy":"2022-07-29T17:09:44.959812Z","iopub.execute_input":"2022-07-29T17:09:44.960132Z","iopub.status.idle":"2022-07-29T17:09:44.966471Z","shell.execute_reply.started":"2022-07-29T17:09:44.960105Z","shell.execute_reply":"2022-07-29T17:09:44.965318Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scores","metadata":{"execution":{"iopub.status.busy":"2022-07-29T17:09:50.996914Z","iopub.execute_input":"2022-07-29T17:09:50.997588Z","iopub.status.idle":"2022-07-29T17:09:51.006632Z","shell.execute_reply.started":"2022-07-29T17:09:50.997552Z","shell.execute_reply":"2022-07-29T17:09:51.005314Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def run(trial):\n    param = {\n                \"n_estimators\" : trial.suggest_int('n_estimators', 100, 500),\n                'max_depth':trial.suggest_int('max_depth', 2, 25),\n                'reg_alpha':trial.suggest_float('reg_alpha', 0.0, 5.0),\n                'reg_lambda':trial.suggest_float('reg_lambda', 0.0, 5.0),\n                'min_child_weight':trial.suggest_float('min_child_weight', 0.0, 5.0),\n                'gamma':trial.suggest_int('gamma', 0, 5),\n                'learning_rate':trial.suggest_loguniform('learning_rate',0.005,0.5),\n                'colsample_bytree':trial.suggest_discrete_uniform('colsample_bytree',0.1,1,0.01),\n                'nthread' : -1,\n                \"random_state\": 0\n            }\n    \n    X_train, X_val, y_train, y_val = train_test_split(X, y)\n    model1 = XGBClassifier(**param)\n    model1.fit(X_train, y_train)\n    \n    return model1.score(X_val, y_val)\n\nstudy = optuna.create_study(direction='maximize')\nstudy.optimize(run, n_trials=200)\n\nprint(study.best_params)","metadata":{"execution":{"iopub.status.busy":"2022-07-29T17:09:51.863526Z","iopub.execute_input":"2022-07-29T17:09:51.863956Z","iopub.status.idle":"2022-07-29T17:19:21.832699Z","shell.execute_reply.started":"2022-07-29T17:09:51.863922Z","shell.execute_reply":"2022-07-29T17:19:21.831409Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"xgb_params = study.best_params\nxgb_params","metadata":{"execution":{"iopub.status.busy":"2022-07-29T17:19:21.835243Z","iopub.execute_input":"2022-07-29T17:19:21.836137Z","iopub.status.idle":"2022-07-29T17:19:21.841250Z","shell.execute_reply.started":"2022-07-29T17:19:21.836091Z","shell.execute_reply":"2022-07-29T17:19:21.840103Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# xgb_params = {'n_estimators': 194,\n#  'max_depth': 17,\n#  'reg_alpha': 0.05985863191374392,\n#  'reg_lambda': 0.8785143132516507,\n#  'min_child_weight': 3.269082659882775,\n#  'gamma': 3,\n#  'learning_rate': 0.12580841291774067,\n#  'colsample_bytree': 0.7}\n# xgb_params","metadata":{"execution":{"iopub.status.busy":"2022-07-29T17:19:21.842981Z","iopub.execute_input":"2022-07-29T17:19:21.843539Z","iopub.status.idle":"2022-07-29T17:19:21.858318Z","shell.execute_reply.started":"2022-07-29T17:19:21.843496Z","shell.execute_reply":"2022-07-29T17:19:21.857043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def run2(trial):\n    criterion = trial.suggest_categorical('criterion', ['gini', 'entropy'])\n    bootstrap = trial.suggest_categorical('bootstrap',['True','False'])\n    max_depth = trial.suggest_int('max_depth', 2, 50)\n    max_features = trial.suggest_categorical('max_features', ['auto', 'sqrt','log2'])\n    max_leaf_nodes = trial.suggest_int('max_leaf_nodes', 2, 100)\n    n_estimators =  trial.suggest_int('n_estimators', 100, 500)\n    \n    X_train, X_val, y_train, y_val = train_test_split(X, y)\n    regr = RandomForestClassifier(bootstrap = bootstrap, criterion = criterion,\n                                 max_depth = max_depth, max_features = max_features,\n                                 max_leaf_nodes = max_leaf_nodes,n_estimators = n_estimators,n_jobs=2)\n    regr.fit(X_train, y_train)\n    \n    return regr.score(X_val, y_val)\n\nstudy2 = optuna.create_study(direction='maximize')\nstudy2.optimize(run2, n_trials=200)\n\nprint(study2.best_params)","metadata":{"execution":{"iopub.status.busy":"2022-07-29T17:19:21.862431Z","iopub.execute_input":"2022-07-29T17:19:21.863125Z","iopub.status.idle":"2022-07-29T17:23:28.007421Z","shell.execute_reply.started":"2022-07-29T17:19:21.863086Z","shell.execute_reply":"2022-07-29T17:23:28.006162Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rf_params = study2.best_params\nrf_params","metadata":{"execution":{"iopub.status.busy":"2022-07-29T17:23:28.008946Z","iopub.execute_input":"2022-07-29T17:23:28.009262Z","iopub.status.idle":"2022-07-29T17:23:28.015067Z","shell.execute_reply.started":"2022-07-29T17:23:28.009235Z","shell.execute_reply":"2022-07-29T17:23:28.013383Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# rf_params = {'criterion': 'entropy',\n#  'bootstrap': 'True',\n#  'max_depth': 33,\n#  'max_features': 'sqrt',\n#  'max_leaf_nodes': 77,\n#  'n_estimators': 107}\n# rf_params","metadata":{"execution":{"iopub.status.busy":"2022-07-29T17:23:28.016783Z","iopub.execute_input":"2022-07-29T17:23:28.017470Z","iopub.status.idle":"2022-07-29T17:23:28.033703Z","shell.execute_reply.started":"2022-07-29T17:23:28.017422Z","shell.execute_reply":"2022-07-29T17:23:28.032487Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from eli5.sklearn import PermutationImportance\nimport eli5\n\nX_train, X_valid, y_train, y_valid = train_test_split(X, y)\n\nmodel1 = XGBClassifier(**xgb_params, nthread = -1, random_state=0)\nmodel1.fit(X_train, y_train)\n\nperm = PermutationImportance(model1, random_state=1).fit(X_valid, y_valid)\neli5.show_weights(perm, feature_names = X_valid.columns.tolist())","metadata":{"execution":{"iopub.status.busy":"2022-07-29T17:23:28.034913Z","iopub.execute_input":"2022-07-29T17:23:28.035598Z","iopub.status.idle":"2022-07-29T17:23:40.209806Z","shell.execute_reply.started":"2022-07-29T17:23:28.035565Z","shell.execute_reply":"2022-07-29T17:23:40.208616Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.scatterplot(X.RoomService, X.SumOfPay)","metadata":{"execution":{"iopub.status.busy":"2022-07-29T17:23:40.211585Z","iopub.execute_input":"2022-07-29T17:23:40.212473Z","iopub.status.idle":"2022-07-29T17:23:40.464550Z","shell.execute_reply.started":"2022-07-29T17:23:40.212433Z","shell.execute_reply":"2022-07-29T17:23:40.463351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model1 = XGBClassifier(**xgb_params, nthread = -1, random_state=0)\nmodel2 = RandomForestClassifier(**rf_params, n_jobs=2, random_state=0)\nmodel3 = MultinomialNB()\n\nmodel1.fit(X, y)\nprint(\"Done!\")\nmodel2.fit(X, y)\nprint(\"Done!\")\nmodel3.fit(X, y)\nprint(\"Done!\")","metadata":{"execution":{"iopub.status.busy":"2022-07-29T17:23:40.466166Z","iopub.execute_input":"2022-07-29T17:23:40.466526Z","iopub.status.idle":"2022-07-29T17:23:44.389133Z","shell.execute_reply.started":"2022-07-29T17:23:40.466497Z","shell.execute_reply":"2022-07-29T17:23:44.387782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = ((model1.predict(df_test_6) + model2.predict(df_test_6) + model3.predict(df_test_6)) / 3).round()\npreds","metadata":{"execution":{"iopub.status.busy":"2022-07-29T17:23:44.393559Z","iopub.execute_input":"2022-07-29T17:23:44.394616Z","iopub.status.idle":"2022-07-29T17:23:44.586473Z","shell.execute_reply.started":"2022-07-29T17:23:44.394555Z","shell.execute_reply":"2022-07-29T17:23:44.585152Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ss = pd.read_csv(\"../input/spaceship-titanic/sample_submission.csv\")\nss.Transported = preds.astype(bool)\nss","metadata":{"execution":{"iopub.status.busy":"2022-07-29T17:23:44.587950Z","iopub.execute_input":"2022-07-29T17:23:44.588682Z","iopub.status.idle":"2022-07-29T17:23:44.615184Z","shell.execute_reply.started":"2022-07-29T17:23:44.588639Z","shell.execute_reply":"2022-07-29T17:23:44.614413Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ss.to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2022-07-29T17:23:44.616515Z","iopub.execute_input":"2022-07-29T17:23:44.617143Z","iopub.status.idle":"2022-07-29T17:23:44.630408Z","shell.execute_reply.started":"2022-07-29T17:23:44.617099Z","shell.execute_reply":"2022-07-29T17:23:44.628992Z"},"trusted":true},"execution_count":null,"outputs":[]}]}