{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-17T06:47:45.121017Z","iopub.execute_input":"2022-07-17T06:47:45.122067Z","iopub.status.idle":"2022-07-17T06:47:45.129965Z","shell.execute_reply.started":"2022-07-17T06:47:45.122026Z","shell.execute_reply":"2022-07-17T06:47:45.129101Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import seaborn as sns","metadata":{"execution":{"iopub.status.busy":"2022-07-17T06:47:47.504408Z","iopub.execute_input":"2022-07-17T06:47:47.504808Z","iopub.status.idle":"2022-07-17T06:47:48.117557Z","shell.execute_reply.started":"2022-07-17T06:47:47.504763Z","shell.execute_reply":"2022-07-17T06:47:48.116489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Cross-check the dataframe and load it into pandas dataframe format","metadata":{}},{"cell_type":"code","source":"df_train = pd.read_csv('/kaggle/input/spaceship-titanic/train.csv')\ndf_test = pd.read_csv('/kaggle/input/spaceship-titanic/test.csv')\nex_submit = pd.read_csv('/kaggle/input/spaceship-titanic/sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2022-07-17T06:47:50.603722Z","iopub.execute_input":"2022-07-17T06:47:50.604092Z","iopub.status.idle":"2022-07-17T06:47:50.696929Z","shell.execute_reply.started":"2022-07-17T06:47:50.604062Z","shell.execute_reply":"2022-07-17T06:47:50.695821Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ex_submit","metadata":{"execution":{"iopub.status.busy":"2022-07-17T06:47:52.730044Z","iopub.execute_input":"2022-07-17T06:47:52.730459Z","iopub.status.idle":"2022-07-17T06:47:52.753555Z","shell.execute_reply.started":"2022-07-17T06:47:52.730424Z","shell.execute_reply":"2022-07-17T06:47:52.752293Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-17T06:47:55.293915Z","iopub.execute_input":"2022-07-17T06:47:55.294872Z","iopub.status.idle":"2022-07-17T06:47:55.318280Z","shell.execute_reply.started":"2022-07-17T06:47:55.294832Z","shell.execute_reply":"2022-07-17T06:47:55.317029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train[['deck', 'num', 'side']] = df_train['Cabin'].str.split('/', expand=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-17T06:47:57.506660Z","iopub.execute_input":"2022-07-17T06:47:57.507146Z","iopub.status.idle":"2022-07-17T06:47:57.540244Z","shell.execute_reply.started":"2022-07-17T06:47:57.507099Z","shell.execute_reply":"2022-07-17T06:47:57.538922Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ax = sns.countplot(x=\"deck\", hue=\"Transported\", data=df_train)","metadata":{"execution":{"iopub.status.busy":"2022-07-17T06:48:00.262188Z","iopub.execute_input":"2022-07-17T06:48:00.262563Z","iopub.status.idle":"2022-07-17T06:48:00.557294Z","shell.execute_reply.started":"2022-07-17T06:48:00.262533Z","shell.execute_reply":"2022-07-17T06:48:00.556178Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train","metadata":{"execution":{"iopub.status.busy":"2022-07-17T06:48:02.413336Z","iopub.execute_input":"2022-07-17T06:48:02.414058Z","iopub.status.idle":"2022-07-17T06:48:02.449438Z","shell.execute_reply.started":"2022-07-17T06:48:02.414020Z","shell.execute_reply":"2022-07-17T06:48:02.448301Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.describe()","metadata":{"execution":{"iopub.status.busy":"2022-07-17T06:48:05.166017Z","iopub.execute_input":"2022-07-17T06:48:05.166441Z","iopub.status.idle":"2022-07-17T06:48:05.206379Z","shell.execute_reply.started":"2022-07-17T06:48:05.166402Z","shell.execute_reply":"2022-07-17T06:48:05.205096Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.info()","metadata":{"execution":{"iopub.status.busy":"2022-07-17T06:48:07.878564Z","iopub.execute_input":"2022-07-17T06:48:07.879538Z","iopub.status.idle":"2022-07-17T06:48:07.907028Z","shell.execute_reply.started":"2022-07-17T06:48:07.879500Z","shell.execute_reply":"2022-07-17T06:48:07.905957Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test[['deck', 'num', 'side']] = df_test['Cabin'].str.split('/', expand=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-17T06:48:10.857283Z","iopub.execute_input":"2022-07-17T06:48:10.857773Z","iopub.status.idle":"2022-07-17T06:48:10.878073Z","shell.execute_reply.started":"2022-07-17T06:48:10.857729Z","shell.execute_reply":"2022-07-17T06:48:10.876886Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = df_train.dropna()","metadata":{"execution":{"iopub.status.busy":"2022-07-17T06:48:14.521420Z","iopub.execute_input":"2022-07-17T06:48:14.521823Z","iopub.status.idle":"2022-07-17T06:48:14.540617Z","shell.execute_reply.started":"2022-07-17T06:48:14.521787Z","shell.execute_reply":"2022-07-17T06:48:14.539572Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# EDA process","metadata":{}},{"cell_type":"markdown","source":"## check if target label data is balance or not?","metadata":{}},{"cell_type":"code","source":"df_train['Transported'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-07-17T06:48:19.067823Z","iopub.execute_input":"2022-07-17T06:48:19.068234Z","iopub.status.idle":"2022-07-17T06:48:19.077534Z","shell.execute_reply.started":"2022-07-17T06:48:19.068178Z","shell.execute_reply":"2022-07-17T06:48:19.076266Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Labels are balance. Now, let's check some correlation","metadata":{}},{"cell_type":"code","source":"pre_encode = df_train.drop(columns = ['Cabin', 'PassengerId', 'Name'])","metadata":{"execution":{"iopub.status.busy":"2022-07-17T06:48:21.166736Z","iopub.execute_input":"2022-07-17T06:48:21.167316Z","iopub.status.idle":"2022-07-17T06:48:21.172986Z","shell.execute_reply.started":"2022-07-17T06:48:21.167279Z","shell.execute_reply":"2022-07-17T06:48:21.172087Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pre_encode.dtypes","metadata":{"execution":{"iopub.status.busy":"2022-07-17T06:48:23.364652Z","iopub.execute_input":"2022-07-17T06:48:23.365234Z","iopub.status.idle":"2022-07-17T06:48:23.373251Z","shell.execute_reply.started":"2022-07-17T06:48:23.365189Z","shell.execute_reply":"2022-07-17T06:48:23.372171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pre_encode[\"num\"] = pre_encode[\"num\"].astype(str).astype(float)","metadata":{"execution":{"iopub.status.busy":"2022-07-17T06:49:06.585529Z","iopub.execute_input":"2022-07-17T06:49:06.585894Z","iopub.status.idle":"2022-07-17T06:49:06.594955Z","shell.execute_reply.started":"2022-07-17T06:49:06.585862Z","shell.execute_reply":"2022-07-17T06:49:06.593620Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pre_encode.dtypes","metadata":{"execution":{"iopub.status.busy":"2022-07-17T06:49:08.388952Z","iopub.execute_input":"2022-07-17T06:49:08.389768Z","iopub.status.idle":"2022-07-17T06:49:08.400681Z","shell.execute_reply.started":"2022-07-17T06:49:08.389717Z","shell.execute_reply":"2022-07-17T06:49:08.399236Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#let encode the data before check correlation\n\nencoded_df  = pd.get_dummies(pre_encode)","metadata":{"execution":{"iopub.status.busy":"2022-07-17T06:49:10.417099Z","iopub.execute_input":"2022-07-17T06:49:10.418045Z","iopub.status.idle":"2022-07-17T06:49:10.441394Z","shell.execute_reply.started":"2022-07-17T06:49:10.417996Z","shell.execute_reply":"2022-07-17T06:49:10.440501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"encoded_df","metadata":{"execution":{"iopub.status.busy":"2022-07-17T06:49:12.717840Z","iopub.execute_input":"2022-07-17T06:49:12.718981Z","iopub.status.idle":"2022-07-17T06:49:12.750695Z","shell.execute_reply.started":"2022-07-17T06:49:12.718943Z","shell.execute_reply":"2022-07-17T06:49:12.749561Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"column = encoded_df.columns\nfor i in column:\n    print(\"Null in column \", i, encoded_df[i].isnull().sum())","metadata":{"execution":{"iopub.status.busy":"2022-07-17T06:49:17.419888Z","iopub.execute_input":"2022-07-17T06:49:17.420929Z","iopub.status.idle":"2022-07-17T06:49:17.435883Z","shell.execute_reply.started":"2022-07-17T06:49:17.420887Z","shell.execute_reply":"2022-07-17T06:49:17.434923Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We need to split test set here\nthe test set need to be:\n* no null value\n* do not transformed","metadata":{}},{"cell_type":"code","source":"# data normalization\n\nfrom sklearn.preprocessing import StandardScaler\nscaler = StandardScaler()\ncontinues_column = encoded_df.describe().columns\nencoded_df[continues_column] = scaler.fit_transform(encoded_df[continues_column].to_numpy())","metadata":{"execution":{"iopub.status.busy":"2022-07-17T06:49:22.990458Z","iopub.execute_input":"2022-07-17T06:49:22.991253Z","iopub.status.idle":"2022-07-17T06:49:23.067844Z","shell.execute_reply.started":"2022-07-17T06:49:22.991186Z","shell.execute_reply":"2022-07-17T06:49:23.066743Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.impute import KNNImputer\nimputer = KNNImputer(n_neighbors=3)\nimputed_result = imputer.fit_transform(encoded_df)\nencoded_df = pd.DataFrame(imputed_result, columns = column)","metadata":{"execution":{"iopub.status.busy":"2022-07-17T06:49:24.945993Z","iopub.execute_input":"2022-07-17T06:49:24.946419Z","iopub.status.idle":"2022-07-17T06:49:25.050296Z","shell.execute_reply.started":"2022-07-17T06:49:24.946381Z","shell.execute_reply":"2022-07-17T06:49:25.049405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"column = encoded_df.columns\nfor i in column:\n    print(\"Null in column \", i, encoded_df[i].isnull().sum())","metadata":{"execution":{"iopub.status.busy":"2022-07-17T06:49:28.877775Z","iopub.execute_input":"2022-07-17T06:49:28.878485Z","iopub.status.idle":"2022-07-17T06:49:28.895194Z","shell.execute_reply.started":"2022-07-17T06:49:28.878431Z","shell.execute_reply":"2022-07-17T06:49:28.894396Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"encoded_df","metadata":{"execution":{"iopub.status.busy":"2022-07-17T06:49:33.482653Z","iopub.execute_input":"2022-07-17T06:49:33.483025Z","iopub.status.idle":"2022-07-17T06:49:33.519734Z","shell.execute_reply.started":"2022-07-17T06:49:33.482995Z","shell.execute_reply":"2022-07-17T06:49:33.518564Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"perform correlation checking","metadata":{}},{"cell_type":"code","source":"corr_label = encoded_df.corrwith(encoded_df['Transported'], axis=0, drop=False, method='pearson')","metadata":{"execution":{"iopub.status.busy":"2022-07-17T06:49:37.582562Z","iopub.execute_input":"2022-07-17T06:49:37.582926Z","iopub.status.idle":"2022-07-17T06:49:37.602843Z","shell.execute_reply.started":"2022-07-17T06:49:37.582897Z","shell.execute_reply":"2022-07-17T06:49:37.601710Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"corr_label.sort_values()","metadata":{"execution":{"iopub.status.busy":"2022-07-17T06:49:40.153762Z","iopub.execute_input":"2022-07-17T06:49:40.154298Z","iopub.status.idle":"2022-07-17T06:49:40.165759Z","shell.execute_reply.started":"2022-07-17T06:49:40.154251Z","shell.execute_reply":"2022-07-17T06:49:40.164340Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"It seem that Cryo sleep status show high correlation to the transported status.\n\nLet explore a bit more in this aspect.","metadata":{}},{"cell_type":"code","source":"ax = sns.countplot(x=\"CryoSleep\", hue=\"Transported\", data=df_train)","metadata":{"execution":{"iopub.status.busy":"2022-07-17T06:49:43.805706Z","iopub.execute_input":"2022-07-17T06:49:43.806324Z","iopub.status.idle":"2022-07-17T06:49:43.994746Z","shell.execute_reply.started":"2022-07-17T06:49:43.806289Z","shell.execute_reply":"2022-07-17T06:49:43.993676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"if cryoSleep status is True, it seem that the change for passenger will be transported is higher.","metadata":{}},{"cell_type":"markdown","source":"### data preparation for logistic regression","metadata":{}},{"cell_type":"code","source":"encoded_df","metadata":{"execution":{"iopub.status.busy":"2022-07-17T06:49:46.375483Z","iopub.execute_input":"2022-07-17T06:49:46.375921Z","iopub.status.idle":"2022-07-17T06:49:46.413193Z","shell.execute_reply.started":"2022-07-17T06:49:46.375883Z","shell.execute_reply":"2022-07-17T06:49:46.411924Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label = encoded_df['Transported']\nfeature = encoded_df.drop(columns = ['Transported'])\n ","metadata":{"execution":{"iopub.status.busy":"2022-07-17T06:49:49.579521Z","iopub.execute_input":"2022-07-17T06:49:49.580191Z","iopub.status.idle":"2022-07-17T06:49:49.586401Z","shell.execute_reply.started":"2022-07-17T06:49:49.580156Z","shell.execute_reply":"2022-07-17T06:49:49.585368Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature","metadata":{"execution":{"iopub.status.busy":"2022-07-17T06:49:51.535194Z","iopub.execute_input":"2022-07-17T06:49:51.536446Z","iopub.status.idle":"2022-07-17T06:49:51.568276Z","shell.execute_reply.started":"2022-07-17T06:49:51.536402Z","shell.execute_reply":"2022-07-17T06:49:51.567466Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.model_selection import KFold\nfrom sklearn.model_selection import cross_val_score\nfrom sklearn.metrics import accuracy_score, confusion_matrix, recall_score, roc_auc_score, precision_score\n\nX_train, X_test, y_train, y_test = train_test_split(feature, label, test_size=0.3,random_state=0)\n","metadata":{"execution":{"iopub.status.busy":"2022-07-17T06:49:54.504240Z","iopub.execute_input":"2022-07-17T06:49:54.505100Z","iopub.status.idle":"2022-07-17T06:49:54.516945Z","shell.execute_reply.started":"2022-07-17T06:49:54.505047Z","shell.execute_reply":"2022-07-17T06:49:54.516195Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pipe = Pipeline([('scaler', StandardScaler())\n                 ,('logis', LogisticRegression(random_state=0))])","metadata":{"execution":{"iopub.status.busy":"2022-07-17T06:49:58.255330Z","iopub.execute_input":"2022-07-17T06:49:58.255716Z","iopub.status.idle":"2022-07-17T06:49:58.261168Z","shell.execute_reply.started":"2022-07-17T06:49:58.255682Z","shell.execute_reply":"2022-07-17T06:49:58.259783Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cv = KFold(n_splits=10)\nscores = cross_val_score(pipe, X_train, y_train, cv = cv)","metadata":{"execution":{"iopub.status.busy":"2022-07-17T06:50:00.277285Z","iopub.execute_input":"2022-07-17T06:50:00.277975Z","iopub.status.idle":"2022-07-17T06:50:00.787964Z","shell.execute_reply.started":"2022-07-17T06:50:00.277923Z","shell.execute_reply":"2022-07-17T06:50:00.786672Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('average score', scores.mean())\nprint('std score', scores.std())","metadata":{"execution":{"iopub.status.busy":"2022-07-17T06:50:03.897940Z","iopub.execute_input":"2022-07-17T06:50:03.898316Z","iopub.status.idle":"2022-07-17T06:50:03.904268Z","shell.execute_reply.started":"2022-07-17T06:50:03.898286Z","shell.execute_reply":"2022-07-17T06:50:03.903479Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pipe.fit(X_train, y_train)","metadata":{"execution":{"iopub.status.busy":"2022-07-17T06:50:06.265707Z","iopub.execute_input":"2022-07-17T06:50:06.266071Z","iopub.status.idle":"2022-07-17T06:50:06.321465Z","shell.execute_reply.started":"2022-07-17T06:50:06.266040Z","shell.execute_reply":"2022-07-17T06:50:06.320332Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_hat = pipe.predict(X_test)","metadata":{"execution":{"iopub.status.busy":"2022-07-17T06:50:08.343185Z","iopub.execute_input":"2022-07-17T06:50:08.343597Z","iopub.status.idle":"2022-07-17T06:50:08.355511Z","shell.execute_reply.started":"2022-07-17T06:50:08.343565Z","shell.execute_reply":"2022-07-17T06:50:08.353774Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import confusion_matrix\n\nconfusion_matrix(y_test, y_hat)","metadata":{"execution":{"iopub.status.busy":"2022-07-17T06:50:10.378620Z","iopub.execute_input":"2022-07-17T06:50:10.379048Z","iopub.status.idle":"2022-07-17T06:50:10.391808Z","shell.execute_reply.started":"2022-07-17T06:50:10.379017Z","shell.execute_reply":"2022-07-17T06:50:10.391023Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.DataFrame(data=[accuracy_score(y_test, y_hat), recall_score(y_test, y_hat),\n                   precision_score(y_test, y_hat), roc_auc_score(y_test, y_hat)], \n             index=[\"accuracy\", \"recall\", \"precision\", \"roc_auc_score\"])","metadata":{"execution":{"iopub.status.busy":"2022-07-17T06:50:12.698253Z","iopub.execute_input":"2022-07-17T06:50:12.698840Z","iopub.status.idle":"2022-07-17T06:50:12.719871Z","shell.execute_reply.started":"2022-07-17T06:50:12.698796Z","shell.execute_reply":"2022-07-17T06:50:12.719011Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import scikitplot as skplt\nimport matplotlib.pyplot as plt\n\ny_probas = pipe.predict_proba(X_test)\nskplt.metrics.plot_roc(y_test, y_probas)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-17T06:50:15.132031Z","iopub.execute_input":"2022-07-17T06:50:15.132405Z","iopub.status.idle":"2022-07-17T06:50:15.398947Z","shell.execute_reply.started":"2022-07-17T06:50:15.132375Z","shell.execute_reply":"2022-07-17T06:50:15.397667Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### hyperparameter tuning","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import GridSearchCV\n\ngrid={\"C\":np.logspace(-3,3,7), \"penalty\":[\"l2\"], \"max_iter\" : [100]}# l1 lasso l2 ridge\nlogreg=LogisticRegression()\nlogreg_cv=GridSearchCV(logreg,grid,cv=10)\n\ncv_pipe =  Pipeline([('scaler', StandardScaler())\n                 ,('logis', logreg_cv)])\n\ncv_pipe.fit(X_train, y_train)\nprint(\"tuned hpyerparameters :(best parameters) \",logreg_cv.best_params_)\nprint(\"accuracy :\",logreg_cv.best_score_)\n","metadata":{"execution":{"iopub.status.busy":"2022-07-17T06:50:19.286478Z","iopub.execute_input":"2022-07-17T06:50:19.286880Z","iopub.status.idle":"2022-07-17T06:50:21.390322Z","shell.execute_reply.started":"2022-07-17T06:50:19.286847Z","shell.execute_reply":"2022-07-17T06:50:21.389070Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"logreg = LogisticRegression(C=100,penalty = 'l2')\ntuned_pipe = Pipeline([('scaler', StandardScaler())\n                 ,('logis', logreg)])\ntuned_pipe.fit(X_train,y_train)","metadata":{"execution":{"iopub.status.busy":"2022-07-17T06:50:25.345975Z","iopub.execute_input":"2022-07-17T06:50:25.346364Z","iopub.status.idle":"2022-07-17T06:50:25.396516Z","shell.execute_reply.started":"2022-07-17T06:50:25.346333Z","shell.execute_reply":"2022-07-17T06:50:25.395303Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_probas = tuned_pipe.predict_proba(X_test)\nskplt.metrics.plot_roc(y_test, y_probas)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-17T06:50:28.766765Z","iopub.execute_input":"2022-07-17T06:50:28.768010Z","iopub.status.idle":"2022-07-17T06:50:28.990880Z","shell.execute_reply.started":"2022-07-17T06:50:28.767968Z","shell.execute_reply":"2022-07-17T06:50:28.990137Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = tuned_pipe.predict(X_test)","metadata":{"execution":{"iopub.status.busy":"2022-07-17T06:50:31.880825Z","iopub.execute_input":"2022-07-17T06:50:31.881693Z","iopub.status.idle":"2022-07-17T06:50:31.895075Z","shell.execute_reply.started":"2022-07-17T06:50:31.881657Z","shell.execute_reply":"2022-07-17T06:50:31.893340Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\npd.DataFrame(data=[accuracy_score(y_test, preds), recall_score(y_test, preds),\n                   precision_score(y_test, preds), roc_auc_score(y_test, preds)], \n             index=[\"accuracy\", \"recall\", \"precision\", \"roc_auc_score\"])","metadata":{"execution":{"iopub.status.busy":"2022-07-17T06:50:34.597977Z","iopub.execute_input":"2022-07-17T06:50:34.598373Z","iopub.status.idle":"2022-07-17T06:50:34.616579Z","shell.execute_reply.started":"2022-07-17T06:50:34.598339Z","shell.execute_reply":"2022-07-17T06:50:34.615800Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Not so much improve, let's try tree-based model","metadata":{}},{"cell_type":"markdown","source":"## Decision Tree classification","metadata":{}},{"cell_type":"code","source":"from sklearn.ensemble import RandomForestClassifier\n\n\ntree_para = {'criterion':['gini','entropy'],'max_depth':[4,5,6,7,8,9,10,11,12,15,20]}\ntree_clf_cv = GridSearchCV(RandomForestClassifier(), tree_para, cv=5)\ntree_pipe = Pipeline([('scaler', StandardScaler())\n                 ,('rf_cv', tree_clf_cv)])\ntree_pipe.fit(X_train, y_train)\nprint(\"tuned hpyerparameters :(best parameters) \",tree_clf_cv.best_params_)\nprint(\"accuracy :\",tree_clf_cv.best_score_)","metadata":{"execution":{"iopub.status.busy":"2022-07-17T06:51:44.958263Z","iopub.execute_input":"2022-07-17T06:51:44.958624Z","iopub.status.idle":"2022-07-17T06:52:34.909497Z","shell.execute_reply.started":"2022-07-17T06:51:44.958596Z","shell.execute_reply":"2022-07-17T06:52:34.908281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rf = RandomForestClassifier(criterion = 'entropy', max_depth= 12)\ntree_pipe = Pipeline([('scaler', StandardScaler())\n                 ,('decision_tree', rf)])\ntree_pipe.fit(X_train,y_train)","metadata":{"execution":{"iopub.status.busy":"2022-07-17T06:55:05.647756Z","iopub.execute_input":"2022-07-17T06:55:05.648153Z","iopub.status.idle":"2022-07-17T06:55:06.259468Z","shell.execute_reply.started":"2022-07-17T06:55:05.648119Z","shell.execute_reply":"2022-07-17T06:55:06.258523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_probas = tree_pipe.predict_proba(X_test)\nskplt.metrics.plot_roc(y_test, y_probas)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-17T06:55:08.251923Z","iopub.execute_input":"2022-07-17T06:55:08.252302Z","iopub.status.idle":"2022-07-17T06:55:08.523510Z","shell.execute_reply.started":"2022-07-17T06:55:08.252269Z","shell.execute_reply":"2022-07-17T06:55:08.522458Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = tree_pipe.predict(X_test)\n\n\npd.DataFrame(data=[accuracy_score(y_test, preds), recall_score(y_test, preds),\n                   precision_score(y_test, preds), roc_auc_score(y_test, preds)], \n             index=[\"accuracy\", \"recall\", \"precision\", \"roc_auc_score\"])","metadata":{"execution":{"iopub.status.busy":"2022-07-17T06:55:14.304341Z","iopub.execute_input":"2022-07-17T06:55:14.304842Z","iopub.status.idle":"2022-07-17T06:55:14.378744Z","shell.execute_reply.started":"2022-07-17T06:55:14.304798Z","shell.execute_reply":"2022-07-17T06:55:14.377497Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### xgboost classifier","metadata":{}},{"cell_type":"code","source":"import xgboost as xgb\n\ncv = KFold(n_splits=10)\nXGB = xgb.XGBClassifier()\npipe_xgb = Pipeline([('scaler', StandardScaler())\n                 ,('XGB', XGB)])\n\nscores = cross_val_score(pipe_xgb, X_train, y_train, cv = cv)","metadata":{"execution":{"iopub.status.busy":"2022-07-17T06:55:20.630008Z","iopub.execute_input":"2022-07-17T06:55:20.631037Z","iopub.status.idle":"2022-07-17T06:55:27.922097Z","shell.execute_reply.started":"2022-07-17T06:55:20.630982Z","shell.execute_reply":"2022-07-17T06:55:27.921227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('average score', scores.mean())\nprint('std score', scores.std())","metadata":{"execution":{"iopub.status.busy":"2022-07-17T06:55:33.939541Z","iopub.execute_input":"2022-07-17T06:55:33.939938Z","iopub.status.idle":"2022-07-17T06:55:33.946129Z","shell.execute_reply.started":"2022-07-17T06:55:33.939905Z","shell.execute_reply":"2022-07-17T06:55:33.945003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pipe_xgb.fit(X_train, y_train)","metadata":{"execution":{"iopub.status.busy":"2022-07-17T06:55:36.405920Z","iopub.execute_input":"2022-07-17T06:55:36.406299Z","iopub.status.idle":"2022-07-17T06:55:37.270484Z","shell.execute_reply.started":"2022-07-17T06:55:36.406268Z","shell.execute_reply":"2022-07-17T06:55:37.269293Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_probas = pipe_xgb.predict_proba(X_test)\nskplt.metrics.plot_roc(y_test, y_probas)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-17T06:55:39.968819Z","iopub.execute_input":"2022-07-17T06:55:39.969578Z","iopub.status.idle":"2022-07-17T06:55:40.201316Z","shell.execute_reply.started":"2022-07-17T06:55:39.969540Z","shell.execute_reply":"2022-07-17T06:55:40.200231Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = pipe_xgb.predict(X_test)\n\n\npd.DataFrame(data=[accuracy_score(y_test, preds), recall_score(y_test, preds),\n                   precision_score(y_test, preds), roc_auc_score(y_test, preds)], \n             index=[\"accuracy\", \"recall\", \"precision\", \"roc_auc_score\"])","metadata":{"execution":{"iopub.status.busy":"2022-07-17T06:55:43.386849Z","iopub.execute_input":"2022-07-17T06:55:43.387593Z","iopub.status.idle":"2022-07-17T06:55:43.415955Z","shell.execute_reply.started":"2022-07-17T06:55:43.387552Z","shell.execute_reply":"2022-07-17T06:55:43.415131Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# test with test set","metadata":{}},{"cell_type":"code","source":"df_test.dtypes","metadata":{"execution":{"iopub.status.busy":"2022-07-17T06:56:00.903515Z","iopub.execute_input":"2022-07-17T06:56:00.904013Z","iopub.status.idle":"2022-07-17T06:56:00.913120Z","shell.execute_reply.started":"2022-07-17T06:56:00.903969Z","shell.execute_reply":"2022-07-17T06:56:00.912103Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pre_encode_test = df_test.drop(columns = ['PassengerId', 'Cabin', 'Name'])","metadata":{"execution":{"iopub.status.busy":"2022-07-17T06:56:04.664658Z","iopub.execute_input":"2022-07-17T06:56:04.665075Z","iopub.status.idle":"2022-07-17T06:56:04.672525Z","shell.execute_reply.started":"2022-07-17T06:56:04.665042Z","shell.execute_reply":"2022-07-17T06:56:04.671732Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pre_encode_test","metadata":{"execution":{"iopub.status.busy":"2022-07-17T06:56:07.461279Z","iopub.execute_input":"2022-07-17T06:56:07.461930Z","iopub.status.idle":"2022-07-17T06:56:07.492643Z","shell.execute_reply.started":"2022-07-17T06:56:07.461892Z","shell.execute_reply":"2022-07-17T06:56:07.491555Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pre_encode_test[\"num\"] = pre_encode_test[\"num\"].astype(float)","metadata":{"execution":{"iopub.status.busy":"2022-07-17T06:56:11.232094Z","iopub.execute_input":"2022-07-17T06:56:11.232496Z","iopub.status.idle":"2022-07-17T06:56:11.239301Z","shell.execute_reply.started":"2022-07-17T06:56:11.232461Z","shell.execute_reply":"2022-07-17T06:56:11.238145Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pre_encode_test.dtypes","metadata":{"execution":{"iopub.status.busy":"2022-07-17T06:56:13.209118Z","iopub.execute_input":"2022-07-17T06:56:13.209668Z","iopub.status.idle":"2022-07-17T06:56:13.221484Z","shell.execute_reply.started":"2022-07-17T06:56:13.209617Z","shell.execute_reply":"2022-07-17T06:56:13.220295Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"encoded_df_test  = pd.get_dummies(pre_encode_test)","metadata":{"execution":{"iopub.status.busy":"2022-07-17T06:56:15.316376Z","iopub.execute_input":"2022-07-17T06:56:15.316771Z","iopub.status.idle":"2022-07-17T06:56:15.335540Z","shell.execute_reply.started":"2022-07-17T06:56:15.316737Z","shell.execute_reply":"2022-07-17T06:56:15.334702Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"encoded_df_test","metadata":{"execution":{"iopub.status.busy":"2022-07-17T06:56:17.338124Z","iopub.execute_input":"2022-07-17T06:56:17.338519Z","iopub.status.idle":"2022-07-17T06:56:17.375362Z","shell.execute_reply.started":"2022-07-17T06:56:17.338489Z","shell.execute_reply":"2022-07-17T06:56:17.374274Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"encoded_value = scaler.transform(encoded_df_test)","metadata":{"execution":{"iopub.status.busy":"2022-07-17T06:56:20.051261Z","iopub.execute_input":"2022-07-17T06:56:20.052050Z","iopub.status.idle":"2022-07-17T06:56:20.062360Z","shell.execute_reply.started":"2022-07-17T06:56:20.052002Z","shell.execute_reply":"2022-07-17T06:56:20.061311Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"encoded_value","metadata":{"execution":{"iopub.status.busy":"2022-07-17T06:56:22.917135Z","iopub.execute_input":"2022-07-17T06:56:22.917613Z","iopub.status.idle":"2022-07-17T06:56:22.926106Z","shell.execute_reply.started":"2022-07-17T06:56:22.917574Z","shell.execute_reply":"2022-07-17T06:56:22.925037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"columns = encoded_df_test.columns\nencoded_df_test = pd.DataFrame(encoded_value, columns = columns)","metadata":{"execution":{"iopub.status.busy":"2022-07-17T06:56:25.288469Z","iopub.execute_input":"2022-07-17T06:56:25.288853Z","iopub.status.idle":"2022-07-17T06:56:25.294074Z","shell.execute_reply.started":"2022-07-17T06:56:25.288821Z","shell.execute_reply":"2022-07-17T06:56:25.292968Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"encoded_df_test","metadata":{"execution":{"iopub.status.busy":"2022-07-17T06:56:27.382339Z","iopub.execute_input":"2022-07-17T06:56:27.382985Z","iopub.status.idle":"2022-07-17T06:56:27.414492Z","shell.execute_reply.started":"2022-07-17T06:56:27.382949Z","shell.execute_reply":"2022-07-17T06:56:27.413734Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"column = encoded_df_test.columns\nfor i in column:\n    print(\"Null in column \", i, encoded_df_test[i].isnull().sum())","metadata":{"execution":{"iopub.status.busy":"2022-07-17T06:56:30.408680Z","iopub.execute_input":"2022-07-17T06:56:30.409177Z","iopub.status.idle":"2022-07-17T06:56:30.430946Z","shell.execute_reply.started":"2022-07-17T06:56:30.409120Z","shell.execute_reply":"2022-07-17T06:56:30.429297Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.impute import KNNImputer\nimputer = KNNImputer(n_neighbors=3)\nimputed_result = imputer.fit_transform(encoded_df_test)\nfeature = pd.DataFrame(imputed_result, columns = column)","metadata":{"execution":{"iopub.status.busy":"2022-07-17T06:56:33.348861Z","iopub.execute_input":"2022-07-17T06:56:33.349970Z","iopub.status.idle":"2022-07-17T06:56:33.670979Z","shell.execute_reply.started":"2022-07-17T06:56:33.349927Z","shell.execute_reply":"2022-07-17T06:56:33.669887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature","metadata":{"execution":{"iopub.status.busy":"2022-07-17T06:56:35.965973Z","iopub.execute_input":"2022-07-17T06:56:35.966347Z","iopub.status.idle":"2022-07-17T06:56:35.999372Z","shell.execute_reply.started":"2022-07-17T06:56:35.966316Z","shell.execute_reply":"2022-07-17T06:56:35.998414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in feature.columns:\n    print(\"Null in column \", i, feature[i].isnull().sum())","metadata":{"execution":{"iopub.status.busy":"2022-07-17T06:56:39.915237Z","iopub.execute_input":"2022-07-17T06:56:39.916207Z","iopub.status.idle":"2022-07-17T06:56:39.931523Z","shell.execute_reply.started":"2022-07-17T06:56:39.916164Z","shell.execute_reply":"2022-07-17T06:56:39.930368Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_hat_test = tree_pipe.predict(feature)","metadata":{"execution":{"iopub.status.busy":"2022-07-17T06:56:47.832558Z","iopub.execute_input":"2022-07-17T06:56:47.832929Z","iopub.status.idle":"2022-07-17T06:56:47.923149Z","shell.execute_reply.started":"2022-07-17T06:56:47.832899Z","shell.execute_reply":"2022-07-17T06:56:47.922144Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test['Transported'] = y_hat_test","metadata":{"execution":{"iopub.status.busy":"2022-07-17T06:56:50.320883Z","iopub.execute_input":"2022-07-17T06:56:50.321302Z","iopub.status.idle":"2022-07-17T06:56:50.327115Z","shell.execute_reply.started":"2022-07-17T06:56:50.321266Z","shell.execute_reply":"2022-07-17T06:56:50.325774Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"summit_result = df_test[['PassengerId', 'Transported']]","metadata":{"execution":{"iopub.status.busy":"2022-07-17T06:56:53.441159Z","iopub.execute_input":"2022-07-17T06:56:53.441560Z","iopub.status.idle":"2022-07-17T06:56:53.450422Z","shell.execute_reply.started":"2022-07-17T06:56:53.441527Z","shell.execute_reply":"2022-07-17T06:56:53.449628Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"summit_result['Transported'] = summit_result['Transported'].astype(bool)","metadata":{"execution":{"iopub.status.busy":"2022-07-17T06:56:55.541569Z","iopub.execute_input":"2022-07-17T06:56:55.541949Z","iopub.status.idle":"2022-07-17T06:56:55.548972Z","shell.execute_reply.started":"2022-07-17T06:56:55.541916Z","shell.execute_reply":"2022-07-17T06:56:55.547788Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"summit_result.dtypes","metadata":{"execution":{"iopub.status.busy":"2022-07-17T06:56:58.214346Z","iopub.execute_input":"2022-07-17T06:56:58.214741Z","iopub.status.idle":"2022-07-17T06:56:58.222865Z","shell.execute_reply.started":"2022-07-17T06:56:58.214710Z","shell.execute_reply":"2022-07-17T06:56:58.221821Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"summit_result.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-07-17T06:57:00.458245Z","iopub.execute_input":"2022-07-17T06:57:00.459434Z","iopub.status.idle":"2022-07-17T06:57:00.474670Z","shell.execute_reply.started":"2022-07-17T06:57:00.459382Z","shell.execute_reply":"2022-07-17T06:57:00.473826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}