{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"https://www.kaggle.com/competitions/titanic/overview\n","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport tensorflow as tf\n\nfrom sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay, f1_score, roc_auc_score\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.preprocessing import StandardScaler, OneHotEncoder, RobustScaler\nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.decomposition import PCA\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.linear_model import LogisticRegression\nfrom xgboost import XGBClassifier, XGBRFClassifier\nfrom sklearn.metrics import accuracy_score","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:27:34.411421Z","iopub.execute_input":"2022-08-05T09:27:34.411922Z","iopub.status.idle":"2022-08-05T09:27:42.110399Z","shell.execute_reply.started":"2022-08-05T09:27:34.411801Z","shell.execute_reply":"2022-08-05T09:27:42.109486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv('../input/titanic/train.csv', index_col='PassengerId')\ntest_df = pd.read_csv('../input/titanic/test.csv', index_col='PassengerId')\n","metadata":{"execution":{"iopub.status.busy":"2022-08-05T09:28:05.397805Z","iopub.execute_input":"2022-08-05T09:28:05.398221Z","iopub.status.idle":"2022-08-05T09:28:05.444263Z","shell.execute_reply.started":"2022-08-05T09:28:05.398189Z","shell.execute_reply":"2022-08-05T09:28:05.442707Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.info()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.nunique()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Removing Columns\nName\n\nTicket\n\nCabin: only 204 non null values and 147 distinct values","metadata":{}},{"cell_type":"code","source":"train_df.drop(['Name', 'Ticket', 'Cabin'], axis=1, inplace=True)\ntest_df.drop(['Name', 'Ticket', 'Cabin'], axis=1, inplace=True)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.info()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_cols = train_df.drop(['Survived'], axis=1).select_dtypes('number').columns","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_cols","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cat_cols = train_df.select_dtypes('object').columns\ncat_cols","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Correlation Matrix","metadata":{}},{"cell_type":"code","source":"corr_mat = train_df.corr()\n\nf = plt.figure(figsize=(19, 15))\nplt.matshow(corr_mat, fignum=f.number)\nplt.xticks(range(corr_mat.shape[1]), corr_mat.columns, fontsize=14, rotation=45)\nplt.yticks(range(corr_mat.shape[1]), corr_mat.columns, fontsize=14)\ncb = plt.colorbar()\ncb.ax.tick_params(labelsize=14)\nplt.title('Correlation Matrix', fontsize=16);","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def subplot_hist(df, cols, no_cols_plot=3, figsize=(15,6)):\n    \n#     plt.figure(figsize=figsize)\n    fig, ax = plt.subplots(15,no_cols_plot, figsize=(15,10))\n\n    for i in range(len(cols)):\n\n        plt.subplot(np.ceil(len(cols)/no_cols_plot).astype(int), no_cols_plot, i+1)\n        sns.histplot(x=df[cols[i]])\n        plt.title(cols[i])\n        \ndef subplot_box_num(df, cols, no_cols_plot=3, figsize=(15,6)):\n    \n#     plt.figure(figsize=figsize)\n    fig, ax = plt.subplots(15,no_cols_plot, figsize=(15,10))\n\n    for i in range(len(cols)):\n\n        plt.subplot(np.ceil(len(cols)/no_cols_plot).astype(int), no_cols_plot, i+1)\n        sns.boxplot(x=df[cols[i]])\n        plt.title(cols[i]) ","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nsubplot_hist(train_df, train_df.columns, no_cols_plot=5, figsize=(15,50))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preprocessing Pipeline","metadata":{}},{"cell_type":"code","source":"cat_transformer = Pipeline(steps=[\n    ('imputer', SimpleImputer(strategy='most_frequent')),\n    ('oh_encoder', OneHotEncoder())\n])\n\nnum_transformer = Pipeline(steps=[\n    ('imputer', SimpleImputer(strategy='median')),\n    ('scaler', StandardScaler()),\n#     ('pca', PCA(0.9))\n])\n\npreprocessor = ColumnTransformer([\n    ('num', num_transformer, num_cols),\n    ('cat', cat_transformer, cat_cols)\n])\n\n# preprocessor = Pipeline(steps=[\n#     ('transformer', preprocessor_cols),\n#     ('pca', PCA())\n# ])","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preprocessor","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train / Validation Split","metadata":{}},{"cell_type":"code","source":"X_full = train_df.drop('Survived', axis=1)\ny_full = train_df['Survived']\n\nX_train, X_valid, y_train, y_valid = train_test_split(X_full,\n                                                     y_full,\n                                                     train_size=0.7,\n                                                     random_state=0)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train_trans = preprocessor.fit_transform(X_train)\nX_valid_trans = preprocessor.transform(X_valid)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Logistic Regression","metadata":{}},{"cell_type":"code","source":"%%time\n\nlr_pipeline = Pipeline(steps=[\n    ('preprocessor', preprocessor),\n    ('lr', LogisticRegression())\n])\n\nlr_pipeline\n\nlr_pipeline.fit(X_train, y_train)\n\ny_hat_train_lr = lr_pipeline.predict(X_train)\ny_hat_valid_lr = lr_pipeline.predict(X_valid)\n\nacc_train_lr = accuracy_score(y_train, y_hat_train_lr)\nacc_valid_lr = accuracy_score(y_valid, y_hat_valid_lr)\n\nprint(f\"Training Data Accuracy:   {acc_train_lr:.2%}\")\nprint(f\"Validation Data Accuracy: {acc_valid_lr:.2%}\")\n\ncm_train_lr = confusion_matrix(y_train, y_hat_train_lr)\ncm_valid_lr = confusion_matrix(y_valid, y_hat_valid_lr)\n\ndisp = ConfusionMatrixDisplay(confusion_matrix=cm_train_lr,\n                              display_labels=lr_pipeline.classes_)\ndisp.plot()\n\nplt.show()\n\ndisp = ConfusionMatrixDisplay(confusion_matrix=cm_valid_lr,\n                              display_labels=lr_pipeline.classes_)\ndisp.plot()\n\nplt.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# XGBClassifier","metadata":{}},{"cell_type":"code","source":"%%time\n\nxgb_pipeline = Pipeline(steps=[\n    ('preprocessor', preprocessor),\n    ('xgb', XGBClassifier(verbose=1,\n#                          eval_set= [(X_valid_trans, y_valid)],\n#                           early_stopping_rounds=50,\n#                           xgb__eval_metric='error',\n                         ))\n])\n\n\n\nxgb_pipeline.fit(X_train, y_train,\n                xgb__eval_set = [(X_valid_trans, y_valid)],\n                xgb__early_stopping_rounds=50,\n                xgb__eval_metric='error'\n                )\n\ny_hat_train_xgb = xgb_pipeline.predict(X_train)\ny_hat_valid_xgb = xgb_pipeline.predict(X_valid)\n\nacc_train_xgb = accuracy_score(y_train, y_hat_train_xgb)\nacc_valid_xgb = accuracy_score(y_valid, y_hat_valid_xgb)\n\nprint(f\"Training Data Accuracy:   {acc_train_xgb:.2%}\")\nprint(f\"Validation Data Accuracy: {acc_valid_xgb:.2%}\")\n\ncm_train_xgb = confusion_matrix(y_train, y_hat_train_xgb)\ncm_valid_xgb = confusion_matrix(y_valid, y_hat_valid_xgb)\n\ndisp = ConfusionMatrixDisplay(confusion_matrix=cm_train_xgb,\n                              display_labels=lr_pipeline.classes_)\ndisp.plot()\n\nplt.show()\n\ndisp = ConfusionMatrixDisplay(confusion_matrix=cm_valid_xgb,\n                              display_labels=lr_pipeline.classes_)\ndisp.plot()\n\nplt.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"xgb_pipeline.steps[-1]","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# XGBRFClassifier","metadata":{}},{"cell_type":"code","source":"%%time\n\nxgbrf_pipeline = Pipeline(steps=[\n    ('preprocessor', preprocessor),\n    ('xgbrf', XGBRFClassifier())\n])\n\n\n\nxgbrf_pipeline.fit(X_train, y_train)\n\ny_hat_train_xgbrf = xgbrf_pipeline.predict(X_train)\ny_hat_valid_xgbrf = xgbrf_pipeline.predict(X_valid)\n\nacc_train_xgbrf = accuracy_score(y_train, y_hat_train_xgbrf)\nacc_valid_xgbrf = accuracy_score(y_valid, y_hat_valid_xgbrf)\n\nprint(f\"Training Data Accuracy:   {acc_train_xgbrf:.2%}\")\nprint(f\"Validation Data Accuracy: {acc_valid_xgbrf:.2%}\")\n\ncm_train_xgbrf = confusion_matrix(y_train, y_hat_train_xgbrf)\ncm_valid_xgbrf = confusion_matrix(y_valid, y_hat_valid_xgbrf)\n\ndisp = ConfusionMatrixDisplay(confusion_matrix=cm_train_xgbrf,\n                              display_labels=lr_pipeline.classes_)\ndisp.plot()\n\nplt.show()\n\ndisp = ConfusionMatrixDisplay(confusion_matrix=cm_valid_xgbrf,\n                              display_labels=lr_pipeline.classes_)\ndisp.plot()\n\nplt.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Neural Network Pipeline","metadata":{}},{"cell_type":"code","source":"X_train_trans[1]","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nnn_model = tf.keras.models.Sequential([\n    tf.keras.layers.Input(shape=X_train_trans.shape[1]),\n    tf.keras.layers.Dense(8, 'relu', kernel_regularizer=tf.keras.regularizers.L2(l2=0.01)),\n    tf.keras.layers.Dropout(0.2),\n    tf.keras.layers.Dense(16, 'relu', kernel_regularizer=tf.keras.regularizers.L2(l2=0.01)),\n    tf.keras.layers.Dropout(0.2),\n    tf.keras.layers.Dense(8, 'relu', kernel_regularizer=tf.keras.regularizers.L2(l2=0.01)),\n    tf.keras.layers.Dropout(0.2),\n    tf.keras.layers.Dense(1, 'sigmoid')\n\n])\n\nnn_model.summary()\n\nnn_model.compile(loss='BinaryCrossentropy',\n                 optimizer='adam',\n                 metrics=['accuracy']\n                )\n\nnn_pipeline = Pipeline([\n    ('preprocessor', preprocessor),\n    ('nn_model', nn_model)\n])\n\nhistory = nn_pipeline.fit(X_train, y_train,\n                         nn_model__epochs=100,\n                         nn_model__verbose=0,\n                         nn_model__validation_data=(X_valid_trans, y_valid))\n\ny_hat_train_nn = (nn_pipeline.predict(X_train) > 0.5).astype('int')\ny_hat_valid_nn = (nn_pipeline.predict(X_valid) > 0.5).astype('int')\n\nacc_train_nn = accuracy_score(y_train, y_hat_train_nn)\nacc_valid_nn = accuracy_score(y_valid, y_hat_valid_nn)\n\nprint(f\"Training Data Accuracy:   {acc_train_nn:.2%}\")\nprint(f\"Validation Data Accuracy: {acc_valid_nn:.2%}\")\n\ncm_train_nn = confusion_matrix(y_train, y_hat_train_nn)\ncm_valid_nn = confusion_matrix(y_valid, y_hat_valid_nn)\n\ndisp = ConfusionMatrixDisplay(confusion_matrix=cm_train_nn,\n                              display_labels=lr_pipeline.classes_)\ndisp.plot()\n\nplt.show()\n\ndisp = ConfusionMatrixDisplay(confusion_matrix=cm_valid_nn,\n                              display_labels=lr_pipeline.classes_)\ndisp.plot()\n\nplt.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Training Data Accuracy:   {acc_train_nn:.2%}\")\nprint(f\"Validation Data Accuracy: {acc_valid_nn:.2%}\")\n\ncm_train_nn = confusion_matrix(y_train, y_hat_train_nn)\ncm_valid_nn = confusion_matrix(y_valid, y_hat_valid_nn)\n\ndisp = ConfusionMatrixDisplay(confusion_matrix=cm_train_nn,\n                              display_labels=lr_pipeline.classes_)\ndisp.plot()\n\nplt.show()\n\ndisp = ConfusionMatrixDisplay(confusion_matrix=cm_valid_nn,\n                              display_labels=lr_pipeline.classes_)\ndisp.plot()\n\nplt.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission","metadata":{}},{"cell_type":"code","source":"def create_sub_file(filename,  y_col):\n    filename = filename + '.csv'\n    submission = pd.DataFrame({\n        'PassengerId': test_df.index,\n        'Survived': y_col\n    })\n    submission.to_csv(filename, index=False)\n    display(pd.read_csv(filename, index_col='PassengerId'))\n    return","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_hat_test_lr = lr_pipeline.predict(test_df)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"create_sub_file('titanic_lr', y_hat_test_lr)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_hat_test_xgb = xgb_pipeline.predict(test_df)\ncreate_sub_file('titanic_xgb', y_hat_test_xgb)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# (nn_pipeline.predict(X_train) > 0.5).astype('int')\ny_hat_test_nn = (nn_pipeline.predict(test_df) > 0.5).astype('int')\ncreate_sub_file('titanic_nn', y_hat_test_nn.flatten())","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Logistic Regression Validation Accuracy:        {acc_valid_lr:.2%}\")\nprint(f\"XGBClassifier Validation Accuracy:              {acc_valid_xgb:.2%}\")\nprint(f\"XGBRFClassifier Regression Validation Accuracy: {acc_valid_xgbrf:.2%}\")\nprint(f\"Neural Network Validation Accuracy:             {acc_valid_nn:.2%}\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}