{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nfrom sklearn.svm import SVC\nfrom sklearn.tree import DecisionTreeClassifier\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.ensemble import AdaBoostClassifier\nfrom xgboost import XGBClassifier\nfrom xgboost import XGBRFClassifier\nfrom sklearn.neural_network import MLPClassifier\n\nfrom sklearn.model_selection import cross_val_score\nfrom sklearn.model_selection import GridSearchCV\n# from sklearn.metrics import accuracy_score","metadata":{"execution":{"iopub.status.busy":"2022-08-01T08:53:37.792093Z","iopub.execute_input":"2022-08-01T08:53:37.792559Z","iopub.status.idle":"2022-08-01T08:53:37.800929Z","shell.execute_reply.started":"2022-08-01T08:53:37.792520Z","shell.execute_reply":"2022-08-01T08:53:37.799698Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('../input/titanic/train.csv')\ntest = pd.read_csv('../input/titanic/test.csv')","metadata":{"execution":{"iopub.status.busy":"2022-08-01T08:53:37.811417Z","iopub.execute_input":"2022-08-01T08:53:37.811956Z","iopub.status.idle":"2022-08-01T08:53:37.833266Z","shell.execute_reply.started":"2022-08-01T08:53:37.811914Z","shell.execute_reply":"2022-08-01T08:53:37.831795Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Look at data","metadata":{}},{"cell_type":"markdown","source":"### Show data","metadata":{}},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-01T08:53:37.835449Z","iopub.execute_input":"2022-08-01T08:53:37.836291Z","iopub.status.idle":"2022-08-01T08:53:37.856757Z","shell.execute_reply.started":"2022-08-01T08:53:37.836243Z","shell.execute_reply":"2022-08-01T08:53:37.855898Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-01T08:53:37.863815Z","iopub.execute_input":"2022-08-01T08:53:37.864371Z","iopub.status.idle":"2022-08-01T08:53:37.881345Z","shell.execute_reply.started":"2022-08-01T08:53:37.864337Z","shell.execute_reply":"2022-08-01T08:53:37.880551Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.info()\nprint(\"\\n\")\ntest.info()","metadata":{"execution":{"iopub.status.busy":"2022-08-01T08:53:37.896686Z","iopub.execute_input":"2022-08-01T08:53:37.897646Z","iopub.status.idle":"2022-08-01T08:53:37.919669Z","shell.execute_reply.started":"2022-08-01T08:53:37.897606Z","shell.execute_reply":"2022-08-01T08:53:37.918561Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Null values","metadata":{}},{"cell_type":"code","source":"plt.subplots(figsize=(12,8))\nsns.heatmap(train.isnull(), cmap='rocket_r', cbar=False)","metadata":{"execution":{"iopub.status.busy":"2022-08-01T08:53:37.925699Z","iopub.execute_input":"2022-08-01T08:53:37.926826Z","iopub.status.idle":"2022-08-01T08:53:38.463690Z","shell.execute_reply.started":"2022-08-01T08:53:37.926773Z","shell.execute_reply":"2022-08-01T08:53:38.462671Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Observations**\n* statistics for train and test set are similar\n* drop Cabin becaouse of nulls\n","metadata":{}},{"cell_type":"markdown","source":"# Analyzing","metadata":{}},{"cell_type":"markdown","source":"### Numerical values","metadata":{}},{"cell_type":"code","source":"train.describe()","metadata":{"execution":{"iopub.status.busy":"2022-08-01T08:53:38.465339Z","iopub.execute_input":"2022-08-01T08:53:38.466544Z","iopub.status.idle":"2022-08-01T08:53:38.503093Z","shell.execute_reply.started":"2022-08-01T08:53:38.466499Z","shell.execute_reply":"2022-08-01T08:53:38.502006Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.describe()","metadata":{"execution":{"iopub.status.busy":"2022-08-01T08:53:38.505833Z","iopub.execute_input":"2022-08-01T08:53:38.506880Z","iopub.status.idle":"2022-08-01T08:53:38.543478Z","shell.execute_reply.started":"2022-08-01T08:53:38.506815Z","shell.execute_reply":"2022-08-01T08:53:38.542257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.drop(\"PassengerId\", axis=1).hist(figsize=(10,8))","metadata":{"execution":{"iopub.status.busy":"2022-08-01T08:53:38.546102Z","iopub.execute_input":"2022-08-01T08:53:38.547010Z","iopub.status.idle":"2022-08-01T08:53:39.352157Z","shell.execute_reply.started":"2022-08-01T08:53:38.546954Z","shell.execute_reply":"2022-08-01T08:53:39.350830Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.corr()","metadata":{"execution":{"iopub.status.busy":"2022-08-01T08:53:39.353856Z","iopub.execute_input":"2022-08-01T08:53:39.354310Z","iopub.status.idle":"2022-08-01T08:53:39.371465Z","shell.execute_reply.started":"2022-08-01T08:53:39.354263Z","shell.execute_reply":"2022-08-01T08:53:39.370311Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.subplots(figsize=(10,8))\nsns.heatmap(train.drop([\"PassengerId\"], axis=1).corr(),\n            cmap='RdBu', annot=True, fmt='.2f', cbar=True, vmin=-1, vmax=1)","metadata":{"execution":{"iopub.status.busy":"2022-08-01T08:53:39.373843Z","iopub.execute_input":"2022-08-01T08:53:39.374317Z","iopub.status.idle":"2022-08-01T08:53:39.798778Z","shell.execute_reply.started":"2022-08-01T08:53:39.374269Z","shell.execute_reply":"2022-08-01T08:53:39.797513Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Pclass and Fare","metadata":{}},{"cell_type":"code","source":"train.groupby('Pclass')['Survived'].mean().plot(kind='bar', ylabel='mean survive')","metadata":{"execution":{"iopub.status.busy":"2022-08-01T08:53:39.800370Z","iopub.execute_input":"2022-08-01T08:53:39.800770Z","iopub.status.idle":"2022-08-01T08:53:39.983743Z","shell.execute_reply.started":"2022-08-01T08:53:39.800734Z","shell.execute_reply":"2022-08-01T08:53:39.982520Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.boxplot(x=\"Pclass\", y=\"Fare\", data=train, showfliers=False)","metadata":{"execution":{"iopub.status.busy":"2022-08-01T08:53:39.985542Z","iopub.execute_input":"2022-08-01T08:53:39.986693Z","iopub.status.idle":"2022-08-01T08:53:40.183971Z","shell.execute_reply.started":"2022-08-01T08:53:39.986649Z","shell.execute_reply":"2022-08-01T08:53:40.182955Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(20,5))\nplt.xlim(0, 50)\nsns.kdeplot(data=train, x=\"Fare\", hue=\"Pclass\")","metadata":{"execution":{"iopub.status.busy":"2022-08-01T08:53:40.185700Z","iopub.execute_input":"2022-08-01T08:53:40.186375Z","iopub.status.idle":"2022-08-01T08:53:40.465060Z","shell.execute_reply.started":"2022-08-01T08:53:40.186330Z","shell.execute_reply":"2022-08-01T08:53:40.464250Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(pd.qcut(train[train['Pclass'] == 1]['Fare'], 4).cat.categories.right)\nprint(pd.qcut(train[train['Pclass'] == 2]['Fare'], 4).cat.categories.right)\nprint(pd.qcut(train[train['Pclass'] == 3]['Fare'], 4).cat.categories.right)","metadata":{"execution":{"iopub.status.busy":"2022-08-01T08:53:40.468643Z","iopub.execute_input":"2022-08-01T08:53:40.469171Z","iopub.status.idle":"2022-08-01T08:53:40.486452Z","shell.execute_reply.started":"2022-08-01T08:53:40.469137Z","shell.execute_reply":"2022-08-01T08:53:40.485560Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"g = sns.FacetGrid(train, row=\"Pclass\", col=\"Survived\")\ng.map_dataframe(sns.histplot, x=\"Age\", bins=10)","metadata":{"execution":{"iopub.status.busy":"2022-08-01T08:53:40.487662Z","iopub.execute_input":"2022-08-01T08:53:40.488585Z","iopub.status.idle":"2022-08-01T08:53:41.752190Z","shell.execute_reply.started":"2022-08-01T08:53:40.488547Z","shell.execute_reply":"2022-08-01T08:53:41.750866Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.qcut(train['Fare'], 4).cat.categories.right","metadata":{"execution":{"iopub.status.busy":"2022-08-01T08:53:41.754141Z","iopub.execute_input":"2022-08-01T08:53:41.754902Z","iopub.status.idle":"2022-08-01T08:53:41.766434Z","shell.execute_reply.started":"2022-08-01T08:53:41.754858Z","shell.execute_reply":"2022-08-01T08:53:41.765266Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Age","metadata":{}},{"cell_type":"code","source":"g = sns.FacetGrid(train, col=\"Survived\")\ng.map_dataframe(sns.histplot, x=\"Age\", bins=20)","metadata":{"execution":{"iopub.status.busy":"2022-08-01T08:53:41.768017Z","iopub.execute_input":"2022-08-01T08:53:41.768407Z","iopub.status.idle":"2022-08-01T08:53:42.254786Z","shell.execute_reply.started":"2022-08-01T08:53:41.768373Z","shell.execute_reply":"2022-08-01T08:53:42.253564Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### SibSp and Parch","metadata":{}},{"cell_type":"code","source":"g = sns.FacetGrid(train, col=\"Survived\")\ng.map_dataframe(sns.histplot, x=\"SibSp\", bins=10)","metadata":{"execution":{"iopub.status.busy":"2022-08-01T08:53:42.256379Z","iopub.execute_input":"2022-08-01T08:53:42.257554Z","iopub.status.idle":"2022-08-01T08:53:42.702637Z","shell.execute_reply.started":"2022-08-01T08:53:42.257511Z","shell.execute_reply":"2022-08-01T08:53:42.701458Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"g = sns.FacetGrid(train, col=\"Survived\")\ng.map_dataframe(sns.histplot, x=\"Parch\", bins=10)","metadata":{"execution":{"iopub.status.busy":"2022-08-01T08:53:42.704633Z","iopub.execute_input":"2022-08-01T08:53:42.705258Z","iopub.status.idle":"2022-08-01T08:53:43.165930Z","shell.execute_reply.started":"2022-08-01T08:53:42.705207Z","shell.execute_reply":"2022-08-01T08:53:43.164503Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_tmp = train.copy()\ndf_tmp[\"WithSibSp\"] = np.where(train[\"SibSp\"] > 0, 1, 0)\n\ndf_tmp.groupby('WithSibSp')['Survived'].mean().plot(kind='bar', ylabel='mean survive')","metadata":{"execution":{"iopub.status.busy":"2022-08-01T08:53:43.167940Z","iopub.execute_input":"2022-08-01T08:53:43.169073Z","iopub.status.idle":"2022-08-01T08:53:43.350844Z","shell.execute_reply.started":"2022-08-01T08:53:43.169012Z","shell.execute_reply":"2022-08-01T08:53:43.349746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_tmp = train.copy()\ndf_tmp[\"WithParch\"] = np.where(train[\"Parch\"] > 0, 1, 0)\n\ndf_tmp.groupby('WithParch')['Survived'].mean().plot(kind='bar', ylabel='mean survive')","metadata":{"execution":{"iopub.status.busy":"2022-08-01T08:53:43.352942Z","iopub.execute_input":"2022-08-01T08:53:43.354853Z","iopub.status.idle":"2022-08-01T08:53:43.529848Z","shell.execute_reply.started":"2022-08-01T08:53:43.354794Z","shell.execute_reply":"2022-08-01T08:53:43.528743Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_tmp = train.copy()\ndf_tmp[\"WithFamily\"] = np.where((train[\"SibSp\"] > 0) | (train[\"Parch\"] > 0), 1, 0)\n\ndf_tmp.groupby('WithFamily')['Survived'].mean().plot(kind='bar', ylabel='mean survive')","metadata":{"execution":{"iopub.status.busy":"2022-08-01T08:53:43.533850Z","iopub.execute_input":"2022-08-01T08:53:43.534693Z","iopub.status.idle":"2022-08-01T08:53:43.715053Z","shell.execute_reply.started":"2022-08-01T08:53:43.534637Z","shell.execute_reply":"2022-08-01T08:53:43.714111Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Observations**\n* higher Pclass = lower survival rate\n* Pclass coreelated with Fare\n* SibSp correlated with Parch (family trips)\n* passengers with family = higher survival rate\n* Age, SibSp, Parch not coerelated","metadata":{}},{"cell_type":"markdown","source":"### Categorical values","metadata":{}},{"cell_type":"markdown","source":"#### Sex","metadata":{}},{"cell_type":"code","source":"train.groupby('Sex')['Survived'].mean().plot(kind='bar', ylabel='mean survive')","metadata":{"execution":{"iopub.status.busy":"2022-08-01T08:53:43.717055Z","iopub.execute_input":"2022-08-01T08:53:43.717950Z","iopub.status.idle":"2022-08-01T08:53:43.898243Z","shell.execute_reply.started":"2022-08-01T08:53:43.717898Z","shell.execute_reply":"2022-08-01T08:53:43.896986Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"g = sns.FacetGrid(train, row=\"Sex\", col=\"Survived\")\ng.map_dataframe(sns.histplot, x=\"Age\", bins=10)","metadata":{"execution":{"iopub.status.busy":"2022-08-01T08:53:43.899772Z","iopub.execute_input":"2022-08-01T08:53:43.900216Z","iopub.status.idle":"2022-08-01T08:53:44.695050Z","shell.execute_reply.started":"2022-08-01T08:53:43.900171Z","shell.execute_reply":"2022-08-01T08:53:44.693546Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Embarked","metadata":{"execution":{"iopub.status.busy":"2022-07-31T13:30:06.288917Z","iopub.execute_input":"2022-07-31T13:30:06.289753Z","iopub.status.idle":"2022-07-31T13:30:06.298365Z","shell.execute_reply.started":"2022-07-31T13:30:06.289705Z","shell.execute_reply":"2022-07-31T13:30:06.297324Z"}}},{"cell_type":"code","source":"train.groupby('Embarked')['Survived'].mean().plot(kind='bar', ylabel='mean survive')","metadata":{"execution":{"iopub.status.busy":"2022-08-01T08:53:44.696799Z","iopub.execute_input":"2022-08-01T08:53:44.697241Z","iopub.status.idle":"2022-08-01T08:53:44.876226Z","shell.execute_reply.started":"2022-08-01T08:53:44.697205Z","shell.execute_reply":"2022-08-01T08:53:44.875012Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.groupby('Embarked')['Fare'].mean().plot(kind='bar', ylabel='mean fare')","metadata":{"execution":{"iopub.status.busy":"2022-08-01T08:53:44.877794Z","iopub.execute_input":"2022-08-01T08:53:44.878433Z","iopub.status.idle":"2022-08-01T08:53:45.065375Z","shell.execute_reply.started":"2022-08-01T08:53:44.878386Z","shell.execute_reply":"2022-08-01T08:53:45.064360Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prepare data","metadata":{}},{"cell_type":"code","source":"X_train = train.drop(\"Survived\", axis=1)\nX_test = test.copy()\ny_train = train[\"Survived\"]\n\nfor df in [X_train, X_test]:\n    # Age fillna\n    df['Age'].fillna(train['Age'].dropna().mean(), inplace=True)\n    # Age cut -> numeric / onehotencoder 0-16-32-rest\n    df.loc[df['Age'] <= 16, 'Age'] = 0\n    df.loc[(df['Age'] > 16) & (df['Age'] <= 32), 'Age'] = 1\n    df.loc[df['Age'] > 32, 'Age'] = 2\n    df['Age'] = df['Age'].astype(int)\n    \n    # SibSp, Parch -> WithFamily\n    df[\"WithFamily\"] = np.where((df[\"SibSp\"] > 0) | (df[\"Parch\"] > 0), 1, 0)\n    \n    # Fare fillna\n    for i_class in range(1,4):\n        df.loc[(df['Fare'].isna()) & (df['Pclass'] == i_class), 'Fare'] = train[train['Pclass'] == i_class]['Fare'].mean()\n    # Fare cut -> numeric / onehotencoder (-0.001, 7.91] < (7.91, 14.454] < (14.454, 31.0] < (31.0, 512.329]\n    df.loc[df['Fare'] <= 7.91, 'Fare'] = 0\n    df.loc[(df['Fare'] > 7.91) & (df['Fare'] <= 14.454), 'Fare'] = 1\n    df.loc[(df['Fare'] > 14.454) & (df['Fare'] <= 31.0), 'Fare'] = 2\n    df.loc[df['Fare'] > 31.0, 'Fare'] = 3\n    df['Fare'] = df['Fare'].astype(int)\n    \n    # Sex -> numeric\n    df['Sex'] = df['Sex'].map({'female': 0, 'male': 1}).astype(int)\n    \n    # Embarked fillna\n    df.loc[df['Embarked'].isna() & (df['Fare'] <= 20), 'Embarked'] = 'Q'\n    df.loc[df['Embarked'].isna() & (df['Fare'] > 20) & (df['Fare'] <= 40), 'Embarked'] = 'S'\n    df.loc[df['Embarked'].isna() & (df['Fare'] > 40), 'Embarked'] = 'C'\n    # Embarked -> numeric / onehotencoder\n    df['Embarked'] = df['Embarked'].map({'C': 0, 'Q': 1, 'S': 2}).astype(int)\n    \n    # Drop 'PassengerId', 'Name', 'SibSp', 'Parch', 'Cabin', 'Ticket'\n    df.drop(['PassengerId', 'Name', 'SibSp', 'Parch', 'Cabin', 'Ticket'], axis=1, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-01T08:53:45.067480Z","iopub.execute_input":"2022-08-01T08:53:45.068371Z","iopub.status.idle":"2022-08-01T08:53:45.121907Z","shell.execute_reply.started":"2022-08-01T08:53:45.068322Z","shell.execute_reply":"2022-08-01T08:53:45.120696Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-01T08:53:45.123865Z","iopub.execute_input":"2022-08-01T08:53:45.124585Z","iopub.status.idle":"2022-08-01T08:53:45.138168Z","shell.execute_reply.started":"2022-08-01T08:53:45.124532Z","shell.execute_reply":"2022-08-01T08:53:45.136988Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{}},{"cell_type":"markdown","source":"### Check base models","metadata":{}},{"cell_type":"code","source":"svm = SVC()\ntree = DecisionTreeClassifier()\nrf = RandomForestClassifier()\nadaboost = AdaBoostClassifier()\nxgb = XGBClassifier()\nxgbrf = XGBRFClassifier()\nmlp = MLPClassifier()","metadata":{"execution":{"iopub.status.busy":"2022-08-01T08:53:45.139686Z","iopub.execute_input":"2022-08-01T08:53:45.140427Z","iopub.status.idle":"2022-08-01T08:53:45.152329Z","shell.execute_reply.started":"2022-08-01T08:53:45.140388Z","shell.execute_reply":"2022-08-01T08:53:45.151051Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for model in [svm, tree, rf, adaboost, xgb, xgbrf, mlp]:\n    print(\"Accuracy for {0:23} = {1}\".format(str(model).split(\"(\")[0], np.round(cross_val_score(model, X_train, y_train, cv=5).mean(), 3)))","metadata":{"execution":{"iopub.status.busy":"2022-08-01T08:53:45.153756Z","iopub.execute_input":"2022-08-01T08:53:45.154121Z","iopub.status.idle":"2022-08-01T08:53:54.686376Z","shell.execute_reply.started":"2022-08-01T08:53:45.154087Z","shell.execute_reply":"2022-08-01T08:53:54.685207Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Hyperparameters","metadata":{}},{"cell_type":"markdown","source":"### SVM","metadata":{}},{"cell_type":"code","source":"params = [\n    {'C': [0.1, 1.0, 10.0], 'kernel': ['linear', 'rbf', 'sigmoid'], 'gamma': ['scale', 'auto']},\n    {'C': [0.1, 1.0, 10.0], 'kernel': ['poly'], 'degree': [2, 3, 4], 'gamma': ['scale', 'auto']}\n]\n\ngrid_svm = GridSearchCV(svm, params, cv=5)\ngrid_svm.fit(X_train, y_train)","metadata":{"execution":{"iopub.status.busy":"2022-08-01T08:59:00.974845Z","iopub.execute_input":"2022-08-01T08:59:00.975463Z","iopub.status.idle":"2022-08-01T08:59:11.554212Z","shell.execute_reply.started":"2022-08-01T08:59:00.975420Z","shell.execute_reply":"2022-08-01T08:59:11.553004Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"grid_svm.best_estimator_","metadata":{"execution":{"iopub.status.busy":"2022-08-01T08:59:14.270503Z","iopub.execute_input":"2022-08-01T08:59:14.270920Z","iopub.status.idle":"2022-08-01T08:59:14.279807Z","shell.execute_reply.started":"2022-08-01T08:59:14.270889Z","shell.execute_reply":"2022-08-01T08:59:14.278175Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"grid_svm.best_score_","metadata":{"execution":{"iopub.status.busy":"2022-08-01T08:59:16.172043Z","iopub.execute_input":"2022-08-01T08:59:16.173525Z","iopub.status.idle":"2022-08-01T08:59:16.180699Z","shell.execute_reply.started":"2022-08-01T08:59:16.173453Z","shell.execute_reply":"2022-08-01T08:59:16.179476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Random Forest","metadata":{}},{"cell_type":"code","source":"params = [\n    {'n_estimators': [50, 100, 200], 'max_features': ['sqrt', None]},\n    {'n_estimators': [500, 1000], 'max_features': ['sqrt'], 'max_depth': [5, 10, None], 'min_samples_leaf': [1, 10, 20]}\n]\n\ngrid_rf = GridSearchCV(rf, params, cv=5)\ngrid_rf.fit(X_train, y_train)","metadata":{"execution":{"iopub.status.busy":"2022-08-01T08:54:06.087060Z","iopub.execute_input":"2022-08-01T08:54:06.087581Z","iopub.status.idle":"2022-08-01T08:56:26.949892Z","shell.execute_reply.started":"2022-08-01T08:54:06.087527Z","shell.execute_reply":"2022-08-01T08:56:26.948575Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"grid_rf.best_estimator_","metadata":{"execution":{"iopub.status.busy":"2022-08-01T08:56:26.952118Z","iopub.execute_input":"2022-08-01T08:56:26.952624Z","iopub.status.idle":"2022-08-01T08:56:26.961025Z","shell.execute_reply.started":"2022-08-01T08:56:26.952575Z","shell.execute_reply":"2022-08-01T08:56:26.959791Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"grid_rf.best_score_","metadata":{"execution":{"iopub.status.busy":"2022-08-01T08:56:26.963242Z","iopub.execute_input":"2022-08-01T08:56:26.963736Z","iopub.status.idle":"2022-08-01T08:56:26.972991Z","shell.execute_reply.started":"2022-08-01T08:56:26.963690Z","shell.execute_reply":"2022-08-01T08:56:26.972171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### XGBClassifier","metadata":{}},{"cell_type":"code","source":"params = [\n    {'learning_rate': [0.8, 1.0, 1.2], 'subsample': [0.6, 0.8, 0.9], 'colsample_bynode': [0.7, 0.8, 0.9], 'reg_lambda': [1e-06, 1e-05, 1e-04]}\n]\n\ngrid_xgb = GridSearchCV(xgb, params, cv=5)\ngrid_xgb.fit(X_train, y_train)","metadata":{"execution":{"iopub.status.busy":"2022-08-01T08:56:26.974556Z","iopub.execute_input":"2022-08-01T08:56:26.975469Z","iopub.status.idle":"2022-08-01T08:58:29.685764Z","shell.execute_reply.started":"2022-08-01T08:56:26.975426Z","shell.execute_reply":"2022-08-01T08:58:29.684578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"grid_xgb.best_estimator_","metadata":{"execution":{"iopub.status.busy":"2022-08-01T08:58:29.687568Z","iopub.execute_input":"2022-08-01T08:58:29.688948Z","iopub.status.idle":"2022-08-01T08:58:29.701920Z","shell.execute_reply.started":"2022-08-01T08:58:29.688853Z","shell.execute_reply":"2022-08-01T08:58:29.700596Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"grid_xgb.best_score_","metadata":{"execution":{"iopub.status.busy":"2022-08-01T08:58:29.703549Z","iopub.execute_input":"2022-08-01T08:58:29.703924Z","iopub.status.idle":"2022-08-01T08:58:29.711607Z","shell.execute_reply.started":"2022-08-01T08:58:29.703885Z","shell.execute_reply":"2022-08-01T08:58:29.710382Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Voting classifier","metadata":{}},{"cell_type":"code","source":"from sklearn.ensemble import VotingClassifier\nclf = VotingClassifier(estimators=[('svm', svm), ('rf', rf), ('xgbrf', xgbrf)], voting='hard')\ncv = cross_val_score(clf, X_train, y_train, cv=5)\ncv, cv.mean()","metadata":{"execution":{"iopub.status.busy":"2022-08-01T09:06:39.221454Z","iopub.execute_input":"2022-08-01T09:06:39.221960Z","iopub.status.idle":"2022-08-01T09:06:41.630442Z","shell.execute_reply.started":"2022-08-01T09:06:39.221920Z","shell.execute_reply":"2022-08-01T09:06:41.629338Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.ensemble import VotingClassifier\nclf = VotingClassifier(estimators=[('svm', SVC(C=10.0, gamma='auto', kernel='poly')),\n                                   ('rf', RandomForestClassifier(max_depth=5, max_features='sqrt', n_estimators=500)),\n                                   ('xgb', XGBClassifier(learning_rate=0.8, colsample_bynode=0.7, reg_lambda=1e-04))],\n                       voting='hard')\ncv = cross_val_score(clf, X_train, y_train, cv=5)\ncv, cv.mean()","metadata":{"execution":{"iopub.status.busy":"2022-08-01T09:06:45.671540Z","iopub.execute_input":"2022-08-01T09:06:45.672041Z","iopub.status.idle":"2022-08-01T09:06:53.024440Z","shell.execute_reply.started":"2022-08-01T09:06:45.672002Z","shell.execute_reply":"2022-08-01T09:06:53.023292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Final predictions","metadata":{}},{"cell_type":"code","source":"clf.fit(X_train, y_train)\ny_pred = clf.predict(X_test)","metadata":{"execution":{"iopub.status.busy":"2022-08-01T09:26:58.853176Z","iopub.execute_input":"2022-08-01T09:26:58.854290Z","iopub.status.idle":"2022-08-01T09:27:00.373094Z","shell.execute_reply.started":"2022-08-01T09:26:58.854228Z","shell.execute_reply":"2022-08-01T09:27:00.371755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred","metadata":{"execution":{"iopub.status.busy":"2022-08-01T09:27:01.652726Z","iopub.execute_input":"2022-08-01T09:27:01.653168Z","iopub.status.idle":"2022-08-01T09:27:01.664596Z","shell.execute_reply.started":"2022-08-01T09:27:01.653136Z","shell.execute_reply":"2022-08-01T09:27:01.663297Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.DataFrame({\"PassengerId\": test['PassengerId'], \"Survived\": y_pred})","metadata":{"execution":{"iopub.status.busy":"2022-08-01T09:27:06.066999Z","iopub.execute_input":"2022-08-01T09:27:06.067428Z","iopub.status.idle":"2022-08-01T09:27:06.074756Z","shell.execute_reply.started":"2022-08-01T09:27:06.067394Z","shell.execute_reply":"2022-08-01T09:27:06.073308Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission","metadata":{"execution":{"iopub.status.busy":"2022-08-01T09:27:08.631802Z","iopub.execute_input":"2022-08-01T09:27:08.632503Z","iopub.status.idle":"2022-08-01T09:27:08.644728Z","shell.execute_reply.started":"2022-08-01T09:27:08.632451Z","shell.execute_reply":"2022-08-01T09:27:08.643772Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-08-01T09:28:05.654795Z","iopub.execute_input":"2022-08-01T09:28:05.655737Z","iopub.status.idle":"2022-08-01T09:28:05.665250Z","shell.execute_reply.started":"2022-08-01T09:28:05.655691Z","shell.execute_reply":"2022-08-01T09:28:05.664167Z"},"trusted":true},"execution_count":null,"outputs":[]}]}