{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# About\nThis notebook contains my progress in Kaggle's Titanic Competition. This notebook covers Exploratory Data Analysis, Data Cleaning, and Model Building with little commentary. \n\nI really had fun exploring this data and attempting to build an effective model. After eleven submissions and trying various models and parameters, I received a score of 0.78947 (top 9%).\n\nThis score can definitely be improved, so, if you have any suggestions, please comment them! I also encourage that you partake in this competition as it was a nice learning experience for a beginner like me.","metadata":{}},{"cell_type":"markdown","source":"# Setup","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport seaborn as sns\nimport numpy as np\n\ntrain_path = \"../input/titanic/train.csv\"\ntest_path = \"./../input/titanic/test.csv\"\n\ndf = pd.read_csv(train_path)\ndf_test = pd.read_csv(test_path)\n\n\ndf.drop(\"Survived\", axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-07-22T19:14:59.084703Z","iopub.execute_input":"2022-07-22T19:14:59.085238Z","iopub.status.idle":"2022-07-22T19:14:59.880006Z","shell.execute_reply.started":"2022-07-22T19:14:59.085137Z","shell.execute_reply":"2022-07-22T19:14:59.878796Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.info()","metadata":{"execution":{"iopub.status.busy":"2022-07-22T19:14:59.882435Z","iopub.execute_input":"2022-07-22T19:14:59.883191Z","iopub.status.idle":"2022-07-22T19:14:59.907542Z","shell.execute_reply.started":"2022-07-22T19:14:59.883140Z","shell.execute_reply":"2022-07-22T19:14:59.906177Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Exploratory Data Analysis","metadata":{}},{"cell_type":"code","source":"df['Embarked'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-07-22T19:14:59.909555Z","iopub.execute_input":"2022-07-22T19:14:59.910060Z","iopub.status.idle":"2022-07-22T19:14:59.923582Z","shell.execute_reply.started":"2022-07-22T19:14:59.910013Z","shell.execute_reply":"2022-07-22T19:14:59.922306Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"embarked_grouped = df.groupby(\"Embarked\").aggregate({\"Survived\": \"sum\"})\nembarked_grouped.reset_index(inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-22T19:14:59.927555Z","iopub.execute_input":"2022-07-22T19:14:59.928648Z","iopub.status.idle":"2022-07-22T19:14:59.939629Z","shell.execute_reply.started":"2022-07-22T19:14:59.928591Z","shell.execute_reply":"2022-07-22T19:14:59.938315Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"embarked_grouped.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-22T19:14:59.941235Z","iopub.execute_input":"2022-07-22T19:14:59.943075Z","iopub.status.idle":"2022-07-22T19:14:59.957791Z","shell.execute_reply.started":"2022-07-22T19:14:59.943033Z","shell.execute_reply":"2022-07-22T19:14:59.955926Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.corr()","metadata":{"execution":{"iopub.status.busy":"2022-07-22T19:14:59.959921Z","iopub.execute_input":"2022-07-22T19:14:59.960989Z","iopub.status.idle":"2022-07-22T19:14:59.980688Z","shell.execute_reply.started":"2022-07-22T19:14:59.960937Z","shell.execute_reply":"2022-07-22T19:14:59.979451Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['cabin_mapped'] = df['Cabin'].map(lambda x: str(x)[0])","metadata":{"execution":{"iopub.status.busy":"2022-07-22T19:14:59.982642Z","iopub.execute_input":"2022-07-22T19:14:59.983457Z","iopub.status.idle":"2022-07-22T19:14:59.992104Z","shell.execute_reply.started":"2022-07-22T19:14:59.983409Z","shell.execute_reply":"2022-07-22T19:14:59.990840Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ncat_columns = ['Pclass', 'Sex', 'SibSp', 'Parch', 'Embarked', 'cabin_mapped']\n\nfor i in cat_columns:\n    print(i)\n    grouped = df.groupby(i).aggregate({\"Survived\": \"sum\"})\n    grouped = pd.concat([grouped, df[i].value_counts()], axis=1)\n    grouped.rename(columns={i: \"Total\"}, inplace=True)\n    grouped['Percentage'] = grouped['Survived'] / grouped['Total']\n    display(grouped)","metadata":{"execution":{"iopub.status.busy":"2022-07-22T19:14:59.994870Z","iopub.execute_input":"2022-07-22T19:14:59.995688Z","iopub.status.idle":"2022-07-22T19:15:00.109803Z","shell.execute_reply.started":"2022-07-22T19:14:59.995639Z","shell.execute_reply":"2022-07-22T19:15:00.107496Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['ticket_mapped'] = df['Ticket'].map(lambda x: x.split(\" \")[-1])\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-22T19:15:00.111965Z","iopub.execute_input":"2022-07-22T19:15:00.112763Z","iopub.status.idle":"2022-07-22T19:15:00.143394Z","shell.execute_reply.started":"2022-07-22T19:15:00.112695Z","shell.execute_reply":"2022-07-22T19:15:00.142114Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = df[df['ticket_mapped'] != 'LINE']\ndf['ticket_mapped'] = df['ticket_mapped'].astype(int)","metadata":{"execution":{"iopub.status.busy":"2022-07-22T19:15:00.148407Z","iopub.execute_input":"2022-07-22T19:15:00.149096Z","iopub.status.idle":"2022-07-22T19:15:00.156664Z","shell.execute_reply.started":"2022-07-22T19:15:00.149058Z","shell.execute_reply":"2022-07-22T19:15:00.155376Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Feature Engineering","metadata":{}},{"cell_type":"code","source":"sns.heatmap(df.corr(), annot=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-22T19:15:00.158172Z","iopub.execute_input":"2022-07-22T19:15:00.158674Z","iopub.status.idle":"2022-07-22T19:15:00.734001Z","shell.execute_reply.started":"2022-07-22T19:15:00.158637Z","shell.execute_reply":"2022-07-22T19:15:00.732771Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['IsAlone'] = (df['SibSp'] + df['Parch']) == 0","metadata":{"execution":{"iopub.status.busy":"2022-07-22T19:15:00.735593Z","iopub.execute_input":"2022-07-22T19:15:00.736617Z","iopub.status.idle":"2022-07-22T19:15:00.742878Z","shell.execute_reply.started":"2022-07-22T19:15:00.736579Z","shell.execute_reply":"2022-07-22T19:15:00.741651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.corr()","metadata":{"execution":{"iopub.status.busy":"2022-07-22T19:15:00.744401Z","iopub.execute_input":"2022-07-22T19:15:00.744801Z","iopub.status.idle":"2022-07-22T19:15:00.773176Z","shell.execute_reply.started":"2022-07-22T19:15:00.744758Z","shell.execute_reply":"2022-07-22T19:15:00.772252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['Age*Pclass'] = df['Age'] * df['Pclass']\ndf.corr()","metadata":{"execution":{"iopub.status.busy":"2022-07-22T19:15:00.774430Z","iopub.execute_input":"2022-07-22T19:15:00.775440Z","iopub.status.idle":"2022-07-22T19:15:00.799041Z","shell.execute_reply.started":"2022-07-22T19:15:00.775404Z","shell.execute_reply":"2022-07-22T19:15:00.798183Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Building the Model","metadata":{}},{"cell_type":"markdown","source":"## Setup","metadata":{}},{"cell_type":"code","source":"df = pd.read_csv(train_path)\ndf_test = pd.read_csv(test_path)","metadata":{"execution":{"iopub.status.busy":"2022-07-22T19:15:00.800459Z","iopub.execute_input":"2022-07-22T19:15:00.800958Z","iopub.status.idle":"2022-07-22T19:15:00.816669Z","shell.execute_reply.started":"2022-07-22T19:15:00.800925Z","shell.execute_reply":"2022-07-22T19:15:00.815784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['train_test'] = 1\ndf_test['train_test'] = 0","metadata":{"execution":{"iopub.status.busy":"2022-07-22T19:15:00.817965Z","iopub.execute_input":"2022-07-22T19:15:00.818454Z","iopub.status.idle":"2022-07-22T19:15:00.824427Z","shell.execute_reply.started":"2022-07-22T19:15:00.818422Z","shell.execute_reply":"2022-07-22T19:15:00.823355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_data = pd.concat([df, df_test])","metadata":{"execution":{"iopub.status.busy":"2022-07-22T19:15:00.825814Z","iopub.execute_input":"2022-07-22T19:15:00.826368Z","iopub.status.idle":"2022-07-22T19:15:00.840024Z","shell.execute_reply.started":"2022-07-22T19:15:00.826333Z","shell.execute_reply":"2022-07-22T19:15:00.838908Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_data['train_test'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-07-22T19:15:00.841796Z","iopub.execute_input":"2022-07-22T19:15:00.842823Z","iopub.status.idle":"2022-07-22T19:15:00.850806Z","shell.execute_reply.started":"2022-07-22T19:15:00.842778Z","shell.execute_reply":"2022-07-22T19:15:00.849749Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_data","metadata":{"execution":{"iopub.status.busy":"2022-07-22T19:15:00.852121Z","iopub.execute_input":"2022-07-22T19:15:00.853481Z","iopub.status.idle":"2022-07-22T19:15:00.882709Z","shell.execute_reply.started":"2022-07-22T19:15:00.853445Z","shell.execute_reply":"2022-07-22T19:15:00.881495Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_data['cabin_mapped'] = all_data['Cabin'].map(lambda x: str(x)[0])\nall_data['ticket_mapped'] = all_data['Ticket'].map(lambda x: x.split(\" \")[-1])\n\nall_data['Title'] = all_data.Name.str.extract(' ([A-Za-z]+)\\.', expand=False)\nall_data['IsAlone'] = (all_data['SibSp'] + all_data['Parch']) == 0\nall_data['Age*Pclass'] = all_data['Age'] * all_data['Pclass']","metadata":{"execution":{"iopub.status.busy":"2022-07-22T19:15:00.883897Z","iopub.execute_input":"2022-07-22T19:15:00.885043Z","iopub.status.idle":"2022-07-22T19:15:00.901553Z","shell.execute_reply.started":"2022-07-22T19:15:00.885003Z","shell.execute_reply":"2022-07-22T19:15:00.900341Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Null Values","metadata":{}},{"cell_type":"code","source":"all_data.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-07-22T19:15:00.903160Z","iopub.execute_input":"2022-07-22T19:15:00.904513Z","iopub.status.idle":"2022-07-22T19:15:00.917159Z","shell.execute_reply.started":"2022-07-22T19:15:00.904473Z","shell.execute_reply":"2022-07-22T19:15:00.916096Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_data['Age'].fillna(all_data['Age'].mean(), inplace=True)\nall_data['Embarked'].fillna(all_data['Embarked'].mode()[0], inplace=True)\nall_data['Fare'].fillna(all_data['Fare'].mean(), inplace=True)\nall_data['Age*Pclass'].fillna(all_data['Age*Pclass'].median(), inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-22T19:15:00.919033Z","iopub.execute_input":"2022-07-22T19:15:00.919379Z","iopub.status.idle":"2022-07-22T19:15:00.931609Z","shell.execute_reply.started":"2022-07-22T19:15:00.919350Z","shell.execute_reply":"2022-07-22T19:15:00.930749Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_data.isnull().sum()","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2022-07-22T19:15:00.933043Z","iopub.execute_input":"2022-07-22T19:15:00.933817Z","iopub.status.idle":"2022-07-22T19:15:00.950930Z","shell.execute_reply.started":"2022-07-22T19:15:00.933775Z","shell.execute_reply":"2022-07-22T19:15:00.949836Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Normalization","metadata":{}},{"cell_type":"code","source":"all_data['norm_fare'] = np.log1p(all_data['Fare'])","metadata":{"execution":{"iopub.status.busy":"2022-07-22T19:15:00.952771Z","iopub.execute_input":"2022-07-22T19:15:00.953142Z","iopub.status.idle":"2022-07-22T19:15:00.960566Z","shell.execute_reply.started":"2022-07-22T19:15:00.953108Z","shell.execute_reply":"2022-07-22T19:15:00.959272Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Categoricals","metadata":{}},{"cell_type":"code","source":"# transform `Pclass` into a categorical\nall_data['Pclass'] = all_data['Pclass'].astype(str)\nall_dummies = pd.get_dummies(all_data[['Pclass', 'Sex', 'Age', 'norm_fare', 'train_test', 'Title', 'ticket_mapped', 'IsAlone', 'Age*Pclass', 'SibSp', 'Parch']])","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2022-07-22T19:15:00.962175Z","iopub.execute_input":"2022-07-22T19:15:00.963073Z","iopub.status.idle":"2022-07-22T19:15:00.997754Z","shell.execute_reply.started":"2022-07-22T19:15:00.963024Z","shell.execute_reply":"2022-07-22T19:15:00.996821Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Scaling","metadata":{}},{"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler\n\nscaler = StandardScaler()\n\nall_dummies_scaled = all_dummies.copy()\nall_dummies_scaled[['norm_fare', 'Age*Pclass', 'SibSp', 'Parch']] = scaler.fit_transform(all_dummies_scaled[['norm_fare', 'Age*Pclass', 'SibSp', 'Parch']])\n\nX_train_scaled = all_dummies_scaled[all_dummies_scaled.train_test == 1].drop(['train_test'], axis =1)\nX_test_scaled = all_dummies_scaled[all_dummies_scaled.train_test == 0].drop(['train_test'], axis =1)\n\ny_train = all_data[all_data.train_test == 1].Survived","metadata":{"execution":{"iopub.status.busy":"2022-07-22T19:15:00.999511Z","iopub.execute_input":"2022-07-22T19:15:01.000239Z","iopub.status.idle":"2022-07-22T19:15:01.101767Z","shell.execute_reply.started":"2022-07-22T19:15:01.000190Z","shell.execute_reply":"2022-07-22T19:15:01.100847Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import cross_val_score\nfrom sklearn.naive_bayes import GaussianNB\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn import tree\nfrom sklearn.neighbors import KNeighborsClassifier\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.svm import SVC\nfrom catboost import CatBoostClassifier","metadata":{"execution":{"iopub.status.busy":"2022-07-22T19:15:01.103481Z","iopub.execute_input":"2022-07-22T19:15:01.104245Z","iopub.status.idle":"2022-07-22T19:15:01.821510Z","shell.execute_reply.started":"2022-07-22T19:15:01.104195Z","shell.execute_reply":"2022-07-22T19:15:01.820153Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gnb = GaussianNB()\ncv = cross_val_score(gnb, X_train_scaled, y_train, cv=5)\nprint(cv)\nprint(cv.mean())","metadata":{"execution":{"iopub.status.busy":"2022-07-22T19:15:01.827799Z","iopub.execute_input":"2022-07-22T19:15:01.828175Z","iopub.status.idle":"2022-07-22T19:15:02.574672Z","shell.execute_reply.started":"2022-07-22T19:15:01.828141Z","shell.execute_reply":"2022-07-22T19:15:02.573552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lr = LogisticRegression(max_iter = 2000)\ncv = cross_val_score(lr, X_train_scaled, y_train, cv=5)\nprint(cv)\nprint(cv.mean())","metadata":{"execution":{"iopub.status.busy":"2022-07-22T19:15:02.576310Z","iopub.execute_input":"2022-07-22T19:15:02.576676Z","iopub.status.idle":"2022-07-22T19:15:05.272964Z","shell.execute_reply.started":"2022-07-22T19:15:02.576644Z","shell.execute_reply":"2022-07-22T19:15:05.271516Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dt = tree.DecisionTreeClassifier(random_state = 1)\ncv = cross_val_score(dt, X_train_scaled, y_train,cv=5)\nprint(cv)\nprint(cv.mean())","metadata":{"execution":{"iopub.status.busy":"2022-07-22T19:15:05.275304Z","iopub.execute_input":"2022-07-22T19:15:05.276352Z","iopub.status.idle":"2022-07-22T19:15:06.132880Z","shell.execute_reply.started":"2022-07-22T19:15:05.276294Z","shell.execute_reply":"2022-07-22T19:15:06.131521Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"knn = KNeighborsClassifier()\ncv = cross_val_score(knn, X_train_scaled, y_train,cv=5)\nprint(cv)\nprint(cv.mean())","metadata":{"execution":{"iopub.status.busy":"2022-07-22T19:15:06.134165Z","iopub.execute_input":"2022-07-22T19:15:06.134577Z","iopub.status.idle":"2022-07-22T19:15:07.140047Z","shell.execute_reply.started":"2022-07-22T19:15:06.134546Z","shell.execute_reply":"2022-07-22T19:15:07.138663Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rf = RandomForestClassifier(random_state = 1)\ncv = cross_val_score(rf, X_train_scaled, y_train,cv=5)\nprint(cv)\nprint(cv.mean())","metadata":{"execution":{"iopub.status.busy":"2022-07-22T19:15:07.142163Z","iopub.execute_input":"2022-07-22T19:15:07.143015Z","iopub.status.idle":"2022-07-22T19:15:09.879374Z","shell.execute_reply.started":"2022-07-22T19:15:07.142959Z","shell.execute_reply":"2022-07-22T19:15:09.878056Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"svc = SVC(probability = True, C = 1, kernel='linear')\ncv = cross_val_score(svc, X_train_scaled, y_train,cv=5)\nprint(cv)\nprint(cv.mean())","metadata":{"execution":{"iopub.status.busy":"2022-07-22T19:15:09.881745Z","iopub.execute_input":"2022-07-22T19:15:09.882210Z","iopub.status.idle":"2022-07-22T19:15:32.180900Z","shell.execute_reply.started":"2022-07-22T19:15:09.882166Z","shell.execute_reply":"2022-07-22T19:15:32.179108Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from xgboost import XGBClassifier\nxgb = XGBClassifier(random_state =1)\ncv = cross_val_score(xgb, X_train_scaled, y_train,cv=5)\nprint(cv)\nprint(cv.mean())","metadata":{"execution":{"iopub.status.busy":"2022-07-22T19:15:32.182674Z","iopub.execute_input":"2022-07-22T19:15:32.183182Z","iopub.status.idle":"2022-07-22T19:15:45.791622Z","shell.execute_reply.started":"2022-07-22T19:15:32.183134Z","shell.execute_reply":"2022-07-22T19:15:45.790668Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.ensemble import VotingClassifier\nvoting_clf = VotingClassifier(estimators = [('lr',lr),('knn',knn),('rf',rf),('gnb',gnb),('svc',svc),('xgb',xgb),('dt', dt)], voting = 'soft') \ncv = cross_val_score(voting_clf, X_train_scaled, y_train,cv=5)\nprint(cv)\nprint(cv.mean())","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2022-07-22T19:15:45.796537Z","iopub.execute_input":"2022-07-22T19:15:45.799526Z","iopub.status.idle":"2022-07-22T19:16:24.334080Z","shell.execute_reply.started":"2022-07-22T19:15:45.799470Z","shell.execute_reply":"2022-07-22T19:16:24.332785Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cbc = CatBoostClassifier(iterations=100, learning_rate=0.01)\ncv = cross_val_score(cbc, X_train_scaled, y_train,cv=5)\nprint(cv)\nprint(cv.mean())","metadata":{"execution":{"iopub.status.busy":"2022-07-22T19:16:24.337041Z","iopub.execute_input":"2022-07-22T19:16:24.337865Z","iopub.status.idle":"2022-07-22T19:16:26.768608Z","shell.execute_reply.started":"2022-07-22T19:16:24.337814Z","shell.execute_reply":"2022-07-22T19:16:26.767278Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Optimization","metadata":{}},{"cell_type":"code","source":"test = pd.read_csv(test_path)","metadata":{"execution":{"iopub.status.busy":"2022-07-22T19:16:26.769845Z","iopub.execute_input":"2022-07-22T19:16:26.770156Z","iopub.status.idle":"2022-07-22T19:16:26.779605Z","shell.execute_reply.started":"2022-07-22T19:16:26.770126Z","shell.execute_reply":"2022-07-22T19:16:26.778148Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import GridSearchCV ","metadata":{"execution":{"iopub.status.busy":"2022-07-22T19:16:26.781043Z","iopub.execute_input":"2022-07-22T19:16:26.781401Z","iopub.status.idle":"2022-07-22T19:16:26.786837Z","shell.execute_reply.started":"2022-07-22T19:16:26.781370Z","shell.execute_reply":"2022-07-22T19:16:26.785891Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clf_performance(classifier, model_name):\n    print(model_name)\n    print('Best Score: ' + str(classifier.best_score_))\n    print('Best Parameters: ' + str(classifier.best_params_))","metadata":{"execution":{"iopub.status.busy":"2022-07-22T19:16:26.788121Z","iopub.execute_input":"2022-07-22T19:16:26.788860Z","iopub.status.idle":"2022-07-22T19:16:26.801630Z","shell.execute_reply.started":"2022-07-22T19:16:26.788824Z","shell.execute_reply":"2022-07-22T19:16:26.800228Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lr = LogisticRegression()\nparam_grid = {'max_iter' : [2000],\n              'penalty' : ['l1', 'l2'],\n              'C' : np.logspace(-4, 4, 20),\n              'solver' : ['liblinear']}\n\nclf_lr = GridSearchCV(lr, param_grid = param_grid, cv = 5, verbose = True, n_jobs = -1)\nbest_clf_lr = clf_lr.fit(X_train_scaled,y_train)\nclf_performance(best_clf_lr,'Logistic Regression')\nideal_lr = LogisticRegression(**clf_lr.best_params_)","metadata":{"execution":{"iopub.status.busy":"2022-07-22T19:16:26.803257Z","iopub.execute_input":"2022-07-22T19:16:26.803629Z","iopub.status.idle":"2022-07-22T19:16:40.024596Z","shell.execute_reply.started":"2022-07-22T19:16:26.803588Z","shell.execute_reply":"2022-07-22T19:16:40.023478Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"knn = KNeighborsClassifier()\nparam_grid = {'n_neighbors' : [3,5,7,9],\n              'weights' : ['uniform', 'distance'],\n              'algorithm' : ['auto', 'ball_tree','kd_tree'],\n              'p' : [1,2]}\nclf_knn = GridSearchCV(knn, param_grid = param_grid, cv = 5, verbose = True, n_jobs = -1)\nbest_clf_knn = clf_knn.fit(X_train_scaled,y_train)\nclf_performance(best_clf_knn,'KNN')","metadata":{"execution":{"iopub.status.busy":"2022-07-22T19:16:40.026757Z","iopub.execute_input":"2022-07-22T19:16:40.027523Z","iopub.status.idle":"2022-07-22T19:17:12.351265Z","shell.execute_reply.started":"2022-07-22T19:16:40.027468Z","shell.execute_reply":"2022-07-22T19:17:12.350173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"voting_soft = VotingClassifier(estimators = [('svc', svc), ('lr', ideal_lr), ('knn', clf_knn.best_estimator_)])","metadata":{"execution":{"iopub.status.busy":"2022-07-22T19:17:12.352915Z","iopub.execute_input":"2022-07-22T19:17:12.353278Z","iopub.status.idle":"2022-07-22T19:17:12.359365Z","shell.execute_reply.started":"2022-07-22T19:17:12.353244Z","shell.execute_reply":"2022-07-22T19:17:12.358059Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"params = {'weights' : [[1,1,1],[1,2,1],[1,1,2],[2,1,1],[2,2,1],[1,2,2],[2,1,2]]}\n\nvote_weight = GridSearchCV(voting_soft, param_grid = params, cv = 5, verbose = True, n_jobs = -1)\nbest_clf_weight = vote_weight.fit(X_train_scaled, y_train)\nclf_performance(best_clf_weight,'VC Weights')","metadata":{"execution":{"iopub.status.busy":"2022-07-22T19:17:12.361125Z","iopub.execute_input":"2022-07-22T19:17:12.361481Z","iopub.status.idle":"2022-07-22T19:18:42.790303Z","shell.execute_reply.started":"2022-07-22T19:17:12.361449Z","shell.execute_reply":"2022-07-22T19:18:42.788736Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission","metadata":{}},{"cell_type":"code","source":"models = [ideal_lr, svc, best_clf_weight.best_estimator_, xgb, cbc, clf_knn.best_estimator_]\n\nfor i in range(len(models)):\n    models[i].fit(X_train_scaled, y_train)\n    preds = models[i].predict(X_test_scaled)\n    final_data = {'PassengerId': test.PassengerId, 'Survived': preds}\n    final_data['Survived'] = final_data['Survived'].astype(int)\n    submission = pd.DataFrame(data=final_data)\n    submission.to_csv(f'submission_{i}.csv', index =False)","metadata":{"execution":{"iopub.status.busy":"2022-07-22T19:18:42.791955Z","iopub.execute_input":"2022-07-22T19:18:42.792633Z","iopub.status.idle":"2022-07-22T19:19:04.287306Z","shell.execute_reply.started":"2022-07-22T19:18:42.792581Z","shell.execute_reply":"2022-07-22T19:19:04.285959Z"},"trusted":true},"execution_count":null,"outputs":[]}]}