{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-input":true,"collapsed":true},"cell_type":"markdown","source":"# Titanic Survival - Exploration + Baseline Model\n\nThis is a simple notebook on exploration and baseline model to predict who will survive the sinking of the Titanic\n\n## **Contents**   \n[1. Load Data](#1)    \n[2. Data Exploration](#2)  \n&nbsp;&nbsp;&nbsp;&nbsp; [2.1 Basic Data Info](#2.1)    \n&nbsp;&nbsp;&nbsp;&nbsp; [2.2 Feature Distributions](#2.2)    \n&nbsp;&nbsp;&nbsp;&nbsp; [2.3 Feature Creation/Deletion](#2.3)    \n&nbsp;&nbsp;&nbsp;&nbsp; [2.4 Impute Missing Data](#2.4)  \n&nbsp;&nbsp;&nbsp;&nbsp; [2.5 Number Conversions](#2.5)    \n&nbsp;&nbsp;&nbsp;&nbsp; [2.6 Feature Extraction](#2.6)\n\n&nbsp;&nbsp;&nbsp;&nbsp; [2.7 Applicants Contract Type](#2.7)   \n&nbsp;&nbsp;&nbsp;&nbsp; [2.8 Education Type and Occupation Type](#2.8)   \n&nbsp;&nbsp;&nbsp;&nbsp; [2.9 Organization Type and Occupation Type](#2.9)   \n&nbsp;&nbsp;&nbsp;&nbsp; [2.10 Walls Material, Foundation and House Type](#2.10)   \n&nbsp;&nbsp;&nbsp;&nbsp; [2.11 Amount Credit Distribution](#2.11)    \n&nbsp;&nbsp;&nbsp;&nbsp; [2.12 Amount Annuity Distribution - Distribution](#2.12)  \n&nbsp;&nbsp;&nbsp;&nbsp; [2.13 Amount Goods Price - Distribution](#2.13)   \n&nbsp;&nbsp;&nbsp;&nbsp; [2.14 Amount Region Population Relative](#2.14)    \n&nbsp;&nbsp;&nbsp;&nbsp; [2.15 Days Birth - Distribution](#2.15)   \n&nbsp;&nbsp;&nbsp;&nbsp; [2.16 Days Employed - Distribution](#2.16)    \n&nbsp;&nbsp;&nbsp;&nbsp; [2.17 Distribution of Num Days Registration](#2.17)  \n&nbsp;&nbsp;&nbsp;&nbsp; [2.18 Applicants Number of Family Members](#2.18)  \n&nbsp;&nbsp;&nbsp;&nbsp; [2.19 Applicants Number of Children](#2.19)  \n[3. Exploration - Bureau Data](#3)  \n&nbsp;&nbsp;&nbsp;&nbsp; [3.1 Snapshot - Bureau Data](#3) \n\n\n\n\n\n## <a id=\"1\">1. Load Data </a>"},{"metadata":{"trusted":true,"_uuid":"965ef1637f3bad085d36563e70344a0bda5a5c6a","scrolled":true},"cell_type":"code","source":"'''\n# Load Python 3 packages and retrieve Titanic data (libraries installed are\n   # defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n'''\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport scipy.stats\n\n# data visualization\nimport seaborn as sns\n%matplotlib inline\nfrom matplotlib import pyplot as plt\nfrom matplotlib import style\n\n# Algorithms\nfrom sklearn import linear_model\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.linear_model import Perceptron\nfrom sklearn.linear_model import SGDClassifier\nfrom sklearn.tree import DecisionTreeClassifier\nfrom sklearn.neighbors import KNeighborsClassifier\nfrom sklearn.svm import SVC, LinearSVC\nfrom sklearn.naive_bayes import GaussianNB\nfrom sklearn.model_selection import train_test_split\nimport lightgbm as lgb\n\n## Function to Hot-Code a categorical variable_\n    # Takes as parameters 1) a dataframe 2) a string variable with the column name to recode\n    # Leaves in tack the initial variable that was recoded\n\ndef HotC(dframe,col):   # Function to Hot-Code a categorical variable\n        \n    if not(isinstance(dframe,pd.DataFrame)):\n        print('!!ERROR!! The first variable in the HotC function must be a dataframe')\n        return\n    if not(isinstance(col,str)):\n        print('!!ERROR!! The second variable in the HotC function must be a string representing a column in the dataframe')\n        return\n    df2=pd.DataFrame(dframe[col].str.get_dummies())\n    df3=pd.concat([dframe,df2],axis=1)\n\n    return df3\n\n\npath='../input/titanic-machine-learning-from-disaster/'\n\ntrain_df = pd.read_csv(path + \"train.csv\")\ntest_df = pd.read_csv(path + 'test.csv')\n\n# Write out Data sets for download\n#train_df.to_csv('train_df_raw.csv', index = False)\n#test_df.to_csv('test_df_raw.csv', index = False)\n\ntrain_df.info()\ntest_df.info()\n\n# Any results you write to the current directory are saved as output.\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"cbb1853f80791e6468ae23c27c397c97394c3ee0"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6d7559a3ccc31548394c6da87ae05bbd7c6a05a1"},"cell_type":"markdown","source":"## <a id=\"2\">2.  Data Exploration </a> \n### &nbsp;&nbsp;  <a id=\"2.1\">2.1  Basic Data Info </a>\n"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true,"scrolled":false},"cell_type":"code","source":"# Set display\npd.options.display.max_columns=15\npd.options.display.max_rows=892\n\n# Some data snapshoots\ndes='''DESCRIPTION OF FEATURES:\nsurvival:    Survival \nPassengerId: Unique Id of a passenger. \npclass:    Ticket class     \nsex:    Sex     \nAge:    Age in years     \nsibsp:    # of siblings / spouses aboard the Titanic     \nparch:    # of parents / children aboard the Titanic     \nticket:    Ticket number     \nfare:    Passenger fare     \ncabin:    Cabin number     \nembarked:    Port of Embarkation'''\nprint(des)\nprint('\\n')\nprint('SNAPSHOOT OF TRAIN_DF')\ntrain_df.info()\nprint('\\n'+'SNAPSHOT OF TEST_DF')\ntest_df.info()\nprint('\\n')\n\nprint('BASIC DESCRIPTION')\nprint(train_df.describe())\nprint('\\n')\n\nprint('SNAPSHOT OF FIRST 8 RECORDS')\nprint(train_df.head(8))\nprint('\\n')\n\n# List missing values\nprint('MISSING VALUE SUMMARY')\ntotal = train_df.isnull().sum().sort_values(ascending=False)\npercent_1 = train_df.isnull().sum()/train_df.isnull().count()*100\npercent_2 = (round(percent_1, 1)).sort_values(ascending=False)\nmissing_data = pd.concat([total, percent_2], axis=1, keys=['Total', '%'])\nprint(missing_data.head(5))\ntrain_df.info()\ntest_df.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"662d37f90458526027c007d5a9d159d52b19d1a9"},"cell_type":"markdown","source":"### &nbsp;&nbsp;  <a id=\"2.2\">2.2  Feature Distributions </a> "},{"metadata":{"trusted":true,"_uuid":"e68b73ab340203be19a8263ec5c431f59051024b","scrolled":false},"cell_type":"code","source":"survived = 'survived'\nnot_survived = 'not survived'\nfig, axes = plt.subplots(nrows=1, ncols=2,figsize=(10, 4))\nwomen = train_df[train_df['Sex']=='female']\nmen = train_df[train_df['Sex']=='male']\nax = sns.distplot(women[women['Survived']==1].Age.dropna(), bins=18, label = survived, ax = axes[0], kde =False)\nax = sns.distplot(women[women['Survived']==0].Age.dropna(), bins=40, label = not_survived, ax = axes[0], kde =False)\nax.legend()\nax.set_title('Female')\nax = sns.distplot(men[men['Survived']==1].Age.dropna(), bins=18, label = survived, ax = axes[1], kde = False)\nax = sns.distplot(men[men['Survived']==0].Age.dropna(), bins=40, label = not_survived, ax = axes[1], kde = False)\nax.legend()\n_ = ax.set_title('Male')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e589ce9ab7d0fa337d3f053fc0c4c09829a935b3"},"cell_type":"code","source":"FacetGrid = sns.FacetGrid(train_df, row='Embarked', size=4.5, aspect=1.6)\nFacetGrid.map(sns.pointplot, 'Pclass', 'Survived', 'Sex', palette=None,  order=None, hue_order=None )\nFacetGrid.add_legend()\n\nprint('Interaction of Embarked & Sex')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"62bd26c9abbf05ba9123cec0de0b23cd63bce927"},"cell_type":"code","source":"sns.barplot(x='Pclass', y='Survived', data=train_df)\nprint('Further breakdown of Class')\n\ngrid = sns.FacetGrid(train_df, col='Survived', row='Pclass', size=2.2, aspect=1.6)\ngrid.map(plt.hist, 'Age', alpha=.5, bins=20)\ngrid.add_legend();\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c7ffce614d989beecc7713a88fffeeaf98abd0bd"},"cell_type":"markdown","source":"### &nbsp;&nbsp;  <a id=\"2.3\">2.3  Feature Creation/Deletion </a>"},{"metadata":{"trusted":true,"_uuid":"14ffc40cf71ca3f20d859de8be2bbd41542db582","collapsed":true},"cell_type":"code","source":"'''# Create/delete some features'''\ntrain_df.info()\ndata = [train_df, test_df]\nfor dataset in data:\n    dataset['relatives'] = dataset['SibSp'] + dataset['Parch']\n    dataset.loc[dataset['relatives'] > 0, 'not_alone'] = 0\n    dataset.loc[dataset['relatives'] == 0, 'not_alone'] = 1\n    dataset['not_alone'] = dataset['not_alone'].astype(int)\n\naxes = sns.factorplot('relatives','Survived', \n                      data=train_df, aspect = 2.5, )\nprint('Impact of Traveling with Relatives')\ntrain_df['not_alone'].value_counts()\ntrain_df.info()\ntest_df.info()\n\n# Delete PassengerId from train_df (not there, so does not need to be deleted)\ntrain_df = train_df.drop(['PassengerId'], axis=1)  \n# Drop Passenger Name\n#train_df = train_df.drop(['Name'], axis=1)   # Don't need to drop; not there","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"431c285141f4a55b5920bc9275eec6cb5e5a35d3"},"cell_type":"code","source":"# Look at correlation between key variables\n\n'''\nprint('Correlation of Pclass & relatives')\n#print(train_df.corr().loc['relatives','Pclass'])\nimport scipy.stats\nprint(scipy.stats.pearsonr(train_df['Pclass'].values,train_df['relatives'].values)[0],'    --using scipy.stats pearsonr')\nprint(train_df.corr().loc['Pclass','relatives'],'    --using pandas pearsonr \\n')\n\nprint('Correlation of relatives & Age')\nprint(train_df.corr().loc['relatives','Age'])\nprint('\\n')\n'''\ncor_dataset=train_df[['Survived','Pclass','Age','Fare','relatives','not_alone']]\nprint('Correlation Matrix')\n\nprint(cor_dataset.corr())\nprint(sns.heatmap(cor_dataset.corr()))\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3854636afb78d3292331e2edb542223601eb913b"},"cell_type":"markdown","source":"### &nbsp;&nbsp;  <a id=\"2.4\">2.4  Missing Data Imputations </a>"},{"metadata":{"trusted":true,"_uuid":"c816979a8b983fe9cc1c4956710444d77e9636c9","collapsed":true},"cell_type":"code","source":"'''\n# Drop PassengerId from the training set\ntrain_df = train_df.drop(['PassengerId'], axis=1)\n'''\n#train_df = train_df.drop(['PassengerId'], axis=1)   # Don't need to drop; not there\n# Convert deck first to alpha (A-), and then to numberic\nimport re\n\ndeck = {\"A\": 1, \"B\": 2, \"C\": 3, \"D\": 4, \"E\": 5, \"F\": 6, \"G\": 7, \"U\": 8}\ndata = [train_df, test_df]\n\nfor dataset in data:\n    dataset['Cabin'] = dataset['Cabin'].fillna(\"U0\")\n    dataset['Deck'] = dataset['Cabin'].map(lambda x: re.compile(\"([a-zA-Z]+)\").search(x).group())\n    dataset['Deck'] = dataset['Deck'].map(deck)\n    dataset['Deck'] = dataset['Deck'].fillna(0)\n    dataset['Deck'] = dataset['Deck'].astype(int)\n# we can now drop the cabin feature\ntrain_df = train_df.drop(['Cabin'], axis=1)\ntest_df = test_df.drop(['Cabin'], axis=1)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"da54efe664ccb65651720c57acf3d55bff4a32f3"},"cell_type":"code","source":"'''## Replace missing data in Age by using\n   # random numbers based on the mean age value in regards to the standard deviation and is_null\n'''\ndata = [train_df, test_df]\nfor dataset in data:\n    mean = train_df[\"Age\"].mean()\n    std = test_df[\"Age\"].std()\n    is_null = dataset[\"Age\"].isnull().sum()\n    # compute random numbers between the mean, std and is_null\n    rand_age = np.random.randint(mean - std, mean + std, size = is_null)\n    # fill NaN values in Age column with random values generated\n    age_slice = dataset[\"Age\"].copy()\n    age_slice[np.isnan(age_slice)] = rand_age\n    dataset[\"Age\"] = age_slice\n    dataset[\"Age\"] = train_df[\"Age\"].astype(int)\ntrain_df[\"Age\"].isnull().sum()     # check there are no null values\ntrain_df.info()\ntest_df.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0261be04e125fb846a51dc8fd4a9a7bfdd97040c","collapsed":true},"cell_type":"code","source":"'''\n## Fill the 2 embarked missing features with the most common values from embarked\n'''\n# Determine the most frequent value\ntrain_df['Embarked'].describe()\n\n# Fill missing values with 'S'\ncommon_value = 'S'\ndata = [train_df, test_df]\nfor dataset in data:\n    dataset['Embarked'] = dataset['Embarked'].fillna(common_value)\ntrain_df.info()\ntest_df.info()\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a8306b36a7095d1df3aff12ab9a16b947e226850"},"cell_type":"markdown","source":"### &nbsp;&nbsp;  <a id=\"2.5\">2.5  Number Conversions </a>"},{"metadata":{"trusted":true,"_uuid":"500fef3d7cd70b49bf4746d1f535c11218e1697b","collapsed":true},"cell_type":"code","source":"'''\n## Convert Fare from float to int64 using 'astype()'\n'''\ndata = [train_df, test_df]\n\nfor dataset in data:\n    dataset['Fare'] = dataset['Fare'].fillna(0)\n    dataset['Fare'] = dataset['Fare'].astype(int)\n    \ntrain_df.info()\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"da276e1d051774ed8cd6ad983c6edb4965fc4aef"},"cell_type":"markdown","source":"### &nbsp;&nbsp;  <a id=\"2.6\">2.6  Feature Extraction </a>"},{"metadata":{"trusted":true,"_uuid":"b0c077bef03f391fa9a3cb8d4ce2a9187955d9d1","collapsed":true},"cell_type":"code","source":"'''\n## Use the Name feature to extract the titles from the Name to build a new feature\n'''\ndata = [train_df, test_df]\ntitles = {\"Mr\": 1, \"Miss\": 2, \"Mrs\": 3, \"Master\": 4, \"Rare\": 5}\n\nfor dataset in data:\n    # extract titles\n    dataset['Title'] = dataset.Name.str.extract(' ([A-Za-z]+)\\.', expand=False)\n    # replace titles with a more common title or as Rare\n    dataset['Title'] = dataset['Title'].replace(['Lady', 'Countess','Capt', 'Col','Don', 'Dr',\\\n                                            'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Rare')\n    dataset['Title'] = dataset['Title'].replace('Mlle', 'Miss')\n    dataset['Title'] = dataset['Title'].replace('Ms', 'Miss')\n    dataset['Title'] = dataset['Title'].replace('Mme', 'Mrs')\n    # convert titles into numbers\n    dataset['Title'] = dataset['Title'].map(titles)\n    # filling NaN with 0, to get safe\n    dataset['Title'] = dataset['Title'].fillna(0)\ntrain_df = train_df.drop(['Name'], axis=1)\ntest_df = test_df.drop(['Name'], axis=1)\n\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8971320f764611d98013e8f69d83bab0db0fba53"},"cell_type":"code","source":"'''\n# Hot-Code Sex feature\n'''\ntrain_df=HotC(train_df,'Sex')\ntrain_df=train_df.drop(['Sex'], axis=1)\n\ntest_df=HotC(test_df,'Sex')\ntest_df=test_df.drop(['Sex'], axis=1)\n\npd.options.display.max_columns=20\nprint(train_df.head(10))\nprint(test_df.head(10))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dbd698f59bae067f83d8d6060329c313a5391c87","collapsed":true},"cell_type":"code","source":"'''\n# Drop Ticket from the data set\n'''\ntrain_df = train_df.drop(['Ticket'], axis=1)\ntest_df = test_df.drop(['Ticket'], axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ba5fb009612dfabebc25b8fea09a877d3b262303"},"cell_type":"code","source":"'''\n## Hot-Code Embarked and re-name columns\n'''\ntrain_df=HotC(train_df,'Embarked')\ntest_df=HotC(test_df,'Embarked')\n\n#Rename Embarked Hot-codes\ntrain_df=train_df.rename(index=str, columns={'C':'Emb_C','Q':'Emb_Q','S':'Emb_S'})\ntest_df=test_df.rename(index=str, columns={'C':'Emb_C','Q':'Emb_Q','S':'Emb_S'})\n\n#Drop Embarked Column\ntrain_df=train_df.drop(['Embarked'], axis=1)\ntest_df=test_df.drop(['Embarked'], axis=1)\n\nprint(train_df.head(10))\nprint(test_df.head(10))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"428bfcbf08271d536997d605c9a41d9a6ff70975","collapsed":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cb8acb2fa1a72a55d832a7d20b5d090d3896c10f"},"cell_type":"code","source":"'''\n## Create Categories for Age Feature\n'''\n\ndata = [train_df, test_df]\nfor dataset in data:\n    dataset['Age'] = dataset['Age'].astype(int)\n    dataset.loc[ dataset['Age'] <= 11, 'Age'] = 0\n    dataset.loc[(dataset['Age'] > 11) & (dataset['Age'] <= 18), 'Age'] = 1\n    dataset.loc[(dataset['Age'] > 18) & (dataset['Age'] <= 22), 'Age'] = 2\n    dataset.loc[(dataset['Age'] > 22) & (dataset['Age'] <= 27), 'Age'] = 3\n    dataset.loc[(dataset['Age'] > 27) & (dataset['Age'] <= 33), 'Age'] = 4\n    dataset.loc[(dataset['Age'] > 33) & (dataset['Age'] <= 40), 'Age'] = 5\n    dataset.loc[(dataset['Age'] > 40) & (dataset['Age'] <= 66), 'Age'] = 6\n    dataset.loc[ dataset['Age'] > 66, 'Age'] = 6\n\n# let's see how it's distributed\nprint('distribution of train_df')\ntrain_df['Age'].value_counts()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a301201ecbd4062224bbf2f9965ea0eb74ff1e12"},"cell_type":"code","source":"'''\n## Create categories for Fare\n'''\ndata = [train_df, test_df]\nfor dataset in data:\n    dataset.loc[ dataset['Fare'] <= 7.91, 'Fare'] = 0\n    dataset.loc[(dataset['Fare'] > 7.91) & (dataset['Fare'] <= 14.454), 'Fare'] = 1\n    dataset.loc[(dataset['Fare'] > 14.454) & (dataset['Fare'] <= 31), 'Fare']   = 2\n    dataset.loc[(dataset['Fare'] > 31) & (dataset['Fare'] <= 99), 'Fare']   = 3\n    dataset.loc[(dataset['Fare'] > 99) & (dataset['Fare'] <= 250), 'Fare']   = 4\n    dataset.loc[ dataset['Fare'] > 250, 'Fare'] = 5\n    dataset['Fare'] = dataset['Fare'].astype(int)\n    \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"45a3ddc209a7eda3c58f4c25b4b3df76995b6ad7","scrolled":true},"cell_type":"code","source":"'''\n## Create some additional variables\n'''\n# Age X Class\ndata = [train_df, test_df]\nfor dataset in data:\n    dataset['Age_Class']= dataset['Age']* dataset['Pclass']\n\n# Fare per Person\nfor dataset in data:\n    dataset['Fare_Per_Person'] = dataset['Fare']/(dataset['relatives']+1)\n    dataset['Fare_Per_Person'] = dataset['Fare_Per_Person'].astype(int)\n# Let's take a last look at the training set, before we start training the models.\ntrain_df.head(10)\n\nprint(train_df.head(10))\nprint(test_df.head(10)) \n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0f8b99824222cfc45e4110fa58b45793514bef5f"},"cell_type":"code","source":"'''## Try several algorithms to find the best'''\n\nprint(\"Results\") # Output Title\n\n## Fit Models to compare effectiveness\n# Define testing dataframes\nX_train = train_df.drop(\"Survived\", axis=1)\nY_train = train_df[\"Survived\"]\nX_test  = test_df.drop(\"PassengerId\", axis=1).copy()\n\n#print(X_test.head(10))\n\n#SGD-Stochastic Gradient Descent\nsgd = linear_model.SGDClassifier(max_iter=50, tol=None)\nsgd.fit(X_train, Y_train)\nY_pred = sgd.predict(X_test)\nsgd.score(X_train, Y_train)\nacc_sgd = round(sgd.score(X_train, Y_train) * 100, 2)\n\n#Random Forest\nrandom_forest = RandomForestClassifier(n_estimators=100)\nrandom_forest.fit(X_train, Y_train)\nY_prediction = random_forest.predict(X_test)\nrandom_forest.score(X_train, Y_train)\nacc_random_forest = round(random_forest.score(X_train, Y_train) * 100, 2)\n\n#Logistic Regression:\nlogreg = LogisticRegression()\nlogreg.fit(X_train, Y_train)\nY_pred = logreg.predict(X_test)\nacc_log = round(logreg.score(X_train, Y_train) * 100, 2)\n\n# K Nearest Neighbor\nknn = KNeighborsClassifier(n_neighbors = 3)\nknn.fit(X_train, Y_train)\nY_pred = knn.predict(X_test)\nacc_knn = round(knn.score(X_train, Y_train) * 100, 2)\n\n# Gaussian Naive Bayes\ngaussian = GaussianNB()\ngaussian.fit(X_train, Y_train)\nY_pred = gaussian.predict(X_test)\nacc_gaussian = round(gaussian.score(X_train, Y_train) * 100, 2)\n    \n# Perceptron:\nperceptron = Perceptron(max_iter=10)\nperceptron.fit(X_train, Y_train)\nY_pred = perceptron.predict(X_test)\nacc_perceptron = round(perceptron.score(X_train, Y_train) * 100, 2)\n\n# Linear Support Vector Machine\nlinear_svc = LinearSVC(max_iter=2000)\nlinear_svc.fit(X_train, Y_train)\nY_pred = linear_svc.predict(X_test)\nacc_linear_svc = round(linear_svc.score(X_train, Y_train) * 100, 2)\n\n'''\n# lgb_light\n# params = {'task': 'train', 'boosting_type': 'gbdt', 'objective': 'binary', 'metric': 'auc', \n          #'learning_rate': 0.01, 'num_leaves': 48, 'num_iteration': 5000, 'verbose': 0 ,\n          #'colsample_bytree':.8, 'subsample':.9, 'max_depth':7, 'reg_alpha':.1, 'reg_lambda':.1, \n          #'min_split_gain':.01, 'min_child_weight':1}\n\n# lgb_light = lgb.train(params, lgb_train, valid_sets=lgb_eval, early_stopping_rounds=150, verbose_eval=200)\nlgb_light = lgb\nlgb_light.fit(x_train,Y_train)\nY_pred = lgb_light.predict(X_test)\nacc_lgb_light = round(lgm_light.score(X_train, Y_train) * 100, 2)\n'''\n# Decision Tree\ndecision_tree = DecisionTreeClassifier()\ndecision_tree.fit(X_train, Y_train)\nY_pred = decision_tree.predict(X_test)\nacc_decision_tree = round(decision_tree.score(X_train, Y_train) * 100, 2)\n\nresults = pd.DataFrame({\n    'Model': ['LinearSVC','KNN', 'Logistic Regression', \n              'Random Forest', 'Naive Bayes', 'Perceptron', \n              'Stochastic Gradient Decent', \n              'Decision Tree'],\n    'Score': [acc_linear_svc, acc_knn, acc_log, \n              acc_random_forest, acc_gaussian, acc_perceptron, \n              acc_sgd, acc_decision_tree]})\nresult_df = results.sort_values(by='Score', ascending=False)\nresult_df = result_df.set_index('Score')\nresult_df.head(9)\n\n  \n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"885e7dd47a2a507142828feeb72b571c4849a3c4","collapsed":true},"cell_type":"code","source":"print (train_df.head(10))\nprint(test_df.head(10))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"af1e596d3ff0fc24520f527ea93ab4041fc61e24","collapsed":true},"cell_type":"code","source":"'''Conduct a K-Fold cross validation'''\nfrom sklearn.model_selection import cross_val_score\nrf = RandomForestClassifier(n_estimators=100)\nscores = cross_val_score(rf, X_train, Y_train, cv=10, scoring = \"accuracy\")\nprint(\"Scores:\", scores)\nprint(\"Mean:\", scores.mean())\nprint(\"Standard Deviation:\", scores.std())\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a1e8c99a71adf010f9e00c4bdb81f08f2eec418d","collapsed":true},"cell_type":"code","source":"# Check feature importance of the random forest\nimportances = pd.DataFrame({'feature':X_train.columns,'importance':np.round(random_forest.feature_importances_,3)})\nimportances = importances.sort_values('importance',ascending=False).set_index('feature')\nprint(importances.head(15))\n# Plot results\nimportances.plot.bar()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5ff3fd8ac0d6762b86ed9c7cb1c762eb0813544d","collapsed":true},"cell_type":"code","source":"\n'''Drop the least important features (Parch, Emb_S, Emb_C,Emb_Q)'''\ntrain_df  = train_df.drop(\"Emb_Q\", axis=1)\ntest_df  = test_df.drop(\"Emb_Q\", axis=1)\n\ntrain_df  = train_df.drop(\"Parch\", axis=1)\ntest_df  = test_df.drop(\"Parch\", axis=1)\n\ntrain_df  = train_df.drop(\"Emb_C\", axis=1)\ntest_df  = test_df.drop(\"Emb_C\", axis=1)\n\ntrain_df  = train_df.drop(\"Emb_S\", axis=1)\ntest_df  = test_df.drop(\"Emb_S\", axis=1)\n\ntrain_df  = train_df.drop(\"not_alone\", axis=1)\ntest_df  = test_df.drop(\"not_alone\", axis=1)\n\nprint(train_df.head(10))\nprint(test_df.head(10))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fa7bbbbf2ba407ebbdda03a806fc7be32db2caeb","collapsed":true},"cell_type":"code","source":"train_df  = train_df.drop(\"Sex\", axis=1)\ntest_df  = test_df.drop(\"Sex\", axis=1)\n\nprint(X_train.head(2))\nprint(Y_train.head(2))\nprint(X_test.head(2))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"eca8a3f27844c4fd59969e15d1eb9e7e3c1ee46e","collapsed":true},"cell_type":"code","source":"# Retrain random forest\n\n# Define testing dataframes\n# Define testing dataframes\nX_train = train_df.drop(\"Survived\", axis=1)\nY_train = train_df[\"Survived\"]\nX_test  = test_df.drop(\"PassengerId\", axis=1).copy()\n\n#Random Forest\nrandom_forest = RandomForestClassifier(n_estimators=100, oob_score = True)\nrandom_forest.fit(X_train, Y_train)\nY_prediction = random_forest.predict(X_test)\nrandom_forest.score(X_train, Y_train)\nacc_random_forest = round(random_forest.score(X_train, Y_train) * 100, 2)\n\nprint(round(acc_random_forest,2,), \"%\")\nprint(round(acc_random_forest,2,), \"%\")\nprint(importances.head(15))\n# Plot results\nimportances.plot.bar()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"95a5832e7f5612985107c89a764254c6ea516552","collapsed":true},"cell_type":"code","source":"print(\"oob score:\", round(random_forest.oob_score_, 4)*100, \"%\")","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}