{ "cells": [ { "cell_type": "markdown", "metadata": { "_cell_guid": "f217caa5-171b-8e17-0576-0b698980029c" }, "source": [ "# Learning Titanic" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "2f939a18-23c8-f1a4-c7d2-78f61c70ddc2" }, "outputs": [], "source": [ "import pandas\n", "import numpy\n", "import re\n", "from sklearn.ensemble import RandomForestRegressor, RandomForestClassifier\n", "from sklearn.model_selection import train_test_split\n", "from sklearn.metrics import mean_absolute_error\n", "from sklearn import preprocessing\n", "from sklearn import linear_model\n", "import matplotlib.pyplot as plt\n", "%matplotlib notebook" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "11de7743-e916-47e2-c787-e27622df55b6" }, "source": [ "\n", "\n", "## load data" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "e0964e8b-6963-4a45-8a13-387d1238b60c" }, "outputs": [], "source": [ "TRAIN_PATH = \"../input/train.csv\"\n", "TEST_PATH = \"../input/test.csv\"\n", "train = pandas.read_csv(TRAIN_PATH)\n", "test = pandas.read_csv(TEST_PATH)" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "0f793914-ccc4-f103-87ee-b83cc986f4a1" }, "source": [ "## identify columns that have blank values" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "4327a1e6-7273-1d34-5457-b4572e1e9748" }, "outputs": [], "source": [ "train.isnull().any()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "bb1f0260-17a6-465e-a142-eb37d3165825" }, "outputs": [], "source": [ "test.isnull().any()" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "e9250a26-5faa-a064-ef23-b970257897d4" }, "source": [ "### age, embarked, fare and cabin all contain blank values: we'll fill later\n", "* age could depend on various features, we'll investigate these then use regression to fill in the blanks\n", "* The majority of passengers embarked from Southampton. We'll just fill the small number of blanks with 'S'.\n", "* First impressions are that fare doesn't seem particularly useful. It probably can't tell us much more than point of embarkation, class of travel and number of cabins/passengers.\n", "* Cabin appears too sparsely populated to fill reliably." ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "f721fd7a-dcc1-1e8c-7cd9-d096aeae7434" }, "source": [ "## derive titles" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "ded54444-1b7d-5134-de7b-44c232680bec" }, "outputs": [], "source": [ "def deriveTitles(s):\n", " title = re.search('(?:\\S )(?P\\w*)',s).group('title')\n", " if title == \"Mr\": return \"adult\"\n", " elif title == \"Don\": return \"gentry\"\n", " elif title == \"Dona\": return \"gentry\"\n", " elif title == \"Miss\": return \"miss\" # we don't know whether miss is an adult or a child\n", " elif title == \"Col\": return \"military\"\n", " elif title == \"Rev\": return \"other\"\n", " elif title == \"Lady\": return \"gentry\"\n", " elif title == \"Master\": return \"child\"\n", " elif title == \"Mme\": return \"adult\"\n", " elif title == \"Captain\": return \"military\"\n", " elif title == \"Dr\": return \"other\"\n", " elif title == \"Mrs\": return \"adult\"\n", " elif title == \"Sir\": return \"gentry\"\n", " elif title == \"Jonkheer\": return \"gentry\"\n", " elif title == \"Mlle\": return \"miss\"\n", " elif title == \"Major\": return \"military\"\n", " elif title == \"Ms\": return \"miss\"\n", " elif title == \"the Countess\": return \"gentry\" \n", " else: return \"other\"\n", " \n", "train['title'] = train.Name.apply(deriveTitles)\n", "test['title'] = test.Name.apply(deriveTitles)\n", "\n", "# and encode these new titles for later\n", "le = preprocessing.LabelEncoder()\n", "titles = ['adult', 'gentry', 'miss', 'military', 'other', 'child']\n", "le.fit(titles)\n", "train['encodedTitle'] = le.transform(train['title']).astype('int')\n", "test['encodedTitle'] = le.transform(test['title']).astype('int')" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "fb95e6fe-5c6e-ae8d-942e-34d5a0fe1053" }, "source": [ "## fill embark\n", "note: embarked only has holes in the training data" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "ad5be62a-4938-3c1b-9980-1fb6cf60671a" }, "outputs": [], "source": [ "train.Embarked.fillna(value = 'S', inplace=True)" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "6d4ef117-2e4c-d644-c019-048c1a19d8fc" }, "source": [ "## does this passenger have more than one cabin?\n", "is this a good indication that they have people to help them?" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "c7896cc9-3cf4-3615-ac3d-36cf860bec08", "collapsed": true }, "outputs": [], "source": [ "# not expected to add significant value because cabin data is so sparse" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "e1f32f74-754b-af9e-def2-f4e8b4fb3a73" }, "source": [ "### Both test and training data sets have missing ages and both have useful insight into how these can be filled" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "8b8a9e66-034a-b153-0c92-0941475385d4" }, "outputs": [], "source": [ "combined = pandas.concat([train, test])\n", "# combining train and test casts Survived from int to float because all Survived values in test are blank\n", "combined.ParChCategories = combined.Parch > 2" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "83c0026f-23ff-5b13-565d-c0710cc7c509" }, "source": [ "## plot features by age to determine which ones might be useful" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "936d85a2-c6a1-c351-d831-f5be8615d7a5" }, "outputs": [], "source": [ "combined.boxplot(column='Age', by='Pclass')" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "5699e084-ee83-5856-d072-a5ca26ddfd49" }, "source": [ "# learn the missing ages\n", "* sex and embarked doesn't seem significant\n", "* class more-or-less linear: use as feature\n", "* SibSp: less than 2: could be anything, 2 and 3 gives medium range, >3 low age. Use these groups as categorical features\n", "* Parch: < 3 could be any age, > 3 and always older. Use these groups as categorical features\n", "* familySize appears to be a combination of the above two so ignore\n", "* title has a huge impact, use it as a categorical feature" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "52b9c84b-2300-bf18-99c7-3e7b3652e6ed" }, "source": [ "### calculate new features based on number of siblings, spouses, parents and children" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "3c797646-895e-a935-635b-0152d387e518" }, "outputs": [], "source": [ "combined = combined.assign(SibSpGroup1 = combined['SibSp'] < 2)\n", "combined = combined.assign(SibSpGroup2 = combined['SibSp'].between(2, 3, inclusive=True))\n", "combined = combined.assign(SibSpGroup3 = combined['SibSp'] > 2)\n", "combined = combined.assign(ParChGT2 = combined['Parch'] > 2)" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "777d9821-de05-4b23-38ba-ecda384171bd" }, "source": [ "### split combined into those with an age and those without\n", "we have just over 1,000 in the training set so let's take 20% for validation\n", "Note: The fact that the split is random affects the accuracy of the two models below. Both vary between 8 and 9.5. It may be worth investigating further to improve results." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "e38d4115-b7f8-d984-eb5f-6fe61edab883" }, "outputs": [], "source": [ "age_train, age_validation = train_test_split(combined[combined.Age.notnull()], test_size = 0.2)\n", "age_learn = combined[combined.Age.isnull()]" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "3643c184-eb95-842f-8c05-6d5d21ea7879" }, "source": [ "### impute ages using a random forest regressor\n", "note that scikit learn treats features of type int as categorical\n", "\n", "http://stackoverflow.com/questions/20095187/regression-trees-or-random-forest-regressor-with-categorical-inputs" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "6ebf43aa-3c9a-093b-f0be-7ad5cf6ff506" }, "source": [ "#### Random Forest" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "38b36ba8-9d94-3c6f-6fee-a865d2f76c3d" }, "outputs": [], "source": [ "age_rf = RandomForestRegressor()\n", "age_rf.fit(age_train[['Pclass', 'encodedTitle', 'SibSpGroup1', 'SibSpGroup2', 'SibSpGroup3', 'ParChGT2']], age_train['Age'])\n", "age_validation = age_validation.assign(rf_age = age_rf.predict(age_validation[['Pclass', 'encodedTitle', 'SibSpGroup1', 'SibSpGroup2', 'SibSpGroup3', 'ParChGT2']]))\n", "mean_absolute_error(age_validation['Age'], age_validation['rf_age'], sample_weight=None, multioutput='uniform_average')" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "eed612c3-c605-5578-23b7-e9922bcbe21a" }, "source": [ "#### Linear Regression\n", "1. onehot encode categorical features\n", "2. scale features - not required since all features are categorical" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "a01197b6-3cb6-6685-c56d-dbcf4ed9fa7d" }, "outputs": [], "source": [ "age_encoder = preprocessing.OneHotEncoder().fit(combined[['Pclass', 'encodedTitle', 'SibSpGroup1', 'SibSpGroup2', 'SibSpGroup3', 'ParChGT2']])\n", "age_training_encoded = age_encoder.transform(age_train[['Pclass', 'encodedTitle', 'SibSpGroup1', 'SibSpGroup2', 'SibSpGroup3', 'ParChGT2']])\n", "age_validation_encoded = age_encoder.transform(age_validation[['Pclass', 'encodedTitle', 'SibSpGroup1', 'SibSpGroup2', 'SibSpGroup3', 'ParChGT2']])\n", "age_model = linear_model.RidgeCV(alphas = [0.1, 0.2, 0.3, 0.4, 0.5])\n", "age_estimator = age_model.fit(age_training_encoded, age_train['Age'])\n", "linear_age_predictions = age_estimator.predict(age_validation_encoded)\n", "mean_absolute_error(age_validation['Age'], linear_age_predictions, sample_weight=None, multioutput='uniform_average')" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "e12b0d66-92da-daff-e69c-2e5587ac6bcf" }, "source": [ "The fandom forest model gets slightly better results most of the time (deoending on the random split of data above) so we'll use it." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "4716bd09-4703-9d57-5582-ad5ede572e8c" }, "outputs": [], "source": [ "age_learn = age_learn.assign(Age = age_rf.predict(age_learn[['Pclass', 'encodedTitle', 'SibSpGroup1', 'SibSpGroup2', 'SibSpGroup3', 'ParChGT2']]))" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "9986849c-c753-c109-4e23-4d90188f91ee", "collapsed": true }, "source": [ "### fill the combined data set with the imputed ages and then split back into training and test\n", "need to add an index to each dataframe to enable the join" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "9ad2363e-3f01-786c-7428-c8c7786b80e7" }, "outputs": [], "source": [ "age_learn.set_index('PassengerId', inplace=True, drop=False)\n", "combined.set_index('PassengerId', inplace=True, drop=False)\n", "combined.update(age_learn, join = 'left', overwrite = False)\n", "# careful here... update changes int columns to floats\n", "# https://github.com/pandas-dev/pandas/issues/4094\n", "# this could be problematic later if they're not changed back since\n", "# int features are treated as categorical and floats are not" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "77c3bc56-bccd-fadd-590e-abf0e8213af5" }, "source": [ "## derived family based features" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "ccd7a5d0-e33e-6982-e31f-63de4b772cf5" }, "outputs": [], "source": [ "combined = combined.assign(familySize = combined['Parch'] + combined['SibSp'])\n", "\n", "def deriveChildren(age, parch):\n", " if(age < 18): return parch\n", " else: return 0\n", "\n", "combined = combined.assign(children = combined.apply(lambda row: deriveChildren(row['Age'], row['Parch']), axis = 1))\n", "# train['children'] = train.apply(lambda row: deriveChildren(row['Age'], row['Parch']), axis = 1)\n", "# test['children'] = test.apply(lambda row: deriveChildren(row['Age'], row['Parch']), axis = 1)\n", "# I think (but am not certain) the commented code above is functionally equivalent to the preceeding two lines,\n", "# but the commented lines gave settingdwithcopy warnings. I think these were false postives but am not certain.\n", "\n", "def deriveParents(age, parch):\n", " if(age > 17): return parch\n", " else: return 0\n", " \n", "combined['parents'] = combined.apply(lambda row: deriveParents(row['Age'], row['Parch']), axis = 1)\n", " \n", "def deriveResponsibleFor(children, SibSp):\n", " if(children > 0): return children / (SibSp + 1)\n", " else: return 0\n", " \n", "combined['responsibleFor'] = combined.apply(lambda row: deriveResponsibleFor(row['children'], row['SibSp']), axis = 1)\n", " \n", "def deriveAccompaniedBy(parents, SibSp):\n", " if(parents > 0): return parents / (SibSp + 1)\n", " else: return 0\n", " \n", "combined['accompaniedBy'] = combined.apply(lambda row: deriveAccompaniedBy(row['parents'], row['SibSp']), axis = 1)\n", " \n", "def unaccompaniedChild(age, parch):\n", " if((age < 16) & (parch == 0)): return True\n", " else: return False\n", "\n", "combined['unaccompaniedChild'] = combined.apply(lambda row: unaccompaniedChild(row['Age'], row['Parch']), axis = 1)" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "a2c176f7-da14-3fa3-cfb2-47a6f6a4ff56" }, "source": [ "### derive passengers likely location aboard the ship based on cabin number" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "c790810d-9bb5-f8f0-3950-f164453d7522" }, "outputs": [], "source": [ "# may not be worth doing given how sparsely populated cabin data is" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "b051ad52-2529-69b6-1640-481957529fb0" }, "source": [ "## Random Forest Survival Prediction\n", "As noted above, scikit learn treats integer features as categorical. Preprocessing has set all integers as floats. These need returning to the correct type so they are handled as expected. Non-numeric values will also need converting.\n", "\n", "Preprocessing of the data frames have left redundant data. The model will use:\n", "- age (continuous)\n", "- embarked (categorical)\n", "- Pclass (continuous) (interesting whether this is actually continuous or whether each class is a category)\n", "- Sex (categorical)\n", "- encodedTitle (categorical)\n", "- SibSpGroups 1 to 3 (categorical)\n", "- familySize (continuous)\n", "- children (continuous)\n", "- parents (continuous)\n", "- responsibleFor (continuous)\n", "- accompaniedBy (continuous)\n", "- unaccompaniedChild (categorical)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "f36a5382-ab71-1777-edbe-3d514029f547" }, "outputs": [], "source": [ "# drop unused columns\n", "combined = combined.drop(['Name', 'Cabin', 'Fare', 'Parch', 'SibSp', 'Ticket', 'title'], axis=1)\n", "# confirm types\n", "combined.dtypes" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "45df3c0a-9e54-1b8d-e834-ae100951492e" }, "outputs": [], "source": [ "# label encode string features\n", "categorical_names = {}\n", "categorical_features = ['Embarked', 'Sex']\n", "for feature in categorical_features:\n", " le = preprocessing.LabelEncoder()\n", " le.fit(combined[feature])\n", " combined[feature] = le.transform(combined[feature])\n", " categorical_names[feature] = le.classes_\n", " \n", "#combined = combined.assign(encodedTitleInt = combined['encodedTitle'].astype(int, copy=False))\n", "combined['title'] = combined['encodedTitle'].astype(int, copy=False)\n", "combined['class'] = combined['Pclass'].astype(int, copy=False)\n", "combined = combined.drop(['Pclass'], axis=1)\n", "combined = combined.drop(['encodedTitle'], axis=1)\n", "\n", "train = combined[combined.PassengerId < 892]\n", "test = combined[combined.PassengerId > 891]\n", "test = test.drop(['Survived'], axis=1)\n", "\n", "train['Survived'] = train['Survived'].astype(int, copy=False)\n", "# the warning below is a false positive since the copy input is set to false" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "a81a26ba-57bc-5732-80ef-400b07272762" }, "outputs": [], "source": [ "test.dtypes" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "3fe19dd4-6587-7682-8edb-0ab2291f1615" }, "outputs": [], "source": [ "rf = RandomForestClassifier()\n", "rf.fit(train[['title', \n", " 'Age', \n", " 'Embarked', \n", " 'class', \n", " 'Sex', \n", " 'SibSpGroup1', \n", " 'SibSpGroup2', \n", " 'SibSpGroup3', \n", " 'familySize', \n", " 'children', \n", " 'parents', \n", " 'responsibleFor', \n", " 'accompaniedBy', \n", " 'unaccompaniedChild']], train['Survived'])\n", "\n", "test = test.assign(Survived = rf.predict(test[['title', \n", " 'Age', \n", " 'Embarked', \n", " 'class', \n", " 'Sex', \n", " 'SibSpGroup1', \n", " 'SibSpGroup2', \n", " 'SibSpGroup3', \n", " 'familySize', \n", " 'children', \n", " 'parents', \n", " 'responsibleFor', \n", " 'accompaniedBy', \n", " 'unaccompaniedChild']]))" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "b3829215-0fd7-f1de-2dbf-f31415559945" }, "source": [ "# Output the results to CSV" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "70ba47b6-f6f1-427a-0ad3-f1cf48b920d0" }, "outputs": [], "source": [ "test[['Survived']].to_csv(path_or_buf='~/output.csv')" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "9076c186-7131-ac9d-30f1-41966b68a5c8" }, "outputs": [], "source": [ "" ] } ], "metadata": { "_change_revision": 0, "_is_fork": false, "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.6.0" } }, "nbformat": 4, "nbformat_minor": 0 }