{ "cells": [ { "cell_type": "markdown", "metadata": { "_cell_guid": "06b6ebc7-3acd-2667-1fa2-b488d417f877" }, "source": [ "**through the task to introduce the normal way to do data science work**" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "32d05eb8-df59-5026-67ed-82ff4e93aa0e" }, "outputs": [], "source": [ "# This Python 3 environment comes with many helpful analytics libraries installed\n", "# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n", "# For example, here's several helpful packages to load in \n", "\n", "import numpy as np # linear algebra\n", "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n", "\n", "# Input data files are available in the \"../input/\" directory.\n", "# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n", "\n", "from subprocess import check_output\n", "print(check_output([\"ls\", \"../input\"]).decode(\"utf8\"))\n", "\n", "# Any results you write to the current directory are saved as output." ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "d4c9869a-e440-3889-0c9a-5b4f5f6d260d" }, "source": [ "**import necessary libraries**" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "c09bb16d-5220-d759-8a1a-2ee0c5dbc869" }, "outputs": [], "source": [ "from sklearn import cross_validation\n", "from sklearn.ensemble import RandomForestClassifier\n", "import re,operator\n", "import matplotlib.pyplot as plt\n", "from sklearn.feature_selection import SelectKBest, f_classif\n", "from sklearn.ensemble import GradientBoostingClassifier\n", "from sklearn.linear_model import LinearRegression,LogisticRegression\n", "from sklearn.cross_validation import KFold" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "0a8e86c3-5338-6565-2cb9-89571f033fe1" }, "source": [ "**load data and have a quick look**" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "60005513-251f-46a0-f767-efef08329fa3" }, "outputs": [], "source": [ "train = pd.read_csv(\"../input/train.csv\" )\n", "test = pd.read_csv(\"../input/test.csv\")\n", "print(\"\\n\\nTop of the training data:\")\n", "print(train.head())\n", "print(\"\\n\\nSummary statistics of training data\")\n", "print(train.describe())\n", "\n", "print(\"\\n\\nTop of the testing data:\")\n", "print(test.head())\n", "print(\"\\n\\nSummary statistics of testing data\")\n", "print(test.describe())" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "aa17f21f-4c4c-27d5-4d78-16c49317af19" }, "source": [ "**clean the original data**" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "bf21ea0d-6fda-46cd-f85b-da29c0628523" }, "outputs": [], "source": [ "train[\"Age\"] = train[\"Age\"].fillna(train[\"Age\"].median())\n", "test[\"Age\"] = test[\"Age\"].fillna(test[\"Age\"].median())\n", "test[\"Fare\"] = test[\"Fare\"].fillna(test[\"Fare\"].median())\n", "\n", "#Handling Non-Numeric Columns\n", "train.loc[train[\"Sex\"] == \"male\", \"Sex\"] = 0\n", "train.loc[train[\"Sex\"] == \"female\", \"Sex\"] = 1 \n", "\n", "train[\"Embarked\"] = train[\"Embarked\"].fillna(\"S\")\n", "\n", "train.loc[train[\"Embarked\"] == \"S\", \"Embarked\"] = 0\n", "train.loc[train[\"Embarked\"] == \"C\", \"Embarked\"] = 1\n", "train.loc[train[\"Embarked\"] == \"Q\", \"Embarked\"] = 2\n", "\n", "test.loc[test[\"Sex\"] == \"male\", \"Sex\"] = 0\n", "test.loc[test[\"Sex\"] == \"female\", \"Sex\"] = 1 \n", "\n", "test[\"Embarked\"] = test[\"Embarked\"].fillna(\"S\")\n", "\n", "test.loc[test[\"Embarked\"] == \"S\", \"Embarked\"] = 0\n", "test.loc[test[\"Embarked\"] == \"C\", \"Embarked\"] = 1\n", "test.loc[test[\"Embarked\"] == \"Q\", \"Embarked\"] = 2" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "f3f90899-bec7-1ade-531d-54f5bebc9f62" }, "source": [ "**add new features**" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "98648f67-735b-3e77-99cc-783778455608" }, "outputs": [], "source": [ "train[\"FamilySize\"] = train[\"SibSp\"] + train[\"Parch\"]\n", "test[\"FamilySize\"] = test[\"SibSp\"] + test[\"Parch\"]\n", "\n", "# The .apply method generates a new series\n", "train[\"NameLength\"] = train[\"Name\"].apply(lambda x: len(x))\n", "test[\"NameLength\"] = test[\"Name\"].apply(lambda x: len(x))" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "12c8c7c7-f4b2-54cb-fa56-c1aa48451ed1" }, "outputs": [], "source": [ "# Extracting The Passengers' Titles With A Regular Expression\n", "# A function to get the title from a name\n", "def get_title(name):\n", " # Use a regular expression to search for a title \n", " # Titles always consist of capital and lowercase letters, and end with a period\n", " title_search = re.search(' ([A-Za-z]+)\\.', name)\n", " # If the title exists, extract and return it\n", " if title_search:\n", " return title_search.group(1)\n", " return \"\"\n", "\n", "titles = train[\"Name\"].apply(get_title)\n", "print(pd.value_counts(titles))" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "69569027-509b-4a56-5548-c460dfd40920" }, "outputs": [], "source": [ "# Map each title to an integer \n", "# Some titles are very rare, so they're compressed into the same codes as other titles\n", "title_mapping = {\"Mr\": 1, \"Miss\": 2, \"Mrs\": 3, \"Master\": 4, \"Dr\": 5, \"Rev\": 6, \"Major\": 7, \"Col\": 7, \"Mlle\": 8, \"Mme\": 8, \"Don\": 9, \"Lady\": 10, \"Countess\": 10, \"Jonkheer\": 10, \"Sir\": 9, \"Capt\": 7, \"Ms\": 2}\n", "for k,v in title_mapping.items():\n", " titles[titles == k] = v\n", "\n", "# Verify that we converted everything\n", "# print(pandas.value_counts(titles))\n", "# Add in the title column\n", "train[\"Title\"] = titles" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "317e2241-ef7a-11d5-1720-070c139cf08d" }, "outputs": [], "source": [ "# First, we'll add titles to the test set\n", "titles = test[\"Name\"].apply(get_title)\n", "print(pd.value_counts(titles))\n", "# We're adding the Dona title to the mapping, because it's in the test set, but not the training set\n", "title_mapping = {\"Mr\": 1, \"Miss\": 2, \"Mrs\": 3, \"Master\": 4, \"Dr\": 5, \"Rev\": 6, \"Major\": 7, \"Col\": 7, \"Mlle\": 8, \"Mme\": 8, \"Don\": 9, \"Lady\": 10, \"Countess\": 10, \"Jonkheer\": 10, \"Sir\": 9, \"Capt\": 7, \"Ms\": 2, \"Dona\": 10}\n", "for k,v in title_mapping.items():\n", " titles[titles == k] = v\n", "test[\"Title\"] = titles" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "766f0afc-40cb-61d9-4183-3ea84b9034fc" }, "outputs": [], "source": [ "# Generating A Feature For Family Groups\n", "# A dictionary mapping family name to ID\n", "family_id_mapping = {}\n", "\n", "# A function to get the ID for a particular row\n", "def get_family_id(row):\n", " # Find the last name by splitting on a comma\n", " last_name = row[\"Name\"].split(\",\")[0]\n", " # Create the family ID\n", " family_id = \"{0}{1}\".format(last_name, row[\"FamilySize\"])\n", " # Look up the ID in the mapping\n", " if family_id not in family_id_mapping:\n", " if len(family_id_mapping) == 0:\n", " current_id = 1\n", " else:\n", " # Get the maximum ID from the mapping, and add 1 to it if we don't have an ID\n", " current_id = (max(family_id_mapping.items(), key=operator.itemgetter(1))[1] + 1)\n", " family_id_mapping[family_id] = current_id\n", " return family_id_mapping[family_id]" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "5103b6b2-f8fa-3e45-e087-479798e732c9" }, "outputs": [], "source": [ "# Get the family IDs with the apply method\n", "family_ids = train.apply(get_family_id, axis=1)\n", "# There are a lot of family IDs, so we'll compress all of the families with less than three members into one code\n", "family_ids[train[\"FamilySize\"] < 3] = -1\n", "# Print the count of each unique ID\n", "#print(pd.value_counts(family_ids))\n", "train[\"FamilyId\"] = family_ids\n", "print(train.head())\n", "# Get the family IDs with the apply method\n", "family_ids = test.apply(get_family_id, axis=1)\n", "# There are a lot of family IDs, so we'll compress all of the families with less than three members into one code\n", "family_ids[test[\"FamilySize\"] < 3] = -1\n", "test[\"FamilyId\"] = family_ids\n", "print(test.head())" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "51aea344-4add-9d6d-8d12-21ef5e5f5e97" }, "source": [ "**Identifying The Best Features To Use**" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "3897f3f8-ccb1-e088-3ff4-396c1a14ef41" }, "outputs": [], "source": [ "predictors = [\"Pclass\", \"Sex\", \"Age\", \"SibSp\", \"Parch\", \"Fare\", \"Embarked\", \"FamilySize\", \"Title\", \"FamilyId\", \"NameLength\"]\n", "\n", "# Perform feature selection\n", "selector = SelectKBest(f_classif, k=5)\n", "selector.fit(train[predictors], train[\"Survived\"])\n", "\n", "scores = -np.log10(selector.pvalues_)\n", "\n", "# Plot the scores \n", "# Do you see how \"Pclass\", \"Sex\", \"Title\", and \"Fare\" are the best features?\n", "plt.bar(range(len(predictors)), scores)\n", "plt.xticks(range(len(predictors)), predictors, rotation='vertical')\n", "plt.show()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "56352e39-0530-7530-a7f1-8c7435e51a63" }, "outputs": [], "source": [ "# Pick only the four best features\n", "#predictors = [\"Pclass\", \"Sex\", \"Fare\", \"Title\"]\n", "predictors = [\"Pclass\", \"Sex\", \"Age\", \"SibSp\", \"Parch\", \"Fare\", \"Embarked\", \"FamilySize\", \"Title\", \"FamilyId\", \"NameLength\"]\n", "alg = RandomForestClassifier(random_state=1, n_estimators=50, min_samples_split=8, min_samples_leaf=4)\n", "# Compute the accuracy score for all the cross-validation folds; this is much simpler than what we did before\n", "scores = cross_validation.cross_val_score(alg, train[predictors], train[\"Survived\"], cv=3)\n", "\n", "# Take the mean of the scores (because we have one for each fold)\n", "print(scores.mean())" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "d91fefe1-4474-59c5-4c18-1abe9514853d" }, "source": [ "**Making Predictions With Multiple Classifiers and cross-validation**" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "4ddf2546-3248-0c30-c1b3-854baccbd7bb" }, "outputs": [], "source": [ "#Making Predictions With Multiple Classifiers\n", "algorithms = [\n", " [GradientBoostingClassifier(random_state=1, n_estimators=25, max_depth=3), [\"Pclass\", \"Sex\", \"Age\", \"Fare\", \"Embarked\", \"FamilySize\", \"Title\", \"FamilyId\"]],\n", " [LogisticRegression(random_state=1), [\"Pclass\", \"Sex\", \"Fare\", \"FamilySize\", \"Title\", \"Age\", \"Embarked\"]]\n", "]\n", "\n", "# Initialize the cross-validation folds\n", "kf = KFold(train.shape[0], n_folds=3, random_state=1)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "e022e523-dcfe-4671-07db-926ac192bdf0" }, "outputs": [], "source": [ "predictions = []\n", "for train_tmp, test_tmp in kf:\n", " train_target = train[\"Survived\"].iloc[train_tmp]\n", " full_test_predictions = []\n", " # Make predictions for each algorithm on each fold\n", " for alg, predictors in algorithms:\n", " # Fit the algorithm on the training data\n", " alg.fit(train[predictors].iloc[train_tmp,:], train_target)\n", " # Select and predict on the test fold \n", " # We need to use .astype(float) to convert the dataframe to all floats and avoid an sklearn error\n", " test_predictions = alg.predict_proba(train[predictors].iloc[test_tmp,:].astype(float))[:,1]\n", " full_test_predictions.append(test_predictions)\n", " # Use a simple ensembling scheme—just average the predictions to get the final classification\n", " \n", " test_predictions = (full_test_predictions[0] + full_test_predictions[1]) / 2\n", " # Any value over .5 is assumed to be a 1 prediction, and below .5 is a 0 prediction\n", " test_predictions[test_predictions <= .5] = 0\n", " test_predictions[test_predictions > .5] = 1\n", " predictions.append(test_predictions)\n", "\n", "# Put all the predictions together into one array\n", "predictions = np.concatenate(predictions, axis=0)\n", "\n", "# Compute accuracy by comparing to the training data\n", "accuracy = sum(predictions[predictions == train[\"Survived\"]]) / len(predictions)\n", "print(accuracy)" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "3ca93651-8762-7ed0-8ad2-3ce51cfe510b" }, "source": [ "**Predicting On The Test Set and submission**" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "cf706229-5333-af5c-d20c-7d68bb0946de" }, "outputs": [], "source": [ "#Predicting On The Test Set\n", "#predictors = [\"Pclass\", \"Sex\", \"Age\", \"Fare\", \"Embarked\", \"FamilySize\", \"Title\", \"FamilyId\"]\n", "algorithms = [\n", " [RandomForestClassifier(random_state=1, n_estimators=50, min_samples_split=8, min_samples_leaf=4), [\"Pclass\", \"Sex\", \"Age\", \"Fare\", \"Embarked\", \"FamilySize\", \"Title\", \"FamilyId\"]],\n", " #[GradientBoostingClassifier(random_state=1, n_estimators=25, max_depth=3),[\"Pclass\", \"Sex\", \"Age\", \"Fare\", \"Embarked\", \"FamilySize\", \"Title\", \"FamilyId\"]]\n", "]\n", "\n", "full_predictions = []\n", "#print(test[predictors].iloc[:])\n", "#print(type(test[predictors]))\n", "for alg, predi in algorithms:\n", " # Fit the algorithm using the full training data.\n", " alg.fit(train[predi], train[\"Survived\"])\n", " # Predict using the test dataset. We have to convert all the columns to floats to avoid an error\n", " predictions = alg.predict_proba(test[predi].astype(float))[:,1]\n", " #predicitons = alg.predict(test[predi])\n", " full_predictions.append(predictions)\n", "\n", "# The gradient boosting classifier generates better predictions, so we weight it higher\n", "#predictions = (full_predictions[0] * 3 + full_predictions[1]) / 4\n", "#predictions[predictions <= .5] = 0\n", "#predictions[predictions > .5] = 1\n", "predictions = predictions.astype(int)\n", "submission = pd.DataFrame({\n", " \"PassengerId\": test[\"PassengerId\"],\n", " \"Survived\": predictions\n", " })\n", "submission.to_csv(\"kaggle.csv\", index=False)" ] } ], "metadata": { "_change_revision": 0, "_is_fork": false, "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.6.0" } }, "nbformat": 4, "nbformat_minor": 0 }