{"cells":[{"metadata":{"trusted":false,"_uuid":"2f3371ca82f7408e4d04de35cb78bafd367d7e2a"},"cell_type":"code","source":"import pandas as pd\nimport numpy as np","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"5c4d0f29c8a1ad102671a48f07f7eb89ba3280d8"},"cell_type":"code","source":"train = pd.read_csv('../input/train.csv')\ntest = pd.read_csv('../input/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"66bf48694070e116ece6a37582f551b9a3593688"},"cell_type":"code","source":"test_key = test.PassengerId","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":false,"_uuid":"d5018e27fab5bbe749d33959927ff066eded1bc7"},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"305c05555a58726b598af86f2add318589493fbe"},"cell_type":"code","source":"train.shape, test.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"b90325f79db3aa05d89bb8957062d1b8206f01d6"},"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\nsns.set()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"c78beab8e6d03c6fcda7c23be0e7ac734946f851"},"cell_type":"code","source":"train_test_data = [train, test]\nfor dataset in train_test_data:\n    dataset['Title'] = dataset['Name'].str.extract(' ([A-Za-z]+)\\.', expand=False)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d638125729bad4fab2bc005284ad6bbed1dda073"},"cell_type":"markdown","source":"# Analyze Missing values with sns heatmap"},{"metadata":{"scrolled":false,"trusted":false,"_uuid":"b083e8c09fdf8ffbd5a9b9784d7d1561c5878311"},"cell_type":"code","source":"nan = dataset.isnull().sum()\nplt.figure(figsize=(20,5))\nidx_nan = nan.mask(nan==0).dropna().index\nsns.heatmap(dataset[idx_nan].transpose().isnull(), cmap = 'binary')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"64ae280533868c7bba34c1f6bd1a7526d0c38c7a"},"cell_type":"markdown","source":"# Categorical features plot info\nWhit this custom funcion I can plot toghere more than 1 categorical feature with this informations:\n% of missing values for classes: (Survived, Dead)\nSee % distribution of the categorial feature on dependent variable"},{"metadata":{"trusted":false,"_uuid":"68d6c57e1d8d111e0c5cb7eebdcc63e653463f91"},"cell_type":"code","source":"def get_categorical_feature_info(data, column='', y_column='', positive_state='', negative_state='',\\\n                                  save_fig=False, path='', figsize=(10,10)):\n    if column != y_column:\n        total_positive = data[y_column].sum()\n        total_negative = len(data)-total_positive\n        positive = data[data[y_column]==1][column].value_counts()/total_positive\n        negative = data[data[y_column]==0][column].value_counts()/total_negative\n        df = pd.DataFrame([positive, negative])\n        ind = (0,1)\n        ax = df.plot(kind='bar',stacked=True, figsize=figsize)\n        plt.xticks(ind, (positive_state, negative_state), fontsize=15)\n        plt.yticks(np.arange(0, 1.1, step=0.1), fontsize=14)\n        plt.ylim((0, 1.2))\n        plt.ylabel('Value Count %', fontsize=15)\n        vals = ax.get_yticks()\n        ax.set_yticklabels(['{:.0%}'.format(x) for x in vals])\n        plt.xlabel('Y value', fontsize=20)\n        positive_missing_value = data[column][data[y_column]==1].isna().sum()/len(data[column][data[y_column]==1])\n        negative_missing_value = data[column][data[y_column]==0].isna().sum()/len(data[column][data[y_column]==0])\n        plt.title('Feature: '+str(column.upper())\\\n                  +'\\nMissing values: '+str('{:,.2%}'.format(positive_missing_value))\\\n                  +positive_state+' and '+ str('{:,.2%}'.format(negative_missing_value)\\\n                                                  +negative_state), fontsize=15)\n        if save_fig:\n            file_name = path+str(column)+'.png'\n            plt.savefig(file_name)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"fe2f2497bcc942e3d88d0cd938573ea86a096ba4"},"cell_type":"code","source":"for dataset in train_test_data:\n    dataset['Family_size'] = dataset[\"SibSp\"] + dataset[\"Parch\"] + 1","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":false,"_uuid":"44070e290cad1b57dc9091fec092c4f3ac78e63c"},"cell_type":"code","source":"title_mapping = {\"Mr\": 1, \"Miss\": 2, \"Mrs\": 3, \n                 \"Master\": 5, \"Dr\": 4, \"Rev\": 4, \"Col\": 4, \"Major\": 4, \"Mlle\": 4,\"Countess\": 4,\n                 \"Ms\": 4, \"Lady\": 4, \"Jonkheer\": 4, \"Don\": 4, \"Dona\" : 4, \"Mme\": 4,\"Capt\": 4,\"Sir\": 4 }\nfor dataset in train_test_data:\n    dataset['Title'] = dataset['Title'].map(title_mapping)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"0b04ff43f91f1e6d8abd76cf00180c9596695e14"},"cell_type":"code","source":"for dataset in train_test_data:\n    dataset['Last_Name'] = dataset['Name'].apply(lambda x: str.split(x, \",\")[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"e292c1b740a6c69aaaf7c87dac255cd8be0f8b79"},"cell_type":"code","source":"total_data = train.append(test, sort=False)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"061b9c7abefa144eadf6a5bdece6f24eb669a493"},"cell_type":"markdown","source":"# Missing cabin custom function\nWith thi function I take cabin info for family and fill nan values"},{"metadata":{"trusted":false,"_uuid":"aa0571b941b904b71c7e4d19a891f68efbc71738"},"cell_type":"code","source":"def fill_missing_cabin(data, match_data):\n    cabin_list = []\n    for i in range(len(data)):\n        if data.Cabin[i] is np.nan:\n            cabin_mode = match_data[(match_data['Last_Name'] == data['Last_Name'][i])\\\n                               & (data['Family_size'][i] == match_data['Family_size'])\\\n                               & (data['Pclass'][i] == match_data['Pclass'])].Cabin.mode()\n            if len(cabin_mode) > 0:\n                cabin_list.append(cabin_mode[0])\n            else:\n                cabin_list.append(np.nan)\n        else:\n            cabin_list.append(data.Cabin[i])\n    return cabin_list","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"dc245255e16be448cf34993853fc08d8371e07ea"},"cell_type":"code","source":"train['Cabin'] = fill_missing_cabin(train, total_data)\ntest['Cabin'] = fill_missing_cabin(test, total_data)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"78d0a10c8e59e33a7e6c9af060c6a6c50d82be6b"},"cell_type":"code","source":"for dataset in train_test_data:\n    dataset['Cabin_letter'] = dataset['Cabin'].str[:1].fillna('Missing_cabin_info')","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"106661f3891bdf76e0163b53d5d34e9be31aa07a"},"cell_type":"code","source":"def get_cabin_count(data, column=''):\n    cabin_count_list=[]\n    for i in range(len(data)):\n        if data[column][i] is np.nan:\n            cabin_count_list.append(0)\n        else:\n            cabin_count_list.append(data[column][i].strip().count(' ')+1)\n    return cabin_count_list","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"880cb0667b1172a3dc4399b84a4702640d8ca9fe"},"cell_type":"code","source":"for dataset in train_test_data:\n    dataset['Cabin_count'] = get_cabin_count(dataset, column='Cabin')","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"b6cfd46c8047dbcfaa6683134a7675b38e59b18b"},"cell_type":"code","source":"sex_mapping = {\"male\": 0, \"female\": 1}\nfor dataset in train_test_data:\n    dataset['Sex'] = dataset['Sex'].map(sex_mapping)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"fa3f5f38b5ca93031bef5e078bea3a93e794c1b0"},"cell_type":"code","source":"categorical_features = ['Pclass', 'Sex', 'Cabin_letter', 'Embarked', 'Title', 'Family_size', 'Parch', 'SibSp']","execution_count":null,"outputs":[]},{"metadata":{"scrolled":false,"trusted":false,"_uuid":"c3f8f564783723d698e6593c0e8560d129c260ce"},"cell_type":"code","source":"for column in categorical_features:\n    get_categorical_feature_info(train, column=column, y_column='Survived', positive_state='Survived',\\\n                                negative_state='Dead')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"61b4e90ff0229d09c93d9cf50d3750ada7af6904"},"cell_type":"markdown","source":"# Filling missing values\nFill nan Age with the median of same Title\nFill nan Fare with media of same Pclass"},{"metadata":{"trusted":false,"_uuid":"04dd8b9af0191f533fa6cdff4f3d5d7f3ba8dd53"},"cell_type":"code","source":"train[\"Age\"].fillna(train.groupby(\"Title\")[\"Age\"].transform(\"median\"), inplace=True)\ntest[\"Age\"].fillna(test.groupby(\"Title\")[\"Age\"].transform(\"median\"), inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":false,"_uuid":"5b86b668a88ed9f27ab042dc144ca3e2ac92d038"},"cell_type":"code","source":"test[\"Fare\"].fillna(test.groupby(\"Pclass\")[\"Fare\"].transform(\"median\"), inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"18e9d713df56eaf141f698e7c7c0db3f1870128c"},"cell_type":"markdown","source":"# Continuous features custom function\nWith this function I can plot more the one continuous features using sns FacetGrid"},{"metadata":{"trusted":false,"_uuid":"e42765492de23fedb3d054c7e38ceaa35be6e022"},"cell_type":"code","source":"def get_continuous_feature_info(data, column='', y_column='', xlim=None):\n    facet = sns.FacetGrid(data, hue=y_column, aspect=4)\n    facet.map(sns.kdeplot,column,shade= True)\n    if xlim is None:\n        facet.set(xlim=(0, data[column].max()))\n    else:\n        facet.set(xlim=xlim)\n    facet.add_legend()\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"ae860dadd73a76f4b2c6bcf3b695b5af1af68e44"},"cell_type":"code","source":"continuous_features = [ 'Pclass', 'Age', 'SibSp', 'Parch', 'Fare', 'Family_size']","execution_count":null,"outputs":[]},{"metadata":{"scrolled":false,"trusted":false,"_uuid":"08c9dbab7bc7f5c1b2f4a1e3171b1800777ebc61"},"cell_type":"code","source":"for column in continuous_features:\n    get_continuous_feature_info(train, column=column, y_column='Survived')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"51c746bfb51724e4981ab946858a5cdfd06a8661"},"cell_type":"markdown","source":"# Filling missing Cabin info\nAfter first function to fill missing value in Cabin column, I saw that don't have a cabin info, is a \"feature\" to pass to\ntrain in the model because is distribution is different on the two classes of dependent variable"},{"metadata":{"trusted":false,"_uuid":"a0d7ee88fe821cb5f313065032b7299e6891cf77"},"cell_type":"code","source":"for dataset in train_test_data:\n    dataset['Cabin_letter'] = dataset['Cabin_letter'].fillna('Missing_cabin_info')","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"d97b64bd1558360f3fb55168770fc674840c4068"},"cell_type":"code","source":"for dataset in train_test_data:\n    dataset['Cabin_count'] = dataset['Cabin_count'].fillna(0)","execution_count":null,"outputs":[]},{"metadata":{"scrolled":false,"trusted":false,"_uuid":"6e6b67bda8ac827ae0891a5d5d6b3155dfa32c26"},"cell_type":"code","source":"for column in ['Cabin_letter', 'Cabin_count']:\n    get_categorical_feature_info(train, column=column, y_column='Survived')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2d9768898988ac0a115307c5df8b6365c9f25e7a"},"cell_type":"markdown","source":"# Creating new features\nAfter analyzed \"Missing cabin info\", I decided to see if do not have a cabin is an important feature"},{"metadata":{"trusted":false,"_uuid":"684cafe6e32f9ff051f1969f251644ba698e2eb2"},"cell_type":"code","source":"for dataset in train_test_data:\n    dataset['Has_a_Cabin'] = dataset[\"Cabin_letter\"].apply(lambda x: 0 if x == 'Missing_cabin_info' else 1)\n    dataset.drop(columns='Cabin_letter', inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"b525d8d1e2a5ec772b354c210c4d81cfd392bc0e"},"cell_type":"code","source":"get_categorical_feature_info(train, column='Has_a_Cabin', y_column='Survived')","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":false,"_uuid":"dabe1a7845d927e3e41790ed9f1698d470bf4d63"},"cell_type":"code","source":"nan = dataset.isnull().sum()\nplt.figure(figsize=(20,5))\nidx_nan = nan.mask(nan==0).dropna().index\nsns.heatmap(dataset[idx_nan].transpose().isnull(), cmap = 'binary')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"082af7e8cdf5e15ee830c0d1a266f14d4a39fcee"},"cell_type":"markdown","source":"# Custom function to plot and extract outliers info\nWith this function I can plot (with sns box plot) and extract outliers info with interquertile, lower boun and upper bound info"},{"metadata":{"trusted":false,"_uuid":"2429d1f4c3d1eae882ed0e798f21187b31cce3e1"},"cell_type":"code","source":"def get_outliers_info(data, column, figsize=(20,8)):\n    plt.figure(figsize=figsize)\n    Q1 = data[column].quantile(0.25)\n    Q3 = data[column].quantile(0.75)\n    IQR = Q3 - Q1\n    lower_bound = Q1 - (IQR * 1.5)\n    upper_bound = Q3 + (IQR * 1.5)\n    bounds_list = [lower_bound, upper_bound]\n    feature_bounds = bounds_list\n    sns.boxplot(x=data[column])\n    plt.title('IQR: '+str(IQR)+', Lower bound: '+str(lower_bound)+', Upper bound: '+str(upper_bound)+\\\n              '\\nMedian: '+str(data[column].median()), fontsize=15)\n    plt.show()\n    return feature_bounds","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":false,"_uuid":"0c5f0d9ddc8a243306e00920e05177b2d00027b9"},"cell_type":"code","source":"outliers_bounds = {}\nfor column in continuous_features:\n    outliers_bounds[column] = get_outliers_info(total_data, column=column)","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":false,"_uuid":"082474ca695fd3bc17de6028cdaa263a74c33fc2"},"cell_type":"code","source":"outliers_bounds","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"d84398b637dd57c80c96eec666c3608f8e68e4c0"},"cell_type":"code","source":"for dataset in train_test_data:\n    dataset.drop('Parch', axis='columns', inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":false,"_uuid":"9561626833d2a5e84f9fb3327f8bdab77c4c676d"},"cell_type":"code","source":"del outliers_bounds['Parch']\ndel outliers_bounds['Pclass']","execution_count":null,"outputs":[]},{"metadata":{"scrolled":false,"trusted":false,"_uuid":"abe68ada98d8ee16638832ae291e7f3fac3c3834"},"cell_type":"code","source":"outliers_bounds","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f7651ce217c9f3f89ba2e2b92515e68cdddf6a5c"},"cell_type":"markdown","source":"# Normalizing outliers\nWith this funciton I normalized upper outliers (because in the plot there are not lower outliers) with upper bound value"},{"metadata":{"trusted":false,"_uuid":"65b874e1b9607b9e7cd67fa4b3ff1e000e7b90d3"},"cell_type":"code","source":"def normalize_outliers(data, outliers_bounds={}):\n    for key in outliers_bounds.keys():\n        median = data[key].median()\n        new_values = []\n        for i in range(len(data)):\n            value = data[key][i]\n            if value < outliers_bounds[key][0] or value > outliers_bounds[key][1]:\n                new_values.append(outliers_bounds[key][1])\n            else:\n                new_values.append(value)\n        data[key] = new_values\n    return data","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"7ee0d5c6fe6905660b2ca45b4806eca02cf5b450"},"cell_type":"code","source":"train = normalize_outliers(train, outliers_bounds=outliers_bounds)\ntest = normalize_outliers(test, outliers_bounds=outliers_bounds)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"fc1d3adb99e5846e93cd6667629517e10bbb9460"},"cell_type":"code","source":"total_data = train.append(test, sort=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"354b18d8a7a5dc30169ede19c34bb8e3d402b56e"},"cell_type":"code","source":"del outliers_bounds['SibSp']","execution_count":null,"outputs":[]},{"metadata":{"scrolled":false,"trusted":false,"_uuid":"12e9e0eb8a0ae9dd1563c35e9eb7d1c56d39b013"},"cell_type":"code","source":"for column in outliers_bounds.keys():\n    get_continuous_feature_info(train, column=column, y_column='Survived')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a523c21d83b616d2a7342e1e753fa8677e02d3f3"},"cell_type":"markdown","source":"# Custom binning for Fare\nHaving info from the plot, I create custom bins for \"Fare\" variable."},{"metadata":{"trusted":false,"_uuid":"5d62bfbea2b14a80a93cd842b669c9c0c75259a8"},"cell_type":"code","source":"for dataset in train_test_data:\n    dataset.loc[ dataset['Fare'] <= 7.91, 'Fare'] = 0\n    dataset.loc[(dataset['Fare'] > 7.91) & (dataset['Fare'] <= 16), 'Fare'] = 1\n    dataset.loc[(dataset['Fare'] > 16) & (dataset['Fare'] <= 31), 'Fare']   = 2\n    dataset.loc[(dataset['Fare'] > 31) & (dataset['Fare'] <= 50), 'Fare']   = 3\n    dataset.loc[ dataset['Fare'] > 50, 'Fare'] = 4\n    dataset['Fare'] = dataset['Fare'].astype(int)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"52302189fe83603244675657c2d9e47210860d64"},"cell_type":"markdown","source":"# Female features engineering\n\nNow it's time to make questions.\nIn first plots we saw that a woman has more probability to survive, but... <br>\n1) What if she's the youngest woman in the family? <br>\nTo answer to this and other questions, we have to match individual info with family info, so we have to have a DataFrame\nwith all train and test info."},{"metadata":{"trusted":false,"_uuid":"50223d0ddd2ef64ad68306a90acb512d95c99ad3"},"cell_type":"code","source":"def is_youngest_girl(data, match_data):\n    match_list = []\n    for i in range(len(data)):\n        if  (data.Sex[i] == 1):\n            match = (data.loc[i].Age <= min(match_data[(match_data['Last_Name'] == data['Last_Name'][i])\\\n                                                   & (data['Family_size'][i] == match_data['Family_size']) \\\n                                                   & (data['Pclass'][i] == match_data['Pclass'])].Age))*1\n\n            match_list.append(match)\n        else:\n            match_list.append(0)\n    return match_list","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"3122bc405f5cf005e59097a032f64b2eff6d6b2a"},"cell_type":"code","source":"train['Is_Youngest_lady'] = is_youngest_girl(train, total_data)\ntest['Is_Youngest_lady'] = is_youngest_girl(test, total_data)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6eec591dcad979a0bcad935a4b45e6186321ce97"},"cell_type":"markdown","source":"2) What if the passengers is a rich wife?"},{"metadata":{"trusted":false,"_uuid":"37fa10def87bac79aa79bdd909133fdb2cbe5457"},"cell_type":"code","source":"for dataset in train_test_data:\n    dataset['Is_WifeandRich'] = ((dataset['Name'].str.find('(') > -1) & (dataset['Sex'] == 1) & (dataset['Pclass'] <3))*1","execution_count":null,"outputs":[]},{"metadata":{"scrolled":false,"trusted":false,"_uuid":"8a44a3737453826a1fce04605b11938aef63f627"},"cell_type":"code","source":"for column in ['Is_WifeandRich', 'Is_Youngest_lady']:\n    get_categorical_feature_info(train, column=column, y_column='Survived', positive_state='Survived', negative_state='Dead')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c52da1dd8efcae3458a8a1e4dccbb25aca519645"},"cell_type":"markdown","source":"3) What is she is a rich mother? (or a woman with children in her family?)"},{"metadata":{"trusted":false,"_uuid":"aa23a9e9106d82c131e39ba6820e45dec28bed33"},"cell_type":"code","source":"def is_a_rich_Mother(data, match_data):\n    match_list = []\n    for i in range(len(data)):\n        if (data.Pclass[i] < 3) & (data.Sex[i] == 1):\n            match = (14 >= min(match_data[(match_data['Last_Name'] == data['Last_Name'][i])\\\n                                                   & (data['Family_size'][i] == match_data['Family_size']) \\\n                                                   & (data['Pclass'][i] == match_data['Pclass'])].Age))*1\n\n            match_list.append(match)\n        else:\n            match_list.append(0)\n    return match_list","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"2088a7f007e881fae09ac30864198d87dda85de8"},"cell_type":"code","source":"train['Is_a_rich_Mother'] = is_a_rich_Mother(train, total_data)\ntest['Is_a_rich_Mother'] = is_a_rich_Mother(test, total_data)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"bbb664d27dbc7605e35022f8e03efaad51f42928"},"cell_type":"markdown","source":"4) What if she is the only woman in the family?"},{"metadata":{"trusted":false,"_uuid":"1b1b313e54ef50f945d53d7e711f6ec2643e63fd"},"cell_type":"code","source":"def is_the_only_woman(data, match_data):\n    match_list = []\n    for i in range(len(data)):\n        if data.Sex[i] == 1:\n            match = (match_data[(match_data['Last_Name'] == data['Last_Name'][i])\\\n                                                   & (data['Family_size'][i] == match_data['Family_size']) \\\n                                                   & (data['Pclass'][i] == match_data['Pclass'])].Sex.sum())\n            if match == 1:\n                match_list.append(match)\n            else:\n                 match_list.append(0)\n        else:\n            match_list.append(0)\n    return match_list","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"13fb39e118a53dc7cf11ba199b055ba397e28a18"},"cell_type":"code","source":"train['Is_the_only_woman'] = is_a_rich_Mother(train, total_data)\ntest['Is_the_only_woman'] = is_a_rich_Mother(test, total_data)","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":false,"_uuid":"da9ba74e4dc1b2c47fc1516f7ce9ef457933656a"},"cell_type":"code","source":"for column in ['Is_a_rich_Mother', 'Is_the_only_woman']:\n    get_categorical_feature_info(train, column=column, y_column='Survived', positive_state='Survived', negative_state='Dead')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"af0a307c0ed06c16ca0f76c701281caf1155aa5e"},"cell_type":"markdown","source":"5) What if she has other women in her family?"},{"metadata":{"trusted":false,"_uuid":"ecbccdefff3339a494b9a32952d5bc9999ba90a5"},"cell_type":"code","source":"def other_women_in_family(data, match_data):\n    match_list = []\n    for i in range(len(data)):\n        if data.Sex[i] == 1:\n            match = (match_data[(match_data['Last_Name'] == data['Last_Name'][i])\\\n                                                   & (data['Family_size'][i] == match_data['Family_size']) \\\n                                                   & (data['Pclass'][i] == match_data['Pclass'])].Sex.sum())\n            if match > 1:\n                match_list.append(1)\n            else:\n                 match_list.append(0)\n        else:\n            match_list.append(0)\n    return match_list","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"54e049d7d9d627ed0a4ec9371e7bec72a0372364"},"cell_type":"code","source":"train['Other_women_in_family'] = other_women_in_family(train, total_data)\ntest['Other_women_in_family'] = other_women_in_family(test, total_data)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"407595c28a5caffd4af3e1008a7a338e6a58623b"},"cell_type":"markdown","source":"6) What if she is the oldest woman in her family?"},{"metadata":{"trusted":false,"_uuid":"c5629e92caf408afcaadce92b8617a502bb4b7b5"},"cell_type":"code","source":"def is_oldest_woman(data, match_data):\n    match_list = []\n    for i in range(len(data)):\n        if  (data.Sex[i] == 1):\n            match = (data.loc[i].Age == max(match_data[(match_data['Last_Name'] == data['Last_Name'][i])\\\n                                                   & (data['Family_size'][i] == match_data['Family_size']) \\\n                                                   & (data['Pclass'][i] == match_data['Pclass'])].Age))*1\n\n            match_list.append(match)\n        else:\n            match_list.append(0)\n    return match_list","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"3da7018128c2975b651d1a99f76f1dadf463c351"},"cell_type":"code","source":"train['Is_oldest_woman'] = is_oldest_woman(train, total_data)\ntest['Is_oldest_woman'] = is_oldest_woman(test, total_data)","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":false,"_uuid":"5ca3869bce995bc2a38c767d6720881d3be01369"},"cell_type":"code","source":"for column in ['Is_oldest_woman', 'Other_women_in_family']:\n    get_categorical_feature_info(train, column=column, y_column='Survived', positive_state='Survived', negative_state='Dead')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a39adbf203735b4902fafa5b8e33f3c88dec4e3d"},"cell_type":"markdown","source":"# Male features engineering\nNow it's Male turn.\n1) What if he is the youngest man in his family?"},{"metadata":{"trusted":false,"_uuid":"e9b4157e33b38ef6ce5669f59a408a690f0f62b0"},"cell_type":"code","source":"def is_youngest_boy(data, match_data):\n    match_list = []\n    for i in range(len(data)):\n        if  (data.Sex[i] == 0):\n            match = (data.loc[i].Age <= min(match_data[(match_data['Last_Name'] == data['Last_Name'][i])\\\n                                                   & (data['Family_size'][i] == match_data['Family_size']) \\\n                                                   & (data['Pclass'][i] == match_data['Pclass'])].Age))*1\n\n            match_list.append(match)\n        else:\n            match_list.append(0)\n    return match_list","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1c3182f259fd215cd317fcaf81c9edbd245a21eb"},"cell_type":"markdown","source":"2) What if he is the oldest man in his family?"},{"metadata":{"trusted":false,"_uuid":"0eaab16a7c0e22cbc9fed7335b239be1171c31b2"},"cell_type":"code","source":"def is_oldest_man(data, match_data):\n    match_list = []\n    for i in range(len(data)):\n        if  (data.Sex[i] == 0):\n            match = (data.loc[i].Age == max(match_data[(match_data['Last_Name'] == data['Last_Name'][i])\\\n                                                   & (data['Family_size'][i] == match_data['Family_size']) \\\n                                                   & (data['Pclass'][i] == match_data['Pclass'])].Age))*1\n\n            match_list.append(match)\n        else:\n            match_list.append(0)\n    return match_list","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"421eec69beb4db78e964062780e650a3d1caa30a"},"cell_type":"markdown","source":"2) What if he has women in his family?"},{"metadata":{"trusted":false,"_uuid":"56c25b18b04e2f50fa303d34fe657f481ce81ad9"},"cell_type":"code","source":"def man_with_woman(data, match_data):\n    match_list = []\n    for i in range(len(data)):\n        if (data.Sex[i] == 0) & (data.Title[i] != 5):\n            match = (match_data[(match_data['Last_Name'] == data['Last_Name'][i])\\\n                                                   & (data['Family_size'][i] == match_data['Family_size']) \\\n                                                   & (data['Pclass'][i] == match_data['Pclass'])].Sex.sum())\n            if match > 1:\n                match_list.append(1)\n            else:\n                 match_list.append(0)\n        else:\n            match_list.append(0)\n    return match_list","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8ec1b13a028eb5eaaaac3e2368b8bf48237903ca"},"cell_type":"markdown","source":"2) What if he has other men in his family?"},{"metadata":{"trusted":false,"_uuid":"0c25553e9d6a5dfcb4f0e3af292119ad8cc915bb"},"cell_type":"code","source":"def man_with_other_men(data, match_data):\n    match_list = []\n    for i in range(len(data)):\n        if data.Sex[i] == 0:\n            match = len(match_data[(match_data['Last_Name'] == data['Last_Name'][i])\\\n                                                   & (data['Family_size'][i] == match_data['Family_size']) \\\n                                                   & (data['Pclass'][i] == match_data['Pclass'])\\\n                                                   & (match_data['Sex'] == 0)].Sex)\n            if match > 1:\n                match_list.append(1)\n            else:\n                 match_list.append(0)\n        else:\n            match_list.append(0)\n    return match_list","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"09591e0b175ab56999db62378693f4b4771f1227"},"cell_type":"code","source":"train['Is_youngest_man'] = is_youngest_boy(train, total_data)\ntest['Is_youngest_man'] = is_youngest_boy(test, total_data)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"042573b9fab11601c3cd7ed5ae13772f473c86b2"},"cell_type":"code","source":"train['Is_oldest_man'] = is_oldest_man(train, total_data)\ntest['Is_oldest_man'] = is_oldest_man(test, total_data)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"b0261ab0d8f1de30c6dd5f55c4737facb3981088"},"cell_type":"code","source":"train['Man_with_woman'] = man_with_woman(train, total_data)\ntest['Man_with_woman'] = man_with_woman(test, total_data)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"bbf503dcc87596e06636216f443a09cc6a2fc702"},"cell_type":"code","source":"train['Man_with_other_men'] = man_with_woman(train, total_data)\ntest['Man_with_other_men'] = man_with_woman(test, total_data)","execution_count":null,"outputs":[]},{"metadata":{"scrolled":false,"trusted":false,"_uuid":"c3e933eed5836c8ecc473c11a64c4ebde56e6198"},"cell_type":"code","source":"for column in ['Is_youngest_man', 'Is_oldest_man', 'Man_with_woman', 'Man_with_other_men']:\n    get_categorical_feature_info(train, column=column, y_column='Survived', positive_state='Survived', negative_state='Dead')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9f0710e4676f923411372307114cf2f9b8d0a353"},"cell_type":"markdown","source":"# Children featuren engineering\nNow It's children turn.\n1) is he/she the youngest one?"},{"metadata":{"trusted":false,"_uuid":"cc4c38f7cd1a022155ff4645dea45d4db3e81a54"},"cell_type":"code","source":"def is_youngest_children(data, match_data):\n    match_list = []\n    for i in range(len(data)):\n        if  (data.Title[i] == 5):\n            match = (data.loc[i].Age <= min(match_data[(match_data['Last_Name'] == data['Last_Name'][i])\\\n                                                   & (data['Family_size'][i] == match_data['Family_size']) \\\n                                                   & (data['Pclass'][i] == match_data['Pclass'])\\\n                                                   & (match_data['Title'] == 5)].Age))*1\n\n            match_list.append(match)\n        else:\n            match_list.append(0)\n    return match_list","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8cefba3b75475609b30cdc0115242f352c491bdf"},"cell_type":"markdown","source":"2) is he/she the oldest one?"},{"metadata":{"trusted":false,"_uuid":"24b2b111a1aca6f00fc18e5e503265fa30de5aab"},"cell_type":"code","source":"def is_oldest_children(data, match_data):\n    match_list = []\n    for i in range(len(data)):\n        if  (data.Title[i] == 5):\n            match = (data.loc[i].Age >= max(match_data[(match_data['Last_Name'] == data['Last_Name'][i])\\\n                                                   & (data['Family_size'][i] == match_data['Family_size']) \\\n                                                   & (data['Pclass'][i] == match_data['Pclass'])\\\n                                                   & (match_data['Title'] == 5)].Age))*1\n\n            match_list.append(match)\n        else:\n            match_list.append(0)\n    return match_list","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"44b905a41ea87e7efbfe5fc3ea448b53c2eaf142"},"cell_type":"markdown","source":"3) is he/she the only one?"},{"metadata":{"trusted":false,"_uuid":"2efe7c6715ef9865a7bdb4b96711951c53c18d76"},"cell_type":"code","source":"def is_the_only_child(data, match_data):\n    match_list = []\n    for i in range(len(data)):\n        if  (data.Title[i] == 5):\n            match = len(match_data[(match_data['Last_Name'] == data['Last_Name'][i])\\\n                                                   & (data['Family_size'][i] == match_data['Family_size']) \\\n                                                   & (data['Pclass'][i] == match_data['Pclass'])\\\n                                                   & (match_data['Title'] == 5)])\n            if match == 1:\n                match_list.append(1)\n            else:\n                match_list.append(0)\n        else:\n            match_list.append(0)\n    return match_list","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"3b4067ec98db6dab247c3946c86a7929b7dc3afc"},"cell_type":"code","source":"train['Is_youngest_children'] = is_youngest_children(train, total_data)\ntest['Is_youngest_children'] = is_youngest_children(test, total_data)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"2be04b559dc12af1307c7e27b5a99310f7695919"},"cell_type":"code","source":"train['Is_oldest_children'] = is_oldest_children(train, total_data)\ntest['Is_oldest_children'] = is_oldest_children(test, total_data)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"2f3929a5b999795fd75a1ec81e7d4ac57bec7e58"},"cell_type":"code","source":"train['Is_the_only_child'] = is_the_only_child(train, total_data)\ntest['Is_the_only_child'] = is_the_only_child(test, total_data)","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":false,"_uuid":"c69e2a7a82d09c92ae89af12067fcc2c9e80c919"},"cell_type":"code","source":"for column in ['Is_youngest_children', 'Is_oldest_children', 'Is_the_only_child']:\n    get_categorical_feature_info(train, column=column, y_column='Survived', positive_state='Survived', negative_state='Dead')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"361927c0f698b8fc18b0d016c42a8e87c04f4277"},"cell_type":"markdown","source":"# Survived Family featuren engineering\nNow we have to analyze info about survived family, but we don't have to take passenger surviving info to do not have <br>\noverfitting result. <br>\nSo, data['PassengerId'][i] != match_data['PassengerId']), exclude this case. <br>\n1) Has survived family?"},{"metadata":{"trusted":false,"_uuid":"43215e032d53dad3614b3919013b84bb6151fb63"},"cell_type":"code","source":"def family_survived(data, match_data):\n    match_list = []\n    for i in range(len(data)):\n        if data.Family_size[i] > 1:\n            survived = (match_data[(match_data['Last_Name'] == data['Last_Name'][i])\\\n                                                   & (data['Family_size'][i] == match_data['Family_size']) \\\n                                                   & (data['Pclass'][i] == match_data['Pclass'])\\\n                                                & (data['PassengerId'][i] != match_data['PassengerId'])].Survived).sum()\n            if survived > 0:\n                match_list.append(1)\n            else:\n                match_list.append(0)\n        else:\n            match_list.append(0)\n    return match_list","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b2fffd25a38b405b9d92936f9247327e36ebdf8e"},"cell_type":"markdown","source":"2) Is she a wife with survived children in her family?"},{"metadata":{"trusted":false,"_uuid":"7789b18c81020550c2f93ddf6f2f067947654eb2"},"cell_type":"code","source":"def is_a_wife_with_survived_children(data, match_data):\n    match_list = []\n    for i in range(len(data)):\n        if (data.Name[i].find('(') > -1) & (data.Sex[i] == 1):\n            survived = (match_data[(match_data['Last_Name'] == data['Last_Name'][i])\\\n                                                   & (data['Family_size'][i] == match_data['Family_size']) \\\n                                                   & (data['Pclass'][i] == match_data['Pclass'])\\\n                                                & (data['PassengerId'][i] != match_data['PassengerId'])\\\n                                                & (match_data['Title'] == 5)].Survived).sum()\n            if survived > 0:\n                match_list.append(1)\n            else:\n                match_list.append(0)\n        else:\n            match_list.append(0)\n    return match_list","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1cc6f307bc9f0610ebcb624e073d740e446e233e"},"cell_type":"markdown","source":"Is He/She a child with other children or women survived?"},{"metadata":{"trusted":false,"_uuid":"0c86f53a6d52e1bf3696c41074a559f31f54ee78"},"cell_type":"code","source":"def is_a_child_with_other_candw_survived(data, match_data):\n    match_list = []\n    for i in range(len(data)):\n        if (data.Title[i] == 5):\n            survived = (match_data[(match_data['Last_Name'] == data['Last_Name'][i])\\\n                                                   & (data['Family_size'][i] == match_data['Family_size']) \\\n                                                   & (data['Pclass'][i] == match_data['Pclass'])\\\n                                                & (data['PassengerId'][i] != match_data['PassengerId'])\\\n                                                & ((match_data['Title'] == 5) | (match_data['Sex'] == 0))].Survived).sum()\n            if survived > 0:\n                match_list.append(1)\n            else:\n                match_list.append(0)\n        else:\n            match_list.append(0)\n    return match_list","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"1f56bf7283374069c19a5ea262224b7c010b9b20"},"cell_type":"code","source":"train['Family_survived'] = family_survived(train, total_data)\ntest['Family_survived'] = family_survived(test, total_data)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"2951ff6019def24d70436408b2c044448f9dc0b1"},"cell_type":"code","source":"train['Is_a_wife_with_sc'] = is_a_wife_with_survived_children(train, total_data)\ntest['Is_a_wife_with_sc'] = is_a_wife_with_survived_children(test, total_data)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"984c374f3d4602600434a4057309956b8547795c"},"cell_type":"code","source":"train['Is_a_child_with_wandc_survived'] = is_a_child_with_other_candw_survived(train, total_data)\ntest['Is_a_child_with_wandc_survived'] = is_a_child_with_other_candw_survived(test, total_data)","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":false,"_uuid":"e78228765f4f3725d3378064becf592d89d06cbb"},"cell_type":"code","source":"for column in ['Family_survived', 'Is_a_wife_with_sc', 'Is_a_child_with_wandc_survived']:\n    get_categorical_feature_info(train, column=column, y_column='Survived', positive_state='Survived', negative_state='Dead')","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":false,"_uuid":"61ae389626ca9b90c02fd6cf8506bfd24ebf68cd"},"cell_type":"code","source":"for value in [(0,15), (15,35), (35,50), (50,65), (65,100)]:\n    get_continuous_feature_info(train, column='Age', y_column='Survived', xlim=value)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2748fff3a5c8cb109ab17ccc19d3a8b017ca7e30"},"cell_type":"markdown","source":"Now, with Age plot, I can bin Age column"},{"metadata":{"trusted":false,"_uuid":"662ab32aa23d9402d051881c934a608deae6b371"},"cell_type":"code","source":"for dataset in train_test_data:    \n    dataset.loc[ dataset['Age'] <= 14, 'Age'] = 0\n    dataset.loc[(dataset['Age'] > 14) & (dataset['Age'] <= 32), 'Age'] = 1\n    dataset.loc[(dataset['Age'] > 32) & (dataset['Age'] <= 48), 'Age'] = 2\n    dataset.loc[(dataset['Age'] > 48) & (dataset['Age'] <= 64), 'Age'] = 3\n    dataset.loc[ dataset['Age'] > 64, 'Age'] = 4","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":false,"_uuid":"85d2f68299a0a74a8fdd2fd56d5c1bdd93844717"},"cell_type":"code","source":"test.isna().sum(), train.isna().sum()","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":false,"_uuid":"8880412c87c97cd5c770f8a20266e8f24ad17d41"},"cell_type":"code","source":"Pclass1 = train[train['Pclass']==1]['Embarked'].value_counts()\nPclass2 = train[train['Pclass']==2]['Embarked'].value_counts()\nPclass3 = train[train['Pclass']==3]['Embarked'].value_counts()\ndf = pd.DataFrame([Pclass1, Pclass2, Pclass3])\ndf.index = ['1st class','2nd class', '3rd class']\ndf.plot(kind='bar',stacked=True, figsize=(10,5))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ae3b6fd7784294cb911260928f2b1cdd11d4251d"},"cell_type":"markdown","source":"# Filling Embarked missing values with the most frequent info"},{"metadata":{"trusted":false,"_uuid":"c56a1e3dbfe2824c51302499ac8ce6af4e687710"},"cell_type":"code","source":"for dataset in train_test_data:\n    dataset['Embarked'] = dataset['Embarked'].fillna('S')","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"fbf3f167779b36eaece2c2f0548c954582ededd6"},"cell_type":"code","source":"embarked_mapping = {\"C\": \"Cherbourg\", \"S\": \"Southampton\", \"Q\": \"Queenstown\"}\nfor dataset in train_test_data:\n    dataset['Embarked'] = dataset['Embarked'].map(embarked_mapping)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e9d0380c508e5c4f6741e3c8ebd33c1ed7241f3a"},"cell_type":"markdown","source":"# Class and status features engineering"},{"metadata":{"trusted":false,"_uuid":"d236fdd8ecf9cab6f51faf489bca9ab122c43333"},"cell_type":"code","source":"for dataset in train_test_data:\n    dataset['Is_AloneandPoor'] = 0\n    dataset.loc[(dataset['Family_size'] == 1) & (dataset['Pclass'] == 3 ),'Is_AloneandPoor'] = 1","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":false,"_uuid":"e9516da9959625ce1ce6e744ab1b4614bbd7c16d"},"cell_type":"code","source":"get_categorical_feature_info(train, column='Is_AloneandPoor', y_column='Survived', positive_state='Survived', negative_state='Dead')","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"f653a37d24d8652b79753ceb1d39d40970445a12"},"cell_type":"code","source":"for dataset in train_test_data:\n    dataset['IsChildandRich'] = 0\n    dataset.loc[(dataset['Title'] == 5) & (dataset['Pclass'] < 3 ),'IsChildandRich'] = 1  ","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":false,"_uuid":"17c380ab36203bcd0e39c9d040b656653d6e755e"},"cell_type":"code","source":"get_categorical_feature_info(train, column='IsChildandRich', y_column='Survived', positive_state='Survived', negative_state='Dead')","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"db77f9608463622bb0c1b7d060f5ee063f879073"},"cell_type":"code","source":"for data in train_test_data:\n    data['Cabin'] = data['Cabin'].fillna('X')\n    data['Cabin'] = data['Cabin'].apply(lambda x: str(x)[0])\n    data['Cabin'] = data['Cabin'].replace(['A', 'D', 'E', 'T'], 'M')\n    data['Cabin'] = data['Cabin'].replace(['B', 'C'], 'H')\n    data['Cabin'] = data['Cabin'].replace(['F', 'G'], 'L')","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"35fff164d6a3b38e19584c7ab71a5c2b6fe3c5a0"},"cell_type":"code","source":"get_categorical_feature_info(train, column='Cabin', y_column='Survived', positive_state='Survived', negative_state='Dead')","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"8f824f32448a901d2242aa8f747b831c4073f526"},"cell_type":"code","source":"for dataset in train_test_data:\n    dataset['Family_size_group'] = 'Small'\n    dataset.loc[dataset['Family_size'] == 1, 'Family_size_group'] = 'Alone'\n    dataset.loc[dataset['Family_size'] > 2, 'Family_size_group'] = 'Big'","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"253c98b3634997997ce9cc15f6c746e997d5d8ad"},"cell_type":"code","source":"train['IsMaleandPoor'] = 0\ntrain.loc[(train['Sex'] == 0) & (train['Pclass'] == 3 ),'IsMaleandPoor'] = 1\ntest['IsMaleandPoor'] = 0\ntest.loc[(test['Sex'] == 0) & (test['Pclass'] == 3 ),'IsMaleandPoor'] = 1 ","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"ab1d9fe42ac12ec10b8993e54433f0b9bbcf7aec"},"cell_type":"code","source":"for column in ['Family_size_group', 'IsMaleandPoor']:\n    get_categorical_feature_info(train, column=column, y_column='Survived', positive_state='Survived', negative_state='Dead')","execution_count":null,"outputs":[]},{"metadata":{"scrolled":false,"trusted":false,"_uuid":"e2cc035df32dbded18598ad12a6b09038b0c1e02"},"cell_type":"code","source":"train.columns","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a8c24c8aeb4dad192fced2d058809546e22083f2"},"cell_type":"markdown","source":"# Dropping useless columns"},{"metadata":{"trusted":false,"_uuid":"94d43d413800a6bff9d258042f922715cfbde368"},"cell_type":"code","source":"for dataset in train_test_data:\n    dataset.drop(['PassengerId', 'Name', 'Ticket', 'Last_Name', 'SibSp', 'Ticket'\\\n                 , 'Family_size'], axis=1, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":false,"_uuid":"fda363031d4059f4d56a2a0bf24393db3a5621d9"},"cell_type":"code","source":"len(train.columns), len(test.columns)","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":false,"_uuid":"8f47081316cd410c5175b567bceeacabb6908fdb"},"cell_type":"code","source":"train.dtypes","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"208fdbf7363673c3652af473507a45651570f2d4"},"cell_type":"code","source":"def get_dummies_from_list(categorical_list):\n    dummies = pd.get_dummies(pd.Categorical(categorical_list), sparse=True)\n    return dummies","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"81a3e2df23d9cefa96a21de36ddbb5e0f14bd83c"},"cell_type":"code","source":"def transform_dummies(data, column_list=[]):\n    for column in column_list:\n        dummy_df = get_dummies_from_list(data[column])\n        data = data.merge(dummy_df, how='left', left_index=True, right_index=True)\n    data.drop(labels=column_list, axis='columns', inplace=True)\n    return data","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7d7c7e264b17de21d0deaa4079879b67fa5c1d4f"},"cell_type":"markdown","source":"# Creationg dummy columns for categorical variables"},{"metadata":{"trusted":false,"_uuid":"9df7dcb4adbc76b2b424399e17410c1702c615a4"},"cell_type":"code","source":"train = transform_dummies(train, column_list=['Embarked', 'Family_size_group', 'Cabin'])\ntest = transform_dummies(test, column_list=['Embarked', 'Family_size_group', 'Cabin'])","execution_count":null,"outputs":[]},{"metadata":{"scrolled":false,"trusted":false,"_uuid":"c8ab814092dba5ac6f3e3c9642372417fa66c14d"},"cell_type":"code","source":"len(train.columns), len(test.columns)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"22aa2ce409690c2059c6da49a4ae4f6fba29c17a"},"cell_type":"markdown","source":"# Custom function to plot info about 2 indipendent variables on dependent variable"},{"metadata":{"trusted":false,"_uuid":"817233c5d6fabe880c9497af32b4f4c180b61d72"},"cell_type":"code","source":"def get_indipendent_variables_mix_info(data, x_column='', y_column='', dependent_variable=''):\n    if (x_column != y_column) & (x_column != dependent_variable):\n        fig, (axis1,axis2) = plt.subplots(1,2,figsize=(14,14))\n        sns.boxplot(x = x_column, y = y_column, hue = dependent_variable, data = data, ax = axis1)\n        axis1.set_title(str(x_column)+' vs '+str(y_column))\n        sns.violinplot(x = x_column, y = y_column, hue = dependent_variable, data = data, ax = axis2, split = True)\n        axis2.set_title(str(x_column)+' vs '+str(y_column))\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"scrolled":false,"trusted":false,"_uuid":"a7dfa00b101e9f147361283fb9640139926165fd"},"cell_type":"code","source":"for column in train.columns:\n    get_indipendent_variables_mix_info(train, x_column=column, y_column='Age', dependent_variable='Survived')","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":false,"_uuid":"723dff36527da47a1ab801df6a98d3b4eee1b921"},"cell_type":"code","source":"corr = train.corr()\nmask = np.zeros_like(corr, dtype=np.bool)\nmask[np.triu_indices_from(mask)] = True\nf, ax = plt.subplots(figsize=(11, 9))\ncmap = sns.diverging_palette(220, 10, as_cmap=True)\nsns.heatmap(corr, mask=mask, cmap=cmap, vmax=.3, center=0,\n            square=True, linewidths=.5, cbar_kws={\"shrink\": .5})","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"edec6bcaf96cb0ecea0e2cb88058a3b9a5e88607"},"cell_type":"code","source":"abs(corr.Survived).sort_values(ascending=False)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"713f1059697df7ec3c22b5660e798f1f761c531f"},"cell_type":"markdown","source":"# PCA Analysis on 2 dimensions"},{"metadata":{"scrolled":true,"trusted":false,"_uuid":"2418cefa2082323d16cb4db1816ce7e8f7d2a44f"},"cell_type":"code","source":"from sklearn.decomposition import PCA\nfrom mpl_toolkits.mplot3d import Axes3D\nimport matplotlib.pyplot as plt, mpl_toolkits.mplot3d\nfrom mpl_toolkits.mplot3d import axes3d\n\nplt.figure(figsize=(10,10))\ntrain2D = PCA(n_components=2).fit_transform(train.drop('Survived', axis='columns'))\nplt.scatter(train2D[train[train.Survived==0].reset_index(drop=True).index, 0], train2D[train[train.Survived==0].reset_index(drop=True).index, 1], c='red', marker='.', alpha=.6, )\nplt.scatter(train2D[train[train.Survived==1].reset_index(drop=True).index, 0], train2D[train[train.Survived==1].reset_index(drop=True).index, 1], c='green', marker='.', alpha=.2)\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c6318c5f51b8fc8fe612d2216bcd7dfcecd0a032"},"cell_type":"markdown","source":"# PCA Analysis on 3 dimensions"},{"metadata":{"scrolled":true,"trusted":false,"_uuid":"c109993fed462145e1a65519f08b76e0641a1ebc"},"cell_type":"code","source":"ax = Axes3D(plt.figure(figsize=(10,10)))\ntrain3D = PCA(n_components=3).fit_transform(train.drop('Survived', axis='columns'))\nax.scatter3D(train3D[train[train.Survived==0].reset_index(drop=True).index, 0], train3D[train[train.Survived==0].reset_index(drop=True).index, 1], train3D[train[train.Survived==0].reset_index(drop=True).index, 2], color='red', marker='.', alpha=.25, s=450)\nax.scatter3D(train3D[train[train.Survived==1].reset_index(drop=True).index, 0], train3D[train[train.Survived==1].reset_index(drop=True).index, 1], train3D[train[train.Survived==1].reset_index(drop=True).index, 2], color='green', marker='.', alpha=.9, s=300)\nax.view_init(elev=25, azim=45)\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"95378f0d20b133745875563e39afd6111532de36"},"cell_type":"code","source":"from sklearn.neighbors import KNeighborsClassifier\nfrom sklearn.tree import DecisionTreeClassifier\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.naive_bayes import GaussianNB\nfrom sklearn.svm import SVC\nimport numpy as np","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"32e978010fdd8e3d2b45a1f1afcac1639c2b0987"},"cell_type":"code","source":"train_data = train.drop(columns='Survived')\ntarget = train['Survived']\n\ntrain_data.shape, target.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"3e28bcb05b2c9640757f494c88a99826e8ed5cf1"},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nfrom sklearn.metrics import precision_recall_curve, make_scorer, recall_score, precision_score, average_precision_score, confusion_matrix, accuracy_score\nfrom sklearn.model_selection import GridSearchCV","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"d8084a20fc8fdf2abd03121c168d7c5cceef46fd"},"cell_type":"code","source":"X_train, X_test, y_train, y_test = train_test_split(train_data, train.Survived, test_size=0.10, stratify=train['Survived'].values, random_state=42)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4ce76c00b1e166504e02390937416a047a07e3de"},"cell_type":"markdown","source":"# CrossValidation on 10 splits to test my model"},{"metadata":{"trusted":false,"_uuid":"898f491f9121732f1723bc1d401394e6e2ca9e30"},"cell_type":"code","source":"from sklearn.model_selection import KFold\nfrom sklearn.model_selection import cross_val_score\nk_fold = KFold(n_splits=10, shuffle=True, random_state=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"8669596cb0eeca9164998911ccc11ac5bc4379a1"},"cell_type":"code","source":"clf = SVC(C=1.0, kernel='rbf', degree=3, gamma='auto', probability=True)\nscoring = 'accuracy'\nscore = cross_val_score(clf, train_data, target, cv=k_fold, n_jobs=1, scoring=scoring)\nprint(score)\nprint('The total score is: '+str(score.mean()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"ceeae9dc1a1e6bcc0d6ee3a8829e4321a35c2fed"},"cell_type":"code","source":"clf.fit(train_data, target)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"efd5a3f72d83d7f899a0d9a954e124f3b80fecac"},"cell_type":"code","source":"predictions = clf.predict(X_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"d514f4f944c06664d4b657ca41ef2e0783a590d3"},"cell_type":"code","source":"y_score = clf.predict_proba(X_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"0e6b4945c46fd5bbb080766e138065cbaea2aa2b"},"cell_type":"code","source":"average_precision = average_precision_score(y_test, y_score[:, 0])\naverage_precision","execution_count":null,"outputs":[]},{"metadata":{"scrolled":false,"trusted":false,"_uuid":"ffbb51bcf0155771cc7768f4f33a426e920f09f1"},"cell_type":"code","source":"import scikitplot as skplt","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":false,"_uuid":"85f067cae3d5fcb3faf0111cdf44db127c18da8f"},"cell_type":"code","source":"skplt.metrics.plot_precision_recall(y_test, y_score, figsize=(15,10))\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"dc5cf885eb168bcb106deb70b8a143bade129ba6"},"cell_type":"code","source":"skplt.metrics.plot_roc(y_test, y_score, figsize=(15,10))\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"60bb05caa967c40f0cac81fd8006cf95d2acabe2"},"cell_type":"code","source":"test_prediction = clf.predict(test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"4bbc93bf05f8b6d27def80039633ba7ecd389b5d"},"cell_type":"code","source":"prediction = clf.predict(test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"eae040e7ab572fee48f174f42f6e225672613ec0"},"cell_type":"code","source":"submission = pd.DataFrame({\n        \"PassengerId\": test_key,\n        \"Survived\": prediction\n    })\n\nsubmission.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"8aafb11c915e1f04ddb3ff1e461da361aba917ab"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.5"}},"nbformat":4,"nbformat_minor":1}