{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Refer to my model training notebook over here:\nhttps://www.kaggle.com/code/pohzixiang/titanic-final-prediction","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-27T05:16:36.141946Z","iopub.execute_input":"2022-07-27T05:16:36.142360Z","iopub.status.idle":"2022-07-27T05:16:36.152223Z","shell.execute_reply.started":"2022-07-27T05:16:36.142326Z","shell.execute_reply":"2022-07-27T05:16:36.151050Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Define our training and test data\ntrain=pd.read_csv('/kaggle/input/titanic/train.csv')\ntest=pd.read_csv('/kaggle/input/titanic/test.csv')","metadata":{"execution":{"iopub.status.busy":"2022-07-27T05:16:37.540856Z","iopub.execute_input":"2022-07-27T05:16:37.542006Z","iopub.status.idle":"2022-07-27T05:16:37.569017Z","shell.execute_reply.started":"2022-07-27T05:16:37.541966Z","shell.execute_reply":"2022-07-27T05:16:37.568168Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-27T05:16:39.068760Z","iopub.execute_input":"2022-07-27T05:16:39.069143Z","iopub.status.idle":"2022-07-27T05:16:39.097634Z","shell.execute_reply.started":"2022-07-27T05:16:39.069113Z","shell.execute_reply":"2022-07-27T05:16:39.096242Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Check number of missing values in each columns for training data\nmissing=train.isnull().sum()\nprint(missing)\nprint('total data:', len(train))","metadata":{"execution":{"iopub.status.busy":"2022-07-27T05:16:40.502740Z","iopub.execute_input":"2022-07-27T05:16:40.503141Z","iopub.status.idle":"2022-07-27T05:16:40.511465Z","shell.execute_reply.started":"2022-07-27T05:16:40.503109Z","shell.execute_reply":"2022-07-27T05:16:40.510617Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"##For Embarked\n\n#Target column\ntarget='Survived'\n#feature column\nfeature='Embarked'\n\nf_index=list(train[feature].value_counts().index)\nt_index=list(train[target].value_counts().index)\n\nfor i in f_index:\n    print('number of',i,':',sum(train[feature]==i))\n    print('Percentage of',i,':', sum(train[feature]==i)/(len(train)-train[feature].isnull().sum()))\n    for j in t_index:\n        print('Percentage of',i,'and',j,':', sum(train[train[target]==j][feature]==i)/sum(train[feature]==i))\n    print('-'*100)","metadata":{"execution":{"iopub.status.busy":"2022-07-27T05:16:41.951643Z","iopub.execute_input":"2022-07-27T05:16:41.952759Z","iopub.status.idle":"2022-07-27T05:16:41.984992Z","shell.execute_reply.started":"2022-07-27T05:16:41.952719Z","shell.execute_reply":"2022-07-27T05:16:41.984084Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"It does appear that Embarked could give us some information on survival rate as well. We would keep it as our feature.","metadata":{}},{"cell_type":"code","source":"##For Pclass\n\n#Target column\ntarget='Survived'\n#feature column\nfeature='Pclass'\n\nf_index=list(train[feature].value_counts().index)\nt_index=list(train[target].value_counts().index)\n\nfor i in f_index:\n    print('number of',i,':',sum(train[feature]==i))\n    print('Percentage of',i,':', sum(train[feature]==i)/(len(train)-train[feature].isnull().sum()))\n    for j in t_index:\n        print('Percentage of',i,'and',j,':', sum(train[train[target]==j][feature]==i)/sum(train[feature]==i))\n    print('-'*100)","metadata":{"execution":{"iopub.status.busy":"2022-07-27T05:16:43.807768Z","iopub.execute_input":"2022-07-27T05:16:43.808352Z","iopub.status.idle":"2022-07-27T05:16:43.827139Z","shell.execute_reply.started":"2022-07-27T05:16:43.808319Z","shell.execute_reply":"2022-07-27T05:16:43.826181Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"It seems that most in Pclass 3 did not survived (75%), and most of Pclass 1 survived (62%). Thus, we would keep Pclass as one of our features.","metadata":{}},{"cell_type":"code","source":"##For survived\n\n#Target column\ntarget='Survived'\n#feature column\nfeature='Sex'\n\nf_index=list(train[feature].value_counts().index)\nt_index=list(train[target].value_counts().index)\n\nfor i in f_index:\n    print('number of',i,':',sum(train[feature]==i))\n    print('Percentage of',i,':', sum(train[feature]==i)/(len(train)-train[feature].isnull().sum()))\n    for j in t_index:\n        print('Percentage of',i,'and',j,':', sum(train[train[target]==j][feature]==i)/sum(train[feature]==i))\n    print('-'*100)","metadata":{"execution":{"iopub.status.busy":"2022-07-27T05:16:45.964466Z","iopub.execute_input":"2022-07-27T05:16:45.964913Z","iopub.status.idle":"2022-07-27T05:16:45.987274Z","shell.execute_reply.started":"2022-07-27T05:16:45.964878Z","shell.execute_reply":"2022-07-27T05:16:45.985893Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Large precntage of males did not survived (81%), and large percentage of females survived (74%).","metadata":{}},{"cell_type":"code","source":"import seaborn as sns","metadata":{"execution":{"iopub.status.busy":"2022-07-27T05:16:47.568404Z","iopub.execute_input":"2022-07-27T05:16:47.568828Z","iopub.status.idle":"2022-07-27T05:16:48.223032Z","shell.execute_reply.started":"2022-07-27T05:16:47.568792Z","shell.execute_reply":"2022-07-27T05:16:48.221970Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Considering that Cabin has so many missing data, it is unlikely to be very useful. We will see if it actually gives us any insight. We split the first letter of the cabin.","metadata":{}},{"cell_type":"code","source":"tcopy=train.copy()\ntcopy['Cabin1']=tcopy['Cabin'].apply(lambda x:str(x)[0])\ntcopy['Cabin1']=tcopy['Cabin1'].replace({'n':np.nan})\nsns.countplot(x='Survived',hue='Cabin1',data=tcopy)","metadata":{"execution":{"iopub.status.busy":"2022-07-27T05:16:49.513893Z","iopub.execute_input":"2022-07-27T05:16:49.514683Z","iopub.status.idle":"2022-07-27T05:16:49.824987Z","shell.execute_reply.started":"2022-07-27T05:16:49.514636Z","shell.execute_reply":"2022-07-27T05:16:49.823845Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Seems like there is no obvious insights from the plot. We will discard Cabin from our features for the model.","metadata":{}},{"cell_type":"code","source":"copy=train.copy()","metadata":{"execution":{"iopub.status.busy":"2022-07-27T05:16:52.073752Z","iopub.execute_input":"2022-07-27T05:16:52.074411Z","iopub.status.idle":"2022-07-27T05:16:52.079640Z","shell.execute_reply.started":"2022-07-27T05:16:52.074376Z","shell.execute_reply":"2022-07-27T05:16:52.078513Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Plot for fare\nsns.histplot(data=copy, x=\"Fare\")","metadata":{"execution":{"iopub.status.busy":"2022-07-27T05:16:54.019143Z","iopub.execute_input":"2022-07-27T05:16:54.019529Z","iopub.status.idle":"2022-07-27T05:16:54.409871Z","shell.execute_reply.started":"2022-07-27T05:16:54.019500Z","shell.execute_reply":"2022-07-27T05:16:54.408704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Plot for age\nsns.histplot(data=copy, x=\"Age\")","metadata":{"execution":{"iopub.status.busy":"2022-07-27T05:16:56.374393Z","iopub.execute_input":"2022-07-27T05:16:56.375301Z","iopub.status.idle":"2022-07-27T05:16:56.613411Z","shell.execute_reply.started":"2022-07-27T05:16:56.375261Z","shell.execute_reply":"2022-07-27T05:16:56.612366Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We try to see what other possible useful information we can extract from the table. Since we would need to impute quite a number of missing age, we see if there are other indicators that can give a better estimation of the individual's age.\n\nWe try the following:\n* Extract their title (Mr, Miss, Mrs etc) \n* Extract their First name (Could possible show us the relationship between families)","metadata":{}},{"cell_type":"code","source":"import re\ndef look_title(x):\n    return re.findall(\"(?<=,\\s).+?(?=\\.)\", x)[0]\n    \ntcopy['title']=tcopy['Name'].apply(lambda x: look_title(x))\ntcopy.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-27T05:16:58.564402Z","iopub.execute_input":"2022-07-27T05:16:58.565242Z","iopub.status.idle":"2022-07-27T05:16:58.591718Z","shell.execute_reply.started":"2022-07-27T05:16:58.565189Z","shell.execute_reply":"2022-07-27T05:16:58.590553Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tcopy['title'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-07-27T05:17:00.496261Z","iopub.execute_input":"2022-07-27T05:17:00.497295Z","iopub.status.idle":"2022-07-27T05:17:00.508036Z","shell.execute_reply.started":"2022-07-27T05:17:00.497234Z","shell.execute_reply":"2022-07-27T05:17:00.506872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"To make out plot clearer, we will focus on those title with >10 individual","metadata":{}},{"cell_type":"code","source":"copy2=tcopy[(tcopy['title']=='Mr') | (tcopy['title']=='Mrs') | (tcopy['title']=='Miss') | (tcopy['title']=='Master')]","metadata":{"execution":{"iopub.status.busy":"2022-07-27T05:17:02.118186Z","iopub.execute_input":"2022-07-27T05:17:02.118924Z","iopub.status.idle":"2022-07-27T05:17:02.126513Z","shell.execute_reply.started":"2022-07-27T05:17:02.118888Z","shell.execute_reply":"2022-07-27T05:17:02.125643Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.boxplot(x='Survived',y='Age',data=copy2,hue='title')","metadata":{"execution":{"iopub.status.busy":"2022-07-27T05:17:04.304441Z","iopub.execute_input":"2022-07-27T05:17:04.305174Z","iopub.status.idle":"2022-07-27T05:17:04.767238Z","shell.execute_reply.started":"2022-07-27T05:17:04.305131Z","shell.execute_reply":"2022-07-27T05:17:04.766067Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Some interesting observation:\n* Those with title 'Master' tend to be very young. \n* Title 'Miss' seems to be relatively younger than Mrs as well. \n\nWe can consider imputing median age based on the title.","metadata":{}},{"cell_type":"code","source":"#Extract First name\ncopy2['Surname']=copy2['Name'].apply(lambda x: x.split(',')[0])\ncopy2.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-27T05:17:07.229057Z","iopub.execute_input":"2022-07-27T05:17:07.229595Z","iopub.status.idle":"2022-07-27T05:17:07.252246Z","shell.execute_reply.started":"2022-07-27T05:17:07.229536Z","shell.execute_reply":"2022-07-27T05:17:07.251320Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(sum(copy2.groupby(by=['Ticket','Surname'])['PassengerId'].count()>1))\na=copy2.groupby(by=['Ticket','Surname'])['PassengerId'].count().sort_values(ascending=False)","metadata":{"execution":{"iopub.status.busy":"2022-07-27T05:20:37.977494Z","iopub.execute_input":"2022-07-27T05:20:37.977909Z","iopub.status.idle":"2022-07-27T05:20:37.997198Z","shell.execute_reply.started":"2022-07-27T05:20:37.977878Z","shell.execute_reply":"2022-07-27T05:20:37.996343Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We can see that there are around 94 groups of people with same ticket and surname. Thus, we can possibly create a feature Ticketgrp that counts the number of people with same number of ticket and surname as the person.","metadata":{}},{"cell_type":"code","source":"b=copy2.groupby(by=['Ticket','Surname'])['Survived'].sum()","metadata":{"execution":{"iopub.status.busy":"2022-07-27T05:21:25.895510Z","iopub.execute_input":"2022-07-27T05:21:25.895909Z","iopub.status.idle":"2022-07-27T05:21:25.910831Z","shell.execute_reply.started":"2022-07-27T05:21:25.895876Z","shell.execute_reply":"2022-07-27T05:21:25.909610Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data=pd.concat([a,b],axis=1)\ndata.rename(columns = {'PassengerId':'FamilySize','Survived':'No. survived'}, inplace = True)\ndata[data['FamilySize']>1]","metadata":{"execution":{"iopub.status.busy":"2022-07-27T05:46:36.808035Z","iopub.execute_input":"2022-07-27T05:46:36.808409Z","iopub.status.idle":"2022-07-27T05:46:36.827807Z","shell.execute_reply.started":"2022-07-27T05:46:36.808380Z","shell.execute_reply":"2022-07-27T05:46:36.826826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.boxplot(data=data, x='FamilySize', y='No. survived')","metadata":{"execution":{"iopub.status.busy":"2022-07-27T05:55:37.278759Z","iopub.execute_input":"2022-07-27T05:55:37.279158Z","iopub.status.idle":"2022-07-27T05:55:37.498505Z","shell.execute_reply.started":"2022-07-27T05:55:37.279128Z","shell.execute_reply":"2022-07-27T05:55:37.497274Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"There seems to be a general trend that:\n* The chance of survival generally increases as family size increase from 1 to 4\n* The chance decrease when family size is above 4","metadata":{}},{"cell_type":"markdown","source":"Objective when processing data:\n* Create Surname column\n* Create Title column\n* Create Ticket group (From above, we see that those with same tickets are mostly from same family)\n* Remove Initial ticket column\n","metadata":{}}]}