{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"**Kindly upvote the notebook if you like. Do drop comments/thoughts for improvement**","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.\ntrain_data = pd.read_csv(\"/kaggle/input/titanic/train.csv\")\ntest_data = pd.read_csv('../input/titanic/test.csv')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-19T12:07:27.477422Z","iopub.execute_input":"2022-07-19T12:07:27.477908Z","iopub.status.idle":"2022-07-19T12:07:27.745846Z","shell.execute_reply.started":"2022-07-19T12:07:27.477862Z","shell.execute_reply":"2022-07-19T12:07:27.744958Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data exploration","metadata":{}},{"cell_type":"code","source":"train_data.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-19T12:07:27.748112Z","iopub.execute_input":"2022-07-19T12:07:27.748420Z","iopub.status.idle":"2022-07-19T12:07:27.778979Z","shell.execute_reply.started":"2022-07-19T12:07:27.748364Z","shell.execute_reply":"2022-07-19T12:07:27.778251Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Data Dictionary**\n> Variable\tDefinition\tKey\n\n> survival \tSurvival \t0 = No, 1 = Yes\n\n> pclass \tTicket class \t1 = 1st, 2 = 2nd, 3 = 3rd\n\n> sex \tSex \t\n\n> Age \tAge in years \t\n> sibsp \t# of siblings / spouses aboard the Titanic \t\n> parch \t# of parents / children aboard the Titanic \t\n> ticket \tTicket number \t\n> fare \tPassenger fare \t\n> cabin \tCabin number \t\n> embarked \tPort of Embarkation \tC = Cherbourg, Q = Queenstown, S = Southampton\n\n\n**Variable Notes**\n\n**pclass**: A proxy for socio-economic status (SES)\n1st = Upper\n2nd = Middle\n3rd = Lower\n\n**age**: Age is fractional if less than 1. If the age is estimated, is it in the form of xx.5\n\n**sibsp**: The dataset defines family relations in this way...\n\n**Sibling** = brother, sister, stepbrother, stepsister\n\n**Spouse** = husband, wife (mistresses and fiancés were ignored)\n\n**parch**: The dataset defines family relations in this way...\n\n**Parent** = mother, father\n\n**Child** = daughter, son, stepdaughter, stepson\n\nSome children travelled only with a nanny, therefore parch=0 for them.","metadata":{}},{"cell_type":"code","source":"test_data.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-19T12:07:27.780660Z","iopub.execute_input":"2022-07-19T12:07:27.781375Z","iopub.status.idle":"2022-07-19T12:07:27.800511Z","shell.execute_reply.started":"2022-07-19T12:07:27.781310Z","shell.execute_reply":"2022-07-19T12:07:27.799497Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data.shape","metadata":{"execution":{"iopub.status.busy":"2022-07-19T12:07:27.801650Z","iopub.execute_input":"2022-07-19T12:07:27.801944Z","iopub.status.idle":"2022-07-19T12:07:27.807509Z","shell.execute_reply.started":"2022-07-19T12:07:27.801865Z","shell.execute_reply":"2022-07-19T12:07:27.806528Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data.shape","metadata":{"execution":{"iopub.status.busy":"2022-07-19T12:07:27.811527Z","iopub.execute_input":"2022-07-19T12:07:27.812276Z","iopub.status.idle":"2022-07-19T12:07:27.821536Z","shell.execute_reply.started":"2022-07-19T12:07:27.812211Z","shell.execute_reply":"2022-07-19T12:07:27.820525Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data.info()","metadata":{"execution":{"iopub.status.busy":"2022-07-19T12:07:27.824679Z","iopub.execute_input":"2022-07-19T12:07:27.825367Z","iopub.status.idle":"2022-07-19T12:07:27.838383Z","shell.execute_reply.started":"2022-07-19T12:07:27.825298Z","shell.execute_reply":"2022-07-19T12:07:27.837379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-07-19T12:07:27.840074Z","iopub.execute_input":"2022-07-19T12:07:27.840699Z","iopub.status.idle":"2022-07-19T12:07:27.853361Z","shell.execute_reply.started":"2022-07-19T12:07:27.840637Z","shell.execute_reply":"2022-07-19T12:07:27.852360Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-07-19T12:07:27.854743Z","iopub.execute_input":"2022-07-19T12:07:27.855014Z","iopub.status.idle":"2022-07-19T12:07:27.868577Z","shell.execute_reply.started":"2022-07-19T12:07:27.854966Z","shell.execute_reply":"2022-07-19T12:07:27.867615Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**We plan on replacing age with average age of repective category,Cabin has too many NaN values to handle.Dropping off cabin**","metadata":{}},{"cell_type":"code","source":"train_data['Sex'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-07-19T12:07:27.870215Z","iopub.execute_input":"2022-07-19T12:07:27.870517Z","iopub.status.idle":"2022-07-19T12:07:27.882623Z","shell.execute_reply.started":"2022-07-19T12:07:27.870458Z","shell.execute_reply":"2022-07-19T12:07:27.881724Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data[train_data['Sex'] == 'male']['Age']","metadata":{"execution":{"iopub.status.busy":"2022-07-19T12:07:27.884250Z","iopub.execute_input":"2022-07-19T12:07:27.884824Z","iopub.status.idle":"2022-07-19T12:07:27.897875Z","shell.execute_reply.started":"2022-07-19T12:07:27.884552Z","shell.execute_reply":"2022-07-19T12:07:27.896867Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"male_average_age = train_data[train_data['Sex'] == 'male']['Age'].mean()\nfemale_average_age = train_data[train_data['Sex'] == 'female']['Age'].mean()\nmale_average_age,female_average_age","metadata":{"execution":{"iopub.status.busy":"2022-07-19T12:07:27.899175Z","iopub.execute_input":"2022-07-19T12:07:27.899526Z","iopub.status.idle":"2022-07-19T12:07:27.908229Z","shell.execute_reply.started":"2022-07-19T12:07:27.899488Z","shell.execute_reply":"2022-07-19T12:07:27.907522Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Let's round the average age to appropriate precision**","metadata":{}},{"cell_type":"code","source":"male_average_age,female_average_age = round(male_average_age,0),round(female_average_age,0)\nmale_average_age,female_average_age","metadata":{"execution":{"iopub.status.busy":"2022-07-19T12:07:27.909310Z","iopub.execute_input":"2022-07-19T12:07:27.909595Z","iopub.status.idle":"2022-07-19T12:07:27.922508Z","shell.execute_reply.started":"2022-07-19T12:07:27.909541Z","shell.execute_reply":"2022-07-19T12:07:27.921439Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data.loc[(train_data['Sex'] == 'male') & (train_data.Age.isnull())]","metadata":{"execution":{"iopub.status.busy":"2022-07-19T12:07:27.924025Z","iopub.execute_input":"2022-07-19T12:07:27.924505Z","iopub.status.idle":"2022-07-19T12:07:27.957352Z","shell.execute_reply.started":"2022-07-19T12:07:27.924456Z","shell.execute_reply":"2022-07-19T12:07:27.956402Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data.loc[(train_data['Sex'] == 'male') & (train_data.Age.isnull()),'Age'] ","metadata":{"execution":{"iopub.status.busy":"2022-07-19T12:07:27.958603Z","iopub.execute_input":"2022-07-19T12:07:27.959004Z","iopub.status.idle":"2022-07-19T12:07:27.969635Z","shell.execute_reply.started":"2022-07-19T12:07:27.958830Z","shell.execute_reply":"2022-07-19T12:07:27.968407Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data.loc[(train_data['Sex'] == 'male') & (train_data.Age.isnull()),'Age'] = male_average_age\ntrain_data.loc[(train_data['Sex'] == 'female') & (train_data.Age.isnull()),'Age'] = female_average_age","metadata":{"execution":{"iopub.status.busy":"2022-07-19T12:07:27.970948Z","iopub.execute_input":"2022-07-19T12:07:27.971412Z","iopub.status.idle":"2022-07-19T12:07:27.989643Z","shell.execute_reply.started":"2022-07-19T12:07:27.971355Z","shell.execute_reply":"2022-07-19T12:07:27.988664Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data.loc[(test_data['Sex'] == 'male') & (test_data.Age.isnull()),'Age'] = male_average_age\ntest_data.loc[(test_data['Sex'] == 'female') & (test_data.Age.isnull()),'Age'] = female_average_age","metadata":{"execution":{"iopub.status.busy":"2022-07-19T12:07:27.991129Z","iopub.execute_input":"2022-07-19T12:07:27.991702Z","iopub.status.idle":"2022-07-19T12:07:28.005506Z","shell.execute_reply.started":"2022-07-19T12:07:27.991639Z","shell.execute_reply":"2022-07-19T12:07:28.004462Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data.loc[(train_data['Sex'] == 'male') & (train_data.Age.isnull()),'Age'] ","metadata":{"execution":{"iopub.status.busy":"2022-07-19T12:07:28.006659Z","iopub.execute_input":"2022-07-19T12:07:28.007076Z","iopub.status.idle":"2022-07-19T12:07:28.017428Z","shell.execute_reply.started":"2022-07-19T12:07:28.006997Z","shell.execute_reply":"2022-07-19T12:07:28.015724Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data.loc[(test_data['Sex'] == 'male') & (test_data.Age.isnull()),'Age'] ","metadata":{"execution":{"iopub.status.busy":"2022-07-19T12:07:28.021733Z","iopub.execute_input":"2022-07-19T12:07:28.022414Z","iopub.status.idle":"2022-07-19T12:07:28.041698Z","shell.execute_reply.started":"2022-07-19T12:07:28.022317Z","shell.execute_reply":"2022-07-19T12:07:28.040255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data.Age.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-07-19T12:07:28.047664Z","iopub.execute_input":"2022-07-19T12:07:28.048097Z","iopub.status.idle":"2022-07-19T12:07:28.056550Z","shell.execute_reply.started":"2022-07-19T12:07:28.048010Z","shell.execute_reply":"2022-07-19T12:07:28.055507Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"How many members from a family are there with the passenger ?","metadata":{}},{"cell_type":"code","source":"train_data['Family members'] = train_data['Parch'] + train_data['SibSp'] +1\n\n# We have included that passenger also.\ntrain_data.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-19T12:07:28.058068Z","iopub.execute_input":"2022-07-19T12:07:28.058604Z","iopub.status.idle":"2022-07-19T12:07:28.119111Z","shell.execute_reply.started":"2022-07-19T12:07:28.058541Z","shell.execute_reply":"2022-07-19T12:07:28.118071Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Doing similar on testing also\ntest_data['Family members'] = test_data['Parch'] + test_data['SibSp'] +1\n\n# We have included that passenger also.\ntest_data.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-19T12:07:28.120588Z","iopub.execute_input":"2022-07-19T12:07:28.121156Z","iopub.status.idle":"2022-07-19T12:07:28.144405Z","shell.execute_reply.started":"2022-07-19T12:07:28.121080Z","shell.execute_reply":"2022-07-19T12:07:28.143344Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data['Family members'].head()","metadata":{"execution":{"iopub.status.busy":"2022-07-19T12:07:28.146168Z","iopub.execute_input":"2022-07-19T12:07:28.146649Z","iopub.status.idle":"2022-07-19T12:07:28.156044Z","shell.execute_reply.started":"2022-07-19T12:07:28.146581Z","shell.execute_reply":"2022-07-19T12:07:28.154905Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def PclassToClass(classNum:int)->str:\n    if classNum == 1:\n        return 'Class 1'\n    elif classNum == 2:\n        return 'Class 2'\n    else:\n        return 'Class 3'","metadata":{"execution":{"iopub.status.busy":"2022-07-19T12:07:28.158685Z","iopub.execute_input":"2022-07-19T12:07:28.160205Z","iopub.status.idle":"2022-07-19T12:07:28.168487Z","shell.execute_reply.started":"2022-07-19T12:07:28.158913Z","shell.execute_reply":"2022-07-19T12:07:28.166986Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data['Class'] = list(map(PclassToClass,train_data.Pclass))\ntrain_data.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-19T12:07:28.171506Z","iopub.execute_input":"2022-07-19T12:07:28.172171Z","iopub.status.idle":"2022-07-19T12:07:28.201334Z","shell.execute_reply.started":"2022-07-19T12:07:28.171842Z","shell.execute_reply":"2022-07-19T12:07:28.200202Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data['Class'] = list(map(PclassToClass,test_data.Pclass))\ntest_data.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-19T12:07:28.206033Z","iopub.execute_input":"2022-07-19T12:07:28.208576Z","iopub.status.idle":"2022-07-19T12:07:28.230624Z","shell.execute_reply.started":"2022-07-19T12:07:28.208497Z","shell.execute_reply":"2022-07-19T12:07:28.229866Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data.drop(columns='Cabin',inplace=True)\ntest_data.drop(columns='Cabin',inplace=True)\n\ntrain_data.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-19T12:07:28.231682Z","iopub.execute_input":"2022-07-19T12:07:28.232037Z","iopub.status.idle":"2022-07-19T12:07:28.258478Z","shell.execute_reply.started":"2022-07-19T12:07:28.231997Z","shell.execute_reply":"2022-07-19T12:07:28.257676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-19T12:07:28.259471Z","iopub.execute_input":"2022-07-19T12:07:28.259811Z","iopub.status.idle":"2022-07-19T12:07:28.277747Z","shell.execute_reply.started":"2022-07-19T12:07:28.259773Z","shell.execute_reply":"2022-07-19T12:07:28.276882Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Visualizing different fields ","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\nimport matplotlib.gridspec as grid_spec\nfrom matplotlib.ticker import FuncFormatter\nimport matplotlib as mpl","metadata":{"execution":{"iopub.status.busy":"2022-07-19T12:07:28.283159Z","iopub.execute_input":"2022-07-19T12:07:28.283781Z","iopub.status.idle":"2022-07-19T12:07:29.081384Z","shell.execute_reply.started":"2022-07-19T12:07:28.283705Z","shell.execute_reply":"2022-07-19T12:07:29.080286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.style.use('seaborn-notebook')\nplt.rcParams['figure.figsize'] = (16,14)\nsns.set(rc={'figure.figsize':(16,14)})","metadata":{"execution":{"iopub.status.busy":"2022-07-19T12:07:29.085242Z","iopub.execute_input":"2022-07-19T12:07:29.085611Z","iopub.status.idle":"2022-07-19T12:07:29.093171Z","shell.execute_reply.started":"2022-07-19T12:07:29.085546Z","shell.execute_reply":"2022-07-19T12:07:29.092098Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def latexTextWriter(text:str):\n    text = text.replace(' ','\\:')\n    return '$' + text + '$'","metadata":{"execution":{"iopub.status.busy":"2022-07-19T12:07:29.095710Z","iopub.execute_input":"2022-07-19T12:07:29.096332Z","iopub.status.idle":"2022-07-19T12:07:29.108541Z","shell.execute_reply.started":"2022-07-19T12:07:29.096249Z","shell.execute_reply":"2022-07-19T12:07:29.107631Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.scatter(train_data['Age'],train_data['Class'])\nplt.xlabel(latexTextWriter('Age of passengers'))\nplt.ylabel(latexTextWriter('Class of travel'))\nplt.xticks(rotation=35)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-19T12:07:29.110049Z","iopub.execute_input":"2022-07-19T12:07:29.110701Z","iopub.status.idle":"2022-07-19T12:07:29.583076Z","shell.execute_reply.started":"2022-07-19T12:07:29.110632Z","shell.execute_reply":"2022-07-19T12:07:29.582196Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.scatterplot(x='Age',y='Class',data=train_data,hue=train_data['Sex'])\nplt.xlabel(latexTextWriter('Age of passengers'))\nplt.ylabel(latexTextWriter('Class of travel'))\nplt.xticks(rotation=35)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-19T12:07:29.584548Z","iopub.execute_input":"2022-07-19T12:07:29.584796Z","iopub.status.idle":"2022-07-19T12:07:29.995406Z","shell.execute_reply.started":"2022-07-19T12:07:29.584752Z","shell.execute_reply":"2022-07-19T12:07:29.994026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Did those who paid high fares get any special benefits to deboard earlier ?","metadata":{}},{"cell_type":"code","source":"farePerCapita = round(train_data.Fare / (train_data['Family members']),2)\nfarePerCapita","metadata":{"execution":{"iopub.status.busy":"2022-07-19T12:07:29.997470Z","iopub.execute_input":"2022-07-19T12:07:29.997896Z","iopub.status.idle":"2022-07-19T12:07:30.008634Z","shell.execute_reply.started":"2022-07-19T12:07:29.997819Z","shell.execute_reply":"2022-07-19T12:07:30.007496Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"avg_perCapita = round(farePerCapita.mean(),2)\navg_perCapita","metadata":{"execution":{"iopub.status.busy":"2022-07-19T12:07:30.011388Z","iopub.execute_input":"2022-07-19T12:07:30.012304Z","iopub.status.idle":"2022-07-19T12:07:30.026998Z","shell.execute_reply.started":"2022-07-19T12:07:30.012233Z","shell.execute_reply":"2022-07-19T12:07:30.025436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"farePerCapita.max(),train_data.Fare.max(),","metadata":{"execution":{"iopub.status.busy":"2022-07-19T12:07:30.028514Z","iopub.execute_input":"2022-07-19T12:07:30.028878Z","iopub.status.idle":"2022-07-19T12:07:30.042746Z","shell.execute_reply.started":"2022-07-19T12:07:30.028825Z","shell.execute_reply":"2022-07-19T12:07:30.041722Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Data is comparablly similar","metadata":{}},{"cell_type":"code","source":"sns.scatterplot(x=farePerCapita,y=train_data.Class,hue=(train_data['Survived']==1))\nplt.xlabel(latexTextWriter('Price paid per member travelled'))\nplt.ylabel(latexTextWriter('Class of travel'))\nplt.xticks(rotation=35)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-19T12:07:30.044230Z","iopub.execute_input":"2022-07-19T12:07:30.044650Z","iopub.status.idle":"2022-07-19T12:07:30.439114Z","shell.execute_reply.started":"2022-07-19T12:07:30.044581Z","shell.execute_reply":"2022-07-19T12:07:30.438064Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nsns.catplot('Family members',\n                   'Survived',\n                   hue='Sex',\n                   data=train_data,\n                   aspect=4,\n                   kind='point')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-19T12:07:30.440863Z","iopub.execute_input":"2022-07-19T12:07:30.441183Z","iopub.status.idle":"2022-07-19T12:07:31.136651Z","shell.execute_reply.started":"2022-07-19T12:07:30.441132Z","shell.execute_reply":"2022-07-19T12:07:31.135679Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Makes sense, \"In the movie they `WOMEN AND CHILDREN FIRST` \"","metadata":{}},{"cell_type":"code","source":"sns.catplot(x='Class',y='Age',data=train_data,height=8)\nplt.ylabel(latexTextWriter('Age of passengers'))\nplt.xlabel(latexTextWriter('Class of travel'))\nplt.xticks(rotation=35)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-19T12:07:31.138378Z","iopub.execute_input":"2022-07-19T12:07:31.138704Z","iopub.status.idle":"2022-07-19T12:07:31.431740Z","shell.execute_reply.started":"2022-07-19T12:07:31.138653Z","shell.execute_reply":"2022-07-19T12:07:31.430615Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.boxplot(x='Class',y='Age',data=train_data, palette='rainbow')\nplt.title(\"Age by Passenger Class \")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-19T12:07:31.433699Z","iopub.execute_input":"2022-07-19T12:07:31.434163Z","iopub.status.idle":"2022-07-19T12:07:31.719650Z","shell.execute_reply.started":"2022-07-19T12:07:31.434047Z","shell.execute_reply":"2022-07-19T12:07:31.718563Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Old people would like to lavishly spend money in the last days huh? It seems so. \n\nClass 3 and class 2 are having lot of outliers, especially in the higher bracket of age.","metadata":{}},{"cell_type":"code","source":"sns.boxplot(x='Class',y='Age',data=train_data, palette='rainbow',hue='Survived')\nplt.title(\"Age by Passenger Class \")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-19T12:07:31.721313Z","iopub.execute_input":"2022-07-19T12:07:31.721615Z","iopub.status.idle":"2022-07-19T12:07:32.064216Z","shell.execute_reply.started":"2022-07-19T12:07:31.721559Z","shell.execute_reply":"2022-07-19T12:07:32.063289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Why Boxen plot ?\n\nWith more quantiles, we can see more info about the distribution shape beyond the central 50% of the data; this extra detail is especially present in the tails, where box plots tend to give limited information.","metadata":{}},{"cell_type":"code","source":"sns.boxenplot(x='Pclass', y='Age', data=train_data, palette='cividis_r',hue='Survived')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-19T12:07:32.065771Z","iopub.execute_input":"2022-07-19T12:07:32.066080Z","iopub.status.idle":"2022-07-19T12:07:32.408684Z","shell.execute_reply.started":"2022-07-19T12:07:32.066007Z","shell.execute_reply":"2022-07-19T12:07:32.407616Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.swarmplot(x='Class',y='Age',data=train_data,dodge=True,hue='Survived')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-19T12:07:32.411113Z","iopub.execute_input":"2022-07-19T12:07:32.411399Z","iopub.status.idle":"2022-07-19T12:07:32.994919Z","shell.execute_reply.started":"2022-07-19T12:07:32.411350Z","shell.execute_reply":"2022-07-19T12:07:32.993844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.violinplot(x='Sex',y='Family members',data=train_data,dodge=True,hue='Survived')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-19T12:07:32.996685Z","iopub.execute_input":"2022-07-19T12:07:32.997027Z","iopub.status.idle":"2022-07-19T12:07:33.342137Z","shell.execute_reply.started":"2022-07-19T12:07:32.996965Z","shell.execute_reply":"2022-07-19T12:07:33.341187Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Combining plots","metadata":{}},{"cell_type":"code","source":"sns.violinplot(x='Class',y=\"Age\", data=train_data, hue='Survived', split='True', palette='cubehelix')\nsns.swarmplot(x='Class',y=\"Age\", data=train_data, hue='Survived', dodge='True', color='#526ED0', alpha=.8, s=4)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-19T12:07:33.343680Z","iopub.execute_input":"2022-07-19T12:07:33.343986Z","iopub.status.idle":"2022-07-19T12:07:34.019211Z","shell.execute_reply.started":"2022-07-19T12:07:33.343931Z","shell.execute_reply":"2022-07-19T12:07:34.018212Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.barplot(x='Embarked',y='Fare',data=train_data, palette='tab20_r', hue='Class')\nsns.stripplot(x='Embarked',y=\"Fare\",data=train_data, hue='Class', dodge='True', color=sns.color_palette('ch:s=-.2,r=.6')[0], alpha=.8, s=2)\nplt.title(\"Fare of Passenger by Embarked Town, Divided by Class\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-19T12:07:34.020832Z","iopub.execute_input":"2022-07-19T12:07:34.021161Z","iopub.status.idle":"2022-07-19T12:07:34.508439Z","shell.execute_reply.started":"2022-07-19T12:07:34.021100Z","shell.execute_reply":"2022-07-19T12:07:34.507618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data.columns","metadata":{"execution":{"iopub.status.busy":"2022-07-19T12:07:34.509604Z","iopub.execute_input":"2022-07-19T12:07:34.510017Z","iopub.status.idle":"2022-07-19T12:07:34.516363Z","shell.execute_reply.started":"2022-07-19T12:07:34.509956Z","shell.execute_reply":"2022-07-19T12:07:34.515378Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"g = sns.catplot(x='Class',y='Survived', col = 'Sex', data=train_data,\n                kind='bar', aspect=.6, palette='Set2')\n\n(g.set_axis_labels(\"Class\", \"Survival Rate\")\n  .set_titles(\"{col_name}\")\n  .set(ylim=(0,1)))\n\nplt.tight_layout()\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-19T12:07:34.517971Z","iopub.execute_input":"2022-07-19T12:07:34.518472Z","iopub.status.idle":"2022-07-19T12:07:34.950119Z","shell.execute_reply.started":"2022-07-19T12:07:34.518268Z","shell.execute_reply":"2022-07-19T12:07:34.949007Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data.columns","metadata":{"execution":{"iopub.status.busy":"2022-07-19T12:07:34.951749Z","iopub.execute_input":"2022-07-19T12:07:34.952044Z","iopub.status.idle":"2022-07-19T12:07:34.959516Z","shell.execute_reply.started":"2022-07-19T12:07:34.951995Z","shell.execute_reply":"2022-07-19T12:07:34.958350Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Updating dataframe to have only necessary features for model training**","metadata":{}},{"cell_type":"code","source":"train_data.drop(columns=['PassengerId','Name','SibSp','Parch','Ticket','Class'],inplace=True)\ntest_data.drop(columns=['PassengerId','Name','SibSp','Parch','Ticket','Class'],inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-19T12:07:34.961433Z","iopub.execute_input":"2022-07-19T12:07:34.961745Z","iopub.status.idle":"2022-07-19T12:07:34.973305Z","shell.execute_reply.started":"2022-07-19T12:07:34.961693Z","shell.execute_reply":"2022-07-19T12:07:34.972333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-19T12:07:34.974901Z","iopub.execute_input":"2022-07-19T12:07:34.975477Z","iopub.status.idle":"2022-07-19T12:07:35.003646Z","shell.execute_reply.started":"2022-07-19T12:07:34.975401Z","shell.execute_reply":"2022-07-19T12:07:35.002783Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = pd.get_dummies(train_data,drop_first=True)\ntrain_data.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-19T12:07:35.005015Z","iopub.execute_input":"2022-07-19T12:07:35.005551Z","iopub.status.idle":"2022-07-19T12:07:35.041841Z","shell.execute_reply.started":"2022-07-19T12:07:35.005355Z","shell.execute_reply":"2022-07-19T12:07:35.040763Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data = pd.get_dummies(test_data,drop_first=True)\ntest_data.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-19T12:07:35.043479Z","iopub.execute_input":"2022-07-19T12:07:35.043803Z","iopub.status.idle":"2022-07-19T12:07:35.075308Z","shell.execute_reply.started":"2022-07-19T12:07:35.043749Z","shell.execute_reply":"2022-07-19T12:07:35.073599Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"corr = train_data.corr()\n\nsns.heatmap(corr,cmap='seismic')\nplt.title(\"Correlation map\",fontsize=22,fontweight='bold')\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-19T12:07:35.077084Z","iopub.execute_input":"2022-07-19T12:07:35.077440Z","iopub.status.idle":"2022-07-19T12:07:35.541333Z","shell.execute_reply.started":"2022-07-19T12:07:35.077377Z","shell.execute_reply":"2022-07-19T12:07:35.540211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We can understand the following from the heatmap\n\n* Embarked_S and Embarked_Q are negatively correlated because a person boarding @ Queenstown;didn't embark@Southampton or Cherbourg. It makes sense for it to have high negative correlation\n\n* As seen before, also as shown in the titanic movie, women and children were evacuvated first, more men died. Hence low survival rate for men\n\n* Pclass represents the tier of travel. Class 3 people paid the most,followed by class 2 and 1 respectively in that order. It makes sense for price to be negatively correalted with class numbers.","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model creation","metadata":{}},{"cell_type":"code","source":"import h2o\nfrom h2o.automl import H2OAutoML\n\nh2o.init()","metadata":{"execution":{"iopub.status.busy":"2022-07-19T12:07:35.542890Z","iopub.execute_input":"2022-07-19T12:07:35.543264Z","iopub.status.idle":"2022-07-19T12:07:41.272149Z","shell.execute_reply.started":"2022-07-19T12:07:35.543200Z","shell.execute_reply":"2022-07-19T12:07:41.270732Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"h2oframe_train = h2o.H2OFrame(train_data)\n\nh2oframe_train","metadata":{"execution":{"iopub.status.busy":"2022-07-19T12:07:41.279111Z","iopub.execute_input":"2022-07-19T12:07:41.282458Z","iopub.status.idle":"2022-07-19T12:07:42.063590Z","shell.execute_reply.started":"2022-07-19T12:07:41.282344Z","shell.execute_reply":"2022-07-19T12:07:42.062741Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"aml = H2OAutoML(max_models=37,nfolds=5,seed=37)\n\n\naml.train(y='Survived',training_frame=h2oframe_train)","metadata":{"execution":{"iopub.status.busy":"2022-07-19T12:24:31.438140Z","iopub.execute_input":"2022-07-19T12:24:31.438478Z","iopub.status.idle":"2022-07-19T12:27:44.971975Z","shell.execute_reply.started":"2022-07-19T12:24:31.438439Z","shell.execute_reply":"2022-07-19T12:27:44.970470Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"aml.leader","metadata":{"execution":{"iopub.status.busy":"2022-07-19T12:11:14.523707Z","iopub.execute_input":"2022-07-19T12:11:14.524318Z","iopub.status.idle":"2022-07-19T12:11:14.678260Z","shell.execute_reply.started":"2022-07-19T12:11:14.524260Z","shell.execute_reply":"2022-07-19T12:11:14.677395Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"h2oframe_test = h2o.H2OFrame(test_data)","metadata":{"execution":{"iopub.status.busy":"2022-07-19T12:11:14.679599Z","iopub.execute_input":"2022-07-19T12:11:14.679923Z","iopub.status.idle":"2022-07-19T12:11:14.948141Z","shell.execute_reply.started":"2022-07-19T12:11:14.679856Z","shell.execute_reply":"2022-07-19T12:11:14.946901Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predict = aml.predict(h2oframe_test)","metadata":{"execution":{"iopub.status.busy":"2022-07-19T12:11:14.949965Z","iopub.execute_input":"2022-07-19T12:11:14.950604Z","iopub.status.idle":"2022-07-19T12:11:15.202297Z","shell.execute_reply.started":"2022-07-19T12:11:14.950542Z","shell.execute_reply":"2022-07-19T12:11:15.201221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predict.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-19T12:11:15.204737Z","iopub.execute_input":"2022-07-19T12:11:15.205039Z","iopub.status.idle":"2022-07-19T12:11:15.252261Z","shell.execute_reply.started":"2022-07-19T12:11:15.204981Z","shell.execute_reply":"2022-07-19T12:11:15.251230Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_predict = predict > 0.5","metadata":{"execution":{"iopub.status.busy":"2022-07-19T12:20:12.000502Z","iopub.execute_input":"2022-07-19T12:20:12.000846Z","iopub.status.idle":"2022-07-19T12:20:12.005276Z","shell.execute_reply.started":"2022-07-19T12:20:12.000792Z","shell.execute_reply":"2022-07-19T12:20:12.004308Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_predict = final_predict.as_data_frame()\nfinal_predict.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-19T12:20:12.733416Z","iopub.execute_input":"2022-07-19T12:20:12.733927Z","iopub.status.idle":"2022-07-19T12:20:12.806972Z","shell.execute_reply.started":"2022-07-19T12:20:12.733856Z","shell.execute_reply":"2022-07-19T12:20:12.806155Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_predict.shape","metadata":{"execution":{"iopub.status.busy":"2022-07-19T12:20:13.816476Z","iopub.execute_input":"2022-07-19T12:20:13.816812Z","iopub.status.idle":"2022-07-19T12:20:13.822274Z","shell.execute_reply.started":"2022-07-19T12:20:13.816759Z","shell.execute_reply":"2022-07-19T12:20:13.821376Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\n# y = train_data[\"Survived\"]\n\n# features = [\"Pclass\", \"Sex\", \"SibSp\", \"Parch\"]\n# X = pd.get_dummies(train_data[features])\n# X_test = pd.get_dummies(test_data[features])\n\n# model = RandomForestClassifier(n_estimators=100, max_depth=5, random_state=12)\n# model.fit(X, y)\n# predictions = model.predict(X_test)","metadata":{"execution":{"iopub.status.busy":"2022-07-19T12:20:14.625415Z","iopub.execute_input":"2022-07-19T12:20:14.625766Z","iopub.status.idle":"2022-07-19T12:20:14.630072Z","shell.execute_reply.started":"2022-07-19T12:20:14.625704Z","shell.execute_reply":"2022-07-19T12:20:14.629180Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# min_index = ar.index(min(ar))\n# print(min_index)\n","metadata":{"execution":{"iopub.status.busy":"2022-07-19T12:20:15.378456Z","iopub.execute_input":"2022-07-19T12:20:15.378821Z","iopub.status.idle":"2022-07-19T12:20:15.382733Z","shell.execute_reply.started":"2022-07-19T12:20:15.378758Z","shell.execute_reply":"2022-07-19T12:20:15.381693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"passengerId = pd.read_csv('../input/titanic/test.csv')['PassengerId']\npassengerId","metadata":{"execution":{"iopub.status.busy":"2022-07-19T12:20:16.083514Z","iopub.execute_input":"2022-07-19T12:20:16.083828Z","iopub.status.idle":"2022-07-19T12:20:16.098456Z","shell.execute_reply.started":"2022-07-19T12:20:16.083768Z","shell.execute_reply":"2022-07-19T12:20:16.097309Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.concat([passengerId,final_predict],axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-07-19T12:20:17.881538Z","iopub.execute_input":"2022-07-19T12:20:17.881852Z","iopub.status.idle":"2022-07-19T12:20:17.896788Z","shell.execute_reply.started":"2022-07-19T12:20:17.881802Z","shell.execute_reply":"2022-07-19T12:20:17.896099Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"output = pd.concat([passengerId,final_predict],axis=1)\n\noutput.rename(columns={'predict':'Survived'})\n","metadata":{"execution":{"iopub.status.busy":"2022-07-19T12:20:18.261517Z","iopub.execute_input":"2022-07-19T12:20:18.262008Z","iopub.status.idle":"2022-07-19T12:20:18.276039Z","shell.execute_reply.started":"2022-07-19T12:20:18.261961Z","shell.execute_reply":"2022-07-19T12:20:18.275166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"output.rename(columns={'predict':'Survived'},inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-19T12:20:18.731759Z","iopub.execute_input":"2022-07-19T12:20:18.732303Z","iopub.status.idle":"2022-07-19T12:20:18.737206Z","shell.execute_reply.started":"2022-07-19T12:20:18.732237Z","shell.execute_reply":"2022-07-19T12:20:18.736486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"output.to_csv('my_submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-07-19T12:20:20.147021Z","iopub.execute_input":"2022-07-19T12:20:20.147572Z","iopub.status.idle":"2022-07-19T12:20:20.154551Z","shell.execute_reply.started":"2022-07-19T12:20:20.147503Z","shell.execute_reply":"2022-07-19T12:20:20.153480Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.read_csv('./my_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2022-07-19T12:18:09.181834Z","iopub.execute_input":"2022-07-19T12:18:09.182248Z","iopub.status.idle":"2022-07-19T12:18:09.197140Z","shell.execute_reply.started":"2022-07-19T12:18:09.182185Z","shell.execute_reply":"2022-07-19T12:18:09.196035Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# output = pd.DataFrame({'PassengerId': test_data.PassengerId, 'Survived': predictions})\n# output.to_csv('my_submission.csv', index=False)\n# print(\"Your submission was successfully saved!\")","metadata":{"execution":{"iopub.status.busy":"2022-07-19T12:08:11.012266Z","iopub.status.idle":"2022-07-19T12:08:11.013126Z"},"trusted":true},"execution_count":null,"outputs":[]}]}