{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n for filename in filenames:\n print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-08-11T03:39:56.112036Z","iopub.execute_input":"2022-08-11T03:39:56.112657Z","iopub.status.idle":"2022-08-11T03:39:56.123066Z","shell.execute_reply.started":"2022-08-11T03:39:56.112614Z","shell.execute_reply":"2022-08-11T03:39:56.121490Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport seaborn as sns\nimport matplotlib.pyplot as plt","metadata":{"execution":{"iopub.status.busy":"2022-08-11T03:39:56.125169Z","iopub.execute_input":"2022-08-11T03:39:56.126216Z","iopub.status.idle":"2022-08-11T03:39:56.894684Z","shell.execute_reply.started":"2022-08-11T03:39:56.126172Z","shell.execute_reply":"2022-08-11T03:39:56.893208Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv('../input/spaceship-titanic/train.csv')\ntarget = train_df.Transported\ntrain_df_all = train_df\ntrain_df = train_df.drop('Transported', axis=1)\ntest_df = pd.read_csv(\"../input/spaceship-titanic/test.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-08-11T03:39:56.896602Z","iopub.execute_input":"2022-08-11T03:39:56.897115Z","iopub.status.idle":"2022-08-11T03:39:56.994460Z","shell.execute_reply.started":"2022-08-11T03:39:56.897055Z","shell.execute_reply":"2022-08-11T03:39:56.992994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### get train_df info","metadata":{}},{"cell_type":"code","source":"train_df.info()","metadata":{"execution":{"iopub.status.busy":"2022-08-11T03:39:56.997249Z","iopub.execute_input":"2022-08-11T03:39:56.997729Z","iopub.status.idle":"2022-08-11T03:39:57.026053Z","shell.execute_reply.started":"2022-08-11T03:39:56.997689Z","shell.execute_reply":"2022-08-11T03:39:57.024608Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### If we drop all of null data, we drop almost 24% of the data","metadata":{}},{"cell_type":"code","source":"def get_null_row_ratio(df):\n all_row_length = len(df)\n null_row_length = len(df.loc[df.isnull().sum(axis=1).astype(bool)])\n return f\"{round(null_row_length/all_row_length*100, 2)}%\"","metadata":{"execution":{"iopub.status.busy":"2022-08-11T03:39:57.027915Z","iopub.execute_input":"2022-08-11T03:39:57.029283Z","iopub.status.idle":"2022-08-11T03:39:57.037210Z","shell.execute_reply.started":"2022-08-11T03:39:57.029218Z","shell.execute_reply":"2022-08-11T03:39:57.035758Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"train : \" + get_null_row_ratio(train_df))\nprint(\"test : \" + get_null_row_ratio(test_df))","metadata":{"execution":{"iopub.status.busy":"2022-08-11T03:39:57.038945Z","iopub.execute_input":"2022-08-11T03:39:57.039466Z","iopub.status.idle":"2022-08-11T03:39:57.062971Z","shell.execute_reply.started":"2022-08-11T03:39:57.039420Z","shell.execute_reply":"2022-08-11T03:39:57.061904Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### get null count in each column","metadata":{}},{"cell_type":"code","source":"def plot_null_value_count(df1, df2):\n null_values = pd.concat([df1.isna().sum().rename(\"train\"), df2.isna().sum().rename(\"test\")], axis=0).rename(\"null count\").reset_index().rename(columns={\"index\":\"column\"})\n null_values['data'] = ['train']*len(df1.columns) + ['test']*len(df2.columns)\n f, ax = plt.subplots(figsize=(5, 5))\n sns.barplot(data = null_values, y=\"column\", x=\"null count\", hue=\"data\", orient=\"h\")","metadata":{"execution":{"iopub.status.busy":"2022-08-11T03:39:57.064589Z","iopub.execute_input":"2022-08-11T03:39:57.065317Z","iopub.status.idle":"2022-08-11T03:39:57.074378Z","shell.execute_reply.started":"2022-08-11T03:39:57.065263Z","shell.execute_reply":"2022-08-11T03:39:57.072849Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_null_value_count(train_df, test_df)","metadata":{"execution":{"iopub.status.busy":"2022-08-11T03:39:57.076271Z","iopub.execute_input":"2022-08-11T03:39:57.076840Z","iopub.status.idle":"2022-08-11T03:39:57.553210Z","shell.execute_reply.started":"2022-08-11T03:39:57.076788Z","shell.execute_reply":"2022-08-11T03:39:57.551933Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### observation\n\n - it's hard to compare null count in train and test
\n
","metadata":{}},{"cell_type":"code","source":"def plot_null_value_ratio(df1, df2):\n null_values = pd.concat([df1.isna().sum().rename(\"train\")/len(df1), df2.isna().sum().rename(\"test\")/len(df2)], axis=0).rename(\"null count\").reset_index().rename(columns={\"index\":\"column\"})\n null_values['data'] = ['train']*len(df1.columns) + ['test']*len(df2.columns)\n f, ax = plt.subplots(figsize=(5, 5))\n sns.barplot(data = null_values, y=\"column\", x=\"null count\", hue=\"data\", orient=\"h\")","metadata":{"execution":{"iopub.status.busy":"2022-08-11T03:39:57.556889Z","iopub.execute_input":"2022-08-11T03:39:57.557627Z","iopub.status.idle":"2022-08-11T03:39:57.566901Z","shell.execute_reply.started":"2022-08-11T03:39:57.557572Z","shell.execute_reply":"2022-08-11T03:39:57.565563Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### get null ratio","metadata":{}},{"cell_type":"code","source":"plot_null_value_ratio(train_df, test_df)","metadata":{"execution":{"iopub.status.busy":"2022-08-11T03:39:57.571234Z","iopub.execute_input":"2022-08-11T03:39:57.572437Z","iopub.status.idle":"2022-08-11T03:39:58.025866Z","shell.execute_reply.started":"2022-08-11T03:39:57.572380Z","shell.execute_reply":"2022-08-11T03:39:58.024604Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"data description says the columns below is paid service","metadata":{}},{"cell_type":"markdown","source":"### observation\n\n - train and test have similar null ratio in each column
\n
","metadata":{}},{"cell_type":"markdown","source":"## Process null data","metadata":{}},{"cell_type":"code","source":"paid_service_columns = ['RoomService', 'FoodCourt', 'ShoppingMall', 'Spa', 'VRDeck']","metadata":{"execution":{"iopub.status.busy":"2022-08-11T03:39:58.027410Z","iopub.execute_input":"2022-08-11T03:39:58.027814Z","iopub.status.idle":"2022-08-11T03:39:58.037507Z","shell.execute_reply.started":"2022-08-11T03:39:58.027779Z","shell.execute_reply":"2022-08-11T03:39:58.036110Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def process_null_data(df):\n temp_df = df.copy()\n # maybe row with NaN in paid_service_columns means person who doesn't use paid service\n for column in paid_service_columns:\n temp_df[column] = temp_df[column].fillna(0)\n temp_df['Age'] = temp_df.Age.fillna(-1)\n temp_df['Cabin'] = temp_df.Cabin.fillna('N/-1/N')\n temp_df['HomePlanet'] = temp_df.HomePlanet.fillna('Unknown')\n temp_df['CryoSleep'] = temp_df.CryoSleep.fillna('Unknown')\n temp_df['Destination'] = temp_df.Destination.fillna('Unknown')\n temp_df['VIP'] = temp_df.VIP.fillna('Unknown')\n temp_df['Name'] = temp_df.Name.fillna('N N')\n \n return temp_df","metadata":{"execution":{"iopub.status.busy":"2022-08-11T03:39:58.039637Z","iopub.execute_input":"2022-08-11T03:39:58.040175Z","iopub.status.idle":"2022-08-11T03:39:58.052478Z","shell.execute_reply.started":"2022-08-11T03:39:58.040120Z","shell.execute_reply":"2022-08-11T03:39:58.051076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"according to data description, PassengerId column is composed to gggg/pp","metadata":{}},{"cell_type":"code","source":"def split_id(df):\n temp_df = df.copy()\n temp_df['group'] = temp_df.PassengerId.apply(lambda x: x[:4])\n temp_df['Id'] = temp_df.PassengerId.apply(lambda x: x[5:])\n temp_df.drop(['PassengerId'], axis=1, inplace=True)\n return temp_df","metadata":{"execution":{"iopub.status.busy":"2022-08-11T03:39:58.054436Z","iopub.execute_input":"2022-08-11T03:39:58.054938Z","iopub.status.idle":"2022-08-11T03:39:58.065191Z","shell.execute_reply.started":"2022-08-11T03:39:58.054888Z","shell.execute_reply":"2022-08-11T03:39:58.064171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def split_name(df):\n temp_df = df.copy()\n temp_df['first_name'] = temp_df.Name.apply(lambda x: str(x).split()[0])\n temp_df['last_name'] = temp_df.Name.apply(lambda x: str(x).split()[1])\n temp_df.drop(['Name'], axis=1, inplace=True)\n return temp_df","metadata":{"execution":{"iopub.status.busy":"2022-08-11T03:39:58.066532Z","iopub.execute_input":"2022-08-11T03:39:58.067004Z","iopub.status.idle":"2022-08-11T03:39:58.079587Z","shell.execute_reply.started":"2022-08-11T03:39:58.066959Z","shell.execute_reply":"2022-08-11T03:39:58.078126Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"according to data description, Cabin column is composed to deck/num/side","metadata":{}},{"cell_type":"code","source":"def split_cabin(df):\n temp_df = df.copy()\n temp_df['deck'] = temp_df.Cabin.apply(lambda x: str(x).split('/')[0])\n temp_df['num'] = temp_df.Cabin.apply(lambda x: str(x).split('/')[1])\n temp_df['side'] = temp_df.Cabin.apply(lambda x: str(x).split('/')[2])\n temp_df.drop(['Cabin'], axis=1, inplace=True)\n return temp_df","metadata":{"execution":{"iopub.status.busy":"2022-08-11T03:39:58.081174Z","iopub.execute_input":"2022-08-11T03:39:58.081757Z","iopub.status.idle":"2022-08-11T03:39:58.097604Z","shell.execute_reply.started":"2022-08-11T03:39:58.081707Z","shell.execute_reply":"2022-08-11T03:39:58.096414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"classify age into six categories.","metadata":{}},{"cell_type":"code","source":"def cat_age(age):\n cat=''\n if age <= -1:\n cat = 'Unknown'\n elif age <= 5: \n cat = 'Baby'\n elif age <= 12:\n cat = 'Child'\n elif age <= 18:\n cat = 'Teenager'\n elif age <= 25:\n cat = 'Student'\n elif age <= 35:\n cat = 'Young Adult'\n elif age <= 60:\n cat = 'Adult'\n else:\n cat = 'Elderly'\n return cat\n\ndef labeling_age(df):\n df['Age_cat'] = df.Age.apply(lambda x: cat_age(x))\n df.drop('Age', axis=1, inplace=True)\n return df","metadata":{"execution":{"iopub.status.busy":"2022-08-11T03:39:58.099634Z","iopub.execute_input":"2022-08-11T03:39:58.100076Z","iopub.status.idle":"2022-08-11T03:39:58.109532Z","shell.execute_reply.started":"2022-08-11T03:39:58.100037Z","shell.execute_reply":"2022-08-11T03:39:58.108386Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We will classify the categories according to the amount of payment for the service used.","metadata":{}},{"cell_type":"code","source":"def cat_paid_service(df):\n df['paid_service_cat'] = round(df.loc[:, paid_service_columns].sum(axis=1)/1000)\n return df","metadata":{"execution":{"iopub.status.busy":"2022-08-11T03:39:58.111389Z","iopub.execute_input":"2022-08-11T03:39:58.111934Z","iopub.status.idle":"2022-08-11T03:39:58.122635Z","shell.execute_reply.started":"2022-08-11T03:39:58.111893Z","shell.execute_reply":"2022-08-11T03:39:58.121603Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### It seems more meaningful to choose whether or not to use the service than to compare the payment for the service used as it is.","metadata":{}},{"cell_type":"code","source":"def bool_paid_service(charge):\n if charge <= 0:\n return False\n else:\n return True\n\ndef judge_use_paid_service(df):\n for column in paid_service_columns:\n df[column] = df[column].apply(lambda x: bool_paid_service(x))\n return df\n\ndef cat_use_paid_service(df):\n df['using_paid_service'] = df.paid_service_cat.apply(lambda x: x != 0 )\n return df","metadata":{"execution":{"iopub.status.busy":"2022-08-11T03:39:58.123888Z","iopub.execute_input":"2022-08-11T03:39:58.124790Z","iopub.status.idle":"2022-08-11T03:39:58.135078Z","shell.execute_reply.started":"2022-08-11T03:39:58.124749Z","shell.execute_reply":"2022-08-11T03:39:58.133942Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def pre_process_df(df):\n return judge_use_paid_service(cat_paid_service(labeling_age(split_cabin(split_name(split_id(process_null_data(df))))))).drop(['num', 'Id', 'first_name'], axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-08-11T03:39:58.136630Z","iopub.execute_input":"2022-08-11T03:39:58.137599Z","iopub.status.idle":"2022-08-11T03:39:58.149127Z","shell.execute_reply.started":"2022-08-11T03:39:58.137557Z","shell.execute_reply":"2022-08-11T03:39:58.147945Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### We will compare the data distributions of train_df and test_df.","metadata":{}},{"cell_type":"code","source":"def val_count_df(df, column_name, sort_by_column_name=False):\n value_count = df[column_name].value_counts().reset_index().rename(columns={column_name:\"count\",\"index\":column_name}).set_index(column_name)\n value_count = value_count.reset_index()\n if sort_by_column_name:\n value_count = value_count.sort_values(column_name)\n return value_count\n\ndef plot_and_display_compare_valuecounts(df1, df2, column_name, sort_by_column_name):\n val_count_1 = val_count_df(df1, column_name, sort_by_column_name)\n val_count_2 = val_count_df(df2, column_name, sort_by_column_name)\n val_count = pd.merge(val_count_1, val_count_2, on=column_name, how=\"outer\")\n val_count = val_count.fillna(0) \n val_count.set_index(column_name).plot.pie(figsize=(12,7), legend=False, ylabel=\"\", subplots=True, title=[\"Train: \"+column_name ,\"Test: \"+column_name]);\n ","metadata":{"execution":{"iopub.status.busy":"2022-08-11T03:46:11.458021Z","iopub.execute_input":"2022-08-11T03:46:11.458717Z","iopub.status.idle":"2022-08-11T03:46:11.469741Z","shell.execute_reply.started":"2022-08-11T03:46:11.458664Z","shell.execute_reply":"2022-08-11T03:46:11.468571Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_test = pre_process_df(train_df)\ntest_test = pre_process_df(test_df)\ncat_features = ['HomePlanet','Destination', 'deck', 'side', 'Age_cat', 'paid_service_cat']","metadata":{"execution":{"iopub.status.busy":"2022-08-11T03:46:12.902815Z","iopub.execute_input":"2022-08-11T03:46:12.904315Z","iopub.status.idle":"2022-08-11T03:46:13.071064Z","shell.execute_reply.started":"2022-08-11T03:46:12.904233Z","shell.execute_reply":"2022-08-11T03:46:13.069656Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for column in cat_features:\n plot_and_display_compare_valuecounts(train_test, test_test, column, True)","metadata":{"execution":{"iopub.status.busy":"2022-08-11T03:39:58.342318Z","iopub.execute_input":"2022-08-11T03:39:58.342763Z","iopub.status.idle":"2022-08-11T03:40:00.608619Z","shell.execute_reply.started":"2022-08-11T03:39:58.342723Z","shell.execute_reply":"2022-08-11T03:40:00.607334Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def bool_to_str(boolean):\n if boolean == True:\n return 'True'\n elif boolean == False:\n return \"False\"\n elif boolean == \"Unknown\":\n return \"Unknown\"\n\ndef plot_bool_column_count(df1, df2):\n for column in paid_service_columns + ['CryoSleep', 'VIP']:\n df1[column] = df1[column].apply(lambda x: bool_to_str(x))\n df2[column] = df2[column].apply(lambda x: bool_to_str(x))\n plot_and_display_compare_valuecounts(df1, df2, column, False)","metadata":{"execution":{"iopub.status.busy":"2022-08-11T03:40:00.610324Z","iopub.execute_input":"2022-08-11T03:40:00.610736Z","iopub.status.idle":"2022-08-11T03:40:00.619137Z","shell.execute_reply.started":"2022-08-11T03:40:00.610699Z","shell.execute_reply":"2022-08-11T03:40:00.617951Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_bool_column_count(train_test, test_test)","metadata":{"execution":{"iopub.status.busy":"2022-08-11T03:40:00.620918Z","iopub.execute_input":"2022-08-11T03:40:00.621706Z","iopub.status.idle":"2022-08-11T03:40:02.219410Z","shell.execute_reply.started":"2022-08-11T03:40:00.621665Z","shell.execute_reply":"2022-08-11T03:40:02.217235Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### observation\n\n - train and test have similar data distribution
\n
","metadata":{}},{"cell_type":"markdown","source":"### which column is important to non-transported","metadata":{}},{"cell_type":"code","source":"train_combined = train_test.join(target)","metadata":{"execution":{"iopub.status.busy":"2022-08-11T03:40:02.223224Z","iopub.execute_input":"2022-08-11T03:40:02.225031Z","iopub.status.idle":"2022-08-11T03:40:02.238704Z","shell.execute_reply.started":"2022-08-11T03:40:02.224935Z","shell.execute_reply":"2022-08-11T03:40:02.236874Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.subplots(figsize=(20, 26))\nfor i, column in enumerate(paid_service_columns + cat_features + ['CryoSleep', 'VIP']):\n plt.subplot(5, 3, i+1)\n sns.barplot(x=column, y='Transported', data=train_combined)","metadata":{"execution":{"iopub.status.busy":"2022-08-11T03:40:12.598740Z","iopub.execute_input":"2022-08-11T03:40:12.599253Z","iopub.status.idle":"2022-08-11T03:40:17.646735Z","shell.execute_reply.started":"2022-08-11T03:40:12.599212Z","shell.execute_reply":"2022-08-11T03:40:17.645668Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### observation\n\n - It can be seen that whether or not the service is used has a significant effect on the results.
\n
","metadata":{}}]}