{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nimport matplotlib.pyplot as plt\nplt.style.use(\"seaborn-dark\")\n\nimport seaborn as sns\n\nfrom collections import Counter\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-22T09:44:01.189523Z","iopub.execute_input":"2022-07-22T09:44:01.190290Z","iopub.status.idle":"2022-07-22T09:44:01.204401Z","shell.execute_reply.started":"2022-07-22T09:44:01.190250Z","shell.execute_reply":"2022-07-22T09:44:01.203069Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Introduction\n1912, was one of the most tragical year for the humanity. A ship named Titanic, sank during its voyage because of a iceberg collide, while it has left 1502 dead out of 2224 passangers and crew member. \n\n<font color = 'blue'>\nContent\n               \n1. [Load and Check Data](#1)\n2. [Variable Description](#2) \n    * [Univariate Variable Analysis](#3)\n        * [Categorical Variable Analysis](#4)\n        * [Numerical Variable Analysis](#5)\n3. [Basic Data Analysis](#6)\n4. [Outlier Detection](#7)\n5. [Missing Value](#8)\n    * [Find Missing Value](#9)\n    * [Fill Missing Value](#10)\n\n    \n    ","metadata":{}},{"cell_type":"markdown","source":"<a id = \"1\"></a><br>\n# Load and Check Data\n","metadata":{}},{"cell_type":"code","source":"train_df = pd.read_csv(\"/kaggle/input/titanic/train.csv\")\ntest_df = pd.read_csv(\"/kaggle/input/titanic/test.csv\")\ntest_PassengerID = test_df[\"PassengerId\"]\n","metadata":{"execution":{"iopub.status.busy":"2022-07-22T09:44:01.295066Z","iopub.execute_input":"2022-07-22T09:44:01.295713Z","iopub.status.idle":"2022-07-22T09:44:01.313977Z","shell.execute_reply.started":"2022-07-22T09:44:01.295677Z","shell.execute_reply":"2022-07-22T09:44:01.313070Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.columns","metadata":{"execution":{"iopub.status.busy":"2022-07-22T09:44:01.400244Z","iopub.execute_input":"2022-07-22T09:44:01.400632Z","iopub.status.idle":"2022-07-22T09:44:01.408021Z","shell.execute_reply.started":"2022-07-22T09:44:01.400602Z","shell.execute_reply":"2022-07-22T09:44:01.406771Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-22T09:44:01.471080Z","iopub.execute_input":"2022-07-22T09:44:01.471666Z","iopub.status.idle":"2022-07-22T09:44:01.490640Z","shell.execute_reply.started":"2022-07-22T09:44:01.471635Z","shell.execute_reply":"2022-07-22T09:44:01.489503Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.describe()","metadata":{"execution":{"iopub.status.busy":"2022-07-22T09:44:01.546841Z","iopub.execute_input":"2022-07-22T09:44:01.547805Z","iopub.status.idle":"2022-07-22T09:44:01.583985Z","shell.execute_reply.started":"2022-07-22T09:44:01.547766Z","shell.execute_reply":"2022-07-22T09:44:01.582769Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id = \"2\"></a><br>\n# Variable Description\n1. PassangerID: unique id number to each passenger\n2. Survived: passenger survived(1) or died(0)\n3. Pclass: passanger class\n4. Name: passanger name\n5. Sex: passanger sex\n6. Age: passanger age\n7. SibSp: siblings,spouses\n8. Parch: parent,children\n9. Ticket: passanger ticket number\n10. Fare: amount of money spent on ticket\n11. Cabin: cabin category\n12. Embarked: port where passanger embarked(C = Cherbourg, Q = Queenstown, S = Southampton)\n","metadata":{}},{"cell_type":"code","source":"train_df.info()","metadata":{"execution":{"iopub.status.busy":"2022-07-22T09:44:01.594752Z","iopub.execute_input":"2022-07-22T09:44:01.595158Z","iopub.status.idle":"2022-07-22T09:44:01.615896Z","shell.execute_reply.started":"2022-07-22T09:44:01.595124Z","shell.execute_reply":"2022-07-22T09:44:01.614486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* float64(2) : Fare and Age\n* int64(5) : PassengerId, Survived, Pclass, SibSp, Parch\n* object(5) : Name, Sex, Ticket, Cabin, Embarked","metadata":{}},{"cell_type":"markdown","source":"<a id = \"3\"></a><br>\n# Univariate Variable Analysis\n* Categorical Variable Analysis: Survived,Sex, Pclass, Embarked, Cabin, Name, Ticket, Sibsp and Parch\n* Numerical Variable Analysis: Fare,Age and PassangerID\n    ","metadata":{}},{"cell_type":"markdown","source":"<a id = \"4\"></a><br>\n## Categorical Variable Analysis","metadata":{}},{"cell_type":"code","source":"def bar_plot(variable):\n    \"\"\"\n        input: variable ex: \"Sex\"\n        output: bar plot & value count\n    \"\"\"\n    var = train_df[variable]\n    varValue = var.value_counts()\n    plt.figure(figsize = (9,3))\n    plt.bar(varValue.index, varValue)\n    plt.xticks(varValue.index, varValue.index.values)\n    plt.ylabel(\"Frequency\")\n    plt.xlabel(\"variable\")\n    plt.show()\n    print(\"{}: \\n {}\".format(variable,varValue))","metadata":{"execution":{"iopub.status.busy":"2022-07-22T09:44:01.673806Z","iopub.execute_input":"2022-07-22T09:44:01.675195Z","iopub.status.idle":"2022-07-22T09:44:01.683079Z","shell.execute_reply.started":"2022-07-22T09:44:01.675145Z","shell.execute_reply":"2022-07-22T09:44:01.682205Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"category1 = [\"Survived\",\"Sex\",\"Pclass\",\"Embarked\",\"SibSp\",\"Parch\"]\nfor c in category1:\n    bar_plot(c)\n    ","metadata":{"execution":{"iopub.status.busy":"2022-07-22T09:44:01.757870Z","iopub.execute_input":"2022-07-22T09:44:01.758937Z","iopub.status.idle":"2022-07-22T09:44:03.042535Z","shell.execute_reply.started":"2022-07-22T09:44:01.758900Z","shell.execute_reply":"2022-07-22T09:44:03.041360Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"category2 = [\"Cabin\", \"Name\", \"Ticket\"]\nfor c in category2:\n    print(\"{} \\n \".format(train_df[c].value_counts()))\n    ","metadata":{"execution":{"iopub.status.busy":"2022-07-22T09:44:03.044630Z","iopub.execute_input":"2022-07-22T09:44:03.045036Z","iopub.status.idle":"2022-07-22T09:44:03.058444Z","shell.execute_reply.started":"2022-07-22T09:44:03.044984Z","shell.execute_reply":"2022-07-22T09:44:03.057032Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id = \"5\"></a><br>\n## Numerical Variable Analysis","metadata":{}},{"cell_type":"code","source":"def plot_hist(variable):\n    plt.figure(figsize = (9,3))\n    plt.hist(train_df[variable], bins = 50)\n    plt.xlabel(variable)\n    plt.ylabel(\"Frequency\")\n    plt.title(\"{} distrubiton with hist\".format(variable))\n    plt.show()\n    ","metadata":{"execution":{"iopub.status.busy":"2022-07-22T09:44:03.059852Z","iopub.execute_input":"2022-07-22T09:44:03.060195Z","iopub.status.idle":"2022-07-22T09:44:03.070638Z","shell.execute_reply.started":"2022-07-22T09:44:03.060165Z","shell.execute_reply":"2022-07-22T09:44:03.069518Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"numericVar = [\"Fare\", \"Age\", \"PassengerId\"]\nfor n in numericVar:\n    plot_hist(n)","metadata":{"execution":{"iopub.status.busy":"2022-07-22T09:44:03.073058Z","iopub.execute_input":"2022-07-22T09:44:03.073382Z","iopub.status.idle":"2022-07-22T09:44:03.952929Z","shell.execute_reply.started":"2022-07-22T09:44:03.073352Z","shell.execute_reply":"2022-07-22T09:44:03.951711Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id = \"6\"></a><br>\n# Basic Data Analysis\n* Pclass - Survived\n* Sex - Survived\n* SibSp - Survived\n* Parch - Survived","metadata":{}},{"cell_type":"code","source":"train_df[[\"Pclass\",\"Survived\"]].groupby([\"Pclass\"], as_index = False).mean()","metadata":{"execution":{"iopub.status.busy":"2022-07-22T09:44:03.954467Z","iopub.execute_input":"2022-07-22T09:44:03.958319Z","iopub.status.idle":"2022-07-22T09:44:03.974583Z","shell.execute_reply.started":"2022-07-22T09:44:03.958275Z","shell.execute_reply":"2022-07-22T09:44:03.973044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df[[\"Pclass\",\"Survived\"]].groupby([\"Pclass\"], as_index = False).mean().sort_values(by=\"Survived\", ascending = True)\n","metadata":{"execution":{"iopub.status.busy":"2022-07-22T09:44:03.977217Z","iopub.execute_input":"2022-07-22T09:44:03.978354Z","iopub.status.idle":"2022-07-22T09:44:03.997392Z","shell.execute_reply.started":"2022-07-22T09:44:03.978288Z","shell.execute_reply":"2022-07-22T09:44:03.996292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df[[\"Sex\",\"Survived\"]].groupby([\"Sex\"], as_index = False).mean().sort_values(by=\"Survived\", ascending = False)","metadata":{"execution":{"iopub.status.busy":"2022-07-22T09:44:03.999120Z","iopub.execute_input":"2022-07-22T09:44:03.999798Z","iopub.status.idle":"2022-07-22T09:44:04.019545Z","shell.execute_reply.started":"2022-07-22T09:44:03.999756Z","shell.execute_reply":"2022-07-22T09:44:04.018265Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df[[\"SibSp\",\"Survived\"]].groupby([\"SibSp\"], as_index = False).mean().sort_values(by=\"Survived\", ascending = False)","metadata":{"execution":{"iopub.status.busy":"2022-07-22T09:44:04.021588Z","iopub.execute_input":"2022-07-22T09:44:04.022040Z","iopub.status.idle":"2022-07-22T09:44:04.042162Z","shell.execute_reply.started":"2022-07-22T09:44:04.021985Z","shell.execute_reply":"2022-07-22T09:44:04.041062Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df[[\"Parch\",\"Survived\"]].groupby([\"Parch\"], as_index = False).mean().sort_values(by=\"Survived\", ascending = False)","metadata":{"execution":{"iopub.status.busy":"2022-07-22T09:44:04.044009Z","iopub.execute_input":"2022-07-22T09:44:04.044906Z","iopub.status.idle":"2022-07-22T09:44:04.068059Z","shell.execute_reply.started":"2022-07-22T09:44:04.044841Z","shell.execute_reply":"2022-07-22T09:44:04.066984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df[[\"Age\",\"Survived\"]].groupby([\"Age\"], as_index = False).mean().sort_values(by=\"Survived\", ascending = False)","metadata":{"execution":{"iopub.status.busy":"2022-07-22T09:44:04.072455Z","iopub.execute_input":"2022-07-22T09:44:04.073190Z","iopub.status.idle":"2022-07-22T09:44:04.099271Z","shell.execute_reply.started":"2022-07-22T09:44:04.073146Z","shell.execute_reply":"2022-07-22T09:44:04.098059Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df[[\"Cabin\",\"Survived\"]].groupby([\"Cabin\"], as_index = False).mean().sort_values(by=\"Survived\", ascending = False)","metadata":{"execution":{"iopub.status.busy":"2022-07-22T09:44:04.101184Z","iopub.execute_input":"2022-07-22T09:44:04.101879Z","iopub.status.idle":"2022-07-22T09:44:04.127640Z","shell.execute_reply.started":"2022-07-22T09:44:04.101834Z","shell.execute_reply":"2022-07-22T09:44:04.126569Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id = \"7\"></a><br>\n# Outlier Detection","metadata":{}},{"cell_type":"code","source":"def detect_outliers(df, features):\n    outlier_indices = []\n    \n    for c in features:\n        # 1st quartile\n        Q1 = np.percentile(df[c], 25)\n        # 3rd quartile\n        Q3 = np.percentile(df[c], 75)\n        # IQR\n        IQR = Q3 - Q1\n        # Outlier Step\n        outlier_step = IQR * 1.5\n        # detect outlier and their indeces\n        outlier_list_col = df[(df[c] < Q1 - outlier_step) | (df[c] > Q3 + outlier_step)].index\n        # store indeces\n        outlier_indices.extend(outlier_list_col)\n        \n    outlier_indices = Counter(outlier_indices)\n    multiple_outliers = list(i for i, v in outlier_indices.items() if v>2)\n    \n    return multiple_outliers\n            ","metadata":{"execution":{"iopub.status.busy":"2022-07-22T09:44:04.129377Z","iopub.execute_input":"2022-07-22T09:44:04.130234Z","iopub.status.idle":"2022-07-22T09:44:04.142049Z","shell.execute_reply.started":"2022-07-22T09:44:04.130184Z","shell.execute_reply":"2022-07-22T09:44:04.140502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.loc[detect_outliers(train_df,[\"Age\",\"SibSp\",\"Parch\",\"Fare\"])]","metadata":{"execution":{"iopub.status.busy":"2022-07-22T09:44:04.145806Z","iopub.execute_input":"2022-07-22T09:44:04.148264Z","iopub.status.idle":"2022-07-22T09:44:04.180854Z","shell.execute_reply.started":"2022-07-22T09:44:04.148158Z","shell.execute_reply":"2022-07-22T09:44:04.179714Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# drop outliers\ntrain_df = train_df.drop(detect_outliers(train_df,[\"Age\",\"SibSp\",\"Parch\",\"Fare\"]), axis=0).reset_index(drop = True)","metadata":{"execution":{"iopub.status.busy":"2022-07-22T09:44:04.182373Z","iopub.execute_input":"2022-07-22T09:44:04.183743Z","iopub.status.idle":"2022-07-22T09:44:04.198986Z","shell.execute_reply.started":"2022-07-22T09:44:04.183691Z","shell.execute_reply":"2022-07-22T09:44:04.197652Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id = \"8\"></a><br>\n# Missing Value\n* Find Missing Value\n* Fill Missing Value\n","metadata":{}},{"cell_type":"code","source":"train_df_len = len(train_df) \ntrain_df = pd.concat([train_df,test_df], axis = 0).reset_index(drop = True)","metadata":{"execution":{"iopub.status.busy":"2022-07-22T09:44:04.200657Z","iopub.execute_input":"2022-07-22T09:44:04.201889Z","iopub.status.idle":"2022-07-22T09:44:04.215351Z","shell.execute_reply.started":"2022-07-22T09:44:04.201840Z","shell.execute_reply":"2022-07-22T09:44:04.214117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-22T09:44:04.217020Z","iopub.execute_input":"2022-07-22T09:44:04.218286Z","iopub.status.idle":"2022-07-22T09:44:04.243828Z","shell.execute_reply.started":"2022-07-22T09:44:04.218112Z","shell.execute_reply":"2022-07-22T09:44:04.242902Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id = \"9\"></a><br>\n## Find Missing Value\n","metadata":{}},{"cell_type":"code","source":"train_df.columns[train_df.isnull().any()]","metadata":{"execution":{"iopub.status.busy":"2022-07-22T09:44:04.245264Z","iopub.execute_input":"2022-07-22T09:44:04.246221Z","iopub.status.idle":"2022-07-22T09:44:04.257342Z","shell.execute_reply.started":"2022-07-22T09:44:04.246185Z","shell.execute_reply":"2022-07-22T09:44:04.256191Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-07-22T09:44:04.258850Z","iopub.execute_input":"2022-07-22T09:44:04.259875Z","iopub.status.idle":"2022-07-22T09:44:04.270525Z","shell.execute_reply.started":"2022-07-22T09:44:04.259832Z","shell.execute_reply":"2022-07-22T09:44:04.269345Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id = \"10\"></a><br>\n## Fill Missing Value\n* Emarked has only 2 missing value\n* Fare has only 1 missing value\n","metadata":{}},{"cell_type":"code","source":"train_df[train_df[\"Embarked\"].isnull()]","metadata":{"execution":{"iopub.status.busy":"2022-07-22T09:50:26.273528Z","iopub.execute_input":"2022-07-22T09:50:26.273893Z","iopub.status.idle":"2022-07-22T09:50:26.291611Z","shell.execute_reply.started":"2022-07-22T09:50:26.273865Z","shell.execute_reply":"2022-07-22T09:50:26.290469Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.boxplot(column = \"Fare\",by = \"Embarked\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-22T09:52:05.519909Z","iopub.execute_input":"2022-07-22T09:52:05.520315Z","iopub.status.idle":"2022-07-22T09:52:05.710576Z","shell.execute_reply.started":"2022-07-22T09:52:05.520274Z","shell.execute_reply":"2022-07-22T09:52:05.709094Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df[\"Embarked\"] = train_df[\"Embarked\"].fillna(\"C\")\ntrain_df[train_df[\"Embarked\"].isnull()]\n\n","metadata":{"execution":{"iopub.status.busy":"2022-07-22T10:18:41.704714Z","iopub.execute_input":"2022-07-22T10:18:41.705162Z","iopub.status.idle":"2022-07-22T10:18:41.723381Z","shell.execute_reply.started":"2022-07-22T10:18:41.705122Z","shell.execute_reply":"2022-07-22T10:18:41.721747Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df[train_df[\"Fare\"].isnull()]","metadata":{"execution":{"iopub.status.busy":"2022-07-22T10:19:03.666945Z","iopub.execute_input":"2022-07-22T10:19:03.667715Z","iopub.status.idle":"2022-07-22T10:19:03.685896Z","shell.execute_reply.started":"2022-07-22T10:19:03.667672Z","shell.execute_reply":"2022-07-22T10:19:03.684702Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df[\"Fare\"] = train_df[\"Fare\"].fillna(np.mean(train_df[train_df[\"Pclass\"] == 3][\"Fare\"]))","metadata":{"execution":{"iopub.status.busy":"2022-07-22T10:26:17.717740Z","iopub.execute_input":"2022-07-22T10:26:17.718176Z","iopub.status.idle":"2022-07-22T10:26:17.727942Z","shell.execute_reply.started":"2022-07-22T10:26:17.718138Z","shell.execute_reply":"2022-07-22T10:26:17.726522Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df[train_df[\"Fare\"].isnull()]","metadata":{"execution":{"iopub.status.busy":"2022-07-22T10:26:55.212626Z","iopub.execute_input":"2022-07-22T10:26:55.213039Z","iopub.status.idle":"2022-07-22T10:26:55.226243Z","shell.execute_reply.started":"2022-07-22T10:26:55.213004Z","shell.execute_reply":"2022-07-22T10:26:55.224961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}