{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Titanic - ProfileReport EDA\n\nAs you get more experienced, the question becomes how to get the most results for minimum effort.\n\nIntroducing `from pandas_profiling import ProfileReport`\n- https://pandas-profiling.github.io/pandas-profiling/docs/master/index.html\n\n**NOTE: This notebook may time some time to load and fully render - it is run twice for both test and train data**","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-05-23T18:18:26.180302Z","iopub.execute_input":"2022-05-23T18:18:26.180717Z","iopub.status.idle":"2022-05-23T18:18:26.212894Z","shell.execute_reply.started":"2022-05-23T18:18:26.180611Z","shell.execute_reply":"2022-05-23T18:18:26.211473Z"}}},{"cell_type":"code","source":"%%html\n<!-- Is there a better way than hard coding the height? -->\n<style> \niframe { height: 16350px; }  <!-- Measured Height == 16318.2px -->\n</style>","metadata":{"execution":{"iopub.status.busy":"2022-06-09T21:46:42.018005Z","iopub.execute_input":"2022-06-09T21:46:42.018649Z","iopub.status.idle":"2022-06-09T21:46:42.049304Z","shell.execute_reply.started":"2022-06-09T21:46:42.018545Z","shell.execute_reply":"2022-06-09T21:46:42.048475Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nfrom pandas_profiling import ProfileReport\n\npd.options.display.max_columns = 999\npd.options.display.max_rows    = 6","metadata":{"execution":{"iopub.status.busy":"2022-06-09T21:46:42.050578Z","iopub.execute_input":"2022-06-09T21:46:42.051349Z","iopub.status.idle":"2022-06-09T21:46:42.05595Z","shell.execute_reply.started":"2022-06-09T21:46:42.051308Z","shell.execute_reply":"2022-06-09T21:46:42.055097Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!find ../input/ -type f -name '*.csv'","metadata":{"execution":{"iopub.status.busy":"2022-06-09T21:46:42.067094Z","iopub.execute_input":"2022-06-09T21:46:42.067636Z","iopub.status.idle":"2022-06-09T21:46:42.875866Z","shell.execute_reply.started":"2022-06-09T21:46:42.0676Z","shell.execute_reply":"2022-06-09T21:46:42.874541Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Preprocessing","metadata":{}},{"cell_type":"code","source":"# Source: https://www.kaggle.com/code/jamesmcguigan/spaceship-titanic-xgboost/\ndef enhance(df):\n    \n    for col in ['HomePlanet', 'Cabin', 'Destination', 'Name']:    \n        df[col] = df[col].astype('category')        \n        \n    for col in ['CryoSleep', 'VIP']: \n        # df[col] = df[col].fillna(False).astype(bool)\n        df[col] = df[col].astype(bool)\n        \n    for col in ['Age', 'RoomService', 'FoodCourt', 'ShoppingMall', 'Spa', 'VRDeck']:\n        df[col] = df[col].fillna(0).astype(int)            \n\n        ### Avoid normalizing for AutoEDA\n        ### Normalizing ints improves XGboost score 0.50783 -> 0.69932\n        ### FillNA(mean) -> FillNA(0) reduces score 0.69932 -> 0.65583 \n        # df[col] = df[col].fillna(train_df[col].mean())    # Fill NA with mean \n        # df[col] = df[col] / train_df[col].max()           # Normalize to range [0-1]\n        \n    # Splitting FirstName + Surname reduces score 0.69932 -> 0.50713\n    df['FirstName'] = df['Name'].str.split(' ', 1).str[0].astype('category')\n    df['LastName']  = df['Name'].str.split(' ', 1).str[-1].astype('category')\n\n    # Split Cabin -> Deck/Num/Side\n    df['Cabin/Deck'] = df['Cabin'].str.split('/', 2).str[0].astype('category')\n    df['Cabin/Num']  = df['Cabin'].str.split('/', 2).str[1].astype('category')\n    df['Cabin/Side'] = df['Cabin'].str.split('/', 2).str[2].astype('category')\n    return df","metadata":{"execution":{"iopub.status.busy":"2022-06-09T21:46:42.878864Z","iopub.execute_input":"2022-06-09T21:46:42.879253Z","iopub.status.idle":"2022-06-09T21:46:42.89306Z","shell.execute_reply.started":"2022-06-09T21:46:42.879202Z","shell.execute_reply":"2022-06-09T21:46:42.892012Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train Dataset","metadata":{}},{"cell_type":"code","source":"train_df = pd.read_csv('../input/spaceship-titanic/train.csv')\ntrain_df = enhance(train_df)\ntrain_df","metadata":{"execution":{"iopub.status.busy":"2022-06-09T21:46:42.894782Z","iopub.execute_input":"2022-06-09T21:46:42.895395Z","iopub.status.idle":"2022-06-09T21:46:43.267837Z","shell.execute_reply.started":"2022-06-09T21:46:42.895352Z","shell.execute_reply":"2022-06-09T21:46:43.266986Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nauto_eda = ProfileReport(train_df, title=\"Spaceship Titanic - Machine Learning Disaster - Train\", explorative=True, minimal=False, progress_bar=False) \n\n# BUGFIX: https://github.com/pandas-profiling/pandas-profiling/issues/493\nauto_eda.to_notebook_iframe() ","metadata":{"execution":{"iopub.status.busy":"2022-06-09T21:46:43.270327Z","iopub.execute_input":"2022-06-09T21:46:43.270924Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Test Dataset","metadata":{}},{"cell_type":"code","source":"test_df = pd.read_csv('../input/spaceship-titanic/test.csv')\ntest_df = enhance(test_df)\ntest_df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nauto_eda = ProfileReport(test_df, title=\"Spaceship Titanic - Machine Learning Disaster - Test\", explorative=True, minimal=False, progress_bar=False)\n\n# BUGFIX: https://github.com/pandas-profiling/pandas-profiling/issues/493\nauto_eda.to_notebook_iframe() ","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}