{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# DON'T FORGET ABOUT UPVOTES :)","metadata":{}},{"cell_type":"markdown","source":"# Table of contents EDA\n1. **Data reading** \n1. **Global info about data: correlations, duplicates, NaN's**\n1. **Unique features and value counts**\n1. **Distribution plots and graphical info**\n1. **Date dtype preprocessing**\n1. **Track conditions**","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"execution":{"iopub.status.busy":"2022-08-14T11:44:45.186622Z","iopub.execute_input":"2022-08-14T11:44:45.186986Z","iopub.status.idle":"2022-08-14T11:44:45.196709Z","shell.execute_reply.started":"2022-08-14T11:44:45.186955Z","shell.execute_reply":"2022-08-14T11:44:45.195631Z"},"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Import necessary libraries\n","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n","metadata":{"execution":{"iopub.status.busy":"2022-08-14T13:44:31.458351Z","iopub.execute_input":"2022-08-14T13:44:31.458716Z","iopub.status.idle":"2022-08-14T13:44:31.466259Z","shell.execute_reply.started":"2022-08-14T13:44:31.458676Z","shell.execute_reply":"2022-08-14T13:44:31.465280Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Complete-file reading\n","metadata":{}},{"cell_type":"code","source":"dataset_complete=pd.read_csv('../input/big-data-derby-2022/nyra_2019_complete.csv')","metadata":{"execution":{"iopub.status.busy":"2022-08-14T13:47:42.713232Z","iopub.execute_input":"2022-08-14T13:47:42.713820Z","iopub.status.idle":"2022-08-14T13:47:50.231681Z","shell.execute_reply.started":"2022-08-14T13:47:42.713783Z","shell.execute_reply":"2022-08-14T13:47:50.230689Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"*TAKE A LOOK*","metadata":{}},{"cell_type":"code","source":"dataset_complete","metadata":{"execution":{"iopub.status.busy":"2022-08-14T11:44:54.256237Z","iopub.execute_input":"2022-08-14T11:44:54.256618Z","iopub.status.idle":"2022-08-14T11:44:54.294689Z","shell.execute_reply.started":"2022-08-14T11:44:54.256581Z","shell.execute_reply":"2022-08-14T11:44:54.293658Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Common info about dataset","metadata":{}},{"cell_type":"code","source":"dataset_complete.info()","metadata":{"execution":{"iopub.status.busy":"2022-08-14T11:44:54.296342Z","iopub.execute_input":"2022-08-14T11:44:54.296698Z","iopub.status.idle":"2022-08-14T11:44:54.308109Z","shell.execute_reply.started":"2022-08-14T11:44:54.296662Z","shell.execute_reply":"2022-08-14T11:44:54.307070Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_complete.describe()","metadata":{"execution":{"iopub.status.busy":"2022-08-14T11:44:54.311073Z","iopub.execute_input":"2022-08-14T11:44:54.311853Z","iopub.status.idle":"2022-08-14T11:44:55.456497Z","shell.execute_reply.started":"2022-08-14T11:44:54.311817Z","shell.execute_reply":"2022-08-14T11:44:55.455450Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Correlation matrix","metadata":{}},{"cell_type":"code","source":"dataset_complete.corr()","metadata":{"execution":{"iopub.status.busy":"2022-08-14T11:44:55.458494Z","iopub.execute_input":"2022-08-14T11:44:55.459196Z","iopub.status.idle":"2022-08-14T11:44:57.001450Z","shell.execute_reply.started":"2022-08-14T11:44:55.459158Z","shell.execute_reply":"2022-08-14T11:44:57.000450Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Distributions btw features","metadata":{}},{"cell_type":"code","source":"dataset_complete.hist(figsize=(16,16))\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-14T13:47:29.091971Z","iopub.execute_input":"2022-08-14T13:47:29.092341Z","iopub.status.idle":"2022-08-14T13:47:31.958264Z","shell.execute_reply.started":"2022-08-14T13:47:29.092308Z","shell.execute_reply":"2022-08-14T13:47:31.957300Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# LIST OF FEATURES","metadata":{}},{"cell_type":"code","source":"dataset_complete.columns.to_list()","metadata":{"execution":{"iopub.status.busy":"2022-08-14T11:44:58.964307Z","iopub.execute_input":"2022-08-14T11:44:58.965055Z","iopub.status.idle":"2022-08-14T11:44:58.973012Z","shell.execute_reply.started":"2022-08-14T11:44:58.965016Z","shell.execute_reply":"2022-08-14T11:44:58.971819Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* IS THERE ANY DUPLICATES ? *","metadata":{}},{"cell_type":"code","source":"dataset_complete.shape, dataset_complete.drop_duplicates().shape","metadata":{"execution":{"iopub.status.busy":"2022-08-14T11:44:58.974685Z","iopub.execute_input":"2022-08-14T11:44:58.975157Z","iopub.status.idle":"2022-08-14T11:45:07.173410Z","shell.execute_reply.started":"2022-08-14T11:44:58.975056Z","shell.execute_reply":"2022-08-14T11:45:07.172484Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"*AS WE CAN SEE, SHAPE IS THE SAME*","metadata":{}},{"cell_type":"markdown","source":"# COUNT UNIQUE VALUES FOR EACH FEATURE","metadata":{}},{"cell_type":"code","source":"features = {}\nfor i in dataset_complete.columns.to_list(): \n    features[i]=len(dataset_complete[i].unique())\nfeatures","metadata":{"execution":{"iopub.status.busy":"2022-08-14T11:45:07.175007Z","iopub.execute_input":"2022-08-14T11:45:07.175364Z","iopub.status.idle":"2022-08-14T11:45:10.580458Z","shell.execute_reply.started":"2022-08-14T11:45:07.175329Z","shell.execute_reply":"2022-08-14T11:45:10.579563Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# PLOTS\n1. **RACE NUMBER**","metadata":{}},{"cell_type":"code","source":"sns.barplot(dataset_complete['race_number'].unique(), dataset_complete['race_number'].value_counts())\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-14T13:43:03.279560Z","iopub.execute_input":"2022-08-14T13:43:03.280619Z","iopub.status.idle":"2022-08-14T13:43:03.585117Z","shell.execute_reply.started":"2022-08-14T13:43:03.280573Z","shell.execute_reply":"2022-08-14T13:43:03.584238Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"2.  **RACE TYPE**","metadata":{}},{"cell_type":"code","source":"sns.barplot(dataset_complete['race_type'].unique(), dataset_complete['race_type'].value_counts())\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-14T13:42:49.319390Z","iopub.execute_input":"2022-08-14T13:42:49.319990Z","iopub.status.idle":"2022-08-14T13:42:50.080744Z","shell.execute_reply.started":"2022-08-14T13:42:49.319952Z","shell.execute_reply":"2022-08-14T13:42:50.079753Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"3.**program number**","metadata":{}},{"cell_type":"code","source":"sns.barplot(dataset_complete['program_number'].unique(), dataset_complete['program_number'].value_counts())\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-14T13:41:28.985226Z","iopub.execute_input":"2022-08-14T13:41:28.985626Z","iopub.status.idle":"2022-08-14T13:41:29.711638Z","shell.execute_reply.started":"2022-08-14T13:41:28.985593Z","shell.execute_reply":"2022-08-14T13:41:29.710686Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"4. **TRACK ID**","metadata":{}},{"cell_type":"code","source":"plt.title('Track id pie')\nplt.pie(dataset_complete['track_id'].value_counts(), labels=dataset_complete['track_id'].unique())\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-14T13:23:41.333138Z","iopub.execute_input":"2022-08-14T13:23:41.334115Z","iopub.status.idle":"2022-08-14T13:23:42.101790Z","shell.execute_reply.started":"2022-08-14T13:23:41.334076Z","shell.execute_reply":"2022-08-14T13:23:42.100145Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**5. Course type**","metadata":{}},{"cell_type":"code","source":"plt.title('Course type pie')\nplt.pie(dataset_complete['course_type'].value_counts(), labels=dataset_complete['course_type'].unique())\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-14T13:23:09.226370Z","iopub.execute_input":"2022-08-14T13:23:09.226810Z","iopub.status.idle":"2022-08-14T13:23:09.818366Z","shell.execute_reply.started":"2022-08-14T13:23:09.226773Z","shell.execute_reply":"2022-08-14T13:23:09.815953Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**6. track_condition**","metadata":{}},{"cell_type":"code","source":"plt.title('Track conditions pie')\nplt.pie(dataset_complete['track_condition'].value_counts(), labels=dataset_complete['track_condition'].unique())\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-14T13:22:58.150141Z","iopub.execute_input":"2022-08-14T13:22:58.150569Z","iopub.status.idle":"2022-08-14T13:22:58.829844Z","shell.execute_reply.started":"2022-08-14T13:22:58.150508Z","shell.execute_reply":"2022-08-14T13:22:58.828605Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# LET'S MAKE A DISPLOT OF SCALAR FEATURES","metadata":{}},{"cell_type":"code","source":"\nsns.displot(dataset_complete['run_up_distance'], kind='kde')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-14T13:48:31.467053Z","iopub.execute_input":"2022-08-14T13:48:31.467728Z","iopub.status.idle":"2022-08-14T13:48:52.828232Z","shell.execute_reply.started":"2022-08-14T13:48:31.467692Z","shell.execute_reply":"2022-08-14T13:48:52.827303Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Correlations between features based on date_time type","metadata":{}},{"cell_type":"code","source":"new_dataset=dataset_complete.assign(week_day=pd.to_datetime(dataset_complete['race_date']).dt.day_name())","metadata":{"execution":{"iopub.status.busy":"2022-08-14T11:55:45.296308Z","iopub.execute_input":"2022-08-14T11:55:45.296989Z","iopub.status.idle":"2022-08-14T11:55:47.896727Z","shell.execute_reply.started":"2022-08-14T11:55:45.296954Z","shell.execute_reply":"2022-08-14T11:55:47.895756Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.title('correlations btw day of week and odds')\nsns.barplot(new_dataset['week_day'], new_dataset['odds'])\nplt.xlabel('day of the week')\nplt.ylabel('odds')\nplt.tight_layout()\nplt.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-08-14T13:43:17.834654Z","iopub.execute_input":"2022-08-14T13:43:17.835014Z","iopub.status.idle":"2022-08-14T13:44:31.456482Z","shell.execute_reply.started":"2022-08-14T13:43:17.834984Z","shell.execute_reply":"2022-08-14T13:44:31.455547Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Correlations btw day of week and odds**","metadata":{}},{"cell_type":"code","source":"plt.title('correlations btw day of week and odds')\nsns.barplot(new_dataset['week_day'], new_dataset['purse'])\nplt.xlabel('day of the week')\nplt.ylabel('purse')\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-14T13:44:55.130244Z","iopub.execute_input":"2022-08-14T13:44:55.130622Z","iopub.status.idle":"2022-08-14T13:46:05.433596Z","shell.execute_reply.started":"2022-08-14T13:44:55.130590Z","shell.execute_reply":"2022-08-14T13:46:05.432459Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"*As we can see, saturday is the most benefitial day*","metadata":{}},{"cell_type":"markdown","source":"**Track condition for each track id**","metadata":{}},{"cell_type":"code","source":"feature_list=dataset_complete['track_id'].unique()\nfig, ax = plt.subplots(1, 3,sharey=True)\nfor i in range(3):\n    ax[i].set_title(feature_list[i])\n    ax[i].bar(dataset_complete[dataset_complete['track_id']==feature_list[i]]['track_condition'].unique(), dataset_complete[dataset_complete['track_id']==feature_list[i]]['track_condition'].value_counts())\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-14T13:00:04.513097Z","iopub.execute_input":"2022-08-14T13:00:04.513469Z","iopub.status.idle":"2022-08-14T13:00:08.952709Z","shell.execute_reply.started":"2022-08-14T13:00:04.513437Z","shell.execute_reply":"2022-08-14T13:00:08.951461Z"},"trusted":true},"execution_count":null,"outputs":[]}]}