{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-10-01T17:10:10.450285Z","iopub.execute_input":"2022-10-01T17:10:10.451081Z","iopub.status.idle":"2022-10-01T17:10:10.462234Z","shell.execute_reply.started":"2022-10-01T17:10:10.451021Z","shell.execute_reply":"2022-10-01T17:10:10.461004Z"},"jupyter":{"outputs_hidden":true,"source_hidden":true},"collapsed":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport matplotlib as mpl\nimport pandas as pd\nimport seaborn as sns","metadata":{"execution":{"iopub.status.busy":"2022-10-01T18:00:21.224536Z","iopub.execute_input":"2022-10-01T18:00:21.224979Z","iopub.status.idle":"2022-10-01T18:00:21.756550Z","shell.execute_reply.started":"2022-10-01T18:00:21.224947Z","shell.execute_reply":"2022-10-01T18:00:21.755305Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Starting off we are going to load in our datasets. These datasets are **very high** in memory so we will examine them and try and understand where we can change the dtypes of our features to reduce the memory requierments for the data.","metadata":{}},{"cell_type":"code","source":"train_dtypes = pd.read_csv(\"/kaggle/input/tabular-playground-series-oct-2022/train_dtypes.csv\").to_dict()\ntrain_data_0_pre_dtype = pd.read_csv(\"/kaggle/input/tabular-playground-series-oct-2022/train_0.csv\")\n\nmemory_usage_before_conversion = train_data_0_pre_dtype[\"game_num\"].memory_usage(index=False, deep=True)\nprint(f\"Train data memory usage on a single column before the dtype conversion: {memory_usage_before_conversion}\", \"\\n\")\n\ndtype_dict_train = {} #initiates a new dictionary\nfor i in range(61): #iterates through the number of columns\n    dtype_dict_train[train_dtypes[\"column\"][i]] = train_dtypes[\"dtype\"][i] # assigns each column name with its respective dtype in the dtype dictionary\n\ntrain_data_0_post_dtype = pd.read_csv(\"/kaggle/input/tabular-playground-series-oct-2022/train_0.csv\", dtype=dtype_dict_train)\nmemory_usage_after_conversion = train_data_0_post_dtype[\"game_num\"].memory_usage(index=False, deep=True)\n\nprint(f\"Train data memory usage on a single column after the dtype conversion: {memory_usage_after_conversion}\", \"\\n\")\nprint(f\"percent reduction in memory usage = {(memory_usage_after_conversion / memory_usage_before_conversion) * 100}%\")","metadata":{"execution":{"iopub.status.busy":"2022-10-01T18:02:59.310156Z","iopub.execute_input":"2022-10-01T18:02:59.310613Z","iopub.status.idle":"2022-10-01T18:03:33.532532Z","shell.execute_reply.started":"2022-10-01T18:02:59.310570Z","shell.execute_reply":"2022-10-01T18:03:33.531262Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"So just by changing the dtypes into what kaggle has kindly provided to us, we can reduce the required memory allocation by **50%**! which is awesome, but i think we can investigate and see further improvments in memory loading. In pandas, when we load data it automaticly uses dtypes of **int64** and **float64** which is quite a larg byte size. however we can get away with some smaller bite sized integer types, which still contain the numbers needed. for example, **int8** or 8 bit integers can hold **2^8** or **256** different numbers, which is **-128 -> 127** range. with **int16** holding somewhere around a **-32,000 -> 32,000** range. finally **int32** holding 2^32 bytes or a range of **-2.1 -> 2.1 billion!** so int 32 is the largest dtype we will need.","metadata":{"execution":{"iopub.status.busy":"2022-10-01T03:09:33.929222Z","iopub.execute_input":"2022-10-01T03:09:33.929655Z","iopub.status.idle":"2022-10-01T03:09:34.075186Z","shell.execute_reply.started":"2022-10-01T03:09:33.929619Z","shell.execute_reply":"2022-10-01T03:09:34.073931Z"}}},{"cell_type":"code","source":"train_data_0_post_dtype[\"team_scoring_next\"].where(train_data_0_post_dtype[\"team_scoring_next\"] == \"b\", 0, inplace=True)\ntrain_data_0_post_dtype[\"team_scoring_next\"].where(train_data_0_post_dtype[\"team_scoring_next\"] == \"a\", 1, inplace=True)\n\nmin_max_df = pd.DataFrame(train_data_0_post_dtype.columns, columns=[\"index\"])\nmin_max_df[\"min\"] = 0\nmin_max_df[\"max\"] = 0\n\nfor column in min_max_df[\"index\"]:\n    min_max_df.loc[min_max_df[\"index\"] == column, [\"min\"]] = min(train_data_0_post_dtype[column])\n    min_max_df.loc[min_max_df[\"index\"] == column, [\"max\"]] = max(train_data_0_post_dtype[column])\n\nprint(min_max_df.head)\nmin_max_df.drop(index=1, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-10-01T18:50:16.292081Z","iopub.execute_input":"2022-10-01T18:50:16.292551Z","iopub.status.idle":"2022-10-01T18:50:41.800693Z","shell.execute_reply.started":"2022-10-01T18:50:16.292507Z","shell.execute_reply":"2022-10-01T18:50:41.799460Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We can see that **event_id** definentaly needs to be float32 and its much larger so we are going to remove that from our group before plotting so the scales are nice.","metadata":{}},{"cell_type":"code","source":"font_color = '#525252'\nhfont = {'fontname':'Calibri'}\nfacecolor = '#eaeaf2'\ncolor_red = '#fd625e'\ncolor_blue = '#01b8aa'\nindex = min_max_df[\"index\"]\ncolumn0 = min_max_df['min']\ncolumn1 = min_max_df['max']\ntitle0 = 'Min'\ntitle1 = 'Max'\n\nfig, axes = plt.subplots(figsize=(16,16),facecolor=facecolor, ncols=2, sharey=True)\nfig.tight_layout()\n\naxes[0].barh(index, column0, align='center', color=color_red, zorder=10)\naxes[0].set_title(title0, fontsize=18, pad=15, color=color_red, **hfont)\naxes[0].axvline(x= -128, color='gray', linestyle='--', label=\"-128 bound\")\naxes[1].barh(index, column1, align='center', color=color_blue, zorder=10)\naxes[1].set_title(title1, fontsize=18, pad=15, color=color_blue, **hfont)\naxes[1].axvline(x= 127, color='black', linestyle='--', label=\"127 bound\")\naxes[0].legend(loc = 'upper left', title_fontsize=\"large\")\naxes[1].legend(loc = 'upper right', title_fontsize=\"large\")\n\n\naxes[0].set(yticks=min_max_df[\"index\"], yticklabels=min_max_df[\"index\"])\naxes[0].yaxis.tick_left()\naxes[0].tick_params(axis='y', colors='white') # tick color\n\naxes[1].set_xticks([0, 100, 200, 300, 400, 500, 600, 700])\naxes[1].set_xticklabels([0, 100, 200, 300, 400, 500, 600, 700])\n\nfor label in (axes[0].get_xticklabels() + axes[0].get_yticklabels()):\n    label.set(fontsize=13, color=font_color, **hfont)\nfor label in (axes[1].get_xticklabels() + axes[1].get_yticklabels()):\n    label.set(fontsize=13, color=font_color, **hfont)\n    \nplt.legend(loc = 'upper center')\n    \nplt.subplots_adjust(wspace=0, top=0.85, bottom=0.1, left=0.18, right=0.95)","metadata":{"execution":{"iopub.status.busy":"2022-10-01T19:07:41.034594Z","iopub.execute_input":"2022-10-01T19:07:41.035143Z","iopub.status.idle":"2022-10-01T19:07:43.113791Z","shell.execute_reply.started":"2022-10-01T19:07:41.035095Z","shell.execute_reply":"2022-10-01T19:07:43.112909Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"So it looks like all columns other than **event_time** and **game_num** can be converted in **float8** dtypes because they all within the 128 range, which will save a lot of space! event time, game_num, and event_id will be converted to float32 as that will be suitable for the number range.","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}