{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Introduction\n\nThis notebook contains a walkthrough of the memory optimization for the train dataset of this Kaggle's competition. The main goal is to reduce as much as posible the memory consumption of the dataset without losing any information by using the correct data types for each column.  ","metadata":{}},{"cell_type":"markdown","source":"> Note that instead of pandas you can also use polars to load the data faster.","metadata":{}},{"cell_type":"markdown","source":"# Setup","metadata":{}},{"cell_type":"code","source":"# Import libraries\nimport numpy as np\nimport pandas as pd","metadata":{"execution":{"iopub.status.busy":"2023-05-01T15:39:15.296492Z","iopub.execute_input":"2023-05-01T15:39:15.296890Z","iopub.status.idle":"2023-05-01T15:39:15.326135Z","shell.execute_reply.started":"2023-05-01T15:39:15.296856Z","shell.execute_reply":"2023-05-01T15:39:15.325158Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Path to files\ntest_csv_path = \"/kaggle/input/predict-student-performance-from-game-play/test.csv\"\ntrain_csv_path = \"/kaggle/input/predict-student-performance-from-game-play/train.csv\"\ntrain_labels_csv = \"/kaggle/input/predict-student-performance-from-game-play/train_labels.csv\"\n\nsample_submission_csv = \"/kaggle/input/predict-student-performance-from-game-play/sample_submission.csv\"","metadata":{"execution":{"iopub.status.busy":"2023-05-01T15:39:15.328068Z","iopub.execute_input":"2023-05-01T15:39:15.328742Z","iopub.status.idle":"2023-05-01T15:39:15.333092Z","shell.execute_reply.started":"2023-05-01T15:39:15.328702Z","shell.execute_reply":"2023-05-01T15:39:15.332285Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> Remember to check the data dictionary to get a better understanding of the data.","metadata":{}},{"cell_type":"markdown","source":"Since the dataset is quite large, we'll start by loading only a subset. This will allow us to perform the memory optimization faster.","metadata":{}},{"cell_type":"code","source":"# Read a subset of the train dataset\ntrain_df = pd.read_csv(train_csv_path, index_col=\"index\", nrows=10_000)\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-05-01T15:39:15.334544Z","iopub.execute_input":"2023-05-01T15:39:15.335205Z","iopub.status.idle":"2023-05-01T15:39:15.453335Z","shell.execute_reply.started":"2023-05-01T15:39:15.335169Z","shell.execute_reply":"2023-05-01T15:39:15.452422Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Memory optimization","metadata":{}},{"cell_type":"code","source":"# Check the shape of the train dataset\ntrain_df.shape","metadata":{"execution":{"iopub.status.busy":"2023-05-01T15:39:15.455676Z","iopub.execute_input":"2023-05-01T15:39:15.457123Z","iopub.status.idle":"2023-05-01T15:39:15.463491Z","shell.execute_reply.started":"2023-05-01T15:39:15.457082Z","shell.execute_reply":"2023-05-01T15:39:15.462071Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check the memory usage of the train dataset subset\ntrain_df.memory_usage(deep=True)","metadata":{"execution":{"iopub.status.busy":"2023-05-01T15:39:15.465322Z","iopub.execute_input":"2023-05-01T15:39:15.466189Z","iopub.status.idle":"2023-05-01T15:39:15.493445Z","shell.execute_reply.started":"2023-05-01T15:39:15.466132Z","shell.execute_reply":"2023-05-01T15:39:15.492437Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check the memory usage of the train dataset subset in mb\ntrain_df.memory_usage(deep=True).sum() / 1024 ** 2","metadata":{"execution":{"iopub.status.busy":"2023-05-01T15:39:15.494687Z","iopub.execute_input":"2023-05-01T15:39:15.495801Z","iopub.status.idle":"2023-05-01T15:39:15.512687Z","shell.execute_reply.started":"2023-05-01T15:39:15.495761Z","shell.execute_reply":"2023-05-01T15:39:15.511847Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Columns names\ntrain_df.columns","metadata":{"execution":{"iopub.status.busy":"2023-05-01T15:39:15.513844Z","iopub.execute_input":"2023-05-01T15:39:15.514699Z","iopub.status.idle":"2023-05-01T15:39:15.526145Z","shell.execute_reply.started":"2023-05-01T15:39:15.514663Z","shell.execute_reply":"2023-05-01T15:39:15.525023Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check the data types of the train dataset subset\ntrain_df.dtypes","metadata":{"execution":{"iopub.status.busy":"2023-05-01T15:39:15.527881Z","iopub.execute_input":"2023-05-01T15:39:15.528326Z","iopub.status.idle":"2023-05-01T15:39:15.540410Z","shell.execute_reply.started":"2023-05-01T15:39:15.528216Z","shell.execute_reply":"2023-05-01T15:39:15.539521Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Number of nan values in each column\ntrain_df.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2023-05-01T15:39:15.541614Z","iopub.execute_input":"2023-05-01T15:39:15.542594Z","iopub.status.idle":"2023-05-01T15:39:15.557524Z","shell.execute_reply.started":"2023-05-01T15:39:15.542557Z","shell.execute_reply":"2023-05-01T15:39:15.556293Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Number of unique values in each column\ntrain_df.nunique()","metadata":{"execution":{"iopub.status.busy":"2023-05-01T15:39:15.564399Z","iopub.execute_input":"2023-05-01T15:39:15.566936Z","iopub.status.idle":"2023-05-01T15:39:15.593797Z","shell.execute_reply.started":"2023-05-01T15:39:15.566860Z","shell.execute_reply":"2023-05-01T15:39:15.592348Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Since some columns like `fullscreen`, `hq` and `music` are binary values (0, 1), we can use the `bool` type instead of `int64`.","metadata":{}},{"cell_type":"code","source":"train_df[\"fullscreen\"] = train_df[\"fullscreen\"].astype(\"bool\")\ntrain_df[\"hq\"] = train_df[\"hq\"].astype(\"bool\")\ntrain_df[\"music\"] = train_df[\"music\"].astype(\"bool\")","metadata":{"execution":{"iopub.status.busy":"2023-05-01T15:39:15.595592Z","iopub.execute_input":"2023-05-01T15:39:15.595912Z","iopub.status.idle":"2023-05-01T15:39:15.603986Z","shell.execute_reply.started":"2023-05-01T15:39:15.595867Z","shell.execute_reply":"2023-05-01T15:39:15.602198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check the event_name column\ntrain_df[\"event_name\"].value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-05-01T15:39:15.606510Z","iopub.execute_input":"2023-05-01T15:39:15.607311Z","iopub.status.idle":"2023-05-01T15:39:15.620398Z","shell.execute_reply.started":"2023-05-01T15:39:15.607272Z","shell.execute_reply":"2023-05-01T15:39:15.619505Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check the name column\ntrain_df[\"name\"].value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-05-01T15:39:15.622290Z","iopub.execute_input":"2023-05-01T15:39:15.623166Z","iopub.status.idle":"2023-05-01T15:39:15.632695Z","shell.execute_reply.started":"2023-05-01T15:39:15.623124Z","shell.execute_reply":"2023-05-01T15:39:15.631660Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The `event_name` and `name` columns are categorical values, so we can use the `category` type instead of `object`.","metadata":{}},{"cell_type":"code","source":"train_df[\"event_name\"] = train_df[\"event_name\"].astype(\"category\")\ntrain_df[\"name\"] = train_df[\"name\"].astype(\"category\")","metadata":{"execution":{"iopub.status.busy":"2023-05-01T15:39:15.634045Z","iopub.execute_input":"2023-05-01T15:39:15.634595Z","iopub.status.idle":"2023-05-01T15:39:15.649977Z","shell.execute_reply.started":"2023-05-01T15:39:15.634558Z","shell.execute_reply":"2023-05-01T15:39:15.648990Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check the level column range of values\ntrain_df[\"level\"].min(), train_df[\"level\"].max()","metadata":{"execution":{"iopub.status.busy":"2023-05-01T15:39:15.651561Z","iopub.execute_input":"2023-05-01T15:39:15.652224Z","iopub.status.idle":"2023-05-01T15:39:15.660249Z","shell.execute_reply.started":"2023-05-01T15:39:15.652188Z","shell.execute_reply":"2023-05-01T15:39:15.657737Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Since the range of the level column is small, we can convert it to int8\ntrain_df[\"level\"] = train_df[\"level\"].astype(\"int8\")","metadata":{"execution":{"iopub.status.busy":"2023-05-01T15:39:15.661704Z","iopub.execute_input":"2023-05-01T15:39:15.666058Z","iopub.status.idle":"2023-05-01T15:39:15.673163Z","shell.execute_reply.started":"2023-05-01T15:39:15.666012Z","shell.execute_reply":"2023-05-01T15:39:15.671979Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check the page column\ntrain_df[\"page\"].value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-05-01T15:39:15.676708Z","iopub.execute_input":"2023-05-01T15:39:15.680208Z","iopub.status.idle":"2023-05-01T15:39:15.693454Z","shell.execute_reply.started":"2023-05-01T15:39:15.680161Z","shell.execute_reply":"2023-05-01T15:39:15.692384Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"As with the `level` column, we can use a smaller integer type for it. In this case, we'll use the `Int8` due to the presence of `NaN` values.","metadata":{}},{"cell_type":"code","source":"train_df[\"page\"] = train_df[\"page\"].astype(\"Int8\")","metadata":{"execution":{"iopub.status.busy":"2023-05-01T15:39:15.696956Z","iopub.execute_input":"2023-05-01T15:39:15.699168Z","iopub.status.idle":"2023-05-01T15:39:15.705604Z","shell.execute_reply.started":"2023-05-01T15:39:15.699123Z","shell.execute_reply":"2023-05-01T15:39:15.704689Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"In the data dictionary we can see that the `level_group` is a categorical value, so we can use the `category` type. But since the order in this case is importart, first we'll create a new category type with the correct order and then we'll use it to convert the column.","metadata":{}},{"cell_type":"code","source":"level_group_cat_type = pd.CategoricalDtype(categories=[\"0-4\", \"5-12\", \"13-22\"], ordered=True)\ntrain_df[\"level_group\"] = train_df[\"level_group\"].astype(level_group_cat_type)","metadata":{"execution":{"iopub.status.busy":"2023-05-01T15:39:15.706740Z","iopub.execute_input":"2023-05-01T15:39:15.709259Z","iopub.status.idle":"2023-05-01T15:39:15.720116Z","shell.execute_reply.started":"2023-05-01T15:39:15.709206Z","shell.execute_reply":"2023-05-01T15:39:15.719071Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now, let's perform a quick check of some columns like `fqid`, `room_fqid`, etc.","metadata":{}},{"cell_type":"code","source":"train_df[\"fqid\"].value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-05-01T15:39:15.721530Z","iopub.execute_input":"2023-05-01T15:39:15.722093Z","iopub.status.idle":"2023-05-01T15:39:15.733195Z","shell.execute_reply.started":"2023-05-01T15:39:15.722059Z","shell.execute_reply":"2023-05-01T15:39:15.732154Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df[\"room_fqid\"].value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-05-01T15:39:15.734459Z","iopub.execute_input":"2023-05-01T15:39:15.738250Z","iopub.status.idle":"2023-05-01T15:39:15.749975Z","shell.execute_reply.started":"2023-05-01T15:39:15.738210Z","shell.execute_reply":"2023-05-01T15:39:15.748980Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df[\"text_fqid\"].value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-05-01T15:39:15.751511Z","iopub.execute_input":"2023-05-01T15:39:15.755031Z","iopub.status.idle":"2023-05-01T15:39:15.764258Z","shell.execute_reply.started":"2023-05-01T15:39:15.754991Z","shell.execute_reply":"2023-05-01T15:39:15.763141Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"As we can see, we can convert this columns to a category type without the loss of any information.","metadata":{}},{"cell_type":"code","source":"train_df[\"fqid\"] = train_df[\"fqid\"].astype(\"category\")\ntrain_df[\"room_fqid\"] = train_df[\"room_fqid\"].astype(\"category\")\ntrain_df[\"text_fqid\"] = train_df[\"text_fqid\"].astype(\"category\")","metadata":{"execution":{"iopub.status.busy":"2023-05-01T15:39:15.765439Z","iopub.execute_input":"2023-05-01T15:39:15.766390Z","iopub.status.idle":"2023-05-01T15:39:15.778503Z","shell.execute_reply.started":"2023-05-01T15:39:15.766340Z","shell.execute_reply":"2023-05-01T15:39:15.777312Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Columns like `room_coor_x`, `room_coor_y`, `screen_coor_x`, `screen_coor_y` and `hover_duration` have a float64 type but looking at the data we can see that maybe they could be converted to float32 without a loss in information.","metadata":{}},{"cell_type":"code","source":"train_df[\"room_coor_x\"].max()","metadata":{"execution":{"iopub.status.busy":"2023-05-01T15:39:15.779856Z","iopub.execute_input":"2023-05-01T15:39:15.780476Z","iopub.status.idle":"2023-05-01T15:39:15.786783Z","shell.execute_reply.started":"2023-05-01T15:39:15.780442Z","shell.execute_reply":"2023-05-01T15:39:15.785801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df[\"room_coor_y\"].max()","metadata":{"execution":{"iopub.status.busy":"2023-05-01T15:39:15.788112Z","iopub.execute_input":"2023-05-01T15:39:15.788758Z","iopub.status.idle":"2023-05-01T15:39:15.797645Z","shell.execute_reply.started":"2023-05-01T15:39:15.788637Z","shell.execute_reply":"2023-05-01T15:39:15.796703Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df[\"screen_coor_x\"].max()","metadata":{"execution":{"iopub.status.busy":"2023-05-01T15:39:15.799068Z","iopub.execute_input":"2023-05-01T15:39:15.799632Z","iopub.status.idle":"2023-05-01T15:39:15.810721Z","shell.execute_reply.started":"2023-05-01T15:39:15.799593Z","shell.execute_reply":"2023-05-01T15:39:15.809868Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df[\"screen_coor_y\"].max()","metadata":{"execution":{"iopub.status.busy":"2023-05-01T15:39:15.813500Z","iopub.execute_input":"2023-05-01T15:39:15.814035Z","iopub.status.idle":"2023-05-01T15:39:15.823727Z","shell.execute_reply.started":"2023-05-01T15:39:15.813994Z","shell.execute_reply":"2023-05-01T15:39:15.822611Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df[\"hover_duration\"].max()","metadata":{"execution":{"iopub.status.busy":"2023-05-01T15:39:15.831415Z","iopub.execute_input":"2023-05-01T15:39:15.831841Z","iopub.status.idle":"2023-05-01T15:39:15.843525Z","shell.execute_reply.started":"2023-05-01T15:39:15.831806Z","shell.execute_reply":"2023-05-01T15:39:15.842046Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"None of the columns needs the precision of a `float64`, so we can convert them to `float32` without losing any information.","metadata":{}},{"cell_type":"code","source":"train_df[\"room_coor_x\"] = train_df[\"room_coor_x\"].astype(\"float32\")\ntrain_df[\"room_coor_y\"] = train_df[\"room_coor_y\"].astype(\"float32\")\ntrain_df[\"screen_coor_x\"] = train_df[\"screen_coor_x\"].astype(\"float32\")\ntrain_df[\"screen_coor_y\"] = train_df[\"screen_coor_y\"].astype(\"float32\")\ntrain_df[\"hover_duration\"] = train_df[\"hover_duration\"].astype(\"float32\")","metadata":{"execution":{"iopub.status.busy":"2023-05-01T15:39:15.846013Z","iopub.execute_input":"2023-05-01T15:39:15.846991Z","iopub.status.idle":"2023-05-01T15:39:15.858088Z","shell.execute_reply.started":"2023-05-01T15:39:15.846952Z","shell.execute_reply":"2023-05-01T15:39:15.857135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The `elapsed_time` columns contains very large values, so we will leave it as an `int64` type. Also, we have to remember that we are working with a subset of the data, so the values in the full dataset could be even larger.","metadata":{}},{"cell_type":"markdown","source":"Finally, we can convert the `text` columns to a category type.","metadata":{}},{"cell_type":"code","source":"train_df[\"text\"] = train_df[\"text\"].astype(\"str\")","metadata":{"execution":{"iopub.status.busy":"2023-05-01T15:39:15.859507Z","iopub.execute_input":"2023-05-01T15:39:15.860036Z","iopub.status.idle":"2023-05-01T15:39:15.870944Z","shell.execute_reply.started":"2023-05-01T15:39:15.860002Z","shell.execute_reply":"2023-05-01T15:39:15.869848Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Finally, let's check the memory usage again.","metadata":{}},{"cell_type":"code","source":"train_df.memory_usage(deep=True)","metadata":{"execution":{"iopub.status.busy":"2023-05-01T15:39:15.872669Z","iopub.execute_input":"2023-05-01T15:39:15.873069Z","iopub.status.idle":"2023-05-01T15:39:15.889708Z","shell.execute_reply.started":"2023-05-01T15:39:15.873031Z","shell.execute_reply":"2023-05-01T15:39:15.888326Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check the memory usage of the train dataset subset in mb\ntrain_df.memory_usage(deep=True).sum() / 1024 ** 2","metadata":{"execution":{"iopub.status.busy":"2023-05-01T15:39:15.893792Z","iopub.execute_input":"2023-05-01T15:39:15.894484Z","iopub.status.idle":"2023-05-01T15:39:15.906230Z","shell.execute_reply.started":"2023-05-01T15:39:15.894447Z","shell.execute_reply":"2023-05-01T15:39:15.904710Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We have reduced the memory usage drastically by just converting some columns to categorical types and the numerical values to more efficient types for the data ranges they contain. This will allow us to load the full dataset later more efficiently and to be able to iterate faster over the data when needed.  ","metadata":{}},{"cell_type":"markdown","source":"# Load the full dataset","metadata":{}},{"cell_type":"code","source":"train_df = pd.read_csv(\n    train_csv_path,\n    index_col=\"index\",\n    dtype={\n        \"session_id\": \"int64\",\n        \"elapsed_time\": \"int64\",\n        \"event_name\": \"category\",\n        \"name\": \"category\",\n        \"level\": \"int8\",\n        \"page\": \"Int8\",\n        \"room_coor_x\": \"float32\",\n        \"room_coor_y\": \"float32\",\n        \"screen_coor_x\": \"float32\",\n        \"screen_coor_y\": \"float32\",\n        \"hover_duration\": \"float32\",\n        \"text\": \"str\",\n        \"fqid\": \"category\",\n        \"room_fqid\": \"category\",\n        \"text_fqid\": \"category\",\n        \"fullscreen\": \"bool\",\n        \"hq\": \"bool\",\n        \"music\": \"bool\",\n        \"level_group\": level_group_cat_type,\n    },\n)","metadata":{"execution":{"iopub.status.busy":"2023-05-01T15:39:15.908206Z","iopub.execute_input":"2023-05-01T15:39:15.908998Z","iopub.status.idle":"2023-05-01T15:41:49.292264Z","shell.execute_reply.started":"2023-05-01T15:39:15.908949Z","shell.execute_reply":"2023-05-01T15:41:49.291227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check the memory usage of the full train dataset in mb\ntrain_df.memory_usage(deep=True).sum() / 1024**2","metadata":{"execution":{"iopub.status.busy":"2023-05-01T15:41:49.293644Z","iopub.execute_input":"2023-05-01T15:41:49.294179Z","iopub.status.idle":"2023-05-01T15:41:51.901661Z","shell.execute_reply.started":"2023-05-01T15:41:49.294145Z","shell.execute_reply":"2023-05-01T15:41:51.900857Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Note that instead of using `bool` for the binary columns, we can also use `int8`.","metadata":{}}]}