{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport os\nimport gc","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-04-21T16:28:49.100034Z","iopub.execute_input":"2022-04-21T16:28:49.100406Z","iopub.status.idle":"2022-04-21T16:28:49.125947Z","shell.execute_reply.started":"2022-04-21T16:28:49.100305Z","shell.execute_reply":"2022-04-21T16:28:49.125205Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_articles():\n    df = pd.read_csv(\"/kaggle/input/h-and-m-personalized-fashion-recommendations/articles.csv\")\n    return df\ndef read_customers():\n    df = pd.read_csv(\"/kaggle/input/h-and-m-personalized-fashion-recommendations/customers.csv\")\n    return df\ndef read_transactions():\n    df = pd.read_csv(\"/kaggle/input/h-and-m-personalized-fashion-recommendations/transactions_train.csv\",\n                                   parse_dates=[\"t_dat\"])\n    return df","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-04-21T16:28:49.145521Z","iopub.execute_input":"2022-04-21T16:28:49.145968Z","iopub.status.idle":"2022-04-21T16:28:49.150680Z","shell.execute_reply.started":"2022-04-21T16:28:49.145935Z","shell.execute_reply":"2022-04-21T16:28:49.149882Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Reducing the Data Types from higher order to lower order","metadata":{}},{"cell_type":"code","source":"def reduce_dtype(df):\n    list_int_type = df.select_dtypes(include=[\"int64\"])\n    for col in list_int_type:\n        if df[col].max()>32767:\n            df[col] = df[col].astype(\"int64\")\n        elif df[col].max()>128:\n            df[col] = df[col].astype(\"int16\")\n        else:\n            df[col] = df[col].astype(\"int8\")\n            \n    list_float_type = df.select_dtypes(include=[\"float64\"])\n    for col in list_float_type:\n        if df[col].max()>np.finfo(np.float16).max:\n            df[col] = df[col].astype(\"float64\")\n        else:\n            df[col] = df[col].astype(\"float16\")       \n    \n    return df","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-04-21T16:28:49.151977Z","iopub.execute_input":"2022-04-21T16:28:49.152248Z","iopub.status.idle":"2022-04-21T16:28:49.163051Z","shell.execute_reply.started":"2022-04-21T16:28:49.152218Z","shell.execute_reply":"2022-04-21T16:28:49.162332Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"| Dataset | Memory Utilization |\n| --- | --- |\n| Articles |  20+ MB |\n| Customer |  73+ MB |\n| Transactions |  1.2+ GB |","metadata":{}},{"cell_type":"code","source":"%%time\narticles_df = read_articles()\narticles_df = reduce_dtype(articles_df)\nprint(articles_df.info(memory_usage=True))\ndel articles_df","metadata":{"execution":{"iopub.status.busy":"2022-04-21T13:38:35.426414Z","iopub.execute_input":"2022-04-21T13:38:35.427019Z","iopub.status.idle":"2022-04-21T13:38:36.236581Z","shell.execute_reply.started":"2022-04-21T13:38:35.426956Z","shell.execute_reply":"2022-04-21T13:38:36.235727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"customers_df = read_customers()\ncustomers_df = reduce_dtype(customers_df)\nprint(customers_df.info())\ndel customers_df","metadata":{"execution":{"iopub.status.busy":"2022-04-21T13:23:38.275785Z","iopub.execute_input":"2022-04-21T13:23:38.276298Z","iopub.status.idle":"2022-04-21T13:23:44.752978Z","shell.execute_reply.started":"2022-04-21T13:23:38.276240Z","shell.execute_reply":"2022-04-21T13:23:44.751989Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transactions_df = read_transactions()\ntransactions_df = reduce_dtype(transactions_df)\nprint(transactions_df.info())\ndel transactions_df","metadata":{"execution":{"iopub.status.busy":"2022-04-21T13:23:44.754484Z","iopub.execute_input":"2022-04-21T13:23:44.754823Z","iopub.status.idle":"2022-04-21T13:24:58.559153Z","shell.execute_reply.started":"2022-04-21T13:23:44.754777Z","shell.execute_reply":"2022-04-21T13:24:58.558318Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## After changing the data type\n\n| Dataset | Memory Utilization |\n| --- | --- |\n| Articles |  14.8+ MB |\n| Customer |  49.7+ MB |\n| Transactions |  818+ MB |","metadata":{}},{"cell_type":"markdown","source":"## Removing the extra columns as per the Intuition","metadata":{}},{"cell_type":"code","source":"def missing_value(data):\n    mis_data = data.isnull().sum().sort_values(ascending=False)\n    per_data = ((data.isnull().sum()/data.isnull().count())*100).sort_values(ascending=False)\n    #nunique_data = mis_data.nunique_data()\n    ret_data = pd.concat([per_data,mis_data],axis=1,keys=[\"Percentage Missing\",\"Missing Count\"])\n    return ret_data[ret_data[\"Missing Count\"]>0] if ret_data[ret_data[\"Missing Count\"]>0].shape[0]>0 else \"No missing Value\"\n\ndef unique_data(data):\n    tot = data.count()\n    nunique = data.nunique().sort_values()\n    ret_data =  pd.concat([tot,nunique],keys=[\"Total\",\"Unique Values\"],axis=1)\n    return ret_data.sort_values(by=\"Unique Values\")","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-04-21T13:24:58.561150Z","iopub.execute_input":"2022-04-21T13:24:58.561378Z","iopub.status.idle":"2022-04-21T13:24:58.568172Z","shell.execute_reply.started":"2022-04-21T13:24:58.561349Z","shell.execute_reply":"2022-04-21T13:24:58.567543Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = read_articles()\nprint(missing_value(df))\nprint(unique_data(df))","metadata":{"execution":{"iopub.status.busy":"2022-04-21T13:39:22.728733Z","iopub.execute_input":"2022-04-21T13:39:22.729393Z","iopub.status.idle":"2022-04-21T13:39:24.150698Z","shell.execute_reply.started":"2022-04-21T13:39:22.729361Z","shell.execute_reply":"2022-04-21T13:39:24.149764Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndf.drop([\"product_type_name\",\"product_group_name\",\"graphical_appearance_name\",\"colour_group_name\",\n         \"perceived_colour_value_name\",\"perceived_colour_master_name\",\"department_name\",\n         \"index_name\",\"index_group_name\",\"section_name\",\"garment_group_name\"],axis=1,inplace=True)\ndf = reduce_dtype(df)\nprint(df.info())\ndel df","metadata":{"execution":{"iopub.status.busy":"2022-04-21T13:39:24.955700Z","iopub.execute_input":"2022-04-21T13:39:24.956248Z","iopub.status.idle":"2022-04-21T13:39:25.044804Z","shell.execute_reply.started":"2022-04-21T13:39:24.956210Z","shell.execute_reply":"2022-04-21T13:39:25.044248Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = read_customers()\nprint(missing_value(df))\nprint(unique_data(df))","metadata":{"execution":{"iopub.status.busy":"2022-04-21T13:25:00.165421Z","iopub.execute_input":"2022-04-21T13:25:00.165621Z","iopub.status.idle":"2022-04-21T13:25:07.726950Z","shell.execute_reply.started":"2022-04-21T13:25:00.165596Z","shell.execute_reply":"2022-04-21T13:25:07.726025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##### This shows Active and FN column contain a single value , so they are not of any use\n##### Also converting age(float64) into age(int8)","metadata":{}},{"cell_type":"code","source":"df.drop([\"FN\",\"Active\"],axis=1,inplace=True)\ndf[\"age\"] = df[\"age\"].apply(lambda x:int(x) if x else None)","metadata":{"execution":{"iopub.status.busy":"2022-04-21T13:25:07.728201Z","iopub.execute_input":"2022-04-21T13:25:07.728437Z","iopub.status.idle":"2022-04-21T13:25:08.002903Z","shell.execute_reply.started":"2022-04-21T13:25:07.728406Z","shell.execute_reply":"2022-04-21T13:25:08.001217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(r\"So, I will be using Nullable Integer 😁\")\ndf[\"age\"] = df[\"age\"].astype(\"Int8\")\nprint(df.info())\ndel df","metadata":{"execution":{"iopub.status.busy":"2022-04-21T13:31:15.076577Z","iopub.execute_input":"2022-04-21T13:31:15.077649Z","iopub.status.idle":"2022-04-21T13:31:16.020048Z","shell.execute_reply.started":"2022-04-21T13:31:15.077575Z","shell.execute_reply":"2022-04-21T13:31:16.019022Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = read_transactions()\ndf = reduce_dtype(df)\nprint(df.info())\ndel df","metadata":{"execution":{"iopub.status.busy":"2022-04-21T13:31:21.491846Z","iopub.execute_input":"2022-04-21T13:31:21.492294Z","iopub.status.idle":"2022-04-21T13:32:15.721242Z","shell.execute_reply.started":"2022-04-21T13:31:21.492251Z","shell.execute_reply":"2022-04-21T13:32:15.720584Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## After removing extra columns\n\n| Dataset | Memory Utilization |\n| --- | --- |\n| Articles |  5.9+ MB |\n| Customer |  44.7+ MB |\n| Transactions |  818+ MB |","metadata":{}},{"cell_type":"markdown","source":"### Will convert csv data into Parquet  ","metadata":{}},{"cell_type":"code","source":"df = read_articles()\ndf.drop([\"product_type_name\",\"product_group_name\",\"graphical_appearance_name\",\"colour_group_name\",\n         \"perceived_colour_value_name\",\"perceived_colour_master_name\",\"department_name\",\n         \"index_name\",\"index_group_name\",\"section_name\",\"garment_group_name\"],axis=1,inplace=True)\ndf = reduce_dtype(df)\ndf.to_parquet(\"articles.parquet\")","metadata":{"execution":{"iopub.status.busy":"2022-04-21T16:28:52.121422Z","iopub.execute_input":"2022-04-21T16:28:52.121731Z","iopub.status.idle":"2022-04-21T16:28:53.454229Z","shell.execute_reply.started":"2022-04-21T16:28:52.121701Z","shell.execute_reply":"2022-04-21T16:28:53.452961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\narticles = pd.read_parquet(\"./articles.parquet\")\nprint(articles.info())\ndel articles","metadata":{"execution":{"iopub.status.busy":"2022-04-21T16:30:06.356270Z","iopub.execute_input":"2022-04-21T16:30:06.356824Z","iopub.status.idle":"2022-04-21T16:30:06.505142Z","shell.execute_reply.started":"2022-04-21T16:30:06.356787Z","shell.execute_reply":"2022-04-21T16:30:06.504337Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cust_df = read_customers()\ncust_df.drop([\"FN\",\"Active\"],axis=1,inplace=True)\ncust_df[\"age\"] = cust_df[\"age\"].astype(\"Int8\")\ncust_df = reduce_dtype(cust_df)\ncust_df.to_parquet(\"customer.parquet\")\ndel cust_df","metadata":{"execution":{"iopub.status.busy":"2022-04-21T16:31:58.599842Z","iopub.execute_input":"2022-04-21T16:31:58.600126Z","iopub.status.idle":"2022-04-21T16:32:04.240909Z","shell.execute_reply.started":"2022-04-21T16:31:58.600095Z","shell.execute_reply":"2022-04-21T16:32:04.239805Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ncust_df = pd.read_parquet(\"./customer.parquet\")\nprint(cust_df.info())\ndel cust_df","metadata":{"execution":{"iopub.status.busy":"2022-04-21T16:32:40.381885Z","iopub.execute_input":"2022-04-21T16:32:40.382518Z","iopub.status.idle":"2022-04-21T16:32:43.645468Z","shell.execute_reply.started":"2022-04-21T16:32:40.382458Z","shell.execute_reply":"2022-04-21T16:32:43.644468Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trans_df = read_transactions()\ntrans_df = reduce_dtype(trans_df)\n# float16 is not supported in parquet\nhalf_floats = list(trans_df.select_dtypes(include=\"float16\"))\ntrans_df[half_floats] = trans_df[half_floats].astype(\"float32\")\ntrans_df.to_parquet(\"transactions.parquet\")\ndel trans_df","metadata":{"execution":{"iopub.status.busy":"2022-04-21T16:45:47.066211Z","iopub.execute_input":"2022-04-21T16:45:47.066521Z","iopub.status.idle":"2022-04-21T16:46:40.955589Z","shell.execute_reply.started":"2022-04-21T16:45:47.066478Z","shell.execute_reply":"2022-04-21T16:46:40.954796Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntrans = pd.read_parquet(\"./transactions.parquet\")\nprint(trans.info())\ndel trans","metadata":{"execution":{"iopub.status.busy":"2022-04-21T16:46:40.960959Z","iopub.execute_input":"2022-04-21T16:46:40.961349Z","iopub.status.idle":"2022-04-21T16:46:53.881938Z","shell.execute_reply.started":"2022-04-21T16:46:40.961303Z","shell.execute_reply":"2022-04-21T16:46:53.881009Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Will update the notebook with new techniques ,if you like it please upvote 🙏🙏","metadata":{}},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}