{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd","metadata":{"_uuid":"ebfbaa7f-af60-4901-812e-d31afaf67f9d","_cell_guid":"2ce0f8f3-18b3-42c4-acdf-f83278123a3b","execution":{"iopub.status.busy":"2022-05-15T19:41:25.821468Z","iopub.execute_input":"2022-05-15T19:41:25.822041Z","iopub.status.idle":"2022-05-15T19:41:25.849846Z","shell.execute_reply.started":"2022-05-15T19:41:25.821901Z","shell.execute_reply":"2022-05-15T19:41:25.849141Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# local environment\n# DATA_INPUT_DIR = 'data'\n# DATA_OUTPUT_DIR = 'data'","metadata":{"execution":{"iopub.status.busy":"2022-05-15T19:41:28.326831Z","iopub.execute_input":"2022-05-15T19:41:28.327125Z","iopub.status.idle":"2022-05-15T19:41:28.330783Z","shell.execute_reply.started":"2022-05-15T19:41:28.327094Z","shell.execute_reply":"2022-05-15T19:41:28.329879Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# kaggle environment\nDATA_INPUT_DIR = '/kaggle/input/h-and-m-personalized-fashion-recommendations'\nDATA_OUTPUT_DIR = '.'","metadata":{"execution":{"iopub.status.busy":"2022-05-15T19:41:36.671457Z","iopub.execute_input":"2022-05-15T19:41:36.672208Z","iopub.status.idle":"2022-05-15T19:41:36.675701Z","shell.execute_reply.started":"2022-05-15T19:41:36.672168Z","shell.execute_reply":"2022-05-15T19:41:36.67507Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# About\n\nThis notebook will create parquet files for the H&M Fashion Recommendation data set. Main purpose is to save memory and make the data load faster. A minimal set of data cleaning and transformation is included:\n\n- customers.age==NA has been mapped to -1\n- customers.customer_id has been converted to integer codes, mappings to original ids in customer_ids.parquet\n- customers.postal_code has been converted to integer codes, no mapping file created to revert this\n- most of the article categories have been converted to categorical variables, original category ids have been dropped\n- transactions_train.price multiplied with 590 (most likely the original price in euros)\n- two new datetime features for transaction data instead of the date\n  - *yearday* - integer representation of the date, ranges from 0 to 733, makes it easy to calculate differences in days\n  - *week* - integer representation of the week, ranges from 0 to 104, 105 is the week to be predicted\n\nResult files:\n\n- customers.parquet (corresponds to customers.csv with new customer_id)\n- customer_ids.parquet (mapping from new customer_id to original customer_id)\n- articles.parquet (corresponds to articles.csv)\n- sales.parquet (corresponds to transactions_train.csv, contains new customer_id)\n- sample_submission.parquet (corresponds to sample_submission.csv, contains original customer_id)","metadata":{}},{"cell_type":"markdown","source":"# Customers","metadata":{}},{"cell_type":"code","source":"customers = pd.read_csv(f'{DATA_INPUT_DIR}/customers.csv')","metadata":{"execution":{"iopub.status.busy":"2022-05-15T19:41:39.590485Z","iopub.execute_input":"2022-05-15T19:41:39.590923Z","iopub.status.idle":"2022-05-15T19:41:44.662767Z","shell.execute_reply.started":"2022-05-15T19:41:39.590886Z","shell.execute_reply":"2022-05-15T19:41:44.661915Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"customers.loc[customers.fashion_news_frequency=='None', 'fashion_news_frequency'] = 'NONE'\ncustomers.fashion_news_frequency = customers.fashion_news_frequency.astype('category')","metadata":{"execution":{"iopub.status.busy":"2022-05-15T19:41:44.664472Z","iopub.execute_input":"2022-05-15T19:41:44.664818Z","iopub.status.idle":"2022-05-15T19:41:45.027049Z","shell.execute_reply.started":"2022-05-15T19:41:44.66476Z","shell.execute_reply":"2022-05-15T19:41:45.026239Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"customers.loc[customers.FN.isna(), 'FN'] = 0\ncustomers.FN = customers.FN.astype('bool')\ncustomers.loc[customers.Active.isna(), 'Active'] = 0\ncustomers.Active = customers.Active.astype('bool')","metadata":{"execution":{"iopub.status.busy":"2022-05-15T19:41:45.028195Z","iopub.execute_input":"2022-05-15T19:41:45.028415Z","iopub.status.idle":"2022-05-15T19:41:45.097024Z","shell.execute_reply.started":"2022-05-15T19:41:45.02839Z","shell.execute_reply":"2022-05-15T19:41:45.096023Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"customers.club_member_status = customers.club_member_status.astype('category')","metadata":{"execution":{"iopub.status.busy":"2022-05-15T19:41:45.099142Z","iopub.execute_input":"2022-05-15T19:41:45.099355Z","iopub.status.idle":"2022-05-15T19:41:45.273098Z","shell.execute_reply.started":"2022-05-15T19:41:45.09933Z","shell.execute_reply":"2022-05-15T19:41:45.272208Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"customers.loc[customers.age.isna(), 'age'] = -1\ncustomers.age = customers.age.astype('int8')","metadata":{"execution":{"iopub.status.busy":"2022-05-15T19:41:45.273993Z","iopub.execute_input":"2022-05-15T19:41:45.274187Z","iopub.status.idle":"2022-05-15T19:41:45.294378Z","shell.execute_reply.started":"2022-05-15T19:41:45.274163Z","shell.execute_reply":"2022-05-15T19:41:45.293604Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"customers.postal_code = customers.postal_code.astype('category').cat.codes","metadata":{"execution":{"iopub.status.busy":"2022-05-15T19:41:45.295595Z","iopub.execute_input":"2022-05-15T19:41:45.296207Z","iopub.status.idle":"2022-05-15T19:41:46.660138Z","shell.execute_reply.started":"2022-05-15T19:41:45.296163Z","shell.execute_reply":"2022-05-15T19:41:46.659145Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"customers['customer_id_original'] = customers.customer_id\ncustomers['customer_id'] = customers.index.values.astype('int32')","metadata":{"execution":{"iopub.status.busy":"2022-05-15T19:41:46.661373Z","iopub.execute_input":"2022-05-15T19:41:46.661574Z","iopub.status.idle":"2022-05-15T19:41:46.703942Z","shell.execute_reply.started":"2022-05-15T19:41:46.661549Z","shell.execute_reply":"2022-05-15T19:41:46.703054Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"customers[['customer_id', 'customer_id_original']].to_parquet(f'{DATA_OUTPUT_DIR}/customer_ids.parquet', compression='gzip')","metadata":{"execution":{"iopub.status.busy":"2022-05-15T19:41:46.705107Z","iopub.execute_input":"2022-05-15T19:41:46.705325Z","iopub.status.idle":"2022-05-15T19:41:55.309554Z","shell.execute_reply.started":"2022-05-15T19:41:46.705298Z","shell.execute_reply":"2022-05-15T19:41:55.30864Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"customers.drop(columns='customer_id_original', inplace=True)\ncustomers.to_parquet(f'{DATA_OUTPUT_DIR}/customers.parquet', compression='gzip')","metadata":{"execution":{"iopub.status.busy":"2022-05-15T19:41:55.312133Z","iopub.execute_input":"2022-05-15T19:41:55.312486Z","iopub.status.idle":"2022-05-15T19:41:57.475563Z","shell.execute_reply.started":"2022-05-15T19:41:55.312441Z","shell.execute_reply":"2022-05-15T19:41:57.474699Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"customers.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-15T19:41:57.477014Z","iopub.execute_input":"2022-05-15T19:41:57.477585Z","iopub.status.idle":"2022-05-15T19:41:57.496835Z","shell.execute_reply.started":"2022-05-15T19:41:57.477542Z","shell.execute_reply":"2022-05-15T19:41:57.496257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Articles","metadata":{"execution":{"iopub.status.busy":"2022-03-01T11:06:38.596302Z","iopub.execute_input":"2022-03-01T11:06:38.596682Z","iopub.status.idle":"2022-03-01T11:06:38.601602Z","shell.execute_reply.started":"2022-03-01T11:06:38.596628Z","shell.execute_reply":"2022-03-01T11:06:38.600654Z"}}},{"cell_type":"code","source":"articles = pd.read_csv(f'{DATA_INPUT_DIR}/articles.csv')","metadata":{"execution":{"iopub.status.busy":"2022-05-14T10:56:28.12215Z","iopub.execute_input":"2022-05-14T10:56:28.122488Z","iopub.status.idle":"2022-05-14T10:56:28.891684Z","shell.execute_reply.started":"2022-05-14T10:56:28.12245Z","shell.execute_reply":"2022-05-14T10:56:28.890721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"articles['product_type'] = articles.product_type_name.astype('category')\narticles.drop(columns=['product_type_no', 'product_type_name'], inplace=True)\n\narticles['graphical_appearance'] = articles.graphical_appearance_name.astype('category')\narticles.drop(columns=['graphical_appearance_no', 'graphical_appearance_name'], inplace=True)\n\narticles['colour_group'] = articles.colour_group_name.astype('category')\narticles.drop(columns=['colour_group_code', 'colour_group_name'], inplace=True)\n\narticles['perceived_colour_value'] = articles.perceived_colour_value_name.astype('category')\narticles.drop(columns=['perceived_colour_value_id', 'perceived_colour_value_name'], inplace=True)\n\narticles['perceived_colour_master'] = articles.perceived_colour_master_name.astype('category')\narticles.drop(columns=['perceived_colour_master_id', 'perceived_colour_master_name'], inplace=True)\n\narticles['index'] = articles.index_name.astype('category')\narticles.drop(columns=['index_code', 'index_name'], inplace=True)\n\narticles['index_group'] = articles.index_group_name.astype('category')\narticles.drop(columns=['index_group_no', 'index_group_name'], inplace=True)\n\narticles['section_name'] = articles.section_name.astype('category')\narticles.drop(columns=['section_no', 'section_name'], inplace=True)\n\narticles['garment_group'] = articles.garment_group_name.astype('category')\narticles.drop(columns=['garment_group_no', 'garment_group_name'], inplace=True)\n\narticles.product_code = articles.product_code.astype('int32')\narticles.department_no = articles.department_no.astype('int32')\n\narticles.product_group_name = articles.product_group_name.astype('category')\n\narticles.article_id = articles.article_id.astype('int32')\narticles.product_code = articles.product_code.astype('int32')","metadata":{"execution":{"iopub.status.busy":"2022-05-14T10:56:29.7847Z","iopub.execute_input":"2022-05-14T10:56:29.785028Z","iopub.status.idle":"2022-05-14T10:56:30.022327Z","shell.execute_reply.started":"2022-05-14T10:56:29.784991Z","shell.execute_reply":"2022-05-14T10:56:30.021307Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"articles.to_parquet(f'{DATA_OUTPUT_DIR}/articles.parquet', compression='gzip')","metadata":{"execution":{"iopub.status.busy":"2022-05-14T10:56:31.12802Z","iopub.execute_input":"2022-05-14T10:56:31.128323Z","iopub.status.idle":"2022-05-14T10:56:31.963209Z","shell.execute_reply.started":"2022-05-14T10:56:31.12829Z","shell.execute_reply":"2022-05-14T10:56:31.962483Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"articles.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-14T10:56:31.964691Z","iopub.execute_input":"2022-05-14T10:56:31.965042Z","iopub.status.idle":"2022-05-14T10:56:31.987149Z","shell.execute_reply.started":"2022-05-14T10:56:31.965012Z","shell.execute_reply":"2022-05-14T10:56:31.986488Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Sales","metadata":{}},{"cell_type":"code","source":"sales = pd.read_csv(f'{DATA_INPUT_DIR}/transactions_train.csv')","metadata":{"execution":{"iopub.status.busy":"2022-05-14T10:56:41.240324Z","iopub.execute_input":"2022-05-14T10:56:41.240765Z","iopub.status.idle":"2022-05-14T10:57:23.479802Z","shell.execute_reply.started":"2022-05-14T10:56:41.240732Z","shell.execute_reply":"2022-05-14T10:57:23.478898Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sales.info(memory_usage='deep')","metadata":{"execution":{"iopub.status.busy":"2022-05-14T10:57:23.481366Z","iopub.execute_input":"2022-05-14T10:57:23.481608Z","iopub.status.idle":"2022-05-14T10:57:35.901266Z","shell.execute_reply.started":"2022-05-14T10:57:23.48158Z","shell.execute_reply":"2022-05-14T10:57:35.899863Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# descramble original price and round to cents\nsales['price'] = (np.round(sales.price*590*100)/100).astype('float32')","metadata":{"execution":{"iopub.status.busy":"2022-05-14T10:57:35.902871Z","iopub.execute_input":"2022-05-14T10:57:35.903148Z","iopub.status.idle":"2022-05-14T10:57:36.176019Z","shell.execute_reply.started":"2022-05-14T10:57:35.903114Z","shell.execute_reply":"2022-05-14T10:57:36.175286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sales.t_dat = sales.t_dat.astype('datetime64')\nsales.article_id = sales.article_id.astype('int32')","metadata":{"execution":{"iopub.status.busy":"2022-05-14T10:57:36.178427Z","iopub.execute_input":"2022-05-14T10:57:36.17874Z","iopub.status.idle":"2022-05-14T10:57:42.99039Z","shell.execute_reply.started":"2022-05-14T10:57:36.178703Z","shell.execute_reply":"2022-05-14T10:57:42.989657Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# sales_channel_id 1: offline, 2: online\nsales['online_channel'] = (sales.sales_channel_id-1).astype('bool')\nsales.drop(columns='sales_channel_id', inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-05-14T10:57:42.991693Z","iopub.execute_input":"2022-05-14T10:57:42.992274Z","iopub.status.idle":"2022-05-14T10:57:44.17498Z","shell.execute_reply.started":"2022-05-14T10:57:42.992226Z","shell.execute_reply":"2022-05-14T10:57:44.173855Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# yearday represents each day with a simple int > faster calculation of diffs, etc.\nsales['yearday'] = (sales.t_dat.dt.day_of_year + (sales.t_dat.dt.year-sales.t_dat.dt.year.min())*365).astype('int16')\nsales['yearday'] = sales.yearday - sales.yearday.min()\n\n# as we need to predict for a week we add a week column\nlast_yearday = sales.yearday.max()\nweek_offset = np.ceil(last_yearday/7)*7 - last_yearday - 1\n\nsales['week'] = np.trunc((sales.yearday+week_offset)/7).astype('int8')\n\n# drop original date column - could be reconstructed by pd.Timestamp('2018-09-20') + sales['yearday'].astype('timedelta64[D]')\nsales.drop(columns='t_dat', inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-05-14T10:57:44.176397Z","iopub.execute_input":"2022-05-14T10:57:44.176644Z","iopub.status.idle":"2022-05-14T10:57:54.756896Z","shell.execute_reply.started":"2022-05-14T10:57:44.176615Z","shell.execute_reply":"2022-05-14T10:57:54.755922Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# map to new customer ids\ncustomer_ids = pd.read_parquet(f'{DATA_OUTPUT_DIR}/customer_ids.parquet')\nsales = sales.rename(columns={'customer_id': 'customer_id_original'}).merge(customer_ids)\nsales = sales.drop(columns='customer_id_original')","metadata":{"execution":{"iopub.status.busy":"2022-05-14T10:57:54.758448Z","iopub.execute_input":"2022-05-14T10:57:54.758704Z","iopub.status.idle":"2022-05-14T10:58:19.63598Z","shell.execute_reply.started":"2022-05-14T10:57:54.758675Z","shell.execute_reply":"2022-05-14T10:58:19.635076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sales = sales.reindex(columns=['yearday', 'week', 'customer_id', 'article_id', 'price', 'online_channel'])","metadata":{"execution":{"iopub.status.busy":"2022-05-14T10:58:19.63737Z","iopub.execute_input":"2022-05-14T10:58:19.637773Z","iopub.status.idle":"2022-05-14T10:58:20.141705Z","shell.execute_reply.started":"2022-05-14T10:58:19.63774Z","shell.execute_reply":"2022-05-14T10:58:20.140844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sales.info(memory_usage='deep')","metadata":{"execution":{"iopub.status.busy":"2022-05-14T10:58:20.14312Z","iopub.execute_input":"2022-05-14T10:58:20.143412Z","iopub.status.idle":"2022-05-14T10:58:20.152818Z","shell.execute_reply.started":"2022-05-14T10:58:20.143377Z","shell.execute_reply":"2022-05-14T10:58:20.152065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sales.to_parquet(f'{DATA_OUTPUT_DIR}/sales.parquet', compression='gzip')","metadata":{"execution":{"iopub.status.busy":"2022-05-14T10:58:20.154729Z","iopub.execute_input":"2022-05-14T10:58:20.155332Z","iopub.status.idle":"2022-05-14T10:59:11.669575Z","shell.execute_reply.started":"2022-05-14T10:58:20.155296Z","shell.execute_reply":"2022-05-14T10:59:11.668255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sales.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-14T10:59:11.671266Z","iopub.execute_input":"2022-05-14T10:59:11.671666Z","iopub.status.idle":"2022-05-14T10:59:11.6841Z","shell.execute_reply.started":"2022-05-14T10:59:11.671633Z","shell.execute_reply":"2022-05-14T10:59:11.683155Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Sample Submission","metadata":{}},{"cell_type":"code","source":"sample_submission = pd.read_csv(f'{DATA_INPUT_DIR}/sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2022-05-14T11:00:35.5983Z","iopub.execute_input":"2022-05-14T11:00:35.598697Z","iopub.status.idle":"2022-05-14T11:00:40.584045Z","shell.execute_reply.started":"2022-05-14T11:00:35.598656Z","shell.execute_reply":"2022-05-14T11:00:40.583102Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission.to_parquet(f'{DATA_OUTPUT_DIR}/sample_submission.parquet', compression='gzip')","metadata":{"execution":{"iopub.status.busy":"2022-05-14T11:00:40.585892Z","iopub.execute_input":"2022-05-14T11:00:40.586166Z","iopub.status.idle":"2022-05-14T11:00:48.726321Z","shell.execute_reply.started":"2022-05-14T11:00:40.586131Z","shell.execute_reply":"2022-05-14T11:00:48.725289Z"},"trusted":true},"execution_count":null,"outputs":[]}]}