{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np","metadata":{"execution":{"iopub.status.busy":"2023-10-25T18:05:29.622534Z","iopub.execute_input":"2023-10-25T18:05:29.623359Z","iopub.status.idle":"2023-10-25T18:05:30.070483Z","shell.execute_reply.started":"2023-10-25T18:05:29.623297Z","shell.execute_reply":"2023-10-25T18:05:30.069601Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## EDA articles","metadata":{}},{"cell_type":"code","source":"articles = pd.read_csv('/kaggle/input/h-and-m-personalized-fashion-recommendations/articles.csv')\narticles.sample(5)","metadata":{"execution":{"iopub.status.busy":"2023-10-25T18:05:32.194272Z","iopub.execute_input":"2023-10-25T18:05:32.195356Z","iopub.status.idle":"2023-10-25T18:05:32.879584Z","shell.execute_reply.started":"2023-10-25T18:05:32.195300Z","shell.execute_reply":"2023-10-25T18:05:32.878378Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"articles.isnull().sum()/len(articles)","metadata":{"execution":{"iopub.status.busy":"2023-10-25T18:05:35.922010Z","iopub.execute_input":"2023-10-25T18:05:35.922789Z","iopub.status.idle":"2023-10-25T18:05:36.062157Z","shell.execute_reply.started":"2023-10-25T18:05:35.922752Z","shell.execute_reply":"2023-10-25T18:05:36.061102Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"articles.nunique()","metadata":{"execution":{"iopub.status.busy":"2023-10-25T18:05:39.872170Z","iopub.execute_input":"2023-10-25T18:05:39.872910Z","iopub.status.idle":"2023-10-25T18:05:40.031699Z","shell.execute_reply.started":"2023-10-25T18:05:39.872865Z","shell.execute_reply":"2023-10-25T18:05:40.030666Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"articles[['product_type_no','product_type_name']].sample(5)","metadata":{"execution":{"iopub.status.busy":"2023-10-25T18:05:44.748206Z","iopub.execute_input":"2023-10-25T18:05:44.749120Z","iopub.status.idle":"2023-10-25T18:05:44.765363Z","shell.execute_reply.started":"2023-10-25T18:05:44.749085Z","shell.execute_reply":"2023-10-25T18:05:44.764065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"articles.product_group_name.unique()","metadata":{"execution":{"iopub.status.busy":"2023-10-25T18:05:48.522502Z","iopub.execute_input":"2023-10-25T18:05:48.522880Z","iopub.status.idle":"2023-10-25T18:05:48.539051Z","shell.execute_reply.started":"2023-10-25T18:05:48.522850Z","shell.execute_reply":"2023-10-25T18:05:48.537880Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"articles[['graphical_appearance_no','graphical_appearance_name']].sample(5)","metadata":{"execution":{"iopub.status.busy":"2023-10-25T18:05:51.801915Z","iopub.execute_input":"2023-10-25T18:05:51.802661Z","iopub.status.idle":"2023-10-25T18:05:51.818438Z","shell.execute_reply.started":"2023-10-25T18:05:51.802627Z","shell.execute_reply":"2023-10-25T18:05:51.817262Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"articles[['colour_group_code','colour_group_name']].sample(5)","metadata":{"execution":{"iopub.status.busy":"2023-10-25T18:05:55.081823Z","iopub.execute_input":"2023-10-25T18:05:55.082727Z","iopub.status.idle":"2023-10-25T18:05:55.097785Z","shell.execute_reply.started":"2023-10-25T18:05:55.082692Z","shell.execute_reply":"2023-10-25T18:05:55.096415Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"articles.perceived_colour_value_id.unique(), articles.perceived_colour_value_name.unique()","metadata":{"execution":{"iopub.status.busy":"2023-10-25T18:05:57.844206Z","iopub.execute_input":"2023-10-25T18:05:57.844618Z","iopub.status.idle":"2023-10-25T18:05:57.863245Z","shell.execute_reply.started":"2023-10-25T18:05:57.844585Z","shell.execute_reply":"2023-10-25T18:05:57.862190Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"articles.perceived_colour_master_id.unique(), articles.perceived_colour_master_name.unique()","metadata":{"execution":{"iopub.status.busy":"2023-10-25T18:06:00.781836Z","iopub.execute_input":"2023-10-25T18:06:00.782799Z","iopub.status.idle":"2023-10-25T18:06:00.801209Z","shell.execute_reply.started":"2023-10-25T18:06:00.782749Z","shell.execute_reply":"2023-10-25T18:06:00.800315Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"articles[['department_no','department_name']].sample(5)","metadata":{"execution":{"iopub.status.busy":"2023-10-25T18:06:03.725991Z","iopub.execute_input":"2023-10-25T18:06:03.726808Z","iopub.status.idle":"2023-10-25T18:06:03.741884Z","shell.execute_reply.started":"2023-10-25T18:06:03.726776Z","shell.execute_reply":"2023-10-25T18:06:03.740846Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"articles.index_code.unique(), articles.index_name.unique()","metadata":{"execution":{"iopub.status.busy":"2023-10-25T18:06:07.194240Z","iopub.execute_input":"2023-10-25T18:06:07.195193Z","iopub.status.idle":"2023-10-25T18:06:07.224678Z","shell.execute_reply.started":"2023-10-25T18:06:07.195157Z","shell.execute_reply":"2023-10-25T18:06:07.223211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"articles.index_group_no.unique(), articles.index_group_name.unique()","metadata":{"execution":{"iopub.status.busy":"2023-10-25T18:06:10.124517Z","iopub.execute_input":"2023-10-25T18:06:10.125410Z","iopub.status.idle":"2023-10-25T18:06:10.144116Z","shell.execute_reply.started":"2023-10-25T18:06:10.125377Z","shell.execute_reply":"2023-10-25T18:06:10.143051Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"articles[['section_no','section_name']].sample(5)","metadata":{"execution":{"iopub.status.busy":"2023-10-25T18:06:13.083923Z","iopub.execute_input":"2023-10-25T18:06:13.084285Z","iopub.status.idle":"2023-10-25T18:06:13.100503Z","shell.execute_reply.started":"2023-10-25T18:06:13.084257Z","shell.execute_reply":"2023-10-25T18:06:13.099313Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"articles.garment_group_no.unique(), articles.garment_group_name.unique()","metadata":{"execution":{"iopub.status.busy":"2023-10-25T18:06:16.394210Z","iopub.execute_input":"2023-10-25T18:06:16.394600Z","iopub.status.idle":"2023-10-25T18:06:16.411954Z","shell.execute_reply.started":"2023-10-25T18:06:16.394568Z","shell.execute_reply":"2023-10-25T18:06:16.410674Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ohe_columns = ['product_group_name', 'graphical_appearance_no','perceived_colour_master_id', 'index_code', \\\n               'garment_group_no']\nemb_columns = ['product_type_name', 'department_name', 'colour_group_name', 'section_name', 'detail_desc']","metadata":{"execution":{"iopub.status.busy":"2023-10-25T18:06:19.373902Z","iopub.execute_input":"2023-10-25T18:06:19.374593Z","iopub.status.idle":"2023-10-25T18:06:19.379883Z","shell.execute_reply.started":"2023-10-25T18:06:19.374558Z","shell.execute_reply":"2023-10-25T18:06:19.378666Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Preprocess articles","metadata":{}},{"cell_type":"markdown","source":"### Preprocess text columns","metadata":{"execution":{"iopub.status.busy":"2023-10-24T19:54:07.588532Z","iopub.execute_input":"2023-10-24T19:54:07.588983Z","iopub.status.idle":"2023-10-24T19:54:07.594885Z","shell.execute_reply.started":"2023-10-24T19:54:07.588951Z","shell.execute_reply":"2023-10-24T19:54:07.593293Z"}}},{"cell_type":"code","source":"articles[['article_id']+emb_columns]","metadata":{"execution":{"iopub.status.busy":"2023-10-25T18:06:25.266231Z","iopub.execute_input":"2023-10-25T18:06:25.266956Z","iopub.status.idle":"2023-10-25T18:06:25.288126Z","shell.execute_reply.started":"2023-10-25T18:06:25.266919Z","shell.execute_reply":"2023-10-25T18:06:25.287056Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"articles['detail_desc'].unique()[-20:]","metadata":{"execution":{"iopub.status.busy":"2023-10-25T18:06:28.667717Z","iopub.execute_input":"2023-10-25T18:06:28.668520Z","iopub.status.idle":"2023-10-25T18:06:28.688971Z","shell.execute_reply.started":"2023-10-25T18:06:28.668486Z","shell.execute_reply":"2023-10-25T18:06:28.687953Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!python3 -m nltk.downloader wordnet\n!unzip /usr/share/nltk_data/corpora/wordnet.zip -d /usr/share/nltk_data/corpora/","metadata":{"execution":{"iopub.status.busy":"2023-10-25T18:05:04.520206Z","iopub.execute_input":"2023-10-25T18:05:04.520780Z","iopub.status.idle":"2023-10-25T18:05:21.168831Z","shell.execute_reply.started":"2023-10-25T18:05:04.520738Z","shell.execute_reply":"2023-10-25T18:05:21.167648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import re\nfrom nltk.stem import WordNetLemmatizer\nimport nltk\nstopwords = set(nltk.corpus.stopwords.words('english'))\nlemmatizer = WordNetLemmatizer()","metadata":{"execution":{"iopub.status.busy":"2023-10-25T18:04:59.592812Z","iopub.execute_input":"2023-10-25T18:04:59.593770Z","iopub.status.idle":"2023-10-25T18:05:00.251888Z","shell.execute_reply.started":"2023-10-25T18:04:59.593724Z","shell.execute_reply":"2023-10-25T18:05:00.250788Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def preprocess(x):\n    x = ' '+x+' '\n    x = x.lower()\n    \n    # remove digits\n    x = re.sub(r'\\d+\\-\\d+', ' ', x)\n    x = re.sub(r'\\d+\\.\\d+', ' ', x)\n    x = re.sub(r'\\d+/\\d+', ' ', x)\n    x = re.sub(r'\\d+', ' ', x)\n    # remove everything in braces\n    x = re.sub(r'\\(.+?\\)', ' ', x)\n    \n    x = re.sub(r' cm ', ' ', x)\n    x = re.sub(r' denier ', ' ', x)\n    x = re.sub(r' t\\-shirt ', ' tshirt ', x)\n    x = re.sub(r'(\\w+)(\\+)', r'\\1 plus', x)\n    # remove everything except letters\n    x = re.sub(r'[^a-z\\s]', ' ', x)\n    tokens = x.strip().split()\n    tokens = ' '.join([lemmatizer.lemmatize(token) for token in tokens if token not in stopwords])\n    return tokens\n\ns = '5-pocket slim-fit t-shirt h&m+'\npreprocess(s)","metadata":{"execution":{"iopub.status.busy":"2023-10-25T18:06:33.887997Z","iopub.execute_input":"2023-10-25T18:06:33.888604Z","iopub.status.idle":"2023-10-25T18:06:35.824444Z","shell.execute_reply.started":"2023-10-25T18:06:33.888569Z","shell.execute_reply":"2023-10-25T18:06:35.823397Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tqdm import tqdm\ntqdm.pandas()\narticles['desc_full'] = articles['product_type_name']+' '+articles['department_name']+' '\\\n                        +articles['colour_group_name']+' '+articles['section_name']+' '\\\n                        +articles['detail_desc'].fillna(' ')\narticles['desc_full_prep'] = articles['desc_full'].progress_apply(preprocess)","metadata":{"execution":{"iopub.status.busy":"2023-10-25T18:06:36.147865Z","iopub.execute_input":"2023-10-25T18:06:36.148762Z","iopub.status.idle":"2023-10-25T18:07:02.534680Z","shell.execute_reply.started":"2023-10-25T18:06:36.148726Z","shell.execute_reply":"2023-10-25T18:07:02.533587Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"texts = articles['desc_full_prep'].unique().tolist()\nwith open('texts_normalized.txt', 'w') as f:\n    for text in texts:\n        f.write(text)\n        f.write('\\n')","metadata":{"execution":{"iopub.status.busy":"2023-10-25T18:07:02.536442Z","iopub.execute_input":"2023-10-25T18:07:02.536733Z","iopub.status.idle":"2023-10-25T18:07:02.648769Z","shell.execute_reply.started":"2023-10-25T18:07:02.536709Z","shell.execute_reply":"2023-10-25T18:07:02.647765Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import fasttext\nfasttext_model = fasttext.train_unsupervised('texts_normalized.txt', minn=2, maxn=5, dim=200)","metadata":{"execution":{"iopub.status.busy":"2023-10-25T18:07:02.649856Z","iopub.execute_input":"2023-10-25T18:07:02.650153Z","iopub.status.idle":"2023-10-25T18:08:13.263173Z","shell.execute_reply.started":"2023-10-25T18:07:02.650129Z","shell.execute_reply":"2023-10-25T18:08:13.262059Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"doc2vec_dict = {}\nfor row in articles[['article_id', 'desc_full_prep']].itertuples():\n    doc2vec_dict[row[1]] = fasttext_model.get_sentence_vector(row[2])","metadata":{"execution":{"iopub.status.busy":"2023-10-25T18:08:13.265926Z","iopub.execute_input":"2023-10-25T18:08:13.266198Z","iopub.status.idle":"2023-10-25T18:08:26.015883Z","shell.execute_reply.started":"2023-10-25T18:08:13.266174Z","shell.execute_reply":"2023-10-25T18:08:26.015093Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"emb_features = np.array(list(doc2vec_dict.values()))\nemb_features.shape","metadata":{"execution":{"iopub.status.busy":"2023-10-25T18:08:26.016930Z","iopub.execute_input":"2023-10-25T18:08:26.017189Z","iopub.status.idle":"2023-10-25T18:08:26.106175Z","shell.execute_reply.started":"2023-10-25T18:08:26.017166Z","shell.execute_reply":"2023-10-25T18:08:26.104936Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### One Hot Encoding rest columns","metadata":{}},{"cell_type":"code","source":"articles[['article_id']+ohe_columns]","metadata":{"execution":{"iopub.status.busy":"2023-10-25T18:08:26.107585Z","iopub.execute_input":"2023-10-25T18:08:26.107991Z","iopub.status.idle":"2023-10-25T18:08:26.133217Z","shell.execute_reply.started":"2023-10-25T18:08:26.107953Z","shell.execute_reply":"2023-10-25T18:08:26.132031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ohe_features = pd.get_dummies(articles[ohe_columns].astype(str)).astype(int).values\nohe_features.shape","metadata":{"execution":{"iopub.status.busy":"2023-10-25T18:08:26.134491Z","iopub.execute_input":"2023-10-25T18:08:26.134782Z","iopub.status.idle":"2023-10-25T18:08:26.551388Z","shell.execute_reply.started":"2023-10-25T18:08:26.134757Z","shell.execute_reply":"2023-10-25T18:08:26.550252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"articles_features = np.concatenate([emb_features, ohe_features], axis=1)\narticles_features.shape","metadata":{"execution":{"iopub.status.busy":"2023-10-25T18:08:26.552828Z","iopub.execute_input":"2023-10-25T18:08:26.553206Z","iopub.status.idle":"2023-10-25T18:08:26.658968Z","shell.execute_reply.started":"2023-10-25T18:08:26.553172Z","shell.execute_reply":"2023-10-25T18:08:26.657823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"articles_features_with_id = {}\nfor article_id, features in zip(articles['article_id'].values, articles_features):\n    articles_features_with_id[article_id] = features","metadata":{"execution":{"iopub.status.busy":"2023-10-25T18:08:26.660075Z","iopub.execute_input":"2023-10-25T18:08:26.660389Z","iopub.status.idle":"2023-10-25T18:08:26.746124Z","shell.execute_reply.started":"2023-10-25T18:08:26.660363Z","shell.execute_reply":"2023-10-25T18:08:26.745057Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## EDA customers","metadata":{}},{"cell_type":"code","source":"customers = pd.read_csv('/kaggle/input/h-and-m-personalized-fashion-recommendations/customers.csv')\ncustomers","metadata":{"execution":{"iopub.status.busy":"2023-10-25T18:08:26.751814Z","iopub.execute_input":"2023-10-25T18:08:26.752137Z","iopub.status.idle":"2023-10-25T18:08:30.377291Z","shell.execute_reply.started":"2023-10-25T18:08:26.752110Z","shell.execute_reply":"2023-10-25T18:08:30.376058Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"customers.isnull().sum()/len(customers)","metadata":{"execution":{"iopub.status.busy":"2023-10-25T18:08:30.378622Z","iopub.execute_input":"2023-10-25T18:08:30.378987Z","iopub.status.idle":"2023-10-25T18:08:30.898603Z","shell.execute_reply.started":"2023-10-25T18:08:30.378954Z","shell.execute_reply":"2023-10-25T18:08:30.897320Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"customers.nunique()","metadata":{"execution":{"iopub.status.busy":"2023-10-25T18:08:30.899909Z","iopub.execute_input":"2023-10-25T18:08:30.900317Z","iopub.status.idle":"2023-10-25T18:08:32.250595Z","shell.execute_reply.started":"2023-10-25T18:08:30.900280Z","shell.execute_reply":"2023-10-25T18:08:32.249036Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"customers['age'].describe()","metadata":{"execution":{"iopub.status.busy":"2023-10-25T18:08:32.251935Z","iopub.execute_input":"2023-10-25T18:08:32.252299Z","iopub.status.idle":"2023-10-25T18:08:32.335542Z","shell.execute_reply.started":"2023-10-25T18:08:32.252264Z","shell.execute_reply":"2023-10-25T18:08:32.334423Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Preprocess customers","metadata":{}},{"cell_type":"code","source":"customers[['FN', 'Active']] = customers[['FN', 'Active']].fillna(0).astype(int)\ncustomers[['club_member_status', 'fashion_news_frequency']] = customers[['club_member_status', \\\n                                                                         'fashion_news_frequency']].fillna('NONE')","metadata":{"execution":{"iopub.status.busy":"2023-10-25T18:08:32.336700Z","iopub.execute_input":"2023-10-25T18:08:32.336998Z","iopub.status.idle":"2023-10-25T18:08:32.706189Z","shell.execute_reply.started":"2023-10-25T18:08:32.336974Z","shell.execute_reply":"2023-10-25T18:08:32.705203Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"customers['age'] = customers['age'].fillna(customers['age'].mean())\n\nbins = [16, 20, 25, 30, 40, 50, 60, 100]\nlabels = [1,2,3,4,5,6,7]\ncustomers['age_bins'] = pd.cut(customers['age'], bins=bins, labels=labels)\ncustomers","metadata":{"execution":{"iopub.status.busy":"2023-10-25T18:08:32.707250Z","iopub.execute_input":"2023-10-25T18:08:32.707542Z","iopub.status.idle":"2023-10-25T18:08:32.784735Z","shell.execute_reply.started":"2023-10-25T18:08:32.707517Z","shell.execute_reply":"2023-10-25T18:08:32.783762Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ohe_columns = ['club_member_status', 'fashion_news_frequency', 'age_bins']\nohe_features = pd.get_dummies(customers[ohe_columns].astype(str)).astype(int)\ncustomer_features = pd.concat([customers[['FN', 'Active']], ohe_features], axis=1).values\ncustomer_features.shape","metadata":{"execution":{"iopub.status.busy":"2023-10-25T18:08:32.785851Z","iopub.execute_input":"2023-10-25T18:08:32.786109Z","iopub.status.idle":"2023-10-25T18:08:33.648240Z","shell.execute_reply.started":"2023-10-25T18:08:32.786088Z","shell.execute_reply":"2023-10-25T18:08:33.647225Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"customer_features_with_id = {}\nfor customer_id, features in zip(customers['customer_id'].values, customer_features):\n    customer_features_with_id[customer_id] = features","metadata":{"execution":{"iopub.status.busy":"2023-10-25T18:08:33.649446Z","iopub.execute_input":"2023-10-25T18:08:33.649733Z","iopub.status.idle":"2023-10-25T18:08:34.561780Z","shell.execute_reply.started":"2023-10-25T18:08:33.649708Z","shell.execute_reply":"2023-10-25T18:08:34.560737Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## EDA transactions","metadata":{}},{"cell_type":"code","source":"transactions = pd.read_csv('/kaggle/input/h-and-m-personalized-fashion-recommendations/transactions_train.csv')\ntransactions","metadata":{"execution":{"iopub.status.busy":"2023-10-25T20:11:00.099540Z","iopub.execute_input":"2023-10-25T20:11:00.099915Z","iopub.status.idle":"2023-10-25T20:11:36.023538Z","shell.execute_reply.started":"2023-10-25T20:11:00.099886Z","shell.execute_reply":"2023-10-25T20:11:36.022263Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.to_datetime(transactions['t_dat']).value_counts().sort_index().plot(kind='line', figsize=(20, 7))","metadata":{"execution":{"iopub.status.busy":"2023-10-25T18:09:10.606204Z","iopub.execute_input":"2023-10-25T18:09:10.606644Z","iopub.status.idle":"2023-10-25T18:09:16.516215Z","shell.execute_reply.started":"2023-10-25T18:09:10.606602Z","shell.execute_reply":"2023-10-25T18:09:16.515243Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Train test split","metadata":{}},{"cell_type":"code","source":"from datetime import datetime, timedelta\nmax_date = transactions['t_dat'].max()\n\n# keep one last 1 day for test dataset\nend_train_date = datetime.strptime(max_date, '%Y-%M-%d') - timedelta(days=1)\nend_train_date = end_train_date.strftime('%Y-%M-%d')\nend_train_date","metadata":{"execution":{"iopub.status.busy":"2023-10-25T18:09:16.520244Z","iopub.execute_input":"2023-10-25T18:09:16.521247Z","iopub.status.idle":"2023-10-25T18:09:20.206759Z","shell.execute_reply.started":"2023-10-25T18:09:16.521208Z","shell.execute_reply":"2023-10-25T18:09:20.205742Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transactions_train = transactions[transactions['t_dat']<=end_train_date]\ntransactions_test = transactions[transactions['t_dat']>end_train_date]\ntransactions_train.shape, transactions_test.shape","metadata":{"execution":{"iopub.status.busy":"2023-10-25T18:09:20.208556Z","iopub.execute_input":"2023-10-25T18:09:20.208926Z","iopub.status.idle":"2023-10-25T18:09:29.447310Z","shell.execute_reply.started":"2023-10-25T18:09:20.208891Z","shell.execute_reply":"2023-10-25T18:09:29.446334Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"articles_ids = pd.Series(list(articles_features_with_id.keys()))\ndef create_zero_target(transactions_df_grouped):\n    cutomers_lst = []\n    articles_lst = []\n    for row in transactions_df_grouped[['customer_id', 'article_id']].itertuples():\n        customer_id, article_ids = row[1], row[2]\n        unord_article_id = articles_ids[~articles_ids.isin(article_ids)].sample(50)\n        cutomers_lst.append(customer_id)\n        articles_lst.append(unord_article_id)\n    df = pd.DataFrame({'customer_id':cutomers_lst, 'article_id':articles_lst, 'target':[0]*len(cutomers_lst)}).\\\n                                                                                            explode('article_id')\n    return df","metadata":{"execution":{"iopub.status.busy":"2023-10-25T18:09:29.448605Z","iopub.execute_input":"2023-10-25T18:09:29.448891Z","iopub.status.idle":"2023-10-25T18:09:29.913101Z","shell.execute_reply.started":"2023-10-25T18:09:29.448867Z","shell.execute_reply":"2023-10-25T18:09:29.911993Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Create test dataset","metadata":{}},{"cell_type":"code","source":"transactions_test['target'] = 1","metadata":{"execution":{"iopub.status.busy":"2023-10-25T18:09:29.914304Z","iopub.execute_input":"2023-10-25T18:09:29.914605Z","iopub.status.idle":"2023-10-25T18:09:29.923257Z","shell.execute_reply.started":"2023-10-25T18:09:29.914581Z","shell.execute_reply":"2023-10-25T18:09:29.922288Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transactions_test_grouped = transactions_test.drop(['article_id'], axis=1).\\\n                                                    merge(transactions[['customer_id', 'article_id']], \\\n                                                    how='left', on='customer_id').\\\n                                                    groupby('customer_id')['article_id'].apply(lambda x: \\\n                                                                                        list(set(x))).reset_index()\nzero_test_target = create_zero_target(transactions_test_grouped)\ntest = pd.concat([transactions_test, zero_test_target])[['customer_id', 'article_id', 'target']].\\\n                                                                    sort_values(by='customer_id').reset_index(drop=True)\ntest","metadata":{"execution":{"iopub.status.busy":"2023-10-25T18:09:29.924902Z","iopub.execute_input":"2023-10-25T18:09:29.925719Z","iopub.status.idle":"2023-10-25T18:10:32.273371Z","shell.execute_reply.started":"2023-10-25T18:09:29.925662Z","shell.execute_reply":"2023-10-25T18:10:32.272123Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_customer_features = np.array([customer_features_with_id[customer_id] for customer_id in \\\n                                   test['customer_id'].values])\ntest_article_features = np.array([articles_features_with_id[article_id] for article_id in \\\n                                  test['article_id'].values])\ntest_target = test['target'].values\ntest_customer_features.shape, test_article_features.shape, test_target.shape","metadata":{"execution":{"iopub.status.busy":"2023-10-25T18:10:32.274569Z","iopub.execute_input":"2023-10-25T18:10:32.274859Z","iopub.status.idle":"2023-10-25T18:10:33.640802Z","shell.execute_reply.started":"2023-10-25T18:10:32.274833Z","shell.execute_reply":"2023-10-25T18:10:33.639707Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Create train dataset","metadata":{}},{"cell_type":"code","source":"transactions_train_grouped = transactions_train.groupby('customer_id')['article_id'].apply(lambda x: len(set(x)))\nselected_customers = transactions_train_grouped[transactions_train_grouped>10].sample(20000, random_state=1).index.values\ntransactions_train = transactions_train[transactions_train['customer_id'].isin(selected_customers)]\ntransactions_train.shape","metadata":{"execution":{"iopub.status.busy":"2023-10-25T18:10:33.641904Z","iopub.execute_input":"2023-10-25T18:10:33.642154Z","iopub.status.idle":"2023-10-25T18:11:35.123601Z","shell.execute_reply.started":"2023-10-25T18:10:33.642132Z","shell.execute_reply":"2023-10-25T18:11:35.122598Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transactions_train['t_dat'] = pd.to_datetime(transactions_train['t_dat'])\ndt_train_max = transactions_train['t_dat'].max()\nrating_dict = {x: 0.5**((dt_train_max - x).days/360) for x in transactions_train['t_dat'].unique()}\ntransactions_train['target'] = transactions_train['t_dat'].map(rating_dict)\ntransactions_train","metadata":{"execution":{"iopub.status.busy":"2023-10-25T18:11:35.125308Z","iopub.execute_input":"2023-10-25T18:11:35.125708Z","iopub.status.idle":"2023-10-25T18:11:35.322713Z","shell.execute_reply.started":"2023-10-25T18:11:35.125671Z","shell.execute_reply":"2023-10-25T18:11:35.321692Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transactions_train_grouped = transactions_train.drop(['article_id'], axis=1).\\\n                                                    merge(transactions[['customer_id', 'article_id']], \\\n                                                    how='left', on='customer_id').\\\n                                                    groupby('customer_id')['article_id'].\\\n                                                    apply(lambda x: list(set(x))).reset_index()\n\nzero_train_target = create_zero_target(transactions_train_grouped)\ntrain = pd.concat([transactions_train, zero_train_target])[['customer_id', 'article_id', 'target']].\\\n                                                                    sort_values(by='customer_id').reset_index(drop=True)\ntrain","metadata":{"execution":{"iopub.status.busy":"2023-10-25T18:11:35.328818Z","iopub.execute_input":"2023-10-25T18:11:35.329189Z","iopub.status.idle":"2023-10-25T18:14:03.845110Z","shell.execute_reply.started":"2023-10-25T18:11:35.329154Z","shell.execute_reply":"2023-10-25T18:14:03.844028Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_customer_features = np.array([customer_features_with_id[customer_id] for customer_id in \\\n                                    train['customer_id'].values])\ntrain_article_features = np.array([articles_features_with_id[article_id] for article_id in \\\n                                   train['article_id'].values])\ntrain_target = train['target'].values\ntrain_customer_features.shape, train_article_features.shape, train_target.shape","metadata":{"execution":{"iopub.status.busy":"2023-10-25T18:14:03.846322Z","iopub.execute_input":"2023-10-25T18:14:03.846691Z","iopub.status.idle":"2023-10-25T18:14:08.527407Z","shell.execute_reply.started":"2023-10-25T18:14:03.846657Z","shell.execute_reply":"2023-10-25T18:14:08.526467Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del transactions_train\ndel transactions_train_grouped\ndel transactions_test\ndel transactions_test_grouped\nimport gc\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-10-25T18:14:08.528829Z","iopub.execute_input":"2023-10-25T18:14:08.529136Z","iopub.status.idle":"2023-10-25T18:14:08.698884Z","shell.execute_reply.started":"2023-10-25T18:14:08.529101Z","shell.execute_reply":"2023-10-25T18:14:08.696400Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Train model","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow import keras\n\nnum_outputs = 32\ntf.random.set_seed(1)\nuser_NN = tf.keras.models.Sequential([    \n    tf.keras.layers.Dense(50, activation='relu'),\n    tf.keras.layers.Dense(40, activation='relu'),\n    tf.keras.layers.Dense(num_outputs) \n])\n\nitem_NN = tf.keras.models.Sequential([   \n    tf.keras.layers.Dense(256, activation='relu'),\n    tf.keras.layers.Dense(128, activation='relu'),\n    tf.keras.layers.Dense(num_outputs)\n])\n\n# create the user input and point to the base network\ninput_user = tf.keras.layers.Input(shape=(train_customer_features.shape[1]))\nvu = user_NN(input_user)\nvu = tf.linalg.l2_normalize(vu, axis=1)\n\n# create the item input and point to the base network\ninput_item = tf.keras.layers.Input(shape=(train_article_features.shape[1]))\nvm = item_NN(input_item)\nvm = tf.linalg.l2_normalize(vm, axis=1)\n\n# compute the dot product of the two vectors vu and vm\noutput = tf.keras.layers.Dot(axes=1)([vu, vm])\n\n# specify the inputs and output of the model\nmodel = tf.keras.Model([input_user, input_item], output)\n\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2023-10-25T18:14:08.700445Z","iopub.execute_input":"2023-10-25T18:14:08.700740Z","iopub.status.idle":"2023-10-25T18:14:13.131651Z","shell.execute_reply.started":"2023-10-25T18:14:08.700718Z","shell.execute_reply":"2023-10-25T18:14:13.130711Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class SaveBestModel(tf.keras.callbacks.Callback):\n    def __init__(self, save_best_metric='val_loss', this_max=False):\n        self.save_best_metric = save_best_metric\n        self.max = this_max\n        if this_max:\n            self.best = float('-inf')\n        else:\n            self.best = float('inf')\n\n    def on_epoch_end(self, epoch, logs=None):\n        metric_value = logs[self.save_best_metric]\n        if self.max:\n            if metric_value > self.best:\n                self.best = metric_value\n                self.best_weights = self.model.get_weights()\n\n        else:\n            if metric_value < self.best:\n                self.best = metric_value\n                self.best_weights= self.model.get_weights()\n                \nsave_best_model = SaveBestModel()","metadata":{"execution":{"iopub.status.busy":"2023-10-25T18:14:13.132778Z","iopub.execute_input":"2023-10-25T18:14:13.133060Z","iopub.status.idle":"2023-10-25T18:14:13.141474Z","shell.execute_reply.started":"2023-10-25T18:14:13.133035Z","shell.execute_reply":"2023-10-25T18:14:13.140400Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.random.set_seed(1)\ncost_fn = tf.keras.losses.MeanSquaredError()\nopt = keras.optimizers.Adam(learning_rate=0.01)\nmodel.compile(optimizer=opt,\n              loss=cost_fn)","metadata":{"execution":{"iopub.status.busy":"2023-10-25T18:14:13.142572Z","iopub.execute_input":"2023-10-25T18:14:13.142819Z","iopub.status.idle":"2023-10-25T18:14:13.180518Z","shell.execute_reply.started":"2023-10-25T18:14:13.142797Z","shell.execute_reply":"2023-10-25T18:14:13.179574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.random.set_seed(1)\nmodel.fit([train_customer_features, train_article_features], train_target, epochs=20, \\\n          validation_data=([test_customer_features, test_article_features], test_target), \\\n          callbacks=[save_best_model])\n#set best weigts\nmodel.set_weights(save_best_model.best_weights)","metadata":{"execution":{"iopub.status.busy":"2023-10-25T18:14:13.181430Z","iopub.execute_input":"2023-10-25T18:14:13.181667Z","iopub.status.idle":"2023-10-25T19:55:42.165037Z","shell.execute_reply.started":"2023-10-25T18:14:13.181646Z","shell.execute_reply":"2023-10-25T19:55:42.163609Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del train_customer_features\ndel train_article_features\ndel train_target\nimport gc\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-10-25T19:55:42.168187Z","iopub.execute_input":"2023-10-25T19:55:42.168500Z","iopub.status.idle":"2023-10-25T19:55:42.658279Z","shell.execute_reply.started":"2023-10-25T19:55:42.168473Z","shell.execute_reply":"2023-10-25T19:55:42.657380Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Predict on single example","metadata":{}},{"cell_type":"code","source":"test_subset = test[test['customer_id']==test['customer_id'].iloc[1]]\ntest_subset.head()","metadata":{"execution":{"iopub.status.busy":"2023-10-25T20:14:08.556378Z","iopub.execute_input":"2023-10-25T20:14:08.557177Z","iopub.status.idle":"2023-10-25T20:14:08.657584Z","shell.execute_reply.started":"2023-10-25T20:14:08.557140Z","shell.execute_reply":"2023-10-25T20:14:08.656425Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_subset_customer_features = np.array([customer_features_with_id[customer_id] for customer_id in \\\n                                   test_subset['customer_id'].values])\ntest_subset_article_features = np.array([articles_features_with_id[article_id] for article_id in \\\n                                  test_subset['article_id'].values])","metadata":{"execution":{"iopub.status.busy":"2023-10-25T19:55:42.762690Z","iopub.execute_input":"2023-10-25T19:55:42.762993Z","iopub.status.idle":"2023-10-25T19:55:42.769588Z","shell.execute_reply.started":"2023-10-25T19:55:42.762965Z","shell.execute_reply":"2023-10-25T19:55:42.768635Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_p = model.predict([test_subset_customer_features, test_subset_article_features])\ny_p[:5]","metadata":{"execution":{"iopub.status.busy":"2023-10-25T20:14:17.075807Z","iopub.execute_input":"2023-10-25T20:14:17.076699Z","iopub.status.idle":"2023-10-25T20:14:17.161625Z","shell.execute_reply.started":"2023-10-25T20:14:17.076665Z","shell.execute_reply":"2023-10-25T20:14:17.160548Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sorted_index = np.argsort(-y_p,axis=0).reshape(-1).tolist()  #negate to get largest rating first\nsorted_y = test_subset['target'].values[sorted_index]\nsorted_y, np.array(sorted_index)","metadata":{"execution":{"iopub.status.busy":"2023-10-25T20:14:37.841828Z","iopub.execute_input":"2023-10-25T20:14:37.842673Z","iopub.status.idle":"2023-10-25T20:14:37.852107Z","shell.execute_reply.started":"2023-10-25T20:14:37.842637Z","shell.execute_reply":"2023-10-25T20:14:37.851071Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Inference","metadata":{}},{"cell_type":"code","source":"predicted_score = model.predict([test_customer_features, test_article_features])","metadata":{"execution":{"iopub.status.busy":"2023-10-25T20:08:51.842061Z","iopub.execute_input":"2023-10-25T20:08:51.842879Z","iopub.status.idle":"2023-10-25T20:09:40.756711Z","shell.execute_reply.started":"2023-10-25T20:08:51.842843Z","shell.execute_reply":"2023-10-25T20:09:40.755671Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test['predicted_score'] = predicted_score.reshape(-1)\ntest_grouped = test.groupby('customer_id').agg({'article_id':list, 'target':list, 'predicted_score':list})\ntest_grouped = test_grouped[test_grouped['target'].apply(lambda x: len(set(x))>1)]\ntest_grouped['predicted_article_id'] = test_grouped[['article_id', 'predicted_score']].\\\n                                    apply(lambda x: list(np.array(x[0])[np.argsort(x[1])[::-1]]), axis=1)\ntest_grouped = test_grouped.reset_index()\ntest_grouped","metadata":{"execution":{"iopub.status.busy":"2023-10-25T20:09:44.984054Z","iopub.execute_input":"2023-10-25T20:09:44.984403Z","iopub.status.idle":"2023-10-25T20:09:46.679321Z","shell.execute_reply.started":"2023-10-25T20:09:44.984377Z","shell.execute_reply":"2023-10-25T20:09:46.678329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Evaluate results","metadata":{}},{"cell_type":"code","source":"def ap_k(actual, predicted, k):\n    \"\"\"\n    Computes the average precision at k.\n    This function computes the average prescision at k between two lists of\n    items.\n    Parameters\n    ----------\n    actual : list\n             A list of elements that are to be predicted (order doesn't matter)\n    predicted : list\n                A list of predicted elements (order does matter)\n    k : int, optional\n        The maximum number of predicted elements\n    Returns\n    -------\n    score : double\n            The average precision at k over the input lists\n    \"\"\"\n    if len(predicted)>k:\n        predicted = predicted[:k]\n\n    score = 0.0\n    num_hits = 0.0\n\n    for i,p in enumerate(predicted):\n        if p in actual and p not in predicted[:i]:\n            num_hits += 1.0\n            score += num_hits / (i+1.0)\n\n    if not actual:\n        return 0.0\n\n    return score / min(len(actual), k)\n\ndef ndcg_k(y_true, y_pred, k):\n    dcg = 0\n    N = len(y_true)\n    if N == 0:\n        return 0\n    for idx, item in enumerate(y_pred[:k], start=1):\n        dcg += (2 ** (item in y_true) - 1) / np.log2(1 + idx)\n    idcg = sum(1 / np.log2(1 + i) for i in range(1, N + 1))\n    return dcg / idcg\n\ndef measure_rank_metrics(test_set, k):\n    \n    group_cols = ['customer_id']\n    explode_cols = ['article_id', 'target']\n    data_full = test_set[group_cols + explode_cols +['predicted_article_id']].explode(explode_cols)\n    data = data_full.sort_values(group_cols, ascending=False)\n    # для сокращения числа строк\n    data[f'rn'] = data.groupby(group_cols).cumcount()+1\n    data = data[(data.rn <= k) | (data.target == 1)]\n\n    data_group = data.groupby(group_cols)\n    map_all = round(data_group.apply(\n                lambda x: ap_k(x.loc[x['target'] == 1, 'article_id'].tolist(), \n                              x['predicted_article_id'].values[0][:k], k)).values.mean(), 5)\n\n    ndcg_all = round(data_group.apply(\n                lambda x: ndcg_k(x.loc[x['target'] == 1, 'article_id'].tolist(), \n                               x['predicted_article_id'].values[0][:k], k)).values.mean(), 5)\n    \n    return map_all, ndcg_all\n\ndef measure_metrics(test_set, k=10):\n    metrics = {}\n    map_all, ndcg_all = measure_rank_metrics(test_set, k)\n    metrics['map_all'] = map_all\n    metrics['ndcg_all'] = ndcg_all\n    \n    return metrics","metadata":{"execution":{"iopub.status.busy":"2023-10-25T20:09:58.148259Z","iopub.execute_input":"2023-10-25T20:09:58.149196Z","iopub.status.idle":"2023-10-25T20:09:58.164313Z","shell.execute_reply.started":"2023-10-25T20:09:58.149159Z","shell.execute_reply":"2023-10-25T20:09:58.163053Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"measure_metrics(test_grouped, 12)","metadata":{"execution":{"iopub.status.busy":"2023-10-25T20:10:00.352191Z","iopub.execute_input":"2023-10-25T20:10:00.352568Z","iopub.status.idle":"2023-10-25T20:10:10.445802Z","shell.execute_reply.started":"2023-10-25T20:10:00.352539Z","shell.execute_reply":"2023-10-25T20:10:10.444780Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"measure_metrics(test_grouped, 20)","metadata":{"execution":{"iopub.status.busy":"2023-10-25T20:10:10.448113Z","iopub.execute_input":"2023-10-25T20:10:10.448535Z","iopub.status.idle":"2023-10-25T20:10:20.542511Z","shell.execute_reply.started":"2023-10-25T20:10:10.448500Z","shell.execute_reply":"2023-10-25T20:10:20.541386Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"measure_metrics(test_grouped, 30)","metadata":{"execution":{"iopub.status.busy":"2023-10-25T20:10:20.543670Z","iopub.execute_input":"2023-10-25T20:10:20.543970Z","iopub.status.idle":"2023-10-25T20:10:31.637792Z","shell.execute_reply.started":"2023-10-25T20:10:20.543944Z","shell.execute_reply":"2023-10-25T20:10:31.636844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del test_customer_features\ndel test_article_features\ndel test_target\nimport gc\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-10-25T20:12:18.617272Z","iopub.execute_input":"2023-10-25T20:12:18.618159Z","iopub.status.idle":"2023-10-25T20:12:18.916534Z","shell.execute_reply.started":"2023-10-25T20:12:18.618124Z","shell.execute_reply":"2023-10-25T20:12:18.915232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Create placeholder for new customers (popular goods)","metadata":{}},{"cell_type":"code","source":"# get last 30 days\nstart_popular_date = datetime.strptime(max_date, '%Y-%M-%d') - timedelta(days=30)\nstart_popular_date = start_popular_date.strftime('%Y-%M-%d')\nstart_popular_date","metadata":{"execution":{"iopub.status.busy":"2023-10-25T20:10:37.004431Z","iopub.execute_input":"2023-10-25T20:10:37.005395Z","iopub.status.idle":"2023-10-25T20:10:37.013313Z","shell.execute_reply.started":"2023-10-25T20:10:37.005334Z","shell.execute_reply":"2023-10-25T20:10:37.012385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transactions_popular = transactions[transactions['t_dat']>=start_popular_date]\npopular_goods = transactions_popular['article_id'].value_counts().head(12).index.tolist()\npopular_goods = ' '.join([str(item) for item in popular_goods])\npopular_goods","metadata":{"execution":{"iopub.status.busy":"2023-10-25T20:11:36.024963Z","iopub.execute_input":"2023-10-25T20:11:36.025216Z","iopub.status.idle":"2023-10-25T20:11:41.122963Z","shell.execute_reply.started":"2023-10-25T20:11:36.025194Z","shell.execute_reply":"2023-10-25T20:11:41.121878Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Predict on sample_submission","metadata":{}},{"cell_type":"code","source":"sample_submission = pd.read_csv('/kaggle/input/h-and-m-personalized-fashion-recommendations/sample_submission.csv')\nsample_submission","metadata":{"execution":{"iopub.status.busy":"2023-10-25T20:12:27.503595Z","iopub.execute_input":"2023-10-25T20:12:27.504323Z","iopub.status.idle":"2023-10-25T20:12:33.050826Z","shell.execute_reply.started":"2023-10-25T20:12:27.504291Z","shell.execute_reply":"2023-10-25T20:12:33.049754Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"known_sample_submission = sample_submission[sample_submission['customer_id'].isin(customer_features_with_id.keys())]\nunknown_sample_submission = sample_submission[~sample_submission['customer_id'].isin(customer_features_with_id.keys())]\nknown_sample_submission.shape, unknown_sample_submission.shape","metadata":{"execution":{"iopub.status.busy":"2023-10-25T20:12:34.660865Z","iopub.execute_input":"2023-10-25T20:12:34.661244Z","iopub.status.idle":"2023-10-25T20:12:36.511807Z","shell.execute_reply.started":"2023-10-25T20:12:34.661213Z","shell.execute_reply":"2023-10-25T20:12:36.510373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"As we can see, all customers are known","metadata":{}},{"cell_type":"markdown","source":"Let's predict on a small chunk","metadata":{}},{"cell_type":"code","source":"import time\nsample_article = list(articles_features_with_id.keys())\nlen_sample_article = len(sample_article)\n\n# split sample submission to not run out of the memory\ndf_subsets = []\nfor df in np.array_split(sample_submission, 100000):\n    start_time = time.time()\n    df_subset = []\n    for customer_id in tqdm(df['customer_id'], total=len(df)):\n        df_subset.append(pd.DataFrame({'customer_id':[customer_id]*len_sample_article, 'article_id':sample_article}))\n    df_subset = pd.concat(df_subset, ignore_index=True)\n    print('df_subset shape', df_subset.shape)\n    subset_customer_features = np.array([customer_features_with_id[customer_id] for customer_id in \\\n                                        df_subset['customer_id'].values])\n    print('subset_customer_features', subset_customer_features.shape)\n    subset_article_features = np.array([articles_features_with_id[article_id] for article_id in \\\n                                       df_subset['article_id'].values])\n    print('subset_article_features', subset_article_features.shape)\n    predicted_score = model.predict([subset_customer_features, subset_article_features])\n    \n    df_subset['predicted_score'] = predicted_score.reshape(-1)\n    df_subset_grouped = df_subset.groupby('customer_id').agg({'article_id':list,'predicted_score':list})\n    df_subset_grouped['prediction'] = df_subset_grouped[['article_id', 'predicted_score']].\\\n                                        apply(lambda x: list(np.array(x[0])[np.argsort(x[1])[::-1]]), axis=1)\n    df_subset_grouped['prediction'] = df_subset_grouped['prediction'].\\\n                                                    apply(lambda x: ' '.join(str(item) for item in x[:12]))\n    df_subset_grouped = df_subset_grouped.reset_index()[['customer_id', 'prediction']]\n    df_subsets.append(df_subset_grouped)\n    end_time = time.time() - start_time\n    break\n    \nend_time","metadata":{"execution":{"iopub.status.busy":"2023-10-25T20:15:19.074423Z","iopub.execute_input":"2023-10-25T20:15:19.075086Z","iopub.status.idle":"2023-10-25T20:17:53.258135Z","shell.execute_reply.started":"2023-10-25T20:15:19.075052Z","shell.execute_reply":"2023-10-25T20:17:53.257195Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_subset_grouped","metadata":{"execution":{"iopub.status.busy":"2023-10-25T20:18:02.197860Z","iopub.execute_input":"2023-10-25T20:18:02.198248Z","iopub.status.idle":"2023-10-25T20:18:02.209711Z","shell.execute_reply.started":"2023-10-25T20:18:02.198217Z","shell.execute_reply":"2023-10-25T20:18:02.208538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"or","metadata":{}},{"cell_type":"code","source":"def inference(customer_id):\n    if not customer_id in customer_features_with_id.keys():\n        return popular_goods\n    else:\n        sample_article_features = np.array(list(articles_features_with_id.values()))\n        len_of_article_features = sample_article_features.shape[0]\n        # repeat user vector\n        sample_customer_features = np.array(list(np.array(customer_features_with_id[customer_id]).reshape(1, -1))*\\\n                                     len_of_article_features).reshape(len_of_article_features, \\\n                                                                customer_features_with_id[customer_id].shape[0])\n        \n        y_p = model.predict([sample_customer_features, sample_article_features]) # get rating\n        sorted_index = np.argsort(-y_p,axis=0).reshape(-1).tolist() # negate to get largest rating first\n        sorted_items = np.array(list(articles_features_with_id.keys()))[sorted_index][:12] # sort articles by rating\n        sorted_items = ' '.join(str(item) for item in sorted_items)\n        return sorted_items\n\nsample_submission['prediction'] = sample_submission['customer_id'].apply(inference)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}