{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"**In this notebook I am using embedding of product descriptions to find product similarity**\n\nThis method might give some redundant results becuase many items have the same descriptions. But it might be useful in a hybrid recommendation system or for comparison of different models.\n\n* Embeddings are produced by 'universal-sentence-encoder' found on TensorFlow Hub\n* Distance metric used is dot product\n\n","metadata":{"papermill":{"duration":0.00778,"end_time":"2023-01-30T15:28:54.945711","exception":false,"start_time":"2023-01-30T15:28:54.937931","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import tensorflow_hub as hub\nimport numpy as np\nimport pandas as pd\nimport pickle\nimport warnings\nimport matplotlib.pyplot as plt\nwarnings.filterwarnings('ignore')\n\npath = '../input/h-and-m-personalized-fashion-recommendations/articles.csv'\n \ndf = pd.read_csv(path).astype(str)","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.execute_input":"2023-01-30T15:28:54.961305Z","iopub.status.busy":"2023-01-30T15:28:54.960793Z","iopub.status.idle":"2023-01-30T15:29:06.023371Z","shell.execute_reply":"2023-01-30T15:29:06.021738Z"},"papermill":{"duration":11.074494,"end_time":"2023-01-30T15:29:06.027012","exception":false,"start_time":"2023-01-30T15:28:54.952518","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#@title Load the Universal Sentence Encoder's TF Hub module\n\nmodule_url = \"https://tfhub.dev/google/universal-sentence-encoder/4\" #@param [\"https://tfhub.dev/google/universal-sentence-encoder/4\", \"https://tfhub.dev/google/universal-sentence-encoder-large/5\"]\nmodel = hub.load(module_url)","metadata":{"execution":{"iopub.execute_input":"2023-01-30T15:29:06.043213Z","iopub.status.busy":"2023-01-30T15:29:06.042639Z","iopub.status.idle":"2023-01-30T15:29:46.159708Z","shell.execute_reply":"2023-01-30T15:29:46.158502Z"},"papermill":{"duration":40.128734,"end_time":"2023-01-30T15:29:46.162898","exception":false,"start_time":"2023-01-30T15:29:06.034164","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"desc = df['detail_desc'].unique()","metadata":{"execution":{"iopub.execute_input":"2023-01-30T15:29:46.180106Z","iopub.status.busy":"2023-01-30T15:29:46.179186Z","iopub.status.idle":"2023-01-30T15:29:46.245693Z","shell.execute_reply":"2023-01-30T15:29:46.243316Z"},"papermill":{"duration":0.079081,"end_time":"2023-01-30T15:29:46.249025","exception":false,"start_time":"2023-01-30T15:29:46.169944","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"desc","metadata":{"execution":{"iopub.execute_input":"2023-01-30T15:29:46.265324Z","iopub.status.busy":"2023-01-30T15:29:46.264874Z","iopub.status.idle":"2023-01-30T15:29:46.277190Z","shell.execute_reply":"2023-01-30T15:29:46.275573Z"},"papermill":{"duration":0.025187,"end_time":"2023-01-30T15:29:46.281218","exception":false,"start_time":"2023-01-30T15:29:46.256031","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"embeds = model(desc)","metadata":{"execution":{"iopub.execute_input":"2023-01-30T15:29:46.297902Z","iopub.status.busy":"2023-01-30T15:29:46.297341Z","iopub.status.idle":"2023-01-30T15:29:58.429212Z","shell.execute_reply":"2023-01-30T15:29:58.428114Z"},"papermill":{"duration":12.143519,"end_time":"2023-01-30T15:29:58.432297","exception":false,"start_time":"2023-01-30T15:29:46.288778","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open('text_desc_embeddings.pickle', 'wb') as f:\n    pickle.dump(embeds, f)\n    \nwith open('text_desc.pickle', 'wb') as f:\n    pickle.dump(desc, f)","metadata":{"execution":{"iopub.execute_input":"2023-01-30T15:29:58.448451Z","iopub.status.busy":"2023-01-30T15:29:58.447889Z","iopub.status.idle":"2023-01-30T15:29:58.758458Z","shell.execute_reply":"2023-01-30T15:29:58.756875Z"},"papermill":{"duration":0.32319,"end_time":"2023-01-30T15:29:58.762432","exception":false,"start_time":"2023-01-30T15:29:58.439242","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"file = open('scores_top20.pkl','wb')\n\nfor embed in embeds: \n    top10 = np.inner(embed, embeds)\n    top10_index = np.argsort(-top10)[:20]\n    top10_score = top10[top10_index]\n\n    pickle.dump([top10_index, top10_score], file)\n\nfile.close()","metadata":{"execution":{"iopub.execute_input":"2023-01-30T15:29:58.780785Z","iopub.status.busy":"2023-01-30T15:29:58.778904Z","iopub.status.idle":"2023-01-30T15:40:55.783645Z","shell.execute_reply":"2023-01-30T15:40:55.781312Z"},"papermill":{"duration":657.0192,"end_time":"2023-01-30T15:40:55.789490","exception":false,"start_time":"2023-01-30T15:29:58.770290","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_top10():\n    \n    top10_indecies = []\n    top10_scores = []\n\n    with open('../input/product-desc-similarity-scores/scores_top10.pkl', 'rb') as f:\n        for i in range(len(desc)):\n            try:\n                row = pickle.load(f)\n                top10_indecies.append(row[0])\n                top10_scores.append(row[1])\n            except:\n                print('Done Loading')\n                \n    return top10_indecies, top10_scores","metadata":{"execution":{"iopub.execute_input":"2023-01-30T15:40:55.826538Z","iopub.status.busy":"2023-01-30T15:40:55.825637Z","iopub.status.idle":"2023-01-30T15:40:55.844590Z","shell.execute_reply":"2023-01-30T15:40:55.842591Z"},"papermill":{"duration":0.044104,"end_time":"2023-01-30T15:40:55.850622","exception":false,"start_time":"2023-01-30T15:40:55.806518","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"top10_indecies, top10_scores = load_top10()","metadata":{"execution":{"iopub.execute_input":"2023-01-30T15:40:55.888574Z","iopub.status.busy":"2023-01-30T15:40:55.887645Z","iopub.status.idle":"2023-01-30T15:40:58.162219Z","shell.execute_reply":"2023-01-30T15:40:58.160902Z"},"papermill":{"duration":2.297485,"end_time":"2023-01-30T15:40:58.165560","exception":false,"start_time":"2023-01-30T15:40:55.868075","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def recommend(article_id):\n    \n    desc_list = desc.tolist()\n    product_desc = df[df.article_id == sample]['detail_desc'].values[0]\n    desc_index = desc_list.index(product_desc)\n    rcmnds_indecies = top10_indecies[desc_index]\n    rcmnds_scores = top10_scores[desc_index]\n    rcmnds_descs = desc[rcmnds_indecies]\n    map_dict = {i:j for i, j in zip(rcmnds_descs, rcmnds_scores)}\n    rcmnds_article_ids = df[df.detail_desc.isin(rcmnds_descs)]\n    rcmnds_article_ids['score'] = rcmnds_article_ids.detail_desc.map(map_dict)\n    rcmnds_article_ids = rcmnds_article_ids[rcmnds_article_ids.score < 0.99]\n    rcmnds_article_ids = rcmnds_article_ids.sort_values(by='score', ascending=False).drop_duplicates('score')\n    \n    \n    return(rcmnds_article_ids[['article_id', 'score']])","metadata":{"execution":{"iopub.execute_input":"2023-01-30T15:40:58.181554Z","iopub.status.busy":"2023-01-30T15:40:58.181049Z","iopub.status.idle":"2023-01-30T15:40:58.191321Z","shell.execute_reply":"2023-01-30T15:40:58.189387Z"},"papermill":{"duration":0.021756,"end_time":"2023-01-30T15:40:58.194379","exception":false,"start_time":"2023-01-30T15:40:58.172623","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_items(items):\n    path = \"../input/h-and-m-personalized-fashion-recommendations/images\"\n\n    k = len(items)\n    fig = plt.figure(figsize=(15, 10))\n    for item, i in zip(items, range(1, k+1)):\n        item = '0' + str(item)\n        sub = item[:3]\n        image = path + \"/\"+ sub + \"/\"+ item +\".jpg\"\n        image = plt.imread(image)\n        fig.add_subplot(1, k, i)\n        plt.imshow(image)\n        ","metadata":{"execution":{"iopub.execute_input":"2023-01-30T15:40:58.211151Z","iopub.status.busy":"2023-01-30T15:40:58.210644Z","iopub.status.idle":"2023-01-30T15:40:58.218906Z","shell.execute_reply":"2023-01-30T15:40:58.217499Z"},"papermill":{"duration":0.02016,"end_time":"2023-01-30T15:40:58.222205","exception":false,"start_time":"2023-01-30T15:40:58.202045","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample = df.article_id.iloc[1]\nrcmnds = recommend(sample)","metadata":{"execution":{"iopub.execute_input":"2023-01-30T15:40:58.239303Z","iopub.status.busy":"2023-01-30T15:40:58.238808Z","iopub.status.idle":"2023-01-30T15:40:58.424455Z","shell.execute_reply":"2023-01-30T15:40:58.423362Z"},"papermill":{"duration":0.197371,"end_time":"2023-01-30T15:40:58.427496","exception":false,"start_time":"2023-01-30T15:40:58.230125","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_items(rcmnds.sample(6).article_id.values)","metadata":{"execution":{"iopub.execute_input":"2023-01-30T15:40:58.443470Z","iopub.status.busy":"2023-01-30T15:40:58.442950Z","iopub.status.idle":"2023-01-30T15:41:01.145280Z","shell.execute_reply":"2023-01-30T15:41:01.144063Z"},"papermill":{"duration":2.715922,"end_time":"2023-01-30T15:41:01.150218","exception":false,"start_time":"2023-01-30T15:40:58.434296","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trans = next(pd.read_csv('../input/h-and-m-personalized-fashion-recommendations/transactions_train.csv', dtype=str, chunksize=10000))\n\ntrans.drop_duplicates(['customer_id', 'article_id'], inplace=True)\ntrans.article_id = trans.article_id.map(lambda x: x[1:])\ngrouped = trans.groupby('customer_id')","metadata":{"execution":{"iopub.execute_input":"2023-01-30T15:41:01.172463Z","iopub.status.busy":"2023-01-30T15:41:01.171899Z","iopub.status.idle":"2023-01-30T15:41:01.239049Z","shell.execute_reply":"2023-01-30T15:41:01.237868Z"},"papermill":{"duration":0.0825,"end_time":"2023-01-30T15:41:01.242323","exception":false,"start_time":"2023-01-30T15:41:01.159823","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"new = []\nfor group in grouped.groups:\n    temp = grouped.get_group(group)\n    if len(temp) >= 12:\n        new.append([group, temp.article_id.values.tolist()[:12]])","metadata":{"execution":{"iopub.execute_input":"2023-01-30T15:41:01.264728Z","iopub.status.busy":"2023-01-30T15:41:01.264195Z","iopub.status.idle":"2023-01-30T15:41:01.545864Z","shell.execute_reply":"2023-01-30T15:41:01.544522Z"},"papermill":{"duration":0.296884,"end_time":"2023-01-30T15:41:01.549184","exception":false,"start_time":"2023-01-30T15:41:01.252300","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_sim(item, items):\n    \n    item_desc = df.detail_desc[df.article_id == item].values[0]\n    item_embed = model([item_desc])[0]\n    \n    items_desc = df[df.article_id.isin(items)].detail_desc\n    items_embed = model(items_desc)\n    scores = []\n    \n    for i in items_embed:\n        sim = np.dot(i, item_embed)\n        scores.append(sim)\n        \n        \n    return np.mean(scores)","metadata":{"execution":{"iopub.execute_input":"2023-01-30T15:41:01.571472Z","iopub.status.busy":"2023-01-30T15:41:01.570973Z","iopub.status.idle":"2023-01-30T15:41:01.579094Z","shell.execute_reply":"2023-01-30T15:41:01.577517Z"},"papermill":{"duration":0.022806,"end_time":"2023-01-30T15:41:01.581869","exception":false,"start_time":"2023-01-30T15:41:01.559063","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = []\nscores = []\n\nfor key, value in new:\n    temp = []\n    for item in value[:6]:\n        temp.append(recommend(item))\n    temp2 = pd.concat(temp).sample(6, random_state=42)\n    temp2['actual'] = value[6:]\n    temp3 = []\n    for item in temp2.actual:\n        sim = get_sim(item, temp2.article_id) \n        temp3.append(sim)\n        \n    scores.append(np.mean(temp3))\n    preds.append([key, temp2])","metadata":{"execution":{"iopub.execute_input":"2023-01-30T15:41:01.602945Z","iopub.status.busy":"2023-01-30T15:41:01.602472Z","iopub.status.idle":"2023-01-30T15:41:21.183927Z","shell.execute_reply":"2023-01-30T15:41:21.182082Z"},"papermill":{"duration":19.596341,"end_time":"2023-01-30T15:41:21.187480","exception":false,"start_time":"2023-01-30T15:41:01.591139","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.mean(scores)","metadata":{"execution":{"iopub.execute_input":"2023-01-30T15:41:21.208991Z","iopub.status.busy":"2023-01-30T15:41:21.208473Z","iopub.status.idle":"2023-01-30T15:41:21.218078Z","shell.execute_reply":"2023-01-30T15:41:21.216876Z"},"papermill":{"duration":0.023325,"end_time":"2023-01-30T15:41:21.220629","exception":false,"start_time":"2023-01-30T15:41:21.197304","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"customer = preds[0][1]","metadata":{"execution":{"iopub.execute_input":"2023-01-30T15:41:21.243311Z","iopub.status.busy":"2023-01-30T15:41:21.241522Z","iopub.status.idle":"2023-01-30T15:41:21.249031Z","shell.execute_reply":"2023-01-30T15:41:21.247703Z"},"papermill":{"duration":0.021718,"end_time":"2023-01-30T15:41:21.251935","exception":false,"start_time":"2023-01-30T15:41:21.230217","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"customer","metadata":{"execution":{"iopub.execute_input":"2023-01-30T15:41:21.273009Z","iopub.status.busy":"2023-01-30T15:41:21.272474Z","iopub.status.idle":"2023-01-30T15:41:21.292903Z","shell.execute_reply":"2023-01-30T15:41:21.291445Z"},"papermill":{"duration":0.034656,"end_time":"2023-01-30T15:41:21.295889","exception":false,"start_time":"2023-01-30T15:41:21.261233","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_items(customer.article_id.values)","metadata":{"execution":{"iopub.execute_input":"2023-01-30T15:41:21.318878Z","iopub.status.busy":"2023-01-30T15:41:21.318391Z","iopub.status.idle":"2023-01-30T15:41:23.959157Z","shell.execute_reply":"2023-01-30T15:41:23.957649Z"},"papermill":{"duration":2.657201,"end_time":"2023-01-30T15:41:23.963422","exception":false,"start_time":"2023-01-30T15:41:21.306221","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_items(customer.actual.values)","metadata":{"execution":{"iopub.execute_input":"2023-01-30T15:41:23.990432Z","iopub.status.busy":"2023-01-30T15:41:23.989924Z","iopub.status.idle":"2023-01-30T15:41:26.678934Z","shell.execute_reply":"2023-01-30T15:41:26.677425Z"},"papermill":{"duration":2.708291,"end_time":"2023-01-30T15:41:26.684238","exception":false,"start_time":"2023-01-30T15:41:23.975947","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]}]}