{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"**In this notebook I am using embedding of product descriptions to find product similarity**\n\nThis method might give some redundant results becuase many items have the same descriptions. But it might be useful in a hybrid recommendation system or for comparison of different models.\n\n* Embeddings are produced by 'universal-sentence-encoder' found on TensorFlow Hub\n* Distance metric used is dot product\n\n","metadata":{"papermill":{"duration":0.025241,"end_time":"2022-05-15T08:28:29.092139","exception":false,"start_time":"2022-05-15T08:28:29.066898","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import tensorflow_hub as hub\nimport numpy as np\nimport pandas as pd\nimport pickle\nimport warnings\nimport matplotlib.pyplot as plt\nwarnings.filterwarnings('ignore')\n\npath = '../input/h-and-m-personalized-fashion-recommendations/articles.csv'\n \ndf = pd.read_csv(path).astype(str)","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":10.357962,"end_time":"2022-05-15T08:28:39.481531","exception":false,"start_time":"2022-05-15T08:28:29.123569","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-01-30T15:13:58.248672Z","iopub.execute_input":"2023-01-30T15:13:58.249140Z","iopub.status.idle":"2023-01-30T15:14:08.478576Z","shell.execute_reply.started":"2023-01-30T15:13:58.249046Z","shell.execute_reply":"2023-01-30T15:14:08.477414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#@title Load the Universal Sentence Encoder's TF Hub module\n\nmodule_url = \"https://tfhub.dev/google/universal-sentence-encoder/4\" #@param [\"https://tfhub.dev/google/universal-sentence-encoder/4\", \"https://tfhub.dev/google/universal-sentence-encoder-large/5\"]\nmodel = hub.load(module_url)","metadata":{"papermill":{"duration":16.455168,"end_time":"2022-05-15T08:28:55.956525","exception":false,"start_time":"2022-05-15T08:28:39.501357","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-01-30T15:14:08.480519Z","iopub.execute_input":"2023-01-30T15:14:08.480861Z","iopub.status.idle":"2023-01-30T15:14:38.370896Z","shell.execute_reply.started":"2023-01-30T15:14:08.480831Z","shell.execute_reply":"2023-01-30T15:14:38.369715Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"desc = df['detail_desc'].unique()","metadata":{"papermill":{"duration":0.082307,"end_time":"2022-05-15T08:28:56.060792","exception":false,"start_time":"2022-05-15T08:28:55.978485","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-01-30T15:14:38.372315Z","iopub.execute_input":"2023-01-30T15:14:38.372657Z","iopub.status.idle":"2023-01-30T15:14:38.430247Z","shell.execute_reply.started":"2023-01-30T15:14:38.372626Z","shell.execute_reply":"2023-01-30T15:14:38.429112Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"desc","metadata":{"papermill":{"duration":0.032858,"end_time":"2022-05-15T08:28:56.114290","exception":false,"start_time":"2022-05-15T08:28:56.081432","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-01-30T15:14:38.433767Z","iopub.execute_input":"2023-01-30T15:14:38.434364Z","iopub.status.idle":"2023-01-30T15:14:38.445363Z","shell.execute_reply.started":"2023-01-30T15:14:38.434323Z","shell.execute_reply":"2023-01-30T15:14:38.443927Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"embeds = model(desc)","metadata":{"papermill":{"duration":8.345826,"end_time":"2022-05-15T08:29:04.482778","exception":false,"start_time":"2022-05-15T08:28:56.136952","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-01-30T15:14:38.447148Z","iopub.execute_input":"2023-01-30T15:14:38.447692Z","iopub.status.idle":"2023-01-30T15:14:48.467321Z","shell.execute_reply.started":"2023-01-30T15:14:38.447646Z","shell.execute_reply":"2023-01-30T15:14:48.466235Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open('text_desc_embeddings.pickle', 'wb') as f:\n    pickle.dump(embeds, f)\n    \nwith open('text_desc.pickle', 'wb') as f:\n    pickle.dump(desc, f)","metadata":{"papermill":{"duration":0.148086,"end_time":"2022-05-15T08:29:04.652118","exception":false,"start_time":"2022-05-15T08:29:04.504032","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-01-30T15:14:48.468901Z","iopub.execute_input":"2023-01-30T15:14:48.469390Z","iopub.status.idle":"2023-01-30T15:14:48.685503Z","shell.execute_reply.started":"2023-01-30T15:14:48.469347Z","shell.execute_reply":"2023-01-30T15:14:48.684233Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"file = open('scores_top20.pkl','wb')\n\nfor embed in embeds: \n    top10 = np.inner(embed, embeds)\n    top10_index = np.argsort(-top10)[:20]\n    top10_score = top10[top10_index]\n\n    pickle.dump([top10_index, top10_score], file)\n\nfile.close()","metadata":{"papermill":{"duration":392.68928,"end_time":"2022-05-15T08:35:37.362224","exception":false,"start_time":"2022-05-15T08:29:04.672944","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-01-30T15:14:48.687025Z","iopub.execute_input":"2023-01-30T15:14:48.687565Z","iopub.status.idle":"2023-01-30T15:22:33.788815Z","shell.execute_reply.started":"2023-01-30T15:14:48.687517Z","shell.execute_reply":"2023-01-30T15:22:33.787511Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_top10():\n    \n    top10_indecies = []\n    top10_scores = []\n\n    with open('../input/product-desc-similarity-scores/scores_top10.pkl', 'rb') as f:\n        for i in range(len(desc)):\n            try:\n                row = pickle.load(f)\n                top10_indecies.append(row[0])\n                top10_scores.append(row[1])\n            except:\n                print('Done Loading')\n                \n    return top10_indecies, top10_scores","metadata":{"papermill":{"duration":0.053348,"end_time":"2022-05-15T08:35:37.460562","exception":false,"start_time":"2022-05-15T08:35:37.407214","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-01-30T15:26:14.561408Z","iopub.execute_input":"2023-01-30T15:26:14.561888Z","iopub.status.idle":"2023-01-30T15:26:14.568977Z","shell.execute_reply.started":"2023-01-30T15:26:14.561848Z","shell.execute_reply":"2023-01-30T15:26:14.567748Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"top10_indecies, top10_scores = load_top10()","metadata":{"papermill":{"duration":1.479513,"end_time":"2022-05-15T08:35:38.962286","exception":false,"start_time":"2022-05-15T08:35:37.482773","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-01-30T15:26:42.431107Z","iopub.execute_input":"2023-01-30T15:26:42.431798Z","iopub.status.idle":"2023-01-30T15:26:44.522866Z","shell.execute_reply.started":"2023-01-30T15:26:42.431757Z","shell.execute_reply":"2023-01-30T15:26:44.521635Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def recommend(article_id):\n    \n    desc_list = desc.tolist()\n    product_desc = df[df.article_id == sample]['detail_desc'].values[0]\n    desc_index = desc_list.index(product_desc)\n    rcmnds_indecies = top10_indecies[desc_index]\n    rcmnds_scores = top10_scores[desc_index]\n    rcmnds_descs = desc[rcmnds_indecies]\n    map_dict = {i:j for i, j in zip(rcmnds_descs, rcmnds_scores)}\n    rcmnds_article_ids = df[df.detail_desc.isin(rcmnds_descs)]\n    rcmnds_article_ids['score'] = rcmnds_article_ids.detail_desc.map(map_dict)\n    rcmnds_article_ids = rcmnds_article_ids[rcmnds_article_ids.score < 0.99]\n    rcmnds_article_ids = rcmnds_article_ids.sort_values(by='score', ascending=False).drop_duplicates('score')\n    \n    \n    return(rcmnds_article_ids[['article_id', 'score']])","metadata":{"papermill":{"duration":0.03431,"end_time":"2022-05-15T08:35:39.019645","exception":false,"start_time":"2022-05-15T08:35:38.985335","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-01-30T15:26:50.458789Z","iopub.execute_input":"2023-01-30T15:26:50.459235Z","iopub.status.idle":"2023-01-30T15:26:50.467909Z","shell.execute_reply.started":"2023-01-30T15:26:50.459199Z","shell.execute_reply":"2023-01-30T15:26:50.466535Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_items(items):\n    path = \"../input/h-and-m-personalized-fashion-recommendations/images\"\n\n    k = len(items)\n    fig = plt.figure(figsize=(15, 10))\n    for item, i in zip(items, range(1, k+1)):\n        item = '0' + str(item)\n        sub = item[:3]\n        image = path + \"/\"+ sub + \"/\"+ item +\".jpg\"\n        image = plt.imread(image)\n        fig.add_subplot(1, k, i)\n        plt.imshow(image)\n        ","metadata":{"papermill":{"duration":0.042773,"end_time":"2022-05-15T08:35:39.094041","exception":false,"start_time":"2022-05-15T08:35:39.051268","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-01-30T15:26:58.961819Z","iopub.execute_input":"2023-01-30T15:26:58.962220Z","iopub.status.idle":"2023-01-30T15:26:58.969693Z","shell.execute_reply.started":"2023-01-30T15:26:58.962188Z","shell.execute_reply":"2023-01-30T15:26:58.968318Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample = df.article_id.iloc[1]\nrcmnds = recommend(sample)","metadata":{"papermill":{"duration":0.222083,"end_time":"2022-05-15T08:35:39.349258","exception":false,"start_time":"2022-05-15T08:35:39.127175","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-01-30T15:27:02.875700Z","iopub.execute_input":"2023-01-30T15:27:02.876130Z","iopub.status.idle":"2023-01-30T15:27:03.042843Z","shell.execute_reply.started":"2023-01-30T15:27:02.876094Z","shell.execute_reply":"2023-01-30T15:27:03.041691Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_items(rcmnds.sample(6).article_id.values)","metadata":{"papermill":{"duration":2.493807,"end_time":"2022-05-15T08:35:41.863863","exception":false,"start_time":"2022-05-15T08:35:39.370056","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-01-30T15:27:06.062656Z","iopub.execute_input":"2023-01-30T15:27:06.063048Z","iopub.status.idle":"2023-01-30T15:27:08.784068Z","shell.execute_reply.started":"2023-01-30T15:27:06.063014Z","shell.execute_reply":"2023-01-30T15:27:08.783138Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trans = next(pd.read_csv('../input/h-and-m-personalized-fashion-recommendations/transactions_train.csv', dtype=str, chunksize=10000))\n\ntrans.drop_duplicates(['customer_id', 'article_id'], inplace=True)\ntrans.article_id = trans.article_id.map(lambda x: x[1:])\ngrouped = trans.groupby('customer_id')","metadata":{"papermill":{"duration":0.084577,"end_time":"2022-05-15T08:35:41.984383","exception":false,"start_time":"2022-05-15T08:35:41.899806","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-01-30T15:27:13.678401Z","iopub.execute_input":"2023-01-30T15:27:13.678810Z","iopub.status.idle":"2023-01-30T15:27:13.771073Z","shell.execute_reply.started":"2023-01-30T15:27:13.678776Z","shell.execute_reply":"2023-01-30T15:27:13.769801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"new = []\nfor group in grouped.groups:\n    temp = grouped.get_group(group)\n    if len(temp) >= 12:\n        new.append([group, temp.article_id.values.tolist()[:12]])","metadata":{"papermill":{"duration":0.306076,"end_time":"2022-05-15T08:35:42.315610","exception":false,"start_time":"2022-05-15T08:35:42.009534","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-01-30T15:27:17.407937Z","iopub.execute_input":"2023-01-30T15:27:17.409332Z","iopub.status.idle":"2023-01-30T15:27:17.765687Z","shell.execute_reply.started":"2023-01-30T15:27:17.409274Z","shell.execute_reply":"2023-01-30T15:27:17.764334Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_sim(item, items):\n    \n    item_desc = df.detail_desc[df.article_id == item].values[0]\n    item_embed = model([item_desc])[0]\n    \n    items_desc = df[df.article_id.isin(items)].detail_desc\n    items_embed = model(items_desc)\n    scores = []\n    \n    for i in items_embed:\n        sim = np.dot(i, item_embed)\n        scores.append(sim)\n        \n        \n    return np.mean(scores)","metadata":{"papermill":{"duration":0.047871,"end_time":"2022-05-15T08:35:42.399775","exception":false,"start_time":"2022-05-15T08:35:42.351904","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-01-30T15:27:20.189794Z","iopub.execute_input":"2023-01-30T15:27:20.190196Z","iopub.status.idle":"2023-01-30T15:27:20.197887Z","shell.execute_reply.started":"2023-01-30T15:27:20.190163Z","shell.execute_reply":"2023-01-30T15:27:20.196455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = []\nscores = []\n\nfor key, value in new:\n    temp = []\n    for item in value[:6]:\n        temp.append(recommend(item))\n    temp2 = pd.concat(temp).sample(6, random_state=42)\n    temp2['actual'] = value[6:]\n    temp3 = []\n    for item in temp2.actual:\n        sim = get_sim(item, temp2.article_id) \n        temp3.append(sim)\n        \n    scores.append(np.mean(temp3))\n    preds.append([key, temp2])","metadata":{"papermill":{"duration":16.109005,"end_time":"2022-05-15T08:35:58.546233","exception":false,"start_time":"2022-05-15T08:35:42.437228","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-01-30T15:27:24.922061Z","iopub.execute_input":"2023-01-30T15:27:24.922492Z","iopub.status.idle":"2023-01-30T15:27:42.600929Z","shell.execute_reply.started":"2023-01-30T15:27:24.922456Z","shell.execute_reply":"2023-01-30T15:27:42.599880Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.mean(scores)","metadata":{"papermill":{"duration":0.04511,"end_time":"2022-05-15T08:35:58.628558","exception":false,"start_time":"2022-05-15T08:35:58.583448","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-01-30T15:27:42.602703Z","iopub.execute_input":"2023-01-30T15:27:42.603080Z","iopub.status.idle":"2023-01-30T15:27:42.610231Z","shell.execute_reply.started":"2023-01-30T15:27:42.603045Z","shell.execute_reply":"2023-01-30T15:27:42.609301Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"customer = preds[0][1]","metadata":{"papermill":{"duration":0.043601,"end_time":"2022-05-15T08:35:58.708634","exception":false,"start_time":"2022-05-15T08:35:58.665033","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-01-30T15:27:52.520459Z","iopub.execute_input":"2023-01-30T15:27:52.520865Z","iopub.status.idle":"2023-01-30T15:27:52.526991Z","shell.execute_reply.started":"2023-01-30T15:27:52.520832Z","shell.execute_reply":"2023-01-30T15:27:52.525240Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"customer","metadata":{"papermill":{"duration":0.068811,"end_time":"2022-05-15T08:35:58.815229","exception":false,"start_time":"2022-05-15T08:35:58.746418","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-01-30T15:27:56.824898Z","iopub.execute_input":"2023-01-30T15:27:56.825356Z","iopub.status.idle":"2023-01-30T15:27:56.843486Z","shell.execute_reply.started":"2023-01-30T15:27:56.825319Z","shell.execute_reply":"2023-01-30T15:27:56.842008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_items(customer.article_id.values)","metadata":{"papermill":{"duration":2.741098,"end_time":"2022-05-15T08:36:01.598185","exception":false,"start_time":"2022-05-15T08:35:58.857087","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-01-30T15:28:00.151782Z","iopub.execute_input":"2023-01-30T15:28:00.152201Z","iopub.status.idle":"2023-01-30T15:28:02.661067Z","shell.execute_reply.started":"2023-01-30T15:28:00.152167Z","shell.execute_reply":"2023-01-30T15:28:02.660039Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_items(customer.actual.values)","metadata":{"papermill":{"duration":2.552911,"end_time":"2022-05-15T08:36:04.194260","exception":false,"start_time":"2022-05-15T08:36:01.641349","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-01-30T15:28:06.770157Z","iopub.execute_input":"2023-01-30T15:28:06.770802Z","iopub.status.idle":"2023-01-30T15:28:09.392575Z","shell.execute_reply.started":"2023-01-30T15:28:06.770758Z","shell.execute_reply":"2023-01-30T15:28:09.391646Z"},"trusted":true},"execution_count":null,"outputs":[]}]}