{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"![](https://miro.medium.com/max/1400/1*gYe2FMr9lKys2Wmo1v-s7A.jpeg)","metadata":{}},{"cell_type":"markdown","source":"# 1. H&M Personalized Fashion Recommendations\n\n## Problem Statement: Given the features and image of a product,finding the similar products for recommendations using weighted nearest neighbors.\n### 1.1.1 we will be using features and image of the product.\n### I will use only 10 points, then you can scale to over 100k","metadata":{}},{"cell_type":"markdown","source":"### 1.1.2 Text featurization. : IDF weighted word2vec of Title","metadata":{}},{"cell_type":"code","source":"#importing required libraries\nimport numpy as np\nimport pandas as pd\nimport matplotlib as plt\nimport seaborn as sns\nfrom collections import Counter\nimport os.path\nfrom sklearn.metrics import pairwise_distances\nfrom PIL import Image\nimport requests\nfrom io import BytesIO\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\nimport pickle\nimport warnings\nfrom bs4 import BeautifulSoup\nfrom nltk.corpus import stopwords\nfrom nltk.tokenize import word_tokenize\nimport nltk\nimport math\nimport time\nimport re\nimport os\nimport seaborn as sns\nfrom collections import Counter\nfrom sklearn.feature_extraction.text import CountVectorizer\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.metrics.pairwise import cosine_similarity  \nfrom sklearn.metrics import pairwise_distances\nfrom matplotlib import gridspec\nfrom scipy.sparse import hstack\nimport plotly\nimport plotly.figure_factory as ff\nfrom plotly.graph_objs import Scatter, Layout\nplotly.offline.init_notebook_mode(connected=True)\nwarnings.filterwarnings(\"ignore\")","metadata":{"execution":{"iopub.status.busy":"2022-02-08T14:24:05.023687Z","iopub.execute_input":"2022-02-08T14:24:05.024854Z","iopub.status.idle":"2022-02-08T14:24:10.316503Z","shell.execute_reply.started":"2022-02-08T14:24:05.024703Z","shell.execute_reply":"2022-02-08T14:24:10.315433Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#improting data as a pandas dataframe\nHM_data = pd.read_csv(\"../input/h-and-m-personalized-fashion-recommendations/articles.csv\")\nHM_data=HM_data.head(10000)","metadata":{"execution":{"iopub.status.busy":"2022-02-08T14:24:10.318318Z","iopub.execute_input":"2022-02-08T14:24:10.318683Z","iopub.status.idle":"2022-02-08T14:24:11.594933Z","shell.execute_reply.started":"2022-02-08T14:24:10.318648Z","shell.execute_reply":"2022-02-08T14:24:11.593815Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#getting the shape of the data\nprint(\"Number of data-points in the data:\" , HM_data.shape[0])\nprint(\"Number of features in the data :\" , HM_data.shape[1])","metadata":{"execution":{"iopub.status.busy":"2022-02-08T14:24:11.596459Z","iopub.execute_input":"2022-02-08T14:24:11.596728Z","iopub.status.idle":"2022-02-08T14:24:11.603669Z","shell.execute_reply.started":"2022-02-08T14:24:11.596695Z","shell.execute_reply":"2022-02-08T14:24:11.602416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#getting the column names\nprint(HM_data.columns)","metadata":{"execution":{"iopub.status.busy":"2022-02-08T14:24:11.607236Z","iopub.execute_input":"2022-02-08T14:24:11.607759Z","iopub.status.idle":"2022-02-08T14:24:11.618238Z","shell.execute_reply.started":"2022-02-08T14:24:11.607701Z","shell.execute_reply":"2022-02-08T14:24:11.617424Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Getting product image path\nimages_path=[]\ni=0\nfor p in HM_data['article_id'].tolist():\n    path= '../input/h-and-m-personalized-fashion-recommendations/images/' + '0' + str(p)[:2] + '/'+ '0' + str(p) +'.jpg'\n\n    if os.path.exists(path):\n        i+=1\n        images_path.append(path)\n    else: images_path.append(None)\nprint(f'There is {i} article with corresponding image')\n    ","metadata":{"execution":{"iopub.status.busy":"2022-02-08T14:24:11.619654Z","iopub.execute_input":"2022-02-08T14:24:11.620302Z","iopub.status.idle":"2022-02-08T14:24:23.824823Z","shell.execute_reply.started":"2022-02-08T14:24:11.620250Z","shell.execute_reply":"2022-02-08T14:24:23.824063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#taking only 6 features and 7th feature will be image url\nHM_data['image_path']=images_path\nHM_data = HM_data[['article_id', 'prod_name', 'colour_group_name', 'image_path',\n             'product_type_name', 'product_group_name', 'detail_desc']]\n#getting the shape of the data\nprint(\"Number of data-points in the data:\" , HM_data.shape[0])\nprint(\"Number of features in the data :\" , HM_data.shape[1])","metadata":{"execution":{"iopub.status.busy":"2022-02-08T14:24:23.826187Z","iopub.execute_input":"2022-02-08T14:24:23.826597Z","iopub.status.idle":"2022-02-08T14:24:23.857388Z","shell.execute_reply.started":"2022-02-08T14:24:23.826563Z","shell.execute_reply":"2022-02-08T14:24:23.856471Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#getting the head of the data\nHM_data.head(3)","metadata":{"execution":{"iopub.status.busy":"2022-02-08T14:24:23.859078Z","iopub.execute_input":"2022-02-08T14:24:23.859617Z","iopub.status.idle":"2022-02-08T14:24:23.887823Z","shell.execute_reply.started":"2022-02-08T14:24:23.859574Z","shell.execute_reply":"2022-02-08T14:24:23.886803Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2.1 Missing data for various features.\n### 2.1.1 Basic stats for the feature: product_type_name","metadata":{}},{"cell_type":"code","source":"print(HM_data['product_type_name'].describe())","metadata":{"execution":{"iopub.status.busy":"2022-02-08T14:24:23.888916Z","iopub.execute_input":"2022-02-08T14:24:23.889406Z","iopub.status.idle":"2022-02-08T14:24:23.903397Z","shell.execute_reply.started":"2022-02-08T14:24:23.889366Z","shell.execute_reply":"2022-02-08T14:24:23.902201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Observations:\n\n* About 12% (11169/105542) of the products are Trousers.\n* There are 131 unique product types.","metadata":{}},{"cell_type":"code","source":"#find the 10 most frequent product_type_names.\nproduct_type_count = Counter(list(HM_data['product_type_name']))\nproduct_type_count.most_common(10)","metadata":{"execution":{"iopub.status.busy":"2022-02-08T14:24:23.904991Z","iopub.execute_input":"2022-02-08T14:24:23.905565Z","iopub.status.idle":"2022-02-08T14:24:23.914744Z","shell.execute_reply.started":"2022-02-08T14:24:23.905523Z","shell.execute_reply":"2022-02-08T14:24:23.913703Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 2.1.2 Basic stats for the feature: color","metadata":{}},{"cell_type":"code","source":"print(HM_data['colour_group_name'].describe())","metadata":{"execution":{"iopub.status.busy":"2022-02-08T14:24:23.918178Z","iopub.execute_input":"2022-02-08T14:24:23.918625Z","iopub.status.idle":"2022-02-08T14:24:23.936195Z","shell.execute_reply.started":"2022-02-08T14:24:23.918587Z","shell.execute_reply":"2022-02-08T14:24:23.935082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Observations:\n\n* we have 50 unique colors\n* Over 20% of products are black in color","metadata":{}},{"cell_type":"code","source":"#find the 10 most frequent colors.\ncolor_count = Counter(list(HM_data['colour_group_name']))\ncolor_count.most_common(10)","metadata":{"execution":{"iopub.status.busy":"2022-02-08T14:24:23.938132Z","iopub.execute_input":"2022-02-08T14:24:23.938722Z","iopub.status.idle":"2022-02-08T14:24:23.954603Z","shell.execute_reply.started":"2022-02-08T14:24:23.938674Z","shell.execute_reply":"2022-02-08T14:24:23.953657Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 2.1.3 Basic stats for the feature: detail_desc","metadata":{}},{"cell_type":"code","source":"print(HM_data['detail_desc'].describe())","metadata":{"execution":{"iopub.status.busy":"2022-02-08T14:24:23.956309Z","iopub.execute_input":"2022-02-08T14:24:23.956897Z","iopub.status.idle":"2022-02-08T14:24:23.977705Z","shell.execute_reply.started":"2022-02-08T14:24:23.956863Z","shell.execute_reply":"2022-02-08T14:24:23.976627Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Observations\n\n* Most of the products have a detail_desc.\n* detail_desc are fairly descriptive of what the product is.\n* We use detail_desc extensively in this notebook as they are short and informative.","metadata":{}},{"cell_type":"markdown","source":"## 2.2 Remove near duplicate items using detail_desc feature","metadata":{}},{"cell_type":"code","source":"print(\"Number of duplicates present in articles: {}\".format(sum(HM_data.duplicated(\"detail_desc\"))))","metadata":{"execution":{"iopub.status.busy":"2022-02-08T14:24:23.979091Z","iopub.execute_input":"2022-02-08T14:24:23.979952Z","iopub.status.idle":"2022-02-08T14:24:23.996955Z","shell.execute_reply.started":"2022-02-08T14:24:23.979895Z","shell.execute_reply":"2022-02-08T14:24:23.996049Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"HM_data","metadata":{"execution":{"iopub.status.busy":"2022-02-08T14:24:23.998255Z","iopub.execute_input":"2022-02-08T14:24:23.999069Z","iopub.status.idle":"2022-02-08T14:24:24.025650Z","shell.execute_reply.started":"2022-02-08T14:24:23.999033Z","shell.execute_reply":"2022-02-08T14:24:24.024435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#removing data-points with short-titles\nHM_data_sorted = HM_data[HM_data['detail_desc'].apply(lambda x: len(str(x).split())>4)]\nprint(\"After removal of products with short description:\", HM_data_sorted.shape[0])","metadata":{"execution":{"iopub.status.busy":"2022-02-08T14:24:24.027287Z","iopub.execute_input":"2022-02-08T14:24:24.028021Z","iopub.status.idle":"2022-02-08T14:24:24.061159Z","shell.execute_reply.started":"2022-02-08T14:24:24.027968Z","shell.execute_reply":"2022-02-08T14:24:24.059913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Sort the whole data based on title (alphabetical order of detail_desc) \nHM_data_sorted.sort_values('detail_desc',inplace=True, ascending=False)","metadata":{"execution":{"iopub.status.busy":"2022-02-08T14:24:24.062925Z","iopub.execute_input":"2022-02-08T14:24:24.064057Z","iopub.status.idle":"2022-02-08T14:24:24.086170Z","shell.execute_reply.started":"2022-02-08T14:24:24.064011Z","shell.execute_reply":"2022-02-08T14:24:24.084914Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 2.2.1 Removing duplicates with Text sorted","metadata":{}},{"cell_type":"code","source":"#getting the duplicates \nindices = []\nfor i,row in HM_data_sorted.iterrows():\n    indices.append(i)\n    \nimport itertools\nstage1_dedupe_asins = []\ni = 0\nj = 0\nnum_data_points = HM_data_sorted.shape[0]\nwhile i < num_data_points and j < num_data_points:\n       \n    previous_i = i\n    a = HM_data['detail_desc'].loc[indices[i]].split()\n\n    # search for the similar products sequentially \n    j = i+1\n    while j < num_data_points:\n        b = HM_data['detail_desc'].loc[indices[j]].split()\n\n        # store the maximum length of two strings\n        length = max(len(a), len(b))\n\n        #count is used to store the number of words that are matched in both strings\n        count  = 0\n\n        for k in itertools.zip_longest(a,b): \n            if (k[0] == k[1]):\n                count += 1\n\n        # if the number of words in which both strings differ are > 2 , we are considering it as those two apperals are different\n        # if the number of words in which both strings differ are < 2 , we are considering it as those two apperals are same, hence we are ignoring them\n        if (length - count) > 2: # number of words in which both sensences differ\n            # if both strings are differ by more than 2 words we include the 1st string index\n            stage1_dedupe_asins.append(HM_data_sorted['article_id'].loc[indices[i]])\n\n            # if the comaprision between is between num_data_points, num_data_points-1 strings and they differ in more than 2 words we include both\n            if j == num_data_points-1: stage1_dedupe_asins.append(HM_data_sorted['article_id'].loc[indices[j]])\n\n            # start searching for similar apperals corresponds 2nd string\n            i = j\n            break\n        else:\n            j += 1\n    if previous_i == i:\n        break    ","metadata":{"execution":{"iopub.status.busy":"2022-02-08T14:24:24.088260Z","iopub.execute_input":"2022-02-08T14:24:24.089004Z","iopub.status.idle":"2022-02-08T14:24:25.192451Z","shell.execute_reply.started":"2022-02-08T14:24:24.088945Z","shell.execute_reply":"2022-02-08T14:24:25.191163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#removing duplicated\nHM_data = HM_data.loc[HM_data['article_id'].isin(stage1_dedupe_asins)]","metadata":{"execution":{"iopub.status.busy":"2022-02-08T14:24:25.193899Z","iopub.execute_input":"2022-02-08T14:24:25.194185Z","iopub.status.idle":"2022-02-08T14:24:25.201603Z","shell.execute_reply.started":"2022-02-08T14:24:25.194151Z","shell.execute_reply":"2022-02-08T14:24:25.200574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#getting the number of data-points after removing the duplicates\nprint('Number of data points : ', HM_data.shape[0])","metadata":{"execution":{"iopub.status.busy":"2022-02-08T14:24:25.203752Z","iopub.execute_input":"2022-02-08T14:24:25.204199Z","iopub.status.idle":"2022-02-08T14:24:25.224843Z","shell.execute_reply.started":"2022-02-08T14:24:25.204146Z","shell.execute_reply":"2022-02-08T14:24:25.223801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 2.2.2 Removing duplicates which are not similar adjacently but very similar","metadata":{}},{"cell_type":"code","source":"indices = []\nfor i,row in HM_data.iterrows():\n    indices.append(i)\n\nstage2_dedupe_asins = []\nwhile len(indices)!=0:\n    i = indices.pop()\n    stage2_dedupe_asins.append(HM_data['article_id'].loc[i])\n    # consider the first apperal's title\n    a = HM_data['detail_desc'].loc[i].split()\n \n    for j in indices:\n        \n        b = HM_data['detail_desc'].loc[j].split()\n        length = max(len(a),len(b))\n        \n        # count is used to store the number of words that are matched in both strings\n        count  = 0\n\n        for k in itertools.zip_longest(a,b): \n            if (k[0]==k[1]):\n                count += 1\n\n        #if the number of words in which both strings differ are < 3 ,\n        #we are considering it as those two apperals are same, hence we are ignoring them\n        if (length - count) < 3:\n            indices.remove(j)","metadata":{"execution":{"iopub.status.busy":"2022-02-08T14:24:25.226456Z","iopub.execute_input":"2022-02-08T14:24:25.226723Z","iopub.status.idle":"2022-02-08T14:26:50.782081Z","shell.execute_reply.started":"2022-02-08T14:24:25.226691Z","shell.execute_reply":"2022-02-08T14:26:50.780699Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#removing duplicate further\nHM_data = HM_data.loc[HM_data['article_id'].isin(stage2_dedupe_asins)]","metadata":{"execution":{"iopub.status.busy":"2022-02-08T14:26:50.783958Z","iopub.execute_input":"2022-02-08T14:26:50.787662Z","iopub.status.idle":"2022-02-08T14:26:50.795651Z","shell.execute_reply.started":"2022-02-08T14:26:50.787604Z","shell.execute_reply":"2022-02-08T14:26:50.794412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 3. Text-PreProcessing:","metadata":{}},{"cell_type":"code","source":"#getting the number of data-points after removing all the duplicates\nprint('Number of data points after stage two of dedupe: ',HM_data.shape[0])","metadata":{"execution":{"iopub.status.busy":"2022-02-08T14:26:50.797206Z","iopub.execute_input":"2022-02-08T14:26:50.798071Z","iopub.status.idle":"2022-02-08T14:26:50.810986Z","shell.execute_reply.started":"2022-02-08T14:26:50.798029Z","shell.execute_reply":"2022-02-08T14:26:50.809533Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"HM_data","metadata":{"execution":{"iopub.status.busy":"2022-02-08T14:26:50.813130Z","iopub.execute_input":"2022-02-08T14:26:50.813862Z","iopub.status.idle":"2022-02-08T14:26:50.844725Z","shell.execute_reply.started":"2022-02-08T14:26:50.813809Z","shell.execute_reply":"2022-02-08T14:26:50.843448Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# removing stop-words\nimport nltk\nfrom nltk.corpus import stopwords\nfrom nltk.tokenize import word_tokenize\n\nstop_words = set(stopwords.words('english'))\nprint ('list of stop words:', stop_words)\n\ndef nlp_preprocessing(total_text, index, column):\n    if type(total_text) is not int:\n        string = \"\"\n        for words in total_text.split():\n            # remove the special chars in review like '\"#$@!%^&*()_+-~?>< etc.\n            word = (\"\".join(e for e in words if e.isalnum()))\n            # Convert all letters to lower-case\n            word = word.lower()\n            # stop-word removal\n            if not word in stop_words:\n                string += word + \" \"\n        HM_data[column][index] = string\n        \n        \n# we take each title and we text-preprocess it.\nfor index, row in HM_data.iterrows():\n    nlp_preprocessing(row['detail_desc'], index, 'detail_desc') ","metadata":{"execution":{"iopub.status.busy":"2022-02-08T14:26:50.846162Z","iopub.execute_input":"2022-02-08T14:26:50.847139Z","iopub.status.idle":"2022-02-08T14:26:52.482552Z","shell.execute_reply.started":"2022-02-08T14:26:50.847103Z","shell.execute_reply":"2022-02-08T14:26:52.481396Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Stemming","metadata":{}},{"cell_type":"code","source":"from nltk.stem.porter import *\nstemmer = PorterStemmer()\nprint(stemmer.stem('arguing'))\nprint(stemmer.stem('fishing'))\n# stemming doesn't convert the tokens into meaningful words ","metadata":{"execution":{"iopub.status.busy":"2022-02-08T14:26:52.484324Z","iopub.execute_input":"2022-02-08T14:26:52.484709Z","iopub.status.idle":"2022-02-08T14:26:52.494642Z","shell.execute_reply.started":"2022-02-08T14:26:52.484661Z","shell.execute_reply":"2022-02-08T14:26:52.492796Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# function to display an image\ndef display_img(url,ax,fig):\n    # we get the url of the apparel and download it\n    #response = requests.get(url)\n    img = Image.open(url)\n    # we will display it in notebook \n    plt.imshow(img)\n\n# function to generate the heatmap\ndef plot_heatmap(keys, values, labels, url, text):\n        # keys: list of words of recommended title\n        # values: len(values) ==  len(keys), values(i) represents the occurence of the word keys(i)\n        # labels: len(labels) == len(keys), the values of labels depends on the model we are using\n                # if model == 'bag of words': labels(i) = values(i)\n                # if model == 'tfidf weighted bag of words':labels(i) = tfidf(keys(i))\n                # if model == 'idf weighted bag of words':labels(i) = idf(keys(i))\n        # url : apparel's url\n        # we will devide the whole figure into two parts\n        gs = gridspec.GridSpec(2, 2, width_ratios=[4,1], height_ratios=[4,1]) \n        fig = plt.figure(figsize=(25,3))\n        # 1st, ploting heat map that represents the count of commonly ocurred words in title2\n        ax = plt.subplot(gs[0])\n        # it displays a cell in white color if the word is intersection(list of words of title1 and list of words of title2), in black if not\n        ax = sns.heatmap(np.array([values]), annot=np.array([labels]))\n        ax.set_xticklabels(keys) # set that axis labels as the words of title\n        ax.set_title(text) # apparel title        \n        # 2nd, plotting image of the the apparel\n        ax = plt.subplot(gs[1])\n        # we don't want any grid lines for image and no labels on x-axis and y-axis\n        ax.grid(False)\n        ax.set_xticks([])\n        ax.set_yticks([])\n        # we call dispaly_img based with paramete url\n        display_img(url, ax, fig)       \n        # displays combine figure ( heat map and image together)\n        plt.show()\n\n# function to display heatmap and image        \ndef plot_heatmap_image(doc_id, vec1, vec2, url, text, model):\n    # doc_id : index of the title1\n    # vec1 : input apparels's vector, it is of a dict type {word:count}\n    # vec2 : recommended apparels's vector, it is of a dict type {word:count}\n    # url : apparels image url\n    # text: title of recomonded apparel (used to keep title of image)\n    # model, it can be any of the models, \n        # 1. bag_of_words\n        # 2. tfidf\n        # 3. idf\n    # we find the common words in both titles, because these only words contribute to the distance between two title vec's\n    intersection = set(vec1.keys()) & set(vec2.keys()) \n    # we set the values of non intersecting words to zero, this is just to show the difference in heatmap\n    for i in vec2:\n        if i not in intersection:\n            vec2[i]=0\n    # for labeling heatmap, keys contains list of all words in title2\n    keys = list(vec2.keys())\n    #  if ith word in intersection(lis of words of title1 and list of words of title2): values(i)=count of that word in title2 else values(i)=0 \n    values = [vec2[x] for x in vec2.keys()]  \n    # labels: len(labels) == len(keys), the values of labels depends on the model we are using\n        # if model == 'bag of words': labels(i) = values(i)\n        # if model == 'tfidf weighted bag of words':labels(i) = tfidf(keys(i))\n        # if model == 'idf weighted bag of words':labels(i) = idf(keys(i))\n    if model == 'bag_of_words':\n        labels = values\n    elif model == 'tfidf':\n        labels = []\n        for x in vec2.keys():\n            # tfidf_title_vectorizer.vocabulary_ it contains all the words in the corpus\n            # tfidf_title_features[doc_id, index_of_word_in_corpus] will give the tfidf value of word in given document (doc_id)\n            if x in  tfidf_title_vectorizer.vocabulary_:\n                labels.append(tfidf_title_features[doc_id, tfidf_title_vectorizer.vocabulary_[x]])\n            else:\n                labels.append(0)\n    elif model == 'idf':\n        labels = []\n        for x in vec2.keys():\n            # idf_title_vectorizer.vocabulary_ it contains all the words in the corpus\n            # idf_title_features[doc_id, index_of_word_in_corpus] will give the idf value of word in given document (doc_id)\n            if x in  idf_title_vectorizer.vocabulary_:\n                labels.append(idf_title_features[doc_id, idf_title_vectorizer.vocabulary_[x]])\n            else:\n                labels.append(0)\n    plot_heatmap(keys, values, labels, url, text)\n\n#function to convert text into vector\ndef text_to_vector(text):\n    word = re.compile(r'\\w+')\n    words = word.findall(text)\n    # words stores list of all words in given string, you can try 'words = text.split()' this will also gives same result\n    return Counter(words) # Counter counts the occurence of each word in list, it returns dict type object {word1:count}\n\n# function to display the result\ndef get_result(doc_id, content_a, content_b, url, model):\n    text1 = content_a\n    text2 = content_b\n    # vector1 = dict{word11:#count, word12:#count, etc.}\n    vector1 = text_to_vector(text1)\n    # vector2 = dict{word21:#count, word22:#count, etc.}\n    vector2 = text_to_vector(text2)\n    plot_heatmap_image(doc_id, vector1, vector2, url, text2, model)","metadata":{"execution":{"iopub.status.busy":"2022-02-08T14:26:52.497485Z","iopub.execute_input":"2022-02-08T14:26:52.497845Z","iopub.status.idle":"2022-02-08T14:26:52.522032Z","shell.execute_reply.started":"2022-02-08T14:26:52.497794Z","shell.execute_reply":"2022-02-08T14:26:52.520887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 5.1 Bag of Words (BoW) on product detail_desc","metadata":{}},{"cell_type":"code","source":"from sklearn.feature_extraction.text import CountVectorizer\ntitle_vectorizer = CountVectorizer()\ntitle_features   = title_vectorizer.fit_transform(HM_data['detail_desc'])\ntitle_features.get_shape()\n# title_features[doc_id, index_of_word_in_corpus] = number of times the word occured in that doc","metadata":{"execution":{"iopub.status.busy":"2022-02-08T14:26:52.524229Z","iopub.execute_input":"2022-02-08T14:26:52.524871Z","iopub.status.idle":"2022-02-08T14:26:52.613732Z","shell.execute_reply.started":"2022-02-08T14:26:52.524802Z","shell.execute_reply":"2022-02-08T14:26:52.612919Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data=HM_data.copy()\ndef bag_of_words_model(doc_id, num_results):\n    # doc_id: apparel's id in given corpus\n    # pairwise_dist will store the distance from given input apparel to all remaining apparels\n    # the metric used here is cosine, the coside distance is mesured as K(X, Y) = <X, Y> / (||X||*||Y||)\n    # http://scikit-learn.org/stable/modules/metrics.html#cosine-similarity\n    pairwise_dist = pairwise_distances(title_features,title_features[doc_id])\n    # np.argsort will return indices of the smallest distances\n    indices = np.argsort(pairwise_dist.flatten())[0:num_results]\n    #pdists will store the smallest distances\n    pdists  = np.sort(pairwise_dist.flatten())[0:num_results]\n    #data frame indices of the 9 smallest distace's\n    df_indices = list(data.index[indices])    \n    for i in range(0,len(indices)):\n        # we will pass 1. doc_id, 2. title1, 3. title2, url, model\n        get_result(indices[i],data['detail_desc'].loc[df_indices[0]], data['detail_desc'].loc[df_indices[i]], data['image_path'].loc[df_indices[i]], 'bag_of_words')\n        print('article_id :',data['article_id'].loc[df_indices[i]])\n        print ('detail_desc:', data['detail_desc'].loc[df_indices[i]])\n        print ('Euclidean similarity with the query image :', pdists[i])\n        print('='*60)\n#call the bag-of-words model for a product to get similar products\nbag_of_words_model(1582, 20)\n# in the output heat map each value represents the tfidf values of the label word, the color represents the intersection with inputs title\n","metadata":{"execution":{"iopub.status.busy":"2022-02-08T14:26:52.617823Z","iopub.execute_input":"2022-02-08T14:26:52.618540Z","iopub.status.idle":"2022-02-08T14:27:05.252604Z","shell.execute_reply.started":"2022-02-08T14:26:52.618490Z","shell.execute_reply":"2022-02-08T14:27:05.251410Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# TF-IDF based product similarity","metadata":{}},{"cell_type":"code","source":"tfidf_title_vectorizer = TfidfVectorizer(min_df = 0)\ntfidf_title_features = tfidf_title_vectorizer.fit_transform(data['detail_desc'])","metadata":{"execution":{"iopub.status.busy":"2022-02-08T14:27:05.254268Z","iopub.execute_input":"2022-02-08T14:27:05.254562Z","iopub.status.idle":"2022-02-08T14:27:05.333212Z","shell.execute_reply.started":"2022-02-08T14:27:05.254529Z","shell.execute_reply":"2022-02-08T14:27:05.331752Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def tfidf_model(doc_id, num_results):\n    # doc_id: apparel's id in given corpus  \n    # pairwise_dist will store the distance from given input apparel to all remaining apparels\n    # the metric used here is cosine, the coside distance is mesured as K(X, Y) = <X, Y> / (||X||*||Y||)\n    # http://scikit-learn.org/stable/modules/metrics.html#cosine-similarity\n    pairwise_dist = pairwise_distances(tfidf_title_features,tfidf_title_features[doc_id])\n    # np.argsort will return indices of 9 smallest distances\n    indices = np.argsort(pairwise_dist.flatten())[0:num_results]\n    #pdists will store the 9 smallest distances\n    pdists  = np.sort(pairwise_dist.flatten())[0:num_results]\n    #data frame indices of the 9 smallest distace's\n    df_indices = list(data.index[indices])\n    for i in range(0,len(indices)):\n        # we will pass 1. doc_id, 2. title1, 3. title2, url, model\n        get_result(indices[i], data['detail_desc'].loc[df_indices[0]], data['detail_desc'].loc[df_indices[i]], data['image_path'].loc[df_indices[i]], 'tfidf')\n        print('article_id :',data['article_id'].loc[df_indices[i]])\n        print ('Eucliden distance from the given image :', pdists[i])\n        print('='*125)\ntfidf_model(1582, 20)\n# in the output heat map each value represents the tfidf values of the label word, the color represents the intersection with inputs title","metadata":{"execution":{"iopub.status.busy":"2022-02-08T14:27:05.334969Z","iopub.execute_input":"2022-02-08T14:27:05.335324Z","iopub.status.idle":"2022-02-08T14:27:17.781525Z","shell.execute_reply.started":"2022-02-08T14:27:05.335277Z","shell.execute_reply":"2022-02-08T14:27:17.780221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# IDF based product similarity","metadata":{}},{"cell_type":"code","source":"idf_title_vectorizer = CountVectorizer()\nidf_title_features = idf_title_vectorizer.fit_transform(data['detail_desc'])","metadata":{"execution":{"iopub.status.busy":"2022-02-08T14:27:17.783290Z","iopub.execute_input":"2022-02-08T14:27:17.783703Z","iopub.status.idle":"2022-02-08T14:27:17.855645Z","shell.execute_reply.started":"2022-02-08T14:27:17.783661Z","shell.execute_reply":"2022-02-08T14:27:17.854684Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def n_containing(word):\n    # return the number of documents which had the given word\n    return sum(1 for blob in data['detail_desc'] if word in blob.split())\ndef idf(word):\n    # idf = log(number of docs / number of docs which had the given word)\n    return math.log(data.shape[0] / (n_containing(word)))","metadata":{"execution":{"iopub.status.busy":"2022-02-08T14:27:17.857293Z","iopub.execute_input":"2022-02-08T14:27:17.857691Z","iopub.status.idle":"2022-02-08T14:27:17.865894Z","shell.execute_reply.started":"2022-02-08T14:27:17.857641Z","shell.execute_reply":"2022-02-08T14:27:17.864674Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# converting the values into float\nidf_title_features  = idf_title_features.astype(np.float)\nfor i in idf_title_vectorizer.vocabulary_.keys():\n    # for every word in whole corpus we will find its idf value\n    idf_val = idf(i)\n    # to calculate idf_title_features we will replace the count values with the idf values of the word\n    # idf_title_features[:, idf_title_vectorizer.vocabulary_[i]].nonzero()[0] will return all documents in which the word i is present\n    for j in idf_title_features[:, idf_title_vectorizer.vocabulary_[i]].nonzero()[0]:        \n        # we will replace the count values of word i in document j with idf_value of word i \n        # idf_title_features[doc_id, index_of_word_in_courpus] = idf value of word\n        idf_title_features[j,idf_title_vectorizer.vocabulary_[i]] = idf_val  ","metadata":{"execution":{"iopub.status.busy":"2022-02-08T14:27:17.868083Z","iopub.execute_input":"2022-02-08T14:27:17.868497Z","iopub.status.idle":"2022-02-08T14:27:29.497133Z","shell.execute_reply.started":"2022-02-08T14:27:17.868446Z","shell.execute_reply":"2022-02-08T14:27:29.495860Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def idf_model(doc_id, num_results):\n    # doc_id: apparel's id in given corpus    \n    # pairwise_dist will store the distance from given input apparel to all remaining apparels\n    # the metric used here is cosine, the coside distance is mesured as K(X, Y) = <X, Y> / (||X||*||Y||)\n    # http://scikit-learn.org/stable/modules/metrics.html#cosine-similarity\n    pairwise_dist = pairwise_distances(idf_title_features,idf_title_features[doc_id])\n    # np.argsort will return indices of 9 smallest distances\n    indices = np.argsort(pairwise_dist.flatten())[0:num_results]\n    #pdists will store the 9 smallest distances\n    pdists  = np.sort(pairwise_dist.flatten())[0:num_results]\n    #data frame indices of the 9 smallest distace's\n    df_indices = list(data.index[indices])\n    for i in range(0,len(indices)):\n        get_result(indices[i],data['detail_desc'].loc[df_indices[0]], data['detail_desc'].loc[df_indices[i]], data['image_path'].loc[df_indices[i]], 'idf')\n        print('article_id :',data['article_id'].loc[df_indices[i]])\n        print ('euclidean distance from the given image :', pdists[i])\n        print('='*125)        \nidf_model(1582,20)\n# in the output heat map each value represents the idf values of the label word, the color represents the intersection with inputs title","metadata":{"execution":{"iopub.status.busy":"2022-02-08T14:27:29.498961Z","iopub.execute_input":"2022-02-08T14:27:29.499268Z","iopub.status.idle":"2022-02-08T14:27:41.968599Z","shell.execute_reply.started":"2022-02-08T14:27:29.499235Z","shell.execute_reply":"2022-02-08T14:27:41.967297Z"},"trusted":true},"execution_count":null,"outputs":[]}]}