{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":31254,"databundleVersionId":3103714,"sourceType":"competition"},{"sourceId":3269741,"sourceType":"datasetVersion","datasetId":1980666},{"sourceId":3270052,"sourceType":"datasetVersion","datasetId":1980859}],"dockerImageVersionId":30170,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport plotly.express as px\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\nimport matplotlib.pyplot as plt\nimport os\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport matplotlib.image as mpimg\npd.options.display.float_format = '{:.4f}'.format\n\n\ndef get_row(n_total, n_cols) :\n    if n_total % n_cols == 0 :\n        n_rows = n_total / n_cols\n    else :\n        n_rows = (n_total // n_cols) + 1\n    return int(n_rows)\ndef visualize_articles(articles , article_list, n_total , n_cols , figsize=(25,10)) :\n    n_rows = get_row(n_total , n_cols)\n    f, ax = plt.subplots(n_rows, n_cols, figsize=figsize)\n    axes = ax.flatten()\n    i = 0\n    for article in article_list:\n        visualize_article(axes , i , articles , article)\n        i += 1\n    else :\n        plt.show()\n        \ndef visualize_article(axes,i, articles , article) :\n    desc = articles[articles['article_id'] == article]['detail_desc'].iloc[0]\n    desc_list = desc.split(' ')\n    for j, elem in enumerate(desc_list):\n        if j > 0 and j % 5 == 0:\n            desc_list[j] = desc_list[j] + '\\n'\n    desc = ' '.join(desc_list)\n    try :\n        img = mpimg.imread(f'../input/h-and-m-personalized-fashion-recommendations/images/0{str(article)[:2]}/0{int(article)}.jpg')\n        axes[i].imshow(img)\n    except :\n        pass\n    axes[i].set_xticks([], [])\n    axes[i].set_yticks([], [])\n    axes[i].set_title(article)\n    axes[i].grid(False)\n    axes[i].set_xlabel(desc, fontsize=10)\n    \ndata_submission_path = \"/kaggle/input/h-and-m-personalized-fashion-recommendations/sample_submission.csv\"\ndata_article_path = \"/kaggle/input/h-and-m-personalized-fashion-recommendations/articles.csv\"\ndata_transaction_path = \"/kaggle/input/h-and-m-personalized-fashion-recommendations/transactions_train.csv\"\ndata_customer_path = \"/kaggle/input/h-and-m-personalized-fashion-recommendations/customers.csv\"\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-03-08T13:38:39.978377Z","iopub.execute_input":"2022-03-08T13:38:39.97881Z","iopub.status.idle":"2022-03-08T13:38:41.519254Z","shell.execute_reply.started":"2022-03-08T13:38:39.978756Z","shell.execute_reply":"2022-03-08T13:38:41.518116Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.read_csv(data_article_path,nrows=1).T","metadata":{"execution":{"iopub.status.busy":"2022-03-08T13:38:41.521022Z","iopub.execute_input":"2022-03-08T13:38:41.521288Z","iopub.status.idle":"2022-03-08T13:38:41.562377Z","shell.execute_reply.started":"2022-03-08T13:38:41.521256Z","shell.execute_reply":"2022-03-08T13:38:41.561243Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Text Preprocessing","metadata":{}},{"cell_type":"code","source":"import re\nimport nltk\nfrom nltk.stem.porter import PorterStemmer\nfrom nltk.stem import WordNetLemmatizer\nimport string \nimport numpy as np\n\ndef remove_punctuation(text):\n    if text != text :\n        punctuationfree = \"\"\n    else :\n        punctuationfree=\"\".join([i for i in text if i not in string.punctuation])\n    return punctuationfree\n\ndef tokenization(text):\n    tokens = re.split('W+',text)\n    return tokens\n\ndef remove_stopwords(text):\n    output= [i for i in text if i not in stopwords]\n    return output\n\n\ndef stemming(text):\n    stem_text = [porter_stemmer.stem(word) for word in text]\n    return stem_text\n\ndef lemmatizer(text):\n    lemm_text = [wordnet_lemmatizer.lemmatize(word) for word in text]\n    return lemm_text\n\nporter_stemmer = PorterStemmer()\nwordnet_lemmatizer = WordNetLemmatizer()\nstopwords = nltk.corpus.stopwords.words('english')","metadata":{"execution":{"iopub.status.busy":"2022-03-08T13:38:41.56441Z","iopub.execute_input":"2022-03-08T13:38:41.564815Z","iopub.status.idle":"2022-03-08T13:38:42.110029Z","shell.execute_reply.started":"2022-03-08T13:38:41.564779Z","shell.execute_reply":"2022-03-08T13:38:42.108782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"articles = pd.read_csv(data_article_path)","metadata":{"execution":{"iopub.status.busy":"2022-03-08T13:38:42.112059Z","iopub.execute_input":"2022-03-08T13:38:42.112428Z","iopub.status.idle":"2022-03-08T13:38:43.516432Z","shell.execute_reply.started":"2022-03-08T13:38:42.112376Z","shell.execute_reply":"2022-03-08T13:38:43.51535Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"articles['detail_desc']= articles['detail_desc'].apply(lambda x:remove_punctuation(x))\narticles['detail_desc'].head()","metadata":{"execution":{"iopub.status.busy":"2022-03-08T13:01:59.900089Z","iopub.execute_input":"2022-03-08T13:01:59.900424Z","iopub.status.idle":"2022-03-08T13:02:02.026788Z","shell.execute_reply.started":"2022-03-08T13:01:59.90037Z","shell.execute_reply":"2022-03-08T13:02:02.025778Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"articles['detail_desc']= articles['detail_desc'].apply(lambda x: x.lower())\narticles['detail_desc'].head()","metadata":{"execution":{"iopub.status.busy":"2022-03-08T13:02:02.02834Z","iopub.execute_input":"2022-03-08T13:02:02.028584Z","iopub.status.idle":"2022-03-08T13:02:02.114149Z","shell.execute_reply.started":"2022-03-08T13:02:02.028536Z","shell.execute_reply":"2022-03-08T13:02:02.112918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"articles['detail_desc']= articles['detail_desc'].apply(lambda x: tokenization(x))\narticles['detail_desc'].head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"articles['detail_desc']= articles['detail_desc'].apply(lambda x: remove_stopwords(x))\narticles['detail_desc'].head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"articles['detail_desc']= articles['detail_desc'].apply(lambda x: stemming(x))\narticles['detail_desc'].head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"articles['detail_desc']= articles['detail_desc'].apply(lambda x: lemmatizer(x))\narticles['detail_desc'].head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"articles['detail_desc'] = articles['detail_desc'].apply(lambda x : x[0].split(\" \"))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"articles_name = articles.filter(regex=\"name$|detail\")\narticles_name.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_list(x) :\n    total_info = []\n    for i in x.values.tolist() :\n        if isinstance( i , (str,)) :\n            total_info.append(i)\n        elif isinstance( i , (list,)) :\n            total_info.extend(i)\n    else :\n        return total_info\n\narticles_name_list = articles_name.apply(lambda x : get_list(x), axis=1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"articles_name_list[0]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Word2Vec","metadata":{}},{"cell_type":"code","source":"# from gensim.models import Word2Vec\n# model = Word2Vec(sentences=articles_name_list, vector_size=50, window=5, min_count=1, workers=4)\n# model.save(\"word2vec.model\")\n# sims = model.wv.most_similar('Black', topn=10) \n# sims","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Setence-Transformer","metadata":{}},{"cell_type":"code","source":"!pip install sentence-transformers","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sentence_transformers import SentenceTransformer\nsbert_model = SentenceTransformer('bert-base-nli-mean-tokens')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Embedding","metadata":{}},{"cell_type":"code","source":"sentence_embeddings  = sbert_model.encode(\" \".join(articles_name_list[0]))\nsentence_embeddings.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"article_sentences = [\" \".join(article) for article in articles_name_list]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Save","metadata":{}},{"cell_type":"code","source":"# result = sbert_model.encode(article_sentences)\n# np.save(\"/kaggle/working/embedding\",result)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# np.load(\"/kaggle/working/embedding.npy\").shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Download","metadata":{}},{"cell_type":"code","source":"# from IPython.display import FileLink\n# import os\n# os.chdir(r'/kaggle/working')\n# FileLink(r'embedding.npy')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"MY GOOGLE DRIVE LINK\n\nhttps://drive.google.com/file/d/1AAI8Bws_9rustIWPoCvz9I3rC7GLsI4J/view?usp=sharing\n","metadata":{}},{"cell_type":"markdown","source":"## Load","metadata":{}},{"cell_type":"code","source":"import numpy as np\n\nembedding_path = \"../input/article-embedding/embedding.npy\"\nembedding_vector = np.load(embedding_path)","metadata":{"execution":{"iopub.status.busy":"2022-03-08T13:02:30.754197Z","iopub.execute_input":"2022-03-08T13:02:30.754534Z","iopub.status.idle":"2022-03-08T13:02:34.731381Z","shell.execute_reply.started":"2022-03-08T13:02:30.754498Z","shell.execute_reply":"2022-03-08T13:02:34.7304Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tqdm import tqdm","metadata":{"execution":{"iopub.status.busy":"2022-03-08T13:02:34.73345Z","iopub.execute_input":"2022-03-08T13:02:34.734203Z","iopub.status.idle":"2022-03-08T13:02:34.738417Z","shell.execute_reply.started":"2022-03-08T13:02:34.734166Z","shell.execute_reply":"2022-03-08T13:02:34.737573Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Cosine Similarity","metadata":{}},{"cell_type":"code","source":"def cosine(u, v):\n    return np.dot(u, v) / (np.linalg.norm(u) * np.linalg.norm(v))\nfrom sklearn.metrics.pairwise import cosine_similarity\n\ndef get_cosine_similarity(embedding_vector,idx) :\n    \n    embedding_cosine = cosine_similarity(embedding_vector[[idx],] , embedding_vector)\n    embedding_cosine[0,idx]=0\n    return embedding_cosine[0]\n\n\ndef get_best_similiarity(embedding_vector,idx, best_n = 3) :\n    emb_cosine = get_cosine_similarity(embedding_vector , idx)\n    return emb_cosine.argsort()[-best_n:][::-1]\n    ","metadata":{"execution":{"iopub.status.busy":"2022-03-08T13:02:40.423728Z","iopub.execute_input":"2022-03-08T13:02:40.424835Z","iopub.status.idle":"2022-03-08T13:02:40.432809Z","shell.execute_reply.started":"2022-03-08T13:02:40.424784Z","shell.execute_reply":"2022-03-08T13:02:40.432166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"articles = pd.read_csv(data_article_path)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"check_idx = 0\ntop_articles = get_best_similiarity(embedding_vector , check_idx , best_n=5)","metadata":{"execution":{"iopub.status.busy":"2022-03-08T13:02:40.752328Z","iopub.execute_input":"2022-03-08T13:02:40.752647Z","iopub.status.idle":"2022-03-08T13:02:41.198039Z","shell.execute_reply.started":"2022-03-08T13:02:40.752615Z","shell.execute_reply":"2022-03-08T13:02:41.196939Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Cosine Similarity (top 6)","metadata":{}},{"cell_type":"code","source":"best_articles = articles[articles.index.isin([check_idx] + top_articles.tolist())]\nvisualize_articles(articles, best_articles['article_id'].values.tolist() ,n_total = len(best_articles) ,n_cols=6 )","metadata":{"execution":{"iopub.status.busy":"2022-03-08T13:02:41.738667Z","iopub.execute_input":"2022-03-08T13:02:41.738963Z","iopub.status.idle":"2022-03-08T13:02:43.835993Z","shell.execute_reply.started":"2022-03-08T13:02:41.738932Z","shell.execute_reply":"2022-03-08T13:02:43.834876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"article_candidates = articles.groupby('index_group_name').sample(1).index.tolist()\narticle_candidates = [58100, 105333, 40738, 23079, 36520]\nfor check_idx in article_candidates :\n    top_articles = get_best_similiarity(embedding_vector , check_idx , best_n=5)\n    best_articles = articles[articles.index.isin(top_articles.tolist())]\n    print(best_articles[\"article_id\"].tolist())\n\n    criterion = articles.iloc[check_idx,]['article_id']\n\n    visualize_articles(articles, [criterion] + best_articles['article_id'].values.tolist() ,n_total = len(best_articles) ,n_cols=6 )","metadata":{"execution":{"iopub.status.busy":"2022-03-08T13:02:45.127185Z","iopub.execute_input":"2022-03-08T13:02:45.127473Z","iopub.status.idle":"2022-03-08T13:02:59.080596Z","shell.execute_reply.started":"2022-03-08T13:02:45.127445Z","shell.execute_reply":"2022-03-08T13:02:59.079395Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install umap-learn","metadata":{"execution":{"iopub.status.busy":"2022-03-08T13:02:59.082494Z","iopub.execute_input":"2022-03-08T13:02:59.082786Z","iopub.status.idle":"2022-03-08T13:03:10.750996Z","shell.execute_reply.started":"2022-03-08T13:02:59.082754Z","shell.execute_reply":"2022-03-08T13:03:10.749027Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# UMAP ","metadata":{}},{"cell_type":"code","source":"import umap","metadata":{"execution":{"iopub.status.busy":"2022-03-08T13:03:10.757605Z","iopub.execute_input":"2022-03-08T13:03:10.758631Z","iopub.status.idle":"2022-03-08T13:03:33.880466Z","shell.execute_reply.started":"2022-03-08T13:03:10.758575Z","shell.execute_reply":"2022-03-08T13:03:33.879227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# mapper = umap.UMAP().fit(embedding_vector)","metadata":{"execution":{"iopub.status.busy":"2022-03-08T13:03:33.882023Z","iopub.execute_input":"2022-03-08T13:03:33.882274Z","iopub.status.idle":"2022-03-08T13:06:46.788464Z","shell.execute_reply.started":"2022-03-08T13:03:33.882246Z","shell.execute_reply":"2022-03-08T13:06:46.78717Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import joblib\n# filename = 'umap_mapper.sav'\n# joblib.dump(mapper, filename)","metadata":{"execution":{"iopub.status.busy":"2022-03-08T13:06:53.341513Z","iopub.execute_input":"2022-03-08T13:06:53.341914Z","iopub.status.idle":"2022-03-08T13:07:20.050703Z","shell.execute_reply.started":"2022-03-08T13:06:53.341882Z","shell.execute_reply":"2022-03-08T13:07:20.049403Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from IPython.display import FileLink\nimport os\nos.chdir(r'/kaggle/working')\nFileLink(r'umap_mapper.sav')","metadata":{"execution":{"iopub.status.busy":"2022-03-08T13:07:57.058087Z","iopub.execute_input":"2022-03-08T13:07:57.058409Z","iopub.status.idle":"2022-03-08T13:07:57.066081Z","shell.execute_reply.started":"2022-03-08T13:07:57.05838Z","shell.execute_reply":"2022-03-08T13:07:57.065416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import joblib\nfilename = '../input/umapmapper/umap_mapper.sav'\nmapper = joblib.load(filename)\n","metadata":{"execution":{"iopub.status.busy":"2022-03-08T13:38:06.594092Z","iopub.execute_input":"2022-03-08T13:38:06.594536Z","iopub.status.idle":"2022-03-08T13:38:39.97583Z","shell.execute_reply.started":"2022-03-08T13:38:06.594495Z","shell.execute_reply":"2022-03-08T13:38:39.974781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"articles.filter(regex=\"name$\")","metadata":{"execution":{"iopub.status.busy":"2022-03-08T13:39:35.243075Z","iopub.execute_input":"2022-03-08T13:39:35.244173Z","iopub.status.idle":"2022-03-08T13:39:35.283301Z","shell.execute_reply.started":"2022-03-08T13:39:35.244119Z","shell.execute_reply":"2022-03-08T13:39:35.282269Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import umap.plot","metadata":{"execution":{"iopub.status.busy":"2022-03-08T13:38:53.861663Z","iopub.execute_input":"2022-03-08T13:38:53.862003Z","iopub.status.idle":"2022-03-08T13:39:01.007084Z","shell.execute_reply.started":"2022-03-08T13:38:53.861964Z","shell.execute_reply":"2022-03-08T13:39:01.006149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# UMAP Visualization 2D","metadata":{}},{"cell_type":"code","source":"umap.plot.points(mapper, color_key_cmap='Paired', background='black')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-03-08T13:41:01.138096Z","iopub.execute_input":"2022-03-08T13:41:01.138543Z","iopub.status.idle":"2022-03-08T13:41:01.427098Z","shell.execute_reply.started":"2022-03-08T13:41:01.138484Z","shell.execute_reply":"2022-03-08T13:41:01.425763Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"umap.plot.points(mapper, labels=articles.index_group_name, color_key_cmap='Paired', background='black')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-03-08T13:40:44.759854Z","iopub.execute_input":"2022-03-08T13:40:44.76028Z","iopub.status.idle":"2022-03-08T13:40:45.547715Z","shell.execute_reply.started":"2022-03-08T13:40:44.760234Z","shell.execute_reply":"2022-03-08T13:40:45.546704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"umap.plot.points(mapper, labels=articles.index_name, color_key_cmap='Paired', background='black')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-03-08T13:42:13.470006Z","iopub.execute_input":"2022-03-08T13:42:13.47082Z","iopub.status.idle":"2022-03-08T13:42:14.920876Z","shell.execute_reply.started":"2022-03-08T13:42:13.470775Z","shell.execute_reply":"2022-03-08T13:42:14.919659Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}