{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Importing Libraries","metadata":{}},{"cell_type":"code","source":"import seaborn as sns\nimport matplotlib.pyplot as plt\nimport pandas as pd\nimport numpy as np\nfrom warnings import filterwarnings\nimport nltk\nfrom nltk.corpus import stopwords\nimport re\nimport string\nfrom nltk.stem import WordNetLemmatizer\nfrom nltk import word_tokenize\nfrom nltk.corpus import stopwords\nfrom sklearn.feature_extraction.text import TfidfVectorizer \nfrom sklearn.metrics.pairwise import cosine_similarity\nfilterwarnings('ignore')","metadata":{"execution":{"iopub.status.busy":"2022-07-24T16:33:47.740972Z","iopub.execute_input":"2022-07-24T16:33:47.741751Z","iopub.status.idle":"2022-07-24T16:33:49.753346Z","shell.execute_reply.started":"2022-07-24T16:33:47.741638Z","shell.execute_reply":"2022-07-24T16:33:49.752307Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Importing Datasets","metadata":{}},{"cell_type":"code","source":"articles = pd.read_csv(\"../input/h-and-m-personalized-fashion-recommendations/articles.csv\")\ncustomers = pd.read_csv(\"../input/h-and-m-personalized-fashion-recommendations/customers.csv\")\ntransactions = pd.read_csv(\"../input/h-and-m-personalized-fashion-recommendations/transactions_train.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-07-24T16:33:49.757646Z","iopub.execute_input":"2022-07-24T16:33:49.757879Z","iopub.status.idle":"2022-07-24T16:34:58.34258Z","shell.execute_reply.started":"2022-07-24T16:33:49.757848Z","shell.execute_reply":"2022-07-24T16:34:58.341281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Articles Dataset","metadata":{}},{"cell_type":"markdown","source":"This dataset contains products and related information about them. Rows with null values were removed from the data set.","metadata":{}},{"cell_type":"code","source":"articles.shape","metadata":{"execution":{"iopub.status.busy":"2022-07-24T16:34:58.344531Z","iopub.execute_input":"2022-07-24T16:34:58.344857Z","iopub.status.idle":"2022-07-24T16:34:58.35508Z","shell.execute_reply.started":"2022-07-24T16:34:58.344809Z","shell.execute_reply":"2022-07-24T16:34:58.353977Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"articles.columns","metadata":{"execution":{"iopub.status.busy":"2022-07-24T16:34:58.358361Z","iopub.execute_input":"2022-07-24T16:34:58.359007Z","iopub.status.idle":"2022-07-24T16:34:58.370582Z","shell.execute_reply.started":"2022-07-24T16:34:58.358961Z","shell.execute_reply":"2022-07-24T16:34:58.369447Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"articles.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-07-24T16:34:58.37213Z","iopub.execute_input":"2022-07-24T16:34:58.373105Z","iopub.status.idle":"2022-07-24T16:34:58.523569Z","shell.execute_reply.started":"2022-07-24T16:34:58.373045Z","shell.execute_reply":"2022-07-24T16:34:58.522645Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"articles = articles.dropna()","metadata":{"execution":{"iopub.status.busy":"2022-07-24T16:34:58.525543Z","iopub.execute_input":"2022-07-24T16:34:58.525861Z","iopub.status.idle":"2022-07-24T16:34:58.690372Z","shell.execute_reply.started":"2022-07-24T16:34:58.525817Z","shell.execute_reply":"2022-07-24T16:34:58.689122Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"articles.shape","metadata":{"execution":{"iopub.status.busy":"2022-07-24T16:34:58.693299Z","iopub.execute_input":"2022-07-24T16:34:58.69394Z","iopub.status.idle":"2022-07-24T16:34:58.704138Z","shell.execute_reply.started":"2022-07-24T16:34:58.693872Z","shell.execute_reply":"2022-07-24T16:34:58.702851Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"articles.info()","metadata":{"tags":[],"execution":{"iopub.status.busy":"2022-07-24T16:34:58.706434Z","iopub.execute_input":"2022-07-24T16:34:58.706953Z","iopub.status.idle":"2022-07-24T16:34:58.868488Z","shell.execute_reply.started":"2022-07-24T16:34:58.706884Z","shell.execute_reply":"2022-07-24T16:34:58.867488Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"article_object = list(articles.select_dtypes(include='object').columns)\narticle_object","metadata":{"execution":{"iopub.status.busy":"2022-07-24T16:34:58.870457Z","iopub.execute_input":"2022-07-24T16:34:58.870782Z","iopub.status.idle":"2022-07-24T16:34:58.894847Z","shell.execute_reply.started":"2022-07-24T16:34:58.870736Z","shell.execute_reply":"2022-07-24T16:34:58.893572Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Only NLP-related variables were selected from the dataset and all those variables containing text were combined in one column with the name \"text\". Since various numeric values will not be used, they were not selected.","metadata":{}},{"cell_type":"code","source":"articles[\"text\"] = articles[\"prod_name\"].map(str) + \" \" + articles[\"product_type_name\"] +\" \"+ articles[\"product_group_name\"]+ \" \"+ articles['graphical_appearance_name']+\" \"+ articles['colour_group_name'] +\" \"+ articles['perceived_colour_value_name']+ \" \" + articles[\"perceived_colour_master_name\"] +\" \"+ articles[\"department_name\"]+ \" \"+ articles['index_name']+\" \"+articles['index_group_name'] +\" \"+articles['section_name']+ \" \"+ articles['garment_group_name']+\" \"+articles['detail_desc']\narticles.head(2)","metadata":{"tags":[],"execution":{"iopub.status.busy":"2022-07-24T16:34:58.899697Z","iopub.execute_input":"2022-07-24T16:34:58.900029Z","iopub.status.idle":"2022-07-24T16:34:59.366802Z","shell.execute_reply.started":"2022-07-24T16:34:58.899996Z","shell.execute_reply":"2022-07-24T16:34:59.365621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Finally, a dataframe created only includes 'article_id', 'product_code', 'text' columns.","metadata":{}},{"cell_type":"code","source":"df_all = articles[['article_id', 'product_code', 'text']]\n#pd.set_option(\"display.max_colwidth\", -1)","metadata":{"execution":{"iopub.status.busy":"2022-07-24T16:34:59.368282Z","iopub.execute_input":"2022-07-24T16:34:59.370261Z","iopub.status.idle":"2022-07-24T16:34:59.399551Z","shell.execute_reply.started":"2022-07-24T16:34:59.37021Z","shell.execute_reply":"2022-07-24T16:34:59.3984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_all.head(2)","metadata":{"execution":{"iopub.status.busy":"2022-07-24T16:34:59.402197Z","iopub.execute_input":"2022-07-24T16:34:59.402478Z","iopub.status.idle":"2022-07-24T16:34:59.416096Z","shell.execute_reply.started":"2022-07-24T16:34:59.402424Z","shell.execute_reply":"2022-07-24T16:34:59.414984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_all[\"text\"][3]","metadata":{"execution":{"iopub.status.busy":"2022-07-24T16:34:59.418205Z","iopub.execute_input":"2022-07-24T16:34:59.419641Z","iopub.status.idle":"2022-07-24T16:34:59.437921Z","shell.execute_reply.started":"2022-07-24T16:34:59.419579Z","shell.execute_reply":"2022-07-24T16:34:59.436923Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The text variable needs to be cleared for NLP implementation. For this reason, the necessary files have been downloaded.","metadata":{}},{"cell_type":"code","source":"nltk.download('punkt')\nnltk.download('stopwords')\nnltk.download('wordnet')\nnltk.download('averaged_perceptron_tagger')","metadata":{"execution":{"iopub.status.busy":"2022-07-24T16:34:59.439648Z","iopub.execute_input":"2022-07-24T16:34:59.440125Z","iopub.status.idle":"2022-07-24T16:34:59.737645Z","shell.execute_reply.started":"2022-07-24T16:34:59.440063Z","shell.execute_reply":"2022-07-24T16:34:59.736462Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Text cleaning function defined and applied on text variable","metadata":{}},{"cell_type":"code","source":"stop = stopwords.words('english')\nstop_words_ = set(stop)\nwn= WordNetLemmatizer()\n\ndef black_txt(token):\n    return  token not in stop_words_ and token not in list(string.punctuation)  and len(token)>2   \n  \ndef clean_txt(text):\n  clean_text = []\n  clean_text2 = []\n  text = re.sub(\"'\", \"\",text)\n  text=re.sub(\"(\\\\d|\\\\W)+\",\" \",text) \n  text = text.replace(\"nbsp\", \"\")\n  clean_text = [ wn.lemmatize(word, pos=\"v\") for word in word_tokenize(text.lower()) if black_txt(word)]\n  clean_text2 = [word for word in clean_text if black_txt(word)]\n  return \" \".join(clean_text2)","metadata":{"execution":{"iopub.status.busy":"2022-07-24T16:34:59.740083Z","iopub.execute_input":"2022-07-24T16:34:59.741346Z","iopub.status.idle":"2022-07-24T16:34:59.752893Z","shell.execute_reply.started":"2022-07-24T16:34:59.741301Z","shell.execute_reply":"2022-07-24T16:34:59.751815Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"text='OP T-shirt (Idro) 1 Bra Underwear 2 Solid Black 3@Dark Black Clean Lingerie Lingeries/Tights Ladieswear Womens Lingerie Under-, Nightwear Microfibre T-shirt bra with underwired, moulded, lightly padded cups that shape the bust and provide good support. Narrow adjustable shoulder straps and a narrow hook-and-eye fastening at the back. Without visible seams for greater comfort.'","metadata":{"execution":{"iopub.status.busy":"2022-07-24T16:34:59.754713Z","iopub.execute_input":"2022-07-24T16:34:59.755196Z","iopub.status.idle":"2022-07-24T16:34:59.764268Z","shell.execute_reply.started":"2022-07-24T16:34:59.755149Z","shell.execute_reply":"2022-07-24T16:34:59.762791Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('\\n step0----',text)\ntext = re.sub(\"'\", \"\",text)\nprint('\\n step1------',text)\ntext=re.sub(\"(\\\\d|\\\\W)+\",\" \",text)\nprint('\\n step2----',text)\ntext = text.replace(\"nbsp\", \"\")\nprint('\\n step3----',text)\n\nl1= word_tokenize(text)\n\nprint('\\n step4----',l1)","metadata":{"execution":{"iopub.status.busy":"2022-07-24T16:34:59.766393Z","iopub.execute_input":"2022-07-24T16:34:59.766764Z","iopub.status.idle":"2022-07-24T16:34:59.791561Z","shell.execute_reply.started":"2022-07-24T16:34:59.766721Z","shell.execute_reply":"2022-07-24T16:34:59.790465Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_all['text'] = df_all['text'].apply(clean_txt)","metadata":{"execution":{"iopub.status.busy":"2022-07-24T16:34:59.793009Z","iopub.execute_input":"2022-07-24T16:34:59.793449Z","iopub.status.idle":"2022-07-24T16:36:29.813849Z","shell.execute_reply.started":"2022-07-24T16:34:59.793402Z","shell.execute_reply":"2022-07-24T16:36:29.812819Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_all.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-24T16:36:29.815533Z","iopub.execute_input":"2022-07-24T16:36:29.816463Z","iopub.status.idle":"2022-07-24T16:36:29.830022Z","shell.execute_reply.started":"2022-07-24T16:36:29.816402Z","shell.execute_reply":"2022-07-24T16:36:29.828747Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Initializing tfidf vectorizer for articles, fitting and transforming the vector","metadata":{}},{"cell_type":"code","source":"tfidf_vectorizer = TfidfVectorizer()\ntfidf_article = tfidf_vectorizer.fit_transform((df_all['text'])) \ntfidf_article.toarray()\n","metadata":{"execution":{"iopub.status.busy":"2022-07-24T16:36:29.831567Z","iopub.execute_input":"2022-07-24T16:36:29.83335Z","iopub.status.idle":"2022-07-24T16:36:42.937221Z","shell.execute_reply.started":"2022-07-24T16:36:29.833307Z","shell.execute_reply":"2022-07-24T16:36:42.935862Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Transactions Dataset","metadata":{}},{"cell_type":"code","source":"transactions.columns","metadata":{"execution":{"iopub.status.busy":"2022-07-24T16:36:42.938969Z","iopub.execute_input":"2022-07-24T16:36:42.939403Z","iopub.status.idle":"2022-07-24T16:36:42.948207Z","shell.execute_reply.started":"2022-07-24T16:36:42.939358Z","shell.execute_reply":"2022-07-24T16:36:42.946964Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# transactions['InvoiceDate'] = pd.to_datetime(transactions['t_dat'],format='%Y-%m-%d')\n# transactions=transactions[[\"InvoiceDate\", \"customer_id\", \"article_id\", \"price\",\"sales_channel_id\"]].drop_duplicates()","metadata":{"execution":{"iopub.status.busy":"2022-07-24T16:36:42.950144Z","iopub.execute_input":"2022-07-24T16:36:42.950742Z","iopub.status.idle":"2022-07-24T16:36:42.958007Z","shell.execute_reply.started":"2022-07-24T16:36:42.9507Z","shell.execute_reply":"2022-07-24T16:36:42.956631Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transactions = transactions.dropna()","metadata":{"execution":{"iopub.status.busy":"2022-07-24T16:36:42.959486Z","iopub.execute_input":"2022-07-24T16:36:42.961073Z","iopub.status.idle":"2022-07-24T16:36:52.718564Z","shell.execute_reply.started":"2022-07-24T16:36:42.961028Z","shell.execute_reply":"2022-07-24T16:36:52.717434Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transactions.shape","metadata":{"execution":{"iopub.status.busy":"2022-07-24T16:36:52.720408Z","iopub.execute_input":"2022-07-24T16:36:52.720718Z","iopub.status.idle":"2022-07-24T16:36:52.729549Z","shell.execute_reply.started":"2022-07-24T16:36:52.720673Z","shell.execute_reply":"2022-07-24T16:36:52.72843Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transactions=transactions[:10000000]","metadata":{"execution":{"iopub.status.busy":"2022-07-24T16:36:52.73125Z","iopub.execute_input":"2022-07-24T16:36:52.732165Z","iopub.status.idle":"2022-07-24T16:36:52.739502Z","shell.execute_reply.started":"2022-07-24T16:36:52.732116Z","shell.execute_reply":"2022-07-24T16:36:52.738237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Sorting the dataset by customer id to see all of a customer's purchases","metadata":{}},{"cell_type":"code","source":"transactions =  transactions.sort_values(by='customer_id')\ntransactions.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-24T16:36:52.741039Z","iopub.execute_input":"2022-07-24T16:36:52.741711Z","iopub.status.idle":"2022-07-24T16:37:11.316072Z","shell.execute_reply.started":"2022-07-24T16:36:52.741666Z","shell.execute_reply":"2022-07-24T16:37:11.314962Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transactions.shape","metadata":{"execution":{"iopub.status.busy":"2022-07-24T16:37:11.317694Z","iopub.execute_input":"2022-07-24T16:37:11.318676Z","iopub.status.idle":"2022-07-24T16:37:11.326147Z","shell.execute_reply.started":"2022-07-24T16:37:11.318627Z","shell.execute_reply":"2022-07-24T16:37:11.325081Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Merging the text&product_id (df_all) dataset with transactions dataset to match article_ids with purchases made by a customer.","metadata":{}},{"cell_type":"code","source":"merged_df = df_all.merge(transactions, how = 'inner', on = ['article_id'])\n","metadata":{"execution":{"iopub.status.busy":"2022-07-24T16:37:11.333709Z","iopub.execute_input":"2022-07-24T16:37:11.334644Z","iopub.status.idle":"2022-07-24T16:37:18.118149Z","shell.execute_reply.started":"2022-07-24T16:37:11.334597Z","shell.execute_reply":"2022-07-24T16:37:18.117141Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"merged_df.head(2)","metadata":{"execution":{"iopub.status.busy":"2022-07-24T16:37:18.119857Z","iopub.execute_input":"2022-07-24T16:37:18.12023Z","iopub.status.idle":"2022-07-24T16:37:18.137369Z","shell.execute_reply.started":"2022-07-24T16:37:18.120187Z","shell.execute_reply":"2022-07-24T16:37:18.135961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The text information of all the products purchased by the user are gathered in the same 'text' variable.","metadata":{}},{"cell_type":"code","source":"merged_df2 = merged_df.groupby('customer_id', sort=False)['text'].apply(' '.join).reset_index()","metadata":{"execution":{"iopub.status.busy":"2022-07-24T16:37:18.139455Z","iopub.execute_input":"2022-07-24T16:37:18.139831Z","iopub.status.idle":"2022-07-24T16:37:50.955955Z","shell.execute_reply.started":"2022-07-24T16:37:18.139786Z","shell.execute_reply":"2022-07-24T16:37:50.954882Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"merged_df2['customer_id'][0]","metadata":{"execution":{"iopub.status.busy":"2022-07-24T16:37:50.957893Z","iopub.execute_input":"2022-07-24T16:37:50.958285Z","iopub.status.idle":"2022-07-24T16:37:50.967089Z","shell.execute_reply.started":"2022-07-24T16:37:50.958225Z","shell.execute_reply":"2022-07-24T16:37:50.965881Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Recommendation**","metadata":{"tags":[]}},{"cell_type":"markdown","source":"A random customer_id was chosen to make a reccommendation","metadata":{}},{"cell_type":"code","source":"u = \"001ae5408a043f64bccd32beffe2730151414cbdf18a6eb3cc8d30bdca605652\" #customer_id\nindex = np.where(merged_df2['customer_id'] == u)[0][0]\ncust_q = merged_df2.iloc[[index]]\ncust_q","metadata":{"execution":{"iopub.status.busy":"2022-07-24T16:41:25.187286Z","iopub.execute_input":"2022-07-24T16:41:25.187617Z","iopub.status.idle":"2022-07-24T16:41:25.438259Z","shell.execute_reply.started":"2022-07-24T16:41:25.187586Z","shell.execute_reply":"2022-07-24T16:41:25.437046Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Products user bought before","metadata":{"tags":[]}},{"cell_type":"code","source":"transactions.loc[transactions['customer_id'] == u]","metadata":{"execution":{"iopub.status.busy":"2022-07-24T16:42:04.22689Z","iopub.execute_input":"2022-07-24T16:42:04.227768Z","iopub.status.idle":"2022-07-24T16:42:05.815297Z","shell.execute_reply.started":"2022-07-24T16:42:04.227729Z","shell.execute_reply":"2022-07-24T16:42:05.814064Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Define a Reccommendation Function ","metadata":{}},{"cell_type":"markdown","source":"Recommendation function includes customer ID, article ID, product code, description and similarity score.","metadata":{}},{"cell_type":"code","source":"def recommendation_product(top, df_all, scores):\n  recommendation = pd.DataFrame(columns = ['customer_id', 'article_id',  'product_code', 'detail_desc', 'score'])\n  count = 0\n  for i in top:\n      recommendation.at[count, 'customer_id'] = u\n      recommendation.at[count, 'article_id'] = df_all['article_id'][i]\n      recommendation.at[count, 'product_code'] = df_all['product_code'][i]\n      recommendation.at[count, 'detail_desc'] = articles['detail_desc'][i]   \n      recommendation.at[count, 'score'] =  scores[count]\n      count += 1\n  return recommendation","metadata":{"execution":{"iopub.status.busy":"2022-07-24T16:37:53.293256Z","iopub.execute_input":"2022-07-24T16:37:53.293905Z","iopub.status.idle":"2022-07-24T16:37:53.302878Z","shell.execute_reply.started":"2022-07-24T16:37:53.293839Z","shell.execute_reply":"2022-07-24T16:37:53.301616Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Calculating Cosine Similarity for the User","metadata":{}},{"cell_type":"code","source":"user_tfidf = tfidf_vectorizer.transform(cust_q['text'])\ncos_similarity_tfidf = map(lambda x: cosine_similarity(user_tfidf, x),tfidf_article)","metadata":{"execution":{"iopub.status.busy":"2022-07-24T16:37:53.30496Z","iopub.execute_input":"2022-07-24T16:37:53.305699Z","iopub.status.idle":"2022-07-24T16:37:53.317463Z","shell.execute_reply.started":"2022-07-24T16:37:53.30565Z","shell.execute_reply":"2022-07-24T16:37:53.316452Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"output2 = list(cos_similarity_tfidf)\n","metadata":{"execution":{"iopub.status.busy":"2022-07-24T16:37:53.321351Z","iopub.execute_input":"2022-07-24T16:37:53.322204Z","iopub.status.idle":"2022-07-24T16:39:39.969806Z","shell.execute_reply.started":"2022-07-24T16:37:53.321948Z","shell.execute_reply":"2022-07-24T16:39:39.968772Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Recommendations with TFIDF","metadata":{}},{"cell_type":"code","source":"top = sorted(range(len(output2)), key=lambda i: output2[i], reverse=True)[:10]\ntf_list_scores = [output2[i][0][0] for i in top]\nrecommendation_product(top, df_all, tf_list_scores)","metadata":{"execution":{"iopub.status.busy":"2022-07-24T16:39:39.971702Z","iopub.execute_input":"2022-07-24T16:39:39.971946Z","iopub.status.idle":"2022-07-24T16:39:41.557896Z","shell.execute_reply.started":"2022-07-24T16:39:39.971915Z","shell.execute_reply":"2022-07-24T16:39:41.556867Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf_idf_score=pd.DataFrame(recommendation_product(top, df_all, tf_list_scores), columns = ['article_id', 'score'])\ntf_idf_score","metadata":{"execution":{"iopub.status.busy":"2022-07-24T16:39:41.559565Z","iopub.execute_input":"2022-07-24T16:39:41.561669Z","iopub.status.idle":"2022-07-24T16:39:41.588874Z","shell.execute_reply.started":"2022-07-24T16:39:41.561636Z","shell.execute_reply":"2022-07-24T16:39:41.587969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Reccomendations with CountVectorizer","metadata":{}},{"cell_type":"code","source":"from sklearn.feature_extraction.text import CountVectorizer\ncount_vectorizer = CountVectorizer()\n\ncount_artid = count_vectorizer.fit_transform((df_all['text'])) #fitting and transforming the vector\ncount_artid","metadata":{"execution":{"iopub.status.busy":"2022-07-24T16:39:41.59047Z","iopub.execute_input":"2022-07-24T16:39:41.590809Z","iopub.status.idle":"2022-07-24T16:39:45.736041Z","shell.execute_reply.started":"2022-07-24T16:39:41.590754Z","shell.execute_reply":"2022-07-24T16:39:45.73492Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics.pairwise import cosine_similarity\nuser_count = count_vectorizer.transform(cust_q['text'])\ncos_similarity_countv = map(lambda x: cosine_similarity(user_count, x),count_artid)","metadata":{"execution":{"iopub.status.busy":"2022-07-24T16:39:45.737959Z","iopub.execute_input":"2022-07-24T16:39:45.738383Z","iopub.status.idle":"2022-07-24T16:39:45.744894Z","shell.execute_reply.started":"2022-07-24T16:39:45.738337Z","shell.execute_reply":"2022-07-24T16:39:45.743813Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"output3 = list(cos_similarity_countv)","metadata":{"execution":{"iopub.status.busy":"2022-07-24T16:39:45.74649Z","iopub.execute_input":"2022-07-24T16:39:45.748225Z","iopub.status.idle":"2022-07-24T16:40:53.431129Z","shell.execute_reply.started":"2022-07-24T16:39:45.748164Z","shell.execute_reply":"2022-07-24T16:40:53.429249Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cv_score=pd.DataFrame(recommendation_product(top, df_all, list_scores_cv), columns = ['article_id', 'score'])","metadata":{"execution":{"iopub.status.busy":"2022-07-24T16:40:53.433077Z","iopub.status.idle":"2022-07-24T16:40:53.433948Z","shell.execute_reply.started":"2022-07-24T16:40:53.433561Z","shell.execute_reply":"2022-07-24T16:40:53.433597Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Reccommendations with KNN","metadata":{}},{"cell_type":"code","source":"from sklearn.neighbors import NearestNeighbors\nKNN = NearestNeighbors(n_neighbors=11)\nKNN.fit(tfidf_article)\nNNs = KNN.kneighbors(user_tfidf, return_distance=True) ","metadata":{"execution":{"iopub.status.busy":"2022-07-24T16:40:53.435511Z","iopub.status.idle":"2022-07-24T16:40:53.436646Z","shell.execute_reply.started":"2022-07-24T16:40:53.436329Z","shell.execute_reply":"2022-07-24T16:40:53.436363Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"top = NNs[1][0][1:]\nindex_score = NNs[0][0][1:]\nrecommendation_product(top, df_all, index_score)","metadata":{"execution":{"iopub.status.busy":"2022-07-24T16:40:53.438487Z","iopub.status.idle":"2022-07-24T16:40:53.439259Z","shell.execute_reply.started":"2022-07-24T16:40:53.438931Z","shell.execute_reply":"2022-07-24T16:40:53.438964Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"knn_score=pd.DataFrame(recommendation_product(top, df_all, index_score), columns = ['article_id', 'score'])","metadata":{"execution":{"iopub.status.busy":"2022-07-24T16:40:53.446123Z","iopub.status.idle":"2022-07-24T16:40:53.447094Z","shell.execute_reply.started":"2022-07-24T16:40:53.44675Z","shell.execute_reply":"2022-07-24T16:40:53.446784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Comparison ","metadata":{}},{"cell_type":"code","source":"tf_idf_score=tf_idf_score.rename(columns={\"score\":\"tf_idf_score\"})\ncv_score=cv_score.rename(columns={\"score\":\"cv_score\"})\nknn_score=knn_score.rename(columns={\"score\":\"knn_score\"})","metadata":{"execution":{"iopub.status.busy":"2022-07-24T16:40:53.448991Z","iopub.status.idle":"2022-07-24T16:40:53.449972Z","shell.execute_reply.started":"2022-07-24T16:40:53.449648Z","shell.execute_reply":"2022-07-24T16:40:53.44968Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.concat([tf_idf_score, cv_score, knn_score], axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-07-24T16:40:53.451692Z","iopub.status.idle":"2022-07-24T16:40:53.452618Z","shell.execute_reply.started":"2022-07-24T16:40:53.452263Z","shell.execute_reply":"2022-07-24T16:40:53.452297Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"It seems that while knn and tf-idf make **almost** the same recommendations, the system based on countvectorizer makes different recommendations.","metadata":{}}]}