{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import random\nimport time\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\nfrom matplotlib.pyplot import figure\nfrom collections import Counter\n\ntransactions = pd.read_csv('/kaggle/input/h-and-m-personalized-fashion-recommendations/transactions_train.csv')\nsample_submission = pd.read_csv('/kaggle/input/h-and-m-personalized-fashion-recommendations/sample_submission.csv')\n# /kaggle/input/h-and-m-personalized-fashion-recommendations/sample_submission.csv\n# /kaggle/input/h-and-m-personalized-fashion-recommendations/articles.csv\n# /kaggle/input/h-and-m-personalized-fashion-recommendations/transactions_train.csv\n# /kaggle/input/h-and-m-personalized-fashion-recommendations/customers.csv\n\nplt.rcParams[\"figure.figsize\"] = (15,5)","metadata":{"execution":{"iopub.status.busy":"2023-01-30T21:15:56.419835Z","iopub.execute_input":"2023-01-30T21:15:56.420758Z","iopub.status.idle":"2023-01-30T21:17:36.492142Z","shell.execute_reply.started":"2023-01-30T21:15:56.420593Z","shell.execute_reply":"2023-01-30T21:17:36.490930Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transactions.head()","metadata":{"execution":{"iopub.status.busy":"2023-01-30T21:17:36.494571Z","iopub.execute_input":"2023-01-30T21:17:36.494885Z","iopub.status.idle":"2023-01-30T21:17:36.522368Z","shell.execute_reply.started":"2023-01-30T21:17:36.494847Z","shell.execute_reply":"2023-01-30T21:17:36.521095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission.head()","metadata":{"execution":{"iopub.status.busy":"2023-01-30T21:17:36.524003Z","iopub.execute_input":"2023-01-30T21:17:36.524368Z","iopub.status.idle":"2023-01-30T21:17:36.535882Z","shell.execute_reply.started":"2023-01-30T21:17:36.524324Z","shell.execute_reply":"2023-01-30T21:17:36.534795Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data analysis","metadata":{}},{"cell_type":"code","source":"# Sold products over the years\ndf = transactions\ndf['year_month'] = df['t_dat'].apply(lambda x : x.split('-')[0]+'-'+x.split('-')[1])\ndf = df.groupby(['year_month']).count().reset_index()\n_ = plt.plot(df['year_month'], df['t_dat'])\n_ = plt.xticks(rotation=90)","metadata":{"execution":{"iopub.status.busy":"2023-01-30T21:17:36.537522Z","iopub.execute_input":"2023-01-30T21:17:36.537868Z","iopub.status.idle":"2023-01-30T21:18:13.413583Z","shell.execute_reply.started":"2023-01-30T21:17:36.537824Z","shell.execute_reply":"2023-01-30T21:18:13.412582Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Sold product each month\ndf = transactions\ndf['month'] = df['t_dat'].apply(lambda x : x.split('-')[1])\ndf = df.groupby(['month']).count().reset_index()\n_ = plt.bar(df['month'], df['t_dat'], align='center')","metadata":{"execution":{"iopub.status.busy":"2023-01-30T21:18:13.415800Z","iopub.execute_input":"2023-01-30T21:18:13.416089Z","iopub.status.idle":"2023-01-30T21:18:58.406386Z","shell.execute_reply.started":"2023-01-30T21:18:13.416053Z","shell.execute_reply":"2023-01-30T21:18:58.402427Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Top products\ndf = transactions.groupby(['article_id']).count().reset_index()[[\"article_id\", \"customer_id\"]].sort_values([\"customer_id\"], ascending=False)\nprint(df)\n_ = plt.bar(df['article_id'], df['customer_id'], align='center')","metadata":{"execution":{"iopub.status.busy":"2023-01-30T21:18:58.408797Z","iopub.execute_input":"2023-01-30T21:18:58.409180Z","iopub.status.idle":"2023-01-30T21:22:32.398398Z","shell.execute_reply.started":"2023-01-30T21:18:58.409142Z","shell.execute_reply":"2023-01-30T21:22:32.397009Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_ = plt.boxplot(df['customer_id'])","metadata":{"execution":{"iopub.status.busy":"2023-01-30T21:22:32.400597Z","iopub.execute_input":"2023-01-30T21:22:32.400852Z","iopub.status.idle":"2023-01-30T21:22:32.576896Z","shell.execute_reply.started":"2023-01-30T21:22:32.400821Z","shell.execute_reply":"2023-01-30T21:22:32.576114Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transactions","metadata":{"execution":{"iopub.status.busy":"2023-01-30T21:22:32.578408Z","iopub.execute_input":"2023-01-30T21:22:32.578867Z","iopub.status.idle":"2023-01-30T21:22:32.599791Z","shell.execute_reply.started":"2023-01-30T21:22:32.578817Z","shell.execute_reply":"2023-01-30T21:22:32.598766Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Number of products by basket (considering one day = one basket as we do not have order_id)\ntransactions_gb = transactions.groupby(['t_dat', 'customer_id'])['article_id'].apply(list).reset_index()  \ntransactions_gb[\"len_basket\"] = transactions_gb[\"article_id\"].apply(lambda x: len(x))\ntransactions_gb = transactions_gb.sort_values([\"len_basket\"])\n\n# Taking off outliers\ntransactions_gb_t = transactions_gb[:len(transactions_gb['len_basket'])-2000]\n\n# Plot\n_ = plt.hist(transactions_gb_t['len_basket'], density=True, bins=range(max(list(set(transactions_gb_t['len_basket'])))), rwidth=0.8)\n_ = plt.xticks(list(set(transactions_gb_t['len_basket'])))","metadata":{"execution":{"iopub.status.busy":"2023-01-30T21:22:32.601252Z","iopub.execute_input":"2023-01-30T21:22:32.601536Z","iopub.status.idle":"2023-01-30T21:27:23.599216Z","shell.execute_reply.started":"2023-01-30T21:22:32.601496Z","shell.execute_reply":"2023-01-30T21:27:23.598252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Cleaning + create train and test dataset","metadata":{}},{"cell_type":"code","source":"# Truncate transactions since the data is too large for Kaggle cluster\ntransactions = transactions[transactions[\"t_dat\"]>='2020-07-01']","metadata":{"execution":{"iopub.status.busy":"2023-01-30T21:27:23.600638Z","iopub.execute_input":"2023-01-30T21:27:23.601977Z","iopub.status.idle":"2023-01-30T21:27:25.803363Z","shell.execute_reply.started":"2023-01-30T21:27:23.601923Z","shell.execute_reply":"2023-01-30T21:27:25.802267Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Considering each article_id as str\ntransactions['article_id'] = transactions['article_id'].astype(str)","metadata":{"execution":{"iopub.status.busy":"2023-01-30T21:27:25.804990Z","iopub.execute_input":"2023-01-30T21:27:25.805282Z","iopub.status.idle":"2023-01-30T21:27:28.030782Z","shell.execute_reply.started":"2023-01-30T21:27:25.805247Z","shell.execute_reply":"2023-01-30T21:27:28.029595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Splitting train and test as 30%\ntest_set = 0.3\ntrain_set = (1-test_set)\nsplit = int(transactions.shape[0]*train_set)\n\ntransactions_train = transactions.sort_values('t_dat').iloc[:split]\ntransactions_test = transactions.sort_values('t_dat').iloc[split:]\nprint(\"Train set size :\", transactions_train.shape[0])\nprint(\"Test set size :\", transactions_test.shape[0])","metadata":{"execution":{"iopub.status.busy":"2023-01-30T21:27:28.032228Z","iopub.execute_input":"2023-01-30T21:27:28.032680Z","iopub.status.idle":"2023-01-30T21:27:33.561575Z","shell.execute_reply.started":"2023-01-30T21:27:28.032645Z","shell.execute_reply":"2023-01-30T21:27:33.559463Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Baseline : Best sellers recommender\n\nOur Baseline will be the well known best sellers.","metadata":{}},{"cell_type":"code","source":"class BestSeller():\n    \"\"\"FIXME\"\"\"\n    def __init__(self):\n        \"\"\"FIXME\"\"\"\n        pass\n    \n    def fit(self, X):\n        \"\"\"FIXME\"\"\"\n        self.X_occur = Counter(list(X['article_id']))\n        self.best_sellers_10 = [article_id for (article_id, occ) in self.X_occur.most_common(10)]\n        return self\n    \n    def predict(self, basket, k=10):\n        \"\"\"FIXME\"\"\"\n        if k==10:\n            return self.best_sellers_10\n        else:\n            best_sellers = [article_id for (article_id, occ) in self.X_occur.most_common(k)]\n            return best_sellers","metadata":{"execution":{"iopub.status.busy":"2023-01-30T21:27:33.563508Z","iopub.execute_input":"2023-01-30T21:27:33.563765Z","iopub.status.idle":"2023-01-30T21:27:33.576525Z","shell.execute_reply.started":"2023-01-30T21:27:33.563735Z","shell.execute_reply":"2023-01-30T21:27:33.574789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bs_model = BestSeller()\nbs_model.fit(transactions_train)","metadata":{"execution":{"iopub.status.busy":"2023-01-30T21:27:33.582581Z","iopub.execute_input":"2023-01-30T21:27:33.582924Z","iopub.status.idle":"2023-01-30T21:27:34.587865Z","shell.execute_reply.started":"2023-01-30T21:27:33.582884Z","shell.execute_reply":"2023-01-30T21:27:34.586550Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bs_model.predict([])","metadata":{"execution":{"iopub.status.busy":"2023-01-30T21:27:34.589632Z","iopub.execute_input":"2023-01-30T21:27:34.589991Z","iopub.status.idle":"2023-01-30T21:27:34.599042Z","shell.execute_reply.started":"2023-01-30T21:27:34.589932Z","shell.execute_reply":"2023-01-30T21:27:34.597612Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Cosine Similarity\n\nCosine similarity is a measure of similarity that can be used to get the similarity of two objects. It uses the cosine value between two vectors to compute the similarity of two data points. As the distances between the vectors increases, the similiraty reduces.","metadata":{}},{"cell_type":"code","source":"from math import sqrt\n\nclass CosineSim():\n    \"\"\"FIXME\"\"\"\n    def __init__(self):\n        \"\"\"FIXME\"\"\"\n        pass\n    \n    def _get_l_of_two_from_l(self, l):\n        \"\"\"FIXME\"\"\"\n        # Dropping duplicates to avoid having more co-occur than occur\n        l = list(set(l))\n        if len(l)>1:\n            l_of_two = []\n            for i in range(0,len(l)):\n                for j in range(i+1,len(l)):\n                    if int(l[i])<int(l[j]):\n                        l_of_two.append(f\"{l[i]}|{l[j]}\")\n                    else:\n                        l_of_two.append(f\"{l[j]}|{l[i]}\")\n            return l_of_two\n        return None\n    \n    def _get_frequency(self, article_id, counter_occur_article_id):\n        \"\"\"FIXME\"\"\"\n        return counter_occur_article_id[article_id]\n    \n    def _get_co_occ_frequency(self, article_id_a, article_id_b, counter_co_occ_article_id):\n        \"\"\"FIXME\"\"\"\n        if int(article_id_a) < int(article_id_b):\n            str_co_occ = article_id_a + '|' + article_id_b\n        else:\n            str_co_occ = article_id_b + '|' + article_id_a\n        return counter_co_occ_article_id[str_co_occ]\n    \n    def _get_cosine_sim(self, freq_a, freq_b, co_occ_freq):\n        \"\"\"FIXME\"\"\"\n        return (co_occ_freq/sqrt(freq_a*freq_b))\n        \n    def fit(self, X, top_n=200, threshold=0.005):\n        \"\"\"FIXME\"\"\"        \n        # Keeping only top N articles\n        start_time = time.time()\n        counter_article_id = Counter(X['article_id'])\n        list_top_n = sorted(counter_article_id, key=counter_article_id.get, reverse=True)[:top_n]\n        X = X[X['article_id'].isin(list_top_n)]\n        print(\"--- %s seconds ---\" % (time.time() - start_time))\n        \n        # Getting baskets\n        start_time = time.time()\n        X_grouped = X.groupby(['t_dat', 'customer_id'])['article_id'].apply(list).reset_index()\n        print(\"--- %s seconds ---\" % (time.time() - start_time))\n        \n        # Getting frequencies of article_ids\n        start_time = time.time()\n        l_baskets_len2 = list(X_grouped[X_grouped['article_id'].map(len) >1]['article_id'])\n        l_baskets_len2_flat_list = [x for xs in l_baskets_len2 for x in xs]\n        counter_occur_article_id = Counter(l_baskets_len2_flat_list)\n        print(\"--- %s seconds ---\" % (time.time() - start_time))\n\n        # Getting co-occur frequencies of article_ids\n        start_time = time.time()\n        l_co_occur_article_id = []\n        for basket in l_baskets_len2:\n            l_of_two = self._get_l_of_two_from_l(basket)\n            if l_of_two:\n                l_co_occur_article_id += l_of_two\n\n        counter_co_occ_article_id = Counter(l_co_occur_article_id)\n        print(\"--- %s seconds ---\" % (time.time() - start_time))\n        \n        # Creating pandas dataframe\n        start_time = time.time()\n        l_article_id_a = [i.split('|', 1)[0] for i in l_co_occur_article_id]\n        l_article_id_b = [i.split('|', 1)[1] for i in l_co_occur_article_id]\n        df_cosine = pd.DataFrame()\n        df_cosine['article_id_a'] = l_article_id_a\n        df_cosine['article_id_b'] = l_article_id_b\n        print(\"--- %s seconds ---\" % (time.time() - start_time))\n        \n        start_time = time.time()\n        df_cosine['freq_a'] = df_cosine['article_id_a'].apply(lambda x : self._get_frequency(x, counter_occur_article_id))\n        print(\"--- %s seconds ---\" % (time.time() - start_time))\n        \n        start_time = time.time()\n        df_cosine['freq_b'] = df_cosine['article_id_b'].apply(lambda x : self._get_frequency(x, counter_occur_article_id))\n        print(\"--- %s seconds ---\" % (time.time() - start_time))\n        \n        start_time = time.time()\n        df_cosine['co_occ_freq'] = df_cosine.apply(lambda x: self._get_co_occ_frequency(x.article_id_a, x.article_id_b, counter_co_occ_article_id), axis=1)\n        print(\"--- %s seconds ---\" % (time.time() - start_time))\n        \n        start_time = time.time()\n        df_cosine['cosine_sim'] = df_cosine.apply(lambda x: self._get_cosine_sim(x.freq_a, x.freq_b, x.co_occ_freq), axis=1)\n        print(\"--- %s seconds ---\" % (time.time() - start_time))\n        \n        df_cosine = df_cosine.sort_values('cosine_sim', ascending=False)\n        \n        # Filtering with threshold\n        self.df_cosine = df_cosine[df_cosine[\"cosine_sim\"]>threshold]\n\n        return self\n    \n    def predict(self, basket, k=10, strat='mean'):\n        \"\"\"FIXME\"\"\"\n        df = self.df_cosine[(self.df_cosine['article_id_a'].isin(basket)) & (~self.df_cosine['article_id_b'].isin(basket))]\n        \n        if strat=='mean':\n            result = list(df[['article_id_b', 'cosine_sim']].groupby(['article_id_b']).mean().reset_index()['article_id_b'])[:k]\n        elif strat=='max':\n            result = list(df[['article_id_b', 'cosine_sim']].groupby(['article_id_b']).max().reset_index()['article_id_b'])[:k]\n        else:\n            result = []\n        return result","metadata":{"execution":{"iopub.status.busy":"2023-01-30T21:27:34.601973Z","iopub.execute_input":"2023-01-30T21:27:34.602544Z","iopub.status.idle":"2023-01-30T21:27:34.636425Z","shell.execute_reply.started":"2023-01-30T21:27:34.602487Z","shell.execute_reply":"2023-01-30T21:27:34.635161Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# FIXME : Optimize for loops, taking ages to run\ncs_model = CosineSim().fit(transactions_train, top_n=500)","metadata":{"execution":{"iopub.status.busy":"2023-01-30T21:27:34.637648Z","iopub.execute_input":"2023-01-30T21:27:34.637943Z","iopub.status.idle":"2023-01-30T21:28:01.571722Z","shell.execute_reply.started":"2023-01-30T21:27:34.637910Z","shell.execute_reply":"2023-01-30T21:28:01.570399Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cs_model.predict(['610776001'])","metadata":{"execution":{"iopub.status.busy":"2023-01-30T21:28:01.573185Z","iopub.execute_input":"2023-01-30T21:28:01.573503Z","iopub.status.idle":"2023-01-30T21:28:01.664348Z","shell.execute_reply.started":"2023-01-30T21:28:01.573467Z","shell.execute_reply":"2023-01-30T21:28:01.663265Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# ALS","metadata":{}},{"cell_type":"code","source":"!pip install pyspark > /dev/null","metadata":{"execution":{"iopub.status.busy":"2023-01-30T21:28:01.666453Z","iopub.execute_input":"2023-01-30T21:28:01.666808Z","iopub.status.idle":"2023-01-30T21:29:10.286598Z","shell.execute_reply.started":"2023-01-30T21:28:01.666759Z","shell.execute_reply":"2023-01-30T21:29:10.285286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pyspark\n\nfrom pyspark import SparkContext, SparkConf\nfrom pyspark.sql import SQLContext, SparkSession","metadata":{"execution":{"iopub.status.busy":"2023-01-30T21:29:10.288774Z","iopub.execute_input":"2023-01-30T21:29:10.289087Z","iopub.status.idle":"2023-01-30T21:29:10.381340Z","shell.execute_reply.started":"2023-01-30T21:29:10.289047Z","shell.execute_reply":"2023-01-30T21:29:10.379983Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"conf = SparkConf().set(\"spark.ui.port\", \"4050\") \\\n                  .set('spark.executor.memory', '9G') \\\n                  .set('spark.driver.memory', '7G') \\\n                  .set('spark.sql.autoBroadcastJoinThreshold', '-1')\n\nsc = pyspark.SparkContext(conf=conf)\nspark = SparkSession.builder.getOrCreate()","metadata":{"execution":{"iopub.status.busy":"2023-01-30T21:29:10.382974Z","iopub.execute_input":"2023-01-30T21:29:10.383341Z","iopub.status.idle":"2023-01-30T21:29:17.677772Z","shell.execute_reply.started":"2023-01-30T21:29:10.383297Z","shell.execute_reply":"2023-01-30T21:29:17.676188Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from pyspark.ml.recommendation import ALS, ALSModel","metadata":{"execution":{"iopub.status.busy":"2023-01-30T21:29:17.679897Z","iopub.execute_input":"2023-01-30T21:29:17.680314Z","iopub.status.idle":"2023-01-30T21:29:17.686353Z","shell.execute_reply.started":"2023-01-30T21:29:17.680261Z","shell.execute_reply":"2023-01-30T21:29:17.685414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Mapping on whole dataset\ncustomer_map = transactions[[\"customer_id\"]].drop_duplicates().reset_index(drop=True)\ncustomer_map[\"USER\"] = customer_map.index\n\nitem_map = transactions[[\"article_id\"]].drop_duplicates().reset_index(drop=True)\nitem_map[\"ITEM\"] = item_map.index\n\ncustomer_map_dict = dict(zip(customer_map['USER'].values, customer_map['customer_id'].values))\nitem_map_dict = dict(zip(item_map['ITEM'].values, item_map['article_id'].values))\n","metadata":{"execution":{"iopub.status.busy":"2023-01-30T21:29:17.688914Z","iopub.execute_input":"2023-01-30T21:29:17.689699Z","iopub.status.idle":"2023-01-30T21:29:21.326565Z","shell.execute_reply.started":"2023-01-30T21:29:17.689624Z","shell.execute_reply":"2023-01-30T21:29:21.325552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transactions_train_als = (\n        transactions_train\n        .merge(customer_map, on=\"customer_id\", how=\"inner\")\n        .merge(item_map, on=\"article_id\", how=\"inner\")\n    )","metadata":{"execution":{"iopub.status.busy":"2023-01-30T21:29:21.328526Z","iopub.execute_input":"2023-01-30T21:29:21.329031Z","iopub.status.idle":"2023-01-30T21:29:28.250830Z","shell.execute_reply.started":"2023-01-30T21:29:21.328994Z","shell.execute_reply":"2023-01-30T21:29:28.249398Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transactions_train_als.head()","metadata":{"execution":{"iopub.status.busy":"2023-01-30T21:29:28.252754Z","iopub.execute_input":"2023-01-30T21:29:28.253127Z","iopub.status.idle":"2023-01-30T21:29:28.273970Z","shell.execute_reply.started":"2023-01-30T21:29:28.253077Z","shell.execute_reply":"2023-01-30T21:29:28.272586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_ratings = transactions_train_als.head(1000).groupby(['USER', 'ITEM']).count().reset_index()[[\"USER\", \"ITEM\", \"price\"]]\ndf_ratings.columns = ['USER', 'ITEM', 'RATING']","metadata":{"execution":{"iopub.status.busy":"2023-01-30T21:29:28.276155Z","iopub.execute_input":"2023-01-30T21:29:28.276549Z","iopub.status.idle":"2023-01-30T21:29:28.297825Z","shell.execute_reply.started":"2023-01-30T21:29:28.276493Z","shell.execute_reply":"2023-01-30T21:29:28.296628Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_ratings","metadata":{"execution":{"iopub.status.busy":"2023-01-30T21:29:28.299975Z","iopub.execute_input":"2023-01-30T21:29:28.300387Z","iopub.status.idle":"2023-01-30T21:29:28.317539Z","shell.execute_reply.started":"2023-01-30T21:29:28.300334Z","shell.execute_reply":"2023-01-30T21:29:28.316231Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_ratings_ps = spark.createDataFrame(df_ratings) \nprint(df_ratings_ps.printSchema())\nprint(df_ratings_ps.show())","metadata":{"execution":{"iopub.status.busy":"2023-01-30T21:29:28.319723Z","iopub.execute_input":"2023-01-30T21:29:28.320138Z","iopub.status.idle":"2023-01-30T21:29:36.123920Z","shell.execute_reply.started":"2023-01-30T21:29:28.320093Z","shell.execute_reply":"2023-01-30T21:29:36.122959Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_mat_sparsity(ratings):\n    # Count the total number of ratings in the dataset\n    count_nonzero = ratings.select(\"RATING\").count()\n\n    # Count the number of distinct userIds and distinct articleIds\n    total_elements = ratings.select(\"USER\").distinct().count() * ratings.select(\"ITEM\").distinct().count()\n\n    # Divide the numerator by the denominator\n    sparsity = (1.0 - (count_nonzero *1.0)/total_elements)*100\n    print(\"The ratings dataframe is \", \"%.2f\" % sparsity + \"% sparse.\")\n    \nget_mat_sparsity(df_ratings_ps)","metadata":{"execution":{"iopub.status.busy":"2023-01-30T21:29:36.126097Z","iopub.execute_input":"2023-01-30T21:29:36.126496Z","iopub.status.idle":"2023-01-30T21:29:40.273592Z","shell.execute_reply.started":"2023-01-30T21:29:36.126448Z","shell.execute_reply":"2023-01-30T21:29:40.272429Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Import the required functions\nfrom pyspark.ml.evaluation import RegressionEvaluator\nfrom pyspark.ml.recommendation import ALS\nfrom pyspark.ml.tuning import ParamGridBuilder, CrossValidator\n\n# Create ALS model\nals = ALS(\n         userCol=\"USER\", \n         itemCol=\"ITEM\",\n         ratingCol=\"RATING\", \n         nonnegative = True, \n         implicitPrefs = False,\n         coldStartStrategy=\"drop\"\n)","metadata":{"execution":{"iopub.status.busy":"2023-01-30T21:29:40.275465Z","iopub.execute_input":"2023-01-30T21:29:40.275834Z","iopub.status.idle":"2023-01-30T21:29:40.713528Z","shell.execute_reply.started":"2023-01-30T21:29:40.275784Z","shell.execute_reply":"2023-01-30T21:29:40.712708Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Import the requisite packages\nfrom pyspark.ml.tuning import ParamGridBuilder, CrossValidator","metadata":{"execution":{"iopub.status.busy":"2023-01-30T21:29:40.714634Z","iopub.execute_input":"2023-01-30T21:29:40.714895Z","iopub.status.idle":"2023-01-30T21:29:41.127832Z","shell.execute_reply.started":"2023-01-30T21:29:40.714863Z","shell.execute_reply":"2023-01-30T21:29:41.126750Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Add hyperparameters and their respective values to param_grid\nparam_grid = ParamGridBuilder() \\\n            .addGrid(als.rank, [10, 50, 100, 150]) \\\n            .addGrid(als.regParam, [.01, .05, .1, .15]) \\\n            .build()","metadata":{"execution":{"iopub.status.busy":"2023-01-30T21:29:41.129240Z","iopub.execute_input":"2023-01-30T21:29:41.130503Z","iopub.status.idle":"2023-01-30T21:29:41.141014Z","shell.execute_reply.started":"2023-01-30T21:29:41.130451Z","shell.execute_reply":"2023-01-30T21:29:41.140319Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from pyspark.ml.evaluation import RegressionEvaluator\n\n# Define evaluator as RMSE and print length of evaluator\nevaluator = RegressionEvaluator(\n           metricName=\"rmse\", \n           labelCol=\"RATING\", \n           predictionCol=\"prediction\") \nprint (\"Num models to be tested: \", len(param_grid))","metadata":{"execution":{"iopub.status.busy":"2023-01-30T22:04:44.141455Z","iopub.execute_input":"2023-01-30T22:04:44.141861Z","iopub.status.idle":"2023-01-30T22:04:44.158648Z","shell.execute_reply.started":"2023-01-30T22:04:44.141824Z","shell.execute_reply":"2023-01-30T22:04:44.157488Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Build cross validation using CrossValidator\ncv = CrossValidator(estimator=als, estimatorParamMaps=param_grid, evaluator=evaluator, numFolds=5)","metadata":{"execution":{"iopub.status.busy":"2023-01-30T22:04:44.560605Z","iopub.execute_input":"2023-01-30T22:04:44.561382Z","iopub.status.idle":"2023-01-30T22:04:44.566659Z","shell.execute_reply.started":"2023-01-30T22:04:44.561320Z","shell.execute_reply":"2023-01-30T22:04:44.565637Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Fit cross validator to the 'train' dataset\nmodel = cv.fit(df_ratings_ps)\n\n# Extract best model from the cv model above\nbest_model = model.bestModel","metadata":{"execution":{"iopub.status.busy":"2023-01-30T22:04:44.828842Z","iopub.execute_input":"2023-01-30T22:04:44.829158Z","iopub.status.idle":"2023-01-30T22:11:33.759322Z","shell.execute_reply.started":"2023-01-30T22:04:44.829123Z","shell.execute_reply":"2023-01-30T22:11:33.757341Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Best Model\")\nprint(\"Rank:\", best_model._java_obj.parent().getRank())\nprint(\"MaxIter:\", best_model._java_obj.parent().getMaxIter())\nprint(\"RegParam:\", best_model._java_obj.parent().getRegParam())","metadata":{"execution":{"iopub.status.busy":"2023-01-30T22:11:33.763669Z","iopub.execute_input":"2023-01-30T22:11:33.764051Z","iopub.status.idle":"2023-01-30T22:11:33.780702Z","shell.execute_reply.started":"2023-01-30T22:11:33.764012Z","shell.execute_reply":"2023-01-30T22:11:33.779329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"recommendations = best_model.recommendForAllUsers(5)","metadata":{"execution":{"iopub.status.busy":"2023-01-30T22:11:33.784501Z","iopub.execute_input":"2023-01-30T22:11:33.786523Z","iopub.status.idle":"2023-01-30T22:11:34.145823Z","shell.execute_reply.started":"2023-01-30T22:11:33.786455Z","shell.execute_reply":"2023-01-30T22:11:34.144414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"recommendations.show(10, False)","metadata":{"execution":{"iopub.status.busy":"2023-01-30T22:13:07.999983Z","iopub.execute_input":"2023-01-30T22:13:08.001239Z","iopub.status.idle":"2023-01-30T22:13:11.586748Z","shell.execute_reply.started":"2023-01-30T22:13:08.001125Z","shell.execute_reply":"2023-01-30T22:13:11.585664Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def history_basket_als(X_test, als):\n#     \"\"\"FIXME\"\"\"\n#     recommendations = als.recommendForAllUsers(5)","metadata":{"execution":{"iopub.status.busy":"2023-01-30T22:11:34.178892Z","iopub.execute_input":"2023-01-30T22:11:34.179439Z","iopub.status.idle":"2023-01-30T22:11:34.185123Z","shell.execute_reply.started":"2023-01-30T22:11:34.179393Z","shell.execute_reply":"2023-01-30T22:11:34.183727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# transactions_test_als = (\n#         transactions_test\n#         .merge(customer_map, on=\"customer_id\", how=\"inner\")\n#         .merge(item_map, on=\"article_id\", how=\"inner\")\n#     )","metadata":{"execution":{"iopub.status.busy":"2023-01-30T21:41:42.846284Z","iopub.status.idle":"2023-01-30T21:41:42.846857Z","shell.execute_reply.started":"2023-01-30T21:41:42.846598Z","shell.execute_reply":"2023-01-30T21:41:42.846627Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# transactions_test_als","metadata":{"execution":{"iopub.status.busy":"2023-01-30T21:41:42.850383Z","iopub.status.idle":"2023-01-30T21:41:42.851100Z","shell.execute_reply.started":"2023-01-30T21:41:42.850737Z","shell.execute_reply":"2023-01-30T21:41:42.850774Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# transactions_test_als.head(1000).groupby(['t_dat','customer_id', 'USER']).ITEM.apply(list).reset_index().head(1000)","metadata":{"execution":{"iopub.status.busy":"2023-01-30T21:41:42.853717Z","iopub.status.idle":"2023-01-30T21:41:42.854379Z","shell.execute_reply.started":"2023-01-30T21:41:42.854014Z","shell.execute_reply":"2023-01-30T21:41:42.854051Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # View the predictions\n# test_predictions = best_model.transform(test)\n# RMSE = evaluator.evaluate(test_predictions)\n# print(RMSE)","metadata":{"execution":{"iopub.status.busy":"2023-01-30T21:41:42.856644Z","iopub.status.idle":"2023-01-30T21:41:42.857291Z","shell.execute_reply.started":"2023-01-30T21:41:42.856916Z","shell.execute_reply":"2023-01-30T21:41:42.856948Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## OLD WORK\n# customer_map = transactions[[\"customer_id\"]].drop_duplicates().reset_index(drop=True)\n# customer_map[\"USER\"] = customer_map.index\n\n# item_map = df[[\"article_id\"]].drop_duplicates().reset_index(drop=True)\n# item_map[\"ITEM\"] = item_map.index\n\n# customer_map_dict = dict(zip(customer_map['USER'].values, customer_map['customer_id'].values))\n# item_map_dict = dict(zip(item_map['ITEM'].values, item_map['article_id'].values))","metadata":{"execution":{"iopub.status.busy":"2023-01-30T21:41:42.860966Z","iopub.status.idle":"2023-01-30T21:41:42.862124Z","shell.execute_reply.started":"2023-01-30T21:41:42.861502Z","shell.execute_reply":"2023-01-30T21:41:42.861557Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# transactions_train['article_id'] = transactions_train['article_id'].astype(str)\n# transactions_train.info()","metadata":{"execution":{"iopub.status.busy":"2023-01-30T21:41:42.864954Z","iopub.status.idle":"2023-01-30T21:41:42.865669Z","shell.execute_reply.started":"2023-01-30T21:41:42.865292Z","shell.execute_reply":"2023-01-30T21:41:42.865331Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# transactions_train_als = (\n#         transactions_train\n#         .merge(customer_map, on=\"customer_id\", how=\"inner\")\n#         .merge(item_map, on=\"article_id\", how=\"inner\")\n#     )\n# transactions_train_als[\"rating\"] = 1","metadata":{"execution":{"iopub.status.busy":"2023-01-30T21:41:42.874121Z","iopub.status.idle":"2023-01-30T21:41:42.875916Z","shell.execute_reply.started":"2023-01-30T21:41:42.875441Z","shell.execute_reply":"2023-01-30T21:41:42.875510Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# transactions_train_als.head()","metadata":{"execution":{"iopub.status.busy":"2023-01-30T21:41:42.882611Z","iopub.status.idle":"2023-01-30T21:41:42.883705Z","shell.execute_reply.started":"2023-01-30T21:41:42.883340Z","shell.execute_reply":"2023-01-30T21:41:42.883384Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# mod = 10000000\n# transactions_train_als = transactions_train[[\"customer_id\", \"article_id\"]].head(1000)\n# transactions_train_als[\"rating\"] = 1\n# transactions_train_als['customer_id_hash'] = transactions_train_als['customer_id'].apply(lambda x : hash(x) % mod)\n# transactions_train_als['article_id_hash'] = transactions_train_als['article_id'].apply(lambda x : hash(x) % mod)","metadata":{"execution":{"iopub.status.busy":"2023-01-30T21:41:42.886980Z","iopub.status.idle":"2023-01-30T21:41:42.888664Z","shell.execute_reply.started":"2023-01-30T21:41:42.888218Z","shell.execute_reply":"2023-01-30T21:41:42.888273Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# transactions_train_als_spark = spark.createDataFrame(transactions_train_als) ","metadata":{"execution":{"iopub.status.busy":"2023-01-30T21:41:42.891059Z","iopub.status.idle":"2023-01-30T21:41:42.892160Z","shell.execute_reply.started":"2023-01-30T21:41:42.891792Z","shell.execute_reply":"2023-01-30T21:41:42.891835Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# transactions_train_als_spark.show()","metadata":{"execution":{"iopub.status.busy":"2023-01-30T21:41:42.895318Z","iopub.status.idle":"2023-01-30T21:41:42.896940Z","shell.execute_reply.started":"2023-01-30T21:41:42.896535Z","shell.execute_reply":"2023-01-30T21:41:42.896585Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# als = ALS(userCol=\"customer_id_hash\", itemCol=\"article_id_hash\", ratingCol=\"rating\", coldStartStrategy=\"drop\")\n\n# # param_grid = ParamGridBuilder().addGrid(als.rank, [25]) \\\n# #                                .addGrid(als.regParam, [0.1, 0.15]) \\\n# #                                .addGrid(als.maxIter, [10]) \\\n# #                                .build()","metadata":{"execution":{"iopub.status.busy":"2023-01-30T21:41:42.900475Z","iopub.status.idle":"2023-01-30T21:41:42.902951Z","shell.execute_reply.started":"2023-01-30T21:41:42.902404Z","shell.execute_reply":"2023-01-30T21:41:42.902487Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# model = als.fit(transactions_train_als_spark)","metadata":{"execution":{"iopub.status.busy":"2023-01-30T21:41:42.906840Z","iopub.status.idle":"2023-01-30T21:41:42.908129Z","shell.execute_reply.started":"2023-01-30T21:41:42.907777Z","shell.execute_reply":"2023-01-30T21:41:42.907837Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# transactions_test_als = transactions_test[[\"customer_id\", \"article_id\"]].head(1000)\n# #transactions_test_als[\"rating\"] = 1\n# transactions_test_als['customer_id_hash'] = transactions_test_als['customer_id'].apply(lambda x : hash(x) % mod)\n# transactions_test_als['article_id_hash'] = transactions_test_als['article_id'].apply(lambda x : hash(x) % mod)\n# transactions_test_als_spark = spark.createDataFrame(transactions_test_als) ","metadata":{"execution":{"iopub.status.busy":"2023-01-30T21:41:42.910856Z","iopub.status.idle":"2023-01-30T21:41:42.912098Z","shell.execute_reply.started":"2023-01-30T21:41:42.911684Z","shell.execute_reply":"2023-01-30T21:41:42.911745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# predictions = model.transform(transactions_test_als_spark)","metadata":{"execution":{"iopub.status.busy":"2023-01-30T21:41:42.915233Z","iopub.status.idle":"2023-01-30T21:41:42.916511Z","shell.execute_reply.started":"2023-01-30T21:41:42.916109Z","shell.execute_reply":"2023-01-30T21:41:42.916150Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# predictions.show()","metadata":{"execution":{"iopub.status.busy":"2023-01-30T21:41:42.919545Z","iopub.status.idle":"2023-01-30T21:41:42.920765Z","shell.execute_reply.started":"2023-01-30T21:41:42.920432Z","shell.execute_reply":"2023-01-30T21:41:42.920472Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !pip install --upgrade implicit","metadata":{"execution":{"iopub.status.busy":"2023-01-30T21:41:42.923536Z","iopub.status.idle":"2023-01-30T21:41:42.924756Z","shell.execute_reply.started":"2023-01-30T21:41:42.924428Z","shell.execute_reply":"2023-01-30T21:41:42.924467Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import implicit","metadata":{"execution":{"iopub.status.busy":"2023-01-30T21:41:42.927743Z","iopub.status.idle":"2023-01-30T21:41:42.929029Z","shell.execute_reply.started":"2023-01-30T21:41:42.928674Z","shell.execute_reply":"2023-01-30T21:41:42.928715Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# model = implicit.als.AlternatingLeastSquares(factors=10, iterations=2)\n# model.fit(coo_train)","metadata":{"execution":{"iopub.status.busy":"2023-01-30T21:41:42.932037Z","iopub.status.idle":"2023-01-30T21:41:42.933296Z","shell.execute_reply.started":"2023-01-30T21:41:42.932901Z","shell.execute_reply":"2023-01-30T21:41:42.932958Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# transactions_train.head()","metadata":{"execution":{"iopub.status.busy":"2023-01-30T21:41:42.938328Z","iopub.status.idle":"2023-01-30T21:41:42.939805Z","shell.execute_reply.started":"2023-01-30T21:41:42.939431Z","shell.execute_reply":"2023-01-30T21:41:42.939471Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# transactions_train_als = transactions_train[[\"customer_id\", \"article_id\"]].head(1000)\n# transactions_train_als[\"rating\"] = 1","metadata":{"execution":{"iopub.status.busy":"2023-01-30T21:41:42.943689Z","iopub.status.idle":"2023-01-30T21:41:42.945347Z","shell.execute_reply.started":"2023-01-30T21:41:42.944917Z","shell.execute_reply":"2023-01-30T21:41:42.944963Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !pip install pyspark","metadata":{"execution":{"iopub.status.busy":"2023-01-30T21:41:42.948526Z","iopub.status.idle":"2023-01-30T21:41:42.949725Z","shell.execute_reply.started":"2023-01-30T21:41:42.949378Z","shell.execute_reply":"2023-01-30T21:41:42.949432Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import pyspark\n\n# from pyspark.ml.recommendation import ALS\n\n# als = ALS(maxIter=5, regParam=0.01, userCol=\"customer_id\", itemCol=\"article_id\", ratingCol=\"rating\", coldStartStrategy=\"drop\")\n# # model = als.fit(transactions_train_als)","metadata":{"execution":{"iopub.status.busy":"2023-01-30T21:41:42.952529Z","iopub.status.idle":"2023-01-30T21:41:42.953737Z","shell.execute_reply.started":"2023-01-30T21:41:42.953374Z","shell.execute_reply":"2023-01-30T21:41:42.953432Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from pyspark.ml.recommendation import ALS\n\n# als = ALS(maxIter=10, regParam=1, userCol=\"USER\", itemCol=\"ITEM\", ratingCol=\"RATING\", coldStartStrategy=\"drop\", implicitPrefs=True, alpha=100, rank=200, seed=1)","metadata":{"execution":{"iopub.status.busy":"2023-01-30T21:41:42.955908Z","iopub.status.idle":"2023-01-30T21:41:42.956861Z","shell.execute_reply.started":"2023-01-30T21:41:42.956515Z","shell.execute_reply":"2023-01-30T21:41:42.956552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Evaluating recommenders","metadata":{}},{"cell_type":"code","source":"def is_predicted(customer_id, order, model, k, debug=False):\n    \"\"\"FIXME\"\"\"\n    if debug:\n        print(\"##\")\n        print(\"order:\", order)\n    order_ = order.copy()\n    hidden_article_id = order_.pop(random.randrange(len(order_)))\n    visible_order = order_\n    \n    start_time = time.time()\n    predictions = model.predict(visible_order, k)\n    if debug:\n        print(time.time() - start_time)\n    \n    if debug:\n        print(\"visible_order:\", visible_order)\n        print(\"predictions:\", predictions)\n        print(\"hidden_article_id:\", hidden_article_id)\n    \n    if hidden_article_id in predictions:\n        if debug:\n            print(1)\n        return 1\n    else:\n        if debug:\n            print(0)\n        return 0","metadata":{"execution":{"iopub.status.busy":"2023-01-30T22:13:33.937765Z","iopub.execute_input":"2023-01-30T22:13:33.938142Z","iopub.status.idle":"2023-01-30T22:13:33.948310Z","shell.execute_reply.started":"2023-01-30T22:13:33.938097Z","shell.execute_reply":"2023-01-30T22:13:33.947293Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def history_basket_performance(X, model, k=10):\n    \"\"\"FIXME\"\"\"\n    # Grouping to have orders by day\n    X_orders = X.groupby(['t_dat', 'customer_id'])['article_id'].apply(list).reset_index()    \n    \n    # Keeping only orders where we can hide at least one article and have at least one item in our basket\n    X_orders['is_long_enough'] = X_orders['article_id'].apply(lambda x: 1 if(len(x)>1) else 0)\n    X_orders = X_orders[X_orders['is_long_enough']==1]\n        \n    v_is_predicted = np.vectorize(lambda x, y, model, k: is_predicted(x, y, model, k))\n\n    X_orders['predicted'] = v_is_predicted(X_orders.customer_id, X_orders.article_id, model, k)\n\n    is_predicted_count = X_orders[X_orders['predicted']==1].shape[0]\n    count = X_orders.shape[0]\n    \n    return (count, is_predicted_count/count)","metadata":{"execution":{"iopub.status.busy":"2023-01-30T22:13:34.255725Z","iopub.execute_input":"2023-01-30T22:13:34.256362Z","iopub.status.idle":"2023-01-30T22:13:34.268350Z","shell.execute_reply.started":"2023-01-30T22:13:34.256315Z","shell.execute_reply":"2023-01-30T22:13:34.266872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history_basket_performance(transactions_test.head(1000), bs_model)","metadata":{"execution":{"iopub.status.busy":"2023-01-30T22:13:34.700642Z","iopub.execute_input":"2023-01-30T22:13:34.700982Z","iopub.status.idle":"2023-01-30T22:13:34.744710Z","shell.execute_reply.started":"2023-01-30T22:13:34.700949Z","shell.execute_reply":"2023-01-30T22:13:34.743524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history_basket_performance(transactions_test.head(1000), cs_model)","metadata":{"execution":{"iopub.status.busy":"2023-01-30T22:13:35.070027Z","iopub.execute_input":"2023-01-30T22:13:35.070426Z","iopub.status.idle":"2023-01-30T22:13:54.174941Z","shell.execute_reply.started":"2023-01-30T22:13:35.070387Z","shell.execute_reply":"2023-01-30T22:13:54.174016Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# (205, 0.24878048780487805)","metadata":{"execution":{"iopub.status.busy":"2023-01-30T21:41:42.970630Z","iopub.status.idle":"2023-01-30T21:41:42.971567Z","shell.execute_reply.started":"2023-01-30T21:41:42.971246Z","shell.execute_reply":"2023-01-30T21:41:42.971284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Evaluating top_n in cosine_sim","metadata":{}},{"cell_type":"code","source":"# [100, 200, 300, 400, 500, 600, 700, 800, 900, 1000]\nl_top_n = [350, 500, 750, 1000, 1250, 1500, 2000]\nl_perf = []\nfor top_n in l_top_n:\n    print(\"##\")\n    print(top_n)\n    cs_model = CosineSim().fit(transactions_train, top_n)\n    perf = history_basket_performance(transactions_test.head(3000), cs_model)\n    l_perf.append(perf[1])\n    print(perf)","metadata":{"execution":{"iopub.status.busy":"2023-01-30T21:41:42.973313Z","iopub.status.idle":"2023-01-30T21:41:42.974247Z","shell.execute_reply.started":"2023-01-30T21:41:42.973863Z","shell.execute_reply":"2023-01-30T21:41:42.973910Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from scipy.interpolate import make_interp_spline\n\nx = np.array(l_top_n)\ny = np.array(l_perf)\n\nX_Y_Spline = make_interp_spline(x, y)\n \n# Returns evenly spaced numbers over a specified interval.\nX_ = np.linspace(x.min(), x.max(), 100)\nY_ = X_Y_Spline(X_)\n\n_ = plt.plot(X_, Y_)\n_ = plt.xticks(rotation=90)","metadata":{"execution":{"iopub.status.busy":"2023-01-30T21:41:42.976926Z","iopub.status.idle":"2023-01-30T21:41:42.977941Z","shell.execute_reply.started":"2023-01-30T21:41:42.977609Z","shell.execute_reply":"2023-01-30T21:41:42.977656Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}