{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Import libraries\n","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport glob\n#import reco\nfrom tqdm import tqdm\nimport datetime\nfrom functools import partial\nfrom dask.diagnostics import ProgressBar\nProgressBar().register()\nimport dask.dataframe as dd\nfrom collections import Counter\nfrom sklearn.preprocessing import OneHotEncoder","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tqdm.pandas()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Read data","metadata":{}},{"cell_type":"code","source":"data = pd.read_csv('transactions_train.csv',dtype={'article_id':str})\nprint(data.shape)","metadata":{"execution":{"iopub.execute_input":"2022-03-04T08:30:05.168987Z","iopub.status.busy":"2022-03-04T08:30:05.16854Z","iopub.status.idle":"2022-03-04T08:30:08.730796Z","shell.execute_reply":"2022-03-04T08:30:08.730119Z","shell.execute_reply.started":"2022-03-04T08:30:05.168956Z"}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Calculate groups of price:","metadata":{}},{"cell_type":"code","source":"data['price2'] = pd.qcut(data['price'], 10)\ndata.head()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.t_dat = pd.to_datetime(data.t_dat)#data.t_dat.progress_apply(lambda x: datetime.datetime.strptime(x,'%Y-%m-%d'))\ndata = data[['t_dat','customer_id','article_id','price2']]","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Select only the last weeks\n\nThis way we will keep the relevant data and keep its size reasonable. We will take 2 weeks for training and leave the last one for validation","metadata":{}},{"cell_type":"code","source":"print(\"All Transactions Date Range: {} to {}\".format(data['t_dat'].min(), data['t_dat'].max()))\n\ntrain1 = data.loc[(data[\"t_dat\"] >= datetime.datetime(2020,9,8)) & (data['t_dat'] < datetime.datetime(2020,9,16))]\ntrain2 = data.loc[(data[\"t_dat\"] >= datetime.datetime(2020,9,1)) & (data['t_dat'] < datetime.datetime(2020,9,8))]\ntrain3 = data.loc[(data[\"t_dat\"] >= datetime.datetime(2020,8,23)) & (data['t_dat'] < datetime.datetime(2020,9,1))]\ntrain4 = data.loc[(data[\"t_dat\"] >= datetime.datetime(2020,8,15)) & (data['t_dat'] < datetime.datetime(2020,8,23))]\n\nval = data.loc[data[\"t_dat\"] >= datetime.datetime(2020,9,16)]\n\n#del data","metadata":{"execution":{"iopub.execute_input":"2022-03-04T08:30:08.732604Z","iopub.status.busy":"2022-03-04T08:30:08.732251Z","iopub.status.idle":"2022-03-04T08:30:08.772891Z","shell.execute_reply":"2022-03-04T08:30:08.772184Z","shell.execute_reply.started":"2022-03-04T08:30:08.732568Z"}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# List of all purchases per user (has repetitions)\npositive_items_per_user1 = train1.groupby(['customer_id'])['article_id'].apply(list)\npositive_items_per_user2 = train2.groupby(['customer_id'])['article_id'].apply(list)\npositive_items_per_user3 = train3.groupby(['customer_id'])['article_id'].apply(list)\npositive_items_per_user4 = train4.groupby(['customer_id'])['article_id'].apply(list)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.concat([train1,train2], axis=0) #train2","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train = train.drop('t_dat', axis = 1).reset_index(drop = True)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"scrolled":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Keep only the transactions for users that have more than 20 transactions for training the FM:","metadata":{}},{"cell_type":"code","source":"train_fm = train[['customer_id','article_id','price2']].copy()\nprint(train_fm.shape)\nv = train_fm.customer_id.value_counts()\ntrain_fm = train_fm[train_fm.customer_id.isin(v.index[v.gt(20)])]\nprint(train_fm.shape)\nprint(f\"There're {len(train_fm.customer_id.value_counts())} users\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Add other characteristics (TRAIN)","metadata":{}},{"cell_type":"markdown","source":"Load age of customers and calculate deciles to add them to train dataset:","metadata":{}},{"cell_type":"code","source":"customers = pd.read_csv('customers.csv')\ncustomers = customers[['customer_id','age']]\ntrain_fm = train_fm.merge(customers, how='left', on='customer_id')\ntrain_fm['age'] = pd.qcut(train_fm['age'], 10)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Load article dataset and add to train the group category of the items and the colour group name:","metadata":{}},{"cell_type":"code","source":"articles = pd.read_csv('articles.csv',dtype={'article_id':str})\narticles = articles[['article_id','product_group_name','perceived_colour_value_name','index_name']]\n\ntrain_fm = train_fm.merge(articles, how='left', on='article_id')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Finally, add the last item bought for every customer:","metadata":{}},{"cell_type":"code","source":"shifted = train_fm[['customer_id','article_id']].groupby(\"customer_id\").shift(+1)\ntrain_fm = train_fm.join(shifted.rename(columns=lambda x: x+\"_lag\"))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_fm","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Add other characteristics (VAL)","metadata":{}},{"cell_type":"code","source":"# val = val.drop('t_dat', axis = 1).reset_index(drop = True)\nval_fm = val.merge(customers, how='left', on='customer_id')\nval_fm['age'] = pd.qcut(val_fm['age'], 10)\n\nval_fm = val_fm.merge(articles, how='left', on='article_id')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"shifted_val = val_fm[['customer_id','article_id']].groupby(\"customer_id\").shift(+1)\nval_fm = val_fm.join(shifted_val.rename(columns=lambda x: x+\"_lag\"))\nval_fm","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Factorization Machine","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt \nimport math\n\nclass Oh_factorization_machine(): \n        \n    def __init__(self,df,df_val, caract,num_components=10):\n        \"\"\" Constructor \"\"\"\n       \n        self.caract = caract\n        self.articles = df[['article_id', 'index_name']].drop_duplicates()\n        self.data     = self.__generatedf__(df[self.caract])\n        self.data_val = self.__generatedf__(df[self.caract].merge(df_val[self.caract], how='inner', on=self.caract))\n        \n        print(self.data_val.shape)\n        \n        \n        #self.all_data = self.__generatedf__(pd.concat([self.data,self.data_val],axis=0))\n    \n        \n        self.ratings = self.data['rating'].values\n        self.ratings_val = self.data_val['rating'].values\n        \n        self.K = num_components\n        \n        self.oh = OneHotEncoder(sparse = True)\n        \n        self.oh_matrix = self.oh.fit_transform(self.data[self.caract])\n        self.oh_matrix_val = self.oh.transform(self.data_val[self.caract])\n        \n        \n        print(self.oh_matrix.shape, self.oh_matrix_val.shape)\n        \n        \n    def get_df(self):\n        return self.data\n        \n    def __generatedf__(self,df):\n        \n        df['rating'] = 1\n        \n        #print(df)\n        \n        data2 = df.sample(frac=0.1, random_state=42).copy()\n        data2['article_id'] = data2['article_id'].sample(frac = 1).values\n        data2 = data2.drop_duplicates(subset=['article_id','customer_id'])\n        #For the rmse:\n        data2['rating'] = 0\n\n#         For the log loss:\n#         data2['rating'] = -1\n        \n        data = pd.concat([df,data2]).sample(frac=1)\n        \n        del data2\n    \n        return data\n        \n        \n    def __sgd__(self,epoch):\n        \"\"\"\n        Perform stochastic gradient descend\n        \"\"\"\n        for idx in tqdm(self.training_indices, leave=True,desc='Epoch {}'.format(epoch),postfix='train_error: {:.3f}  val_error: {:.3f}'.format(self.train_rmse[-1],self.val_rmse[-1])):\n            \n            row = self.oh_matrix[idx]\n            y = self.ratings[idx]\n            \n            #print(row.todense())\n            prediction = self.predict(row)\n        \n            error  = (y - prediction)\n            \n            self.w0 += self.learning_rate * error\n            \n            self.bias += self.learning_rate * error * row \n            \n            self.V[row.nonzero()[1]] += (self.learning_rate * error * (-self.V[row.nonzero()[1]] + row@self.V))\n            \n                \n    def fit(self, n_epochs = 50, learning_rate = 0.001, lmbda=0.1, verbose =True, patience=15, early=True):\n        \"\"\" Train the model. \"\"\"\n        self.verbose = verbose\n        self.learning_rate = learning_rate\n        self.lmbda = lmbda\n      \n        self.n_cols = self.oh_matrix.shape[1]\n        \n        self.train_rmse =[]\n        self.val_rmse = []\n        iter_diff = 0\n        \n        # initialize latent vectors\n        self.V = np.random.normal(scale=1./self.K,\n                                          size=(self.n_cols,self.K))      \n        \n        self.bias = np.random.normal(scale=1/self.n_cols,size=(self.n_cols)) \n        \n        self.w0 = 1\n    \n        for epoch in range(n_epochs):\n            \n            self.training_indices = np.arange(self.n_cols)\n            \n            #shuffle training samples\n            np.random.shuffle(self.training_indices)\n            \n            self.train_rmse.append(self.evaluate(self.predict,self.ratings,self.oh_matrix))\n            self.val_rmse.append(self.evaluate(self.predict,self.ratings_val,self.oh_matrix_val))\n            \n            self.__sgd__(epoch)\n            \n            \n            if early and len(self.val_rmse) > patience and self.val_rmse[-1] >= self.val_rmse[-patience]:\n                if verbose: \n                    print(\"Early stopping at epoch {}\".format(epoch))\n                break\n            \n        \n        if(self.verbose):\n            self.__plot_learning_curves__()\n            \n    def __plot_learning_curves__(self):\n        plt.plot(self.train_rmse,'--o',label=\"train_error\")\n        plt.plot(self.val_rmse,'--o',label=\"val_error\")\n        plt.legend()\n        plt.show()\n    \n    def predict(self, row):\n        \"\"\" Single transaction.\"\"\"\n        row = np.ravel(row.todense())\n        a = self.w0 \n        b = np.dot(row,self.bias.T)\n        c = 0\n        \n        try: \n            b = np.ravel(b)\n        except:\n            pass\n        \n        #print(row.shape,row[1:].nonzero()[0])\n        \n        index = row.nonzero()[0]\n        c = np.dot(self.V[index[0],:],self.V[index[1],:])\n                \n        sigmoid = lambda x : 1 / (1 + math.exp(-x))\n        \n        \n        return sigmoid(a + b + c)\n    \n    def compute_rmse(self,y_pred, y_true):\n        \"\"\" Compute Root Mean Squared Error. \"\"\"\n        \n        self.error = (y_pred - y_true)\n        return np.median(np.abs(y_pred - y_true))\n    \n    def log_loss(self,y_pred, y_true):\n        \"\"\" Compute Log Loss. \"\"\"\n        return np.log(np.exp(-y_pred * y_true) + 1.0)\n    \n    def evaluate(self,predict_f,y_set,data_train):\n        \"\"\"\n        data_train = oh_matrix\n        y_set = rating (val,train)\n        \n        \"\"\"\n        #For every transaction row:\n        #For rmse:\n        prediction = [self.predict(data_train[idx]) for idx in range(data_train.shape[0])]\n        return self.compute_rmse(prediction, y_set)\n        \n        #For log-loss:\n        #loss = [self.log_loss(self.predict(data_train[idx]), y_set[idx]) for idx in range(data_train.shape[0])]\n        \n        #return np.mean(loss)\n\n    \n    \n    def __top_12__(self,user):\n        \n        #For index_name:\n        article_df = self.articles[['article_id', 'index_name']]\n            \n        article_df['customer_id'] = user\n        \n        article_df = article_df[['customer_id', 'article_id', 'index_name']]\n        \n        to_predict = self.oh.transform(article_df[self.caract])\n        article_df['pred'] = [self.predict(to_predict[i]) for i in range(to_predict.shape[0])]\n        article_df = article_df.sort_values(by=['pred'], ascending = False)[:12]\n        \n        return list(article_df['article_id'])\n    ","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We will only add the index name, because we think is a very representative feature of the item:","metadata":{}},{"cell_type":"code","source":"fact_machine = Oh_factorization_machine(train_fm,val_fm,caract=['customer_id','article_id','index_name'],num_components=50)","metadata":{"scrolled":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fact_machine.fit(patience=8)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Predict top12 items for every user trained","metadata":{}},{"cell_type":"code","source":"users = dict(map(lambda x: x[::-1],enumerate(train_fm.customer_id.unique())))\nrecom = pd.DataFrame(list(users.keys()),columns=['customer_id'])\nrecom['recom'] = recom.customer_id.map(lambda x: fact_machine.__top_12__(x))\nrecom","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Non-personalized \n\nFor users that aren't in our training we will use a non-personalized method. The method will be the same as the last assingment, so we will compute the top items in our data as well as stratify the customer population in ages (divided in deciles), as we suppose different intervals will buy different items. \n\nThe best items will have a score sensible to time, as items that have not been bought in the recent weeks might not be as relevant as new ones. ","metadata":{}},{"cell_type":"markdown","source":"Let's read the customer dataset to find the age:","metadata":{}},{"cell_type":"code","source":"customers = pd.read_csv('customers.csv')\ncustomers = customers[['customer_id','age']]","metadata":{"execution":{"iopub.execute_input":"2022-03-04T08:30:13.578933Z","iopub.status.busy":"2022-03-04T08:30:13.578681Z","iopub.status.idle":"2022-03-04T08:30:13.815258Z","shell.execute_reply":"2022-03-04T08:30:13.814542Z","shell.execute_reply.started":"2022-03-04T08:30:13.578906Z"}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Merge it with the training dataset:","metadata":{}},{"cell_type":"code","source":"train = train.merge(customers, how='left', on='customer_id')\nval = val.merge(customers, how='left', on='customer_id')","metadata":{"execution":{"iopub.execute_input":"2022-03-04T08:30:16.796227Z","iopub.status.busy":"2022-03-04T08:30:16.795981Z","iopub.status.idle":"2022-03-04T08:30:17.303134Z","shell.execute_reply":"2022-03-04T08:30:17.302434Z","shell.execute_reply.started":"2022-03-04T08:30:16.7962Z"}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Calculate deciles of age and a popularity factor based on the time of the transaction:","metadata":{}},{"cell_type":"code","source":"train['age2'] = pd.qcut(train['age'], 10)\ntrain['age2'].value_counts()\ntrain['pop_factor'] = train['t_dat'].apply(lambda x: 1/(datetime.datetime(2020,9,16) - x).days)","metadata":{"execution":{"iopub.execute_input":"2022-03-04T08:30:18.647795Z","iopub.status.busy":"2022-03-04T08:30:18.646071Z","iopub.status.idle":"2022-03-04T08:30:23.849316Z","shell.execute_reply":"2022-03-04T08:30:23.848589Z","shell.execute_reply.started":"2022-03-04T08:30:18.647749Z"},"scrolled":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We define the function that calculates the most frequent bought items on the training dataset given an age that is in the training set:","metadata":{}},{"cell_type":"code","source":"intervals = train.age2.unique().dropna()\n\ntop_by_age = {}\nfor inter in intervals: \n    train_age = train.loc[train.age2 == inter]\n    popular_items_group = train_age.groupby(['article_id'])['pop_factor'].sum()\n    _, popular_items = zip(*sorted(zip(popular_items_group, popular_items_group.keys()))[::-1][:12])\n    \n    top_by_age.setdefault(inter,popular_items)\n\nage_interval = {age : interval for age in range(15,100) for interval in intervals if age in interval}\n\npopular_items_group = train.groupby(['article_id'])['pop_factor'].sum()\n_, top = zip(*sorted(zip(popular_items_group, popular_items_group.keys()))[::-1][:12])\n","metadata":{"execution":{"iopub.execute_input":"2022-03-04T08:30:23.858169Z","iopub.status.busy":"2022-03-04T08:30:23.857237Z","iopub.status.idle":"2022-03-04T08:30:24.022599Z","shell.execute_reply":"2022-03-04T08:30:24.021845Z","shell.execute_reply.started":"2022-03-04T08:30:23.85813Z"}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.pop_factor.describe()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"top","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# RecSys Implementation\n\nWe implement the recommendation system. We also add code to simulate the score. The implementation is the following:\n\nFor a user, if it's found in the training set that we use to compute the factorization machine (users that buy >=20 items) we will give the recommendation based on this algorithm computed before. If in the contrary, the user is found in the training set but not in the one used for CF, we recommend the most common items for this user. The most common items is calculated first looking at the items bought in the last week, then if no item is found we look at the second week, and so on. Finally, if the user is not in any of the sets used for training, we give him the most popular items based on the age (if it's found) or in general.","metadata":{}},{"cell_type":"markdown","source":"Calculation of the score:","metadata":{}},{"cell_type":"code","source":"def apk(actual, predicted, k=12):\n    if len(predicted)>k:\n        predicted = predicted[:k]\n\n    score = 0.0\n    num_hits = 0.0\n\n    for i,p in enumerate(predicted):\n        if p in actual and p not in predicted[:i]:\n            num_hits += 1.0\n            score += num_hits / (i+1.0)\n\n    if not actual:\n        return 0.0\n\n    return score / min(len(actual), k)\n\ndef mapk(actual, predicted, k=12):\n    return np.mean([apk(a,p,k) for a,p in zip(actual, predicted)])","metadata":{"execution":{"iopub.execute_input":"2022-03-03T23:40:37.703614Z","iopub.status.busy":"2022-03-03T23:40:37.703342Z","iopub.status.idle":"2022-03-03T23:40:37.71022Z","shell.execute_reply":"2022-03-03T23:40:37.709498Z","shell.execute_reply.started":"2022-03-03T23:40:37.703586Z"}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Implement the recommender system in the validation dataset:","metadata":{}},{"cell_type":"code","source":"#Comparem el validation items (actual output del validation) amb el predit del validation a partir del training!\npositive_items_val = val.groupby(['customer_id'])['article_id'].apply(list)\nval_users = positive_items_val.keys()\nval_items = []\n\nfor i,user in tqdm(enumerate(val_users)):\n    val_items.append(positive_items_val[user])","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from collections import Counter\noutputs = []\ncnt = 0\ncnt2 = 0\n\nuser_age = dict(zip(val.customer_id,val.age))\npopular_items = list(popular_items)\n\nfor user in tqdm(val_users):\n    if user not in users.keys():\n        \n        user_output = []\n        \n        if user in positive_items_per_user1.keys():\n            most_common_items_of_user = {k:v for k, v in Counter(positive_items_per_user1[user]).most_common()}\n            user_output += list(most_common_items_of_user.keys())[:12]\n            \n        if user in positive_items_per_user2.keys():\n            most_common_items_of_user = {k:v for k, v in Counter(positive_items_per_user2[user]).most_common()}\n            user_output += list(most_common_items_of_user.keys())[:12]\n            \n        if user in positive_items_per_user3.keys():\n            most_common_items_of_user = {k:v for k, v in Counter(positive_items_per_user3[user]).most_common()}\n            user_output += list(most_common_items_of_user.keys())[:12]\n            \n        if user in positive_items_per_user4.keys():\n            most_common_items_of_user = {k:v for k, v in Counter(positive_items_per_user4[user]).most_common()}\n            user_output += list(most_common_items_of_user.keys())[:12]\n            \n        if user in user_age.keys() and ~np.isnan(user_age[user]) and user_age[user] >= 15.0 and user_age[user] < max(age_interval.keys()):\n            cnt2 += 1\n            user_output += list(top_by_age[age_interval[int(user_age[user])]][:12 - len(user_output)])\n            outputs.append(user_output)\n        \n        \n        else: \n            user_output += list(top[:12 - len(user_output)])\n            outputs.append(user_output)\n    else:\n        user_output = list(recom.loc[recom['customer_id'] == user,'recom'].values)\n        user_output += list(top[:12 - len(user_output)])\n        outputs.append(list(user_output))\n        cnt+=1\n        \n    \n        \nprint(cnt2,cnt)\nprint(\"mAP Score on Validation set:\", mapk(val_items, outputs))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Test submission:","metadata":{}},{"cell_type":"markdown","source":"Implement the RecSyst for the test dataset:","metadata":{}},{"cell_type":"code","source":"train1_t = data.loc[(data[\"t_dat\"] >= datetime.datetime(2020,9,16)) & (data['t_dat'] < datetime.datetime(2020,9,23))]\ntrain2_t = data.loc[(data[\"t_dat\"] >= datetime.datetime(2020,9,8)) & (data['t_dat'] < datetime.datetime(2020,9,16))]\ntrain3_t = data.loc[(data[\"t_dat\"] >= datetime.datetime(2020,8,31)) & (data['t_dat'] < datetime.datetime(2020,9,8))]\ntrain4_t = data.loc[(data[\"t_dat\"] >= datetime.datetime(2020,8,23)) & (data['t_dat'] < datetime.datetime(2020,8,31))]\n\npositive_items_per_user1_t = train1_t.groupby(['customer_id'])['article_id'].apply(list)\npositive_items_per_user2_t = train2_t.groupby(['customer_id'])['article_id'].apply(list)\npositive_items_per_user3_t = train3_t.groupby(['customer_id'])['article_id'].apply(list)\npositive_items_per_user4_t = train4_t.groupby(['customer_id'])['article_id'].apply(list)\n\ntrain_t = pd.concat([train1_t,train2_t], axis=0) #train2_t\ntrain_t = train_t.merge(customers, how='left', on='customer_id')\ntrain_t['pop_factor'] = train_t['t_dat'].apply(lambda x: 1/(datetime.datetime(2020,9,23) - x).days)\n\ntrain_t['age2'] = pd.qcut(train_t['age'], 10)\ntrain_t","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"intervals_t = train_t.age2.unique().dropna()\n\ntop_by_age = {}\nfor inter in intervals_t: \n    train_age_t = train_t.loc[train_t.age2 == inter]\n    popular_items_group_t = train_age_t.groupby(['article_id'])['pop_factor'].sum()\n    _, popular_items_t = zip(*sorted(zip(popular_items_group_t, popular_items_group_t.keys()))[::-1][:12])\n    \n    top_by_age.setdefault(inter,popular_items_t)\n\nage_interval = {age : interval for age in range(15,100) for interval in intervals_t if age in interval}\n\npopular_items_group_t = train.groupby(['article_id'])['pop_factor'].sum()\n_, top = zip(*sorted(zip(popular_items_group_t, popular_items_group_t.keys()))[::-1][:12])\n\nuser_group = pd.concat([train1, train2, train3, train4], axis=0).groupby(['customer_id'])['article_id'].apply(list)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Load test submission:","metadata":{}},{"cell_type":"code","source":"test = pd.read_csv(\"sample_submission.csv\")\ntest = test[['customer_id']]\ntest.head()","metadata":{"execution":{"iopub.execute_input":"2022-03-03T22:34:50.927694Z","iopub.status.busy":"2022-03-03T22:34:50.927436Z","iopub.status.idle":"2022-03-03T22:34:51.201421Z","shell.execute_reply":"2022-03-03T22:34:51.20068Z","shell.execute_reply.started":"2022-03-03T22:34:50.927666Z"}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Apply the RecSys on these customers ids to get predictions:","metadata":{}},{"cell_type":"code","source":"test1 = test.copy()","metadata":{"execution":{"iopub.execute_input":"2022-03-03T22:34:53.731125Z","iopub.status.busy":"2022-03-03T22:34:53.730727Z","iopub.status.idle":"2022-03-03T22:34:54.609539Z","shell.execute_reply":"2022-03-03T22:34:54.608806Z","shell.execute_reply.started":"2022-03-03T22:34:53.731089Z"}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = test1.merge(customers, how='left', on='customer_id')\n\ndef to_submission(data):\n    return \" \".join([str(x) for x in data])\n        \ndef recommend(user,age): \n    recommendation = []\n    if user not in users.keys():\n        user_output = []\n        if user in positive_items_per_user1_t.keys():\n            most_common_items_of_user = {k:v for k, v in Counter(positive_items_per_user1_t[user]).most_common()}\n            user_output += list(most_common_items_of_user.keys())[:12]\n            \n        if user in positive_items_per_user2_t.keys():\n            most_common_items_of_user = {k:v for k, v in Counter(positive_items_per_user2_t[user]).most_common()}\n            user_output += list(most_common_items_of_user.keys())[:12]\n            \n        if user in positive_items_per_user3_t.keys():\n            most_common_items_of_user = {k:v for k, v in Counter(positive_items_per_user3_t[user]).most_common()}\n            user_output += list(most_common_items_of_user.keys())[:12]\n            \n        if user in positive_items_per_user4_t.keys():\n            most_common_items_of_user = {k:v for k, v in Counter(positive_items_per_user4_t[user]).most_common()}\n            user_output += list(most_common_items_of_user.keys())[:12]\n\n        if user in user_age.keys() and ~np.isnan(user_age[user]) and user_age[user] >= 15.0 and user_age[user] < max(age_interval.keys()):\n            user_output += list(top_by_age[age_interval[int(user_age[user])]][:12 - len(user_output)])\n        else: \n            user_output += list(top[:12 - len(user_output)])\n            \n        return user_output\n    else:\n        user_output = list(recom.loc[recom['customer_id'] == user,'recom'].values)\n        user_output += list(top[:12 - len(user_output)])\n        \n        return user_output\n\n\ntest['prediction'] = test.progress_apply(lambda x: to_submission(recommend(x.customer_id,x.age)),axis=1)\n","metadata":{"execution":{"iopub.execute_input":"2022-03-03T23:23:45.8425Z","iopub.status.busy":"2022-03-03T23:23:45.842228Z","iopub.status.idle":"2022-03-03T23:26:54.222689Z","shell.execute_reply":"2022-03-03T23:26:54.221163Z","shell.execute_reply.started":"2022-03-03T23:23:45.842468Z"}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test","metadata":{"execution":{"iopub.execute_input":"2022-03-03T23:27:25.240315Z","iopub.status.busy":"2022-03-03T23:27:25.239753Z","iopub.status.idle":"2022-03-03T23:27:25.250787Z","shell.execute_reply":"2022-03-03T23:27:25.25016Z","shell.execute_reply.started":"2022-03-03T23:27:25.240279Z"}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Add the predictions to the test dataset to create the final submission:","metadata":{}},{"cell_type":"code","source":"del test['age']\n\ntest.to_csv(f'submission.csv',index=False)\ntest.head()","metadata":{"execution":{"iopub.execute_input":"2022-03-03T23:27:40.150123Z","iopub.status.busy":"2022-03-03T23:27:40.149618Z","iopub.status.idle":"2022-03-03T23:27:51.416633Z","shell.execute_reply":"2022-03-03T23:27:51.415897Z","shell.execute_reply.started":"2022-03-03T23:27:40.150079Z"}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}