{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nfrom tqdm.notebook import tqdm\nimport gc\nimport sklearn\n\n\n# load data\n# ../input/h-and-m-personalized-fashion-recommendations/\noriginal_df_customers = pd.read_csv('./customers.csv')\noriginal_df_items = pd.read_csv('./articles.csv')\noriginal_df_customers_items = pd.read_csv('./transactions_train.csv')\n\nsubmission_file = pd.read_csv('./sample_submission.csv')","metadata":{"pycharm":{"is_executing":true},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# CLEANING DATA CUSTOMERS","metadata":{}},{"cell_type":"code","source":"original_df_customers = original_df_customers.drop(\n    columns=['FN', 'Active', 'club_member_status', 'fashion_news_frequency', 'postal_code'])\n","metadata":{"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"original_df_customers['age'].fillna(\n    (original_df_customers['age'].mean()), inplace=True)\noriginal_df_customers['age'] = original_df_customers['age'].astype(np.int8)\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"original_df_customers[\"age\"].plot.hist(bins=50)\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bins = [20, 21, 55, 110]\nlabels = ['Teen', 'Adult', 'Senior']\noriginal_df_customers['AgeGroup'] = pd.cut(\n    original_df_customers['age'], bins=bins, labels=labels, right=False)\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"original_df_customers\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"original_df_customers = original_df_customers.drop(columns=['age'])\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# CLEANING DATA ITEMS","metadata":{}},{"cell_type":"code","source":"original_df_items.columns\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(len(original_df_items.columns)):\n    print(original_df_items.columns[i] + \" \" + str(\n        len(original_df_items[original_df_items.columns[i]].unique())))\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"original_df_items['index_name'].unique()\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"original_df_items.head(10)\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"original_df_items = original_df_items[[\"article_id\", \"product_group_name\",\n                                       \"perceived_colour_value_name\", \"perceived_colour_master_name\", \"index_name\", \"garment_group_name\"]]\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# CLEANING TRANSACTIONS","metadata":{}},{"cell_type":"code","source":"original_df_customers_items = original_df_customers_items.drop(\n    columns=['price', 'sales_channel_id'])\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# original_df_customers_items['day'] = original_df_customers_items['t_dat'].dt.day\noriginal_df_customers_items['t_dat'] = pd.to_datetime(\n    original_df_customers_items['t_dat'], format='%Y-%m-%d')\noriginal_df_customers_items['day'] = original_df_customers_items['t_dat'].dt.day\noriginal_df_customers_items['month'] = original_df_customers_items['t_dat'].dt.month\noriginal_df_customers_items['year'] = original_df_customers_items['t_dat'].dt.year\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"original_df_customers_items['season'] = pd.Categorical.from_codes(\n    original_df_customers_items['t_dat'].dt.month % 12 // 3,\n    categories=['winter', 'spring', 'summer', 'fall'],\n    ordered=True\n)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"original_df_customers_items\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"original_df_customers_items = original_df_customers_items.drop(columns=['t_dat', 'month'])\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"value_counts = original_df_customers_items.article_id.value_counts()\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#counts_items = original_df_items.merge(value_counts, left_on='article_id', right_on='article_id')\ncounts_items = pd.DataFrame(original_df_customers_items.article_id.value_counts().reset_index())\ncounts_items.columns = ['article_id', 'count']\n\n# merge 'df' & 'brands_count'\noriginal_df_customers_items = pd.merge(\n    original_df_customers_items, counts_items, on='article_id')\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# MERGE DATASETS","metadata":{}},{"cell_type":"code","source":"df_merged_1 = original_df_customers_items.merge(original_df_customers, left_on='customer_id', right_on='customer_id')\ndel original_df_customers_items\ndel original_df_customers\ngc.collect()\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_merged_final = df_merged_1.merge(original_df_items, left_on='article_id', right_on='article_id')\ndel original_df_items\ngc.collect()\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_merged_final.columns\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_merged_final = df_merged_final[['customer_id', 'AgeGroup', 'article_id', 'product_group_name', 'perceived_colour_value_name',\n                                   'perceived_colour_master_name', 'index_name', 'garment_group_name', 'count', 'day', 'year', 'season']]\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_merged_final\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"article_ids = np.unique(df_merged_final.article_id.values)\ncustomer_ids = np.unique(df_merged_final.customer_id.values)\n\nnum_articles = len(article_ids)\nnum_customers = len(customer_ids)\n\n# Create dictionaries with mapping keys\narticles_id_to_idx = dict(zip(article_ids, range(num_articles)))\ncustomers_id_to_idx = dict(zip(customer_ids, range(num_customers)))\n\narticles_idx_to_id = dict(zip(range(num_articles), article_ids))\ncustomers_idx_to_id = dict(zip(range(num_customers), customer_ids))\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"articles_id_to_idx","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def top_bought_articles(df, n):\n    return df.value_counts().head(n)\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"top_12 = top_bought_articles(df_merged_final['article_id'], 12)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"top_12 = top_12.to_numpy()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Assign the customer and article indices to the transactions and drop the ids\ndf_merged_final['customer_index'] = df_merged_final.customer_id.map(\n    customers_id_to_idx)\ndf_merged_final['article_index'] = df_merged_final.article_id.map(\n    articles_id_to_idx)\n\ndf_merged_final.drop(['customer_id', 'article_id'], inplace=True, axis=1)\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_merged_final = df_merged_final[[\n    'customer_index', 'AgeGroup', 'article_index', 'product_group_name']]\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_merged_final\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_merged_final['sale'] = 1\ndata_aux = df_merged_final.copy()\ndata_aux['sale'] = 0  # -1\n\ndata_aux['customer_index'] = data_aux['customer_index'].sample(frac=1, ignore_index=True)\ndata_aux['article_index'] = data_aux['article_index'].sample(frac=1, ignore_index=True)\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_merged_final = df_merged_final.merge(data_aux, how='outer')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# One Hot Encoding","metadata":{}},{"cell_type":"code","source":"y = df_merged_final['sale']\nX = df_merged_final.drop(['sale'], axis=1)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"columns = X.columns","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\n\n\ndef one_hot_encoding(data, categorical_cols, sparse=True, df=False):\n    # instantiate labelencoder object\n    le = LabelEncoder()\n\n    # apply le on categorical feature columns\n    data[categorical_cols] = data[categorical_cols].apply(\n        lambda col: le.fit_transform(col))\n\n    from sklearn.preprocessing import OneHotEncoder\n    ohe = OneHotEncoder(sparse=True)\n\n    # One-hot-encode the categorical columns.\n    # Unfortunately outputs an array instead of dataframe.\n    array_hot_encoded = ohe.fit_transform(data[categorical_cols])\n    print(type(array_hot_encoded))\n\n    if df == True:\n        # Convert it to df\n        data_hot_encoded = pd.DataFrame(array_hot_encoded, index=data.index)\n\n        # Extract only the columns that didnt need to be encoded\n        data_other_cols = data.drop(columns=categorical_cols)\n\n        # Concatenate the two dataframes :\n        data_out = pd.concat([data_hot_encoded, data_other_cols], axis=1)\n        return le, ohe, data_out\n    else:\n        return le, ohe, array_hot_encoded\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"le, ohe_func, array_hot_encoded = one_hot_encoding(\n    X, columns, sparse=True, df=False)\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import scipy\nscipy.sparse.save_npz('./sparse_matrix.npz', array_hot_encoded)\n# sparse_matrix = scipy.sparse.load_npz('./sparse_matrix.npz')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n## Lesson 4: Collaborative-based recommender systems: Factorization MODELS","metadata":{}},{"cell_type":"markdown","source":"Matrix factorization is a simple embedding model, which decomposes the user-item intraction matrix, $R \\in R^{m\\times n}$ matrix, where $m$ is the number of users and $n$ the number of items, into the product of two lower dimensionality rectangular matrice. The the goal of the factorization models is to learn:\n* A user embedding (or user latent factor) $P \\in R^{m\\times k}$, where row $i$ is the embedding of user $i$.\n* A item embedding (or item latent factor) $Q \\in R^{n\\times k}$, where row $j$ is the embedding of user $j$.\n\n![alt factorization models](https://miro.medium.com/max/988/1*nIVWl2ROaxOY23hHajkTTg.png)\n\nThese methods became really popular due to the shown efficiency in the Netflix Prize Challenge. \n\nThere are several variants of these methods:\n* SVD \n* Vanilla Factorization Matrix (also knwon as Funk SVD)\n* Vanilla Factorization Matrix with biases \n* SVD++\n* ...\n\nIn these notewook we will the application of these method to the transactionLens dataset.","metadata":{}},{"cell_type":"code","source":"from scipy.sparse import csr_matrix, dok_matrix\n\n\ndef to_dense(array):\n    \"\"\"\n    Accepta una csr_matrix, dok_matrix o matrix i la converteix en una \n    np.array normal, densa.\n\n    :param array: Array a convertir\n    :return: np.array densa, sense cap dimensió de tamany 1\n    \"\"\"\n    try:\n        array = array.todense()\n    except:\n        pass\n\n    return np.array(array).squeeze()\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def evaluate(predict_f, data_train, data_test):\n    \"\"\" RMSE-based predictive performance evaluation with pandas. \"\"\"\n    ids_to_estimate = zip(data_test.customer_id, data_test.article_id)\n    list_users = set(data_train.customer_id)\n    estimated = np.array(\n        [predict_f(u, i) if u in list_users else 0.5 for (u, i) in ids_to_estimate])\n    real = data_test['sale'].values\n    return compute_rmse(estimated, real)\n\n\ndef compute_rmse(y_pred, y_true):\n    \"\"\" Compute Root Mean Squared Error. \"\"\"\n    return np.sqrt(np.mean(np.power(y_pred - y_true, 2)))\n\n\n# Divide the data in two sets: training and test\ndef assign_to_set(df):\n    sampled_ids = np.random.choice(df.index,\n                                   size=np.int64(np.ceil(df.index.size * 0.2)),\n                                   replace=False)\n    df.loc[sampled_ids, 'for_testing'] = True\n    return df\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### The Vanilla Matrix Factorization Model \n* Also know as **Funk SVD**\n* * Despite its name, in Funk SVD, no singular value decomposition is applied.\n* * https://sifter.org/simon/journal/20061211.html\n\nA straightforward matrix factorization model maps both users and items to a joint latent factor space of dimensionality D. User-item interaction are modeled as inner products in that space\n$$R = UV$$\n\nEach item j is associated with a vector $v_j$ from $V$, and each user $i$ is associated with a vecor $u_i$ from $U$.\nThe resulting dot product $u_i\\cdot v_j$ captures the interaction between the user $i$ and item $j$:\n$$ \\hat{r} = u_i\\cdot v_j$$\n\nThe goal of the matrix factorization consist on finding the mapping of each item and user to factors $u_i$ and $v_j$. To do so, the minimization the of squarred error function is performed:\n$$ \\sum(R_{ui} - u_i\\cdot v_j)^2$$\n\nThis factorization can be learnt using **only those known ratings**. We do not need to infer missing values.","metadata":{}},{"cell_type":"markdown","source":"![alt Amazon](https://miro.medium.com/max/4800/1*b4M7o7W8bfRRxdMxtFoVBQ.png)","metadata":{}},{"cell_type":"code","source":"from scipy import sparse\nimport matplotlib.pyplot as plt\nfrom numba import njit, jit\n\n\nclass RecSys_mf():\n    def __init__(self, article_ids, customer_ids, num_components=10):\n        \"\"\" Constructor \"\"\"\n        self.num_components = num_components\n\n        self.article_ids = article_ids\n        self.customer_ids = customer_ids\n\n        self.num_articles = len(self.article_ids)\n        self.num_customers = len(self.customer_ids)\n\n        # Create dictionaries with mapping keys\n        self.article_id2index = dict(zip(article_ids, range(num_articles)))\n        self.customer_id2index = dict(zip(customer_ids, range(num_customers)))\n        self.article_index2id = dict(zip(range(num_articles), article_ids))\n        self.customer_index2id = dict(zip(range(num_customers), customer_ids))\n\n\nclass RecSys_vanilla_mf(RecSys_mf):\n\n    def __init__(self, matrix, sales, article_ids, customer_ids, top_12, reg_w=0.01, reg_v=0.01, num_components=10, seed=42):\n        \"\"\" Constructor \"\"\"\n        super().__init__(article_ids, customer_ids, num_components)\n        self.matrix = matrix\n        self.y = sales.to_numpy(dtype=np.int8)\n        self.top_12 = top_12\n        self.seed = seed   # Random seed\n\n        self.reg_w = reg_w\n        self.reg_v = reg_v\n\n    def fit(self, n_epochs=10, learning_rate=0.001, lmbda=0.1, verbose=True):\n        \"\"\" We decompose the R matrix into to submatrices using the training data \"\"\"\n        self.verbose = verbose\n        self.learning_rate = learning_rate\n        self.lmbda = lmbda\n\n        self.n_samples, self.n_features = self.matrix.shape\n        # Initialize weights\n        self.w0 = np.random.normal(scale=1, size=1)\n        self.w = np.random.normal(\n            scale=1/self.n_features, size=self.n_features)\n        self.v = np.random.normal(\n            scale=1./self.n_features, size=(self.num_components, self.n_features))\n\n        self.train_rmse = []\n        \n        print(type(self.y))\n\n        for epoch in tqdm(range(n_epochs)):\n            print('Epoch: {}'.format(epoch))\n\n            self.training_indices = np.arange(self.n_samples)\n\n            # self.__shuffle_sales__()\n            # shuffle training samples\n            np.random.shuffle(self.training_indices)\n            __sdg__(self.matrix.data, self.matrix.indptr, self.matrix.indices, self.y, self.n_samples, self.n_features,\n                    self.w0, self.w, self.v, self.num_components, self.learning_rate, self.reg_w, self.reg_v)\n\n            # self.train_rmse.append(evaluate(self.predict, self.matrix, self.matrix))\n\n            # print('\\tTrain rmse: %s' % self.train_rmse[-1])\n\n        if(self.verbose):\n            self.__plot_learning_curves__()\n\n    def __shuffle_sales__(self):\n        \"\"\" Shuffle negative samples \"\"\"\n        self.y = np.random.shuffle(self.y)\n\n    def __plot_learning_curves__(self):\n        plt.plot(self.train_rmse, '--o', label=\"train_error\")\n        plt.plot(self.test_rmse, '--o', label=\"test_error\")\n        plt.legend()\n        plt.show()\n\n    def predict(self, customers):\n        \"\"\"Similar to _predict_instance but vectorized for all samples\"\"\"\n        \n        results = pd.DataFrame(columns=['customer_id', 'predict'])\n\n        for idx, customer in enumerate(tqdm(customers.customer_id.unique())):\n            if customer not in self.customer_id2index.keys():\n                # If the the customer is not in the trained ones return the default recommendation\n                article_ids = self.top_12\n            else:\n                customer_idx = self.customer_id2index[customer]\n                \n                summed = np.zeros(self.num_components)\n                summed_squared = np.zeros(self.num_components)\n\n                # linear output w * x\n                pred = self.w0\n                for index in range(self.matrix.indptr[customer_idx], self.matrix.indptr[customer_idx + 1]):\n                    feature = self.matrix.indices[index]\n                    pred += self.w[feature] * self.matrix.data[index]\n\n                # factor output\n                for factor in range(self.num_components):\n                    for index in range(self.matrix.indptr[customer_idx], self.matrix.indptr[customer_idx + 1]):\n                        feature = self.indices[index]\n                        term = self.v[factor, feature] * self.matrix.data[index]\n                        summed[factor] += term\n                        summed_squared[factor] += term * term\n\n                    pred += 0.5 * (summed[factor] * summed[factor] - summed_squared[factor])\n                \n                recommended = pred.argsort()[-12:]\n                \n                article_ids = ' '.join([self.article_index2id[item_idx] for item_idx in recommended])\n            \n            results.loc[idx] = (customer, ''.join(article_ids))\n            \n        return results\n\n\ndef __sdg__(data, indptr, indices, y, n_samples, n_features, w0, w, v, n_factors, learning_rate, reg_w, reg_v):\n    \"\"\"\n    Compute the loss of the current iteration and update\n    gradients accordingly.\n    \"\"\"\n    loss = 0.0\n    for i in range(n_samples):\n        pred, summed = _predict_instance(\n            data, indptr, indices, w0, w, v, n_factors, i)\n\n        # calculate loss and its gradient\n        loss += _log_loss(pred, y[i])\n        loss_gradient = -y[i] / (np.exp(y[i] * pred) + 1.0)\n\n        # update bias/intercept term\n        w0 -= learning_rate * loss_gradient\n\n        # update weight\n        for index in range(indptr[i], indptr[i + 1]):\n            feature = indices[index]\n            w[feature] -= learning_rate * \\\n                (loss_gradient * data[index] + 2 * reg_w * w[feature])\n\n        # update factor\n        for factor in range(n_factors):\n            for index in range(indptr[i], indptr[i + 1]):\n                feature = indices[index]\n                term = summed[factor] - v[factor, feature] * data[index]\n                v_gradient = loss_gradient * data[index] * term\n                v[factor, feature] -= learning_rate * \\\n                    (v_gradient + 2 * reg_v * v[factor, feature])\n\n    loss /= n_samples\n    return loss\n\n\ndef _predict_instance(data, indptr, indices, w0, w, v, n_factors, i):\n    \"\"\"predicting a single instance\"\"\"\n    summed = np.zeros(n_factors)\n    summed_squared = np.zeros(n_factors)\n\n    # linear output w * x\n    pred = w0\n    for index in range(indptr[i], indptr[i + 1]):\n        feature = indices[index]\n        pred += w[feature] * data[index]\n\n    # factor output\n    for factor in range(n_factors):\n        for index in range(indptr[i], indptr[i + 1]):\n            feature = indices[index]\n            term = v[factor, feature] * data[index]\n            summed[factor] += term\n            summed_squared[factor] += term * term\n\n        pred += 0.5 * (summed[factor] * summed[factor] -\n                       summed_squared[factor])\n\n    # summed is the independent term that can be re-used\n    # during the gradient update stage\n    return pred, summed\n\n\ndef _log_loss(pred, y):\n    \"\"\"\n    negative log likelihood of the\n    current prediction and label, y.\n    \"\"\"\n    return np.log(np.exp(-pred * y) + 1.0)\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"reco = RecSys_vanilla_mf(array_hot_encoded, y, article_ids, customer_ids,\n                         top_12, reg_w=0.01, reg_v=0.01, num_components=10, seed=42)\nreco.fit(n_epochs=20, learning_rate=0.01, lmbda=0.5, verbose=True)\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result = reco._predict(submission_file)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_aux = result.explode(column='prediction')\ndf_aux.prediction = '0' + df_aux.prediction\ndf_aux.head()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def group(list_of_strings):\n    return \" \".join(list_of_strings)\n    \nsub = df_aux.groupby('customer_id')['prediction'].agg(lambda x: group(x)).reset_index()\n\nsub.to_csv('submission.csv', index=False)","metadata":{},"execution_count":null,"outputs":[]}]}