{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"!ls /kaggle/input/test-recsys/","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport json\nfrom scipy import sparse as sp\nfrom tqdm.notebook import tqdm\nfrom collections import defaultdict","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import hashlib\nfrom typing import List, Set\nimport numpy as np\nimport pandas as pd\nfrom scipy import sparse as sp\n\nclass ProductEncoder:\n    def __init__(self, product_df):\n        self.product_idx = {}\n        self.product_pid = {}\n        for idx, pid in enumerate(product_df.product_id.values):\n            self.product_idx[pid] = idx\n            self.product_pid[idx] = pid\n\n    def toIdx(self, x):\n        if type(x) == int:\n            pid = x\n            return self.product_idx[pid]\n        return [self.product_idx[pid] for pid in x]\n\n    def toPid(self, x):\n        if type(x) == int:\n            idx = x\n            return self.product_pid[idx]\n        return [self.product_pid[idx] for idx in x]\n\n    @property\n    def num_products(self):\n        return len(self.product_idx)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def average_precision(actual, recommended, k=50):\n    ap_sum = 0\n    hits = 0\n    for i in range(k):\n        product_id = recommended[i] if i < len(recommended) else None\n        if product_id is not None and product_id in actual:\n            hits += 1\n            ap_sum += hits / (i + 1)\n    return ap_sum / k\n\n\ndef normalized_average_precision(actual, recommended, k=50):\n    actual = set(actual)\n    if len(actual) == 0:\n        return 0.0\n\n    ap = average_precision(actual, recommended, k=k)\n    ap_ideal = average_precision(actual, list(actual)[:k], k=k)\n    return ap / ap_ideal","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"root = '/kaggle/input/test-recsys/'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"orders = pd.read_csv('/kaggle/input/test-recsys/kaggle_tab_1345/tab_1_orders.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"product_table = pd.DataFrame(columns=['product_id'])\n\nfor folder in tqdm(os.listdir(root)):\n    if 'sbermarket' in folder:\n        for file in os.listdir(os.path.join(root, folder)):\n            df = pd.read_csv(os.path.join(root, folder, file))\n            product_table = pd.concat([product_table, df[['product_id']]]).drop_duplicates()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"enc = ProductEncoder(product_table.sort_values(by='product_id'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nclass Order:\n    def __init__(self, order_id, order_datetime, **kwargs):\n        self.data = {\n            **{\"order_id\": order_id, \"datetime\": order_datetime, \"products\": [],},\n            **kwargs,\n        }\n\n    def add_item(\n        self, product_id: str, product_quantity: float, price: float, master_category_id : int, parent_category_id: int\n    ) -> None:\n        p = {\n            \"product_id\": product_id,\n            \"quantity\": product_quantity,\n            'price': price,\n            'master_category_id':master_category_id,\n            'parent_category_id':parent_category_id\n        }\n        self.data[\"products\"].append(p)\n\n    def as_dict(self,):\n        return self.data\n\n    def order_id(self,):\n        return self.data[\"order_id\"]\n\n\nclass UserHistory:\n    def __init__(\n        self, user_id,\n    ):\n        self.data = {\n            \"user_id\": user_id,\n            \"order_history\": [],\n        }\n\n    def add_order(\n        self, order,\n    ):\n        self.data[\"order_history\"].append(order)\n\n    def as_dict(self,):\n        return self.data\n\n    def user_id(self,):\n        return self.data[\"user_id\"]\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"orders[orders.order_id == 15696861].order_created_time\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ndef md5_hash(x):\n    return int(hashlib.md5(str(x).encode()).hexdigest(), 16)\nclass RowSplitter:\n    def __init__(\n        self, output_path, n_shards=16,\n    ):\n        self.n_shards = n_shards\n        os.makedirs(\n            output_path, exist_ok=True,\n        )\n        self.outs = []\n        for i in range(self.n_shards):\n            self.outs.append(open(output_path + \"/{:02d}.jsons\".format(i), \"w\",))\n        self._client = None\n        self._transaction = None\n\n    def finish(self,):\n        self.flush()\n        for outs in self.outs:\n            outs.close()\n\n    def flush(self,):\n        if self._client is not None:\n            self._client.add_order(self._transaction.as_dict())\n            # rows are sharded by cliend_id\n            shard_idx = md5_hash(self._client.user_id()) % self.n_shards\n            data = self._client.as_dict()\n            self.outs[shard_idx].write(json.dumps(data) + \"\\n\")\n\n            self._client = None\n            self._transaction = None\n\n    def consume_row(\n        self, row,\n    ):\n        if self._client is not None and self._client.user_id() != row.user_id:\n            self.flush()\n\n        if self._client is None:\n            self._client = UserHistory(user_id=int(row.user_id))\n\n        if self._transaction is not None and self._transaction.order_id() != row.order_id:\n            self._client.add_order(self._transaction.as_dict())\n            self._transaction = None\n\n        if self._transaction is None:\n            if len(orders[orders.order_id == row.order_id]) > 0:\n                self._transaction = Order(\n                    order_id=int(row.order_id),\n                    order_datetime=orders[orders.order_id == row.order_id].order_created_time.values[0],\n                    store_id=int(orders[orders.order_id == row.order_id].store_id.values[0]),\n                    platform=orders[orders.order_id == row.order_id].platform.values[0],\n                    retailer=orders[orders.order_id == row.order_id].retailer.values[0]\n                )\n            else:\n                self._transaction = Order(\n                    order_id=int(row.order_id),\n                    order_datetime=None,\n                    store_id=None,\n                    platform=None,\n                    retailer=None\n                )\n\n        self._transaction.add_item(\n            product_id=int(row.product_id) if not np.isnan(row.product_id) else row.product_id,\n            product_quantity=int(row.quantity) if not np.isnan(row.quantity) else row.quantity,\n            price=float(row.price),\n            master_category_id=int(row.master_category_id) if not np.isnan(row.master_category_id) else row.master_category_id,\n            parent_category_id=int(row.parent_category_id) if not np.isnan(row.parent_category_id) else row.parent_category_id\n        )\n\n\ndef split_data_to_chunks(\n    output_dir, n_shards\n):\n    splitter = RowSplitter(output_path=output_dir, n_shards=n_shards,)\n    for folder in tqdm(os.listdir(root)):\n        if 'sbermarket' in folder:\n            for file in tqdm(os.listdir(os.path.join(root, folder))):\n                for df in tqdm(pd.read_csv(os.path.join(root, folder, file), chunksize=500000, )):\n                    for row in tqdm(df.itertuples()):\n                        splitter.consume_row(row)\n    splitter.finish()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ndef md5_hash(x):\n    return int(hashlib.md5(str(x).encode()).hexdigest(), 16)\nclass RowSplitter:\n    def __init__(\n        self, output_path, n_shards=16,\n    ):\n        self.n_shards = n_shards\n        os.makedirs(\n            output_path, exist_ok=True,\n        )\n        self.outs = []\n        self._client = None\n        self._transaction = None\n\n    def finish(self,):\n        self.flush()\n\n    def flush(self,):\n        if self._client is not None:\n            self._client.add_order(self._transaction.as_dict())\n            data = self._client.as_dict()\n            self.outs.append(make_coo_row(data['order_history'], enc))\n\n    def consume_row(\n        self, row,\n    ):\n        if self._client is not None and self._client.user_id() != row.user_id:\n            self.flush()\n\n        if self._client is None:\n            self._client = UserHistory(user_id=int(row.user_id))\n\n        if self._transaction is not None and self._transaction.order_id() != row.order_id:\n            self._client.add_order(self._transaction.as_dict())\n            self._transaction = None\n\n        if self._transaction is None:\n                self._transaction = Order(\n                    order_id=int(row.order_id),\n                    order_datetime=None,\n                    store_id=None,\n                    platform=None,\n                    retailer=None\n                )\n\n        self._transaction.add_item(\n            product_id=int(row.product_id) if not np.isnan(row.product_id) else row.product_id,\n            product_quantity=None,\n            price=None,\n            master_category_id=None,\n            parent_category_id=None\n        )\n\ndef make_coo_row(transaction_history, product_encoder: ProductEncoder):\n    idx = []\n    values = []\n\n    items = []\n    for trans in transaction_history:\n        items.extend([i[\"product_id\"] for i in trans[\"products\"]])\n    n_items = len(items)\n\n    for pid in items:\n        idx.append(product_encoder.toIdx(pid))\n        values.append(1.0)\n\n    return sp.coo_matrix(\n        (np.array(values).astype(np.float32), ([0] * len(idx), idx)), shape=(1, product_encoder.num_products),\n    )\n        \ndef split_data_to_chunks(\n    output_dir, n_shards\n):\n    splitter = RowSplitter(output_path=output_dir, n_shards=n_shards,)\n    for folder in tqdm(os.listdir(root)):\n        if 'sbermarket' in folder:\n            for file in tqdm(os.listdir(os.path.join(root, folder))):\n                print(file)\n                df = pd.read_csv(os.path.join(root, folder, file))\n                for row in tqdm(df.itertuples(), total=len(df)):\n                    splitter.consume_row(row)\n    splitter.finish()\n    return splitter.outs","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"X_sparse = split_data_to_chunks('jsons/', n_shards=16)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def make_coo_row(transaction_history, product_encoder: ProductEncoder):\n    idx = []\n    values = []\n\n    items = [pid for i in trans[\"products\"]]\n    n_items = len(items)\n\n    for pid in items:\n        idx.append(product_encoder.toIdx(pid))\n        values.append(1.0)\n\n    return sp.coo_matrix(\n        (np.array(values).astype(np.float32), ([0] * len(idx), idx)), shape=(1, product_encoder.num_products),\n    )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"user_hist = defaultdict(lambda: defaultdict(list))\nfor folder in tqdm(os.listdir(root)):\n    if 'sbermarket' in folder:\n        for file in os.listdir(os.path.join(root, folder)):\n            df = pd.read_csv(os.path.join(root, folder, file))\n            for user in tqdm(df.iterrows(), total=len(df)):\n                trans = {'order_id': user[1]['order_id'], 'product_id': user[1]['product_id'], 'quantity':user[1]['quantity']}\n                user_hist[user[1]['user_id']]['products'].append(user[1]['product_id'])\n                user_hist[user[1]['user_id']]['quantities'].append(user[1]['quantity'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sys import getsizeof","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!mkdir jsons","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!touch './jsons/user_items.json'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with open('./jsons/user_items.json', 'w') as f:\n    json.dump(user_hist, f)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"getsizeof(user_hist) / 1024 / 1024","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def make_coo_row(transaction_history, product_encoder: ProductEncoder):\n    idx = []\n    values = []\n\n    items = [product_id for product_id in transaction_history['products']]\n    values = [quantity for quantity in transaction_history['quantities']]\n    \n    n_items = len(items)\n\n    for pid in items:\n        idx.append(product_encoder.toIdx(pid))\n\n    return sp.coo_matrix(\n        (np.array(values).astype(np.float32), ([0] * len(idx), idx)), shape=(1, product_encoder.num_products),\n    )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def merge_products(history):\n    products = history['products']\n    quantities = history['quantities']\n    result_dict = defaultdict(int)\n    for i in range(len(products)):\n        result_dict[products[i]] += quantities[i]\n    return zip(*result_dict.items()) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for user_id in tqdm(user_hist.keys()):\n    user_hist[user_id]['products'], user_hist[user_id]['quantities'] = merge_products(user_hist[user_id])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_sparse = []\nfor user, history in tqdm(user_hist.items()):\n    merged_history = defaultdict(list)\n    try:\n        merged_history['products'], merged_history['quantities'] = merge_products(history)\n    except:\n        pass\n    X_sparse.append(make_coo_row(merged_history, enc))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_sparse = sp.vstack(X_sparse).tocsr()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"np.save?","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"np.save('sparse_user_item', X_sparse)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import implicit","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = implicit.als.AlternatingLeastSquares(factors=32, regularization=0.0, iterations=16)\nmodel.fit(X_sparse.T)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.recommend?","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def predict(user_id):\n    row_sparse = make_coo_row(user_hist[user_id], enc)\n    user_id = list(user_hist.keys()).index(user_id)\n    preds = model.recommend(user_id, row_sparse, N=50, filter_already_liked_items=False, recalculate_user=False)\n    return np.array([pred[0] for pred in preds])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_top_50(user_id):\n    prods, quants = merge_products(user_hist[user_id])\n    return np.array(prods)[np.argsort(quants)[-50:]]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"' '.join([1, 2])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def arr2str(arr):\n    return ' '.join(map(lambda x: str(x), arr))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"arr2str(get_top_50(65))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.read_csv('/kaggle/input/test-recsys/sample_submission.csv', index_col=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission[submission.Id == 65].Predicted.values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"recommends = model.recommend_all(X_sparse, N=50, filter_already_liked_items=False, recalculate_user=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"recommends","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"user_idxs = list(user_hist.keys())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"user_id_map = {}\nfor i, idx in enumerate(user_idxs):\n    user_id_map[idx] = i","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def predict(user_id):\n    user_id = user_id_map[user_id]\n    return recommends[user_id]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds = []\nfor idx in tqdm(submission.index.values):\n    submission.loc[idx].Predicted = arr2str(predict(idx))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds = []\nfor idx in tqdm(submission.index.values):\n    submission.loc[idx].Predicted = arr2str(predict(idx))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.to_csv('submission_als.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pd.read_csv('/kaggle/input/test-recsys/sample_submission.csv', index_col=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"rows = []\nfor user_id, transaction_history in tqdm(user_hist.items()):\n    rows.append(make_coo_row(transaction_history, enc))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(user_hist.items())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}