{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Setup","metadata":{}},{"cell_type":"code","source":"GLOBAL_SEED = 42\n\nimport os\nos.environ['PYTHONHASHSEED'] = str(GLOBAL_SEED)\nimport sys\n\nimport pandas as pd\nimport numpy as np\nfrom numpy import random as np_rnd\nimport random as rnd\nimport gc\nfrom collections import defaultdict\nimport datetime\nimport copy\nimport pickle\nfrom tqdm import tqdm\nfrom collections import Counter\n\ntry:\n    import cudf\n    import cupy\n    import cuml\nexcept:\n    pass\n\nimport torch\nfrom torch import nn","metadata":{"execution":{"iopub.status.busy":"2023-01-16T04:14:31.908378Z","iopub.execute_input":"2023-01-16T04:14:31.908735Z","iopub.status.idle":"2023-01-16T04:14:33.255280Z","shell.execute_reply.started":"2023-01-16T04:14:31.908706Z","shell.execute_reply":"2023-01-16T04:14:33.254206Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def seed_everything(seed=42):\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    # python random\n    rnd.seed(seed)\n    # numpy random\n    np_rnd.seed(seed)\n    # tf random\n    try:\n        tf_rnd.set_seed(seed)\n    except:\n        pass\n    # RAPIDS random\n    try:\n        cupy.random.seed(seed)\n    except:\n        pass\n    # pytorch random\n    try:\n        torch.manual_seed(seed)\n    except:\n        pass\n\ndef pickleIO(obj, src, op=\"w\"):\n    if op==\"w\":\n        with open(src, op + \"b\") as f:\n            pickle.dump(obj, f)\n    elif op==\"r\":\n        with open(src, op + \"b\") as f:\n            tmp = pickle.load(f)\n        return tmp\n    else:\n        print(\"unknown operation\")\n        return obj\n    \ndef findIdx(data_x, col_names):\n    return [int(i) for i, j in enumerate(data_x) if j in col_names]\n\ndef createFolder(directory):\n    try:\n        if not os.path.exists(directory):\n            os.makedirs(directory)\n    except OSError:\n        print('Error: Creating directory. ' + directory)\n\ndef create_get_ts(ts):\n    return int((ts.replace(tzinfo=CFG.tz) - CFG.ts_zero).total_seconds())","metadata":{"execution":{"iopub.status.busy":"2023-01-16T04:13:07.824025Z","iopub.execute_input":"2023-01-16T04:13:07.824375Z","iopub.status.idle":"2023-01-16T04:13:07.838253Z","shell.execute_reply.started":"2023-01-16T04:13:07.824342Z","shell.execute_reply":"2023-01-16T04:13:07.837427Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CFG:\n    local = False\n    debug = False\n    tz = datetime.timezone.utc\n    ts_zero = datetime.datetime(1970, 1, 1, tzinfo=tz)\n    contentType_mapper = pd.Series([\"clicks\", \"carts\", \"orders\"], index=[0, 1, 2])\n    target_weight = (0.1, 0.3, 0.6)\n\nif CFG.local:\n    CFG.folder_path = \"./dataset/\"\nelse:\n    CFG.folder_path = \"/kaggle/input/\"\n    \nif CFG.debug:\n    CFG.epochs = 5","metadata":{"execution":{"iopub.status.busy":"2023-01-16T04:13:07.841216Z","iopub.execute_input":"2023-01-16T04:13:07.841558Z","iopub.status.idle":"2023-01-16T04:13:07.852693Z","shell.execute_reply.started":"2023-01-16T04:13:07.841520Z","shell.execute_reply":"2023-01-16T04:13:07.851768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CFG.train_dates = (create_get_ts(datetime.datetime(2022, 8, 15, 0, 0)), create_get_ts(datetime.datetime(2022, 8, 22, 0, 0)))\nCFG.valid_dates = (create_get_ts(datetime.datetime(2022, 8, 22, 0, 0)), create_get_ts(datetime.datetime(2022, 8, 29, 0, 0)))","metadata":{"execution":{"iopub.status.busy":"2023-01-16T04:13:07.855323Z","iopub.execute_input":"2023-01-16T04:13:07.855677Z","iopub.status.idle":"2023-01-16T04:13:07.862441Z","shell.execute_reply.started":"2023-01-16T04:13:07.855643Z","shell.execute_reply":"2023-01-16T04:13:07.861080Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Loading Data","metadata":{}},{"cell_type":"code","source":"metadata = pickleIO(None, \"/kaggle/input/otto-create-dataset-matrix-factorization/metadata.pkl\", \"r\")\ntest = pd.read_parquet('../input/otto-full-optimized-memory-footprint/test.parquet')\nsessions = test.session.unique()","metadata":{"execution":{"iopub.status.busy":"2023-01-16T04:13:07.863761Z","iopub.execute_input":"2023-01-16T04:13:07.864314Z","iopub.status.idle":"2023-01-16T04:13:09.094490Z","shell.execute_reply.started":"2023-01-16T04:13:07.864281Z","shell.execute_reply":"2023-01-16T04:13:09.093543Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test 데이터 세션별 aid 및 event 리스트화\nsession_types = ['clicks', 'carts', 'orders']\ntest_session_AIDs = test.reset_index(drop=True).groupby('session')['aid'].apply(list)\ntest_session_types = test.reset_index(drop=True).groupby('session')['type'].apply(list)","metadata":{"execution":{"iopub.status.busy":"2023-01-16T04:13:09.096089Z","iopub.execute_input":"2023-01-16T04:13:09.096445Z","iopub.status.idle":"2023-01-16T04:14:04.681641Z","shell.execute_reply.started":"2023-01-16T04:13:09.096407Z","shell.execute_reply":"2023-01-16T04:14:04.680681Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Loading Model & Building Neighbor Searcher","metadata":{}},{"cell_type":"code","source":"class MatrixFactorization(nn.Module):\n    def __init__(self, n_aids, embed_size):\n        super().__init__()\n        self.aid_factors = nn.Embedding(n_aids, embed_size, sparse=True)\n        \n    def forward(self, aid1, aid2):\n        aid1 = self.aid_factors(aid1).squeeze(axis=1)\n        aid2 = self.aid_factors(aid2).squeeze(axis=1)\n        return (aid1 * aid2).sum(dim=-1)","metadata":{"execution":{"iopub.status.busy":"2023-01-16T04:14:36.461882Z","iopub.execute_input":"2023-01-16T04:14:36.462614Z","iopub.status.idle":"2023-01-16T04:14:36.470465Z","shell.execute_reply.started":"2023-01-16T04:14:36.462574Z","shell.execute_reply":"2023-01-16T04:14:36.469305Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = MatrixFactorization(metadata[\"cardinality_aids\"], 32)\nmodel.load_state_dict(torch.load(\"/kaggle/input/otto-create-matrix-factorization-model-with-gpu/model_factorization.pth\", map_location=\"cpu\")[\"model\"])\nembeddings = model.aid_factors.weight.detach().cpu().numpy()\ndel model","metadata":{"execution":{"iopub.status.busy":"2023-01-16T04:14:41.186091Z","iopub.execute_input":"2023-01-16T04:14:41.186547Z","iopub.status.idle":"2023-01-16T04:14:43.959359Z","shell.execute_reply.started":"2023-01-16T04:14:41.186504Z","shell.execute_reply":"2023-01-16T04:14:43.958371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nsearcher = cuml.neighbors.NearestNeighbors(21)\nsearcher.fit(embeddings)\ndistances, indices = searcher.kneighbors(embeddings)","metadata":{"execution":{"iopub.status.busy":"2023-01-16T04:15:18.050558Z","iopub.execute_input":"2023-01-16T04:15:18.051000Z","iopub.status.idle":"2023-01-16T04:16:32.757448Z","shell.execute_reply.started":"2023-01-16T04:15:18.050962Z","shell.execute_reply":"2023-01-16T04:16:32.756452Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"distances","metadata":{"execution":{"iopub.status.busy":"2023-01-16T04:16:32.759454Z","iopub.execute_input":"2023-01-16T04:16:32.759821Z","iopub.status.idle":"2023-01-16T04:16:32.768245Z","shell.execute_reply.started":"2023-01-16T04:16:32.759787Z","shell.execute_reply":"2023-01-16T04:16:32.767278Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"indices","metadata":{"execution":{"iopub.status.busy":"2023-01-16T04:16:32.769934Z","iopub.execute_input":"2023-01-16T04:16:32.770522Z","iopub.status.idle":"2023-01-16T04:16:32.778712Z","shell.execute_reply.started":"2023-01-16T04:16:32.770487Z","shell.execute_reply":"2023-01-16T04:16:32.777593Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Inference","metadata":{}},{"cell_type":"code","source":"%%time\n\nn_aids = 20\n\noutput = {\n    \"session\": [],\n    \"type\": [],\n    \"rec\": [],\n    \"score\": [],\n}\n\n# ZERO INDEX IS UNKNOWN AID\nfor SESS, AIDs, types in tqdm(zip(test_session_AIDs.index, test_session_AIDs.values, test_session_types.values), total=len(test_session_AIDs.index)):\n\n    neighbors = indices[AIDs, 1:]\n    candidates = Counter(neighbors.flatten())\n    \n    rec, score = zip(*candidates.most_common(n_aids))\n\n    output[\"session\"].extend([SESS] * 3)\n    output[\"type\"].extend([0, 1, 2])\n    output[\"rec\"].extend([\" \".join(pd.Series(rec, dtype=\"str\").values)] * 3)\n    output[\"score\"].extend([\" \".join(pd.Series(np.round(score, 5), dtype=\"str\").values)] * 3)\n\noutput = pd.DataFrame(output).set_index([\"session\", \"type\"])","metadata":{"execution":{"iopub.status.busy":"2023-01-16T04:21:24.367940Z","iopub.execute_input":"2023-01-16T04:21:24.368342Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"output","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"output.reset_index().to_parquet(\"./raw_output.parquet\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Submission","metadata":{}},{"cell_type":"code","source":"output[\"session_type\"] = [str(i[0]) + \"_\" + str(CFG.contentType_mapper[i[1]]) for i in output.index]","metadata":{"execution":{"iopub.status.busy":"2023-01-05T14:11:14.598344Z","iopub.status.idle":"2023-01-05T14:11:14.598726Z","shell.execute_reply.started":"2023-01-05T14:11:14.598543Z","shell.execute_reply":"2023-01-05T14:11:14.598561Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.read_csv(\"/kaggle/input/otto-recommender-system/sample_submission.csv\")\nsubmission = submission.set_index(\"session_type\")\nsubmission.loc[output[\"session_type\"].values, \"labels\"] = output[\"rec\"].values\nsubmission = submission.reset_index()\nsubmission.to_csv(\"./submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2023-01-05T14:11:14.599744Z","iopub.status.idle":"2023-01-05T14:11:14.600136Z","shell.execute_reply.started":"2023-01-05T14:11:14.599956Z","shell.execute_reply":"2023-01-05T14:11:14.599973Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission","metadata":{"execution":{"iopub.status.busy":"2023-01-05T14:11:14.601453Z","iopub.status.idle":"2023-01-05T14:11:14.601837Z","shell.execute_reply.started":"2023-01-05T14:11:14.601639Z","shell.execute_reply":"2023-01-05T14:11:14.601656Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}