{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[],"dockerImageVersionId":30786,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false},"papermill":{"default_parameters":{},"duration":647.243798,"end_time":"2024-11-19T19:34:11.526167","environment_variables":{},"exception":true,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2024-11-19T19:23:24.282369","version":"2.3.3"},"widgets":{"application/vnd.jupyter.widget-state+json":{"state":{"0df3ed86370d459db46123229ecf04bb":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"0fbd04ea6f714486ad90ba24650978a8":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"FloatProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_7956931ef2794431a99eba75a06ee7ca","max":200,"min":0,"orientation":"horizontal","style":"IPY_MODEL_7687f218339a4a788833040820d73b09","value":200}},"233eed4160e947f09e2fa7ace2566826":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"26f24ce80bf94f41af31c6e718ba2095":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_f7995c16c8f64224b01fd1418da63f9d","IPY_MODEL_0fbd04ea6f714486ad90ba24650978a8","IPY_MODEL_8ab5199cae8f42aca5da93a93c8d50e2"],"layout":"IPY_MODEL_233eed4160e947f09e2fa7ace2566826"}},"3d9d5868e1cc4794887768fd57622ecc":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"42f6a7f7447f43c0bfb303d3e89ea923":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"7687f218339a4a788833040820d73b09":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"7956931ef2794431a99eba75a06ee7ca":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"7c1d237360c14624957edc71b5bf2107":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"8ab5199cae8f42aca5da93a93c8d50e2":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_3d9d5868e1cc4794887768fd57622ecc","placeholder":"​","style":"IPY_MODEL_0df3ed86370d459db46123229ecf04bb","value":" 200/200 [06:16&lt;00:00,  1.88s/it]"}},"f7995c16c8f64224b01fd1418da63f9d":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_7c1d237360c14624957edc71b5bf2107","placeholder":"​","style":"IPY_MODEL_42f6a7f7447f43c0bfb303d3e89ea923","value":"100%"}}},"version_major":2,"version_minor":0}}},"nbformat_minor":4,"nbformat":4,"cells":[{"id":"9d07e428","cell_type":"markdown","source":"# Imports","metadata":{"papermill":{"duration":0.022528,"end_time":"2024-11-19T19:23:31.710726","exception":false,"start_time":"2024-11-19T19:23:31.688198","status":"completed"},"tags":[]}},{"id":"e0800295-d54a-4a6b-8f69-4f22c99b3dc4","cell_type":"code","source":"!pip install implicit","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T14:23:12.707774Z","iopub.execute_input":"2024-12-01T14:23:12.708348Z","iopub.status.idle":"2024-12-01T14:23:25.641881Z","shell.execute_reply.started":"2024-12-01T14:23:12.708257Z","shell.execute_reply":"2024-12-01T14:23:25.640193Z"}},"outputs":[],"execution_count":null},{"id":"952dc1fd","cell_type":"code","source":"import os; os.environ['OPENBLAS_NUM_THREADS']='1'\nimport numpy as np\nimport pandas as pd\nimport implicit\nfrom scipy.sparse import coo_matrix\nfrom implicit.evaluation import mean_average_precision_at_k\nfrom implicit.evaluation import ndcg_at_k","metadata":{"papermill":{"duration":0.347443,"end_time":"2024-11-19T19:23:44.485494","exception":false,"start_time":"2024-11-19T19:23:44.138051","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T14:23:25.644564Z","iopub.execute_input":"2024-12-01T14:23:25.64503Z","iopub.status.idle":"2024-12-01T14:23:26.280294Z","shell.execute_reply.started":"2024-12-01T14:23:25.644988Z","shell.execute_reply":"2024-12-01T14:23:26.279122Z"}},"outputs":[],"execution_count":null},{"id":"893d3dac","cell_type":"markdown","source":"# Load dataframes","metadata":{"papermill":{"duration":0.021689,"end_time":"2024-11-19T19:23:44.529518","exception":false,"start_time":"2024-11-19T19:23:44.507829","status":"completed"},"tags":[]}},{"id":"5376270a-d928-43af-8d35-3e3573f9565e","cell_type":"code","source":"!wget https://ebnerd-dataset.s3.eu-west-1.amazonaws.com/ebnerd_large.zip \n!unzip ebnerd_large.zip ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T14:23:26.281698Z","iopub.execute_input":"2024-12-01T14:23:26.28211Z","iopub.status.idle":"2024-12-01T14:28:19.648438Z","shell.execute_reply.started":"2024-12-01T14:23:26.28208Z","shell.execute_reply":"2024-12-01T14:28:19.645665Z"}},"outputs":[],"execution_count":null},{"id":"b82ac3f7-6d45-4c17-94e1-1c244c37636d","cell_type":"code","source":"import pandas as pd\ndf_history = pd.read_parquet('train/history.parquet')\ndf_history","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T14:28:19.652754Z","iopub.execute_input":"2024-12-01T14:28:19.653297Z","iopub.status.idle":"2024-12-01T14:28:35.7113Z","shell.execute_reply.started":"2024-12-01T14:28:19.653247Z","shell.execute_reply":"2024-12-01T14:28:35.709714Z"}},"outputs":[],"execution_count":null},{"id":"4b6e14dc-9af4-4ed0-bdd5-9b4a516cbefe","cell_type":"code","source":"df = {'UserId':[], 'ItemId':[], 'rating':[]}\nfor id, row in df_history.iterrows():\n    user_id = row['user_id']\n    article_id_fixed = row['article_id_fixed']\n\n    for ai in article_id_fixed:\n        df['UserId'].append(user_id)\n        df['ItemId'].append(ai)\n        df['rating'].append(5)\ndf = pd.DataFrame(df)\ndf = df[['UserId', 'ItemId', 'rating']]\n\nuser_list = df['UserId'].unique()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T14:45:01.398594Z","iopub.execute_input":"2024-12-01T14:45:01.399635Z","iopub.status.idle":"2024-12-01T14:59:37.176344Z","shell.execute_reply.started":"2024-12-01T14:45:01.399587Z","shell.execute_reply":"2024-12-01T14:59:37.17436Z"}},"outputs":[],"execution_count":null},{"id":"57059e5f","cell_type":"code","source":"# test = pd.read_csv('data/public_testset.csv', names=['user_id'] + [f'item_id_{i}' for i in range(1,1001)])\n# test_user_id = test['user_id'].values","metadata":{"papermill":{"duration":1.554158,"end_time":"2024-11-19T19:23:46.666579","exception":false,"start_time":"2024-11-19T19:23:45.112421","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T14:59:37.179206Z","iopub.execute_input":"2024-12-01T14:59:37.179655Z","iopub.status.idle":"2024-12-01T14:59:37.186134Z","shell.execute_reply.started":"2024-12-01T14:59:37.179613Z","shell.execute_reply":"2024-12-01T14:59:37.184993Z"}},"outputs":[],"execution_count":null},{"id":"72135050","cell_type":"code","source":"user_map = {UserId: index for index, UserId in enumerate(user_list)}\nuser_map = pd.DataFrame(list(user_map.items()), columns=['UserId', 'index'])\nuser_map.head()","metadata":{"papermill":{"duration":0.056867,"end_time":"2024-11-19T19:23:46.745517","exception":false,"start_time":"2024-11-19T19:23:46.68865","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T14:59:37.188013Z","iopub.execute_input":"2024-12-01T14:59:37.188492Z","iopub.status.idle":"2024-12-01T14:59:39.42733Z","shell.execute_reply.started":"2024-12-01T14:59:37.188451Z","shell.execute_reply":"2024-12-01T14:59:39.42594Z"}},"outputs":[],"execution_count":null},{"id":"0e3f378c","cell_type":"markdown","source":"## Assign autoincrementing ids starting from 0 to both users and items","metadata":{"papermill":{"duration":0.021732,"end_time":"2024-11-19T19:23:46.887557","exception":false,"start_time":"2024-11-19T19:23:46.865825","status":"completed"},"tags":[]}},{"id":"8e0ad17e","cell_type":"code","source":"ALL_USERS = df['UserId'].unique().tolist()\nALL_ITEMS = df['ItemId'].unique().tolist()\n\nuser_ids = dict(list(enumerate(ALL_USERS)))\nitem_ids = dict(list(enumerate(ALL_ITEMS)))\n\nuser_map = {u: uidx for uidx, u in user_ids.items()}\nitem_map = {i: iidx for iidx, i in item_ids.items()}\n\ndf['UserId'] = df['UserId'].map(user_map)\ndf['ItemId'] = df['ItemId'].map(item_map)\n","metadata":{"papermill":{"duration":0.231213,"end_time":"2024-11-19T19:23:47.140779","exception":false,"start_time":"2024-11-19T19:23:46.909566","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T14:59:39.430454Z","iopub.execute_input":"2024-12-01T14:59:39.431013Z","iopub.status.idle":"2024-12-01T14:59:46.61093Z","shell.execute_reply.started":"2024-12-01T14:59:39.430959Z","shell.execute_reply":"2024-12-01T14:59:46.609629Z"}},"outputs":[],"execution_count":null},{"id":"e124e4df","cell_type":"code","source":"# test['user_index'] = test['user_id'].map(user_map)\n# test_user_index = test['user_index'].values","metadata":{"papermill":{"duration":0.040706,"end_time":"2024-11-19T19:23:47.203947","exception":false,"start_time":"2024-11-19T19:23:47.163241","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T14:59:46.612367Z","iopub.execute_input":"2024-12-01T14:59:46.612752Z","iopub.status.idle":"2024-12-01T14:59:46.618368Z","shell.execute_reply.started":"2024-12-01T14:59:46.612717Z","shell.execute_reply":"2024-12-01T14:59:46.616854Z"}},"outputs":[],"execution_count":null},{"id":"4dce8af8","cell_type":"code","source":"# Split data into train and test sets (90% for train, 10% for test)\ntrain_df = df.sample(frac=0.95, random_state=42)  # 90% for train\ntest_df = df.drop(train_df.index)  # Remaining 10% for test","metadata":{"papermill":{"duration":0.123062,"end_time":"2024-11-19T19:23:47.349507","exception":false,"start_time":"2024-11-19T19:23:47.226445","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T14:59:46.620248Z","iopub.execute_input":"2024-12-01T14:59:46.620722Z","iopub.status.idle":"2024-12-01T15:00:14.22438Z","shell.execute_reply.started":"2024-12-01T14:59:46.620684Z","shell.execute_reply":"2024-12-01T15:00:14.223251Z"}},"outputs":[],"execution_count":null},{"id":"1d26050c","cell_type":"markdown","source":"## Create coo_matrix (user x item) and csr matrix (user x item)\n\nIt is common to use scipy sparse matrices in recommender systems, because the main core of the problem is typically modeled as a matrix with users and items, with the values representing whether the user purchased (or liked) an items. Since each user purchases only a small fraction of the catalog of products, this matrix is full of zero (aka: it's sparse).\n\nIn a very recent release they did an API breaking change, so be aware of that: https://github.com/benfred/implicit/releases\nIn this notebook we are using the latest version, so everything is aligned with (user x item)\n\n**We are using (user x item) matrices, both for training and for evaluating/recommender.**\n\nIn the previous versions the training procedure required a COO item x user\n\nFor evaluation and prediction, on the other hand, CSR matrices with users x items format should be provided.\n\n\n### About COO matrices\nCOO matrices are a kind of sparse matrix.\nThey store their values as tuples of `(row, column, value)` (the coordinates)\n\nYou can read more about them here: \n* https://en.wikipedia.org/wiki/Sparse_matrix#Coordinate_list_(COO)\n* https://scipy-lectures.org/advanced/scipy_sparse/coo_matrix.html\n\nFrom https://het.as.utexas.edu/HET/Software/Scipy/generated/scipy.sparse.coo_matrix.html\n\n```python\n>>> row  = np.array([0,3,1,0]) # user_ids\n>>> col  = np.array([0,3,1,2]) # item_ids\n>>> data = np.array([4,5,7,9]) # a bunch of ones of lenght unique(user) x unique(items)\n>>> coo_matrix((data,(row,col)), shape=(4,4)).todense()\nmatrix([[4, 0, 9, 0],\n        [0, 7, 0, 0],\n        [0, 0, 0, 0],\n        [0, 0, 0, 5]])\n```\n\n## About CSR matrices\n* https://en.wikipedia.org/wiki/Sparse_matrix#Compressed_sparse_row_(CSR,_CRS_or_Yale_format)","metadata":{"papermill":{"duration":0.021693,"end_time":"2024-11-19T19:23:47.393396","exception":false,"start_time":"2024-11-19T19:23:47.371703","status":"completed"},"tags":[]}},{"id":"ca72ed9c","cell_type":"code","source":"row = df['UserId'].values\ncol = df['ItemId'].values\n# data = np.ones(train_df.shape[0])\ndata = df['rating'].values\ncoo_train = coo_matrix((data, (row, col)), shape=(len(ALL_USERS), len(ALL_ITEMS)))\ncoo_train","metadata":{"papermill":{"duration":0.032788,"end_time":"2024-11-19T19:23:47.447938","exception":false,"start_time":"2024-11-19T19:23:47.41515","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T15:00:14.226273Z","iopub.execute_input":"2024-12-01T15:00:14.227152Z","iopub.status.idle":"2024-12-01T15:00:15.403162Z","shell.execute_reply.started":"2024-12-01T15:00:14.227102Z","shell.execute_reply":"2024-12-01T15:00:15.401694Z"}},"outputs":[],"execution_count":null},{"id":"1de67588","cell_type":"markdown","source":"# Validation","metadata":{"papermill":{"duration":0.022148,"end_time":"2024-11-19T19:23:47.492589","exception":false,"start_time":"2024-11-19T19:23:47.470441","status":"completed"},"tags":[]}},{"id":"667b0762","cell_type":"markdown","source":"## Functions required for validation","metadata":{"papermill":{"duration":0.02199,"end_time":"2024-11-19T19:23:47.536837","exception":false,"start_time":"2024-11-19T19:23:47.514847","status":"completed"},"tags":[]}},{"id":"ba1e264b","cell_type":"code","source":"def to_user_item_coo(df):\n    \"\"\" Turn a dataframe with transactions into a COO sparse items x users matrix\"\"\"\n    row = df['UserId'].values\n    col = df['ItemId'].values\n    # data = np.ones(df.shape[0])\n    data = df['rating'].values\n    coo = coo_matrix((data, (row, col)), shape=(len(ALL_USERS), len(ALL_ITEMS)))\n    return coo\n\n\ndef split_data(df, validation_days=7):\n    \"\"\" Split a pandas dataframe into training and validation data, using <<validation_days>>\n    \"\"\"\n    validation_cut = df['t_dat'].max() - pd.Timedelta(validation_days)\n\n    df_train = df[df['t_dat'] < validation_cut]\n    df_val = df[df['t_dat'] >= validation_cut]\n    return df_train, df_val\n\ndef get_val_matrices(df, validation_days=7):\n    \"\"\" Split into training and validation and create various matrices\n        \n        Returns a dictionary with the following keys:\n            coo_train: training data in COO sparse format and as (users x items)\n            csr_train: training data in CSR sparse format and as (users x items)\n            csr_val:  validation data in CSR sparse format and as (users x items)\n    \n    \"\"\"\n    df_train, df_val = train_df, test_df\n    coo_train = to_user_item_coo(df_train)\n    coo_val = to_user_item_coo(df_val)\n\n    csr_train = coo_train.tocsr()\n    csr_val = coo_val.tocsr()\n    \n    return {'coo_train': coo_train,\n            'csr_train': csr_train,\n            'csr_val': csr_val\n          }\n\n\ndef validate(model, matrices, factors=200, iterations=20, regularization=0.01, show_progress=True):\n    \"\"\" Train an ALS model with <<factors>> (embeddings dimension) \n    for <<iterations>> over matrices and validate with MAP@12\n    \"\"\"\n    coo_train, csr_train, csr_val = matrices['coo_train'], matrices['csr_train'], matrices['csr_val']\n    \n    # model = implicit.als.AlternatingLeastSquares(factors=factors, \n    #                                              iterations=iterations, \n    #                                              regularization=regularization, \n    #                                              random_state=42)\n    # model.fit(coo_train, show_progress=show_progress)\n    \n    # The MAPK by implicit doesn't allow to calculate allowing repeated items, which is the case.\n    # TODO: change MAP@12 to a library that allows repeated items in prediction\n    # map12 = mean_average_precision_at_k(model, csr_train, csr_val, K=12, show_progress=show_progress, num_threads=4)\n    ndcg = ndcg_at_k(model, csr_train, csr_val, K=10, show_progress=show_progress, num_threads=4)\n    print(f\"Factors: {factors:>3} - Iterations: {iterations:>2} - Regularization: {regularization:4.3f} ==> NDCG@10: {ndcg:6.5f}\")\n    return ndcg","metadata":{"papermill":{"duration":0.032794,"end_time":"2024-11-19T19:23:47.591628","exception":false,"start_time":"2024-11-19T19:23:47.558834","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T15:00:15.40553Z","iopub.execute_input":"2024-12-01T15:00:15.405936Z","iopub.status.idle":"2024-12-01T15:00:15.417587Z","shell.execute_reply.started":"2024-12-01T15:00:15.405901Z","shell.execute_reply":"2024-12-01T15:00:15.415879Z"}},"outputs":[],"execution_count":null},{"id":"4da6b512","cell_type":"code","source":"matrices = get_val_matrices(df)","metadata":{"papermill":{"duration":0.050387,"end_time":"2024-11-19T19:23:47.664069","exception":false,"start_time":"2024-11-19T19:23:47.613682","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T15:00:15.419348Z","iopub.execute_input":"2024-12-01T15:00:15.420809Z","iopub.status.idle":"2024-12-01T15:00:52.815874Z","shell.execute_reply.started":"2024-12-01T15:00:15.42075Z","shell.execute_reply":"2024-12-01T15:00:52.814499Z"}},"outputs":[],"execution_count":null},{"id":"e0a0d69c","cell_type":"code","source":"# %%time\n# best_map12 = 0\n# for factors in [40, 50, 60, 100, 200, 500, 1000]:\n#     for regularization in [0.01]:\n#         map12 = validate(matrices, factors, iterations, regularization, show_progress=False)\n#         if map12 > best_map12:\n#             best_map12 = map12\n#             best_params = {'factors': factors, 'iterations': iterations, 'regularization': regularization}\n#             print(f\"Best MAP@12 found. Updating: {best_params}\")\n\nfactors = 1000\niterations = 200\nregularization = 0.01\nbest_params = {'factors': factors, 'iterations': iterations, 'regularization': regularization}","metadata":{"papermill":{"duration":0.028094,"end_time":"2024-11-19T19:23:47.714489","exception":false,"start_time":"2024-11-19T19:23:47.686395","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T15:00:52.820579Z","iopub.execute_input":"2024-12-01T15:00:52.82105Z","iopub.status.idle":"2024-12-01T15:00:52.830869Z","shell.execute_reply.started":"2024-12-01T15:00:52.821013Z","shell.execute_reply":"2024-12-01T15:00:52.828235Z"}},"outputs":[],"execution_count":null},{"id":"4d02b97c","cell_type":"markdown","source":"# Training over the full dataset","metadata":{"papermill":{"duration":0.021944,"end_time":"2024-11-19T19:23:47.758625","exception":false,"start_time":"2024-11-19T19:23:47.736681","status":"completed"},"tags":[]}},{"id":"2e502253","cell_type":"code","source":"coo_train = to_user_item_coo(df)\ncsr_train = coo_train.tocsr()","metadata":{"papermill":{"duration":0.045829,"end_time":"2024-11-19T19:23:47.826671","exception":false,"start_time":"2024-11-19T19:23:47.780842","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T15:00:52.833241Z","iopub.execute_input":"2024-12-01T15:00:52.833706Z","iopub.status.idle":"2024-12-01T15:01:01.962131Z","shell.execute_reply.started":"2024-12-01T15:00:52.833656Z","shell.execute_reply":"2024-12-01T15:01:01.960157Z"}},"outputs":[],"execution_count":null},{"id":"5b50dd97","cell_type":"code","source":"def train(coo_train, factors=200, iterations=15, regularization=0.01, show_progress=True):\n    model = implicit.als.AlternatingLeastSquares(factors=factors, \n                                                 iterations=iterations, \n                                                 regularization=regularization, \n                                                 random_state=42)\n    model.fit(coo_train, show_progress=show_progress)\n    return model","metadata":{"papermill":{"duration":0.028121,"end_time":"2024-11-19T19:23:47.876912","exception":false,"start_time":"2024-11-19T19:23:47.848791","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T15:01:01.963879Z","iopub.execute_input":"2024-12-01T15:01:01.964262Z","iopub.status.idle":"2024-12-01T15:01:01.97166Z","shell.execute_reply.started":"2024-12-01T15:01:01.964225Z","shell.execute_reply":"2024-12-01T15:01:01.970092Z"}},"outputs":[],"execution_count":null},{"id":"92539d04","cell_type":"code","source":"model = train(coo_train, **best_params)","metadata":{"papermill":{"duration":376.92678,"end_time":"2024-11-19T19:30:04.825669","exception":false,"start_time":"2024-11-19T19:23:47.898889","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2024-12-01T15:01:01.973307Z","iopub.execute_input":"2024-12-01T15:01:01.97373Z"}},"outputs":[],"execution_count":null},{"id":"45da96f7-ad9c-4a87-89d2-31767fa8adf8","cell_type":"code","source":"save_path = \"runs/ALS\"\nos.makedirs(save_path, exist_ok=True)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"4922dc99","cell_type":"code","source":"import pickle\nwith open(f\"{save_path}/user_embedding.pkl\", \"wb\") as f:\n    pickle.dump(model.user_factors, f)\n    \nwith open(f\"{save_path}/item_embedding.pkl\", \"wb\") as f:\n    pickle.dump(model.user_factors, f)\n    \nwith open(f\"{save_path}/model.pkl\", \"wb\") as f:\n    pickle.dump(model, f)\n\nwith open(f\"{save_path}/usermap.pkl\", \"wb\") as f:\n    pickle.dump(user_map, f)\n\nwith open(f\"{save_path}/itemmap.pkl\", \"wb\") as f:\n    pickle.dump(item_map, f)\n\nwith open(f\"{save_path}/csr_train.pkl\", \"wb\") as f:\n    pickle.dump(csr_train, f)","metadata":{"papermill":{"duration":2.099837,"end_time":"2024-11-19T19:30:06.94829","exception":false,"start_time":"2024-11-19T19:30:04.848453","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"5f293d6a-85e3-4c86-865d-4b0759aa2630","cell_type":"code","source":"user_ids","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"ea40b2fb","cell_type":"code","source":"ndcg10 = validate(model, matrices, factors, iterations, regularization, show_progress=False)\nprint(ndcg10)","metadata":{"papermill":{"duration":0.318396,"end_time":"2024-11-19T19:30:07.289424","exception":false,"start_time":"2024-11-19T19:30:06.971028","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"a425a58a-aff9-4930-8886-a9a69159e046","cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}