{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Motivation","metadata":{}},{"cell_type":"markdown","source":"Suppose we have (user_id × content_id)-wise features saved as csv files:","metadata":{"papermill":{"duration":0.012262,"end_time":"2021-01-08T08:38:21.043924","exception":false,"start_time":"2021-01-08T08:38:21.031662","status":"completed"},"tags":[]}},{"cell_type":"code","source":"%%time\nimport cudf\n\ntypes = {\n        'row_id': 'int64',\n        'timestamp': 'int64',\n        'user_id': 'int32',\n        'content_id': 'int16',\n        'content_type_id': 'int8',\n        'task_container_id': 'int16',\n        'user_answer': 'int8',\n        'answered_correctly': 'int8',\n        'prior_question_elapsed_time': 'float32',\n        'prior_question_had_explanation': 'int8'\n}\ndatapath = '/kaggle/input/riiid-test-answer-prediction/train.csv'\n\ntrain_X = cudf.read_csv(datapath, dtype=types)\ntrain_X = train_X[train_X['content_type_id'] == 0]\nfeat_float = train_X.groupby(['user_id', 'content_id'])['answered_correctly'].mean().astype('float32')\nfeat_int = train_X.groupby(['user_id', 'content_id'])['answered_correctly'].count().astype('int16')\n# del train_X\n\nprint('number of keys : ', len(feat_int))\nfeat_float.reset_index().to_csv('user_content_wise_float.csv', index=False)\nfeat_int.reset_index().to_csv('user_content_wise_int.csv', index=False)","metadata":{"papermill":{"duration":85.997121,"end_time":"2021-01-08T08:39:47.052234","exception":false,"start_time":"2021-01-08T08:38:21.055113","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-19T10:04:07.895475Z","iopub.execute_input":"2021-12-19T10:04:07.895815Z","iopub.status.idle":"2021-12-19T10:05:25.104076Z","shell.execute_reply.started":"2021-12-19T10:04:07.895783Z","shell.execute_reply":"2021-12-19T10:05:25.103089Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_X.user_id.nunique(), train_X.content_id.nunique()","metadata":{"execution":{"iopub.status.busy":"2021-12-19T10:05:25.105946Z","iopub.execute_input":"2021-12-19T10:05:25.106211Z","iopub.status.idle":"2021-12-19T10:05:25.597002Z","shell.execute_reply.started":"2021-12-19T10:05:25.106184Z","shell.execute_reply":"2021-12-19T10:05:25.596301Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install memory_profiler\n%load_ext memory_profiler","metadata":{"execution":{"iopub.status.busy":"2021-12-19T10:05:25.598316Z","iopub.execute_input":"2021-12-19T10:05:25.598689Z","iopub.status.idle":"2021-12-19T10:05:42.479104Z","shell.execute_reply.started":"2021-12-19T10:05:25.598652Z","shell.execute_reply":"2021-12-19T10:05:42.477731Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def all_dict():\n    num_user_id = len(feat_float.index.get_level_values(level=0).unique())\n    \n    print('num_user_id :', num_user_id)\n\n    feat_dict = dict()\n    for cnt, (user_id, data) in enumerate(feat_float.groupby(level=0)):\n        df = data.to_frame().to_pandas()\n        feat_dict[user_id] = df.reset_index().drop('user_id',axis=1).set_index('content_id').to_dict()['answered_correctly']\n        del df\n    return feat_dict\n\n%memit feat_dict = all_dict()\ndel feat_dict","metadata":{"execution":{"iopub.status.busy":"2021-12-19T09:15:54.344562Z","iopub.execute_input":"2021-12-19T09:15:54.347796Z","iopub.status.idle":"2021-12-19T09:32:05.379159Z","shell.execute_reply.started":"2021-12-19T09:15:54.347738Z","shell.execute_reply":"2021-12-19T09:32:05.378158Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We have a huge dict with the number of keys 86867031, and the estimated RAM usage with Python dict (64-bit float value) is 8588MiB.\n\nIt is hard to put this data on RAM as a usual Python dict, but we can borrow the power of parallel-hashmap implemented in C++ through pybind11 to put it on RAM.","metadata":{"papermill":{"duration":0.134787,"end_time":"2021-01-08T08:39:47.233037","exception":false,"start_time":"2021-01-08T08:39:47.09825","status":"completed"},"tags":[]}},{"cell_type":"code","source":"del feat_float, feat_int\n\npath_to_parallelmap_folder = \"/kaggle/input/parallel-hashmap\"\npath_to_cppfile = \"/kaggle/input/pybind11demo/mydicts.cpp\"\nmodule_name = \"my_module\"","metadata":{"papermill":{"duration":7.122486,"end_time":"2021-01-08T08:40:13.625519","exception":false,"start_time":"2021-01-08T08:40:06.503033","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-05T10:10:16.217668Z","iopub.execute_input":"2021-06-05T10:10:16.218051Z","iopub.status.idle":"2021-06-05T10:10:16.224626Z","shell.execute_reply.started":"2021-06-05T10:10:16.218016Z","shell.execute_reply":"2021-06-05T10:10:16.22198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Compiling and importing","metadata":{}},{"cell_type":"markdown","source":"We first need to compile the cpp file with the following command, and no, I don't fully understand what this long command does neither. See [pybind11 tutorial](https://pybind11.readthedocs.io/en/stable/basics.html) for details.","metadata":{"papermill":{"duration":0.013605,"end_time":"2021-01-08T08:40:13.659433","exception":false,"start_time":"2021-01-08T08:40:13.645828","status":"completed"},"tags":[]}},{"cell_type":"code","source":"!c++ -O3 -Wall -shared -std=c++11 -fPIC `python3 -m pybind11 --includes` $path_to_cppfile -I$path_to_parallelmap_folder -o $module_name`python3-config --extension-suffix`","metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","papermill":{"duration":7.872361,"end_time":"2021-01-08T08:40:21.545533","exception":false,"start_time":"2021-01-08T08:40:13.673172","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-05T10:10:17.963515Z","iopub.execute_input":"2021-06-05T10:10:17.963908Z","iopub.status.idle":"2021-06-05T10:10:26.170209Z","shell.execute_reply.started":"2021-06-05T10:10:17.963874Z","shell.execute_reply":"2021-06-05T10:10:26.168776Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now we can import the classes defined in the cpp file (``mydicts.cpp``).","metadata":{"papermill":{"duration":0.013417,"end_time":"2021-01-08T08:40:21.572881","exception":false,"start_time":"2021-01-08T08:40:21.559464","status":"completed"},"tags":[]}},{"cell_type":"code","source":"!ls","metadata":{"execution":{"iopub.status.busy":"2021-06-05T10:10:29.46832Z","iopub.execute_input":"2021-06-05T10:10:29.46875Z","iopub.status.idle":"2021-06-05T10:10:30.500615Z","shell.execute_reply.started":"2021-06-05T10:10:29.468697Z","shell.execute_reply":"2021-06-05T10:10:30.498918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from my_module import my_dict_int, my_dict_float","metadata":{"papermill":{"duration":0.022565,"end_time":"2021-01-08T08:40:21.608331","exception":false,"start_time":"2021-01-08T08:40:21.585766","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-05T10:10:35.334308Z","iopub.execute_input":"2021-06-05T10:10:35.334731Z","iopub.status.idle":"2021-06-05T10:10:35.34126Z","shell.execute_reply.started":"2021-06-05T10:10:35.334685Z","shell.execute_reply":"2021-06-05T10:10:35.340016Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## How to use","metadata":{}},{"cell_type":"markdown","source":"To instantiate the hash map, we pass the datapath of csv to the constuctor. Then, on C++ side, the csv file is parsed, and the hash map is constructed.\n\nIn ``mydicts.cpp``, I defined two operations:\n\n* ``setval(user_id, content_id, value)`` of my_dict_int/float ⇔ ``update({(user_id, content_id): value})`` of Python 3 dict\n* ``getval(user_id, content_id)`` of my_dict_int/float ⇔ ``setdefault((user_id, content_id), 0/0.0)`` of Python 3 dict\n\nThese operations just reflect my taste; you can modify the cpp file (mydicts.cpp) and use any API you prefer.\n\nAlso, you can use any data format (not only csv) if you can parse it on C++ side. You need to modify the constructor in ``mydicts.cpp`` of each class accordingly.","metadata":{"papermill":{"duration":0.012614,"end_time":"2021-01-08T08:40:21.63385","exception":false,"start_time":"2021-01-08T08:40:21.621236","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"## memory/time efficiency","metadata":{}},{"cell_type":"markdown","source":"[The parallel hashmap](https://github.com/greg7mdp/parallel-hashmap) is really RAM-friendly: \n\n* ~700MiB for each 16-bit integer (user_id × content_id)-wise feature\n* ~1300MiB for each 32-bit float (user_id × content_id)-wise feature","metadata":{"papermill":{"duration":0.013179,"end_time":"2021-01-08T08:40:21.660202","exception":false,"start_time":"2021-01-08T08:40:21.647023","status":"completed"},"tags":[]}},{"cell_type":"code","source":"%%time\n%memit user_content_feat_float = my_dict_float(\"/kaggle/working/user_content_wise_float.csv\")","metadata":{"execution":{"iopub.execute_input":"2021-01-08T08:40:48.629243Z","iopub.status.busy":"2021-01-08T08:40:48.628592Z","iopub.status.idle":"2021-01-08T08:41:19.620175Z","shell.execute_reply":"2021-01-08T08:41:19.620824Z"},"papermill":{"duration":31.138183,"end_time":"2021-01-08T08:41:19.621","exception":false,"start_time":"2021-01-08T08:40:48.482817","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n%memit user_content_feat_int = my_dict_int(\"/kaggle/working/user_content_wise_int.csv\")","metadata":{"execution":{"iopub.execute_input":"2021-01-08T08:40:21.817719Z","iopub.status.busy":"2021-01-08T08:40:21.817054Z","iopub.status.idle":"2021-01-08T08:40:48.468456Z","shell.execute_reply":"2021-01-08T08:40:48.467929Z"},"papermill":{"duration":26.795754,"end_time":"2021-01-08T08:40:48.468607","exception":false,"start_time":"2021-01-08T08:40:21.672853","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Accessing and modifying the value is very fast:","metadata":{"papermill":{"duration":0.014521,"end_time":"2021-01-08T08:41:19.650127","exception":false,"start_time":"2021-01-08T08:41:19.635606","status":"completed"},"tags":[]}},{"cell_type":"code","source":"%timeit user_content_feat_int.setval(user_id=5, content_id=16, value=123)\n%timeit user_content_feat_int.getval(user_id=5, content_id=16)\n\nprint(user_content_feat_int.getval(user_id=5, content_id=16))\n\n%timeit user_content_feat_float.setval(user_id=5, content_id=16, value=123.193)\n%timeit user_content_feat_float.getval(user_id=5, content_id=16)\n\nprint(user_content_feat_float.getval(user_id=5, content_id=16))","metadata":{"execution":{"iopub.execute_input":"2021-01-08T08:41:19.729129Z","iopub.status.busy":"2021-01-08T08:41:19.708713Z","iopub.status.idle":"2021-01-08T08:42:15.870355Z","shell.execute_reply":"2021-01-08T08:42:15.869244Z"},"papermill":{"duration":56.20591,"end_time":"2021-01-08T08:42:15.870594","exception":false,"start_time":"2021-01-08T08:41:19.664684","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## positional/keyword arguments","metadata":{}},{"cell_type":"markdown","source":"We can use both positional and keyword arguments, thanks to pybind11:","metadata":{"papermill":{"duration":0.016244,"end_time":"2021-01-08T08:42:15.903684","exception":false,"start_time":"2021-01-08T08:42:15.88744","status":"completed"},"tags":[]}},{"cell_type":"code","source":"print(user_content_feat_int.getval(user_id=2147482888, content_id=9788))\n\nassert user_content_feat_int.getval(user_id=2147482888, content_id=9788) == user_content_feat_int.getval(2147482888, 9788)","metadata":{"execution":{"iopub.execute_input":"2021-01-08T08:42:15.943576Z","iopub.status.busy":"2021-01-08T08:42:15.941648Z","iopub.status.idle":"2021-01-08T08:42:15.946093Z","shell.execute_reply":"2021-01-08T08:42:15.945525Z"},"papermill":{"duration":0.02634,"end_time":"2021-01-08T08:42:15.946205","exception":false,"start_time":"2021-01-08T08:42:15.919865","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]}]}