{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Riiidコンペの実際のデータから(ユーザ×問題)の特徴量のcsv生成","metadata":{}},{"cell_type":"code","source":"%%time\nimport cudf\n\ntypes = {\n        'row_id': 'int64',\n        'timestamp': 'int64',\n        'user_id': 'int32',\n        'content_id': 'int16',\n        'content_type_id': 'int8',\n        'task_container_id': 'int16',\n        'user_answer': 'int8',\n        'answered_correctly': 'int8',\n        'prior_question_elapsed_time': 'float32',\n        'prior_question_had_explanation': 'int8'\n}\ndatapath = '/kaggle/input/riiid-test-answer-prediction/train.csv'\n\ntrain_X = cudf.read_csv(datapath, dtype=types)\ntrain_X = train_X[train_X['content_type_id'] == 0]\nfeat_float = train_X.groupby(['user_id', 'content_id'])['answered_correctly'].mean().astype('float32')\nfeat_int = train_X.groupby(['user_id', 'content_id'])['answered_correctly'].count().astype('int16')\ndel train_X\n\nprint('number of keys : ', len(feat_int))\nfeat_float.reset_index().to_csv('user_content_wise_float.csv', index=False)\nfeat_int.reset_index().to_csv('user_content_wise_int.csv', index=False)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## C++のコードをriiid_module.cppに保存","metadata":{}},{"cell_type":"code","source":"%%file riiid_module.cpp\n\n#include <parallel_hashmap/phmap.h>\n#include <string>\n#include <fstream>\n#include <iostream>\n\nusing phmap::flat_hash_map;\n\nstruct riiid_user_x_content_int {\n  riiid_user_x_content_int(const std::string& datapath) {\n    std::ifstream mycsvfile (datapath);\n    if (mycsvfile.is_open()) {\n      std::string line; getline(mycsvfile, line);\n      while (mycsvfile.good()) {\n          getline(mycsvfile, line);\n          if (line.size() == 0) break;\n \n          int comma_pos = line.find(',', 0);\n          int comma_pos2 = line.find(',', comma_pos+1);\n \n          uint32_t user_id = stoi(line.substr(0, comma_pos));\n          uint16_t content_id = stoi(line.substr(comma_pos+1,\n            comma_pos2-comma_pos-1));\n          uint8_t value = stoi(line.substr(comma_pos2+1));\n          m_dict[user_id][content_id] = value;\n      }\n      std::cout << \"csv file loaded\" << std::endl;\n    } else {\n      std::cout << \"Error in opening file\" << std::endl;\n    }\n  }\n\n  int getval(int user_id, int content_id) {\n    return m_dict[user_id][content_id];\n  }\n\n  void setval(int user_id, int content_id, int value) {\n    m_dict[user_id][content_id] = value;\n  }\n\n  flat_hash_map<uint32_t, flat_hash_map<uint16_t, uint8_t>> m_dict;\n};\n\n#include <pybind11/pybind11.h>\nnamespace py = pybind11;\n\nPYBIND11_MODULE(riiid_module, m) {\n  py::class_<riiid_user_x_content_int>(m, \"user_x_content_int\")\n    .def(py::init<const std::string &>())\n    .def(\"get\", &riiid_user_x_content_int::getval, \n      py::arg(\"user_id\"), py::arg(\"content_id\"))\n    .def(\"set\", &riiid_user_x_content_int::setval,\n      py::arg(\"user_id\"), py::arg(\"content_id\"), py::arg(\"value\"));\n}","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-12-19T11:03:39.232593Z","iopub.execute_input":"2021-12-19T11:03:39.233106Z","iopub.status.idle":"2021-12-19T11:03:39.265348Z","shell.execute_reply.started":"2021-12-19T11:03:39.232978Z","shell.execute_reply":"2021-12-19T11:03:39.264635Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path_to_parallelmap_folder = \"/kaggle/input/parallel-hashmap\"\npath_to_cppfile = \"./riiid_module.cpp\"\nmodule_name = \"riiid_module\"","metadata":{"execution":{"iopub.status.busy":"2021-12-19T11:03:39.268632Z","iopub.execute_input":"2021-12-19T11:03:39.268845Z","iopub.status.idle":"2021-12-19T11:03:39.27271Z","shell.execute_reply.started":"2021-12-19T11:03:39.268819Z","shell.execute_reply":"2021-12-19T11:03:39.272157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 共有ライブラリにコンパイル","metadata":{}},{"cell_type":"code","source":"!c++ -O3 -Wall -shared -std=c++11 -fPIC `python3 -m pybind11 --includes` $path_to_cppfile -I$path_to_parallelmap_folder -o $module_name`python3-config --extension-suffix`","metadata":{"execution":{"iopub.status.busy":"2021-12-19T11:03:39.273808Z","iopub.execute_input":"2021-12-19T11:03:39.274465Z","iopub.status.idle":"2021-12-19T11:03:48.347377Z","shell.execute_reply.started":"2021-12-19T11:03:39.274436Z","shell.execute_reply":"2021-12-19T11:03:48.346507Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### メモリプロファイラインストール","metadata":{}},{"cell_type":"code","source":"!pip install memory_profiler\n%load_ext memory_profiler","metadata":{"execution":{"iopub.status.busy":"2021-12-19T11:03:48.349516Z","iopub.execute_input":"2021-12-19T11:03:48.349793Z","iopub.status.idle":"2021-12-19T11:03:58.069865Z","shell.execute_reply.started":"2021-12-19T11:03:48.349762Z","shell.execute_reply":"2021-12-19T11:03:58.068892Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Pythonから拡張モジュールをimport","metadata":{}},{"cell_type":"code","source":"%%time\nfrom riiid_module import user_x_content_int\n\npath_to_csv = \"./user_content_wise_int.csv\"\n%memit cumulative_count = user_x_content_int(path_to_csv)\n\ncumulative_count.set(user_id=42, content_id=4, value=3) # セット\nassert cumulative_count.get(user_id=42, content_id=4) == 3 # キーワード引数\nassert cumulative_count.get(42, 4) == 3 # ポジション引数","metadata":{"execution":{"iopub.status.busy":"2021-12-19T11:03:58.07143Z","iopub.execute_input":"2021-12-19T11:03:58.071761Z","iopub.status.idle":"2021-12-19T11:03:58.401632Z","shell.execute_reply.started":"2021-12-19T11:03:58.071716Z","shell.execute_reply":"2021-12-19T11:03:58.400951Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}