{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport gc\n\nDEBUG = True\nTARGET = \"answered_correctly\"\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_load(debug=True):\n    rows = 10 ** 4 if debug else 10 ** 7\n    train = pd.read_csv(\"../input/riiid-test-answer-prediction/train.csv\",\n                        low_memory=False,\n                        nrows=rows,\n                        dtype={'row_id': 'int64',\n                               'timestamp': 'int64',\n                               'user_id': 'int32',\n                               'content_id': 'int16',\n                               'content_type_id': 'int8',\n                               'task_container_id': 'int16',\n                               'user_answer': 'int8',\n                               'answered_correctly': 'int8',\n                               'prior_question_elapsed_time': 'float32',\n                               'prior_question_had_explanation': 'boolean',\n                               }\n                        ).drop(columns=\"row_id\")\n\n    train[\"prior_question_elapsed_time\"] = train[\"prior_question_elapsed_time\"].replace(np.nan, 0).astype(\"int32\")\n\n    print(train.info())\n    return train\n\ndef question():\n    questions = pd.read_csv(\"../input/riiid-test-answer-prediction/questions.csv\")\n    print(questions.info())\n    return questions\n\n\ndef lecture():\n    lectures = pd.read_csv(\"../input/riiid-test-answer-prediction/lectures.csv\")\n    print(lectures.info())\n    return lectures","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","collapsed":true,"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":false},"cell_type":"code","source":"def features_engineering(data):\n    data[\"count\"] = data.groupby(\"user_id\")[TARGET].transform(\"count\")\n    data[\"answered_num\"] = data.groupby(\"user_id\")[TARGET].transform(\"sum\")\n    data[\"correct_ratio\"] = data[\"answered_num\"] / data[\"count\"]\n\n    return data\n\n\ndef questions_preprocess(data):\n    data = data.reset_index(drop=True)\n    data = data.drop(columns=[\"bundle_id\", \"correct_answer\"])\n\n    data = data.drop(columns=\"tags\")\n    data = data.rename(columns={\"question_id\": \"content_id\"})\n\n    data[\"content_id\"] = data[\"content_id\"].astype(\"int16\")\n    return data\n\n\ndef lectures_preprocess(data):\n    data = data.reset_index(drop=True)\n    data[\"type_of\"] = data[\"type_of\"].astype(\"category\").cat.codes\n    data = data.rename(columns={\"lecture_id\": \"content_id\"})\n\n    data[\"content_id\"] = data[\"content_id\"].astype(\"int16\")\n    return data\n\n\ndef concat(data):\n    questions = questions_preprocess(question())\n    lectures = lectures_preprocess(lecture())\n\n    q_data = data.loc[data[\"content_type_id\"] == 0, :].copy()\n    q_data = q_data.merge(questions, on=\"content_id\", how=\"left\")\n\n    l_data = data.loc[data[\"content_type_id\"] == 1, :]\n    l_data = l_data.merge(lectures, on=\"content_id\", how=\"left\")\n    del data\n    gc.collect()\n\n    data = pd.concat([q_data, l_data])\n\n    del q_data\n    del l_data\n    gc.collect()\n\n    return data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = train_load(debug=DEBUG)\ntrain","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = features_engineering(train)\ntrain","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = concat(train)\ntrain","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if not DEBUG:\n    train.to_pickle(\"train_v1.pkl\")","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}