{"cells":[{"metadata":{},"cell_type":"markdown","source":"This is a folk of [This notebook](https://www.kaggle.com/its7171/cv-strategy). I randomsampled 1/10 of user to reduce dataset size."},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd\nimport random\nimport gc\nfrom sklearn.model_selection import GroupShuffleSplit\nrandom.seed(1)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/train.csv',\n                   dtype={'row_id': 'int64',\n                          'timestamp': 'int64',\n                          'user_id': 'int32',\n                          'content_id': 'int16',\n                          'content_type_id': 'int8',\n                          'task_container_id': 'int16',\n                          'user_answer': 'int8',\n                          'answered_correctly':'int8',\n                          'prior_question_elapsed_time': 'float32',\n                          'prior_question_had_explanation': 'boolean'}\n                   )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"reduced_train_size=0.1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_idx, test_idx =next(GroupShuffleSplit(n_splits=1, train_size=reduced_train_size, random_state=42).split(train,groups=train.user_id))\ntrain=train.iloc[train_idx]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"valid_split1 = train.groupby('user_id').tail(5)\ntrain_split1 = train[~train.row_id.isin(valid_split1.row_id)]\nvalid_split1 = valid_split1[valid_split1.content_type_id == 0]\ntrain_split1 = train_split1[train_split1.content_type_id == 0]\nprint(f'{train_split1.answered_correctly.mean():.3f} {valid_split1.answered_correctly.mean():.3f}')","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"del valid_split1, train_split1\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Since training data and test data are split by time, the validation data should also be split by time.\nHowever, the given timestamp is the time that has elapsed since the user's first event, not the actual time.\nSo I set a random first access time for each user within a certain interval."},{"metadata":{"trusted":true},"cell_type":"code","source":"max_timestamp_u = train[['user_id','timestamp']].groupby(['user_id']).agg(['max']).reset_index()\nmax_timestamp_u.columns = ['user_id', 'max_time_stamp']\nMAX_TIME_STAMP = max_timestamp_u.max_time_stamp.max()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"`(MAX_TIME_STAMP for all users) - (max_time_stamp for each user)` is used for this interval."},{"metadata":{"trusted":true},"cell_type":"code","source":"def rand_time(max_time_stamp):\n    interval = MAX_TIME_STAMP - max_time_stamp\n    rand_time_stamp = random.randint(0,interval)\n    return rand_time_stamp\n\nmax_timestamp_u['rand_time_stamp'] = max_timestamp_u.max_time_stamp.apply(rand_time)\ntrain = train.merge(max_timestamp_u, on='user_id', how='left')\ntrain['viretual_time_stamp'] = train.timestamp + train['rand_time_stamp']","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"del train['max_time_stamp']\ndel train['rand_time_stamp']\ndel max_timestamp_u\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"kaggle_env = False\nif kaggle_env:\n    # Full dataframe can not be sorted on kaggle kernel due to lack of memory.\n    train = train[:10000000]\ntrain = train.sort_values(['viretual_time_stamp', 'row_id']).reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if kaggle_env:\n    val_size = 250000\nelse:\n    val_size = 2500000\n\nfor cv in range(5):\n    valid = train[-val_size:]\n    train = train[:-val_size]\n    # check new users and new contents\n    new_users = len(valid[~valid.user_id.isin(train.user_id)].user_id.unique())\n    valid_question = valid[valid.content_type_id == 0]\n    train_question = train[train.content_type_id == 0]\n    new_contents = len(valid_question[~valid_question.content_id.isin(train_question.content_id)].content_id.unique())    \n    print(f'cv{cv} {train_question.answered_correctly.mean():.3f} {valid_question.answered_correctly.mean():.3f} {new_users} {new_contents}')\n    valid.to_pickle(f'cv{cv+1}_valid.pickle')\n    train.to_pickle(f'cv{cv+1}_train.pickle')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}