{"cells":[{"metadata":{},"cell_type":"markdown","source":"**Have a (random) subset of users' data to enable quick model experiments, split into train, val w/o much hassle**\n\nprinciples:\n* keep each user's time course healthy, i.e. do not downsample for the sake of smaller data\n* first part of time course should go into train - remaining part into val\n* across users, have a fixed portion go into train and val, respectively"},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd\nimport random\n\nrandom.seed(33)\n\nfraction_users = 0.001\nsplit = 0.8","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"%%time \n\ndata = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/train.csv',\n                   dtype={'row_id': 'int64',\n                          'timestamp': 'int64',\n                          'user_id': 'int32',\n                          'content_id': 'int16',\n                          'content_type_id': 'int8',\n                          'task_container_id': 'int16',\n                          'user_answer': 'int8',\n                          'answered_correctly':'int8',\n                          'prior_question_elapsed_time': 'float32',\n                          'prior_question_had_explanation': 'boolean'\n                         })","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"take a subset of users"},{"metadata":{"trusted":true},"cell_type":"code","source":"users = data.user_id.unique()\nno_users_sample = int(round(len(users)*fraction_users,0))\n\nprint(f'no. of unique users: {len(users)}')\nprint(f'no. of users in sample: {no_users_sample}')\n\nusers = random.sample(list(users), no_users_sample)\nmask = data.user_id.isin(users)\ndata = data[mask]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"get xth percentile of timestamp, to prepare split"},{"metadata":{"trusted":true},"cell_type":"code","source":"my_planet = data[['timestamp', 'user_id']].groupby('user_id').quantile(split).reset_index()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"mark rows with train, val flag"},{"metadata":{"trusted":true},"cell_type":"code","source":"my_planet.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"my_planet.columns = ['user_id', 'quartile']\ndata = pd.merge(data, my_planet)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"mask = data['timestamp']>data['quartile']\ndata['mask'] = mask","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data['split'] = 'none'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ndata['split'].where(data['mask'] is True) = 'val'\n#data['split'][data['mask'] is False] = 'train'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\n# https://stackoverflow.com/questions/20625582/how-to-deal-with-settingwithcopywarning-in-pandas\npd.options.mode.chained_assignment = None\n\nfrom tqdm import tqdm\n\ndata['split'] = 'na'\n\nfor i in tqdm(range(len(data))):\n    current_user = data.user_id.iloc[i]\n    percentile = user_percentile[current_user]\n    if data.timestamp.iloc[i] <= percentile:\n        data.at[i, split] = 'train'\n    else:\n        data.at[i, split] = 'val'\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"split"},{"metadata":{"trusted":true},"cell_type":"code","source":"train = data[data.split == 'train']\nval = data[data.split == 'val']\n\ntrain = train.drop('split', axis=1)\nval = val.drop('split', axis=1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"check properties for train and val"},{"metadata":{"trusted":true},"cell_type":"code","source":"print(f'no. of users in train: {len(train.user_id.unique())}')\nprint(f'no. of users in val: {len(val.user_id.unique())}')\nprint('-> should be the same')\nprint(f'no. of records in train: {len(train)}')\nprint(f'no. of records in val: {len(val)}')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"oops - what's this?"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_ids = set(train['user_id'].unique())\nval_ids = set(val['user_id'].unique()) \ncnt=0\n\nmissing_ids = train_ids - val_ids\n\nfor id in missing_ids:\n    print(data[['timestamp', 'user_id']][data['user_id']==id])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"interesting that there are a number of entries with equal timestamp for this user. But for now let us remove them just from train, val"},{"metadata":{"trusted":true},"cell_type":"code","source":"train = train[~train.user_id.isin(missing_ids)]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"check again"},{"metadata":{"trusted":true},"cell_type":"code","source":"print(f'no. of users in train: {len(train.user_id.unique())}')\nprint(f'no. of users in val: {len(val.user_id.unique())}')\nprint('-> should be the same')\nprint(f'no. of records in train: {len(train)}')\nprint(f'no. of records in val: {len(val)}')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"look at mean, standard deviation"},{"metadata":{"trusted":true},"cell_type":"code","source":"valid_question = val[val.content_type_id == 0]\ntrain_question = train[train.content_type_id == 0]\n\nprint(f'accuracy of answering in train: {round(train_question.answered_correctly.mean(), 2)} (stdev={round(train_question.answered_correctly.std(), 2)})')\nprint(f'accuracy of answering in val: {round(valid_question.answered_correctly.mean(), 2)} (stdev={round(valid_question.answered_correctly.std(), 2)})')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"write train, val to csv files"},{"metadata":{"trusted":true},"cell_type":"code","source":"train.to_csv(f'/kaggle/working/train_{int(fraction_users*100)}percent.csv')\nval.to_csv(f'/kaggle/working/val_{int(fraction_users*100)}percent.csv')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"\nthank you, tito, https://www.kaggle.com/its7171/cv-strategy for valuable insights"}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}