{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"source:\nhttps://github.com/fielddaylab/jo_wilder/tree/master/src/scenes > data.js","metadata":{}},{"cell_type":"code","source":"import pandas as pd","metadata":{"execution":{"iopub.status.busy":"2023-03-21T07:51:44.029045Z","iopub.execute_input":"2023-03-21T07:51:44.029593Z","iopub.status.idle":"2023-03-21T07:51:44.059714Z","shell.execute_reply.started":"2023-03-21T07:51:44.029558Z","shell.execute_reply":"2023-03-21T07:51:44.058716Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# https://www.kaggle.com/competitions/predict-student-performance-from-game-play/discussion/393571\nemoji_dict = {\n    '\\\\u00f0\\\\u0178\\\\u02dc\\\\u00b4': \"😴\",\n    '\\\\u00f0\\\\u0178\\\\u00a6\\\\u2014': \"🦗\",\n    '\\\\u00f0\\\\u0178\\\\u02dc\\\\u0160': \"😊\",\n    '\\\\u00f0\\\\u0178\\\\u02dc\\\\u00ad': \"😭\",\n    '\\\\u00e2\\\\u009d\\\\u00a4\\\\u00ef\\\\u00b8\\\\u008f': \"❤️\",\n    '\\\\u00f0\\\\u0178\\\\u02dc\\\\u0090': \"😐\",\n    '\\\\u00f0\\\\u0178\\\\u02dc\\\\u009d': \"😝\",\n    '\\\\u00f0\\\\u0178\\\\u00a7\\\\u02dc': \"🧘\",\n    '\\\\u00f0\\\\u0178\\\\u008d\\\\u00a9': \"🍩\",\n    '\\\\u00f0\\\\u0178\\\\u2122\\\\u201e': \"🙄\",\n    'I used to have a magnifying glass around here\\\\u00e2\\\\u20ac\\\\u00a6': \"I used to have a magnifying glass around here…\",\n}\nemoji_dict2enc = {v:k for k, v in emoji_dict.items()}","metadata":{"execution":{"iopub.status.busy":"2023-03-21T07:51:44.061847Z","iopub.execute_input":"2023-03-21T07:51:44.062165Z","iopub.status.idle":"2023-03-21T07:51:44.069513Z","shell.execute_reply.started":"2023-03-21T07:51:44.062126Z","shell.execute_reply":"2023-03-21T07:51:44.068440Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"raw_text={}\ntmp1=[]\nwith open('/kaggle/input/src-scenes-data-js/data.txt') as f:\n  for line in f:\n    if str(line).split('=')[0].endswith('raw_text ') and str(line).split('=')[1].startswith(' \"'):\n      tmp1.append(line.split('=')[1][2:-3])\ntmp1_ = list(set(tmp1))\ntmp1_.sort(key=tmp1.index)\n\ntmp1_ = [emoji_dict2enc.get(t, t)  for t in tmp1_]\n\nfor i,t in enumerate(tmp1_):\n    raw_text[i] = t\n\n\nraw_atext={}\ntmp2=[]\nwith open('/kaggle/input/src-scenes-data-js/data.txt') as f:\n  for line in f:\n    if str(line).split('=')[0].endswith('raw_atext ') and str(line).split('=')[1].startswith(' \"'):\n      tmp2.append(line.split('=')[1][2:-3])\ntmp2_ = list(set(tmp2))\ntmp2_.sort(key=tmp2.index)\n\ntmp2_ = [emoji_dict2enc.get(t, t)  for t in tmp2_]\n\nfor i,t in enumerate(tmp2_):\n    raw_atext[i] = t\n\n\ntext_all={}\nfor i,t in enumerate(tmp1_+tmp2_):\n    text_all[i] = t","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-03-21T07:51:44.071087Z","iopub.execute_input":"2023-03-21T07:51:44.072190Z","iopub.status.idle":"2023-03-21T07:51:44.164669Z","shell.execute_reply.started":"2023-03-21T07:51:44.072151Z","shell.execute_reply":"2023-03-21T07:51:44.163804Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del f","metadata":{"execution":{"iopub.status.busy":"2023-03-21T07:51:44.167634Z","iopub.execute_input":"2023-03-21T07:51:44.168959Z","iopub.status.idle":"2023-03-21T07:51:44.174408Z","shell.execute_reply.started":"2023-03-21T07:51:44.168919Z","shell.execute_reply":"2023-03-21T07:51:44.173410Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"raw_text","metadata":{"execution":{"iopub.status.busy":"2023-03-21T07:51:44.176029Z","iopub.execute_input":"2023-03-21T07:51:44.176758Z","iopub.status.idle":"2023-03-21T07:51:44.192460Z","shell.execute_reply.started":"2023-03-21T07:51:44.176667Z","shell.execute_reply":"2023-03-21T07:51:44.191467Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"raw_atext","metadata":{"execution":{"iopub.status.busy":"2023-03-21T07:51:44.193943Z","iopub.execute_input":"2023-03-21T07:51:44.194811Z","iopub.status.idle":"2023-03-21T07:51:44.214444Z","shell.execute_reply.started":"2023-03-21T07:51:44.194755Z","shell.execute_reply":"2023-03-21T07:51:44.213263Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"text_all","metadata":{"execution":{"iopub.status.busy":"2023-03-21T07:51:44.218044Z","iopub.execute_input":"2023-03-21T07:51:44.218305Z","iopub.status.idle":"2023-03-21T07:51:44.244682Z","shell.execute_reply.started":"2023-03-21T07:51:44.218279Z","shell.execute_reply":"2023-03-21T07:51:44.243822Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/predict-student-performance-from-game-play/train.csv')\ntrain_text_ = list(train.text)\ntrain_text_l = list(set(train_text_))\ntrain_text_l.sort(key=train_text_.index)\n\ntrain_text_d={}\nfor i,t in enumerate(train_text_l):\n    train_text_d[i] = t","metadata":{"execution":{"iopub.status.busy":"2023-03-21T07:51:44.245840Z","iopub.execute_input":"2023-03-21T07:51:44.246173Z","iopub.status.idle":"2023-03-21T07:53:52.444786Z","shell.execute_reply.started":"2023-03-21T07:51:44.246135Z","shell.execute_reply":"2023-03-21T07:53:52.436656Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_text_d","metadata":{"execution":{"iopub.status.busy":"2023-03-21T07:53:52.454441Z","iopub.execute_input":"2023-03-21T07:53:52.458059Z","iopub.status.idle":"2023-03-21T07:53:52.485420Z","shell.execute_reply.started":"2023-03-21T07:53:52.458013Z","shell.execute_reply":"2023-03-21T07:53:52.484133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f'not included in the training data: {len(list(set(tmp1_+tmp2_) - set(train_text_l)))}')\nlist(set(tmp1_+tmp2_) - set(train_text_l))","metadata":{"execution":{"iopub.status.busy":"2023-03-21T07:53:52.489429Z","iopub.execute_input":"2023-03-21T07:53:52.489790Z","iopub.status.idle":"2023-03-21T07:53:52.512185Z","shell.execute_reply.started":"2023-03-21T07:53:52.489748Z","shell.execute_reply":"2023-03-21T07:53:52.510791Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Text not included in the train data may appear in the hidden test data.  \nThere may be other sources of games that can be useful.  \nI will keep looking :)  ","metadata":{}}]}