{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!python ../input/ai4code-reg-cls-loss-dataset/submission_code_reg_cls_code_loss/code/preprocess.py --data_dir /kaggle/input/AI4Code/","metadata":{"execution":{"iopub.status.busy":"2022-08-11T15:18:38.988855Z","iopub.execute_input":"2022-08-11T15:18:38.989562Z","iopub.status.idle":"2022-08-11T15:18:48.117810Z","shell.execute_reply.started":"2022-08-11T15:18:38.989475Z","shell.execute_reply":"2022-08-11T15:18:48.116626Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport os \nimport gc\n\ntest_df = pd.read_pickle(\"tmp/data/test.pkl\")\ntest_df['cell_id'] = test_df.index.get_level_values('id')+'#'+test_df['cell_id']\n#del test_df['source']\n#tmp=test_df.groupby(\"id\").cell_id.count().reset_index()\n#tmp = tmp[(tmp.cell_id>60)]\n\ntest_df['idx'] = test_df.index.get_level_values('id')\n# test_df1 = test_df[test_df['idx'].isin(tmp['id'])]\n#test_df2 = test_df[~test_df['idx'].isin(tmp['id'])]\n\n# del test_df1['idx']\n#del test_df2['idx']\n# del test_df['idx']\n\n# if not os.path.exists('tmp/data/test1'):\n#     os.makedirs('tmp/data/test1')\n#if not os.path.exists('tmp/data/test2'):\n#    os.makedirs('tmp/data/test2')\n# test_df1.to_pickle(\"tmp/data/test1/test.pkl\")\n#test_df2.to_pickle(\"tmp/data/test2/test.pkl\")\ntest_df.to_pickle(\"tmp/data/test.pkl\")\ndel test_df\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-08-11T15:18:48.120879Z","iopub.execute_input":"2022-08-11T15:18:48.121449Z","iopub.status.idle":"2022-08-11T15:18:48.246344Z","shell.execute_reply.started":"2022-08-11T15:18:48.121406Z","shell.execute_reply":"2022-08-11T15:18:48.245487Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!python ../input/ai4code-reg-cls-loss-dataset/submission_code_reg_cls_code_loss/code/extract_feature.py --model_path ../input/baseline/checkpoints","metadata":{"execution":{"iopub.status.busy":"2022-08-11T15:18:48.247826Z","iopub.execute_input":"2022-08-11T15:18:48.248189Z","iopub.status.idle":"2022-08-11T15:19:06.772003Z","shell.execute_reply.started":"2022-08-11T15:18:48.248153Z","shell.execute_reply":"2022-08-11T15:19:06.770991Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!python ../input/ai4code-reg-cls-loss-dataset/code/run.py --eval_batch_size 8 --md_number 512 --code_number 512 --data_dir ./tmp/data --feature_dir ./tmp/feature-epoch10 --result_path sub1.pkl --reloaded_model_path ../input/ai4code-reg-cls-loss-dataset/round_52_model_ext.bin --do_test","metadata":{"execution":{"iopub.status.busy":"2022-08-11T15:19:06.776186Z","iopub.execute_input":"2022-08-11T15:19:06.776505Z","iopub.status.idle":"2022-08-11T15:19:17.281530Z","shell.execute_reply.started":"2022-08-11T15:19:06.776475Z","shell.execute_reply":"2022-08-11T15:19:17.280524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!python ../input/ai4code-reg-cls-loss-dataset/code/run.py --eval_batch_size 8 --md_number 512 --code_number 512 --data_dir ./tmp/data --feature_dir ./tmp/feature-epoch10 --result_path sub2.pkl --reloaded_model_path ../input/baseline/round_67_model_ext_all.bin --do_test","metadata":{"execution":{"iopub.status.busy":"2022-08-11T15:19:17.282940Z","iopub.execute_input":"2022-08-11T15:19:17.283344Z","iopub.status.idle":"2022-08-11T15:19:27.528800Z","shell.execute_reply.started":"2022-08-11T15:19:17.283302Z","shell.execute_reply":"2022-08-11T15:19:27.527753Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!python ../input/ai4code-reg-cls-loss-dataset/code/run.py --eval_batch_size 8 --md_number 512 --code_number 512 --data_dir ./tmp/data --feature_dir ./tmp/feature-epoch10 --result_path sub3.pkl --reloaded_model_path ../input/baseline/round_70_model_ext_2.bin --do_test","metadata":{"execution":{"iopub.status.busy":"2022-08-11T15:19:27.530316Z","iopub.execute_input":"2022-08-11T15:19:27.530713Z","iopub.status.idle":"2022-08-11T15:19:38.520360Z","shell.execute_reply.started":"2022-08-11T15:19:27.530680Z","shell.execute_reply":"2022-08-11T15:19:38.519322Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !python ../input/ai4code-reg-cls-loss-dataset/cls_code/run.py --eval_batch_size 4 --md_number 512 --code_number 512 --data_dir ./tmp/data/test2 --feature_dir ./tmp/feature-epoch10 --result_path sub3.pkl --reloaded_model_path ../input/ai4code-reg-cls-loss-dataset/round_75_cls_model.bin --do_test","metadata":{"execution":{"iopub.status.busy":"2022-08-11T15:19:38.521898Z","iopub.execute_input":"2022-08-11T15:19:38.522567Z","iopub.status.idle":"2022-08-11T15:19:38.527453Z","shell.execute_reply.started":"2022-08-11T15:19:38.522526Z","shell.execute_reply":"2022-08-11T15:19:38.526285Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\ndf=pd.read_pickle(\"sub1.pkl\")\ndf2=pd.read_pickle(\"sub2.pkl\")\ndf3=pd.read_pickle(\"sub3.pkl\")\n#df3['idx']=df3.index.get_level_values('id')\n#df3['pred']=(df3['pred']+1)/2/df3['idx'].map(df3[df3.cell_type=='code'].groupby('id')['cell_id'].count()+1)\n\n#print(df3)\n#df=pd.concat([df1,df2])\n#df2=pd.read_pickle(\"sub2.pkl\")\ndf['pred']=df['pred']+df2['pred']+df3['pred']\n#cls_dict=df3.set_index('cell_id').pred.to_dict()\n#df['pred'][df['cell_id'].isin(df3['cell_id'])]=df['pred']+df['cell_id'].map(cls_dict)*0.5\n\ndf['cell_id']=df['cell_id'].apply(lambda x:x.split(\"#\")[1])\nsub_df = df.sort_values(\"pred\").groupby(\"id\")[\"cell_id\"].apply(lambda x: \" \".join(x)).reset_index()\nsub_df.rename(columns={\"cell_id\": \"cell_order\"}, inplace=True)\nprint(sub_df.head())\nsub_df.to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2022-08-11T15:19:38.528866Z","iopub.execute_input":"2022-08-11T15:19:38.529515Z","iopub.status.idle":"2022-08-11T15:19:38.559873Z","shell.execute_reply.started":"2022-08-11T15:19:38.529480Z","shell.execute_reply":"2022-08-11T15:19:38.559114Z"},"trusted":true},"execution_count":null,"outputs":[]}]}