{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"##### Train Notebook\n\nhttps://www.kaggle.com/code/anantgupt/pytorch-feedback-eda-train","metadata":{}},{"cell_type":"code","source":"import os\nimport joblib\nimport pandas as pd\nimport numpy as np\n\nimport torch\nimport torch.nn as nn\nfrom torch.nn.functional import softmax\nfrom transformers import AutoConfig, AutoModel, AutoTokenizer\n                        ","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-30T16:03:01.907531Z","iopub.execute_input":"2022-07-30T16:03:01.908407Z","iopub.status.idle":"2022-07-30T16:03:04.500083Z","shell.execute_reply.started":"2022-07-30T16:03:01.908313Z","shell.execute_reply":"2022-07-30T16:03:04.498973Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SAVE_MODEL_PATH = \"../input/trained-competition-models/Feedback Prize - Predicting Effective Arguments/best_feedback_8.bin\"\nENCODER_PATH = \"../input/trained-competition-models/Feedback Prize - Predicting Effective Arguments/le.pkl\"\nTEST_CSV_PATH = \"../input/feedback-prize-effectiveness/test.csv\"\nTEST_DATA_PATH = \"../input/feedback-prize-effectiveness/test\"\n","metadata":{"execution":{"iopub.status.busy":"2022-07-30T16:03:04.505486Z","iopub.execute_input":"2022-07-30T16:03:04.507983Z","iopub.status.idle":"2022-07-30T16:03:04.514618Z","shell.execute_reply.started":"2022-07-30T16:03:04.507952Z","shell.execute_reply":"2022-07-30T16:03:04.513058Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class MeanPooling(nn.Module):\n    def __init__(self):\n        super(MeanPooling, self).__init__()\n        \n    def forward(self, last_hidden_state, attention_mask):\n        input_mask_expanded = attention_mask.unsqueeze(-1).expand(last_hidden_state.size()).float()\n        sum_embeddings = torch.sum(last_hidden_state * input_mask_expanded, 1)\n        sum_mask = input_mask_expanded.sum(1)\n        sum_mask = torch.clamp(sum_mask, min=1e-9)\n        mean_embeddings = sum_embeddings / sum_mask\n        return mean_embeddings\n    \n\nclass FeedBackModel(nn.Module):\n    def __init__(self, model_name):\n        super(FeedBackModel, self).__init__()\n        self.drop = nn.Dropout(p=0.05)\n\n        self.model = AutoModel.from_pretrained(model_name)\n        self.config = AutoConfig.from_pretrained(model_name)\n        self.mpool = MeanPooling()\n        \n        self.fc = nn.Sequential(\n            nn.Linear(self.config.hidden_size, CONFIG['num_classes']),\n        )\n#         self.fc2 = nn.Sequential(\n#             nn.Linear(1024, CONFIG['num_classes']),\n#         )\n        \n    def forward(self, ids, mask):        \n        out = self.model(input_ids=ids,attention_mask=mask,\n                         output_hidden_states=False)\n        out = self.mpool(out.last_hidden_state, mask)\n        out = self.drop(out)\n        out = self.fc(out)\n#         out = self.drop(out)\n#         out = self.fc2\n        return out","metadata":{"execution":{"iopub.status.busy":"2022-07-30T16:03:04.516442Z","iopub.execute_input":"2022-07-30T16:03:04.517143Z","iopub.status.idle":"2022-07-30T16:03:04.530594Z","shell.execute_reply.started":"2022-07-30T16:03:04.517105Z","shell.execute_reply":"2022-07-30T16:03:04.529678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CONFIG = {\n    \"model_name\": \"../input/debertav3base\",\n    \"num_classes\": 3,\n    \"max_length\": 512,\n    \"device\": torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\"),\n  }\n\nCONFIG[\"tokenizer\"] = AutoTokenizer.from_pretrained(CONFIG['model_name'])","metadata":{"execution":{"iopub.status.busy":"2022-07-30T16:03:04.535974Z","iopub.execute_input":"2022-07-30T16:03:04.538302Z","iopub.status.idle":"2022-07-30T16:03:05.793263Z","shell.execute_reply.started":"2022-07-30T16:03:04.538254Z","shell.execute_reply":"2022-07-30T16:03:05.791620Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"encoder = joblib.load(ENCODER_PATH)\nencoder.classes_","metadata":{"execution":{"iopub.status.busy":"2022-07-30T16:06:14.882666Z","iopub.execute_input":"2022-07-30T16:06:14.883304Z","iopub.status.idle":"2022-07-30T16:06:14.892193Z","shell.execute_reply.started":"2022-07-30T16:06:14.883261Z","shell.execute_reply":"2022-07-30T16:06:14.891017Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = FeedBackModel(CONFIG[\"model_name\"])\nmodel.load_state_dict(torch.load(SAVE_MODEL_PATH))\nmodel.eval();","metadata":{"execution":{"iopub.status.busy":"2022-07-30T16:06:26.114609Z","iopub.execute_input":"2022-07-30T16:06:26.115036Z","iopub.status.idle":"2022-07-30T16:06:41.066662Z","shell.execute_reply.started":"2022-07-30T16:06:26.114997Z","shell.execute_reply":"2022-07-30T16:06:41.065647Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = pd.read_csv(TEST_CSV_PATH)\ntest_df","metadata":{"execution":{"iopub.status.busy":"2022-07-30T16:06:41.068504Z","iopub.execute_input":"2022-07-30T16:06:41.068948Z","iopub.status.idle":"2022-07-30T16:06:41.097338Z","shell.execute_reply.started":"2022-07-30T16:06:41.068911Z","shell.execute_reply":"2022-07-30T16:06:41.096476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def infer(essay_id, discourse_type, discourse_text, data_path=TEST_DATA_PATH, max_len=CONFIG[\"max_length\"]):\n    \n    essay_path = os.path.join(data_path, f\"{essay_id}.txt\")\n    essay = open(essay_path, 'r').read()\n    text = discourse_type + \" \" + tokenizer.sep_token + \\\n            discourse_text + \" \" + tokenizer.sep_token + \" \" + essay\n    inputs = tokenizer.encode_plus(\n                text,\n                truncation=True,\n                add_special_tokens=True,\n                max_length=max_len\n            )\n    ids = torch.tensor(inputs['input_ids'], dtype=torch.long).unsqueeze(0)\n    mask = torch.tensor(inputs['attention_mask'], dtype=torch.long).unsqueeze(0)\n    \n    output = model(ids, mask)\n    return softmax(output, dim=1).detach().numpy()[0]","metadata":{"execution":{"iopub.status.busy":"2022-07-30T16:06:44.588072Z","iopub.execute_input":"2022-07-30T16:06:44.588641Z","iopub.status.idle":"2022-07-30T16:06:44.597116Z","shell.execute_reply.started":"2022-07-30T16:06:44.588603Z","shell.execute_reply":"2022-07-30T16:06:44.596122Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = []\ntokenizer = CONFIG[\"tokenizer\"]\n\nfor i in range(len(test_df)):\n    essay_id = test_df.essay_id[i]\n    discourse_text = test_df.discourse_text[i]\n    discourse_type = test_df.discourse_type[i]\n    \n    preds.append(infer(essay_id, discourse_type, discourse_text))\n    \npreds = np.array(preds)\npreds","metadata":{"execution":{"iopub.status.busy":"2022-07-30T16:06:44.925425Z","iopub.execute_input":"2022-07-30T16:06:44.926352Z","iopub.status.idle":"2022-07-30T16:07:06.026928Z","shell.execute_reply.started":"2022-07-30T16:06:44.926303Z","shell.execute_reply":"2022-07-30T16:07:06.026025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = np.array(preds)\npreds","metadata":{"execution":{"iopub.status.busy":"2022-07-30T16:07:06.029075Z","iopub.execute_input":"2022-07-30T16:07:06.029439Z","iopub.status.idle":"2022-07-30T16:07:06.037072Z","shell.execute_reply.started":"2022-07-30T16:07:06.029401Z","shell.execute_reply":"2022-07-30T16:07:06.035890Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample = pd.read_csv(\"../input/feedback-prize-effectiveness/sample_submission.csv\")\nsample.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-30T16:07:06.038937Z","iopub.execute_input":"2022-07-30T16:07:06.039500Z","iopub.status.idle":"2022-07-30T16:07:06.063152Z","shell.execute_reply.started":"2022-07-30T16:07:06.039464Z","shell.execute_reply":"2022-07-30T16:07:06.062071Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample['Adequate'] = preds[:, 0]\nsample['Effective'] = preds[:, 1]\nsample['Ineffective'] = preds[:, 2]\n\nsample.to_csv('submission.csv', index=False)\nsample","metadata":{"execution":{"iopub.status.busy":"2022-07-30T16:07:06.066647Z","iopub.execute_input":"2022-07-30T16:07:06.066902Z","iopub.status.idle":"2022-07-30T16:07:06.085970Z","shell.execute_reply.started":"2022-07-30T16:07:06.066878Z","shell.execute_reply":"2022-07-30T16:07:06.085133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}