{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\ntrain_df=pd.read_csv('../input/feedback-prize-effectiveness/train.csv')\ntest_df=pd.read_csv('../input/feedback-prize-effectiveness/test.csv')\nsub_df=pd.read_csv('../input/feedback-prize-effectiveness/sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2022-07-15T19:32:29.034422Z","iopub.execute_input":"2022-07-15T19:32:29.035077Z","iopub.status.idle":"2022-07-15T19:32:29.383428Z","shell.execute_reply.started":"2022-07-15T19:32:29.035015Z","shell.execute_reply":"2022-07-15T19:32:29.382305Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head(10)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:18:28.915992Z","iopub.execute_input":"2022-07-15T18:18:28.916382Z","iopub.status.idle":"2022-07-15T18:18:28.944724Z","shell.execute_reply.started":"2022-07-15T18:18:28.916334Z","shell.execute_reply":"2022-07-15T18:18:28.943686Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['discourse_text'][2]","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:18:28.945981Z","iopub.execute_input":"2022-07-15T18:18:28.946867Z","iopub.status.idle":"2022-07-15T18:18:28.957754Z","shell.execute_reply.started":"2022-07-15T18:18:28.946821Z","shell.execute_reply":"2022-07-15T18:18:28.957097Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.shape","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:18:28.959444Z","iopub.execute_input":"2022-07-15T18:18:28.959753Z","iopub.status.idle":"2022-07-15T18:18:28.97413Z","shell.execute_reply.started":"2022-07-15T18:18:28.95972Z","shell.execute_reply":"2022-07-15T18:18:28.973001Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:18:28.977796Z","iopub.execute_input":"2022-07-15T18:18:28.978126Z","iopub.status.idle":"2022-07-15T18:18:28.995197Z","shell.execute_reply.started":"2022-07-15T18:18:28.978066Z","shell.execute_reply":"2022-07-15T18:18:28.994351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.shape","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:18:28.99689Z","iopub.execute_input":"2022-07-15T18:18:28.997283Z","iopub.status.idle":"2022-07-15T18:18:29.008882Z","shell.execute_reply.started":"2022-07-15T18:18:28.997238Z","shell.execute_reply":"2022-07-15T18:18:29.008208Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:18:29.011682Z","iopub.execute_input":"2022-07-15T18:18:29.011961Z","iopub.status.idle":"2022-07-15T18:18:29.03033Z","shell.execute_reply.started":"2022-07-15T18:18:29.011932Z","shell.execute_reply":"2022-07-15T18:18:29.029392Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df.shape","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:18:29.031801Z","iopub.execute_input":"2022-07-15T18:18:29.032056Z","iopub.status.idle":"2022-07-15T18:18:29.046788Z","shell.execute_reply.started":"2022-07-15T18:18:29.032026Z","shell.execute_reply":"2022-07-15T18:18:29.045923Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df['discourse_text'][0]","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:18:29.048267Z","iopub.execute_input":"2022-07-15T18:18:29.049051Z","iopub.status.idle":"2022-07-15T18:18:29.060718Z","shell.execute_reply.started":"2022-07-15T18:18:29.049014Z","shell.execute_reply":"2022-07-15T18:18:29.059848Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.tail()","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:18:29.061991Z","iopub.execute_input":"2022-07-15T18:18:29.062921Z","iopub.status.idle":"2022-07-15T18:18:29.084377Z","shell.execute_reply.started":"2022-07-15T18:18:29.062877Z","shell.execute_reply":"2022-07-15T18:18:29.083408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df=train_df.drop(columns=['discourse_id','essay_id','discourse_type'],axis=1)\ntrain_df.head(1)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:18:29.085916Z","iopub.execute_input":"2022-07-15T18:18:29.086926Z","iopub.status.idle":"2022-07-15T18:18:29.108962Z","shell.execute_reply.started":"2022-07-15T18:18:29.086874Z","shell.execute_reply":"2022-07-15T18:18:29.107724Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['discourse_effectiveness'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:18:29.110604Z","iopub.execute_input":"2022-07-15T18:18:29.111626Z","iopub.status.idle":"2022-07-15T18:18:29.13054Z","shell.execute_reply.started":"2022-07-15T18:18:29.11158Z","shell.execute_reply":"2022-07-15T18:18:29.129809Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['discourse_effectiveness']=train_df['discourse_effectiveness'].map({'Ineffective':0,'Adequate':1,'Effective':2})\ntrain_df['discourse_effectiveness'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:18:29.131907Z","iopub.execute_input":"2022-07-15T18:18:29.133092Z","iopub.status.idle":"2022-07-15T18:18:29.16141Z","shell.execute_reply.started":"2022-07-15T18:18:29.133032Z","shell.execute_reply":"2022-07-15T18:18:29.160423Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import warnings\nwarnings.filterwarnings('ignore')","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:18:29.167326Z","iopub.execute_input":"2022-07-15T18:18:29.167845Z","iopub.status.idle":"2022-07-15T18:18:29.171998Z","shell.execute_reply.started":"2022-07-15T18:18:29.167798Z","shell.execute_reply":"2022-07-15T18:18:29.171369Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import transformers\nfrom transformers import DistilBertTokenizer, TFDistilBertModel\ntokenizer = DistilBertTokenizer.from_pretrained('distilbert-base-uncased')\nbert= TFDistilBertModel.from_pretrained(\"distilbert-base-uncased\")","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:18:29.17307Z","iopub.execute_input":"2022-07-15T18:18:29.173433Z","iopub.status.idle":"2022-07-15T18:18:51.6891Z","shell.execute_reply.started":"2022-07-15T18:18:29.173405Z","shell.execute_reply":"2022-07-15T18:18:51.688121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('max length of word',max([len(x.split()) for x in train_df['discourse_text']]))","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:18:51.690654Z","iopub.execute_input":"2022-07-15T18:18:51.690969Z","iopub.status.idle":"2022-07-15T18:18:51.825966Z","shell.execute_reply.started":"2022-07-15T18:18:51.69093Z","shell.execute_reply":"2022-07-15T18:18:51.824947Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['discourse_text'].apply(lambda x: len(x.split())).mean()","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:18:51.828887Z","iopub.execute_input":"2022-07-15T18:18:51.829144Z","iopub.status.idle":"2022-07-15T18:18:51.98245Z","shell.execute_reply.started":"2022-07-15T18:18:51.829117Z","shell.execute_reply":"2022-07-15T18:18:51.981589Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"max_length=50","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:18:51.983738Z","iopub.execute_input":"2022-07-15T18:18:51.983995Z","iopub.status.idle":"2022-07-15T18:18:51.989289Z","shell.execute_reply.started":"2022-07-15T18:18:51.983967Z","shell.execute_reply":"2022-07-15T18:18:51.988258Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bert","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:18:51.990652Z","iopub.execute_input":"2022-07-15T18:18:51.991113Z","iopub.status.idle":"2022-07-15T18:18:52.00275Z","shell.execute_reply.started":"2022-07-15T18:18:51.991064Z","shell.execute_reply":"2022-07-15T18:18:52.001741Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tokenizer","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:18:52.004274Z","iopub.execute_input":"2022-07-15T18:18:52.004583Z","iopub.status.idle":"2022-07-15T18:18:52.01958Z","shell.execute_reply.started":"2022-07-15T18:18:52.004552Z","shell.execute_reply":"2022-07-15T18:18:52.018592Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tokenizer('i love baby')","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:18:52.020906Z","iopub.execute_input":"2022-07-15T18:18:52.021687Z","iopub.status.idle":"2022-07-15T18:18:52.036684Z","shell.execute_reply.started":"2022-07-15T18:18:52.021647Z","shell.execute_reply":"2022-07-15T18:18:52.036067Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_train=tokenizer(\n      \n    text=train_df['discourse_text'].tolist(),\n    add_special_tokens=True,\n    max_length=max_length,\n    truncation=True,\n    padding=True,\n    return_tensors='tf',\n    return_attention_mask=False,\n    verbose=True\n    \n    )","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:18:52.037885Z","iopub.execute_input":"2022-07-15T18:18:52.038674Z","iopub.status.idle":"2022-07-15T18:19:51.824904Z","shell.execute_reply.started":"2022-07-15T18:18:52.038644Z","shell.execute_reply":"2022-07-15T18:19:51.823957Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_train","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:19:51.82619Z","iopub.execute_input":"2022-07-15T18:19:51.826432Z","iopub.status.idle":"2022-07-15T18:19:51.834064Z","shell.execute_reply.started":"2022-07-15T18:19:51.826404Z","shell.execute_reply":"2022-07-15T18:19:51.833337Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow.keras import Model,layers,optimizers,callbacks,losses,metrics,utils","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:19:51.83522Z","iopub.execute_input":"2022-07-15T18:19:51.835953Z","iopub.status.idle":"2022-07-15T18:19:51.852054Z","shell.execute_reply.started":"2022-07-15T18:19:51.835917Z","shell.execute_reply":"2022-07-15T18:19:51.850996Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"input_ids=layers.Input(shape=(max_length,),dtype=tf.int32,name='input_ids')\nembedding=bert(input_ids)[0]","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:19:51.853883Z","iopub.execute_input":"2022-07-15T18:19:51.854225Z","iopub.status.idle":"2022-07-15T18:19:57.875856Z","shell.execute_reply.started":"2022-07-15T18:19:51.854183Z","shell.execute_reply":"2022-07-15T18:19:57.875061Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"embedding[0]","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:19:57.877569Z","iopub.execute_input":"2022-07-15T18:19:57.877864Z","iopub.status.idle":"2022-07-15T18:19:57.89046Z","shell.execute_reply.started":"2022-07-15T18:19:57.877827Z","shell.execute_reply":"2022-07-15T18:19:57.889599Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"out=layers.GlobalMaxPool1D()(embedding)\nout=layers.Dropout(0.1)(out)\nout=layers.Dense(399,activation='relu')(out)\n\nout=layers.Dropout(0.15)(out)\nout=layers.Dense(67,activation='relu')(out)\nout=layers.Dense(17,activation='relu')(out)\ny=layers.Dense(3,activation='softmax')(out)\n\nnn=Model(inputs=[input_ids],outputs=y)\nnn.layers[1].trainable=True\noptimizer=optimizers.Adam(learning_rate = 2e-5)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:19:57.891765Z","iopub.execute_input":"2022-07-15T18:19:57.892279Z","iopub.status.idle":"2022-07-15T18:19:57.94994Z","shell.execute_reply.started":"2022-07-15T18:19:57.892242Z","shell.execute_reply":"2022-07-15T18:19:57.949287Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nn.summary()","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:19:57.95138Z","iopub.execute_input":"2022-07-15T18:19:57.951839Z","iopub.status.idle":"2022-07-15T18:19:57.971103Z","shell.execute_reply.started":"2022-07-15T18:19:57.9518Z","shell.execute_reply":"2022-07-15T18:19:57.97022Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train=train_df['discourse_effectiveness']\ny_train","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:19:57.972647Z","iopub.execute_input":"2022-07-15T18:19:57.973174Z","iopub.status.idle":"2022-07-15T18:19:57.98161Z","shell.execute_reply.started":"2022-07-15T18:19:57.973131Z","shell.execute_reply":"2022-07-15T18:19:57.980647Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:19:57.983558Z","iopub.execute_input":"2022-07-15T18:19:57.984066Z","iopub.status.idle":"2022-07-15T18:19:57.995478Z","shell.execute_reply.started":"2022-07-15T18:19:57.984015Z","shell.execute_reply":"2022-07-15T18:19:57.994552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"utils.plot_model(nn,show_shapes=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:19:57.996921Z","iopub.execute_input":"2022-07-15T18:19:57.99755Z","iopub.status.idle":"2022-07-15T18:19:59.467726Z","shell.execute_reply.started":"2022-07-15T18:19:57.997506Z","shell.execute_reply":"2022-07-15T18:19:59.466757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nn.compile(optimizer=optimizer,loss='sparse_categorical_crossentropy',metrics=['accuracy'])\nnn.fit(\n\n    x={'input_ids':x_train['input_ids']},\n    y=y_train,\n    validation_split=0.20,\n    epochs=1,\n    batch_size=45\n\n)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T18:20:33.873637Z","iopub.execute_input":"2022-07-15T18:20:33.873941Z","iopub.status.idle":"2022-07-15T19:29:00.910284Z","shell.execute_reply.started":"2022-07-15T18:20:33.873913Z","shell.execute_reply":"2022-07-15T19:29:00.909121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Working with test data","metadata":{}},{"cell_type":"code","source":"test_df=test_df.drop(columns=['discourse_id','essay_id','discourse_type'],axis=1)\ntest_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-15T19:33:30.194299Z","iopub.execute_input":"2022-07-15T19:33:30.195443Z","iopub.status.idle":"2022-07-15T19:33:30.212879Z","shell.execute_reply.started":"2022-07-15T19:33:30.195388Z","shell.execute_reply":"2022-07-15T19:33:30.212144Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_test=tokenizer(\n\n    text=test_df['discourse_text'].tolist(),\n    add_special_tokens=True,\n    max_length=max_length,\n    truncation=True,\n    padding=True,\n    return_tensors='tf',\n    return_attention_mask=False,\n    verbose=True\n\n\n)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T19:33:42.389272Z","iopub.execute_input":"2022-07-15T19:33:42.39049Z","iopub.status.idle":"2022-07-15T19:33:42.416939Z","shell.execute_reply.started":"2022-07-15T19:33:42.390398Z","shell.execute_reply":"2022-07-15T19:33:42.416078Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_test","metadata":{"execution":{"iopub.status.busy":"2022-07-15T19:33:45.544557Z","iopub.execute_input":"2022-07-15T19:33:45.544944Z","iopub.status.idle":"2022-07-15T19:33:45.555344Z","shell.execute_reply.started":"2022-07-15T19:33:45.5449Z","shell.execute_reply":"2022-07-15T19:33:45.554245Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predicted=nn.predict({'input_ids':x_test['input_ids']})","metadata":{"execution":{"iopub.status.busy":"2022-07-15T19:33:53.584998Z","iopub.execute_input":"2022-07-15T19:33:53.585411Z","iopub.status.idle":"2022-07-15T19:33:55.649782Z","shell.execute_reply.started":"2022-07-15T19:33:53.585372Z","shell.execute_reply":"2022-07-15T19:33:55.648923Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predicted","metadata":{"execution":{"iopub.status.busy":"2022-07-15T19:33:58.034675Z","iopub.execute_input":"2022-07-15T19:33:58.035609Z","iopub.status.idle":"2022-07-15T19:33:58.042544Z","shell.execute_reply.started":"2022-07-15T19:33:58.035565Z","shell.execute_reply":"2022-07-15T19:33:58.041571Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df=sub_df.drop(columns=['Adequate','Ineffective','Effective'],axis=1)\nsub_df","metadata":{"execution":{"iopub.status.busy":"2022-07-15T19:34:06.825587Z","iopub.execute_input":"2022-07-15T19:34:06.826111Z","iopub.status.idle":"2022-07-15T19:34:06.839611Z","shell.execute_reply.started":"2022-07-15T19:34:06.826037Z","shell.execute_reply":"2022-07-15T19:34:06.838653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df['Ineffective'] = predicted[:,0]\nsub_df['Adequate'] = predicted[:,1]\nsub_df['Effective'] = predicted[:,2]\nsub_df","metadata":{"execution":{"iopub.status.busy":"2022-07-15T19:34:15.695774Z","iopub.execute_input":"2022-07-15T19:34:15.696819Z","iopub.status.idle":"2022-07-15T19:34:15.716451Z","shell.execute_reply.started":"2022-07-15T19:34:15.696772Z","shell.execute_reply":"2022-07-15T19:34:15.715481Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df.to_csv('feedback_prize_1.csv',index=None)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T19:34:39.85555Z","iopub.execute_input":"2022-07-15T19:34:39.856237Z","iopub.status.idle":"2022-07-15T19:34:39.870512Z","shell.execute_reply.started":"2022-07-15T19:34:39.856194Z","shell.execute_reply":"2022-07-15T19:34:39.869262Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}