{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Importing the libraries and data","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\ntrain_df=pd.read_csv('../input/feedback-prize-effectiveness/train.csv')\ntest_df=pd.read_csv('../input/feedback-prize-effectiveness/test.csv')\nsub_df=pd.read_csv('../input/feedback-prize-effectiveness/sample_submission.csv')","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2022-07-17T05:22:24.247795Z","iopub.execute_input":"2022-07-17T05:22:24.248388Z","iopub.status.idle":"2022-07-17T05:22:24.593099Z","shell.execute_reply.started":"2022-07-17T05:22:24.248221Z","shell.execute_reply":"2022-07-17T05:22:24.592125Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head(10)","metadata":{"execution":{"iopub.status.busy":"2022-07-17T05:22:24.595411Z","iopub.execute_input":"2022-07-17T05:22:24.595783Z","iopub.status.idle":"2022-07-17T05:22:24.617283Z","shell.execute_reply.started":"2022-07-17T05:22:24.595748Z","shell.execute_reply":"2022-07-17T05:22:24.616268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['discourse_text'][2]","metadata":{"execution":{"iopub.status.busy":"2022-07-17T05:22:24.618763Z","iopub.execute_input":"2022-07-17T05:22:24.619232Z","iopub.status.idle":"2022-07-17T05:22:24.628649Z","shell.execute_reply.started":"2022-07-17T05:22:24.619188Z","shell.execute_reply":"2022-07-17T05:22:24.627608Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.shape","metadata":{"execution":{"iopub.status.busy":"2022-07-17T05:22:24.630376Z","iopub.execute_input":"2022-07-17T05:22:24.631090Z","iopub.status.idle":"2022-07-17T05:22:24.640528Z","shell.execute_reply.started":"2022-07-17T05:22:24.631055Z","shell.execute_reply":"2022-07-17T05:22:24.639406Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df","metadata":{"execution":{"iopub.status.busy":"2022-07-17T05:22:24.644946Z","iopub.execute_input":"2022-07-17T05:22:24.645598Z","iopub.status.idle":"2022-07-17T05:22:24.660533Z","shell.execute_reply.started":"2022-07-17T05:22:24.645569Z","shell.execute_reply":"2022-07-17T05:22:24.659328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.shape","metadata":{"execution":{"iopub.status.busy":"2022-07-17T05:22:24.662500Z","iopub.execute_input":"2022-07-17T05:22:24.663264Z","iopub.status.idle":"2022-07-17T05:22:24.671487Z","shell.execute_reply.started":"2022-07-17T05:22:24.663228Z","shell.execute_reply":"2022-07-17T05:22:24.670198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df","metadata":{"execution":{"iopub.status.busy":"2022-07-17T05:22:24.673062Z","iopub.execute_input":"2022-07-17T05:22:24.674242Z","iopub.status.idle":"2022-07-17T05:22:24.690076Z","shell.execute_reply.started":"2022-07-17T05:22:24.674196Z","shell.execute_reply":"2022-07-17T05:22:24.689149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df.shape","metadata":{"execution":{"iopub.status.busy":"2022-07-17T05:22:24.691510Z","iopub.execute_input":"2022-07-17T05:22:24.692515Z","iopub.status.idle":"2022-07-17T05:22:24.699034Z","shell.execute_reply.started":"2022-07-17T05:22:24.692480Z","shell.execute_reply":"2022-07-17T05:22:24.697788Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df['discourse_text'][0]","metadata":{"execution":{"iopub.status.busy":"2022-07-17T05:22:24.701404Z","iopub.execute_input":"2022-07-17T05:22:24.702023Z","iopub.status.idle":"2022-07-17T05:22:24.709908Z","shell.execute_reply.started":"2022-07-17T05:22:24.701987Z","shell.execute_reply":"2022-07-17T05:22:24.708753Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.tail()","metadata":{"execution":{"iopub.status.busy":"2022-07-17T05:22:24.711399Z","iopub.execute_input":"2022-07-17T05:22:24.712630Z","iopub.status.idle":"2022-07-17T05:22:24.727505Z","shell.execute_reply.started":"2022-07-17T05:22:24.712584Z","shell.execute_reply":"2022-07-17T05:22:24.726252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df=train_df.drop(columns=['discourse_id','essay_id','discourse_type'],axis=1)\ntrain_df.head(1)","metadata":{"execution":{"iopub.status.busy":"2022-07-17T05:22:24.729159Z","iopub.execute_input":"2022-07-17T05:22:24.729782Z","iopub.status.idle":"2022-07-17T05:22:24.743636Z","shell.execute_reply.started":"2022-07-17T05:22:24.729745Z","shell.execute_reply":"2022-07-17T05:22:24.742535Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['discourse_effectiveness'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-07-17T05:22:24.745310Z","iopub.execute_input":"2022-07-17T05:22:24.745797Z","iopub.status.idle":"2022-07-17T05:22:24.761590Z","shell.execute_reply.started":"2022-07-17T05:22:24.745761Z","shell.execute_reply":"2022-07-17T05:22:24.760556Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['discourse_effectiveness']=train_df['discourse_effectiveness'].map({'Ineffective':0,'Adequate':1,'Effective':2})\ntrain_df['discourse_effectiveness'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-07-17T05:22:24.763225Z","iopub.execute_input":"2022-07-17T05:22:24.763876Z","iopub.status.idle":"2022-07-17T05:22:24.782952Z","shell.execute_reply.started":"2022-07-17T05:22:24.763837Z","shell.execute_reply":"2022-07-17T05:22:24.781759Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import warnings\nwarnings.filterwarnings('ignore')","metadata":{"execution":{"iopub.status.busy":"2022-07-17T05:22:24.788090Z","iopub.execute_input":"2022-07-17T05:22:24.788399Z","iopub.status.idle":"2022-07-17T05:22:24.794421Z","shell.execute_reply.started":"2022-07-17T05:22:24.788368Z","shell.execute_reply":"2022-07-17T05:22:24.793033Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Loading pretrained model \"DISTILBERT BASE UNCASED\"","metadata":{}},{"cell_type":"code","source":"import transformers\nfrom transformers import DistilBertTokenizer, TFDistilBertModel\ntokenizer = DistilBertTokenizer.from_pretrained('distilbert-base-uncased')\nbert= TFDistilBertModel.from_pretrained(\"distilbert-base-uncased\")","metadata":{"execution":{"iopub.status.busy":"2022-07-17T05:22:24.796248Z","iopub.execute_input":"2022-07-17T05:22:24.796703Z","iopub.status.idle":"2022-07-17T05:22:59.567870Z","shell.execute_reply.started":"2022-07-17T05:22:24.796666Z","shell.execute_reply":"2022-07-17T05:22:59.566864Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('max length of word',max([len(x.split()) for x in train_df['discourse_text']]))","metadata":{"execution":{"iopub.status.busy":"2022-07-17T05:22:59.569523Z","iopub.execute_input":"2022-07-17T05:22:59.569883Z","iopub.status.idle":"2022-07-17T05:22:59.681290Z","shell.execute_reply.started":"2022-07-17T05:22:59.569846Z","shell.execute_reply":"2022-07-17T05:22:59.679434Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['discourse_text'].apply(lambda x: len(x.split())).mean()","metadata":{"execution":{"iopub.status.busy":"2022-07-17T05:22:59.682774Z","iopub.execute_input":"2022-07-17T05:22:59.683111Z","iopub.status.idle":"2022-07-17T05:22:59.808231Z","shell.execute_reply.started":"2022-07-17T05:22:59.683076Z","shell.execute_reply":"2022-07-17T05:22:59.807293Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"max_length=50","metadata":{"execution":{"iopub.status.busy":"2022-07-17T05:22:59.810201Z","iopub.execute_input":"2022-07-17T05:22:59.810966Z","iopub.status.idle":"2022-07-17T05:22:59.815381Z","shell.execute_reply.started":"2022-07-17T05:22:59.810921Z","shell.execute_reply":"2022-07-17T05:22:59.814411Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bert","metadata":{"execution":{"iopub.status.busy":"2022-07-17T05:22:59.816945Z","iopub.execute_input":"2022-07-17T05:22:59.817579Z","iopub.status.idle":"2022-07-17T05:22:59.828051Z","shell.execute_reply.started":"2022-07-17T05:22:59.817545Z","shell.execute_reply":"2022-07-17T05:22:59.827069Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tokenizer","metadata":{"execution":{"iopub.status.busy":"2022-07-17T05:22:59.830148Z","iopub.execute_input":"2022-07-17T05:22:59.831074Z","iopub.status.idle":"2022-07-17T05:22:59.839397Z","shell.execute_reply.started":"2022-07-17T05:22:59.831036Z","shell.execute_reply":"2022-07-17T05:22:59.838401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tokenizer('i love baby')","metadata":{"execution":{"iopub.status.busy":"2022-07-17T05:22:59.842058Z","iopub.execute_input":"2022-07-17T05:22:59.843327Z","iopub.status.idle":"2022-07-17T05:22:59.852472Z","shell.execute_reply.started":"2022-07-17T05:22:59.843295Z","shell.execute_reply":"2022-07-17T05:22:59.851412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Finding x_train from the tokenizer","metadata":{}},{"cell_type":"code","source":"x_train=tokenizer(\n      \n    text=train_df['discourse_text'].tolist(),\n    add_special_tokens=True,\n    max_length=max_length,\n    truncation=True,\n    padding=True,\n    return_tensors='tf',\n    return_attention_mask=False,\n    verbose=True\n    \n    )","metadata":{"execution":{"iopub.status.busy":"2022-07-17T05:22:59.854017Z","iopub.execute_input":"2022-07-17T05:22:59.854909Z","iopub.status.idle":"2022-07-17T05:23:58.094378Z","shell.execute_reply.started":"2022-07-17T05:22:59.854860Z","shell.execute_reply":"2022-07-17T05:23:58.093397Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_train","metadata":{"execution":{"iopub.status.busy":"2022-07-17T05:23:58.096255Z","iopub.execute_input":"2022-07-17T05:23:58.096659Z","iopub.status.idle":"2022-07-17T05:23:58.105879Z","shell.execute_reply.started":"2022-07-17T05:23:58.096614Z","shell.execute_reply":"2022-07-17T05:23:58.104817Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Importing tensorflow","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow.keras import Model,layers,optimizers,callbacks,losses,metrics,utils","metadata":{"execution":{"iopub.status.busy":"2022-07-17T05:23:58.107487Z","iopub.execute_input":"2022-07-17T05:23:58.107856Z","iopub.status.idle":"2022-07-17T05:23:58.116047Z","shell.execute_reply.started":"2022-07-17T05:23:58.107819Z","shell.execute_reply":"2022-07-17T05:23:58.115014Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"input_ids=layers.Input(shape=(max_length,),dtype=tf.int32,name='input_ids')\nembedding=bert(input_ids)[0]","metadata":{"execution":{"iopub.status.busy":"2022-07-17T05:23:58.117852Z","iopub.execute_input":"2022-07-17T05:23:58.118521Z","iopub.status.idle":"2022-07-17T05:24:02.948533Z","shell.execute_reply.started":"2022-07-17T05:23:58.118468Z","shell.execute_reply":"2022-07-17T05:24:02.947552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"embedding[0]","metadata":{"execution":{"iopub.status.busy":"2022-07-17T05:24:02.950001Z","iopub.execute_input":"2022-07-17T05:24:02.951051Z","iopub.status.idle":"2022-07-17T05:24:02.964245Z","shell.execute_reply.started":"2022-07-17T05:24:02.951012Z","shell.execute_reply":"2022-07-17T05:24:02.963367Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Building the model","metadata":{}},{"cell_type":"code","source":"out=layers.GlobalMaxPool1D()(embedding)\nout=layers.Dropout(0.1)(out)\nout=layers.Dense(399,activation='relu')(out)\n\nout=layers.Dropout(0.15)(out)\nout=layers.Dense(67,activation='relu')(out)\nout=layers.Dense(17,activation='relu')(out)\ny=layers.Dense(3,activation='softmax')(out)\n\nnn=Model(inputs=[input_ids],outputs=y)\nnn.layers[1].trainable=True\noptimizer=optimizers.Adam(learning_rate = 2e-5)","metadata":{"execution":{"iopub.status.busy":"2022-07-17T05:24:02.965853Z","iopub.execute_input":"2022-07-17T05:24:02.966842Z","iopub.status.idle":"2022-07-17T05:24:03.019471Z","shell.execute_reply.started":"2022-07-17T05:24:02.966806Z","shell.execute_reply":"2022-07-17T05:24:03.018526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nn.summary()","metadata":{"execution":{"iopub.status.busy":"2022-07-17T05:24:03.021173Z","iopub.execute_input":"2022-07-17T05:24:03.021551Z","iopub.status.idle":"2022-07-17T05:24:03.037136Z","shell.execute_reply.started":"2022-07-17T05:24:03.021515Z","shell.execute_reply":"2022-07-17T05:24:03.036014Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train=train_df['discourse_effectiveness']\ny_train","metadata":{"execution":{"iopub.status.busy":"2022-07-17T05:24:03.039072Z","iopub.execute_input":"2022-07-17T05:24:03.039802Z","iopub.status.idle":"2022-07-17T05:24:03.049619Z","shell.execute_reply.started":"2022-07-17T05:24:03.039732Z","shell.execute_reply":"2022-07-17T05:24:03.048417Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-07-17T05:24:03.051311Z","iopub.execute_input":"2022-07-17T05:24:03.051738Z","iopub.status.idle":"2022-07-17T05:24:03.063744Z","shell.execute_reply.started":"2022-07-17T05:24:03.051667Z","shell.execute_reply":"2022-07-17T05:24:03.062723Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Plotting the model architecture","metadata":{}},{"cell_type":"code","source":"utils.plot_model(nn,show_shapes=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-17T05:24:03.065203Z","iopub.execute_input":"2022-07-17T05:24:03.066078Z","iopub.status.idle":"2022-07-17T05:24:04.212481Z","shell.execute_reply.started":"2022-07-17T05:24:03.066038Z","shell.execute_reply":"2022-07-17T05:24:04.211326Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Fitting the model with x_train and y_train","metadata":{}},{"cell_type":"code","source":"nn.compile(optimizer=optimizer,loss='sparse_categorical_crossentropy',metrics=['accuracy'])\nnn.fit(\n\n    x={'input_ids':x_train['input_ids']},\n    y=y_train,\n    validation_split=0.20,\n    epochs=3,\n    batch_size=45\n\n)","metadata":{"execution":{"iopub.status.busy":"2022-07-17T05:24:04.213905Z","iopub.execute_input":"2022-07-17T05:24:04.214669Z","iopub.status.idle":"2022-07-17T05:29:13.106479Z","shell.execute_reply.started":"2022-07-17T05:24:04.214611Z","shell.execute_reply":"2022-07-17T05:29:13.105544Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Working with test data","metadata":{}},{"cell_type":"code","source":"test_df=test_df.drop(columns=['discourse_id','essay_id','discourse_type'],axis=1)\ntest_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-17T05:29:13.108291Z","iopub.execute_input":"2022-07-17T05:29:13.108997Z","iopub.status.idle":"2022-07-17T05:29:13.122469Z","shell.execute_reply.started":"2022-07-17T05:29:13.108957Z","shell.execute_reply":"2022-07-17T05:29:13.121473Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Finding x_test from tokenizer","metadata":{}},{"cell_type":"code","source":"x_test=tokenizer(\n\n    text=test_df['discourse_text'].tolist(),\n    add_special_tokens=True,\n    max_length=max_length,\n    truncation=True,\n    padding=True,\n    return_tensors='tf',\n    return_attention_mask=False,\n    verbose=True\n\n\n)","metadata":{"execution":{"iopub.status.busy":"2022-07-17T05:29:13.124569Z","iopub.execute_input":"2022-07-17T05:29:13.125666Z","iopub.status.idle":"2022-07-17T05:29:13.146787Z","shell.execute_reply.started":"2022-07-17T05:29:13.125612Z","shell.execute_reply":"2022-07-17T05:29:13.145612Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_test","metadata":{"execution":{"iopub.status.busy":"2022-07-17T05:29:13.147917Z","iopub.execute_input":"2022-07-17T05:29:13.148200Z","iopub.status.idle":"2022-07-17T05:29:13.161171Z","shell.execute_reply.started":"2022-07-17T05:29:13.148175Z","shell.execute_reply":"2022-07-17T05:29:13.160033Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model prediction","metadata":{}},{"cell_type":"code","source":"predicted=nn.predict({'input_ids':x_test['input_ids']})","metadata":{"execution":{"iopub.status.busy":"2022-07-17T05:29:13.162634Z","iopub.execute_input":"2022-07-17T05:29:13.163649Z","iopub.status.idle":"2022-07-17T05:29:14.526618Z","shell.execute_reply.started":"2022-07-17T05:29:13.163610Z","shell.execute_reply":"2022-07-17T05:29:14.525580Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predicted","metadata":{"execution":{"iopub.status.busy":"2022-07-17T05:29:14.528184Z","iopub.execute_input":"2022-07-17T05:29:14.529038Z","iopub.status.idle":"2022-07-17T05:29:14.536953Z","shell.execute_reply.started":"2022-07-17T05:29:14.529000Z","shell.execute_reply":"2022-07-17T05:29:14.535909Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df=sub_df.drop(columns=['Adequate','Ineffective','Effective'],axis=1)\nsub_df","metadata":{"execution":{"iopub.status.busy":"2022-07-17T05:29:14.538292Z","iopub.execute_input":"2022-07-17T05:29:14.539507Z","iopub.status.idle":"2022-07-17T05:29:14.554707Z","shell.execute_reply.started":"2022-07-17T05:29:14.539465Z","shell.execute_reply":"2022-07-17T05:29:14.553717Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df['Ineffective'] = predicted[:,0]\nsub_df['Adequate'] = predicted[:,1]\nsub_df['Effective'] = predicted[:,2]\nsub_df","metadata":{"execution":{"iopub.status.busy":"2022-07-17T05:29:14.555961Z","iopub.execute_input":"2022-07-17T05:29:14.556470Z","iopub.status.idle":"2022-07-17T05:29:14.575130Z","shell.execute_reply.started":"2022-07-17T05:29:14.556436Z","shell.execute_reply":"2022-07-17T05:29:14.574210Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df.to_csv('submission.csv',index=None)","metadata":{"execution":{"iopub.status.busy":"2022-07-17T05:29:14.577191Z","iopub.execute_input":"2022-07-17T05:29:14.577558Z","iopub.status.idle":"2022-07-17T05:29:14.587887Z","shell.execute_reply.started":"2022-07-17T05:29:14.577524Z","shell.execute_reply":"2022-07-17T05:29:14.586969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.read_csv('submission.csv')","metadata":{"execution":{"iopub.status.busy":"2022-07-17T05:40:04.552068Z","iopub.execute_input":"2022-07-17T05:40:04.552566Z","iopub.status.idle":"2022-07-17T05:40:04.574716Z","shell.execute_reply.started":"2022-07-17T05:40:04.552524Z","shell.execute_reply":"2022-07-17T05:40:04.573702Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}