{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Import the libraries","metadata":{}},{"cell_type":"code","source":"\nfrom sklearn.model_selection import train_test_split\n\nimport re\nimport os\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport time\nimport datetime\nimport tensorflow.keras as keras\nfrom scipy import sparse\nimport tensorflow as tf\nfrom tensorflow.keras.layers import Dense, Input, Dropout\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.models import Model\n\nimport transformers\n\nimport numpy as np\nimport matplotlib.pylab as plt\nfrom matplotlib.pyplot import figure\nfrom sklearn import preprocessing\nimport pandas as pd\nfrom keras.layers import Dense # Dense layers are \"fully connected\" layers\nfrom keras.models import Sequential # Documentation: https://keras.io/models/sequential/\nfrom keras.layers import  Flatten\nfrom keras.utils.np_utils import to_categorical\n#from keras.optimizers import SGD\nfrom keras.callbacks import EarlyStopping\nfrom tensorflow.keras.preprocessing.text import Tokenizer\nfrom tensorflow.keras.preprocessing.sequence import pad_sequences\n#from keras.optimizers import SGD, Adam\nfrom keras.callbacks import EarlyStopping\nfrom sklearn.model_selection import KFold\nfrom sklearn.model_selection import RandomizedSearchCV\nfrom sklearn.model_selection import cross_val_score\nfrom keras.wrappers.scikit_learn import KerasClassifier\n\n\n","metadata":{"execution":{"iopub.status.busy":"2022-07-03T10:16:22.734880Z","iopub.execute_input":"2022-07-03T10:16:22.735276Z","iopub.status.idle":"2022-07-03T10:16:22.744545Z","shell.execute_reply.started":"2022-07-03T10:16:22.735245Z","shell.execute_reply":"2022-07-03T10:16:22.743313Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Creating the Pandas Dataframe","metadata":{}},{"cell_type":"code","source":"df = pd.read_csv('../input/feedback-prize-effectiveness/train.csv')\ndt = pd.read_csv('../input/feedback-prize-effectiveness/test.csv')\ndf[\"text\"] = df[\"essay_id\"].apply(lambda x: open(f'/kaggle/input/feedback-prize-effectiveness/train/{x}.txt').read())\ndt[\"text\"] = dt[\"essay_id\"].apply(lambda x: open(f'/kaggle/input/feedback-prize-effectiveness/test/{x}.txt').read())\nfeedback_map = {\"Adequate\":1,\"Effective\":2,\"Ineffective\":0}\ndf[\"feedback\"] = df[\"discourse_effectiveness\"].map(feedback_map)\n#print(df.head())","metadata":{"execution":{"iopub.status.busy":"2022-07-03T10:16:22.764666Z","iopub.execute_input":"2022-07-03T10:16:22.765550Z","iopub.status.idle":"2022-07-03T10:16:36.850464Z","shell.execute_reply.started":"2022-07-03T10:16:22.765500Z","shell.execute_reply":"2022-07-03T10:16:36.848989Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Plot the charts","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(15,10))\nsns.countplot(df['discourse_type'])","metadata":{"_uuid":"ba4fa79f-e47e-4a5f-a4b5-b3e5b383123c","_cell_guid":"9aff6461-cc84-4bf9-b05f-3bfa2eaa61d4","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-07-03T10:16:36.852665Z","iopub.execute_input":"2022-07-03T10:16:36.853033Z","iopub.status.idle":"2022-07-03T10:16:37.038887Z","shell.execute_reply.started":"2022-07-03T10:16:36.852999Z","shell.execute_reply":"2022-07-03T10:16:37.037683Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10,5))\nsns.countplot(df['discourse_effectiveness'])","metadata":{"execution":{"iopub.status.busy":"2022-07-03T10:16:37.040312Z","iopub.execute_input":"2022-07-03T10:16:37.041002Z","iopub.status.idle":"2022-07-03T10:16:37.231523Z","shell.execute_reply.started":"2022-07-03T10:16:37.040959Z","shell.execute_reply":"2022-07-03T10:16:37.230409Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Tokenizing ,Padding and seperating by features and labels","metadata":{}},{"cell_type":"code","source":"#plt.figure(figsize=(10,5))\n#sns.countplot(df['discourse_effectiveness'])\n\n\ntraining_sentences = df.discourse_type.values + ' ' + df.text.values\nlabels = df.feedback.values\ntesting_sentences = dt.discourse_type.values + ' ' + dt.text.values\ntokenizer = Tokenizer()       #out of vocabulary token\ntokenizer.fit_on_texts(training_sentences)\ntraining_sequences = tokenizer.texts_to_sequences(training_sentences)\nx_train = pad_sequences(training_sequences,maxlen=100,padding='post', truncating='post')\n#print(labels.shape)\ntest_sequences = tokenizer.texts_to_sequences(testing_sentences)\nx_test = pad_sequences(test_sequences,maxlen=100,padding='post', truncating='post')\n\nx_train = np.array(x_train)\ny_train = np.array(labels)\nx_test = np.array(x_test)\n#print(x_train.shape)\n","metadata":{"_uuid":"d5a8f8bf-2831-4c23-a747-19127094faf1","_cell_guid":"c630c142-64a0-4672-a8bd-72dddf688be7","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-07-03T10:16:37.233873Z","iopub.execute_input":"2022-07-03T10:16:37.234183Z","iopub.status.idle":"2022-07-03T10:17:01.535220Z","shell.execute_reply.started":"2022-07-03T10:16:37.234155Z","shell.execute_reply":"2022-07-03T10:17:01.534111Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Creating the Random search grid and searching for the best parameters","metadata":{}},{"cell_type":"code","source":"def create_model(learning_rate, activation):\n     opt = Adam(lr = learning_rate)\n     model = Sequential()\n     model.add(Dense(1600, input_shape = (x_train.shape[1],), activation = activation))\n     model.add(Dense(800, activation = activation))\n     model.add(Dense(3, activation = 'softmax'))\n     model.compile(optimizer = opt, loss = 'sparse_categorical_crossentropy', metrics = ['accuracy'])\n     return model\n    ","metadata":{"execution":{"iopub.status.busy":"2022-07-03T10:17:01.536726Z","iopub.execute_input":"2022-07-03T10:17:01.537051Z","iopub.status.idle":"2022-07-03T10:17:01.545016Z","shell.execute_reply.started":"2022-07-03T10:17:01.537022Z","shell.execute_reply":"2022-07-03T10:17:01.543956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = KerasClassifier(build_fn = create_model)\n\n# Define the parameters to try out\nparams = {'activation': ['relu', 'tanh'], 'batch_size': [32, 128, 256], \n          'epochs': [5, 10, 15], 'learning_rate': [0.1, .01, .001]}\n\n# Create a randomize search cv object passing in the parameters to try\nrandom_search = RandomizedSearchCV(model, param_distributions = params, cv = KFold(3))\nrandom_search.fit(x_train, y_train)\n(random_search.best_params_)","metadata":{"execution":{"iopub.status.busy":"2022-07-03T10:17:01.546238Z","iopub.execute_input":"2022-07-03T10:17:01.546533Z","iopub.status.idle":"2022-07-03T10:38:52.305671Z","shell.execute_reply.started":"2022-07-03T10:17:01.546498Z","shell.execute_reply":"2022-07-03T10:38:52.304891Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Once we get the best parameters then apply those parameters in the model","metadata":{}},{"cell_type":"code","source":"\n\nn_cols = x_train.shape[1]\nmodel = Sequential()\n\n# Add the first hidden layer\nmodel.add(Dense(128, activation='relu',input_shape = (n_cols,)))\n\n\n# Add the second hidden layer\nmodel.add(Dense(64, activation='relu'))\nmodel.add(Dense(26, activation='tanh'))\n#model.add(Flatten())\n# Add the output layer\nmodel.add(Dense(3,activation='softmax'))\n#print(\"input shape \",model.input_shape)\n#print(\"output shape \",model.output_shape)\n\n# Compile the model with learnign reate 0.1 ,here early stopping is applied too\nopt = keras.optimizers.Adam(learning_rate=0.01)\nmodel.compile(optimizer=opt, loss='sparse_categorical_crossentropy' , metrics=['accuracy'])\n\n# Fit the model\n\n#print(model.summary())\n\n\nmodel.fit(x_train,y_train,epochs = 5,verbose=2)\npred = model.predict(x_test)\n\n\nsample_submission = pd.read_csv('../input/feedback-prize-effectiveness/sample_submission.csv')\n#sample_submission.head()\nsample_submission['discourse_id'] = dt['discourse_id']\nsample_submission['Ineffective'] = pred[:,0]\nsample_submission['Adequate'] = pred[:,1]\nsample_submission['Effective'] = pred[:,2]\nsample_submission.to_csv(\"submission.csv\", index=False)","metadata":{"_uuid":"5af9af59-2e3f-4a35-a5c5-080aab79487b","_cell_guid":"ec993ad5-0b29-4ac1-8abd-e13d4f961f60","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-07-03T10:38:52.307269Z","iopub.execute_input":"2022-07-03T10:38:52.307948Z","iopub.status.idle":"2022-07-03T10:39:03.499066Z","shell.execute_reply.started":"2022-07-03T10:38:52.307896Z","shell.execute_reply":"2022-07-03T10:39:03.497759Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Plotting the loss function of train data","metadata":{}},{"cell_type":"code","source":"history = model.fit(x_train,y_train,validation_split=0.33,epochs = 5,batch_size=10,verbose=2)\nfigure(figsize=(15, 10))\nplt.plot(history.history['loss'])\nplt.plot(history.history['val_loss'])\nplt.title('model loss')\nplt.ylabel('loss')\nplt.xlabel('epoch')\nplt.legend(['train', 'test'], loc='upper left')\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2022-07-03T10:39:03.500207Z","iopub.execute_input":"2022-07-03T10:39:03.500506Z","iopub.status.idle":"2022-07-03T10:39:33.829538Z","shell.execute_reply.started":"2022-07-03T10:39:03.500481Z","shell.execute_reply":"2022-07-03T10:39:33.828403Z"},"trusted":true},"execution_count":null,"outputs":[]}]}