{"cells":[{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom matplotlib import pyplot as plt\nimport seaborn as sns\nimport os\nimport warnings\nwarnings.filterwarnings(\"ignore\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## EDA"},{"metadata":{"trusted":true},"cell_type":"code","source":"os.listdir('../input/riiid-test-answer-prediction')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lectCsv = pd.read_csv(\"../input/riiid-test-answer-prediction/lectures.csv\")\nexampleTestCsv = pd.read_csv(\"../input/riiid-test-answer-prediction/example_test.csv\")\ntrainCsv = pd.read_csv(\"../input/riiid-test-answer-prediction/train.csv\", low_memory=False, nrows=1000000)\nquestionsCsv = pd.read_csv(\"../input/riiid-test-answer-prediction/questions.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"trainCsv.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"trainCsv.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"trainCsv.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"trainCsv.nunique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"trainCsv.isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#trainCsv[\"prior_question_elapsed_time\"] = trainCsv.groupby([\"user_id\", \"content_id\"]).transform(lambda x: x.fillna(x.mean()))\n#trainCsv[\"prior_question_had_explanation\"] = trainCsv.groupby([\"user_id\", \"content_id\"]).transform(lambda x: x.fillna(x.mean()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#trainCsv.isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"trainCsv['timestamp'].hist(bins = 50)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(15, 7))\nax = sns.countplot(trainCsv.groupby('user_id')['user_answer'].count().value_counts(), palette=\"hls\")\nplt.title(\"Count of answers per user\", fontsize=12)\nplt.xticks(rotation=90, fontsize=13)\nplt.ylabel('Number of answers')\nplt.xlabel('Count of users')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(15, 7))\nax = sns.countplot(trainCsv.user_answer)\nplt.title(\"Distribution of Mean's answer per user\", fontsize=12)\nplt.xticks(rotation=90, fontsize=13)\nplt.ylabel('Frequency')\nplt.xlabel('Average answer')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Preprocessing"},{"metadata":{"trusted":true},"cell_type":"code","source":"# 0 if the event was a question being posed to the user, 1 if the event was the user watching a lecture. So, let's keep just the questions\ntrainCsv = trainCsv[trainCsv.content_type_id == 0]\n# read -1 as null, for lectures\ntrainCsv = trainCsv[trainCsv.answered_correctly != -1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"trainCsv = trainCsv.sort_values(['timestamp'], ascending=True).reset_index(drop = True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"trainCsv.head(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"content_mean_final = trainCsv[['content_id','answered_correctly']].groupby(['content_id']).agg(['mean'])\ncontent_mean_final.columns = [\"answered_correctly_content_mean\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"user_mean_final = trainCsv[['user_id','answered_correctly']].groupby(['user_id']).agg(['mean', 'sum', 'count'])\nuser_mean_final.columns = [\"answered_correctly_user_mean\", 'sum_correct', 'count']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#saving value to fillna\nelapsed_time_mean_final = trainCsv.prior_question_elapsed_time.mean()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"trainCsv.drop(['timestamp', 'content_type_id'], axis=1, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"validation = pd.DataFrame()\nfor i in range(4):\n    last_records = trainCsv.drop_duplicates('user_id', keep = 'last')\n    train_csv = trainCsv[~trainCsv.index.isin(last_records.index)]\n    validation = validation.append(last_records)\n    #print(validation)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X = pd.DataFrame()\nfor i in range(15):\n    last_records = train_csv.drop_duplicates('user_id', keep = 'last')\n    train_csv = train_csv[~train_csv.index.isin(last_records.index)]\n    X = X.append(last_records)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"results_c = trainCsv[['content_id','answered_correctly']].groupby(['content_id']).agg(['mean'])\nresults_c.columns = [\"answered_correctly_content_mean\"]\n\nresults_u = trainCsv[['user_id','answered_correctly']].groupby(['user_id']).agg(['mean', 'sum', 'count'])\nresults_u.columns = [\"answered_correctly_user_mean\", 'sum_correct', 'count']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"results_c.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"results_c.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"results_u.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"results_u.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"result_time_mean = trainCsv.prior_question_elapsed_time.mean()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X = pd.merge(X, results_u, on=['user_id'], how=\"left\")\nX = pd.merge(X, results_c, on=['content_id'], how=\"left\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"validation = pd.merge(validation, results_u, on=['user_id'], how=\"left\")\nvalidation = pd.merge(validation, results_c, on=['content_id'], how=\"left\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"validation.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y = X['answered_correctly']\nX = X.drop(['answered_correctly'], axis=1)\n\ny_val = validation['answered_correctly']\nX_val = validation.drop(['answered_correctly'], axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\nlencoder = LabelEncoder()\n\nX['prior_question_had_explanation'].fillna(False, inplace = True)\nX['prior_question_had_explanation_enc'] = lencoder.fit_transform(X['prior_question_had_explanation'])\nX['answered_correctly_user_mean'].fillna(0.5,  inplace=True)\nX['answered_correctly_content_mean'].fillna(0.5,  inplace=True)\nX['sum_correct'].fillna(0, inplace = True)\nX['count'].fillna(0, inplace = True)\nX['prior_question_elapsed_time'].fillna(result_time_mean, inplace = True)\n\nX_val['prior_question_had_explanation'].fillna(False, inplace = True)\nX_val['prior_question_had_explanation_enc'] = lencoder.fit_transform(X_val['prior_question_had_explanation'])\nX_val['answered_correctly_user_mean'].fillna(0.5,  inplace=True)\nX_val['answered_correctly_content_mean'].fillna(0.5,  inplace=True)\nX_val['sum_correct'].fillna(0, inplace = True)\nX_val['count'].fillna(0, inplace = True)\nX_val['prior_question_elapsed_time'].fillna(result_time_mean, inplace = True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from pandas.plotting import scatter_matrix\nattributes = [\"content_id\",\"prior_question_elapsed_time\",\"answered_correctly_user_mean\", \"answered_correctly_content_mean\"]\nscatter_matrix(X[attributes], figsize=(12, 8))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#import matplotlib.pyplot as plt\n#X.hist(bins=50, figsize=(20,15))\n#plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X = X[['answered_correctly_user_mean', 'answered_correctly_content_mean', 'sum_correct', 'count',\n       'prior_question_elapsed_time','prior_question_had_explanation_enc']]\nX_val = X_val[['answered_correctly_user_mean', 'answered_correctly_content_mean', 'sum_correct', 'count',\n       'prior_question_elapsed_time','prior_question_had_explanation_enc']]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_val","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler\nscaler = StandardScaler()\nX = scaler.fit_transform(X)\nX_val = scaler.transform(X_val)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Model: 1D-CNN"},{"metadata":{"trusted":true},"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow.keras import Sequential\nfrom tensorflow.keras.layers import BatchNormalization,Dropout,Dense,Flatten,Conv1D\nfrom tensorflow.keras.optimizers import Adam\nfrom keras.metrics import BinaryAccuracy\nfrom keras import backend as K","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X1 = X\nX_val1 = X_val","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"K.clear_session()\nX_train = X1.reshape(X1.shape[0], X1.shape[1], 1)\nX_test = X_val1.reshape(X_val1.shape[0], X_val1.shape[1], 1)\n\n    \nmodel=Sequential()\nmodel.add(Conv1D(32, 2, activation='relu', input_shape=X_train[0].shape))\nmodel.add(Conv1D(64, 2, activation='relu'))#, padding='causal'))\nmodel.add(Dropout(0.5))\nmodel.add(Flatten())\nmodel.add(Dense(32, activation='relu'))\nmodel.add(Dense(1, activation='sigmoid'))\n\nmodel.compile(optimizer=Adam(learning_rate=0.01), loss='binary_crossentropy', metrics=['accuracy',tf.keras.metrics.BinaryAccuracy()])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"hist = model.fit(X_train, y, epochs=100, batch_size=50000,validation_data =(X_test,y_val))\nprint(hist)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plot\nplot.plot(hist.history['accuracy'])\nplot.plot(hist.history['val_accuracy'])\nplot.title('Regularized Model accuracy')\nplot.ylabel('Accuracy')\nplot.xlabel('Epoch')\nplot.legend(['Train', 'Test'], loc='upper left')\nplot.grid()\nplot.show()\n\nplot.plot(hist.history['loss'])\nplot.plot(hist.history['val_loss'])\nplot.title('Model loss')\nplot.ylabel('Loss')\nplot.xlabel('Epoch')\nplot.legend(['Train', 'Test'], loc='upper left')\nplot.grid()\nplot.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_pred = model.predict(X_test)\ny_true = np.array(y_val)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import roc_auc_score\nroc_auc_score(y_true, y_pred)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import classification_report, confusion_matrix\n\ny_proba = model.predict(X_test) \ny_pred = model.predict_classes(X_test)\nprint('Confusion Matrix')\nprint(confusion_matrix(y_pred, y_val))\nprint('Classification Report')\nprint(classification_report(y_pred, y_val))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x1=X_test\ny1=y_val\nscore = model.evaluate(x1, y1, verbose=0)\nprint('Test loss:', score[0])\nprint('Test accuracy:', score[1])\n\nxtrain1 = X_train\nytrain1 = y\nscore = model.evaluate(xtrain1, ytrain1, verbose=0)\nprint('Train loss:', score[0])\nprint('Train accuracy:', score[1])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## RNN"},{"metadata":{"trusted":true},"cell_type":"code","source":"X3 = X\nX_val3 = X_val","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"K.clear_session()\nX_train = X3.reshape(X3.shape[0], X3.shape[1], 1)\nX_test = X_val3.reshape(X_val3.shape[0], X_val3.shape[1], 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import keras\nmodelRNN = keras.models.Sequential([\nkeras.layers.SimpleRNN(100, return_sequences=True, input_shape=X_train[0].shape),\nDropout(0.5),\nkeras.layers.SimpleRNN(100, return_sequences=True),\nDropout(0.5),\n\nkeras.layers.SimpleRNN(1)\n])\nmodelRNN.compile(optimizer=Adam(learning_rate=0.01), loss='binary_crossentropy', metrics=['accuracy',tf.keras.metrics.BinaryAccuracy()])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history1 = modelRNN.fit(X_train, y, epochs=100, batch_size=50000,validation_data =(X_test,y_val))\nprint(history1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plot\nplot.plot(history1.history['accuracy'])\nplot.plot(history1.history['val_accuracy'])\nplot.title('Regularized Model accuracy')\nplot.ylabel('Accuracy')\nplot.xlabel('Epoch')\nplot.legend(['Train', 'Test'], loc='upper left')\nplot.grid()\nplot.show()\n\nplot.plot(history1.history['loss'])\nplot.plot(history1.history['val_loss'])\nplot.title('Model loss')\nplot.ylabel('Loss')\nplot.xlabel('Epoch')\nplot.legend(['Train', 'Test'], loc='upper left')\nplot.grid()\nplot.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import classification_report, confusion_matrix\n\ny_proba = modelRNN.predict(X_test) \ny_pred = modelRNN.predict_classes(X_test)\nprint('Confusion Matrix')\nprint(confusion_matrix(y_pred, y_val))\nprint('Classification Report')\nprint(classification_report(y_pred, y_val))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_pred1 = modelRNN.predict(X_test)\ny_true1 = np.array(y_val)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import roc_auc_score\nroc_auc_score(y_true1, y_pred1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x2=X_test\ny2=y_val\nscore = modelRNN.evaluate(x2, y2, verbose=0)\nprint('Test loss:', score[0])\nprint('Test accuracy:', score[1])\n\nxtrain2 = X_train\nytrain2 = y\nscore = modelRNN.evaluate(xtrain2, ytrain2, verbose=0)\nprint('Train loss:', score[0])\nprint('Train accuracy:', score[1])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## LSTM"},{"metadata":{"trusted":true},"cell_type":"code","source":"X4 = X\nX_val4 = X_val\nK.clear_session()\nX_train4 = X4.reshape(X4.shape[0], X4.shape[1], 1)\nX_test4 = X_val4.reshape(X_val4.shape[0], X_val4.shape[1], 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"modelLSTM = keras.models.Sequential([\nkeras.layers.LSTM(20, return_sequences=True, input_shape=X_train4[0].shape),\nkeras.layers.LSTM(20, return_sequences=True),\nkeras.layers.TimeDistributed(keras.layers.Dense(1))\n])\nmodelLSTM.compile(optimizer=Adam(learning_rate=0.001), loss='binary_crossentropy', metrics=['accuracy',tf.keras.metrics.BinaryAccuracy()])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history2 = modelLSTM.fit(X_train, y, epochs=150, batch_size=50000,validation_data =(X_test,y_val))\nprint(history2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_pred2 = modelLSTM.predict(X_test)\ny_true2 = np.array(y_val)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plot\nplot.plot(history2.history['accuracy'])\nplot.plot(history2.history['val_accuracy'])\nplot.title('Regularized Model accuracy')\nplot.ylabel('Accuracy')\nplot.xlabel('Epoch')\nplot.legend(['Train', 'Test'], loc='upper left')\nplot.grid()\nplot.show()\n\nplot.plot(history2.history['loss'])\nplot.plot(history2.history['val_loss'])\nplot.title('Model loss')\nplot.ylabel('Loss')\nplot.xlabel('Epoch')\nplot.legend(['Train', 'Test'], loc='upper left')\nplot.grid()\nplot.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x3=X_test\ny3=y_val\nscore = modelLSTM.evaluate(x3, y3, verbose=0)\nprint('Test loss:', score[0])\nprint('Test accuracy:', score[1])\n\nxtrain3 = X_train\nytrain3 = y\nscore = modelLSTM.evaluate(xtrain3, ytrain3, verbose=0)\nprint('Train loss:', score[0])\nprint('Train accuracy:', score[1])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Parameter Tuning\n\nParameter tuning is done on the 1D-CNN Model since the model accuracy plot shows the train and test curves underfitting....."},{"metadata":{"trusted":true},"cell_type":"code","source":"X2 = X\nX_val2 = X_val","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.layers.convolutional import MaxPooling1D\n\nK.clear_session()\nX_train2 = X2.reshape(X2.shape[0], X2.shape[1], 1)\nX_test2 = X_val2.reshape(X_val2.shape[0], X_val2.shape[1], 1)\n\n\nmodelTuning = Sequential()\nmodelTuning.add(Conv1D(256, 3, activation='relu', input_shape=X_train2[0].shape))\nmodelTuning.add(Dropout(0.35))\nmodelTuning.add(Conv1D(128, 3, activation='relu', padding='valid'))\nmodelTuning.add(Dropout(0.35))\nmodelTuning.add(Flatten())\nmodelTuning.add(Dense(128, activation='relu'))\nmodelTuning.add(Dropout(0.35))\nmodelTuning.add(Dense(64, activation='relu'))\nmodelTuning.add(Dropout(0.35))\nmodelTuning.add(Dense(1, activation='sigmoid'))\n\nmodelTuning.compile(optimizer=Adam(learning_rate=0.001), loss='binary_crossentropy', metrics=['accuracy',tf.keras.metrics.BinaryAccuracy()])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"hist3 = modelTuning.fit(X_train, y, epochs=100, batch_size=100000,validation_data =(X_test2,y_val))\nprint(hist3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plot\nplot.plot(hist3.history['accuracy'])\nplot.plot(hist3.history['val_accuracy'])\nplot.title('Regularized Model accuracy')\nplot.ylabel('Accuracy')\nplot.xlabel('Epoch')\nplot.legend(['Train', 'Test'], loc='upper left')\nplot.grid()\nplot.show()\n\nplot.plot(hist3.history['loss'])\nplot.plot(hist3.history['val_loss'])\nplot.title('Model loss')\nplot.ylabel('Loss')\nplot.xlabel('Epoch')\nplot.legend(['Train', 'Test'], loc='upper left')\nplot.grid()\nplot.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import classification_report, confusion_matrix\n\ny_proba = modelTuning.predict(X_test) \ny_pred = modelTuning.predict_classes(X_test)\nprint('Confusion Matrix')\nprint(confusion_matrix(y_pred, y_val))\nprint('Classification Report')\nprint(classification_report(y_pred, y_val))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_pred3 = modelTuning.predict(X_test)\ny_true3 = np.array(y_val)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import roc_auc_score\nroc_auc_score(y_true3, y_pred3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x3=X_test\ny3=y_val\nscore = modelTuning.evaluate(x3, y3, verbose=0)\nprint('Test loss:', score[0])\nprint('Test accuracy:', score[1])\n\nxtrain3 = X_train\nytrain3 = y\nscore = modelTuning.evaluate(xtrain3, ytrain3, verbose=0)\nprint('Train loss:', score[0])\nprint('Train accuracy:', score[1])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Parameter Tuning LSTM"},{"metadata":{"trusted":true},"cell_type":"code","source":"X5 = X\nX_val5 = X_val\nK.clear_session()\nX_train5 = X5.reshape(X5.shape[0], X5.shape[1], 1)\nX_test5 = X_val5.reshape(X_val5.shape[0], X_val5.shape[1], 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"modelLSTMTuned = keras.models.Sequential([\nkeras.layers.RNN(keras.layers.LSTMCell(50), return_sequences=True,\ninput_shape=X_train4[0].shape),\nkeras.layers.RNN(keras.layers.LSTMCell(50), return_sequences=True),\nkeras.layers.TimeDistributed(keras.layers.Dense(1))\n])\nmodelLSTMTuned.compile(optimizer=Adam(learning_rate=0.001), loss='binary_crossentropy', metrics=['accuracy',tf.keras.metrics.BinaryAccuracy()])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history5 = modelLSTMTuned.fit(X_train, y, epochs=350, batch_size=50000,validation_data =(X_test,y_val))\nprint(history5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_pred5 = modelLSTMTuned.predict(X_test)\ny_true5 = np.array(y_val)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot.plot(history5.history['accuracy'])\nplot.plot(history5.history['val_accuracy'])\nplot.title('Regularized Model accuracy')\nplot.ylabel('Accuracy')\nplot.xlabel('Epoch')\nplot.legend(['Train', 'Test'], loc='upper left')\nplot.grid()\nplot.show()\n\nplot.plot(history5.history['loss'])\nplot.plot(history5.history['val_loss'])\nplot.title('Model loss')\nplot.ylabel('Loss')\nplot.xlabel('Epoch')\nplot.legend(['Train', 'Test'], loc='upper left')\nplot.grid()\nplot.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x5=X_test\ny5=y_val\nscore = modelLSTMTuned.evaluate(x5, y5, verbose=0)\nprint('Test loss:', score[0])\nprint('Test accuracy:', score[1])\n\nxtrain5 = X_train\nytrain5 = y\nscore = modelLSTMTuned.evaluate(xtrain5, ytrain5, verbose=0)\nprint('Train loss:', score[0])\nprint('Train accuracy:', score[1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#y_proba5 = modelLSTMTuned.predict(X_test) \n#y_pred5 = modelLSTMTuned.predict_classes(X_test)\n#print('Confusion Matrix')\n#print(confusion_matrix(y_pred5, y5))\n#print('Classification Report')\n#print(classification_report(y_pred5, y5))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"import riiideducation\nenv = riiideducation.make_env()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"iter_test = env.iter_test()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for (test_df, sample_prediction_df) in iter_test:\n    test_df = pd.merge(test_df, user_mean_final, on=['user_id'],  how=\"left\")\n    test_df = pd.merge(test_df, content_mean_final, on=['content_id'],  how=\"left\")\n    \n    test_df['answered_correctly_user_mean'].fillna(0.5,  inplace=True)\n    test_df['answered_correctly_content_mean'].fillna(0.5,  inplace=True)\n    test_df['sum_correct'].fillna(0, inplace=True)\n    test_df['count'].fillna(0, inplace=True)\n    test_df['prior_question_elapsed_time'].fillna(elapsed_time_mean_final, inplace = True)\n    test_df['prior_question_had_explanation'].fillna(False, inplace=True)\n    test_df[\"prior_question_had_explanation_enc\"] = lencoder.transform(test_df[\"prior_question_had_explanation\"])\n\n    # fit transform cnn\n    X = scaler.transform(test_df[['answered_correctly_user_mean', 'answered_correctly_content_mean', 'sum_correct', 'count',\n                                  'prior_question_elapsed_time', 'prior_question_had_explanation_enc']])\n    test_df['answered_correctly'] = model.predict(X.reshape(X.shape[0], X.shape[1], 1))\n    \n    env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']])","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}