{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Project in Keras\n\nBased on: https://github.com/rachhek/speech_recognition_using_lstm/blob/master/speech_recognition_using_lstm_experiment.ipynb\n","metadata":{}},{"cell_type":"code","source":"! pip install python_speech_features","metadata":{"execution":{"iopub.status.busy":"2023-04-24T15:52:09.925773Z","iopub.execute_input":"2023-04-24T15:52:09.926245Z","iopub.status.idle":"2023-04-24T15:52:24.575786Z","shell.execute_reply.started":"2023-04-24T15:52:09.926202Z","shell.execute_reply":"2023-04-24T15:52:24.574056Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras.layers import LSTM, Dense, Dropout, Embedding, Masking, Bidirectional,Flatten\nfrom keras.layers import SpatialDropout1D, SpatialDropout2D, SpatialDropout3D, Bidirectional\nfrom keras.layers import Conv1D, BatchNormalization, Conv2D, MaxPooling2D, MaxPooling1D, Flatten, Dropout, Reshape\nfrom python_speech_features import mfcc\nfrom python_speech_features import logfbank\nfrom keras.models import Sequential, load_model\nfrom keras.optimizers import Adam\nfrom keras.utils import plot_model\nfrom keras.preprocessing.text import Tokenizer\nfrom tensorflow.keras.utils import pad_sequences\nfrom sklearn.preprocessing import LabelEncoder,normalize\nfrom matplotlib import pyplot\nfrom keras.callbacks import EarlyStopping\nfrom sklearn.preprocessing import MinMaxScaler\nimport scipy.io.wavfile as wav\nimport numpy as np\nimport keras\nimport csv\nimport os\nfrom tqdm import tqdm\nfrom sklearn.metrics import confusion_matrix, accuracy_score\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport time\nimport tensorflow as tf\nimport librosa\nimport soundfile as sf\nimport pickle","metadata":{"collapsed":false,"pycharm":{"is_executing":true},"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-04-24T15:52:24.578693Z","iopub.execute_input":"2023-04-24T15:52:24.579592Z","iopub.status.idle":"2023-04-24T15:52:33.673444Z","shell.execute_reply.started":"2023-04-24T15:52:24.579520Z","shell.execute_reply":"2023-04-24T15:52:33.671591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"collapsed":false,"jupyter":{"outputs_hidden":false}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preprocessing with spectogram (can be run only once)","metadata":{}},{"cell_type":"code","source":"#! cp /kaggle/input/tensorflow-speech-recognition-challenge/train.7z ./\n#! 7za x train.7z","metadata":{"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-04-24T15:52:33.677204Z","iopub.execute_input":"2023-04-24T15:52:33.678905Z","iopub.status.idle":"2023-04-24T15:52:33.684506Z","shell.execute_reply.started":"2023-04-24T15:52:33.678832Z","shell.execute_reply":"2023-04-24T15:52:33.683096Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def convert_background_noise(root_path='./train', input_folder='_background_noise_', output_folder='silence'):\n    audio_path = os.path.join(root_path, 'audio')\n    input_path = os.path.join(audio_path, input_folder)\n    output_path = os.path.join(audio_path, output_folder)\n    \n    sample_rate = 16000\n    sample_length = 1\n\n    audio_files = [d for d in os.listdir(input_path)\n                   if os.path.isfile(os.path.join(input_path, d)) and d.endswith('.wav')]\n    samples = []\n\n    for f in audio_files:\n        path = os.path.join(input_path, f)\n        s, _ = librosa.load(path, sr=sample_rate)\n        samples.append(s)\n\n    samples = np.hstack(samples)\n    c = int(sample_rate * sample_length)\n    r = len(samples) // c\n    names = [f'recording_{i}.wav' for i in range(r-1)]\n\n    if not os.path.exists(output_path):\n        os.makedirs(output_path)\n\n    for i in range(r - 1):\n        y = samples[c*i:c*(i+1)]\n        sf.write(os.path.join(output_path, names[i]), y, sample_rate)\n\n    val_choice = np.random.choice(names, int(0.1*len(names)), replace=False).tolist()\n    with open(os.path.join(root_path, 'validation_list.txt'), 'a') as f:\n        for name in val_choice:\n            p = os.path.join(output_folder, name)\n            p = p.replace('./', '')\n            f.write(p)\n            f.write('\\n')\n\n    test_choice = np.random.choice([n for n in names if n not in val_choice], int(0.1*len(names)), replace=False).tolist()\n    with open(os.path.join(root_path, 'testing_list.txt'), 'a') as f:\n        for name in test_choice:\n            p = os.path.join(output_folder, name)\n            p = p.replace('./', '')\n            f.write(p)\n            f.write('\\n')\n","metadata":{"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-04-24T15:52:33.686050Z","iopub.execute_input":"2023-04-24T15:52:33.686424Z","iopub.status.idle":"2023-04-24T15:52:33.704172Z","shell.execute_reply.started":"2023-04-24T15:52:33.686389Z","shell.execute_reply":"2023-04-24T15:52:33.702820Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#convert_background_noise(root_path='./train', input_folder='_background_noise_', output_folder='silence')","metadata":{"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-04-24T15:52:33.707766Z","iopub.execute_input":"2023-04-24T15:52:33.708440Z","iopub.status.idle":"2023-04-24T15:52:33.728396Z","shell.execute_reply.started":"2023-04-24T15:52:33.708372Z","shell.execute_reply":"2023-04-24T15:52:33.726859Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_dataset(root_path_files, files_names_list_name, label_encoder=None):\n    #Calculating x_test and y_test        \n    test_labels = []\n    test_data = []\n\n    #test_labels.txt is a txt file with all labels for the speech samples that is required for the evaluation. We loop through it to calculate the MFCC value for each speech sample and then normalize it\n    with open(os.path.join(root_path_files, files_names_list_name), newline='') as tsvfile:\n        reader = csv.DictReader(tsvfile)\n        reader = csv.reader(tsvfile, delimiter=' ')\n        for row in reader:\n            wav_file = os.path.join(root_path_files, \"audio/\", row[0])\n\n            row.append(row[0].split(\"/\")[0])\n            (rate,sig) = wav.read(wav_file)\n\n            # pad to 1s of length using pad_sequences\n            sig = pad_sequences([sig], maxlen=16000, dtype='float', padding='post', truncating='post', value=0.0)\n\n            #Getting the MFCC value from the .wav files.\n            mfcc_feat = mfcc(sig,rate)\n            \n            scaler = MinMaxScaler(feature_range=(0,1))\n            scaler = scaler.fit(mfcc_feat)\n\n            #Normalizing the MFCC values.\n            normalized = scaler.transform(mfcc_feat)\n            test_data.append(normalized)\n            test_labels.append(str(row[1]))\n        \n        if label_encoder is None:\n            label_encoder_test = LabelEncoder().fit(test_labels)\n        else:\n            label_encoder_test = label_encoder\n        vec_test = label_encoder_test.transform(test_labels)\n\n        #One hot encoding the labels\n        one_hot_labels_test = keras.utils.to_categorical(vec_test, num_classes=len(label_encoder_test.classes_))\n        Y_test = one_hot_labels_test\n        X_test = np.array(test_data,dtype=np.float32)\n        return X_test, Y_test, label_encoder_test","metadata":{"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-04-24T15:52:33.730115Z","iopub.execute_input":"2023-04-24T15:52:33.730613Z","iopub.status.idle":"2023-04-24T15:52:33.743702Z","shell.execute_reply.started":"2023-04-24T15:52:33.730559Z","shell.execute_reply":"2023-04-24T15:52:33.742164Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def generate_train_txt(root_path_files, files_names_list_name):\n    omit = []\n    train = []\n    for f in files_names_list_name:\n        with open(os.path.join(root_path_files, f)) as fileobj:\n            omit += [line.strip() for line in fileobj]\n    for target in os.listdir(os.path.join(root_path_files, 'audio')):\n        if not target.startswith('_'):\n            for file in os.listdir(os.path.join(root_path_files, 'audio', target)):\n                p = os.path.join(target, file)\n                if p not in omit:\n                    train.append(p)\n    with open(os.path.join(root_path_files, 'training_list.txt'), 'wb') as file:\n        for t in train:\n            file.write(t.encode())\n            file.write('\\n'.encode())","metadata":{"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-04-24T15:52:33.745370Z","iopub.execute_input":"2023-04-24T15:52:33.746027Z","iopub.status.idle":"2023-04-24T15:52:33.762729Z","shell.execute_reply.started":"2023-04-24T15:52:33.745978Z","shell.execute_reply":"2023-04-24T15:52:33.761333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#! mkdir ./preprocessed","metadata":{"execution":{"iopub.status.busy":"2023-04-24T15:52:33.764404Z","iopub.execute_input":"2023-04-24T15:52:33.764814Z","iopub.status.idle":"2023-04-24T15:52:33.780319Z","shell.execute_reply.started":"2023-04-24T15:52:33.764774Z","shell.execute_reply":"2023-04-24T15:52:33.779017Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#root_path_files = './train'\n#root_saved_files = './preprocessed'\n\n#generate_train_txt(root_path_files, ['validation_list.txt', 'testing_list.txt'])\n\n#train_files_names_list_name = 'training_list.txt'\n#X_train, Y_train, label_encoder = load_dataset(root_path_files=root_path_files, files_names_list_name=train_files_names_list_name)\n#np.save(os.path.join(root_saved_files, 'X_train'), X_train)\n#np.save(os.path.join(root_saved_files, 'Y_train'), Y_train)\n\n#valid_files_names_list_name = 'validation_list.txt'\n#X_valid, Y_valid, _ = load_dataset(root_path_files=root_path_files, files_names_list_name=valid_files_names_list_name, \n#                                   label_encoder=label_encoder)\n#np.save(os.path.join(root_saved_files, 'X_valid'), X_valid)\n#np.save(os.path.join(root_saved_files, 'Y_valid'), Y_valid)\n\n#test_files_names_list_name = 'testing_list.txt'\n#X_test, Y_test, _ = load_dataset(root_path_files=root_path_files, files_names_list_name=test_files_names_list_name, \n#                                 label_encoder=label_encoder)\n#np.save(os.path.join(root_saved_files, 'X_test'), X_test)\n#np.save(os.path.join(root_saved_files, 'Y_test'), Y_test)","metadata":{"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-04-24T15:52:33.781645Z","iopub.execute_input":"2023-04-24T15:52:33.782111Z","iopub.status.idle":"2023-04-24T15:52:33.791600Z","shell.execute_reply.started":"2023-04-24T15:52:33.782059Z","shell.execute_reply":"2023-04-24T15:52:33.790240Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#with open('encoder', 'wb') as f:\n#    pickle.dump(label_encoder, f)","metadata":{"execution":{"iopub.status.busy":"2023-04-24T15:52:33.793177Z","iopub.execute_input":"2023-04-24T15:52:33.793584Z","iopub.status.idle":"2023-04-24T15:52:33.803617Z","shell.execute_reply.started":"2023-04-24T15:52:33.793534Z","shell.execute_reply":"2023-04-24T15:52:33.802348Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"collapsed":false,"jupyter":{"outputs_hidden":false}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Loading data","metadata":{}},{"cell_type":"code","source":"! pip install gdown\n! gdown https://drive.google.com/uc?id=1S0ZWGTnKzyaYfLUOzii_LFnrqdygLDtf\n! mkdir ./preprocessed\n! unzip -o preprocessed.zip -d ./preprocessed\n! mv ./preprocessed/encoder ./","metadata":{"execution":{"iopub.status.busy":"2023-04-24T19:58:26.244300Z","iopub.execute_input":"2023-04-24T19:58:26.244776Z","iopub.status.idle":"2023-04-24T19:58:49.918040Z","shell.execute_reply.started":"2023-04-24T19:58:26.244725Z","shell.execute_reply":"2023-04-24T19:58:49.916546Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport os\nimport pickle","metadata":{"execution":{"iopub.status.busy":"2023-04-24T19:59:06.979883Z","iopub.execute_input":"2023-04-24T19:59:06.980432Z","iopub.status.idle":"2023-04-24T19:59:06.986910Z","shell.execute_reply.started":"2023-04-24T19:59:06.980375Z","shell.execute_reply":"2023-04-24T19:59:06.985473Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# load the test data and labels\nroot_saved_files = './preprocessed'\n\nX_train = np.load(os.path.join(root_saved_files,'X_train.npy'))\nY_train = np.load(os.path.join(root_saved_files, 'Y_train.npy'))\n\nX_test = np.load(os.path.join(root_saved_files, 'X_test.npy'))\nY_test = np.load(os.path.join(root_saved_files, 'Y_test.npy'))\n\nX_valid = np.load(os.path.join(root_saved_files, 'X_valid.npy'))\nY_valid = np.load(os.path.join(root_saved_files, 'Y_valid.npy'))","metadata":{"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-04-24T19:59:00.098051Z","iopub.execute_input":"2023-04-24T19:59:00.098492Z","iopub.status.idle":"2023-04-24T19:59:00.299551Z","shell.execute_reply.started":"2023-04-24T19:59:00.098455Z","shell.execute_reply":"2023-04-24T19:59:00.297939Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open('encoder', 'rb') as f:\n    encoder = pickle.load(f)\n    CLASSES = encoder.classes_","metadata":{"execution":{"iopub.status.busy":"2023-04-24T19:59:09.004403Z","iopub.execute_input":"2023-04-24T19:59:09.004877Z","iopub.status.idle":"2023-04-24T19:59:09.601606Z","shell.execute_reply.started":"2023-04-24T19:59:09.004837Z","shell.execute_reply":"2023-04-24T19:59:09.600039Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"collapsed":false,"jupyter":{"outputs_hidden":false}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Experiments","metadata":{}},{"cell_type":"code","source":"n_classes = len(CLASSES)","metadata":{"execution":{"iopub.status.busy":"2023-04-24T15:53:05.152759Z","iopub.execute_input":"2023-04-24T15:53:05.153189Z","iopub.status.idle":"2023-04-24T15:53:05.159047Z","shell.execute_reply.started":"2023-04-24T15:53:05.153150Z","shell.execute_reply":"2023-04-24T15:53:05.157729Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def update_seed(new_random_seed):\n    np.random.seed(new_random_seed)\n    tf.keras.utils.set_random_seed(new_random_seed)","metadata":{"execution":{"iopub.status.busy":"2023-04-24T15:53:05.523609Z","iopub.execute_input":"2023-04-24T15:53:05.524481Z","iopub.status.idle":"2023-04-24T15:53:05.530461Z","shell.execute_reply.started":"2023-04-24T15:53:05.524433Z","shell.execute_reply":"2023-04-24T15:53:05.529131Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_model(model, train_data, val_data, test_data, lr, epochs, batch, labels=CLASSES, path='checkpoint'):\n    callbacks = [\n        EarlyStopping(monitor='val_loss', min_delta=0.01, patience=5, mode = 'min')\n    ]\n    m = model()\n    m.compile(optimizer=Adam(amsgrad=True, learning_rate=lr),loss='categorical_crossentropy',metrics=['accuracy'])\n    history = m.fit(train_data[0], train_data[1],\n                    epochs=epochs,\n                    callbacks=callbacks,\n                    batch_size=batch,\n                    validation_data=val_data,\n                    verbose=1,\n                    shuffle=True)\n\n    datetime = time.strftime(\"%Y%m%d-%H%M%S\")\n    m.save(os.path.join(path, 'model_' + datetime))\n\n    #plotting the loss\n    history = m.history\n    pyplot.plot(history.history['loss'])\n    pyplot.plot(history.history[\"val_loss\"])\n    pyplot.title(\"train and validation loss\")\n    pyplot.ylabel(\"value\")\n    pyplot.xlabel(\"epoch\")\n    pyplot.legend(['train','validation'])\n    plt.show()\n\n    #plotting the loss\n    pyplot.plot(history.history['accuracy'])\n    pyplot.plot(history.history[\"val_accuracy\"])\n    pyplot.title(\"train and validation accuracy\")\n    pyplot.ylabel(\"accuracy value\")\n    pyplot.xlabel(\"epoch\")\n    pyplot.legend(['train','validation'])\n    plt.show()\n\n    y_prediction = m.predict(test_data[0])\n    y_prediction = np.argmax(y_prediction, axis = 1)\n    y_test_single_column=np.argmax(test_data[1], axis=1)\n    result = confusion_matrix(y_test_single_column, y_prediction , normalize='pred')\n    plt.figure(figsize=(20,20))\n    labels = labels\n    sns.heatmap(result, annot=True, fmt='.2f', xticklabels=labels, yticklabels=labels)\n    plt.ylabel('True label')\n    plt.xlabel('Predicted label')\n    plt.title('Confusion matrix on test data')\n    plt.show()\n    \n    result = confusion_matrix(y_test_single_column, y_prediction)\n    plt.figure(figsize=(20,20))\n    labels = labels\n    sns.heatmap(result, annot=True, fmt='.2f', xticklabels=labels, yticklabels=labels)\n    plt.ylabel('True label')\n    plt.xlabel('Predicted label')\n    plt.title('Confusion matrix on test data')\n    plt.show()\n\n    \n    acc_train = accuracy_score(np.argmax(train_data[1], axis=1), np.argmax(m.predict(train_data[0]), axis = 1))\n    print(f\"Accuracy score on train dataset: {acc_train}\")\n    acc_val = accuracy_score(np.argmax(val_data[1], axis=1), np.argmax(m.predict(val_data[0]), axis = 1))\n    print(f\"Accuracy score on validation dataset: {acc_val}\")\n    acc_test = accuracy_score(y_test_single_column, y_prediction)\n    print(f\"Accuracy score on test dataset: {acc_test}\")\n\n    return [acc_train, acc_val, acc_test]","metadata":{"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-04-24T15:53:05.948673Z","iopub.execute_input":"2023-04-24T15:53:05.949367Z","iopub.status.idle":"2023-04-24T15:53:05.967223Z","shell.execute_reply.started":"2023-04-24T15:53:05.949300Z","shell.execute_reply":"2023-04-24T15:53:05.966154Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def repeat_train(model, train_data, val_data, test_data, lr, epochs, batch, seeds, labels=CLASSES, path='checkpoint'):\n    accuracy = []\n    for seed in seeds:\n        print(f\"Training with seed {seed}\")\n        p = os.path.join(path, str(seed))\n        if not os.path.exists(path):\n            os.mkdir(path)\n        if not os.path.exists(p):\n            os.mkdir(p)\n        update_seed(seed)\n        acc = train_model(model, train_data, val_data, test_data, lr, epochs, batch, labels, path=p)\n        accuracy.append(acc)\n    with open(os.path.join(path, 'accuracy'), 'wb') as f:\n        pickle.dump(accuracy, f)","metadata":{"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-04-24T15:53:06.477133Z","iopub.execute_input":"2023-04-24T15:53:06.477578Z","iopub.status.idle":"2023-04-24T15:53:06.485746Z","shell.execute_reply.started":"2023-04-24T15:53:06.477526Z","shell.execute_reply":"2023-04-24T15:53:06.484571Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"collapsed":false,"jupyter":{"outputs_hidden":false}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Experiments running!","metadata":{}},{"cell_type":"code","source":"def model1():\n    model = Sequential()\n    model.add(LSTM(200,input_shape=(99,13),return_sequences=False))\n    model.add(Dropout(0.2))\n    model.add(Dense(Y_test.shape[1], activation='softmax'))\n    return model","metadata":{"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-04-24T15:53:07.739120Z","iopub.execute_input":"2023-04-24T15:53:07.739560Z","iopub.status.idle":"2023-04-24T15:53:07.746515Z","shell.execute_reply.started":"2023-04-24T15:53:07.739519Z","shell.execute_reply":"2023-04-24T15:53:07.744929Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#repeat_train(model1, (X_train, Y_train), (X_valid, Y_valid), (X_test, Y_test), lr=0.001, epochs=100, batch=32,\n#             seeds=[0, 10, 20, 30, 40], path='checkpoint_lstm_one_layer')","metadata":{"execution":{"iopub.status.busy":"2023-04-24T15:53:07.938467Z","iopub.execute_input":"2023-04-24T15:53:07.938895Z","iopub.status.idle":"2023-04-24T15:53:07.944728Z","shell.execute_reply.started":"2023-04-24T15:53:07.938853Z","shell.execute_reply":"2023-04-24T15:53:07.943229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def model2():\n    model = Sequential()\n    model.add(LSTM(200,input_shape=(99,13),return_sequences=True))\n    model.add(LSTM(200,return_sequences=False))\n    model.add(Dropout(0.2))\n    model.add(Dense(n_classes, activation='softmax'))\n    return model","metadata":{"execution":{"iopub.status.busy":"2023-04-24T15:53:08.115098Z","iopub.execute_input":"2023-04-24T15:53:08.115875Z","iopub.status.idle":"2023-04-24T15:53:08.121558Z","shell.execute_reply.started":"2023-04-24T15:53:08.115830Z","shell.execute_reply":"2023-04-24T15:53:08.120602Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#repeat_train(model2, (X_train, Y_train), (X_valid, Y_valid), (X_test, Y_test), lr=0.001, epochs=100, batch=32,\n#             seeds=[0, 10, 20, 30, 40], path='checkpoint_lstm_two_layer')","metadata":{"execution":{"iopub.status.busy":"2023-04-24T15:53:08.282942Z","iopub.execute_input":"2023-04-24T15:53:08.284516Z","iopub.status.idle":"2023-04-24T15:53:08.291676Z","shell.execute_reply.started":"2023-04-24T15:53:08.284468Z","shell.execute_reply":"2023-04-24T15:53:08.289845Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def model3():\n    model = Sequential()\n    model.add(LSTM(200,input_shape=(99,13),return_sequences=True))\n    model.add(LSTM(200,return_sequences=True))\n    model.add(LSTM(200,return_sequences=False))\n    model.add(Dropout(0.2))\n    model.add(Dense(n_classes, activation='softmax'))\n    return model","metadata":{"execution":{"iopub.status.busy":"2023-04-24T15:53:08.414941Z","iopub.execute_input":"2023-04-24T15:53:08.415610Z","iopub.status.idle":"2023-04-24T15:53:08.422058Z","shell.execute_reply.started":"2023-04-24T15:53:08.415570Z","shell.execute_reply":"2023-04-24T15:53:08.420820Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#repeat_train(model3, (X_train, Y_train), (X_valid, Y_valid), (X_test, Y_test), lr=0.001, epochs=100, batch=32,\n#             seeds=[0, 10, 20, 30, 40], path='checkpoint_lstm_three_layer')","metadata":{"execution":{"iopub.status.busy":"2023-04-24T15:53:08.571793Z","iopub.execute_input":"2023-04-24T15:53:08.572597Z","iopub.status.idle":"2023-04-24T15:53:08.578065Z","shell.execute_reply.started":"2023-04-24T15:53:08.572532Z","shell.execute_reply":"2023-04-24T15:53:08.576661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def model4():\n    model = Sequential()\n    model.add(Bidirectional(LSTM(200, input_shape=(99,13), return_sequences=True)))\n    model.add(Bidirectional(LSTM(200, return_sequences=False)))\n    model.add(Dropout(0.2))\n    model.add(Dense(n_classes, activation='softmax'))\n    return model","metadata":{"execution":{"iopub.status.busy":"2023-04-24T15:53:08.696207Z","iopub.execute_input":"2023-04-24T15:53:08.696935Z","iopub.status.idle":"2023-04-24T15:53:08.703606Z","shell.execute_reply.started":"2023-04-24T15:53:08.696892Z","shell.execute_reply":"2023-04-24T15:53:08.702134Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#repeat_train(model4, (X_train, Y_train), (X_valid, Y_valid), (X_test, Y_test), lr=0.001, epochs=100, batch=32,\n#             seeds=[0, 10, 20, 30, 40], path='checkpoint_lstm_bidirectional')","metadata":{"execution":{"iopub.status.busy":"2023-04-24T15:53:08.848997Z","iopub.execute_input":"2023-04-24T15:53:08.849679Z","iopub.status.idle":"2023-04-24T15:53:08.853908Z","shell.execute_reply.started":"2023-04-24T15:53:08.849636Z","shell.execute_reply":"2023-04-24T15:53:08.852733Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def model5():\n    model = Sequential()\n    model.add(LSTM(20, input_shape=(99,13),return_sequences=True))\n    model.add(LSTM(20, return_sequences=False))\n    model.add(Dropout(0.2))\n    model.add(Dense(n_classes, activation='softmax'))\n    return model","metadata":{"execution":{"iopub.status.busy":"2023-04-24T15:53:08.986039Z","iopub.execute_input":"2023-04-24T15:53:08.986705Z","iopub.status.idle":"2023-04-24T15:53:08.995268Z","shell.execute_reply.started":"2023-04-24T15:53:08.986650Z","shell.execute_reply":"2023-04-24T15:53:08.994033Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#repeat_train(model5, (X_train, Y_train), (X_valid, Y_valid), (X_test, Y_test), lr=0.001, epochs=100, batch=32,\n#             seeds=[0, 10, 20, 30, 40], path='checkpoint_lstm_hidden_20')","metadata":{"execution":{"iopub.status.busy":"2023-04-24T15:53:09.121179Z","iopub.execute_input":"2023-04-24T15:53:09.121643Z","iopub.status.idle":"2023-04-24T15:53:09.127276Z","shell.execute_reply.started":"2023-04-24T15:53:09.121602Z","shell.execute_reply":"2023-04-24T15:53:09.125539Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def model6():\n    model = Sequential()\n    model.add(LSTM(100,input_shape=(99,13),return_sequences=True))\n    model.add(LSTM(100,return_sequences=False))\n    model.add(Dropout(0.2))\n    model.add(Dense(n_classes, activation='softmax'))\n    return model","metadata":{"execution":{"iopub.status.busy":"2023-04-24T15:53:09.410282Z","iopub.execute_input":"2023-04-24T15:53:09.410990Z","iopub.status.idle":"2023-04-24T15:53:09.418808Z","shell.execute_reply.started":"2023-04-24T15:53:09.410934Z","shell.execute_reply":"2023-04-24T15:53:09.417409Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#repeat_train(model6, (X_train, Y_train), (X_valid, Y_valid), (X_test, Y_test), lr=0.001, epochs=100, batch=32,\n#             seeds=[0, 10, 20, 30, 40], path='checkpoint_lstm_hidden_100')","metadata":{"execution":{"iopub.status.busy":"2023-04-24T15:53:10.208733Z","iopub.execute_input":"2023-04-24T15:53:10.209439Z","iopub.status.idle":"2023-04-24T15:53:10.213397Z","shell.execute_reply.started":"2023-04-24T15:53:10.209397Z","shell.execute_reply":"2023-04-24T15:53:10.212418Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def model7():\n    model = Sequential()\n    model.add(LSTM(300,input_shape=(99,13),return_sequences=True))\n    model.add(LSTM(300,return_sequences=False))\n    model.add(Dropout(0.2))\n    model.add(Dense(n_classes, activation='softmax'))\n    return model","metadata":{"execution":{"iopub.status.busy":"2023-04-24T15:53:10.398007Z","iopub.execute_input":"2023-04-24T15:53:10.398471Z","iopub.status.idle":"2023-04-24T15:53:10.405007Z","shell.execute_reply.started":"2023-04-24T15:53:10.398424Z","shell.execute_reply":"2023-04-24T15:53:10.403708Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#repeat_train(model7, (X_train, Y_train), (X_valid, Y_valid), (X_test, Y_test), lr=0.001, epochs=100, batch=32,\n#             seeds=[0, 10, 20, 30, 40], path='checkpoint_lstm_hidden_300')","metadata":{"execution":{"iopub.status.busy":"2023-04-24T15:53:10.978643Z","iopub.execute_input":"2023-04-24T15:53:10.979341Z","iopub.status.idle":"2023-04-24T15:53:10.983598Z","shell.execute_reply.started":"2023-04-24T15:53:10.979286Z","shell.execute_reply":"2023-04-24T15:53:10.982654Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def model8():\n    model = Sequential()\n    model.add(Conv1D(filters=256, kernel_size=10, strides=4, input_shape=(99,13)))\n    model.add(BatchNormalization())\n    model.add(LSTM(128,return_sequences=True))\n    model.add(LSTM(128,return_sequences=False))\n    model.add(Dropout(0.2))\n    model.add(Dense(n_classes, activation='softmax'))\n    return model","metadata":{"execution":{"iopub.status.busy":"2023-04-24T15:53:11.168921Z","iopub.execute_input":"2023-04-24T15:53:11.169645Z","iopub.status.idle":"2023-04-24T15:53:11.176997Z","shell.execute_reply.started":"2023-04-24T15:53:11.169605Z","shell.execute_reply":"2023-04-24T15:53:11.175674Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#repeat_train(model8, (X_train, Y_train), (X_valid, Y_valid), (X_test, Y_test), lr=0.001, epochs=100, batch=32,\n#             seeds=[0, 10, 20, 30, 40], path='checkpoint_conv1d_lstm')","metadata":{"execution":{"iopub.status.busy":"2023-04-24T15:53:11.603358Z","iopub.execute_input":"2023-04-24T15:53:11.603833Z","iopub.status.idle":"2023-04-24T15:53:11.610345Z","shell.execute_reply.started":"2023-04-24T15:53:11.603793Z","shell.execute_reply":"2023-04-24T15:53:11.608520Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def model9():\n    model = Sequential()\n    model.add(Reshape((99, 13, 1), input_shape=(99, 13)))\n    model.add(Conv2D(32, (3, 3), activation='relu'))\n    model.add(MaxPooling2D((2, 2)))\n    model.add(Conv2D(64, (3, 3), activation='relu'))\n    model.add(MaxPooling2D((2, 2)))\n    model.add(Flatten())\n    model.add(Dropout(0.2))\n    model.add(Dense(64, activation='relu'))\n    model.add(Dropout(0.2))\n    model.add(Dense(n_classes, activation='softmax'))\n    return model","metadata":{"execution":{"iopub.status.busy":"2023-04-24T15:53:11.767236Z","iopub.execute_input":"2023-04-24T15:53:11.767703Z","iopub.status.idle":"2023-04-24T15:53:11.776706Z","shell.execute_reply.started":"2023-04-24T15:53:11.767653Z","shell.execute_reply":"2023-04-24T15:53:11.775363Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#repeat_train(model9, (X_train, Y_train), (X_valid, Y_valid), (X_test, Y_test), lr=0.001, epochs=100, batch=32,\n#             seeds=[0, 10, 20, 30, 40], path='checkpoint_conv2d')","metadata":{"execution":{"iopub.status.busy":"2023-04-24T15:53:12.116187Z","iopub.execute_input":"2023-04-24T15:53:12.116639Z","iopub.status.idle":"2023-04-24T15:53:12.122019Z","shell.execute_reply.started":"2023-04-24T15:53:12.116600Z","shell.execute_reply":"2023-04-24T15:53:12.120568Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def model10():\n    model = Sequential()\n    model.add(Conv1D(32, 3, activation='relu'))\n    model.add(MaxPooling1D(2))\n    model.add(Conv1D(64, 3, activation='relu'))\n    model.add(MaxPooling1D(2))\n    model.add(Flatten())\n    model.add(Dropout(0.2))\n    model.add(Dense(64, activation='relu'))\n    model.add(Dropout(0.2))\n    model.add(Dense(n_classes, activation='softmax'))\n    return model","metadata":{"execution":{"iopub.status.busy":"2023-04-24T15:53:12.316048Z","iopub.execute_input":"2023-04-24T15:53:12.316847Z","iopub.status.idle":"2023-04-24T15:53:12.324799Z","shell.execute_reply.started":"2023-04-24T15:53:12.316784Z","shell.execute_reply":"2023-04-24T15:53:12.323370Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#repeat_train(model10, (X_train, Y_train), (X_valid, Y_valid), (X_test, Y_test), lr=0.001, epochs=100, batch=32,\n#             seeds=[0, 10, 20, 30, 40], path='checkpoint_conv1d')","metadata":{"execution":{"iopub.status.busy":"2023-04-24T15:53:13.140747Z","iopub.execute_input":"2023-04-24T15:53:13.141145Z","iopub.status.idle":"2023-04-24T15:53:13.145760Z","shell.execute_reply.started":"2023-04-24T15:53:13.141110Z","shell.execute_reply":"2023-04-24T15:53:13.144681Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"collapsed":false,"jupyter":{"outputs_hidden":false}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Other approaches\n\n1. The main goal of the Kaggle competition is to make predictions on 12 target classes (10 commands + silence + Unknown). In above experiments, we considered all 31 classes, so we have dealt with a bigger task. For the final predictions, we would aggregate predictions for all not considered classes into target \"Unknown\" label. Let's now consider using the \"Unknown\" label for not considered classes.","metadata":{}},{"cell_type":"code","source":"def transform_y(y, encoder, new_classes, new_encoder=None):\n    def translate(x, new=new_classes):\n        if x in new:\n            return x\n        else:\n            return 'unknown'\n    \n    y_new = encoder.inverse_transform(np.argmax(y, axis=1))\n    y_new = np.array(list(map(translate, y_new)))\n    \n    if new_encoder is None:\n        new_encoder = LabelEncoder().fit(y_new)\n        \n    y_new = new_encoder.transform(y_new)\n    return keras.utils.to_categorical(y_new, num_classes=len(np.unique(y_new))), new_encoder","metadata":{"execution":{"iopub.status.busy":"2023-04-24T16:09:18.408351Z","iopub.execute_input":"2023-04-24T16:09:18.408784Z","iopub.status.idle":"2023-04-24T16:09:18.415440Z","shell.execute_reply.started":"2023-04-24T16:09:18.408749Z","shell.execute_reply":"2023-04-24T16:09:18.414335Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"new_classes = 'unknown, silence, yes, no, up, down, left, right, on, off, stop, go'.split(', ')","metadata":{"execution":{"iopub.status.busy":"2023-04-24T16:09:18.958238Z","iopub.execute_input":"2023-04-24T16:09:18.958693Z","iopub.status.idle":"2023-04-24T16:09:18.964300Z","shell.execute_reply.started":"2023-04-24T16:09:18.958650Z","shell.execute_reply":"2023-04-24T16:09:18.962910Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Y_train_with_unknown, new_encoder1 = transform_y(Y_train, encoder, new_classes)\nY_valid_with_unknown, _ = transform_y(Y_valid, encoder, new_classes, new_encoder1)\nY_test_with_unknown, _ = transform_y(Y_test, encoder, new_classes, new_encoder1)","metadata":{"execution":{"iopub.status.busy":"2023-04-24T16:09:20.755430Z","iopub.execute_input":"2023-04-24T16:09:20.755847Z","iopub.status.idle":"2023-04-24T16:09:20.815209Z","shell.execute_reply.started":"2023-04-24T16:09:20.755813Z","shell.execute_reply":"2023-04-24T16:09:20.813975Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def model11():\n    model = Sequential()\n    model.add(LSTM(200,input_shape=(99,13),return_sequences=False))\n    model.add(Dropout(0.2))\n    model.add(Dense(Y_train_with_unknown.shape[1], activation='softmax'))\n    return model","metadata":{"execution":{"iopub.status.busy":"2023-04-24T16:09:22.098528Z","iopub.execute_input":"2023-04-24T16:09:22.098943Z","iopub.status.idle":"2023-04-24T16:09:22.105997Z","shell.execute_reply.started":"2023-04-24T16:09:22.098908Z","shell.execute_reply":"2023-04-24T16:09:22.104685Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#repeat_train(model11, (X_train, Y_train_with_unknown), (X_valid, Y_valid_with_unknown), (X_test, Y_test_with_unknown), lr=0.001, epochs=100, batch=32,\n#             seeds=[0, 10, 20, 30, 40], labels=new_encoder1.classes_, path='checkpoint_lstm_with_unknown')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"2. The model trained on all classes had quite a problem with silence class. This is why we wanted to train a model to recognize silence from all other possible classes.","metadata":{}},{"cell_type":"code","source":"Y_train_silence, new_encoder2 = transform_y(Y_train, encoder, ['silence'])\nY_valid_silence, _ = transform_y(Y_valid, encoder, ['silence'], new_encoder2)\nY_test_silence, _ = transform_y(Y_test, encoder, ['silence'], new_encoder2)","metadata":{"execution":{"iopub.status.busy":"2023-04-24T16:11:44.656559Z","iopub.execute_input":"2023-04-24T16:11:44.657003Z","iopub.status.idle":"2023-04-24T16:11:44.704816Z","shell.execute_reply.started":"2023-04-24T16:11:44.656962Z","shell.execute_reply":"2023-04-24T16:11:44.703631Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def model12():\n    model = Sequential()\n    model.add(LSTM(200,input_shape=(99,13),return_sequences=False))\n    model.add(Dropout(0.2))\n    model.add(Dense(Y_train_silence.shape[1], activation='softmax'))\n    return model","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#repeat_train(model12, (X_train, Y_train_silence), (X_valid, Y_valid_silence), (X_test, Y_test_silence), lr=0.001, epochs=100, batch=32,\n#             seeds=[0, 10, 20, 30, 40], labels=new_encoder2.classes_, path='checkpoint_lstm_only_silence')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"3. As both approaches above cause class imbalance, we will add weights. We hope it will diminish the effects of the accuracy paradox. This experiment requires changes in the training function","metadata":{}},{"cell_type":"code","source":"from sklearn.utils.class_weight import compute_class_weight","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_model2(model, train_data, val_data, test_data, lr, epochs, batch, labels, path='checkpoint'):\n    # class weights\n    targets = np.argmax(train_data[1], axis=1).tolist()\n    classes_unique = sorted(np.unique(targets).tolist())\n    class_weights = compute_class_weight('balanced', classes=classes_unique, y=targets)\n    class_weights = {classes_unique[i]: class_weights[i] for i in range(len(class_weights))}\n\n    \n    callbacks = [\n        EarlyStopping(monitor='val_loss', min_delta=0.01, patience=5, mode = 'min')\n    ]\n    m = model()\n    m.compile(optimizer=Adam(amsgrad=True, learning_rate=lr),loss='categorical_crossentropy',metrics=['accuracy'])\n    history = m.fit(train_data[0], train_data[1],\n                    epochs=epochs,\n                    callbacks=callbacks,\n                    batch_size=batch,\n                    validation_data=val_data,\n                    verbose=1,\n                    shuffle=True,\n                    class_weight=class_weights)\n\n    datetime = time.strftime(\"%Y%m%d-%H%M%S\")\n    m.save(os.path.join(path, 'model_' + datetime))\n\n    #plotting the loss\n    history = m.history\n    pyplot.plot(history.history['loss'])\n    pyplot.plot(history.history[\"val_loss\"])\n    pyplot.title(\"train and validation loss\")\n    pyplot.ylabel(\"value\")\n    pyplot.xlabel(\"epoch\")\n    pyplot.legend(['train','validation'])\n    plt.show()\n\n    #plotting the loss\n    pyplot.plot(history.history['accuracy'])\n    pyplot.plot(history.history[\"val_accuracy\"])\n    pyplot.title(\"train and validation accuracy\")\n    pyplot.ylabel(\"accuracy value\")\n    pyplot.xlabel(\"epoch\")\n    pyplot.legend(['train','validation'])\n    plt.show()\n\n    y_prediction = m.predict(test_data[0])\n    y_prediction = np.argmax(y_prediction, axis = 1)\n    y_test_single_column=np.argmax(test_data[1], axis=1)\n    result = confusion_matrix(y_test_single_column, y_prediction , normalize='pred')\n    plt.figure(figsize=(20,20))\n    labels = labels\n    sns.heatmap(result, annot=True, fmt='.2f', xticklabels=labels, yticklabels=labels)\n    plt.ylabel('True label')\n    plt.xlabel('Predicted label')\n    plt.title('Confusion matrix on test data')\n    plt.show()\n    \n    result = confusion_matrix(y_test_single_column, y_prediction)\n    plt.figure(figsize=(20,20))\n    labels = labels\n    sns.heatmap(result, annot=True, fmt='.2f', xticklabels=labels, yticklabels=labels)\n    plt.ylabel('True label')\n    plt.xlabel('Predicted label')\n    plt.title('Confusion matrix on test data')\n    plt.show()\n\n    \n    acc_train = accuracy_score(np.argmax(train_data[1], axis=1), np.argmax(m.predict(train_data[0]), axis = 1))\n    print(f\"Accuracy score on train dataset: {acc_train}\")\n    acc_val = accuracy_score(np.argmax(val_data[1], axis=1), np.argmax(m.predict(val_data[0]), axis = 1))\n    print(f\"Accuracy score on validation dataset: {acc_val}\")\n    acc_test = accuracy_score(y_test_single_column, y_prediction)\n    print(f\"Accuracy score on test dataset: {acc_test}\")\n\n    return [acc_train, acc_val, acc_test]","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def repeat_train2(model, train_data, val_data, test_data, lr, epochs, batch, seeds, labels, path='checkpoint'):\n    accuracy = []\n    for seed in seeds:\n        print(f\"Training with seed {seed}\")\n        p = os.path.join(path, str(seed))\n        if not os.path.exists(path):\n            os.mkdir(path)\n        if not os.path.exists(p):\n            os.mkdir(p)\n        update_seed(seed)\n        acc = train_model2(model, train_data, val_data, test_data, lr, epochs, batch, labels, path=p)\n        accuracy.append(acc)\n    with open(os.path.join(path, 'accuracy'), 'wb') as f:\n        pickle.dump(accuracy, f)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"3a. Repeat for the first approach","metadata":{}},{"cell_type":"code","source":"#repeat_train2(model11, (X_train, Y_train_with_unknown), (X_valid, Y_valid_with_unknown), (X_test, Y_test_with_unknown), lr=0.001, epochs=100, batch=32,\n#              seeds=[0, 10, 20, 30, 40], labels=new_classes, path='checkpoint_lstm_with_unknown_weighted')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"3b. Repeat for the second approach","metadata":{}},{"cell_type":"code","source":"#repeat_train2(model12, (X_train, Y_train_silence), (X_valid, Y_valid_silence), (X_test, Y_test_silence), lr=0.001, epochs=100, batch=32,\n#              seeds=[0, 10, 20, 30, 40], labels=['unknown', 'silence'], path='checkpoint_lstm_only_silence')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Confusion matrices with removed diagonal for the most interesting models","metadata":{}},{"cell_type":"code","source":"! pip install gdown\n! gdown https://drive.google.com/uc?id=1TSexiQS2-k3fk-WDkHjf6fxYSFFErQSI\n! unzip -o matrices.zip","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import seaborn as sns\nimport matplotlib.pyplot as plt","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_conf_matrix_no_diagonal(path):\n    model = keras.models.load_model(path)\n    y_prediction = model.predict(X_test)\n    y_prediction = np.argmax(y_prediction, axis = 1)\n    y_test_single_column=np.argmax(Y_test, axis=1)\n    \n    result = confusion_matrix(y_test_single_column, y_prediction)\n    np.fill_diagonal(result, 0)\n    plt.figure(figsize=(20,20))\n    labels = CLASSES\n    sns.heatmap(result, annot=True, fmt='.2f', xticklabels=labels, yticklabels=labels)\n    plt.ylabel('True label')\n    plt.xlabel('Predicted label')\n    plt.title('Confusion matrix without diagonal on test data')\n    plt.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_conf_matrix_no_diagonal('checkpoint_lstm_three_layers/0/model_20230423-101725')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_conf_matrix_no_diagonal('checkpoint_conv1d/10/model_20230423-193025')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_conf_matrix_no_diagonal('checkpoint_conv2d/10/model_20230423-190231')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Boxplots summarising accuracy scores ","metadata":{}},{"cell_type":"code","source":"# results taken from the calculations above","metadata":{"execution":{"iopub.status.busy":"2023-04-24T20:53:47.118859Z","iopub.execute_input":"2023-04-24T20:53:47.119267Z","iopub.status.idle":"2023-04-24T20:53:47.146350Z","shell.execute_reply.started":"2023-04-24T20:53:47.119229Z","shell.execute_reply":"2023-04-24T20:53:47.145389Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import seaborn as sns\nimport matplotlib.pyplot as plt\nimport pandas as pd","metadata":{"execution":{"iopub.status.busy":"2023-04-24T20:53:47.160506Z","iopub.execute_input":"2023-04-24T20:53:47.161426Z","iopub.status.idle":"2023-04-24T20:53:48.299818Z","shell.execute_reply.started":"2023-04-24T20:53:47.161390Z","shell.execute_reply":"2023-04-24T20:53:48.298414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results_df = {\n    'model': ['one layer LSTM (200)' for _ in range(5)] +\n             ['two layer LSTM (200)' for _ in range(5)] +\n             ['three layer LSTM (200)' for _ in range(5)] +\n             ['bidirectional two layer LSTM (200)' for _ in range(5)] +\n             ['two layer LSTM (20)' for _ in range(5)] +\n             ['two layer LSTM (100)' for _ in range(5)] +\n             ['two layer LSTM (300)' for _ in range(5)] +\n             ['convolution 1D + two layer LSTM (128)' for _ in range(5)] +\n             ['convolution 2D' for _ in range(5)] +\n             ['convolution 1D' for _ in range(5)],\n    'accuracy': [0.9047, 0.9132, 0.9090, 0.9073, 0.9003] +\n                [0.9150, 0.9166, 0.9187, 0.9112, 0.9196] +\n                [0.9217, 0.9207, 0.9172, 0.9203, 0.9162] +\n                [0.9150, 0.9179, 0.9252, 0.9210, 0.9172] +\n                [0.8462, 0.8459, 0.8322, 0.8468, 0.8347] +\n                [0.9047, 0.9051, 0.9135, 0.9100, 0.9035] +\n                [0.9090, 0.9176, 0.9205, 0.9172, 0.9207] +\n                [0.8977, 0.9010, 0.9020, 0.8983, 0.8990] +\n                [0.8066, 0.8052, 0.7849, 0.7967, 0.7880] +\n                [0.8489, 0.8433, 0.8452, 0.8408, 0.8434]\n}","metadata":{"execution":{"iopub.status.busy":"2023-04-24T20:53:48.302503Z","iopub.execute_input":"2023-04-24T20:53:48.303949Z","iopub.status.idle":"2023-04-24T20:53:48.315318Z","shell.execute_reply.started":"2023-04-24T20:53:48.303899Z","shell.execute_reply":"2023-04-24T20:53:48.314012Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results_df = pd.DataFrame(results_df)","metadata":{"execution":{"iopub.status.busy":"2023-04-24T20:53:48.316703Z","iopub.execute_input":"2023-04-24T20:53:48.317271Z","iopub.status.idle":"2023-04-24T20:53:48.330092Z","shell.execute_reply.started":"2023-04-24T20:53:48.317238Z","shell.execute_reply":"2023-04-24T20:53:48.328775Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.set_style('whitegrid')","metadata":{"execution":{"iopub.status.busy":"2023-04-24T20:54:06.558701Z","iopub.execute_input":"2023-04-24T20:54:06.559116Z","iopub.status.idle":"2023-04-24T20:54:06.564696Z","shell.execute_reply.started":"2023-04-24T20:54:06.559081Z","shell.execute_reply":"2023-04-24T20:54:06.563337Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.boxplot(y='model', x='accuracy', data=results_df)\nplt.xlabel('Test accuracy')\nplt.ylabel('Model type')\nplt.title('Test accuracy on different network architectures')\n\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-04-24T20:54:07.809763Z","iopub.execute_input":"2023-04-24T20:54:07.810134Z","iopub.status.idle":"2023-04-24T20:54:08.298164Z","shell.execute_reply.started":"2023-04-24T20:54:07.810101Z","shell.execute_reply":"2023-04-24T20:54:08.297313Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.boxplot(y='model', x='accuracy', data=results_df[results_df.model.isin([\n    'one layer LSTM (200)', 'two layer LSTM (200)', 'three layer LSTM (200)'\n])])\nplt.xlabel('Test accuracy')\nplt.ylabel('Model type')\nplt.title('Test accuracy on different network architectures')\n\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-04-24T20:55:49.855897Z","iopub.execute_input":"2023-04-24T20:55:49.857172Z","iopub.status.idle":"2023-04-24T20:55:50.094987Z","shell.execute_reply.started":"2023-04-24T20:55:49.857123Z","shell.execute_reply":"2023-04-24T20:55:50.093723Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.boxplot(y='model', x='accuracy', data=results_df[results_df.model.isin([\n    'two layer LSTM (200)', 'bidirectional two layer LSTM (200)'\n])])\nplt.xlabel('Test accuracy')\nplt.ylabel('Model type')\nplt.title('Test accuracy on different network architectures')\n\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-04-24T20:56:37.465286Z","iopub.execute_input":"2023-04-24T20:56:37.466356Z","iopub.status.idle":"2023-04-24T20:56:37.753937Z","shell.execute_reply.started":"2023-04-24T20:56:37.466305Z","shell.execute_reply":"2023-04-24T20:56:37.752537Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ranks = {'two layer LSTM (20)':0, 'two layer LSTM (100)':1, 'two layer LSTM (200)':2, 'two layer LSTM (300)':3}  \nd = results_df[results_df.model.isin([\n    'two layer LSTM (20)', 'two layer LSTM (100)', 'two layer LSTM (200)', 'two layer LSTM (300)'\n])].copy()\nd['rank'] = d.model.map(ranks)\nd = d.sort_values('rank')\nsns.boxplot(y='model', x='accuracy', data=d)\nplt.xlabel('Test accuracy')\nplt.ylabel('Model type')\nplt.title('Test accuracy on different network architectures')\n\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-04-24T21:03:24.494847Z","iopub.execute_input":"2023-04-24T21:03:24.495506Z","iopub.status.idle":"2023-04-24T21:03:24.859686Z","shell.execute_reply.started":"2023-04-24T21:03:24.495468Z","shell.execute_reply":"2023-04-24T21:03:24.858481Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Heatmaps with Welsh's statistical test results","metadata":{}},{"cell_type":"code","source":"from scipy.stats import ttest_ind\nfrom functools import partial\nimport numpy as np","metadata":{"execution":{"iopub.status.busy":"2023-04-24T21:06:03.561087Z","iopub.execute_input":"2023-04-24T21:06:03.561617Z","iopub.status.idle":"2023-04-24T21:06:03.567317Z","shell.execute_reply.started":"2023-04-24T21:06:03.561573Z","shell.execute_reply":"2023-04-24T21:06:03.566205Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"welsh_test = partial(ttest_ind, equal_var=False)\n\ndef extract_results_for_architecture(arch):\n    return results_df[results_df.model == arch]['accuracy'].tolist()\n\narchitectures = np.unique(results_df.model).tolist()","metadata":{"execution":{"iopub.status.busy":"2023-04-24T21:06:18.566504Z","iopub.execute_input":"2023-04-24T21:06:18.566982Z","iopub.status.idle":"2023-04-24T21:06:18.575252Z","shell.execute_reply.started":"2023-04-24T21:06:18.566945Z","shell.execute_reply":"2023-04-24T21:06:18.573718Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"p_values = {}\n\nfor a in architectures:\n    p_values[a] = {}","metadata":{"execution":{"iopub.status.busy":"2023-04-24T21:06:20.148762Z","iopub.execute_input":"2023-04-24T21:06:20.149691Z","iopub.status.idle":"2023-04-24T21:06:20.154818Z","shell.execute_reply.started":"2023-04-24T21:06:20.149629Z","shell.execute_reply":"2023-04-24T21:06:20.153630Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for arch1 in architectures:\n    x1 = extract_results_for_architecture(arch1)\n    for arch2 in architectures:\n        x2 = extract_results_for_architecture(arch2)\n        _, p = welsh_test(x1, x2)\n\n        p_values[arch1][arch2] = p\n        p_values[arch2][arch1] = p","metadata":{"execution":{"iopub.status.busy":"2023-04-24T21:06:28.768270Z","iopub.execute_input":"2023-04-24T21:06:28.768968Z","iopub.status.idle":"2023-04-24T21:06:28.901582Z","shell.execute_reply.started":"2023-04-24T21:06:28.768903Z","shell.execute_reply":"2023-04-24T21:06:28.899792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"p_values = pd.DataFrame.from_dict(p_values)","metadata":{"execution":{"iopub.status.busy":"2023-04-24T21:06:41.453283Z","iopub.execute_input":"2023-04-24T21:06:41.453822Z","iopub.status.idle":"2023-04-24T21:06:41.460855Z","shell.execute_reply.started":"2023-04-24T21:06:41.453776Z","shell.execute_reply":"2023-04-24T21:06:41.459078Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"order = ['two layer LSTM (20)', 'two layer LSTM (100)', 'two layer LSTM (200)', 'two layer LSTM (300)',\n          'one layer LSTM (200)', 'three layer LSTM (200)', 'bidirectional two layer LSTM (200)',\n          'convolution 1D', 'convolution 1D + two layer LSTM (128)', 'convolution 2D'\n       ]\n\np_values = p_values.loc[order, order]","metadata":{"execution":{"iopub.status.busy":"2023-04-24T21:13:15.579231Z","iopub.execute_input":"2023-04-24T21:13:15.579822Z","iopub.status.idle":"2023-04-24T21:13:15.589803Z","shell.execute_reply.started":"2023-04-24T21:13:15.579772Z","shell.execute_reply":"2023-04-24T21:13:15.587769Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.subplots(figsize=(8, 8))\nsns.heatmap(p_values, annot=True, fmt=\".2f\", cmap='Blues')\nplt.title(\"p-values of Welsh's test on accuracies of models\")\n\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-04-24T21:13:25.527342Z","iopub.execute_input":"2023-04-24T21:13:25.527889Z","iopub.status.idle":"2023-04-24T21:13:26.631199Z","shell.execute_reply.started":"2023-04-24T21:13:25.527842Z","shell.execute_reply":"2023-04-24T21:13:26.629829Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}