{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":59093,"databundleVersionId":7469972,"sourceType":"competition"},{"sourceId":7976031,"sourceType":"datasetVersion","datasetId":4617907},{"sourceId":167498142,"sourceType":"kernelVersion"},{"sourceId":169052058,"sourceType":"kernelVersion"},{"sourceId":169211386,"sourceType":"kernelVersion"}],"dockerImageVersionId":30646,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import sys\nsys.path.append('/kaggle/input/bottleneck-package')\nimport bottleneck\n\nimport pandas as pd\nimport numpy as np\nimport os\nimport psutil  \nimport random\n\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nfrom tensorflow.keras.optimizers import Adam\n\nfrom pathlib import Path\nimport shutil\nfrom tqdm import tqdm\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import accuracy_score\nimport matplotlib.pyplot as plt\nfrom functools import lru_cache\nfrom pympler import asizeof\n\nimport typing\nfrom dataclasses import dataclass\nfrom collections.abc import Callable\nfrom collections.abc import Sequence\n\nimport itertools","metadata":{"execution":{"iopub.status.busy":"2024-03-29T16:37:55.278164Z","iopub.execute_input":"2024-03-29T16:37:55.278549Z","iopub.status.idle":"2024-03-29T16:38:00.184486Z","shell.execute_reply.started":"2024-03-29T16:37:55.278519Z","shell.execute_reply":"2024-03-29T16:38:00.183127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SUBMISSION = True\nSUBMISSION = False\nDEBUG = False\n#DEBUG = True\n\nVALIDATION_FRAC = 0.05\nif SUBMISSION:\n    DEBUG = False\n    VALIDATION_FRAC = None\n    \nTRAIN_SIZE = 512 if DEBUG else None\nSKIP_ASSERT = SUBMISSION or not DEBUG","metadata":{"execution":{"iopub.status.busy":"2024-03-29T16:38:00.187165Z","iopub.execute_input":"2024-03-29T16:38:00.187820Z","iopub.status.idle":"2024-03-29T16:38:00.194015Z","shell.execute_reply.started":"2024-03-29T16:38:00.187790Z","shell.execute_reply":"2024-03-29T16:38:00.192719Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%run -i '/kaggle/input/hms-share/data-utils.py'\n%run -i '/kaggle/input/hms-share/eeg.py'\n%run -i '/kaggle/input/hms-share/spectr.py'\n%run -i '/kaggle/input/hms-share/eeg_id.py'\n%run -i '/kaggle/input/hms-share/target.py'\n%run -i '/kaggle/input/hms-share/data.py'","metadata":{"execution":{"iopub.status.busy":"2024-03-29T16:38:00.195254Z","iopub.execute_input":"2024-03-29T16:38:00.195563Z","iopub.status.idle":"2024-03-29T16:38:00.238381Z","shell.execute_reply.started":"2024-03-29T16:38:00.195537Z","shell.execute_reply":"2024-03-29T16:38:00.237226Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ADAM_LEARNING_RATE = 0.0001","metadata":{"execution":{"iopub.status.busy":"2024-03-29T16:38:00.241351Z","iopub.execute_input":"2024-03-29T16:38:00.241901Z","iopub.status.idle":"2024-03-29T16:38:00.247106Z","shell.execute_reply.started":"2024-03-29T16:38:00.241859Z","shell.execute_reply":"2024-03-29T16:38:00.245986Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"NUM_EPOCHS = 2\nNUM_SUB_EPOCHS = 5 if not DEBUG else 2\nBATCH_SIZE = 128","metadata":{"execution":{"iopub.status.busy":"2024-03-29T16:38:00.262883Z","iopub.execute_input":"2024-03-29T16:38:00.263391Z","iopub.status.idle":"2024-03-29T16:38:00.275205Z","shell.execute_reply.started":"2024-03-29T16:38:00.263352Z","shell.execute_reply":"2024-03-29T16:38:00.273888Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"USE_GPU = False","metadata":{"execution":{"iopub.status.busy":"2024-03-29T16:38:00.276844Z","iopub.execute_input":"2024-03-29T16:38:00.277275Z","iopub.status.idle":"2024-03-29T16:38:00.288820Z","shell.execute_reply.started":"2024-03-29T16:38:00.277244Z","shell.execute_reply":"2024-03-29T16:38:00.287724Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('SUBMISSION =', SUBMISSION)\nprint('USE_GPU =', USE_GPU)\nprint('DEBUG = ', DEBUG)","metadata":{"execution":{"iopub.status.busy":"2024-03-29T16:38:00.290369Z","iopub.execute_input":"2024-03-29T16:38:00.290732Z","iopub.status.idle":"2024-03-29T16:38:00.305085Z","shell.execute_reply.started":"2024-03-29T16:38:00.290703Z","shell.execute_reply":"2024-03-29T16:38:00.303978Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def size_2_str(value):\n    if value < 5 * 1024:\n        return str(value) + ' bytes'\n    if value < 5 * 1024 * 1024:\n        return str(value//1024) + ' KB'\n    return str(value//(1024*1024)) +' MB'\n\ndef get_mem_usage():\n    pid = os.getpid()\n    py = psutil.Process(pid)\n    return py.memory_info()[0] // 2 ** 20","metadata":{"execution":{"iopub.status.busy":"2024-03-29T16:38:00.307148Z","iopub.execute_input":"2024-03-29T16:38:00.307590Z","iopub.status.idle":"2024-03-29T16:38:00.317739Z","shell.execute_reply.started":"2024-03-29T16:38:00.307551Z","shell.execute_reply":"2024-03-29T16:38:00.316472Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def scale_probs(probs):\n    s = np.sum(probs, axis = -1, keepdims = True)\n    return probs/s","metadata":{"execution":{"iopub.status.busy":"2024-03-29T16:38:00.322326Z","iopub.execute_input":"2024-03-29T16:38:00.322706Z","iopub.status.idle":"2024-03-29T16:38:00.331365Z","shell.execute_reply.started":"2024-03-29T16:38:00.322668Z","shell.execute_reply":"2024-03-29T16:38:00.330376Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def assert_no_nan(x):\n    assert not np.isnan(x).any(), 'has NAN!!!'\n    return x","metadata":{"execution":{"iopub.status.busy":"2024-03-29T16:38:00.332804Z","iopub.execute_input":"2024-03-29T16:38:00.333421Z","iopub.status.idle":"2024-03-29T16:38:00.351408Z","shell.execute_reply.started":"2024-03-29T16:38:00.333389Z","shell.execute_reply":"2024-03-29T16:38:00.350207Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_decision(probs):\n    decision = np.zeros_like(probs)\n    decision[np.arange(len(probs)), probs.argmax(1)] = 1\n    return decision   ","metadata":{"execution":{"iopub.status.busy":"2024-03-29T16:38:00.353373Z","iopub.execute_input":"2024-03-29T16:38:00.354098Z","iopub.status.idle":"2024-03-29T16:38:00.364187Z","shell.execute_reply.started":"2024-03-29T16:38:00.354057Z","shell.execute_reply":"2024-03-29T16:38:00.363326Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#if USE_TPU:\n#    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n#    tf.tpu.experimental.initialize_tpu_system(tpu)\n#    tpu_strategy = tf.distribute.TPUStrategy(tpu)","metadata":{"execution":{"iopub.status.busy":"2024-03-29T16:38:00.366043Z","iopub.execute_input":"2024-03-29T16:38:00.366806Z","iopub.status.idle":"2024-03-29T16:38:00.377078Z","shell.execute_reply.started":"2024-03-29T16:38:00.366766Z","shell.execute_reply":"2024-03-29T16:38:00.376016Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_and_compile_model():\n    eeg_model =  tf.keras.models.load_model(\n        '/kaggle/input/hsm-eeg-model/eeg_model.keras'\n    )\n    spectr_model =  tf.keras.models.load_model(\n        '/kaggle/input/hsm-spectr-model/spectr_model.keras'\n    )\n    \n    eeg_model.trainable = False\n    spectr_model.trainable = False\n    model = tf.keras.layers.Concatenate(\n        name = 'merge.merge'\n    )([\n        eeg_model.layers[-2].output, \n        spectr_model.layers[-2].output\n    ])\n    model  = keras.layers.Flatten(name = 'merge.flatten')(model)\n    model  = keras.layers.Dense(\n        units = 23, activation='relu', name = 'merge.dense.1'\n    )(model)\n    model  = keras.layers.Dense(\n        units = 11, activation='relu', name = 'merge.dense.2'\n    )(model)\n    model  = keras.layers.Dense(\n        units = 11, activation='relu', name = 'merge.dense.3'\n    )(model)\n    model  = keras.layers.Dense(\n        units = len(Target.FEATURES), activation='softmax',name = 'merge.dense.output'\n    )(model)   \n    model = keras.models.Model(\n        inputs = [eeg_model.input, spectr_model.input],\n        outputs = model)\n    optimizer = Adam(learning_rate = ADAM_LEARNING_RATE)\n    model.compile(loss='categorical_crossentropy', optimizer= optimizer, metrics=['acc'])\n    return model","metadata":{"execution":{"iopub.status.busy":"2024-03-29T16:38:00.378354Z","iopub.execute_input":"2024-03-29T16:38:00.379399Z","iopub.status.idle":"2024-03-29T16:38:00.392160Z","shell.execute_reply.started":"2024-03-29T16:38:00.379349Z","shell.execute_reply":"2024-03-29T16:38:00.390989Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#if USE_TPU:\n#    with tpu_strategy.scope():\n#        model = create_and_compile_model()\n#\n#else:\n\nmodel = create_and_compile_model()\n    \nmodel.summary()\nkeras.utils.plot_model(model, 'model.png', show_shapes=True)","metadata":{"execution":{"iopub.status.busy":"2024-03-29T16:38:00.393506Z","iopub.execute_input":"2024-03-29T16:38:00.394639Z","iopub.status.idle":"2024-03-29T16:38:01.677762Z","shell.execute_reply.started":"2024-03-29T16:38:00.394598Z","shell.execute_reply":"2024-03-29T16:38:01.676670Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def build_train_data(\n    ids, \n    train_size, \n    eeg_loader, \n    spectr_loader, \n    target_loader\n):\n    eeg = EEG.create_model_input(ids, eeg_loader, BATCH_SIZE)\n    spectr = SPECTR.create_model_input(ids, train_spectr_loader, BATCH_SIZE)\n    target = Target.create_model_data(ids, target_loader, BATCH_SIZE)\n\n    features_seq = DataUtils.JoinSequence(eeg,spectr)\n    features_target_seq = DataUtils.JoinSequence(features_seq, target)\n    return DataUtils.AsKerasSequence(features_target_seq, lambda : random.shuffle(ids))\n    ","metadata":{"execution":{"iopub.status.busy":"2024-03-29T16:38:01.679312Z","iopub.execute_input":"2024-03-29T16:38:01.680587Z","iopub.status.idle":"2024-03-29T16:38:01.688439Z","shell.execute_reply.started":"2024-03-29T16:38:01.680546Z","shell.execute_reply":"2024-03-29T16:38:01.687324Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_size, train_eeg_loader, train_spectr_loader, train_target_loader = Data.load_train(TRAIN_SIZE)\n\ntrain_ids = list([x for x in range(train_size)])\nif VALIDATION_FRAC > 0:\n    train_ids, valid_ids = train_test_split(train_ids, test_size = VALIDATION_FRAC)\n    \ntrain_data = build_train_data(\n    train_ids, \n    train_size, \n    train_eeg_loader,\n    train_spectr_loader, \n    train_target_loader\n)\nif VALIDATION_FRAC > 0:\n    valid_data = build_train_data(\n        valid_ids, \n        train_size, \n        train_eeg_loader, \n        train_spectr_loader, \n        train_target_loader\n    ) \nelse:\n    valid_data = None    \ntrain_data = DataUtils.SplitSubEpoches(train_data, NUM_SUB_EPOCHS)","metadata":{"execution":{"iopub.status.busy":"2024-03-29T16:38:01.690134Z","iopub.execute_input":"2024-03-29T16:38:01.690785Z","iopub.status.idle":"2024-03-29T16:38:01.937711Z","shell.execute_reply.started":"2024-03-29T16:38:01.690747Z","shell.execute_reply":"2024-03-29T16:38:01.936143Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device_name = tf.test.gpu_device_name()\nif \"GPU\" not in device_name:\n    print(\"GPU device not found\")\nprint('Found GPU at: {}'.format(device_name))","metadata":{"execution":{"iopub.status.busy":"2024-03-29T16:38:01.939274Z","iopub.execute_input":"2024-03-29T16:38:01.939821Z","iopub.status.idle":"2024-03-29T16:38:01.947039Z","shell.execute_reply.started":"2024-03-29T16:38:01.939677Z","shell.execute_reply":"2024-03-29T16:38:01.945789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\ndef fit_model(model, train_data, epochs, validation_data):\n    if USE_GPU:\n        with tf.device('/gpu:0'):\n            return model.fit(\n                train_data, \n                epochs = epochs, \n                validation_data = validation_data)\n    else:\n        return model.fit(\n            train_data, \n            epochs = epochs, \n            validation_data = validation_data)\n        \nhistory = fit_model(\n    model,\n    train_data, \n    NUM_EPOCHS * NUM_SUB_EPOCHS, \n    valid_data\n)","metadata":{"execution":{"iopub.status.busy":"2024-03-29T16:38:01.948951Z","iopub.execute_input":"2024-03-29T16:38:01.949397Z","iopub.status.idle":"2024-03-29T16:41:35.851248Z","shell.execute_reply.started":"2024-03-29T16:38:01.949360Z","shell.execute_reply":"2024-03-29T16:41:35.849983Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.save('hms_model.keras')","metadata":{"execution":{"iopub.status.busy":"2024-03-29T16:41:35.852576Z","iopub.execute_input":"2024-03-29T16:41:35.852900Z","iopub.status.idle":"2024-03-29T16:41:36.007156Z","shell.execute_reply.started":"2024-03-29T16:41:35.852874Z","shell.execute_reply":"2024-03-29T16:41:36.006243Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(history.history['loss'], 'r', label='Training loss')\nif VALIDATION_FRAC > 0:\n    plt.plot(history.history['val_loss'], 'g', label='Validation loss')\nplt.title('Training VS Validation loss')\nplt.xlabel('No. of Epochs')\nplt.ylabel('Loss')\nplt.legend()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-03-29T16:41:36.008811Z","iopub.execute_input":"2024-03-29T16:41:36.010028Z","iopub.status.idle":"2024-03-29T16:41:36.313026Z","shell.execute_reply.started":"2024-03-29T16:41:36.009992Z","shell.execute_reply":"2024-03-29T16:41:36.311819Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(history.history['acc'], 'r', label='Training accuracy')\nif VALIDATION_FRAC > 0:\n    plt.plot(history.history['val_acc'], 'g', label='Validation accuracy')\nplt.title('Training Vs Validation Accuracy')\nplt.xlabel('No. of Epochs')\nplt.ylabel('Accuracy')\nplt.legend()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-03-29T16:41:36.314891Z","iopub.execute_input":"2024-03-29T16:41:36.315647Z","iopub.status.idle":"2024-03-29T16:41:36.595179Z","shell.execute_reply.started":"2024-03-29T16:41:36.315606Z","shell.execute_reply":"2024-03-29T16:41:36.593964Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('--------------- model fitted ----------------------')","metadata":{"execution":{"iopub.status.busy":"2024-03-29T16:41:36.597028Z","iopub.execute_input":"2024-03-29T16:41:36.597772Z","iopub.status.idle":"2024-03-29T16:41:36.604062Z","shell.execute_reply.started":"2024-03-29T16:41:36.597729Z","shell.execute_reply":"2024-03-29T16:41:36.602621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def check_validation():\n    y_true = []\n    y_predict = []\n    for index in range(len(valid_data)):\n        train_batch, target_batch = valid_data[index]\n        predict = model.predict(train_batch)\n        decision = make_decision(predict)\n        target_decision = make_decision(target_batch)\n        for i in range(len(decision)):\n            y_predict.append(decision[i])\n            y_true.append(target_decision[i])\n    print('accuracy =', accuracy_score(y_true, y_predict))","metadata":{"execution":{"iopub.status.busy":"2024-03-29T16:41:36.606053Z","iopub.execute_input":"2024-03-29T16:41:36.606839Z","iopub.status.idle":"2024-03-29T16:41:36.620502Z","shell.execute_reply.started":"2024-03-29T16:41:36.606794Z","shell.execute_reply":"2024-03-29T16:41:36.619395Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nif VALIDATION_FRAC > 0:\n    if USE_GPU:\n        with tf.device('/gpu:0'):\n            check_validation()\n    else:\n        check_validation()","metadata":{"execution":{"iopub.status.busy":"2024-03-29T16:41:36.622714Z","iopub.execute_input":"2024-03-29T16:41:36.623254Z","iopub.status.idle":"2024-03-29T16:41:38.481715Z","shell.execute_reply.started":"2024-03-29T16:41:36.623199Z","shell.execute_reply":"2024-03-29T16:41:38.480650Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('mem usage =', size_2_str(get_mem_usage()))\ndel train_eeg_loader,train_spectr_loader, train_target_loader","metadata":{"execution":{"iopub.status.busy":"2024-03-29T16:41:38.482864Z","iopub.execute_input":"2024-03-29T16:41:38.483191Z","iopub.status.idle":"2024-03-29T16:41:38.491005Z","shell.execute_reply.started":"2024-03-29T16:41:38.483165Z","shell.execute_reply":"2024-03-29T16:41:38.489645Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.save('hms_model.keras')","metadata":{"execution":{"iopub.status.busy":"2024-03-29T16:41:38.492366Z","iopub.execute_input":"2024-03-29T16:41:38.492673Z","iopub.status.idle":"2024-03-29T16:41:38.620296Z","shell.execute_reply.started":"2024-03-29T16:41:38.492648Z","shell.execute_reply":"2024-03-29T16:41:38.618888Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def build_test_data(test_size, eeg_loader, spectr_loader, eeg_id_loader):\n    indexes = list([x for x in range(test_size)])\n\n    eeg = EEG.create_model_input(indexes, eeg_loader, BATCH_SIZE)\n    spectr = SPECTR.create_model_input(indexes, spectr_loader, BATCH_SIZE)  \n    eeg_id = EEG_ID.create_model_data(indexes, eeg_id_loader, BATCH_SIZE)\n    \n    features_seq = DataUtils.JoinSequence(eeg, spectr)\n    features_eeg_id_seq = DataUtils.JoinSequence(features_seq, eeg_id)\n    return DataUtils.AsKerasSequence(features_eeg_id_seq)","metadata":{"execution":{"iopub.status.busy":"2024-03-29T16:41:38.625719Z","iopub.execute_input":"2024-03-29T16:41:38.626135Z","iopub.status.idle":"2024-03-29T16:41:38.634728Z","shell.execute_reply.started":"2024-03-29T16:41:38.626105Z","shell.execute_reply":"2024-03-29T16:41:38.633302Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_size, test_eeg_loader, test_spectr_loader, test_eeg_id_loader = Data.load_test()\ntest_data = build_test_data(test_size, test_eeg_loader, test_spectr_loader, test_eeg_id_loader)","metadata":{"execution":{"iopub.status.busy":"2024-03-29T16:41:38.636311Z","iopub.execute_input":"2024-03-29T16:41:38.636663Z","iopub.status.idle":"2024-03-29T16:41:38.650407Z","shell.execute_reply.started":"2024-03-29T16:41:38.636626Z","shell.execute_reply":"2024-03-29T16:41:38.649018Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\noutput = []\nfor index in range(len(test_data)):\n    batch, eeg_ids = test_data[index]\n    predict = model.predict(batch)\n    for i in range(len(eeg_ids)):\n        res = [*eeg_ids[i]]\n        res.extend(predict[i].tolist())\n        output.append(res)\noutput = pd.DataFrame(\n    data = output,\n    columns = ['eeg_id'] + Target.COLUMNS\n)\noutput.to_csv('submission.csv', index = False)\n","metadata":{"execution":{"iopub.status.busy":"2024-03-29T16:41:38.652022Z","iopub.execute_input":"2024-03-29T16:41:38.653637Z","iopub.status.idle":"2024-03-29T16:41:38.849139Z","shell.execute_reply.started":"2024-03-29T16:41:38.653590Z","shell.execute_reply":"2024-03-29T16:41:38.847864Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('--------------- submission done ----------------------')","metadata":{"execution":{"iopub.status.busy":"2024-03-29T16:41:38.850726Z","iopub.execute_input":"2024-03-29T16:41:38.851143Z","iopub.status.idle":"2024-03-29T16:41:38.857332Z","shell.execute_reply.started":"2024-03-29T16:41:38.851112Z","shell.execute_reply":"2024-03-29T16:41:38.856010Z"},"trusted":true},"execution_count":null,"outputs":[]}]}