{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":59093,"databundleVersionId":7469972,"sourceType":"competition"},{"sourceId":7976031,"sourceType":"datasetVersion","datasetId":4617907},{"sourceId":167498142,"sourceType":"kernelVersion"}],"dockerImageVersionId":30646,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import sys\nsys.path.append('/kaggle/input/bottleneck-package')\nimport bottleneck\n\nimport pandas as pd\nimport numpy as np\nimport os\nimport psutil  \nimport random\n\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nfrom tensorflow.keras.optimizers import Adam\n\nfrom pathlib import Path\nimport shutil\nfrom tqdm import tqdm\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import accuracy_score\nimport matplotlib.pyplot as plt\nfrom functools import lru_cache\nfrom pympler import asizeof\n\nimport typing\nfrom dataclasses import dataclass\nfrom collections.abc import Callable\nfrom collections.abc import Sequence\n\nimport itertools\nfrom tqdm import tqdm","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SUBMISSION = True\nSUBMISSION = False\nDEBUG = False\n#DEBUG = True\n\nVALIDATION_FRAC = 0.05\nif SUBMISSION:\n    DEBUG = False\n    VALIDATION_FRAC = None\n    \nTRAIN_SIZE = 512 if DEBUG else None\nSKIP_ASSERT = SUBMISSION or not DEBUG","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"USE_GPU = False \nUSE_GPU = True","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ADAM_LEARNING_RATE = 0.0001","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%run -i '/kaggle/input/hms-share/data-utils.py'\n%run -i '/kaggle/input/hms-share/eeg.py'\n%run -i '/kaggle/input/hms-share/spectr.py'\n%run -i '/kaggle/input/hms-share/target.py'\n%run -i '/kaggle/input/hms-share/eeg_id.py'\n%run -i '/kaggle/input/hms-share/data.py'","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"NUM_EPOCHS = 150\nNUM_SUB_EPOCHS = 5 if not DEBUG else 2\nBATCH_SIZE = 128","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('SUBMISSION =', SUBMISSION)\nprint('USE_GPU =', USE_GPU)\nprint('DEBUG = ', DEBUG)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def size_2_str(value):\n    if value < 5 * 1024:\n        return str(value) + ' bytes'\n    if value < 5 * 1024 * 1024:\n        return str(value//1024) + ' KB'\n    return str(value//(1024*1024)) +' MB'\n\ndef get_mem_usage():\n    pid = os.getpid()\n    py = psutil.Process(pid)\n    return py.memory_info()[0] // 2 ** 20","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_eeg_model():\n    input = keras.layers.Input(\n        shape = (EEG.MODEL_WINDOW, len(EEG.FEATURES)), name = 'eeg.input'\n    )\n    model = keras.layers.Conv1D(\n        filters = 29, kernel_size = 5, padding = 'valid', \n        name = 'eeg.1', data_format=\"channels_last\", \n        activation = 'tanh'\n    )(input)\n    model = keras.layers.MaxPooling1D(\n        pool_size = 7, strides = 3, name = 'eeg.1.max'\n    )(model)\n    model = keras.layers.Conv1D(\n        filters = 29, kernel_size = 5, padding = 'valid', \n        name = 'eeg.2', data_format=\"channels_last\", \n        activation = 'tanh'\n    )(model)\n    model = keras.layers.MaxPooling1D(\n        pool_size = 7, strides = 3, name = 'eeg.2.max'\n    )(model)\n    model = keras.layers.Conv1D(\n        filters = 31, kernel_size = 5, padding = 'valid', name = 'eeg.3',\n        activation = 'tanh'\n    )(model)\n    model = keras.layers.MaxPooling1D(\n        pool_size = 7, strides = 3, name = 'eeg.3.max'\n    )(model)\n    model = keras.layers.Conv1D(\n        filters = 41, kernel_size = 5, padding = 'valid', name = 'eeg.4',\n        activation = 'tanh'\n    )(model)\n    model = keras.layers.MaxPooling1D(\n        pool_size = 7, strides = 3, name = 'eeg.4.max'\n    )(model)\n    model = keras.layers.Flatten(name = 'eeg.flatten')(model)\n    model = keras.layers.Dropout(name = 'eeg.dropout.1', rate = 0.1)(model)\n    model = keras.layers.Dense(\n        units = 11, activation='relu', name = 'eeg.dense.1', \n    )(model)\n    model = keras.layers.Dropout(name = 'eeg.dropout.2', rate = 0.1)(model)\n    model = keras.layers.Dense(\n        units = 11, activation='relu', name = 'eeg.dense.2', \n    )(model)\n    model = keras.layers.Dense(\n        units = len(Target.FEATURES), activation='softmax',name = 'eeg.output'\n    )(model)\n    model = keras.models.Model(inputs=input, outputs=model)\n    return model    ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#if USE_TPU:\n#    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n#    tf.tpu.experimental.initialize_tpu_system(tpu)\n#    tpu_strategy = tf.distribute.TPUStrategy(tpu)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_and_compile_model():\n    model = create_eeg_model()\n    optimizer = Adam(learning_rate = ADAM_LEARNING_RATE)\n    model.compile(loss='categorical_crossentropy', optimizer= optimizer, metrics=['acc'])\n    return model","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#if USE_TPU:\n#    with tpu_strategy.scope():\n#        model = create_and_compile_model()\n#\n#else:\n\nmodel = create_and_compile_model()\nmodel.summary()\nkeras.utils.plot_model(model, 'model.png', show_shapes=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def build_train_data(\n    ids, \n    eeg_loader, \n    target_loader\n):\n    eeg = EEG.create_model_input(ids, eeg_loader, BATCH_SIZE)\n    target = Target.create_model_data(ids, target_loader, BATCH_SIZE)\n    \n    eeg_target = DataUtils.JoinSequence(eeg, target)\n    return DataUtils.AsKerasSequence(eeg_target, lambda : random.shuffle(ids))\n    ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class EEGTrainLoader :\n    CACHE_FILE = 'eeg_cache.npy'\n    def __init__(self, train_info):\n        self.data = pd.DataFrame(\n            {\n                c : DataUtils.decrease_int_type(train_info[c]) \n                for c in ['eeg_id','eeg_label_offset_seconds']\n            }\n        )\n        try:\n            self.cache = np.load(EEGTrainLoader.CACHE_FILE, 'r')\n            if not len(self.cache) == len(self.data):\n                raise ValueError('cache size invalid!')\n        except Exception:\n            self.cache = self.build_cache()\n            print('Cache size =', size_2_str(asizeof.asizeof(self.cache)))\n            np.save(EEGTrainLoader.CACHE_FILE, self.cache)\n            self.cache = np.load(EEGTrainLoader.CACHE_FILE, 'r')\n        print('Cache loaded, shape =', self.cache.shape)\n                  \n    def build_model_data(self, index):\n        eeg_id, start = self.data.iloc[index] \n        start = start * EEG.FRAME_PER_SECOND\n        end = start + EEG.FRAME\n        data = EEG.load_train_frame(eeg_id)\n        if not SKIP_ASSERT:\n            assert start >=0 and start <= len(data), 'inlvalid start = {}, len = {}'.format(start, len(data))\n            assert end <= len(data) and end >=0, 'invalid end = {}, len = {}'.format(end, len(data))\n        data = EEG.filter_signals(data[EEG.FEATURES].iloc[start:end].to_numpy())\n        return data.astype(dtype = np.float32)        \n    \n    def build_cache(self):\n        return np.fromiter(\n            [\n                self.build_model_data(index) for index in tqdm(\n                    range(len(self.data)),\n                    desc = 'Build eeg cache'\n                )\n            ],\n            dtype = (np.float32, (EEG.MODEL_WINDOW, len(EEG.FEATURES))),\n            count = len(self.data)\n        )\n\n    @lru_cache(maxsize = None)\n    def __call__(self, index):\n        return self.cache[index]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_size, train_eeg_loader, _, train_target_loader \\\n    = Data.load_train(TRAIN_SIZE)\n\ntrain_info = Data.load_train_info(TRAIN_SIZE)\ntrain_eeg_loader = EEGTrainLoader(train_info)\n\ntrain_ids = list([x for x in range(train_size)])\nif VALIDATION_FRAC > 0:\n    train_ids, valid_ids = train_test_split(train_ids, test_size = VALIDATION_FRAC)\n    \ntrain_data = build_train_data(train_ids, train_eeg_loader, train_target_loader)\nif VALIDATION_FRAC > 0:\n    valid_data = build_train_data(valid_ids, train_eeg_loader, train_target_loader) \nelse:\n    valid_data = None    \ntrain_data = DataUtils.SplitSubEpoches(train_data, NUM_SUB_EPOCHS)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device_name = tf.test.gpu_device_name()\nif \"GPU\" not in device_name:\n    print(\"GPU device not found\")\nprint('Found GPU at: {}'.format(device_name))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\ndef fit_model(model, train_data, epochs, validation_data):\n    if USE_GPU:\n        with tf.device('/gpu:0'):\n            return model.fit(\n                train_data, \n                epochs = epochs, \n                validation_data = validation_data)\n    else:\n        return model.fit(\n            train_data, \n            epochs = epochs, \n            validation_data = validation_data)\n        \nhistory = fit_model(\n    model,\n    train_data, \n    NUM_EPOCHS * NUM_SUB_EPOCHS, \n    valid_data\n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(history.history['loss'], 'r', label='Training loss')\nif VALIDATION_FRAC > 0:\n    plt.plot(history.history['val_loss'], 'g', label='Validation loss')\nplt.title('Training VS Validation loss')\nplt.xlabel('No. of Epochs')\nplt.ylabel('Loss')\nplt.legend()\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(history.history['acc'], 'r', label='Training accuracy')\nif VALIDATION_FRAC > 0:\n    plt.plot(history.history['val_acc'], 'g', label='Validation accuracy')\nplt.title('Training Vs Validation Accuracy')\nplt.xlabel('No. of Epochs')\nplt.ylabel('Accuracy')\nplt.legend()\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('--------------- model fitted ----------------------')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def check_validation():\n    y_true = []\n    y_predict = []\n    for index in range(len(valid_data)):\n        train_batch, target_batch = valid_data[index]\n        predict = model.predict(train_batch)\n        decision = Target.make_decision(predict)\n        target_decision = Target.make_decision(target_batch)\n        for i in range(len(decision)):\n            y_predict.append(decision[i])\n            y_true.append(target_decision[i])\n    print('accuracy =', accuracy_score(y_true, y_predict))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nif VALIDATION_FRAC > 0:\n    if USE_GPU:\n        with tf.device('/gpu:0'):\n            check_validation()\n    else:\n        check_validation()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('mem usage =', size_2_str(get_mem_usage()))\ndel train_eeg_loader,train_target_loader","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def build_test_data(ids, eeg_loader, eeg_id_loader):\n    eeg = EEG.create_model_input(ids, eeg_loader, BATCH_SIZE)\n    eeg_id = EEG_ID.create_model_data(ids, eeg_id_loader, BATCH_SIZE)\n    \n    eeg_eeg_id_seq = DataUtils.JoinSequence(eeg, eeg_id)\n    return DataUtils.AsKerasSequence(eeg_eeg_id_seq)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_size, test_eeg_loader, _, test_eeg_id_loader = Data.load_test()\ntest_data = build_test_data(\n    list([x for x in range(test_size)]), \n    test_eeg_loader, \n    test_eeg_id_loader\n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\noutput = []\nfor index in range(len(test_data)):\n    batch, eeg_ids = test_data[index]\n    predict = model.predict(batch)\n    for i in range(len(eeg_ids)):\n        res = [*eeg_ids[i]]\n        res.extend(predict[i].tolist())\n        output.append(res)\noutput = pd.DataFrame(\n    data = output,\n    columns = ['eeg_id'] + Target.FEATURES\n)\noutput.to_csv('submission.csv', index = False)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.save('eeg_model.keras')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('--------------- submission done ----------------------')","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}