{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":59093,"databundleVersionId":7469972,"sourceType":"competition"},{"sourceId":163569236,"sourceType":"kernelVersion"}],"dockerImageVersionId":30664,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os,sys\nimport shutil\nimport torch\n\nCODE_DIR = '/kaggle/working/code'\nos.makedirs(CODE_DIR, exist_ok=True)\n\nCODE_DIR_SRC = '/kaggle/input/datapreprocessing/code'\n# List all files in the source directory\nfiles = os.listdir(CODE_DIR_SRC)\n# Iterate over each file and copy it to the destination directory\nfor file in files:\n    # Construct full file paths\n    source_file = os.path.join(CODE_DIR_SRC, file)\n    if source_file.endswith('__pycache__'):\n        continue\n    destination_file = os.path.join(CODE_DIR, file)  \n    # Copy the file\n    shutil.copy(source_file, destination_file)\n\nsys.path.append(CODE_DIR)\n\nMODEL_DIR = '/kaggle/working/models'\nos.makedirs(MODEL_DIR, exist_ok=True)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\nTRAIN_FILE = '/kaggle/input/datapreprocessing/preprocessed/train.csv'\nSPC_DIR = '/kaggle/input/datapreprocessing/preprocessed/spectrogram'\n\nTEST_SPC_DIR = '/kaggle/input/hms-harmful-brain-activity-classification/test_spectrograms'\nTEST_PROCESSED_SPC_DIR = \"/kaggle/working/preprocessed/spectrogram\"\nos.makedirs(TEST_PROCESSED_SPC_DIR, exist_ok=True)\n\nTRAIN_EEG_DIR = '/kaggle/input/hms-harmful-brain-activity-classification/train_eegs'\nTRAIN_SPC_DIR = '/kaggle/input/hms-harmful-brain-activity-classification/train_spectrograms'\n\nTEST_FILE = '/kaggle/input/hms-harmful-brain-activity-classification/test.csv'\nSUBMISSION_FILE = '/kaggle/input/hms-harmful-brain-activity-classification/sample_submission.csv'\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-03-10T11:12:47.036378Z","iopub.execute_input":"2024-03-10T11:12:47.036735Z","iopub.status.idle":"2024-03-10T11:12:49.166747Z","shell.execute_reply.started":"2024-03-10T11:12:47.036690Z","shell.execute_reply":"2024-03-10T11:12:49.165900Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Hyperparameters\nBATCH_SIZE = 50\nHIDDEN_SIZE = 100\nLEARNING_RATE = 0.001\nNUM_EPOCHS = 300\nNUM_SPLITS = 10","metadata":{"execution":{"iopub.status.busy":"2024-03-10T11:12:49.168295Z","iopub.execute_input":"2024-03-10T11:12:49.168688Z","iopub.status.idle":"2024-03-10T11:12:49.173066Z","shell.execute_reply.started":"2024-03-10T11:12:49.168662Z","shell.execute_reply":"2024-03-10T11:12:49.172159Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\n\ndata = pd.read_csv(TRAIN_FILE)\ndel data['Unnamed: 0']\ndata.head()","metadata":{"execution":{"iopub.status.busy":"2024-03-10T11:12:49.174103Z","iopub.execute_input":"2024-03-10T11:12:49.174364Z","iopub.status.idle":"2024-03-10T11:12:49.573449Z","shell.execute_reply.started":"2024-03-10T11:12:49.174341Z","shell.execute_reply":"2024-03-10T11:12:49.572459Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%writefile {CODE_DIR}/spec_dataset.py\n\nimport os\nimport numpy as np\nimport torch\nfrom torch.utils.data import Dataset, DataLoader\n\n# Define custom dataset\nclass SpectrogramDataset(Dataset):\n    def __init__(self, train_df, SPC_DIR):\n        self.train = train_df  \n        self.SPC_DIR = SPC_DIR\n\n    def __len__(self):\n        return len(self.train)\n\n    def __getitem__(self, idx):\n        spc_file = f\"{self.train.spectrogram_id[idx]}.pt\"\n        spc_path = os.path.join(self.SPC_DIR, spc_file)\n        spc = torch.load(spc_path)\n        \n        spc_start = self.train.spec_start_idx[idx]\n        spc_end = self.train.spec_end_idx[idx]\n        spc = spc[spc_start:spc_end]\n        \n        labels = ['seizure_vote','lpd_vote','gpd_vote','lrda_vote','grda_vote',\t'other_vote']\n        y = self.train.loc[idx,labels].astype('float').values\n        y = y/np.sum(y)\n        return spc, y","metadata":{"execution":{"iopub.status.busy":"2024-03-10T11:12:49.575517Z","iopub.execute_input":"2024-03-10T11:12:49.575826Z","iopub.status.idle":"2024-03-10T11:12:49.582824Z","shell.execute_reply.started":"2024-03-10T11:12:49.575801Z","shell.execute_reply":"2024-03-10T11:12:49.581781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%writefile {CODE_DIR}/spec_test_dataset.py\n\nimport os\nimport numpy as np\nimport torch\nfrom torch.utils.data import Dataset, DataLoader\n\n# Define custom dataset\nclass SpectrogramTestDataset(Dataset):\n    def __init__(self, test_df, SPC_DIR):\n        self.test = test_df  \n        self.SPC_DIR = SPC_DIR\n\n    def __len__(self):\n        return len(self.test)\n\n    def __getitem__(self, idx):\n        spc_file = f\"{self.test.spectrogram_id[idx]}.pt\"\n        spc_path = os.path.join(self.SPC_DIR, spc_file)\n        spc = torch.load(spc_path)\n        return spc","metadata":{"execution":{"iopub.status.busy":"2024-03-10T11:12:49.584017Z","iopub.execute_input":"2024-03-10T11:12:49.584291Z","iopub.status.idle":"2024-03-10T11:12:49.594346Z","shell.execute_reply.started":"2024-03-10T11:12:49.584269Z","shell.execute_reply":"2024-03-10T11:12:49.593428Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%writefile {CODE_DIR}/spec_dataset_builder.py\n\nimport pandas as pd\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.preprocessing import OrdinalEncoder\nfrom tqdm.auto import tqdm\nfrom torch.utils.data import DataLoader\n\nfrom spec_dataset import SpectrogramDataset\nfrom spec_test_dataset import SpectrogramTestDataset\n\nSPC_DIR = '/kaggle/input/datapreprocessing/preprocessed/spectrogram'\n\nclass DataSetBuilder:\n    def __init__(self, \n                 data, \n                 target_column= 'expert_consensus', \n                 n_splits=10, \n                 random_state=42, \n                 batch_size=32, \n                 SPC_DIR=SPC_DIR,\n                ):\n        self.data = data\n        self.target_column = target_column\n        self.n_splits = n_splits\n        self.random_state = random_state\n        self.batch_size = batch_size\n        self.SPC_DIR = SPC_DIR\n        self.encoder = OrdinalEncoder()\n\n    def stratified_split(self, data):\n        y = self.encoder.fit_transform(data[[self.target_column]])\n        X = data.drop(columns=[self.target_column])\n        skf = StratifiedKFold(n_splits=self.n_splits, shuffle=True, random_state=self.random_state)\n        stratified_splits = []\n        for train_index, _ in skf.split(X, y):\n            train_data = self.data.iloc[train_index].copy().reset_index(drop=True)\n            stratified_splits.append(train_data)\n        return stratified_splits\n    \n    def make_datasets(self, data):\n        data_splits = self.stratified_split(data)\n        dataset_list = [SpectrogramDataset(df, self.SPC_DIR) for df in data_splits]\n        return dataset_list\n    \n    def make_test_dataloader(test_data, TEST_SPC_DIR):\n        dataset = SpectrogramTestDataset(test_data, TEST_SPC_DIR)\n        dataloader = DataLoader(dataset, batch_size=1, pin_memory=True) \n        return dataloader\n        \n    def make_dataloaders(self):\n        k_datasets = self.make_datasets(self.data)\n        dataloaders = [DataLoader(dataset, batch_size=self.batch_size, shuffle=True, pin_memory=True) for dataset in k_datasets]\n        return dataloaders","metadata":{"execution":{"iopub.status.busy":"2024-03-10T11:12:49.595785Z","iopub.execute_input":"2024-03-10T11:12:49.596361Z","iopub.status.idle":"2024-03-10T11:12:49.606321Z","shell.execute_reply.started":"2024-03-10T11:12:49.596329Z","shell.execute_reply":"2024-03-10T11:12:49.605477Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from spec_dataset_builder import DataSetBuilder\n\nspec_datamaker = DataSetBuilder(\n    data = data,\n    target_column = 'expert_consensus',\n    n_splits = NUM_SPLITS,\n    batch_size=BATCH_SIZE,\n    SPC_DIR = SPC_DIR\n)\n\ndata_loaders = spec_datamaker.make_dataloaders()","metadata":{"execution":{"iopub.status.busy":"2024-03-10T11:12:49.607618Z","iopub.execute_input":"2024-03-10T11:12:49.608125Z","iopub.status.idle":"2024-03-10T11:12:50.501594Z","shell.execute_reply.started":"2024-03-10T11:12:49.608100Z","shell.execute_reply":"2024-03-10T11:12:50.500768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from preprocess_specs import SpectrogramDatasetBuilder\n\ntest_data = pd.read_csv(TEST_FILE)\nspectrogram_builder = SpectrogramDatasetBuilder()\nspectrogram_builder.buildAll(TEST_SPC_DIR)\n\n#subm_spec_file = f\"{TEST_PROCESSED_SPC_DIR}/{test_df['spectrogram_id'][0]}.pt\"\n\nsub_dataloader = DataSetBuilder.make_test_dataloader(test_data, \n                                                     TEST_SPC_DIR = TEST_PROCESSED_SPC_DIR)","metadata":{"execution":{"iopub.status.busy":"2024-03-10T11:12:50.502814Z","iopub.execute_input":"2024-03-10T11:12:50.503296Z","iopub.status.idle":"2024-03-10T11:12:52.020184Z","shell.execute_reply.started":"2024-03-10T11:12:50.503260Z","shell.execute_reply":"2024-03-10T11:12:52.019258Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# %%writefile {CODE_DIR}/model.py\n\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nimport math\n\n#MODEL PARAMETERS\nINPUT_SIZE = 400\nOUTPUT_SIZE = 6\n\nclass TransformerEncoderModel(nn.Module):\n    def __init__(self,batch_size,\n                 input_dim = INPUT_SIZE,\n                 num_layers =4, \n                 num_heads = 8, \n                 dropout = 0.1, \n                 max_seq_length=300, \n                 output_dim = 6):\n        super(TransformerEncoderModel, self).__init__()\n        \n        self.input_dim = input_dim\n        self.batch_size = batch_size\n        self.max_seq_length = max_seq_length\n        \n        self.pos_encoding = PositionalEncoding(input_dim, dropout, max_seq_length)\n        self.encoder_layer = nn.TransformerEncoderLayer(input_dim, num_heads, dropout=dropout, batch_first=False, norm_first = True)\n        self.transformer_encoder = nn.TransformerEncoder(self.encoder_layer, num_layers)\n        self.fc = nn.Linear(input_dim, output_dim)\n        self.softmax = nn.Softmax(dim=1)\n\n    def forward(self, src):\n        src = src.reshape([self.batch_size, self.max_seq_length, self.input_dim])\n        src = src.permute(1,0,2)\n        embedded_src = src * math.sqrt(self.input_dim)\n        embedded_src = self.pos_encoding(embedded_src)\n        src_mask = self.generate_square_subsequent_mask().to(embedded_src.device)\n        encoded_src = self.transformer_encoder(embedded_src, src_key_padding_mask=src_mask)\n        output = self.fc(encoded_src[-1])\n        output = self.softmax(output)\n        return output\n    \n    def generate_square_subsequent_mask(self):\n        mask = (torch.triu(torch.ones(self.max_seq_length, self.batch_size)) == 1).transpose(0, 1)\n        mask = mask.float().masked_fill(mask == 0, float('-inf')).masked_fill(mask == 1, float(0.0))\n        return mask\n\nclass PositionalEncoding(nn.Module):\n    def __init__(self, d_model, dropout=0.1, max_len=5000):\n        super(PositionalEncoding, self).__init__()\n        self.dropout = nn.Dropout(p=dropout)\n        \n        pe = torch.zeros(max_len, d_model)\n        position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1)\n        div_term = torch.exp(torch.arange(0, d_model, 2).float() * (-math.log(10000.0) / d_model))\n        pe[:, 0::2] = torch.sin(position * div_term)\n        pe[:, 1::2] = torch.cos(position * div_term)\n        pe = pe.unsqueeze(0).transpose(0, 1)\n        self.register_buffer('pe', pe)\n    \n    def forward(self, x):\n        x = x + self.pe[:x.size(0), :]\n        return self.dropout(x)","metadata":{"execution":{"iopub.status.busy":"2024-03-10T11:12:52.021689Z","iopub.execute_input":"2024-03-10T11:12:52.022130Z","iopub.status.idle":"2024-03-10T11:12:52.037443Z","shell.execute_reply.started":"2024-03-10T11:12:52.022104Z","shell.execute_reply":"2024-03-10T11:12:52.036584Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#from model import TransformerEncoderModel\n\n# Example usage\ninput_dim = 400  # Size of the vocabulary\nnum_layers = 4    # Number of encoder layers\nnum_heads = 8     # Number of attention heads\ndropout = 0.1     # Dropout rate\nmax_seq_length = 300\noutput_dim = 6\n\nmodel = TransformerEncoderModel(BATCH_SIZE,\n                                input_dim,  \n                                num_layers, \n                                num_heads, \n                                dropout,max_seq_length,\n                               output_dim)\nmodel = model.to(device)","metadata":{"execution":{"iopub.status.busy":"2024-03-10T11:12:52.040444Z","iopub.execute_input":"2024-03-10T11:12:52.040816Z","iopub.status.idle":"2024-03-10T11:12:52.210935Z","shell.execute_reply.started":"2024-03-10T11:12:52.040781Z","shell.execute_reply":"2024-03-10T11:12:52.209905Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"i = 0\nfor X,y in data_loaders[0]:\n    X = X.to(device).long()\n    y = model(X)\n    print(y.shape)\n    \n    i+=1\n    if i == 1:\n        break\n    \ny","metadata":{"execution":{"iopub.status.busy":"2024-03-10T11:12:52.212024Z","iopub.execute_input":"2024-03-10T11:12:52.212318Z","iopub.status.idle":"2024-03-10T11:12:52.714314Z","shell.execute_reply.started":"2024-03-10T11:12:52.212292Z","shell.execute_reply":"2024-03-10T11:12:52.713405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nfrom tqdm.auto import tqdm\n\ndef predict(model, pred_loader):\n    preds = []\n    \n    for X in pred_loader:\n        X = X.to(device).long()\n        print(X.shape)\n        pred = model(X)[0]\n        preds.append(pred)\n        \n    return preds\n\ndef get_accuracy(model, test_loader):\n    score = 0\n    model.eval()\n    i = 0\n    for testX, testY in tqdm(test_loader):\n        if not list(testX.shape) == [BATCH_SIZE, 300, 100, 4]:\n            continue\n        X = testX.to(device).long()\n        Y = testY.to(device)\n        \n        preds = model(X)\n        _, pred_labels = torch.max(preds, dim=1)\n        _, target_labels = torch.max(Y, dim=1)\n        score += torch.sum(pred_labels == target_labels)\n\n\n    return score/len(test_loader.dataset)","metadata":{"execution":{"iopub.status.busy":"2024-03-10T11:13:08.033912Z","iopub.execute_input":"2024-03-10T11:13:08.034290Z","iopub.status.idle":"2024-03-10T11:13:08.043021Z","shell.execute_reply.started":"2024-03-10T11:13:08.034260Z","shell.execute_reply":"2024-03-10T11:13:08.041933Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_loader = data_loaders[9]","metadata":{"execution":{"iopub.status.busy":"2024-03-10T11:13:11.511360Z","iopub.execute_input":"2024-03-10T11:13:11.512090Z","iopub.status.idle":"2024-03-10T11:13:11.516316Z","shell.execute_reply.started":"2024-03-10T11:13:11.512059Z","shell.execute_reply":"2024-03-10T11:13:11.515310Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom tqdm.auto import tqdm\nfrom torch.optim.lr_scheduler import StepLR\n\nmodel = model.to(device)\n\n# Define optimizer\noptimizer = optim.Adam(model.parameters(), lr=LEARNING_RATE)\nscheduler = StepLR(optimizer, step_size=5, gamma=0.1)\n\n# Define loss function\ncriterion = nn.KLDivLoss(reduction='batchmean')\n\n# Training loop\ndef train_model(model, train_loader, optimizer, criterion):\n    model.train()\n    running_loss = 0.0\n    for inputs, targets in tqdm(train_loader):\n        if not list(inputs.shape) == [BATCH_SIZE, 300, 100, 4]:\n            continue\n        optimizer.zero_grad()\n        inputs = inputs.to(device).long()\n        targets = targets.to(device)\n        outputs = model(inputs)\n        loss = criterion(outputs, targets)\n        loss.backward()\n        optimizer.step()\n        running_loss += loss.item() * inputs.size(0)\n        \n    return running_loss / len(train_loader.dataset)\n\n# Validation loop\ndef validate_model(model, valid_loader, criterion):\n    model.eval()\n    running_loss = 0.0\n    with torch.no_grad():\n        for inputs, targets in tqdm(valid_loader):\n            if not list(inputs.shape) == [BATCH_SIZE, 300, 100, 4]:\n                continue\n            inputs = inputs.to(device).long()\n            outputs = model(inputs)\n            loss = criterion(outputs, targets)\n            running_loss += loss.item() * inputs.size(0)\n    return running_loss / len(valid_loader.dataset)\n\naccuracy = get_accuracy(model, test_loader)\nprint(f\"base accuracy: {accuracy}\")\n\n# Assuming you have train_loader and valid_loader DataLoader objects for training and validation data\nfor epoch in tqdm(range(NUM_EPOCHS)):\n    \n    loader_idx = epoch%10\n    train_loader = data_loaders[loader_idx]\n    \n    train_loss = train_model(model, train_loader, optimizer, criterion)\n    print(f\"Epoch {epoch+1}/{NUM_EPOCHS}, Train Loss: {train_loss}\")","metadata":{"execution":{"iopub.status.busy":"2024-03-10T11:13:13.458665Z","iopub.execute_input":"2024-03-10T11:13:13.459052Z","iopub.status.idle":"2024-03-10T11:13:20.950025Z","shell.execute_reply.started":"2024-03-10T11:13:13.459022Z","shell.execute_reply":"2024-03-10T11:13:20.948737Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"accuracy = get_accuracy(model, test_loader)\nprint(f\"Accuracy after {NUM_EPOCHS} epochs: {accuracy}\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.save(model.state_dict(), 'transformer_v1.pth')","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}