{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":59093,"databundleVersionId":7469972,"sourceType":"competition"},{"sourceId":7679150,"sourceType":"datasetVersion","datasetId":4479990},{"sourceId":7679170,"sourceType":"datasetVersion","datasetId":4479998},{"sourceId":7679242,"sourceType":"datasetVersion","datasetId":4480059},{"sourceId":7679288,"sourceType":"datasetVersion","datasetId":4480089},{"sourceId":7679679,"sourceType":"datasetVersion","datasetId":4480381},{"sourceId":165866387,"sourceType":"kernelVersion"}],"dockerImageVersionId":30664,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install mne","metadata":{"execution":{"iopub.status.busy":"2024-03-14T18:52:10.614145Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\n\ncurrent_directory = os.getcwd()\nprint(\"Current Working Directory:\", current_directory)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BASE_PATH = '/kaggle/input/sparcnet/SPaRCNet'","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport re\nfrom mne.filter import filter_data, notch_filter\nimport time\nimport numpy as np\nimport random\nimport re\nimport pandas as pd\nfrom collections import OrderedDict\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom collections import Counter\nimport os\nimport albumentations as alb\nfrom torch.utils.data import Dataset, DataLoader\n\nprint (\"\")\nprint (\"lib finish\")\nprint (\"\")\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Configuration class","metadata":{}},{"cell_type":"code","source":"class Config:\n\n use_aug = False\n num_classes = 6\n batch_size = 88\n epochs = 20\n PRECISION = 16 \n PATIENCE = 20 \n seed = 2024\n pretrained = False \n weight_decay = 1e-2\n use_mixup = False\n mixup_alpha = 0.1 \n num_channels = 8\n data_root = \"/kaggle/input/hms-harmful-brain-activity-classification/\"\n raw_eeg_path = None#\"/home/nischay/brain/Data/raw_eeg/eegs.npy\" #\"/home/nischay/brain/Data/eegs_20ch.npy\"#\n \n LR = 8e-3\n processed_train = None\n output_dir = '/kaggle/working/sparcnet/SPaRCNet/Data/loader'\n trn_folds = [0,1,2,3,4]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not os.path.exists(Config.output_dir):\n os.makedirs(Config.output_dir)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(BASE_PATH)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"## Load training csv","metadata":{}},{"cell_type":"code","source":"import pandas as pd, numpy as np, os\nimport matplotlib.pyplot as plt\n\ndf = pd.read_csv(f'{Config.data_root}train.csv')\nprint( df.shape )","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Load train data","metadata":{}},{"cell_type":"code","source":"EEG_IDS = df.eeg_id.unique()\n\nTARGETS = df.columns[-6:]\nTARS = {'Seizure':0, 'LPD':1, 'GPD':2, 'LRDA':3, 'GRDA':4, 'Other':5}\nTARS_INV = {x:y for y,x in TARS.items()}\n\ntrain = df.groupby('eeg_id')[['patient_id']].agg('first')\n\ntmp = df.groupby('eeg_id')[TARGETS].agg('sum')\nfor t in TARGETS:\n train[t] = tmp[t].values\n \ny_data = train[TARGETS].values\ny_data = y_data / y_data.sum(axis=1,keepdims=True)\ntrain[TARGETS] = y_data\n\ntmp = df.groupby('eeg_id')[['expert_consensus']].agg('first')\ntrain['target'] = tmp\n\ntrain = train.reset_index()\ntrain = train.loc[train.eeg_id.isin(EEG_IDS)]\nprint('Train Data with unique eeg_id shape:', train.shape )","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Config.num_classes = len(TARS.keys())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.describe()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## EEG from Parquet functions","metadata":{}},{"cell_type":"code","source":"'''def eeg_from_parquet(parquet_path, display=False):\n \n # EXTRACT MIDDLE 50 SECONDS\n eeg = pd.read_parquet(parquet_path, columns=FEATS)\n rows = len(eeg)\n offset = (rows-10_000)//2\n eeg = eeg.iloc[offset:offset+10_000]\n\n if display: \n plt.figure(figsize=(10,5))\n offset = 0\n\n # CONVERT TO NUMPY\n data = np.zeros((10_000,len(FEATS)))\n for j,col in enumerate(FEATS):\n\n # FILL NAN\n x = eeg[col].values.astype('float32')\n m = np.nanmean(x)\n if np.isnan(x).mean()<1: x = np.nan_to_num(x,nan=m)\n else: x[:] = 0\n\n data[:,j] = x\n\n if display: \n if j!=0: offset += x.max()\n plt.plot(range(10_000),x-offset,label=col)\n offset -= x.min()\n\n if display:\n plt.legend()\n name = parquet_path.split('/')[-1]\n name = name.split('.')[0]\n plt.title(f'EEG {name}',size=16)\n plt.show()\n\n return data'''","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''%%time\n\nCREATE_EEGS = True\n\ndf = pd.read_parquet(f'{Config.data_root}train_eegs/1000913311.parquet')\nFEATS = df.columns\nprint(f'There are {len(FEATS)} raw eeg features')\nprint( list(FEATS) )\n\nif Config.raw_eeg_path is not None:\n raw_eegs = np.load(Config.raw_eeg_path, allow_pickle=True).item()\nelse:\n\n all_eegs = {}\n DISPLAY = 4\n EEG_IDS = train.eeg_id.unique()\n PATH = f'{Config.data_root}train_eegs/'\n \n for i,eeg_id in enumerate(EEG_IDS):\n if (i%100==0)&(i!=0): print(i,', ',end='') \n \n # SAVE EEG TO PYTHON DICTIONARY OF NUMPY ARRAYS\n data = eeg_from_parquet(f'{PATH}{eeg_id}.parquet', display=i 0:\n\t\t\tnew_features = F.dropout(new_features, p=self.drop_rate, training=self.training)\n\t\treturn torch.cat([x, new_features], 1)\n\n\nclass _DenseBlock(nn.Sequential):\n\tdef __init__(self, num_layers, num_input_features, bn_size, growth_rate, drop_rate, conv_bias, batch_norm):\n\t\tsuper(_DenseBlock, self).__init__()\n\t\tfor i in range(num_layers):\n\t\t\tlayer = _DenseLayer(num_input_features + i * growth_rate, growth_rate, bn_size, drop_rate, conv_bias, batch_norm)\n\t\t\tself.add_module('denselayer%d' % (i + 1), layer)\n\n\nclass _Transition(nn.Sequential):\n\tdef __init__(self, num_input_features, num_output_features, conv_bias, batch_norm):\n\t\tsuper(_Transition, self).__init__()\n\t\tif batch_norm:\n\t\t\tself.add_module('norm', nn.BatchNorm1d(num_input_features))\n\t\t# self.add_module('relu', nn.ReLU())\n\t\tself.add_module('elu', nn.ELU())\n\t\tself.add_module('conv', nn.Conv1d(num_input_features, num_output_features, kernel_size=1, stride=1, bias=conv_bias))\n\t\tself.add_module('pool', nn.AvgPool1d(kernel_size=2, stride=2))\n\n\nclass DenseNetEnconder(nn.Module):\n\tdef __init__(self, growth_rate=32, block_config=(4, 4, 4, 4, 4, 4, 4), #block_config=(6, 12, 24, 48, 24, 20, 16), #block_config=(6, 12, 24, 16),\n\t\t\t\t in_channels=16, num_init_features=64, bn_size=4, drop_rate=0.2, conv_bias=True, batch_norm=False):\n\n\t\tsuper(DenseNetEnconder, self).__init__()\n\n\t\t# First convolution\n\t\tfirst_conv = OrderedDict([('conv0', nn.Conv1d(in_channels, num_init_features, kernel_size=7, stride=2, padding=3, bias=conv_bias))])\n\t\t# first_conv = OrderedDict([('conv0', nn.Conv1d(in_channels, num_init_features, groups=in_channels, kernel_size=7, stride=2, padding=3, bias=conv_bias))])\n\t\t# first_conv = OrderedDict([('conv0', nn.Conv1d(in_channels, num_init_features, kernel_size=15, stride=2, padding=7, bias=conv_bias))])\n\n\t\t# first_conv = OrderedDict([\n\t\t# \t('conv0-depth', nn.Conv1d(in_channels, 32, groups=in_channels, kernel_size=7, stride=2, padding=3, bias=conv_bias)),\n\t\t# \t('conv0-point', nn.Conv1d(32, num_init_features, kernel_size=1, stride=1, bias=conv_bias)),\n\t\t# ])\n\n\t\tif batch_norm:\n\t\t\tfirst_conv['norm0'] = nn.BatchNorm1d(num_init_features)\n\t\t# first_conv['relu0'] = nn.ReLU()\n\t\tfirst_conv['elu0'] = nn.ELU()\n\t\tfirst_conv['pool0'] = nn.MaxPool1d(kernel_size=3, stride=2, padding=1)\n\n\t\tself.densenet = nn.Sequential(first_conv)\n\n\t\tnum_features = num_init_features\n\t\tfor i, num_layers in enumerate(block_config):\n\t\t\tblock = _DenseBlock(num_layers=num_layers, num_input_features=num_features,\n\t\t\t\t\t\t\t\tbn_size=bn_size, growth_rate=growth_rate, drop_rate=drop_rate, conv_bias=conv_bias, batch_norm=batch_norm)\n\t\t\tself.densenet.add_module('denseblock%d' % (i + 1), block)\n\t\t\tnum_features = num_features + num_layers * growth_rate\n\t\t\tif i != len(block_config) - 1:\n\t\t\t\ttrans = _Transition(num_input_features=num_features, num_output_features=num_features // 2, conv_bias=conv_bias, batch_norm=batch_norm)\n\t\t\t\tself.densenet.add_module('transition%d' % (i + 1), trans)\n\t\t\t\tnum_features = num_features // 2\n\n\t\t# Final batch norm\n\t\tif batch_norm:\n\t\t\tself.densenet.add_module('norm{}'.format(len(block_config) + 1), nn.BatchNorm1d(num_features))\n\t\t# self.features.add_module('norm5', BatchReNorm1d(num_features))\n\n\t\tself.densenet.add_module('relu{}'.format(len(block_config) + 1), nn.ReLU())\n\t\tself.densenet.add_module('pool{}'.format(len(block_config) + 1), nn.AvgPool1d(kernel_size=7, stride=3)) # stride originally 1\n\n\t\tself.num_features = num_features\n\n\t\t# Official init from torch repo.\n\t\tfor m in self.modules():\n\t\t\tif isinstance(m, nn.Conv1d):\n\t\t\t\tnn.init.kaiming_normal_(m.weight.data)\n\t\t\telif isinstance(m, nn.BatchNorm1d):\n\t\t\t\tm.weight.data.fill_(1)\n\t\t\t\tm.bias.data.zero_()\n\t\t\telif isinstance(m, nn.Linear):\n\t\t\t\tm.bias.data.zero_()\n\n\tdef forward(self, x):\n\t\tfeatures = self.densenet(x)\n\t\t# print(\"Final Output\")\n\t\t# print(features.size())\n\t\treturn features.view(features.size(0), -1)\n\n\nclass DenseNetClassifier(nn.Module):\n\t# def __init__(self, growth_rate=16, block_config=(3, 6, 12, 8), #block_config=(6, 12, 24, 48, 24, 20, 16), #block_config=(6, 12, 24, 16),\n\t# \t\t\t in_channels=16, num_init_features=32, bn_size=2, drop_rate=0, conv_bias=False, drop_fc=0.5, num_classes=6):\n\tdef __init__(self, growth_rate=32, block_config=(4, 4, 4, 4, 4, 4, 4),\n\t\t\t\t in_channels=16, num_init_features=64, bn_size=4, drop_rate=0.2, conv_bias=True, batch_norm=False, drop_fc=0.5, num_classes=6):\n\n\t\tsuper(DenseNetClassifier, self).__init__()\n\n\t\tself.features = DenseNetEnconder(growth_rate=growth_rate, block_config=block_config, in_channels=in_channels,\n\t\t\t\t\t\t\t\t\t\t num_init_features=num_init_features, bn_size=bn_size, drop_rate=drop_rate,\n\t\t\t\t\t\t\t\t\t\t conv_bias=conv_bias, batch_norm=batch_norm)\n\n\t\t# Linear layer\n\t\tself.classifier = nn.Sequential(\n\t\t\tnn.Dropout(p=drop_fc),\n\t\t\tnn.Linear(self.features.num_features, num_classes)\n\t\t)\n\n\t\t# Official init from torch repo.\n\t\tfor m in self.modules():\n\t\t\tif isinstance(m, nn.Conv1d):\n\t\t\t\tnn.init.kaiming_normal_(m.weight.data)\n\t\t\telif isinstance(m, nn.BatchNorm1d):\n\t\t\t\tm.weight.data.fill_(1)\n\t\t\t\tm.bias.data.zero_()\n\t\t\telif isinstance(m, nn.Linear):\n\t\t\t\tm.bias.data.zero_()\n\n\tdef forward(self, x):\n\t\tfeatures = self.features(x)\n\t\tout = self.classifier(features)\n\t\treturn out, features\n\n \ndevice = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n\nprint (\"device: \", device)\nprint (\"\")\n\nDATA_RAW_PATH = os.path.join(BASE_PATH, 'Data', 'Raw/') \nmodel_cnn = torch.load('/kaggle/input/iiic-sparcnet/SPaRCNet/model_1130.pt', map_location=torch.device('cpu'))\nmodel_cnn.train()\n\nprint('so far so good!')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_cnn.to(device)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DATA_RAW_PATH = os.path.join('/kaggle/input/hms-harmful-brain-activity-classification/train_eegs/')\nDATA_RAW_PATH = os.path.join('/kaggle/input/hms-harmful-brain-activity-classification/test_eegs/')\nprint(DATA_RAW_PATH)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(BASE_PATH)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(BASE_PATH + \"/model_1130.pt\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DATA_PATH = '/kaggle/input/sparcnet/SPaRCNet/Data'\nprint(DATA_PATH)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\n\n# Define the directory path\noutput_directory = '/kaggle/working/sparcnet/SPaRCNet/Data/iiic'\n\n# Create the directory if it doesn't exist\nos.makedirs(output_directory, exist_ok=True)\n\"\"\"\n# Define the file path\nfile_path = os.path.join(output_directory, 'sample_cEEG_score.csv')\n\n# Create the file\nwith open(file_path, 'w'):\n pass # This will create an empty file-\n\nprint(\"File created successfully at:\", file_path)\n\"\"\"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"PATH = '/kaggle/input/hms-harmful-brain-activity-classification/train_eegs/'\ntrain = pd.read_csv('/kaggle/input/hms-harmful-brain-activity-classification/train.csv')\nEEG_IDS = train.eeg_id.unique()\ndf = pd.read_parquet(f'{PATH}{EEG_IDS[1]}.parquet')\nFEATS = df.columns\n\ndef eeg_from_parquet(parquet_path, display=False):\n \n # EXTRACT MIDDLE 50 SECONDS\n eeg = pd.read_parquet(parquet_path, columns=FEATS)\n rows = len(eeg)\n offset = (rows-10_000)//2\n eeg = eeg.iloc[offset:offset+10_000]\n eeg = eeg.iloc[4000:6000]\n if display: \n plt.figure(figsize=(10,5))\n offset = 0\n\n # CONVERT TO NUMPY\n #data = np.zeros((10_000,len(FEATS)))\n data = np.zeros((2_000,len(FEATS)))\n for j,col in enumerate(FEATS):\n\n # FILL NAN\n x = eeg[col].values.astype('float32')\n m = np.nanmean(x)\n if np.isnan(x).mean()<1: x = np.nan_to_num(x,nan=m)\n else: x[:] = 0\n\n data[:,j] = x\n\n if display: \n if j!=0: offset += x.max()\n #plt.plot(range(10_000),x-offset,label=col)\n plt.plot(range(2_000),x-offset,label=col)\n offset -= x.min()\n\n if display:\n plt.legend()\n name = parquet_path.split('/')[-1]\n name = name.split('.')[0]\n plt.title(f'EEG {name}',size=16)\n plt.show()\n\n return data.T","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Train model\n","metadata":{}},{"cell_type":"code","source":"'''class EEGDataset(torch.utils.data.Dataset):\n\n def __init__(self, data, eegs=None, augmentations = None, test = False): \n\n self.data = data\n self.eegs = eegs\n self.augmentations = augmentations\n self.test = test\n \n def __len__(self):\n return len(self.data)\n \n def __getitem__(self, index):\n\n row = self.data.iloc[index] \n data = self.eegs[row.eeg_id]\n\n data = np.clip(data,-1024,1024)\n data = np.nan_to_num(data, nan=0) / 32.0\n \n data = butter_lowpass_filter(data)\n data = quantize_data(data,1)\n\n samples = torch.from_numpy(data).float()\n \n samples,_ = self.augmentations(samples.unsqueeze(0), None)\n samples = samples.squeeze()\n\n samples = samples.permute(1,0)\n if not self.test:\n label = row[TARGETS] \n label = torch.tensor(label).float() \n return samples, label\n else:\n return samples'''\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''import torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import TensorDataset, DataLoader\nimport numpy as np\n\n# Assuming your data is in numpy arrays\nX_train = np.array(...) # Your training data\ny_train = np.array(...) # Your training labels\n\n# Convert numpy arrays to PyTorch tensors\nX_train_tensor = torch.tensor(X_train, dtype=torch.float32)\ny_train_tensor = torch.tensor(y_train, dtype=torch.long)\n\n# Create a dataset and dataloader\ntrain_dataset = TensorDataset(X_train_tensor, y_train_tensor)\ntrain_loader = DataLoader(train_dataset, batch_size=64, shuffle=True)\n\n# Load the pretrained model\n# Ensure DenseNetClassifier class is defined in your script\nmodel = DenseNetClassifier(...)\nmodel.load_state_dict(torch.load('path_to_model_1130.pt'))\nmodel.to(device)\n\n# Loss function and optimizer\ncriterion = nn.CrossEntropyLoss()\noptimizer = optim.Adam(model.parameters(), lr=0.001)\n\n# Training loop\nnum_epochs = 10\nfor epoch in range(num_epochs):\n model.train()\n for inputs, labels in train_loader:\n inputs, labels = inputs.to(device), labels.to(device)\n\n # Forward pass\n outputs, _ = model(inputs)\n loss = criterion(outputs, labels)\n\n # Backward and optimize\n optimizer.zero_grad()\n loss.backward()\n optimizer.step()\n\n print(f'Epoch {epoch+1}/{num_epochs}, Loss: {loss.item()}')\n\n# Save the retrained model\ntorch.save(model.state_dict(), 'retrained_model.pt')'''\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_predictions(DATA_RAW_PATH):\n \n ################################################################################################################################## \n\n print (\"\")\n print (\"$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$\")\n print (\"read data\")\n print (\"$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$\")\n print (\"\") \n\n ######################################################################################\n\n\n\n # file_name_array = np.loadtxt(\"file_path_24h.txt\", delimiter = ',', dtype = np.str)\n\n total_file_list = os.listdir(DATA_RAW_PATH)\n all_predictions = []\n\n print (\"len(total_file_list):\" , len(total_file_list))\n print (\"\")\n\n T = len(total_file_list)\n\n print (\"T: \", T)\n print (\"\")\n\n\n for t in range(T):\n# for t in range(min(5, len(total_file_list))):\n\n print (\"***************************************\")\n print (\"t: \", t)\n print (\"\")\n\n\n file_name = total_file_list[t]\n save_name = file_name.rstrip(\".parquet\")\n\n if os.path.isfile(DATA_PATH+\"/iiic/\"+ save_name + \"_score.csv\"):\n print('--alr done ' + save_name)\n\n else:\n print (\"file_name: \", file_name)\n print (\"\")\n print (\"save_name: \", save_name)\n print (\"\")\n\n path1 = DATA_RAW_PATH + file_name\n print (\"path1: \", path1)\n print (\"\")\n\n X = eeg_from_parquet(f'{PATH}{EEG_IDS[t]}.parquet')\n print (\"X.shape: \", X.shape)\n print (\"\")\n \n print (\"************ Montage\")\n print (\"\")\n\n X2 = X[[0,4,5,6, 11,15,16,17, 0,1,2,3, 11,12,13,14]] - X[[4,5,6,7, 15,16,17,18, 1,2,3,7, 12,13,14,18]]\n print (\"X2.shape: \", X2.shape)\n\n print (\"************ filtering\")\n print (\"\")\n\n X2 = notch_filter(X2, 200, 60, n_jobs=-1, verbose='ERROR')\n X2 = filter_data(X2, 200, 0.5, 40, n_jobs=-1, verbose='ERROR') \n\n print (\"X2.shape: \", X2.shape)\n N = 1 #int(X2.shape[1]/400)\n\n print (\"N: \", N)\n print (\"\")\n\n print (\"************ reshaping\")\n print (\"\")\n\n #X3 = np.zeros((N-5,16,2000))\n X3 = np.zeros((N,16,2000))\n\n for n in range(N):\n #start_sn = n*400\n start_sn = n\n end_sn = start_sn + 2000\n x = X2[:,start_sn:end_sn]\n X3[n,:,:] = x\n\n print (\"X3.shape: \", X3.shape)\n\n X = X3\n\n print (\"X.shape: \", X.shape)\n print (\"\")\n print (\"np.isnan(X).sum(): \", np.isnan(X).sum())\n print (\"np.max(X): \", np.max(X))\n print (\"np.min(X): \", np.min(X))\n print (\"\")\n\n X = np.where(X<=500, X, 500)\n X = np.where(X>=-500, X, -500)\n\n print (\"X.shape: \", X.shape)\n print (\"\")\n print (\"np.isnan(X).sum(): \", np.isnan(X).sum())\n print (\"np.max(X): \", np.max(X))\n print (\"np.min(X): \", np.min(X))\n print (\"\")\n\n X4 = X\n\n del X\n del X2\n del X3\n\n print (\"X4.shape: \", X4.shape)\n print (\"\")\n\n\n ######################### evaluation\n batch_size = 1000\n def get_unlabeled_batch_list(X_train,batch_size):\n N = X_train.shape[0]\n sn_list = list(range(N))\n K = int(N/batch_size)\n X_list = list()\n end_sn = 0\n\n for k in range(K):\n start_sn = k*batch_size\n end_sn = start_sn + batch_size\n\n X = X_train[start_sn:end_sn,:,:]\n X_list.append(X) \n if not end_sn == N:\n X = X_train[end_sn:N,:,:]\n X_list.append(X) \n\n return (X_list)\n\n ################### scanning\n model_cnn.eval() #*\n # print (\"unlabeled losses.avg: \", losses.avg)\n\n (X_batch_list) = get_unlabeled_batch_list(X4,batch_size)\n K = len(X_batch_list)\n\n print (\"K: \", K)\n print (\"\")\n\n S_list = list() \n V_list = list()\n\n for k in range(K):\n if k%100 == 0:\n print (k)\n\n X = X_batch_list[k]\n\n #print (\"X.shape: \", X.shape)\n #print (\"Y.shape: \", Y.shape)\n\n X = torch.from_numpy(X).float()\n X = X.to(device)\n\n output, v = model_cnn(X)\n\n S_list.append(output.detach().to('cpu'))\n V_list.append(v.detach().to('cpu'))\n\n del X\n del output\n del v\n\n\n #print (\"unlabeled losses.avg: \", losses.avg)\n #print (\"\")\n\n S2 = torch.cat(S_list,dim=0)\n prob = F.softmax(S2, 1)\n unlabeled_score = prob.numpy()\n averaged_unlabeled_score = np.mean(unlabeled_score, axis=0)\n all_predictions.append(averaged_unlabeled_score)\n \n\n print (\"\")\n print (\"average_unlabeled_score.shape: \", averaged_unlabeled_score.shape)\n print (\"\")\n\n V2 = torch.cat(V_list,dim=0)\n unlabeled_V = V2.numpy()\n\n print (\"unlabeled_V.shape: \", unlabeled_V.shape)\n print (\"\")\n\n# path3 = output_directory + save_name + \"_score.csv\"\n# np.savetxt(path3, unlabeled_score, delimiter=',')\n\n #path4 = \"./Data/iiic/\"+ save_name + \"_vector.csv\"\n #np.savetxt(path4, unlabeled_V, delimiter=',')\n \n \n\n print (\"writing finish\")\n print (\"\")\n\n del X4\n del X_batch_list\n \n all_predictions_df = pd.DataFrame(np.vstack(all_predictions))\n #all_predictions_df = pd.DataFrame(all_predictions)\n\n\n # Save the DataFrame to a single CSV file\n output_path = '/kaggle/working/sparcnet/combined_predictions.csv'\n all_predictions_csv = all_predictions_df.to_csv(output_path, index=False)\n\n print(\"All predictions saved to:\", output_path)\n\n print (\"\")\n print (\"Done!\")\n print (\"\")\n return all_predictions_df\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# make_predictions(DATA_RAW_PATH)\npredictions = make_predictions('/kaggle/input/hms-harmful-brain-activity-classification/test_eegs')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#pd.read_csv(\"/kaggle/working/sparcnet/combined_predictions.csv\")\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"raw","source":"","metadata":{}},{"cell_type":"code","source":"#print(predictions\ndf = pd.read_csv('/kaggle/input/hms-harmful-brain-activity-classification/train.csv')\nTARGETS = df.columns[-6:]\nprint(TARGETS)\n\ntest = pd.read_csv('/kaggle/input/hms-harmful-brain-activity-classification/test.csv')\n\nsub = pd.DataFrame({'eeg_id':test.eeg_id.values})\nsub[TARGETS] = predictions\nsub.to_csv('submission.csv',index=False)\nprint('Submissionn shape',sub.shape)\nsub.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\neeg_id\tseizure_vote\tlpd_vote\tgpd_vote\tlrda_vote\tgrda_vote\tother_vote\n0\t3911565283\t0.334435\t0.476884\t0.08896\t0.009496\t0.063273\t0.026953","metadata":{}}]}