{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":59093,"databundleVersionId":7469972,"sourceType":"competition"}],"dockerImageVersionId":30674,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-03-28T19:53:18.613853Z","iopub.execute_input":"2024-03-28T19:53:18.614585Z","iopub.status.idle":"2024-03-28T19:53:39.773771Z","shell.execute_reply.started":"2024-03-28T19:53:18.614556Z","shell.execute_reply":"2024-03-28T19:53:39.772749Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"import gc\nimport os\nimport random\nimport warnings\nimport numpy as np\nimport pandas as pd\nfrom IPython.display import display\n\n# PyTorch for deep learning\nimport timm\nimport torch\nimport torch.nn as nn  \nimport torch.optim as optim\nimport torch.nn.functional as F\nfrom torch.optim.lr_scheduler import CosineAnnealingLR\n\n# torchvision for image processing and augmentation\nimport torchvision.transforms as transforms\n\n# Suppressing minor warnings to keep the output clean\nwarnings.filterwarnings('ignore', category=Warning)\n\n# Reclaim memory no longer in use.\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-03-28T19:53:39.775466Z","iopub.execute_input":"2024-03-28T19:53:39.775772Z","iopub.status.idle":"2024-03-28T19:53:39.957329Z","shell.execute_reply.started":"2024-03-28T19:53:39.775747Z","shell.execute_reply":"2024-03-28T19:53:39.956441Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"code defines a configuration class (Config) and a function (set_seed) to set a seed for reproducibility across different libraries. It also defines a custom loss function (KL_loss) for Kullback-Leibler Divergence.\n\nThe Config class stores various parameters like seed, image transformation size, batch size, number of epochs, and number of folds.\nThe set_seed function ensures that random number generation in libraries like PyTorch, NumPy, and Python's built-in random module will produce the same results each time the code is run, facilitating reproducibility.\nThe KL_loss function calculates the Kullback-Leibler Divergence loss between two probability distributions p and q.\nFinally, the code calls gc.collect() to reclaim memory that is no longer in use.","metadata":{}},{"cell_type":"code","source":"class Config:\n    seed = 42 \n    image_transform = transforms.Resize((512,512))  \n    batch_size = 16\n    num_epochs = 9\n    num_folds = 5\n\n# Set the seed for reproducibility across multiple libraries\ndef set_seed(seed):\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n    torch.manual_seed(seed)\n    np.random.seed(seed)\n    random.seed(seed)\n    \nset_seed(Config.seed)\n\n# Define the 'Kullback Leibler Divergence' loss function\ndef KL_loss(p,q):\n    epsilon=10**(-15)\n    p=torch.clip(p,epsilon,1-epsilon)\n    q = nn.functional.log_softmax(q,dim=1)\n    return torch.mean(torch.sum(p*(torch.log(p)-q),dim=1))\n\n# Reclaim memory no longer in use.\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-03-28T19:53:39.958503Z","iopub.execute_input":"2024-03-28T19:53:39.958861Z","iopub.status.idle":"2024-03-28T19:53:40.132792Z","shell.execute_reply.started":"2024-03-28T19:53:39.958828Z","shell.execute_reply":"2024-03-28T19:53:40.131661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"Define Labels: Initializes a list of labels for classification, including 'seizure', 'lpd', 'gpd', 'lrda', 'grda', and 'other'.\n\nFeature Aggregation: Iterates through each label, groups the data by 'spectrogram_id', and sums the votes for each label.\n\nMerge DataFrames: Merges the aggregated data for each label into a single DataFrame called train_feats.\n\nCalculate Total Votes: Adds a column to train_feats to calculate the total number of votes across all labels.\n\nNormalize Votes: Calculates and stores the normalized vote for each label by dividing the vote count for each label by the total votes.\n\nSelect Relevant Columns: Selects relevant columns, including 'spectrogram_id' and the normalized vote columns for each label.\n\nAdd Spectrogram File Paths: Adds a column with the file path to the spectrogram files based on the 'spectrogram_id'.\n\nMemory Reclamation: Calls gc.collect() to reclaim memory that is no longer in use.","metadata":{}},{"cell_type":"code","source":"train_df = pd.read_csv(\"/kaggle/input/hms-harmful-brain-activity-classification/train.csv\")\n\n# Define labels for classification\nlabels = ['seizure', 'lpd', 'gpd', 'lrda', 'grda', 'other']\n\n# Initialize an empty DataFrame for storing features\ntrain_feats = pd.DataFrame()\n\n# Aggregate votes for each label and merge into train_feats DataFrame\nfor label in labels:\n    # Group by 'spectrogram_id' and sum the votes for the current label\n    group = train_df[f'{label}_vote'].groupby(train_df['spectrogram_id']).sum()\n\n    # Create a DataFrame from the grouped data\n    label_vote_sum = pd.DataFrame({'spectrogram_id': group.index, f'{label}_vote_sum': group.values})\n\n    # Initialize train_feats with the first label or merge subsequent labels\n    if label == 'seizure':\n        train_feats = label_vote_sum\n    else:\n        train_feats = train_feats.merge(label_vote_sum, on='spectrogram_id', how='left')\n\n# Add a column to sum all votes\ntrain_feats['total_vote'] = 0\nfor label in labels:\n    train_feats['total_vote'] += train_feats[f'{label}_vote_sum']\n\n# Calculate and store the normalized vote for each label\nfor label in labels:\n    train_feats[f'{label}_vote'] = train_feats[f'{label}_vote_sum'] / train_feats['total_vote']\n\n# Select relevant columns for the training features\nchoose_cols = ['spectrogram_id']\nfor label in labels:\n    choose_cols += [f'{label}_vote']\ntrain_feats = train_feats[choose_cols]\n\n# Add a column with the path to the spectrogram files\ntrain_feats['path'] = train_feats['spectrogram_id'].apply(lambda x: \"/kaggle/input/hms-harmful-brain-activity-classification/train_spectrograms/\" + str(x) + \".parquet\")\n\n# Reclaim memory no longer in use.\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-03-28T19:53:40.134804Z","iopub.execute_input":"2024-03-28T19:53:40.135517Z","iopub.status.idle":"2024-03-28T19:53:40.621450Z","shell.execute_reply.started":"2024-03-28T19:53:40.135484Z","shell.execute_reply":"2024-03-28T19:53:40.620579Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"function, get_batch, is designed to process a batch of spectrogram data from parquet files. Here's a breakdown of what it does:\n\nSet Epsilon: Defines a very small value (epsilon) to avoid division by zero when normalizing data.\n\nInitialize Batch Data List: Creates an empty list to store processed batch data.\n\nIterate Over Paths: Loops through each path provided in the paths argument.\n\nRead Parquet File: Reads data from the parquet file specified by the current path.\n\nData Preprocessing:\n\nFills missing values in the data with -1.\nRemoves the time column and transposes the data.\nClips data values to a range between exp(-6) and exp(10).\nApplies a logarithmic transformation to the data.\nNormalizes the data by subtracting the mean and dividing by the standard deviation along each axis.\nConvert to PyTorch Tensor: Converts the processed data to a PyTorch tensor and adds an extra dimension using unsqueeze.\n\nApply Image Transformations: Applies the image transformation defined in Config.image_transform to the data.\n\nAppend Processed Data: Adds the processed data to the batch_data list.\n\nStack Batch Data: Stacks all the processed data tensors in the batch_data list into a single tensor using torch.stack.\n\nReturn Batch Data: Returns the processed batch data as a PyTorch tensor.","metadata":{}},{"cell_type":"code","source":"def get_batch(paths, batch_size=Config.batch_size):\n    # Set a small epsilon to avoid division by zero\n    eps = 1e-6\n\n    # Initialize a list to store batch data\n    batch_data = []\n\n    # Iterate over each path in the provided paths\n    for path in paths:\n        # Read data from parquet file\n        data = pd.read_parquet(path[0])\n\n        # Fill missing values, remove time column, and transpose\n        data = data.fillna(-1).values[:, 1:].T\n\n        # Clip values and apply logarithmic transformation\n        data = np.clip(data, np.exp(-6), np.exp(10))\n        data = np.log(data)\n\n        # Normalize the data\n        data_mean = data.mean(axis=(0, 1))\n        data_std = data.std(axis=(0, 1))\n        data = (data - data_mean) / (data_std + eps)\n\n        # Convert data to a PyTorch tensor and apply transformations\n        data_tensor = torch.unsqueeze(torch.Tensor(data), dim=0)\n        data = Config.image_transform(data_tensor)\n\n        # Append the processed data to the batch_data list\n        batch_data.append(data)\n\n    # Stack all the batch data into a single tensor\n    batch_data = torch.stack(batch_data)\n\n    # Return the batch data\n    return batch_data","metadata":{"execution":{"iopub.status.busy":"2024-03-28T19:53:40.622648Z","iopub.execute_input":"2024-03-28T19:53:40.622923Z","iopub.status.idle":"2024-03-28T19:53:40.630623Z","shell.execute_reply.started":"2024-03-28T19:53:40.622893Z","shell.execute_reply":"2024-03-28T19:53:40.629839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Check Device Availability: Checks whether CUDA (GPU) is available and assigns the device accordingly.\n\nCross-validation Loop:\n\nSplits the data into train and test sets for each fold.\nInitializes the ResNet18d model with pretrained weights and moves it to the selected device.\nSets up the optimizer (AdamW) and scheduler (CosineAnnealingLR) for learning rate adjustment.\nTraining Loop:\n\nIterates over the specified number of epochs.\nRandomizes the order of training indices for each epoch.\nDivides the training data into batches using the specified batch size.\nComputes predictions using the model, calculates the loss using the KL divergence loss function (KL_loss), and performs backpropagation to update model parameters.\nPrints and records the training loss for each epoch.\nAdjusts the learning rate using the scheduler.\nEvaluation Loop:\n\nSwitches the model to evaluation mode (no gradient calculation).\nDivides the test data into batches.\nComputes predictions using the model and calculates the test loss.\nPrints and records the test loss for each epoch.\nModel Saving:\n\nSaves the model's state dictionary if it achieves the best test loss so far.\nMemory Reclamation: Calls gc.collect() to reclaim memory no longer in use.\n\nFold-wise Reporting:\n\nPrints the best test loss achieved for each fold.","metadata":{}},{"cell_type":"code","source":"# Determine device availability\ndevice = 'cuda' if torch.cuda.is_available() else 'cpu'\nprint(f\"Using device: {device}\")\n\n# Assuming train_feats is defined and contains the training features and labels\ntotal_idx = np.arange(len(train_feats))\nnp.random.shuffle(total_idx)\n\ngc.collect()\n\n# Cross-validation loop\nfor fold in range(Config.num_folds):\n    # Split data into train and test sets for this fold\n    test_idx = total_idx[fold * len(total_idx) // Config.num_folds:(fold + 1) * len(total_idx) // Config.num_folds]\n    train_idx = np.array([idx for idx in total_idx if idx not in test_idx])\n\n    # Initialize ResNet34d model with pretrained weights\n    model = timm.create_model('resnet18d', pretrained=True, num_classes=6, in_chans=1)\n    model.to(device)\n\n    optimizer = optim.AdamW(model.parameters(), lr=0.001, betas=(0.5, 0.999), weight_decay=0.01)\n    scheduler = CosineAnnealingLR(optimizer, T_max=Config.num_epochs)\n\n    best_test_loss = float('inf')\n    train_losses = []\n    test_losses = []\n\n    print(f\"Starting training for fold {fold + 1}\")\n\n    # Training loop\n    for epoch in range(Config.num_epochs):\n        model.train()\n        train_loss = []\n        random_num = np.arange(len(train_idx))\n        np.random.shuffle(random_num)\n        train_idx = train_idx[random_num]\n\n        # Iterate over batches in the training set\n        for idx in range(0, len(train_idx), Config.batch_size):\n            optimizer.zero_grad()\n            train_idx1 = train_idx[idx:idx + Config.batch_size]\n            train_X1_path = train_feats[['path']].iloc[train_idx1].values\n            train_X1 = get_batch(train_X1_path, batch_size=Config.batch_size)\n            train_y1 = train_feats[['seizure_vote', 'lpd_vote', 'gpd_vote', 'lrda_vote', 'grda_vote', 'other_vote']].iloc[train_idx1].values\n            train_y1 = torch.Tensor(train_y1)\n\n            train_pred = model(train_X1.to(device))\n            loss = KL_loss(train_y1.to(device), train_pred)\n            loss.backward()\n            optimizer.step()\n            train_loss.append(loss.item())\n\n        epoch_train_loss = np.mean(train_loss)\n        train_losses.append(epoch_train_loss)\n        print(f\"Epoch {epoch + 1}: Train Loss = {epoch_train_loss:.2f}\")\n\n        scheduler.step()\n\n        # Evaluation loop\n        model.eval()\n        test_loss = []\n        with torch.no_grad():\n            for idx in range(0, len(test_idx), Config.batch_size):\n                test_idx1 = test_idx[idx:idx + Config.batch_size]\n                test_X1_path = train_feats[['path']].iloc[test_idx1].values\n                test_X1 = get_batch(test_X1_path, batch_size=Config.batch_size)\n                test_y1 = train_feats[['seizure_vote', 'lpd_vote', 'gpd_vote', 'lrda_vote', 'grda_vote', 'other_vote']].iloc[test_idx1].values\n                test_y1 = torch.Tensor(test_y1)\n\n                test_pred = model(test_X1.to(device))\n                loss = KL_loss(test_y1.to(device), test_pred)\n                test_loss.append(loss.item())\n\n        epoch_test_loss = np.mean(test_loss)\n        test_losses.append(epoch_test_loss)\n        print(f\"Epoch {epoch + 1}: Test Loss = {epoch_test_loss:.2f}\")\n\n        # Save the model if it has the best test loss so far\n        if epoch_test_loss < best_test_loss:\n            best_test_loss = epoch_test_loss\n            torch.save(model.state_dict(), f\"resnet34d_fold{fold}.pth\")\n\n        gc.collect()\n\n    print(f\"Fold {fold + 1} Best Test Loss: {best_test_loss:.2f}\")","metadata":{"execution":{"iopub.status.busy":"2024-03-28T19:53:40.632000Z","iopub.execute_input":"2024-03-28T19:53:40.632329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}