{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":59093,"databundleVersionId":7469972,"sourceType":"competition"},{"sourceId":6259291,"sourceType":"datasetVersion","datasetId":3597559},{"sourceId":7867347,"sourceType":"datasetVersion","datasetId":4615838},{"sourceId":7979752,"sourceType":"datasetVersion","datasetId":4696459},{"sourceId":7979770,"sourceType":"datasetVersion","datasetId":4696472},{"sourceId":7979791,"sourceType":"datasetVersion","datasetId":4696485},{"sourceId":8105078,"sourceType":"datasetVersion","datasetId":4701635},{"sourceId":171463847,"sourceType":"kernelVersion"},{"sourceId":171471367,"sourceType":"kernelVersion"},{"sourceId":171471576,"sourceType":"kernelVersion"},{"sourceId":171471577,"sourceType":"kernelVersion"},{"sourceId":171471579,"sourceType":"kernelVersion"},{"sourceId":171471582,"sourceType":"kernelVersion"},{"sourceId":171496762,"sourceType":"kernelVersion"},{"sourceId":171496971,"sourceType":"kernelVersion"},{"sourceId":171496973,"sourceType":"kernelVersion"},{"sourceId":171496976,"sourceType":"kernelVersion"},{"sourceId":171496978,"sourceType":"kernelVersion"},{"sourceId":171503794,"sourceType":"kernelVersion"},{"sourceId":171504057,"sourceType":"kernelVersion"},{"sourceId":171504059,"sourceType":"kernelVersion"},{"sourceId":171504064,"sourceType":"kernelVersion"},{"sourceId":171504065,"sourceType":"kernelVersion"},{"sourceId":171509978,"sourceType":"kernelVersion"},{"sourceId":171510378,"sourceType":"kernelVersion"},{"sourceId":171510382,"sourceType":"kernelVersion"},{"sourceId":171510384,"sourceType":"kernelVersion"},{"sourceId":171510387,"sourceType":"kernelVersion"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This notebook is for finetune, trained by clean data(vote_sum >= 10).\nAs you can see from the input NOTEBOOKS, I will use spectrograms created in the kaggle notebooks. 4 different models will be trained using 4 different types of data. For models with the string '_spe_and_eeg', the kaggle spectrograms are concatenated and input into the model. This is the same as chris's method as shown in the published notebook.  \nnote: I committed in 'debug mode' (50sample, 1epoch), The finetune weight used in inference notebook ( https://www.kaggle.com/code/yujiariyasu/4th-place-solution ) is what I learned locally.","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport os","metadata":{"execution":{"iopub.status.busy":"2024-04-13T05:04:50.045146Z","iopub.execute_input":"2024-04-13T05:04:50.045529Z","iopub.status.idle":"2024-04-13T05:04:50.900781Z","shell.execute_reply.started":"2024-04-13T05:04:50.045499Z","shell.execute_reply":"2024-04-13T05:04:50.900050Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%capture\n!ln -s /kaggle/input/timm092/pytorch-image-models-0.9.2/timm\nimport timm\n!pip install einops\n!pip install albumentations==1.4.3","metadata":{"execution":{"iopub.status.busy":"2024-04-13T05:04:50.902460Z","iopub.execute_input":"2024-04-13T05:04:50.903167Z","iopub.status.idle":"2024-04-13T05:05:29.385285Z","shell.execute_reply.started":"2024-04-13T05:04:50.903133Z","shell.execute_reply":"2024-04-13T05:05:29.384121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!cp -r /kaggle/input/hms-pipeline ./","metadata":{"execution":{"iopub.status.busy":"2024-04-13T05:05:29.386674Z","iopub.execute_input":"2024-04-13T05:05:29.386957Z","iopub.status.idle":"2024-04-13T05:05:31.221885Z","shell.execute_reply.started":"2024-04-13T05:05:29.386931Z","shell.execute_reply":"2024-04-13T05:05:31.220818Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir -p /kaggle/working/hms/results","metadata":{"execution":{"iopub.status.busy":"2024-04-13T05:05:31.224409Z","iopub.execute_input":"2024-04-13T05:05:31.224747Z","iopub.status.idle":"2024-04-13T05:05:32.167010Z","shell.execute_reply.started":"2024-04-13T05:05:31.224716Z","shell.execute_reply":"2024-04-13T05:05:32.165816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%cd /kaggle/working/hms-pipeline/hms_pipeline/scripts","metadata":{"execution":{"iopub.status.busy":"2024-04-13T05:05:32.168349Z","iopub.execute_input":"2024-04-13T05:05:32.168636Z","iopub.status.idle":"2024-04-13T05:05:32.175646Z","shell.execute_reply.started":"2024-04-13T05:05:32.168602Z","shell.execute_reply":"2024-04-13T05:05:32.174784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%writefile /kaggle/working/hms-pipeline/hms_pipeline/src/configs.py\n\nfrom pathlib import Path\nfrom pprint import pprint\nimport timm\nfrom src.utils.metric_learning_loss import *\nfrom src.utils.metrics import *\nfrom src.utils.loss import *\nimport os\nimport torch.nn as nn\nimport pandas as pd\nimport numpy as np\nfrom pdb import set_trace as st\nfrom sklearn.preprocessing import OneHotEncoder\nimport torch.nn.functional as F\nimport pytorch_lightning as pl\nfrom types import MethodType\n\nfrom src.models.layers import AdaptiveConcatPool2d, Flatten\nfrom src.models.ch_mdl_dolg_efficientnet import ChMdlDolgEfficientnet, ArcFaceLossAdaptiveMargin\nfrom src.models.rsna_multi_image import MultiLevelModel2\nfrom src.models.backbones import *\nfrom src.models.group_norm import convert_groupnorm\nfrom src.models.batch_renorm import convert_batchrenorm\nfrom src.models.multi_instance import MultiInstanceModel, MetaMIL, AttentionMILModel, MultiInstanceModelWithWataruAttention\nfrom src.models.resnet import resnet18, resnet34, resnet101, resnet152\nfrom src.models.nextvit import NextVitNet\nfrom src.models.model_4channels import get_attention, get_resnet34, get_attention_inceptionv3\nfrom src.models.vae import VAE, ResNet_VAE\nfrom src.models.model_with_arcface import ArcMarginProduct, AddMarginProduct, ArcMarginProductSubcenter, ArcMarginProductOutCosine, ArcMarginProductSubcenterOutCosine, PudaeArcNet, WithArcface, WhalePrev1stModel, Guie2\nfrom src.models.with_meta_models import WithMetaModel\n\nfrom src.utils.augmentations.strong_aug import *\nfrom src.utils.augmentations.augmentation import *\nfrom src.utils.augmentations.policy_transform import policy_transform\nfrom sklearn.metrics import roc_auc_score, confusion_matrix, mean_squared_error, average_precision_score, accuracy_score\n\n\n################\n\nclass Baseline:\n    def __init__(self):\n        self.gpu = 'small'\n        self.compe = 'rsna'\n        self.batch_size = 16\n        self.grad_accumulations = 1\n        self.lr = 0.0001\n        self.epochs = 20\n        self.resume = False\n        self.seed = 2023\n        self.tta = 1\n        self.model_name = 'convnext_small.fb_in22k_ft_in1k_384'\n        # self.model_name = 'resnet50'\n        self.num_classes = 1\n        self.model = timm.create_model(self.model_name, pretrained=False, num_classes=self.num_classes)\n        self.criterion = torch.nn.BCEWithLogitsLoss()\n        # self.criterion = torch.nn.BCELoss()\n        # self.transform = medical_v1\n        self.transform = kuma_aug\n        self.image_size = 384\n        self.label_features = ['target']\n        self.metric = roc_auc_score # AUC().torch # MultiAP().torch # MultiAUC().torch\n        self.fp16 = True\n        self.optimizer = 'adam'\n        self.scheduler = 'CosineAnnealingWarmRestarts'\n        self.eta_min = 5e-7\n        self.train_by_all_data = False\n        self.early_stop_patience = 1000\n        self.inference = False\n        self.predict_valid = False\n        self.predict_test = False\n        self.logit_to = None\n        self.pretrained_path = None\n        self.sync_batchnorm = True\n        # self.sync_batchnorm = False\n        self.warmup_epochs = -1\n        self.finetune_transform = base_aug_v1\n        self.mixup = False\n        self.arcface = False\n        self.box_crop = None\n        self.predicted_mask_crop = None\n        self.pad_square = False\n        self.resume_epoch = 0\n        self.t_max=30\n        self.save_top_k = 1\n        self.meta_cols = []\n        self.output_features = False\n        self.force_use_model_path_config_when_inf = None\n        self.reset_classifier_when_inf = False\n        self.upsample = None\n        self.in_chans = 3\n        self.add_imsizes_when_inference = [(0, 0)]\n        self.inf_fp16 = False\n        self.distill = False\n        self.reload_dataloaders_every_n_epochs = 0\n        self.tranform_dataset_version = None\n        self.no_trained_model_when_inf = False\n        self.normalize_horiz_orientation = False\n        self.upsample_batch_pos_n = None\n        self.cut_200 = False\n        self.affine_for_gbr = False\n        self.half_dark = False\n        self.crop_by_left_right_line_text = False\n        self.use_wandb = True\n        self.memo = ''\n        self.inference_only = False\n        self.valid_df = None\n        self.valid_df_path = None\n\nclass hms_criterion(nn.Module):\n    def __init__(self):\n        super(hms_criterion, self).__init__()\n        # self.criterion = nn.KLDivLoss(reduction=\"batchmean\")\n        self.criterion = nn.KLDivLoss(reduction=\"mean\")\n    def forward(self, logits, targets, weights=None):\n        logits = F.log_softmax(logits, dim=1)\n        return self.criterion(logits, targets)\n\nclass hms_criterion(nn.Module):\n    def __init__(self):\n        super(hms_criterion, self).__init__()\n        self.criterion = nn.KLDivLoss(reduction=\"none\")  # 'none'を使用して個々の損失を保持\n        self.criterion_aux = torch.nn.BCEWithLogitsLoss()\n\n    def forward(self, logits, targets, weights=None):\n        aux = False\n        if logits.shape[1]>6:\n            targets_aux = targets[:, 6:]\n            logits_aux = logits[:, 6:]\n            loss_aux = self.criterion_aux(logits_aux, targets_aux)\n            logits = logits[:, :6]\n            targets = targets[:, :6]\n            aux = True\n\n        logits = F.log_softmax(logits, dim=1)\n        loss = self.criterion(logits, targets)\n        if weights is not None:\n            loss = loss * weights.unsqueeze(1)\n        loss = loss.mean()\n        if aux:\n            loss = loss + loss_aux*0.2\n        return loss\n\ndef kl_divergence(true, preds):\n    epsilon = 1e-10\n    preds = F.softmax(preds.float(), dim=1) + epsilon\n    true = true + epsilon\n    kl_div = true * torch.log(true / preds)\n    return - torch.sum(torch.mean(kl_div, dim=0))\n\n\nclass hms_base(Baseline):\n    def __init__(self):\n        super().__init__()\n        self.compe = 'hms'\n        self.predict_valid = False\n        self.predict_test = False\n        self.model_name = 'convnext_small.fb_in22k_ft_in1k_384'\n        self.label_features = ['seizure','lpd','gpd','lrda','grda','other']\n        self.num_classes = len(self.label_features)\n        self.in_chans = 3\n        self.model = timm.create_model(self.model_name, pretrained=False, num_classes=self.num_classes, in_chans=self.in_chans)\n        self.transform = hms_aug_v12\n        self.batch_size = 2\n        self.grad_accumulations = 8\n        self.lr = 1e-4\n        self.grad_accumulations = 1\n        self.metric = None\n        self.criterion = hms_criterion()\n        self.use_eeg_spectrograms = False\n        self.use_center_sec = None\n        self.clip_exp = np.exp(8)\n        self.epochs = 8\n        self.mixup = True\n        self.warmup_epochs = -1\n        self.ch3_zero_or_one = False\n        self.spe_and_eeg = False\n        self.use_one_ll = None\n        self.mu_std = None\n        self.vote_over9_weight = 1\n        self.metric = kl_divergence\n        self.train_df = pd.read_csv('/kaggle/working/train.csv')\n\nclass hms_swin_base(hms_base):\n    def __init__(self):\n        super().__init__()\n        self.image_size = 384\n        self.use_wandb = False\n        self.use_eeg_spectrograms = True\n        self.mu_std = (9, 3.5)\n        self.transform = hms_aug_v12\n        self.model_name = 'swinv2_large_window12to24_192to384.ms_in22k_ft_in1k'\n        self.model = timm.create_model(self.model_name, pretrained=False, num_classes=self.num_classes, in_chans=self.in_chans)\n        self.use_eeg_spectrograms = True\n        self.predict_valid = False\n        self.predict_test = False\n\n        \n### 4configs ###        \n\nclass newdata_hms_chris_fmax60_40sec_16ims_bandpass_9ep(hms_swin_base):\n    def __init__(self, fold):\n        super().__init__()\n        self.pretrained_path = f'/kaggle/input/hms-chris-fmax60-30sec-16ims-bandpass/last_fold{fold}.ckpt' # Since the model was created at the end of the competition, there was not enough time, so I used a pretrain weight that was trained on different data that I originally had.\n        self.epochs = 9\n        self.lr = 1e-5\n\nclass newdata_hms_chris_fmax30_50sec_16ims_bandpass_9ep(hms_swin_base):\n    def __init__(self, fold):\n        super().__init__()\n        self.pretrained_path = f'/kaggle/input/first-fmax30-50sec-16ims-bandpass/last_fold{fold}.ckpt'\n        self.epochs = 9\n        self.lr = 1e-5\n\nclass newdata_hms_chris_fmax30_30sec_8ims_bandpass_spe_and_eeg_9ep(hms_swin_base):\n    def __init__(self, fold):\n        super().__init__()\n        self.pretrained_path = f'/kaggle/input/first-fmax30-30sec-8ims-bandpass-spe-and-eeg/last_fold{fold}.ckpt'\n        self.epochs = 9\n        self.lr = 1e-5\n        self.spe_and_eeg = True\n\nclass newdata_hms_chris_fmax90_10sec_8ims_bandpass_spe_and_eeg_9ep(hms_swin_base):\n    def __init__(self, fold):\n        super().__init__()\n        self.pretrained_path = f'/kaggle/input/first-fmax90-10sec-8ims-bandpass-spe-and-eeg/last_fold{fold}.ckpt'\n        self.epochs = 9\n        self.lr = 1e-5\n        self.spe_and_eeg = True\n        \n### 4configs ###        \n","metadata":{"execution":{"iopub.status.busy":"2024-04-13T05:05:32.177278Z","iopub.execute_input":"2024-04-13T05:05:32.177575Z","iopub.status.idle":"2024-04-13T05:05:32.194110Z","shell.execute_reply.started":"2024-04-13T05:05:32.177552Z","shell.execute_reply":"2024-04-13T05:05:32.193231Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv('/kaggle/input/preprocess-train-csv/train_with_group_in_eeg.csv')\n\ndfs = []\nfor i, idf in df.groupby(['eeg_id', 'group_in_eeg']):\n    sec = (idf.spectrogram_label_offset_seconds.min() + idf.spectrogram_label_offset_seconds.max())//2\n    idf['spectrogram_label_offset_seconds'] = sec\n    dfs.append(idf.iloc[:1])\ndf = pd.concat(dfs)  \ndf = df.sort_values('eeg_id')","metadata":{"execution":{"iopub.status.busy":"2024-04-13T05:05:32.195155Z","iopub.execute_input":"2024-04-13T05:05:32.195436Z","iopub.status.idle":"2024-04-13T05:06:05.950178Z","shell.execute_reply.started":"2024-04-13T05:05:32.195414Z","shell.execute_reply":"2024-04-13T05:06:05.949417Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_eeg_ids = df.eeg_id.unique()\nall_data_n = 5\nchunk = len(all_eeg_ids) // all_data_n + 1\ndf['data_n'] = 0\nfor data_n in range(5):\n    use_eeg_ids = all_eeg_ids[data_n*chunk:(data_n+1)*chunk]\n    df.loc[df.eeg_id.isin(use_eeg_ids), 'data_n'] = data_n\ndf.data_n.value_counts()","metadata":{"execution":{"iopub.status.busy":"2024-04-13T05:06:05.951187Z","iopub.execute_input":"2024-04-13T05:06:05.951482Z","iopub.status.idle":"2024-04-13T05:06:05.971992Z","shell.execute_reply.started":"2024-04-13T05:06:05.951458Z","shell.execute_reply":"2024-04-13T05:06:05.971185Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head()","metadata":{"execution":{"iopub.status.busy":"2024-04-13T05:06:05.973066Z","iopub.execute_input":"2024-04-13T05:06:05.973359Z","iopub.status.idle":"2024-04-13T05:06:05.999785Z","shell.execute_reply.started":"2024-04-13T05:06:05.973334Z","shell.execute_reply":"2024-04-13T05:06:05.998952Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"configs = [\n    'newdata_hms_chris_fmax30_30sec_8ims_bandpass_spe_and_eeg_9ep',\n    'newdata_hms_chris_fmax90_10sec_8ims_bandpass_spe_and_eeg_9ep',\n    'newdata_hms_chris_fmax60_40sec_16ims_bandpass_9ep',\n    'newdata_hms_chris_fmax30_50sec_16ims_bandpass_9ep',\n]","metadata":{"execution":{"iopub.status.busy":"2024-04-13T05:06:06.002300Z","iopub.execute_input":"2024-04-13T05:06:06.002553Z","iopub.status.idle":"2024-04-13T05:06:06.006763Z","shell.execute_reply.started":"2024-04-13T05:06:06.002531Z","shell.execute_reply":"2024-04-13T05:06:06.005911Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path_dirs = [\n    '/kaggle/input/fmax30-sec30-8feats',\n    '/kaggle/input/fmax90-sec10-8feats',\n    '/kaggle/input/fmax60-sec40-16feats',\n    '/kaggle/input/fmax30-sec50-16feats',\n]","metadata":{"execution":{"iopub.status.busy":"2024-04-13T05:06:06.007795Z","iopub.execute_input":"2024-04-13T05:06:06.008022Z","iopub.status.idle":"2024-04-13T05:06:06.017058Z","shell.execute_reply.started":"2024-04-13T05:06:06.008002Z","shell.execute_reply":"2024-04-13T05:06:06.016225Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = df[df.vote_sum >= 10]\ndf[['seizure','lpd','gpd','lrda','grda','other']] = df[['seizure_vote','lpd_vote','gpd_vote','lrda_vote','grda_vote','other_vote']] / np.array([df[['seizure_vote','lpd_vote','gpd_vote','lrda_vote','grda_vote','other_vote']].values.sum(1).tolist()]*6).T\nlen(df)","metadata":{"execution":{"iopub.status.busy":"2024-04-13T05:06:06.018127Z","iopub.execute_input":"2024-04-13T05:06:06.018537Z","iopub.status.idle":"2024-04-13T05:06:06.038175Z","shell.execute_reply.started":"2024-04-13T05:06:06.018508Z","shell.execute_reply":"2024-04-13T05:06:06.037368Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"debug = True\nif debug:\n    df = df.sample(50)\n    epoch = 1\nelse:\n    epoch = 9\n\nfor config, path_dir in zip(configs, path_dirs):\n    df['path'] = path_dir + '-' + df.data_n.astype(str) + '/' + df.eeg_id.astype(str) + '_' + df.group_in_eeg.astype(str) + '.npy'\n    print('config:', config)\n    print('path sample:', df.path.values[0])\n    print('train len:', len(df))\n    df.to_csv('/kaggle/working/train.csv', index=False)    \n    for fold in range(5):\n        print('fold:', fold)\n        !python3 train_one_fold.py -c {config} -e {epoch} --fold {fold}\n        print('-'*100)        ","metadata":{"execution":{"iopub.status.busy":"2024-04-13T05:06:06.039160Z","iopub.execute_input":"2024-04-13T05:06:06.039437Z","iopub.status.idle":"2024-04-13T05:24:15.608696Z","shell.execute_reply.started":"2024-04-13T05:06:06.039415Z","shell.execute_reply":"2024-04-13T05:24:15.607529Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm -rf /kaggle/working/hms-pipeline","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from glob import glob\nfor p in sorted(glob('/kaggle/working/hms/results/*/last_fold*.ckpt')):\n    print(p)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}