{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":59093,"databundleVersionId":7469972,"sourceType":"competition"},{"sourceId":6259291,"sourceType":"datasetVersion","datasetId":3597559},{"sourceId":7392733,"sourceType":"datasetVersion","datasetId":4297749},{"sourceId":7979206,"sourceType":"datasetVersion","datasetId":4596419},{"sourceId":7981321,"sourceType":"datasetVersion","datasetId":4419986},{"sourceId":7989915,"sourceType":"datasetVersion","datasetId":4703503},{"sourceId":8015225,"sourceType":"datasetVersion","datasetId":4435105},{"sourceId":8016808,"sourceType":"datasetVersion","datasetId":4723312},{"sourceId":8025136,"sourceType":"datasetVersion","datasetId":4729369},{"sourceId":8035920,"sourceType":"datasetVersion","datasetId":4737176},{"sourceId":8054458,"sourceType":"datasetVersion","datasetId":4724042},{"sourceId":8055494,"sourceType":"datasetVersion","datasetId":4687290},{"sourceId":8077221,"sourceType":"datasetVersion","datasetId":4708755},{"sourceId":8109988,"sourceType":"datasetVersion","datasetId":4652811},{"sourceId":8114219,"sourceType":"datasetVersion","datasetId":4701635},{"sourceId":162276846,"sourceType":"kernelVersion"},{"sourceId":170279329,"sourceType":"kernelVersion"},{"sourceId":170279396,"sourceType":"kernelVersion"},{"sourceId":170388873,"sourceType":"kernelVersion"},{"sourceId":170625834,"sourceType":"kernelVersion"},{"sourceId":170714546,"sourceType":"kernelVersion"},{"sourceId":170714548,"sourceType":"kernelVersion"},{"sourceId":170715357,"sourceType":"kernelVersion"},{"sourceId":170739282,"sourceType":"kernelVersion"},{"sourceId":170742054,"sourceType":"kernelVersion"},{"sourceId":170742073,"sourceType":"kernelVersion"},{"sourceId":170763234,"sourceType":"kernelVersion"},{"sourceId":170763274,"sourceType":"kernelVersion"},{"sourceId":170787277,"sourceType":"kernelVersion"},{"sourceId":170806449,"sourceType":"kernelVersion"},{"sourceId":170882149,"sourceType":"kernelVersion"},{"sourceId":170883254,"sourceType":"kernelVersion"},{"sourceId":170883282,"sourceType":"kernelVersion"},{"sourceId":170883536,"sourceType":"kernelVersion"},{"sourceId":170883539,"sourceType":"kernelVersion"},{"sourceId":170912829,"sourceType":"kernelVersion"},{"sourceId":171749895,"sourceType":"kernelVersion"}],"dockerImageVersionId":30636,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"We did stacking with 8 models oof.  \nIn this notebook, we used the oof prepared by four members at their local env in advance.  \npytorch nn.Linear model was trained by 5939 clean label data (vote_sum>=10).  \nLB Probing results showed that there seemed to be more Seizure/LRDA/GRDA labels in test compared to train, so we did upsampling and adjusted the model to focus more on labels such as Seizure/LRDA/GRDA.  \nNote: To save kaggle gpu, we are training during submission in the final submission notebook ( https://www.kaggle.com/code/yujiariyasu/4th-place-solution ). This means that our final submission notebook will not use the weights of the models trained in this notebook. This notebook is a demonstration.","metadata":{}},{"cell_type":"markdown","source":"# (model code is simple)\n```\nclass MultiLayerPerceptronBase(nn.Module):\n    def __init__(self, num_classes, input_num, dropout=0.3):\n        super(MultiLayerPerceptronBase, self).__init__()\n        self.mlp = nn.Sequential(\n            nn.Linear(input_num, num_classes),\n        )\n\n    def forward(self, x, labels=None):\n        x = self.mlp(x)\n        return x\n```    ","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom glob import glob\nfrom tqdm import tqdm\nimport warnings\nwarnings.simplefilter('ignore')","metadata":{"execution":{"iopub.status.busy":"2024-04-14T07:32:57.445987Z","iopub.execute_input":"2024-04-14T07:32:57.446957Z","iopub.status.idle":"2024-04-14T07:32:57.452072Z","shell.execute_reply.started":"2024-04-14T07:32:57.446901Z","shell.execute_reply":"2024-04-14T07:32:57.451094Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cols = ['seizure_vote','lpd_vote','gpd_vote','lrda_vote','grda_vote','other_vote']\ndf = pd.read_csv('/kaggle/input/hms-harmful-brain-activity-classification/train.csv')\ndf['n'] = df[cols].sum(1)\ndf = df[df.n>8]\ndfs = []\nfor i, idf in df.groupby('eeg_id'):\n    idf[cols] = idf[cols].mean(0)\n    dfs.append(idf.iloc[:1])\ndf = pd.concat(dfs)\ndf[cols] = df[cols] / np.array([df[cols].values.sum(1).tolist()]*6).T\ntrue = df.sort_values('eeg_id')[cols].values+1e-10\nlen(true)","metadata":{"execution":{"iopub.status.busy":"2024-04-14T07:32:57.454356Z","iopub.execute_input":"2024-04-14T07:32:57.454851Z","iopub.status.idle":"2024-04-14T07:33:20.232685Z","shell.execute_reply.started":"2024-04-14T07:32:57.454825Z","shell.execute_reply":"2024-04-14T07:33:20.231736Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ariyasu\nconfigs=[\n    'newdata_hms_chris_fmax30_50sec_16ims_bandpass_9ep',\n    'newdata_hms_chris_fmax60_40sec_16ims_bandpass_9ep',\n    'newdata_hms_chris_fmax30_30sec_8ims_bandpass_spe_and_eeg_9ep',\n    'newdata_hms_chris_fmax90_10sec_8ims_bandpass_spe_and_eeg_9ep',\n]\ntrue_cols = ['seizure','lpd','gpd','lrda','grda','other']\npred_cols = [f'pred_{c}' for c in true_cols]\npreds = []\nfolds = range(5)\nfor config in configs:\n    if config == 'newdata_hms_chris_fmax30_50sec_16ims_bandpass_9ep':\n        oof = pd.read_csv('/kaggle/input/ariyasu-newdata-oof-v3/finetune_hms_chris_fmax30_50sec_16ims_bandpass_ep9.csv')\n        cs = [f'finetune_hms_chris_fmax30_50sec_16ims_bandpass_ep9_pred_{c}' for c in true_cols]\n    elif config == 'newdata_hms_chris_fmax60_40sec_16ims_bandpass_9ep':\n        oof = pd.read_csv('/kaggle/input/fork-of-ariyasu-newdata-oof-v4/finetune_hms_chris_fmax60_40sec_16ims_bandpass_ep9.csv')\n        cs = [f'finetune_hms_chris_fmax60_40sec_16ims_bandpass_ep9_pred_{c}' for c in true_cols]\n    elif config == 'newdata_hms_chris_fmax30_30sec_8ims_bandpass_spe_and_eeg_9ep':\n        oof = pd.read_csv('/kaggle/input/ariyasu-newdata-oof-v4/finetune_hms_chris_fmax30_30sec_8ims_bandpass_spe_and_eeg_ep9.csv')\n        cs = [f'finetune_hms_chris_fmax30_30sec_8ims_bandpass_spe_and_eeg_ep9_pred_{c}' for c in true_cols]\n    elif config == 'newdata_hms_chris_fmax90_10sec_8ims_bandpass_spe_and_eeg_9ep':\n        oof = pd.read_csv('/kaggle/input/ariyasu-newdata-oof-v4/finetune_hms_chris_fmax90_10sec_8ims_bandpass_spe_and_eeg_ep9.csv')\n        cs = [f'finetune_hms_chris_fmax90_10sec_8ims_bandpass_spe_and_eeg_ep9_pred_{c}' for c in true_cols]\n    oof = pd.DataFrame(oof.groupby('eeg_id').first()).reset_index().sort_values('eeg_id')\n    expert_consensus = oof['expert_consensus'].values\n    oof[true_cols] = true-1e-10\n    pr = oof[cs].values\n    oof[pred_cols] = pr\n    preds.append(pr)\n\n# tattaka\ncols = ['oof_logits_seizure_vote','oof_logits_lpd_vote','oof_logits_gpd_vote','oof_logits_lrda_vote','oof_logits_grda_vote','oof_logits_other_vote']\nexp_dirs = [\n    \"/kaggle/input/hms-weights-2/exp094/caformer_s18_2_5d_256_el30_mixup_100ep\",\n    \"/kaggle/input/hms-weights-3/exp147/tiny_vit_21m_512_el30_mixup_50ep\",\n]\nfor exp_dir in exp_dirs:\n    df = pd.read_csv(glob(f\"{exp_dir}/**/result_df.csv\", recursive=True)[0])\n    df = df[df.num_votes > 7].groupby(\"eeg_id\").first().reset_index()\n    c = exp_dir.split('/')[-1]\n    df = df[df.eeg_id.isin(oof.eeg_id)].sort_values('eeg_id')\n    pr = df[cols].values\n    preds.append(pr)\n    configs.append(c)\n\n# bilzard\ncols = ['pl_seizure_vote','pl_lpd_vote','pl_gpd_vote','pl_lrda_vote','pl_grda_vote','pl_other_vote']\nfor c in [\n'v5_eeg_24ep_cutmix',\n]:\n    df = pd.read_parquet(f'/kaggle/input/hms-bilzard-oof/{c}/train_pseudo_label.pqt').sort_values('eeg_id')\n    df = df[df.eeg_id.isin(oof.eeg_id)].sort_values('eeg_id')\n    pr = df[cols].values\n    preds.append(pr)\n    configs.append(c)\n\n# yu4u\nc = \"yu4u\"\ndf = pd.read_csv(\"/kaggle/input/hms-oof/oof.csv\")\npr = df.sort_values(\"eeg_id\")[[\"seizure_vote\", \"lpd_vote\", \"gpd_vote\", \"lrda_vote\", \"grda_vote\", \"other_vote\"]].values\npreds.append(pr)\nconfigs.append(c)\nconfigs","metadata":{"execution":{"iopub.status.busy":"2024-04-14T07:33:20.234169Z","iopub.execute_input":"2024-04-14T07:33:20.234443Z","iopub.status.idle":"2024-04-14T07:33:21.154224Z","shell.execute_reply.started":"2024-04-14T07:33:20.234417Z","shell.execute_reply":"2024-04-14T07:33:21.153246Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# upsample","metadata":{}},{"cell_type":"code","source":"features = []\nassert len(configs) == len(preds)\nfor c, pr in zip(configs, preds):\n    oof[[f'{c}_{col}' for col in pred_cols]] = pr\n    features += [f'{c}_{col}' for col in pred_cols]\n\n# upsample\ndfs = []\nfor sample_n, v in zip(\n    [11, 8, 8, 11, 11, 6],\n    ['Seizure','LPD','GPD','LRDA','GRDA','Other']\n):\n    for _ in range(sample_n):\n        dfs.append(oof[oof.expert_consensus==v].copy())\noof = pd.concat(dfs)\nlen(oof)","metadata":{"execution":{"iopub.status.busy":"2024-04-14T07:33:21.155383Z","iopub.execute_input":"2024-04-14T07:33:21.155676Z","iopub.status.idle":"2024-04-14T07:33:21.526031Z","shell.execute_reply.started":"2024-04-14T07:33:21.155650Z","shell.execute_reply":"2024-04-14T07:33:21.525117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(preds))\np = f'oof_for_mlp.csv'\noof.to_csv(p, index=False)\nconfigs","metadata":{"execution":{"iopub.status.busy":"2024-04-14T07:33:21.528314Z","iopub.execute_input":"2024-04-14T07:33:21.528604Z","iopub.status.idle":"2024-04-14T07:33:27.322828Z","shell.execute_reply.started":"2024-04-14T07:33:21.528579Z","shell.execute_reply":"2024-04-14T07:33:27.321983Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!cp -r /kaggle/input/hms-pipeline ./","metadata":{"execution":{"iopub.status.busy":"2024-04-14T07:33:27.323826Z","iopub.execute_input":"2024-04-14T07:33:27.324112Z","iopub.status.idle":"2024-04-14T07:33:29.153499Z","shell.execute_reply.started":"2024-04-14T07:33:27.324087Z","shell.execute_reply":"2024-04-14T07:33:29.152197Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir -p /kaggle/working/hms/results","metadata":{"execution":{"iopub.status.busy":"2024-04-14T07:33:29.155533Z","iopub.execute_input":"2024-04-14T07:33:29.155872Z","iopub.status.idle":"2024-04-14T07:33:30.094071Z","shell.execute_reply.started":"2024-04-14T07:33:29.155841Z","shell.execute_reply":"2024-04-14T07:33:30.092715Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%writefile /kaggle/working/hms-pipeline/hms_pipeline/src/mlp_configs.py\n\nfrom pathlib import Path\nfrom pprint import pprint\nimport timm\nfrom src.utils.metrics import *\nfrom src.utils.loss import *\nimport os\nimport torch\nimport torch.nn as nn\nimport pandas as pd\nimport numpy as np\nfrom pdb import set_trace as st\n\nfrom sklearn.metrics import roc_auc_score, confusion_matrix, mean_squared_error, average_precision_score\nfrom src.models.mlp import *\n\nclass hms_criterion(nn.Module):\n    def __init__(self):\n        super(hms_criterion, self).__init__()\n        self.criterion = nn.KLDivLoss(reduction=\"none\")  # 'none'を使用して個々の損失を保持\n\n    def forward(self, logits, targets, weights=None):\n        logits = F.log_softmax(logits, dim=1)\n        loss = self.criterion(logits, targets)\n        if weights is not None:\n            loss = loss * weights.unsqueeze(1)\n        return loss.mean()\ndef kl_divergence(true, preds):\n    epsilon = 1e-10  # 0による除算を避けるための小さな値\n    preds = F.softmax(preds.float(), dim=1) + epsilon\n    true = true + epsilon\n    kl_div = true * torch.log(true / preds)\n    return - torch.sum(torch.mean(kl_div, dim=0))\n\n\nclass Baseline:\n    def __init__(self):\n        self.compe = 'kaggle_days'\n        self.batch_size = 8\n        self.grad_accumulations = 1\n        self.lr = 0.0001\n        self.epochs = 300\n        self.resume = False\n        self.seed = 2022\n        self.tta = 1\n        self.predict_valid = True\n        self.predict_test = True\n        self.valid_df = None\n        self.num_classes = 1\n        self.criterion = torch.nn.BCEWithLogitsLoss()\n        # self.criterion = torch.nn.BCELoss()\n        self.label_features = ['target']\n        self.metric = roc_auc_score # AUC().torch # MultiAP().torch\n        self.fp16 = False\n        self.optimizer = 'adam'\n        # self.scheduler = 'linear_schedule_with_warmup'\n        self.scheduler = 'CosineAnnealingWarmRestarts'\n        self.train_by_all_data = False\n        self.early_stop_patience = 30\n        self.inference = False\n        self.logit_to = None\n        self.pretrained_path = None\n        self.sync_batchnorm = False\n        self.finetune_transform = None\n        self.gpu = 'v100'\n        self.weight_decay = 0.1 # | 0.01\n        self.inference_only = False\n        self.num_train_optimization_steps = 3000\n        self.resume_epoch = 0\n        self.t_max = 30\n        self.save_top_k = 1\n        self.output_features = False\n        self.eta_min = 5e-7\n        self.mixup = False\n        self.add_imsizes_when_inference = [(0, 0)] # dummy\n\n################\nclass hms_stacking_base(Baseline):\n    def __init__(self):\n        super().__init__()\n        self.compe = 'hms'\n        self.train_df_path = '/kaggle/working/oof_for_mlp.csv'\n        self.train_df = pd.read_csv(self.train_df_path)\n        configs = [\n         'newdata_hms_chris_fmax30_50sec_16ims_bandpass_9ep',\n         'newdata_hms_chris_fmax60_40sec_16ims_bandpass_9ep',\n         'newdata_hms_chris_fmax30_30sec_8ims_bandpass_spe_and_eeg_9ep',\n         'newdata_hms_chris_fmax90_10sec_8ims_bandpass_spe_and_eeg_9ep',\n         'caformer_s18_2_5d_256_el30_mixup_100ep',\n         'tiny_vit_21m_512_el30_mixup_50ep',            \n         'v5_eeg_24ep_cutmix',\n         'yu4u',\n        ]\n        print('len(configs):', len(configs))\n        self.label_features = ['seizure','lpd','gpd','lrda','grda','other']\n\n        self.meta_cols = []\n        for config in configs:\n            for col in self.label_features:\n                if f'{config}_pred_{col}' in list(self.train_df):\n                    self.meta_cols.append(f'{config}_pred_{col}')\n\n        self.num_classes = len(self.label_features)\n        self.batch_size = 16\n        self.grad_accumulations = 2\n        self.lr = 1e-4\n        self.criterion = hms_criterion()\n        self.metric = kl_divergence\n        self.warmup_epochs = 0\n        self.predict_test = False\n        self.predict_valid = True\n        self.gpu = 'small'\n        self.valid_df = self.train_df.copy()\n        self.train_df['fold'] = -1\n        self.valid_df['fold'] = 0\n        self.t_max = 30\n\nclass mlp_config_0(hms_stacking_base):\n    def __init__(self):\n        super().__init__()\n        self.model = MultiLayerPerceptronBase(num_classes=self.num_classes, input_num=len(self.meta_cols))\nclass mlp_config_1(hms_stacking_base):\n    def __init__(self):\n        super().__init__()\n        self.model = MultiLayerPerceptronBaseNoBias(num_classes=self.num_classes, input_num=len(self.meta_cols))\n","metadata":{"execution":{"iopub.status.busy":"2024-04-14T07:33:30.096297Z","iopub.execute_input":"2024-04-14T07:33:30.096628Z","iopub.status.idle":"2024-04-14T07:33:30.106845Z","shell.execute_reply.started":"2024-04-14T07:33:30.096598Z","shell.execute_reply":"2024-04-14T07:33:30.105911Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%cd /kaggle/working/hms-pipeline/hms_pipeline/scripts","metadata":{"execution":{"iopub.status.busy":"2024-04-14T07:33:30.108301Z","iopub.execute_input":"2024-04-14T07:33:30.108627Z","iopub.status.idle":"2024-04-14T07:33:30.121566Z","shell.execute_reply.started":"2024-04-14T07:33:30.108595Z","shell.execute_reply":"2024-04-14T07:33:30.120695Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nif len(pd.read_csv('/kaggle/input/hms-harmful-brain-activity-classification/test.csv')) == 1:\n    !python3 train_one_fold.py -c mlp_config_0 -t mlp -e 1\n    !python3 train_one_fold.py -c mlp_config_1 -t mlp -e 1\nelse:\n    !python3 train_one_fold.py -c mlp_config_0 -t mlp -e 300\n    !python3 train_one_fold.py -c mlp_config_1 -t mlp -e 300","metadata":{"execution":{"iopub.status.busy":"2024-04-14T07:33:30.122739Z","iopub.execute_input":"2024-04-14T07:33:30.123041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}