{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":71549,"databundleVersionId":8561470,"sourceType":"competition"},{"sourceId":9069868,"sourceType":"datasetVersion","datasetId":5459726}],"dockerImageVersionId":30747,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Optimizing the evaluation metric\n\nSince we are competing for the best metric scores, directly optimizing the evaluation loss function seems natural. Binary cross entropy, for example, is often good enough for evaluation metrics like ROC AUC or Dice, but sometimes building a custom loss for the evaluation metric is crucial. The SenNet competition ([1st solution](https://www.kaggle.com/competitions/blood-vessel-segmentation/discussion/475522)) was such a case and I regret not doing that.\n\nIn this competition, we use a special evaluation metric that emphasizes the \"severe\" status.\n\n- The score is an average of 4 losses\n- 3 of them are weighted cross-entropy loss for spinal, foraminal, and subarticular\n  * The weights are: Normal/Mild: 1, Moderate: 2, Severe: 4\n- The 4th loss is any_severe_spinal_loss\n\nSee the [official notebook](https://www.kaggle.com/code/metric/rsna-lumbar-metric-71549).\n\nIn this notebook, I train a very simple model with a loss that optimizes this metric. Take a look at `class SevereLoss` and you can replace CrossEntropyLoss with it.\n\n\n#### Version 2\n- Fix bug in evaluate() score (thanks to @quan0095)\n- Fix pretrained weight was not pretrained weight but random\n\n#### Version 3\n- Fix crucial randomness from polars group_by maintain_order in src/data.py; train/val split was irreproducible.\n","metadata":{}},{"cell_type":"code","source":"! cp -r /kaggle/input/rsna2024-public/src .\n! ls","metadata":{"execution":{"iopub.status.busy":"2024-07-30T22:04:14.488390Z","iopub.execute_input":"2024-07-30T22:04:14.489182Z","iopub.status.idle":"2024-07-30T22:04:16.598180Z","shell.execute_reply.started":"2024-07-30T22:04:14.489142Z","shell.execute_reply":"2024-07-30T22:04:16.596990Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport polars as pl\nimport sys\nimport time\nimport yaml\nfrom sklearn.model_selection import KFold\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.nn.modules.loss import _Loss\nfrom transformers import get_cosine_schedule_with_warmup\n\ndi = '/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification'\ndevice = torch.device('cuda')\n\ntb_global = time.time()","metadata":{"execution":{"iopub.status.busy":"2024-07-30T22:04:16.600834Z","iopub.execute_input":"2024-07-30T22:04:16.601220Z","iopub.status.idle":"2024-07-30T22:04:22.082518Z","shell.execute_reply.started":"2024-07-30T22:04:16.601183Z","shell.execute_reply":"2024-07-30T22:04:22.081574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# SevereLoss\n\n### Notation\n\n```\ny_pred (Tensor[float32]): logit with shape (batch_size, 3, 25)\ny      (Tensor[int]):     ground truth index (batch_size, 25)\n                          indices are 0, 1, 2 for Normal/Mild, Moderate, and Severe, respectively\n```\n\nI assume 25 values are in the order of columns in train.csv.\n\n\"spinal\" columns are 0:5, \"foraminal\" columns are 5:15 and \"subarticular\" are 15:25.","metadata":{}},{"cell_type":"code","source":"train = pl.read_csv(di + '/train.csv',\n                    schema_overrides={'study_id': str, 'series_id': str})\ncolumns = train.columns[1:]\n\nprint('spinal:      ', columns[:5][:2], '...');     assert all(['spinal' in c for c in columns[:5]])\nprint('foraminal:   ', columns[5:15][:2], '...');   assert all(['foraminal' in c for c in columns[5:15]])\nprint('subarticular:', columns[15:25][:2], '...');  assert all(['subarticular' in c for c in columns[15:25]])","metadata":{"execution":{"iopub.status.busy":"2024-07-30T22:04:22.090368Z","iopub.execute_input":"2024-07-30T22:04:22.090631Z","iopub.status.idle":"2024-07-30T22:04:22.234107Z","shell.execute_reply.started":"2024-07-30T22:04:22.090608Z","shell.execute_reply":"2024-07-30T22:04:22.233182Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Custom Loss for this competition\n\nclass SevereLoss(_Loss):\n    \"\"\"\n    For RSNA 2024\n    criterion = SevereLoss()     # you can replace nn.CrossEntropyLoss\n    loss = criterion(y_pred, y)\n    \"\"\"\n    def __init__(self, temperature=1.0):\n        \"\"\"\n        Use max if temperature = 0\n        \"\"\"\n        super().__init__()\n        self.t = temperature\n        assert self.t >= 0\n    \n    def __repr__(self):\n        return 'SevereLoss(t=%.1f)' % self.t\n\n    def forward(self, y_pred: torch.Tensor, y: torch.Tensor) -> torch.Tensor:\n        \"\"\"\n        Args:\n          y_pred (Tensor[float]): logit             (batch_size, 3, 25)\n          y      (Tensor[int]):   true label index  (batch_size, 25)\n        \"\"\"\n        assert y_pred.size(0) == y.size(0)\n        assert y_pred.size(1) == 3 and y_pred.size(2) == 25\n        assert y.size(1) == 25\n        assert y.size(0) > 0\n        \n        slices = [slice(0, 5), slice(5, 15), slice(15, 25)] \n        w = 2 ** y  # sample_weight w = (1, 2, 4) for y = 0, 1, 2 (batch_size, 25)\n\n        loss = F.cross_entropy(y_pred, y, reduction='none')  # (batch_size, 25)\n\n        # Weighted sum of losses for spinal (:5), foraminal (5:15), and subarticular (15:25)\n        wloss_sums = []\n        for k, idx in enumerate(slices):\n            wloss_sums.append((w[:, idx] * loss[:, idx]).sum())\n\n        # any_severe_spinal\n        #   True label y_max:      Is any of 5 spinal severe? true/false\n        #   Prediction y_pred_max: Max of 5 spinal severe probabilities y_pred[:, 2, :5].max(dim=1)\n        #   any_severe_spinal_loss is the binary cross entropy between these two.\n        y_spinal_prob = y_pred[:, :, :5].softmax(dim=1)             # (batch_size, 3,  5)\n        w_max = torch.amax(w[:, :5], dim=1)                         # (batch_size, )\n        #y_max = torch.amax(y[:, :5] == 2, dim=1).to(torch.float32)  # 0 or 1\n        y_max = torch.amax(y[:, :5] == 2, dim=1).to(y_pred.dtype)   # 0 or 1\n\n        if self.t > 0:\n            # Attention for the maximum value\n            attn = F.softmax(y_spinal_prob[:, 2, :] / self.t, dim=1)  # (batch_size, 5)\n\n            # Pick the sofmax among 5 severe=2 y_spinal_probs with attn\n            y_pred_max = (attn * y_spinal_prob[:, 2, :]).sum(dim=1)   # weighted average among 5 spinal columns \n        else:\n            # Exact max; this works too\n            y_pred_max = y_spinal_prob[:, 2, :].amax(dim=1)\n\n        loss_max = F.binary_cross_entropy(y_pred_max, y_max, reduction='none')\n        wloss_sums.append((w_max * loss_max).sum())\n\n        # See below about these numbers\n        loss = (wloss_sums[0] / 6.084050632911392 +\n                wloss_sums[1] / 12.962531645569621 + \n                wloss_sums[2] / 14.38632911392405 +\n                wloss_sums[3] / 1.729113924050633) / (4 * y.size(0))\n\n        return loss\n","metadata":{"execution":{"iopub.status.busy":"2024-07-30T22:04:22.235228Z","iopub.execute_input":"2024-07-30T22:04:22.235513Z","iopub.status.idle":"2024-07-30T22:04:22.249559Z","shell.execute_reply.started":"2024-07-30T22:04:22.235489Z","shell.execute_reply":"2024-07-30T22:04:22.248454Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Softmax\n\nI thought taking a softmax `attn` among 5 severe spinal predictions is a good idea, but maybe it does not matter. You can use max by setting temperature=0. There is no reason to use temperature=1 as it is a free hyperparameter.\n\n### Normalization\n\nThe evaluation metric is an average over all data, while we compute loss within mini-batch during training. The weighted average over all data is not equal to the average of local weighted averages:\n\n$$\n\\frac{\\sum_i^N A_i}{\\sum_j^N B_j} \\neq \\frac{1}{N} \\sum_i^N \\frac{A_i}{B_i}.\n$$\n\nI use the global averaged normalization factors:\n\n$$\n\\frac{\\sum_i^N A_i}{\\sum_j^N B_j} = \\frac{1}{N} \\sum_i^N \\frac{A_i}{\\sum_j B_j / N}.\n$$\n\nI do not say local weighted average is wrong and I guess there are little differences.\n","metadata":{}},{"cell_type":"code","source":"# Compute the weight global average\nweight_map = {'Normal/Mild': 1,\n              'Moderate': 2,\n              'Severe': 4,\n              None: 0}\n\nw_sum = [0, ] * 4\n\nfor r in train.iter_rows():\n    w = np.array([weight_map[x] for x in r[1:]])  # array[int] (25, )\n    assert len(w) == 25\n\n    w_sum[0] += w[:5].sum()    # spinal\n    w_sum[1] += w[5:15].sum()  # foraminal\n    w_sum[2] += w[15:25].sum() # subarticular \n    w_sum[3] += w[:5].max()    # any_severe_spinal\n\nfor k in range(4):\n    w_sum[k] /= len(train)\n\nw_sum\n# (6.084050632911392, 12.962531645569621, 14.38632911392405, 1.729113924050633)","metadata":{"execution":{"iopub.status.busy":"2024-07-30T22:04:22.250629Z","iopub.execute_input":"2024-07-30T22:04:22.250930Z","iopub.status.idle":"2024-07-30T22:04:22.320986Z","shell.execute_reply.started":"2024-07-30T22:04:22.250898Z","shell.execute_reply":"2024-07-30T22:04:22.320115Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Configuration","metadata":{}},{"cell_type":"code","source":"debug = False  # Train only 2 epochs if True\n\ncfg = yaml.safe_load(\"\"\"\ndata:\n  image_size_in: 224\n\nmodel:\n  encoder: convnext_tiny.in12k_ft_in1k\n\nkfold:\n  k: 5\n  folds: [0, ]\n\ntrain:\n  lr: 1e-4\n  epochs: 20\n  weight_decay: 1e-4   # these two are negligibly weak.\n  max_grad_norm: 1000  # Maybe stronger are better for transformer models\n\nvalidate:\n  every_n_epoch: 1\n\nloader:\n  batch_size: 8\n  num_workers: 2\n\"\"\")","metadata":{"execution":{"iopub.status.busy":"2024-07-30T22:04:22.322109Z","iopub.execute_input":"2024-07-30T22:04:22.322430Z","iopub.status.idle":"2024-07-30T22:04:22.331675Z","shell.execute_reply.started":"2024-07-30T22:04:22.322397Z","shell.execute_reply":"2024-07-30T22:04:22.330865Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Hyperparameters\nweight_decay = float(cfg['train']['weight_decay'])\nmax_grad_norm = cfg['train']['max_grad_norm']\nval_every = cfg['validate']['every_n_epoch']\n\n# Criterion\ncriterion = SevereLoss(temperature=0)\n# criterion = nn.CrossEntropyLoss()       # for standard unweighted cross entropy\n\nprint('Criterion:', criterion)","metadata":{"execution":{"iopub.status.busy":"2024-07-30T22:04:22.332928Z","iopub.execute_input":"2024-07-30T22:04:22.333331Z","iopub.status.idle":"2024-07-30T22:04:22.342128Z","shell.execute_reply.started":"2024-07-30T22:04:22.333299Z","shell.execute_reply":"2024-07-30T22:04:22.341254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data and Model\n\nData and Model are *terribly simple*, and you do not have to look at them. I just started this competition.\n\nData: I only use one \"Sagittal T1\" series per study_id. One random image per study per epoch during training, and one middle image for validation and test prediction.\n\nModel: timm convnext_tiny that outputs y_pred (batch_size, 3, 25)\n","metadata":{}},{"cell_type":"code","source":"# Load from src/data.py image.py model.py\nsys.path.append('src')\nfrom data import load_series_descriptions, load_labels, Dataset\nfrom model import Model\nimport official_metric\n\ndatad = load_series_descriptions('train')\nload_labels(datad)","metadata":{"execution":{"iopub.status.busy":"2024-07-30T22:04:22.343245Z","iopub.execute_input":"2024-07-30T22:04:22.343572Z","iopub.status.idle":"2024-07-30T22:04:42.086168Z","shell.execute_reply.started":"2024-07-30T22:04:22.343542Z","shell.execute_reply":"2024-07-30T22:04:42.085277Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Evaluate\n\n","metadata":{}},{"cell_type":"code","source":"# Compute the loss and score for the validation set.\n# Weight normalization here is using the weights in the val set, not approximated by the global mean\ndef evaluate(model, loader_val):\n    tb = time.time()\n\n    was_training = model.training\n    model.eval()\n\n    n_sum = 0\n    loss0_sum = 0.0  # loss for the criterion\n    \n    # 4 losses for the evaluation metric\n    loss4_sum = torch.zeros(4, device=device)\n    w_sum = torch.zeros(4, device=device)\n    slices = [slice(0, 5), slice(5, 15), slice(15, 25)]  # spinal, foraminal, subarticular\n\n    for d in loader_val:\n        x = d['x'].to(device)  # input image\n        y = d['y'].to(device)  # int (batch_size, 25)\n        batch_size = len(x)\n\n        # Predict\n        with torch.no_grad():\n            y_pred = model(x)  # (batch_size, 3, 25)\n\n        w = 2 ** y  # sample_weight w = (1, 2, 4) for y = 0, 1, 2 (batch_size, 25)\n\n        loss0 = criterion(y_pred, y)\n\n        n_sum += batch_size\n        loss0_sum += loss0.item() * batch_size\n\n        # Compute score\n        # - weighted loss for spinal, foraminal, subarticular\n        # - binary cross entropy for maximum spinal severe\n        ce_loss = F.cross_entropy(y_pred, y, reduction='none')  # (batch_size, 25)\n        for k, idx in enumerate(slices):\n            w_sum[k] += w[:, idx].sum()\n            loss4_sum[k] += (w[:, idx] * ce_loss[:, idx]).sum()\n\n        # Spinal max\n        y_spinal_prob = y_pred[:, :, :5].softmax(dim=1)            # (batch_size, 3,  5)\n        w_max = torch.amax(w[:, :5], dim=1)                        # (batch_size, )\n        y_max = torch.amax(y[:, :5] == 2, dim=1).to(torch.float)   # 0 or 1\n        y_pred_max = y_spinal_prob[:, 2, :].amax(dim=1)            # max in severe (class=2)\n\n        loss_max = F.binary_cross_entropy(y_pred_max, y_max, reduction='none')\n        loss4_sum[3] += (w_max * loss_max).sum()\n        w_sum[3] += w_max.sum()\n\n    # Average over spinal, foraminal, subarticular, and any_severe_spinal\n    score = (loss4_sum / w_sum).sum().item() / 4\n\n    model.train(was_training)\n\n    dt = time.time() - tb\n    ret = {'loss': loss0_sum / n_sum,\n           'score': score,\n           'dt': dt}\n    return ret","metadata":{"execution":{"iopub.status.busy":"2024-07-30T22:04:42.087575Z","iopub.execute_input":"2024-07-30T22:04:42.088051Z","iopub.status.idle":"2024-07-30T22:04:42.101008Z","shell.execute_reply.started":"2024-07-30T22:04:42.088023Z","shell.execute_reply":"2024-07-30T22:04:42.099989Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training","metadata":{"execution":{"iopub.status.busy":"2024-07-29T00:54:16.941926Z","iopub.execute_input":"2024-07-29T00:54:16.942587Z","iopub.status.idle":"2024-07-29T00:54:16.953613Z","shell.execute_reply.started":"2024-07-29T00:54:16.942547Z","shell.execute_reply":"2024-07-29T00:54:16.952372Z"}}},{"cell_type":"code","source":"# KFold\nstudy_ids = [sid for sid, d in datad.items() if d['filenames'] is not None]\ndf = pl.DataFrame({'study_id': study_ids})\n\nnfolds = cfg['kfold']['k']\nfolds = cfg['kfold']['folds']  # list[int]\nkfold = KFold(n_splits=nfolds, shuffle=True, random_state=42)\nprint('Folds', folds, '/', nfolds)\n\n\n#\n# Training loop\n#\nstudy_ids = [sid for sid, d in datad.items() if d['filenames'] is not None]\ndf = pl.DataFrame({'study_id': study_ids})\n\nfor ifold, (idx_train, idx_val) in enumerate(kfold.split(df)):\n    if ifold not in folds:\n        continue\n        \n    # Data\n    ds_train = Dataset(df[idx_train], datad, cfg, pick='random', augment=True)\n    ds_val =   Dataset(df[idx_val],   datad, cfg, pick='middle')\n\n    loader_train = ds_train.loader(cfg, shuffle=True, drop_last=True)\n    loader_val = ds_val.loader(cfg)\n\n    nbatch = len(loader_train)\n\n    # Model\n    model = Model(cfg, pretrained=False)  # Just use pretrained=True with internet\n    \n    # Read pretrained model because internet=off\n    model_filename = '/kaggle/input/rsna2024-public/weights/model_pretrained.pytorch'\n    model.load_state_dict(torch.load(model_filename))  \n    # model.encoder.head.fc.reset_parameters()  # random initialize the classification head\n\n    model.to(device)\n    model.train()\n\n    # Optimizer\n    lr = float(cfg['train']['lr'])\n    optimizer = torch.optim.AdamW(model.parameters(), lr=lr,\n                                  weight_decay=weight_decay)\n\n    epochs = 2 if debug else cfg['train']['epochs']\n    scheduler = get_cosine_schedule_with_warmup(optimizer,\n                    num_warmup_steps=nbatch,\n                    num_training_steps=(epochs * nbatch))\n\n    print('%d epochs' % epochs)\n\n    # n-epoch loop\n    tb = time.time()\n    dt_val = 0\n    loss_sum, n_sum = 0, 0\n    \n    print('Epoch  loss          score   lr      time')\n    for iepoch in range(epochs):\n        for ibatch, d in enumerate(loader_train):\n            x = d['x'].to(device)  # input image\n            y = d['y'].to(device)  # segmentation label\n            batch_size = len(x)\n\n            optimizer.zero_grad()\n\n            # Predict\n            y_pred = model(x)      # (batch_size, 3, 25)\n            loss = criterion(y_pred, y)\n\n            # Backpropagate\n            loss.backward()\n            n_sum += batch_size\n            loss_sum += batch_size * loss.item()\n\n            nn.utils.clip_grad_norm_(model.parameters(), max_grad_norm)\n            optimizer.step()\n\n            scheduler.step()\n        \n        # Validation\n        if (iepoch + 1) % val_every == 0:\n            loss_train = loss_sum / n_sum\n\n            val = evaluate(model, loader_val)\n            dt_val += val['dt']\n            \n            lr = optimizer.param_groups[0]['lr']\n\n            dt = time.time() - tb\n            print('%3d %7.4f %7.4f  %.4f  %5.1e %.2f %.2f min' % (iepoch + 1,\n                  loss_train, val['loss'], val['score'],\n                  lr, dt_val / 60, dt / 60))\n\n            loss_sum, n_sum = 0, 0\n\n    # Save model\n    model.to('cpu')\n    model.eval()\n    ofilename = 'model%d.pytorch' % ifold\n    torch.save(model.state_dict(), ofilename)\n\n    print(ofilename, 'written')\n\n\nif debug:\n    print('\\nDebug %r: train only %d epochs.' % (debug, epochs))","metadata":{"execution":{"iopub.status.busy":"2024-07-30T22:04:42.102246Z","iopub.execute_input":"2024-07-30T22:04:42.102526Z","iopub.status.idle":"2024-07-30T22:22:07.471279Z","shell.execute_reply.started":"2024-07-30T22:04:42.102498Z","shell.execute_reply":"2024-07-30T22:22:07.470071Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Inference","metadata":{}},{"cell_type":"code","source":"def add_missing_studies(datad, preds):\n    # I skip study_ids with no Sagittal T1; filling such study_id with 1/3.\n    # Well, I forgot to use this and submission worked, so looks like\n    # Sagittal T1 exists for all test study_ids\n    y_pred = torch.ones((3, 25), dtype=torch.float32) / 3\n    for study_id, d in datad.items():\n        if d['filenames'] is None:\n            pred = {'study_id': study_id,\n                    'y_pred': y_pred}\n            preds.append(pred)\n\n    assert len(preds) == len(datad)\n\n\ndef predict(model, loader, device):\n    preds = []\n    for d in loader:\n        x = d['x'].to(device)  # input image\n        batch_size = len(x)\n\n        # Predict\n        with torch.no_grad():\n            y_pred = model(x)  # (batch_size, 3, 25)\n\n        study_ids = d['study_id']\n        y_pred = y_pred.cpu()  # Tensor[float32] (batch_size, 3, 25)\n        for i in range(batch_size):\n            pred = {'study_id': study_ids[i],\n                    'y_pred': y_pred[i].clone(),  # Tensor[float32] (3, 25)\n            }\n            preds.append(pred)\n\n    return preds\n\n\ndef check_submission(submit):\n    # Check row_ids exactly match those in sample_submission.csv\n    sample = pl.read_csv(di + '/sample_submission.csv')\n\n    assert submit.shape == sample.shape\n    assert submit.columns == sample.columns\n    assert (submit['row_id'] == sample['row_id']).all()\n\n\ndef create_submission(preds, columns):\n    \"\"\"\n    columns list[str]: name of 25 targets, train.columns[1:]\n    \"\"\"\n    assert len(columns) == 25\n\n    rows = []\n    for pred in preds:\n        study_id = pred['study_id']\n        y_pred = pred['y_pred'].to(torch.float64)  # Tensor[float] (3, 25)\n        y_pred = y_pred.softmax(dim=0).numpy()     # logit -> normalized probability\n        assert y_pred.shape == (3, 25)\n\n        for j, name in enumerate(columns):\n            row_id = '%s_%s' % (study_id, name)\n            row = (row_id, ) + tuple(y_pred[:, j])\n            assert len(row) == 4\n            rows.append(row)\n\n    # Sort like the sample submission; not necessary but I double check row_ids match exactly.\n    rows.sort()\n\n    col_names = ('row_id', 'normal_mild', 'moderate', 'severe')\n    submit = pl.DataFrame(rows, schema=col_names, orient='row')\n\n    return submit","metadata":{"execution":{"iopub.status.busy":"2024-07-30T22:22:07.473355Z","iopub.execute_input":"2024-07-30T22:22:07.474200Z","iopub.status.idle":"2024-07-30T22:22:07.488034Z","shell.execute_reply.started":"2024-07-30T22:22:07.474159Z","shell.execute_reply":"2024-07-30T22:22:07.487093Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Compare with the official metric\n\nCheck if my evaluate() function correctly compute the score.","metadata":{}},{"cell_type":"code","source":"model.to(device)\nmodel.eval()\n\n# Check evaluate() score\nval = evaluate(model, loader_val)\nprint('My evaluate() score: %.6f' % val['score'])\n\n# Check the prediction with val data\npreds = predict(model, loader_val, device)\nsubmission = create_submission(preds, train.columns[1:])\n\n# Use the official metric\n# https://www.kaggle.com/code/metric/rsna-lumbar-metric-71549 (version 10)\nsolution1, submission1 = official_metric.create_solution(train, submission)  # this is my code, not official\nscore_official = official_metric.score(solution1.copy(),   # this function destroys dataframes\n                                       submission1.copy(),\n                                       'row_id', any_severe_scalar=1.0)\nprint('Official score for val: %.6f' % score_official)\n\nerr = abs(val['score'] - score_official)\nok = 'OK' if err < 1e-6 else 'too large!'\nprint('Difference %e %s' % (err, ok))","metadata":{"execution":{"iopub.status.busy":"2024-07-30T22:22:07.492418Z","iopub.execute_input":"2024-07-30T22:22:07.492767Z","iopub.status.idle":"2024-07-30T22:22:16.005449Z","shell.execute_reply.started":"2024-07-30T22:22:07.492737Z","shell.execute_reply":"2024-07-30T22:22:16.004217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submit test predictions","metadata":{}},{"cell_type":"code","source":"# Test data\ndatad = load_series_descriptions('test')\nds_test = Dataset(None, datad, cfg, pick='middle')\nloader_test = ds_test.loader(cfg)\nprint('Data', len(ds_test))\n\n# Model trained locally for 20 epochs\nmodel_filename = '/kaggle/input/rsna2024-public/weights/model0.pytorch'\n\nmodel = Model(cfg, pretrained=False)\nmodel.load_state_dict(torch.load(model_filename))\nmodel.to(device)\nmodel.eval()\n\n# Predict\npreds = predict(model, loader_test, device)\n\n# Write submission.csv\nofilename = 'submission.csv'\nsubmit = create_submission(preds, train.columns[1:])\nsubmit.write_csv(ofilename, float_precision=16)\ncheck_submission(submit)\n\nprint('%s written. %d rows' % (ofilename, len(submit)))\n\ndt = time.time() - tb_global\nprint('Total %.1f min' % (dt / 60))","metadata":{"execution":{"iopub.status.busy":"2024-07-30T22:22:16.007186Z","iopub.execute_input":"2024-07-30T22:22:16.007622Z","iopub.status.idle":"2024-07-30T22:22:17.657214Z","shell.execute_reply.started":"2024-07-30T22:22:16.007576Z","shell.execute_reply":"2024-07-30T22:22:17.656102Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! head -n 3 submission.csv","metadata":{"execution":{"iopub.status.busy":"2024-07-30T22:22:17.658501Z","iopub.execute_input":"2024-07-30T22:22:17.658802Z","iopub.status.idle":"2024-07-30T22:22:18.724310Z","shell.execute_reply.started":"2024-07-30T22:22:17.658774Z","shell.execute_reply":"2024-07-30T22:22:18.723007Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}