{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":71549,"databundleVersionId":8561470,"sourceType":"competition"},{"sourceId":9358862,"sourceType":"datasetVersion","datasetId":5674107},{"sourceId":183439689,"sourceType":"kernelVersion"}],"dockerImageVersionId":30733,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"#### RSNA2024 LSDC Training Baseline\nIn the [previous notebook](https://www.kaggle.com/code/itsuki9180/rsna2024-lsdc-making-dataset), We selected the images we wanted to use and exported them to png.\n\nThis notebook will use those images for training.\n\n- version 1 to 5 are old versions\n- version 6 is fixed something about validation metrics\n- version 7 is used efn-b3\n\n### My other Notebooks\n- [RSNA2024 LSDC Making Dataset](https://www.kaggle.com/code/itsuki9180/rsna2024-lsdc-making-dataset) \n- [RSNA2024 LSDC Training Baseline](https://www.kaggle.com/code/itsuki9180/rsna2024-lsdc-training-baseline) <- you're reading now\n- [RSNA2024 LSDC Submission Baseline](https://www.kaggle.com/code/itsuki9180/rsna2024-lsdc-submission-baseline)","metadata":{}},{"cell_type":"markdown","source":"# Import Libralies","metadata":{}},{"cell_type":"code","source":"#  !unzip -q /kaggle/input/rsna2024-lsdc-making-dataset/_output_.zip ","metadata":{"execution":{"iopub.status.busy":"2024-09-17T17:16:46.814557Z","iopub.execute_input":"2024-09-17T17:16:46.81497Z","iopub.status.idle":"2024-09-17T17:17:01.661256Z","shell.execute_reply.started":"2024-09-17T17:16:46.814936Z","shell.execute_reply":"2024-09-17T17:17:01.659733Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport gc\nimport sys\nfrom PIL import Image\nimport cv2\nimport math, random\nimport numpy as np\nimport pandas as pd\nfrom glob import glob\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import KFold\n","metadata":{"execution":{"iopub.status.busy":"2024-09-17T17:17:08.561882Z","iopub.execute_input":"2024-09-17T17:17:08.562329Z","iopub.status.idle":"2024-09-17T17:17:09.871225Z","shell.execute_reply.started":"2024-09-17T17:17:08.562291Z","shell.execute_reply":"2024-09-17T17:17:09.870148Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rd = '/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification'","metadata":{"execution":{"iopub.status.busy":"2024-09-17T17:17:12.380465Z","iopub.execute_input":"2024-09-17T17:17:12.38104Z","iopub.status.idle":"2024-09-17T17:17:12.386152Z","shell.execute_reply.started":"2024-09-17T17:17:12.381004Z","shell.execute_reply":"2024-09-17T17:17:12.385022Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Config","metadata":{}},{"cell_type":"markdown","source":"# Open Dataframes","metadata":{}},{"cell_type":"code","source":"df = pd.read_csv(f'{rd}/train.csv')\ndf","metadata":{"execution":{"iopub.status.busy":"2024-09-17T17:17:15.408193Z","iopub.execute_input":"2024-09-17T17:17:15.409079Z","iopub.status.idle":"2024-09-17T17:17:15.495597Z","shell.execute_reply.started":"2024-09-17T17:17:15.409035Z","shell.execute_reply":"2024-09-17T17:17:15.494522Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Change the state to Label.\n\nThe dataframe contains some Nans, which we will replace with -100 so that We and function can ignore them when calculating the loss and score.","metadata":{}},{"cell_type":"code","source":"df = df.fillna(0)","metadata":{"execution":{"iopub.status.busy":"2024-09-17T17:17:18.651575Z","iopub.execute_input":"2024-09-17T17:17:18.652382Z","iopub.status.idle":"2024-09-17T17:17:18.668144Z","shell.execute_reply.started":"2024-09-17T17:17:18.652342Z","shell.execute_reply":"2024-09-17T17:17:18.666876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label2id = {'Normal/Mild': 0, 'Moderate':1, 'Severe':2}\ndf = df.replace(label2id)\ndf","metadata":{"execution":{"iopub.status.busy":"2024-09-17T17:17:20.567348Z","iopub.execute_input":"2024-09-17T17:17:20.567815Z","iopub.status.idle":"2024-09-17T17:17:20.641683Z","shell.execute_reply.started":"2024-09-17T17:17:20.567779Z","shell.execute_reply":"2024-09-17T17:17:20.640589Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CONDITIONS = [\n    'Spinal Canal Stenosis', \n    'Left Neural Foraminal Narrowing', \n    'Right Neural Foraminal Narrowing',\n    'Left Subarticular Stenosis',\n    'Right Subarticular Stenosis'\n]\n\nLEVELS = [\n    'L1/L2',\n    'L2/L3',\n    'L3/L4',\n    'L4/L5',\n    'L5/S1',\n]\nIMG_SIZE = (256, 256)","metadata":{"execution":{"iopub.status.busy":"2024-09-17T17:17:30.365756Z","iopub.execute_input":"2024-09-17T17:17:30.369357Z","iopub.status.idle":"2024-09-17T17:17:30.382006Z","shell.execute_reply.started":"2024-09-17T17:17:30.369246Z","shell.execute_reply":"2024-09-17T17:17:30.380448Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Define Dataset\n\nThis implementation is very slow and leaves a lot of room for improvement.","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf\nimport numpy as np\nfrom PIL import Image\nimport glob\n\n\nclass RSNA24Dataset(tf.keras.utils.Sequence):\n    def __init__(self, df, phase='train'):\n        self.df = df\n        self.phase = phase\n    \n    def __len__(self):\n        return int(np.ceil(len(self.df)))\n    \n    def __getitem__(self, idx):\n        t = self.df.iloc[idx]      \n        x = np.zeros((IMG_SIZE[0], IMG_SIZE[1], 30), dtype=np.uint8)  # Equivalent to IN_CHANS = 30\n        st_id = int(t['study_id'])\n        label = t[1:].values.astype(np.int64)\n        \n\n        # Sagittal T1\n        for i in range(10):\n            try:\n                p = f'./cvt_png/{st_id}/Sagittal T1/{i:03d}.png'\n                img = Image.open(p).convert('L')\n                img = img.resize(IMG_SIZE)\n                x[..., i] = np.array(img, dtype=np.uint8)\n            except:\n                pass\n        \n        # Sagittal T2/STIR\n        for i in range(10):\n            try:\n                p = f'./cvt_png/{st_id}/Sagittal T2_STIR/{i:03d}.png'\n                img = Image.open(p).convert('L')\n                img = img.resize(IMG_SIZE)\n                x[..., i + 10] = np.array(img, dtype=np.uint8)\n            except:\n                pass\n\n        # Axial T2\n        axt2 = sorted(glob.glob(f'./cvt_png/{st_id}/Axial T2/*.png'))\n        step = len(axt2) / 10.0\n        st = len(axt2) / 2.0 - 4.0 * step\n        end = len(axt2) + 0.0001\n\n        for i, j in enumerate(np.arange(st, end, step)):\n            try:\n                p = axt2[max(0, int(round(j-0.5001)))]\n                img = Image.open(p).convert('L')\n                img = img.resize(IMG_SIZE)\n                x[..., i + 20] = np.array(img, dtype=np.uint8)\n            except:\n                pass\n\n\n\n        x = np.transpose(x, (0, 1, 2))  \n        \n        return x.astype(np.float32), label\n","metadata":{"execution":{"iopub.status.busy":"2024-09-17T17:17:35.362231Z","iopub.execute_input":"2024-09-17T17:17:35.362769Z","iopub.status.idle":"2024-09-17T17:17:50.923734Z","shell.execute_reply.started":"2024-09-17T17:17:35.362732Z","shell.execute_reply":"2024-09-17T17:17:50.922434Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\n\ndataset = RSNA24Dataset(df)\n\ndata_augmentation = tf.keras.Sequential([\n    tf.keras.layers.GaussianNoise(stddev=0.2), \n    tf.keras.layers.RandomTranslation(height_factor=0.1, width_factor=0.1),\n    tf.keras.layers.RandomZoom(height_factor=(-0.1, 0.1), width_factor=(-0.1, 0.1)),\n    tf.keras.layers.RandomRotation(factor=0.1),  # Rotate by 15 degrees (0.1 radians)\n    \n\n])\n\n\nfor i,(x,t) in enumerate(dataset):\n        if i ==1:\n            break\n        x = tf.expand_dims(x, axis=0)\n        x = data_augmentation(x)\n        print('x stat:', x.shape)\n        print(t, t.shape)\n\n\n        y = x.numpy()[0, :, :, 3]/255\n        print(type(x), type(t))\n\n\n        # Display the image\n        plt.imshow(y)\n        plt.show()\n        \n        y = x.numpy()[0, :, :, :3]/255\n        print(type(x), type(t))\n\n\n        # Display the image\n        plt.imshow(y)\n        plt.show()\n        print('y stat:', y.shape, tf.math.reduce_mean(\n    y, axis=None, keepdims=False, name=None\n))\n        print()\n\n        plt.close()\n\n","metadata":{"execution":{"iopub.status.busy":"2024-09-17T17:17:54.785464Z","iopub.execute_input":"2024-09-17T17:17:54.786227Z","iopub.status.idle":"2024-09-17T17:17:56.064631Z","shell.execute_reply.started":"2024-09-17T17:17:54.786181Z","shell.execute_reply":"2024-09-17T17:17:56.063452Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Define Data Augmentaion\nData augmentation is important because the number of images used for training is extremely small.\nSee [this notebook](https://www.kaggle.com/code/haqishen/1st-place-soluiton-code-small-ver) by [Qishen Ha](https://www.kaggle.com/haqishen) for help setting up this augmentation.","metadata":{}},{"cell_type":"markdown","source":"# Trying Data Loader\nChecking if the data loader works properly.","metadata":{}},{"cell_type":"markdown","source":"# Define Model\nWe use timm, which is commonly used for image classification.","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow.keras.applications import InceptionResNetV2\nfrom tensorflow.keras.saving import register_keras_serializable\n\n@register_keras_serializable()\nclass MyModel(tf.keras.Model):  # Inherit from tf.keras.Model\n    def __init__(self):\n        super(MyModel, self).__init__()  # Call the parent constructor\n\n        # Conv2D to reduce the channels from 30 to 3\n        self.conv = tf.keras.layers.Conv2D(3, (1, 1), padding='same')\n        self.bn = tf.keras.layers.BatchNormalization()\n        self.relu = tf.keras.layers.Activation('relu')\n\n        # Load InceptionResNetV2 with pre-trained weights\n        self.base_model = InceptionResNetV2(weights=None, input_shape=(IMG_SIZE[0], IMG_SIZE[0], 3), include_top=False, pooling='avg')\n        self.base_model.load_weights('/kaggle/input/modelweights/RadImageNet-IRV2_notop.h5', by_name=True)\n\n        for layer in self.base_model.layers[:-25]:  # Freeze all layers except the last 10\n            layer.trainable = False\n\n        # Add a dropout layer\n        self.dropout = tf.keras.layers.Dropout(0.1)\n\n        # Output layer for 75 classes (25 diseases * 3 severity levels)\n        self.pred = tf.keras.layers.Dense(75)\n\n    def call(self, inputs):\n        # Forward pass\n        x = self.conv(inputs)\n        x = self.bn(x)\n        x = self.relu(x)\n        x = self.base_model(x)\n        x = self.dropout(x)\n        return self.pred(x)\n\n        \n","metadata":{"execution":{"iopub.status.busy":"2024-09-17T17:18:28.86097Z","iopub.execute_input":"2024-09-17T17:18:28.861451Z","iopub.status.idle":"2024-09-17T17:18:28.876271Z","shell.execute_reply.started":"2024-09-17T17:18:28.861385Z","shell.execute_reply":"2024-09-17T17:18:28.875056Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndef create_tf_dataset(rsna_dataset, batch_size=32, shuffle_buffer_size=100, augment=False):\n    def generator():\n        for i in range(len(rsna_dataset)):\n            x, y = rsna_dataset[i]\n            yield x, y\n\n    dataset = tf.data.Dataset.from_generator(generator,\n                                             output_signature=(\n                                                 tf.TensorSpec(shape=(IMG_SIZE[0], IMG_SIZE[0], 30), dtype=tf.float32),\n                                                 tf.TensorSpec(shape=(25,), dtype=tf.int64)\n                                             ))\n\n    def preprocess_img(x, y):\n        x = x / 255.0 \n\n        if augment:  \n           \n            x = tf.expand_dims(x, axis=0)  \n            x = data_augmentation(x, training=True)\n            x = tf.squeeze(x, axis=0) \n\n        return x, y\n\n    dataset = dataset.map(preprocess_img, num_parallel_calls=tf.data.experimental.AUTOTUNE)\n    dataset = dataset.shuffle(buffer_size=shuffle_buffer_size)\n    dataset = dataset.batch(batch_size)\n    dataset = dataset.prefetch(buffer_size=tf.data.experimental.AUTOTUNE)\n\n    return dataset\n","metadata":{"execution":{"iopub.status.busy":"2024-09-17T17:18:32.541799Z","iopub.execute_input":"2024-09-17T17:18:32.542862Z","iopub.status.idle":"2024-09-17T17:18:32.554171Z","shell.execute_reply.started":"2024-09-17T17:18:32.542822Z","shell.execute_reply":"2024-09-17T17:18:32.552713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\n# Create the dataset object\ndataset = RSNA24Dataset(df, phase='train')\n\n# Assuming total dataset size is 1900\ntotal_size = 1975\n\nindices = np.arange(total_size)\n\n# Split indices for training and testing\ntrain_indices, test_indices = train_test_split(indices, test_size=0.2, random_state=42)\n# Create RSNA24Dataset objects\ntrain_dataset = RSNA24Dataset(df.iloc[train_indices], phase='train')\ntest_dataset = RSNA24Dataset(df.iloc[test_indices], phase='train')\n\n# Create tf.data.Dataset for train and test\ntrain_dataset = create_tf_dataset(rsna_dataset=train_dataset, batch_size=16, augment=True)\n\n# Example of using the function for validation without augmentation\nval_dataset = create_tf_dataset(rsna_dataset=test_dataset, batch_size=8, augment=True)\n\n","metadata":{"execution":{"iopub.status.busy":"2024-09-17T17:18:37.831799Z","iopub.execute_input":"2024-09-17T17:18:37.832988Z","iopub.status.idle":"2024-09-17T17:18:38.485809Z","shell.execute_reply.started":"2024-09-17T17:18:37.832935Z","shell.execute_reply":"2024-09-17T17:18:38.48448Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"   for i,(x,t) in enumerate(val_dataset):\n        if i ==1:\n            break\n        \n\n        print('x stat:', x.shape)\n        print(t, t.shape)\n\n\n        y = x.numpy()[0, :, :, 6] \n        print(type(x), type(t))\n        print(y.shape)\n\n        # Display the image\n        plt.imshow(y)\n        plt.show()\n\n        print('y stat:', y.shape)\n        print()\n\n        plt.close()\n","metadata":{"execution":{"iopub.status.busy":"2024-09-17T17:19:19.890069Z","iopub.execute_input":"2024-09-17T17:19:19.891368Z","iopub.status.idle":"2024-09-17T17:19:44.291898Z","shell.execute_reply.started":"2024-09-17T17:19:19.891319Z","shell.execute_reply":"2024-09-17T17:19:44.290475Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def gradient_step(l,x, model):\n    loss=0\n    with tf.GradientTape() as tape:\n        y = model(x)\n        for col in range(25):\n            pred = y[:,col*3:col*3+3]\n            gt = l[:,col]\n            loss = loss + loss_object(gt, pred) / 25\n                        \n    \n        \n    gradients = tape.gradient(loss, model.trainable_weights)\n    gradients = [tf.clip_by_value(grad, -1.0, 1.0) for grad in gradients]\n    optimizer.apply_gradients(zip(gradients, model.trainable_weights))\n\n    return loss, y    \n    ","metadata":{"execution":{"iopub.status.busy":"2024-09-17T17:20:12.371161Z","iopub.execute_input":"2024-09-17T17:20:12.371751Z","iopub.status.idle":"2024-09-17T17:20:12.380698Z","shell.execute_reply.started":"2024-09-17T17:20:12.371711Z","shell.execute_reply":"2024-09-17T17:20:12.379244Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train loop","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow.keras.optimizers.schedules import CosineDecay\nfrom tqdm import tqdm\nimport numpy as np\n\n# Set mixed precision policy for TensorFlow\ntf.keras.mixed_precision.set_global_policy('mixed_float16')\n\n# Initialize your model\nmodel = MyModel()\nnum_epoch = 15\n# Hyperparameters for CosineDecay\ntotal_steps = num_epoch * 1580\nmin_lr = 1e-6\nlr_schedule = CosineDecay(\n    initial_learning_rate=1e-3,  # Starting LR\n    decay_steps=total_steps,\n    alpha=min_lr / 1e-3  # Final LR after decay\n)\n\n# Initialize the optimizer with the cosine decay scheduler\noptimizer = tf.keras.optimizers.Adam(learning_rate=lr_schedule)\n\n# Initialize loss function\nloss_object = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)\n\n# Training loop with mixed precision and cosine scheduler\npatience = 5  \nwait = 0     \nbest_val_loss = np.Inf\nearly_stop = False\nmin_delta = 0  \n\nlr_patience = 3 \nlr_wait = 0\nlr_factor = 0.5\n\nepochs_val_losses, epochs_train_losses = [], []\n\nfor epoch in range(num_epoch):\n    if early_stop:\n        print(\"Early stopping triggered. Stopping training.\")\n        break\n\n    print(f'Start of epoch {epoch}')\n    losses_t = []\n    pbar = tqdm(total=1580//16, position=0, leave=True, bar_format='{l_bar}{bar}| {n_fmt}/{total_fmt} ')\n\n    for i, (x, l) in enumerate(train_dataset):\n        loss, y = gradient_step(l, x, model)\n        losses_t.append(loss)  \n        pbar.set_description(f\"Training loss for step {i}: {float(loss):.4f}\")\n        pbar.update()\n    \n    print('...............................................................................................................')\n    \n    # Validation\n    losses_v = []\n    for i, (x, l) in enumerate(val_dataset):\n        y = model(x)\n        loss=0\n        for col in range(25):\n            pred = y[:,col*3:col*3+3]\n            gt = l[:,col]\n            loss = loss + loss_object(gt, pred) / 25\n        losses_v.append(loss)\n        \n    losses_train_mean = np.mean(losses_t)\n    losses_val_mean = np.mean(losses_v)\n\n    epochs_val_losses.append(losses_val_mean)\n    epochs_train_losses.append(losses_train_mean)\n\n    print(f'\\nEpoch {epoch}: Train loss: {float(losses_train_mean):.4f}  Validation Loss: {float(losses_val_mean):.4f}')\n\n    # Early stopping check\n    if losses_val_mean < best_val_loss - min_delta:\n        print(f\"Validation loss improved from {best_val_loss:.4f} to {losses_val_mean:.4f}\")\n        best_val_loss = losses_val_mean\n        wait = 0  # Reset patience counter\n    else:\n        wait += 1\n        print(f\"Validation loss did not improve. Patience: {wait}/{patience}\")\n\n        if wait >= patience:\n            print(f\"Early stopping after {epoch+1} epochs\")\n            early_stop = True\n            break\n\n    # Learning rate scheduler - this is handled by the CosineDecay automatically\n    if wait >= lr_patience:\n        # No manual adjustment needed, since the CosineDecay automatically adjusts\n        print(f\"Cosine schedule adjusted \")\n        lr_wait = 0  # Reset learning rate patience\n    else:\n        lr_wait += 1\n","metadata":{"execution":{"iopub.status.busy":"2024-09-17T17:20:51.872502Z","iopub.execute_input":"2024-09-17T17:20:51.873589Z","iopub.status.idle":"2024-09-17T17:29:16.672849Z","shell.execute_reply.started":"2024-09-17T17:20:51.873544Z","shell.execute_reply":"2024-09-17T17:29:16.66951Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_json = model.to_json()\nwith open(\"model_architecture.json\", \"w\") as json_file:\n    json_file.write(model_json)","metadata":{"execution":{"iopub.status.busy":"2024-09-17T12:48:11.882709Z","iopub.execute_input":"2024-09-17T12:48:11.883435Z","iopub.status.idle":"2024-09-17T12:48:11.891717Z","shell.execute_reply.started":"2024-09-17T12:48:11.883401Z","shell.execute_reply":"2024-09-17T12:48:11.890741Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.save_weights('model.weights.h5')","metadata":{"execution":{"iopub.status.busy":"2024-09-17T12:50:56.206227Z","iopub.execute_input":"2024-09-17T12:50:56.206601Z","iopub.status.idle":"2024-09-17T12:50:57.550913Z","shell.execute_reply.started":"2024-09-17T12:50:56.206573Z","shell.execute_reply":"2024-09-17T12:50:57.549914Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"model.save_weights('model_weights.h5')","metadata":{}},{"cell_type":"markdown","source":"# ","metadata":{}},{"cell_type":"markdown","source":"# Calculation CV","metadata":{}},{"cell_type":"code","source":"cv = 0\ny_preds = []\nlabels = []\nweights = torch.tensor([1.0, 2.0, 4.0])\ncriterion2 = nn.CrossEntropyLoss(weight=weights)\n\nfor fold, (trn_idx, val_idx) in enumerate(skf.split(range(len(df)))):\n    print('#'*30)\n    print(f'start fold{fold}')\n    print('#'*30)\n    df_valid = df.iloc[val_idx]\n    valid_ds = RSNA24Dataset(df_valid, phase='valid', transform=transforms_val)\n    valid_dl = DataLoader(\n                valid_ds,\n                batch_size=1,\n                shuffle=False,\n                pin_memory=True,\n                drop_last=False,\n                num_workers=N_WORKERS\n                )\n\n    model = RSNA24Model(MODEL_NAME, IN_CHANS, N_CLASSES, pretrained=False)\n    fname = f'{OUTPUT_DIR}/best_wll_model_fold-{fold}.pt'\n    model.load_state_dict(torch.load(fname))\n    model.to(device)   \n    \n    model.eval()\n    with tqdm(valid_dl, leave=True) as pbar:\n        with torch.no_grad():\n            for idx, (x, t) in enumerate(pbar):\n                \n                x = x.to(device)\n                t = t.to(device)\n                    \n                with autocast:\n                    y = model(x)\n                    for col in range(N_LABELS):\n                        pred = y[:,col*3:col*3+3]\n                        gt = t[:,col] \n                        y_pred = pred.float()\n                        y_preds.append(y_pred.cpu())\n                        labels.append(gt.cpu())\n\ny_preds = torch.cat(y_preds)\nlabels = torch.cat(labels)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cv = criterion2(y_preds, labels)\nprint('cv score:', cv.item())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Calculation Competition Metrics\nThis will give a slightly different score, probably due to the different behavior for nan.","metadata":{}},{"cell_type":"code","source":"from sklearn.metrics import log_loss\ny_pred_np = y_preds.softmax(1).numpy()\nlabels_np = labels.numpy()\ny_pred_nan = np.zeros((y_preds.shape[0], 1))\ny_pred2 = np.concatenate([y_pred_nan, y_pred_np],axis=1)\nweights = []\nfor l in labels:\n    if l==0: weights.append(1)\n    elif l==1: weights.append(2)\n    elif l==2: weights.append(4)\n    else: weights.append(0)\ncv2 = log_loss(labels, y_pred2, normalize=True, sample_weight=weights)\nprint('cv score:', cv2)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.save(f'{OUTPUT_DIR}/labels.npy', labels_np)\nnp.save(f'{OUTPUT_DIR}/final_oof.npy', y_pred2)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# When predictions are random\ngets around 1.1.","metadata":{}},{"cell_type":"code","source":"random_pred = np.ones((y_preds.shape[0], 3)) / 3.0\ny_pred3 = np.concatenate([y_pred_nan, random_pred],axis=1)\ncv3 = log_loss(labels, y_pred3, normalize=True, sample_weight=weights)\nprint('random score:', cv3)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm -r cvt_png","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Continuing with the [Submission Baseline...](https://www.kaggle.com/code/itsuki9180/rsna2024-lsdc-submission-baseline)","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}