{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":71549,"databundleVersionId":8561470,"sourceType":"competition"},{"sourceId":184324498,"sourceType":"kernelVersion"}],"dockerImageVersionId":30732,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Downloading model weights from `wandb`","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"code","source":"import os, gc, sys, copy, pickle\nfrom pathlib import Path\nimport glob\nfrom tqdm.auto import tqdm\ntqdm.pandas()\n\nimport math\nimport random\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\n\nfrom joblib import Parallel, delayed\nimport multiprocessing as mp\n\nimport albumentations as A\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nimport torch.cuda.amp as amp\nimport torch.nn.functional as F\nfrom torch.utils.data import DataLoader, Dataset\nimport torchvision.transforms as transforms\n\nimport pytorch_lightning as pl\nfrom pytorch_lightning.loggers import WandbLogger\nfrom pytorch_lightning.callbacks import LearningRateMonitor, ModelCheckpoint\nfrom pytorch_lightning.callbacks.early_stopping import EarlyStopping\n\nfrom torch.utils.data import WeightedRandomSampler\nfrom sklearn.utils.class_weight import compute_class_weight\n\nimport timm\n\nimport cv2\ncv2.setNumThreads(0)\nimport PIL\nimport pydicom\nimport warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"execution":{"iopub.status.busy":"2024-07-17T08:02:43.346426Z","iopub.execute_input":"2024-07-17T08:02:43.346801Z","iopub.status.idle":"2024-07-17T08:02:55.438990Z","shell.execute_reply.started":"2024-07-17T08:02:43.346772Z","shell.execute_reply":"2024-07-17T08:02:55.437637Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import wandb\nfrom kaggle_secrets import UserSecretsClient\nuser_secrets = UserSecretsClient()\nsecret_value_0 = user_secrets.get_secret(\"WANDB_API_KEY\")\nwandb.login(key=secret_value_0)","metadata":{"execution":{"iopub.status.busy":"2024-07-17T08:02:55.441064Z","iopub.execute_input":"2024-07-17T08:02:55.441401Z","iopub.status.idle":"2024-07-17T08:02:57.506327Z","shell.execute_reply.started":"2024-07-17T08:02:55.441374Z","shell.execute_reply":"2024-07-17T08:02:57.504966Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def seeding(SEED):\n    np.random.seed(SEED)\n    random.seed(SEED)\n    os.environ['PYTHONHASHSEED'] = str(SEED)\n    torch.manual_seed(SEED)\n    if torch.cuda.is_available(): \n        torch.cuda.manual_seed(SEED)\n        torch.cuda.manual_seed_all(SEED)\n        torch.backends.cudnn.deterministic = True\n        torch.backends.cudnn.benchmark = False\n#     os.environ['TF_CUDNN_DETERMINISTIC'] = str(SEED)\n#     tf.random.set_seed(SEED)\n#     keras.utils.set_random_seed(seed=SEED)\n    print('seeding done!!!')\n\ndef flush():\n    gc.collect()\n    if torch.cuda.is_available():\n        torch.cuda.empty_cache()\n        torch.cuda.reset_peak_memory_stats()","metadata":{"execution":{"iopub.status.busy":"2024-07-17T08:02:57.507723Z","iopub.execute_input":"2024-07-17T08:02:57.508270Z","iopub.status.idle":"2024-07-17T08:02:57.515750Z","shell.execute_reply.started":"2024-07-17T08:02:57.508246Z","shell.execute_reply":"2024-07-17T08:02:57.514325Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CONFIG = dict(\n    project_name = \"PL-RSNA-2024-Lumbar-Spine-Classification\",\n    artifact_name = \"rsnaEffNetModel\",\n    load_kernel = None,\n    load_last = True,\n    n_folds = 5,\n    backbone = \"efficientnet_b0.ra_in1k\", # tf_efficientnetv2_s_in21ft1k\n    img_size = 512,\n    n_slice_per_c = 16,\n    in_chans = 1,\n\n    drop_rate = 0.,\n    drop_rate_last = 0.3,\n    drop_path_rate = 0.,\n    p_mixup = 0.5,\n    p_rand_order_v1 = 0.2,\n    lr = 1e-3,\n\n    out_dim = 3,\n    epochs = 15,\n    batch_size = 8,\n    device = torch.device(\"cuda\") if torch.cuda.is_available() else \"cpu\",\n    seed = 2024\n)\n\nseeding(CONFIG['seed'])","metadata":{"execution":{"iopub.status.busy":"2024-07-17T08:02:57.519187Z","iopub.execute_input":"2024-07-17T08:02:57.520397Z","iopub.status.idle":"2024-07-17T08:02:57.532783Z","shell.execute_reply.started":"2024-07-17T08:02:57.520363Z","shell.execute_reply":"2024-07-17T08:02:57.531548Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class TimmModel(nn.Module):\n    def __init__(self, backbone, pretrained=False):\n        super(TimmModel, self).__init__()\n\n        self.encoder = timm.create_model(\n            backbone,\n            in_chans=CONFIG[\"in_chans\"],\n            num_classes=CONFIG[\"out_dim\"],\n            features_only=False,\n            drop_rate=CONFIG[\"drop_rate\"],\n            drop_path_rate=CONFIG[\"drop_path_rate\"],\n            pretrained=pretrained\n        )\n\n        if 'efficient' in backbone:\n            hdim = self.encoder.conv_head.out_channels\n            self.encoder.classifier = nn.Identity()\n        elif 'convnext' in backbone:\n            hdim = self.encoder.head.fc.in_features\n            self.encoder.head.fc = nn.Identity()\n\n\n        self.lstm = nn.LSTM(hdim, 256, num_layers=2, dropout=CONFIG[\"drop_rate\"], bidirectional=True, batch_first=True)\n        self.head = nn.Sequential(\n            nn.Linear(512, 256),\n            nn.BatchNorm1d(256),\n            nn.Dropout(CONFIG[\"drop_rate_last\"]),\n            nn.LeakyReLU(0.1),\n            nn.Linear(256, CONFIG[\"out_dim\"]),\n        )\n\n    def forward(self, x):  # (bs, nslice, ch, sz, sz)\n        bs = x.shape[0]\n        x = x.view(bs * CONFIG[\"n_slice_per_c\"], CONFIG[\"in_chans\"], CONFIG[\"img_size\"], CONFIG[\"img_size\"])\n        feat = self.encoder(x)\n        feat = feat.view(bs, CONFIG[\"n_slice_per_c\"], -1)\n        feat, _ = self.lstm(feat)\n        feat = feat.contiguous().view(bs * CONFIG[\"n_slice_per_c\"], -1)\n        feat = self.head(feat)\n        feat = feat.view(bs, CONFIG[\"n_slice_per_c\"], CONFIG[\"out_dim\"]).contiguous()\n\n        return feat","metadata":{"execution":{"iopub.status.busy":"2024-07-17T08:02:57.534153Z","iopub.execute_input":"2024-07-17T08:02:57.534509Z","iopub.status.idle":"2024-07-17T08:02:57.547737Z","shell.execute_reply.started":"2024-07-17T08:02:57.534483Z","shell.execute_reply":"2024-07-17T08:02:57.546660Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class TimmModel2(nn.Module):\n    def __init__(self, backbone, pretrained=False):\n        super(TimmModel2, self).__init__()\n\n        self.encoder = timm.create_model(\n            backbone,\n            num_classes=CONFIG[\"out_dim\"],\n            features_only=False,\n            drop_rate=CONFIG[\"drop_rate\"],\n            drop_path_rate=CONFIG[\"drop_path_rate\"],\n            pretrained=pretrained\n        )\n\n        if 'efficient' in backbone:\n            hdim = self.encoder.conv_head.out_channels\n            self.encoder.classifier = nn.Identity()\n        elif 'convnext' in backbone:\n            hdim = self.encoder.head.fc.in_features\n            self.encoder.head.fc = nn.Identity()\n\n\n        self.lstm = nn.LSTM(hdim, 256, num_layers=2, dropout=CONFIG[\"drop_rate\"], bidirectional=True, batch_first=True)\n        self.head = nn.Sequential(\n            nn.Linear(512, 256),\n            nn.BatchNorm1d(256),\n            nn.Dropout(CONFIG[\"drop_rate_last\"]),\n            nn.LeakyReLU(0.1),\n            nn.Linear(256, CONFIG[\"out_dim\"]),\n        )\n\n    def forward(self, x):\n        feat = self.encoder(x)\n        feat, _ = self.lstm(feat)\n        feat = self.head(feat)\n        return feat","metadata":{"execution":{"iopub.status.busy":"2024-07-17T08:02:57.549000Z","iopub.execute_input":"2024-07-17T08:02:57.549317Z","iopub.status.idle":"2024-07-17T08:02:57.567015Z","shell.execute_reply.started":"2024-07-17T08:02:57.549292Z","shell.execute_reply":"2024-07-17T08:02:57.565744Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class LumbarLightningModel2(pl.LightningModule):\n    def __init__(self, pretrained=False):\n        self.save_hyperparameters()\n        super().__init__()\n        self.model = TimmModel2(backbone=CONFIG[\"backbone\"], pretrained=pretrained)\n        class_weights = torch.tensor([1, 2, 4], dtype=torch.float32)\n        self.loss_fn = nn.CrossEntropyLoss(weight=class_weights)\n    \n    def forward(self, images):\n        return self.model(images)\n    \n    def shared_step(self, batch):\n        images, labels = batch[0], batch[1]\n        logits = self.forward(images)\n        loss = self.loss_fn(logits, labels.to(torch.int64))\n        return loss\n    \n    def training_step(self, batch, batch_idx):\n        loss = self.shared_step(batch)\n        self.log(\"train_loss\", loss, on_step=True, on_epoch=True, prog_bar=True)\n        return loss\n    \n    def validation_step(self, batch, batch_idx):\n        loss = self.shared_step(batch)\n        self.log(\"valid_loss\", loss, on_step=True, on_epoch=True, prog_bar=True)\n        return loss\n    \n    def configure_optimizers(self):\n        optimizer = torch.optim.Adam(self.parameters(), lr=CONFIG['lr'])\n        scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=CONFIG[\"epochs\"], eta_min=0)\n        return [optimizer], [scheduler]","metadata":{"execution":{"iopub.status.busy":"2024-07-17T08:02:57.568380Z","iopub.execute_input":"2024-07-17T08:02:57.568869Z","iopub.status.idle":"2024-07-17T08:02:57.588388Z","shell.execute_reply.started":"2024-07-17T08:02:57.568846Z","shell.execute_reply":"2024-07-17T08:02:57.587139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Cross validation","metadata":{}},{"cell_type":"code","source":"DATA_PATH = Path(\"/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification\")\nos.listdir(DATA_PATH)","metadata":{"execution":{"iopub.status.busy":"2024-07-17T08:02:57.590149Z","iopub.execute_input":"2024-07-17T08:02:57.590605Z","iopub.status.idle":"2024-07-17T08:02:57.610615Z","shell.execute_reply.started":"2024-07-17T08:02:57.590573Z","shell.execute_reply":"2024-07-17T08:02:57.609213Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_main = pd.read_csv(DATA_PATH/\"train.csv\")\ntrain_desc = pd.read_csv(DATA_PATH/\"train_series_descriptions.csv\")\ntrain_label_coordinates = pd.read_csv(DATA_PATH/\"train_label_coordinates.csv\")","metadata":{"execution":{"iopub.status.busy":"2024-07-17T08:02:57.612475Z","iopub.execute_input":"2024-07-17T08:02:57.612965Z","iopub.status.idle":"2024-07-17T08:02:57.768715Z","shell.execute_reply.started":"2024-07-17T08:02:57.612931Z","shell.execute_reply":"2024-07-17T08:02:57.767461Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define function to reshape a single row of the DataFrame\ndef reshape_row(row):\n    data = {'study_id': [], 'condition': [], 'level': [], 'severity': []}\n    \n    for column, value in row.items():\n        if column not in ['study_id', 'series_id', 'instance_number', 'x', 'y', 'series_description']:\n            parts = column.split('_')\n            condition = ' '.join([word.capitalize() for word in parts[:-2]])\n            level = parts[-2].capitalize() + '/' + parts[-1].capitalize()\n            data['study_id'].append(row['study_id'])\n            data['condition'].append(condition)\n            data['level'].append(level)\n            data['severity'].append(value)\n    \n    return pd.DataFrame(data)\n\n# Reshape the DataFrame for all rows\nnew_train_df = pd.concat([reshape_row(row) for _, row in train_main.iterrows()], ignore_index=True)\n\n# Display the first few rows of the reshaped dataframe\nnew_train_df.head(5)","metadata":{"execution":{"iopub.status.busy":"2024-07-17T08:02:57.772986Z","iopub.execute_input":"2024-07-17T08:02:57.773414Z","iopub.status.idle":"2024-07-17T08:02:58.981134Z","shell.execute_reply.started":"2024-07-17T08:02:57.773378Z","shell.execute_reply":"2024-07-17T08:02:58.979581Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Merge the dataframes on the common columns\nmerged_df = pd.merge(new_train_df, train_label_coordinates, on=['study_id', 'condition', 'level'], how='inner')\nfinal_merged_df = pd.merge(merged_df, train_desc, on=['series_id','study_id'], how='inner')\n\n# Create the row_id column\nfinal_merged_df['row_id'] = (\n    final_merged_df['study_id'].astype(str) + '_' +\n    final_merged_df['condition'].str.lower().str.replace(' ', '_') + '_' +\n    final_merged_df['level'].str.lower().str.replace('/', '_')\n)\n\n# Create the image_path column\nfinal_merged_df['image_path'] = (\n    f'{str(DATA_PATH)}/train_images/' + \n    final_merged_df['study_id'].astype(str) + '/' +\n    final_merged_df['series_id'].astype(str) + '/' +\n    final_merged_df['instance_number'].astype(str) + '.dcm'\n)\n\nfinal_merged_df['severity'] = final_merged_df['severity'].map(\n    {'Normal/Mild': 'normal_mild', 'Moderate': 'moderate', 'Severe': 'severe'}\n)\n\ntrain_data = final_merged_df.copy()\n# Display the updated dataframe\ntrain_data.head(5)","metadata":{"execution":{"iopub.status.busy":"2024-07-17T08:02:58.982479Z","iopub.execute_input":"2024-07-17T08:02:58.982801Z","iopub.status.idle":"2024-07-17T08:02:59.240486Z","shell.execute_reply.started":"2024-07-17T08:02:58.982774Z","shell.execute_reply":"2024-07-17T08:02:59.239254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define a function to check if a path exists\ndef check_exists(path):\n    return os.path.exists(path)\n\n# Define a function to check if a study ID directory exists\ndef check_study_id(row):\n    study_id = row['study_id']\n    path = f'{str(DATA_PATH)}/train_images/{study_id}'\n    return check_exists(path)\n\n# Define a function to check if a series ID directory exists\ndef check_series_id(row):\n    study_id = row['study_id']\n    series_id = row['series_id']\n    path = f'{str(DATA_PATH)}/train_images/{study_id}/{series_id}'\n    return check_exists(path)\n\n# Define a function to check if an image file exists\ndef check_image_exists(row):\n    image_path = row['image_path']\n    return check_exists(image_path)\n\n# Apply the functions to the train_data dataframe\ntrain_data['study_id_exists'] = train_data.progress_apply(check_study_id, axis=1)\ntrain_data['series_id_exists'] = train_data.progress_apply(check_series_id, axis=1)\ntrain_data['image_exists'] = train_data.progress_apply(check_image_exists, axis=1)\n\n# Filter train_data\ntrain_data = train_data[(train_data['study_id_exists']) & (train_data['series_id_exists']) & (train_data['image_exists'])]","metadata":{"execution":{"iopub.status.busy":"2024-07-17T08:02:59.241794Z","iopub.execute_input":"2024-07-17T08:02:59.242141Z","iopub.status.idle":"2024-07-17T08:03:43.060780Z","shell.execute_reply.started":"2024-07-17T08:02:59.242116Z","shell.execute_reply":"2024-07-17T08:03:43.059756Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label2id = {v: i for i, v in enumerate(train_data['severity'].unique())}\nid2label = {v:k for k,v in label2id.items()}\ntrain_data['target'] = train_data['severity'].map(label2id)\ntrain_data = train_data.dropna(subset=['severity']).reset_index(drop=True)\n\n# series2id = {v:i for i, v in enumerate(train_data['series_description'].unique().tolist())}\n# id2series = {v:k for k,v in series2id.items()}\n# train_data['series2id'] = train_desc['series_description'].map(series2id)\n# train_data = train_data.dropna(subset=['series2id']).reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2024-07-17T08:03:43.062069Z","iopub.execute_input":"2024-07-17T08:03:43.062333Z","iopub.status.idle":"2024-07-17T08:03:43.092494Z","shell.execute_reply.started":"2024-07-17T08:03:43.062312Z","shell.execute_reply":"2024-07-17T08:03:43.091054Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_dicom(path):\n    dicom = pydicom.read_file(path)\n    data = dicom.pixel_array\n    data = data - np.min(data)\n    if np.max(data) != 0:\n        data = data / np.max(data)\n    data = (data * 255).astype(np.uint8)\n    return data","metadata":{"execution":{"iopub.status.busy":"2024-07-17T08:03:43.093648Z","iopub.execute_input":"2024-07-17T08:03:43.093911Z","iopub.status.idle":"2024-07-17T08:03:43.099424Z","shell.execute_reply.started":"2024-07-17T08:03:43.093890Z","shell.execute_reply":"2024-07-17T08:03:43.098278Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CustomDataset(Dataset):\n    def __init__(self, dataframe, transform=None, label_name='target'):\n        self.dataframe = dataframe\n        self.transform = transform\n        self.label = dataframe.loc[:, label_name]\n\n    def __len__(self):\n        return len(self.dataframe)\n\n    def __getitem__(self, index):\n        image_path = self.dataframe['image_path'][index]\n        image = load_dicom(image_path)  # Define this function to load your DICOM images\n        target = self.dataframe['target'][index]\n        \n        if self.transform:\n            image = cv2.cvtColor(image, cv2.COLOR_GRAY2BGR)\n            image = self.transform(image=image)['image']\n            image = image.transpose(2, 0, 1).astype(np.float32) / 255.\n\n        return image, torch.tensor(target).float()\n    \n    def get_labels(self):\n        return self.label","metadata":{"execution":{"iopub.status.busy":"2024-07-17T08:03:43.101029Z","iopub.execute_input":"2024-07-17T08:03:43.101430Z","iopub.status.idle":"2024-07-17T08:03:43.115183Z","shell.execute_reply.started":"2024-07-17T08:03:43.101401Z","shell.execute_reply":"2024-07-17T08:03:43.113728Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_transforms(height, width):\n    train_tsfm = A.Compose([\n        # Geometric augmentations\n#         A.Perspective(p=0.5),\n        A.HorizontalFlip(p=0.5),\n        A.VerticalFlip(p=0.5),\n#         A.RandomRotate90(p=0.5),\n        A.Rotate(-30, 30, p=0.5),\n        \n        A.Resize(height=height, width=width),\n    ])\n    \n    valid_tsfm = A.Compose([\n        A.Resize(height=height, width=width),\n    ])\n    return {\"train\": train_tsfm, \"eval\": valid_tsfm}\n\ndef get_dataloaders(data, cfg, split=\"train\"):\n    img_size = cfg['img_size']\n    height, width = img_size, img_size\n    tsfm = get_transforms(height=height, width=width)\n    if split == 'train':\n        tr_tsfm = tsfm['train']\n        ds = CustomDataset(data, transform=tr_tsfm)\n        labels = ds.get_labels()\n#         class_weights = torch.tensor(compute_class_weight(class_weight=\"balanced\", classes=np.unique(labels), y=labels))\n        class_weights = torch.tensor([1, 2, 4])\n        samples_weights = class_weights[labels]\n#         print(class_weights)\n        sampler = WeightedRandomSampler(weights=samples_weights, \n                                        num_samples=len(samples_weights), \n                                        replacement=True)\n\n        dls = DataLoader(ds, \n                         batch_size=cfg['batch_size'], \n                         sampler=sampler, \n                         num_workers=os.cpu_count(), \n                         drop_last=True, \n                         pin_memory=True)\n        \n    elif split == 'valid' or split == 'test':\n        eval_tsfm = tsfm['eval']\n        ds = CustomDataset(data, transform=eval_tsfm)\n        dls = DataLoader(ds, \n                         batch_size=2*cfg['batch_size'], \n                         shuffle=False, \n                         num_workers=os.cpu_count(), \n                         drop_last=False, \n                         pin_memory=True)\n    else:\n        raise Exception(\"Split should be 'train' or 'valid' or 'test'!!!\")\n    return dls","metadata":{"execution":{"iopub.status.busy":"2024-07-17T08:03:43.116881Z","iopub.execute_input":"2024-07-17T08:03:43.117335Z","iopub.status.idle":"2024-07-17T08:03:43.135166Z","shell.execute_reply.started":"2024-07-17T08:03:43.117296Z","shell.execute_reply":"2024-07-17T08:03:43.134119Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn import model_selection\n\nkfold = model_selection.StratifiedKFold(n_splits=5, shuffle=True, random_state=2024)\nx = train_data.index.values\ny = train_data['target'].values.astype(int)\n# g = train_data['series2id'].values.astype(int)\n\ntrain_data['fold'] = -1\nfor fold, (tr_idx, val_idx) in enumerate(kfold.split(x,y)):\n    train_data.loc[val_idx, 'fold'] = fold\n    \ntrain_data['fold'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2024-07-17T08:03:43.136279Z","iopub.execute_input":"2024-07-17T08:03:43.136607Z","iopub.status.idle":"2024-07-17T08:03:43.178645Z","shell.execute_reply.started":"2024-07-17T08:03:43.136580Z","shell.execute_reply":"2024-07-17T08:03:43.176816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"FLIPS = [None, [-1], [-2], [-2, -1]]\n\ndef inference_loop(model, loader):\n    model.to(CONFIG[\"device\"])\n    model.eval()\n    preds = np.empty((0, 3))\n    with torch.no_grad():\n        for batch in tqdm(loader):\n            images, labels = batch\n            images = images.to(CONFIG[\"device\"], non_blocking=True)\n            with torch.autocast(device_type=\"cuda\", dtype=torch.float16):\n#                 logits = model(images.to(torch.float32))\n                logits = model(images)\n                logits = logits.softmax(dim=-1)\n                preds = np.concatenate([preds, logits.detach().cpu().numpy()])\n#     np.save('preds.npy', preds)\n    return preds\n\n\n\ndef tta_inference_loop(model, loader):\n    model.to(CONFIG[\"device\"])\n    model.eval()\n    preds = np.empty((0, 3))\n    with torch.no_grad():\n        for batch in tqdm(loader):\n            images, labels = batch\n            images = images.to(CONFIG[\"device\"], non_blocking=True)\n            pred_tta = []\n            with torch.autocast(device_type=\"cuda\", dtype=torch.float16):\n                for f in FLIPS:\n                    logits = model(torch.flip(images, f) if f is not None else images)\n                    logits = logits.softmax(dim=-1)\n                    pred_tta.append(logits.detach().cpu().numpy())\n#                 preds = np.concatenate([preds, logits.detach().cpu().numpy()])\n                preds = np.concatenate([preds, np.mean(pred_tta, 0)])\n#     np.save('preds.npy', preds)\n    return preds","metadata":{"execution":{"iopub.status.busy":"2024-07-17T08:03:43.180484Z","iopub.execute_input":"2024-07-17T08:03:43.180982Z","iopub.status.idle":"2024-07-17T08:03:43.192871Z","shell.execute_reply.started":"2024-07-17T08:03:43.180942Z","shell.execute_reply":"2024-07-17T08:03:43.191501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"LB: 0.98, CV: 81, LR=8e-4\n```python\nimport wandb\nrun = wandb.init()\nartifact = run.use_artifact('samu2505/RSNA-2024-Lumbar-Spine-Classification-3Chan/rsnaEffNetModel_0:v19', type='model')\nartifact_dir = artifact.download()\n```","metadata":{}},{"cell_type":"code","source":"import wandb\nrun = wandb.init()\nartifact = run.use_artifact('samu2505/RSNA-2024-Lumbar-Spine-Classification-Torch/rsnaEffNetModel_0:v71', type='model')\nartifact_dir = artifact.download()","metadata":{"execution":{"iopub.status.busy":"2024-07-17T08:03:43.194506Z","iopub.execute_input":"2024-07-17T08:03:43.194921Z","iopub.status.idle":"2024-07-17T08:04:01.533233Z","shell.execute_reply.started":"2024-07-17T08:03:43.194869Z","shell.execute_reply":"2024-07-17T08:04:01.531894Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # checkpoints = glob.glob(f\"{artifact_dir}/*.ckpt\")[0]\n# # lit_model2 = LumbarLightningModel2.load_from_checkpoint(checkpoints)\n# # torch.save(lit_model2.model.state_dict(), \"model_weights.pth\")\n# # weights_path = \"/kaggle/working/model_weights.pth\"\nmodel = TimmModel2(backbone=CONFIG[\"backbone\"], pretrained=False)\nweights_path = glob.glob(f\"{artifact_dir}/*.pth\")[0]\n# weights_path = \"/kaggle/input/rsna-pytorch-train-lb-0-84-cv-0-54/rsna_2024_lumbar_spine_fold_0_epoch_14.pth\"\nweights = torch.load(weights_path, map_location=torch.device(\"cpu\"))\nmodel.load_state_dict(weights)\ntorch.save(model.state_dict(), \"model_weights.pth\")","metadata":{"execution":{"iopub.status.busy":"2024-07-17T08:04:01.535226Z","iopub.execute_input":"2024-07-17T08:04:01.535568Z","iopub.status.idle":"2024-07-17T08:04:01.849418Z","shell.execute_reply.started":"2024-07-17T08:04:01.535537Z","shell.execute_reply":"2024-07-17T08:04:01.848518Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# label_arr = train_data['target'].values\n# oof_pred_arr = np.zeros((len(train_data), 3))\n# score_list = []\n\n# for fold in range(5):\n#     print(f\"\\nFold {fold}\")\n#     valid_ds = train_data[train_data['fold'] == fold]\n#     val_idx = valid_ds.index.values\n#     valid_ds = valid_ds.reset_index(drop=True)\n#     valid_loader = get_dataloaders(valid_ds, CONFIG, split=\"valid\")\n    \n# #     checkpoints = glob.glob(f\"{artifact_dir}/*.ckpt\")[0]\n# #     lit_model2 = LumbarLightningModel2.load_from_checkpoint(checkpoints)\n# #     torch.save(lit_model2.model.state_dict(), \"model_weights.pth\")\n#     model = TimmModel2(backbone=CONFIG[\"backbone\"], pretrained=False)\n# #     weights_path = \"/kaggle/working/model_weights.pth\"\n# #     weights_path = glob.glob(f\"{artifact_dir}/*.pth\")[0]\n#     weights_path = \"/kaggle/input/rsna-pytorch-train-lb-0-84-cv-0-54/rsna_2024_lumbar_spine_fold_0_epoch_14.pth\"\n#     weights = torch.load(weights_path, map_location=torch.device(\"cpu\"))\n#     model.load_state_dict(weights)\n    \n#     # inference\n# #     val_pred = inference_loop(model, valid_loader)\n#     val_pred = tta_inference_loop(model, valid_loader)\n#     oof_pred_arr[val_idx] = val_pred\n    \n#     del valid_ds, val_idx, valid_loader\n#     del model\n#     flush()","metadata":{"execution":{"iopub.status.busy":"2024-07-17T08:04:01.850602Z","iopub.execute_input":"2024-07-17T08:04:01.850923Z","iopub.status.idle":"2024-07-17T08:04:01.858347Z","shell.execute_reply.started":"2024-07-17T08:04:01.850897Z","shell.execute_reply":"2024-07-17T08:04:01.857125Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from sklearn.metrics import log_loss\n# log_loss(label_arr, oof_pred_arr)","metadata":{"execution":{"iopub.status.busy":"2024-07-17T08:04:01.860083Z","iopub.execute_input":"2024-07-17T08:04:01.860705Z","iopub.status.idle":"2024-07-17T08:04:01.875885Z","shell.execute_reply.started":"2024-07-17T08:04:01.860624Z","shell.execute_reply":"2024-07-17T08:04:01.874440Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}