{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.14"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":71549,"databundleVersionId":8561470,"sourceType":"competition"},{"sourceId":1188216,"sourceType":"datasetVersion","datasetId":675800},{"sourceId":9255944,"sourceType":"datasetVersion","datasetId":5600033},{"sourceId":9255958,"sourceType":"datasetVersion","datasetId":5600041},{"sourceId":9268478,"sourceType":"datasetVersion","datasetId":5608848},{"sourceId":9268489,"sourceType":"datasetVersion","datasetId":5608856}],"dockerImageVersionId":30761,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"! pip install -q --no-index --find-links /kaggle/input/lightningwheels lightning \n! pip install -q --no-index --find-links /kaggle/input/torchevalwheels torcheval ","metadata":{"execution":{"iopub.status.busy":"2024-09-10T17:29:15.308629Z","iopub.execute_input":"2024-09-10T17:29:15.308911Z","iopub.status.idle":"2024-09-10T17:29:43.539305Z","shell.execute_reply.started":"2024-09-10T17:29:15.308880Z","shell.execute_reply":"2024-09-10T17:29:43.538277Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport sys\nimport torch\nimport shutil\n\nsys.path.insert(0, \"/kaggle/input/rsna-src\")\nsys.path.insert(0, \"/kaggle/input/omegaconf\")\n#sys.path.insert(0, \"../\")\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\nfrom tqdm.notebook import tqdm\nfrom collections import defaultdict\nfrom omegaconf import OmegaConf, DictConfig\nfrom torcheval.metrics.functional import multiclass_confusion_matrix \nfrom sklearn.metrics import ConfusionMatrixDisplay, precision_recall_fscore_support\nfrom pandas.api.types import is_numeric_dtype\n\n\nfrom rsna.model import Model, ModelConfig, MLPConfig, MLP, Unet\nfrom rsna.data import DataModule, DataConfig, Metadata\nfrom rsna.data.io import StudyData\nfrom rsna.transformations import Preprocessing\n\ntorch.set_float32_matmul_precision(\"high\")","metadata":{"execution":{"iopub.status.busy":"2024-09-10T17:29:43.541128Z","iopub.execute_input":"2024-09-10T17:29:43.541477Z","iopub.status.idle":"2024-09-10T17:29:51.690387Z","shell.execute_reply.started":"2024-09-10T17:29:43.541430Z","shell.execute_reply":"2024-09-10T17:29:51.689467Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_root = \"/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification\"\ncheckpoint_path = \"/kaggle/input/rsna-unet-v0-weights/version_0/checkpoints/epoch=9-step=26240.ckpt\"\nhparams_file = \"/kaggle/input/rsna-unet-v0-weights/version_0/logs/hparams.yaml\"\nconfig_path = \"/kaggle/input/rsna-unet-v0-weights/version_0/17-04-19/.hydra/config.yaml\"","metadata":{"execution":{"iopub.status.busy":"2024-09-10T17:29:51.691611Z","iopub.execute_input":"2024-09-10T17:29:51.692046Z","iopub.status.idle":"2024-09-10T17:29:51.696991Z","shell.execute_reply.started":"2024-09-10T17:29:51.692012Z","shell.execute_reply":"2024-09-10T17:29:51.695940Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## initializing the metadata\nconfig = OmegaConf.load(config_path)\ndata_config = config.data\ndata_config[\"data_root\"] = \"/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification\"\ndata_config = DataConfig(**data_config)\nmetadata = Metadata(data_config)","metadata":{"execution":{"iopub.status.busy":"2024-09-10T17:29:51.699018Z","iopub.execute_input":"2024-09-10T17:29:51.699307Z","iopub.status.idle":"2024-09-10T17:29:52.105459Z","shell.execute_reply.started":"2024-09-10T17:29:51.699275Z","shell.execute_reply":"2024-09-10T17:29:52.104670Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## loading sample submission\nsample_submission_path = os.path.join(data_config.data_root, \"sample_submission.csv\")\nsample_submission_df = pd.read_csv(sample_submission_path)","metadata":{"execution":{"iopub.status.busy":"2024-09-10T17:29:52.106586Z","iopub.execute_input":"2024-09-10T17:29:52.106953Z","iopub.status.idle":"2024-09-10T17:29:52.115694Z","shell.execute_reply.started":"2024-09-10T17:29:52.106912Z","shell.execute_reply":"2024-09-10T17:29:52.114760Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## initializing model\nmodel = Model.load_from_checkpoint(checkpoint_path = checkpoint_path, hparams_file = hparams_file)\nmodel = model.eval()","metadata":{"execution":{"iopub.status.busy":"2024-09-10T17:29:52.116868Z","iopub.execute_input":"2024-09-10T17:29:52.117179Z","iopub.status.idle":"2024-09-10T17:29:54.039283Z","shell.execute_reply.started":"2024-09-10T17:29:52.117145Z","shell.execute_reply":"2024-09-10T17:29:54.038505Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## initializing the preprocessing\npreprocessing = Preprocessing(data_config)\nstudy_pp = preprocessing.init_fixed_pp()","metadata":{"execution":{"iopub.status.busy":"2024-09-10T17:29:54.040507Z","iopub.execute_input":"2024-09-10T17:29:54.040876Z","iopub.status.idle":"2024-09-10T17:29:54.046292Z","shell.execute_reply.started":"2024-09-10T17:29:54.040835Z","shell.execute_reply":"2024-09-10T17:29:54.045380Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## running predictions on test data\ntest_preds = defaultdict(list)\n## retrieving the test studies\ntest_images_dir = os.path.join(data_root, \"test_images\")\ntest_studies = [int(study_id) for study_id in os.listdir(test_images_dir)]\n## iterating over the test studies\nfor study_id in test_studies:\n    study = StudyData(\n        study_id, \n        test_images_dir,\n        voi_lut = data_config.voi_lut,\n        fix_monochrome = data_config.fix_monochrome,\n        metadata_df = None,\n        label_df = None\n    )\n    study_data = study.load_study()\n    study_data = study_pp(study_data)\n    ## iterating over each volume of the study\n    for series_id, series_data in study_data.items():\n        ## iterating over each slice of the volume\n        for instance_id, instance_data in series_data.items():\n            ## retrieving the input image\n            image = instance_data[\"image_data\"]\n            ## adding dummy batch dimension to the input\n            image = torch.unsqueeze(image, dim = 0)\n            ## moving the input tensor to the target device\n            image = image.to(model.device)\n            with torch.no_grad():\n                ## encoding image\n                bottleneck, enc1, enc2, enc3, enc4 = model.unet.encode(image)\n                flattened_bottleneck = model.unet.flatten_bottleneck(bottleneck)\n            ## flattening the bottleneck\n            logits = model.classification_head(flattened_bottleneck)\n            ## storing the predicted logits\n            test_preds[study_id].append(logits)","metadata":{"execution":{"iopub.status.busy":"2024-09-10T17:29:54.047409Z","iopub.execute_input":"2024-09-10T17:29:54.047790Z","iopub.status.idle":"2024-09-10T17:29:57.866163Z","shell.execute_reply.started":"2024-09-10T17:29:54.047759Z","shell.execute_reply":"2024-09-10T17:29:57.865391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## defining example label for referencing its shape\nexample_label = torch.zeros((1, model.config.num_classes))\n## aggregating predictions\ntest_preds_avg = {key: torch.mean(torch.stack(val, dim = 0), dim = (0, 1)) for key, val in test_preds.items()}\ntest_preds_probs = {key: torch.squeeze(model.predict_probs(val, example_label)) for key, val in test_preds_avg.items()}\n## retrieving conditions and labels\nconditions = sorted(metadata.conditions_map.keys())\nlabels = [\"normal_mild\", \"moderate\", \"severe\"]\n## setting the display format for the floating point numbers\npd.set_option('display.float_format', lambda x: '%.6f' % x)\nrows = []\n## iterating over the number of studies\nfor study_id in test_studies:\n    for condition in conditions:\n        condition_id = metadata.conditions_map[condition]\n        row_id = f\"{study_id}_{condition}\"\n        row_data = {\"row_id\": row_id}\n        for label_id, label in enumerate(labels):\n            row_data[label] = test_preds_probs[study_id][condition_id, label_id].item()\n            #row_data[label] = 1/3\n        rows.append(row_data)\nsubmission_df = pd.DataFrame(rows)\n## retrieving the row-wise sums\n#tot_rows = np.sum(submission_df[[\"normal_mild\", \"moderate\", \"severe\"]].values, axis = 1)\n## normalizing columns to sum to 1\n#submission_df[\"normal_mild\"] = np.true_divide(submission_df[\"normal_mild\"], tot_rows)\n#submission_df[\"moderate\"] = np.true_divide(submission_df[\"moderate\"], tot_rows)\n#submission_df[\"severe\"] = np.true_divide(submission_df[\"severe\"], tot_rows)\nsubmission_df = submission_df.fillna(1./3) #Rohit Chaudhari\nv = submission_df[['normal_mild','moderate','severe']].values\nv = v/v.sum(1).reshape(-1,1)\nsubmission_df[['normal_mild','moderate','severe']] = v\n## dumping submission data\nsubmission_df.to_csv(\"submission.csv\", index = False) \n#submission_df","metadata":{"execution":{"iopub.status.busy":"2024-09-10T17:29:57.867317Z","iopub.execute_input":"2024-09-10T17:29:57.867643Z","iopub.status.idle":"2024-09-10T17:29:57.926200Z","shell.execute_reply.started":"2024-09-10T17:29:57.867610Z","shell.execute_reply":"2024-09-10T17:29:57.925225Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## asserting that the row_id column has unique values\nassert len(test_studies) * model.config.num_classes == submission_df[\"row_id\"].nunique()\n## asserting that all the values in the submission df are numerics\nfor label in labels:\n    assert is_numeric_dtype(submission_df[label])\n## asserting that all the values in the submission df are at least zero\nfor label in labels:\n    assert (submission_df[label] >= 0).all()\n## asserting that all the values in the submission df are finite\nfor label in labels:\n    assert (np.isfinite(submission_df[label])).all()\n## asserting that all the values in the submission df are not NAN\n#for label in labels:\n#    assert not (np.isnan(submission_df[label])).all()\n## checking against sample submission\nassert set(sample_submission_df.row_id.to_list()) == set(submission_df.row_id.to_list())\n## assert uniqueness of row_id\nassert submission_df[\"row_id\"].is_unique\n## assert that the rows sum to 1\n#assert np.isclose(np.sum(submission_df[[\"normal_mild\", \"moderate\", \"severe\"]].values, axis = 1), 1).all()\n## assert that the rows of sample submission sum to 1\nassert (np.sum(sample_submission_df[[\"normal_mild\", \"moderate\", \"severe\"]].values, axis = 1) == 1).all()","metadata":{"execution":{"iopub.status.busy":"2024-09-10T17:29:57.928822Z","iopub.execute_input":"2024-09-10T17:29:57.929128Z","iopub.status.idle":"2024-09-10T17:29:57.939645Z","shell.execute_reply.started":"2024-09-10T17:29:57.929095Z","shell.execute_reply":"2024-09-10T17:29:57.938505Z"},"trusted":true},"execution_count":null,"outputs":[]}]}