{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":71549,"databundleVersionId":8561470,"sourceType":"competition"},{"sourceId":9612988,"sourceType":"datasetVersion","datasetId":5837762},{"sourceId":9612998,"sourceType":"datasetVersion","datasetId":5823579},{"sourceId":9613008,"sourceType":"datasetVersion","datasetId":5823786},{"sourceId":9612983,"sourceType":"datasetVersion","datasetId":5850614},{"sourceId":191111786,"sourceType":"kernelVersion"}],"dockerImageVersionId":30747,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install keras==\"2.15.0\" --no-index --find-links=file:///kaggle/input/keras215/keras2150/ ","metadata":{"execution":{"iopub.status.busy":"2024-10-13T07:09:03.819338Z","iopub.execute_input":"2024-10-13T07:09:03.819699Z","iopub.status.idle":"2024-10-13T07:09:19.122576Z","shell.execute_reply.started":"2024-10-13T07:09:03.819672Z","shell.execute_reply":"2024-10-13T07:09:19.121549Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport glob\nimport warnings\nimport sys\nimport json\nimport time\nimport psutil\nfrom contextlib import contextmanager\nimport shutil\nimport glob\nimport copy\n\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport cv2\nimport pydicom\nimport numpy as np\nfrom tqdm import tqdm","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-10-13T07:09:19.124893Z","iopub.execute_input":"2024-10-13T07:09:19.125306Z","iopub.status.idle":"2024-10-13T07:09:19.864399Z","shell.execute_reply.started":"2024-10-13T07:09:19.125270Z","shell.execute_reply":"2024-10-13T07:09:19.863572Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def show_memory_usage(name = \"unknown\"):\n    vm = psutil.virtual_memory()\n    print(f\"[MEMUSE] memory usage (in {name}): {vm.used/1024/1024:.2f}MB ({vm.percent}%)\")\n\n@contextmanager\ndef timer(name: str):\n    show_memory_usage(f\"before {name}\")\n    s = time.time()\n    yield\n    elapsed = time.time() - s\n    print(f\"[{name}] {elapsed:.3f}sec\")\n    show_memory_usage(f\"after {name}\")\n\n\nclass Config:\n    phase = \"test\"\n    debug = False\n    run_all_models = True\n    \n    path = {\"cache\": \"cache/\",\n            \"sample_sub\": \"/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/sample_submission.csv\",            \n            \"train\": \"/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train.csv\",\n            \"train_coord\": \"/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_label_coordinates.csv\",\n            \"train_description\": \"/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_series_descriptions.csv\",\n            \"test_description\": \"/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/test_series_descriptions.csv\",\n            \"train_img\": \"/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_images/\",\n            \"test_img\": \"/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/test_images/\",\n           }\n\n    # data params (duplicate\n    seed = 111\n    num_label = 25\n    num_cls = 3\n    num_cls_det = 25\n    data_path = \"/kaggle/working/train_merge_3rows.csv\"\n    img_dir = \"/workspaces/rsna2024l/data/train_images\"\n\n    # cls\n    crop_1st_pred = True\n    conf_thresh_to_run_classifier = 0.1\n    \n    # fold split\n    base_fold_path = \"/kaggle/working/fold.csv\"\n    fold_split_by = \"study_id\"\n    use_3ch = False\n    \n\n\nwarnings.simplefilter('ignore')\nsys.path.insert(1, \"/kaggle/input/rsna-src/src\")\nConfig.img_dir = f\"/kaggle/{Config.phase}_images/\"\nos.makedirs(Config.path[\"cache\"], exist_ok=True)","metadata":{"execution":{"iopub.status.busy":"2024-10-13T07:09:19.865475Z","iopub.execute_input":"2024-10-13T07:09:19.865897Z","iopub.status.idle":"2024-10-13T07:09:19.875973Z","shell.execute_reply.started":"2024-10-13T07:09:19.865871Z","shell.execute_reply":"2024-10-13T07:09:19.874847Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## prediction","metadata":{}},{"cell_type":"code","source":"from train import inference, postprocess, score_validation, make_crop_dataset_from_1st_stage, inference_classifier, merge_clspred_with_1st_stage, inference_classifier_w_crop_and_merge\n\ndef single_prediction(config:object, \n                      weight_dir_1st:str, \n                      weight_dir_cls:str,\n                      weight_dir_2nd:str) -> pd.DataFrame:\n    \n    cache_val_output_path = \"/kaggle/working/df_test_outputs.csv\"\n    inference(config, weight_dir_1st, cache_val_output_path, test_mode=True)\n#     display(pd.read_csv(cache_val_output_path).iloc[:,25:35])\n    if config.crop_1st_pred:  # use 1st stage prediction to crop instead of original images croppped by true label\n        save_path_crop = \"cache/df_crop_dataset.csv\"\n        save_crop_image_dir = \"cache/crop_images/\"\n        path_cls_infer = \"cache/df_val_outputs_cls.csv\"\n        \n        inference_classifier_w_crop_and_merge(\n                config=config, weight_dir_cls=weight_dir_cls, path_1st_pred=cache_val_output_path, test_mode=True\n            )\n\n#     display(pd.read_csv(cache_val_output_path).iloc[:,25:35])\n    df_pred = postprocess(config, cache_val_output_path, second_weight_dir=weight_dir_2nd, test_mode=True)\n    return df_pred\n\ndef ensemble_prediction(config:object, \n                      weight_dirs_1st:list[str], \n                      weight_dirs_cls:list[str], \n                      weight_dirs_2nd:list[str], \n                        num_channels:list[int], # temporal\n                      ) -> pd.DataFrame:\n    df_pred_ensemble = None\n    num_ensemble = len(weight_dirs_1st)\n    print(f\"ensemble {num_ensemble} models\")\n    pred_cols = [\"pred_0\",\"pred_1\",\"pred_2\"]\n    for w1,wc,w2,ch in zip(weight_dirs_1st, weight_dirs_cls, weight_dirs_2nd, num_channels):\n        config.use_3ch = ch==3\n        df_pred = single_prediction(config, w1, wc, w2)\n        if df_pred_ensemble is None:\n            df_pred_ensemble = df_pred.copy()\n        else:\n            df_pred_ensemble[pred_cols] += df_pred[pred_cols].to_numpy()\n    df_pred_ensemble[pred_cols] /= num_ensemble\n    return df_pred_ensemble\n","metadata":{"execution":{"iopub.status.busy":"2024-10-13T07:12:24.278950Z","iopub.execute_input":"2024-10-13T07:12:24.279745Z","iopub.status.idle":"2024-10-13T07:12:43.200227Z","shell.execute_reply.started":"2024-10-13T07:12:24.279714Z","shell.execute_reply":"2024-10-13T07:12:43.199333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## postprocess (severe only)","metadata":{}},{"cell_type":"code","source":"def severe_simple_postprocess(df, clip_value=1e-3):\n    \n    weight_base = 0.5\n    weight_update = 1- weight_base\n\n    not_pred2 = 1.0 - df[\"pred_2\"].values\n    not_pred2 = np.reshape(not_pred2, [-1,5,5])\n    not_pred2 = np.prod(not_pred2, axis=-1)\n    not_pred2 = np.tile(not_pred2[:,:,np.newaxis], (1,1,5))\n    df[\"not_pred_2\"] = not_pred2.flatten()\n    max_pred2 = df.groupby([\"study_id\", \"cond_id\"])[[\"pred_2\"]].max().reset_index()\n    max_pred2.columns = [\"study_id\", \"cond_id\", \"pred_2_max\"]\n    df = df.merge(max_pred2, on=[\"study_id\", \"cond_id\"], how=\"left\")\n    df[\"is_max_pred_2\"] = df[\"pred_2\"] == df[\"pred_2_max\"]    \n\n    scs_max_mask = np.logical_and(df[\"cond_id\"] == 0, df[\"is_max_pred_2\"])\n    df.loc[scs_max_mask, \"pred_2\"] = (1 - df[\"not_pred_2\"]) * weight_update + df[\"pred_2\"] * weight_base\n    p0 = df[\"pred_0\"].values\n    p1 = df[\"pred_1\"].values\n    p2 = df[\"pred_2\"].values\n    df[\"pred_0\"] = p0/(p0+p1) * (1-p2)\n    df[\"pred_1\"] = p1/(p0+p1) * (1-p2)\n    df = df.drop(columns=[\"not_pred_2\", \"is_max_pred_2\"])\n\n    if clip_value is not None:\n        preds = df[[\"pred_0\",\"pred_1\",\"pred_2\"]].values\n        print(\"min_pred:\", np.min(preds))\n        print(\"clip by \", clip_value)\n        preds = np.maximum(preds, clip_value)\n        preds = preds / preds.sum(axis=-1, keepdims=True)\n        df[[\"pred_0\",\"pred_1\",\"pred_2\"]] = preds\n                       \n    return df\n\n","metadata":{"execution":{"iopub.status.busy":"2024-10-13T07:12:43.201906Z","iopub.execute_input":"2024-10-13T07:12:43.202564Z","iopub.status.idle":"2024-10-13T07:12:43.214316Z","shell.execute_reply.started":"2024-10-13T07:12:43.202536Z","shell.execute_reply":"2024-10-13T07:12:43.213228Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## train_data -> solution","metadata":{}},{"cell_type":"code","source":"def add_sample_weight_on_gt(df):\n    df[\"sample_weight\"] = df[\"normal_mild\"] * 1 + df[\"moderate\"] * 2 + df[\"severe\"] * 4\n    return df\n\ndef train_df_to_solution_df(df):\n    new_df = []\n    df = df.fillna(\"no_label\") # temporary\n    for col in df.columns:\n        if col == \"study_id\":\n            continue\n        mini_df = df[[\"study_id\", col]].copy()\n        mini_df[\"study_id\"] = mini_df[\"study_id\"].astype(str)\n        mini_df[\"normal_mild\"] = (mini_df[col] == \"Normal/Mild\").astype(float)\n        mini_df[\"moderate\"] = (mini_df[col] == \"Moderate\").astype(float)\n        mini_df[\"severe\"] = (mini_df[col] == \"Severe\").astype(float)\n        mini_df[\"row_id\"] = mini_df['study_id'] + (\"_\"+col)\n        mini_df[\"target\"] = mini_df[col].replace({\"Normal/Mild\":0, \"Moderate\":1, \"Severe\":2})\n        new_df.append(mini_df[[\"row_id\",\"normal_mild\",\"moderate\",\"severe\"]])#,\"target\"]])\n    new_df = pd.concat(new_df, axis=0).reset_index(drop=True)\n    new_df = add_sample_weight_on_gt(new_df)\n    return new_df\n\n","metadata":{"execution":{"iopub.status.busy":"2024-10-13T07:12:43.215771Z","iopub.execute_input":"2024-10-13T07:12:43.216136Z","iopub.status.idle":"2024-10-13T07:12:43.239679Z","shell.execute_reply.started":"2024-10-13T07:12:43.216096Z","shell.execute_reply":"2024-10-13T07:12:43.238730Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## pred->submission","metadata":{}},{"cell_type":"code","source":"def map_cond_id(df):\n    map_dict = {0:'spinal_canal_stenosis', \n         1:'right_neural_foraminal_narrowing',\n         2:'left_neural_foraminal_narrowing', \n         3:'left_subarticular_stenosis',\n         4:'right_subarticular_stenosis'\n        }\n    df[\"cond_id\"] = df[\"cond_id\"].replace(map_dict).astype(str)\n    return df\n\ndef map_level_id(df):\n    map_dict = {0:\"l1_l2\",1: \"l2_l3\", 2:\"l3_l4\", 3:\"l4_l5\", 4:\"l5_s1\"}\n    df[\"level_id\"] = df[\"level_id\"].replace(map_dict).astype(str)\n    return df\n\ndef map_study_id(df):\n    # tuple?? something wrong in my code?\n    df[\"study_id\"] = df[\"study_id\"].apply(lambda x: str(x)[1:-2]).astype(str)\n    return df\n\ndef make_row_id(df):\n    df['row_id'] = df['study_id'].str.cat(df['cond_id'], sep='_').str.cat(df['level_id'], sep='_')\n    return df\n\ndef rename_preds(df):\n    df = df.rename(columns={\"pred_0\":\"normal_mild\", \"pred_1\":\"moderate\", \"pred_2\":\"severe\"})\n    return df\n    \ndef convert_local_pred_to_submission(df):\n    df = map_study_id(df)\n    df = map_cond_id(df)\n    df = map_level_id(df)\n    df = rename_preds(df)\n    df = make_row_id(df)\n    return df[[\"row_id\",\"normal_mild\",\"moderate\",\"severe\"]]#,\"target\"]]\n\n    \ndef load_local_preds(paths):\n    df_val = pd.concat([pd.read_csv(p) for p in paths], axis=0).reset_index(drop=True)\n    return df_val\n\ndef sort_submit(sample_df, submission_df, save_as=\"submission.csv\"):\n    sample_df_row = sample_df[[\"row_id\"]].copy()\n    submission = pd.merge(sample_df_row, submission_df, on='row_id', how='left')\n    submission = submission.fillna(1./3.)    \n    print(submission.shape)\n    display(submission)\n    submission.to_csv(save_as, index=False)\n    \n","metadata":{"execution":{"iopub.status.busy":"2024-10-13T07:12:43.242050Z","iopub.execute_input":"2024-10-13T07:12:43.242637Z","iopub.status.idle":"2024-10-13T07:12:43.256425Z","shell.execute_reply.started":"2024-10-13T07:12:43.242599Z","shell.execute_reply":"2024-10-13T07:12:43.255525Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## MAIN","metadata":{}},{"cell_type":"markdown","source":"### dataset","metadata":{}},{"cell_type":"code","source":"from data.data_preparation import prepare_dataset, add_sub_labels\n\nprepare_dataset(\n        img_dir=Config.path[f\"{Config.phase}_img\"],#\"/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_images\",\n        description_path=Config.path[f\"{Config.phase}_description\"],#\"/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_series_descriptions.csv\",\n        img_save_dir=Config.img_dir,\n        dataframe_save_path=Config.data_path,\n        label_path=Config.path[\"train\"] if Config.phase==\"train\" else None,\n        coords_path=Config.path[\"train_coord\"] if Config.phase==\"train\" else None,\n        debug=Config.debug\n    )\nif Config.phase==\"train\":\n    add_sub_labels(\n        df_train=pd.read_csv(Config.data_path),\n        save_path=Config.data_path,\n        dist_threshold=1.0,\n    )","metadata":{"execution":{"iopub.status.busy":"2024-10-13T07:12:43.257565Z","iopub.execute_input":"2024-10-13T07:12:43.257940Z","iopub.status.idle":"2024-10-13T07:12:46.191207Z","shell.execute_reply.started":"2024-10-13T07:12:43.257907Z","shell.execute_reply":"2024-10-13T07:12:46.190086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# for sumall(debug) test data\ntmp = pd.read_csv(Config.data_path)\ntmp[[\"relative_x0\", \"relative_y0\", \"relative_z0\"]] = tmp[[\"relative_x0\", \"relative_y0\", \"relative_z0\"]].fillna(0.5)\nfor key in tmp.columns:\n    if tmp[key].isna().sum()>0:\n        print(key)\ntmp.to_csv(Config.data_path, index=False)","metadata":{"execution":{"iopub.status.busy":"2024-10-11T13:05:17.080978Z","iopub.execute_input":"2024-10-11T13:05:17.081287Z","iopub.status.idle":"2024-10-11T13:05:17.101838Z","shell.execute_reply.started":"2024-10-11T13:05:17.081233Z","shell.execute_reply":"2024-10-11T13:05:17.101066Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\nnum_ensemble = 5\nweight_dirs_1st = [f\"/kaggle/input/rsna1006-effv2b1-448-1ch/exp016_002_effv2b1_448_1ch_unlabel50_sigmoid_accm1_f{foldno}/\" for foldno in range(num_ensemble)]\nweight_dirs_cls = [f\"/kaggle/input/rsna1006-effv2b1-448-1ch/exp016_002_CLS_effv2b1_128_s10_6ep_crop2016_1ch_batch224_R_f{foldno}/\" for foldno in range(num_ensemble)]\nweight_dirs_2nd = [f\"/kaggle/input/rsna1006-effv2b1-448-1ch/exp016_002_2nd_3body_f{foldno}/\" for foldno in range(num_ensemble)]\nnum_channels = [1] * num_ensemble\n\nif Config.run_all_models:\n    num_ensemble = 6\n    weight_dirs_1st += [f\"/kaggle/input/rsna1006-convnext-480-3ch/exp016_007_convnexttiny_480_3ch_unlabel50_sigmoid_accm2_f{foldno}/\" for foldno in range(num_ensemble)]\n    weight_dirs_cls += [f\"/kaggle/input/rsna1006-convnext-480-3ch/exp016_007_CLS_effv2b1_128_s10_6ep_crop2016_3ch_batch224_R_f{foldno}/\" for foldno in range(num_ensemble)]\n    weight_dirs_2nd += [f\"/kaggle/input/rsna1006-convnext-480-3ch/exp016_007_2nd_3body_f{foldno}/\" for foldno in range(num_ensemble)]\n    num_channels += [3] * num_ensemble\n\n    num_ensemble = 5\n    weight_dirs_1st += [f\"/kaggle/input/rsna1006-effv2b1-448-3ch/exp016_003_effv2b1_448_3ch_unlabel50_sigmoid_accm2_f{foldno}/\" for foldno in range(num_ensemble)]\n    weight_dirs_cls += [f\"/kaggle/input/rsna1006-effv2b1-448-3ch/exp016_003_CLS_effv2b1_128_s10_6ep_crop2016_3ch_batch224_R_f{foldno}/\" for foldno in range(num_ensemble)]\n    weight_dirs_2nd += [f\"/kaggle/input/rsna1006-effv2b1-448-3ch/exp016_003_2nd_3body_f{foldno}/\" for foldno in range(num_ensemble)]\n    num_channels += [3] * num_ensemble\n\ndf_pred_pre = ensemble_prediction(Config, weight_dirs_1st, weight_dirs_cls, weight_dirs_2nd, num_channels)\ndf_pred = severe_simple_postprocess(df_pred_pre.copy())\nif Config.phase==\"train\":\n    score = score_validation(df_pred_pre)\n    print(score)\n    score_pp = score_validation(df_pred)\n    print(score_pp)\n","metadata":{"execution":{"iopub.status.busy":"2024-10-11T13:05:17.103199Z","iopub.execute_input":"2024-10-11T13:05:17.103514Z","iopub.status.idle":"2024-10-11T13:06:54.579983Z","shell.execute_reply.started":"2024-10-11T13:05:17.103491Z","shell.execute_reply":"2024-10-11T13:06:54.578458Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### prediction","metadata":{}},{"cell_type":"markdown","source":"### make submission","metadata":{}},{"cell_type":"code","source":"df_pred_all = convert_local_pred_to_submission(df_pred)\ndisplay(df_pred_all)\nprint(df_pred_all.shape)\nsub_sample = pd.read_csv(Config.path[\"sample_sub\"])\nsort_submit(sub_sample, df_pred_all)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}