{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":71549,"databundleVersionId":8561470,"sourceType":"competition"},{"sourceId":9195731,"sourceType":"datasetVersion","datasetId":5559249},{"sourceId":193161758,"sourceType":"kernelVersion"}],"dockerImageVersionId":30746,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport pandas as pd\nimport numpy as np\nimport pydicom\nfrom tqdm.auto import tqdm\nimport matplotlib.pyplot as plt\nimport cv2\nimport glob","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-09-27T11:31:28.878861Z","iopub.execute_input":"2024-09-27T11:31:28.879604Z","iopub.status.idle":"2024-09-27T11:31:29.923817Z","shell.execute_reply.started":"2024-09-27T11:31:28.879552Z","shell.execute_reply":"2024-09-27T11:31:29.922637Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IMG_DIR = \"/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_images\"","metadata":{"execution":{"iopub.status.busy":"2024-09-27T11:31:29.926069Z","iopub.execute_input":"2024-09-27T11:31:29.926691Z","iopub.status.idle":"2024-09-27T11:31:29.936353Z","shell.execute_reply.started":"2024-09-27T11:31:29.926650Z","shell.execute_reply":"2024-09-27T11:31:29.934621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"FOLDS = [0,1,2,3,4]\nOD_INPUT_SIZE = 384\nSTD_BOX_SIZE = 20\nSAMPLE = None\nCONDITIONS = ['Left Subarticular Stenosis', 'Right Subarticular Stenosis']\nSEVERITIES = ['Normal/Mild', 'Moderate', 'Severe']\nLEVELS = ['l1_l2', 'l2_l3', 'l3_l4', 'l4_l5', 'l5_s1']\n","metadata":{"execution":{"iopub.status.busy":"2024-09-27T11:31:29.938609Z","iopub.execute_input":"2024-09-27T11:31:29.939351Z","iopub.status.idle":"2024-09-27T11:31:29.949546Z","shell.execute_reply.started":"2024-09-27T11:31:29.939318Z","shell.execute_reply":"2024-09-27T11:31:29.948290Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# rm -rf val_fold0","metadata":{"execution":{"iopub.status.busy":"2024-09-27T11:31:29.952336Z","iopub.execute_input":"2024-09-27T11:31:29.953292Z","iopub.status.idle":"2024-09-27T11:31:29.961274Z","shell.execute_reply.started":"2024-09-27T11:31:29.953258Z","shell.execute_reply":"2024-09-27T11:31:29.960204Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_val_df = pd.read_csv('/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train.csv')\ntrain_xy = pd.read_csv('/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_label_coordinates.csv')\ntrain_des = pd.read_csv('/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_series_descriptions.csv')","metadata":{"execution":{"iopub.status.busy":"2024-09-27T11:31:29.962617Z","iopub.execute_input":"2024-09-27T11:31:29.962955Z","iopub.status.idle":"2024-09-27T11:31:30.152503Z","shell.execute_reply.started":"2024-09-27T11:31:29.962927Z","shell.execute_reply":"2024-09-27T11:31:30.150707Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if SAMPLE:\n    train_val_df = train_val_df.sample(SAMPLE, random_state=2698)","metadata":{"execution":{"iopub.status.busy":"2024-09-27T11:31:30.154269Z","iopub.execute_input":"2024-09-27T11:31:30.155665Z","iopub.status.idle":"2024-09-27T11:31:30.161708Z","shell.execute_reply.started":"2024-09-27T11:31:30.155623Z","shell.execute_reply":"2024-09-27T11:31:30.160214Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fold_df = pd.read_csv('/kaggle/input/lsdc-fold-split/5folds.csv')","metadata":{"execution":{"iopub.status.busy":"2024-09-27T11:31:30.164290Z","iopub.execute_input":"2024-09-27T11:31:30.164825Z","iopub.status.idle":"2024-09-27T11:31:30.187506Z","shell.execute_reply.started":"2024-09-27T11:31:30.164781Z","shell.execute_reply":"2024-09-27T11:31:30.186166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_xy.head(3)","metadata":{"execution":{"iopub.status.busy":"2024-09-27T11:31:30.328629Z","iopub.execute_input":"2024-09-27T11:31:30.329610Z","iopub.status.idle":"2024-09-27T11:31:30.353542Z","shell.execute_reply.started":"2024-09-27T11:31:30.329563Z","shell.execute_reply":"2024-09-27T11:31:30.352432Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_level(text):\n    for lev in ['l1_l2', 'l2_l3', 'l3_l4', 'l4_l5', 'l5_s1']:\n        if lev in text:\n            split = lev.split('_')\n            split[0] = split[0].capitalize()\n            split[1] = split[1].capitalize()\n            return '/'.join(split)\n    raise ValueError('Level not found '+ lev)\n    \ndef get_condition(text):\n    split = text.split('_')\n    for i in range(len(split)):\n        split[i] = split[i].capitalize()\n    split = split[:-2]\n    return ' '.join(split)\n#     raise ValueError('Condition not found '+ lev)","metadata":{"execution":{"iopub.status.busy":"2024-09-27T11:31:30.959270Z","iopub.execute_input":"2024-09-27T11:31:30.959673Z","iopub.status.idle":"2024-09-27T11:31:30.968277Z","shell.execute_reply.started":"2024-09-27T11:31:30.959643Z","shell.execute_reply":"2024-09-27T11:31:30.967112Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_xy['condition'].unique()","metadata":{"execution":{"iopub.status.busy":"2024-09-27T11:31:31.148616Z","iopub.execute_input":"2024-09-27T11:31:31.149444Z","iopub.status.idle":"2024-09-27T11:31:31.164662Z","shell.execute_reply.started":"2024-09-27T11:31:31.149410Z","shell.execute_reply":"2024-09-27T11:31:31.163396Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_df = train_df.dropna()","metadata":{"execution":{"iopub.status.busy":"2024-09-27T11:31:31.368192Z","iopub.execute_input":"2024-09-27T11:31:31.368600Z","iopub.status.idle":"2024-09-27T11:31:31.374000Z","shell.execute_reply.started":"2024-09-27T11:31:31.368551Z","shell.execute_reply":"2024-09-27T11:31:31.372805Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label_df = {'study_id':[], 'condition': [], 'level':[], 'label':[]}\n\nfor i, row in train_val_df.iterrows():\n    study_id = row['study_id']\n    for k, label in row.iloc[1:].to_dict().items():\n        level = get_level(k)\n        condition = get_condition(k)\n        label_df['study_id'].append(study_id)\n        label_df['condition'].append(condition)\n        label_df['level'].append(level)\n        label_df['label'].append(label)\n#         break\n#     break\n\nlabel_df = pd.DataFrame(label_df)\nlabel_df = label_df.merge(fold_df, on='study_id')","metadata":{"execution":{"iopub.status.busy":"2024-09-27T11:31:31.566557Z","iopub.execute_input":"2024-09-27T11:31:31.567465Z","iopub.status.idle":"2024-09-27T11:31:32.299823Z","shell.execute_reply.started":"2024-09-27T11:31:31.567431Z","shell.execute_reply":"2024-09-27T11:31:32.298551Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_xy = train_xy.merge(train_des, how='inner', on=['study_id', 'series_id'])\nlabel_df = label_df.merge(train_xy, how='inner', on=['study_id', 'condition', 'level'])","metadata":{"execution":{"iopub.status.busy":"2024-09-27T11:31:32.302169Z","iopub.execute_input":"2024-09-27T11:31:32.302931Z","iopub.status.idle":"2024-09-27T11:31:32.402341Z","shell.execute_reply.started":"2024-09-27T11:31:32.302870Z","shell.execute_reply":"2024-09-27T11:31:32.401178Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# cnt[cnt>1]","metadata":{"execution":{"iopub.status.busy":"2024-09-27T11:31:32.403816Z","iopub.execute_input":"2024-09-27T11:31:32.404253Z","iopub.status.idle":"2024-09-27T11:31:32.409271Z","shell.execute_reply.started":"2024-09-27T11:31:32.404216Z","shell.execute_reply":"2024-09-27T11:31:32.408141Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def query_train_xy_row(study_id, series_id=None, instance_num=None):\n    if series_id is not None and instance_num is not None:\n        return label_df[(label_df.study_id==study_id) & (label_df.series_id==series_id) &\n            (label_df.instance_number==instance_num)]\n    elif series_id is None and instance_num is None:\n        return label_df[(label_df.study_id==study_id)]\n    else:\n        return label_df[(train_xy.study_id==study_id) & (label_df.series_id==series_id)]","metadata":{"execution":{"iopub.status.busy":"2024-09-27T11:31:32.411513Z","iopub.execute_input":"2024-09-27T11:31:32.411883Z","iopub.status.idle":"2024-09-27T11:31:32.422394Z","shell.execute_reply.started":"2024-09-27T11:31:32.411854Z","shell.execute_reply":"2024-09-27T11:31:32.421132Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import os\n\n# def count_dcm_files(directory):\n#     dcm_count = 0\n#     for root, dirs, files in os.walk(directory):\n#         for file in files:\n#             if file.endswith('.dcm'):\n#                 dcm_count += 1\n#     return dcm_count\n\n# dcm_files_count = count_dcm_files(IMG_DIR)\n\n# print(f\"Number of .dcm files: {dcm_files_count}\")","metadata":{"execution":{"iopub.status.busy":"2024-09-27T11:31:32.525940Z","iopub.execute_input":"2024-09-27T11:31:32.526367Z","iopub.status.idle":"2024-09-27T11:31:32.531482Z","shell.execute_reply.started":"2024-09-27T11:31:32.526333Z","shell.execute_reply":"2024-09-27T11:31:32.530101Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_dcm(src_path):\n    dicom_data = pydicom.dcmread(src_path)\n    image = dicom_data.pixel_array\n    image = (image - image.min()) / (image.max() - image.min() +1e-6) * 255\n    image = np.stack([image]*3, axis=-1).astype('uint8')\n    return image\n\ndef get_accronym(text):\n    split = text.split(' ')\n    return ''.join([x[0] for x in split])","metadata":{"execution":{"iopub.status.busy":"2024-09-27T11:31:32.738033Z","iopub.execute_input":"2024-09-27T11:31:32.738484Z","iopub.status.idle":"2024-09-27T11:31:32.745920Z","shell.execute_reply.started":"2024-09-27T11:31:32.738453Z","shell.execute_reply":"2024-09-27T11:31:32.744782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# study_id = 4003253 \n# series_id = 2448190387\n# instance_num = 28\n\nex = label_df.sample(1).iloc[0]\nstudy_id = ex.study_id\nseries_id = ex.series_id\ninstance_num = ex.instance_number\n\nWIDTH = 10\n\npath = os.path.join(IMG_DIR, str(study_id), str(series_id), f'{instance_num}.dcm')","metadata":{"execution":{"iopub.status.busy":"2024-09-27T11:31:34.298237Z","iopub.execute_input":"2024-09-27T11:31:34.298679Z","iopub.status.idle":"2024-09-27T11:31:34.309134Z","shell.execute_reply.started":"2024-09-27T11:31:34.298645Z","shell.execute_reply":"2024-09-27T11:31:34.307906Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img = read_dcm(path)\nxWIDTH = img.shape[0]*10/384\nyWIDTH = img.shape[1]*10/384\n\ntmp_df = query_train_xy_row(study_id, series_id, instance_num)\nfor i, row in tmp_df.iterrows():\n    lbl = f\"{get_accronym(row['condition'])}_{row['level']}\"\n    x, y = row['x'], row['y']\n    x1 = int(x - xWIDTH)\n    x2 = int(x + xWIDTH)\n    y1 = int(y - yWIDTH)\n    y2 = int(y + yWIDTH)\n    color = None\n    if row['label'] == 'Normal/Mild':\n        color =  (0, 255, 0)\n    elif row['label'] == 'Moderate':\n        color = (255,255,0) \n    elif row['label'] == 'Severe':\n        color = (255,0,0)\n        \n    fontFace = cv2.FONT_HERSHEY_SIMPLEX\n    fontScale = 0.5\n    thickness = 1\n    cv2.rectangle(img, (x1,y1), (x2,y2), color, 2)\n    cv2.putText(img, lbl, (x1,y1), fontFace, fontScale, color, thickness, cv2.LINE_AA)\n\ntmp_df","metadata":{"execution":{"iopub.status.busy":"2024-09-27T11:31:36.426118Z","iopub.execute_input":"2024-09-27T11:31:36.426882Z","iopub.status.idle":"2024-09-27T11:31:36.506701Z","shell.execute_reply.started":"2024-09-27T11:31:36.426847Z","shell.execute_reply":"2024-09-27T11:31:36.505494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(img)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-09-27T11:31:38.205935Z","iopub.execute_input":"2024-09-27T11:31:38.206325Z","iopub.status.idle":"2024-09-27T11:31:38.562874Z","shell.execute_reply.started":"2024-09-27T11:31:38.206297Z","shell.execute_reply":"2024-09-27T11:31:38.561495Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# label_df[['study_id', 'series_id']].drop_duplicates()","metadata":{"execution":{"iopub.status.busy":"2024-09-27T11:31:38.693035Z","iopub.execute_input":"2024-09-27T11:31:38.693435Z","iopub.status.idle":"2024-09-27T11:31:38.698396Z","shell.execute_reply.started":"2024-09-27T11:31:38.693407Z","shell.execute_reply":"2024-09-27T11:31:38.697241Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from scipy.ndimage import zoom\n\nimport albumentations as A\nimport cv2\n\n# 定义数据增强变换\ntransform = A.Compose([\n#     A.RandomRotate90(p=0.5),  # 有 50% 的概率随机旋转 90 度\n#     A.HorizontalFlip(p=0.5),  # 有 50% 的概率水平翻转\n#     A.VerticalFlip(p=0.2),  # 有 20% 的概率垂直翻转\n#     A.RandomBrightnessContrast(p=1.0),  # 有 50% 的概率随机调整亮度和对比度\n#     A.ElasticTransform(p=0.3, alpha=120, sigma=120 * 0.05, alpha_affine=120 * 0.03),  # 有 30% 的概率进行弹性变形\n    A.CLAHE(p=1.0),  # 对比度受限的自适应直方图均衡化\n], \n)\n\ndef resize_image_to_target(image_to_resize, target_image):\n    zoom_factors = [target_image.shape[0] / image_to_resize.shape[0],\n                    target_image.shape[1] / image_to_resize.shape[1]]\n    resized_image = zoom(image_to_resize, zoom_factors, order=1)\n    return resized_image\n\n\n# def read_dcm(src_path1,src_path,src_path2):\n#     dicom_data = pydicom.dcmread(src_path)\n#     if os.path.exists(src_path1):\n#         dicom_data1 = pydicom.dcmread(src_path1)\n#     else:\n#         dicom_data1 = dicom_data\n#     if os.path.exists(src_path2):\n#         dicom_data2 = pydicom.dcmread(src_path2)\n#     else:\n#         dicom_data2 = dicom_data\n#     image = dicom_data.pixel_array\n#     image1 = dicom_data1.pixel_array\n#     image2 = dicom_data2.pixel_array\n#     image1 = resize_image_to_target(image1, image)\n#     image2 = resize_image_to_target(image2, image)\n#     image = (image - image.min()) / (image.max() - image.min() +1e-6) * 255\n#     image1 = (image1 - image1.min()) / (image1.max() - image1.min() +1e-6) * 255\n#     image2 = (image2 - image2.min()) / (image2.max() - image2.min() +1e-6) * 255\n#     image = np.stack([image1,image,image2], axis=-1).astype('uint8')\n#     return image\n\n\n\ndef read_dcm(src_path):\n    dicom_data = pydicom.dcmread(src_path)\n    image = dicom_data.pixel_array\n    image = (image - image.min()) / (image.max() - image.min() +1e-6) * 255\n    image = np.stack([image,image,image], axis=-1).astype('uint8')\n    image = transform(image=image)\n    image = image['image']\n    return image","metadata":{"execution":{"iopub.status.busy":"2024-09-27T11:33:01.378971Z","iopub.execute_input":"2024-09-27T11:33:01.379431Z","iopub.status.idle":"2024-09-27T11:33:02.220802Z","shell.execute_reply.started":"2024-09-27T11:33:01.379402Z","shell.execute_reply":"2024-09-27T11:33:02.219776Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"filtered_df = label_df[label_df.condition.map(lambda x: x in CONDITIONS)]\n# filtered_df = filtered_df[:100]\nfiltered_df","metadata":{"execution":{"iopub.status.busy":"2024-09-27T11:33:18.018730Z","iopub.execute_input":"2024-09-27T11:33:18.019833Z","iopub.status.idle":"2024-09-27T11:33:18.066404Z","shell.execute_reply.started":"2024-09-27T11:33:18.019787Z","shell.execute_reply":"2024-09-27T11:33:18.064931Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**将Severe和Moderate 进行重复处理**","metadata":{}},{"cell_type":"code","source":"# severe_ = [str(i)+str(j) for i,j in zip(filtered_df[filtered_df['label']=='Severe']['series_id'],filtered_df[filtered_df['label']=='Severe']['instance_number'])]\n# moderate_ = [str(i)+str(j) for i,j in zip(filtered_df[filtered_df['label']=='Moderate']['series_id'],filtered_df[filtered_df['label']=='Moderate']['instance_number'])]\n","metadata":{"execution":{"iopub.status.busy":"2024-09-27T11:33:18.877849Z","iopub.execute_input":"2024-09-27T11:33:18.878275Z","iopub.status.idle":"2024-09-27T11:33:18.883453Z","shell.execute_reply.started":"2024-09-27T11:33:18.878244Z","shell.execute_reply":"2024-09-27T11:33:18.882178Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# severe_index = []\n# moderate_index = []\n# for item in filtered_df.T.items():\n#     if str(item[1]['series_id'])+str(item[1]['instance_number']) in severe_:\n#         severe_index.append(item[0])\n    \n#     elif str(item[1]['series_id'])+str(item[1]['instance_number']) in moderate_:\n#         moderate_index.append(item[0])\n","metadata":{"execution":{"iopub.status.busy":"2024-09-27T11:33:19.098191Z","iopub.execute_input":"2024-09-27T11:33:19.098612Z","iopub.status.idle":"2024-09-27T11:33:19.104094Z","shell.execute_reply.started":"2024-09-27T11:33:19.098561Z","shell.execute_reply":"2024-09-27T11:33:19.102678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df_Severe = filtered_df.loc[severe_index,:]\n# df_Moderate = filtered_df.loc[moderate_index,:]\n\n\n# # # 使用 join 进行连接，连接条件为两列相等\n# # joined_df = df_Moderate.set_index(['series_id', 'instance_number']).join(df_Severe.set_index(['series_id', 'instance_number']), how='inner', lsuffix='_df_Moderate', rsuffix='_df_Severe')\n# # print(len(joined_df))\n# # # 获取重复的series_id和instance_number\n# # series_id_j,instance_number_j = [],[]\n\n# # for i,j in joined_df.index:\n# #     series_id_j.append(str(i)+str(j))\n\n# # # 按index去重\n# # df_Moderate = df_Moderate.drop(df_Moderate[(df_Moderate['series_id']+df_Moderate['instance_number']).astype(str).isin(series_id_j)].index)\n\n# # print(len(df_Moderate))\n# df_Severe1 = df_Severe.copy()\n# df_Severe2 = df_Severe.copy()\n# df_Severe3 = df_Severe.copy()\n# df_Moderate1 = df_Moderate.copy()\n\n# df_Severe1['series_id'] = df_Severe['series_id'].astype(str)+'a'\n# df_Severe2['series_id'] = df_Severe['series_id'].astype(str)+'b'\n# df_Severe3['series_id'] = df_Severe['series_id'].astype(str)+'c'\n\n# df_Moderate1['series_id'] = df_Moderate['series_id'].astype(str)+'a'\n\n# repeated_df = pd.concat([df_Severe1,df_Severe2,df_Severe3,df_Moderate1])\n# filtered_df = pd.concat([filtered_df,repeated_df])\n# filtered_df","metadata":{"execution":{"iopub.status.busy":"2024-09-27T11:33:19.778228Z","iopub.execute_input":"2024-09-27T11:33:19.778747Z","iopub.status.idle":"2024-09-27T11:33:19.786245Z","shell.execute_reply.started":"2024-09-27T11:33:19.778707Z","shell.execute_reply":"2024-09-27T11:33:19.784605Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label2id = {}\nid2label = {}\ni = 0\nfor cond in CONDITIONS:\n    for level in LEVELS:\n        for severity in SEVERITIES:\n            cls_ = f\"{cond.lower().replace(' ', '_')}_{level}_{severity.lower()}\"\n            label2id[cls_] = i\n            id2label[i] = cls_\n            i+=1","metadata":{"execution":{"iopub.status.busy":"2024-09-27T11:33:20.307933Z","iopub.execute_input":"2024-09-27T11:33:20.308365Z","iopub.status.idle":"2024-09-27T11:33:20.315172Z","shell.execute_reply.started":"2024-09-27T11:33:20.308334Z","shell.execute_reply":"2024-09-27T11:33:20.313667Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"id2label","metadata":{"execution":{"iopub.status.busy":"2024-09-27T11:33:21.019390Z","iopub.execute_input":"2024-09-27T11:33:21.019835Z","iopub.status.idle":"2024-09-27T11:33:21.029075Z","shell.execute_reply.started":"2024-09-27T11:33:21.019788Z","shell.execute_reply":"2024-09-27T11:33:21.027801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def gen_yolo_format(ann_df, phase='train'):\n    for name, group in tqdm(ann_df.groupby(['study_id', 'series_id', 'instance_number'])):\n        study_id, series_id, instance_num = name[0], name[1], name[2]\n        if str(series_id).endswith('a') or str(series_id).endswith('b') or str(series_id).endswith('c'):\n            series_idr = series_id[:-1]\n        else :\n            series_idr = series_id\n        path1 = f'{IMG_DIR}/{study_id}/{series_idr}/{int(instance_num)-1}.dcm'\n        path = f'{IMG_DIR}/{study_id}/{series_idr}/{instance_num}.dcm'\n        path2 = f'{IMG_DIR}/{study_id}/{series_idr}/{int(instance_num)+1}.dcm'\n        img = read_dcm(path)\n        H, W = img.shape[:2]\n\n        img_dir = os.path.join(OUT_DIR, 'images', phase)\n        os.makedirs(img_dir, exist_ok=True)\n        img_path = os.path.join(img_dir, f'{study_id}_{series_id}_{instance_num}.jpg')\n        cv2.imwrite(img_path, img)\n\n        ann_dir = os.path.join(OUT_DIR, 'labels', phase)\n        os.makedirs(ann_dir, exist_ok=True)\n        ann_path = os.path.join(ann_dir, f'{study_id}_{series_id}_{instance_num}.txt')\n        \n        contain_nulls = False\n        \n        with open(ann_path, 'w') as f:\n            for i, row in group.iterrows():\n                cond = row['condition']\n                level = row['level']\n                severity = row['label']\n                if pd.isnull(severity):\n                    contain_nulls = True\n                    break\n                class_label = f\"{cond.lower().replace(' ', '_')}_{level.lower().replace('/', '_')}_{severity.lower()}\"\n                class_id = label2id[class_label]\n                x_center = row['x'] / W\n                y_center = row['y'] / H\n                width = W / OD_INPUT_SIZE * STD_BOX_SIZE / W\n                height = H /  OD_INPUT_SIZE * STD_BOX_SIZE / H\n                f.write(f'{class_id} {x_center} {y_center} {width} {height}\\n')\n        \n        if not contain_nulls:\n            cv2.imwrite(img_path, img)\n#         break","metadata":{"execution":{"iopub.status.busy":"2024-09-27T11:33:31.818754Z","iopub.execute_input":"2024-09-27T11:33:31.819569Z","iopub.status.idle":"2024-09-27T11:33:31.833785Z","shell.execute_reply.started":"2024-09-27T11:33:31.819534Z","shell.execute_reply":"2024-09-27T11:33:31.832669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for FOLD in FOLDS:\n    print('Gen data fold', FOLD)\n    OUT_DIR = f'data_fold{FOLD}'\n    os.makedirs(OUT_DIR, exist_ok=True)\n    \n    train_df = filtered_df[filtered_df.fold != FOLD]\n    val_df = filtered_df[filtered_df.fold == FOLD]\n    \n    gen_yolo_format(train_df, phase='train')\n    gen_yolo_format(val_df, phase='val')","metadata":{"execution":{"iopub.status.busy":"2024-09-27T11:33:34.074159Z","iopub.execute_input":"2024-09-27T11:33:34.074571Z","iopub.status.idle":"2024-09-27T11:33:43.322686Z","shell.execute_reply.started":"2024-09-27T11:33:34.074540Z","shell.execute_reply":"2024-09-27T11:33:43.321497Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# cat  train_fold0/labels/4646740_3666319702_9.txt","metadata":{"execution":{"iopub.status.busy":"2024-09-27T06:08:32.208084Z","iopub.execute_input":"2024-09-27T06:08:32.208479Z","iopub.status.idle":"2024-09-27T06:08:32.214018Z","shell.execute_reply.started":"2024-09-27T06:08:32.208447Z","shell.execute_reply":"2024-09-27T06:08:32.212762Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# cat val_fold0/labels/4003253_702807833_8.txt","metadata":{"execution":{"iopub.status.busy":"2024-09-27T06:08:32.215479Z","iopub.execute_input":"2024-09-27T06:08:32.215931Z","iopub.status.idle":"2024-09-27T06:08:32.230217Z","shell.execute_reply.started":"2024-09-27T06:08:32.215893Z","shell.execute_reply":"2024-09-27T06:08:32.228913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # test generated annotations\n\n_IM_DIR = f'{OUT_DIR}/images/train'\n_ANN_DIR = f'{OUT_DIR}/labels/train'\nname = np.random.choice(os.listdir(_IM_DIR))[:-4]\n\nim = plt.imread(os.path.join(_IM_DIR, name+'.jpg')).copy()\nH,W = im.shape[:2]\nanns = np.loadtxt(os.path.join(_ANN_DIR, name+'.txt')).reshape(-1, 5)\n\nfor _cls, x,y,w,h in anns.tolist():\n    x *= W\n    y *= H\n    w *= W\n    h *= H\n    x1 = int(x-w/2)\n    x2 = int(x+w/2)\n    y1 = int(y-h/2)\n    y2 = int(y+h/2)\n    label = id2label[_cls]\n    \n#     if _cls == 0:\n#         c = (255,0,0)\n#     elif _cls == 1:\n#         c = (0,255,0)\n#     else:\n#         c = (255,255,0)\n    c = (0,255,255)\n\n    im = cv2.rectangle(im, (x1,y1), (x2,y2), c, 2)\n    cv2.putText(im, label, (x1,y1), fontFace, 0.3, c, 1, cv2.LINE_AA)\n\n\nplt.imshow(im)","metadata":{"execution":{"iopub.status.busy":"2024-09-27T11:34:17.458471Z","iopub.execute_input":"2024-09-27T11:34:17.458947Z","iopub.status.idle":"2024-09-27T11:34:17.828760Z","shell.execute_reply.started":"2024-09-27T11:34:17.458913Z","shell.execute_reply":"2024-09-27T11:34:17.827621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import albumentations as A\n# import cv2\n\n# # 定义数据增强变换\n# transform = A.Compose([\n# #     A.RandomRotate90(p=0.5),  # 有 50% 的概率随机旋转 90 度\n# #     A.HorizontalFlip(p=0.5),  # 有 50% 的概率水平翻转\n# #     A.VerticalFlip(p=0.2),  # 有 20% 的概率垂直翻转\n# #     A.RandomBrightnessContrast(p=1.0),  # 有 50% 的概率随机调整亮度和对比度\n# #     A.ElasticTransform(p=0.3, alpha=120, sigma=120 * 0.05, alpha_affine=120 * 0.03),  # 有 30% 的概率进行弹性变形\n#     A.CLAHE(p=1.0),  # 有 30% 的概率应用对比度受限的自适应直方图均衡化\n#     A.GaussNoise(p=1.0),  # 有 20% 的概率添加高斯噪声\n# ], \n# )\n\n\n\n# im = plt.imread(os.path.join(_IM_DIR, name+'.jpg')).copy()\n# im = transform(image=im)\n# im = im['image']\n# H,W = im.shape[:2]\n# anns = np.loadtxt(os.path.join(_ANN_DIR, name+'.txt')).reshape(-1, 5)\n\n\n# for _cls, x,y,w,h in anns.tolist():\n#     x *= W\n#     y *= H\n#     w *= W\n#     h *= H\n#     x1 = int(x-w/2)\n#     x2 = int(x+w/2)\n#     y1 = int(y-h/2)\n#     y2 = int(y+h/2)\n#     label = id2label[_cls]\n    \n# #     if _cls == 0:\n# #         c = (255,0,0)\n# #     elif _cls == 1:\n# #         c = (0,255,0)\n# #     else:\n# #         c = (255,255,0)\n#     c = (0,255,255)\n\n#     im = cv2.rectangle(im, (x1,y1), (x2,y2), c, 2)\n#     cv2.putText(im, label, (x1,y1), fontFace, 0.3, c, 1, cv2.LINE_AA)\n\n\n# plt.imshow(im)","metadata":{"execution":{"iopub.status.busy":"2024-09-27T06:28:34.165170Z","iopub.execute_input":"2024-09-27T06:28:34.165587Z","iopub.status.idle":"2024-09-27T06:28:34.534224Z","shell.execute_reply.started":"2024-09-27T06:28:34.165555Z","shell.execute_reply":"2024-09-27T06:28:34.532991Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ls data_fold0/labels/val","metadata":{"execution":{"iopub.status.busy":"2024-09-27T05:51:04.190414Z","iopub.status.idle":"2024-09-27T05:51:04.190964Z","shell.execute_reply.started":"2024-09-27T05:51:04.190680Z","shell.execute_reply":"2024-09-27T05:51:04.190722Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# os.path.join(_ANN_DIR, name+'.txt')","metadata":{"execution":{"iopub.status.busy":"2024-09-27T05:51:04.192651Z","iopub.status.idle":"2024-09-27T05:51:04.193188Z","shell.execute_reply.started":"2024-09-27T05:51:04.192926Z","shell.execute_reply":"2024-09-27T05:51:04.192949Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# cat 'train_fold0/labels/404602713_1230697721_12.txt'","metadata":{"execution":{"iopub.status.busy":"2024-09-27T05:51:04.195275Z","iopub.status.idle":"2024-09-27T05:51:04.195891Z","shell.execute_reply.started":"2024-09-27T05:51:04.195543Z","shell.execute_reply":"2024-09-27T05:51:04.195565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for k, v in id2label.items():\n    print(f'{k}: {v}')","metadata":{"execution":{"iopub.status.busy":"2024-09-27T05:51:04.198158Z","iopub.status.idle":"2024-09-27T05:51:04.198542Z","shell.execute_reply.started":"2024-09-27T05:51:04.198367Z","shell.execute_reply":"2024-09-27T05:51:04.198383Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!zip -r -q data_fold0.zip data_fold0\n!zip -r -q data_fold1.zip data_fold1\n!zip -r -q data_fold2.zip data_fold2\n!zip -r -q data_fold3.zip data_fold3\n!zip -r -q data_fold4.zip data_fold4","metadata":{"execution":{"iopub.status.busy":"2024-09-27T05:51:04.199776Z","iopub.status.idle":"2024-09-27T05:51:04.200122Z","shell.execute_reply.started":"2024-09-27T05:51:04.199957Z","shell.execute_reply":"2024-09-27T05:51:04.199972Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm -rf data_fold0\n!rm -rf data_fold1\n!rm -rf data_fold2\n!rm -rf data_fold3\n!rm -rf data_fold4","metadata":{"execution":{"iopub.status.busy":"2024-09-27T05:51:04.201940Z","iopub.status.idle":"2024-09-27T05:51:04.202447Z","shell.execute_reply.started":"2024-09-27T05:51:04.202184Z","shell.execute_reply":"2024-09-27T05:51:04.202206Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}