{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport pydicom\nimport cv2\nimport random\nimport numpy as np\nimport pandas as pd\nimport PIL.Image as Image\nfrom matplotlib import pyplot as plt","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-22T12:18:52.169425Z","iopub.execute_input":"2022-07-22T12:18:52.170519Z","iopub.status.idle":"2022-07-22T12:18:52.545553Z","shell.execute_reply.started":"2022-07-22T12:18:52.170417Z","shell.execute_reply":"2022-07-22T12:18:52.544218Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Get train and test images details","metadata":{}},{"cell_type":"code","source":"train_root = '../input/unifesp-x-ray-body-part-classifier/train/train/'\ntrain_files = []\n\nfor i, folder in enumerate(os.listdir(train_root)):\n    lv_1 = os.path.join(train_root,folder)\n    \n    if os.path.isdir(lv_1):\n        for f_lv_1 in os.listdir(lv_1):\n            lv_2 = os.path.join(lv_1, f_lv_1)\n            \n            if os.path.isdir(lv_2):\n                for f_lv_2 in os.listdir(lv_2):\n                    lv_3 = os.path.join(lv_2, f_lv_2)\n                    \n                    if os.path.isdir(lv_3):\n                        for file in os.listdir(lv_3):\n                            train_files.append([file, os.path.join(lv_3,file)])\n                            \n    print(f'{i+1}/{len(os.listdir(train_root))}',end='\\r')","metadata":{"execution":{"iopub.status.busy":"2022-07-22T12:30:56.348138Z","iopub.execute_input":"2022-07-22T12:30:56.348720Z","iopub.status.idle":"2022-07-22T12:31:00.532187Z","shell.execute_reply.started":"2022-07-22T12:30:56.348691Z","shell.execute_reply":"2022-07-22T12:31:00.530926Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_root = '../input/unifesp-x-ray-body-part-classifier/test/test/'\ntest_files = []\n\nfor i, folder in enumerate(os.listdir(test_root)):\n    lv_1 = os.path.join(test_root,folder)\n    \n    if os.path.isdir(lv_1):\n        for f_lv_1 in os.listdir(lv_1):\n            lv_2 = os.path.join(lv_1, f_lv_1)\n            \n            if os.path.isdir(lv_2):\n                for f_lv_2 in os.listdir(lv_2):\n                    lv_3 = os.path.join(lv_2, f_lv_2)\n                    \n                    if os.path.isdir(lv_3):\n                        for file in os.listdir(lv_3):\n                            test_files.append([file, os.path.join(lv_3,file)])\n                            \n    print(f'{i+1}/{len(os.listdir(test_root))}',end='\\r')","metadata":{"execution":{"iopub.status.busy":"2022-07-22T12:31:00.534457Z","iopub.execute_input":"2022-07-22T12:31:00.534814Z","iopub.status.idle":"2022-07-22T12:31:02.476580Z","shell.execute_reply.started":"2022-07-22T12:31:00.534772Z","shell.execute_reply":"2022-07-22T12:31:02.475216Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_files = pd.DataFrame(train_files, columns=['file_name', 'file_path'])\ntest_files = pd.DataFrame(test_files, columns=['file_name', 'file_path'])","metadata":{"execution":{"iopub.status.busy":"2022-07-22T12:31:04.717246Z","iopub.execute_input":"2022-07-22T12:31:04.718021Z","iopub.status.idle":"2022-07-22T12:31:04.727044Z","shell.execute_reply.started":"2022-07-22T12:31:04.717947Z","shell.execute_reply":"2022-07-22T12:31:04.726093Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_files['file_name'] = [value.split('-')[0].split('.')[-1] for value in train_files['file_name'].values]","metadata":{"execution":{"iopub.status.busy":"2022-07-22T12:31:05.958036Z","iopub.execute_input":"2022-07-22T12:31:05.958799Z","iopub.status.idle":"2022-07-22T12:31:05.966314Z","shell.execute_reply.started":"2022-07-22T12:31:05.958761Z","shell.execute_reply":"2022-07-22T12:31:05.965343Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_files","metadata":{"execution":{"iopub.status.busy":"2022-07-22T12:31:11.747558Z","iopub.execute_input":"2022-07-22T12:31:11.748601Z","iopub.status.idle":"2022-07-22T12:31:11.761029Z","shell.execute_reply.started":"2022-07-22T12:31:11.748558Z","shell.execute_reply":"2022-07-22T12:31:11.760204Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_meta = pd.read_csv('../input/unifesp-x-ray-body-part-classifier/train.csv')\ntrain_meta","metadata":{"execution":{"iopub.status.busy":"2022-07-22T12:31:13.374655Z","iopub.execute_input":"2022-07-22T12:31:13.375434Z","iopub.status.idle":"2022-07-22T12:31:13.396053Z","shell.execute_reply.started":"2022-07-22T12:31:13.375380Z","shell.execute_reply":"2022-07-22T12:31:13.395144Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_meta['SOPInstanceUID'] = [value.split('.')[-1] for value in train_meta['SOPInstanceUID'].values]","metadata":{"execution":{"iopub.status.busy":"2022-07-22T12:31:18.337783Z","iopub.execute_input":"2022-07-22T12:31:18.338482Z","iopub.status.idle":"2022-07-22T12:31:18.346299Z","shell.execute_reply.started":"2022-07-22T12:31:18.338432Z","shell.execute_reply":"2022-07-22T12:31:18.345374Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"full_train_dtls = pd.merge(train_files, train_meta, left_on='file_name', right_on='SOPInstanceUID').drop('SOPInstanceUID', axis=1)\nfull_train_dtls","metadata":{"execution":{"iopub.status.busy":"2022-07-22T12:31:20.648065Z","iopub.execute_input":"2022-07-22T12:31:20.648480Z","iopub.status.idle":"2022-07-22T12:31:20.667261Z","shell.execute_reply.started":"2022-07-22T12:31:20.648445Z","shell.execute_reply":"2022-07-22T12:31:20.666108Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_files","metadata":{"execution":{"iopub.status.busy":"2022-07-22T12:31:23.414840Z","iopub.execute_input":"2022-07-22T12:31:23.415279Z","iopub.status.idle":"2022-07-22T12:31:23.428770Z","shell.execute_reply.started":"2022-07-22T12:31:23.415243Z","shell.execute_reply":"2022-07-22T12:31:23.427456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"correct = pd.DataFrame(os.listdir('../input/unifesp-xray-body-part-classifier-dataset-jpeg/data/test'), columns=['file_name'])\ncorrect","metadata":{"execution":{"iopub.status.busy":"2022-07-22T12:31:26.248620Z","iopub.execute_input":"2022-07-22T12:31:26.249747Z","iopub.status.idle":"2022-07-22T12:31:26.266495Z","shell.execute_reply.started":"2022-07-22T12:31:26.249704Z","shell.execute_reply":"2022-07-22T12:31:26.265361Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"correct['file_name'] = correct['file_name'].str.replace('1.2.826.0.1.3680043.8.498.','')","metadata":{"execution":{"iopub.status.busy":"2022-07-22T12:31:28.954747Z","iopub.execute_input":"2022-07-22T12:31:28.955638Z","iopub.status.idle":"2022-07-22T12:31:28.964258Z","shell.execute_reply.started":"2022-07-22T12:31:28.955588Z","shell.execute_reply":"2022-07-22T12:31:28.963307Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_files['file_name'] = test_files['file_name'].str.replace('1.2.826.0.1.3680043.8.498.','')\ntest_files['file_name'] = test_files['file_name'].str.replace('-c.dcm', '.jpg')","metadata":{"execution":{"iopub.status.busy":"2022-07-22T12:31:30.459038Z","iopub.execute_input":"2022-07-22T12:31:30.459627Z","iopub.status.idle":"2022-07-22T12:31:30.469768Z","shell.execute_reply.started":"2022-07-22T12:31:30.459593Z","shell.execute_reply":"2022-07-22T12:31:30.468793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_files","metadata":{"execution":{"iopub.status.busy":"2022-07-22T12:31:33.208835Z","iopub.execute_input":"2022-07-22T12:31:33.209569Z","iopub.status.idle":"2022-07-22T12:31:33.224285Z","shell.execute_reply.started":"2022-07-22T12:31:33.209533Z","shell.execute_reply":"2022-07-22T12:31:33.222852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"correct","metadata":{"execution":{"iopub.status.busy":"2022-07-22T12:32:04.987900Z","iopub.execute_input":"2022-07-22T12:32:04.988333Z","iopub.status.idle":"2022-07-22T12:32:05.001200Z","shell.execute_reply.started":"2022-07-22T12:32:04.988298Z","shell.execute_reply":"2022-07-22T12:32:04.999870Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"damage_test_files = [test_files[test_files['file_name']==x_file]['file_path'].values[0] for x_file in [file for file in test_files.file_name.values if file not in correct.file_name.values]]","metadata":{"execution":{"iopub.status.busy":"2022-07-22T12:44:17.543168Z","iopub.execute_input":"2022-07-22T12:44:17.543577Z","iopub.status.idle":"2022-07-22T12:44:17.620198Z","shell.execute_reply.started":"2022-07-22T12:44:17.543542Z","shell.execute_reply":"2022-07-22T12:44:17.618966Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Convert to Jpg and save","metadata":{}},{"cell_type":"code","source":"# check sample image\nfile_path_sample = full_train_dtls['file_path'][0]\ndcm_file = pydicom.dcmread(file_path_sample)\n\nplt.figure(figsize=[8,8], dpi=100)\nplt.imshow(dcm_file.pixel_array, cmap=plt.cm.gray)","metadata":{"execution":{"iopub.status.busy":"2022-07-20T03:04:47.474524Z","iopub.execute_input":"2022-07-20T03:04:47.474931Z","iopub.status.idle":"2022-07-20T03:04:53.827646Z","shell.execute_reply.started":"2022-07-20T03:04:47.474898Z","shell.execute_reply":"2022-07-20T03:04:53.826743Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dc_img = dcm_file.pixel_array\ndc_img.shape","metadata":{"execution":{"iopub.status.busy":"2022-07-20T03:04:59.918785Z","iopub.execute_input":"2022-07-20T03:04:59.919188Z","iopub.status.idle":"2022-07-20T03:04:59.926193Z","shell.execute_reply.started":"2022-07-20T03:04:59.919149Z","shell.execute_reply":"2022-07-20T03:04:59.925152Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"asp_ratio = dc_img.shape[0]/dc_img.shape[1]\nnew_w = 720\nnew_h = int(new_w*asp_ratio)\ncrop_window = (int(new_w*0.85), int(new_h*0.85))\nanchor_point = ((new_w//2 - crop_window[0]//2), (new_h//2 - crop_window[1]//2))\nresized_dc_img = cv2.resize(dc_img, (new_w, new_h))\n\nfig, axes = plt.subplots(nrows=1, ncols=2, figsize=[10,6], dpi=100)\naxes = axes.ravel()\n\naxes[0].imshow(resized_dc_img, cmap=plt.cm.gray)\naxes[1].imshow(resized_dc_img[anchor_point[1]: crop_window[1], anchor_point[0]: crop_window[0]], cmap=plt.cm.gray)\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-20T04:07:33.449757Z","iopub.execute_input":"2022-07-20T04:07:33.450190Z","iopub.status.idle":"2022-07-20T04:07:33.889199Z","shell.execute_reply.started":"2022-07-20T04:07:33.450155Z","shell.execute_reply":"2022-07-20T04:07:33.888092Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"resized_dc_img.shape, resized_dc_img.reshape(resized_dc_img.shape[0], resized_dc_img.shape[1], 1).shape","metadata":{"execution":{"iopub.status.busy":"2022-07-20T04:14:16.269979Z","iopub.execute_input":"2022-07-20T04:14:16.270610Z","iopub.status.idle":"2022-07-20T04:14:16.279390Z","shell.execute_reply.started":"2022-07-20T04:14:16.270564Z","shell.execute_reply":"2022-07-20T04:14:16.278186Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### create image converting pipeline","metadata":{}},{"cell_type":"code","source":"def convert_and_resize(file_path):\n    # load dcm file\n    dcm_file = pydicom.dcmread(file_path)\n    dc_img = dcm_file.pixel_array\n    \n    # resize with keeping aspect ratio\n    asp_ratio = dc_img.shape[0]/dc_img.shape[1]\n    new_w = 720\n    new_h = int(new_w*asp_ratio)\n    resized_dc_img = cv2.resize(dc_img, (new_w, new_h))\n    \n    return resized_dc_img","metadata":{"execution":{"iopub.status.busy":"2022-07-22T12:45:24.519548Z","iopub.execute_input":"2022-07-22T12:45:24.520869Z","iopub.status.idle":"2022-07-22T12:45:24.527964Z","shell.execute_reply.started":"2022-07-22T12:45:24.520825Z","shell.execute_reply":"2022-07-22T12:45:24.526546Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(len(damage_test_files)):\n    cv2.imwrite(f\"./test/{damage_test_files[i].split('/')[-1].replace('-c.dcm', '.jpg')}\", convert_and_resize(damage_test_files[i]))","metadata":{"execution":{"iopub.status.busy":"2022-07-22T12:53:56.573627Z","iopub.execute_input":"2022-07-22T12:53:56.574299Z","iopub.status.idle":"2022-07-22T12:54:08.571644Z","shell.execute_reply.started":"2022-07-22T12:53:56.574263Z","shell.execute_reply":"2022-07-22T12:54:08.570396Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"file_list = os.listdir('./test/')\nfig, axes = plt.subplots(nrows=5, ncols=7, figsize=[24,24], dpi=200)\naxes = axes.ravel()\n\nfor i in range(len(axes)):\n    axes[i].imshow(cv2.imread(f'./test/{file_list[i]}', cv2.IMREAD_GRAYSCALE), cmap=plt.cm.gray)","metadata":{"execution":{"iopub.status.busy":"2022-07-22T13:01:43.190036Z","iopub.execute_input":"2022-07-22T13:01:43.190465Z","iopub.status.idle":"2022-07-22T13:01:50.932895Z","shell.execute_reply.started":"2022-07-22T13:01:43.190429Z","shell.execute_reply":"2022-07-22T13:01:50.931966Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# destination folders\n# if not os.path.isdir('train'):\n#     os.mkdir('train')\n    \nif not os.path.isdir('test'):\n    os.mkdir('test')","metadata":{"execution":{"iopub.status.busy":"2022-07-22T12:46:28.213794Z","iopub.execute_input":"2022-07-22T12:46:28.214208Z","iopub.status.idle":"2022-07-22T12:46:28.220191Z","shell.execute_reply.started":"2022-07-22T12:46:28.214174Z","shell.execute_reply":"2022-07-22T12:46:28.219035Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train images\n\nfor i, (file_name, file_path) in enumerate(full_train_dtls[['file_name','file_path']].values):\n    dst_path = f'train/{file_name}.jpg'\n    img = convert_and_resize(file_path)\n    cv2.imwrite(dst_path, img)\n    \n    print(f'{i + 1}/{full_train_dtls.shape[0]}', end='\\r')","metadata":{"execution":{"iopub.status.busy":"2022-07-20T05:35:45.420457Z","iopub.execute_input":"2022-07-20T05:35:45.420840Z","iopub.status.idle":"2022-07-20T05:38:00.264715Z","shell.execute_reply.started":"2022-07-20T05:35:45.420810Z","shell.execute_reply":"2022-07-20T05:38:00.262817Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"full_train_dtls.to_csv('train.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-07-20T05:51:22.705821Z","iopub.execute_input":"2022-07-20T05:51:22.706256Z","iopub.status.idle":"2022-07-20T05:51:22.739716Z","shell.execute_reply.started":"2022-07-20T05:51:22.706207Z","shell.execute_reply":"2022-07-20T05:51:22.738857Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"files_list = os.listdir('./train/')\nrand_trans_files = random.choices(files_list, k=5)\nfig, axes = plt.subplots(nrows=1, ncols=5, figsize=[24, 6], dpi=100)\naxes = axes.ravel()\n\nfor i, file in enumerate(rand_trans_files):\n    axes[i].imshow(Image.open(f'./train/{file}'), cmap=plt.cm.gray)\n    \nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-20T05:45:59.109721Z","iopub.execute_input":"2022-07-20T05:45:59.110403Z","iopub.status.idle":"2022-07-20T05:46:00.119227Z","shell.execute_reply.started":"2022-07-20T05:45:59.110367Z","shell.execute_reply":"2022-07-20T05:46:00.118067Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test images\n\nfor i, (file_name, file_path) in enumerate(test_files[['file_name','file_path']].values):\n    dst_path = f'test/{file_name[:-6]}.jpg'\n    img = convert_and_resize(file_path)\n    cv2.imwrite(dst_path, img)\n    \n    print(f'{i + 1}/{test_files.shape[0]}', end='\\r')\n    ","metadata":{"execution":{"iopub.status.busy":"2022-07-20T05:46:33.328157Z","iopub.execute_input":"2022-07-20T05:46:33.328561Z","iopub.status.idle":"2022-07-20T05:48:49.498052Z","shell.execute_reply.started":"2022-07-20T05:46:33.328528Z","shell.execute_reply":"2022-07-20T05:48:49.496384Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"files_list = os.listdir('./test/')\nrand_trans_files = random.choices(files_list, k=5)\nfig, axes = plt.subplots(nrows=1, ncols=5, figsize=[24, 6], dpi=100)\naxes = axes.ravel()\n\nfor i, file in enumerate(rand_trans_files):\n    axes[i].imshow(Image.open(f'./test/{file}'), cmap=plt.cm.gray)\n    \nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-20T05:49:21.501012Z","iopub.execute_input":"2022-07-20T05:49:21.501646Z","iopub.status.idle":"2022-07-20T05:49:22.488594Z","shell.execute_reply.started":"2022-07-20T05:49:21.501612Z","shell.execute_reply":"2022-07-20T05:49:22.487512Z"},"trusted":true},"execution_count":null,"outputs":[]}]}