{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install python-gdcm\nimport numpy as np\nimport pandas as pd\nimport cv2,random\nimport pydicom\nimport gdcm\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\nimport matplotlib.pyplot as plt\nimport glob\nimport os\nfrom PIL import Image, ImageStat\nimport albumentations as A","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-07-28T10:47:57.394586Z","iopub.execute_input":"2021-07-28T10:47:57.395114Z","iopub.status.idle":"2021-07-28T10:48:09.339421Z","shell.execute_reply.started":"2021-07-28T10:47:57.395028Z","shell.execute_reply":"2021-07-28T10:48:09.338461Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_xray(path, voi_lut = True, fix_monochrome = True):\n    # Original from: https://www.kaggle.com/raddar/convert-dicom-to-np-array-the-correct-way\n    dicom = pydicom.read_file(path)\n    # VOI LUT (if available by DICOM device) is used to transform raw DICOM data to \n    # \"human-friendly\" view\n    if voi_lut:\n        data = apply_voi_lut(dicom.pixel_array, dicom)\n    else:\n        data = dicom.pixel_array\n               \n    # depending on this value, X-ray may look inverted - fix that:\n    if fix_monochrome and dicom.PhotometricInterpretation == \"MONOCHROME1\":\n        data = np.amax(data) - data\n        \n    data = data - np.min(data)\n    data = data / np.max(data)\n    data = (data * 255).astype(np.uint8)\n        \n    return data\n\ndef expand2square(pil_img, background_color):\n    width, height = pil_img.size\n    if width == height:\n        return pil_img\n    elif width > height:\n        result = Image.new(pil_img.mode, (width, width), background_color)\n        result.paste(pil_img, (0, (width - height) // 2))\n        return result\n    else:\n        result = Image.new(pil_img.mode, (height, height), background_color)\n        result.paste(pil_img, ((height - width) // 2, 0))\n        return result\n\ndef resize(array, size1,size2, keep_ratio=True, resample=Image.LANCZOS):\n    # Original from: https://www.kaggle.com/xhlulu/vinbigdata-process-and-resize-to-image\n    im = Image.fromarray(array)\n    \n    if keep_ratio:\n        im.thumbnail((size1, size2), resample)\n        im = expand2square(im, background_color=0)\n    else:\n        im = im.resize((size1, size2), resample)\n    \n    return im","metadata":{"execution":{"iopub.status.busy":"2021-07-28T10:48:09.340952Z","iopub.execute_input":"2021-07-28T10:48:09.34135Z","iopub.status.idle":"2021-07-28T10:48:09.352762Z","shell.execute_reply.started":"2021-07-28T10:48:09.341304Z","shell.execute_reply":"2021-07-28T10:48:09.352014Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# get approciate image statics value\n\n## step1: about augflag==0 data ","metadata":{}},{"cell_type":"code","source":"df = pd.read_csv(\"../input/step2-make-dataframe/train_image.csv\")\ntargetmean, targetstd = df[\"mean\"].values.mean(),df[\"var\"].values.mean()\ndf = df[df.augflag==0]\nyind = df[df.crop_ymax == 0].index\nxind = df[df.crop_xmax == 0].index\ndf.loc[xind,\"crop_xmax\"]= df.loc[xind,\"dim1\"]\ndf.loc[yind,\"crop_ymax\"]= df.loc[yind,\"dim2\"]\ndisplay(df)","metadata":{"execution":{"iopub.status.busy":"2021-07-28T11:29:36.878091Z","iopub.execute_input":"2021-07-28T11:29:36.878582Z","iopub.status.idle":"2021-07-28T11:29:36.965069Z","shell.execute_reply.started":"2021-07-28T11:29:36.878531Z","shell.execute_reply":"2021-07-28T11:29:36.964034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"targetmean,targetstd","metadata":{"execution":{"iopub.status.busy":"2021-07-28T12:35:24.768838Z","iopub.execute_input":"2021-07-28T12:35:24.769319Z","iopub.status.idle":"2021-07-28T12:35:24.775734Z","shell.execute_reply.started":"2021-07-28T12:35:24.769273Z","shell.execute_reply":"2021-07-28T12:35:24.774829Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_transform(image_size):\n    return A.Compose(\n            [\n\n            A.CLAHE(clip_limit=(1,3),p=0.5),\n            A.RandomBrightnessContrast(\n                    brightness_limit=0.1, \n                    contrast_limit=0.1,\n                    p=1.0),\n            A.RandomGamma(gamma_limit = (90, 110),p=1.0)\n            ],p=1.0)\n\n            \ndef optimization_imagehist(row,image, trytimes=100):\n    best_loss = targetmean**2 + targetstd**2\n    for _ in range(trytimes):\n        if random.uniform(0,1)<0.5:\n            xmin, xmax = int(row.crop_xmin), int(row.crop_xmax)\n            ymin, ymax = int(row.crop_ymin), int(row.crop_ymax)\n            images = image[xmin:xmax,ymin:ymax]\n        else:\n            images = image\n        transform = get_transform((images.shape[0],images.shape[1]))\n        sample = transform(image=images)#, bboxes=boxes, label=labels)\n        calc_loss = (sample['image'].mean() - targetmean)**2 + (sample['image'].std() - targetstd)**2\n        if calc_loss < best_loss:\n            best_loss = calc_loss\n            outimg = sample['image']\n            #bboxes = sample['bboxes']\n    return outimg","metadata":{"execution":{"iopub.status.busy":"2021-07-28T13:01:57.148311Z","iopub.status.idle":"2021-07-28T13:01:57.148908Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.mkdir(\"comp\")","metadata":{"execution":{"iopub.status.busy":"2021-07-28T13:01:57.149784Z","iopub.status.idle":"2021-07-28T13:01:57.15029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for idx,(index,row) in enumerate(df.iterrows()):\n    image = read_xray(row.path)\n    fig, ax= plt.subplots(1,2)\n    print(row.id)\n    ax[0].imshow(image)\n    ax[0].axis('off')\n    ax[0].set_title(\"before\")\n    ax[1].imshow(optimization_imagehist(row,image))\n    ax[1].axis('off')\n    ax[1].set_title(\"after\")\n    plt.title(f\"img_path:{row.id}\", x=0, y=1.3, size=15)\n    #plt.show()\n    fig.savefig(\"comp/\" + str(ids)+\".png\",dpi=80)\n    plt.close()\n    #if idx==2:\n        #break","metadata":{"execution":{"iopub.status.busy":"2021-07-28T13:01:57.151108Z","iopub.status.idle":"2021-07-28T13:01:57.151665Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## step2: bounding box .vs. black frame","metadata":{}},{"cell_type":"code","source":"df = pd.read_csv(\"../input/step2-make-dataframe/train_image.csv\")\ncnd1 = df.fracxmax1 < df.crop_fxmax\ncnd2 = df.fracxmax2 < df.crop_fxmax\ncnd3 = df.fracxmax2 < df.crop_fxmax\ncnd4 = df.fracxmin1 > df.crop_fxmin\ncnd5 = (df.fracxmin2 > df.crop_fxmin) | (df.fracxmin2 == 0)\ncnd6 = (df.fracxmin3 > df.crop_fxmin) | (df.fracxmin3 == 0)\ncndx = cnd1 & cnd2 & cnd3 & cnd4 & cnd5 & cnd6\n\ncnd1 = df.fracymax1 < df.crop_fymax \ncnd2 = df.fracymax2 < df.crop_fymax \ncnd3 = df.fracymax2 < df.crop_fymax \ncnd4 = df.fracymin1 > df.crop_fymin \ncnd5 = (df.fracymin2 > df.crop_fymin) | (df.fracymin2 == 0)\ncnd6 = (df.fracymin3 > df.crop_fymin) | (df.fracymin3 == 0)\ncndy = cnd1 & cnd2 & cnd3 & cnd4 & cnd5 & cnd6\n\ndf = df[~(cndx & cndy)]\ndf","metadata":{"execution":{"iopub.status.busy":"2021-07-28T12:16:53.041299Z","iopub.execute_input":"2021-07-28T12:16:53.041824Z","iopub.status.idle":"2021-07-28T12:16:53.133219Z","shell.execute_reply.started":"2021-07-28T12:16:53.04177Z","shell.execute_reply":"2021-07-28T12:16:53.132165Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\ncnd1 = df.fracxmax1 > df.crop_fxmax\ndf.loc[cnd1,\"fracxmax1\"] = df.loc[cnd1,\"crop_fxmax\"]\n\ncnd2 = df.fracxmax2 > df.crop_fxmax\ndf.loc[cnd2,\"fracxmax2\"] = df.loc[cnd2,\"crop_fxmax\"]\n\ncnd3 = df.fracxmax3 > df.crop_fxmax\ndf.loc[cnd3,\"fracxmax3\"] = df.loc[cnd3,\"crop_fxmax\"]\n\ncnd4 = df.fracxmin1 < df.crop_fxmin\ndf.loc[cnd4,\"fracxmin1\"] = df.loc[cnd4,\"crop_fxmin\"]\n\n\ncnd5 = (df.fracxmin2 < df.crop_fxmin) | (df.fracxmin2 == 0)\ndf.loc[cnd5,\"fracxmin2\"] = df.loc[cnd5,\"crop_fxmin\"]\n\ncnd6 = (df.fracxmin3 < df.crop_fxmin) | (df.fracxmin3 == 0)\ndf.loc[cnd6,\"fracxmin3\"] = df.loc[cnd6,\"crop_fxmin\"]\n\ncnd1 = df.fracymax1 > df.crop_fymax\ndf.loc[cnd1,\"fracymax1\"] = df.loc[cnd1,\"crop_fymax\"]\n\n\ncnd2 = df.fracymax2 > df.crop_fymax\ndf.loc[cnd2,\"fracymax2\"] = df.loc[cnd2,\"crop_fymax\"]\n\n\ncnd3 = df.fracymax3 > df.crop_fymax\ndf.loc[cnd3,\"fracymax3\"] = df.loc[cnd3,\"crop_fymax\"]\n\ncnd4 = df.fracymin1 < df.crop_fymin\ndf.loc[cnd4,\"fracymin1\"] = df.loc[cnd4,\"crop_fymin\"]\n\ncnd5 = (df.fracymin2 < df.crop_fymin) | (df.fracymin2 == 0)\ndf.loc[cnd5,\"fracymin2\"] = df.loc[cnd5,\"crop_fymin\"]\n\ncnd6 = (df.fracymin3 < df.crop_fymin) | (df.fracymin3 == 0)\ndf.loc[cnd6,\"fracymin3\"] = df.loc[cnd6,\"crop_fymin\"]\n\"\"\"","metadata":{"execution":{"iopub.status.busy":"2021-07-28T12:16:53.508793Z","iopub.execute_input":"2021-07-28T12:16:53.509313Z","iopub.status.idle":"2021-07-28T12:16:53.518174Z","shell.execute_reply.started":"2021-07-28T12:16:53.509264Z","shell.execute_reply":"2021-07-28T12:16:53.517033Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df","metadata":{"execution":{"iopub.status.busy":"2021-07-28T12:16:54.033012Z","iopub.execute_input":"2021-07-28T12:16:54.033371Z","iopub.status.idle":"2021-07-28T12:16:54.07702Z","shell.execute_reply.started":"2021-07-28T12:16:54.03334Z","shell.execute_reply":"2021-07-28T12:16:54.07631Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv(\"../input/step2-make-dataframe/train_image.csv\")\ndf = df[(df[\"mean\"]< 90) & (df[\"var\"]<40)]\ndf","metadata":{"execution":{"iopub.status.busy":"2021-07-28T12:47:00.699188Z","iopub.execute_input":"2021-07-28T12:47:00.699607Z","iopub.status.idle":"2021-07-28T12:47:00.774994Z","shell.execute_reply.started":"2021-07-28T12:47:00.699571Z","shell.execute_reply":"2021-07-28T12:47:00.774222Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_transform(image_size):\n    return A.Compose(\n            [\n\n            A.CLAHE(clip_limit=(1,3),p=0.5),\n            A.RandomBrightnessContrast(\n                    brightness_limit=0.1, \n                    contrast_limit=0.1,\n                    p=1.0),\n            A.RandomGamma(gamma_limit = (90, 110),p=1.0)\n            ],p=1.0)\n\n            \ndef optimization_imagehist(row,image, trytimes=100):\n    best_loss = targetmean**2 + targetstd**2\n    for _ in range(trytimes):\n        transform = get_transform((image.shape[0],image.shape[1]))\n        sample = transform(image=image)#, bboxes=boxes, label=labels)\n        calc_loss = (sample['image'].mean() - targetmean)**2 + (sample['image'].std() - targetstd)**2\n        if calc_loss < best_loss:\n            best_loss = calc_loss\n            outimg = sample['image']\n            #bboxes = sample['bboxes']\n    return outimg","metadata":{"execution":{"iopub.status.busy":"2021-07-28T13:02:08.945159Z","iopub.execute_input":"2021-07-28T13:02:08.945779Z","iopub.status.idle":"2021-07-28T13:02:08.954038Z","shell.execute_reply.started":"2021-07-28T13:02:08.945742Z","shell.execute_reply":"2021-07-28T13:02:08.953136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.mkdir(\"comp2\")","metadata":{"execution":{"iopub.status.busy":"2021-07-28T13:34:52.036898Z","iopub.execute_input":"2021-07-28T13:34:52.037331Z","iopub.status.idle":"2021-07-28T13:34:52.044397Z","shell.execute_reply.started":"2021-07-28T13:34:52.037295Z","shell.execute_reply":"2021-07-28T13:34:52.043509Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for idx,(index,row) in enumerate(df.iterrows()):\n    image = read_xray(row.path)\n    cv2.rectangle(image, (int(row.fracxmin1*row.dim1),int(row.fracymin1*row.dim1)),(int(row.fracxmax1*row.dim2),int(row.fracymax1*row.dim2)), (0, 0, 255), 15)\n    cv2.rectangle(image, (int(row.fracxmin2*row.dim1),int(row.fracymin2*row.dim1)),(int(row.fracxmax2*row.dim2),int(row.fracymax2*row.dim2)), (0, 0, 255), 15)\n    cv2.rectangle(image, (int(row.fracxmin3*row.dim1),int(row.fracymin3*row.dim1)),(int(row.fracxmax3*row.dim2),int(row.fracymax3*row.dim2)), (0, 0, 255), 15)\n    \n    #plt.imshow(image)\n    #plt.show()\n    fig, ax= plt.subplots(1,2)\n    augimage = optimization_imagehist(row,image)\n    ax[0].imshow(image)\n    ax[0].axis('off')\n    ax[0].set_title(\"before\")\n    ax[1].imshow(augimage)\n    ax[1].axis('off')\n    ax[1].set_title(\"after\")\n    plt.title(f\"img_path:{row.id}\", x=0, y=1.3, size=15)\n    #plt.show()\n    fig.savefig(\"comp2/\" + str(row.id)+\".png\",dpi=80)\n    plt.close()\n    #if idx ==50:\n        #break","metadata":{"execution":{"iopub.status.busy":"2021-07-28T13:35:12.642369Z","iopub.execute_input":"2021-07-28T13:35:12.642756Z","iopub.status.idle":"2021-07-28T13:35:44.82067Z","shell.execute_reply.started":"2021-07-28T13:35:12.642724Z","shell.execute_reply":"2021-07-28T13:35:44.818131Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!zip comp.zip comp\n!rm -r comp\n!zip comp2.zip comp2\n!rm -r comp2","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}