{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n!pip install python-gdcm\n#!pip install pylibjpeg-libjpeg\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport cv2,random\nimport pydicom\nimport gdcm\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\nimport matplotlib.pyplot as plt\nimport glob\nimport os\nfrom PIL import Image, ImageStat\nimport albumentations as A","metadata":{"execution":{"iopub.status.busy":"2021-07-12T22:28:01.334549Z","iopub.execute_input":"2021-07-12T22:28:01.335027Z","iopub.status.idle":"2021-07-12T22:28:14.660276Z","shell.execute_reply.started":"2021-07-12T22:28:01.334903Z","shell.execute_reply":"2021-07-12T22:28:14.659038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.mkdir(\"image512\")\n#os.mkdir(\"image600_addeqhist\")\nos.mkdir(\"image600\")\n#os.mkdir(\"image600_both\")","metadata":{"execution":{"iopub.status.busy":"2021-07-12T22:28:14.662072Z","iopub.execute_input":"2021-07-12T22:28:14.662386Z","iopub.status.idle":"2021-07-12T22:28:14.668638Z","shell.execute_reply.started":"2021-07-12T22:28:14.662357Z","shell.execute_reply":"2021-07-12T22:28:14.667584Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ids = [\"0b858129adb4_image\",\n       \"3d12cb6aad8b_image\",\n       \"41e9a794b342_image\",\n       \"681ed0b5dff2_image\",\n       \"0eb641cb0dcd_image\",\n       \"0f9709784c19_image\",\n       \"3b982073ec16_image\",\n       \"4f3d52d652dd_image\"]","metadata":{"execution":{"iopub.status.busy":"2021-07-12T22:28:14.67024Z","iopub.execute_input":"2021-07-12T22:28:14.670545Z","iopub.status.idle":"2021-07-12T22:28:14.686365Z","shell.execute_reply.started":"2021-07-12T22:28:14.670515Z","shell.execute_reply":"2021-07-12T22:28:14.684947Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv(\"../input/step2-make-dataframe/train_study.csv\")\nprint(len(df),len(ids))\ndf = df[~df.id.isin(ids)]\nprint(len(df))\ndf = df[~df.duplicated(keep='last',subset=['mean', 'var'])]\nprint(len(df))\ndf = df.sort_values(\"norm_var\",ascending=False)\ndf['fold'] = 0\ndf.fold = df.fold.apply(lambda x: random.randint(0,3))\n#df.to_csv(\"train.csv\")\ndf","metadata":{"execution":{"iopub.status.busy":"2021-07-12T22:28:28.340112Z","iopub.execute_input":"2021-07-12T22:28:28.340741Z","iopub.status.idle":"2021-07-12T22:28:28.437401Z","shell.execute_reply.started":"2021-07-12T22:28:28.340704Z","shell.execute_reply":"2021-07-12T22:28:28.436319Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_transform(ori_size):\n    transform = A.Compose([\n        #A.Rotate(p=1.0,limit=5),\n        A.RandomSizedCrop(\n                         min_max_height=[int(0.8*ori_size), int(0.95*ori_size)],\n                         height=ori_size,\n                         width=ori_size, \n                         p=1.0)\n    ],p=1.0)\n    return transform\n\ndef exclude_blackflame(image, ori_size, trytimes=10):\n    best_var = image.std()\n    transform = get_transform(ori_size)\n    for _ in range(trytimes):\n        sample = transform(image=image)\n        if sample['image'].std() < best_var:\n            best_var = sample['image'].std()\n            images = sample['image']\n    if best_var >= image.std():\n        images = image\n    return images","metadata":{"execution":{"iopub.status.busy":"2021-07-12T22:33:19.097624Z","iopub.execute_input":"2021-07-12T22:33:19.098106Z","iopub.status.idle":"2021-07-12T22:33:19.106257Z","shell.execute_reply.started":"2021-07-12T22:33:19.098059Z","shell.execute_reply":"2021-07-12T22:33:19.105207Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_xray(path, voi_lut = True, fix_monochrome = True):\n    # Original from: https://www.kaggle.com/raddar/convert-dicom-to-np-array-the-correct-way\n    dicom = pydicom.read_file(path)\n    # VOI LUT (if available by DICOM device) is used to transform raw DICOM data to \n    # \"human-friendly\" view\n    if voi_lut:\n        data = apply_voi_lut(dicom.pixel_array, dicom)\n    else:\n        data = dicom.pixel_array\n               \n    # depending on this value, X-ray may look inverted - fix that:\n    if fix_monochrome and dicom.PhotometricInterpretation == \"MONOCHROME1\":\n        data = np.amax(data) - data\n        \n    data = data - np.min(data)\n    data = data / np.max(data)\n    data = (data * 255).astype(np.uint8)\n        \n    return data\n\ndef expand2square(pil_img, background_color):\n    width, height = pil_img.size\n    if width == height:\n        return pil_img\n    elif width > height:\n        result = Image.new(pil_img.mode, (width, width), background_color)\n        result.paste(pil_img, (0, (width - height) // 2))\n        return result\n    else:\n        result = Image.new(pil_img.mode, (height, height), background_color)\n        result.paste(pil_img, ((height - width) // 2, 0))\n        return result\n\ndef resize(array, size1,size2, keep_ratio=True, resample=Image.LANCZOS):\n    # Original from: https://www.kaggle.com/xhlulu/vinbigdata-process-and-resize-to-image\n    im = Image.fromarray(array)\n    \n    if keep_ratio:\n        im.thumbnail((size1, size2), resample)\n        im = expand2square(im, background_color=0)\n    else:\n        im = im.resize((size1, size2), resample)\n    \n    return im","metadata":{"execution":{"iopub.status.busy":"2021-07-12T22:33:19.226119Z","iopub.execute_input":"2021-07-12T22:33:19.226488Z","iopub.status.idle":"2021-07-12T22:33:19.238906Z","shell.execute_reply.started":"2021-07-12T22:33:19.226459Z","shell.execute_reply":"2021-07-12T22:33:19.237639Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfs = []\ndfs.append(df[df.fold ==0])\ndfs.append(df[df.fold ==1])\ndfs.append(df[df.fold ==2])\ndfs.append(df[df.fold ==3])\nprint(len(dfs[0]),len(dfs[1]),len(dfs[2]),len(dfs[3]))\nprint(len(df))","metadata":{"execution":{"iopub.status.busy":"2021-07-12T22:33:19.830133Z","iopub.execute_input":"2021-07-12T22:33:19.830768Z","iopub.status.idle":"2021-07-12T22:33:19.845769Z","shell.execute_reply.started":"2021-07-12T22:33:19.830707Z","shell.execute_reply":"2021-07-12T22:33:19.844676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def run(fold):\n    for idx,(index,row) in enumerate(dfs[fold].iterrows()):\n        image = read_xray(row.path)\n        #image = cv2.resize(image,(2048,2048),interpolation = cv2.INTER_AREA)\n        image = exclude_blackflame(image, ori_size=2048, trytimes=10)\n        save_img = cv2.resize(image,(512,512),interpolation = cv2.INTER_AREA)\n        cv2.imwrite(\"image512\" + \"/\" + row.id + \".png\",save_img)\n        \n        save_img = cv2.resize(image,(600,600),interpolation = cv2.INTER_AREA)\n        cv2.imwrite(\"image600\" + \"/\" + row.id + \".png\",save_img)\n        \n        #plt.imshow(images)\n        #plt.show()\n        \n        #save_img = cv2.resize(image,(600,600),interpolation = cv2.INTER_AREA)\n        #cv2.imwrite(\"image600_both\" + \"/\" + row.id + \".png\",save_img)\n        #plt.imshow(images)\n        #plt.show()\n        #cv2.imwrite(\"image512\" + \"/\" + row.id + \".png\",images)\n        #images = exclude_blackflame(image, ori_size=1024, trytimes=10)\n        #plt.imshow(images)\n        #plt.show()\n        #cv2.imwrite(\"image600\" + \"/\" + row.id + \".png\",images)\n        #images = exclude_blackflame(image, ori_size=1024, trytimes=10)\n        #plt.imshow(images)\n        #plt.show()\n        #cv2.imwrite(\"image640\" + \"/\" + row.id + \".png\",images)\n        #break\n        #display(im)\n        #stat = ImageStat.Stat(im)\n        #df.loc[index,'mean'] = stat.mean\n        #df.loc[index,'var'] = stat.stddev\n        #print(np.array(im).shape)\n        #image = cv2.resize(image,(512,512),interpolation = cv2.INTER_AREA)\n        #im.save(\"image\" + \"/\" + row.id_x + \".png\")\n        if idx%500==0:\n            print(idx)","metadata":{"execution":{"iopub.status.busy":"2021-07-12T22:33:20.96794Z","iopub.execute_input":"2021-07-12T22:33:20.968573Z","iopub.status.idle":"2021-07-12T22:33:20.978953Z","shell.execute_reply.started":"2021-07-12T22:33:20.968522Z","shell.execute_reply":"2021-07-12T22:33:20.97816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#run(1)","metadata":{"execution":{"iopub.status.busy":"2021-07-12T22:33:21.732315Z","iopub.execute_input":"2021-07-12T22:33:21.732887Z","iopub.status.idle":"2021-07-12T22:33:21.738023Z","shell.execute_reply.started":"2021-07-12T22:33:21.73284Z","shell.execute_reply":"2021-07-12T22:33:21.736682Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from joblib import Parallel, delayed\nParallel(n_jobs=4, backend=\"threading\")(delayed(run)(i) for i in range(4))","metadata":{"execution":{"iopub.status.busy":"2021-07-12T22:33:21.990545Z","iopub.execute_input":"2021-07-12T22:33:21.991093Z","iopub.status.idle":"2021-07-12T22:35:07.067855Z","shell.execute_reply.started":"2021-07-12T22:33:21.991055Z","shell.execute_reply":"2021-07-12T22:35:07.058322Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!zip -qr image600.zip image600\n!rm -r image600_normal\n!zip -qr image512.zip image512\n!rm -r image512\n#!zip -qr image600_excludeblackframe.zip image600_excludeblackframe\n#!rm -r image600_excludeblackframe\n#!zip -qr image600_keep.zip image600_keep\n#!rm -r image600_keep\n#!zip -qr image600_both.zip image600_both\n#!rm -r image600_both\n#!zip -qr image640_keep.zip image640_keep\n#!rm -r image640_keep","metadata":{"execution":{"iopub.status.busy":"2021-07-12T22:35:09.122436Z","iopub.execute_input":"2021-07-12T22:35:09.122955Z","iopub.status.idle":"2021-07-12T22:35:27.871867Z","shell.execute_reply.started":"2021-07-12T22:35:09.122915Z","shell.execute_reply":"2021-07-12T22:35:27.868262Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.to_csv(\"train.csv\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}