{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# instalar as bibliotecas\n!pip install fast_slic\n!conda install '/kaggle/input/pydicom-helper/certifi-2020.12.5-py37h89c1867_1.tar.bz2' -c conda-forge -y\n!conda install '/kaggle/input/pydicom-helper/libgcc-ng-9.3.0-h2828fa1_19.tar.bz2' -c conda-forge -y\n!conda install '/kaggle/input/pydicom-helper/gdcm-2.8.9-py37h500ead1_1.tar.bz2' -c conda-forge -y\n!conda install '/kaggle/input/pydicom-helper/conda-4.10.1-py37h89c1867_0.tar.bz2' -c conda-forge -y\n!conda install '/kaggle/input/pydicom-helper/certifi-2020.12.5-py37h89c1867_1.tar.bz2' -c conda-forge -y\n!conda install '/kaggle/input/pydicom-helper/openssl-1.1.1k-h7f98852_0.tar.bz2' -c conda-forge -y","metadata":{"execution":{"iopub.status.busy":"2021-07-22T05:09:41.683942Z","iopub.execute_input":"2021-07-22T05:09:41.689532Z","iopub.status.idle":"2021-07-22T05:11:00.835475Z","shell.execute_reply.started":"2021-07-22T05:09:41.689293Z","shell.execute_reply":"2021-07-22T05:11:00.830384Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import SimpleITK as sitk\nimport numpy as np\nimport pandas as pd\n\nimport imageio\nimport random\n\nfrom fast_slic import Slic\nfrom PIL import Image\n\nimport glob\nimport ujson\nimport time\nimport os\n\nimport multiprocessing\n\nfrom multiprocessing import Pool, Manager, Process, Lock\n\nimport pydicom\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\n\ndef rand_color(n):\n    colors = []\n    for i in range(n):\n        r = random.randint(0, 255)\n        g = random.randint(0, 255)\n        b = random.randint(0, 255)\n        colors.append([r,g,b])\n    return colors\n\ndef read_xray(path, voi_lut = True, fix_monochrome = True):\n    # Original from: https://www.kaggle.com/raddar/convert-dicom-to-np-array-the-correct-way\n    dicom = pydicom.read_file(path)\n    \n    # VOI LUT (if available by DICOM device) is used to transform raw DICOM data to \n    # \"human-friendly\" view\n    if voi_lut:\n        data = apply_voi_lut(dicom.pixel_array, dicom)\n    else:\n        data = dicom.pixel_array\n               \n    # depending on this value, X-ray may look inverted - fix that:\n    if fix_monochrome and dicom.PhotometricInterpretation == \"MONOCHROME1\":\n        data = np.amax(data) - data\n        \n    data = data - np.min(data)\n    data = data / np.max(data)\n    data = (data * 255).astype(np.uint8)\n    return data\n\ndef make_sp_from_array(image, side=10, comp=10):\n    nsp = int((image.shape[0]*image.shape[1])/(side*side))\n    slic = Slic(num_components=nsp, compactness=comp)       \n    ma_arr = slic.iterate(image) # Cluster Map\n    ma_arr = ma_arr+1\n    return ma_arr\n\ndef makesp(imagefi=None, imagefo=None, side=10, comp=10, issave=False):\n    ma_arr = None\n   \n    #with Image.open(imagefi) as f:       \n    if True:\n        f = sitk.ReadImage(imagefi, sitk.sitkVectorUInt8)\n        #image = np.array(f)\n        \n        image = sitk.GetArrayFromImage(f)\n        print(\"nda***\", image)\n\n        #image = np.zeros([1000, 800, 3], dtype=np.uint8)\n\n        nsp = int((image.shape[0]*image.shape[1])/(side*side))\n        print(side, nsp)\n        print(\"image\",image.shape)\n        # import cv2; image = cv2.cvtColor(image, cv2.COLOR_RGB2LAB)   # You can convert the image to CIELAB space if you need.\n        slic = Slic(num_components=nsp, compactness=comp)\n       \n        ma_arr = slic.iterate(image) # Cluster Map\n        ma_arr = ma_arr+1\n        \n        print(ma_arr)\n        \n        if issave:\n            ma = sitk.GetImageFromArray(ma_arr)\n            #ma.CopyInformation(image_sitk)\n            sitk.WriteImage(ma, imagefo, True)\n\n    return ma_arr\n\ndef drawsp(fmask, imagefo):\n    # read mask\n    mask = sitk.ReadImage(fmask)\n    mask_array = sitk.GetArrayFromImage(mask)\n\n    lsif = sitk.LabelShapeStatisticsImageFilter()\n    lsif.Execute(mask)\n\n    labels = lsif.GetLabels()\n    colors = rand_color(len(labels)+100)\n\n    image_out = np.zeros((mask_array.shape[0], mask_array.shape[1], 3), dtype=np.uint8)\n    for l in labels:\n        index = np.where(mask_array == int(l))        \n        image_out[index] = colors[int(l)]\n\n    # write out image\n    imageio.imsave(imagefo, image_out)\n\n    \ndef process_sp(arg):\n    c = arg[\"c\"]\n    spmap = arg[\"sp\"]\n    gt_arr = arg[\"gt\"]\n    la = arg[\"la\"]\n\n    pcs=[0.0, 0.0, 0.0, 0.0]\n    if c>1:            \n        sp_count = len(spmap[spmap==la])\n        c_count = float(np.count_nonzero(gt_arr[spmap==la] == c))\n        pcs[c-1] = 0.0 if c_count==0.0 else c_count/sp_count\n    return la, pcs\n    \ndef make_superpixels_train(side=50, l_inf=None, l_sup=None, save_gt=False):\n    path_gt = \"./groundtruth/\"\n    if not os.path.exists(path_gt):\n        os.makedirs(path_gt)\n\n    path_sp = \"./superpixels/\"+str(side)\n    if not os.path.exists(path_sp):\n        os.makedirs(path_sp)\n\n    # make combine train csv\n    df_image_level = pd.read_csv(\"../input/siim-covid19-detection/train_image_level.csv\")\n    df_study_level = pd.read_csv(\"../input/siim-covid19-detection/train_study_level.csv\")\n    \n    df_image_level[\"image_id\"] = df_image_level.id.map(lambda x: x.split('_')[0])\n    df_study_level[\"StudyInstanceUID\"] = df_study_level.id.map(lambda x: x.split('_')[0])\n    \n    df_image_level.rename(columns={\"id\": \"id_image\"}, inplace=True)\n    df_study_level.rename(columns={\"id\": \"id_study\"}, inplace=True)\n    \n    #print(\"df_image_level\", df_image_level.shape[0])\n\n    df_com = pd.merge(df_image_level, df_study_level, how=\"left\", on=[\"StudyInstanceUID\"])\n    #df_com.to_csv('train_image_study_level.csv',index=False)\n    \n    dcm_path = glob.glob('/kaggle/input/siim-covid19-detection/train/**/*dcm', recursive=True)\n    img_meta = pd.DataFrame({'dcm_path':dcm_path})\n    img_meta['image_id'] = img_meta.dcm_path.map(lambda x: x.split('/')[-1].replace('.dcm', ''))\n    img_meta['study_id'] = img_meta.dcm_path.map(lambda x: x.split('/')[-3].replace('.dcm', ''))\n    #img_meta.to_csv('train_img_meta.csv',index=False)\n    \n    df_com = pd.merge(df_com, img_meta, how=\"left\", on=[\"image_id\"])\n    df_com.to_csv('train_combine.csv',index=False)\n\n    #print(df_study_level)\n    #print(\"df_com\", df_com)\n    \n    #print(\"df_com\", df_com.shape[0])\n    #print(df_com.head())\n    \n    \n    # make superpixels and csv file\n    dfs_image_id = []\n    dfs_image_study = []\n    dfs_label_sp = []\n    dfs_label_1 = []\n    dfs_label_2 = []\n    dfs_label_3 = []\n    dfs_label_4 = []    \n    dfs_label_1_p = []\n    dfs_label_2_p = []\n    dfs_label_3_p = []\n    dfs_label_4_p = []\n\n    ii = 0\n    # for each image\n    if l_inf!=None and l_sup!=None:\n        df_com = df_com.iloc[l_inf:l_sup,]\n        \n    for index, row in df_com.iterrows():\n        tic = time.time()\n        #get label           \n        c = 0\n        if row[\"Negative for Pneumonia\"] > 0:\n            c = 1\n        elif row[\"Typical Appearance\"] > 0:\n            c = 2\n        elif row[\"Indeterminate Appearance\"] > 0:\n            c = 3\n        elif row[\"Atypical Appearance\"] > 0:\n            c = 4\n            \n            \n        # make superpixels\n        #print(row[\"dcm_path\"])\n        xray = read_xray(row['dcm_path'])\n        rgb_img = np.stack((xray,)*3, axis=-1)\n        spmap = make_sp_from_array(rgb_img, side=side)\n        # save superpixels\n        sitk.WriteImage(sitk.GetImageFromArray(spmap), \"./superpixels/\"+str(side)+\"/\"+row[\"id_image\"]+\".nrrd\", True)\n\n        hs, ws = spmap.shape[0],spmap.shape[1]\n        \n        #print(row['boxes'])\n        #args = ujson.loads(\"\"\"+\"\"\"+str(row['boxes'])+\"\"\"+\"\"\")\n        #print(args)\n        \n        #make boxes\n        bboxes = []\n        bbox = []\n        scale = 1\n        for i, l in enumerate(row['label'].split(' ')):\n            if (i % 6 == 0) | (i % 6 == 1):\n                continue\n            bbox.append(float(l)/scale)\n            if i % 6 == 5:\n                bboxes.append(bbox)\n                bbox = []    \n                \n        #if c==1:\n        #    print(bboxes)\n\n        gt_arr = np.zeros((hs, ws), dtype=np.uint16)\n        for box in bboxes:\n            x = int(box[0])\n            y = int(box[1])\n            h = int(box[2])\n            w = int(box[3])\n            sub_img = gt_arr[y:y+h, x:x+w]\n            sub_img = c\n            gt_arr[y:y+h, x:x+w] = sub_img\n        if save_gt:\n            sitk.WriteImage(sitk.GetImageFromArray(gt_arr), \"./groundtruth/\"+row[\"id_image\"]+\".nrrd\", True)\n        #print(gt_arr)\n        \n        #print(spmap.shape, gt_arr.shape)\n        \n        \n        # read each superpixel\n        spimg = sitk.GetImageFromArray(spmap)\n        lsif = sitk.LabelShapeStatisticsImageFilter()\n        lsif.Execute(spimg)\n        labels = lsif.GetLabels()\n        #print(labels)\n        \n        \n        argpass = []\n        for la in labels:\n            argpass.append({\n                \"c\":c,\n                \"la\":la,\n                \"sp\":spmap,\n                \"gt\":gt_arr\n            })\n        \n        \n        ncpus = 20\n        #ncpus = multiprocessing.cpu_count()-1\n        pool = Pool(processes=ncpus)\n        rr = pool.map(process_sp, argpass)\n        pool.close()\n\n        for rs in rr:\n            la = rs[0]\n            pcs = rs[1]\n            dfs_image_id.append(row['id_image'])\n            dfs_image_study.append(row['StudyInstanceUID'])\n            dfs_label_sp.append(la)\n            dfs_label_1.append(row['Negative for Pneumonia'])\n            dfs_label_2.append(row['Typical Appearance'])\n            dfs_label_3.append(row['Indeterminate Appearance'])\n            dfs_label_4.append(row['Atypical Appearance'])\n            dfs_label_1_p.append(pcs[0])\n            dfs_label_2_p.append(pcs[1])\n            dfs_label_3_p.append(pcs[2])\n            dfs_label_4_p.append(pcs[3])\n\n            \n            \n        \"\"\"\n        for la in labels:\n            #box = lsif.GetBoundingBox(la)\n            #x = int(box[0])\n            #y = int(box[1])\n            #h = int(box[2])\n            #w = int(box[3])          \n            #sp_count = np.count_nonzero(spmap[y:y+h, x:x+w] == la)\n            pcs=[0.0, 0.0, 0.0, 0.0]\n            if c>1:            \n                sp_count = len(spmap[spmap==la])\n                c_count = float(np.count_nonzero(gt_arr[spmap==la] == c))\n\n                pcs[c-1] = 0.0 if c_count==0.0 else c_count/sp_count\n            dfs_image_id.append(row['id_image'])\n            dfs_image_study.append(row['StudyInstanceUID'])\n            dfs_label_sp.append(la)\n            dfs_label_1.append(row['Negative for Pneumonia'])\n            dfs_label_2.append(row['Typical Appearance'])\n            dfs_label_3.append(row['Indeterminate Appearance'])\n            dfs_label_4.append(row['Atypical Appearance'])\n            dfs_label_1_p.append(pcs[0])\n            dfs_label_2_p.append(pcs[1])\n            dfs_label_3_p.append(pcs[2])\n            dfs_label_4_p.append(pcs[3])\n        \"\"\"\n        print(ii, row['id_image'], time.time()-tic)\n\n        #if ii==1:\n        #   break\n        ii+=1\n    \n    #save superpixels\n    data_sp =  {\n                'image_id': dfs_image_id,\n                'image_study': dfs_image_study,\n                'label_sp': dfs_label_sp,\n                'Negative_for_Pneumonia': dfs_label_1,\n                'Typical_Appearance': dfs_label_2,\n                'Indeterminate_Appearance': dfs_label_3,\n                'Atypical_Appearance': dfs_label_4,\n                'Negative_for_Pneumonia_percentage': dfs_label_1_p,\n                'Typical_Appearance_percentage': dfs_label_2_p,\n                'Indeterminate_Appearance_percentage': dfs_label_3_p,\n                'Atypical_Appearance_percentage': dfs_label_4_p\n            }\n\n    df_superpixels = pd.DataFrame.from_dict(data_sp)\n    df_superpixels.to_csv('train_superpixels.csv',index=False)\n\nmake_superpixels_train(side=100, l_inf=0, l_sup=10, save_gt=True)\n\n# make_superpixels_train(side=50, l_inf=0, l_sup=1000, save_gt=True)\n# make_superpixels_train(side=50, l_inf=1000, l_sup=2000, save_gt=True)\n# make_superpixels_train(side=50, l_inf=2000, l_sup=3000, save_gt=True)\n# make_superpixels_train(side=50, l_inf=3000, l_sup=4000, save_gt=True)\n# make_superpixels_train(side=50, l_inf=5000, l_sup=6000, save_gt=True)\n# make_superpixels_train(side=50, l_inf=6000, l_sup=7000, save_gt=True)\n        ","metadata":{"execution":{"iopub.status.busy":"2021-07-22T05:11:22.495204Z","iopub.execute_input":"2021-07-22T05:11:22.497136Z","iopub.status.idle":"2021-07-22T05:17:52.100873Z","shell.execute_reply.started":"2021-07-22T05:11:22.497034Z","shell.execute_reply":"2021-07-22T05:17:52.094288Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# imginputs = [\"../input/imagesample/bear.jpg\",\"../input/imagesample/bear.jpg\"]\n# i=0\n# for imginput in imginputs:\n#     imgoutput = \"bear\"+str(i)+\".nrrd\"\n#     sp = makesp(imagefi=imginput, imagefo=imgoutput, side=30, comp=100, issave=True)\n#     drawsp(imgoutput,\"bearr\"+str(i)+\".png\")\n#     i+=1\n#     #import urllib.request\n#     # urllib.request.urlretrieve(\n#     #     'https://criptoativo.com.br/wp-content/uploads/2021/01/Grayscale-1536x984.jpeg',\n#     #     \"gfg.jpeg\")\n#     # makesp(\"gfg.jpeg\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from IPython.display import Image\n# Image(\"bearr0.png\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Image(\"bearr1.png\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#         argpass = []\n#         for la in labels:\n#             argpass.append({\n#                 \"c\":c,\n#                 \"la\":la,\n#                 \"sp\":spmap,\n#                 \"gt\":gt_arr\n#             })\n        \n        \n#         ncpus = 15\n#         #ncpus = multiprocessing.cpu_count()-1\n#         pool = Pool(processes=ncpus)\n#         rr = pool.map(process_sp, argpass)\n#         pool.close()\n\n#         for rs in rr:\n#             la = rs[0]\n#             pcs = rs[1]\n#             dfs_image_id.append(row['id_image'])\n#             dfs_image_study.append(row['StudyInstanceUID'])\n#             dfs_label_sp.append(la)\n#             dfs_label_1.append(row['Negative for Pneumonia'])\n#             dfs_label_2.append(row['Typical Appearance'])\n#             dfs_label_3.append(row['Indeterminate Appearance'])\n#             dfs_label_4.append(row['Atypical Appearance'])\n#             dfs_label_1_p.append(pcs[0])\n#             dfs_label_2_p.append(pcs[1])\n#             dfs_label_3_p.append(pcs[2])\n#             dfs_label_4_p.append(pcs[3])\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}