{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"\n#20 temmuz: this will be the ensemble version...","metadata":{"execution":{"iopub.status.busy":"2021-07-21T11:56:00.950686Z","iopub.execute_input":"2021-07-21T11:56:00.95122Z","iopub.status.idle":"2021-07-21T11:56:00.956261Z","shell.execute_reply.started":"2021-07-21T11:56:00.951097Z","shell.execute_reply":"2021-07-21T11:56:00.955142Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Example:\n# Changed from: 'libjpeg-turbo-2.1.0-h7f98852_0.tar.bz2'\n# Changed to: '../input/GDCM-notebook/libjpeg-turbo-2.1.0-h7f98852_0.tar.bz2'\n# ../input/k/abdulkadirguner/installs-gdcm/certifi-2020.12.5-py37h89c1867_1.tar.bz2\n# ../input/k/abdulkadirguner/installs-gdcm/conda-4.10.1-py37h89c1867_0.tar.bz2\n!conda install '../input/k/abdulkadirguner/installs-gdcm/libjpeg-turbo-2.1.0-h7f98852_0.tar.bz2' -c conda-forge -y\n!conda install '../input/k/abdulkadirguner/installs-gdcm/libgcc-ng-9.3.0-h2828fa1_19.tar.bz2' -c conda-forge -y\n!conda install '../input/k/abdulkadirguner/installs-gdcm/gdcm-2.8.9-py37h500ead1_1.tar.bz2' -c conda-forge -y\n!conda install '../input/k/abdulkadirguner/installs-gdcm/conda-4.10.1-py37h89c1867_0.tar.bz2' -c conda-forge -y\n!conda install '../input/k/abdulkadirguner/installs-gdcm/certifi-2020.12.5-py37h89c1867_1.tar.bz2' -c conda-forge -y\n!conda install '../input/k/abdulkadirguner/installs-gdcm/openssl-1.1.1k-h7f98852_0.tar.bz2' -c conda-forge -y","metadata":{"execution":{"iopub.status.busy":"2021-07-21T11:56:00.958274Z","iopub.execute_input":"2021-07-21T11:56:00.958965Z","iopub.status.idle":"2021-07-21T11:57:12.577135Z","shell.execute_reply.started":"2021-07-21T11:56:00.958926Z","shell.execute_reply":"2021-07-21T11:57:12.57605Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pydicom\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut","metadata":{"execution":{"iopub.status.busy":"2021-07-21T11:57:12.580553Z","iopub.execute_input":"2021-07-21T11:57:12.580919Z","iopub.status.idle":"2021-07-21T11:57:12.845078Z","shell.execute_reply.started":"2021-07-21T11:57:12.580883Z","shell.execute_reply":"2021-07-21T11:57:12.844175Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import glob\nfrom shutil import copyfile\n\n\n\n!mkdir -p /tmp/pip/cache/\n\n\n\nfor file in glob.glob('../input/segmentation-models/*.*'):\n    filem=file.replace('../input/segmentation-models','/tmp/pip/cache').replace('xyz','tar.gz')\n    print(file,filem)\n    copyfile(file,filem)","metadata":{"execution":{"iopub.status.busy":"2021-07-21T11:57:12.846892Z","iopub.execute_input":"2021-07-21T11:57:12.847299Z","iopub.status.idle":"2021-07-21T11:57:14.389716Z","shell.execute_reply.started":"2021-07-21T11:57:12.847258Z","shell.execute_reply":"2021-07-21T11:57:14.388674Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install --no-index --find-links /tmp/pip/cache/ segmentation-models-pytorch","metadata":{"execution":{"iopub.status.busy":"2021-07-21T11:57:14.391548Z","iopub.execute_input":"2021-07-21T11:57:14.391957Z","iopub.status.idle":"2021-07-21T11:57:24.864374Z","shell.execute_reply.started":"2021-07-21T11:57:14.391913Z","shell.execute_reply":"2021-07-21T11:57:24.863429Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import segmentation_models_pytorch as smp\nimport cv2\n\nimport albumentations as A\nfrom pathlib import Path\nfrom PIL import Image\nfrom tqdm import tqdm\nimport os","metadata":{"execution":{"iopub.status.busy":"2021-07-21T11:57:24.86775Z","iopub.execute_input":"2021-07-21T11:57:24.868067Z","iopub.status.idle":"2021-07-21T11:57:28.891465Z","shell.execute_reply.started":"2021-07-21T11:57:24.868039Z","shell.execute_reply":"2021-07-21T11:57:28.890621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from fastai.vision.widgets import *\n#from tqdm import tqdm\nfrom fastai.data.all import *\nfrom fastai.vision.all import *\nfrom fastai import *\nfrom fastai.callback.fp16 import *\nimport gc","metadata":{"execution":{"iopub.status.busy":"2021-07-21T11:57:28.893804Z","iopub.execute_input":"2021-07-21T11:57:28.894173Z","iopub.status.idle":"2021-07-21T11:57:29.418639Z","shell.execute_reply.started":"2021-07-21T11:57:28.894137Z","shell.execute_reply":"2021-07-21T11:57:29.41775Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## revised read_xray: returns data and its shape\ndef read_xray_x(dcmfile, voi_lut = True, fix_monochrome = True):\n    gc.collect()\n    dicom = pydicom.dcmread(dcmfile)\n    \n    # VOI LUT (if available by DICOM device) is used to transform raw DICOM data to \"human-friendly\" view\n    if voi_lut:\n        data = apply_voi_lut(dicom.pixel_array, dicom)\n    else:\n        data = dicom.pixel_array\n               \n    # depending on this value, X-ray may look inverted - fix that:\n    if fix_monochrome and dicom.PhotometricInterpretation == \"MONOCHROME1\":\n        data = np.amax(data) - data\n        \n    data = data - np.min(data)\n    data = data / np.max(data)\n    data = (data * 255).astype(np.uint8)\n    \n    #Image.fromarray(data).save('./images/' + dcmfile.stem + '.png',format = 'png')\n    return data   #height width gondermiyorum....\n\n\n# im = Image.fromarray(A)\n# im.save(\"your_file.jpeg\")","metadata":{"execution":{"iopub.status.busy":"2021-07-21T11:57:29.420501Z","iopub.execute_input":"2021-07-21T11:57:29.420833Z","iopub.status.idle":"2021-07-21T11:57:29.427094Z","shell.execute_reply.started":"2021-07-21T11:57:29.420799Z","shell.execute_reply":"2021-07-21T11:57:29.426083Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## this class makes it possible to use custom augmentations\nclass AlbumentationsTransform(DisplayedTransform):  #displayed transform fastai icinden galiba....\n    split_idx,order=0,2\n    def __init__(self, train_aug): store_attr()\n    \n    def encodes(self, img: PILImage):\n        aug_img = self.train_aug(image=np.array(img))['image']\n        return PILImage.create(aug_img)","metadata":{"execution":{"iopub.status.busy":"2021-07-21T11:57:29.428491Z","iopub.execute_input":"2021-07-21T11:57:29.429005Z","iopub.status.idle":"2021-07-21T11:57:29.44309Z","shell.execute_reply.started":"2021-07-21T11:57:29.428968Z","shell.execute_reply":"2021-07-21T11:57:29.442243Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def xfunc(o):\n    return o.files","metadata":{"execution":{"iopub.status.busy":"2021-07-21T11:57:29.444362Z","iopub.execute_input":"2021-07-21T11:57:29.444922Z","iopub.status.idle":"2021-07-21T11:57:29.452808Z","shell.execute_reply.started":"2021-07-21T11:57:29.444876Z","shell.execute_reply":"2021-07-21T11:57:29.451818Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def yfunc(o):\n    return o.s_label","metadata":{"execution":{"iopub.status.busy":"2021-07-21T11:57:29.45429Z","iopub.execute_input":"2021-07-21T11:57:29.454723Z","iopub.status.idle":"2021-07-21T11:57:29.462501Z","shell.execute_reply.started":"2021-07-21T11:57:29.454687Z","shell.execute_reply":"2021-07-21T11:57:29.461631Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#x ray classification model\n\ndef label_func(filiem): \n    file=filiem.parts[1].split('_')[0] #+ '.png'\n    #print(file) #bu iki print (asagidaki ile) hangi file hangi label gormek icin iyi oldu...\n    returnee=merged_annotations.loc[merged_annotations.filename==file].iloc[0].values[-1]\n    #print(returnee)\n    return returnee\n\n#learn_xray=load_learner('../input/07temmuz-augmented-enhanced-xrayclassifier/07temmuz_xrayclassifier_augmented_baseline_finetuned_3.pkl',cpu=False)\nlearn_xray=load_learner('../input/09temmuz-submissions-01/09temmuz_xrayclassifier_augmented_baseline_finetuned_4.pkl',cpu=False) #V2 OLARAK SAVE EDIYORUM..\n    #BASELINE UZERINE SADECE XRAY CLASSIFIER FINETUNE3 IDI... ONU BASE DATA ILE TEKRAR TRAIN ETTIM....KISA OLARAK, TYPICAL VE NONE DAHA IYI GORSUN DIYE,, GORUYOR\n    \n","metadata":{"execution":{"iopub.status.busy":"2021-07-21T11:57:29.463799Z","iopub.execute_input":"2021-07-21T11:57:29.464478Z","iopub.status.idle":"2021-07-21T11:57:36.193333Z","shell.execute_reply.started":"2021-07-21T11:57:29.464436Z","shell.execute_reply":"2021-07-21T11:57:36.192419Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def label_func(filiem): #burada taframeden okuyorum str olarak geliyor... parts kullanmak icin once path yap\n    #print(Path(filiem[0]).parts[1].split('_')[1])  #opacity_boxes/6640cb880cec_indeterminate__6466.png olarak geldi onu parse et asagida\n    return Path(filiem[0]).parts[1].split('_')[1]#buraya gelen pandas series object, bu nedenle [0] togetstring\n                        #and then you can path it and extract the albel\n##V1:    \n#learn_box=load_learner('../input/09temmuz-submissions-01/06temmuz_BOXclassifier_224_baseline_leanNEGATIFES.pkl',cpu=False) #baselinedan farki boxclassifier lean\n                #NEDEN LEAN: DAHA AZ HATA YAPYOR TYPICAL VE NEGATIVE OLARAK, EGER TEST DE TRAIN GIBI SKEWED ISE.. BU DURUMDA SONUC DAHA IYI GELEBILIR...\nlearn_box=load_learner('../input/06temmuz-corrrected-atypical5-indeter2-finetuned/06temmuz_leannegatives_atypical5_indet2_finetuned__XXX.pkl',cpu=False)","metadata":{"execution":{"iopub.status.busy":"2021-07-21T11:57:36.194672Z","iopub.execute_input":"2021-07-21T11:57:36.195023Z","iopub.status.idle":"2021-07-21T11:57:36.942663Z","shell.execute_reply.started":"2021-07-21T11:57:36.194989Z","shell.execute_reply":"2021-07-21T11:57:36.941795Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_data_segment(preds,allinfo,xray_learner,box_learner,detection_resize,detection_thold):\n    #gelmisken hemen xray level classification yap...\n    preds_xray=[xray_learner.predict(x[0]) for x in allinfo]\n    \n    #hemen dataframeini kur,, sonuclar burada olacak\n    detect_image=pd.DataFrame(columns=['ImageID','width','height',\n                                       'atypical','indeterminate','negative','typical','studyID'])\n    \n    detect_box=pd.DataFrame(columns=['ImageID','iskalp', 'boxconf','XMin','YMin','XMax','YMax',\n                                     'atypical','indeterminate','negative','typical','studyID'])\n    im_counter=0\n    box_counter=0\n    for detection,xray,info in zip(preds,preds_xray,allinfo):\n        imageid=info[2] + '_image'\n        studyid=info[1] + '_study'\n        width=info[0].shape[1]\n        height=info[0].shape[0]\n        atypical_xray = xray[2][0].numpy()\n        indeterminate_xray = xray[2][1].numpy()\n        negative_xray = xray[2][2].numpy()\n        typical_xray = xray[2][3].numpy()\n        \n        detect_image.loc[im_counter] = [imageid,width,height,atypical_xray,indeterminate_xray,\n                                       negative_xray,typical_xray,studyid]\n        im_counter+=1\n        \n        \n        if preds[detection]['kalp'][0][0]==0 and preds[detection]['diger'][0][0]==0: #her iki lung da bos\n        #if detections.detection.bboxes==[]:                                      #xminlerini kontrol ediyorum\n            atypical_box = 0\n            indeterminate_box = 0\n            negative_box = 1\n            typical_box = 0            \n            xmin=0\n            xmax=1\n            ymin=0\n            ymax=1\n            boxconf=1\n            iskalp=0  #box box yani hicbir lunga ait degil anlaminda....\n            \n            detect_box.loc[box_counter] = [imageid, iskalp, boxconf, xmin,ymin,xmax,ymax, \n                                           atypical_box, indeterminate_box, negative_box, typical_box,\n                                          studyid]\n            \n            box_counter+=1\n            \n        else: # DIKKAT KALP VE DIGER LUNGLARI AYRI AYRI BOXLARA ILAVE EDECEGIM (KALP IS KALP IS 1, DIGER 2)\n            for sm,box in enumerate(preds[detection]['kalp']):\n                if box[0]==0: continue    ## herikisi de degil ama biri bossa?.... o zaman bu kontrol olmali!!\n                iskalp=1    #kalplung oldugunu belirten column... 0 ise zaten bos box demek....\n                xmin=box[0]\n                xmax=box[2]\n                ymin=box[1]\n                ymax=box[3]\n                \n                boxconf=box[4] \n                \n                ##iste burada ilave olarak her bir box icin box classifier sonuclarini bulacagiz....\n                crop = info[0][max(0,ymin):min(ymax,height), max(0,xmin):min(xmax,width)].copy()\n                boxpred=box_learner.predict(crop)\n                \n                atypical_box = boxpred[2][0].numpy()\n                indeterminate_box = boxpred[2][1].numpy()\n                negative_box = boxpred[2][2].numpy()\n                typical_box = boxpred[2][3].numpy()\n                \n                detect_box.loc[box_counter] = [imageid, iskalp, boxconf, xmin,ymin,xmax,ymax, \n                                           atypical_box, indeterminate_box, negative_box, typical_box,\n                                          studyid]\n            \n                box_counter+=1\n                \n            ## do the same for diger lung\n            for sm,box in enumerate(preds[detection]['diger']):\n                if box[0]==0: continue    ## herikisi de degil ama biri bossa?.... o zaman bu kontrol olmali!!\n                iskalp=2    #kalplung oldugunu belirten column... 0 ise zaten bos box demek....\n                xmin=box[0]\n                xmax=box[2]\n                ymin=box[1]\n                ymax=box[3]\n                \n                boxconf=box[4] \n                \n                ##iste burada ilave olarak her bir box icin box classifier sonuclarini bulacagiz....\n                crop = info[0][max(0,ymin):min(ymax,height), max(0,xmin):min(xmax,width)].copy()\n                boxpred=box_learner.predict(crop)\n                \n                atypical_box = boxpred[2][0].numpy()\n                indeterminate_box = boxpred[2][1].numpy()\n                negative_box = boxpred[2][2].numpy()\n                typical_box = boxpred[2][3].numpy()\n                \n                detect_box.loc[box_counter] = [imageid, iskalp, boxconf, xmin,ymin,xmax,ymax, \n                                           atypical_box, indeterminate_box, negative_box, typical_box,\n                                          studyid]\n            \n                box_counter+=1\n                \n                \n    return detect_image,detect_box","metadata":{"execution":{"iopub.status.busy":"2021-07-21T11:57:36.944032Z","iopub.execute_input":"2021-07-21T11:57:36.944408Z","iopub.status.idle":"2021-07-21T11:57:36.965933Z","shell.execute_reply.started":"2021-07-21T11:57:36.944367Z","shell.execute_reply":"2021-07-21T11:57:36.965177Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from itertools import islice\n\ndef chunked(generator, size):\n    \"\"\"Read parts of the generator, pause each time after a chunk\"\"\"\n    # islice returns results until 'size',\n    # make_chunk gets repeatedly called by iter(callable).\n    gen = iter(generator)\n    make_chunk = lambda: list(islice(gen, size))\n    return iter(make_chunk, [])\n\n# for files in chunked(allinfo,10):","metadata":{"execution":{"iopub.status.busy":"2021-07-21T11:57:36.967417Z","iopub.execute_input":"2021-07-21T11:57:36.967804Z","iopub.status.idle":"2021-07-21T11:57:36.977383Z","shell.execute_reply.started":"2021-07-21T11:57:36.967767Z","shell.execute_reply":"2021-07-21T11:57:36.976486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"paths=Path('../input/siim-covid19-detection/test')\ntumliste=list(paths.glob('**/*.dcm'))\n#tumliste = tumliste[:10]","metadata":{"execution":{"iopub.status.busy":"2021-07-21T11:57:36.978785Z","iopub.execute_input":"2021-07-21T11:57:36.979266Z","iopub.status.idle":"2021-07-21T11:57:42.886489Z","shell.execute_reply.started":"2021-07-21T11:57:36.979229Z","shell.execute_reply":"2021-07-21T11:57:42.878112Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# tumliste.index(Path('../input/siim-covid19-detection/train/e4e62d4e8849/07388331bba9/9c24e37a0ef5.dcm'))\n#tumliste","metadata":{"execution":{"iopub.status.busy":"2021-07-21T11:57:42.887676Z","iopub.execute_input":"2021-07-21T11:57:42.888005Z","iopub.status.idle":"2021-07-21T11:57:42.900317Z","shell.execute_reply.started":"2021-07-21T11:57:42.887971Z","shell.execute_reply":"2021-07-21T11:57:42.895221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def fillpoly(predsim): ## returns filledpoly masks from predsim....\n    ##fillpoly the preds\n    filledpoly=[]\n    for img_np in predsim:\n        img2li=img_np.copy()\n        img2li[img2li!=2]=0\n        conturs,_=cv2.findContours(img2li,cv2.RETR_EXTERNAL,cv2.CHAIN_APPROX_SIMPLE)\n        maskem2 = np.zeros(img_np.shape,dtype=np.uint8)\n        for cc in conturs: #nonzero pixels are considered as one, so you dont need \n            maskem2=cv2.fillPoly(maskem2, [cc], 2)\n\n        img1li=img_np.copy()\n        img1li[img1li!=1]=0\n        conturs,_=cv2.findContours(img1li,cv2.RETR_EXTERNAL,cv2.CHAIN_APPROX_SIMPLE)\n        maskem1 = np.zeros(img_np.shape,dtype=np.uint8)\n        for cc in conturs:\n            maskem1=cv2.fillPoly(maskem1, [cc], 1)\n        maskem=maskem1 + maskem2\n        filledpoly.append(maskem)\n    return filledpoly","metadata":{"execution":{"iopub.status.busy":"2021-07-21T11:57:42.901623Z","iopub.execute_input":"2021-07-21T11:57:42.902167Z","iopub.status.idle":"2021-07-21T11:57:42.922505Z","shell.execute_reply.started":"2021-07-21T11:57:42.902102Z","shell.execute_reply":"2021-07-21T11:57:42.921661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## BURADA PREDS BIR LIST OF LISTS \n### ITS A LIST BECAUSE EACH TEST FILE PREDICTION IS IN ONE LINE AS A LIST..\n#### AND EACH PREDICTION IS A LIST CONTAINING ALL THE BOXES IN THAT PREDICTION...\n##### EACH BOX IS A TUPLE OF 5, PROBABILITY, XMIN,YMIN,XMAX,YMAX,....\n###### IF THERE IS NO BOX,,, STILL THERE IS ONE TUPLE OF 5, (1, 0, 0, 1, 1)\n######## BOYLE GELSIN BUNA GORE GUNCELLERSIN.......................................................\n\nfind_cigers=load_learner('../input/17temmuz-baseline/14temmuz_lungclassifier_finetune_3_EFFNET_b4__choosen.pkl',cpu=False)\nfind_kalp_2=load_learner('../input/kalptrain-inclhistdata-b4/20temmuz_kalp_TEST_INCHISTDATA_finetune_2_512_EFFNET_b4.pkl',cpu=False)\nfind_kalp_1=load_learner('../input/18temmuz-models/18temmuz_all_TEST_finetune_2_EFFNET_b5.pkl',cpu=False)\nfind_kalp_3=load_learner('../input/20temmuz-b2-b3-all-models/19temmuz_all_TEST_finetune_2_EFFNET_b3.pkl',cpu=False)\n#find_kalp_4=load_learner('../input/20temmuz-b2-b3-all-models/19temmuz_all_TEST_finetune_2_EFFNET_b2.pkl',cpu=False)\n\n\n\nfind_diger_2=load_learner('../input/19-temmuz-singlemodels/18temmuz_diger_TEST_finetune_2_512_EFFNET_b4.pkl',cpu=False)\nfind_diger_1=load_learner('../input/18temmuz-models/18temmuz_all_TEST_finetune_2_EFFNET_b5.pkl',cpu=False)\nfind_diger_3=load_learner('../input/20temmuz-b2-b3-all-models/19temmuz_all_TEST_finetune_2_EFFNET_b3.pkl',cpu=False)\n#find_diger_4=load_learner('../input/20temmuz-b2-b3-all-models/19temmuz_all_TEST_finetune_2_EFFNET_b2.pkl',cpu=False)\n\n\n\ndef prediction(datachunk,thold):  #thold segmentator thold ama ihtiyac yok ki direct decoded olarak al..\n    \n    ##datachunk is the data in np.array format and with bs of 64 and \n    ##each member of list datachunk is a tuple of 3: first the data, studyid, imageid\n    ##you might use imageid for the width and height of the that image to cv2.resize back to the original...\n    ## dont forget to select the right interpolation for resizing....\n    \n    preds = [find_cigers.predict(np.dstack((x[0],x[0],x[0])))[0].cpu().numpy().astype(np.uint8) for x in datachunk]\n    \n    preds = fillpoly(preds)\n    \n    #resize preds to original size.\n    sizes=[x[0].shape for x in datachunk] #x[0] larin icinde tek channel data vardi... height, widht formati\n    resized=[cv2.resize(x[0],(x[1][1],x[1][0]),interpolation = cv2.INTER_NEAREST) for x in zip(preds,sizes)]\n    \n    #now cut the datachunk xrays into left and right lungs and feet to both segmentators:\n    #and findexternal countours and find bboxes for all of fillpolyed cc in countours..\n    allboxes={}\n    for img_np,data in zip(resized,datachunk):\n        \n        \n####### diger ciger tarafindanki boxlar digerboxes icinde toplaniyor... alist of tuples        \n        img2li=img_np.copy()\n        img2li[img2li!=2]=0\n        conturs,_=cv2.findContours(img2li,cv2.RETR_EXTERNAL,cv2.CHAIN_APPROX_SIMPLE)\n        rects_diger=[]\n        digerboxes=[]\n\n        \n######....DIKKAT asagidakilerin hepsini conturs varsa yapacak....bak teee asagida bir tane daha else var...\n        if conturs:\n            for cc in conturs:\n                x,y,width,height=cv2.boundingRect(cc)\n                rects_diger.append((x,y,width,height))\n            biggest=np.argmax([x[2]*x[3] for x in rects_diger]) #en buyuk bounding recti bul...2lipixeller icin...\n            x2,y2,w2,h2=rects_diger[biggest]\n            digerlung=data[0][y2:y2+h2,x2:x2+w2]  #diger ciger burada.......................segmentatora girecek\n                                                  # X2 Y2 W2 H2 LAZIM...XLER WIDTH, Y IS HEIGHT AND FIRST IN NUMPY\n                                                  # LUNG MUSK BULDUN: BUNU RESIZE TO W2H2...FILLPOLYDEN SONRA..\n                                                  # BOYLECE SU AN LUNG MASK ICINDE DOGRU YERDE...\n                                                  # BBOX BULABILIRSIN... AMA BULDUKTAN SONRA X2 Y2 EKLEYECEKSIN\n                                                  # WIDTH HEIGHT ORJINAL AMA.. XY ICIN,,, X2 Y2 EKLE\n                                                  # CUNKU GERCEK XRAYDEKI YERINE GORE BULMAN LAZIM BOXLARI...\n\n            #resim verirsen problem yapmiyor,, ama numpy array veriyorsan ya 3le, ya da normalize in tfms                       \n            #diger=find_diger.predict(np.dstack((digerlung,digerlung,digerlung)))[0].cpu().numpy().astype(np.uint8)                       \n\n            ### BURADA THOLD KULLANIP OPACITY BOXLARINI DEGERLENDIRMEK ISTIYORSAN.. [0] DEGIL [2][1] AL VE...\n            TO_PREDICT=np.dstack((digerlung,digerlung,digerlung))\n            diger_1=find_diger_1.predict(TO_PREDICT)[2][1].cpu().numpy().astype(np.float32)\n            diger_2=find_diger_2.predict(TO_PREDICT)[2][1].cpu().numpy().astype(np.float32)\n            diger_3=find_diger_3.predict(TO_PREDICT)[2][1].cpu().numpy().astype(np.float32)\n#             diger_4=find_diger_4.predict(TO_PREDICT)[2][1].cpu().numpy().astype(np.float32)\n\n#             diger  = (diger_1 + diger_2 + diger_3 + diger_4) / 4            #diger=(diger>thold).astype(np.uint8) #diger orjinal olarak kullanilacak, bu nedenle keep it as is\n            diger  = (diger_1 + diger_2 + diger_3) / 3            #diger=(diger>thold).astype(np.uint8) #diger orjinal olarak kullanilacak, bu nedenle keep it as is\n\n            ##fillpoly before resize\n            conturs,_=cv2.findContours((diger>thold).astype(np.uint8),cv2.RETR_EXTERNAL,cv2.CHAIN_APPROX_SIMPLE)\n            maskediger = np.zeros(diger.shape,dtype=np.uint8)\n            for cc in conturs:\n                maskediger=cv2.fillPoly(maskediger, [cc], 1)  #SADECE 1LE DOLACAK HEPSI.....\n            #resize the mask to original lung shape\n            maskediger=cv2.resize(maskediger, (w2,h2)) #BURADA IYA DEFAULT YA DA ,interpolation = cv2.INTER_NEAREST\n            #find bboxes\n            conturs,_=cv2.findContours(maskediger,cv2.RETR_EXTERNAL,cv2.CHAIN_APPROX_SIMPLE)\n            if conturs:\n                for cc in conturs:\n                    ### how about adding a snippet to find average probability of the countour from score[2][1]\n                    cimg = np.zeros_like(maskediger)\n                    cimg=cv2.fillPoly(cimg, [cc], 1)  #simdi 1 ile dolu olan bu yerlerin prob topla ve suma bol..\n                    confidence=np.mean(cv2.resize(diger, (w2,h2))[cimg==1])\n\n                    x,y,width,height=cv2.boundingRect(cc)\n                    x,y = x + x2, y + y2   #ana resimdeki yerlerini bulmak icin, digerlung x2vey2 degerlerini ekle.\n                    digerboxes.append((x,y,x + width,y + height,confidence))  #FORMATI XMIN YMIN XMAX YMAX'A CEVIRDIM.....\n                                                       #from boxclassifier and xray classifier.....!!!\n                                                       #her boxun kendi opacity degeri var zaten... boxclassifer\n                                                       #buna belki xray classifier eklersin....\n                                                       #ortalamasini almak icin....\n                                                       #DOLAYISIYLA SEGMENTATOR VEYA DETECTOR BOXCONFIDENCE bosver\n\n            else:\n                digerboxes.append((0,0,1,1,1))\n        else:\n            digerboxes.append((0,0,1,1,1))\n                                \n####### kalp ciger tarafindanki boxlar kalpboxes icinde toplaniyor... alist of tuples        \n        img1li=img_np.copy()\n        img1li[img1li!=1]=0\n        conturs,_=cv2.findContours(img1li,cv2.RETR_EXTERNAL,cv2.CHAIN_APPROX_SIMPLE)\n        rects_kalp=[]\n        kalpboxes=[]\n\n######....DIKKAT asagidakilerin hepsini conturs varsa yapacak....bak teee asagida bir tane daha else var...\n        if conturs:\n            for cc in conturs:\n                x,y,width,height=cv2.boundingRect(cc)\n                rects_kalp.append((x,y,width,height))\n            biggest=np.argmax([x[2]*x[3] for x in rects_kalp]) #en buyuk bounding recti bul...1lipixeller icin...\n            x1,y1,w1,h1=rects_kalp[biggest]\n            kalplung=data[0][y1:y1+h1,x1:x1+w1]   #kalp ciger burada.......................segmentatora girecek\n                                                  # X2 Y2 W2 H2 LAZIM...XLER WIDTH, Y IS HEIGHT AND FIRST IN NUMPY\n                                                  # LUNG MUSK BULDUN: BUNU RESIZE TO W2H2...FILLPOLYDEN SONRA..\n                                                  # BOYLECE SU AN LUNG MASK ICINDE DOGRU YERDE...\n                                                  # BBOX BULABILIRSIN... AMA BULDUKTAN SONRA X2 Y2 EKLEYECEKSIN\n                                                  # WIDTH HEIGHT ORJINAL AMA.. XY ICIN,,, X2 Y2 EKLE\n                                                  # CUNKU GERCEK XRAYDEKI YERINE GORE BULMAN LAZIM BOXLARI...\n\n\n            #resim verirsen problem yapmiyor,, ama numpy array veriyorsan ya 3le, ya da normalize in tfms                       \n            #kalp=find_kalp.predict(np.dstack((kalplung,kalplung,kalplung)))[0].cpu().numpy().astype(np.uint8)                       \n            ### BURADA THOLD KULLANIP OPACITY BOXLARINI DEGERLENDIRMEK ISTIYORSAN.. [0] DEGIL [2][1] AL VE...\n            TO_PREDICT=np.dstack((kalplung,kalplung,kalplung))\n            kalp_1=find_kalp_1.predict(TO_PREDICT)[2][1].cpu().numpy().astype(np.float32)\n            kalp_2=find_kalp_2.predict(TO_PREDICT)[2][1].cpu().numpy().astype(np.float32)\n            kalp_3=find_kalp_3.predict(TO_PREDICT)[2][1].cpu().numpy().astype(np.float32)\n#             kalp_4=find_kalp_4.predict(TO_PREDICT)[2][1].cpu().numpy().astype(np.float32)\n\n#             kalp  = (kalp_1 + kalp_2 + kalp_3 + kalp_4) / 4            #diger=(diger>thold).astype(np.uint8) #diger orjinal olarak kullanilacak, bu nedenle keep it as is\n            kalp  = (kalp_1 + kalp_2 + kalp_3) / 3            #diger=(diger>thold).astype(np.uint8) #diger orjinal olarak kullanilacak, bu nedenle keep it as is\n\n\n            ##fillpoly before resize\n            conturs,_=cv2.findContours((kalp>thold).astype(np.uint8),cv2.RETR_EXTERNAL,cv2.CHAIN_APPROX_SIMPLE)\n            maskekalp = np.zeros(kalp.shape,dtype=np.uint8)\n            for cc in conturs:\n                maskekalp=cv2.fillPoly(maskekalp, [cc], 1)  #SADECE 1LE DOLACAK HEPSI.....\n            #resize the mask to original lung shape\n            maskekalp=cv2.resize(maskekalp, (w1,h1)) #BURADA IYA DEFAULT YA DA ,interpolation = cv2.INTER_NEAREST\n            #find bboxes\n            conturs,_=cv2.findContours(maskekalp,cv2.RETR_EXTERNAL,cv2.CHAIN_APPROX_SIMPLE)\n            if conturs:\n                for cc in conturs:\n\n                    ### how about adding a snippet to find average probability of the countour from score[2][1]\n                    cimg = np.zeros_like(maskekalp)\n                    cimg=cv2.fillPoly(cimg, [cc], 1)  #simdi 1 ile dolu olan bu yerlerin prob topla ve suma bol..\n                    confidence=np.mean(cv2.resize(kalp, (w1,h1))[cimg==1])\n\n                    x,y,width,height=cv2.boundingRect(cc)\n                    x,y = x + x1, y + y1   #ana resimdeki yerlerini bulmak icin, digerlung x2vey2 degerlerini ekle.\n                    kalpboxes.append((x,y,x + width,y + height,confidence))  #I will not use box confidence,, that can come\n                                                           #from boxclassifier and xray classifier.....!!!\n                                                       #her boxun kendi opacity degeri var zaten... boxclassifer\n                                                       #buna belki xray classifier eklersin....\n                                                       #ortalamasini almak icin....\n            else:                                          #DOLAYISIYLA SEGMENTATOR VEYA DETECTOR BOXCONFIDENCE bosver\n                kalpboxes.append((0,0,1,1,1))\n        else:\n            kalpboxes.append((0,0,1,1,1))\n            \n        #bunlari birbirine ekleyecegim ama nasil....\n        allboxes[(data[1],data[2])]={'kalp':kalpboxes,'diger':digerboxes}\n    \n    \n    return allboxes\n\n","metadata":{"execution":{"iopub.status.busy":"2021-07-21T11:57:42.932046Z","iopub.execute_input":"2021-07-21T11:57:42.937053Z","iopub.status.idle":"2021-07-21T11:57:51.720062Z","shell.execute_reply.started":"2021-07-21T11:57:42.937011Z","shell.execute_reply":"2021-07-21T11:57:51.719216Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import copy\n\ndef move(preds):\n    kalanlar = copy.deepcopy(preds)\n\n    for key in preds:\n        for lung in preds[key]:\n            #print('su an baktigim lung...',preds[key][lung])\n            for box in preds[key][lung]:\n                #print('suanki box.....',box)\n                if box[4]<.55:\n    #                 print(preds[key][lung][i])\n                    #print('cikarmam gereken box...',box)\n                    kalanlar[key][lung].remove(box)\n    #                 kalanlar[key][lung].pop(kalanlar[key][lung].index(box))\n    #                 print(kalanlar[key][lung].pop(kalanlar[key][lung].index(preds[key][lung][i])))\n            if kalanlar[key][lung]==[]: kalanlar[key][lung]=[(0,0,1,1,1)]\n    return kalanlar","metadata":{"execution":{"iopub.status.busy":"2021-07-21T11:57:51.721787Z","iopub.execute_input":"2021-07-21T11:57:51.72214Z","iopub.status.idle":"2021-07-21T11:57:51.730592Z","shell.execute_reply.started":"2021-07-21T11:57:51.722088Z","shell.execute_reply":"2021-07-21T11:57:51.729823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"###17 temmuz main func segmentation version\nxdata_image=pd.DataFrame(columns=['ImageID','width','height',\n                                       'atypical','indeterminate','negative','typical','studyID'])\n\nxdata_box=pd.DataFrame(columns=['ImageID', 'iskalp' ,'boxconf', 'XMin','YMin','XMax','YMax',\n                                     'atypical','indeterminate','negative','typical','studyID'])\n\n\nfor mm, xxx in tqdm(enumerate(chunked(tumliste,64))): ### burada xxx icin analizi yapip asagidaki dataframe'e ekleyecegim...\n    yyy=xxx.copy()\n    print(mm)\n    \n    #allinfo=[(read_xray_x(x),x.parts[1],x.stem) for x in yyy]\n    #KAGGLE VERSION\n    allinfo=[(read_xray_x(x),x.parts[4],x.stem) for x in yyy]\n    \n    threshold=.48 ## threshold for segmentation of boxes not in find_cigers,,,yani ciger bulurken degil, boxbul\n    preds=prediction(allinfo,threshold)\n    preds=move(preds)\n    \n    #break\n    detection_image_resize = image_size #boxun gercek yerlerini create_data icinde bulmak icin...\n    detection_thold=threshold  #bu threshold ile .50/thold normalize yapiyordum.. bu da bir opsiyon\n    adding_image,adding_box=create_data_segment(preds, allinfo,learn_xray,learn_box,detection_image_resize,detection_thold)\n    xdata_image = xdata_image.append(adding_image,ignore_index=True)\n    xdata_box = xdata_box.append(adding_box,ignore_index=True)\n    \ngc.collect()\n#break\n\n#IMAGE LEVEL:\n#OPACITY VE NONE VAR...\n#BOX YOKSA: NONE 1100 VE XRAY CLASSIFIER NEGATIVE DEGERI ORTALAMASINI NONE CONFIDENCE OLARAK YAZ...\n#### BU DURUMDA ZATEN OPACITY YAZMAYAAKSIN...\n#BOX VARSA: OPACITY OLARAK, HER BOXUN OPACITYSI KENDI OPACITY'SI SADECE KENDI....VE NEGATIF HARIC TOPLAM...\n#### NEGATIF OLARAK: TUM BOXLAR VE XRAY CLASSIFIER'IN NEGATIF TOPLAMLARININ ORTALAMASINI AL....\n\n\nimage_dict={}\nimages=np.unique(xdata_image.ImageID.values)  \n\nfor line in xdata_image.itertuples():\n    #image_dict[line.ImageID] = ' '.join(['none','1 0 0 1 1'])  #v23\n\n    #box yoksa bunu bir kac yerden anlarsin, mesela: negative column degeri 1 olur, veya xmax'da 1 olur...\n    if xdata_box.loc[xdata_box.ImageID==line.ImageID].iloc[0].negative == 1:\n        image_dict[line.ImageID] = ' '.join(['none',str((1+line.negative)/2),'0 0 1 1'])\n        #image_dict[line.ImageID] = ' '.join(['none','1 0 0 1 1'])\n\n    else:\n        image_boxes_result=[]\n        nonepred=[line.negative] #burada negatif degerleri toplayacagim... none yazmak icin line.negativedenbasla\n        for row in xdata_box.loc[xdata_box.ImageID==line.ImageID].itertuples():\n            rowresult=f'opacity {(row.atypical + row.indeterminate + row.typical + row.boxconf)/2} {row.XMin} {row.YMin} {row.XMax} {row.YMax}'\n            #option2 boxconf devredisi olursa... asagidaki gibi bir submission opsiyonu olur...\n            #rowresult=f'opacity {(row.atypical + row.indeterminate + row.typical} {row.XMin} {row.YMin} {row.XMax} {row.YMax}'\n            #option3: bu degerlere line degerlerini de ekleme...\n            #xray_boxvar=line.atypical + line.indeterminate + line.typical\n            #rowresult=f'opacity {(row.atypical + row.indeterminate + row.typical + row.boxconf + xray_boxvar)/3} {row.XMin} {row.YMin} {row.XMax} {row.YMax}'\n            #option4: bu degerlere line degerlerini de ekleme...\n            #xray_boxvar=line.atypical + line.indeterminate + line.typical\n            #rowresult=f'opacity {(row.atypical + row.indeterminate + row.typical + xray_boxvar)/2} {row.XMin} {row.YMin} {row.XMax} {row.YMax}'\n            \n            image_boxes_result.append(rowresult)\n            nonepred.append(row.negative)        \n            \n        nonesresult=f'none {sum(nonepred)/len(nonepred)} 0 0 1 1'\n        image_boxes_result.append(nonesresult)\n        image_result= ' '.join(image_boxes_result)\n                #burada line.negative ile row.negatiflerin herbirinin agirliklari ayni\n                #xray.negativi bir birim, kalanlarin da ortalamasini bir birim alarak da bir opsiyon olabilir\n                #tum boxlarin negatif oranlari ile xrayin ayri ayri degerlendirip, normalize ediyorum....\n                #burada bir opsiyon da line.negative hic devreye sokmamak\n        \n        image_dict[line.ImageID]=image_result\n\n        \nstudies=np.unique(xdata_image.studyID.values)\nstudy_dict={}\n\nfor study_id in studies: #guzel olan bu sekilde ayri ayri toplarladigimda... box level negatif bile olsa\n                         #bu boxdan negatif:1 geliyor ve image level ile birlikte yumusatiliyor...\n                         #box yoksa dogrudan yok demiyor.... bunun\n    \n    studyimage=xdata_image.loc[xdata_image.studyID == study_id] #her bir studynin kendi image ve box dataframei\n    studybox= xdata_box.loc[xdata_box.studyID == study_id]\n    study_results=[]\n    for row in studyimage.itertuples():\n        study_results.append([row.atypical,row.indeterminate,row.negative,row.typical])\n    for row in studybox.itertuples():\n        study_results.append([row.atypical,row.indeterminate,row.negative,row.typical])\n\n    study_results=np.array(study_results).T #her bir sonuc bir row ken her rowda annotation bazli sonuclar..\n                                            #ki bunlarin ortalamasi dogrudan study id icin girecegin deger..\n    results = [sum(x)/len(x) for x in study_results]\n#     s = sum(results) #aslinda normalize gerek yok.. cunku her satir already probabilty ve tamamini aliyorsun..\n#     results = [x/s for x in results]\n    study_dict[study_id]=f'atypical {results[0]} 0 0 1 1 indeterminate {results[1]} 0 0 1 1 negative {results[2]} 0 0 1 1 typical {results[3]} 0 0 1 1'","metadata":{"execution":{"iopub.status.busy":"2021-07-21T11:57:51.731865Z","iopub.execute_input":"2021-07-21T11:57:51.732236Z","iopub.status.idle":"2021-07-21T12:01:24.104208Z","shell.execute_reply.started":"2021-07-21T11:57:51.732199Z","shell.execute_reply":"2021-07-21T12:01:24.101547Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds","metadata":{"execution":{"iopub.status.busy":"2021-07-21T12:01:24.105413Z","iopub.status.idle":"2021-07-21T12:01:24.105976Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_data = pd.DataFrame(image_dict.items(), columns=['Id', 'PredictionString'])","metadata":{"execution":{"iopub.status.busy":"2021-07-21T12:01:24.107302Z","iopub.status.idle":"2021-07-21T12:01:24.108034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"study_data = pd.DataFrame(study_dict.items(),columns=['Id','PredictionString'])","metadata":{"execution":{"iopub.status.busy":"2021-07-21T12:01:24.109278Z","iopub.status.idle":"2021-07-21T12:01:24.109894Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"full_data=image_data.append(study_data,ignore_index=True)","metadata":{"execution":{"iopub.status.busy":"2021-07-21T12:01:24.111074Z","iopub.status.idle":"2021-07-21T12:01:24.111718Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"full_data","metadata":{"execution":{"iopub.status.busy":"2021-07-21T12:01:24.112869Z","iopub.status.idle":"2021-07-21T12:01:24.113504Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"full_data.to_csv('/kaggle/working/submission.csv',index = False)  \n","metadata":{"execution":{"iopub.status.busy":"2021-07-21T12:01:24.114532Z","iopub.status.idle":"2021-07-21T12:01:24.115272Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### asagida ram check konusu:","metadata":{}},{"cell_type":"code","source":"# import sys\n\n# # These are the usual ipython objects, including this one you are creating\n# ipython_vars = ['In', 'Out', 'exit', 'quit', 'get_ipython', 'ipython_vars']\n\n# # Get a sorted list of the objects and their sizes\n# sorted([(x, sys.getsizeof(globals().get(x))) for x in dir() if not x.startswith('_') and x not in sys.modules and x not in ipython_vars], key=lambda x: x[1], reverse=True)","metadata":{"execution":{"iopub.status.busy":"2021-07-21T12:01:24.116408Z","iopub.status.idle":"2021-07-21T12:01:24.117027Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import sys\n# def sizeof_fmt(num, suffix='B'):\n#     ''' by Fred Cirera,  https://stackoverflow.com/a/1094933/1870254, modified'''\n#     for unit in ['','Ki','Mi','Gi','Ti','Pi','Ei','Zi']:\n#         if abs(num) < 1024.0:\n#             return \"%3.1f %s%s\" % (num, unit, suffix)\n#         num /= 1024.0\n#     return \"%.1f %s%s\" % (num, 'Yi', suffix)\n\n# for name, size in sorted(((name, sys.getsizeof(value)) for name, value in locals().items()),\n#                          key= lambda x: -x[1])[:10]:\n#     print(\"{:>30}: {:>8}\".format(name, sizeof_fmt(size)))","metadata":{"execution":{"iopub.status.busy":"2021-07-21T12:01:24.118256Z","iopub.status.idle":"2021-07-21T12:01:24.118865Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import os\n# import psutil\n# pid = os.getpid()\n# python_process = psutil.Process(pid)\n# memoryUse = python_process.memory_info()[0]/2.**30  # memory use in GB...I think\n# print('memory use:', memoryUse)","metadata":{"execution":{"iopub.status.busy":"2021-07-21T12:01:24.119942Z","iopub.status.idle":"2021-07-21T12:01:24.120562Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}