{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os #ceate folders\nfrom glob import glob # get paths and use the to have floders name\nfrom tqdm.notebook import tqdm # get nice bar\nimport sys\nimport pydicom as pdc  # read dicom images\nimport numpy as np\nimport imageio    # save to PNG images\nimport matplotlib.pyplot as plt  # plot some PNG images\nimport cv2 as cv  # read PNG images\nfrom random import sample \nfrom joblib import Parallel,delayed\nimport subprocess\nfrom ast import literal_eval\n\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-08-02T10:08:46.801934Z","iopub.execute_input":"2021-08-02T10:08:46.802665Z","iopub.status.idle":"2021-08-02T10:08:47.589092Z","shell.execute_reply.started":"2021-08-02T10:08:46.802525Z","shell.execute_reply":"2021-08-02T10:08:47.587932Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# get CPU informations \n\ndef run(command):\n    process = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE)\n    out, err = process.communicate()\n    print(out.decode('utf-8').strip())\n    \nprint('# CPU')\nrun('cat /proc/cpuinfo | egrep -m 1 \"^model name\"')\nrun('cat /proc/cpuinfo | egrep -m 1 \"^cpu MHz\"')\nrun('cat /proc/cpuinfo | egrep -m 1 \"^cpu cores\"')","metadata":{"execution":{"iopub.status.busy":"2021-08-02T10:08:47.593083Z","iopub.execute_input":"2021-08-02T10:08:47.593392Z","iopub.status.idle":"2021-08-02T10:08:47.640713Z","shell.execute_reply.started":"2021-08-02T10:08:47.593361Z","shell.execute_reply":"2021-08-02T10:08:47.639456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train and test folder names \ntrain_path_list=glob('../input/rsna-miccai-brain-tumor-radiogenomic-classification/train/*')\ntest_path_list= glob('../input/rsna-miccai-brain-tumor-radiogenomic-classification/test/*')\nkaggle_input_path= '../input/'\n\n#list of name of subdirectorys\ntrain_d=list(map(lambda path:path.split('/')[-1],train_path_list))\ntest_d=list(map(lambda path:path.split('/')[-1],test_path_list))\nmpMRI_scans=[\"FLAIR\",\"T1w\",\"T1wCE\",\"T2w\"] \n\n# sample of names\nprint(train_d[:4])","metadata":{"execution":{"iopub.status.busy":"2021-08-02T10:08:47.642945Z","iopub.execute_input":"2021-08-02T10:08:47.643272Z","iopub.status.idle":"2021-08-02T10:08:47.703987Z","shell.execute_reply.started":"2021-08-02T10:08:47.643236Z","shell.execute_reply":"2021-08-02T10:08:47.702619Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# read dico image and return the image array\n\ndef read_dcm(img):\n    \"\"\"reading a dicom image with preprocessing\"\"\"\n    dcm_img=pdc.dcmread(img)\n    img_array=dcm_img.pixel_array\n    img_array = img_array - np.min(img_array)\n    if np.max(img_array) != 0:\n        img_array = img_array / np.max(img_array)\n    img_array = (img_array * 255).astype(np.uint8)\n    return img_array","metadata":{"execution":{"iopub.status.busy":"2021-08-02T10:08:47.705661Z","iopub.execute_input":"2021-08-02T10:08:47.706021Z","iopub.status.idle":"2021-08-02T10:08:47.713942Z","shell.execute_reply.started":"2021-08-02T10:08:47.705986Z","shell.execute_reply":"2021-08-02T10:08:47.712358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#delete old folder (if you run the code twice)\n\n!rm -rf  Png-rsna-miccai-brain-tumor-radiogenomic-classification/\n\n# create the main PNG folder with the test and train \n\npng_test_path='Png-rsna-miccai-brain-tumor-radiogenomic-classification/test'\npng_train_path='Png-rsna-miccai-brain-tumor-radiogenomic-classification/train'\n\nos.mkdir('Png-rsna-miccai-brain-tumor-radiogenomic-classification')\nos.mkdir(png_train_path)\nos.mkdir(png_test_path)\n\n\n# floders creation \n\nfor trfold in train_d:\n    os.mkdir(png_train_path+'/'+trfold)\n    for mp in mpMRI_scans:\n        os.mkdir(png_train_path+'/'+trfold+'/'+mp)\n        \nfor tsfold in test_d: \n    os.mkdir(png_test_path+'/'+tsfold)\n    for mp in mpMRI_scans:\n        os.mkdir(png_test_path+'/'+tsfold+'/'+mp)","metadata":{"execution":{"iopub.status.busy":"2021-08-02T10:08:47.715497Z","iopub.execute_input":"2021-08-02T10:08:47.715821Z","iopub.status.idle":"2021-08-02T10:08:48.585214Z","shell.execute_reply.started":"2021-08-02T10:08:47.715788Z","shell.execute_reply":"2021-08-02T10:08:48.583902Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# list the paths and name directory \n\npng_train_path_list=glob('Png-rsna-miccai-brain-tumor-radiogenomic-classification/test/*')\npng_test_path_list= glob('Png-rsna-miccai-brain-tumor-radiogenomic-classification/train/*')\n\npng_train_d=list(map(lambda path:path.split('/')[-1],png_train_path_list))\npng_test_d=list(map(lambda path:path.split('/')[-1],png_test_path_list))\n\n#print sample of Png path folders and there names\nprint(png_train_path_list[:3])\nprint(png_train_d[:3])\n\n\n# compare the folder result \n\nprint(train_d.sort()==png_train_d.sort())\nprint(test_d.sort()==png_test_d.sort())\n\n# list of all dicom iamges paths\ntrain_images_path=glob('../input/rsna-miccai-brain-tumor-radiogenomic-classification/train/*/*/*')\ntest_images_path=glob('../input/rsna-miccai-brain-tumor-radiogenomic-classification/test/*/*/*')\n","metadata":{"execution":{"iopub.status.busy":"2021-08-02T10:08:48.587026Z","iopub.execute_input":"2021-08-02T10:08:48.587358Z","iopub.status.idle":"2021-08-02T10:09:49.263606Z","shell.execute_reply.started":"2021-08-02T10:08:48.587322Z","shell.execute_reply":"2021-08-02T10:09:49.262408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# the logic to create the new PNG path \n\nimage_name=test_images_path[0].split('/')[4:][-1].split('.')[0]\nnew_png_path=png_train_path+'/'+'/'.join(train_images_path[0].split('/')[4:-1])+'/'+image_name+'.PNG'\nprint(new_png_path)\n","metadata":{"execution":{"iopub.status.busy":"2021-08-02T10:09:49.265382Z","iopub.execute_input":"2021-08-02T10:09:49.265834Z","iopub.status.idle":"2021-08-02T10:09:49.273954Z","shell.execute_reply.started":"2021-08-02T10:09:49.265756Z","shell.execute_reply":"2021-08-02T10:09:49.272501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(train_images_path),len(test_images_path))\n","metadata":{"execution":{"iopub.status.busy":"2021-08-02T10:09:49.277150Z","iopub.execute_input":"2021-08-02T10:09:49.277622Z","iopub.status.idle":"2021-08-02T10:09:49.288422Z","shell.execute_reply.started":"2021-08-02T10:09:49.277567Z","shell.execute_reply":"2021-08-02T10:09:49.287586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# PNG save function \n\ndef train_png(train_path):\n    train_array_img=read_dcm(train_path)                                             \n    train_image_name=train_path.split('/')[4:][-1].split('.')[0]                               \n    new_train_png_path=png_train_path+'/'+'/'.join(train_path.split('/')[4:-1])+'/'+train_image_name+'.PNG'    \n    imageio.imsave(new_train_png_path,train_array_img)\n    \ndef test_png(test_path):\n    test_array_path=read_dcm(test_path) \n    test_image_name=test_path.split('/')[4:][-1].split('.')[0]\n    new_test_png_path=png_test_path+'/'+'/'.join(test_path.split('/')[4:-1])+'/'+test_image_name+'.PNG'\n    imageio.imsave(new_test_png_path,test_array_path)","metadata":{"execution":{"iopub.status.busy":"2021-08-02T10:09:49.289795Z","iopub.execute_input":"2021-08-02T10:09:49.290281Z","iopub.status.idle":"2021-08-02T10:09:49.300309Z","shell.execute_reply.started":"2021-08-02T10:09:49.290242Z","shell.execute_reply":"2021-08-02T10:09:49.299106Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#FIXED : not all the training data was converted and the slow conversion \n# test for 20000 for tarain and 10000 for test \nfb_train=Parallel(n_jobs=4,verbose=1,prefer='threads')(delayed(train_png)(train_path) for train_path in tqdm(train_images_path[:20000],total=len(train_images_path[:20000])))\nfb_test=Parallel(n_jobs=4,verbose=1,prefer='threads') (delayed(test_png) (test_path) for test_path in tqdm(test_images_path[:20000],total=len(test_images_path[:20000])))\n","metadata":{"execution":{"iopub.status.busy":"2021-08-02T10:09:49.302016Z","iopub.execute_input":"2021-08-02T10:09:49.302352Z","iopub.status.idle":"2021-08-02T10:15:07.888480Z","shell.execute_reply.started":"2021-08-02T10:09:49.302320Z","shell.execute_reply":"2021-08-02T10:15:07.886954Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Image creation and saving part","metadata":{}},{"cell_type":"code","source":"# FIXED : printing some black images \n\n#print(PNG random PNG images)\npng_images=glob('Png-rsna-miccai-brain-tumor-radiogenomic-classification/test/*/*/*')\nprint(png_images[:10])\n\nplt.figure(figsize=(15,15))\nplot_indicator=0\nsub_in=0\nfor index,image in tqdm(enumerate(sample(png_images,5000))) :\n    img = cv.imread(image,cv.IMREAD_GRAYSCALE)\n    img=cv.resize(img,(200,200))\n    if np.max(img)!= 0 and np.mean(img)>=30:  \n        plot_indicator+=1\n        if plot_indicator==5:\n            break\n        else :    \n            sub_in+=1\n            plt.subplot(2,2,sub_in)  \n            plt.imshow(img)\n            \n    else:\n        continue ","metadata":{"execution":{"iopub.status.busy":"2021-08-02T10:15:07.891211Z","iopub.execute_input":"2021-08-02T10:15:07.891723Z","iopub.status.idle":"2021-08-02T10:15:09.151225Z","shell.execute_reply.started":"2021-08-02T10:15:07.891669Z","shell.execute_reply":"2021-08-02T10:15:09.150119Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!tar -zcf dcm_to_png.tar.gz -C \"/kaggle/working/Png-rsna-miccai-brain-tumor-radiogenomic-classification/\" .","metadata":{"execution":{"iopub.status.busy":"2021-08-02T10:15:09.152572Z","iopub.execute_input":"2021-08-02T10:15:09.152972Z","iopub.status.idle":"2021-08-02T10:15:31.429951Z","shell.execute_reply.started":"2021-08-02T10:15:09.152928Z","shell.execute_reply":"2021-08-02T10:15:31.428444Z"},"trusted":true},"execution_count":null,"outputs":[]}]}