{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Images for training baseline CNNs\n- ## Training data for both Typical (Covid) and Negative classes after dropping other classes\n- ## Testing data for covid & negative (80:20)\n- ## label encoding classes in dfs\n\n- v1 downsampled covid to match negative each (Negative , Covid =  1709, 1709)\n- v2 kept both classes intact and slight imbalance to be handled using k-fold training and precion-recall metrics (Negative , Covid =  1709, 2957)","metadata":{}},{"cell_type":"code","source":"#!pip install -q efficientnet >> /dev/null","metadata":{"execution":{"iopub.status.busy":"2021-08-17T13:06:31.294251Z","iopub.execute_input":"2021-08-17T13:06:31.294643Z","iopub.status.idle":"2021-08-17T13:06:31.300106Z","shell.execute_reply.started":"2021-08-17T13:06:31.294610Z","shell.execute_reply":"2021-08-17T13:06:31.298367Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd, numpy as np, random,os, shutil\nfrom glob import glob\nfrom kaggle_datasets import KaggleDatasets\nimport tensorflow as tf, re, math\nimport tensorflow.keras.backend as K\n#import efficientnet.tfkeras as efn\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import roc_auc_score\nimport matplotlib.pyplot as plt\nimport tensorflow_addons as tfa\nimport cv2\n\n\nprint('tf:',tf.__version__)","metadata":{"execution":{"iopub.status.busy":"2021-08-17T13:06:33.367819Z","iopub.execute_input":"2021-08-17T13:06:33.368237Z","iopub.status.idle":"2021-08-17T13:06:33.376345Z","shell.execute_reply.started":"2021-08-17T13:06:33.368203Z","shell.execute_reply":"2021-08-17T13:06:33.374799Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"NP_RANDOM = 2021\nSAMPLE_VERSION = 'V2' #change to V1 for downsampling\n\n#clear and make output dir\nfrom shutil import copyfile\nimport shutil\nimport os\n\nvfolder = '/kaggle/working/' + SAMPLE_VERSION + '/'\n\ntry:\n    shutil.rmtree(vfolder)\nexcept Exception as e:\n    print(str(e))\nfinally:\n    if not os.path.exists(vfolder):\n        os.mkdir(vfolder)\n        print(\"Created \" + vfolder)\n        ","metadata":{"execution":{"iopub.status.busy":"2021-08-17T13:06:36.079619Z","iopub.execute_input":"2021-08-17T13:06:36.080569Z","iopub.status.idle":"2021-08-17T13:06:36.090009Z","shell.execute_reply.started":"2021-08-17T13:06:36.080517Z","shell.execute_reply":"2021-08-17T13:06:36.088362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# for competition test folder we don't have labels so can't use them\n# from train folder, we will split train-val-test\n# remove the duplicate image-ids we found from ljmu-2-preprocessing notebook\nimg_df = pd.read_csv('../input/siim-covid19-detection/train_image_level.csv')\nimg_df.head(1)","metadata":{"execution":{"iopub.status.busy":"2021-08-17T13:06:39.770398Z","iopub.execute_input":"2021-08-17T13:06:39.770761Z","iopub.status.idle":"2021-08-17T13:06:39.817695Z","shell.execute_reply.started":"2021-08-17T13:06:39.770731Z","shell.execute_reply":"2021-08-17T13:06:39.816364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"stu_df = pd.read_csv('../input/siim-covid19-detection/train_study_level.csv')\nstu_df.head(1)","metadata":{"execution":{"iopub.status.busy":"2021-08-17T13:06:42.126721Z","iopub.execute_input":"2021-08-17T13:06:42.127067Z","iopub.status.idle":"2021-08-17T13:06:42.148007Z","shell.execute_reply.started":"2021-08-17T13:06:42.127039Z","shell.execute_reply":"2021-08-17T13:06:42.147257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#merge dfs\nstu_df['StudyInstanceUID'] = stu_df['id'].apply(lambda x: x.replace('_study', ''))\ndel stu_df['id']\nimg_df = img_df.merge(stu_df, on='StudyInstanceUID')\nimg_df.sample(3)","metadata":{"execution":{"iopub.status.busy":"2021-08-17T13:06:44.302301Z","iopub.execute_input":"2021-08-17T13:06:44.302812Z","iopub.status.idle":"2021-08-17T13:06:44.331765Z","shell.execute_reply.started":"2021-08-17T13:06:44.302781Z","shell.execute_reply":"2021-08-17T13:06:44.331083Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#label encoding 0,1,2,3\nimg_df.loc[img_df['Negative for Pneumonia']==1, 'label'] = 0\nimg_df.loc[img_df['Typical Appearance']==1, 'label'] = 1\nimg_df.loc[img_df['Indeterminate Appearance']==1, 'label'] = 2\nimg_df.loc[img_df['Atypical Appearance']==1, 'label'] = 3\nimg_df.head(3)","metadata":{"execution":{"iopub.status.busy":"2021-08-17T13:06:47.043149Z","iopub.execute_input":"2021-08-17T13:06:47.043843Z","iopub.status.idle":"2021-08-17T13:06:47.069662Z","shell.execute_reply.started":"2021-08-17T13:06:47.043800Z","shell.execute_reply":"2021-08-17T13:06:47.068554Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cols_to_drop = ['Negative for Pneumonia','Typical Appearance','Indeterminate Appearance','Atypical Appearance','StudyInstanceUID', 'boxes']\nimg_df.drop(cols_to_drop, axis=1, inplace=True)\nimg_df.head(3)","metadata":{"execution":{"iopub.status.busy":"2021-08-17T13:06:49.936205Z","iopub.execute_input":"2021-08-17T13:06:49.936840Z","iopub.status.idle":"2021-08-17T13:06:49.951752Z","shell.execute_reply.started":"2021-08-17T13:06:49.936789Z","shell.execute_reply":"2021-08-17T13:06:49.950602Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img_df[\"id\"].replace(\"_image\", \".jpg\",regex=True, inplace=True)\nimg_df.head(3)","metadata":{"execution":{"iopub.status.busy":"2021-08-17T13:06:52.352030Z","iopub.execute_input":"2021-08-17T13:06:52.352480Z","iopub.status.idle":"2021-08-17T13:06:52.382029Z","shell.execute_reply.started":"2021-08-17T13:06:52.352436Z","shell.execute_reply":"2021-08-17T13:06:52.380323Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"drop_df = pd.read_csv('../input/images-to-drop/images_to_drop.csv')\ndrop_df[\"p1\"].replace(\"_image\", \".jpg\",regex=True, inplace=True)\ndrop_df.head(3)\n","metadata":{"execution":{"iopub.status.busy":"2021-08-17T13:06:55.630794Z","iopub.execute_input":"2021-08-17T13:06:55.631178Z","iopub.status.idle":"2021-08-17T13:06:55.653723Z","shell.execute_reply.started":"2021-08-17T13:06:55.631123Z","shell.execute_reply":"2021-08-17T13:06:55.652557Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#drop images marked as duplicates\nbefore = len(img_df)\nimg_df = img_df.loc[~img_df['id'].isin(drop_df['p1'])]\nafter = len(img_df)\nprint(\"before , after = \", before, \",\", after)","metadata":{"execution":{"iopub.status.busy":"2021-08-17T13:06:58.154960Z","iopub.execute_input":"2021-08-17T13:06:58.155344Z","iopub.status.idle":"2021-08-17T13:06:58.164825Z","shell.execute_reply.started":"2021-08-17T13:06:58.155313Z","shell.execute_reply":"2021-08-17T13:06:58.163992Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#filter 0,1 only for further experiments (binary classification)\n#../input/siimcovid19256jpg/256-jpg\nbefore = len(img_df)\nimg_df = img_df.loc[img_df['label'].isin([0,1])]\nafter = len(img_df)\nprint(\"before , after = \", before, \",\", after)","metadata":{"execution":{"iopub.status.busy":"2021-08-17T13:07:00.778508Z","iopub.execute_input":"2021-08-17T13:07:00.778892Z","iopub.status.idle":"2021-08-17T13:07:00.788663Z","shell.execute_reply.started":"2021-08-17T13:07:00.778860Z","shell.execute_reply":"2021-08-17T13:07:00.786910Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Negative , Covid = \", len( img_df[(img_df['label']==0)]), len( img_df[(img_df['label']==1)]))","metadata":{"execution":{"iopub.status.busy":"2021-08-17T13:08:59.989584Z","iopub.execute_input":"2021-08-17T13:08:59.989920Z","iopub.status.idle":"2021-08-17T13:09:00.005031Z","shell.execute_reply.started":"2021-08-17T13:08:59.989891Z","shell.execute_reply":"2021-08-17T13:09:00.002969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#keep a copy of all rows before split\ncovid_all = img_df[(img_df['label']==1)]\nnegative_all = img_df[(img_df['label']==0)]\n#shuffle before save\ncovid_all = covid_all.sample(frac = 1).reset_index(drop=True)\nnegative_all = negative_all.sample(frac = 1).reset_index(drop=True)\n#save\ncovid_all.to_csv(vfolder + 'covid_all.csv', index=False)\nnegative_all.to_csv(vfolder + 'negative_all.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2021-08-17T13:07:09.196388Z","iopub.execute_input":"2021-08-17T13:07:09.196713Z","iopub.status.idle":"2021-08-17T13:07:09.228645Z","shell.execute_reply.started":"2021-08-17T13:07:09.196685Z","shell.execute_reply":"2021-08-17T13:07:09.227604Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# for v1, use downsampling to address slight imbalanced classes based on RARE class count (negative:1709)\n# for v2, don't downsample.. use all available data\nRARE_CLASS_COUNT = ( min( len( img_df[(img_df['label']==0)]), len( img_df[(img_df['label']==1)])) )\nif SAMPLE_VERSION=='V1':\n    np.random.seed(NP_RANDOM)\n    img_df = img_df.groupby('label').apply(lambda x : x.sample(RARE_CLASS_COUNT)).reset_index(drop=True)\n\nprint (len(img_df))\nimg_df.head(5)\n","metadata":{"execution":{"iopub.status.busy":"2021-08-17T13:10:50.405628Z","iopub.execute_input":"2021-08-17T13:10:50.405971Z","iopub.status.idle":"2021-08-17T13:10:50.425521Z","shell.execute_reply.started":"2021-08-17T13:10:50.405942Z","shell.execute_reply":"2021-08-17T13:10:50.424518Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train:80% (covid*0.8 + negative*0.8) test:20% (covid*0.2 + negative*0.2)\n#from sklearn. model_selection import train_test_split\nimg_df_0 = img_df[(img_df['label']==0)]\nimg_df_1 = img_df[(img_df['label']==1)]\n\ntrain_0 = img_df_0.sample(frac=0.8,random_state=NP_RANDOM) \ntrain_1 = img_df_1.sample(frac=0.8,random_state=NP_RANDOM) \ntest_0 = img_df_0.loc[~img_df_0['id'].isin(train_0['id'])]\ntest_1 = img_df_1.loc[~img_df_1['id'].isin(train_1['id'])]\n\ntrain_df = train_0.append(train_1)\ntest_df = test_0.append(test_1)\n\ntrain_df['label'] = train_df.label.astype('int')\ntest_df['label'] = test_df.label.astype('int')\n\n# shuffle the rows\ntrain_df = train_df.sample(frac = 1).reset_index(drop=True)\ntest_df = test_df.sample(frac = 1).reset_index(drop=True)\n\n#save to file for later loading\ntrain_df.to_csv(vfolder + 'train_df.csv', index=False)\ntest_df.to_csv(vfolder + 'test_df.csv', index=False)\n\nprint(\"Train has Negative , Covid = \", len( train_df[(train_df['label']==0)]), len( train_df[(train_df['label']==1)]))\nprint(\"Test has Negative , Covid = \", len( test_df[(test_df['label']==0)]), len( test_df[(test_df['label']==1)]))","metadata":{"execution":{"iopub.status.busy":"2021-08-17T13:11:02.261427Z","iopub.execute_input":"2021-08-17T13:11:02.261838Z","iopub.status.idle":"2021-08-17T13:11:02.322097Z","shell.execute_reply.started":"2021-08-17T13:11:02.261804Z","shell.execute_reply":"2021-08-17T13:11:02.320328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#copy train & train images to folders\n\ndef copyImgsToFolder(folder, tlist):\n    src = '../input/siimcovid19256jpg/256-jpg/'\n    dest = vfolder + folder + '/'\n    #print(os. getcwd()) \n    #/kaggle/working\n    \n    try:\n        shutil.rmtree(dest)\n    except Exception as e:\n        print(str(e))\n    finally:\n        if not os.path.exists(dest):\n            os.mkdir(dest)\n            print(\"Created \"+ dest)\n        \n    for img in tlist:\n        copyfile(src+img, dest+img)\n        \n    print(\"Copied to \" + dest)","metadata":{"execution":{"iopub.status.busy":"2021-08-17T13:14:11.938110Z","iopub.execute_input":"2021-08-17T13:14:11.938679Z","iopub.status.idle":"2021-08-17T13:14:11.945040Z","shell.execute_reply.started":"2021-08-17T13:14:11.938627Z","shell.execute_reply":"2021-08-17T13:14:11.944112Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_covid_df = train_df[(train_df['label'] == 1)]\ntest_covid_df = test_df[(test_df['label'] == 1)]\n\ntrain_negative_df = train_df[(train_df['label'] == 0)]\ntest_negative_df = test_df[(test_df['label'] == 0)]\n\n#includes both classes (for CNN)\ncopyImgsToFolder('train', train_df['id'].tolist())\ncopyImgsToFolder('test', test_df['id'].tolist())\n\n#includes covid class only (for GAN)\ncopyImgsToFolder('train-covid', train_covid_df['id'].tolist())\ncopyImgsToFolder('test-covid', test_covid_df['id'].tolist())\n\n#includes negative class only (for GAN)\ncopyImgsToFolder('train-negative', train_negative_df['id'].tolist())\ncopyImgsToFolder('test-negative', test_negative_df['id'].tolist())\n\n#for references only\ncopyImgsToFolder('covid-all', covid_all['id'].tolist())\ncopyImgsToFolder('negative-all', negative_all['id'].tolist())\n\nprint(\"Images copied ...\")","metadata":{"execution":{"iopub.status.busy":"2021-08-17T13:14:17.092347Z","iopub.execute_input":"2021-08-17T13:14:17.092985Z","iopub.status.idle":"2021-08-17T13:15:17.600531Z","shell.execute_reply.started":"2021-08-17T13:14:17.092928Z","shell.execute_reply":"2021-08-17T13:15:17.598284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"# TFRecords creation for GAN network training ...\n- ## TFRecord for training data (covid, negative)\n- ## TFRecord for testing data (covid, negative)","metadata":{}},{"cell_type":"code","source":"# PATHS TO IMAGES\nTRPATH = vfolder+ 'train/'\nTEPATH = vfolder+ 'test/'\n\nTRPATH_C = vfolder+ 'train-covid/'\nTEPATH_C = vfolder+ 'test-covid/'\n\nTRPATH_N = vfolder+ 'train-negative/'\nTEPATH_N = vfolder+ 'test-negative/'\n\nC_ALL = vfolder+ 'covid-all/'\nN_ALL = vfolder+ 'negative-all/'\n\nTRIMGS = os.listdir(TRPATH)\nTEIMGS = os.listdir(TEPATH)\n\nTRIMGS_C = os.listdir(TRPATH_C)\nTEIMGS_C = os.listdir(TEPATH_C)\n\nTRIMGS_N = os.listdir(TRPATH_N)\nTEIMGS_N = os.listdir(TEPATH_N)\n\nC_IMGS_ALL = os.listdir(C_ALL)\nN_IMGS_ALL = os.listdir(N_ALL)\n\nprint (str(len(TRIMGS)) + \" \" + str(len(TEIMGS)))\nTRIMGS[0]","metadata":{"execution":{"iopub.status.busy":"2021-08-17T13:21:11.170302Z","iopub.execute_input":"2021-08-17T13:21:11.170813Z","iopub.status.idle":"2021-08-17T13:21:11.204303Z","shell.execute_reply.started":"2021-08-17T13:21:11.170773Z","shell.execute_reply":"2021-08-17T13:21:11.202905Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SIZE = 200\n#CT = len(TRIMGS)//SIZE + int(len(TRIMGS)%SIZE!=0)\nCT = len(TEIMGS)//SIZE + int(len(TEIMGS)%SIZE!=0)\nprint(len(TEIMGS) // CT)","metadata":{"execution":{"iopub.status.busy":"2021-08-17T13:21:18.247843Z","iopub.execute_input":"2021-08-17T13:21:18.248282Z","iopub.status.idle":"2021-08-17T13:21:18.256289Z","shell.execute_reply.started":"2021-08-17T13:21:18.248249Z","shell.execute_reply":"2021-08-17T13:21:18.254355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#TF website\ndef _bytes_feature(value):\n  \"\"\"Returns a bytes_list from a string / byte.\"\"\"\n  if isinstance(value, type(tf.constant(0))):\n    value = value.numpy() # BytesList won't unpack a string from an EagerTensor.\n  return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))\n\ndef _float_feature(value):\n  \"\"\"Returns a float_list from a float / double.\"\"\"\n  return tf.train.Feature(float_list=tf.train.FloatList(value=[value]))\n\ndef _int64_feature(value):\n  \"\"\"Returns an int64_list from a bool / enum / int / uint.\"\"\"\n  return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))\n\n\n","metadata":{"execution":{"iopub.status.busy":"2021-08-17T13:21:21.792091Z","iopub.execute_input":"2021-08-17T13:21:21.792607Z","iopub.status.idle":"2021-08-17T13:21:21.801514Z","shell.execute_reply.started":"2021-08-17T13:21:21.792567Z","shell.execute_reply":"2021-08-17T13:21:21.800080Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def serialize_example(feature0, feature1, feature2):\n  feature = {\n      'image': _bytes_feature(feature0), #image data sent in as byte\n      'id': _bytes_feature(feature1),    #string data sent in as byte\n      'label': _int64_feature(feature2)  #int data sent in as int\n  }\n  example_proto = tf.train.Example(features=tf.train.Features(feature=feature))\n  return example_proto.SerializeToString()\n\n","metadata":{"execution":{"iopub.status.busy":"2021-08-17T13:21:24.437820Z","iopub.execute_input":"2021-08-17T13:21:24.438226Z","iopub.status.idle":"2021-08-17T13:21:24.444818Z","shell.execute_reply.started":"2021-08-17T13:21:24.438189Z","shell.execute_reply":"2021-08-17T13:21:24.443537Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_tfrecord(df, IMGS, SIZE, PATH, folder):\n    \n    dest = vfolder + folder + '/'\n    try:\n        shutil.rmtree(dest)\n    except Exception as e:\n        print(str(e))\n    finally:\n        if not os.path.exists(dest):\n            os.mkdir(dest)\n            print(\"Created \"+dest)\n    \n    CT = len(IMGS)//SIZE + int(len(IMGS)%SIZE!=0)\n    for j in range(CT):\n        print(); print('Writing TFRecord %i of %i...'%(j,CT))\n        CT2 = min(SIZE,len(IMGS)-j*SIZE)\n        with tf.io.TFRecordWriter(dest + 'img%.2i-%i.tfrec'%(j,CT2)) as writer:\n            for k in range(CT2):\n                img = cv2.imread(PATH+IMGS[SIZE*j+k])\n                img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR) # Fix incorrect colors\n                img = cv2.imencode('.jpg', img, (cv2.IMWRITE_JPEG_QUALITY, 94))[1].tostring()\n                name = IMGS[SIZE*j+k]\n                row = df.loc[df.id==name]\n                example = serialize_example(\n                    img, str.encode(row.id.values[0]),\n                    row.label.values[0])\n                writer.write(example)\n                if k%100==0: print(k,', ',end='')\n    \n    print(\"TFRecords created for \"+ folder)","metadata":{"execution":{"iopub.status.busy":"2021-08-17T13:25:51.846524Z","iopub.execute_input":"2021-08-17T13:25:51.846933Z","iopub.status.idle":"2021-08-17T13:25:51.861325Z","shell.execute_reply.started":"2021-08-17T13:25:51.846899Z","shell.execute_reply":"2021-08-17T13:25:51.859544Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"create_tfrecord(train_df, TRIMGS, 200, TRPATH, 'train-tfrec')\n\nprint()\ncreate_tfrecord(test_df, TEIMGS, 200, TEPATH, 'test-tfrec')\n\ncreate_tfrecord(train_covid_df, TRIMGS_C, 200, TRPATH_C, 'train-covid-tfrec')\nprint()\ncreate_tfrecord(test_covid_df, TEIMGS_C, 200, TEPATH_C, 'test-covid-tfrec')\n\ncreate_tfrecord(train_negative_df, TRIMGS_N, 200, TRPATH_N, 'train-negative-tfrec')\nprint()\ncreate_tfrecord(test_negative_df, TEIMGS_N, 200, TEPATH_N, 'test-negative-tfrec')\n\ncreate_tfrecord(covid_all, C_IMGS_ALL, 200, C_ALL, 'covid-all-tfrec')\nprint()\ncreate_tfrecord(negative_all, N_IMGS_ALL, 200, N_ALL, 'negative-all-tfrec')","metadata":{"execution":{"iopub.status.busy":"2021-08-17T13:24:46.825002Z","iopub.execute_input":"2021-08-17T13:24:46.825427Z","iopub.status.idle":"2021-08-17T13:25:39.308349Z","shell.execute_reply.started":"2021-08-17T13:24:46.825388Z","shell.execute_reply":"2021-08-17T13:25:39.306272Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! ls -l","metadata":{"execution":{"iopub.status.busy":"2021-08-17T13:26:30.760368Z","iopub.execute_input":"2021-08-17T13:26:30.760744Z","iopub.status.idle":"2021-08-17T13:26:31.546791Z","shell.execute_reply.started":"2021-08-17T13:26:30.760711Z","shell.execute_reply":"2021-08-17T13:26:31.545906Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Verify TFRecords","metadata":{}},{"cell_type":"code","source":"# numpy and matplotlib defaults\nnp.set_printoptions(threshold=15, linewidth=80)\nCLASSES = [0,1]\n\ndef batch_to_numpy_images_and_labels(data):\n    images, labels = data\n    numpy_images = images.numpy()\n    numpy_labels = labels.numpy()\n    #if numpy_labels.dtype == object: # binary string in this case, these are image ID strings\n    #    numpy_labels = [None for _ in enumerate(numpy_images)]\n    # If no labels, only image IDs, return None for labels (this is the case for test data)\n    return numpy_images, numpy_labels\n\ndef title_from_label_and_target(label, correct_label):\n    if correct_label is None:\n        return CLASSES[label], True\n    correct = (label == correct_label)\n    return \"{} [{}{}{}]\".format(CLASSES[label], 'OK' if correct else 'NO', u\"\\u2192\" if not correct else '',\n                                CLASSES[correct_label] if not correct else ''), correct\n\ndef display_one_xray(image, title, subplot, red=False, titlesize=16):\n    \n    plt.subplot(*subplot)\n    plt.axis('off')\n    plt.imshow(image)\n    #if len(title) > 0:\n    plt.title(title, fontsize=int(titlesize) if not red else int(titlesize/1.2), color='red' if red else 'black', fontdict={'verticalalignment':'center'}, pad=int(titlesize/1.5))\n    return (subplot[0], subplot[1], subplot[2]+1)\n    \ndef display_batch_of_images(databatch, predictions=None):\n    \"\"\"This will work with:\n    display_batch_of_images(images)\n    display_batch_of_images(images, predictions)\n    display_batch_of_images((images, labels))\n    display_batch_of_images((images, labels), predictions)\n    \"\"\"\n    \n    # data\n    images, labels = batch_to_numpy_images_and_labels(databatch)\n    if labels is None:\n        labels = [None for _ in enumerate(images)]\n\n        \n    #print(labels)\n    # auto-squaring: this will drop data that does not fit into square or square-ish rectangle\n    rows = int(math.sqrt(len(images)))\n    cols = len(images)//rows\n        \n    # size and spacing\n    FIGSIZE = 13.0\n    SPACING = 0.1\n    subplot=(rows,cols,1)\n    if rows < cols:\n        plt.figure(figsize=(FIGSIZE,FIGSIZE/cols*rows))\n    else:\n        plt.figure(figsize=(FIGSIZE/rows*cols,FIGSIZE))\n    \n    # display\n    for i, (image, label) in enumerate(zip(images[:rows*cols], labels[:rows*cols])):\n        title = label\n        correct = True\n        if predictions is not None:\n            title, correct = title_from_label_and_target(predictions[i], label)\n            \n        dynamic_titlesize = FIGSIZE*SPACING/max(rows,cols)*40+3 # magic formula tested to work from 1x1 to 10x10 images\n        subplot = display_one_xray(image, title, subplot, not correct, titlesize=dynamic_titlesize)\n    \n    #layout\n    plt.tight_layout()\n    if label is None and predictions is None:\n        plt.subplots_adjust(wspace=0, hspace=0)\n    else:\n        plt.subplots_adjust(wspace=SPACING, hspace=SPACING)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2021-08-17T13:27:29.316522Z","iopub.execute_input":"2021-08-17T13:27:29.317006Z","iopub.status.idle":"2021-08-17T13:27:29.337202Z","shell.execute_reply.started":"2021-08-17T13:27:29.316957Z","shell.execute_reply":"2021-08-17T13:27:29.336257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def decode_image(image_data):\n    image = tf.image.decode_jpeg(image_data, channels=3)\n    image = tf.cast(image, tf.float32) / 255.0  # convert image to floats in [0, 1] range\n    image = tf.reshape(image, [*IMAGE_SIZE, 3]) # explicit size needed for TPU\n    return image\n\ndef read_labeled_tfrecord(example):\n    LABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string), # tf.string means bytestring\n        \"label\": tf.io.FixedLenFeature([], tf.int64),  # shape [] means single element\n    }\n    example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    label = example['label']\n    return image, label # returns a dataset of (image, label) pairs\n\ndef load_dataset(filenames, labeled=True, ordered=False):\n    # Read from TFRecords. For optimal performance, reading from multiple files at once and\n    # disregarding data order. Order does not matter since we will be shuffling the data anyway.\n\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False # disable order, increase speed\n\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO) # automatically interleaves reads from multiple files\n    dataset = dataset.with_options(ignore_order) # uses data as soon as it streams in, rather than in its original order\n    dataset = dataset.map(read_labeled_tfrecord)\n    # returns a dataset of (image, label) pairs if labeled=True or (image, id) pairs if labeled=False\n    return dataset\n\ndef get_training_dataset():\n    dataset = load_dataset(TRAINING_FILENAMES, labeled=True)\n    dataset = dataset.repeat() # the training dataset must repeat for several epochs\n    dataset = dataset.shuffle(2048)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n    return dataset\n\ndef count_data_items(filenames):\n    # the number of data items is written in the name of the .tfrec files, i.e. img00-200.tfrec = 200 data items\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)","metadata":{"execution":{"iopub.status.busy":"2021-08-17T13:27:35.450116Z","iopub.execute_input":"2021-08-17T13:27:35.450757Z","iopub.status.idle":"2021-08-17T13:27:35.463525Z","shell.execute_reply.started":"2021-08-17T13:27:35.450693Z","shell.execute_reply":"2021-08-17T13:27:35.462387Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# INITIALIZE VARIABLES  //VERIFY STAGE\nIMAGE_SIZE= [256,256]; BATCH_SIZE = 32\nAUTO = tf.data.experimental.AUTOTUNE\nTRAINING_FILENAMES = tf.io.gfile.glob(vfolder+ 'train-tfrec/img*.tfrec')\nTESTING_FILENAMES = tf.io.gfile.glob(vfolder + 'test-tfrec/img*.tfrec')\nprint('There are %i train and %i test images' % (count_data_items(TRAINING_FILENAMES), count_data_items(TESTING_FILENAMES)))","metadata":{"execution":{"iopub.status.busy":"2021-08-17T13:27:44.685057Z","iopub.execute_input":"2021-08-17T13:27:44.685693Z","iopub.status.idle":"2021-08-17T13:27:44.697028Z","shell.execute_reply.started":"2021-08-17T13:27:44.685632Z","shell.execute_reply":"2021-08-17T13:27:44.695487Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# DISPLAY TRAIN IMAGES //VERIFY STAGE\ntraining_dataset = get_training_dataset()\ntraining_dataset = training_dataset.unbatch().batch(3)\ntrain_batch = iter(training_dataset)\ndisplay_batch_of_images(next(train_batch))","metadata":{"execution":{"iopub.status.busy":"2021-08-17T13:27:50.795471Z","iopub.execute_input":"2021-08-17T13:27:50.795834Z","iopub.status.idle":"2021-08-17T13:27:55.713795Z","shell.execute_reply.started":"2021-08-17T13:27:50.795803Z","shell.execute_reply":"2021-08-17T13:27:55.712914Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%cd '/kaggle/working'\nvfolder","metadata":{"execution":{"iopub.status.busy":"2021-08-17T13:31:31.461459Z","iopub.execute_input":"2021-08-17T13:31:31.462755Z","iopub.status.idle":"2021-08-17T13:31:31.472908Z","shell.execute_reply.started":"2021-08-17T13:31:31.462637Z","shell.execute_reply":"2021-08-17T13:31:31.471324Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\n# zip the output and download\ntarname = SAMPLE_VERSION+\".tar.gz\"\ntarfolder = '/kaggle/working/'+SAMPLE_VERSION\n!tar -zcf {tarname} -C {tarfolder} .\n\nimport os\nfrom IPython.display import FileLink\n\nFileLink('/kaggle/working/'+tarname)\n","metadata":{"execution":{"iopub.status.busy":"2021-08-17T13:35:34.236991Z","iopub.execute_input":"2021-08-17T13:35:34.237583Z","iopub.status.idle":"2021-08-17T13:35:51.423993Z","shell.execute_reply.started":"2021-08-17T13:35:34.237543Z","shell.execute_reply":"2021-08-17T13:35:51.422182Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\n%%time\n!tar -zcf train-tfrec.tar.gz -C \"/kaggle/working/train-tfrec/\" .\n!tar -zcf test-tfrec.tar.gz -C \"/kaggle/working/test-tfrec/\" .\n\n!tar -zcf train-covid-tfrec.tar.gz -C \"/kaggle/working/train-covid-tfrec/\" .\n!tar -zcf test-covid-tfrec.tar.gz -C \"/kaggle/working/test-covid-tfrec/\" .\n\n!tar -zcf train-negative-tfrec.tar.gz -C \"/kaggle/working/train-negative-tfrec/\" .\n!tar -zcf test-negative-tfrec.tar.gz -C \"/kaggle/working/test-negative-tfrec/\" .\n\n!tar -zcf covid-train-all-tfrec.tar.gz -C \"/kaggle/working/covid-train-all-tfrec/\" .\n!tar -zcf negative-train-all-tfrec.tar.gz -C \"/kaggle/working/negative-train-all-tfrec/\" .\n\n!tar -zcf train.tar.gz -C \"/kaggle/working/train/\" .\n!tar -zcf test.tar.gz -C \"/kaggle/working/test/\" .\n\n!tar -zcf train-covid.tar.gz -C \"/kaggle/working/train-covid/\" .\n!tar -zcf test-covid.tar.gz -C \"/kaggle/working/test-covid/\" .\n\n!tar -zcf train-negative.tar.gz -C \"/kaggle/working/train-negative/\" .\n!tar -zcf test-negative.tar.gz -C \"/kaggle/working/test-negative/\" .\n\n!tar -zcf covid-train-all.tar.gz -C \"/kaggle/working/covid-train-all/\" .\n!tar -zcf negative-train-all.tar.gz -C \"/kaggle/working/negative-train-all/\" .\n'''","metadata":{"execution":{"iopub.status.busy":"2021-08-15T23:11:55.360412Z","iopub.execute_input":"2021-08-15T23:11:55.360812Z","iopub.status.idle":"2021-08-15T23:12:10.23209Z","shell.execute_reply.started":"2021-08-15T23:11:55.36078Z","shell.execute_reply":"2021-08-15T23:12:10.230927Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Completed ...\")\n","metadata":{"execution":{"iopub.status.busy":"2021-08-15T23:12:34.81668Z","iopub.execute_input":"2021-08-15T23:12:34.817062Z","iopub.status.idle":"2021-08-15T23:12:34.822092Z","shell.execute_reply.started":"2021-08-15T23:12:34.817029Z","shell.execute_reply":"2021-08-15T23:12:34.821289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}}]}