{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install -q efficientnet >> /dev/null","metadata":{"execution":{"iopub.status.busy":"2021-08-13T08:42:30.85765Z","iopub.execute_input":"2021-08-13T08:42:30.858012Z","iopub.status.idle":"2021-08-13T08:42:36.740747Z","shell.execute_reply.started":"2021-08-13T08:42:30.857976Z","shell.execute_reply":"2021-08-13T08:42:36.739781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Preprocessing","metadata":{}},{"cell_type":"markdown","source":"Converting the DICOM X-ray images to 512 jpeg images. Creating and saving the dataset with 512x512 images and a merged csv data with study and image level id.","metadata":{}},{"cell_type":"code","source":"!conda install gdcm -c conda-forge -y","metadata":{"execution":{"iopub.status.busy":"2021-08-13T08:53:47.151631Z","iopub.execute_input":"2021-08-13T08:53:47.152201Z","iopub.status.idle":"2021-08-13T08:54:49.993426Z","shell.execute_reply.started":"2021-08-13T08:53:47.152077Z","shell.execute_reply":"2021-08-13T08:54:49.992306Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nfrom PIL import Image\nimport pandas as pd\nfrom tqdm.auto import tqdm\nimport numpy as np\nimport pydicom\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\nimport pandas as pd\n","metadata":{"execution":{"iopub.status.busy":"2021-08-14T19:23:09.684609Z","iopub.execute_input":"2021-08-14T19:23:09.685014Z","iopub.status.idle":"2021-08-14T19:23:10.097314Z","shell.execute_reply.started":"2021-08-14T19:23:09.684935Z","shell.execute_reply":"2021-08-14T19:23:10.096364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np \nimport pandas as pd \nimport os, shutil\nfrom glob import glob\nfrom sklearn.cluster import KMeans\nfrom tqdm.notebook import tqdm\nfrom sklearn.preprocessing import LabelEncoder\nimport random\ntqdm.pandas()","metadata":{"execution":{"iopub.status.busy":"2021-08-14T19:23:14.536603Z","iopub.execute_input":"2021-08-14T19:23:14.536991Z","iopub.status.idle":"2021-08-14T19:23:15.72572Z","shell.execute_reply.started":"2021-08-14T19:23:14.536959Z","shell.execute_reply":"2021-08-14T19:23:15.724748Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_level = pd.read_csv(\"../input/siim-covid19-detection/train_image_level.csv\")  \nstudy_level = pd.read_csv(\"../input/siim-covid19-detection/train_study_level.csv\")\n\nstudy_level['id'] = study_level['id'].str.replace('_study','')\nstudy_level.rename(columns={\"id\":\"StudyID\"},inplace=True)\nimage_level.rename(columns={\"StudyInstanceUID\":\"StudyID\"},inplace=True)\nmerged_df = pd.merge(image_level,study_level,on=['StudyID'])\nmerged_df['id'] = merged_df['id'].str.replace('_image','')\nmerged_df.rename(columns={\"id\":\"ImageID\"},inplace=True)","metadata":{"execution":{"iopub.status.busy":"2021-08-14T19:23:29.283797Z","iopub.execute_input":"2021-08-14T19:23:29.284208Z","iopub.status.idle":"2021-08-14T19:23:29.399025Z","shell.execute_reply.started":"2021-08-14T19:23:29.284175Z","shell.execute_reply":"2021-08-14T19:23:29.397846Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_xray(path, voi_lut = True, fix_monochrome = True):\n    # Original from: https://www.kaggle.com/raddar/convert-dicom-to-np-array-the-correct-way\n    dicom = pydicom.read_file(path)\n    \n    # VOI LUT (if available by DICOM device) is used to transform raw DICOM data to \n    # \"human-friendly\" view\n    if voi_lut:\n        data = apply_voi_lut(dicom.pixel_array, dicom)\n    else:\n        data = dicom.pixel_array\n               \n    # depending on this value, X-ray may look inverted - fix that:\n    if fix_monochrome and dicom.PhotometricInterpretation == \"MONOCHROME1\":\n        data = np.amax(data) - data\n        \n    data = data - np.min(data)\n    data = data / np.max(data)\n    data = (data * 255).astype(np.uint8)\n        \n    return data\n\ndef resize(array, size, keep_ratio=False, resample=Image.LANCZOS):\n    # Original from: https://www.kaggle.com/xhlulu/vinbigdata-process-and-resize-to-image\n    im = Image.fromarray(array)\n    \n    if keep_ratio:\n        im.thumbnail((size, size), resample)\n    else:\n        im = im.resize((size, size), resample)\n    \n    return im","metadata":{"execution":{"iopub.status.busy":"2021-08-14T19:23:36.589401Z","iopub.execute_input":"2021-08-14T19:23:36.589789Z","iopub.status.idle":"2021-08-14T19:23:36.598324Z","shell.execute_reply.started":"2021-08-14T19:23:36.589755Z","shell.execute_reply":"2021-08-14T19:23:36.597167Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for split in ['test', 'train']:\n    save_dir = f'/kaggle/tmp/{split}/'\n\n    os.makedirs(save_dir, exist_ok=True)\n    \n    for dirname, _, filenames in tqdm(os.walk(f'../input/siim-covid19-detection/{split}')):\n        for file in filenames:\n            # set keep_ratio=True to have original aspect ratio\n            xray = read_xray(os.path.join(dirname, file))\n            im = resize(xray, size=2048)  \n            im.save(os.path.join(save_dir, file.replace('dcm', 'jpg')))\n            merged_df.loc[merged_df.index[merged_df['ImageID']==file],\"ImagePath\"] = os.path.join(save_dir, file.replace('dcm', 'jpg'))\n            merged_df.loc[merged_df.index[merged_df['ImageID']==file],\"Split\"] = split","metadata":{"execution":{"iopub.status.busy":"2021-08-13T08:55:20.927092Z","iopub.execute_input":"2021-08-13T08:55:20.927618Z","iopub.status.idle":"2021-08-13T10:12:08.154757Z","shell.execute_reply.started":"2021-08-13T08:55:20.927578Z","shell.execute_reply":"2021-08-13T10:12:08.152741Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nfrom tqdm import tqdm\nsplitarr = []\nimageid = []\nimagepath = []\nfor split in ['test']:\n    save_dir = f'/kaggle/tmp/{split}/'\n\n    os.makedirs(save_dir, exist_ok=True)\n    \n    for dirname, _, filenames in tqdm(os.walk(f'../input/siim-covid19-detection/{split}')):\n        for file in filenames:\n            # set keep_ratio=True to have original aspect ratio\n            splitarr.append(split)\n            imageid.append(file.replace('.dcm',''))\n            imagepath.append(os.path.join('./test', file.replace('dcm', 'jpg')))","metadata":{"execution":{"iopub.status.busy":"2021-08-15T07:59:29.019498Z","iopub.execute_input":"2021-08-15T07:59:29.019996Z","iopub.status.idle":"2021-08-15T07:59:37.121721Z","shell.execute_reply.started":"2021-08-15T07:59:29.019965Z","shell.execute_reply":"2021-08-15T07:59:37.120757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\ntestdf = pd.DataFrame(columns=['ImageID','Image','Split'])\n","metadata":{"execution":{"iopub.status.busy":"2021-08-15T07:59:38.473366Z","iopub.execute_input":"2021-08-15T07:59:38.473752Z","iopub.status.idle":"2021-08-15T07:59:38.486181Z","shell.execute_reply.started":"2021-08-15T07:59:38.473718Z","shell.execute_reply":"2021-08-15T07:59:38.479959Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"testdf['ImageID'] = imageid\ntestdf['Image'] = imagepath\ntestdf['Split'] = splitarr\n\n","metadata":{"execution":{"iopub.status.busy":"2021-08-15T07:59:40.918191Z","iopub.execute_input":"2021-08-15T07:59:40.918565Z","iopub.status.idle":"2021-08-15T07:59:40.927419Z","shell.execute_reply.started":"2021-08-15T07:59:40.918532Z","shell.execute_reply":"2021-08-15T07:59:40.926636Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"testdf.head()","metadata":{"execution":{"iopub.status.busy":"2021-08-14T19:24:08.366348Z","iopub.execute_input":"2021-08-14T19:24:08.366746Z","iopub.status.idle":"2021-08-14T19:24:08.384041Z","shell.execute_reply.started":"2021-08-14T19:24:08.366711Z","shell.execute_reply":"2021-08-14T19:24:08.383339Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"splitdf[splitdf.Split=='train'].shape","metadata":{"execution":{"iopub.status.busy":"2021-08-14T09:00:21.179215Z","iopub.execute_input":"2021-08-14T09:00:21.179796Z","iopub.status.idle":"2021-08-14T09:00:21.188611Z","shell.execute_reply.started":"2021-08-14T09:00:21.179764Z","shell.execute_reply":"2021-08-14T09:00:21.187562Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"merged_df.to_csv(\"/kaggle/working/merged.csv\")","metadata":{"execution":{"iopub.status.busy":"2021-08-13T10:12:08.158656Z","iopub.execute_input":"2021-08-13T10:12:08.15903Z","iopub.status.idle":"2021-08-13T10:12:08.252123Z","shell.execute_reply.started":"2021-08-13T10:12:08.15898Z","shell.execute_reply":"2021-08-13T10:12:08.251248Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n!tar -zcf dataset.tar.gz -C \"/kaggle/tmp/\" .","metadata":{"execution":{"iopub.status.busy":"2021-08-13T10:12:08.254147Z","iopub.execute_input":"2021-08-13T10:12:08.254588Z","iopub.status.idle":"2021-08-13T10:13:56.177125Z","shell.execute_reply.started":"2021-08-13T10:12:08.254544Z","shell.execute_reply":"2021-08-13T10:13:56.176206Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# TF Records creation","metadata":{}},{"cell_type":"code","source":"IMG_SIZES = [[512, 512]]*5 \nfor i,k in enumerate(IMG_SIZES):\n    print(k[0],k[1],i)","metadata":{"execution":{"iopub.status.busy":"2021-08-13T22:27:56.860822Z","iopub.execute_input":"2021-08-13T22:27:56.861623Z","iopub.status.idle":"2021-08-13T22:27:56.875658Z","shell.execute_reply.started":"2021-08-13T22:27:56.861527Z","shell.execute_reply":"2021-08-13T22:27:56.874517Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tarfile\nimport cv2\n\nopenedfile = tarfile.open('../input/datasetcreation/dataset.tar.gz')\nopenedfile.extractall()","metadata":{"execution":{"iopub.status.busy":"2021-08-15T07:59:00.354968Z","iopub.execute_input":"2021-08-15T07:59:00.355325Z","iopub.status.idle":"2021-08-15T07:59:22.359847Z","shell.execute_reply.started":"2021-08-15T07:59:00.355296Z","shell.execute_reply":"2021-08-15T07:59:22.358424Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport pandas as pd\ntestdf = pd.DataFrame(columns=['ImageID','ImagePath','Split'])\ntraindf = pd.DataFrame(columns=['ImageID','ImagePath','Split'])\nimageIdtest = []\nimagePathtest = []\nimageIdtrain = []\nimagePathtrain = []\nsplittrain = []\nsplittest = []\nfor filename in os.listdir('./test'):\n    imageIdtest.append(filename.replace('.jpg',''))\n    imagePathtest.append(os.path.join('./test',filename))\nfor filename in os.listdir('./train'):\n    imageIdtrain.append(filename.replace('.jpg',''))\n    imagePathtrain.append(os.path.join('./train',filename))\ntestdf['ImageID'] = imageIdtest\ntestdf['ImagePath'] = imagePathtest\ntraindf['ImageID'] = imageIdtrain\ntraindf['ImagePath'] = imagePathtrain\nsplittrain.extend(['train' for i in range(traindf.shape[0])])\nsplittest.extend(['test' for i in range(testdf.shape[0])])\ntraindf['Split'] = splittrain  \ntestdf['Split'] = splittest","metadata":{"execution":{"iopub.status.busy":"2021-08-15T07:39:03.138843Z","iopub.execute_input":"2021-08-15T07:39:03.139334Z","iopub.status.idle":"2021-08-15T07:39:03.200095Z","shell.execute_reply.started":"2021-08-15T07:39:03.139290Z","shell.execute_reply":"2021-08-15T07:39:03.199125Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mergeddf = pd.read_csv('../input/datasetcreation/merged.csv')\nmergeddf.drop(columns=['ImagePath','Split'],inplace=True)\nmergeddf.drop(columns='Unnamed: 0',inplace=True)","metadata":{"execution":{"iopub.status.busy":"2021-08-15T07:39:03.201807Z","iopub.execute_input":"2021-08-15T07:39:03.202114Z","iopub.status.idle":"2021-08-15T07:39:03.274090Z","shell.execute_reply.started":"2021-08-15T07:39:03.202086Z","shell.execute_reply":"2021-08-15T07:39:03.273269Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Final DataFrame merging the two dataframes based on ImageID to get the ImagePath within the dataframe","metadata":{}},{"cell_type":"code","source":"finaltrain = pd.merge(traindf,mergeddf,on=['ImageID'])","metadata":{"execution":{"iopub.status.busy":"2021-08-15T07:39:03.275604Z","iopub.execute_input":"2021-08-15T07:39:03.276184Z","iopub.status.idle":"2021-08-15T07:39:03.298942Z","shell.execute_reply.started":"2021-08-15T07:39:03.276143Z","shell.execute_reply":"2021-08-15T07:39:03.297896Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"finaltrain.to_csv('./finaltrain.csv')","metadata":{"execution":{"iopub.status.busy":"2021-08-14T07:44:43.801972Z","iopub.execute_input":"2021-08-14T07:44:43.802466Z","iopub.status.idle":"2021-08-14T07:44:43.892268Z","shell.execute_reply.started":"2021-08-14T07:44:43.802432Z","shell.execute_reply":"2021-08-14T07:44:43.891533Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#run through the folder and get all the image paths in a column and image id in the other\ndef normalise_row(row):\n    if row['Negative for Pneumonia'] == 1:\n        return 'Negative for Pneumonia'\n    elif row['Typical Appearance']==1:\n        return 'Typical Appearance'\n    elif row['Atypical Appearance'] == 1:\n        return 'Atypical Appearance'\n    elif row['Indeterminate Appearance']==1:\n        return 'Indeterminate Appearance'\n    return result\n\nfinaltrain['classLabel'] = finaltrain.apply(lambda row : normalise_row(row), axis=1) ","metadata":{"execution":{"iopub.status.busy":"2021-08-15T07:39:03.300312Z","iopub.execute_input":"2021-08-15T07:39:03.300698Z","iopub.status.idle":"2021-08-15T07:39:03.429276Z","shell.execute_reply.started":"2021-08-15T07:39:03.300660Z","shell.execute_reply":"2021-08-15T07:39:03.428148Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"classmap = {'Negative for Pneumonia': 0,\n               'Indeterminate Appearance': 1,\n 'Atypical Appearance': 2,\n    'Typical Appearance': 3\n }\nfinaltrain['class'] = finaltrain.classLabel.map(classmap)","metadata":{"execution":{"iopub.status.busy":"2021-08-15T07:39:03.430733Z","iopub.execute_input":"2021-08-15T07:39:03.431340Z","iopub.status.idle":"2021-08-15T07:39:03.440478Z","shell.execute_reply.started":"2021-08-15T07:39:03.431298Z","shell.execute_reply":"2021-08-15T07:39:03.439251Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"name2label = {'Typical Appearance': 3,\n 'Indeterminate Appearance': 1,\n 'Atypical Appearance': 2,\n 'Negative for Pneumonia': 0}\nclass_names = list(name2label.keys())\nlabel2name = {v:k for k, v in name2label.items()}","metadata":{"execution":{"iopub.status.busy":"2021-08-15T07:47:37.078486Z","iopub.execute_input":"2021-08-15T07:47:37.078995Z","iopub.status.idle":"2021-08-15T07:47:37.084013Z","shell.execute_reply.started":"2021-08-15T07:47:37.078966Z","shell.execute_reply":"2021-08-15T07:47:37.083185Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#We actually get the train and test indices at each iteration\nfor fold, (train_idx, val_idx) in enumerate(gkf.split(finaltrain, groups=finaltrain.StudyID.tolist())):\n    print(len(train_idx), len(val_idx))","metadata":{"execution":{"iopub.status.busy":"2021-08-14T07:44:44.030793Z","iopub.execute_input":"2021-08-14T07:44:44.031313Z","iopub.status.idle":"2021-08-14T07:44:44.111334Z","shell.execute_reply.started":"2021-08-14T07:44:44.031261Z","shell.execute_reply":"2021-08-14T07:44:44.110067Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#we use the split method in GroupKFold to actually get training and test(in our case validation) indices for the data.\n#For each iteration we find the validation indices and mark them up in the 'fold' column so that when we perform k-fold cross validation,\n#we could use the data points at these indices to validate\n#https://scikit-learn.org/stable/modules/generated/sklearn.model_selection.GroupKFold.html\n#https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.loc.html\nfrom sklearn.model_selection import GroupKFold\ngkf = GroupKFold(n_splits=5)\nfinaltrain['fold'] = -1\nfor fold, (train_idx, val_idx) in enumerate(gkf.split(finaltrain, groups=finaltrain.StudyID.tolist())):\n    finaltrain.loc[val_idx,'fold'] = fold\nfinaltrain.head()","metadata":{"execution":{"iopub.status.busy":"2021-08-15T07:39:03.441991Z","iopub.execute_input":"2021-08-15T07:39:03.442488Z","iopub.status.idle":"2021-08-15T07:39:04.438150Z","shell.execute_reply.started":"2021-08-15T07:39:03.442308Z","shell.execute_reply":"2021-08-15T07:39:04.437229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\n\ndef _bytes_feature(value):\n    \"\"\"Returns a bytes_list from a string / byte.\"\"\"\n    if isinstance(value, type(tf.constant(0))):\n        value = value.numpy() # BytesList won't unpack a string from an EagerTensor.\n    return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))\n\ndef _float_feature(value):\n    \"\"\"Returns a float_list from a float / double.\"\"\"\n    return tf.train.Feature(float_list=tf.train.FloatList(value=[value]))\n\ndef _int64_feature(value):\n    \"\"\"Returns an int64_list from a bool / enum / int / uint.\"\"\"\n    return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))","metadata":{"execution":{"iopub.status.busy":"2021-08-15T07:39:08.841017Z","iopub.execute_input":"2021-08-15T07:39:08.841376Z","iopub.status.idle":"2021-08-15T07:39:14.791577Z","shell.execute_reply.started":"2021-08-15T07:39:08.841348Z","shell.execute_reply":"2021-08-15T07:39:14.790515Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_example(imageid,image,group,target):\n    feature_samples = {\n      'image_id': _bytes_feature(imageid),\n      'image'   : _bytes_feature(image),\n      'group'   : _bytes_feature(group),    \n      'target'  : _int64_feature(target),\n  }\n    example = tf.train.Example(features=tf.train.Features(feature=feature_samples))\n    return example","metadata":{"execution":{"iopub.status.busy":"2021-08-15T07:39:14.793251Z","iopub.execute_input":"2021-08-15T07:39:14.793551Z","iopub.status.idle":"2021-08-15T07:39:14.799180Z","shell.execute_reply.started":"2021-08-15T07:39:14.793524Z","shell.execute_reply":"2021-08-15T07:39:14.798177Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DIM=512\nimport matplotlib.pyplot as plt\nimport cv2\ndef load_image(path, dim=DIM, ch=3):\n    img = cv2.imread(path, cv2.IMREAD_GRAYSCALE if ch==None else cv2.IMREAD_COLOR)\n    if img.shape[:2]!=(dim,dim) and dim!=-1:\n        img = cv2.resize(img, dsize=(dim,dim), interpolation=cv2.INTER_AREA)\n    return img","metadata":{"execution":{"iopub.status.busy":"2021-08-15T07:39:48.515370Z","iopub.execute_input":"2021-08-15T07:39:48.515880Z","iopub.status.idle":"2021-08-15T07:39:48.521897Z","shell.execute_reply.started":"2021-08-15T07:39:48.515849Z","shell.execute_reply":"2021-08-15T07:39:48.520955Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tqdm import tqdm\nimport numpy as np\n\nfolds = finaltrain.fold.unique().tolist()\nfor fold in tqdm(folds): # create tfrecord for each fold\n    fold_df = finaltrain[finaltrain.fold==fold]\n    with tf.io.TFRecordWriter('train%.2i-%i.tfrec'%(fold,fold_df.shape[0])) as writer:\n        samples = fold_df.shape[0] \n        for k in range(samples): # images in fold\n            row = fold_df.iloc[k,:]\n            image      = load_image(row['ImagePath'], dim=DIM)\n            imageid   = row['ImageID']\n            group      = row['StudyID']\n            target     = np.array(row['class'], dtype=np.uint8)\n            example  = train_example(str.encode(imageid),cv2.imencode('.jpg', image, (cv2.IMWRITE_JPEG_QUALITY, 96))[1].tobytes(),str.encode(group),target,)\n            writer.write(example.SerializeToString())","metadata":{"execution":{"iopub.status.busy":"2021-08-15T07:39:57.646966Z","iopub.execute_input":"2021-08-15T07:39:57.647463Z","iopub.status.idle":"2021-08-15T07:44:55.010208Z","shell.execute_reply.started":"2021-08-15T07:39:57.647433Z","shell.execute_reply":"2021-08-15T07:44:55.009156Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def test_example(imageid,image):\n    feature_samples = {\n        'image_id':_bytes_feature(imageid),\n        'image':_bytes_feature(image)\n    }\n    example = tf.train.Example(features=tf.train.Features(feature=feature_samples))\n    return example","metadata":{"execution":{"iopub.status.busy":"2021-08-15T07:51:35.654607Z","iopub.execute_input":"2021-08-15T07:51:35.655032Z","iopub.status.idle":"2021-08-15T07:51:35.660057Z","shell.execute_reply.started":"2021-08-15T07:51:35.654994Z","shell.execute_reply":"2021-08-15T07:51:35.659134Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"folds = 10\nl = int(np.ceil(testdf.shape[0]/folds))\nfor fold in tqdm(range(folds)): # create tfrecord for each fold\n    fold_df = testdf.iloc[l*fold:l*(fold+1)]\n    with tf.io.TFRecordWriter('test%.2i-%i.tfrec'%(fold,fold_df.shape[0])) as writer:\n        samples = fold_df.shape[0] \n        for k in range(samples): # images in fold\n            row = fold_df.iloc[k,:]\n            imageid   = row['ImageID']\n            image      = load_image(row['ImagePath'], dim=DIM)\n            example  = test_example(str.encode(imageid),cv2.imencode('.jpg', image, (cv2.IMWRITE_JPEG_QUALITY, 96))[1].tobytes())\n            writer.write(example.SerializeToString())","metadata":{"execution":{"iopub.status.busy":"2021-08-15T07:51:36.968033Z","iopub.execute_input":"2021-08-15T07:51:36.968451Z","iopub.status.idle":"2021-08-15T07:52:41.406925Z","shell.execute_reply.started":"2021-08-15T07:51:36.968418Z","shell.execute_reply":"2021-08-15T07:52:41.406012Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2021-08-15T07:04:31.897289Z","iopub.execute_input":"2021-08-15T07:04:31.897622Z","iopub.status.idle":"2021-08-15T07:04:31.924581Z","shell.execute_reply.started":"2021-08-15T07:04:31.897596Z","shell.execute_reply":"2021-08-15T07:04:31.923358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd, numpy as np, random,os, shutil\nfrom glob import glob\nfrom kaggle_datasets import KaggleDatasets\nimport tensorflow as tf, re, math\nimport tensorflow.keras.backend as K\n#import efficientnet.tfkeras as efn\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import roc_auc_score\nimport matplotlib.pyplot as plt\nimport tensorflow_addons as tfa\nprint('tf:',tf.__version__)\nimport math","metadata":{"execution":{"iopub.status.busy":"2021-08-15T07:45:01.186147Z","iopub.execute_input":"2021-08-15T07:45:01.186511Z","iopub.status.idle":"2021-08-15T07:45:01.553842Z","shell.execute_reply.started":"2021-08-15T07:45:01.186481Z","shell.execute_reply":"2021-08-15T07:45:01.552889Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"GCS_PATH = KaggleDatasets().get_gcs_path('siimcovid19tfrecords')\nfiles_train = np.sort(np.array(tf.io.gfile.glob(GCS_PATH+'/train*.tfrec')))\nfiles_test = np.sort(np.array(tf.io.gfile.glob(GCS_PATH+'/test*.tfrec')))","metadata":{"execution":{"iopub.status.busy":"2021-08-15T07:09:52.164134Z","iopub.execute_input":"2021-08-15T07:09:52.164532Z","iopub.status.idle":"2021-08-15T07:09:52.848125Z","shell.execute_reply.started":"2021-08-15T07:09:52.164498Z","shell.execute_reply":"2021-08-15T07:09:52.847004Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import re, math\ndef decode_image(image_data):\n    image = tf.image.decode_png(image_data, channels=3)\n    image = tf.cast(image, tf.float32) / 255.0  # convert image to floats in [0, 1] range\n    image = tf.reshape(image, [*IMAGE_SIZE, 3]) # explicit size needed for TPU\n    return image\ndef prepare_target(target):    \n    target = tf.cast(target, tf.float32)            \n    target = tf.reshape(target, [1])         \n    return target\n\ndef read_labeled_tfrecord(example):\n    LABELED_TFREC_FORMAT = {\n        \"image\" : tf.io.FixedLenFeature([], tf.string), # tf.string means bytestring\n        \"target\": tf.io.FixedLenFeature([], tf.int64),  # shape [] means single element\n    }\n    example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    image  = tf.reshape(image, [DIM, DIM, 3])\n    target = prepare_target(example['target'])\n    return image, target # returns a dataset of (image, label) pairs\n\ndef load_dataset(fileids, labeled=True, ordered=False):\n    # Read from TFRecords. For optimal performance, reading from multiple files at once and\n    # disregarding data order. Order does not matter since we will be shuffling the data anyway.\n\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False # disable order, increase speed\n\n    dataset = tf.data.TFRecordDataset(fileids, num_parallel_reads=AUTO) # automatically interleaves reads from multiple files\n    dataset = dataset.with_options(ignore_order) # uses data as soon as it streams in, rather than in its original order\n    dataset = dataset.map(read_labeled_tfrecord)\n    # returns a dataset of (image, label) pairs if labeled=True or (image, id) pairs if labeled=False\n    return dataset\n\ndef get_training_dataset():\n    dataset = load_dataset(TRAINING_FILENAMES, labeled=True)\n    dataset = dataset.repeat() # the training dataset must repeat for several epochs\n    dataset = dataset.shuffle(20, seed=SEED)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n    return dataset\n\ndef count_data_items(fileids):\n    # the number of data items is written in the id of the .tfrec files, i.e. flowers00-230.tfrec = 230 data items\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(fileid).group(1)) for fileid in fileids]\n    return np.sum(n)","metadata":{"execution":{"iopub.status.busy":"2021-08-15T07:56:19.075663Z","iopub.execute_input":"2021-08-15T07:56:19.076028Z","iopub.status.idle":"2021-08-15T07:56:19.091198Z","shell.execute_reply.started":"2021-08-15T07:56:19.075998Z","shell.execute_reply":"2021-08-15T07:56:19.090246Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def display_batch(batch, size=2):\n    imgs, tars = batch\n    plt.figure(figsize=(size*5, 5))\n    for img_idx in range(size):\n        plt.subplot(1, size, img_idx+1)\n        plt.title(f'class: {label2name[tars[img_idx].numpy()[0]]}', fontsize=15)\n        plt.imshow(imgs[img_idx,:, :, :])\n        plt.xticks([])\n        plt.yticks([])\n    plt.tight_layout()\n    plt.show() ","metadata":{"execution":{"iopub.status.busy":"2021-08-15T07:56:21.049519Z","iopub.execute_input":"2021-08-15T07:56:21.049888Z","iopub.status.idle":"2021-08-15T07:56:21.057099Z","shell.execute_reply.started":"2021-08-15T07:56:21.049837Z","shell.execute_reply":"2021-08-15T07:56:21.055899Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DIM = 512\nIMAGE_SIZE= [DIM,DIM];\nBATCH_SIZE = 32\nSEED=42\nAUTO = tf.data.experimental.AUTOTUNE\n# DISPLAY TRAIN IMAGES\nTRAINING_FILENAMES = tf.io.gfile.glob('train*.tfrec')\nTEST_FILENAMES     = tf.io.gfile.glob('test*.tfrec')\nprint('There are %i train & %i test images'%(count_data_items(TRAINING_FILENAMES), count_data_items(TEST_FILENAMES)))\ntraining_dataset = get_training_dataset()\ntraining_dataset = training_dataset.unbatch().batch(20)\ntrain_batch = next(iter(training_dataset))\ndisplay_batch(train_batch, 5);","metadata":{"execution":{"iopub.status.busy":"2021-08-15T07:56:29.787685Z","iopub.execute_input":"2021-08-15T07:56:29.788189Z","iopub.status.idle":"2021-08-15T07:56:31.263635Z","shell.execute_reply.started":"2021-08-15T07:56:29.788129Z","shell.execute_reply":"2021-08-15T07:56:31.262408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm -rf train","metadata":{"execution":{"iopub.status.busy":"2021-08-15T07:55:49.811166Z","iopub.execute_input":"2021-08-15T07:55:49.811548Z","iopub.status.idle":"2021-08-15T07:55:51.000936Z","shell.execute_reply.started":"2021-08-15T07:55:49.811516Z","shell.execute_reply":"2021-08-15T07:55:50.999479Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"finaltrain.to_csv('./train.csv')","metadata":{"execution":{"iopub.status.busy":"2021-08-15T07:57:29.318362Z","iopub.execute_input":"2021-08-15T07:57:29.318743Z","iopub.status.idle":"2021-08-15T07:57:29.420934Z","shell.execute_reply.started":"2021-08-15T07:57:29.318713Z","shell.execute_reply":"2021-08-15T07:57:29.420018Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv('./train.csv')","metadata":{"execution":{"iopub.status.busy":"2021-08-15T07:58:00.260018Z","iopub.execute_input":"2021-08-15T07:58:00.260385Z","iopub.status.idle":"2021-08-15T07:58:00.298206Z","shell.execute_reply.started":"2021-08-15T07:58:00.260357Z","shell.execute_reply":"2021-08-15T07:58:00.297074Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head()","metadata":{"execution":{"iopub.status.busy":"2021-08-15T07:58:05.249320Z","iopub.execute_input":"2021-08-15T07:58:05.249695Z","iopub.status.idle":"2021-08-15T07:58:05.271023Z","shell.execute_reply.started":"2021-08-15T07:58:05.249654Z","shell.execute_reply":"2021-08-15T07:58:05.270149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"testdf.to_csv('./test.csv')","metadata":{"execution":{"iopub.status.busy":"2021-08-15T08:00:18.646060Z","iopub.execute_input":"2021-08-15T08:00:18.646405Z","iopub.status.idle":"2021-08-15T08:00:18.656763Z","shell.execute_reply.started":"2021-08-15T08:00:18.646378Z","shell.execute_reply":"2021-08-15T08:00:18.655951Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"testdf.head()","metadata":{"execution":{"iopub.status.busy":"2021-08-15T08:00:30.227443Z","iopub.execute_input":"2021-08-15T08:00:30.227823Z","iopub.status.idle":"2021-08-15T08:00:30.239879Z","shell.execute_reply.started":"2021-08-15T08:00:30.227793Z","shell.execute_reply":"2021-08-15T08:00:30.238732Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm -rf train","metadata":{"execution":{"iopub.status.busy":"2021-08-15T08:01:19.078921Z","iopub.execute_input":"2021-08-15T08:01:19.079331Z","iopub.status.idle":"2021-08-15T08:01:20.238094Z","shell.execute_reply.started":"2021-08-15T08:01:19.079297Z","shell.execute_reply":"2021-08-15T08:01:20.236968Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}