{"cells":[{"metadata":{"papermill":{"duration":0.020972,"end_time":"2020-12-05T18:11:25.673408","exception":false,"start_time":"2020-12-05T18:11:25.652436","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Create TFRecords\n\n--- Hamed/Ali --- Jan 10\n- 1) oversample with cutmix\n- 2) cut ratio 0.4\n\n--- Hamed --- Jan 02\n- 1) not copying images to working directory\n- 2) no undersampling for major class\n- 3) cut off 70%\n\n--- Hamed --- Jan 01\n- 1) shuffled images in each file\n\n--- Hamed --- Dec 31\n- 1) fixed the empty TFR files\n- 2) changed the N_FILE to 30 for HDD problem\n- 3) fixed size 600x800\n- 4) cut-off to 0.6 for HDD problem\n\n--- Hamed --- Dec 27\n- 1) added merged dataset 2019-20 from Tom's notebook\n- 2) from merged_data.csv only Images to remove: ['train-cmd-2399.jpg']\n- 3) \"N_FILES\" TFRecord files are ready (over/undersampled with no common images between files, no duplicates)\n- 4) sizes vary! not 600*800!\n\n--- Hamed --- Dec 24\n- 1) kfold\n- 2) Random over/undersampling on each fold (cut_ratio can be defined)\n\n\n- Based on: https://www.kaggle.com/dimitreoliveira/cassava-leaf-disease-stratified-tfrecords-256x256/data\n- Discussion [thread](https://www.kaggle.com/c/cassava-leaf-disease-classification/discussion/198744)\n- Reference: [How To Create TFRecords](https://www.kaggle.com/cdeotte/how-to-create-tfrecords)"},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.execute_input":"2020-12-05T18:11:25.720088Z","iopub.status.busy":"2020-12-05T18:11:25.719222Z","iopub.status.idle":"2020-12-05T18:11:33.311182Z","shell.execute_reply":"2020-12-05T18:11:33.310436Z"},"papermill":{"duration":7.619603,"end_time":"2020-12-05T18:11:33.311321","exception":false,"start_time":"2020-12-05T18:11:25.691718","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"import shutil\nimport re, math, os, cv2, random, warnings\nimport math\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport tensorflow as tf\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import StratifiedKFold\n\nimport glob, torch, imagehash\nfrom tqdm.auto import tqdm\nfrom PIL import Image\n\ndef seed_everything(seed=0):\n    random.seed(seed)\n    np.random.seed(seed)\n    tf.random.set_seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    os.environ['TF_DETERMINISTIC_OPS'] = '1'\n\nGOAL = 'valid'\n#GOAL = 'train'\nseed = 2020\nseed_everything(seed)\nwarnings.filterwarnings('ignore')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"base_path = '../input/cassavapreprocessed'\nimages_dir = '../input/cassavapreprocessed/train_images/train_images/'\ngen_dir = './gen_images/'\n#working_dir = './working/'\ntrain_tfrecords_dir = './train_data_tf/'\nvalid_tfrecords_dir = './valid_data_tf/'\n\nif GOAL == 'train':\n    if not os.path.exists(train_tfrecords_dir):\n        os.mkdir(train_tfrecords_dir)\n    if not os.path.exists(gen_dir):\n        os.mkdir(gen_dir)\n        for i in range(N_FILES):\n            os.mkdir(gen_dir+f'{i}/')\n        \n    # Resampling\n    # if 0.6, all classes will be over/undersampled\n    # to the 0.6 of the major class\n    cut_ratio = 0.4\n\nelif GOAL == 'valid':\n    if not os.path.exists(valid_tfrecords_dir):\n        os.mkdir(valid_tfrecords_dir)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"DIMS = (800, 600)\nN_FILES = 30\nWIDTH, HEIGHT  = DIMS\nIMG_QUALITY = 100","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# LOAD DATA"},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv(base_path + '/merged_data.csv')\ntrain.head()\nprint('Train samples: %d' % len(train))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Remove duplicates from train data\nremove_pd = ['train-cmd-2399.jpg']\ntrain = train[~train['image_id'].isin(remove_pd)]\ntrain.reset_index(inplace=True)\nprint('Train samples: %d' % len(train))","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.019744,"end_time":"2020-12-05T18:41:33.489169","exception":false,"start_time":"2020-12-05T18:41:33.469425","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Helper functions"},{"metadata":{"_kg_hide-input":true,"execution":{"iopub.execute_input":"2020-12-05T18:41:33.557508Z","iopub.status.busy":"2020-12-05T18:41:33.548741Z","iopub.status.idle":"2020-12-05T18:41:33.560646Z","shell.execute_reply":"2020-12-05T18:41:33.559845Z"},"papermill":{"duration":0.051443,"end_time":"2020-12-05T18:41:33.560772","exception":false,"start_time":"2020-12-05T18:41:33.509329","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"def decode_image(image_data):\n    image = tf.image.decode_jpeg(image_data, channels=3)\n    image = tf.cast(image, tf.float32) / 255.0\n                      \n    # image = tf.image.resize(image, [HEIGHT, WIDTH])\n    # image = tf.reshape(image, [HEIGHT, WIDTH, 3])\n    return image\n\ndef read_tfrecord(example):\n    TFREC_FORMAT = {\n        'image': tf.io.FixedLenFeature([], tf.string), \n        'target': tf.io.FixedLenFeature([], tf.int64), \n        'image_name': tf.io.FixedLenFeature([], tf.string), \n    }\n    example = tf.io.parse_single_example(example, TFREC_FORMAT)\n    image = decode_image(example['image'])\n    target = example['target']\n    name = example['image_name']\n    return image, target, name\n\ndef load_dataset(filenames, HEIGHT, WIDTH, CHANNELS=3):\n    dataset = tf.data.TFRecordDataset(filenames)\n    dataset = dataset.map(read_tfrecord, num_parallel_calls=AUTO)\n    return dataset\n\ndef display_samples(ds, row, col):\n    ds_iter = iter(ds)\n    plt.figure(figsize=(15, int(15*row/col)))\n    for j in range(row*col):\n        image, label, name = next(ds_iter)\n        plt.subplot(row,col,j+1)\n        plt.axis('off')\n        plt.imshow(image[0])\n        plt.title(f\"{label[0]}: {name[0].numpy().decode('utf-8')}\", fontsize=12)\n    plt.show()\n\ndef count_data_items(filenames):\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)\n\n\n# Create TF Records\ndef _bytes_feature(value):\n  \"\"\"Returns a bytes_list from a string / byte.\"\"\"\n  if isinstance(value, type(tf.constant(0))):\n    value = value.numpy() # BytesList won't unpack a string from an EagerTensor.\n  return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))\n\ndef _int64_feature(value):\n  \"\"\"Returns an int64_list from a bool / enum / int / uint.\"\"\"\n  return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))\n\ndef serialize_example(image, target, image_name):\n  feature = {\n      'image': _bytes_feature(image),\n      'target': _int64_feature(target),\n      'image_name': _bytes_feature(image_name),\n  }\n  example_proto = tf.train.Example(features=tf.train.Features(feature=feature))\n  return example_proto.SerializeToString()","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.021549,"end_time":"2020-12-05T18:41:33.961276","exception":false,"start_time":"2020-12-05T18:41:33.939727","status":"completed"},"tags":[]},"cell_type":"markdown","source":"## Split samples into \"N_FILES\" different files"},{"metadata":{"_kg_hide-input":true,"execution":{"iopub.execute_input":"2020-12-05T18:41:34.017817Z","iopub.status.busy":"2020-12-05T18:41:34.017045Z","iopub.status.idle":"2020-12-05T18:41:34.508532Z","shell.execute_reply":"2020-12-05T18:41:34.507714Z"},"papermill":{"duration":0.525356,"end_time":"2020-12-05T18:41:34.508661","exception":false,"start_time":"2020-12-05T18:41:33.983305","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"folds = StratifiedKFold(n_splits=N_FILES, shuffle=True, random_state=seed)\ntrain['file'] = -1\n\nfor fold_n, (train_idx, val_idx) in enumerate(folds.split(train, train['label'])):\n    print('File: %s has %s samples' % (fold_n+1, len(val_idx)))\n    train['file'].loc[val_idx] = fold_n\n    \ndisplay(train.head())\ndisplay(train.describe())\ntrain.to_csv('train.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Cutmix function"},{"metadata":{"trusted":true},"cell_type":"code","source":"def gen_cutmix(df, label = None, file = None, probability = 0.2):\n    \n    f_uni = df.file.unique()\n    l_uni = df.label.unique()\n    print(f'DataFrame has {len(f_uni)} files!')\n    max_label = df.label.value_counts().idxmax()\n    sampling_num = max(df.label.value_counts())*cut_ratio // len(f_uni)\n    print('sampling_num is', sampling_num)\n    rows = []    \n    \n    if label:\n        df = df.loc[df.label == label]\n        \n    if file:\n        df = df.loc[df.file == file]\n    f_counter = 0    \n    for f in f_uni:\n        f_counter += 1\n        path = f\"{gen_dir}{f}/\"\n        for l in l_uni:\n            \n            n = len(df.loc[(df.file == f) & (df.label == l)])\n            i = 0\n            while (n + i) < sampling_num:\n                \n                P = np.random.rand()                \n                if (P < probability) & (P > (1 - probability)):\n                    continue\n                \n                row = []    \n                # CHOOSE RANDOM PHOTOS OF THE SAME FILE AND CLASS\n                img1_name = df.iloc[np.random.randint(len(df))].image_id\n                img1_path = f'{images_dir}{img1_name}'\n                img1 = cv2.imread(img1_path)\n                if img1.shape[:2] != (HEIGHT, WIDTH):\n                    img1 = cv2.resize(img1, (WIDTH, HEIGHT))              \n\n                img2_name = df.iloc[np.random.randint(len(df))].image_id\n                img2_path = f'{images_dir}{img2_name}'\n                img2 = cv2.imread(img2_path)\n                if img2.shape[:2] != (HEIGHT, WIDTH):\n                    img2 = cv2.resize(img2, (WIDTH, HEIGHT))\n\n                # CHOOSE RANDOM LOCATION\n                x = (np.random.rand() * WIDTH)//1\n                y = (np.random.rand() * HEIGHT)//1\n                width = (WIDTH * P)//1\n                height = (HEIGHT * P)//1\n\n                ya = int(max(0,y-height//2))\n                yb = int(max(HEIGHT,y+height//2))\n                xa = int(max(0,x-width//2))\n                xb = int(max(WIDTH,x+width//2))\n\n                # MAKE CUTMIX IMAGE\n                one = img1[ya:yb,0:xa,:]\n                two = img2[ya:yb,xa:xb,:]\n                three = img1[ya:yb,xb:WIDTH,:]\n                middle = np.concatenate((one,two,three),axis=1)\n                img = np.concatenate((img1[0:ya,:,:],middle,img1[yb:HEIGHT,:,:]),axis=0)\n                i = i + 1    \n                img_name = f\"x{i}_{img1_name}\"\n                cv2.imwrite(f\"{path}{img_name}\",img)\n\n                row.append(img_name)\n                row.append(l)\n                row.append(f)\n                rows.append(row)\n        print(f\"{f_counter} files are done!\")\n    new_df = pd.DataFrame(rows, columns = ['image_id','label','file'])\n    return new_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if GOAL == 'train':\n    if os.path.exists(gen_dir):\n        shutil.rmtree(gen_dir)\n    else:\n        os.mkdir(gen_dir)\n        for i in range(N_FILES):\n            os.mkdir(gen_dir+f'{i}/')\n    imgs = gen_cutmix(train)\n    imgs.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if GOAL == 'train':\n    _, ax = plt.subplots(10, figsize = (10,50))\n    for i in range(10):\n        img = plt.imread(f\"{gen_dir}{imgs.file[i]}/{imgs.image_id[i]}\")\n        ax[i].imshow(img)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if GOAL == 'train':\n    bl_train = pd.concat([train[['image_id','label','file']],imgs],ignore_index=True)\n    bl_train.label.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.02331,"end_time":"2020-12-05T18:41:34.556106","exception":false,"start_time":"2020-12-05T18:41:34.532796","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Generate TF records"},{"metadata":{"_kg_hide-input":true,"execution":{"iopub.execute_input":"2020-12-05T18:41:34.615797Z","iopub.status.busy":"2020-12-05T18:41:34.615052Z","iopub.status.idle":"2020-12-05T18:46:37.070472Z","shell.execute_reply":"2020-12-05T18:46:37.069767Z"},"papermill":{"duration":302.490823,"end_time":"2020-12-05T18:46:37.070634","exception":false,"start_time":"2020-12-05T18:41:34.579811","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"if GOAL == 'train':\n    for tfrec_num in range(N_FILES):\n        print('\\nWriting TFRecord %i of %i...'%(tfrec_num, N_FILES))\n        samples = bl_train[bl_train['file'] == tfrec_num]\n        n_samples = len(samples)\n        print(f'{n_samples} samples')\n        with tf.io.TFRecordWriter(train_tfrecords_dir + 'Id_train%.2i-%i.tfrec'%(tfrec_num, n_samples)) as writer:\n            for row in samples.itertuples():\n                label = row.label\n                image_name = row.image_id\n                if image_name[0] == 'x':\n                    img_path = f'{gen_dir}{tfrec_num}/{image_name}'\n                else:\n                    img_path = f'{images_dir}{image_name}'\n\n                img = cv2.imread(img_path)\n                img = cv2.resize(img, (WIDTH, HEIGHT))\n                img = cv2.imencode('.jpg', img, (cv2.IMWRITE_JPEG_QUALITY, IMG_QUALITY))[1].tostring()\n\n                example = serialize_example(img, label, str.encode(image_name))\n                writer.write(example)\n        shutil.rmtree(f\"{gen_dir}{tfrec_num}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if os.path.exists(gen_dir):\n    shutil.rmtree(gen_dir)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if GOAL == 'valid':\n    for tfrec_num in range(N_FILES):\n        print('\\nWriting TFRecord %i of %i...'%(tfrec_num, N_FILES))\n        samples = train[train['file'] == tfrec_num]\n        n_samples = len(samples)\n        print(f'{n_samples} samples')\n        with tf.io.TFRecordWriter(valid_tfrecords_dir + 'Id_valid%.2i-%i.tfrec'%(tfrec_num, n_samples)) as writer:\n            for row in samples.itertuples():\n                label = row.label\n                image_name = row.image_id\n                img_path = f'{images_dir}{image_name}'\n\n                img = cv2.imread(img_path)\n                img = cv2.resize(img, (WIDTH, HEIGHT))\n                img = cv2.imencode('.jpg', img, (cv2.IMWRITE_JPEG_QUALITY, IMG_QUALITY))[1].tostring()\n\n                example = serialize_example(img, label, str.encode(image_name))\n                writer.write(example)","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.029813,"end_time":"2020-12-05T18:46:37.130801","exception":false,"start_time":"2020-12-05T18:46:37.100988","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Visualize created TF records\n\n## Class map\n\n```\n0: Cassava Bacterial Blight (CBB)\n1: Cassava Brown Streak Disease (CBSD)\n2: Cassava Green Mottle (CGM)\n3: Cassava Mosaic Disease (CMD)\n4: Healthy\n```"},{"metadata":{"_kg_hide-input":true,"execution":{"iopub.execute_input":"2020-12-05T18:46:37.198465Z","iopub.status.busy":"2020-12-05T18:46:37.19773Z","iopub.status.idle":"2020-12-05T18:46:39.942076Z","shell.execute_reply":"2020-12-05T18:46:39.94268Z"},"papermill":{"duration":2.781566,"end_time":"2020-12-05T18:46:39.942843","exception":false,"start_time":"2020-12-05T18:46:37.161277","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"if GOAL == 'train':\n    path = train_tfrecords_dir + \nelif GAL == 'valid':\n    path = valid_tfrecords_dir\n    \nAUTO = tf.data.experimental.AUTOTUNE\nFILENAMES = tf.io.gfile.glob(path + '*.tfrec')\nprint(f'TFRecords files: {FILENAMES}')\nprint(f'Created image samples: {count_data_items(FILENAMES)}')\n\ndisplay_samples(load_dataset(FILENAMES, WIDTH, HEIGHT).batch(1), 6, 6)","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.068022,"end_time":"2020-12-05T18:46:40.079223","exception":false,"start_time":"2020-12-05T18:46:40.011201","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Complete set label distribution"},{"metadata":{"_kg_hide-input":true,"execution":{"iopub.execute_input":"2020-12-05T18:46:40.230619Z","iopub.status.busy":"2020-12-05T18:46:40.229816Z","iopub.status.idle":"2020-12-05T18:46:40.431475Z","shell.execute_reply":"2020-12-05T18:46:40.430813Z"},"papermill":{"duration":0.284844,"end_time":"2020-12-05T18:46:40.431602","exception":false,"start_time":"2020-12-05T18:46:40.146758","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"CLASSES = ['Cassava Bacterial Blight', \n           'Cassava Brown Streak Disease', \n           'Cassava Green Mottle', \n           'Cassava Mosaic Disease', \n           'Healthy']\nif GOAL == 'train':\n    df = bl_train\nelif GOAL == 'valid':\n    df = train\n\nlabel_count = df.groupby('label', as_index=False).count()\nlabel_count.rename(columns={'image_id': 'Count', 'label': 'Label'}, inplace=True)\nlabel_count['Label'] = label_count['Label'].apply(lambda x: CLASSES[x])\n\nfig, ax = plt.subplots(1, 1, figsize=(14, 8))\nax = sns.barplot(x=label_count['Count'], y=label_count['Label'], palette='viridis')\nax.tick_params(labelsize=16)\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.067973,"end_time":"2020-12-05T18:46:40.568192","exception":false,"start_time":"2020-12-05T18:46:40.500219","status":"completed"},"tags":[]},"cell_type":"markdown","source":"## Labels distribution for each file"},{"metadata":{"_kg_hide-input":true,"execution":{"iopub.execute_input":"2020-12-05T18:46:40.717152Z","iopub.status.busy":"2020-12-05T18:46:40.716067Z","iopub.status.idle":"2020-12-05T18:46:43.791836Z","shell.execute_reply":"2020-12-05T18:46:43.791057Z"},"papermill":{"duration":3.155411,"end_time":"2020-12-05T18:46:43.791994","exception":false,"start_time":"2020-12-05T18:46:40.636583","status":"completed"},"scrolled":false,"tags":[],"trusted":true},"cell_type":"code","source":"for fold_n in range(folds.n_splits):\n    label_count = df[df['file'] == fold_n].groupby('label', as_index=False).count()\n    label_count.rename(columns={'image_id': 'Count', 'label': 'Label'}, inplace=True)\n    label_count['Label'] = label_count['Label'].apply(lambda x: CLASSES[x])\n\n    fig, ax = plt.subplots(1, 1, figsize=(14, 8))\n    fig.suptitle(f'File {fold_n+1}', fontsize=22)\n    ax = sns.barplot(x=label_count['Count'], y=label_count['Label'], palette='viridis')\n    ax.tick_params(labelsize=16)\n\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}