{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"#Preprocessing the Data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.execute_input":"2021-01-31T11:44:38.778565Z","iopub.status.busy":"2021-01-31T11:44:38.773342Z","iopub.status.idle":"2021-01-31T11:45:56.057076Z","shell.execute_reply":"2021-01-31T11:45:56.056378Z"},"papermill":{"duration":77.315718,"end_time":"2021-01-31T11:45:56.057286","exception":false,"start_time":"2021-01-31T11:44:38.741568","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"# Try and get keras plot to work\n!pip install -q pydot\n!pip install -q pydotplus\n!apt-get install -q graphviz\n\nprint(\"\\n... OTHER IMPORTS STARTING ...\\n\")\nprint(\"\\n\\tVERSION INFORMATION\")\n\n# Machine Learning and Data Science Imports\nimport tensorflow_addons as tfa; print(f\"\\t\\t– TENSORFLOW ADDONS VERSION: {tfa.__version__}\");\nimport tensorflow as tf; print(f\"\\t\\t– TENSORFLOW VERSION: {tf.__version__}\");\nimport pandas as pd; pd.options.mode.chained_assignment = None;\nimport numpy as np; print(f\"\\t\\t– NUMPY VERSION: {np.__version__}\");\nimport scipy; print(f\"\\t\\t– SCIPY VERSION: {scipy.__version__}\");\n\n# Built In Imports\nfrom collections import Counter\nfrom datetime import datetime\nimport multiprocessing\nfrom glob import glob\nimport warnings\nimport requests\nimport imageio\nimport IPython\nimport urllib\nimport zipfile\nimport pickle\nimport random\nimport shutil\nimport string\nimport math\nimport tqdm\nimport time\nimport gzip\nimport io\nimport os\nimport gc\nimport re\n\n# Visualization Imports\nfrom matplotlib.colors import ListedColormap\nimport matplotlib.patches as patches\nimport plotly.graph_objects as go\nimport matplotlib.pyplot as plt\nfrom tqdm.notebook import tqdm\nimport plotly.express as px\nimport seaborn as sns\nfrom PIL import Image\nimport matplotlib; print(f\"\\t\\t– MATPLOTLIB VERSION: {matplotlib.__version__}\");\nimport plotly\nimport PIL\nimport cv2\nimport ast\n\n# PRESETS\nLBL_NAMES = [\"Nucleoplasm\", \"Nuclear Membrane\", \"Nucleoli\", \"Nucleoli Fibrillar Center\", \"Nuclear Speckles\", \"Nuclear Bodies\", \"Endoplasmic Reticulum\", \"Golgi Apparatus\", \"Intermediate Filaments\", \"Actin Filaments\", \"Microtubules\", \"Mitotic Spindle\", \"Centrosome\", \"Plasma Membrane\", \"Mitochondria\", \"Aggresome\", \"Cytosol\", \"Vesicles\", \"Negative\"]\nINT_2_STR = {x:LBL_NAMES[x] for x in np.arange(19)}\nINT_2_STR_LOWER = {k:v.lower().replace(\" \", \"_\") for k,v in INT_2_STR.items()}\nSTR_2_INT_LOWER = {v:k for k,v in INT_2_STR_LOWER.items()}\nSTR_2_INT = {v:k for k,v in INT_2_STR.items()}\nFIG_FONT = dict(family=\"Helvetica, Arial\", size=14, color=\"#7f7f7f\")\nLABEL_COLORS = [px.colors.label_rgb(px.colors.convert_to_RGB_255(x)) for x in sns.color_palette(\"Spectral\", len(LBL_NAMES))]\nLABEL_COL_MAP = {str(i):x for i,x in enumerate(LABEL_COLORS)}\n\nprint(\"\\n\\n... IMPORTS COMPLETE ...\\n\")","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-01-31T11:45:56.320967Z","iopub.status.busy":"2021-01-31T11:45:56.319646Z","iopub.status.idle":"2021-01-31T11:45:58.177889Z","shell.execute_reply":"2021-01-31T11:45:58.177292Z"},"papermill":{"duration":1.893314,"end_time":"2021-01-31T11:45:58.178032","exception":false,"start_time":"2021-01-31T11:45:56.284718","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"# Define the path to the root data directory\nROOT_DIR = \"/kaggle/input\"\n\n# Define the path to the competition data directory\nCOMP_DIR = os.path.join(ROOT_DIR, \"hpa-single-cell-image-classification\")\n\n# Define path to the filtered TP IDs for each class\nPKL_DIR = os.path.join(ROOT_DIR, \"hpa-rule-based-single-cell-filtering\")\n\n# Define the paths to the training tiles for the cell-wise classification dataset\nRED_TILE_DIR = os.path.join(ROOT_DIR, \"human-protein-atlas-red-cell-tile-dataset\")\nGREEN_TILE_DIR = os.path.join(ROOT_DIR, \"human-protein-atlas-green-cell-tile-dataset\")\nBLUE_TILE_DIR = os.path.join(ROOT_DIR, \"human-protein-atlas-blue-cell-tile-dataset\")\nYELLOW_TILE_DIR = os.path.join(ROOT_DIR, \"human-protein-atlas-yellow-cell-tile-dataset\")\n\n# Define the paths to the training and testing tfrecord and \n# image folders respectively for the competition data\nTRAIN_IMG_DIR = os.path.join(COMP_DIR, \"train\")\nTRAIN_TFREC_DIR = os.path.join(COMP_DIR, \"train_tfrecords\")\nTEST_IMG_DIR = os.path.join(COMP_DIR, \"test\")\nTEST_TFREC_DIR = os.path.join(COMP_DIR, \"test_tfrecords\")\n\n# Capture all the relevant full image paths for the competition dataset\nTRAIN_IMG_PATHS = sorted([os.path.join(TRAIN_IMG_DIR, f_name) for f_name in os.listdir(TRAIN_IMG_DIR)])\nTEST_IMG_PATHS = sorted([os.path.join(TEST_IMG_DIR, f_name) for f_name in os.listdir(TEST_IMG_DIR)])\nprint(f\"\\n... Recall that 4 training images compose one example (R,G,B,Y) ...\")\nprint(f\"... \\t– i.e. The first 4 training files are:\")\nfor path in [x.rsplit('/',1)[1] for x in TRAIN_IMG_PATHS[:4]]: print(f\"... \\t\\t– {path}\")\nprint(f\"\\n... The number of training images is {len(TRAIN_IMG_PATHS)} i.e. {len(TRAIN_IMG_PATHS)//4} 4-channel images ...\")\nprint(f\"... The number of testing images is {len(TEST_IMG_PATHS)} i.e. {len(TEST_IMG_PATHS)//4} 4-channel images ...\")\n\n# Capture all the relevant full tfrec paths\nTRAIN_TFREC_PATHS = sorted([os.path.join(TRAIN_TFREC_DIR, f_name) for f_name in os.listdir(TRAIN_TFREC_DIR)])\nTEST_TFREC_PATHS = sorted([os.path.join(TEST_TFREC_DIR, f_name) for f_name in os.listdir(TEST_TFREC_DIR)])\nprint(f\"\\n... The number of training tfrecord files is {len(TRAIN_TFREC_PATHS)} ...\")\nprint(f\"... The number of testing tfrecord files is {len(TEST_TFREC_PATHS)} ...\\n\")\n\n# Random Useful Info\nORIGINAL_DIST_MAP = {0: 37472, 1: 4845, 2: 12672, 3: 12882, 4: 17527, 5: 15337, 6: 10198, 7: 18825, 8: 11194, 9: 5322, 10: 7789, 11: 10, 12: 13952, 13: 21168, 14: 27494, 15: 2275, 16: 22738, 17: 5619, 18: 952}\n\n# Define paths to the relevant csv files\nTRAIN_CSV = os.path.join(ROOT_DIR, \"hpa-train-data-with-additional-metadata/updated_train.csv\")\n\nprint(\"\\n... Loading massive train dataframe ...\\n\")\n# Create the relevant dataframe objects\ntrain_df = pd.read_csv(TRAIN_CSV)\n# train_df.mask_rles = train_df.mask_rles.apply(lambda x: ast.literal_eval(x))\n# train_df.mask_bboxes = train_df.mask_bboxes.apply(lambda x: ast.literal_eval(x))\n    \nprint(\"\\n\\nTRAIN DATAFRAME\\n\\n\")\ndisplay(train_df.head(3))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_image_scaled(img_id, img_dir, img_size=512, load_style=\"tf\"):\n    \"\"\" Load An Image Using ID and Directory Path - Composes 4 Individual Images \"\"\"\n    def __load_with_tf(path, img_size=512):\n        img = tf.io.read_file(path)\n        img = tf.image.decode_png(img, channels=1)\n        return tf.image.resize(img, (img_size, img_size))[..., 0]\n    \n    def __load_with_pil(path, img_size=512):\n        img = Image.open(path)\n        img = img.resize((img_size, img_size))\n        return np.asarray(img)\n    \n    def __load_with_cv2(path, img_size=512):\n        img = cv2.imread(path, 0)\n        img = cv2.resize(img, (img_size, img_size))\n        return img\n        \n    if load_style is \"tf\":\n        load_fn = __load_with_tf\n    elif load_style is \"PIL\":\n        load_fn = __load_with_pil\n    else:\n        load_fn = __load_with_cv2\n    \n    return np.stack(\n        [np.asarray(load_fn(os.path.join(img_dir, img_id+f\"_{c}.png\"), img_size)/255.) for c in [\"red\", \"yellow\", \"blue\"]], axis=2\n    )\n\n\ndef decode_img(img, img_size=(224,224)):\n    \"\"\"TBD\"\"\"\n    \n    # convert the compressed string to a 3D uint8 tensor\n    img = tf.image.decode_png(img, channels=1)\n\n    # resize the image to the desired size\n    return tf.cast(tf.image.resize(img, img_size), tf.uint8)\n\n\ndef get_color_path_maps(color_dirs, tp_id_map):\n    c_p_maps = [{k:[] for k in INT_2_STR.keys()} for _ in range(len(color_dirs))]\n    color_d_paths = [\n        [d_path for d_path in os.listdir(color_dir) if d_path.endswith(\"_256\")] \\\n        for color_dir in color_dirs\n    ]\n    for c in tqdm(color_d_paths[0], total=len(color_d_paths[0])):\n        \n        # Get class stuff\n        cls = c.split(\"_\", 1)[1].rsplit(\"_\",1)[0]\n        cls_idx = STR_2_INT_LOWER[cls]\n        \n        # Get the relevant color directories\n        c_dirs = [\n            os.path.join(color_dir, c.replace(\"red\", clr), \"data\", \"train_tiles\", cls) \\\n            for clr, color_dir in zip([\"red\", \"green\", \"blue\", \"yellow\"], color_dirs)\n        ]\n\n        # Update map\n        for f_name in tqdm(os.listdir(c_dirs[0]), total=len(os.listdir(c_dirs[0]))):\n            # get the relevant full paths\n            full_paths = [os.path.join(c_dir, f_name.replace(\"red\", clr)) for clr, c_dir in zip([\"red\", \"green\", \"blue\", \"yellow\"], c_dirs)]\n            if tp_id_map==None:\n                for c_p_map, full_path in zip(c_p_maps, full_paths):\n                    c_p_map[cls_idx].append(full_path)\n            elif (f_name.endswith(\".png\") and (\"negative\" in full_paths[0] or f_name.rsplit(\"_\", 1)[0] in tp_id_map[cls_idx])):\n                for c_p_map, full_path in zip(c_p_maps, full_paths):\n                    c_p_map[cls_idx].append(full_path)\n            else:\n                for c_p_map, full_path in zip(c_p_maps, full_paths):\n                    c_p_map[STR_2_INT[\"Negative\"]].append(full_path)\n    return [{k:sorted(v) for k,v in c_p_map.items()} for c_p_map in c_p_maps]\n\n\ndef get_tp_id_map(pkl_dir):\n    \"\"\" TBD \"\"\"\n    # Capture all relevant paths\n    pkl_paths = [\n        os.path.join(pkl_dir, f_name) \\\n        for f_name in os.listdir(pkl_dir) \\\n        if f_name.endswith(\".pkl\")\n    ]\n    \n    # REMOVE AFTER UPDATING CLASSBASED NOTEBOOK\n    pkl_paths.append(\"/kaggle/input/tmp-intermediate-filaments-pkl-file/intermediate_filaments_tp_list.pkl\")\n    \n    # Initialize\n    tp_id_map = {}\n    for path in pkl_paths:\n        class_id = STR_2_INT_LOWER[path.rsplit(\"/\", 1)[1].replace(\"_tp_list.pkl\", \"\")]\n        with open(path, \"rb\") as f:\n            tp_id_map[class_id] = pickle.load(f)\n    return tp_id_map\n\n    \ndef plot_rgb(arr, figsize=(12,12)):\n    \"\"\" Plot 3 Channel Microscopy Image \"\"\"\n    plt.figure(figsize=figsize)\n    plt.title(f\"RGB Composite Image\", fontweight=\"bold\")\n    plt.imshow(arr)\n    plt.axis(False)\n    plt.show()    \n\n    \ndef convert_rgby_to_rgb(arr):\n    \"\"\" Convert a 4 channel (RGBY) image to a 3 channel RGB image.\n    \n    Advice From Competition Host/User: lnhtrang\n\n    For annotation (by experts) and for the model, I guess we agree that individual \n    channels with full range px values are better. \n    In annotation, we toggled the channels. \n    For visualization purpose only, you can try blending the channels. \n    For example, \n        - red = red + yellow\n        - green = green + yellow/2\n        - blue=blue.\n        \n    Args:\n        arr (numpy array): The RGBY, 4 channel numpy array for a given image\n    \n    Returns:\n        RGB Image\n    \"\"\"\n    \n    rgb_arr = np.zeros_like(arr[..., :-1])\n    rgb_arr[..., 0] = arr[..., 0]\n    rgb_arr[..., 1] = arr[..., 1]+arr[..., 3]/2\n    rgb_arr[..., 2] = arr[..., 2]\n    \n    return rgb_arr\n    \n    \ndef plot_ex(arr, figsize=(20,6), title=None, plot_merged=True, rgb_only=False):\n    \"\"\" Plot 4 Channels Side by Side \"\"\"\n    if plot_merged and not rgb_only:\n        n_images=5 \n    elif plot_merged and rgb_only:\n        n_images=4\n    elif not plot_merged and rgb_only:\n        n_images=4\n    else:\n        n_images=3\n    plt.figure(figsize=figsize)\n    if type(title) == str:\n        plt.suptitle(title, fontsize=20, fontweight=\"bold\")\n\n    for i, c in enumerate([\"Red Channel – Microtubles\", \"Green Channel – Protein of Interest\", \"Blue - Nucleus\", \"Yellow – Endoplasmic Reticulum\"]):\n        if not rgb_only:\n            ch_arr = np.zeros_like(arr[..., :-1])        \n        else:\n            ch_arr = np.zeros_like(arr)\n        if c in [\"Red Channel – Microtubles\", \"Green Channel – Protein of Interest\", \"Blue - Nucleus\"]:\n            ch_arr[..., i] = arr[..., i]\n        else:\n            if rgb_only:\n                continue\n            ch_arr[..., 0] = arr[..., i]\n            ch_arr[..., 1] = arr[..., i]\n        plt.subplot(1,n_images,i+1)\n        plt.title(f\"{c.title()}\", fontweight=\"bold\")\n        plt.imshow(ch_arr)\n        plt.axis(False)\n        \n    if plot_merged:\n        plt.subplot(1,n_images,n_images)\n        \n        if rgb_only:\n            plt.title(f\"Merged RGB\", fontweight=\"bold\")\n            plt.imshow(arr)\n        else:\n            plt.title(f\"Merged RGBY into RGB\", fontweight=\"bold\")\n            plt.imshow(convert_rgby_to_rgb(arr))\n        plt.axis(False)\n        \n    plt.tight_layout(rect=[0, 0.2, 1, 0.97])\n    plt.show()\n    \n    \ndef flatten_list_of_lists(l_o_l):\n    return [item for sublist in l_o_l for item in sublist]\n\n\ndef create_input_list(crp, cgp, cbp, cyp, shuffle=True, val_split=0.025):\n    lbl_arr = flatten_list_of_lists([[k,]*len(v) for k, v in sorted(crp.items())])\n    cr_arr = flatten_list_of_lists([v for k,v in sorted(crp.items())])\n    cg_arr = flatten_list_of_lists([v for k,v in sorted(cgp.items())])\n    cb_arr = flatten_list_of_lists([v for k,v in sorted(cbp.items())])\n    cy_arr = flatten_list_of_lists([v for k,v in sorted(cyp.items())])\n    \n    if val_split is not None:\n        val_lbl_arr = lbl_arr[:int(len(lbl_arr)*val_split)]\n        lbl_arr = lbl_arr[int(len(lbl_arr)*val_split):]\n        \n        val_cr_arr = cr_arr[:int(len(cr_arr)*val_split)]\n        cr_arr = cr_arr[int(len(cr_arr)*val_split):]\n        \n        val_cg_arr = cg_arr[:int(len(cg_arr)*val_split)]\n        cg_arr = cg_arr[int(len(cg_arr)*val_split):]\n        \n        val_cb_arr = cb_arr[:int(len(cb_arr)*val_split)]\n        cb_arr = cb_arr[int(len(cb_arr)*val_split):]\n\n        val_cy_arr = cy_arr[:int(len(cy_arr)*val_split)]\n        cy_arr = cy_arr[int(len(cy_arr)*val_split):]\n        \n    if shuffle:\n        to_shuffle = list(zip(cr_arr, cg_arr, cb_arr, cy_arr, lbl_arr))\n        random.shuffle(to_shuffle)\n        cr_arr, cg_arr, cb_arr, cy_arr, lbl_arr = zip(*to_shuffle)\n        \n        if val_split is not None:\n            val_to_shuffle = list(zip(val_cr_arr, val_cg_arr, val_cb_arr, val_cy_arr, val_lbl_arr))\n            random.shuffle(val_to_shuffle)\n            val_cr_arr, val_cg_arr, val_cb_arr, val_cy_arr, val_lbl_arr = zip(*val_to_shuffle)\n    \n    if val_split is None:\n        return list(cr_arr), list(cg_arr), list(cb_arr), list(cy_arr), list(lbl_arr)\n    else:\n        return (list(cr_arr), list(cg_arr), list(cb_arr), list(cy_arr), list(lbl_arr)), \\\n               (list(val_cr_arr), list(val_cg_arr), list(val_cb_arr), list(val_cy_arr), list(val_lbl_arr))\n\n\ndef get_class_wts(single_ch_paths, n_classes=19, exclude_mitotic=True, multiplier=10, return_counts=False):\n    \"\"\" TBD \"\"\"\n    # Get class counts\n    class_counts = {c_idx:len(single_ch_paths[c_idx]) for c_idx in range(n_classes)}\n\n    # Exclude mitotic spindle\n    if exclude_mitotic:\n        real_min_count = list(sorted(class_counts.values(), reverse=True))[-2]\n    else:\n        real_min_count = list(sorted(class_counts.values(), reverse=True))[-1]\n\n    # Calculate weights\n    class_wts = {k:min(1, multiplier*(real_min_count/v)) for k,v in class_counts.items()}\n\n    if exclude_mitotic:\n        # Manually adjust mitotic spindle to a more appropriate value\n        class_wts[min(class_counts, key=class_counts.get)] = 1.0\n\n    if return_counts:\n        return class_wts, class_counts\n    else:\n        return class_wts","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"TILE_DIRS = [RED_TILE_DIR, GREEN_TILE_DIR, BLUE_TILE_DIR, YELLOW_TILE_DIR]\n# TP_ID_MAP = get_tp_id_map(PKL_DIR)\n\n# Define the paths to the training files for the tile dataset as a map from class index to list of paths\nRED_FILE_MAP, GREEN_FILE_MAP, BLUE_FILE_MAP, YELLOW_FILE_MAP = \\\n    get_color_path_maps(TILE_DIRS, None)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-01-31T11:46:01.474688Z","iopub.status.busy":"2021-01-31T11:46:01.47402Z","iopub.status.idle":"2021-01-31T11:46:01.584122Z","shell.execute_reply":"2021-01-31T11:46:01.583505Z"},"papermill":{"duration":0.153587,"end_time":"2021-01-31T11:46:01.584256","exception":false,"start_time":"2021-01-31T11:46:01.430669","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"VAL_FRAC = 0.075\n\n# red_inputs, green_inputs, blue_inputs, yellow_inputs, labels\ntrain_inputs, val_inputs = create_input_list(\n    RED_FILE_MAP, \n    GREEN_FILE_MAP, \n    BLUE_FILE_MAP, \n    YELLOW_FILE_MAP, \n    shuffle=True,\n    val_split=VAL_FRAC,\n)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-01-31T11:46:01.732027Z","iopub.status.busy":"2021-01-31T11:46:01.719926Z","iopub.status.idle":"2021-01-31T11:46:02.842637Z","shell.execute_reply":"2021-01-31T11:46:02.84207Z"},"papermill":{"duration":1.172573,"end_time":"2021-01-31T11:46:02.842773","exception":false,"start_time":"2021-01-31T11:46:01.6702","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"# class_wts, class_cnts = get_class_wts(RED_FILE_MAP, return_counts=True, multiplier=23.203)\nclass_wts, class_cnts = get_class_wts(RED_FILE_MAP, return_counts=True, multiplier=50)\nprint(\"\\n ... CLASSWISE COUNTS ... \\n\")\ndisplay(class_cnts)\n\nprint(\"\\n ... CLASS WEIGHTING ... \\n\")\ndisplay(class_wts)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"N_EPOCHS=10\nLR_START = 0.0005\nLR_MAX = 0.0011\nLR_MIN = 0.0005\nLR_RAMPUP_EPOCHS = 3\nLR_SUSTAIN_EPOCHS = 2\nLR_EXP_DECAY = 0.75\n\ndef lrfn(epoch):\n    if epoch < LR_RAMPUP_EPOCHS:\n        lr = (LR_MAX - LR_START) / LR_RAMPUP_EPOCHS * epoch + LR_START\n    elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:\n        lr = LR_MAX\n    else:\n        lr = (LR_MAX - LR_MIN) * LR_EXP_DECAY**(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS) + LR_MIN\n    return lr\n\n# VIEW SCHEDULE\nrng = [i for i in range(N_EPOCHS)]\ny = [lrfn(x) for x in rng]\n\nplt.figure(figsize=(10,4))\nplt.plot(rng, y)\nplt.title(\"CUSTOM LR SCHEDULE\", fontweight=\"bold\")\nplt.show()\n\nprint(f\"Learning rate schedule: {y[0]:.3g} to {max(y):.3g} to {y[-1]:.3g}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#PARAMS\nMODEL_CKPT_DIR = \"/kaggle/working/ebnet_b2_wdensehead\"\nDROP_YELLOW = True\nNO_NEG_CLASS = False\n\nif NO_NEG_CLASS:\n    class_wts = {k:v for k,v in class_wts.items() if k!=18}\n    class_cnts = {k:v for k,v in class_cnts.items() if k!=18}\n    n_classes = 18\nelse:\n    n_classes=19\n    \nBATCH_SIZE=32\nOPTIMIZER = tf.keras.optimizers.Adam(lr=LR_START)\nLOSS_FN = \"binary_crossentropy\"\nSHUFF_BUFF = 500\n\n\n# AUTO-CALCULATED\nN_EX = len(RED_FILE_MAP[0])\nN_VAL = int(VAL_FRAC*N_EX)\nN_TRAIN = N_EX-N_VAL\n\nif not os.path.isdir(MODEL_CKPT_DIR):\n    os.makedirs(MODEL_CKPT_DIR, exist_ok=True)\n    \nprint(f\"{N_TRAIN:<7} TRAINING EXAMPLES\")\nprint(f\"{N_VAL:<7} VALIDATION EXAMPLES\")","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}