{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Issue\n\nTunning this notebook, I got the following exception in Keras model evaluation method:\nOutOfRangeError: 7 root error(s) found.\n  (0) Out of range: {{function_node __inference_test_function_9112}} End of sequence\n\t [[{{node cond_9/else/_84/cond_9/IteratorGetNext}}]]\n\t [[strided_slice_33/_322]]\n  (1) Out of range: {{function_node __inference_test_function_9112}} End of sequence\n\t [[{{node cond_9/else/_84/cond_9/IteratorGetNext}}]]\n\t [[tpu_compile_succeeded_assert/_5221419028491475094/_5/_351]]\n  (2) Out of range: {{function_node __inference_test_function_9112}} End of sequence\n\t [[{{node cond_9/else/_84/cond_9/IteratorGetNext}}]]\n\t [[TPUReplicate/_compile/_12011285436106554079/_4/_300]]\n  (3) Out of range: {{function_node __inference_test_function_9112}} End of sequence\n\t [[{{node cond_9/else/_84/cond_9/IteratorGetNext}}]]\n\t [[ConstantFoldingCtrl/cond_2/switch_pred/_20_1/_83]]\n  (4) Out of range: {{function_node __inference_test_function_9112}} End of sequence\n\t [[{{node cond_9/else/_84/cond_9/IteratorGetNext}}]]\n  (5) Out of range: {{function_node __inference_test_function_9112}} End of sequence\n\t [[{{node cond_9/else/_84/cond_9/IteratorGetNext}}]]\n\t [[Shape_23/_220]]\n  (6) Out of range: {{function_node __inference_test_function_9112}} End of sequence\n\t [[{{node cond_9/else/_84/cond_9/IteratorGetNext}}]]\n\t [[GroupCrossDeviceControlEdges_1/Identity_7/_395]]\n0 successful operations.\n2 derived errors ignored.\n\nThis notebook is a subset of my defaut [working notebook](https://www.kaggle.com/yanngarcia/human-protein-atlas) which is alredy shared. I removed all the useless code from Machine Learning and all my blabla ;)\n\nI would like some help to understand what's going wrong.\n\nThanks a lot\n\n**NOTE: Please, feel free to correct and enhance this notebook ;)**"},{"metadata":{},"cell_type":"markdown","source":"Loading modules..."},{"metadata":{"trusted":true},"cell_type":"code","source":"from __future__ import division # Import floating-point division (1/4=0.25) instead of Euclidian division (1/4=0)\n\n# 1. Prepare Problem\n\n# a) Load libraries\nimport os, warnings, argparse, io, operator, requests, math, random, tempfile\nimport re # Regular expressions support\nfrom datetime import datetime # Date & Time support\n\nimport numpy as np # Linear algebra\nimport matplotlib.pyplot as plt # Data visualization\nimport seaborn as sns # Enhanced data visualization\n#import seaborn_image as isns # Enhanced image data visualization\nimport pandas as pd # Data processing, CSV file I/O (e.g. pd.read_csv)\n\nfrom pandas_profiling import ProfileReport\n\nimport sklearn\nfrom sklearn import model_selection\nfrom sklearn import linear_model # Regression\nfrom sklearn import discriminant_analysis\nfrom sklearn import neighbors # Clustering\nfrom sklearn import naive_bayes\nfrom sklearn import tree # Decisional tree learning\nfrom sklearn import svm # Support Vector Machines\nfrom sklearn import ensemble # Support RandomForestRegressor, GradientBoostingRegressor, ExtraTreesRegressor, AdaBoostRegressor\n\nimport xgboost as xgb # Gradient Boosted Decision Trees algorithm\n\nimport lightgbm as lgb # Light Gradient Boost model\n\nfrom sklearn.base import is_classifier, is_regressor # To check if the model is for regression or classification (do not work for Keras)\n\nfrom sklearn.impute import SimpleImputer \n\nfrom sklearn.preprocessing import LabelEncoder # Labelling categorical feature from 0 to N_class - 1('object' type)\nfrom sklearn.preprocessing import LabelBinarizer # Binary labelling of categorical feature\nfrom sklearn.preprocessing import OneHotEncoder\nfrom sklearn.preprocessing import MultiLabelBinarizer\n\nfrom sklearn.preprocessing import StandardScaler # Data normalization\nfrom sklearn.preprocessing import MinMaxScaler # Data normalization\nfrom sklearn.preprocessing import MaxAbsScaler # Data normalization\n\nfrom sklearn.pipeline import Pipeline\n\nfrom sklearn.metrics import classification_report\nfrom sklearn.metrics import confusion_matrix\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.metrics import r2_score\nfrom sklearn.metrics import f1_score\nfrom sklearn.metrics import precision_score\nfrom sklearn.metrics import recall_score\nfrom sklearn.metrics import mean_absolute_error\nfrom sklearn.metrics import mean_squared_error\n\nfrom sklearn.model_selection import GridSearchCV\n\nfrom sklearn.feature_selection import mutual_info_regression, mutual_info_classif # To build Mutual Information plots\n\nfrom sklearn.inspection import permutation_importance\n\nimport pickle # Use to save and load models\n\nimport eli5\nfrom eli5.sklearn import PermutationImportance\n\n# Neural Network\nimport tensorflow as tf\nfrom tensorflow.keras.callbacks import EarlyStopping\nfrom tensorflow.keras.callbacks import ModelCheckpoint\nimport keras\nfrom keras.wrappers.scikit_learn import KerasClassifier, KerasRegressor\nfrom keras.preprocessing.image import ImageDataGenerator, load_img, img_to_array, array_to_img\n\nfrom PIL import Image","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Description of the different projects of my playground"},{"metadata":{"trusted":true},"cell_type":"code","source":"\n# Human Protein Atlas - Single Cell Classification (https://www.kaggle.com/c/hpa-single-cell-image-classification)\n# Require ExecutionFlags.USE_CNN_NEURAL_NETWORK_FLAG\nML_NAME = 'HumanProteinAtlas' # https://www.kaggle.com/c/hpa-single-cell-image-classification\nDATABASE_NAME = 'hpa-single-cell-image-classification'\nCOLUMNS_LABEL = ['image_id', 'class']\nIMAGES_SAMPLES_NUM = 2048 # Total number of images to be used for all datasets. None: Use all images\nIMAGE_NUM_PIXELS = 512 # Image size in pixels\nIMAGE_SIZE = [IMAGE_NUM_PIXELS, IMAGE_NUM_PIXELS] # (Heigh,Width) image size in pixels\nIMAGE_CLASSES = [ # Classes name\n                    'Nucleoplasm', \n                    'Nuclear membrane', \n                    'Nucleoli', \n                    'Nucleoli fibrillar center', \n                    'Nuclear speckles', \n                    'Nuclear bodies', \n                    'Endoplasmic reticulum', \n                    'Golgi apparatus', \n                    'Intermediate filaments', \n                    'Actin filaments', \n                    'Microtubules', \n                    'Mitotic spindle', \n                    'Centrosome', \n                    'Plasma membrane', \n                    'Mitochondria', \n                    'Aggresome', \n                    'Cytosol', \n                    'Vesicles and punctate cytosolic patterns', \n                    'Negative'\n                ]\nTARGET_COLUMNS = None\nOUTPUT_IS_REGRESSION = False\nDATE_TIME_COLUMNS = None\nDATE_TIME_COLUMNS = None\nEXCLUDE_FROM_OULIERS = None\nNON_TRANSFORMABLE_COLUMNS = None\n# Channels:\n# RED: Microtubule channels\n# GREEN: Protein of interest\n# BLUE: Nuclei channels\n# YELLOW: Endoplasmic reticulum","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Pre and Post processings..."},{"metadata":{"trusted":true},"cell_type":"code","source":"def kaggle_pre_load_images_datasets(p_train_url: str, \n                                    p_labels: list = None,\n                                    p_global_path: str = None,\n                                    p_validation_url: str = None,\n                                    p_test_url: str = None,\n                                    p_train_size: float = 0.9\n                                    ) -> list:\n    \"\"\"\n    This function is called by kaggle_load_images_datasets() just before to start processing\n    The Images database shall be organized as follow:\n    1) In Tfrecord format\n        <current_folder>/tfrecords-jpeg-<n>x<n>/train\n        <current_folder>/tfrecords-jpeg-<n>x<n>/val\n        <current_folder>/tfrecords-jpeg-<n>x<n>/test\n    2) Otherwise:\n        <current_folder>/train_labels.csv\n        <current_folder>/train\n        [<current_folder>/train], could not exist\n        <current_folder>/test\n    :parameters p_labels: The label of the columns to be used. Default: None\n    :parameter p_train_size: Size ratio of Training dataset vs. Validation dataset\n    \"\"\"\n    print('----------------------------- kaggle_pre_load_images_datasets -----------------------------')\n    train_df = None\n    validation_df = None\n    test_df = None\n    \n    if not p_train_url is None: # Case of separated Images and labels\n        print('kaggle_pre_load_images_datasets: Processing %s' % ML_NAME)\n        path = p_global_path #os.path.join(p_global_path, p_train_url) # p_train_url = DATABASE_NAME\n        def _convert_multi_labels(p_labels: list, p_length: int, p_splitter: str = '|') -> list:\n            labels = []\n            for label in p_labels:\n                l = np.zeros(p_length).astype(dtype = np.uint8)\n                for item in label:\n                    if(item != p_splitter):\n                        l[int(item)] = int(1)\n                labels.append(list(l))\n            return labels\n            # End of function _convert_multi_labels\n\n        # Add a 'full path' feature with 4 color channels full path\n        p = os.path.join(path, 'train.csv')\n        train_df = pd.read_csv(p)\n        # Set labels\n        if not p_labels is None:\n            train_df.columns = p_labels\n        # Prepare Training, Validation and Test datasets\n        print('----------------------------- training dataset')\n        # Build full path images\n        p = os.path.join(path, 'train')\n        train_df['image_path'] = train_df['image_id'].apply(lambda x: p + '/%s' % x)\n        train_df.drop(['image_id'], inplace = True, axis = 1)\n        # Split labels into classes as descibed by IMAGE_CLASSES\n        l = _convert_multi_labels(train_df['class'], p_length = len(IMAGE_CLASSES))\n        classes = pd.DataFrame(l, columns = IMAGE_CLASSES)\n        train_df.drop(['class'], inplace = True, axis = 1)\n        train_df = pd.concat([train_df, classes], axis = 1)\n        # Read the test files and build the files list\n        print('----------------------------- kaggle_pre_load_images_datasets: test dataset')\n        if not p_test_url is None:\n            p = os.path.join(path, 'test')\n            test_files = tf.io.gfile.glob(os.path.join(p, '*'))\n            test_files = [(test_files[i].split('_')[0]) for i in range(0, len(test_files), 4)]\n            # Put them in dataset\n            test_df = pd.DataFrame(test_files, columns = ['image_path'])\n        else:\n            raise Exception('kaggle_pre_load_images_datasets', 'Test url not provided')\n    \n        if not p_validation_url is None:\n            p = os.path.join(path, p_validation_url)\n            if  ML_NAME == 'BMS-MolecularTranslation':\n                # Add a 'full path' feature\n                p = os.path.join(path, 'validation_labels.csv')\n                validation_df = pd.read_csv(p)\n                # Set labels\n                if not p_labels is None:\n                    validation_df.columns = p_labels\n                validation_df['full_path'] = validation_df['image_id'].apply(lambda x: p + '/%c/%c/%c/%s.png' % (x[0], x[1], x[2], x))\n            else:\n                raise Exception('kaggle_pre_load_images_datasets', 'To be implemented')\n        else: # Need to extract randomly Validation datatset from Training dataset\n            train_df, validation_df = model_selection.train_test_split(train_df, test_size = p_train_size)\n            # reindex after split\n            train_df.reset_index(inplace = True)\n            validation_df.reset_index(inplace = True)\n\n        #print('----------------------------- kaggle_pre_load_images_datasets: training dataset')\n        #print(train_df.head())\n        #print('----------------------------- kaggle_pre_load_images_datasets: validation dataset')\n        #print(validation_df.head())\n        #print('----------------------------- kaggle_pre_load_images_datasets: test dataset')\n        #print(test_df.head())\n    else:\n        pass\n\n    print('kaggle_pre_load_images_datasets: Done')\n    return train_df, validation_df, test_df","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Loading the Datasets/Images"},{"metadata":{"trusted":true},"cell_type":"code","source":"# b.2) Set some defaults\ndef kaggle_set_mp_default() -> None:\n    \"\"\"\n    Some default setting for Matplotlib plots\n    \"\"\"\n    warnings.filterwarnings(\"ignore\") # to clean up output cells\n    pd.set_option('precision', 3)\n    # Set Matplotlib defaults\n    plt.rc('figure', autolayout=True)\n    plt.rc('axes', labelweight='bold', labelsize='large', titleweight='bold', titlesize=18, titlepad=10)\n    plt.rc('image', cmap='magma')\n    # End of function set_mp_default\n\n# Fix random values for reproductibility\nSEED_HARCODED_VALUE = 42\n\ndef kaggle_set_seed(p_seed: int = SEED_HARCODED_VALUE) -> None:\n    \"\"\"\n    Random reproducability\n    :parameter p_seed: Set the seed value for random functions reproductibility\n    \"\"\"\n    seed = p_seed\n    if seed is None:\n        seed = random.randint(0, 10000) \n    np.random.seed(seed)\n    sklearn.utils.check_random_state(seed)\n    tf.random.set_seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    os.environ['TF_DETERMINISTIC_OPS'] = '1'\n    # End of function set_seed\n\ndef kaggle_modules_versions() -> None:\n    \"\"\"\n    Print the different modules version\n    \"\"\"\n    print('----------------------------- modules_versions -----------------------------')\n    print(\"Numpy version: \" + np.__version__)\n    print('seaborn: %s' % sns.__version__)\n    print(\"Pandas version: \" + pd.__version__)\n    print(\"Sklearn version: \" + sklearn.__version__)\n    print(\"Tensorflow version: \" + tf.__version__)\n    print('modules_versions: Done')\n    # End of function modules_versions","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"In case of working with Neural Networks, set some additional defaults and doing a TPU detection are required."},{"metadata":{"trusted":true},"cell_type":"code","source":"# b.4) Set additional defaults for Neural Networks\nDL_BATCH_SIZE = 32 # Default batch size for DL models \nDL_EPOCH_NUM = 32 # Default epoch number for DL models \nDL_LEARNING_RATE = 0.002 # Default learning rate for DL models \nDL_DROP_RATE = 0.2 # Default drop rate for DL models \nDL_INPUT_SHAPE = None # Default input shape size for DL models, will be used for cross_validation (see kaggle_check_models)\n\n# Doing TPU detection\ndef kaggle_tpu_detection():\n    \"\"\"\n    This function provides a TPU detection. If TPU is not supported, a default strategy is returned.\n    Note: This method also setup the global path to access datasets when TPU (from Kaggle environment) is detected.\n    :return: The appropriate distribution strategy and the global path to access datasets (None if no TPU support)\n    \"\"\"\n    print('----------------------------- kaggle_tpu_detection -----------------------------')\n    global_path = None\n    try:\n        tpu = tf.distribute.cluster_resolver.TPUClusterResolver() \n        print('kaggle_tpu_detection: Running on TPU ', tpu.master())\n    except ValueError:\n        tpu = None\n    if tpu:\n        print(\"kaggle_tpu_detection: List of devices: \", tf.config.list_logical_devices('TPU'))\n        tf.config.experimental_connect_to_cluster(tpu)\n        tf.tpu.experimental.initialize_tpu_system(tpu)\n        strategy = tf.distribute.experimental.TPUStrategy(tpu)\n        if strategy.num_replicas_in_sync > 1:\n            # Update paths accordingly\n            from kaggle_datasets import KaggleDatasets\n            global_path = KaggleDatasets().get_gcs_path(DATABASE_NAME)\n    else:\n        strategy = tf.distribute.get_strategy() \n    print('kaggle_tpu_detection: replica=%s' % str(strategy.num_replicas_in_sync))\n    print('kaggle_tpu_detection: global_path: ', global_path)\n    \n    print('kaggle_tpu_detection Done')\n    return strategy, global_path\n    # End of function kaggle_tpu_detection\n\n# Tensorflow specific helper functions\ndef kaggle_bytes_to_tfrecord(p_value):\n    \"\"\"\n    Returns a bytes_list from a string/byte\n    \"\"\"\n    if isinstance(p_value, type(tf.constant(0))):\n        p_value = p_value.numpy()\n    return tf.train.Feature(bytes_list = tf.train.BytesList(value = [p_value]))\n    # End of function kaggle_bytes_to_tfrecord\n\ndef kaggle_floats_to_tfrecord(p_value):\n    \"\"\"\n    Returns a float_list from a float/double\n    \"\"\"\n    return tf.train.Feature(float_list = tf.train.FloatList(value = [p_value]))\n    # End of function kaggle_floats_to_tfrecord\n\ndef kaggle_ints_to_tfrecord(p_value):\n    \"\"\"\n    Returns an int64_list from a bool/enum/int/uint\n    \"\"\"\n    return tf.train.Feature(int64_list = tf.train.Int64List(value = [p_value]))\n    # End of function kaggle_ints_to_tfrecord\n\ndef kaggle_dataset_size(p_filenames: list) -> int:\n    \"\"\"\n    The number of data items in the dataset is written in the name of the .tfrec files (i.g. features00-230.tfrec = 230 data items in the file features00-230.tfrec)\n    \"\"\"\n    print('----------------------------- kaggle_dataset_size -----------------------------')\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in p_filenames]\n    print('kaggle_dataset_size: n=', n)\n    print('kaggle_dataset_size: np.sum(n)=', np.sum(n))\n    return np.sum(n)\n    # End of function kaggle_dataset_size\n\ndef kaggle_decode_png_image(p_image_data, p_channels: int = 3):\n    \"\"\"\n    This function decodes the PNG image and applies required Tensorflow adjustment\n    Note: The image was stored in a Tfrecord dataset\n    :parameter p_image_data: \n    :parameter p_channels: \n    :return: The decoded image\n    \"\"\"\n    image = tf.image.decode_png(p_image_data, channels = p_channels)\n    # Convert image to floats in [0, 1] range\n    image = tf.cast(image, tf.float32) / 255.0\n    # Explicit size needed for TPU\n    image = tf.reshape(image, [*IMAGE_SIZE, p_channels])\n    return image\n    # End of function kaggle_decode_png_image\n\ndef kaggle_decode_jpeg_image(p_image_data, p_channels: int = 3):\n    \"\"\"\n    This function decodes the JPEG image and applies required Tensorflow adjustment\n    Note: The image was stored in a Tfrecord dataset\n    :parameter p_image_data: \n    :parameter p_channels: \n    :return: The decoded image\n    \"\"\"\n    image = tf.image.decode_jpeg(p_image_data, channels = p_channels)\n    # Convert image to floats in [0, 1] range\n    image = tf.cast(image, tf.float32) / 255.0\n    # Explicit size needed for TPU\n    image = tf.reshape(image, [*IMAGE_SIZE, p_channels])\n    return image\n    # End of function kaggle_decode_jpeg_image\n\n# Tensorflow Protocol buffer for unlabeled images stroed into a Tfrecord dataset.\n# Labels are replaced by identifiers\nunlabeled_image_feature_description = {\n    'image': tf.io.FixedLenFeature([], tf.string), # tf.string means bytestring\n    'id': tf.io.FixedLenFeature([], tf.string),  # shape [] means single element\n}\ndef kaggle_read_unlabeled_tfrecord(p_data):\n    data = tf.io.parse_single_example(p_data, unlabeled_image_feature_description)\n    image = kaggle_decode_jpeg_image(data['image'])\n    idnum = data['id']\n    return image, idnum # returns a dataset of image(s)\n    # End of function kaggle_read_unlabeled_tfrecord\n\n# Tensorflow Protocol buffer for labeled images stroed into a Tfrecord dataset\nlabeled_image_feature_description = {\n    'image': tf.io.FixedLenFeature([], tf.string), # tf.string means bytestring\n    'class': tf.io.FixedLenFeature([], tf.int64),  # shape [] means single element\n}\ndef kaggle_read_labeled_tfrecord(p_data):\n    data = tf.io.parse_single_example(p_data, labeled_image_feature_description)\n    image = kaggle_decode_jpeg_image(data['image'])\n    label = tf.cast(data['class'], tf.int32)\n    return image, label # returns a dataset of (image, label) pairs\n    # End of function kaggle_read_labeled_tfrecord","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Loads images dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"# c.2) Load 'Images' dataset\ndef kaggle_load_images_datasets(p_train_url: str, \n                                p_labels: list = None,\n                                p_global_path: str = None,\n                                p_validation_url: str = None,\n                                p_test_url: str = None, \n                                p_train_path: str = None, \n                                p_validation_path: str = None,\n                                p_test_path: str = None,\n                                p_train_size: float = 0.9,\n                                p_ordered: bool = False,\n                                p_seed: int = SEED_HARCODED_VALUE\n                                ) -> list:\n    \"\"\"\n    This function loads Tensorflow Tfrecord datasets.\n    \n    :parameters p_labels: The label of the columns to be used. Default: None\n    \n    \n    :parameter p_train_path: Path of the Training 'Tfrec' folder  \n    :parameter p_validation_path: Path of the Validation 'Tfrec' folder\n    :parameter p_test_path: Path of the Test 'Tfrec' folder\n    :parameter p_train_size: Size ratio of Training dataset vs. Validation dataset\n    :parameter p_seed: The seed value for reproductibility\n    :return: Four datasets: The Training, Test and Validation datasets. The Test dataset does no contain the outputs, it acts as unseen data for the model. This is the fourth dataset returned\n    \"\"\"\n    print('----------------------------- kaggle_load_images_datasets -----------------------------')\n\n    train_df, validation_df, test_df = kaggle_pre_load_images_datasets(p_train_url = p_train_url, \n                                                                       p_labels = p_labels, \n                                                                       p_global_path = p_global_path,\n                                                                       p_validation_url = p_validation_url, \n                                                                       p_test_url = p_test_url,\n                                                                       p_train_size = p_train_size\n                                                                       )\n    #print('----------------------------- kaggle_load_images_datasets: training dataset')\n    #print(train_df.head())\n    #print('----------------------------- kaggle_load_images_datasets: validation dataset')\n    #print(validation_df.head())\n    #print('----------------------------- kaggle_load_images_datasets: test dataset')\n    #print(test_df.head())\n\n    train_df_num_images = None\n    validation_df_num_images = None\n    test_df_num_images = None\n    if not train_df is None:\n        print('kaggle_load_images_datasets: Processing compete %s:' % ML_NAME)\n        if ML_NAME == 'HumanProteinAtlas':\n            def _rebuild_image_from_channels(p_image_path: str, p_label: list = None, p_seed: int = SEED_HARCODED_VALUE) -> list:\n                \"\"\"\n                This function convert image channels into one RGB image\n                \"\"\"\n                red = tf.io.read_file(p_image_path + \"_red.png\")\n                blue = tf.io.read_file(p_image_path + \"_blue.png\")\n                green = tf.io.read_file(p_image_path + \"_green.png\")\n                yellow = tf.io.read_file(p_image_path + \"_yellow.png\")\n\n                red = tf.io.decode_png(red, channels = 1) # Grayscale image\n                blue = tf.io.decode_png(blue, channels = 1) # Grayscale image\n                green = tf.io.decode_png(green, channels = 1) # Grayscale image\n                yellow = tf.io.decode_png(yellow, channels = 1) # Grayscale image\n                \n                red = tf.math.maximum(red, yellow)\n                blue = tf.math.maximum(blue, yellow)\n\n                # Convert image to floats in [0, 1] range\n                red = tf.cast(red, tf.float32) / 255.0\n                blue = tf.cast(blue, tf.float32) / 255.0\n                green = tf.cast(green, tf.float32) / 255.0\n                \n                # Explicit size needed for TPU\n                red = tf.image.resize(red, [*IMAGE_SIZE])\n                blue = tf.image.resize(blue, [*IMAGE_SIZE])\n                green = tf.image.resize(green, [*IMAGE_SIZE])\n\n                red = tf.squeeze(red)\n                blue = tf.squeeze(blue)\n                green = tf.squeeze(green)\n                \n                # Build RGB image, channels last: shape is (width, high, channel)\n                # In this case, shape is (1024, 1024, 3):\n                #     1024 entries of (1024 lines per 3 columns)\n                #         First entry, first line:   [ R[0,0], G[0,0], B[0,0] ]\n                #         First entry, second line:  [ R[0,1], G[0,1], B[0,1] ]\n                #         ...\n                #         Second entry, first line:  [ R[1,0], G[1,0], B[1,0] ]\n                #         ....\n                # Stack structure is: each entry of the stack is the tuple (R, G, B) of  \n                # Axis value: 2 for channels last, -1 for channel first\n                # From stack, to extract one channel (R, G or B): stack[:, :, n], n = 0 for R, 1, for G and 2 for B\n                image = tf.stack([red, green, blue], axis = 2) # RGB channels last\n\n                return image, p_label\n                # End of _rebuild_image_from_channels\n\n            # Sample elements from Tests and Validation dataset to reduce execution time\n            if not IMAGES_SAMPLES_NUM is None: # Use a subset of images from the training Dataset\n                # Shuffle row from the Training dataset\n                train_df = train_df.sample(IMAGES_SAMPLES_NUM, random_state = p_seed)\n                train_df.reset_index(inplace = True);\n                # Shuffle row from the Validation dataset\n                validation_df = validation_df.sample(int((1 - p_train_size) * IMAGES_SAMPLES_NUM // p_train_size), random_state = p_seed)\n                validation_df.reset_index(inplace = True);\n            else: # Use all images from the training Dataset\n                # Nothing to do\n                pass\n            # Set sizes\n            train_df_num_images = train_df.shape[0]\n            validation_df_num_images = validation_df.shape[0]\n            test_df_num_images = test_df.shape[0]\n            # Build Tensoflow dataset for Training\n            image_path = train_df['image_path'] # List of image paths as pd.Series\n            labels = train_df[IMAGE_CLASSES] # List of labels as pd.DataFrame\n            print('Training image_path shape ==> ', image_path.shape)\n            print('Training labels shape ==> ', labels.shape)\n            # Conver Series/DataFrame into np.darray before to convert them into tensors\n            image_path = np.array(image_path.values.tolist())\n            #print(type(image_path))\n            #print(image_path)\n            labels = np.array(labels.values.tolist())\n            #print(type(labels))\n            #print(labels)\n            # Testing _rebuild_image_from_channels function\n            #i, t = _rebuild_image_from_channels(image_path[0], labels[0])\n            #print(i.shape)\n            #print(i)\n            #print(t)\n            #raise Exception('Stop')\n            # Convert them into tensors\n            train_df = tf.data.Dataset.from_tensor_slices((image_path, labels))\n            # Shuffle the training dataset\n#            train_df = train_df.shuffle(len(image_path) + 1)  # buffer_size >= dataset length\n            train_df = train_df.map(_rebuild_image_from_channels, num_parallel_calls = tf.data.AUTOTUNE)\n            # Inspect Training dataset\n            for image, label in train_df.take(3):\n                print('kaggle_load_images_datasets: Training data label inspection:', label.numpy())\n                print(image.numpy().shape, len(image.numpy()), label.numpy().shape)\n            # Build Tensoflow dataset for Validation\n            image_path = validation_df['image_path'] # List of image paths as pd.Series\n            labels = validation_df[IMAGE_CLASSES] # List of labels as pd.DataFrame\n            print('Validation image_path shape ==> ', image_path.shape)\n            print('Validation labels shape == ', labels.shape)\n            # Conver Series/DataFrame into np.darray before to convert them into tensors\n            image_path = np.array(image_path.values.tolist())\n            #print(type(image_path))\n            #print(image_path)\n            labels = np.array(labels.values.tolist())\n            #print(type(labels))\n            #print(labels)\n            # Convert them into tensors\n            validation_df = tf.data.Dataset.from_tensor_slices((image_path, labels))\n            # Shuffle the validation dataset\n#            validation_df = validation_df.shuffle(len(image_path) + 1)  # buffer_size >= dataset length\n            validation_df = validation_df.map(_rebuild_image_from_channels, num_parallel_calls = tf.data.AUTOTUNE)\n            # Inspect Validation dataset\n            for image, label in validation_df.take(3):\n                print('kaggle_load_images_datasets: Validation data label inspection:', label.numpy())\n                print(image.numpy().shape, len(image.numpy()), label.numpy().shape)\n            # Build Tensoflow dataset for Validation\n            image_path = test_df['image_path'] # List of image paths\n            # Conver Series/DataFrame into np.darray before to convert them into tensors\n            image_path = np.array(image_path.values.tolist())\n            #print('type(image_path) ==> ', type(image_path))\n            #print(image_path)\n            test_df = tf.data.Dataset.from_tensor_slices(image_path)\n            # Shuffle the test dataset\n#            test_df = test_df.shuffle(len(image_path) + 1)  # buffer_size >= dataset length\n            test_df = test_df.map(_rebuild_image_from_channels, num_parallel_calls = tf.data.AUTOTUNE)\n            # Inspect Validation dataset\n            for image, label in validation_df.take(3):\n                print('kaggle_load_images_datasets: Test data inspection:')\n                print(image.numpy().shape, len(image.numpy()))\n        #elif ML_NAME == 'BMS-MolecularTranslation':\n        #    pass\n        else:\n            raise Exception('kaggle_load_images_datasets', 'Unsupported ML_NAME: %s' % ML_NAME)\n    elif ML_NAME == 'FlowerClassification':\n        print('kaggle_load_images_datasets: Processing compete %s:' % ML_NAME)\n        # Load file names for Train, Validation and Test images folders\n        train_files = tf.io.gfile.glob(os.path.join(p_global_path, p_train_path))\n        # Sample elements from Tests and Validation dataset to reduce execution time\n        # FIXME How to sample Tensorflow datasets?\n        #if not IMAGES_SAMPLES_NUM is None: # Use a subset of images from the training Dataset\n        #    # Shuffle row from the Training dataset\n        #    train_files = random.sample(train_files, IMAGES_SAMPLES_NUM)\n        #    train_files.reset_index(inplace = True);\n        #else: # Use all images from the training Dataset\n        #    # Nothing to do\n        #    pass\n        # Disabling order increases speed\n        ignore_order = tf.data.Options()\n        if not p_ordered:\n            ignore_order.experimental_deterministic = False # disable order, increase speed\n        # Build the dataset with the images\n        train_df = tf.data.TFRecordDataset(train_files, num_parallel_reads = tf.data.experimental.AUTOTUNE)\n        # To use data as soon as it streams in\n        train_df = train_df.with_options(ignore_order)\n        # Decode tfrecord images into (jpeg, label)\n        train_df = train_df.map(kaggle_read_labeled_tfrecord, num_parallel_calls = tf.data.experimental.AUTOTUNE)\n        print('kaggle_load_images_datasets: Training data shapes:', train_df.cardinality().numpy())\n        # Inspect Training dataset\n        for image, label in train_df.take(3):\n            print('kaggle_load_images_datasets: Training data label inspection:', label.numpy())\n            print(image.numpy().shape, len(image.numpy()), label.numpy().shape)\n        train_df_num_images = kaggle_dataset_size(train_files)\n        if not p_validation_path is None:\n            validation_files = tf.io.gfile.glob(os.path.join(p_global_path, p_validation_path))\n            # Sample elements from Tests and Validation dataset to reduce execution time\n            # FIXME How to sample Tensorflow datasets?\n            #if not IMAGES_SAMPLES_NUM is None: # Use a subset of images from the training Dataset\n            #    # Shuffle row from the Validation dataset\n            #    validation_df = random.sample(validation_df, int((1 - p_train_size) * IMAGES_SAMPLES_NUM // p_train_size))\n            #    validation_df.reset_index(inplace = True);\n            #else: # Use all images from the validation Dataset\n            #    # Nothing to do\n            #    pass\n            # Build the dataset with the images\n            validation_df = tf.data.TFRecordDataset(validation_files, num_parallel_reads = tf.data.experimental.AUTOTUNE)\n            # To use data as soon as it streams in\n            validation_df = validation_df.with_options(ignore_order)\n            # Decode tfrecord images into (jpeg, label)\n            validation_df = validation_df.map(kaggle_read_labeled_tfrecord, num_parallel_calls = tf.data.experimental.AUTOTUNE)\n            print('kaggle_load_images_datasets: Validation data shapes:', validation_df.cardinality().numpy())\n            # Inspect Validation dataset\n            for image, label in validation_df.take(3):\n                print('kaggle_load_images_datasets: Validation data label inspection:', label.numpy())\n                print(image.numpy().shape, len(image.numpy()), label.numpy().shape)\n            validation_df_num_images = kaggle_dataset_size(validation_files)\n        if not p_test_path is None:\n            test_files = tf.io.gfile.glob(os.path.join(p_global_path, p_test_path))\n            print('----------------------------- kaggle_load_images_datasets: test_files')\n            # Build the dataset with the images\n            test_df = tf.data.TFRecordDataset(test_files, num_parallel_reads = tf.data.experimental.AUTOTUNE)\n            # To use data as soon as it streams in\n            test_df = test_df.with_options(ignore_order)\n            # Decode tfrecord images into (jpeg, label)\n            test_df = test_df.map(kaggle_read_unlabeled_tfrecord, num_parallel_calls = tf.data.experimental.AUTOTUNE)\n            # Inspect Test dataset\n            for image in test_df.take(3):\n                print('kaggle_load_images_datasets: Test data inspection:')\n                print(image.numpy().shape, len(image.numpy()))\n            print('kaggle_load_images_datasets: Test data shapes:', test_df.cardinality().numpy())\n            test_df_num_images = kaggle_dataset_size(test_files)\n\n    print('kaggle_load_images_datasets: Done: %s' % (p_train_url if not p_train_url is None else p_train_path))\n    return train_df, validation_df, test_df, train_df_num_images, validation_df_num_images, test_df_num_images\n    # End of function kaggle_load_images_datasets","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"Image visualization..."},{"metadata":{"trusted":true},"cell_type":"code","source":"# b.2) Images visualization\ndef kaggle_image_visualization(p_dataset, p_num_images: int = 10, p_prediction = None):\n    \"\"\"\n    This method show a number of images from p_dataset\n    :parameter p_dataset: The dataset containing the images to show. Note the dataset is usually already 'batch'\n    :parameter p_num_images: The number of images to show\n    :parameter p_prediction: \n    \"\"\"\n    print('----------------------------- kaggle_image_visualization -----------------------------')\n    # Peek some data from the dataset\n    it = iter(p_dataset.unbatch().batch(p_num_images))\n    batch = next(it)\n    kaggle_display_batch(batch, p_prediction)\n    # End of function kaggle_image_visualization\n\ndef kaggle_batch_to_numpy(p_dataset):\n    print('----------------------------- kaggle_batch_to_numpy -----------------------------')\n    images, labels = p_dataset\n    numpy_images = images.numpy()\n    numpy_labels = labels.numpy()\n    if numpy_labels.dtype == object: # binary string in this case,\n                                     # these are image ID strings\n        numpy_labels = [None for _ in enumerate(numpy_images)]\n    # If no labels, only image IDs, return None for labels (this is\n    # the case for test data)\n    return numpy_images, numpy_labels\n    # End of function kaggle_batch_to_numpy\n\ndef kaggle_title_from_label(p_label: str, p_correct_label: str):\n    print('----------------------------- kaggle_title_from_label -----------------------------')\n    if p_correct_label is None:\n        return IMAGE_CLASSES[p_label], True\n    correct = (p_label == p_correct_label)\n    return \"{} [{}{}{}]\".format(IMAGE_CLASSES[p_label], 'OK' if correct else 'NO', u\"\\u2192\" if not correct else '',\n                                IMAGE_CLASSES[p_correct_label] if not correct else ''), correct\n    # End of function kaggle_title_from_label\n\ndef kaggle_display_batch_image(p_image, p_title:str, subplot, red = False, titlesize = 16, p_cmap = 'viridis'):\n    plt.subplot(*subplot)\n    plt.axis('off')\n    plt.imshow(p_image, cmap = p_cmap)\n    if len(p_title) > 0:\n        plt.title(p_title, fontsize=int(titlesize) if not red else int(titlesize/1.2), color='red' if red else 'black', fontdict={'verticalalignment':'center'}, pad=int(titlesize/1.5))\n    return (subplot[0], subplot[1], subplot[2]+1)\n    # End of function kaggle_display_batch_image\n    \ndef kaggle_display_batch(p_databatch, p_predictions = None):\n    \"\"\"\n    \"\"\"\n    print('----------------------------- kaggle_display_batch -----------------------------')\n    # Images\n    images, labels = kaggle_batch_to_numpy(p_databatch)\n    if labels is None: # Fill with None\n        labels = [None for _ in enumerate(images)] \n    # Auto-squaring: this will drop data that does not fit into square or square-ish rectangle\n    rows = int(math.sqrt(len(images)))\n    cols = len(images)//rows\n    # Sizing and spacing\n    FIGSIZE = 13.0\n    SPACING = 0.1\n    subplot=(rows, cols, 1)\n    if rows < cols:\n        plt.figure(figsize=(FIGSIZE,FIGSIZE / cols * rows))\n    else:\n        plt.figure(figsize=(FIGSIZE / rows * cols,FIGSIZE))\n    # Display\n    for i, (image, label) in enumerate(zip(images[:rows * cols], labels[:rows * cols])):\n        title = 'No label' if label is None or type(label) != 'int' else IMAGE_CLASSES[label]\n        correct = True\n        if not p_predictions is None:\n            title, correct = kaggle_title_from_label(p_predictions[i], label)\n        dynamic_titlesize = FIGSIZE * SPACING / max(rows, cols) * 40 + 3 # magic formula tested to work from 1x1 to 10x10 images\n        subplot = kaggle_display_batch_image(image, title, subplot, not correct, titlesize=dynamic_titlesize)\n    # Layout\n    plt.tight_layout()\n    if label is None and p_predictions is None:\n        plt.subplots_adjust(wspace=0, hspace=0)\n    else:\n        plt.subplots_adjust(wspace=SPACING, hspace=SPACING)\n    plt.show()\n    # End of function kaggle_display_batch\n\ndef kaggle_display_image_and_component(p_image, p_title:str = None) -> None:\n    \"\"\"\n    This function displays an image and its RGB components separatly\n    :parameter p_image: The image to display (RGB format)\n    :parameter p_title: The title of the display\n    \"\"\"\n    # Extract RGB components\n    pixels = img_to_array(p_image)\n    red = pixels[:, :, 0]\n    green = pixels[:, :, 1]\n    blue = pixels[:, :, 2]\n    # Sizing and spacing\n    FIGSIZE = 13.0\n    SPACING = 0.1\n    rows = 1\n    cols = 4\n    subplot=(rows, cols, 1)\n    if rows < cols:\n        plt.figure(figsize=(FIGSIZE,FIGSIZE / cols * rows))\n    else:\n        plt.figure(figsize=(FIGSIZE / rows * cols,FIGSIZE))\n    # Display image and its components\n    images = (pixels, red, green, blue)\n    titles = (p_title, 'red component', 'green component', 'blue component')\n    for i, (image, title) in enumerate(zip(images, titles)):\n        dynamic_titlesize = FIGSIZE * SPACING / max(rows, cols) * 40 + 3 # magic formula tested to work from 1x1 to 10x10 images\n        subplot = kaggle_display_batch_image(image, title, subplot, True, titlesize = dynamic_titlesize, p_cmap = 'viridis' if i == 0 else 'gray')    \n    # Layout\n    plt.tight_layout()\n    plt.subplots_adjust(wspace=0, hspace=0)\n    plt.show()\n    # End of function kaggle_display_image_and_component\n\ndef kaggle_display_confusion_matrix(cmat, score, precision, recall) -> None:\n    \"\"\"\n    \"\"\"\n    print('----------------------------- kaggle_display_confusion_matrix -----------------------------')\n    plt.figure(figsize=(15,15))\n    ax = plt.gca()\n    ax.matshow(cmat, cmap='Reds')\n    ax.set_xticks(range(len(IMAGE_CLASSES)))\n    ax.set_xticklabels(IMAGE_CLASSES, fontdict={'fontsize': 7})\n    plt.setp(ax.get_xticklabels(), rotation=45, ha=\"left\", rotation_mode=\"anchor\")\n    ax.set_yticks(range(len(IMAGE_CLASSES)))\n    ax.set_yticklabels(IMAGE_CLASSES, fontdict={'fontsize': 7})\n    plt.setp(ax.get_yticklabels(), rotation=45, ha=\"right\", rotation_mode=\"anchor\")\n    titlestring = \"\"\n    if score is not None:\n        titlestring += 'f1 = {:.3f} '.format(score)\n    if precision is not None:\n        titlestring += '\\nprecision = {:.3f} '.format(precision)\n    if recall is not None:\n        titlestring += '\\nrecall = {:.3f} '.format(recall)\n    if len(titlestring) > 0:\n        ax.text(101, 1, titlestring, fontdict={'fontsize': 18, 'horizontalalignment':'right', 'verticalalignment':'top', 'color':'#804040'})\n    plt.show()\n    print('kaggle_display_confusion_matrix: Done')\n    # End of function kaggle_display_confusion_matrix\n\ndef kaggle_display_learning_curves(training, validation, title, subplot) -> None:\n    print('----------------------------- kaggle_display_learning_curves -----------------------------')\n    if subplot % 10 == 1: # set up the subplots on the first call\n        plt.subplots(figsize = (10,10), facecolor = '#F0F0F0')\n        plt.tight_layout()\n    ax = plt.subplot(subplot)\n    ax.set_facecolor('#F8F8F8')\n    ax.plot(training)\n    ax.plot(validation)\n    ax.set_title('model '+ title)\n    ax.set_ylabel(title)\n    #ax.set_ylim(0.28,1.05)\n    ax.set_xlabel('epoch')\n    ax.legend(['train', 'valid.'])\n    plt.show()\n    print('kaggle_display_learning_curves: Done')\n    # End of function kaggle_display_learning_curves","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Callbacks to create DL models and functions to visualize learning curves or learning rate curves"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Helper functions for kaggle_dl_quick_and_dirty()\ndef kaggle_build_pretrained_model(\n                                  p_pretrained_layers:list, \n                                  p_image_size: list,\n                                  p_channels: int = 3\n                                  ) -> tf.keras.Sequential:\n    \"\"\"\n    This function builds the pretrained models stacks for the Neural Network\n    :parameter p_pretrained_layers: The pretrained models to be used\n    :parameter p_image_size: The image size\n    :parameter p_channels: The number of channels (e.g. 1 for grayscale, 3 for RGB)\n    :return: The pretrained models stacks to build the final Neural Network\n    \"\"\"\n    model = tf.keras.Sequential();\n    for p in p_pretrained_layers:\n        if p == 'DenseNet201': \n            pretrained_model = tf.keras.applications.DenseNet201(\n                weights = 'imagenet',\n                include_top = False,\n                pooling = 'max',\n                input_shape = [*p_image_size, p_channels]\n            )\n            pretrained_model.trainable = False\n            model.add(pretrained_model)\n            model.add(GlobalAveragePooling2D())\n        elif p == 'InceptionV3': \n            # See https://github.com/fchollet/deep-learning-models/blob/ccd0eb24996b4cbff4231b90cd44b057c0b20f14/inception_v3.py\n            pretrained_model = tf.keras.applications.InceptionV3(\n                weights = 'imagenet',\n                include_top = False,\n                pooling = 'max',\n                input_shape = [*p_image_size, p_channels]\n            )\n            pretrained_model.trainable = False\n            model.add(pretrained_model)\n        elif p == 'MobileNetV2':\n            pretrained_model = tf.keras.applications.MobileNetV2(\n                weights = 'imagenet',\n                include_top = False,\n                pooling = 'max',\n                input_shape = [*p_image_size, p_channels]\n            )\n            pretrained_model.trainable = False\n            model.add(pretrained_model)\n        elif p == 'ResNet50':\n            # See https://github.com/fchollet/deep-learning-models/blob/ccd0eb24996b4cbff4231b90cd44b057c0b20f14/resnet50.py\n            pretrained_model = tf.keras.applications.ResNet50(\n                weights = 'imagenet',\n                include_top = False ,\n                input_shape = [*p_image_size, p_channels]\n            )\n            pretrained_model.trainable = False\n            model.add(pretrained_model)\n        elif p == 'VGG16': \n            # See https://github.com/fchollet/deep-learning-models/blob/master/vgg16.py\n            pretrained_model = tf.keras.applications.VGG16(\n                weights = 'imagenet',\n                include_top = False ,\n                input_shape = [*p_image_size, p_channels]\n            )\n            pretrained_model.trainable = False\n            model.add(pretrained_model)\n        elif p == 'VGG19':\n            # See https://github.com/fchollet/deep-learning-models/blob/master/vgg19.py\n            pretrained_model = tf.keras.applications.VGG19(\n                weights = 'imagenet',\n                include_top = False,\n                pooling = 'max',\n                input_shape = [*p_image_size, p_channels]\n            )\n            pretrained_model.trainable = False\n            model.add(pretrained_model)\n        else:\n            raise Exception('kaggle_build_pretrained_model', 'Undefined pretrained model')\n        # End of 'for' statement\n    return model\n    # End of function kaggle_build_pretrained_model\n\ndef kaggle_create_sequential_classifier_model(\n                                              #p_strategy\n                                              #p_input_shape: int = DL_INPUT_SHAPE,\n                                              #p_drop_rate: float = DL_DROP_RATE,\n                                              p_optimizer: str = 'adam', \n                                              p_loss: str = 'binary_crossentropy', \n                                              p_metrics: list = ['AUC', 'accuracy'],\n                                              p_pretrained_layers:list = None,\n                                              p_image_size: list = None,\n                                              p_class_num: int = 1\n                                             ) -> tf.keras.Sequential:\n    \"\"\"\n    Build a Neural network model for classification\n    \"\"\"\n    print('----------------------------- kaggle_create_sequential_classifier_model -----------------------------')\n    global DL_INPUT_SHAPE, DL_DROP_RATE\n\n    print('kaggle_create_sequential_classifier_model: p_class_num=%d' % p_class_num)\n    model = None\n    if p_pretrained_layers is None:\n        model = tf.keras.Sequential([\n                                    tf.keras.layers.BatchNormalization(input_shape = DL_INPUT_SHAPE),\n                                    tf.keras.layers.Dense(32, activation = 'relu'),\n                                    tf.keras.layers.Dropout(rate = DL_DROP_RATE),\n                                    tf.keras.layers.BatchNormalization(),\n                                    tf.keras.layers.Dense(64, activation = 'relu'),\n                                    tf.keras.layers.BatchNormalization(),\n                                    tf.keras.layers.Dropout(rate = DL_DROP_RATE),\n                                    tf.keras.layers.Dense(p_class_num, activation = 'sigmoid'), # Binary classes\n                                    ])\n    else:\n        model = kaggle_build_pretrained_model(p_pretrained_layers, p_image_size)\n        model.add(tf.keras.layers.Dense(p_class_num, activation = 'softmax')) # Multi classes\n\n    model.compile(optimizer=p_optimizer, loss = p_loss, metrics = p_metrics)\n    # End of 'with' statement\n\n    print('kaggle_create_sequential_classifier_model: Model summary:')\n    model.summary()\n    #plot_model(model, to_file='model_plot.png', show_shapes=True, show_layer_names=True)\n    return model\n    # End of function kaggle_create_sequential_classifier_model\n\ndef kaggle_create_sequential_regressor_model(\n                                             p_optimizer:str = 'adam', \n                                             p_loss:str = 'mae', \n                                             p_metrics:list = ['mae'],\n                                             p_pretrained_layers: list = None,\n                                             p_image_size: list = None,\n                                             p_class_num: int = 1\n                                            ) -> tf.keras.Sequential:\n    \"\"\"\n    Build a Neural network model for regression\n    \"\"\"\n    print('----------------------------- kaggle_create_sequential_regressor_model -----------------------------')\n    global DL_INPUT_SHAPE, DL_DROP_RATE\n\n    model = None\n    if p_pretrained_layers is None:\n        model = tf.keras.Sequential([\n                                    tf.keras.layers.BatchNormalization(input_shape = DL_INPUT_SHAPE),\n                                    tf.keras.layers.Dense(32, activation = 'relu'),\n                                    tf.keras.layers.Dropout(rate = DL_DROP_RATE),\n                                    tf.keras.layers.BatchNormalization(),\n                                    tf.keras.layers.Dense(64, activation = 'relu'),\n                                    tf.keras.layers.BatchNormalization(),\n                                    tf.keras.layers.Dropout(rate = DL_DROP_RATE),\n                                    tf.keras.layers.Dense(p_class_num, activation = 'relu'),\n                                    ])\n    else:\n        model = kaggle_build_pretrained_model(p_pretrained_layers, p_image_size);\n        model.add(tf.keras.layers.Dense(p_class_num, activation='relu'))\n\n    model.compile(optimizer=p_optimizer, loss = p_loss, metrics = p_metrics)\n\n    print('kaggle_create_sequential_regressor_model: Model summary:')\n    model.summary()\n    return model\n    # End of function kaggle_create_sequential_regressor_model\n\ndef kaggle_build_learning_rate_schedule(epoch,\n                                        start_lr = 0.0001, \n                                        min_lr = 0.0001, \n                                        max_lr = 0.0005,\n                                        rampup_epochs = 5, \n                                        sustain_epochs = 0,\n                                        exp_decay = 0.8\n                                       ):\n    \"\"\"\n    \"\"\"\n    def _exponential_lr(epoch, start_lr, min_lr, max_lr, rampup_epochs, sustain_epochs, exp_decay):\n        \"\"\"\n        \"\"\"\n        # linear increase from start to rampup_epochs\n        if epoch < rampup_epochs:\n            lr = ((max_lr - start_lr) /\n                  rampup_epochs * epoch + start_lr)\n        # constant max_lr during sustain_epochs\n        elif epoch < rampup_epochs + sustain_epochs:\n            lr = max_lr\n        # exponential decay towards min_lr\n        else:\n            lr = ((max_lr - min_lr) *\n                  exp_decay**(epoch - rampup_epochs - sustain_epochs) +\n                  min_lr)\n        return lr\n    l = _exponential_lr(epoch, start_lr, min_lr, max_lr, rampup_epochs, sustain_epochs, exp_decay)\n    return l\n    # End of function kaggle_build_learning_rate_schedule\n\ndef kaggle_display_learning_rate_curve(p_lr):\n    print('----------------------------- kaggle_display_learning_rate_curve -----------------------------')\n    rng = [i for i in range(DL_EPOCH_NUM)]\n    y = [p_lr(x) for x in rng]\n    plt.plot(rng, y)\n    plt.show()\n    print(\"kaggle_display_learning_rate_curve: Done: Schedule: {:.3g} to {:.3g} to {:.3g}\".format(y[0], max(y), y[-1]))\n    # End of function kaggle_display_learning_rate_curve","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Data augmentation... "},{"metadata":{"trusted":true},"cell_type":"code","source":"def kaggle_image_augmentation(\n                              p_dataset, \n                              p_strategy, \n                              p_augmentations: list = None, \n                              p_shuffle_buffer_size: int = 2048, \n                              p_seed: int = SEED_HARCODED_VALUE\n                              ):\n    \"\"\"\n    This function proceeds to data augmentation in order to enhance Training dataset by creating new images\n    :parameter p_seed: The seed value for reproductibility\n    Note: Using method dataset.prefetch(tf.data.experimental.AUTOTUNE), \n          this processing happens essentially for free on TPU. \n          Data pipeline code is executed on the \"CPU\" part of the TPU \n          while the TPU itself is computing gradients.\n    \"\"\"\n    print('----------------------------- kaggle_image_augmentation -----------------------------')\n    if not p_augmentations is None:\n        for augmentation in p_augmentations:\n            if augmentation == 'flip_right_left':\n                dataset = p_dataset.map(kaggle_flip_left_right_data_augmentation, num_parallel_calls = tf.data.experimental.AUTOTUNE)\n            elif augmentation == 'flip_up_down':\n                dataset = p_dataset.map(kaggle_flip_up_down_data_augmentation, num_parallel_calls = tf.data.experimental.AUTOTUNE)\n            elif augmentation == 'random_contrast':\n                dataset = p_dataset.map(kaggle_random_contrast_data_augmentation, num_parallel_calls = tf.data.experimental.AUTOTUNE)\n            elif augmentation == 'random_brightness':\n                dataset = p_dataset.map(kaggle_random_brightness_data_augmentation, num_parallel_calls = tf.data.experimental.AUTOTUNE)\n            else:\n                raise Exception('kaggle_image_augmentation', 'Wrong parameter')\n            # End of 'for' statement\n        dataset = dataset.repeat() # the training dataset must repeat for several epochs\n        dataset = dataset.shuffle(p_shuffle_buffer_size, seed = p_seed) # Shuffle images of the batch at each iteration\n        dataset = dataset.batch(DL_BATCH_SIZE * p_strategy.num_replicas_in_sync) # Prepare batches for DL\n    else:\n        dataset = p_dataset.batch(DL_BATCH_SIZE * p_strategy.num_replicas_in_sync) # Prepare batches for DL\n        \n    if p_augmentations is None:\n        dataset = dataset.cache()\n    dataset = dataset.prefetch(tf.data.experimental.AUTOTUNE) # The next batch will always be ready for processing for the next iteration\n    print('kaggle_image_augmentation: type(dataset) => ', type(dataset))\n    print('kaggle_image_augmentation: Done')\n    return dataset\n    # End of function kaggle_image_augmentation  \n\ndef kaggle_flip_left_right_data_augmentation(p_image, p_label):\n    \"\"\"\n    \"\"\"\n    print('----------------------------- kaggle_flip_left_right_data_augmentation -----------------------------')\n    image = tf.image.random_flip_left_right(p_image)\n    print('kaggle_flip_left_right_data_augmentation: Done')\n    return image, p_label\n    # End of function kaggle_flip_left_right_data_augmentation\n\ndef kaggle_flip_up_down_data_augmentation(p_image, p_label):\n    \"\"\"\n    \"\"\"\n    print('----------------------------- kaggle_flip_up_down_data_augmentation -----------------------------')\n    image = tf.image.random_flip_left_right(p_image)\n    print('kaggle_flip_up_down_data_augmentation: Done')\n    return image, p_label\n    # End of function kaggle_flip_up_down_data_augmentation\n\ndef kaggle_random_contrast_data_augmentation(p_image, p_label):\n    \"\"\"\n    \"\"\"\n    print('----------------------------- kaggle_flip_left_right_data_augmentation -----------------------------')\n    image = tf.image.random_contrast(p_image, lower = 0.3, upper = 1.2)\n    print('kaggle_random_contrast_data_augmentation: Done')\n    return image, p_label\n    # End of function kaggle_random_contrast_data_augmentation\n\ndef kaggle_random_brightness_data_augmentation(p_image, p_label):\n    \"\"\"\n    \"\"\"\n    print('----------------------------- kaggle_random_brightness_data_augmentation -----------------------------')\n    image = tf.image.random_brightness(p_image, lower = 0.3, upper = 1.2)\n    print('kaggle_random_brightness_data_augmentation: Done')\n    return image, p_label\n    # End of function kaggle_random_brightness_data_augmentation","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# b.2) Check models for CNN\ndef kaggle_check_dl_model(p_strategy, p_model, p_train_df, p_train_df_size, p_validation_df, p_validation_df_size):\n    \"\"\"\n    \"\"\"\n    print('----------------------------- kaggle_check_dl_model -----------------------------')\n    with p_strategy.scope():\n        print('kaggle_check_dl_model: Setup LearningRateScheduler:')\n        \n        lr_callback = tf.keras.callbacks.LearningRateScheduler(kaggle_build_learning_rate_schedule, verbose = True)\n        kaggle_display_learning_rate_curve(kaggle_build_learning_rate_schedule)\n        temp_name = next(tempfile._get_candidate_names()) + '.h5'\n        # Training the model\n        print('kaggle_check_dl_model: Train the model:')\n        print('kaggle_check_dl_model: steps_per_epoch = ', p_train_df_size // (DL_BATCH_SIZE * p_strategy.num_replicas_in_sync))\n        history = p_model.fit(\n            p_train_df,\n            validation_data = p_validation_df,\n            epochs = DL_EPOCH_NUM,\n            steps_per_epoch = p_train_df_size // (DL_BATCH_SIZE * p_strategy.num_replicas_in_sync), # p_train_df_size is the initial size of the Train dataset.\n                                                                                                    # After data augmentation, this size is much much higher\n            callbacks = [lr_callback, ModelCheckpoint(filepath = temp_name, monitor = 'val_loss', save_best_only = True)],\n            verbose = 2\n        )\n        # End of 'with' statement\n\n    # Display leaning curve\n    kaggle_display_learning_curves(\n        history.history['loss'],\n        history.history['val_loss'],\n        'loss',\n        211,\n    )\n    kaggle_display_learning_curves(\n        history.history['sparse_categorical_accuracy'],\n        history.history['val_sparse_categorical_accuracy'],\n        'accuracy',\n        212,\n    )\n    # Show scores\n    # Split Validation images & labels\n    print('kaggle_check_dl_model: Split images & labels for validation_df')\n    validation_images_df = p_validation_df.map(lambda image, label: image)\n    validation_labels_df = p_validation_df.map(lambda image, label: label).unbatch()\n    # Convert labels into a Pandas dataframe\n    #l = []\n    #for i, e in validation_labels_df.enumerate():\n    #    l[i] = int(e.numpy())\n    #validation_labels_df = pd.DataFrame(l)\n    #print('type(validation_labels_df) ==>', type(validation_labels_df))\n    #print('Main: validation_labels_df.shape = ', len(validation_labels_df), ' / ', validation_df_size)\n    #raise Exception('Stop')\n\n    # Display confusion matrix\n    cm_correct_labels = next(iter(validation_labels_df.batch(p_validation_df_size))).numpy() #np.darray\n    print('kaggle_check_dl_model: cm_correct_labels ==>', cm_correct_labels)\n    cm_probabilities = kaggle_prediction(p_model, validation_images_df)\n    print('kaggle_check_dl_model: cm_probabilities ==>', cm_probabilities)\n    y_predictions = np.argmax(cm_probabilities, axis = -1)\n    print('kaggle_check_dl_model: y_predictions ==>', y_predictions)\n    labels = range(len(IMAGE_CLASSES))\n    cmat = confusion_matrix(\n                            cm_correct_labels,\n                            y_predictions,\n                            labels = labels,\n                            )\n    cmat = (cmat.T / cmat.sum(axis=1)).T # normalize\n    print('kaggle_check_dl_model: cmat = ', cmat)\n    # Display scores\n    score = f1_score(\n        cm_correct_labels,\n        y_predictions,\n        labels=labels,\n        average = 'macro',\n    )\n    print('kaggle_check_dl_model: Score = ', score)\n    precision = precision_score(\n                                cm_correct_labels,\n                                y_predictions,\n                                labels = labels,\n                                average = 'macro',\n    )\n    print('kaggle_check_dl_model: Precision score = ', precision)\n    recall = recall_score(\n                          cm_correct_labels,\n                          y_predictions,\n                          labels = labels,\n                          average='macro',\n    )\n    print('kaggle_check_dl_model: Recall score = ', recall)\n    kaggle_display_confusion_matrix(cmat, score, precision, recall)\n\n    # Retrieve the best weights for the model\n    print('kaggle_check_dl_model: Retrieve the best weights for the model')\n    model = tf.keras.models.load_model(temp_name)\n    os.remove(temp_name)\n\n    print('kaggle_check_dl_model: Done')\n    return (score, precision, recall), history.history, model\n    # End of function kaggle_check_dl_model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Main function"},{"metadata":{"trusted":true},"cell_type":"code","source":"def kaggle_main() -> None:\n    global DL_INPUT_SHAPE\n    \n    # Set defaults\n    kaggle_set_seed()\n    kaggle_set_mp_default()\n    \n    # Current path\n    print(os.path.abspath(os.getcwd()))\n    # Kaggle current path and files\n    #for dirname, _, filenames in os.walk('/kaggle/input'):\n    #    for filename in filenames:\n    #        print(os.path.join(dirname, filename))\n\n    # Modules version\n    kaggle_modules_versions()\n\n    print('Main: Playgrounf name: ', ML_NAME)\n\n    strategy, global_path = kaggle_tpu_detection()\n    if global_path is None: # Force global_path to local path when datasets not copied to gs://kds-xxx\n        global_path = os.path.join(os.path.abspath(os.getcwd()), '../input')\n        global_path = os.path.join(global_path, DATABASE_NAME)\n\n    # Images and lables are separated\n    train_df, validation_df, test_df, train_df_size, validation_df_size, test_df_size = kaggle_load_images_datasets(p_train_url = DATABASE_NAME,\n                                                                                                                    p_labels = COLUMNS_LABEL, \n                                                                                                                    p_global_path = global_path,\n                                                                                                                    p_test_url = DATABASE_NAME\n                                                                                                                    )\n\n    print('type(train_df) ==> ', type(train_df))\n    print('Main: train_df_size', train_df_size)\n    print('type(validation_df) ==> ', type(validation_df))\n    print('Main: validation_df_size', validation_df_size)\n    print('type(test_df_size) ==> ', type(train_df))\n    print('Main: test_df_size', test_df_size)\n    # End of datasets loading operation\n\n#    raise Exception('Stop')\n    # 1. Data augmentation and batches setup\n    augmentations_list = ['flip_right_left']\n    train_df = kaggle_image_augmentation(train_df, strategy, augmentations_list)\n    train_df_size *= len(augmentations_list) \n    # Inspect Validation dataset\n    for image_batch, label_batch in train_df.take(1):\n        pass\n    print('Main: Train data label inspection:', label_batch.numpy())\n    print(image_batch.numpy().shape, len(image_batch.numpy()), label_batch.numpy().shape)\n    validation_df = kaggle_image_augmentation(validation_df, strategy) # No data augmentation for Validation dataset\n    # Inspect Validation dataset\n    for image_batch, label_batch in validation_df.take(1):\n        pass\n    print('Main: Validation data label inspection:', label_batch.numpy())\n    print(image_batch.numpy().shape, len(image_batch.numpy()), label_batch.numpy().shape)\n    copy_validation_df = validation_df # Keep a copy of the original validation_df for model evaluation\n    test_df = test_df.batch(DL_BATCH_SIZE * strategy.num_replicas_in_sync) # Prepare batches for DL\n    test_df = test_df.prefetch(tf.data.experimental.AUTOTUNE) # The next batch will always be ready for processing for the next iteration\n    # Inspect test dataset\n    for image_batch in test_df.take(1):\n        pass\n    print('Main: Test inspection:')\n    print('type(image_batch) ==> ', type(image_batch))\n    print('len(image_batch) ==> ', len(image_batch))\n    #print(image_batch)\n    print('Main: Dataset: %d training images, %d validation images, %d unlabeled test images' % (train_df_size, validation_df_size, test_df_size))\n    # 2. Do a basic DL evaluation as reference for the end\n    print('----------------------------- kaggle_dl_quick_and_dirty -----------------------------')\n    # FIXME Enhance function kaggle_dl_quick_and_dirty to support CNN and more\n    model = None\n    if OUTPUT_IS_REGRESSION: # Use regression algorithms\n        with strategy.scope():\n            pass # TODO\n            # End of 'with' statement\n    else:\n        with strategy.scope():\n            model = kaggle_create_sequential_classifier_model(\n                                                              #strategy\n                                                              #p_input_shape = DL_INPUT_SHAPE, \n                                                              #p_drop_rate = DL_DROP_RATE,\n                                                              p_loss = 'categorical_crossentropy', \n                                                              p_metrics = ['categorical_accuracy'],\n                                                              p_pretrained_layers = ['VGG19'], \n                                                              p_image_size = IMAGE_SIZE, \n                                                              p_class_num = len(IMAGE_CLASSES)\n                                                              )\n    with strategy.scope():\n        loss, accuracy = model.evaluate(validation_df, steps = 16, verbose = 2)\n        print('Loss: %.2f' % loss)\n        print('Accuracy: %.2f' % accuracy)\n        # End of 'with' statement\n\n    raise Exception('Stop')\n\n    if OUTPUT_IS_REGRESSION: # Use regression algorithms\n        with strategy.scope():\n            pass # TODO\n        # End of 'with' statement\n    else:\n        # Using kaggle_check_models() and KerasClassifier require too many processing time, just use standard fit() method\n        with strategy.scope():\n            models.append(('VGG19', kaggle_create_sequential_classifier_model(\n                                                                              #strategy\n                                                                              #p_input_shape = DL_INPUT_SHAPE, \n                                                                              #p_drop_rate = DL_DROP_RATE,\n                                                                              p_loss = 'categorical_crossentropy', \n                                                                              p_metrics = ['categorical_accuracy'],\n                                                                              p_pretrained_layers = ['VGG19'], \n                                                                              p_image_size = IMAGE_SIZE, \n                                                                              p_class_num = len(IMAGE_CLASSES)\n                                                                              )))\n            models.append(('DenseNet201', kaggle_create_sequential_classifier_model(\n                                                                                 #strategy\n                                                                                 #p_input_shape = DL_INPUT_SHAPE, \n                                                                                 #p_drop_rate = DL_DROP_RATE,\n                                                                                 p_loss = 'categorical_crossentropy', \n                                                                                 p_metrics = ['categorical_accuracy'],\n                                                                                 p_pretrained_layers = ['DenseNet201'], \n                                                                                 p_image_size = IMAGE_SIZE, \n                                                                                 p_class_num = len(IMAGE_CLASSES)\n                                                                                 )))\n    with strategy.scope():\n        print('----------------------------- kaggle_check_dl_models -----------------------------')\n        results = []\n        histories = []\n        trained_models = []\n        for name, model in models:\n            print('kaggle_check_dl_models: Processing %s with type %s' % (name, type(model)))\n            result, history, trained_model = kaggle_check_dl_model(strategy, model, train_df, train_df_size, validation_df, validation_df_size)\n            results.append(result)\n            histories.append(history)\n            trained_models.append(trained_model)\n            print('kaggle_check_dl_models: result: ', result)\n            #print('kaggle_check_dl_models: history: ', history)\n            #break # For debug\n            # End of 'for' statement\n        ml = trained_models[0]\n        # TODO Add support of regressor!\n    # End of 'with' statement\n\n    print('Main: End of processing')\n    # End of function kaggle_main","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Entry point"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Entry point\nprint(\"Starting at \", datetime.now().strftime(\"%d/%m/%Y %H:%M:%S\"))\nkaggle_main()\nprint(\"Ending at \", datetime.now().strftime(\"%d/%m/%Y %H:%M:%S\"))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"***If you liked this Notebook, please upvote.\nGives Motivation to make new Notebooks :)***"}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}