{"cells":[{"metadata":{"execution":{"iopub.execute_input":"2021-02-19T02:20:48.873372Z","iopub.status.busy":"2021-02-19T02:20:48.872553Z","iopub.status.idle":"2021-02-19T02:20:49.234019Z","shell.execute_reply":"2021-02-19T02:20:49.234643Z"},"papermill":{"duration":0.376575,"end_time":"2021-02-19T02:20:49.23501","exception":false,"start_time":"2021-02-19T02:20:48.858435","status":"completed"},"tags":[],"id":"OEnHkhkgHF4f","outputId":"70e7a1a3-38e9-49ff-9dcf-79111970a2dd","trusted":true},"cell_type":"code","source":"colab = False\nif colab:\n  !pip install fsspec\n  !pip install gcsfs\n  GCS_PATH= 'gs://kds-c75d129c15f5ca0a5d6b5efb3159606da6866597fee1d43cd4de083e'\nelse:\n  from kaggle_datasets import KaggleDatasets\n  GCS_PATH = KaggleDatasets().get_gcs_path()\nprint ('GCS_PATH=',GCS_PATH)\n#GCS_PATH= gs://kds-c75d129c15f5ca0a5d6b5efb3159606da6866597fee1d43cd4de083e","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.008731,"end_time":"2021-02-19T02:20:49.254439","exception":false,"start_time":"2021-02-19T02:20:49.245708","status":"completed"},"tags":[],"id":"vLJW4qSGHF4n"},"cell_type":"markdown","source":"# Dependencies"},{"metadata":{"execution":{"iopub.execute_input":"2021-02-19T02:20:49.27951Z","iopub.status.busy":"2021-02-19T02:20:49.278864Z","iopub.status.idle":"2021-02-19T02:20:56.35687Z","shell.execute_reply":"2021-02-19T02:20:56.357786Z"},"papermill":{"duration":7.094639,"end_time":"2021-02-19T02:20:56.357977","exception":false,"start_time":"2021-02-19T02:20:49.263338","status":"completed"},"tags":[],"id":"LlV-2wM-HF4n","outputId":"507488de-e216-4361-e66b-720d31ac48d1","trusted":true},"cell_type":"code","source":"import math, os, re, warnings, random, time\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\n\nfrom matplotlib import pyplot as plt\n\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import classification_report, confusion_matrix\n\nimport tensorflow as tf\nimport tensorflow.keras as keras\nimport tensorflow.keras.layers as L\nimport tensorflow.keras.backend as K\n\nfrom tensorflow.keras import optimizers, Sequential, losses, metrics, Model\nfrom tensorflow.keras.callbacks import EarlyStopping\n#import efficientnet.tfkeras as efn\n\ndef seed_everything(seed=0):\n    random.seed(seed)\n    np.random.seed(seed)\n    tf.random.set_seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    os.environ['TF_DETERMINISTIC_OPS'] = '1'\n\nseed = 0\nseed_everything(seed)\nwarnings.filterwarnings('ignore')\n\nprint('Tensorflor:', tf.__version__)","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.009581,"end_time":"2021-02-19T02:20:56.378552","exception":false,"start_time":"2021-02-19T02:20:56.368971","status":"completed"},"tags":[],"id":"xljPAdwMHF4n"},"cell_type":"markdown","source":"### Hardware configuration\n\nNote that we have `8` cores, this is because the `TPU v3-8` have more workers than a single `TPU v2` which has `8` cores."},{"metadata":{"execution":{"iopub.execute_input":"2021-02-19T02:20:56.402599Z","iopub.status.busy":"2021-02-19T02:20:56.401994Z","iopub.status.idle":"2021-02-19T02:20:56.411989Z","shell.execute_reply":"2021-02-19T02:20:56.412698Z"},"papermill":{"duration":0.02511,"end_time":"2021-02-19T02:20:56.412868","exception":false,"start_time":"2021-02-19T02:20:56.387758","status":"completed"},"tags":[],"id":"LFZHaGL0HF4o","outputId":"9cc79547-f075-4aeb-ac86-6f7e1a900683","trusted":true},"cell_type":"code","source":"# TPU or GPU detection\n# Detect hardware, return appropriate distribution strategy\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print(f'Running on TPU {tpu.master()}')\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.TPUStrategy(tpu)\nelse:\n    strategy = tf.distribute.get_strategy()\n\nAUTO = tf.data.experimental.AUTOTUNE\nREPLICAS = strategy.num_replicas_in_sync\nprint(f'REPLICAS: {REPLICAS}')","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.009528,"end_time":"2021-02-19T02:20:56.432459","exception":false,"start_time":"2021-02-19T02:20:56.422931","status":"completed"},"tags":[],"id":"-TqUTer_HF4o"},"cell_type":"markdown","source":"# Model parameters"},{"metadata":{"execution":{"iopub.execute_input":"2021-02-19T02:20:56.456281Z","iopub.status.busy":"2021-02-19T02:20:56.455681Z","iopub.status.idle":"2021-02-19T02:20:56.458567Z","shell.execute_reply":"2021-02-19T02:20:56.458069Z"},"papermill":{"duration":0.016601,"end_time":"2021-02-19T02:20:56.458667","exception":false,"start_time":"2021-02-19T02:20:56.442066","status":"completed"},"tags":[],"id":"Wk62ctT0HF4o","trusted":true},"cell_type":"code","source":"BATCH_SIZE = 8 * REPLICAS\nLEARNING_RATE = 1e-5 * REPLICAS\nEPOCHS = 20\nHEIGHT = 2048\nWIDTH = 2048\nHEIGHT_RS = 2048\nWIDTH_RS = 2048","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.00969,"end_time":"2021-02-19T02:20:56.478073","exception":false,"start_time":"2021-02-19T02:20:56.468383","status":"completed"},"tags":[],"id":"BhD6RbAnHF4o"},"cell_type":"markdown","source":"# Load data"},{"metadata":{"execution":{"iopub.execute_input":"2021-02-19T02:20:56.501955Z","iopub.status.busy":"2021-02-19T02:20:56.501158Z","iopub.status.idle":"2021-02-19T02:21:05.864775Z","shell.execute_reply":"2021-02-19T02:21:05.864312Z"},"papermill":{"duration":9.377004,"end_time":"2021-02-19T02:21:05.864904","exception":false,"start_time":"2021-02-19T02:20:56.4879","status":"completed"},"tags":[],"id":"R-Rz6CNQHF4p","outputId":"498ef5d5-affe-401e-b3c2-177aaedea661","trusted":true},"cell_type":"code","source":"train = pd.read_csv(f'{GCS_PATH}/train.csv')\ndisplay(train.head())","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-02-19T02:21:05.892029Z","iopub.status.busy":"2021-02-19T02:21:05.891512Z","iopub.status.idle":"2021-02-19T02:21:06.125248Z","shell.execute_reply":"2021-02-19T02:21:06.125847Z"},"papermill":{"duration":0.250421,"end_time":"2021-02-19T02:21:06.126037","exception":false,"start_time":"2021-02-19T02:21:05.875616","status":"completed"},"tags":[],"id":"yb-bEE0wHF4p","outputId":"119200be-a1f3-4964-dfa3-eb7b08283371","trusted":true},"cell_type":"code","source":"def count_data_items(filenames):\n    n = [int(re.compile(r'-([0-9]*)\\.').search(filename).group(1)) for filename in filenames]\n    return np.sum(n)\n\nFILENAMES_COMP = tf.io.gfile.glob(GCS_PATH + '/train_tfrecords/*.tfrec') # Original TFRecords\n\nNUM_TRAINING_IMAGES = count_data_items(FILENAMES_COMP)\n\nprint(f'GCS: train images: {NUM_TRAINING_IMAGES}')\n\nCLASSES = [\n            'Nucleoplasm', \n            'Nuclear membrane', \n            'Nucleoli', \n            'Nucleoli fibrillar center', \n            'Nuclear speckles',\n            'Nuclear bodies',\n            'Endoplasmic reticulum',\n            'Golgi apparatus',\n            'Intermediate filaments',\n            'Actin filaments',\n            'Microtubules',\n            'Mitotic spindle',\n            'Centrosome',\n            'Plasma membrane',\n            'Mitochondria',\n            'Aggresome',\n            'Cytosol',\n            'Vesicles and punctate cytosolic patterns',\n            'Negative']","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-02-19T02:21:07.166462Z","iopub.status.busy":"2021-02-19T02:21:07.165649Z","iopub.status.idle":"2021-02-19T02:21:11.404538Z","shell.execute_reply":"2021-02-19T02:21:11.403555Z"},"papermill":{"duration":5.266896,"end_time":"2021-02-19T02:21:11.404708","exception":false,"start_time":"2021-02-19T02:21:06.137812","status":"completed"},"tags":[],"id":"yEcYVmWQHF4p","outputId":"69eeb55e-98f2-4713-ee92-cc83857b2fae","trusted":true},"cell_type":"code","source":"# num_parallel_reads=None forces the order to be preserved\nraw_train_ds = tf.data.TFRecordDataset(FILENAMES_COMP, num_parallel_reads=None)\nprint('TFRecord Features')\n# See an example\nfor raw in raw_train_ds.take(1):\n    example = tf.train.Example()\n    example.ParseFromString(raw.numpy())\n    for k,v in example.features.feature.items():\n        print(k)\n        if k!=\"image\":\n            print(f\"\\t--> {v.bytes_list.value[0]}\")\n        else:\n            print(f\"\\t-->{str(v.bytes_list.value[0][:25])+' ... '}\")","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.012832,"end_time":"2021-02-19T02:21:11.431912","exception":false,"start_time":"2021-02-19T02:21:11.41908","status":"completed"},"tags":[],"id":"sYQBb8BgHF4q"},"cell_type":"markdown","source":"# Datasets utility functions"},{"metadata":{"execution":{"iopub.execute_input":"2021-02-19T02:21:11.466482Z","iopub.status.busy":"2021-02-19T02:21:11.46579Z","iopub.status.idle":"2021-02-19T02:21:11.468426Z","shell.execute_reply":"2021-02-19T02:21:11.468865Z"},"papermill":{"duration":0.024619,"end_time":"2021-02-19T02:21:11.468986","exception":false,"start_time":"2021-02-19T02:21:11.444367","status":"completed"},"tags":[],"id":"AwcxyenoHF4q","trusted":true},"cell_type":"code","source":"def decode_image(image_data):\n    image = tf.io.decode_png(image_data, channels=3)\n    return image\n\ndef prepare_image(image, label, name):\n    \"\"\"\n        Resize and reshape images to the expected size.\n    \"\"\"\n    image = tf.image.resize_with_pad(image, HEIGHT_RS, WIDTH_RS)\n    image = tf.reshape(image, [HEIGHT_RS, WIDTH_RS, 3])\n    print(image)\n\n    return image, label, name\n\ndef scale_image(image, label, name):\n    \"\"\"\n        Cast tensor to float and normalizes (range between 0 and 1).\n    \"\"\"\n    image = tf.cast(image, tf.float32)\n    image /= 255.0\n    \n    return image, label, name\n\ndef read_tfrecord(example_proto):\n    \"\"\"\n        1. Parse data based on the 'TFREC_FORMAT' map.\n        2. Decode image.\n    \"\"\"\n            \n    TFREC_FORMAT = {\n        'target': tf.io.FixedLenFeature([], tf.string), \n        'image': tf.io.FixedLenFeature([], tf.string), \n        'image_name': tf.io.FixedLenFeature([], tf.string), \n    }\n    \n    example = tf.io.parse_single_example(example_proto, TFREC_FORMAT)\n    \n    image = decode_image(example['image'])\n    \n    label = tf.cast(example['target'], tf.string)\n    name  = tf.cast(example['image_name'], tf.string)\n    return image, label, name\n\ndef get_dataset(FILENAMES):\n    \"\"\"\n        Return a Tensorflow dataset ready for training or inference.\n    \"\"\"\n    dataset = tf.data.TFRecordDataset(FILENAMES, num_parallel_reads=None)\n        \n    \n    dataset = dataset.map(lambda x: read_tfrecord(x), num_parallel_calls=None)\n    dataset = dataset.map(scale_image, num_parallel_calls=AUTO)\n    dataset = dataset.map(prepare_image, num_parallel_calls=AUTO)\n    dataset = dataset.batch(BATCH_SIZE)\n    \n    dataset = dataset.prefetch(AUTO)\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.011009,"end_time":"2021-02-19T02:21:11.491963","exception":false,"start_time":"2021-02-19T02:21:11.480954","status":"completed"},"tags":[],"id":"tg0FXnV2HF4q"},"cell_type":"markdown","source":"# Visualization utility functions"},{"metadata":{"execution":{"iopub.execute_input":"2021-02-19T02:21:11.527698Z","iopub.status.busy":"2021-02-19T02:21:11.526998Z","iopub.status.idle":"2021-02-19T02:21:11.529252Z","shell.execute_reply":"2021-02-19T02:21:11.529679Z"},"papermill":{"duration":0.026043,"end_time":"2021-02-19T02:21:11.529803","exception":false,"start_time":"2021-02-19T02:21:11.50376","status":"completed"},"tags":[],"id":"RHLiSrT5HF4q","trusted":true},"cell_type":"code","source":"np.set_printoptions(threshold=15, linewidth=80)\n\ndef batch_to_numpy_images_and_labels(data):\n    images, labels, names = data\n    \n    numpy_images = images.numpy()\n    numpy_labels = labels.numpy()\n    numpy_names = names.numpy()\n    return numpy_images, numpy_labels, numpy_names\n\ndef display_batch_of_images(databatch, predictions=None):\n    # data\n    images, labels, names = batch_to_numpy_images_and_labels(databatch)\n    # auto-squaring: this will drop data that does not fit into square or square-ish rectangle\n    rows = int(math.sqrt(len(images)))\n    cols = len(images)//rows\n        \n    # size and spacing\n    FIGSIZE = 13.0\n    SPACING = 0.1\n    subplot=(rows,cols,1)\n    if rows < cols:\n        plt.figure(figsize=(FIGSIZE,FIGSIZE/cols*rows))\n    else:\n        plt.figure(figsize=(FIGSIZE/rows*cols,FIGSIZE))\n    \n    # display\n    for i, (image, label, name) in enumerate(zip(images[:rows*cols], labels[:rows*cols], names[:rows*cols],)):\n        print(' Name:', name.decode())\n        title = name.decode() #if label is None else CLASSES[label]\n        correct = True\n        dynamic_titlesize = FIGSIZE*SPACING/max(rows,cols)*15+3 # magic formula tested to work from 1x1 to 10x10 images\n        subplot = display_one_flower(image, title, subplot, not correct, titlesize=dynamic_titlesize)\n    \n    #layout\n    plt.tight_layout()\n    if label is None and predictions is None:\n        plt.subplots_adjust(wspace=0, hspace=0)\n    else:\n        plt.subplots_adjust(wspace=SPACING, hspace=SPACING)\n    plt.show()\n    \ndef display_one_flower(image, title, subplot, red=False, titlesize=8):\n    plt.subplot(*subplot)\n    plt.axis('off')\n    plt.imshow(image)\n    if len(title) > 0:\n        plt.title(title, fontsize=int(titlesize) if not red else int(titlesize/1.2), color='red' if red else 'black', \n                  fontdict={'verticalalignment':'center'}, pad=int(titlesize/1.5))\n    return (subplot[0], subplot[1], subplot[2]+1)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-02-19T02:21:11.556321Z","iopub.status.busy":"2021-02-19T02:21:11.555697Z","iopub.status.idle":"2021-02-19T02:21:11.558846Z","shell.execute_reply":"2021-02-19T02:21:11.55845Z"},"papermill":{"duration":0.01772,"end_time":"2021-02-19T02:21:11.558947","exception":false,"start_time":"2021-02-19T02:21:11.541227","status":"completed"},"tags":[],"id":"hAoQ-E9cHF4r","trusted":true},"cell_type":"code","source":"#stacked_images = np.transpose(np.array([red_image, green_image, blue_image, yellow_image]), (1,2,0))","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.011693,"end_time":"2021-02-19T02:21:11.582","exception":false,"start_time":"2021-02-19T02:21:11.570307","status":"completed"},"tags":[],"id":"SK5VC-N4HF4r"},"cell_type":"markdown","source":"# Training data samples"},{"metadata":{"execution":{"iopub.execute_input":"2021-02-19T02:21:11.611057Z","iopub.status.busy":"2021-02-19T02:21:11.610304Z","iopub.status.idle":"2021-02-19T02:21:15.44038Z","shell.execute_reply":"2021-02-19T02:21:15.440797Z"},"papermill":{"duration":3.846328,"end_time":"2021-02-19T02:21:15.440944","exception":false,"start_time":"2021-02-19T02:21:11.594616","status":"completed"},"tags":[],"id":"qLxEQBpBHF4r","outputId":"c08b95a5-194c-4af2-84cd-63bcad42e254","trusted":true},"cell_type":"code","source":"dataset = get_dataset(FILENAMES_COMP)\ntrain_iter = iter(dataset.unbatch().batch(16))\n\ndisplay_batch_of_images(next(train_iter))","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}