{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n#for dirname, _, filenames in os.walk('/kaggle/input'):\n    #for filename in filenames:\n        #print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from IPython.core.magic import register_cell_magic\n@register_cell_magic\ndef skip(line, cell=None):\n    '''Skips execution of the current line/cell if line evaluates to True.'''\n    if eval(line):\n        return\n        \n    get_ipython().run_cell(cell)","metadata":{"_kg_hide-output":true,"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport tensorflow as tf\nfrom tensorflow import keras \nfrom tensorflow.keras import backend as K \nfrom tensorflow.keras.preprocessing.text import Tokenizer\nfrom tensorflow.keras.preprocessing.sequence import pad_sequences\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport os, cv2\nimport random\nimport dill\nimport gc\nfrom sklearn.model_selection import train_test_split\nimport time\nfrom kaggle_datasets import KaggleDatasets\nimport tensorflow_datasets.public_api as tfds\nimport math\nfrom tqdm.notebook import tqdm\nimport tensorflow_addons as tfa\nfrom mt_utils import *","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Dataset links : https://www.kaggle.com/tchaye59/mt-tfrecord-custom-vocab & https://www.kaggle.com/tchaye59/mtcustomvocabimg\n#### Pretraining : https://www.kaggle.com/tchaye59/mt-pretraining\n#### Training: https://www.kaggle.com/tchaye59/mt-fast-distributed-training-tpu","metadata":{}},{"cell_type":"code","source":"seed=123\nrandom.seed(seed)\nnp.random.seed(seed)\ntf.random.set_seed(seed)","metadata":{"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# NEW on TPU in TensorFlow 24: shorter cross-compatible TPU/GPU/multi-GPU/cluster-GPU detection code\ntpu = None\ntry: # detect TPUs\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver.connect() # TPU detection\n    strategy = tf.distribute.TPUStrategy(tpu)\nexcept ValueError: # detect GPUs\n    strategy = tf.distribute.MirroredStrategy() # for GPU or multi-GPU machines\n    #strategy = tf.distribute.get_strategy() # default strategy that works on CPU and single GPU\n    #strategy = tf.distribute.experimental.MultiWorkerMirroredStrategy() # for clusters of multi-GPU machines\n\nprint(\"Number of accelerators: \", strategy.num_replicas_in_sync)\n\n\nAUTO = tf.data.experimental.AUTOTUNE\nREPLICAS = strategy.num_replicas_in_sync\nprint(f'REPLICAS: {REPLICAS}')","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"GCS_PATH = KaggleDatasets().get_gcs_path('mtcustomvocabimg')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tokenizer = CstTokenizer()\ntokenizer.word_index","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Initialization\nmax_seq = 393","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset","metadata":{}},{"cell_type":"code","source":"class TrainDataset(tfds.core.GeneratorBasedBuilder):\n    VERSION = tfds.core.Version('0.1.0')\n    \n    def _split_generators(self, dl_manager):\n        return [\n            tfds.core.SplitGenerator(\n                    name=f'train',\n                    gen_kwargs={\n                    },\n            )\n        ]\n    \n    def _info(self):\n        return tfds.core.DatasetInfo(\n            builder=self,\n            description=(\"\"),\n            features=tfds.features.FeaturesDict({\n                \"image\": tfds.features.Image(shape=(None,None,1)),\n                \"target\": tfds.features.Tensor(shape=(max_seq,),dtype=tf.int8),\n                \"count\": tfds.features.Tensor(dtype=tf.int32,shape=()),\n            }),\n        )\n    \n    def _generate_examples(self,**args):\n        pass","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BATCH_SIZE = 512\nSTEPS_PER_TRAIN = 10\ntrain_steps = 2424186//(BATCH_SIZE*REPLICAS)\nBUFFER_SIZE = 20000\n\nprefetch = 20\nHEIGHT = 300\nWIDTH = 300","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def data_augment(image):\n    p_rotation = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_noise = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_flip1 = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_flip2 = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n            \n    # Rotation\n    if p_rotation > .1:\n        image = rotation(image)\n        \n    # Flip\n    if p_flip1 > .4:\n        image = tf.image.random_flip_left_right(image, seed)\n        \n    # Flip\n    if p_flip2 > .4:\n        image = tf.image.random_flip_up_down(image, seed)\n        \n    # Resize \n    image = tf.image.resize(image,(WIDTH, HEIGHT), method='nearest')\n            \n    # Noise\n    if p_noise >= .4:\n        image = random_noise(image)\n        \n    return image\n\ndef rotation(img, rotation=0.2):\n    rotation = tf.random.uniform([], -1.0, 1.0, dtype=tf.float32)*rotation\n    shape = tf.shape(img)\n    h,w = shape[0],shape[1]\n    # Pad the image with zeros to avoid losing some pixels after rotation. \n    # This will double the image width and height\n    img = tf.image.pad_to_bounding_box(img,h//2, w//2,h*2, w*2)\n    img = tfa.image.rotate(img,rotation,fill_value=0)\n    # Now remove the zero pads\n    return remove_pad(img)\n\n\ndef remove_pad(arr,pad_value = 0.0):\n    arr_masked = tf.reduce_all(arr != pad_value , axis=-1)\n    #x\n    y = tf.argmax(arr_masked, axis=1)\n    y = tf.where(y)\n    y_min,y_max = y[0,0],y[-1,0]+1\n    #y\n    x = tf.argmax(arr_masked, axis=0)\n    x = tf.where(x)\n    x_min,x_max = x[0,0],x[-1,0]+1\n    arr = arr[y_min:y_max,x_min:x_max]\n    return arr\n\ndef random_noise(img,p=0.01):\n    shape = tf.shape(img)\n    choice = tf.random.categorical(tf.math.log([[p, 1-p]]), tf.size(img),dtype=tf.int32)\n    noise = tf.random.categorical(tf.math.log([[1., 1.]]), tf.size(img),dtype=tf.int32)\n    choice = tf.reshape(choice,shape)\n    noise = tf.reshape(noise,shape)\n    noise = tf.abs(choice-1)*noise\n    choice = tf.cast(choice,img.dtype)\n    noise = tf.cast(noise,img.dtype)\n    return (choice*img)+noise","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_dataset(_):\n    builder = TrainDataset(data_dir=GCS_PATH)\n    # The following line download the dataset\n    builder.download_and_prepare()\n    dataset = builder.as_dataset()['train']\n\n    # normalize, shuffle and bacth\n    def preprecoss(x):\n        img,target = x['image'],x['target']\n        # Normalize : There are two pixels 0 and 255\n        img = tf.cast(img == 0,tf.float32)\n        return data_augment(img),target\n    dataset = dataset.repeat().shuffle(BUFFER_SIZE).map(preprecoss,num_parallel_calls=AUTO)\n    dataset = dataset.batch(BATCH_SIZE).prefetch(prefetch)\n    return dataset\n\nwith strategy.scope():\n    if tpu is None:\n        dataset = get_dataset(0)\n    else:\n        dataset = strategy.experimental_distribute_datasets_from_function(get_dataset)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nfor x,_ in get_dataset(0).take(1):\n    pass\nplt.imshow(x[0].numpy())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model ","metadata":{}},{"cell_type":"code","source":"name = 'EfficientNetB0'","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with strategy.scope():\n    image_input = tf.keras.layers.Input(shape=(WIDTH,HEIGHT,1))\n    backbone_model = tf.keras.applications.EfficientNetB0(include_top=False,weights=None,input_shape=(WIDTH,HEIGHT,1),)\n\n\n    backbone_model = backbone_model(image_input)\n    backbone_model = tf.keras.layers.Dropout(0.3)(backbone_model)\n    backbone_model = tf.keras.layers.GlobalAveragePooling2D()(backbone_model)\n\n    output = tf.keras.layers.Dense(1,activation='linear')(backbone_model)\n\n    model = tf.keras.Model(image_input,output)\n\n    model.summary()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with strategy.scope():\n    model.compile(loss='mse',\n                  experimental_steps_per_execution = STEPS_PER_TRAIN,\n                  optimizer=keras.optimizers.Adam(0.001))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with strategy.scope():\n    save_locally = tf.saved_model.SaveOptions(experimental_io_device='/job:localhost')\n    load_locally = tf.saved_model.LoadOptions(experimental_io_device='/job:localhost')\n    #callbacks\n    filepath=f\"{name}.h5\"\n    callbacks_list = [\n            keras.callbacks.ModelCheckpoint(filepath, \n                                            verbose=1,\n                                            monitor='loss', \n                                            save_best_only=True,\n                                            options=save_locally,\n                                            mode='min'),\n            keras.callbacks.EarlyStopping(monitor='loss',\n                                          patience=20,\n                                          mode='min'),\n            keras.callbacks.ReduceLROnPlateau(monitor='loss',\n                                              factor=0.2,\n                                              patience=1,\n                                              min_lr=0.00001)\n    ]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#! cp ../input/mt-pretraining/*.h5 .","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with strategy.scope():\n    if os.path.exists(f'last_{name}.h5'):\n        print(\"Loading...\")\n        model = tf.keras.models.load_model(f'last_{name}.h5',options=load_locally)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model.fit(dataset,\n                    steps_per_epoch=train_steps,\n                    epochs=10,\n                    callbacks=callbacks_list,)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.save(f'last_{name}.h5')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.DataFrame(history.history)[['loss']].plot()","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}