{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n!pip install efficientnet -q\n!pip install iterative-stratification -q\n!pip install tensorflow_addons -q\n!pip install focal-loss\nimport numpy as np\nimport pandas\nimport tensorflow as tf\nimport os\nimport tensorflow_addons as tfa\nimport efficientnet.tfkeras as efn\nimport numpy as np\nimport pandas as pd\nfrom sklearn.model_selection import train_test_split\nimport tensorflow as tf\nfrom sklearn.model_selection import GroupKFold\nimport matplotlib.pyplot as plt\nimport glob\nfrom tqdm import tqdm\nfrom iterstrat.ml_stratifiers import MultilabelStratifiedKFold,MultilabelStratifiedShuffleSplit\nfrom focal_loss import BinaryFocalLoss","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def onehot(image,label):\n    CLASSES = 19\n    return image,tf.one_hot(label,CLASSES)\ndef cutmix(image, label, PROBABILITY = 1.0):\n    # input image - is a batch of images of size [n,dim,dim,3] not a single image of [dim,dim,3]\n    # output - a batch of images with cutmix applied\n    DIM = IMSIZE[IMS]\n    CLASSES = 19\n    \n    imgs = []; labs = []\n    for j in range(AUG_BATCH):\n        # DO CUTMIX WITH PROBABILITY DEFINED ABOVE\n        P = tf.cast( tf.random.uniform([],0,1)<=PROBABILITY, tf.int32)\n        # CHOOSE RANDOM IMAGE TO CUTMIX WITH\n        k = tf.cast( tf.random.uniform([],0,AUG_BATCH),tf.int32)\n        # CHOOSE RANDOM LOCATION\n        x = tf.cast( tf.random.uniform([],0,DIM),tf.int32)\n        y = tf.cast( tf.random.uniform([],0,DIM),tf.int32)\n        b = tf.random.uniform([],0,1) # this is beta dist with alpha=1.0\n        WIDTH = tf.cast( DIM * tf.math.sqrt(1-b),tf.int32) * P\n        ya = tf.math.maximum(0,y-WIDTH//2)\n        yb = tf.math.minimum(DIM,y+WIDTH//2)\n        xa = tf.math.maximum(0,x-WIDTH//2)\n        xb = tf.math.minimum(DIM,x+WIDTH//2)\n        # MAKE CUTMIX IMAGE\n        one = image[j,ya:yb,0:xa,:]\n        two = image[k,ya:yb,xa:xb,:]\n        three = image[j,ya:yb,xb:DIM,:]\n        middle = tf.concat([one,two,three],axis=1)\n        img = tf.concat([image[j,0:ya,:,:],middle,image[j,yb:DIM,:,:]],axis=0)\n        imgs.append(img)\n        # MAKE CUTMIX LABEL\n        a = tf.cast(WIDTH*WIDTH/DIM/DIM,tf.float32)\n        if len(label.shape)==1:\n            lab1 = tf.one_hot(label[j],CLASSES)\n            lab2 = tf.one_hot(label[k],CLASSES)\n        else:\n            lab1 = label[j,]\n            lab2 = label[k,]\n        lab1 = tf.cast(lab1,tf.float32)\n        lab2 = tf.cast(lab2,tf.float32)\n        labs.append((1-a)*lab1 + a*lab2)\n            \n    # RESHAPE HACK SO TPU COMPILER KNOWS SHAPE OF OUTPUT TENSOR (maybe use Python typing instead?)\n    image2 = tf.reshape(tf.stack(imgs),(AUG_BATCH,DIM,DIM,3))\n    label2 = tf.reshape(tf.stack(labs),(AUG_BATCH,CLASSES))\n    return image2,label2\n\n\ndef mixup(image, label, PROBABILITY = 1.0):\n    # input image - is a batch of images of size [n,dim,dim,3] not a single image of [dim,dim,3]\n    # output - a batch of images with mixup applied\n    DIM = IMSIZE[IMS]\n    CLASSES = 19\n    \n    imgs = []; labs = []\n    for j in range(AUG_BATCH):\n        # DO MIXUP WITH PROBABILITY DEFINED ABOVE\n        P = tf.cast( tf.random.uniform([],0,1)<=PROBABILITY, tf.float32)\n        # CHOOSE RANDOM\n        k = tf.cast( tf.random.uniform([],0,AUG_BATCH),tf.int32)\n        a = tf.random.uniform([],0,1)*P # this is beta dist with alpha=1.0\n        # MAKE MIXUP IMAGE\n        img1 = image[j,]\n        img2 = image[k,]\n        imgs.append((1-a)*img1 + a*img2)\n        # MAKE CUTMIX LABEL\n        if len(label.shape)==1:\n            lab1 = tf.one_hot(label[j],CLASSES)\n            lab2 = tf.one_hot(label[k],CLASSES)\n        else:\n            lab1 = label[j,]\n            lab2 = label[k,]\n        lab1 = tf.cast(lab1,tf.float32)\n        lab2 = tf.cast(lab2,tf.float32)\n        labs.append((1-a)*lab1 + a*lab2)\n            \n    # RESHAPE HACK SO TPU COMPILER KNOWS SHAPE OF OUTPUT TENSOR (maybe use Python typing instead?)\n    image2 = tf.reshape(tf.stack(imgs),(AUG_BATCH,DIM,DIM,3))\n    label2 = tf.reshape(tf.stack(labs),(AUG_BATCH,CLASSES))\n    return image2,label2\n\n\ndef transform(image,label):\n    # THIS FUNCTION APPLIES BOTH CUTMIX AND MIXUP\n    DIM = IMSIZE[IMS]\n    CLASSES = 19\n    SWITCH = 0.5\n    CUTMIX_PROB = 0.666\n    MIXUP_PROB = 0.666\n    # FOR SWITCH PERCENT OF TIME WE DO CUTMIX AND (1-SWITCH) WE DO MIXUP\n    image2, label2 = cutmix(image, label, CUTMIX_PROB)\n    image3, label3 = mixup(image, label, MIXUP_PROB)\n    imgs = []; labs = []\n    for j in range(AUG_BATCH):\n        P = tf.cast( tf.random.uniform([],0,1)<=SWITCH, tf.float32)\n        imgs.append(P*image2[j,]+(1-P)*image3[j,])\n        labs.append(P*label2[j,]+(1-P)*label3[j,])\n    # RESHAPE HACK SO TPU COMPILER KNOWS SHAPE OF OUTPUT TENSOR (maybe use Python typing instead?)\n    image4 = tf.reshape(tf.stack(imgs),(AUG_BATCH,DIM,DIM,3))\n    label4 = tf.reshape(tf.stack(labs),(AUG_BATCH,CLASSES))\n    return image4,label4\n\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def auto_select_accelerator():\n    try:\n        tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n        tf.config.experimental_connect_to_cluster(tpu)\n        tf.tpu.experimental.initialize_tpu_system(tpu)\n        strategy = tf.distribute.experimental.TPUStrategy(tpu)\n        print(\"Running on TPU:\", tpu.master())\n    except ValueError:\n        strategy = tf.distribute.get_strategy()\n    print(f\"Running on {strategy.num_replicas_in_sync} replicas\")\n    \n    return strategy\n\n\ndef build_decoder(with_labels=True, target_size=(256, 256), ext='jpg'):\n    def decode(path):\n        file_bytes = tf.io.read_file(path)\n\n        if ext == 'png':\n            img = tf.image.decode_png(file_bytes, channels=3)\n        elif ext in ['jpg', 'jpeg']:\n            img = tf.image.decode_jpeg(file_bytes, channels=3)\n        else:\n            raise ValueError(\"Image extension not supported\")\n        img = tf.cast(img, tf.float32) / 255.0\n        img = tf.image.resize(img, target_size)\n\n        return img\n    \n    def decode_with_labels(path, label):\n        return decode(path), label\n    \n    return decode_with_labels if with_labels else decode\n\n\ndef build_augmenter(with_labels=True):\n    def augment(img):\n        img = tf.image.random_flip_left_right(img)\n        img = tf.image.random_flip_up_down(img)\n        return img\n    \n    def augment_with_labels(img, label):\n        return augment(img), label\n    \n    return augment_with_labels if with_labels else augment\n\nDIM =256\nn_class = 19\ndef _parse_image_function(example_proto,augment = True):\n    image_feature_description = {\n        'image': tf.io.FixedLenFeature([], tf.string),\n        'label': tf.io.FixedLenFeature([], tf.string)\n    }\n    single_example = tf.io.parse_single_example(example_proto, image_feature_description)\n    image = tf.reshape( tf.io.decode_raw(single_example['image'],out_type=np.dtype('uint8')), (DIM,DIM, 3))\n    mask =  tf.reshape(tf.io.decode_raw(single_example['label'],out_type=np.dtype('uint8')),[n_class])\n    image = tf.dtypes.cast(image, tf.float32)\n    mask = tf.dtypes.cast(mask, tf.float32)\n    image = image/255.\n    if augment: # https://www.kaggle.com/kool777/training-hubmap-eda-tf-keras-tpu\n\n        if tf.random.uniform(()) > 0.5:\n            image = tf.image.flip_left_right(image)\n            mask = tf.image.flip_left_right(mask)\n\n        if tf.random.uniform(()) > 0.4:\n            image = tf.image.flip_up_down(image)\n            mask = tf.image.flip_up_down(mask)\n\n        if tf.random.uniform(()) > 0.5:\n            image = tf.image.rot90(image, k=1)\n            mask = tf.image.rot90(mask, k=1)\n\n        if tf.random.uniform(()) > 0.45:\n            image = tf.image.random_saturation(image, 0.7, 1.3)\n\n        if tf.random.uniform(()) > 0.45:\n            image = tf.image.random_contrast(image, 0.8, 1.2)\n    \n    return tf.cast(image, tf.float32),tf.cast(mask, tf.float32)\n\ndef load_dataset(filenames, ordered=False, augment = False):\n    AUTO = tf.data.experimental.AUTOTUNE\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO)\n    dataset = dataset.with_options(ignore_order)\n    dataset = dataset.map(lambda ex: _parse_image_function(ex, augment = augment), num_parallel_calls=AUTO)\n    return dataset\n\n\n\n\n\n\ndef build_dataset(paths, labels=None, bsize=128, cache=True,\n                  decode_fn=None, augment_fn=None,\n                  augment=True,augment_mixup_cutmix=False\n                  , repeat=True, shuffle=1024, \n                  cache_dir=\"\"):\n    if cache_dir != \"\" and cache is True:\n        os.makedirs(cache_dir, exist_ok=True)\n    \n    if decode_fn is None:\n        decode_fn = build_decoder(labels is not None)\n    \n    if augment_fn is None:\n        augment_fn = build_augmenter(True)\n    \n    AUTO = tf.data.experimental.AUTOTUNE\n    #slices = paths if labels is None else (paths, labels)\n    dset = load_dataset(paths)\n    #dset = tf.data.Dataset.from_tensor_slices(slices)\n    #dset = dset.map(decode_fn, num_parallel_calls=AUTO)\n    dset = dset.cache(cache_dir) if cache else dset\n    #dset = dset.map(transform, num_parallel_calls=AUTO) if augment else dset\n    dset = dset.map(augment_fn, num_parallel_calls=AUTO) if augment else dset\n    dset = dset.repeat() if repeat else dset\n    dset = dset.shuffle(shuffle) if shuffle else dset\n    dset = dset.batch(bsize)\n    dset = dset.map(transform,num_parallel_calls=AUTO) if augment_mixup_cutmix else dset\n    dset = dset.prefetch(AUTO)\n    \n    return dset","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def count_data_items(filenames):\n    img_num = 0\n    for i in filenames:\n        img_num +=int(i.split('-')[-1].split('.tf')[0])\n    return img_num","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''jpg\ngs_list=[\n'gs://kds-a5e7e2dcae43f711e19239e1403cdcb801efc84fbed5c79873c8e0e0',\n'gs://kds-b79fc4bc0e8a9d183d87e6a9ed366335e32f053e3b43d89dbb444402',\n'gs://kds-9df711802837f97da8bb5ba364a7d923b628992cc836779524f186bf',\n'gs://kds-fb1626515f2d252abe62121eaf1b55af664a395f0480eaaa94862ac3',\n'gs://kds-a6e4790428249fa3b1e531e73da79f60d1e573a79689cc200de28bb2'\n]\n''' \n'''\ngs_list = [\n'gs://kds-ee2bbb7b1b93a3aca6594b057105767bf7740a87e2c9e737df2bd14e',\n'gs://kds-0127ffcbb8111d1467f791a3c1827f88fd682f420d074fcf3f5d5114',\n'gs://kds-45c77f30a5cdf65369189f85ee40fd4a0f2cf13eb58e6be037942c27',\n'gs://kds-90dcc93f1059ed7ab006ccc06e8013f83dc003e3c2000c6f7fa1e220',\n'gs://kds-fb300e8fd046e2a7ae9baa24ab75f0e2fffdd643d56429fa3bf52edb'           \n           \n]\n'''\n#['gs://kds-a3cbdd21eaaacbeed971d397811f1d0c5629f5f3cd94676470c11403'] hpasinglecell0png\n#gs://kds-37d0364b7a76e93071da034c72923c99e801a09c94f2aef2e6e257ea hpasinglecell1png\n#'gs://kds-19fc2671d87d00f0318a60fe2a3b9dea6ad96536a1faa53532417be9' hpasinglecell2png\n#gs://kds-783a2216109b925edb061147dd4bd7a5fd9fd22499918bc30af3caa7 hpasinglecell3png\n#'gs://kds-7ab9ae4e26848338cea01aa1d8ceef869749ed8aa28b8b7452e3ef2f' hpasinglecell4png","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# glob the data\n'''\nx_list=[]\nfor i in gs_list:\n  x_list.extend(tf.io.gfile.glob(i+'/cell_all_128/*'))\n  '''","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\ndf = pd.read_csv('../input/hpa-single-cell-image-classification/train.csv')\ndf = df[['ID','Label']]\nfor i in range(19):\n    df[f'{i}'] =0\n\nfor i in tqdm(range(df.shape[0])):\n    #df.loc[i,'ID']=df.loc[i,'Image'].split('/')[-1]+'_green'\n    a = str(df.loc[i,'Label']).split('|')\n    a_len = len(str(df.loc[i,'Label']).split('|'))\n    for j in range(a_len ):\n        df.loc[i,a[j]] = 1\n\ndf_onehot = df.copy()[['ID','Label','0','1','2','3','4','5','6','7','8','9','10','11','12','13','14','15','16','17','18']]\n'''","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nlabel_cols = df_onehot.columns[2:21]\ny_list=[]\nfor i in x_list:\n  ID = i.split('.png_')[0].split('/')[-1].split('_')[-2]\n  y_list.append(df_onehot[df_onehot['ID']==ID][label_cols].values)\n\n#y_list = np.load('y_list.npz')['arr_0']\n\n#y_list = y_list.astype(np.float32)\n'''","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#y_list = np.array(y_list)\n#y_list = y_list.astype(np.float32)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#y_list = np.squeeze(y_list)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#x_list = np.array(x_list)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\n(\n    train_paths, valid_paths, \n    train_labels, valid_labels\n) = train_test_split(x_list, y_list, test_size=0.2, random_state=42)\n'''","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#kf = MultilabelStratifiedShuffleSplit(n_splits=2, test_size=0.2, random_state=100)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nfor train_index, test_index in kf.split(x_list, y_list):\n   \n   train_paths, valid_paths = x_list[train_index], x_list[test_index]\n   train_labels, valid_labels = y_list[train_index], y_list[test_index]\n   break\n'''","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#'gs://kds-d2130e02868047c380c97c6ae5d31a4180f74d21ceefbf5d4c9e531e' 0\n#'gs://kds-53777409f1b5814b19e2bc8b542d4af1163b9ac03623463085b45210' ../input/hpatrain-tfrec-1\n#'gs://kds-37f6a0e35a9bb77b47960da902f2fad44df121760c080ff2a9a04257' hpatrain-tfrec-2\n#'gs://kds-e3c610c144b38135540fa8471f82968eab5f181530e876ba09d20fad' hpatrain-tfrec-3\n\ntrain_list=[]\n#ori\ntrain_list.extend(tf.io.gfile.glob('gs://kds-18467a34620ba1afe1f389fd285bf108447b09f584cbe7c55a500ebe/MYS*'))\ntrain_list.extend(tf.io.gfile.glob('gs://kds-099b48978fad967933337af13a66cb036007b1310d5b231711934bb2/MYS*'))\ntrain_list.extend(tf.io.gfile.glob('gs://kds-695d786331c70c00dde5644812f3b8ea8f2ae892cab2a4f48a2f9766/MYS*'))\n\n\n#ex\ntrain_list.extend(tf.io.gfile.glob('gs://kds-7b0babb233b7ca137d3618494467528c94939ab3ed6700474cc134e3/MYS*'))\ntrain_list.extend(tf.io.gfile.glob('gs://kds-1f4282d99006b8514585b07f4a361bcfb8d19454bb7872b81e7d37e7/MYS*'))\ntrain_list.extend(tf.io.gfile.glob('gs://kds-ac7d763f7f8eb506cb0b1868e31b45301e079bb1d509df6237aaf8db/MYS*'))\ntrain_list.extend(tf.io.gfile.glob('gs://kds-f2ed388129c9136f54e2645a1d8b30622c264e9dfcf1d59daab50c4f/MYS*'))\ntrain_list.extend(tf.io.gfile.glob('gs://kds-52c5b56223349671ebe5f640fb11981cd40a1c052da071daeca2ba8a/MYS*'))\ntrain_list.extend(tf.io.gfile.glob('gs://kds-81e6dfe132eb32b60d9d018d968e64a7633e530b5a212a50d6e9ded9/MYS*'))\n\ntrain_list.extend(tf.io.gfile.glob('gs://kds-f2ed388129c9136f54e2645a1d8b30622c264e9dfcf1d59daab50c4f/MYS*'))\ntrain_list.extend(tf.io.gfile.glob('gs://kds-52c5b56223349671ebe5f640fb11981cd40a1c052da071daeca2ba8a/MYS*'))\ntrain_list.extend(tf.io.gfile.glob('gs://kds-81e6dfe132eb32b60d9d018d968e64a7633e530b5a212a50d6e9ded9/MYS*'))\n#train_list = tf.io.gfile.glob('gs://kds-6922a5ae3ea86268c632cb56f65631de76976e8d47d9b775ff699612/train_*')\n\n#ex\nvalid_list = []\ntrain_list.extend(tf.io.gfile.glob('gs://kds-da16be69b2e2a909bc95a2cffd9efede5d035fcb9793591520d96332/MYS*'))\ntrain_list.extend(tf.io.gfile.glob('gs://kds-a28dfde12325e031ae832c7e818f84c660a297284d409b20ed496c63/MYS*'))\ntrain_list.extend(tf.io.gfile.glob('gs://kds-dd992e3678f2e10af911cd0f85c5925d9bf9ed530161c790b780c23f/MYS*'))\nvalid_list.extend(tf.io.gfile.glob('gs://kds-df3fc43230853e82f685f4c3029c6238b8d29cfb98d20ae635a0d2d2/MYS*'))\ntrain_list.extend(tf.io.gfile.glob('gs://kds-30b9f186e0fe2978ad22c7f59109e4cc4e65ec9896839e5ca0b10ccb/MYS*'))\ntrain_list.extend(tf.io.gfile.glob('gs://kds-06683f7151402f921a76bdb3dbee0b711d13e8de449343e09a8404dd/MYS*'))\ntrain_list.extend(tf.io.gfile.glob('gs://kds-60c69c7f8822b869b35a0c1390c61041abf3b2932c1caced16df832d/MYS*'))\ntrain_list.extend(tf.io.gfile.glob('gs://kds-c9807109d89d8da8c8077075206ea4c048cd05fce211953e601a5f75/MYS*'))\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\n'gs://kds-18467a34620ba1afe1f389fd285bf108447b09f584cbe7c55a500ebe' MYS0101\n'gs://kds-099b48978fad967933337af13a66cb036007b1310d5b231711934bb2' MYS0102\n'gs://kds-695d786331c70c00dde5644812f3b8ea8f2ae892cab2a4f48a2f9766' MYS0103\n'gs://kds-7b0babb233b7ca137d3618494467528c94939ab3ed6700474cc134e3' MYS0104\n'gs://kds-1f4282d99006b8514585b07f4a361bcfb8d19454bb7872b81e7d37e7' MYS0105\n'gs://kds-ac7d763f7f8eb506cb0b1868e31b45301e079bb1d509df6237aaf8db' MYS0106\n'gs://kds-f2ed388129c9136f54e2645a1d8b30622c264e9dfcf1d59daab50c4f' MYS0107\n'gs://kds-52c5b56223349671ebe5f640fb11981cd40a1c052da071daeca2ba8a' MYS0108\n\n'gs://kds-81e6dfe132eb32b60d9d018d968e64a7633e530b5a212a50d6e9ded9' MYS0201\n'gs://kds-da16be69b2e2a909bc95a2cffd9efede5d035fcb9793591520d96332' MYS0202\n'gs://kds-a28dfde12325e031ae832c7e818f84c660a297284d409b20ed496c63' MYS0203\n'gs://kds-dd992e3678f2e10af911cd0f85c5925d9bf9ed530161c790b780c23f' MYS0204\n'gs://kds-df3fc43230853e82f685f4c3029c6238b8d29cfb98d20ae635a0d2d2' MYS0205\n'gs://kds-30b9f186e0fe2978ad22c7f59109e4cc4e65ec9896839e5ca0b10ccb' MYS0206\n'gs://kds-06683f7151402f921a76bdb3dbee0b711d13e8de449343e09a8404dd' MYS0207\n'gs://kds-60c69c7f8822b869b35a0c1390c61041abf3b2932c1caced16df832d' MYS0208\n'gs://kds-c9807109d89d8da8c8077075206ea4c048cd05fce211953e601a5f75' MYS0209\n'''","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#'gs://kds-c90356b0c1a5565426b91f2ffea10276766a0b17367e43f88882d164' hpaexset-02tfrec\n#'gs://kds-42aa49ce9fc8212676e97da0be4f4b4ddbcb0f04aa5a7481a97aac08' hpaexset-3-5tfrec\n#'gs://kds-bc31e123b5089b0128a4f3e872c3dfbb6b3545a2e87468449cbda814' hpaexset-6-8tfrec\n#'gs://kds-871d044987171af8883d7c7f98174dc75853a4a5eb5f28f6a30d0eb7' hpaexset-9-11\n#'gs://kds-75bbca92fb91ce73f2e1c5eaa27578caa91308d0f702d9aa0b77fbce' hpaexset-12-14tfrec\n#'gs://kds-cd5df74554678136acf099d07fdc0b632dd162396e04f51616040fa8' hpaexset-15-17tfrec\n#'gs://kds-96c49accad963db1c4068dadc7132d6495323b4785eae6522b537115' hpaexset-18-20tfrec\n#'gs://kds-c0fa84156d8e93032e87c17985a0c4051c300863de62a3f341d94acc' hpaexset-21-23tfrec\n# 'gs://kds-a6c822533c6fa25a6358bb47e20d988305def64a9790104bfb4e3596' hpaexset-24-26tfrec\n#'gs://kds-de5b3c0fdfbe51f9fc2e36af8665b2b76bf50daed0f7b2b705c8e0c6' hpaexset-27-30tfrec\n# 'gs://kds-5566487c260e590cfab3f0b9c79fc433988134436a6ee939860b5d44' hpaexset-31-34tfrec\n#'gs://kds-bc6e8d8aeb83e22f6839287353c7ee17784d3c1d852528a12679a6a8' hpaexset-35-37tfrec","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IMSIZE = (224, 240, 260, 300, 380, 456, 528, 600, 256)\nIMS = 8\n\nstrategy = auto_select_accelerator()\nBATCH_SIZE = strategy.num_replicas_in_sync * 32\nAUG_BATCH = BATCH_SIZE\n\n\n\n\ntrain_dataset = build_dataset(\n    train_list,  bsize=BATCH_SIZE,cache=False,augment_mixup_cutmix=True\n)\n\nvalid_dataset = build_dataset(\n    valid_list,  bsize=BATCH_SIZE,\n    cache=False,repeat=False, shuffle=False, augment=False\n)\n\n#decoder = build_decoder(with_labels=True, target_size=(IMSIZE[IMS], IMSIZE[IMS]))\n#test_decoder = build_decoder(with_labels=False, target_size=(IMSIZE[IMS], IMSIZE[IMS]))\n'''\ntrain_dataset = build_dataset(\n    train_paths, train_labels, bsize=BATCH_SIZE, decode_fn=decoder,cache=False,augment_mixup_cutmix=True\n)\n'''\n''' dataset checking\nfrom tqdm import tqdm\nrow = 6; col = 4;\nrow = min(row,BATCH_SIZE//col)\nfor (img,label) in tqdm(train_dataset):\n    plt.figure(figsize=(15,int(15*row/col)))\n    for j in range(row*col):\n        plt.subplot(row,col,j+1)\n        plt.axis('off')\n        plt.imshow(img[j,])\n    plt.show()\n    break\n'''\n'''\nvalid_dataset = build_dataset(\n    valid_paths, valid_labels, bsize=BATCH_SIZE, decode_fn=decoder,\n    cache=False,repeat=False, shuffle=False, augment=False\n)\n'''","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nwith strategy.scope():\n    \n    model = tf.keras.models.load_model(\n            '../input/hpa-training-tpu/model_RGB_0.2186.h5'\n        )\n'''","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tqdm import tqdm","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nhigh_confidence=0\nimg_num=[]\nlabel=[]\ntotal= len(result)*19\nfor i in tqdm(range(len(result))):\n    first=1\n    for j in range(len(result[i])):\n        if result[i][j]>0.8:\n            \n            if first:\n                img_num.append(i)\n                label.append([])\n                first=0\n            label[-1].append(j)\n            high_confidence+=1\n'''","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# The following functions can be used to convert a value to a type compatible\n# with tf.Example.\n\ndef _bytes_feature(value):\n  \"\"\"Returns a bytes_list from a string / byte.\"\"\"\n  if isinstance(value, type(tf.constant(0))):\n    value = value.numpy() # BytesList won't unpack a string from an EagerTensor.\n  return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))\n\ndef _float_feature(value):\n  \"\"\"Returns a float_list from a float / double.\"\"\"\n  return tf.train.Feature(float_list=tf.train.FloatList(value=[value]))\n\ndef _int64_feature(value):\n  \"\"\"Returns an int64_list from a bool / enum / int / uint.\"\"\"\n  return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def en_example(l,img):\n    feature = {\n      'label': _bytes_feature(l.tobytes()),\n      'image': _bytes_feature(img.tobytes()),\n    }\n    return tf.train.Example(features=tf.train.Features(feature=feature))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nkt=[]\nrecord_file='temp.tfrecords'\ncnt = 0\nwith tf.io.TFRecordWriter(record_file) as writer:\n    for i in valid_dataset:\n    \n        kt.append(model.predict(i[0]))\n\n        for value,pred,true in zip(i[0],kt[-1],i[1]):\n\n                true_amount_b = np.sum(true)\n                true += pred>0.8\n                true -= pred<0.1\n                true = np.array(true)\n                true[true<0] = 0.\n                true[true>1] = 1.\n                if np.sum(true) ==0:\n                    if true[18]>0.2:\n                        true[18]=1\n                true_amount_a = np.sum(true)\n                print(true_amount_a)\n                if true_amount_a == 0:\n                    continue\n                \n                true = np.array(true,dtype='uint8')\n                value = np.array(value,dtype='uint8')\n\n                exp = en_example(true,value)\n                writer.write(exp.SerializeToString())\n                cnt+=1\n                print(cnt)\n            \n        \n        break\nos.rename(record_file,'-'+str(cnt)+'.tfrecords')\n'''","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# define the model\n\ntry:\n    n_labels = 19#train_labels.shape[1]\nexcept:\n    n_labels = 1\n    \nwith strategy.scope():\n    '''\n    model = tf.keras.models.load_model(\n            '/content/gdrive/MyDrive/Kaggle/model_RGB_0.0051.h5'\n        )\n    '''\n    \n    model = tf.keras.Sequential([\n        efn.EfficientNetB0(\n            input_shape=(IMSIZE[IMS], IMSIZE[IMS], 3),\n            weights='imagenet',\n            include_top=False),\n        tf.keras.layers.GlobalAveragePooling2D(),\n        tf.keras.layers.Dense(n_labels, activation='sigmoid')\n    ])\n    \n    model.compile(\n        optimizer=tf.keras.optimizers.Adam(),\n        loss='binary_crossentropy',#BinaryFocalLoss(gamma=5),#'binary_crossentropy',#tfa.losses.SigmoidFocalCrossEntropy(),\n        metrics=[tf.keras.metrics.AUC(multi_label=True)])\n        \n    model.summary()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"colour = '_RGB'\n#steps_per_epoch = len(train_paths) // BATCH_SIZE\nsteps_per_epoch = count_data_items(train_list) // BATCH_SIZE\ncheckpoint = tf.keras.callbacks.ModelCheckpoint(\n    \"model_RGB_{val_loss:.4f}.h5\", save_best_only=False, monitor='val_loss', mode='min')\nlr_reducer = tf.keras.callbacks.ReduceLROnPlateau(\n    monitor=\"val_loss\", patience=3, min_lr=1e-6, mode='min')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model.fit(\n    train_dataset, \n    epochs=2,\n    verbose=1,\n    callbacks=[checkpoint, lr_reducer],\n    steps_per_epoch=steps_per_epoch,\n    validation_data=valid_dataset)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hist_df = pd.DataFrame(history.history)\nhist_df.to_csv(f'history{colour}.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}