{"cells":[{"metadata":{"_uuid":"85c95b80d8ec01e7e50b8637b5676ac6a667b0d9"},"cell_type":"markdown","source":"* no data expand, but augment  \n* try batch_size=128  \n* pre-train\n* tanh\n* no cache"},{"metadata":{"trusted":true,"_uuid":"71f01dbdab05a84cd863524166ca5e43b21ed1b6"},"cell_type":"code","source":"ls -la ../input","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"501bc5c48766ec5964873f6dc7ebd4007adae413"},"cell_type":"code","source":"%matplotlib inline\nfrom IPython.display import SVG\nfrom keras.utils.vis_utils import model_to_dot","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport shutil\nprint(os.listdir(\"../input\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d8bb2129a2d5f5316ddb07c5eda9cd4da84af18e"},"cell_type":"code","source":"ls -la ../input/keras-pretrained-models","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"de51410914c593a4eebf89a5635a09c7f96c0969"},"cell_type":"code","source":"try:\n    os.makedirs('/tmp/.keras/datasets')\nexcept FileExistsError:\n    pass","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4d7cb1b9173e0653687b633f695ff57c182e8cc4"},"cell_type":"code","source":"try:\n    shutil.copytree(\"../input/keras-pretrained-models\", \"/tmp/.keras/models\")\nexcept FileExistsError:\n    pass","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"import os.path\nimport itertools\nfrom itertools import chain\n\nimport numpy as np\nimport pandas as pd\nfrom sklearn import datasets\nfrom sklearn import preprocessing\nfrom sklearn.decomposition import PCA\nfrom sklearn import cluster, datasets, mixture\nfrom sklearn.datasets import load_digits\nfrom sklearn.preprocessing import StandardScaler, OneHotEncoder\nfrom sklearn.metrics import f1_score, classification_report, confusion_matrix\nfrom sklearn.model_selection import train_test_split\n\nimport matplotlib.pyplot as plt\nfrom matplotlib.colors import ListedColormap\nimport seaborn as sns\n\nimport tensorflow as tf\n\nfrom keras.layers import Input, Embedding, LSTM, GRU, Dense, Dropout, Lambda, \\\n    Conv1D, Conv2D, Conv3D, \\\n    Conv2DTranspose, \\\n    AveragePooling1D, AveragePooling2D, \\\n    MaxPooling1D, MaxPooling2D, MaxPooling3D, \\\n    GlobalAveragePooling1D, \\\n    GlobalMaxPooling1D, GlobalMaxPooling2D, GlobalMaxPooling3D, \\\n    LocallyConnected1D, LocallyConnected2D, \\\n    concatenate, Flatten, Average, Activation, \\\n    RepeatVector, Permute, Reshape, Dot, \\\n    multiply, dot, add, \\\n    PReLU, \\\n    Bidirectional, TimeDistributed, \\\n    SpatialDropout1D, \\\n    BatchNormalization\nfrom keras.models import Model, Sequential\nfrom keras import losses\nfrom keras.callbacks import BaseLogger, ProgbarLogger, Callback, History\nfrom keras.callbacks import ModelCheckpoint, LearningRateScheduler, ReduceLROnPlateau\nfrom keras.wrappers.scikit_learn import KerasClassifier\nfrom keras import regularizers\nfrom keras import initializers\nfrom keras.metrics import categorical_accuracy\nfrom keras.constraints import maxnorm, non_neg\nfrom keras.optimizers import RMSprop\nfrom keras.utils import to_categorical, plot_model\nfrom keras import backend as K","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"50fefbf82ee590bd43c1c0e8cc5034b3796652cc"},"cell_type":"code","source":"from PIL import Image\nfrom zipfile import ZipFile\nimport h5py\nimport cv2\nfrom tqdm import tqdm","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"26885390f9a1d5f8b1be5f30b9933a44ac200dbe"},"cell_type":"code","source":"src_dir = '../input/human-protein-atlas-image-classification'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"256ab9500ecf1492676ac5a56d630b916dce093a"},"cell_type":"code","source":"train_labels = pd.read_csv(os.path.join(src_dir, \"train.csv\"))\nprint(train_labels.shape)\ntrain_labels.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"27681f3b3e0ae9b0fe682270acdf0fe9aa2d8738"},"cell_type":"code","source":"test_labels = pd.read_csv(os.path.join(src_dir, \"sample_submission.csv\"))\nprint(test_labels.shape)\ntest_labels.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"085662b1058aa000cdbe2cf7ab923b46910b0004"},"cell_type":"code","source":"def show_arr(arr, nrows = 1, ncols = 4, figsize=(15, 5)):\n    fig, subs = plt.subplots(nrows=nrows, ncols=ncols, figsize=figsize)\n    for ii in range(ncols):\n        iplt = subs[ii]\n        try:\n            img_array = arr[:,:,ii]\n            if ii == 0:\n                cp = 'Greens'\n            elif ii == 1:\n                cp = 'Blues'\n            elif ii == 2:\n                cp = 'Reds'\n            else:\n                cp = 'Oranges'\n            iplt.imshow(img_array, cmap=cp)\n        except:\n            pass","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"66967a6fc0a40138dcf8f7cc80a7f17a5fc3426b"},"cell_type":"code","source":"def get_arr0(Id, test=False):\n    def fn(Id, color, test=False):\n        if test:\n            tgt = 'test'\n        else:\n            tgt = 'train'\n        with open(os.path.join(src_dir, tgt, Id+'_{}.png'.format(color)), 'rb') as fp:\n            img = Image.open(fp)\n            arr = (np.asarray(img) / 255.)\n        return arr\n    res = []\n    for icolor in ['green', 'blue', 'red', 'yellow']:\n        arr0 = fn(Id, icolor, test)\n        res.append(arr0)\n    arr = np.stack(res, axis=-1)\n    return arr","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"aaef824e54b9e397a2f8a980337f266a1fe7e413"},"cell_type":"code","source":"arr = get_arr0('00008af0-bad0-11e8-b2b8-ac1f6b6435d0', test=True)\nprint(arr.shape)\nshow_arr(arr)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"26e4815ccced55f2d31830e2134c63c52d592db3"},"cell_type":"code","source":"arr = get_arr0('00070df0-bbc3-11e8-b2bc-ac1f6b6435d0')\nprint(arr.shape)\nshow_arr(arr)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2906351bbf846b82ea7c2d7d12b83a3443baf984"},"cell_type":"code","source":"SH = (256, 256)\nID_LIST_TRAIN = train_labels.Id.tolist()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7b0eb63dc51ce06412c7be3db185adae129f4d39"},"cell_type":"code","source":"def get_arr(Id, test=False):\n    if test:\n        arr = get_arr0(Id, test=True)\n    else:\n        arr = get_arr0(Id)\n    arr = cv2.resize(arr, SH).astype('float32')\n    return arr","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"567e3046dd0a4a0fcc641b46e4ac6b1d99da7d04"},"cell_type":"code","source":"arr = get_arr('00070df0-bbc3-11e8-b2bc-ac1f6b6435d0')\nprint(arr.shape)\nshow_arr(arr)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8220a9292dc9ba8977d491f55a3d7a5236dd03e8"},"cell_type":"code","source":"arr = get_arr('00008af0-bad0-11e8-b2b8-ac1f6b6435d0', test=True)\nprint(arr.shape)\nshow_arr(arr)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6a2dfc516268368e147cc176ba74392f08cd445f"},"cell_type":"code","source":"y_cat_train_dic = {}\nfor icat in range(28):\n    target = str(icat)\n    y_cat_train_5 = np.array([int(target in ee.split()) for ee in train_labels.Target.tolist()])\n    y_cat_train_dic[icat] = y_cat_train_5","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9b22a3224e0683d580593af45fcf81f3a1c8c083"},"cell_type":"code","source":"up_sample = {}\nfor k in y_cat_train_dic:\n    v = y_cat_train_dic[k].sum()\n    up_sample[k] = np.ceil((train_labels.shape[0]/28) / v)\n\nup_sample","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"79abf3891d5990af8297cb75b01afb2bb263ffd5"},"cell_type":"code","source":"up_sample2 = list(zip(*sorted(list(up_sample.items()), key=lambda x: x[0])))[1]\nup_sample2 = np.array(up_sample2)\nup_sample2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"aee7e5a06bfe0ceaf47218f3c3610429a5c1c035"},"cell_type":"code","source":"import random\n\nclass Seq(object):\n    sections = None\n    index = None\n    \n    def __init__(self, df, extend=False, aug=False, test=False, batch_size=32):\n        self.shaffle = None\n        self.extend = extend\n        self.aug = aug\n        self.test = test\n        self.batch_size = batch_size\n        self.df = df\n        \n        # proccess\n        self.ids = self.df.Id.tolist()\n        self.reversed = sorted(range(SH[0]), reverse=True)\n        \n        # estimate self length\n        self.initialize_it()\n        self.len = 1\n        for _ in self.it:\n            self.len += 1\n        \n        self.initialize_it()\n    \n    def initialize_it(self):\n        if self.shaffle:\n            '''not implemented yet'''\n            raise NotImplementedError\n            #random.seed(self.state)\n            #random.shuffle(self.ids)\n        \n        self.it = iter(range(0, len(self.ids), self.batch_size))\n        self.idx_next = self.it.__next__()\n    \n    def __len__(self):\n        return self.len\n    \n    def __iter__(self):\n        return self\n    \n    def __next__(self):\n        idx = self.idx_next\n        self.ids_part = self.ids[idx:((idx+self.batch_size) if idx+self.batch_size<len(self.ids) else len(self.ids))]\n        res = self.getpart(self.ids_part)\n        try:\n            self.idx_next = self.it.__next__()\n        except StopIteration:\n            self.initialize_it()\n        return res\n    \n    def __getitem__(self, id0):\n        arr, tgts = self.get_data(id0)\n        cat = self.convert_tgts(tgts)\n        return arr, cat\n    \n    k_list = list(range(4))\n    def random_transform(self, arr):\n        k = random.choice(self.k_list)\n        arr0 = np.rot90(arr, k=k)\n        if random.randint(0,1):\n            arr0 = arr0[self.reversed,:,:]\n        if random.randint(0,1):\n            arr0 = arr0[:,self.reversed,:]\n        return arr0\n    \n    def convert_tgts(self, tgts):\n        try:\n            cats = to_categorical(tgts, num_classes=28)\n            cat = cats.sum(axis=0)\n        except TypeError:\n            cat = np.zeros((28,))\n        return cat\n    \n    def get_data(self, id0):\n        arr = get_arr(id0, test=self.test)\n        \n        try:\n            y0 = (self.df.Target[self.df.Id == id0]).tolist()[0]\n            y1 = y0.split()\n            y = [int(ee) for ee in y1]\n        except AttributeError:\n            y = None\n        return arr, y\n    \n    def getpart(self, ids):\n        xs = []\n        ys = []\n        for id0 in ids:\n            self.extend_data(id0, xs, ys)\n        \n        x = np.stack(xs)\n        y = np.stack(ys)\n        x_dummy = np.zeros((len(x), 1))\n        x_ret = {\n            'input': x,\n            'input_cls': y,\n        }\n        y_ret = {\n            'path_fit_cls_img': x_dummy,\n            'path_cls_img_cls': y,\n            'path_fit_imgA': x_dummy,\n            'path_fit_img_cls_img': x_dummy,\n            'path_cls_cls': y,\n            'path_img_cls': y,\n            'path_img_img_cls': y,\n            'path_fit_cls_img_imgE': x_dummy,\n        }\n        return (x_ret, y_ret)\n    \n    def split(self, arr, sections=sections):\n        res0 = np.vsplit(arr, sections)\n        res = [np.hsplit(ee, sections) for ee in res0]\n        res = list(chain.from_iterable(res))\n        return res\n    \n    def extend_data(self, id0, xs, ys):\n        arr0, cat = self[id0]\n        \n        # data augmentation\n        if self.extend:\n            mm = up_sample2[cat==1].max()\n            mm = int(mm)\n            #print(mm)\n            for ii in range(mm):\n                if self.aug:\n                    img = self.random_transform(arr0)\n                else:\n                    img = arr0\n                xs.append(img.flatten())\n                ys.append(cat)\n        else:\n            if self.aug:\n                img = self.random_transform(arr0)\n            else:\n                img = arr0\n            xs.append(img.flatten())\n            ys.append(cat)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d98c20522bcf8014f0b9e4d2cee8d269bfcfdfaa"},"cell_type":"markdown","source":"### make model"},{"metadata":{"trusted":true,"_uuid":"163f04800116988c66e6aa65fe3caf65c57d3cee"},"cell_type":"code","source":"from keras import applications","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1f76a77d9d9608e0aef1db29cd147c40d9f511cd","scrolled":true},"cell_type":"code","source":"def make_trainable_false(model_resnet, trainable=False):\n    layers = model_resnet.layers\n    for ilayer in layers:\n        ilayer.trainable = trainable\n    return","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0b3c366671d5f7913952e10b35182ad3c867c4cb"},"cell_type":"code","source":"img_shape = tuple(list(SH) + [4])\nprint(img_shape)\nimg_dim = np.array(img_shape).prod()\nprint(img_dim)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4edf6b659cfc1a206db6ef6705be72fa34f80c9e"},"cell_type":"code","source":"def make_model_cnvt(img_dim, img_shape):\n    '''==============================\n    inputs\n    =============================='''\n    inp = Input(shape=(img_dim,))\n    oup = Reshape(img_shape)(inp)\n    #oup = Conv2D(3, kernel_size=1, strides=1, padding='same')(oup)\n    #oup = Conv2D(3, kernel_size=1, strides=1, padding='same', activation='sigmoid')(oup)\n    oup = Conv2D(3,\n                 kernel_size=1,\n                 strides=1,\n                 padding='same',\n                 activation='tanh',\n                 kernel_regularizer=regularizers.l2(1e-4))(oup)\n    #kernel_regularizer=regularizers.l2(1e-4)\n    model_cnvt = Model(inp, oup, name='model_cnvt')\n    return model_cnvt","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a62bfbcfc54d796359f72f3bc7eda9b20b8f9be6"},"cell_type":"code","source":"def make_model_classifier(input_dim=1536):\n    inp_cls = Input((input_dim,))\n    oup_cls = Dense(28)(inp_cls)\n    oup_cls = Activation('sigmoid')(oup_cls)\n    model_classifier = Model(inp_cls, oup_cls, name='classifier')\n    return model_classifier","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"632e6ed34edbce32760989c1750748fe125e7f7a"},"cell_type":"code","source":"def make_model(img_dim, model_cnvt, model_resnet, model_classifier):\n    '''==============================\n    inputs\n    =============================='''\n    inp = Input(shape=(img_dim,), name='input')\n    oup = model_cnvt(inp)\n    oup = model_resnet(oup)\n    oup = model_classifier(oup)\n    oup = Activation('linear', name='path_cls_cls')(oup)\n    \n    model = Model(inp, oup, name='model')\n    model.compile(loss='binary_crossentropy',\n                  optimizer='adam',\n                  metrics=['categorical_accuracy', 'binary_accuracy'])\n    \n    return {\n        'model_classifier': model_classifier,\n        'model_resnet': model_resnet,\n        'model_cnvt': model_cnvt,\n        'model': model\n    }","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b4545428eb68c11036c26f110a23e36675d86e83"},"cell_type":"code","source":"model_cnvt = make_model_cnvt(img_dim, img_shape)\nmodel_cnvt.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5cc70e0aeaf657838050bf2e9a8530eb7190dff5"},"cell_type":"code","source":"model_cnvt.layers[2].get_weights()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"df914f5b737824bbebe88647db823e118f4e2bf9"},"cell_type":"code","source":"model_cnvt.load_weights('../input/keras-inceptionresnetv2-resize139x139-005focal/model_5_cnvt.h5')\nmodel_cnvt.layers[2].get_weights()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8f5b82c71943f783049cd450dbc7904bca13dc49"},"cell_type":"code","source":"model_resnet = applications.inception_resnet_v2.InceptionResNetV2(\n    include_top=False,\n    weights='imagenet',\n    input_tensor=None,\n    input_shape=list(SH) + [3],\n    pooling='avg',\n    classes=None)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e86f3d63580b9e860341ba4c673fa982b946fd30"},"cell_type":"code","source":"# model_resnet.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"44de2ae03fd5e30c5cc509f0d1ef2a416551c291"},"cell_type":"code","source":"model_resnet.layers[1].get_weights()[0][0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2df25fe1d4ad2010f16e8365262129a6bea50d71"},"cell_type":"code","source":"model_resnet.load_weights('../input/keras-inceptionresnetv2-resize139x139-005focal/model_5_resnet.h5')\nmodel_resnet.layers[1].get_weights()[0][0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0383dd3682e287aa53a2129c7586cb3f731d1ab1"},"cell_type":"code","source":"model_classifier = make_model_classifier()\nmodel_classifier.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1b30e9a80a42bdeb0c55a36994a7ce2bbdbffe8e"},"cell_type":"code","source":"model_classifier.layers[1].get_weights()[0][0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"20179bb767bce79b515e652c6ed690d50f29be17"},"cell_type":"code","source":"model_classifier.load_weights('../input/keras-inceptionresnetv2-resize139x139-005focal/model_5_classifier.h5')\nmodel_classifier.layers[1].get_weights()[0][0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"296359eccad432b4e71ebf0305c676655a71d6a7"},"cell_type":"code","source":"models = make_model(img_dim, model_cnvt, model_resnet, model_classifier)\nmodels['model'].summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"22e93af07d24ace94e7972eaff4825cb026a2e59"},"cell_type":"code","source":"THRESHOLD = 0.5\n\n# credits: https://www.kaggle.com/guglielmocamporese/macro-f1-score-keras\n\nK_epsilon = K.epsilon()\ndef f1(y_true, y_pred):\n    #y_pred = K.round(y_pred)\n    y_pred = K.cast(K.greater(K.clip(y_pred, 0, 1), THRESHOLD), K.floatx())\n    tp = K.sum(K.cast(y_true*y_pred, 'float'), axis=0)\n    tn = K.sum(K.cast((1-y_true)*(1-y_pred), 'float'), axis=0)\n    fp = K.sum(K.cast((1-y_true)*y_pred, 'float'), axis=0)\n    fn = K.sum(K.cast(y_true*(1-y_pred), 'float'), axis=0)\n\n    p = tp / (tp + fp + K_epsilon)\n    r = tp / (tp + fn + K_epsilon)\n\n    f1 = 2*p*r / (p+r+K_epsilon)\n    f1 = tf.where(tf.is_nan(f1), tf.zeros_like(f1), f1)\n    return K.mean(f1)\n\ndef f1_loss(y_true, y_pred):\n    \n    #y_pred = K.cast(K.greater(K.clip(y_pred, 0, 1), THRESHOLD), K.floatx())\n    tp = K.sum(K.cast(y_true*y_pred, 'float'), axis=0)\n    tn = K.sum(K.cast((1-y_true)*(1-y_pred), 'float'), axis=0)\n    fp = K.sum(K.cast((1-y_true)*y_pred, 'float'), axis=0)\n    fn = K.sum(K.cast(y_true*(1-y_pred), 'float'), axis=0)\n\n    p = tp / (tp + fp + K_epsilon)\n    r = tp / (tp + fn + K_epsilon)\n\n    f1 = 2*p*r / (p+r+K_epsilon)\n    f1 = tf.where(tf.is_nan(f1), tf.zeros_like(f1), f1)\n    return 1-K.mean(f1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2e97be362cd6d1a8be3dff38f3bee75e741c9fbe"},"cell_type":"code","source":"models['model'].compile(loss=f1_loss,\n                        optimizer='adam',\n                        metrics=['categorical_accuracy', 'binary_accuracy', f1])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7e4672b15a810c37a7daea0dd28556cc4e21c690"},"cell_type":"markdown","source":"## #1"},{"metadata":{"trusted":true,"_uuid":"3c48abcccc9f42e8171ece01ac68753b4e330412","scrolled":false},"cell_type":"code","source":"seq = Seq(train_labels, extend=False, aug=True, batch_size=32)\nprint(len(seq))\n\nhst = models['model'].fit_generator(seq, epochs=3,\n                              steps_per_epoch=len(seq),\n                              callbacks=[])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1fbaaf76d643c90b547203367487383bcc953145"},"cell_type":"code","source":"fig, ax = plt.subplots(1, 2, figsize=(15,5))\nax[0].set_title('loss')\nax[0].plot(hst.epoch, hst.history[\"loss\"], label=\"Train loss\")\nax[1].set_title('acc')\nax[1].plot(hst.epoch, hst.history[\"categorical_accuracy\"], label=\"categorical_accuracy\")\nax[1].plot(hst.epoch, hst.history[\"binary_accuracy\"], label=\"binary_accuracy\")\nax[0].legend()\nax[1].legend()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7c34349ced0780c1f2da41c46c2817a2bd060c79"},"cell_type":"code","source":"seq = Seq(train_labels, extend=False, aug=False, batch_size=32)\nprint(len(seq))\nxs, ys = next(seq)\nprint(xs['input'].shape)\ny_pred = models['model_cnvt'].predict(xs['input'])\nprint(y_pred.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d24ee6e3d1f01a589bfb99439ff0bdd0352c2c9d"},"cell_type":"code","source":"y_pred[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"48d34f6e60f0b188105f0b52e6c8e6fee694f289"},"cell_type":"code","source":"show_arr(y_pred[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b7af3b04154d25bad46e786b168960b5d5e72dc7"},"cell_type":"code","source":"Image.fromarray(np.uint8((y_pred[0]+1)/2*255))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e5382d82e02e36aad43064fb0f46275115c43f59"},"cell_type":"markdown","source":"### predict and submit"},{"metadata":{"trusted":true,"_uuid":"a545b725c69cdc9c0305066b5d0e3456837cab27"},"cell_type":"code","source":"seq_pred = Seq(train_labels, test=False, aug=False, batch_size=128)\nlen(seq_pred)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"902ce70dd767b9317fcc50377c84b7ffbb037fa0"},"cell_type":"code","source":"pred = models['model'].predict_generator(seq_pred, steps=len(seq_pred), verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6b902e80ea41a107f7e827bc53753d6e0877e0fd"},"cell_type":"code","source":"def calc_threshold(pred):\n    ### calc threshold\n    threshold_dic = {}\n    for idx in tqdm(range(28)):\n        m = 0\n        for ii in range(100):\n            threshold0 = ii*0.01\n            f1_val = f1_score(y_cat_train_dic[idx], threshold0<(pred[:,idx]))\n            if m < f1_val:\n                threshold_dic[idx] = threshold0+0.005\n                m = f1_val\n    return threshold_dic","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5d8c7a7f99e2291e5b90cfdedd2fc7011c4844e0"},"cell_type":"code","source":"threshold_dic = calc_threshold(pred)\nthreshold_dic","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"beb2ae7253557b2a4a054ffe4bde530b82c13a6f"},"cell_type":"code","source":"seq_test = Seq(test_labels, test=True, aug=False, batch_size=128)\nseq_test","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a921f7ddefb2099a4d4332047fb8f69ea33e98cf"},"cell_type":"code","source":"pred_test = models['model'].predict_generator(seq_test, steps=len(seq_test), verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f8bbdce55d142033b6315f6a29f61a9c188ab0b4"},"cell_type":"code","source":"def make_test(pred):\n    test_labels1 = test_labels.copy()\n    test_labels1['Predicted'] = [str(ee) for ee in np.argmax(pred, axis=1)]\n    print(test_labels1.head())\n    #test_labels1.to_csv(fn0, index=False)\n    \n    test_labels2 = test_labels1.copy()\n    for ii in range(test_labels2.shape[0]):\n        threshold = list(zip(*sorted(list(threshold_dic.items()), key=lambda x:x[0], reverse=False)))[1]\n        idx = threshold < pred[ii,:]\n        tgt = test_labels2['Predicted'][ii]\n        tgt = [tgt] + [str(ee) for ee in np.arange(28)[idx]]\n        tgt = set(tgt)\n        tgt = ' '.join(tgt)\n        test_labels2['Predicted'][ii] = tgt\n    print(test_labels2.head())\n    #test_labels2.to_csv(fn, index=False)\n    return test_labels1, test_labels2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"87ef7ff12c86c548726785589c0f9977986ef1b3"},"cell_type":"code","source":"test_labels1_1, test_labels1_2 = make_test(pred_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"54a87bc7d9f288aca9bc864ec7f55da9062741a7"},"cell_type":"code","source":"test_labels1_2.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d85574855df017b70cc6ed75361f75ca630c69aa"},"cell_type":"code","source":"test_labels1_2.to_csv('InceptionResNetV1_2.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"83d2124b8f7d67f334e9087446147b20c3c15411"},"cell_type":"markdown","source":"## save weights for later loading"},{"metadata":{"trusted":true,"_uuid":"5de2c0d6f3c2fe522480dd79414b424ff7139774"},"cell_type":"code","source":"'''save weights for later loading'''\nNo = 1\nmodels['model_cnvt'].save_weights('model_{}_cnvt.h5'.format(No))\nmodels['model_resnet'].save_weights('model_{}_resnet.h5'.format(No))\nmodels['model_classifier'].save_weights('model_{}_classifier.h5'.format(No))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3804822d6103523ee070b695d367ec6d9d65aa32"},"cell_type":"code","source":"ls -la","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a7aa49c828b3153d5382504c31311da10ff7fd0f"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f60b43f1c21b1dc29b1d314db5f2d6bc757cb5ee"},"cell_type":"markdown","source":"## #2"},{"metadata":{"trusted":true,"_uuid":"4ad1678bcd5fa32c6dfadae3d72f877b47afb8a2"},"cell_type":"code","source":"seq = Seq(train_labels, extend=False, aug=True, batch_size=32)\nprint(len(seq))\n\nhst = models['model'].fit_generator(seq, epochs=2,\n                              steps_per_epoch=len(seq),\n                              callbacks=[])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0012fb54eb256d327e56a05845989a3ae3619652"},"cell_type":"code","source":"fig, ax = plt.subplots(1, 2, figsize=(15,5))\nax[0].set_title('loss')\nax[0].plot(hst.epoch, hst.history[\"loss\"], label=\"Train loss\")\nax[1].set_title('acc')\nax[1].plot(hst.epoch, hst.history[\"categorical_accuracy\"], label=\"categorical_accuracy\")\nax[1].plot(hst.epoch, hst.history[\"binary_accuracy\"], label=\"binary_accuracy\")\nax[0].legend()\nax[1].legend()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"db147403b2a31a0b28c4497afa85e2ea3e2674b4"},"cell_type":"code","source":"seq_pred = Seq(train_labels, test=False, aug=False, batch_size=128)\nlen(seq_pred)\npred = models['model'].predict_generator(seq_pred, steps=len(seq_pred), verbose=1)\nthreshold_dic = calc_threshold(pred)\nthreshold_dic","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"aecb300e3ccd22b9d70d287974d91bbeb82dc4c4"},"cell_type":"code","source":"seq_test = Seq(test_labels, test=True, aug=False, batch_size=128)\nseq_test\npred_test = models['model'].predict_generator(seq_test, steps=len(seq_test), verbose=1)\ntest_labels2_1, test_labels2_2 = make_test(pred_test)\ntest_labels2_2.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f2960ca4417198d888cea4ef923390efb7f47d76"},"cell_type":"code","source":"test_labels2_2.to_csv('InceptionResNetV2_2.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"30ea019e7f5009b1e33bd7a3c98bbdd22c784b2a"},"cell_type":"code","source":"'''save weights for later loading'''\nNo = 2\nmodels['model_cnvt'].save_weights('model_{}_cnvt.h5'.format(No))\nmodels['model_resnet'].save_weights('model_{}_resnet.h5'.format(No))\nmodels['model_classifier'].save_weights('model_{}_classifier.h5'.format(No))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0e6e7484df85d7c1204c6dd6a0d97a5732f77a2b"},"cell_type":"markdown","source":"## #3"},{"metadata":{"trusted":true,"_uuid":"78c83f61bcb968704eb941443be0c525fee726dd"},"cell_type":"code","source":"seq = Seq(train_labels, aug=True, batch_size=32)\nprint(len(seq))\n\nhst = models['model'].fit_generator(seq, epochs=2,\n                              steps_per_epoch=len(seq),\n                              callbacks=[])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"33e5b6f2636721cf25fc0d445831ba9fbaf4ae49"},"cell_type":"code","source":"fig, ax = plt.subplots(1, 2, figsize=(15,5))\nax[0].set_title('loss')\nax[0].plot(hst.epoch, hst.history[\"loss\"], label=\"Train loss\")\nax[1].set_title('acc')\nax[1].plot(hst.epoch, hst.history[\"categorical_accuracy\"], label=\"categorical_accuracy\")\nax[1].plot(hst.epoch, hst.history[\"binary_accuracy\"], label=\"binary_accuracy\")\nax[0].legend()\nax[1].legend()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3a3f1e2d6510e40ea6b72d4bd335d24e3d00d663"},"cell_type":"code","source":"seq_pred = Seq(train_labels, test=False, aug=False, batch_size=128)\nlen(seq_pred)\npred = models['model'].predict_generator(seq_pred, steps=len(seq_pred), verbose=1)\nthreshold_dic = calc_threshold(pred)\nthreshold_dic","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f10baf1dc7d8ffa329c79a156b45f873c77896e0"},"cell_type":"code","source":"seq_test = Seq(test_labels, test=True, aug=False, batch_size=128)\nseq_test\npred_test = models['model'].predict_generator(seq_test, steps=len(seq_test), verbose=1)\ntest_labels3_1, test_labels3_2 = make_test(pred_test)\ntest_labels3_2.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"df506068a2addbf3b0457d9a1f41da276901f89e"},"cell_type":"code","source":"test_labels3_2.to_csv('InceptionResNetV3_2.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4b3512fa38d1f7ba1bdd27d093cfdb9e9e426e17"},"cell_type":"code","source":"'''save weights for later loading'''\nNo = 3\nmodels['model_cnvt'].save_weights('model_{}_cnvt.h5'.format(No))\nmodels['model_resnet'].save_weights('model_{}_resnet.h5'.format(No))\nmodels['model_classifier'].save_weights('model_{}_classifier.h5'.format(No))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"fcf3be1d973010dc81d04316baf3153604181a41"},"cell_type":"markdown","source":"## #4"},{"metadata":{"trusted":true,"_uuid":"2e2e4db3b0aa954d9767ce51e408495815e10dc7"},"cell_type":"code","source":"def lr_schedule(epoch):\n    lr = 1e-4\n    print('Learning rate: ', lr)\n    return lr","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e8c5da8cb45900edd7d0447316915e50ca8b5d32"},"cell_type":"code","source":"lr_scheduler = LearningRateScheduler(lr_schedule)\ncallbacks = [lr_scheduler]\n\nseq = Seq(train_labels, aug=True, batch_size=32)\nprint(len(seq))\n\nhst = models['model'].fit_generator(seq, epochs=2,\n                              steps_per_epoch=len(seq),\n                              callbacks=callbacks)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ba669d4cee23e20680bdf92785fe759a8043a4c4"},"cell_type":"code","source":"fig, ax = plt.subplots(1, 2, figsize=(15,5))\nax[0].set_title('loss')\nax[0].plot(hst.epoch, hst.history[\"loss\"], label=\"Train loss\")\nax[1].set_title('acc')\nax[1].plot(hst.epoch, hst.history[\"categorical_accuracy\"], label=\"categorical_accuracy\")\nax[1].plot(hst.epoch, hst.history[\"binary_accuracy\"], label=\"binary_accuracy\")\nax[0].legend()\nax[1].legend()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3dacd52d67af586fa0f721a0804a4fe3f0ef1f03"},"cell_type":"code","source":"seq_pred = Seq(train_labels, test=False, aug=False, batch_size=128)\nlen(seq_pred)\npred = models['model'].predict_generator(seq_pred, steps=len(seq_pred), verbose=1)\nthreshold_dic = calc_threshold(pred)\nthreshold_dic","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"37fa85bbc0493040233dc367e8dd3f0a2ee5ea6d"},"cell_type":"code","source":"seq_test = Seq(test_labels, test=True, aug=False, batch_size=128)\nseq_test\npred_test = models['model'].predict_generator(seq_test, steps=len(seq_test), verbose=1)\ntest_labels4_1, test_labels4_2 = make_test(pred_test)\ntest_labels4_2.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"83d5ec762880c18c481105d320b793adc8f080ca"},"cell_type":"code","source":"test_labels4_2.to_csv('InceptionResNetV4_2.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6233156003bc299b785280a8e6c4f0453b17c469"},"cell_type":"code","source":"'''save weights for later loading'''\nNo = 4\nmodels['model_cnvt'].save_weights('model_{}_cnvt.h5'.format(No))\nmodels['model_resnet'].save_weights('model_{}_resnet.h5'.format(No))\nmodels['model_classifier'].save_weights('model_{}_classifier.h5'.format(No))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"36295fe44818d075d8a065c4fc0395430abd127e"},"cell_type":"markdown","source":"## #5"},{"metadata":{"trusted":true,"_uuid":"3e8960f6b624dbb86fcf281cbccea0846ca898fa"},"cell_type":"code","source":"def lr_schedule(epoch):\n    lr = 1e-4\n    print('Learning rate: ', lr)\n    return lr","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6eee4b2907cfbce18e79cb9d41134ceadf177f51"},"cell_type":"code","source":"lr_scheduler = LearningRateScheduler(lr_schedule)\ncallbacks = [lr_scheduler]\n\nseq = Seq(train_labels, aug=True, batch_size=32)\nprint(len(seq))\n\nhst = models['model'].fit_generator(seq, epochs=2,\n                              steps_per_epoch=len(seq),\n                              callbacks=callbacks)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f2ff5e28da6c9dedc083651ec5ea87b99a6d2ca3"},"cell_type":"code","source":"fig, ax = plt.subplots(1, 2, figsize=(15,5))\nax[0].set_title('loss')\nax[0].plot(hst.epoch, hst.history[\"loss\"], label=\"Train loss\")\nax[1].set_title('acc')\nax[1].plot(hst.epoch, hst.history[\"categorical_accuracy\"], label=\"categorical_accuracy\")\nax[1].plot(hst.epoch, hst.history[\"binary_accuracy\"], label=\"binary_accuracy\")\nax[0].legend()\nax[1].legend()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"98a6875654acbe30615b55025dc2d0c3e8b40a48"},"cell_type":"code","source":"seq_pred = Seq(train_labels, test=False, aug=False, batch_size=128)\nlen(seq_pred)\npred = models['model'].predict_generator(seq_pred, steps=len(seq_pred), verbose=1)\nthreshold_dic = calc_threshold(pred)\nthreshold_dic","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9bc8c0387dcb26fbcd5e2b71843ca5a1cf911a65"},"cell_type":"code","source":"seq_test = Seq(test_labels, test=True, aug=False, batch_size=128)\nseq_test\npred_test = models['model'].predict_generator(seq_test, steps=len(seq_test), verbose=1)\ntest_labels5_1, test_labels5_2 = make_test(pred_test)\ntest_labels5_2.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2e2280813ccc464653c8c42302fbf98817861620"},"cell_type":"code","source":"test_labels5_2.to_csv('InceptionResNetV5_2.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"43b835d43a448afe5e1b6753da6a0e6b5303048a"},"cell_type":"code","source":"'''save weights for later loading'''\nNo = 5\nmodels['model_cnvt'].save_weights('model_{}_cnvt.h5'.format(No))\nmodels['model_resnet'].save_weights('model_{}_resnet.h5'.format(No))\nmodels['model_classifier'].save_weights('model_{}_classifier.h5'.format(No))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5b3c77c5de9a5b999b5e4fd502342aba6190dd77"},"cell_type":"code","source":"ls -la","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0eb628f77709fdac57564bfbfd99ea066a302828"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"17e6f5f040f048c6712917528fb139840a849302"},"cell_type":"code","source":"seq = Seq(train_labels, extend=False, aug=False, batch_size=32)\nprint(len(seq))\nxs, ys = next(seq)\nprint(xs['input'].shape)\ny_pred = models['model_cnvt'].predict(xs['input'])\nprint(y_pred.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"51b1e950bb92e4e81d3f66b958afbfe76d7596f4"},"cell_type":"code","source":"y_pred[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5bd6d2706736988dfadff3e876214269c8a9eccd"},"cell_type":"code","source":"show_arr(y_pred[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"aeac0ef8be3a0d99573c47b00bd0fad524ac6dd9"},"cell_type":"code","source":"Image.fromarray(np.uint8((y_pred[0]+1)/2*255))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3126189ffc44b13571bf2aa356441525a0e85e01"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9c8ccba1717dbeb59dc64dda8cafeaca054dcd1c"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1ccafc5ba97f79e04ae9b30c779ec638226850a0"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"071987babcbe794b712b76d844b68cec5ad92ae7"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}