{"cells":[{"metadata":{"_uuid":"85c95b80d8ec01e7e50b8637b5676ac6a667b0d9"},"cell_type":"markdown","source":"* Please refer to the comments for the overview."},{"metadata":{"trusted":true,"_uuid":"7c356814c097abb220ef3135a688e4c48230ac1a"},"cell_type":"code","source":"import datetime\nnow = datetime.datetime.now()\nprint(now)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"501bc5c48766ec5964873f6dc7ebd4007adae413"},"cell_type":"code","source":"%matplotlib inline\nfrom IPython.display import SVG\nfrom keras.utils.vis_utils import model_to_dot","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport shutil\nprint(os.listdir(\"../input\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"de51410914c593a4eebf89a5635a09c7f96c0969"},"cell_type":"code","source":"try:\n    os.makedirs('/tmp/.keras/datasets')\nexcept FileExistsError:\n    pass","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4d7cb1b9173e0653687b633f695ff57c182e8cc4"},"cell_type":"code","source":"try:\n    shutil.copytree(\"../input/keras-pretrained-models\", \"/tmp/.keras/models\")\nexcept FileExistsError:\n    pass","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"import os.path\nimport itertools\nfrom itertools import chain\n\nimport numpy as np\nimport pandas as pd\nfrom sklearn import datasets\nfrom sklearn import preprocessing\nfrom sklearn.decomposition import PCA\nfrom sklearn import cluster, datasets, mixture\nfrom sklearn.datasets import load_digits\nfrom sklearn.preprocessing import StandardScaler, OneHotEncoder\nfrom sklearn.metrics import f1_score, classification_report, confusion_matrix\nfrom sklearn.model_selection import train_test_split\n\nimport matplotlib.pyplot as plt\nfrom matplotlib.colors import ListedColormap\nimport seaborn as sns\n\nimport tensorflow as tf\n\nfrom keras.layers import Input, Embedding, LSTM, GRU, Dense, Dropout, Lambda, \\\n    Conv1D, Conv2D, Conv3D, \\\n    Conv2DTranspose, \\\n    AveragePooling1D, AveragePooling2D, \\\n    MaxPooling1D, MaxPooling2D, MaxPooling3D, \\\n    GlobalAveragePooling1D, \\\n    GlobalMaxPooling1D, GlobalMaxPooling2D, GlobalMaxPooling3D, \\\n    LocallyConnected1D, LocallyConnected2D, \\\n    concatenate, Flatten, Average, Activation, \\\n    RepeatVector, Permute, Reshape, Dot, \\\n    multiply, dot, add, \\\n    PReLU, \\\n    Bidirectional, TimeDistributed, \\\n    SpatialDropout1D, \\\n    BatchNormalization\nfrom keras.models import Model, Sequential\nfrom keras import losses\nfrom keras.callbacks import BaseLogger, ProgbarLogger, Callback, History\nfrom keras.callbacks import ModelCheckpoint, LearningRateScheduler, ReduceLROnPlateau\nfrom keras.wrappers.scikit_learn import KerasClassifier\nfrom keras import regularizers\nfrom keras import initializers\nfrom keras.metrics import categorical_accuracy\nfrom keras.constraints import maxnorm, non_neg\nfrom keras.optimizers import RMSprop\nfrom keras.utils import to_categorical, plot_model\nfrom keras import backend as K","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"50fefbf82ee590bd43c1c0e8cc5034b3796652cc"},"cell_type":"code","source":"from PIL import Image\nfrom zipfile import ZipFile\nimport h5py\nimport cv2\nfrom tqdm import tqdm","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"26885390f9a1d5f8b1be5f30b9933a44ac200dbe"},"cell_type":"code","source":"src_dir = '../input/human-protein-atlas-image-classification'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"256ab9500ecf1492676ac5a56d630b916dce093a"},"cell_type":"code","source":"train_labels = pd.read_csv(os.path.join(src_dir, \"train.csv\"))\nprint(train_labels.shape)\ntrain_labels.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"27681f3b3e0ae9b0fe682270acdf0fe9aa2d8738"},"cell_type":"code","source":"test_labels = pd.read_csv(os.path.join(src_dir, \"sample_submission.csv\"))\nprint(test_labels.shape)\ntest_labels.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"085662b1058aa000cdbe2cf7ab923b46910b0004"},"cell_type":"code","source":"def show_arr(arr, nrows = 1, ncols = 4, figsize=(15, 5)):\n    fig, subs = plt.subplots(nrows=nrows, ncols=ncols, figsize=figsize)\n    for ii in range(ncols):\n        iplt = subs[ii]\n        try:\n            img_array = arr[:,:,ii]\n            if ii == 0:\n                cp = 'Greens'\n            elif ii == 1:\n                cp = 'Blues'\n            elif ii == 2:\n                cp = 'Reds'\n            else:\n                cp = 'Oranges'\n            iplt.imshow(img_array, cmap=cp)\n        except:\n            pass","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"66967a6fc0a40138dcf8f7cc80a7f17a5fc3426b"},"cell_type":"code","source":"def get_arr0(Id, test=False):\n    def fn(Id, color, test=False):\n        if test:\n            tgt = 'test'\n        else:\n            tgt = 'train'\n        with open(os.path.join(src_dir, tgt, Id+'_{}.png'.format(color)), 'rb') as fp:\n            img = Image.open(fp)\n            arr = (np.asarray(img) / 255.)\n        return arr\n    res = []\n    for icolor in ['green', 'blue', 'red', 'yellow']:\n        arr0 = fn(Id, icolor, test)\n        res.append(arr0)\n    arr = np.stack(res, axis=-1)\n    return arr","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"aaef824e54b9e397a2f8a980337f266a1fe7e413"},"cell_type":"code","source":"arr = get_arr0('00008af0-bad0-11e8-b2b8-ac1f6b6435d0', test=True)\nprint(arr.shape)\nshow_arr(arr)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"26e4815ccced55f2d31830e2134c63c52d592db3"},"cell_type":"code","source":"arr = get_arr0('00070df0-bbc3-11e8-b2bc-ac1f6b6435d0')\nprint(arr.shape)\nshow_arr(arr)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2906351bbf846b82ea7c2d7d12b83a3443baf984"},"cell_type":"code","source":"SH_ALL = (512, 512)\nSH = (256, 256)\nID_LIST_TRAIN = train_labels.Id.tolist()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7b0eb63dc51ce06412c7be3db185adae129f4d39"},"cell_type":"code","source":"# def get_arr(Id, test=False, spl=2):\n#     if test:\n#         arr = get_arr0(Id, test=True)\n#     else:\n#         arr = get_arr0(Id)\n#     arr = arr[:].astype('float32')\n#     res = []\n#     arr2 = [res.extend(np.hsplit(ee, spl)) for ee in np.vsplit(arr, spl)]\n#     return np.stack(res)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2dde4da74c0b11d6c7e175ba47df733e9860a77f"},"cell_type":"code","source":"# get_arr('00070df0-bbc3-11e8-b2bc-ac1f6b6435d0').shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"567e3046dd0a4a0fcc641b46e4ac6b1d99da7d04"},"cell_type":"code","source":"# arr = get_arr('00070df0-bbc3-11e8-b2bc-ac1f6b6435d0')\n# print(arr.shape)\n# show_arr(arr[0])\n# show_arr(arr[1])\n# show_arr(arr[2])\n# show_arr(arr[3])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8220a9292dc9ba8977d491f55a3d7a5236dd03e8"},"cell_type":"code","source":"# arr = get_arr('00008af0-bad0-11e8-b2b8-ac1f6b6435d0', test=True)\n# print(arr.shape)\n# show_arr(arr[0])\n# show_arr(arr[1])\n# show_arr(arr[2])\n# show_arr(arr[3])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6a2dfc516268368e147cc176ba74392f08cd445f"},"cell_type":"code","source":"y_cat_train_dic = {}\nfor icat in range(28):\n    target = str(icat)\n    y_cat_train_5 = np.array([int(target in ee.split()) for ee in train_labels.Target.tolist()])\n    y_cat_train_dic[icat] = y_cat_train_5","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9b22a3224e0683d580593af45fcf81f3a1c8c083"},"cell_type":"code","source":"up_sample = {}\nfor k in y_cat_train_dic:\n    v = y_cat_train_dic[k].sum()\n    up_sample[k] = np.ceil((train_labels.shape[0]/28) / v)\n\nup_sample","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"79abf3891d5990af8297cb75b01afb2bb263ffd5"},"cell_type":"code","source":"up_sample2 = list(zip(*sorted(list(up_sample.items()), key=lambda x: x[0])))[1]\nup_sample2 = np.array(up_sample2)\nup_sample2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9e0cc74d19319fff20756df1d38e1bc14a969c0a"},"cell_type":"code","source":"import random\n\nclass Seq(object):\n    sections = None\n    index = None\n    SPLIT_LIST = [\n        np.array(range(256)),\n        np.array(range(256)) + 256\n    ]\n    \n    def __init__(self, df, extend=False, aug=False, test=False, batch_size=32):\n        self.shaffle = None\n        self.extend = extend\n        self.aug = aug\n        self.test = test\n        self.batch_size = batch_size\n        self.df = df\n        \n        # proccess\n        self.ids = self.df.Id.tolist()\n        self.reversed = sorted(range(SH_ALL[0]), reverse=True)\n        \n        # estimate self length\n        self.initialize_it()\n        self.len = 1\n        for _ in self.it:\n            self.len += 1\n        \n        self.initialize_it()\n    \n    def initialize_it(self):\n        if self.shaffle:\n            '''not implemented yet'''\n            raise NotImplementedError\n            #random.seed(self.state)\n            #random.shuffle(self.ids)\n        \n        self.it = iter(range(0, len(self.ids), self.batch_size))\n        self.idx_next = self.it.__next__()\n    \n    def __len__(self):\n        return self.len\n    \n    def __iter__(self):\n        return self\n    \n    def __next__(self):\n        idx = self.idx_next\n        self.ids_part = self.ids[idx:((idx+self.batch_size) if idx+self.batch_size<len(self.ids) else len(self.ids))]\n        res = self.getpart(self.ids_part)\n        try:\n            self.idx_next = self.it.__next__()\n        except StopIteration:\n            self.initialize_it()\n        return res\n    \n    def __getitem__(self, id0):\n        arr, tgts = self.get_data(id0)\n        cat = self.convert_tgts(tgts)\n        return arr, cat\n    \n    k_list = list(range(4))\n    def random_transform(self, arr):\n        k = random.choice(self.k_list)\n        arr0 = np.rot90(arr, k=k)\n        if random.randint(0,1):\n            arr0 = arr0[self.reversed,:,:]\n        if random.randint(0,1):\n            arr0 = arr0[:,self.reversed,:]\n        return arr0\n    \n    def convert_tgts(self, tgts):\n        try:\n            cats = to_categorical(tgts, num_classes=28)\n            cat = cats.sum(axis=0)\n        except TypeError:\n            cat = np.zeros((28,))\n        return cat\n    \n    def get_data(self, id0):\n        arr = get_arr0(id0, test=self.test)\n        \n        try:\n            y0 = (self.df.Target[self.df.Id == id0]).tolist()[0]\n            y1 = y0.split()\n            y = [int(ee) for ee in y1]\n        except AttributeError:\n            y = None\n        return arr, y\n    \n    def getpart(self, ids):\n        xs = []\n        ys = []\n        for id0 in ids:\n            self.extend_data(id0, xs, ys)\n        \n        x = np.stack(xs)\n        y = np.stack(ys)\n        x_dummy = np.zeros((len(x), 1))\n        x_ret = {\n            'input': x,\n            'input_cls': y,\n        }\n        y_ret = {\n            'path_fit_cls_img': x_dummy,\n            'path_cls_img_cls': y,\n            'path_fit_imgA': x_dummy,\n            'path_fit_img_cls_img': x_dummy,\n            'path_cls_cls': y,\n            'path_img_cls': y,\n            'path_img_img_cls': y,\n            'path_fit_cls_img_imgE': x_dummy,\n        }\n        return (x_ret, y_ret)\n    \n    def split(self, arr):\n        res = []\n        for row_idx in self.SPLIT_LIST:\n            for col_idx in self.SPLIT_LIST:\n                #print(arr.shape, row_idx, col_idx)\n                res.append(arr[:, col_idx][row_idx].flatten())\n        return res\n    \n    def extend_data(self, id0, xs, ys):\n        arr0, cat = self[id0]\n        \n        # data augmentation\n        if self.extend:\n            mm = up_sample2[cat==1].max()\n            mm = int(mm)\n            #print(mm)\n            for ii in range(mm):\n                if self.aug:\n                    img = self.random_transform(arr0)\n                else:\n                    img = arr0\n                xs.append(img.flatten())\n                ys.append(cat)\n        else:\n            if self.aug:\n                img0 = self.random_transform(arr0)\n            else:\n                img0 = arr0\n            res = self.split(img0)\n            res = np.concatenate(res)\n            xs.append(res)\n            ys.append(cat)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a72c0bb0d0d1799e80b5e2cbcd0c82d7809f9378"},"cell_type":"code","source":"seq = Seq(train_labels, extend=False, aug=True, batch_size=8)\nprint(len(seq))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e8d963b414f324fff0f0ad46baad7658edffa14a"},"cell_type":"code","source":"print(len(seq.ids))\nlen(seq)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7abf6eb73f763e04fda03cb13560984d8c92981e"},"cell_type":"code","source":"arr, y = seq.get_data('ad5a4858-bb9d-11e8-b2b9-ac1f6b6435d0')\nprint(arr.shape)\nshow_arr(arr)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a929e002002a2a423e0375d07ef97f16553ec510"},"cell_type":"code","source":"x, y = seq['ad5a4858-bb9d-11e8-b2b9-ac1f6b6435d0']\nprint(x.shape)\nshow_arr(x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ef3fd71aadb7d18de30787eec4cc749c9dd91651"},"cell_type":"code","source":"xs, ys = next(seq)\nprint(xs['input'].shape)\nprint(xs['input_cls'].shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"212dbbd981519efaf824e136da5d30aa443b9d60"},"cell_type":"code","source":"show_arr(xs['input'][0].reshape((4,256,256,4))[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4436e7679bc6016fae01088239c01f45644d6135"},"cell_type":"code","source":"show_arr(xs['input'][0].reshape((4,256,256,4))[1])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d98c20522bcf8014f0b9e4d2cee8d269bfcfdfaa"},"cell_type":"markdown","source":"### make model"},{"metadata":{"trusted":true,"_uuid":"163f04800116988c66e6aa65fe3caf65c57d3cee"},"cell_type":"code","source":"from keras import applications","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1f76a77d9d9608e0aef1db29cd147c40d9f511cd","scrolled":true},"cell_type":"code","source":"def make_trainable_false(model_resnet, trainable=False):\n    layers = model_resnet.layers\n    for ilayer in layers:\n        ilayer.trainable = trainable\n    return","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0b3c366671d5f7913952e10b35182ad3c867c4cb"},"cell_type":"code","source":"img_shape = tuple(list(SH) + [4])\nprint(img_shape)\nimg_dim = 4 * np.array(img_shape).prod()\nprint(img_dim)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4edf6b659cfc1a206db6ef6705be72fa34f80c9e"},"cell_type":"code","source":"def make_model_cnvt(img_dim, img_shape):\n    '''==============================\n    inputs\n    =============================='''\n    inp = Input(shape=(img_dim,))\n    oup = Reshape(img_shape)(inp)\n    #oup = Conv2D(3, kernel_size=1, strides=1, padding='same')(oup)\n    #oup = Conv2D(3, kernel_size=1, strides=1, padding='same', activation='sigmoid')(oup)\n    oup = Conv2D(3,\n                 kernel_size=1,\n                 strides=1,\n                 padding='same',\n                 activation='tanh',\n                 kernel_regularizer=regularizers.l2(0))(oup)\n    #kernel_regularizer=regularizers.l2(1e-4)\n    model_cnvt = Model(inp, oup, name='model_cnvt')\n    return model_cnvt","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a62bfbcfc54d796359f72f3bc7eda9b20b8f9be6"},"cell_type":"code","source":"def make_model_classifier(input_dim=1536):\n    inp_cls = Input((input_dim,))\n    oup_cls = Dense(28)(inp_cls)\n    oup_cls = Activation('sigmoid')(oup_cls)\n    model_classifier = Model(inp_cls, oup_cls, name='classifier')\n    return model_classifier","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"632e6ed34edbce32760989c1750748fe125e7f7a"},"cell_type":"code","source":"def make_model(img_dim, model_cnvt, model_resnet, model_classifier):\n    inp0 = Input(shape=(int(img_dim/4),), name='input0')\n    oup0 = model_cnvt(inp0)\n    oup0 = model_resnet(oup0)\n    oup0 = model_classifier(oup0)\n    oup0 = Activation('linear', name='path_cls_cls')(oup0)\n    model0 = Model(inp0, oup0, name='model0')\n    \n    '''==============================\n    inputs\n    =============================='''\n    def fn(x, idx):\n        x1 = K.permute_dimensions(x, (1,0,2,))\n        x2 = K.gather(x1, idx)\n        return x2\n    inp = Input(shape=(img_dim,), name='input')\n    oup = Reshape((4,int(img_dim/4)))(inp)\n    img0 = Lambda(lambda x: fn(x, 0))(oup)\n    img1 = Lambda(lambda x: fn(x, 1))(oup)\n    img2 = Lambda(lambda x: fn(x, 2))(oup)\n    img3 = Lambda(lambda x: fn(x, 3))(oup)\n    oup = Lambda(lambda x: K.stack(x, 1))([model0(img0), model0(img1), model0(img2), model0(img3)])\n    oup = GlobalMaxPooling1D()(oup)\n    oup = Activation('linear', name='path_cls_cls')(oup)\n    model = Model(inp, oup, name='model')\n    \n    return {\n        'model_classifier': model_classifier,\n        'model_resnet': model_resnet,\n        'model_cnvt': model_cnvt,\n        'model': model,\n        'model0': model0\n    }","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ecbdefe8b5c3805783dbc0244b8c0ddad74346f2"},"cell_type":"code","source":"# models = make_model(img_dim, model_cnvt, model_resnet, model_classifier)\n# models['model0'].summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"65df347849950a9566fe190c379e7a9583e152cd"},"cell_type":"code","source":"# models['model'].summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b4545428eb68c11036c26f110a23e36675d86e83"},"cell_type":"code","source":"model_cnvt = make_model_cnvt(int(img_dim/4), img_shape)\nmodel_cnvt.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5cc70e0aeaf657838050bf2e9a8530eb7190dff5"},"cell_type":"code","source":"model_cnvt.layers[2].get_weights()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"df914f5b737824bbebe88647db823e118f4e2bf9"},"cell_type":"code","source":"model_cnvt.load_weights('../input/resnet50-4x256-max-1/model_1_cnvt.h5')\nmodel_cnvt.layers[2].get_weights()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8f5b82c71943f783049cd450dbc7904bca13dc49"},"cell_type":"code","source":"model_resnet = applications.resnet50.ResNet50(\n    include_top=False,\n    weights='imagenet',\n    input_tensor=None,\n    input_shape=list(SH)+[3],\n    pooling='avg',\n    classes=None)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e86f3d63580b9e860341ba4c673fa982b946fd30"},"cell_type":"code","source":"# model_resnet.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"44de2ae03fd5e30c5cc509f0d1ef2a416551c291"},"cell_type":"code","source":"model_resnet.layers[2].get_weights()[0][0,0,0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2df25fe1d4ad2010f16e8365262129a6bea50d71"},"cell_type":"code","source":"model_resnet.load_weights('../input/resnet50-4x256-max-1/model_1_resnet.h5')\nmodel_resnet.layers[2].get_weights()[0][0,0,0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0383dd3682e287aa53a2129c7586cb3f731d1ab1"},"cell_type":"code","source":"model_classifier = make_model_classifier(1024*2)\nmodel_classifier.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1b30e9a80a42bdeb0c55a36994a7ce2bbdbffe8e"},"cell_type":"code","source":"model_classifier.layers[1].get_weights()[0][0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"20179bb767bce79b515e652c6ed690d50f29be17"},"cell_type":"code","source":"model_classifier.load_weights('../input/resnet50-4x256-max-1/model_1_classifier.h5')\nmodel_classifier.layers[1].get_weights()[0][0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"296359eccad432b4e71ebf0305c676655a71d6a7"},"cell_type":"code","source":"models = make_model(img_dim, model_cnvt, model_resnet, model_classifier)\nmodels['model0'].summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"22e93af07d24ace94e7972eaff4825cb026a2e59","scrolled":false},"cell_type":"code","source":"models['model'].summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cf509018fdd890d992e5c6ec0fa46b929ad39414"},"cell_type":"code","source":"THRESHOLD = 0.5\n\n# credits: https://www.kaggle.com/guglielmocamporese/macro-f1-score-keras\n\nK_epsilon = K.epsilon()\ndef f1(y_true, y_pred):\n    #y_pred = K.round(y_pred)\n    y_pred = K.cast(K.greater(K.clip(y_pred, 0, 1), THRESHOLD), K.floatx())\n    tp = K.sum(K.cast(y_true*y_pred, 'float'), axis=0)\n    tn = K.sum(K.cast((1-y_true)*(1-y_pred), 'float'), axis=0)\n    fp = K.sum(K.cast((1-y_true)*y_pred, 'float'), axis=0)\n    fn = K.sum(K.cast(y_true*(1-y_pred), 'float'), axis=0)\n\n    p = tp / (tp + fp + K_epsilon)\n    r = tp / (tp + fn + K_epsilon)\n\n    f1 = 2*p*r / (p+r+K_epsilon)\n    f1 = tf.where(tf.is_nan(f1), tf.zeros_like(f1), f1)\n    return K.mean(f1)\n\ndef f1_loss(y_true, y_pred):\n    \n    #y_pred = K.cast(K.greater(K.clip(y_pred, 0, 1), THRESHOLD), K.floatx())\n    tp = K.sum(K.cast(y_true*y_pred, 'float'), axis=0)\n    tn = K.sum(K.cast((1-y_true)*(1-y_pred), 'float'), axis=0)\n    fp = K.sum(K.cast((1-y_true)*y_pred, 'float'), axis=0)\n    fn = K.sum(K.cast(y_true*(1-y_pred), 'float'), axis=0)\n\n    p = tp / (tp + fp + K_epsilon)\n    r = tp / (tp + fn + K_epsilon)\n\n    f1 = 2*p*r / (p+r+K_epsilon)\n    f1 = tf.where(tf.is_nan(f1), tf.zeros_like(f1), f1)\n    return 1-K.mean(f1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9bcd317b665814fcfd8bc4a5814e60ccc472ba70"},"cell_type":"code","source":"'''\nThanks Iafoss.\npretrained ResNet34 with RGBY\nhttps://www.kaggle.com/iafoss/pretrained-resnet34-with-rgby-0-460-public-lb\n'''\ngamma = 2.0\nepsilon = K.epsilon()\ndef focal_loss(y_true, y_pred):\n    pt = y_pred * y_true + (1-y_pred) * (1-y_true)\n    pt = K.clip(pt, epsilon, 1-epsilon)\n    CE = -K.log(pt)\n    FL = K.pow(1-pt, gamma) * CE\n    loss = K.sum(FL, axis=1)\n    return loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9fde77f510bde3766dcb9ca4d987727e5eab3e0a"},"cell_type":"code","source":"class LearningRateReducer(Callback):\n\n    def __init__(self, schedule,\n                 cycle_epoch=2, steps_per_epoch=100,\n                 start_lr=0.001, end_lr=0.0001, verbose=0):\n        super(LearningRateReducer, self).__init__()\n        self.schedule = schedule\n        self.steps_per_epoch = steps_per_epoch\n        self.cycle_epoch = cycle_epoch\n        self.start_lr = start_lr\n        self.end_lr = end_lr\n        self.verbose = verbose\n\n    def on_train_begin(self, epoch, logs=None):\n        self.lrf = {'lr': [], 'loss': []}\n        self.epoch = None\n\n    def on_epoch_begin(self, epoch, logs=None):\n        self.epoch = epoch\n    \n    def on_batch_begin(self, batch, logs=None):\n        #print(batch)\n        lr = self.schedule(self.epoch, batch,\n                           cycle_epoch=self.cycle_epoch,\n                           steps_per_epoch=self.steps_per_epoch,\n                           start_lr=self.start_lr, end_lr=self.end_lr)\n        K.set_value(self.model.optimizer.lr, lr)\n        self.lrf['lr'].append(lr)\n        return\n    \n    def on_batch_end(self, batch, logs={}):\n        loss = logs.get('loss')\n        self.lrf['loss'].append(loss)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"917bfde303141c1d0325a9dcd7f1cd838d868a69"},"cell_type":"code","source":"def lr_schedule(epoch, batch,\n                cycle_epoch=2, steps_per_epoch=100,\n                start_lr=0.002, end_lr=0.0002):\n    by = (np.log(start_lr) - np.log(end_lr)) / (steps_per_epoch*cycle_epoch-1)\n    ii = divmod(epoch, cycle_epoch)[1]*steps_per_epoch + batch # use amari\n    lr = np.exp(np.log(start_lr) - ii * by)\n    #print('Learning rate: ', lr)\n    return lr","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fb8c118a6e0fe477757a19f23b9f288d3037dc1f"},"cell_type":"code","source":"LEARNING_RATE = 0.001\nprint(datetime.datetime.now() - now)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2e97be362cd6d1a8be3dff38f3bee75e741c9fbe"},"cell_type":"code","source":"models['model'].compile(loss='binary_crossentropy',\n                        optimizer='adam',\n                        metrics=['categorical_accuracy', 'binary_accuracy', f1])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7e4672b15a810c37a7daea0dd28556cc4e21c690"},"cell_type":"markdown","source":"## #1"},{"metadata":{"trusted":true,"_uuid":"df50d98162b837b60a64284cf41ff5d0934326b0","scrolled":true},"cell_type":"code","source":"seq = Seq(train_labels, extend=False, aug=True, batch_size=8)\nprint(len(seq))\n\nlr_scheduler = LearningRateReducer(lr_schedule,\n                                   cycle_epoch=1, steps_per_epoch=len(seq),\n                                   start_lr=LEARNING_RATE, end_lr=LEARNING_RATE/10)\ncallbacks = [lr_scheduler]\n\nhst = models['model'].fit_generator(seq, epochs=1,\n                              steps_per_epoch=len(seq),\n                              callbacks=callbacks)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1fbaaf76d643c90b547203367487383bcc953145"},"cell_type":"code","source":"plt.subplots(1, 1, figsize=(10,10))\nplt.plot(lr_scheduler.lrf['lr'], lr_scheduler.lrf[\"loss\"])\nplt.xscale('log')\nplt.title('learning rate')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7c34349ced0780c1f2da41c46c2817a2bd060c79"},"cell_type":"code","source":"seq = Seq(train_labels, extend=False, aug=False, batch_size=32)\nprint(len(seq))\nxs, ys = next(seq)\nprint(xs['input'].shape)\nxs['input'].reshape((32,4,-1)).shape\ny_pred = models['model_cnvt'].predict(xs['input'].reshape((32,4,-1))[0])\nprint(y_pred.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d24ee6e3d1f01a589bfb99439ff0bdd0352c2c9d"},"cell_type":"code","source":"y_pred[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3071e5cdef6cb0cfea8b0b917d0227a728b3dd1b"},"cell_type":"code","source":"tmp = np.vstack(\n    [\n        np.hstack([y_pred[0], y_pred[1]]),\n        np.hstack([y_pred[2], y_pred[3]])\n    ])\ntmp.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"48d34f6e60f0b188105f0b52e6c8e6fee694f289"},"cell_type":"code","source":"show_arr(tmp)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b7af3b04154d25bad46e786b168960b5d5e72dc7"},"cell_type":"code","source":"Image.fromarray(np.uint8((tmp+1)/2*255))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e5382d82e02e36aad43064fb0f46275115c43f59"},"cell_type":"markdown","source":"### predict and submit"},{"metadata":{"trusted":true,"_uuid":"a545b725c69cdc9c0305066b5d0e3456837cab27"},"cell_type":"code","source":"seq_pred = Seq(train_labels, test=False, aug=False, batch_size=32)\nlen(seq_pred)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"902ce70dd767b9317fcc50377c84b7ffbb037fa0"},"cell_type":"code","source":"pred = models['model'].predict_generator(seq_pred, steps=len(seq_pred), verbose=1)\npred.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6b902e80ea41a107f7e827bc53753d6e0877e0fd"},"cell_type":"code","source":"def calc_threshold(pred):\n    ### calc threshold\n    threshold_dic = {}\n    for idx in tqdm(range(28)):\n        threshold_dic[idx] = 0.5\n        m = 0\n        for ii in range(100):\n            threshold0 = ii*0.01\n            f1_val = f1_score(y_cat_train_dic[idx], threshold0<(pred[:,idx]))\n            if m < f1_val:\n                threshold_dic[idx] = threshold0+0.005\n                m = f1_val\n    return threshold_dic","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5d8c7a7f99e2291e5b90cfdedd2fc7011c4844e0"},"cell_type":"code","source":"import warnings\nwarnings.filterwarnings('ignore')\n\nthreshold_dic = calc_threshold(pred)\nthreshold_dic","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"172deecf04f7565ef2f9e876d94998421c65d436"},"cell_type":"code","source":"'''use threshold_dic'''\nfor ii in range(28):\n    print(ii, f1_score(y_cat_train_dic[ii], threshold_dic[ii]<pred[:,ii]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"690cd8c1b2deb3d7e04874488ec38b354f901404"},"cell_type":"code","source":"'''threshold = 0.5'''\nfor ii in range(28):\n    print(ii, f1_score(y_cat_train_dic[ii], 0.5<pred[:,ii]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"beb2ae7253557b2a4a054ffe4bde530b82c13a6f"},"cell_type":"code","source":"seq_test = Seq(test_labels, test=True, aug=False, batch_size=32)\nseq_test","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a921f7ddefb2099a4d4332047fb8f69ea33e98cf","scrolled":true},"cell_type":"code","source":"pred_test = models['model'].predict_generator(seq_test, steps=len(seq_test), verbose=1)\npred_test.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f8bbdce55d142033b6315f6a29f61a9c188ab0b4"},"cell_type":"code","source":"def make_test(pred):\n    test_labels1 = test_labels.copy()\n    test_labels1['Predicted'] = [str(ee) for ee in np.argmax(pred, axis=1)]\n    print(test_labels1.head())\n    #test_labels1.to_csv(fn0, index=False)\n    \n    test_labels2 = test_labels1.copy()\n    for ii in range(test_labels2.shape[0]):\n        threshold = list(zip(*sorted(list(threshold_dic.items()), key=lambda x:x[0], reverse=False)))[1]\n        idx = threshold < pred[ii,:]\n        tgt = test_labels2['Predicted'][ii]\n        tgt = [tgt] + [str(ee) for ee in np.arange(28)[idx]]\n        tgt = set(tgt)\n        tgt = ' '.join(tgt)\n        test_labels2['Predicted'][ii] = tgt\n    print(test_labels2.head())\n    #test_labels2.to_csv(fn, index=False)\n    return test_labels1, test_labels2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"87ef7ff12c86c548726785589c0f9977986ef1b3"},"cell_type":"code","source":"test_labels1_1, test_labels1_2 = make_test(pred_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"54a87bc7d9f288aca9bc864ec7f55da9062741a7"},"cell_type":"code","source":"test_labels1_2.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d85574855df017b70cc6ed75361f75ca630c69aa"},"cell_type":"code","source":"test_labels1_2.to_csv('InceptionResNetV1_2.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"83d2124b8f7d67f334e9087446147b20c3c15411"},"cell_type":"markdown","source":"## save weights for later loading"},{"metadata":{"trusted":true,"_uuid":"5de2c0d6f3c2fe522480dd79414b424ff7139774"},"cell_type":"code","source":"'''save weights for later loading'''\nNo = 1\nmodels['model_cnvt'].save_weights('model_{}_cnvt.h5'.format(No))\nmodels['model_resnet'].save_weights('model_{}_resnet.h5'.format(No))\nmodels['model_classifier'].save_weights('model_{}_classifier.h5'.format(No))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5d3496d3f1606ca24feaa5183e5ffb6746af5569"},"cell_type":"code","source":"df_pred1 = pd.DataFrame(pred)\ndf_pred1.columns = ['cls'+ str(ii) for ii in range(28)]\ndf_pred1 = pd.concat([train_labels.copy(), df_pred1], axis=1)\n\ndf_pred1.head()\ndf_pred1.to_csv('proba_{}.csv'.format(No), index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1aab61fd0374e4b538b5cacb3c5738cad36521df"},"cell_type":"code","source":"df_pred1_test = pd.DataFrame(pred_test)\ndf_pred1_test.columns = ['cls'+ str(ii) for ii in range(28)]\ndf_pred1_test = pd.concat([test_labels.copy(), df_pred1_test], axis=1)\n\ndf_pred1_test.head()\ndf_pred1_test.to_csv('proba_test_{}.csv'.format(No), index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a7aa49c828b3153d5382504c31311da10ff7fd0f"},"cell_type":"code","source":"print(datetime.datetime.now() - now)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0eb628f77709fdac57564bfbfd99ea066a302828"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3126189ffc44b13571bf2aa356441525a0e85e01"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9c8ccba1717dbeb59dc64dda8cafeaca054dcd1c"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1ccafc5ba97f79e04ae9b30c779ec638226850a0"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"071987babcbe794b712b76d844b68cec5ad92ae7"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}