{"cells":[{"metadata":{"_uuid":"85c95b80d8ec01e7e50b8637b5676ac6a667b0d9"},"cell_type":"markdown","source":"gkernel -> gkernel"},{"metadata":{"trusted":true,"_uuid":"9d0a5b44107243a23c4d6a3ad0ca62daf678673a"},"cell_type":"code","source":"!pip install keras==2.1.6","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2656074fa92b2183a3068465813a2281b37c1ae3"},"cell_type":"code","source":"!pip install git+https://github.com/darecophoenixx/wordroid.sblo.jp","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0647fd23a18f9410198e5c5ac857df88cf9dd876"},"cell_type":"code","source":"ls -la /opt/conda/lib/python3.6/site-packages/keras_ex","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1226d9f0d5c79cabd74bc7a166b8c60e234dd4b2"},"cell_type":"code","source":"import datetime\nnow = datetime.datetime.now()\nprint(now)\n\nrs = 10002","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"501bc5c48766ec5964873f6dc7ebd4007adae413"},"cell_type":"code","source":"%matplotlib inline\nfrom IPython.display import SVG\nfrom keras.utils.vis_utils import model_to_dot","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os, io\nimport shutil\nprint(os.listdir(\"../input\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"de51410914c593a4eebf89a5635a09c7f96c0969"},"cell_type":"code","source":"try:\n    os.makedirs('/tmp/.keras/datasets')\nexcept FileExistsError:\n    pass","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4d7cb1b9173e0653687b633f695ff57c182e8cc4"},"cell_type":"code","source":"try:\n    shutil.copytree(\"../input/keras-pretrained-models\", \"/tmp/.keras/models\")\nexcept FileExistsError:\n    pass","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"import datetime\nimport os.path\nimport itertools\nfrom itertools import chain\n\nimport numpy as np\nimport pandas as pd\nfrom sklearn import datasets\nfrom sklearn import preprocessing\nfrom sklearn.decomposition import PCA\nfrom sklearn import cluster, datasets, mixture\nfrom sklearn.datasets import load_digits\nfrom sklearn.preprocessing import StandardScaler, OneHotEncoder\nfrom sklearn.metrics import f1_score, classification_report, confusion_matrix\nfrom sklearn.model_selection import train_test_split\n\nimport matplotlib\nimport matplotlib.pyplot as plt\nfrom matplotlib.colors import ListedColormap\nimport seaborn as sns\n\nimport tensorflow as tf\n\nfrom keras.layers import Input, Embedding, LSTM, GRU, Dense, Dropout, Lambda, \\\n    Conv1D, Conv2D, Conv3D, \\\n    Conv2DTranspose, \\\n    AveragePooling1D, AveragePooling2D, \\\n    MaxPooling1D, MaxPooling2D, MaxPooling3D, \\\n    GlobalAveragePooling1D, GlobalAveragePooling2D, \\\n    GlobalMaxPooling1D, GlobalMaxPooling2D, GlobalMaxPooling3D, \\\n    LocallyConnected1D, LocallyConnected2D, \\\n    concatenate, Flatten, Average, Activation, \\\n    RepeatVector, Permute, Reshape, Dot, \\\n    multiply, dot, add, \\\n    PReLU, \\\n    Bidirectional, TimeDistributed, \\\n    SpatialDropout1D, \\\n    BatchNormalization\nfrom keras.models import Model, Sequential\nfrom keras import losses\nfrom keras.callbacks import BaseLogger, ProgbarLogger, Callback, History\nfrom keras.callbacks import ModelCheckpoint, LearningRateScheduler, ReduceLROnPlateau\nfrom keras.wrappers.scikit_learn import KerasClassifier\nfrom keras import regularizers\nfrom keras import initializers\nfrom keras.metrics import categorical_accuracy\nfrom keras.constraints import maxnorm, non_neg\nfrom keras.optimizers import RMSprop\nfrom keras.utils import to_categorical, plot_model\nfrom keras import backend as K\nimport keras","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"50fefbf82ee590bd43c1c0e8cc5034b3796652cc"},"cell_type":"code","source":"from PIL import Image\nfrom zipfile import ZipFile\nimport h5py\nimport cv2\nfrom tqdm import tqdm\nimport datetime","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7e1f3ba9bd21954516ad65624e3290afe7123474"},"cell_type":"code","source":"from keras_ex.HumanisticML import HML, HMLx\nfrom keras_ex.gkernel import GaussianKernel, GaussianKernel2, GaussianKernel3","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2dd70dabe72458b9b5d1004eb2e2bd1fc4dc8f1c"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2b9167f6d6b3a1964bf9e389f70c1816e61b9b3d"},"cell_type":"code","source":"# Load the data\ntrain = pd.read_csv(\"../input/digit-recognizer/train.csv\")\ntest = pd.read_csv(\"../input/digit-recognizer/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6619d8f5da36e0e7455ec299f65df2b9ad5bf24b"},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1b2a6245ab12279bea772004c83cdd0421199c27"},"cell_type":"code","source":"test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ea8044e099445d033b11c7c91ab3c044def933e3"},"cell_type":"code","source":"y_train = train['label'].values\nprint(y_train.shape)\n\ny_cat = to_categorical(y_train)\ny_cat.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"72c1d0008273d0b659d37e60cbc90c0f7cba37fd"},"cell_type":"code","source":"x_train = train.iloc[:,1:].values / 255.0\nprint((x_train.min(), x_train.max()))\nx_train.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7025fbfd69777c577ef7b73dcd085533254ce344"},"cell_type":"code","source":"x_train_fl = x_train.reshape((x_train.shape[0], -1))\nx_train_fl.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"417897373015722347be086aaf611d2f57b7ce88"},"cell_type":"code","source":"plt.imshow(x_train[0].reshape((28,28)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3087425073721e87221a64c823b601c08658557f"},"cell_type":"code","source":"x_test = test.iloc[:,:].values / 255.0\nprint((x_test.min(), x_test.max()))\nx_test.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9919f0461f51933b8b30e457331b29eeab1db53d"},"cell_type":"code","source":"x_test_fl = x_test.reshape((x_test.shape[0], -1))\nx_test_fl.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"580cfee9c6c8161e425a14a8fb9319913bcec1fd"},"cell_type":"code","source":"plt.imshow(x_test[0].reshape((28,28)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"29cc82a94773e10a10712ee05957ad9a37f44560"},"cell_type":"code","source":"sample_submit = pd.read_csv(\"../input/digit-recognizer/sample_submission.csv\")\nsample_submit.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f37f63447467ad445ae9b4b1dfc2f28bbb373d45"},"cell_type":"markdown","source":"## make model"},{"metadata":{"trusted":true,"_uuid":"2180dc6425eb9af46fb0cb868d1d6cb080c09fe1"},"cell_type":"code","source":"def make_trainable_false(model, trainable=False):\n    layers = model.layers\n    for ilayer in layers:\n        ilayer.trainable = trainable\n    return\n\nclass TrainableCtrl(object):\n    \n    def __init__(self, model_dic):\n        self.model_dic = model_dic\n        self.trainable_dic = {}\n        self.get_trainable()\n        \n    def get_trainable(self):\n        for k in self.model_dic:\n            model = self.model_dic[k]\n            res = []\n            for ilayer in model.layers:\n                res.append(ilayer.trainable)\n            self.trainable_dic[k] = res\n    \n    def set_trainable_false(self, model_key):\n        model = self.model_dic[model_key]\n        make_trainable_false(model)\n    \n    def set_trainable_true(self, model_key):\n        model = self.model_dic[model_key]\n        for ii, ilayer in enumerate(model.layers):\n            ilayer.trainable = self.trainable_dic[model_key][ii]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"06e9057a221e3c27e08e3ec1f0e887b65c324452"},"cell_type":"code","source":"img_shape = (28, 28, 1)\nimg_dim = np.array(img_shape).prod()\nprint(img_dim)\n\nnn = 64*2 # img_cnvtの出力の次元\n\nnum_cnvt_lm = 2 # 1段目の特徴量の次元\nnum_cls = 10","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9e11e915d0951f604a45d930c3669a343b370a55"},"cell_type":"code","source":"def make_model_img_convert0(img_dim, img_shape):\n\n    '''==============================\n    inputs\n    =============================='''\n    inp = Input(shape=(img_dim,))\n    \n    '''==============================\n    layers\n    =============================='''\n    oup_ae = Reshape(img_shape)(inp)\n    oup_ae = Conv2D(filters=32, kernel_size=(5,5), padding='same', activation='elu')(oup_ae)\n    oup_ae = Conv2D(filters=32, kernel_size=(5,5), padding='same', activation='elu')(oup_ae)\n    oup_ae = MaxPooling2D(pool_size=(2, 2))(oup_ae)\n    #oup_ae = Dropout(0.25)(oup_ae)\n    oup_ae = Conv2D(filters=64, kernel_size=(3,3), padding='same', activation='elu')(oup_ae)\n    oup_ae = Conv2D(filters=64, kernel_size=(3,3), padding='same')(oup_ae)\n    \n    oup_ae1 = GlobalMaxPooling2D()(oup_ae)\n    oup_ae2 = GlobalAveragePooling2D()(oup_ae)\n    oup = concatenate([oup_ae1, oup_ae2])\n    model_cnvt_org = Model(inp, oup, name='model_img_cnvt_org')\n    # oup1 = Activation('sigmoid')(oup)\n    oup1 = BatchNormalization()(oup)\n    model_cnvt = Model(inp, oup1, name='model_img_cnvt')\n    \n    oup = Dense(10, activation='sigmoid')(oup1)\n    model = Model(inp, oup, name='model')\n    \n    return {\n        'model_img_cnvt': model_cnvt,\n        'model_img_cnvt_org': model_cnvt_org,\n        'model': model\n    }\n\n\nmodels_img_cnvt0 = make_model_img_convert0(img_dim, img_shape)\nmodels_img_cnvt0['model_img_cnvt'].summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f534609c75cccac968648337386eed913abcef99"},"cell_type":"code","source":"models_img_cnvt0['model'].summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e60298f250e821fc58c8ba86ff3be12a2bf39b25"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fcdcf2e3c70e8b6dbadb2363bbe700d17e978306"},"cell_type":"code","source":"def make_model_cnvt(nn=nn, num_lm=num_cnvt_lm, random_state=0, scale=1):\n    inp = Input(shape=(nn,), name='inp')\n    oup = inp\n    \n    np.random.seed(random_state)\n    init_wgt = (np.random.random_sample((num_lm, nn))-0.5) * scale\n    \n#     weights2 = [init_wgt, np.log(np.array([1/(2*nn*0.1*scale)]))]\n#     oup = GaussianKernel3(num_landmark=num_lm, num_feature=nn, weights=weights2, name='gkernel')(oup)\n    weights2 = [np.log(np.array([1/(2*nn*0.1*scale)]))]\n    oup = GaussianKernel2(init_wgt, weights=weights2, name='gkernel')(oup)\n    model = Model(inp, oup, name='model_cnvt')\n    return init_wgt, model\n\nlm_cnvt, model_cnvt = make_model_cnvt(random_state=rs, scale=4)\nmodel_cnvt.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"553cfd3033142897dee289a3eaffa5b41111f931"},"cell_type":"code","source":"#lm_cnvt = model_cnvt.layers[1].get_weights()[0]\nprint(lm_cnvt.shape)\n\ndf = pd.DataFrame(lm_cnvt[:,:5])\ndf.head()\nfig = sns.pairplot(df, markers=['o'], height=2.2, diag_kind='hist')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7dc17282c8334f073c74579f2b9f62c0905db79e"},"cell_type":"code","source":"def make_models_dense(nn=num_cnvt_lm, num_cls=num_cls):\n    inp = Input(shape=(nn,), name='inp')\n    # oup = Dense(num_cls, activation='sigmoid')(inp)\n    init_wgt = np.random.random_sample((num_cls, nn))\n    weights = [init_wgt, np.log(np.array([1/(2*nn*0.1)]))]\n    oup = GaussianKernel3(num_landmark=num_cls, num_feature=nn, weights=weights, name='gkernel3')(inp)\n    model = Model(inp, oup, name='model_dense')\n    return {\n        'model': model,\n    }\nmodels_dense = make_models_dense()\nmodels_dense['model'].summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2d057cb4f5f5726c5424300a9c8a11795049d645"},"cell_type":"code","source":"model_dic = {\n    'model_img_cnvt': models_img_cnvt0['model_img_cnvt'],\n    'model_cnvt': model_cnvt,\n    'model_dense': models_dense['model']\n}\ntrain_ctrl = TrainableCtrl(model_dic)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3953b654fc78c1d8fb30ab9e2031c8a0a8b783ef"},"cell_type":"code","source":"def make_modelz(img_dim, model_cnvt0, model_cnvt, model_dense):\n    inp = Input(shape=(img_dim,), name='inp')\n    oup = model_cnvt0(inp)\n    oup = model_cnvt(oup)\n    oup1 = model_dense(oup)\n    pre_model = Model(inp, oup1)\n    pre_model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n    return {\n        'pre_model': pre_model,\n        'model_cnvt0': model_cnvt0,\n        'model_cnvt': model_cnvt,\n    }\n\nmodels = make_modelz(img_dim, models_img_cnvt0['model_img_cnvt'], model_cnvt, models_dense['model'])\nmodels['pre_model'].summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"02ae14cf05d21a5fd9f41859aeba99a46dd9af61"},"cell_type":"code","source":"THRESHOLD = 0.5\n\n# credits: https://www.kaggle.com/guglielmocamporese/macro-f1-score-keras\n\nK_epsilon = K.epsilon()\ndef f1(y_true, y_pred):\n    #y_pred = K.round(y_pred)\n    y_pred = K.cast(K.greater(K.clip(y_pred, 0, 1), THRESHOLD), K.floatx())\n    tp = K.sum(K.cast(y_true*y_pred, 'float'), axis=0)\n    tn = K.sum(K.cast((1-y_true)*(1-y_pred), 'float'), axis=0)\n    fp = K.sum(K.cast((1-y_true)*y_pred, 'float'), axis=0)\n    fn = K.sum(K.cast(y_true*(1-y_pred), 'float'), axis=0)\n\n    p = tp / (tp + fp + K_epsilon)\n    r = tp / (tp + fn + K_epsilon)\n\n    f1 = 2*p*r / (p+r+K_epsilon)\n    f1 = tf.where(tf.is_nan(f1), tf.zeros_like(f1), f1)\n    return K.mean(f1)\n\ndef f1_loss(y_true, y_pred):\n    \n    #y_pred = K.cast(K.greater(K.clip(y_pred, 0, 1), THRESHOLD), K.floatx())\n    tp = K.sum(K.cast(y_true*y_pred, 'float'), axis=0)\n    tn = K.sum(K.cast((1-y_true)*(1-y_pred), 'float'), axis=0)\n    fp = K.sum(K.cast((1-y_true)*y_pred, 'float'), axis=0)\n    fn = K.sum(K.cast(y_true*(1-y_pred), 'float'), axis=0)\n\n    p = tp / (tp + fp + K_epsilon)\n    r = tp / (tp + fn + K_epsilon)\n\n    f1 = 2*p*r / (p+r+K_epsilon)\n    f1 = tf.where(tf.is_nan(f1), tf.zeros_like(f1), f1)\n    return 1-K.mean(f1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3e34bf5869549a8d79f1a2deffde335bfe2381a9"},"cell_type":"code","source":"'''\nThanks Iafoss.\npretrained ResNet34 with RGBY\nhttps://www.kaggle.com/iafoss/pretrained-resnet34-with-rgby-0-460-public-lb\n'''\ngamma = 2.0\nepsilon = K.epsilon()\ndef focal_loss(y_true, y_pred):\n    pt = y_pred * y_true + (1-y_pred) * (1-y_true)\n    pt = K.clip(pt, epsilon, 1-epsilon)\n    CE = -K.log(pt)\n    FL = K.pow(1-pt, gamma) * CE\n    loss = K.sum(FL, axis=1)\n    return loss","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9cbe3ab1689e9285867cb2d459a760aba625fefe"},"cell_type":"markdown","source":"### train"},{"metadata":{"trusted":true,"_uuid":"f771716e318c4af129a6ca0ae35ff9186f46d3e0"},"cell_type":"code","source":"# train_ctrl.set_trainable_true('model_img_cnvt')\n# train_ctrl.set_trainable_false('model_cnvt')\n# train_ctrl.set_trainable_true('model_dense')\n\nmodels['pre_model'].compile(loss=focal_loss,\n                            optimizer='adam',\n                            metrics=['categorical_accuracy', 'binary_accuracy', f1])\nmodels['pre_model'].summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"25105534e921f9875164b863c5591be90f7ec0c4"},"cell_type":"code","source":"hst = models['pre_model'].fit(x_train_fl, y_cat, verbose=2,\n                          epochs=10, batch_size=64)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"720db2e71f1f4be781eef95a5e2a1cf758e07ade"},"cell_type":"code","source":"hst_history = hst.history","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9045f051c1a2a611b83e01677cd0704ef40852a3"},"cell_type":"code","source":"fig, ax = plt.subplots(1, 3, figsize=(20,5))\nax[0].set_title('loss')\nax[0].plot(list(range(len(hst_history[\"loss\"]))), hst_history[\"loss\"], label=\"Train loss\")\nax[1].set_title('acc')\nax[1].plot(list(range(len(hst_history[\"loss\"]))), hst_history[\"categorical_accuracy\"], label=\"categorical_accuracy\")\nax[1].plot(list(range(len(hst_history[\"loss\"]))), hst_history[\"binary_accuracy\"], label=\"binary_accuracy\")\nax[2].set_title('f1_score')\nax[2].plot(list(range(len(hst_history[\"loss\"]))), hst_history[\"f1\"], label=\"f1 score\")\nax[0].legend()\nax[1].legend()\nax[2].legend()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6d70a30f0250e8c9cd1db8013a83916529f3b30f"},"cell_type":"code","source":"#lm_cnvt = model_cnvt.layers[1].get_weights()[0]\nprint(lm_cnvt.shape)\n\ndf = pd.DataFrame(lm_cnvt[:,:5])\ndf.head()\nfig = sns.pairplot(df, markers=['o'], height=2.2, diag_kind='hist')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"24cd9931056c69c92ca829e33902094d3727c59e"},"cell_type":"code","source":"pred_img_cnvt_org = models_img_cnvt0['model_img_cnvt_org'].predict(x_train_fl, batch_size=128, verbose=1)\nprint(pred_img_cnvt_org.shape)\npred_img_cnvt_org","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9e65ac77e4e0096d96ed2fd2cb3188ea9681cdab"},"cell_type":"code","source":"df = pd.DataFrame(pred_img_cnvt_org[:,:5])\ndf['cls'] = ['c'+str(ee) for ee in y_train]\ndf.head()\nfig = sns.pairplot(df, markers='o', hue='cls', height=2.2, diag_kind='hist')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6b2cc28f81afca02f0029e673a7ab71e7f8bed96"},"cell_type":"code","source":"pred_img_cnvt = models_img_cnvt0['model_img_cnvt'].predict(x_train_fl, batch_size=128, verbose=1)\nprint(pred_img_cnvt.shape)\npred_img_cnvt","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"66863af2b1d3fa009e3952de3f0331dc9f0734c4"},"cell_type":"code","source":"df = pd.DataFrame(pred_img_cnvt[:,:5])\ndf['cls'] = ['c'+str(ee) for ee in y_train]\ndf.head()\nfig = sns.pairplot(df, markers='o', hue='cls', height=2.2, diag_kind='hist')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"12f0fe42f53425e48d4114116c67afaf236efeec"},"cell_type":"code","source":"#lm_cnvt = model_cnvt.layers[1].get_weights()[0]\nlm_cnvt.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2aedc1e4ccfa8141e459487e2668380774770901"},"cell_type":"code","source":"df = pd.DataFrame(np.vstack([pred_img_cnvt, lm_cnvt])[:,:5])\ndf['cls'] = ['c'+str(ee) for ee in y_train] + ['LM']*lm_cnvt.shape[0]\ndf.head()\nfig = sns.pairplot(df, markers=['.']*num_cls + ['s'], hue='cls', height=2.2, diag_kind='hist')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0f83c8e8fa7f66d951498f025f3a704563562fdd"},"cell_type":"code","source":"y_pred0 = models['pre_model'].predict(x_train_fl, batch_size=128, verbose=1)\ny_pred0.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"93e5619e79d6bc6d207c2c7b789e4bb814e1102f"},"cell_type":"code","source":"print(classification_report(y_train, np.argmax(y_pred0, axis=1)))\nconfusion_matrix(y_train, np.argmax(y_pred0, axis=1))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"67c6fa8b8f506429ff46f2152e897d0dd7d3b8fa"},"cell_type":"code","source":"pred_cnvt = model_cnvt.predict(pred_img_cnvt, batch_size=128, verbose=1)\nprint(pred_cnvt.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"452484e0683217aa1eb04cd7a9023997772839db"},"cell_type":"code","source":"df = pd.DataFrame(pred_cnvt)\ndf.columns = [\"comp_1\", \"comp_2\"]\ndf['cls'] = [str(ee) for ee in y_train]\ndf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ca5fa164760c1b1980430b0e7020895d6f10ac14"},"cell_type":"code","source":"matplotlib.rcParams['figure.figsize'] = (10.0, 10.0)\nsns.lmplot(\"comp_1\", \"comp_2\", hue=\"cls\", data=df, fit_reg=False, markers='.')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"db198c69b82271870d63b96b14e9ffc513b8e6bd"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8e9a788a232e2fdfa114d003b91dc2ef1cf2dc3d"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f771716e318c4af129a6ca0ae35ff9186f46d3e0"},"cell_type":"code","source":"# train_ctrl.set_trainable_true('model_img_cnvt')\n# train_ctrl.set_trainable_false('model_cnvt')\n# train_ctrl.set_trainable_true('model_dense')\n\n# models['pre_model'].compile(loss=focal_loss,\n#                             optimizer='adam',\n#                             metrics=['categorical_accuracy', 'binary_accuracy', f1])\n# models['pre_model'].summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d0b0640aa8c9be0159dea861b8dd965e8f6272f1"},"cell_type":"code","source":"# def lr_schedule(epoch):\n#     lr = 0.001\n#     if divmod(epoch,2)[1] == 1:\n#         lr *= (1/8)\n#     elif divmod(epoch,2)[1] == 0:\n#         pass\n#     print('Learning rate: ', lr)\n#     return lr\ndef lr_schedule(epoch):\n    lr = 0.001\n    if divmod(epoch,4)[1] == 3:\n        lr *= (1/8)\n    elif divmod(epoch,4)[1] == 2:\n        lr *= (1/4)\n    elif divmod(epoch,4)[1] == 1:\n        lr *= (1/2)\n    elif divmod(epoch,4)[1] == 0:\n        pass\n    print('Learning rate: ', lr)\n    return lr\n\nlr_scheduler = LearningRateScheduler(lr_schedule)\ncallbacks = [lr_scheduler]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"25105534e921f9875164b863c5591be90f7ec0c4"},"cell_type":"code","source":"hst = models['pre_model'].fit(x_train_fl, y_cat, verbose=2,\n                        epochs=8, batch_size=64,\n                        callbacks=callbacks)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6af96512b680cdfce1571d700d6815a9e7185b61"},"cell_type":"code","source":"'''merge hst_history'''\nfor k in hst.history:\n    hst_history[k].extend(hst.history[k])\n\nhst_history","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ec8e867285ff7781abb8ca1075875bd89d13b9dd"},"cell_type":"code","source":"fig, ax = plt.subplots(1, 3, figsize=(20,5))\nax[0].set_title('loss')\nax[0].plot(list(range(len(hst_history[\"loss\"]))), hst_history[\"loss\"], label=\"Train loss\")\nax[1].set_title('acc')\nax[1].plot(list(range(len(hst_history[\"loss\"]))), hst_history[\"categorical_accuracy\"], label=\"categorical_accuracy\")\nax[1].plot(list(range(len(hst_history[\"loss\"]))), hst_history[\"binary_accuracy\"], label=\"binary_accuracy\")\nax[2].set_title('f1_score')\nax[2].plot(list(range(len(hst_history[\"loss\"]))), hst_history[\"f1\"], label=\"f1 score\")\nax[0].legend()\nax[1].legend()\nax[2].legend()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6d70a30f0250e8c9cd1db8013a83916529f3b30f"},"cell_type":"code","source":"#lm_cnvt = model_cnvt.layers[1].get_weights()[0]\nprint(lm_cnvt.shape)\n\ndf = pd.DataFrame(lm_cnvt[:,:5])\ndf.head()\nfig = sns.pairplot(df, markers=['o'], height=2.2, diag_kind='hist')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"24cd9931056c69c92ca829e33902094d3727c59e"},"cell_type":"code","source":"pred_img_cnvt_org = models_img_cnvt0['model_img_cnvt_org'].predict(x_train_fl, batch_size=128, verbose=1)\nprint(pred_img_cnvt_org.shape)\npred_img_cnvt_org","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9e65ac77e4e0096d96ed2fd2cb3188ea9681cdab"},"cell_type":"code","source":"df = pd.DataFrame(pred_img_cnvt_org[:,:5])\ndf['cls'] = ['c'+str(ee) for ee in y_train]\ndf.head()\nfig = sns.pairplot(df, markers='o', hue='cls', height=2.2, diag_kind='hist')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6b2cc28f81afca02f0029e673a7ab71e7f8bed96"},"cell_type":"code","source":"pred_img_cnvt = models_img_cnvt0['model_img_cnvt'].predict(x_train_fl, batch_size=128, verbose=1)\nprint(pred_img_cnvt.shape)\npred_img_cnvt","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"66863af2b1d3fa009e3952de3f0331dc9f0734c4"},"cell_type":"code","source":"df = pd.DataFrame(pred_img_cnvt[:,:5])\ndf['cls'] = ['c'+str(ee) for ee in y_train]\ndf.head()\nfig = sns.pairplot(df, markers='o', hue='cls', height=2.2, diag_kind='hist')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"12f0fe42f53425e48d4114116c67afaf236efeec"},"cell_type":"code","source":"#lm_cnvt = model_cnvt.layers[1].get_weights()[0]\nlm_cnvt.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2aedc1e4ccfa8141e459487e2668380774770901"},"cell_type":"code","source":"df = pd.DataFrame(np.vstack([pred_img_cnvt, lm_cnvt])[:,:5])\ndf['cls'] = ['c'+str(ee) for ee in y_train] + ['LM']*lm_cnvt.shape[0]\ndf.head()\nfig = sns.pairplot(df, markers=['.']*num_cls + ['s'], hue='cls', height=2.2, diag_kind='hist')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0f83c8e8fa7f66d951498f025f3a704563562fdd"},"cell_type":"code","source":"y_pred0 = models['pre_model'].predict(x_train_fl, batch_size=128, verbose=1)\ny_pred0.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"93e5619e79d6bc6d207c2c7b789e4bb814e1102f"},"cell_type":"code","source":"print(classification_report(y_train, np.argmax(y_pred0, axis=1)))\nconfusion_matrix(y_train, np.argmax(y_pred0, axis=1))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"67c6fa8b8f506429ff46f2152e897d0dd7d3b8fa"},"cell_type":"code","source":"pred_cnvt = model_cnvt.predict(pred_img_cnvt, batch_size=128, verbose=1)\nprint(pred_cnvt.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"452484e0683217aa1eb04cd7a9023997772839db"},"cell_type":"code","source":"df = pd.DataFrame(pred_cnvt)\ndf.columns = [\"comp_1\", \"comp_2\"]\ndf['cls'] = [str(ee) for ee in y_train]\ndf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ca5fa164760c1b1980430b0e7020895d6f10ac14"},"cell_type":"code","source":"matplotlib.rcParams['figure.figsize'] = (10.0, 10.0)\nsns.lmplot(\"comp_1\", \"comp_2\", hue=\"cls\", data=df, fit_reg=False, markers='.')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4c83c6e9b50d4341f59e2bacdb685daa14f6a199"},"cell_type":"markdown","source":"### 2回目"},{"metadata":{"trusted":true,"_uuid":"c6cd446d5b09ad4661558df4210dc2b6cd9df954"},"cell_type":"code","source":"# train_ctrl.set_trainable_true('model_img_cnvt')\n# train_ctrl.set_trainable_true('model_cnvt')\n# train_ctrl.set_trainable_true('model_dense')\n\n# models['pre_model'].compile(loss=focal_loss,\n#                             optimizer='adam',\n#                             metrics=['categorical_accuracy', 'binary_accuracy', f1])\n# models['pre_model'].summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f771716e318c4af129a6ca0ae35ff9186f46d3e0"},"cell_type":"code","source":"def lr_schedule(epoch):\n    lr = 0.001/2\n    if divmod(epoch,2)[1] == 1:\n        lr *= (1/8)\n    elif divmod(epoch,2)[1] == 0:\n        pass\n    print('Learning rate: ', lr)\n    return lr\n\nlr_scheduler = LearningRateScheduler(lr_schedule)\ncallbacks = [lr_scheduler]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"25105534e921f9875164b863c5591be90f7ec0c4","scrolled":false},"cell_type":"code","source":"hst = models['pre_model'].fit(x_train_fl, y_cat, verbose=2,\n                        epochs=10, batch_size=64,\n                        callbacks=callbacks)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8983e0821488d1f5733fa9fa1f09856f3c9a8627"},"cell_type":"code","source":"'''merge hst_history'''\nfor k in hst.history:\n    hst_history[k].extend(hst.history[k])\n\nhst_history","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dee9b2352e19e1824046a6e9ccb18558a2386490"},"cell_type":"code","source":"fig, ax = plt.subplots(1, 3, figsize=(20,5))\nax[0].set_title('loss')\nax[0].plot(list(range(len(hst_history[\"loss\"]))), hst_history[\"loss\"], label=\"Train loss\")\nax[1].set_title('acc')\nax[1].plot(list(range(len(hst_history[\"loss\"]))), hst_history[\"categorical_accuracy\"], label=\"categorical_accuracy\")\nax[1].plot(list(range(len(hst_history[\"loss\"]))), hst_history[\"binary_accuracy\"], label=\"binary_accuracy\")\nax[2].set_title('f1_score')\nax[2].plot(list(range(len(hst_history[\"loss\"]))), hst_history[\"f1\"], label=\"f1 score\")\nax[0].legend()\nax[1].legend()\nax[2].legend()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5c4deec270fa51a2ca061c5bbc39c1e7c5d22b0d"},"cell_type":"code","source":"#lm_cnvt = model_cnvt.layers[1].get_weights()[0]\nprint(lm_cnvt.shape)\n\ndf = pd.DataFrame(lm_cnvt[:,:5])\ndf.head()\nfig = sns.pairplot(df, markers=['o'], height=2.2, diag_kind='hist')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"24cd9931056c69c92ca829e33902094d3727c59e"},"cell_type":"code","source":"pred_img_cnvt_org = models_img_cnvt0['model_img_cnvt_org'].predict(x_train_fl, batch_size=128, verbose=1)\nprint(pred_img_cnvt_org.shape)\npred_img_cnvt_org","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9e65ac77e4e0096d96ed2fd2cb3188ea9681cdab"},"cell_type":"code","source":"df = pd.DataFrame(pred_img_cnvt_org[:,:5])\ndf['cls'] = ['c'+str(ee) for ee in y_train]\ndf.head()\nfig = sns.pairplot(df, markers='o', hue='cls', height=2.2, diag_kind='hist')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6b2cc28f81afca02f0029e673a7ab71e7f8bed96"},"cell_type":"code","source":"pred_img_cnvt = models_img_cnvt0['model_img_cnvt'].predict(x_train_fl, batch_size=128, verbose=1)\nprint(pred_img_cnvt.shape)\npred_img_cnvt","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"66863af2b1d3fa009e3952de3f0331dc9f0734c4"},"cell_type":"code","source":"df = pd.DataFrame(pred_img_cnvt[:,:5])\ndf['cls'] = ['c'+str(ee) for ee in y_train]\ndf.head()\nfig = sns.pairplot(df, markers='o', hue='cls', height=2.2, diag_kind='hist')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"12f0fe42f53425e48d4114116c67afaf236efeec"},"cell_type":"code","source":"#lm_cnvt = model_cnvt.layers[1].get_weights()[0]\nlm_cnvt.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2aedc1e4ccfa8141e459487e2668380774770901"},"cell_type":"code","source":"df = pd.DataFrame(np.vstack([pred_img_cnvt, lm_cnvt])[:,:5])\ndf['cls'] = ['c'+str(ee) for ee in y_train] + ['LM']*lm_cnvt.shape[0]\ndf.head()\nfig = sns.pairplot(df, markers=['.']*num_cls + ['s'], hue='cls', height=2.2, diag_kind='hist')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0f83c8e8fa7f66d951498f025f3a704563562fdd"},"cell_type":"code","source":"y_pred = models['pre_model'].predict(x_train_fl, batch_size=128, verbose=1)\ny_pred.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"aa16b3c54e666ea702f0a4c7a81aefade17d245f"},"cell_type":"code","source":"y_pred_test = models['pre_model'].predict(x_test_fl, batch_size=128, verbose=1)\ny_pred_test.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"93e5619e79d6bc6d207c2c7b789e4bb814e1102f"},"cell_type":"code","source":"print(classification_report(y_train, np.argmax(y_pred, axis=1)))\nconfusion_matrix(y_train, np.argmax(y_pred, axis=1))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"67c6fa8b8f506429ff46f2152e897d0dd7d3b8fa"},"cell_type":"code","source":"pred_cnvt = model_cnvt.predict(pred_img_cnvt, batch_size=128, verbose=1)\nprint(pred_cnvt.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"452484e0683217aa1eb04cd7a9023997772839db"},"cell_type":"code","source":"df = pd.DataFrame(pred_cnvt)\ndf.columns = [\"comp_1\", \"comp_2\"]\ndf['cls'] = [str(ee) for ee in y_train]\ndf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ca5fa164760c1b1980430b0e7020895d6f10ac14"},"cell_type":"code","source":"matplotlib.rcParams['figure.figsize'] = (10.0, 10.0)\nsns.lmplot(\"comp_1\", \"comp_2\", hue=\"cls\", data=df, fit_reg=False, markers='.')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e3eaa4eb6c1c406adb270c8b7b4ec40b27d2efb4"},"cell_type":"code","source":"df_pred1 = pd.DataFrame(train[['label']])\ndf_pred1 = pd.concat([df_pred1, pd.DataFrame(y_pred)], axis=1)\nprint(df_pred1.shape)\ndf_pred1.to_csv('proba.csv', index=False)\ndf_pred1.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1cf833ab35299f630686ff090758feda50527ca3"},"cell_type":"code","source":"df_pred1_test = pd.DataFrame({'ImageId': np.array(range(test.shape[0]))+1})\ndf_pred1_test = pd.concat([df_pred1_test, pd.DataFrame(y_pred_test)], axis=1)\nprint(df_pred1_test.shape)\ndf_pred1_test.to_csv('proba_test.csv', index=False)\ndf_pred1_test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c8ec15c152b2c0a8db0774cdff2fb6beea820ca8"},"cell_type":"code","source":"submit_csv = pd.DataFrame({\n    'ImageId': np.array(range(test.shape[0]))+1,\n    'Label': np.argmax(y_pred_test, axis=1)\n})\nsubmit_csv.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b543fbab90358dee6afc81bb14ddc0e0f2b8da70"},"cell_type":"code","source":"submit_csv.to_csv('submit.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a955cbb71983c58b374505645028a7b4d8c6055c"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4a5c034fc88a46fa90c9523cec1017c8044d0aa3"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}