{"cells":[{"metadata":{"_uuid":"5e0db6a2c0980b155648e11e30fbdc9ac134db31"},"cell_type":"markdown","source":"# Freesound Dataset Kaggle 2018 Solution\n\nThis is private 0.917 / public 0.950 solution."},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"collapsed":true},"cell_type":"code","source":"# Kernel preparation\n\n# Warning suppression\nimport warnings\nwarnings.simplefilter('ignore')\nimport numpy as np\nnp.warnings.filterwarnings('ignore')\nnp.random.seed(1001)\n# Cliche\nimport os\nimport tensorflow as tf\nos.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' \nimport sys\nimport shutil\nfrom pathlib import Path\nimport IPython\nimport matplotlib\nimport matplotlib.pyplot as plt\nimport pandas as pd\nimport seaborn as sns\nfrom tqdm import tqdm_notebook\nfrom sklearn.model_selection import StratifiedKFold\n%matplotlib inline\nmatplotlib.style.use('ggplot')\n\ndef play_audio(data):\n    IPython.display.display(IPython.display.Audio(data=data))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"# Dataset preparation\n\n# Root folder that contains entire dataset\n#DATAROOT = Path.home() / '.kaggle/competitions/freesound-audio-tagging'\nDATAROOT = Path('../input') / 'freesound-audio-tagging'\n# Root folder that contains extra data provided by this kernel\nimport tarfile\ntar = tarfile.open(Path('../input') / 'fsd2018extra' / 'fsd2018_extra.tgz')\ntar.extractall()\ntar.close()\nEXTRA = Path('.') / 'fsd2018_extra'\n# Data frames for train/test dataset\ndf_train = pd.read_csv(DATAROOT / 'train.csv')\ndf_test = pd.read_csv(DATAROOT / 'sample_submission.csv')\n# Labels and integer converter\nlabels = df_train.label.unique()\nlabel2int = {l:i for i, l in enumerate(labels)}\nnum_classes = len(labels)\n# Train data sample index of manually verified ones\ntrain_verified_idx = np.array(df_train[df_train.manually_verified == 1].index)\n# Plain y_train label\nplain_y_train = np.array([label2int[label] for label in df_train.label])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bcf0ed6c1fe3bd5b4ff549f1f13263f61bf3df92"},"cell_type":"code","source":"## External dependencies - mixup & random eraser\n#\n# Downloaded from https://github.com/yu4u/mixup-generator\n#\n\nimport numpy as np\n\n\nclass MixupGenerator():\n    def __init__(self, X_train, y_train, batch_size=32, alpha=0.2, shuffle=True, datagen=None):\n        self.X_train = X_train\n        self.y_train = y_train\n        self.batch_size = batch_size\n        self.alpha = alpha\n        self.shuffle = shuffle\n        self.sample_num = len(X_train)\n        self.datagen = datagen\n\n    def __call__(self):\n        while True:\n            indexes = self.__get_exploration_order()\n            itr_num = int(len(indexes) // (self.batch_size * 2))\n\n            for i in range(itr_num):\n                batch_ids = indexes[i * self.batch_size * 2:(i + 1) * self.batch_size * 2]\n                X, y = self.__data_generation(batch_ids)\n\n                yield X, y\n\n    def __get_exploration_order(self):\n        indexes = np.arange(self.sample_num)\n\n        if self.shuffle:\n            np.random.shuffle(indexes)\n\n        return indexes\n\n    def __data_generation(self, batch_ids):\n        _, h, w, c = self.X_train.shape\n        l = np.random.beta(self.alpha, self.alpha, self.batch_size)\n        X_l = l.reshape(self.batch_size, 1, 1, 1)\n        y_l = l.reshape(self.batch_size, 1)\n\n        X1 = self.X_train[batch_ids[:self.batch_size]]\n        X2 = self.X_train[batch_ids[self.batch_size:]]\n        X = X1 * X_l + X2 * (1 - X_l)\n\n        if self.datagen:\n            for i in range(self.batch_size):\n                X[i] = self.datagen.random_transform(X[i])\n                X[i] = self.datagen.standardize(X[i])\n\n        if isinstance(self.y_train, list):\n            y = []\n\n            for y_train_ in self.y_train:\n                y1 = y_train_[batch_ids[:self.batch_size]]\n                y2 = y_train_[batch_ids[self.batch_size:]]\n                y.append(y1 * y_l + y2 * (1 - y_l))\n        else:\n            y1 = self.y_train[batch_ids[:self.batch_size]]\n            y2 = self.y_train[batch_ids[self.batch_size:]]\n            y = y1 * y_l + y2 * (1 - y_l)\n\n        return X, y\n\n\ndef get_random_eraser(p=0.5, s_l=0.02, s_h=0.4, r_1=0.3, r_2=1/0.3, v_l=0, v_h=255):\n    def eraser(input_img):\n        img_h, img_w, _ = input_img.shape\n        p_1 = np.random.rand()\n\n        if p_1 > p:\n            return input_img\n\n        while True:\n            s = np.random.uniform(s_l, s_h) * img_h * img_w\n            r = np.random.uniform(r_1, r_2)\n            w = int(np.sqrt(s / r))\n            h = int(np.sqrt(s * r))\n            left = np.random.randint(0, img_w)\n            top = np.random.randint(0, img_h)\n\n            if left + w <= img_w and top + h <= img_h:\n                break\n\n        c = np.random.uniform(v_l, v_h)\n        input_img[top:top + h, left:left + w, :] = c\n\n        return input_img\n\n    return eraser","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"edb28189ed2472948b1550a9d8c78d14a2001faa"},"cell_type":"markdown","source":"## Strategy\n\nThis solution has two different approaches, and overall result is the ensemble of both results.\n\n- Approach LH: Uses highest feature resolution, but only with beginning part of sounds.\n- Approach X: Splits samples and uses all the samples long enough, but with coarser feature resolution.\n\nMotivation to have these two is based on observation of how models performs well and fails.\n\n- Useful information seems to be around the beginning part of sample. This should be natural considering how we record and edit target sound. This assumption was supported by good results of approach LH which uses only the beginning some sconds.\n- But it fails with such samples that has important content in the middle or later part. To save these cases, we need another approach that uses entire sample wave. This is approach X.\n\nHere starting with making configuration variables for these approaches."},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"ed65ca4db836a81832aad4f694ccf1f2efa9f81d"},"cell_type":"code","source":"confLH, confX = {}, {}\nconfs = [confLH, confX]\nconfLH['folder'] = Path('LH')\nconfX['folder'] = Path('X')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"fec426dc755672640d1a213dcb180c43a40482df"},"cell_type":"markdown","source":"## Blacklist for Making Clean Training Dataset\n\nTraining dataset contains followings:\n\n1. Manually verified clearly labeled data.\n2. Weak but clean data which almost cosistent with their labels.\n3. Dirty data that are one or multiple of: silent, bad sound quality, too much confusing, or nothing to do with its label.\n\nNo. 3 is harmful for performance. Then manually removed them by following procedure:\n\n1. Create best model at the time.\n2. Predict and make a list of samples which the model failed to predict.\n3. Check the failed samples one by one by hand, put dirty samples on blacklist.\n4. Repeat from top until we don't find bad samples any more."},{"metadata":{"trusted":true,"_uuid":"9e88b96b49c81202b3aa272f05dd8063c4e0cc22"},"cell_type":"code","source":"# This is the blacklist manually picked.\ntrain_blacklist_index = np.load(EXTRA / 'train_blacklist.npy')\nprint('Blacklist has %d items.' % len(train_blacklist_index))\nprint(train_blacklist_index)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b44c403e5e29bf03ee78265eb39f839682606f0b"},"cell_type":"markdown","source":"## Preprocess and Convert Data\n\nAll the train and test data are converted in advance, so that we can simply load it when training.\n\nWe have bunch of audio parameters here which performance of models fairly depends on."},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"095906497da03223767b57d9cca6dc54684d8167"},"cell_type":"code","source":"# Approach LH parameters: highest resolutions\nconfLH['sampling_rate'] = 44100\nconfLH['duration'] = 4\nconfLH['hop_length'] = 882 # 20ms\nconfLH['fmin'] = 20\nconfLH['fmax'] = confLH['sampling_rate'] // 2\nconfLH['n_mels'] = 128\nconfLH['n_fft'] = confLH['n_mels'] * 20\nconfLH['audio_split'] = 'head'\n\n# Approach X uses longer sound, then it uses suppressed \nconfX['sampling_rate'] = 26000\nconfX['duration'] = 6\nconfX['hop_length'] = 520 # 20ms\nconfX['fmin'] = 20\nconfX['fmax'] = confX['sampling_rate'] // 2\nconfX['n_mels'] = 48\nconfX['n_fft'] = confX['n_mels'] * 20\nconfX['audio_split'] = 'dont_crop'\n\n# Auto calculate other configuration paramters\n\ndef auto_complete_conf(conf):\n    conf['samples'] = conf['sampling_rate'] * conf['duration']\n    conf['dims'] = (conf['n_mels'], 1 + int(np.floor(conf['samples']/conf['hop_length'])), 1)\n\nauto_complete_conf(confLH)\nauto_complete_conf(confX)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0df19e50d8c4635ad5ee56d6c7ccf04cd8b4a240"},"cell_type":"markdown","source":"### Preprocessing functions\n\n- read_audio: Trims silence of both ends, then split if longer or pad if shorter.\n- audio_to_melspectrogram: Convert audio samples into mel-spectrogram data.\n- read_as_melspectrogram: Do both above."},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"992a4659e604cc0c8b575ca91be99ed1c5907df7"},"cell_type":"code","source":"import librosa\nimport librosa.display\n\ndef read_audio(conf, pathname):\n    y, sr = librosa.load(pathname, sr=conf['sampling_rate'])\n    # trim silence\n    if 0 < len(y): # workaround: 0 length causes error\n        y, _ = librosa.effects.trim(y) # trim, top_db=default(60)\n    # make it unified length to conf.samples\n    if len(y) > conf['samples']: # long enough\n        if conf['audio_split'] == 'head':\n            y = y[0:0+conf['samples']]\n    else: # pad blank\n        padding = conf['samples'] - len(y)    # add padding at both ends\n        offset = padding // 2\n        y = np.pad(y, (offset, conf['samples'] - len(y) - offset), 'constant')\n    return y\n\ndef audio_to_melspectrogram(conf, audio):\n    spectrogram = librosa.feature.melspectrogram(audio, \n                                                 sr=conf['sampling_rate'],\n                                                 n_mels=conf['n_mels'],\n                                                 hop_length=conf['hop_length'],\n                                                 n_fft=conf['n_fft'],\n                                                 fmin=conf['fmin'],\n                                                 fmax=conf['fmax'])\n    spectrogram = librosa.power_to_db(spectrogram)\n    spectrogram = spectrogram.astype(np.float32)\n    return spectrogram\n\ndef show_melspectrogram(mels, conf):\n    librosa.display.specshow(mels, x_axis='time', y_axis='mel', \n                             sr=conf['sampling_rate'], hop_length=conf['hop_length'],\n                            fmin=conf['fmin'], fmax=conf['fmax'])\n    plt.colorbar(format='%+2.0f dB')\n    plt.title('Log-frequency power spectrogram')\n    plt.show()\n\ndef read_as_melspectrogram(conf, pathname, debug_display=False):\n    x = read_audio(conf, pathname)\n    mels = audio_to_melspectrogram(conf, x)\n    if debug_display:\n        IPython.display.display(IPython.display.Audio(x, rate=conf['sampling_rate']))\n        show_melspectrogram(mels, conf)\n    return mels","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a9858ce913c16c33a2dd9c52f6be488c40392bbf"},"cell_type":"code","source":"# Preprocessing examples\n_ = read_as_melspectrogram(confLH, DATAROOT / 'audio_train' / df_train.fname[0], debug_display=True)\n_ = read_as_melspectrogram(confX, DATAROOT / 'audio_train' / df_train.fname[0], debug_display=True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0b68120665585e19934e048cb47d6a686a9ae94b"},"cell_type":"markdown","source":"LH has fine-grained features for 4 seconds, and  X has more abstract whole sample (for now).\n\n### Converter"},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"d92cc6b2a3dda2dd54259876b2574fcb562409f4"},"cell_type":"code","source":"def split_long_data(conf, X):\n    # Splits long mel-spectrogram data with small overlap\n    L = X.shape[1]\n    one_length = conf['dims'][1]\n    loop_length = int(one_length * 0.9)\n    min_length = int(one_length * 0.2)\n    print(' sample length', L, 'to cut every', one_length)\n    for idx in range(L // loop_length):\n        cur = loop_length * idx\n        rest = L - cur\n        if one_length <= rest:\n            yield X[:, cur:cur+one_length]\n        elif min_length <= rest:\n            cur = L - one_length\n            yield X[:, cur:cur+one_length]\n\ndef convert_X(df, conf, datapath):\n    # Convert all files listed on df.fname\n    # Then generates X (contains mel-spectrograms)\n    # and index mapping to original sample order\n    X = []\n    index_map = []\n    for i, fname in enumerate(df.fname):\n        print('processing', fname)\n        data = read_as_melspectrogram(conf, datapath / fname)\n        for chunk in split_long_data(conf, data):\n            X.append(np.expand_dims(chunk, axis=-1))\n            index_map.append(i)\n    return np.array(X), np.array(index_map)\n\ndef convert_y_train(idx_train, plain_y_train):\n    return np.array( [plain_y_train[idx] for idx in idx_train])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ecb0e9e097370dcec002dc5b08056441bcea6dae"},"cell_type":"code","source":"# Example - process up to 10 samples\nprint('All samples will be cut per split length (=duration)')\nX_train, idx_train = convert_X(df_train[:10], confX, DATAROOT / 'audio_train')\ny_train = convert_y_train(idx_train, plain_y_train)\nprint('Now original 10 samples were cut into ', len(idx_train), 'samples.')\nprint()\nprint('idx_train holds original sample index, y_train is also converted to have the same length with X_train/idx_train.')\nprint('idx_train', idx_train)\nprint('y_train', y_train)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"05d6cfeb065a114f9c521bfa825f7462c023fc40"},"cell_type":"markdown","source":"## Create Dataset Files\n\nNow convert all the data and store them on files.\n\nToy example dataset which is subset of whole samples will be prepared by default, for running on Kaggle kernel."},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"aa13b3d352f58aba270b7496a5a26ee3065362b7"},"cell_type":"code","source":"# Data utilities\ndef datapath(conf, filename):\n    return conf['folder'] / filename\n\ndef loaddata(conf, filename):\n    return np.load(conf['folder'] / filename)\n\n#### This is Toy example by default ####\nTRYING_AS_TOY = True # False if you like creating full set\n\nfor conf in confs:\n    conf['folder'].mkdir(parents=True, exist_ok=True)\n    if TRYING_AS_TOY:\n        for file in ['X_train', 'y_train', 'idx_train', 'X_test', 'idx_test']:\n            shutil.copy(EXTRA/datapath(conf, file+'.npy'), datapath(conf, file+'.npy'))\n        plain_y_train = np.load(EXTRA/'toy_plain_y_train.npy')\n        train_verified_idx = np.load(EXTRA/'toy_train_verified_idx.npy')\n        train_blacklist_index = np.load(EXTRA / 'toy_train_blacklist.npy')\n    else:\n        if not os.path.exists(datapath(conf, 'X_train.npy')):\n            X_train, idx_train = convert_X(df_train, conf, DATAROOT / 'audio_train')\n            y_train = convert_y_train(idx_train, plain_y_train)\n            np.save(datapath(conf, 'X_train.npy'), X_train)\n            np.save(datapath(conf, 'y_train.npy'), y_train)\n            np.save(datapath(conf, 'idx_train.npy'), idx_train)\n\n            X_test, idx_test = convert_X(df_test, conf, DATAROOT / 'audio_test')\n            np.save(datapath(conf, 'X_test.npy'), X_test)\n            np.save(datapath(conf, 'idx_test.npy'), idx_test)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"743c40d628109a8a4f2fadd8db161969869301a2"},"cell_type":"markdown","source":"## Models\n\nSEResNet was basically used for competition submission. Here AlexNet based model is enabled by default for Kaggle kernel."},{"metadata":{"trusted":true,"_uuid":"6614ab715e8baa9b2169433829a4709d0b539293"},"cell_type":"code","source":"import keras\nfrom keras.layers import Dense, Conv2D, AveragePooling2D\nfrom keras.layers import MaxPooling2D, GlobalAveragePooling2D, Dropout, BatchNormalization, Flatten\nfrom keras.models import Model, Sequential\n\ndef create_model(conf, num_classes):\n    input_shape = conf['dims']\n    if False:\n        # This is the model used for competition\n        # Thanks to: https://github.com/titu1994/keras-squeeze-excite-network\n        from se_resnet import SEResNet\n        model = SEResNet(input_shape=input_shape,\n               width=1,\n               bottleneck=True,\n               weight_decay=1e-4,\n               include_top=True,\n               depth=[3, 4, 4, 3],\n               filters=[32, 64, 128, 256],\n               weights=None,\n               input_tensor=None,\n               pooling=None,\n               classes=num_classes)\n    elif False: # Or if you prefer VGG16\n        from keras.applications.vgg16 import VGG16\n        # create new empty model with desired input_shape\n        base_model = VGG16(weights=None, input_shape=input_shape, include_top=False)\n        # you can also set this as: weights='imagenet' if you want transfer learning from imagenet\n        x = GlobalAveragePooling2D()(base_model.output)\n        x = Dense(1024, activation='relu')(x)\n        predictions = Dense(num_classes, activation='softmax')(x)\n        model = Model(inputs=base_model.input, outputs=predictions)\n    else: # AlexNet based model\n        model = Sequential()\n        model.add(Conv2D(48, 11,  input_shape=input_shape, strides=(2,3), activation='relu', padding='same'))\n        model.add(MaxPooling2D(3, strides=(1,2)))\n        model.add(BatchNormalization())\n        model.add(Conv2D(128, 5, strides=(2,3), activation='relu', padding='same'))\n        model.add(MaxPooling2D(3, strides=2))\n        model.add(BatchNormalization())\n        model.add(Conv2D(192, 3, strides=(1, 2), activation='relu', padding='same'))\n        model.add(Conv2D(192, 3, strides=(1, 1), activation='relu', padding='same'))\n        model.add(Conv2D(128, 3, strides=(1, 1), activation='relu', padding='same'))\n        model.add(MaxPooling2D(3, strides=(1,2)))\n        model.add(BatchNormalization())\n        model.add(Flatten())\n        model.add(Dense(256, activation='relu'))\n        model.add(Dropout(0.5))\n        model.add(Dense(256, activation='relu'))\n        model.add(Dropout(0.5))\n        model.add(Dense(num_classes, activation='softmax'))\n\n    model.compile(loss='categorical_crossentropy',\n              optimizer=keras.optimizers.Adam(lr=conf['learning_rate']),\n              metrics=['accuracy'])\n    model.summary()\n    return model","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ed7393e36792411b56b9c7a3e42d4180d4879e56"},"cell_type":"markdown","source":"## Training\n\nDatasets and model are ready, now prepare for training.\n\n\n\n### Utilities for Balancing Category Distribution"},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"8e4c47c94868be6ea2885c18eaa134801d827995"},"cell_type":"code","source":"def get_class_distribution(y):\n    # y_cls can be one of [OH label, index of class, class label name]\n    # convert OH to index of class\n    y_cls = [np.argmax(one) for one in y] if len(np.array(y).shape) == 2 else y\n    # y_cls can be one of [index of class, class label name]\n    classset = sorted(list(set(y_cls)))\n    sample_distribution = {cur_cls:len([one for one in y_cls if one == cur_cls]) for cur_cls in classset}\n    return sample_distribution\n\ndef get_class_distribution_list(y, num_classes):\n    dist = get_class_distribution(y)\n    assert(y[0].__class__ != str) # class index or class OH label only\n    list_dist = np.zeros((num_classes))\n    for i in range(num_classes):\n        if i in dist:\n            list_dist[i] = dist[i]\n    return list_dist\n\nfrom imblearn.over_sampling import RandomOverSampler\ndef balance_class_by_over_sampling(X, y):\n    Xidx = [[xidx] for xidx in range(len(X))]\n    y_cls = [np.argmax(one) for one in y]\n    classset = sorted(list(set(y_cls)))\n    sample_distribution = [len([one for one in y_cls if one == cur_cls]) for cur_cls in classset]\n    nsamples = np.max(sample_distribution)\n    flat_ratio = {cls:nsamples for cls in classset}\n    Xidx_resampled, y_cls_resampled = RandomOverSampler(ratio=flat_ratio, random_state=42).fit_sample(Xidx, y_cls)\n    sampled_index = [idx[0] for idx in Xidx_resampled]\n    return np.array([X[idx] for idx in sampled_index]), np.array([y[idx] for idx in sampled_index])\n\ndef visualize_class_balance(title, y, labels):\n    sample_dist_list = get_class_distribution_list(y, len(labels))\n    index = range(len(labels))\n    fig, ax = plt.subplots(1, 1, figsize = (16, 5))\n    ax.bar(index, sample_dist_list)\n    ax.set_xlabel('Label')\n    ax.set_xticks(index)\n    ax.set_xticklabels(labels, rotation='vertical')\n    ax.set_ylabel('Number of Samples')\n    ax.set_title(title)\n    fig.show()\n\ndef print_class_balance(title, y, labels):\n    distributions = get_class_distribution(y)\n    dist_dic = {labels[cls]:distributions[cls] for cls in distributions}\n    print(title, '=', dist_dic)\n    zeroclasses = [label for i, label in enumerate(labels) if i not in distributions.keys()]\n    if 0 < len(zeroclasses):\n        print(' 0 sample classes:', zeroclasses)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"eea7560337a8e5ec99abb13f1a196f44a1042cee"},"cell_type":"markdown","source":"### Dataset related Utilities\n\nAll the augmentatoion is done by data generators."},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"aaa3cbf068900cd883a16bd735bd4af6319c128f"},"cell_type":"code","source":"from keras.preprocessing.image import ImageDataGenerator\n\ndef create_generators(conf, _Xtrain, _ytrain, _Xvalid, _yvalid):\n    # Create Keras ImageDataGenerator\n    aug_datagen = ImageDataGenerator(\n        featurewise_center=conf['normalize'] == 'featurewise',\n        featurewise_std_normalization=conf['normalize'] == 'featurewise',\n        rotation_range=0,\n        width_shift_range=0.4,\n        height_shift_range=0.0,\n        horizontal_flip=True,\n        preprocessing_function=get_random_eraser(v_l=-1, v_h=1)\n    )\n    plain_datagen = ImageDataGenerator(\n        featurewise_center=aug_datagen.featurewise_center,\n        featurewise_std_normalization=aug_datagen.featurewise_std_normalization,\n    )\n    # Set featurewise normalization mean/std\n    if aug_datagen.featurewise_center:\n        print(' normalize featurewise')\n        aug_datagen.mean, aug_datagen.std = np.mean(_Xtrain), np.std(_Xtrain)\n        plain_datagen.mean, plain_datagen.std = aug_datagen.mean, aug_datagen.std\n    # Create Generators\n    train_generator = MixupGenerator(_Xtrain, _ytrain, \n                                     alpha=1.0, batch_size=conf['batch_size'], datagen=aug_datagen)()\n    valid_generator = plain_datagen.flow(_Xvalid, _yvalid,\n                                         batch_size=conf['batch_size'], shuffle=False)\n    return train_generator, valid_generator, plain_datagen\n\ndef get_steps_per_epoch(conf, _Xtrain, _Xvalid):\n    train_steps_per_epoch = len(_Xtrain) // conf['batch_size']\n    valid_steps_per_epoch = len(_Xvalid) // conf['batch_size']\n    return train_steps_per_epoch, valid_steps_per_epoch\n\ndef samplewise_mean_X(X):\n    for i in range(len(X)):\n        X[i] -= np.mean(X[i], keepdims=True)\n        X[i] /= (np.std(X[i], keepdims=True) + 1.0) # Kind of Compressor effect","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"40806735272676a3d12c3463fa723eaedaae5fc4"},"cell_type":"markdown","source":"### Fold Generator & Trainer"},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"4b071d8fe35e9401393cfa3ae147528206a72dd3"},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\ndef get_cross_valid_fold_balanced(conf, fold, X_train, y_train, idx_train):\n    indices = np.array(range(len(X_train)))\n    # Cross validation split -> _Xtrain|_ytrain, _Xvalid|_yvalid\n    _, _, _, _, train_fold, valid_fold = train_test_split(X_train, y_train, indices,\n                                                          test_size=conf['test_size'],\n                                                          random_state=conf['random_state'] + fold*10)\n    _Xtrain, _ytrain = X_train[train_fold], y_train[train_fold]\n\n    # Validation set to filter non-verified samples if requested\n    if conf['valid_limit'] == 'manually_verified_only':\n        filtered = [idx for idx in valid_fold if idx_train[idx] in train_verified_idx]\n        print(' valid set is filtered to verified samples only, %d -> %d' % (len(valid_fold), len(filtered)))\n        valid_fold = filtered\n    _Xvalid, _yvalid = X_train[valid_fold], y_train[valid_fold]\n\n    # Balance distribution -> _Xtrain|_ytrain (overwritten)\n    print_class_balance('Current fold category distribution', _ytrain, labels)\n    _Xtrain, _ytrain = balance_class_by_over_sampling(_Xtrain, _ytrain)\n    print_class_balance('after balanced', _ytrain, labels)\n\n    return _Xtrain, _ytrain, _Xvalid, _yvalid\n\nfrom keras.callbacks import (EarlyStopping, LearningRateScheduler,\n                             ModelCheckpoint, TensorBoard, ReduceLROnPlateau)\nfrom keras import backend as K\ndef train_model(conf, fold, model, train_generator, valid_generator,\n                train_steps_per_epoch, valid_steps_per_epoch,\n                init_best_weights=False, this_epochs=None):\n    callbacks = [\n        ModelCheckpoint(str(datapath(conf, 'best_%d.h5' % fold)),\n                        monitor='val_loss', verbose=1, save_best_only=True, save_weights_only=True),\n        TensorBoard(log_dir=str(datapath(conf, 'logs%s/fold_%d' % (conf['folder'], fold))), write_graph=True)\n    ]\n    # Create model\n    if model is None:\n        model = create_model(conf, num_classes)\n        #if fold == 0:\n        #    model.summary()\n        # Load weights\n        weight_filename = str(init_best_weights) # for when file name was set\n        if weight_filename == 'True': weight_filename = str(datapath(conf, 'best_%d.h5' % fold))\n        if weight_filename is not 'False':\n            print(' Initializing model with last best weights:', weight_filename)\n            model.load_weights(weight_filename)\n    # Train model\n    history = model.fit_generator(train_generator,\n                    steps_per_epoch=train_steps_per_epoch,\n                    epochs=conf['epochs'] if this_epochs is None else this_epochs,\n                    validation_data=valid_generator, \n                    validation_steps=valid_steps_per_epoch,\n                    callbacks=callbacks,\n                    verbose=conf['verbose'])\n    return model, history","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1340efafd6ff9fea75a89b447295db8bb52c154c"},"cell_type":"markdown","source":"### More training utilities"},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"74eb41b15fd1f3512990e069eead0ecc1ca0e86a"},"cell_type":"code","source":"def geometric_mean_preds(_preds):\n    preds = _preds.copy()\n    for i in range(1, preds.shape[0]):\n        preds[0] = np.multiply(preds[0], preds[i])\n    return preds[0]\n\ndef get_unified_preds(preds, pred_idx, N):\n    mean_results = []\n    for idx in range(N):\n        this_preds = preds[np.where(pred_idx == idx)]\n        if len(this_preds) <= 0:\n            print(' no result: %d' % idx)\n            mean_results.append(np.ones((preds.shape[1],)))\n        else:\n            mean_results.append(geometric_mean_preds(this_preds))\n    return np.array(mean_results)\n\ndef evaluate_pred_acc(y, uni_preds, idx_map, N):\n    uni_y = []\n    for idx in range(N):\n        uni_y.append(y[np.where(idx_map == idx)[0]][0])\n    uni_y = np.array(uni_y)\n\n    refs = np.argmax(uni_y, axis=1)\n    results = np.argmax(uni_preds, axis=1)\n    acc = np.sum(refs == results) / len(refs)\n    n_verified = len(refs[train_verified_idx])\n    acc_verified = np.sum(refs[train_verified_idx] == results[train_verified_idx]) / n_verified\n    return acc, acc_verified\n\ndef evaluate_fold(conf, fold, filenametmpl, model, plain_datagen, X, idx_map, y=None, verified_idx=None):\n    # predict\n    _y = keras.utils.to_categorical(np.ones((len(X)))) if y is None else y\n    test_generator = plain_datagen.flow(X, _y, batch_size=conf['batch_size'], shuffle=False)\n    preds = model.predict_generator(test_generator)\n    preds = get_unified_preds(preds, idx_map, np.max(idx_map) + 1)\n    # save & return acc\n    np.save(datapath(conf, filenametmpl % fold), preds)\n    # evaluate \n    if y is not None:\n        return evaluate_pred_acc(y, preds, idx_map, len(plain_y_train))\n    return None, None\n\ndef run_fold(conf, fold, dataset, model=None, init_best_weights=False, eval_only=False):\n    X_train, y_train, idx_train, all_X_train, all_y_train, all_idx_train, X_test, idx_test = dataset\n    print('----- Fold#%d ----' % fold)\n    # c. Cross validation split & balance # of samples\n    _Xtrain, _ytrain, _Xvalid, _yvalid = \\\n        get_cross_valid_fold_balanced(conf, fold, X_train, y_train, idx_train)\n\n    # d. Train model\n    train_generator, valid_generator, plain_datagen = \\\n        create_generators(conf, _Xtrain, _ytrain, _Xvalid, _yvalid)\n    train_steps_per_epoch, valid_steps_per_epoch = \\\n        get_steps_per_epoch(conf, _Xtrain, _Xvalid)\n    model, history = train_model(conf, fold, model, train_generator, valid_generator,\n                                train_steps_per_epoch, valid_steps_per_epoch,\n                                 init_best_weights=init_best_weights,\n                                this_epochs=0 if eval_only else None)\n\n    # e. Evaluate with all train sample\n    model.load_weights(datapath(conf, 'best_%d.h5' % fold))\n    acc, acc_v = evaluate_fold(conf, fold, 'train_predictions_%d.npy', model, plain_datagen,\n                               all_X_train, all_idx_train, all_y_train, train_verified_idx)\n    evaluate_fold(conf, fold, 'test_predictions_%d.npy', model, plain_datagen, X_test, idx_test)\n\n    print('Trainset accuracy =', acc, '(tested all over the original training set)')\n    print('Verified samples accuracy =', acc_v, '(tested over manually verified samples only)')\n    return acc, acc_v, history, model, plain_datagen","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"adaeb18a7e4abc09c511a347afc0ce7af1ce6d06"},"cell_type":"markdown","source":"### Training Parameters"},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"ca6c80900fe8af0e51f71268a43821ca1a4b30d8"},"cell_type":"code","source":"confLH['n_fold'] = 2 if TRYING_AS_TOY else 5\nconfLH['normalize'] = 'samplewise'\nconfLH['valid_limit'] = 'manually_verified_only'\nconfLH['random_state'] = 42\nconfLH['test_size'] = 0.2\nconfLH['batch_size'] = 32\nconfLH['learning_rate'] = 0.0001\nconfLH['epochs'] = 10 if TRYING_AS_TOY else 200\nconfLH['verbose'] = 1\n\nconfX['n_fold'] = 2 if TRYING_AS_TOY else 5\nconfX['normalize'] = 'featurewise'\nconfX['valid_limit'] = None\nconfX['random_state'] = 42\nconfX['test_size'] = 0.2\nconfX['batch_size'] = 32\nconfX['learning_rate'] = 0.0001\nconfX['epochs'] = 10 if TRYING_AS_TOY else 200\nconfX['verbose'] = 1","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"54a4365c0c6163b9ac423f6dea9dfb31c42c6811"},"cell_type":"markdown","source":"### Now let's begin\n\nTraining will do followings.\n\n- a. Load all dataset -> all_(X|y|idx)_train, (X|idx)_test\n- b. Removing samples on the blacklist -> X|y|idx\n- c. Cross validation split & balance # of samples\n- d. Train model with data (augmentation) generators\n- e. Evaluate with all train sample"},{"metadata":{"trusted":true,"_uuid":"9ec4ebd3e4f1a16d9b4d7d9d0c630c6803f7f4d7"},"cell_type":"code","source":"for conf in [confX]: # Running confX only, change this to confs if you need running both confX and confLH\n    print('== Attempt [%s] ==' % conf['folder'])\n\n    # a. Load all dataset -> all_(X|y|idx)_train, (X|idx)_test\n    all_X_train, all_y_train, all_idx_train = \\\n        loaddata(conf, 'X_train.npy'), \\\n        keras.utils.to_categorical(loaddata(conf, 'y_train.npy')), \\\n        loaddata(conf, 'idx_train.npy')\n    X_test, idx_test = loaddata(conf, 'X_test.npy'), loaddata(conf, 'idx_test.npy')\n    print('Loaded trainset:%d, testset:%d samples.' % (len(all_X_train), len(X_test)))\n\n    # a'. Normalize samplewise if requested\n    if conf['normalize'] == 'samplewise':\n        print(' normalize samplewise')\n        samplewise_mean_X(all_X_train)\n        samplewise_mean_X(X_test)\n\n    # b. Removing samples on the blacklist -> X|y|idx\n    whitelist = [idx for idx in range(len(all_idx_train)) if all_idx_train[idx] not in train_blacklist_index]\n    X_train, y_train, idx_train = \\\n        all_X_train[whitelist], all_y_train[whitelist], all_idx_train[whitelist]\n    print('Filtered samples on blacklist, now trainset has %d samples' % len(idx_train))\n\n    # Train folds\n    work = {'train_acc': [],\n            'train_acc_verified': [],\n            'history': []}\n    for fold in range(conf['n_fold']):\n        acc, acc_verified, history, model, _ = run_fold(conf, fold,\n                [X_train, y_train, idx_train, all_X_train, all_y_train, all_idx_train, X_test, idx_test],\n                model=None,\n                init_best_weights=EXTRA / 'X48_AlexNet_00696.h5',\n                eval_only=False)\n        work['history'].append(history)\n        work['train_acc'].append(acc)\n        work['train_acc_verified'].append(acc_verified)\n\n    print('___ training finished ___')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ca2b731a0cc50d9cc62fb60a15f0e00bb79c1dea"},"cell_type":"markdown","source":"## Ensemble"},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"b9e622212b5075f8b8a9466b0cdfc49c2feeba71"},"cell_type":"code","source":"def pred_geometric_mean(preds_set):\n    predictions = np.ones_like(preds_set[0])\n    for preds in preds_set:\n        predictions = predictions*preds\n    predictions = predictions**(1./len(preds_set))\n    return predictions\n\ndef pred_geometric_mean_by_files(npy_pred_files):\n    preds_set = np.array([np.load(file) for file in npy_pred_files])\n    predictions = pred_geometric_mean(preds_set)\n    return predictions","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a682eb758d0414d95d37e7ac88f59daef4278609"},"cell_type":"code","source":"for conf in [confX]: # Running confX only, change this to confs if you need running both confX and confLH\n    print('== Attempt [%s] ==' % conf['folder'])\n    train_pred_files = list(conf['folder'].glob('train_pred*.npy'))\n    print('Train set ensemble = ', train_pred_files)\n    ensembled_train_preds = pred_geometric_mean_by_files(train_pred_files)\n    y_train = keras.utils.to_categorical(loaddata(conf, 'y_train.npy'))\n    acc, acc_v = evaluate_pred_acc(y_train, ensembled_train_preds, all_idx_train, len(plain_y_train))\n    print('Ensemble train set accuracy =', acc)\n    print('Ensemble verified samples accuracy =', acc_v)\n    np.save(datapath(conf, 'ensemble_train_preds.npy'), ensembled_train_preds)\n\n    test_pred_files = list(conf['folder'].glob('test_pred*.npy'))\n    print('Test set ensemble = ', test_pred_files)\n    ensembled_test_preds = pred_geometric_mean_by_files(test_pred_files)\n    np.save(datapath(conf, 'ensemble_test_preds.npy'), ensembled_test_preds)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"cf658436b3905a919a7f0190e91b868906632279"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}