{"cells":[{"metadata":{},"cell_type":"markdown","source":"This is the fork of the inference kernel that got 80-th place on public LB with 70.2 LWLRAP and will likely be disqulified due to OOM from stage 2 evaluation - too large batch size was chosen.\nBrief summary:\n* PCEN used as features\n* Pure CNN with global average pooling to handle different input size at training and inference stage\n* Tiling used instead of padding for short files. At training stage maxshape = 500 was used, at inference - 4893\n* Using SpatialDropout in CNN helped a lot. Using PRelu helped a bit.\n* Model was first trained on curated data. Then curated + noisy with sample_weight 0.8 for noisy data.\n* 5 fold cv was used using skmultilearn.model_selection.IterativeStratification"},{"metadata":{},"cell_type":"markdown","source":"based on : \n* https://www.kaggle.com/CVxTz/keras-cnn-starter\n* https://www.kaggle.com/jmourad100/keras-eda-and-cnn-starter\n\n# || Loading Packages"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport os, time, random, cv2, glob, pickle, librosa\nfrom pathlib import Path\nfrom PIL import Image\nimport imgaug as ia\nfrom imgaug import augmenters as iaa\nfrom tqdm import tqdm\n\nfrom keras.models import Model, Sequential\nfrom keras.layers import (Convolution1D, Input, Dense, Flatten, Dropout, GlobalAveragePooling1D, concatenate,\n                          Activation, MaxPool1D, GlobalMaxPool1D, BatchNormalization, Concatenate, ReLU, LeakyReLU, GRU, Masking, \n                          Conv2D,MaxPooling2D, GlobalAveragePooling2D,GlobalMaxPooling2D, Conv1D,MaxPooling1D, Reshape, SpatialDropout2D, PReLU)\nfrom keras.callbacks import ModelCheckpoint, EarlyStopping, LearningRateScheduler, Callback\nfrom keras.optimizers import Adam, SGD, RMSprop\nfrom keras.losses import sparse_categorical_crossentropy\nfrom keras.utils.np_utils import to_categorical\nfrom sklearn.model_selection import train_test_split\nfrom keras.constraints import max_norm, MinMaxNorm\nprint(os.listdir(\"../input\"))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","collapsed":true,"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":false},"cell_type":"markdown","source":"# || Configuration"},{"metadata":{"trusted":true},"cell_type":"code","source":"t_start = time.time()\n\n# Keras reproduce score (then init all model seed)\nseed_nb=14\nimport numpy as np \nnp.random.seed(seed_nb)\nimport tensorflow as tf\ntf.set_random_seed(seed_nb)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# || Data Preparation"},{"metadata":{"trusted":true},"cell_type":"code","source":"maxshape = 4893 #inference mod, for training 500 was used\nimport random\n\nfrom scipy import ndimage\ndef pad2d(inp, use_mfcc=True, use_conv2d=True, augment=False, maxshape=maxshape, channels=1):\n    inp = inp.astype(np.float64)\n\n    if augment:\n        scale = random.uniform(0.7, 1.3)\n        inp = ndimage.zoom(inp, (1, scale))\n    if inp.shape[1] >= maxshape:\n        if augment:\n            offset = int(random.uniform(0, 1) * (inp.shape[1] - maxshape))\n            out =  inp[:, offset:offset + maxshape] \n        else:\n            out =  inp[:, :maxshape] \n    else:\n        offset = int(random.uniform(0, 1) * inp.shape[1] * 0.1)\n        out =  inp[:, offset:] \n        ntiles = int(maxshape / inp.shape[1]) + 1\n        out = np.tile(inp, (1, ntiles))[:, :maxshape]\n    if use_conv2d:\n        res = out.T.reshape((out.T.shape[0],out.T.shape[1], 1))\n        if channels > 1:\n            res = np.repeat(res, 3, axis=-1)\n        return res\n    else:\n        return out.T","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import sklearn.metrics\n# Accumulator object version.\n# Core calculation of label precisions for one test sample.\n\ndef _one_sample_positive_class_precisions(scores, truth):\n  \"\"\"Calculate precisions for each true class for a single sample.\n  \n  Args:\n    scores: np.array of (num_classes,) giving the individual classifier scores.\n    truth: np.array of (num_classes,) bools indicating which classes are true.\n\n  Returns:\n    pos_class_indices: np.array of indices of the true classes for this sample.\n    pos_class_precisions: np.array of precisions corresponding to each of those\n      classes.\n  \"\"\"\n  num_classes = scores.shape[0]\n  pos_class_indices = np.flatnonzero(truth > 0)\n  # Only calculate precisions if there are some true classes.\n  if not len(pos_class_indices):\n    return pos_class_indices, np.zeros(0)\n  # Retrieval list of classes for this sample. \n  retrieved_classes = np.argsort(scores)[::-1]\n  # class_rankings[top_scoring_class_index] == 0 etc.\n  class_rankings = np.zeros(num_classes, dtype=np.int)\n  class_rankings[retrieved_classes] = range(num_classes)\n  # Which of these is a true label?\n  retrieved_class_true = np.zeros(num_classes, dtype=np.bool)\n  retrieved_class_true[class_rankings[pos_class_indices]] = True\n  # Num hits for every truncated retrieval list.\n  retrieved_cumulative_hits = np.cumsum(retrieved_class_true)\n  # Precision of retrieval list truncated at each hit, in order of pos_labels.\n  precision_at_hits = (\n      retrieved_cumulative_hits[class_rankings[pos_class_indices]] / \n      (1 + class_rankings[pos_class_indices].astype(np.float)))\n  return pos_class_indices, precision_at_hits\n\nclass lwlrap_accumulator(object):\n  \"\"\"Accumulate batches of test samples into per-class and overall lwlrap.\"\"\"  \n\n  def __init__(self):\n    self.num_classes = 0\n    self.total_num_samples = 0\n  \n  def accumulate_samples(self, batch_truth, batch_scores):\n    \"\"\"Cumulate a new batch of samples into the metric.\n    \n    Args:\n      truth: np.array of (num_samples, num_classes) giving boolean\n        ground-truth of presence of that class in that sample for this batch.\n      scores: np.array of (num_samples, num_classes) giving the \n        classifier-under-test's real-valued score for each class for each\n        sample.\n    \"\"\"\n    assert batch_scores.shape == batch_truth.shape\n    num_samples, num_classes = batch_truth.shape\n    if not self.num_classes:\n      self.num_classes = num_classes\n      self._per_class_cumulative_precision = np.zeros(self.num_classes)\n      self._per_class_cumulative_count = np.zeros(self.num_classes, \n                                                  dtype=np.int)\n    assert num_classes == self.num_classes\n    for truth, scores in zip(batch_truth, batch_scores):\n      pos_class_indices, precision_at_hits = (\n        _one_sample_positive_class_precisions(scores, truth))\n      self._per_class_cumulative_precision[pos_class_indices] += (\n        precision_at_hits)\n      self._per_class_cumulative_count[pos_class_indices] += 1\n    self.total_num_samples += num_samples\n\n  def per_class_lwlrap(self):\n    \"\"\"Return a vector of the per-class lwlraps for the accumulated samples.\"\"\"\n    return (self._per_class_cumulative_precision / \n            np.maximum(1, self._per_class_cumulative_count))\n\n  def per_class_weight(self):\n    \"\"\"Return a normalized weight vector for the contributions of each class.\"\"\"\n    return (self._per_class_cumulative_count / \n            float(np.sum(self._per_class_cumulative_count)))\n\n  def overall_lwlrap(self):\n    \"\"\"Return the scalar overall lwlrap for cumulated samples.\"\"\"\n    return np.sum(self.per_class_lwlrap() * self.per_class_weight())\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class LwrapEvaluation(Callback):\n    def __init__(self, valgen):\n        super(Callback, self).__init__()\n        self.valgen = valgen\n\n    def on_epoch_end(self, epoch, logs={}):\n        lwrap = lwlrap_accumulator()\n        for i in range(len(self.valgen)):\n            inp, truth = self.valgen[i][:2]\n            pred = self.model.predict(inp)\n            lwrap.accumulate_samples(truth, pred)\n        print(\"validation lwrap: \", lwrap.overall_lwlrap())\n        \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_model(lr=0.001, trainable = True, print_model=True):\n    nclass = 80\n    model = Sequential()\n    model.add(BatchNormalization(input_shape=(None, 128, 1), trainable = trainable, axis=2))\n    model.add(Conv2D(filters=192, kernel_size=3, input_shape=(maxshape, 128, 1), padding=\"same\", trainable = trainable,  kernel_initializer='he_uniform', \n                     kernel_constraint=max_norm(2.)))\n    model.add(BatchNormalization(trainable = trainable))\n    model.add(PReLU(shared_axes=[1,2]))\n    model.add(SpatialDropout2D(0.25))\n    model.add(Conv2D(filters=192, kernel_size=3, padding=\"same\", trainable = trainable,  kernel_initializer='he_uniform', \n              kernel_constraint=max_norm(2.)))\n    model.add(BatchNormalization(trainable = trainable))\n    model.add(PReLU(shared_axes=[1,2]))\n    model.add(SpatialDropout2D(0.25))\n    model.add(MaxPooling2D(pool_size=2))\n    \n\n    model.add(Conv2D(filters=128, kernel_size=3, padding=\"same\", trainable = trainable,  kernel_initializer='he_uniform', kernel_constraint=max_norm(2.)))\n    model.add(BatchNormalization(trainable = trainable))\n    model.add(PReLU(shared_axes=[1,2]))\n    model.add(SpatialDropout2D(0.25))\n    model.add(Conv2D(filters=128, kernel_size=3, padding=\"same\", trainable = trainable,  kernel_initializer='he_uniform', kernel_constraint=max_norm(2.)))\n    model.add(BatchNormalization(trainable = trainable))\n    model.add(PReLU(shared_axes=[1,2]))\n    model.add(SpatialDropout2D(0.25))\n    model.add(MaxPooling2D(pool_size=2))\n    \n\n    model.add(Conv2D(filters=256, kernel_size=3, padding=\"same\", trainable = trainable,  kernel_initializer='he_uniform', kernel_constraint=max_norm(2.)))\n    model.add(BatchNormalization(trainable = trainable))\n    model.add(PReLU(shared_axes=[1,2]))\n    model.add(SpatialDropout2D(0.25))\n    model.add(MaxPooling2D(pool_size=(2,2)))\n    \n\n    model.add(Conv2D(filters=512, kernel_size=3, padding=\"same\", trainable = trainable,  kernel_initializer='he_uniform', kernel_constraint=max_norm(2.)))\n    model.add(BatchNormalization(trainable = trainable))\n    model.add(PReLU(shared_axes=[1,2]))\n    model.add(MaxPooling2D(pool_size=(2,2)))\n    model.add(SpatialDropout2D(0.25))\n    \n    model.add(Conv2D(filters=1024, kernel_size=3, padding=\"same\", trainable = trainable,  kernel_initializer='he_uniform', kernel_constraint=max_norm(2.)))\n    model.add(BatchNormalization())\n    model.add(PReLU(shared_axes=[1,2]))\n    model.add(MaxPooling2D(pool_size=2))\n    model.add(SpatialDropout2D(0.25))\n    \n    model.add(Conv2D(filters=512, kernel_size=3, padding=\"same\", trainable = trainable,  kernel_initializer='he_uniform', kernel_constraint=max_norm(2.)))\n    model.add(BatchNormalization())\n    model.add(PReLU(shared_axes=[1,2]))\n    model.add(MaxPooling2D(pool_size=(2,1)))\n    model.add(SpatialDropout2D(0.25))\n    \n    model.add(Conv2D(filters=256, kernel_size=3, padding=\"same\", trainable = trainable,  kernel_initializer='he_uniform', kernel_constraint=max_norm(2.)))\n    model.add(BatchNormalization())\n    model.add(PReLU(shared_axes=[1,2]))\n    model.add(MaxPooling2D(pool_size=(2,1)))\n    model.add(SpatialDropout2D(0.25))\n    \n    model.add(Conv2D(filters=nclass, kernel_size=(1,1), padding=\"valid\", trainable = trainable,  kernel_initializer='he_uniform', activation=\"sigmoid\"))\n    model.add(GlobalAveragePooling2D())\n    \n    model.compile(optimizer=Adam(lr), loss='binary_crossentropy', metrics=['categorical_accuracy'])\n    if print_model:\n        model.summary()\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"softened_weights = {0:\"../input/softened-noisy-kfold-fold-0/validation_lwrap_curated_noisy_0_best_weight.h5\",\n                   1:\"../input/softened-noisy-kfold-fold-1/validation_lwrap_curated_noisy_1_best_weight.h5\",\n                   2:\"../input/softened-noisy-kfold-fold-2/validation_lwrap_curated_noisy_2_best_weight.h5\",\n                   3:\"../input/softened-noisy-kfold-fold-3/validation_lwrap_curated_noisy_3_best_weight.h5\",\n                   4:\"../input/softened-noisy-kfold-fold-4/validation_lwrap_curated_noisy_4_best_weight.h5\"}\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"models = []\nfor i in range(5):\n    model = get_model()\n    model.load_weights(softened_weights[i])\n    models.append(model)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = pd.read_csv('../input/freesound-audio-tagging-2019/sample_submission.csv')\ndf.head(), df.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_file_names = glob.glob(\"../input/freesound-audio-tagging-2019/test/*.wav\")\nlen(test_file_names), sorted(test_file_names)[:10]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"testfiles = [\"../input/freesound-audio-tagging-2019/test/\" + f for f in df.fname.values]\nlen(testfiles), testfiles[:10]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import librosa\ndef load_audio_file(file_path):\n    data, sr = librosa.core.load(file_path, sr=44100)\n    data_t, index = librosa.effects.trim(data)\n    S = librosa.feature.melspectrogram(data_t, sr=sr, power=1)\n    pcen = librosa.core.pcen(S, sr=sr,  gain=0.7, bias=0.1, power=0.4, time_constant=0.4, eps=1e-9)\n    return pcen.astype(np.float16)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tqdm import tqdm_notebook\nfeatures_test = [load_audio_file(f) for f in tqdm_notebook(testfiles)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import gc\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"nbatches = 400\ndef test_generator():\n    for s in np.array_split(np.arange(len(features_test)), nbatches):\n        yield np.array([pad2d(features_test[f]) for f in s])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds_test = None\nfor m in tqdm_notebook(models):\n    preds = m.predict_generator(test_generator(), nbatches, verbose=1)\n    if preds_test is None:\n        preds_test = preds\n    else:\n        preds_test += preds\npreds_test = preds_test / len(models)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for i in range(preds_test.shape[1]):\n    df.iloc[:, i + 1] = preds_test[:, i]\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}