{"cells":[{"metadata":{},"cell_type":"markdown","source":"### References:\n* https://www.kaggle.com/maxwell110/beginner-s-guide-to-audio-data-2\n* https://www.kaggle.com/daisukelab/cnn-2d-basic-solution-powered-by-fast-ai\n* https://www.kaggle.com/christofhenkel/keras-baseline-lstm-attention-5-fold\n* https://yerevann.github.io/2016/06/26/combining-cnn-and-rnn-for-spoken-language-identification/\n\n### In this kernel, only train curated will be used.\n\nI'm taking 5 seconds of spectrograms for each video -> likely an overkill, to be fine-tuned later.\n\nTo use the noisy set for training, a data generator is required, as the complete spectograms won't fit into the memory.\n"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nimport librosa\nimport matplotlib.pyplot as plt\nimport gc\n\nfrom tqdm import tqdm, tqdm_notebook\nfrom sklearn.metrics import label_ranking_average_precision_score\nfrom sklearn.model_selection import train_test_split\n\ntqdm.pandas()","execution_count":27,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def calculate_overall_lwlrap_sklearn(truth, scores):\n    \"\"\"Calculate the overall lwlrap using sklearn.metrics.lrap.\"\"\"\n    # sklearn doesn't correctly apply weighting to samples with no labels, so just skip them.\n    sample_weight = np.sum(truth > 0, axis=1)\n    nonzero_weight_sample_indices = np.flatnonzero(sample_weight > 0)\n    overall_lwlrap = label_ranking_average_precision_score(\n        truth[nonzero_weight_sample_indices, :] > 0, \n        scores[nonzero_weight_sample_indices, :], \n        sample_weight=sample_weight[nonzero_weight_sample_indices])\n    return overall_lwlrap","execution_count":2,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def split_and_label(rows_labels):\n    \n    row_labels_list = []\n    for row in rows_labels:\n        row_labels = row.split(',')\n        labels_array = np.zeros((80))\n        \n        for label in row_labels:\n            index = label_mapping[label]\n            labels_array[index] = 1\n        \n        row_labels_list.append(labels_array)\n    \n    return row_labels_list","execution_count":3,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train_curated = pd.read_csv('../input/train_curated.csv')\ntrain_noisy = pd.read_csv('../input/train_noisy.csv')\ntest = pd.read_csv('../input/sample_submission.csv')","execution_count":4,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train_curated.shape, train_noisy.shape, test.shape)","execution_count":5,"outputs":[{"output_type":"stream","text":"(4970, 2) (19815, 2) (1120, 81)\n","name":"stdout"}]},{"metadata":{"trusted":true},"cell_type":"code","source":"label_columns = test.columns[1:]","execution_count":6,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"label_mapping = dict((label, index) for index, label in enumerate(label_columns))","execution_count":7,"outputs":[]},{"metadata":{"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"label_mapping","execution_count":8,"outputs":[{"output_type":"execute_result","execution_count":8,"data":{"text/plain":"{'Accelerating_and_revving_and_vroom': 0,\n 'Accordion': 1,\n 'Acoustic_guitar': 2,\n 'Applause': 3,\n 'Bark': 4,\n 'Bass_drum': 5,\n 'Bass_guitar': 6,\n 'Bathtub_(filling_or_washing)': 7,\n 'Bicycle_bell': 8,\n 'Burping_and_eructation': 9,\n 'Bus': 10,\n 'Buzz': 11,\n 'Car_passing_by': 12,\n 'Cheering': 13,\n 'Chewing_and_mastication': 14,\n 'Child_speech_and_kid_speaking': 15,\n 'Chink_and_clink': 16,\n 'Chirp_and_tweet': 17,\n 'Church_bell': 18,\n 'Clapping': 19,\n 'Computer_keyboard': 20,\n 'Crackle': 21,\n 'Cricket': 22,\n 'Crowd': 23,\n 'Cupboard_open_or_close': 24,\n 'Cutlery_and_silverware': 25,\n 'Dishes_and_pots_and_pans': 26,\n 'Drawer_open_or_close': 27,\n 'Drip': 28,\n 'Electric_guitar': 29,\n 'Fart': 30,\n 'Female_singing': 31,\n 'Female_speech_and_woman_speaking': 32,\n 'Fill_(with_liquid)': 33,\n 'Finger_snapping': 34,\n 'Frying_(food)': 35,\n 'Gasp': 36,\n 'Glockenspiel': 37,\n 'Gong': 38,\n 'Gurgling': 39,\n 'Harmonica': 40,\n 'Hi-hat': 41,\n 'Hiss': 42,\n 'Keys_jangling': 43,\n 'Knock': 44,\n 'Male_singing': 45,\n 'Male_speech_and_man_speaking': 46,\n 'Marimba_and_xylophone': 47,\n 'Mechanical_fan': 48,\n 'Meow': 49,\n 'Microwave_oven': 50,\n 'Motorcycle': 51,\n 'Printer': 52,\n 'Purr': 53,\n 'Race_car_and_auto_racing': 54,\n 'Raindrop': 55,\n 'Run': 56,\n 'Scissors': 57,\n 'Screaming': 58,\n 'Shatter': 59,\n 'Sigh': 60,\n 'Sink_(filling_or_washing)': 61,\n 'Skateboard': 62,\n 'Slam': 63,\n 'Sneeze': 64,\n 'Squeak': 65,\n 'Stream': 66,\n 'Strum': 67,\n 'Tap': 68,\n 'Tick-tock': 69,\n 'Toilet_flush': 70,\n 'Traffic_noise_and_roadway_noise': 71,\n 'Trickle_and_dribble': 72,\n 'Walk_and_footsteps': 73,\n 'Water_tap_and_faucet': 74,\n 'Waves_and_surf': 75,\n 'Whispering': 76,\n 'Writing': 77,\n 'Yell': 78,\n 'Zipper_(clothing)': 79}"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"for col in tqdm(label_columns):\n    train_curated[col] = 0\n    train_noisy[col] = 0\n    \nprint(train_curated.shape, train_noisy.shape)","execution_count":9,"outputs":[{"output_type":"stream","text":"100%|██████████| 80/80 [00:00<00:00, 1054.05it/s]","name":"stderr"},{"output_type":"stream","text":"(4970, 82) (19815, 82)\n","name":"stdout"},{"output_type":"stream","text":"\n","name":"stderr"}]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_curated_labels = split_and_label(train_curated['labels'])\ntrain_noisy_labels = split_and_label(train_noisy['labels'])","execution_count":11,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_curated[label_columns] = train_curated_labels\ntrain_noisy[label_columns] = train_noisy_labels","execution_count":12,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_curated['num_labels'] = train_curated[label_columns].sum(axis=1)\ntrain_noisy['num_labels'] = train_noisy[label_columns].sum(axis=1)","execution_count":13,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"plt.figure(figsize=(18,6))\n\nplt.subplot(121)\nax1 = train_curated['num_labels'].value_counts().plot(kind='bar')\nplt.xlabel('Number of labels')\nplt.ylabel('Counts')\nplt.xticks(rotation=0)\nplt.title('Curated Training Set')\n\nfor p in ax1.patches:\n    ax1.annotate(str(p.get_height()), \n                (p.get_x() + p.get_width()/2., p.get_height() * 1.005), \n                ha='center',\n                va='center',\n                xytext=(0,5), \n                textcoords='offset points')\n\nplt.subplot(122)\nax2 = train_noisy['num_labels'].value_counts().sort_index().plot(kind='bar', )\nplt.xlabel('Number of labels')\nplt.ylabel('Counts')\nplt.xticks(rotation=0)\nplt.title('Noisy Training Set')\n\nfor p in ax2.patches:\n    ax2.annotate(str(p.get_height()), \n                (p.get_x() + p.get_width()/2., p.get_height() * 1.005), \n                ha='center',\n                va='center',\n                xytext=(0,5), \n                textcoords='offset points')\n\n    \nplt.show()","execution_count":14,"outputs":[{"output_type":"display_data","data":{"text/plain":"<Figure size 1296x432 with 2 Axes>","image/png":"iVBORw0KGgoAAAANSUhEUgAABCwAAAGDCAYAAADgYIEMAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzs3Xu4VmWZ+PHvzUFSK0FAQ8HAU3LQQSXFnNR0VHScME+DWmJijmUzlpWHfpPawTKbPFY6puSpEalsZIw0x0Nq4wkVzwWEqJgpCmhqqeD9+2M9G1+3e8MG97v3y97fz3W9117rXs9a61kvf7wP93oOkZlIkiRJkiQ1kh6dXQFJkiRJkqTmTFhIkiRJkqSGY8JCkiRJkiQ1HBMWkiRJkiSp4ZiwkCRJkiRJDceEhSRJkiRJajgmLCQtV0RkRGzaQff6Q0R8tL3LSpKkjhMRv46IiZ1dj9ZExMYR8XJ7l5XU/kxYSB0sIg6JiBkR8XJEPFN+1P++Tvc6PCJur9O1HynP8HJELI2Iv9Xsf3VVrpmZH8rM29q77MqIiD4RcXZEPB0Rf4mIxyPi+20891sRcUl710mSpI4UEfMi4rmIWLsmdmRE3NKW8zNzr8y8tJ3qslFN++Ll8iLllZr9lX55kZlzM/O97V12ZZVn+2VEPB8RL0bEQxHxqTaee3tEHF6PekmNxISF1IEi4jjgbODbwPrARsCPgPGrcK1e7Vu7lZOZIzPzveVH/Dbg8037mfnt5uU7u74r4d+BrYBtgfcDuwL3d2qNJEnqeD2BYzu7Epn5ZE37oilx8Hc1sXe8vIiInh1czVX1U2AuVXuwPzAReK5TayQ1GBMWUgeJiHWAbwDHZObVmflKZr6Rmf+TmV8pZS6JiG/VnLNLRMyv2Z8XESdExIPAKxHRKyJOjIg/lt4Aj0bEJ0rZ4cAFwA7lDcTiEu8TEf8REU9GxLMRcUFErFlzj6+Unh9/iogj3sXzHhkRt0bEuRGxEPj3iNgsIm6OiIXlbcLl5XtpOmd+ROxStr8VEVdGxBXl2R6OiG1WseyYiJhZjk2JiJ9FxKmtVP3DwNWZ+eesPJ6ZV9Rca3B5G7Kg9L44psT3AY4HDi3f972r+t1JktQAvgd8OSL6tnQwIj4SEfeUngH3RMRHao7dEhFHlu1NI+K3pdzzEXFVif+weQ/GiJgWEV9c2YqW3/8fRsR1EfEK8NGI+Hj57X+ptHm+VlN+04jImv3bI+LrEfF/pa1wXUSsu7Jly/FPl/s9HxFfrW2vtODDwE8y89XMXJKZ92Xm9TXX2jEi7oyIxeVZdirx7wI7ABeUNsfZK/udSasLExZSx9kBeA/wy3d5nYOBfwT6ZuYS4I/AR4F1gK8DV0TEoMx8DDgauKO8gWhqcJwObA6MBjYFNgROBoiIccCXgd2BzYB/eJd1/QjwGDAQ+C4QwLeADwAjgI2Br7V6NuwLXA70BX4NnLuyZSOiD/DfwEXAusAvStnW3Al8JSI+GxGjIiKaDkRED+Ba4B6q7233Una3zLwWOAP4afm+t13OPSRJanQzgFuo2gVvU/6D/iuq39r+wJnAryKifwvX+SbwG6AfMBg4r8QvBQ4uv61ExACqdsd/rWJ9D6FqB70PuAN4GTiUql3wT8Cx5eXC8s6fSNUDdm3guJUtGxFbUn0nE6jaCQOp2jytuRM4PyL+OSKG1B4o+9OAU6jaLycCV0dE/8w8oTzj0aXN8YXl3ENarZmwkDpOf+D5kmR4N87NzKcy868AmfmzzPxTZr6ZmVcBs4HtWjqx/Of7KOCLmbkwM/9CNTxlQilyEFWm/+HMfAU49V3W9cnMPD8zl2bmXzNzVmbemJmvZ+ZzwFnAzss5/7eZeX1mLqVKRoxehbI7Am9m5g9Kj5afAcvr/fAt4D+AT5Vy8yPik+XYDsD7M/Pb5RnmABfz1vcnSVJXcjLwrxExsFn8H4HZmXl56RlwJfB7qsRAc28AHwQ2yMy/ZebtAJl5N/AisFspNwG4JTOfXcW6/jIz7yjtodcy86bMfKTsPwBMYfltjoszc3Zmvgr8jOW3OVoreyDw35n5f5n5GtUw0+XZjyrxcArwRETcFxFNLzwOA6aVts2bmXkd8AAwbgXXlLoUExZSx3kBGBDvfi6Hp2p3IuKw0k1wcVTDPkYBA1o5dyCwFnBvTfnrShxgg2bXf6Kd6/qBiJga1YSWLwGXLKeuAH+u2X6V6i3GypbdAJjfrOxTtKI0vM7LzI9QvZU5A7gkIjananBt1PTdle/veJb/9kSSpNVSZj5M1bPwxGaHNuCdbYQnqHoVNHc8VQ/Lu6OasLt2uOmlQNNLgU9SvXBYVc3bHDuUoSkLIuJF4EhWrs2xvIk2Wyv7tnZUefmzqLWLlJdHx2fmCKreGo/wVk/cD1L1QKltc4wt95C6DRMWUse5A3iN5Q9HeIUqodCkpf8I146j/CDwY+DzQP8y7ONhqobB28oWzwN/BUZmZt/yWadmEqtngNouiRst/5FWqPn9v0v1HWyZme8HDq+pa708wzsbUENaKthc6RVyDlW30uFUjZDZNd9d38x8X2Y2vVFq/rySJK3uTgE+w9t/S/9E9R/qWhsBTzc/ucwJ9ZnM3AD4F+BH8dZy6VcA4yPi76h+Z//7XdSz+W/wFKphoEMycx2qoaEd0eYY3LQT1Sor/dpyYmYuAL4PDIlqfq+nqHq91rY51s7M7zWd0s51lxqSCQupg2Tmi1RdK38YEftGxFoR0Tsi9oqIM0qxmcDeEbFuRHwAWNGYxLWpfrAWQDXRE1UPiybPAoMjYo1ShzepEhxnRcR65ZwNI2LPUn4qcHhEjIiItagaKe3pfVRJmRfL2Mx3jIutg9uBXmVOil4RsT/VCiAtiogvRsROEbFmKX8E1dwjM6mSTq9HxJci4j0R0TMitqzpvvksMLR23gtJklZnZfjjVcC/1YSnA5tHtVR7r4j4Z6q5qa5tfn5EHBgRTf+JX0TVbnmzXHs+1bxQlwO/aBru2k7eByzMzL9FxFg6Zvjmz4B9I2JsaXt9Y3mFI+KMiBhZ2hPvBz4L/L60GS8HPhERu5fj74mIj0VEUw+LZ6nmApO6NBMWUgfKzO9TTcz071RJhqeoekc0vVG4nGp84jyqCaquWsH1HqXKxt9B9cO1JfC7miI3UXUv/HNEPF9iJwBzgDvLsIz/BT5UrvdrqmVXbyplblrlh23ZKVTza7xINZHUL9r5+u9QxpB+gmoC0kVU83RMp+rp0ZK/UX0Hz1L1SPkXYL/MfKLMP7I31TPMK8f/k2r5U6j+vdYAFkbE3fV4HkmSOsE3qBmWmZkvAPsAX6Ia8no8sE9mPt/CuR8G7oqIl6l++4/NzLk1xy+lar+8m+EgLfks8J2I+AvwVaqXMnWVmQ8CX6RKXPyJ6rt5gdbbHO8FrqFqF/2RarjHvuVa86jaL1+jajM+SfV9N/3/7WzeGjJyZh0eR2oIkWlvIkndS1RLjp6dme3dOJIkSSuhLNV5BfDB7GL/MSm9JhZTPVur82dJap09LCR1eRGxS0SsX7qtTgK2AK5f0XmSJKl+IqI3cCxwUVdJVkTEx8uw3/dS9YK9z2SFtOpMWEjqDoYDD1K95fg3YP+yrKokSeoEETGc6nd5ENXwhq7iE1TDQeYDQ4GDO7U20mrOISGSJEmSJKnh2MNCkiRJkiQ1HBMWkiRJkiSp4fTq7ArUw4ABA3Lo0KGdXQ1JkhrOvffe+3xmDuzsenQHtkckSWpZW9sjXTJhMXToUGbMmNHZ1ZAkqeFExBOdXYfuwvaIJEkta2t7xCEhkiRJkiSp4ZiwkCRJkiRJDceERQdYunQpW2+9Nfvssw8Ahx56KB/60IcYNWoURxxxBG+88caysrfccgujR49m5MiR7Lzzzsvi55xzDqNGjWLkyJGcfXZXWqpakiQtzxFHHMF6663HqFGj3hY/77zz2GKLLRg5ciTHH388APPmzWPNNddk9OjRjB49mqOPPnpZ+ddff52jjjqKzTffnC222IJf/OIXy45NnTqVESNGMHLkSA455JCOeTBJklagS85h0WjOOecchg8fzksvvQRUCYsrrrgCgEMOOYSLLrqIz372syxevJjPfe5zXHfddWy00UY899xzADz88MP8+Mc/5u6772aNNdZg3Lhx7LPPPmy66aad9kySJKljHH744Xz+85/nsMMOWxa7+eabueaaa3jggQfo06fPsjYDwCabbMLMmTPfcZ3TTjuN9dZbj1mzZvHmm2+ycOFCAGbPns13vvMdfve739GvX7+3XUuSpM5kD4s6mz9/Pr/61a848sgjl8X23ntvIoKIYLvttmP+/PkA/Nd//Rf77bcfG220EQDrrbceAI899hjbb789a621Fr169WLnnXfm6quv7viHkSRJHW6nnXZi3XXXfVvs/PPP58QTT6RPnz7AW22G5Zk8eTInnXQSAD169GDAgAEA/PjHP+aYY46hX79+bb6WJEkdwYRFnX3hC1/gjDPOoEePd37Vb7zxBpdffjnjxo0DYNasWSxatIhddtmFbbfdlssuuwyAUaNGcdttt/HCCy/w6quvMn36dJ566qkOfQ5JktQ4Zs2axW233cb222/PzjvvzD333LPs2OOPP87WW2/NzjvvzG233QbA4sWLAfja177GNttsw4EHHsizzz677FqzZs1ixx13ZOzYsVx33XUd/0CSJLXAISF1dO2117Leeuux7bbbcsstt7zj+Oc+9zl22mknPvrRjwKwZMkS7r33Xm688Ub++te/ssMOOzB27FiGDx/OCSecwB577MHaa6/N6NGj6dmzZwc/jSRJahRLlixh4cKF3Hnnndxzzz0cdNBBzJ07l0GDBvHkk0/Sv39/7r33Xvbdd18eeeQRlixZwvz58/nIRz7CmWeeyZlnnsmXv/xlLr/8cpYsWcLs2bO55ZZbmD9/PjvttBMPPfQQffv27ezHlCR1c/awqKPf/e53TJs2jaFDhzJhwgRuuukmPvnJTwLw9a9/nQULFnDmmWcuKz948GD23HNP1l57bQYMGMBOO+3EAw88AMCkSZO49957ufXWW+nXrx+bb755pzyTJEnqfIMHD2a//fZbNry0R48ePP/88/Tp04f+/fsDsO2227LJJpswa9Ys+vfvz1prrcV+++0HwIEHHsh999237Fof//jH6d27N8OGDWPzzTdn9uzZnfZskiQ1MWFRR9/5zneYP38+8+bNY8qUKey6665cccUVXHTRRVx//fVceeWVbxsqMn78eG6//XaWLFnCq6++yl133cXw4cMBlk2A9eSTT3L11Vc7g7ckSd3Yvvvuy8033wxUQzpef/11BgwYwIIFC1i6dCkAc+fOZfbs2Wy88cZEBP/0T/+0rMfnjTfeyIgRI5Zdqyn+/PPPM2vWLDbeeOMOfyZJkppzSEgnOProo/ngBz/IDjvsAMB+++3HySefzPDhwxk3bhxbbbUVPXr04Mgjj1y2hNn+++/PCy+8QO/evfnhD39oN01JkrqJgw8+mFtuuYXnn3+ewYMH8/Wvf50jjjiCI444glGjRrHGGmtw6aWXEhHceuutnHzyyfTu3ZsePXpwwQUXLJuw87vf/S6f+tSn+MIXvsDAgQP5yU9+AsCee+7Jb37zG0aMGEHPnj353ve+t6yXhiRJnSkys7Pr0O7GjBmTM2bM6OxqSJLUcCLi3swc09n16A5sj0iS1LK2tkccEiJJkiRJkhqOCQtJkiRJktRwnMNiJQ098VedXYVVMu/0f+zsKkiSpJXU2e0O2w+SpM5kDwtJkiRJktRwTFhIkiRJkqSGY8JCkiRJkiQ1HBMWkiRJkiSp4ZiwkCRJkiRJDceEhSRJkiRJajgmLCRJ0movIiZHxHMR8XCz+L9GxO8j4pGIOKMmflJEzImIP0TEnjXxcSU2JyJOrIkPi4i7SvyqiFijY55MkqTuy4SFJEnqCi4BxtUGIuJjwHjg7zJzJPAfJT4CmACMLOf8KCJ6RkRP4IfAXsAI4OBSFuC7wFmZuSmwCJhU9yeSJKmbM2EhSZJWe5l5K7CwWfizwOmZ+Vop81yJjwemZOZrmfk4MAfYrnzmZObczHwdmAKMj4gAdgV+Xs6/FNi3rg8kSZJMWEiSpC5rc+CjZSjHbyPiwyW+IfBUTbn5JdZavD+wODOXNItLkqQ66tXZFZAkSaqTXsC6wFjgw8DUiNi4njeMiKOAowA22mijet5KkqQuzx4WkiSpq5oPXJ2Vu4E3gQHA08CQmnKDS6y1+AtA34jo1Sz+Dpl5YWaOycwxAwcObNeHkSSpuzFhIUmSuqr/Bj4GEBGbA2sAzwPTgAkR0ScihgGbAXcD9wCblRVB1qCamHNaZiZwM3BAue5E4JoOfRJJkrohh4RIkqTVXkRcCewCDIiI+cApwGRgclnq9HVgYkk+PBIRU4FHgSXAMZm5tFzn88D1QE9gcmY+Um5xAjAlIr4F3A9c3GEPJ0lSN2XCQpIkrfYy8+BWDn2ylfKnAae1EJ8OTG8hPpdqFRFJktRB6j4kpKxrfn9EXFv2h5XZuudExFWlyyWlW+ZVJX5XRAytucZJJf6HiNiz3nWWJEmSJEmdqyPmsDgWeKxm/7vAWZm5KbAImFTik4BFJX5WKUdEjKAaQzoSGAf8KCJ6dkC9JUmSJElSJ6lrwiIiBgP/CFxU9gPYFfh5KXIpsG/ZHl/2Kcd3K+XHA1My87XMfByYg10yJUmSJEnq0urdw+Js4HiqZcQA+gOLM3NJ2Z8PbFi2NwSeAijHXyzll8VbOEeSJEmSJHVBdUtYRMQ+wHOZeW+97tHsfkdFxIyImLFgwYKOuKUkSZIkSaqTevaw2BH4eETMA6ZQDQU5B+gbEU2rkwwGni7bTwNDAMrxdYAXauMtnLNMZl6YmWMyc8zAgQPb/2kkSZIkSVKHqVvCIjNPyszBmTmUatLMmzLzUOBm4IBSbCJwTdmeVvYpx28qa6VPAyaUVUSGAZsBd9er3pIkSZIkqfP1WnGRdncCMCUivgXcD1xc4hcDl0fEHGAhVZKDzHwkIqYCjwJLgGMyc2nHV1uSJEmSJHWUDklYZOYtwC1ley4trPKRmX8DDmzl/NOA0+pXQ0mSJEmS1EjqvUqIJEmSJEnSSjNhIUmSJEmSGo4JC0mSJEmS1HBMWEiSJEmSpIZjwkKSJEmSJDUcExaSJEmSJKnhmLCQJEmSJEkNx4SFJEmSJElqOCYsJEmSJElSwzFhIUmSJEmSGo4JC0mSJEmS1HBMWEiSJEmSpIZjwkKSJEmSJDUcExaSJEmSJKnhmLCQJEmSJEkNx4SFJEmSJElqOCYsJEmSJElSwzFhIUmSJEmSGo4JC0mSJEmS1HBMWEiSJEmSpIZjwkKSJK32ImJyRDwXEQ+3cOxLEZERMaDsR0ScGxFzIuLBiNimpuzEiJhdPhNr4ttGxEPlnHMjIjrmySRJ6r5MWEiSpK7gEmBc82BEDAH2AJ6sCe8FbFY+RwHnl7LrAqcA2wPbAadERL9yzvnAZ2rOe8e9JElS+zJhIUmSVnuZeSuwsIVDZwHHA1kTGw9clpU7gb4RMQjYE7ghMxdm5iLgBmBcOfb+zLwzMxO4DNi3ns8jSZJMWEiSpC4qIsYDT2fmA80ObQg8VbM/v8SWF5/fQrylex4VETMiYsaCBQve5RNIktS9mbCQJEldTkSsBXwVOLkj75uZF2bmmMwcM3DgwI68tSRJXY4JC0mS1BVtAgwDHoiIecBg4L6I+ADwNDCkpuzgEltefHALcUmSVEcmLCRJUpeTmQ9l5nqZOTQzh1IN49gmM/8MTAMOK6uFjAVezMxngOuBPSKiX5lscw/g+nLspYgYW1YHOQy4plMeTJKkbsSEhSRJWu1FxJXAHcCHImJ+RExaTvHpwFxgDvBj4HMAmbkQ+CZwT/l8o8QoZS4q5/wR+HU9nkOSJL2lV70uHBHvAW4F+pT7/DwzT4mIS4CdgRdL0cMzc2Z5Y3EOsDfwaonfV641Efj3Uv5bmXlpveotSZJWP5l58AqOD63ZTuCYVspNBia3EJ8BjHp3tZQkSSujbgkL4DVg18x8OSJ6A7dHRNPbiK9k5s+bla9dE317qvXOt69ZE30M1ZJk90bEtLLcmCRJkiRJ6oLqNiSkrG3+ctntXT65nFNWak30etVbkiRJkiR1vrrOYRERPSNiJvAcVdLhrnLotIh4MCLOiog+Jbaya6I3v5frnkuSJEmS1EXUNWGRmUszczTV8l/bRcQo4CRgC+DDwLrACe10L9c9lyRJkiSpi+iQVUIyczFwMzAuM58pwz5eA34CbFeKreya6JIkSZIkqYuqW8IiIgZGRN+yvSawO/D7Mi8FZVWQfYGHyykrtSZ6veotSZIkSZI6Xz1XCRkEXBoRPakSI1Mz89qIuCkiBgIBzASOLuWnUy1pOodqWdNPQ7UmekQ0rYkOb18TXZIkSZIkdUF1S1hk5oPA1i3Ed22l/EqviS5JkiRJkrqmDpnDQpIkSZIkaWWYsJAkSZIkSQ3HhIUkSZIkSWo4JiwkSZIkSVLDMWEhSZIkSZIajgkLSZIkSZLUcExYSJIkSZKkhmPCQpIkSZIkNRwTFpIkSZIkqeGYsJAkSZIkSQ3HhIUkSZIkSWo4JiwkSZIkSVLDMWEhSZIkSZIajgkLSZIkSZLUcExYSJIkSZKkhmPCQpIkSZIkNRwTFpIkSZIkqeGYsJAkSZIkSQ3HhIUkSZIkSWo4JiwkSZIkSVLDMWEhSZJWexExOSKei4iHa2Lfi4jfR8SDEfHLiOhbc+ykiJgTEX+IiD1r4uNKbE5EnFgTHxYRd5X4VRGxRsc9nSRJ3ZMJC0mS1BVcAoxrFrsBGJWZWwGzgJMAImIEMAEYWc75UUT0jIiewA+BvYARwMGlLMB3gbMyc1NgETCpvo8jSZJMWEiSpNVeZt4KLGwW+01mLim7dwKDy/Z4YEpmvpaZjwNzgO3KZ05mzs3M14EpwPiICGBX4Ofl/EuBfev6QJIkyYSFJEnqFo4Afl22NwSeqjk2v8Rai/cHFtckP5rikiSpjkxYSJKkLi0i/h+wBPhpB9zrqIiYEREzFixYUO/bSZLUpZmwkCRJXVZEHA7sAxyamVnCTwNDaooNLrHW4i8AfSOiV7P4O2TmhZk5JjPHDBw4sN2eQ5Kk7siEhSRJ6pIiYhxwPPDxzHy15tA0YEJE9ImIYcBmwN3APcBmZUWQNagm5pxWEh03AweU8ycC13TUc0iS1F3VLWEREe+JiLsj4oGIeCQivl7iLS4LVhoNV5X4XRExtOZaLS49JkmSBBARVwJ3AB+KiPkRMQn4AfA+4IaImBkRFwBk5iPAVOBR4DrgmMxcWuao+DxwPfAYMLWUBTgBOC4i5lDNaXFxBz6eJEndUq8VF1llrwG7ZubLEdEbuD0ifg0cR7Us2JTScJgEnF/+LsrMTSNiAtXyYf/cbOmxDYD/jYjNM3NpHesuSZJWI5l5cAvhVpMKmXkacFoL8enA9Bbic6lWEZEkSR2kbj0ssvJy2e1dPknry4KNL/uU47uVZcRaW3pMkiRJkiR1UXWdwyIiekbETOA54Abgj7S+LNiypcTK8Repuly2tsSYJEmSJEnqouqasCjjQUdTzaa9HbBFve7lMmKSJEmSJHUdHbJKSGYupppdewdaXxZs2VJi5fg6VMuItbbEWPN7uIyYJEmSJEldRD1XCRkYEX3L9prA7lQzbre2LNi0sk85flNZRqy1pcckSZIkSVIXVc9VQgYBl0ZET6rEyNTMvDYiHgWmRMS3gPt5awbvi4HLy3JhC6lWBiEzH4mIpqXHllCWHqtjvSVJkiRJUierW8IiMx8Etm4h3uKyYJn5N+DAVq7V4tJjkiRJkiSpa+qQOSwkSZIkSZJWhgkLSZIkSZLUcExYSJIkSZKkhmPCQpIkSZIkNRwTFpIkSZIkqeGYsJAkSZIkSQ3HhIUkSZIkSWo4JiwkSZIkSVLDMWEhSZIkSZIajgkLSZIkSZLUcExYSJIkSZKkhmPCQpIkSZIkNRwTFpIkSZIkqeGYsJAkSZIkSQ3HhIUkSZIkSWo4JiwkSZIkSVLDMWEhSZIkSZIajgkLSZIkSZLUcExYSJIkSZKkhmPCQpIkSZIkNRwTFpIkSZIkqeGYsJAkSZIkSQ3HhIUkSVrtRcTkiHguIh6uia0bETdExOzyt1+JR0ScGxFzIuLBiNim5pyJpfzsiJhYE982Ih4q55wbEdGxTyhJUvdjwkKSJHUFlwDjmsVOBG7MzM2AG8s+wF7AZuVzFHA+VAkO4BRge2A74JSmJEcp85ma85rfS5IktTMTFpIkabWXmbcCC5uFxwOXlu1LgX1r4pdl5U6gb0QMAvYEbsjMhZm5CLgBGFeOvT8z78zMBC6ruZYkSaoTExaSJKmrWj8znynbfwbWL9sbAk/VlJtfYsuLz28h/g4RcVREzIiIGQsWLHj3TyBJUjdmwkKSJHV5pWdEdsB9LszMMZk5ZuDAgfW+nSRJXVrdEhYRMSQibo6IRyPikYg4tsRPjYinI2Jm+exdc85JZTKrP0TEnjXxcSU2JyJObOl+kiRJzTxbhnNQ/j5X4k8DQ2rKDS6x5cUHtxCXJEl1VM8eFkuAL2XmCGAscExEjCjHzsrM0eUzHaAcmwCMpJrI6kcR0TMiegI/pJogawRwcM11JEmSWjMNaFrpYyJwTU38sLJayFjgxTJ05Hpgj4joVybb3AO4vhx7KSLGltVBDqu5liRJqpNe9bpw+XF/pmz/JSIeo5XxnsV4YEpmvgY8HhFzqGboBpiTmXMBImJKKftoveouSZJWLxFxJbALMCAi5lOt9nE6MDUiJgFPAAeV4tOBvYE5wKvApwEyc2FEfBO4p5T7RmY2TeT5OaqVSNYEfl0+kiSpjuqWsKgVEUOBrYG7gB2Bz0fEYcAMql4Yi6iSGXfWnFY7oVXzCbC2r3OVJUnSaiQzD27l0G4tlE3gmFauMxmY3EJ8BjDq3dRRkiStnLpPuhkR7wV+AXwhM1+iWsd8E2A0VQ+M77fTfZyVW5IkSZKkLqKuCYuI6E2VrPhpZl4NkJnPZubSzHwT+DFvDftY2Qmw3sZZuSWcuu6fAAAgAElEQVRJkiRJ6jrquUpIABcDj2XmmTXxQTXFPgE8XLanARMiok9EDAM2A+6mGke6WUQMi4g1qCbmnFavekuSJEmSpM5XzzksdgQ+BTwUETNL7KtUq3yMploLfR7wLwCZ+UhETKWaTHMJcExmLgWIiM9TzdzdE5icmY/Usd6SJEmSJKmTrXTCoizzNSQzH1xeucy8HYgWDk1fzjmnAae1EJ++vPMkSVLX09Y2hyRJ6praNCQkIm6JiPdHxLrAfcCPI+LMFZ0nSZK0MmxzSJKkJm2dw2KdssLHfsBlmbk98A/1q5YkSeqmbHNIkiSg7QmLXmWyzIOAa+tYH0mS1L3Z5pAkSUDbExZfp5r0ck5m3hMRGwOz61ctSZLUTdnmkCRJQNsn3XwmM7dq2snMuY4nlSRJdWCbQ5IkAW3vYXFeG2OSJEnvhm0OSZIErKCHRUTsAHwEGBgRx9Ucej/Qs54VkyRJ3YdtDkmS1NyKhoSsAby3lHtfTfwl4IB6VUqSJHU7tjkkSdLbLDdhkZm/BX4bEZdk5hMdVCdJktTN2OaQJEnNtXXSzT4RcSEwtPaczNy1HpWSJEndlm0OSZIEtD1h8TPgAuAiYGn9qiNJkro52xySJAloe8JiSWaeX9eaSJIk2eaQJElFW5c1/Z+I+FxEDIqIdZs+da2ZJEnqjmxzSJIkoO09LCaWv1+piSWwcftWR5IkdXO2OSRJEtDGhEVmDqt3RSRJkmxzSJKkJm1KWETEYS3FM/Oy9q2OJEnqzmxzSJKkJm0dEvLhmu33ALsB9wE2HiRJUnuyzSFJkoC2Dwn519r9iOgLTKlLjSRJUrdlm0OSJDVp6yohzb0COMZUkiTVm20OSZK6qbbOYfE/VDN0A/QEhgNT61UpSZLUPdnmkCRJTdo6h8V/1GwvAZ7IzPl1qI8kSerebHNIkiSgjUNCMvO3wO+B9wH9gNfrWSlJktQ92eaQJElN2pSwiIiDgLuBA4GDgLsi4oB6VkySJHU/tjkkSVKTtg4J+X/AhzPzOYCIGAj8L/DzelVMkiR1S+3e5oiILwJHUs2N8RDwaWAQ1eoj/YF7gU9l5usR0YdqCdVtgReAf87MeeU6JwGTgKXAv2Xm9ataJ0mStGJtXSWkR1PDoXhhJc6VJElqq3Ztc0TEhsC/AWMycxTVRJ4TgO8CZ2XmpsAiqkQE5e+iEj+rlCMiRpTzRgLjgB9FRM9VrZckSVqxtjYArouI6yPi8Ig4HPgVML1+1ZIkSd1UPdocvYA1I6IXsBbwDLArb/XauBTYt2yPL/uU47tFRJT4lMx8LTMfB+YA273LekmSpOVY7pCQiNgUWD8zvxIR+wF/Xw7dAfy03pWTJEndQ73aHJn5dET8B/Ak8FfgN1RDQBZn5pJSbD6wYdneEHiqnLskIl6kGjayIXBnzaVrz5EkSXWwoh4WZwMvAWTm1Zl5XGYeB/yyHJMkSWoPdWlzREQ/qt4Rw4ANgLWphnTURUQcFREzImLGggUL6nUbSZK6hRUlLNbPzIeaB0ts6PJOjIghEXFzRDwaEY9ExLElvm5E3BARs8vffiUeEXFuRMyJiAcjYpuaa00s5WdHxMSVfkpJktToVrnNsQL/ADyemQsy8w3gamBHoG8ZIgIwGHi6bD8NDAEox9ehmkdjWbyFc2rre2FmjsnMMQMHDnwX1ZYkSStKWPRdzrE1V3DuEuBLmTkCGAscUyasOhG4MTM3A24s+wB7AZuVz1HA+VAlOIBTgO2pxoqe0pTkkCRJXca7aXMsz5PA2IhYq8xFsRvwKHAz0LRc6kTgmrI9rexTjt+UmVniEyKiT0QMo2qv3P0u6iVJklZgRQmLGRHxmebBiDiSavxnqzLzmcy8r2z/BXiMaqxn7WRWzSe5uiwrd1K9+RgE7AnckJkLM3MRcAN17MopSZI6xSq3OZYnM++imjzzPqolTXsAFwInAMdFxByqOSouLqdcDPQv8eMoL1Yy8xFgKlWy4zrgmMxcuqr1kiRJK7bcSTeBLwC/jIhDeauxMAZYA/hEW28SEUOBrYG7qLp8PlMO/RlYv2wvm+SqaJrMqrV483scRdUzg4022qitVZMkSY2hXdocLcnMU6h6a9aaSwurfGTm34ADW7nOacBp76YukiSp7ZabsMjMZ4GPRMTHgFEl/KvMvKmtN4iI9wK/AL6QmS9VvTGXXT8jIle+2i3W9UKqNyaMGTOmXa4pSZI6Rnu0OSRJUteyoh4WAGTmzVRjPVdKRPSmSlb8NDOvLuFnI2JQZj5Thnw8V+KtTWb1NLBLs/gtK1sXSZLU+Fa1zSFJkrqeFc1hscrKxFYXA49l5pk1h2ons2o+ydVhZbWQscCLZejI9cAeEdGvTLa5R4lJkiRJkqQuqk09LFbRjsCngIciYmaJfRU4HZgaEZOAJ4CDyrHpwN7AHOBV4NMAmbkwIr4J3FPKfSMzF9ax3pIkSZIkqZPVLWGRmbcD0crh3Voon8AxrVxrMjC5/WonSZIkSZIaWd2GhEiSJEmSJK0qExaSJEmSJKnhmLCQJEmSJEkNx4SFJEmSJElqOCYsJEmSJElSwzFhIUmSJEmSGo4JC0mSJEmS1HBMWEiSJEmSpIZjwkKSJEmSJDUcExaSJEmSJKnhmLCQJEmSJEkNx4SFJEmSJElqOCYsJEmSJElSwzFhIUmSJEmSGo4JC0mSJEmS1HBMWEiSJEmSpIZjwkKSJEmSJDUcExaSJEmSJKnhmLCQJEmSJEkNx4SFJEmSJElqOCYsJEmSJElSwzFhIUmSJEmSGo4JC0mSJEmS1HBMWEiSJEmSpIZjwkKSJHVpEdE3In4eEb+PiMciYoeIWDciboiI2eVvv1I2IuLciJgTEQ9GxDY115lYys+OiImd90SSJHUPJiwkSVJXdw5wXWZuAfwd8BhwInBjZm4G3Fj2AfYCNiufo4DzASJiXeAUYHtgO+CUpiSHJEmqDxMWkiSpy4qIdYCdgIsBMvP1zFwMjAcuLcUuBfYt2+OBy7JyJ9A3IgYBewI3ZObCzFwE3ACM68BHkSSp26lbwiIiJkfEcxHxcE3s1Ih4OiJmls/eNcdOKt0v/xARe9bEx5XYnIg4sfl9JEmSlmMYsAD4SUTcHxEXRcTawPqZ+Uwp82dg/bK9IfBUzfnzS6y1+NtExFERMSMiZixYsKCdH0WSpO6lnj0sLqHlNw9nZebo8pkOEBEjgAnAyHLOjyKiZ0T0BH5I1T1zBHBwKStJktQWvYBtgPMzc2vgFd4a/gFAZiaQ7XGzzLwwM8dk5piBAwe2xyUlSeq26pawyMxbgYVtLD4emJKZr2Xm48AcqvGh2wFzMnNuZr4OTCllJUmS2mI+MD8z7yr7P6dKYDxbhnpQ/j5Xjj8NDKk5f3CJtRaXJEl10hlzWHy+zLo9uWayqnfV/VKSJKklmfln4KmI+FAJ7QY8CkwDmlb6mAhcU7anAYeV1ULGAi+WoSPXA3tERL/SftmjxCRJUp306uD7nQ98k6rb5TeB7wNHtMeFI+Ioqtm82WijjdrjkpIkqWv4V+CnEbEGMBf4NNVLm6kRMQl4AjiolJ0O7E3V2/PVUpbMXBgR3wTuKeW+kZlt7UkqSZJWQYcmLDLz2abtiPgxcG3ZXV43yzZ1v8zMC4ELAcaMGdMu41AlSdLqLzNnAmNaOLRbC2UTOKaV60wGJrdv7SRJUms6dEhI01jR4hNA0woi04AJEdEnIoZRrX1+N9VbjM0iYlh5KzKhlJUkSZIkSV1Y3XpYRMSVwC7AgIiYD5wC7BIRo6mGhMwD/gUgMx+JiKlUY0qXAMdk5tJync9TjRHtCUzOzEfqVWdJkiRJktQY6pawyMyDWwhfvJzypwGntRCfTjWeVJIkSZIkdROdsUqIJEmSJEnScpmwkCRJkiRJDceEhSRJkiRJajgmLCRJkiRJUsMxYSFJkiRJkhqOCQtJkiRJktRwTFhIkiRJkqSGY8JCkiRJkiQ1HBMWkiRJkiSp4ZiwkCRJkiRJDceEhSRJkiRJajgmLCRJkiRJUsMxYSFJkiRJkhqOCQtJkiRJktRwTFhIkiRJkqSGY8JCkiRJkiQ1HBMWkiRJkiSp4ZiwkCRJkiRJDceEhSRJkiRJajgmLCRJkiRJUsMxYSFJkiRJkhqOCQtJkiRJktRwTFhIkiRJkqSGY8JCkiRJkiQ1HBMWkiRJkiSp4ZiwkCRJXV5E9IyI+yPi2rI/LCLuiog5EXFVRKxR4n3K/pxyfGjNNU4q8T9ExJ6d8ySSJHUfJiwkSVJ3cCzwWM3+d4GzMnNTYBEwqcQnAYtK/KxSjogYAUwARgLjgB9FRM8OqrskSd2SCQtJktSlRcRg4B+Bi8p+ALsCPy9FLgX2Ldvjyz7l+G6l/HhgSma+lpmPA3OA7TrmCSRJ6p7qlrCIiMkR8VxEPFwTWzciboiI2eVvvxKPiDi3dLN8MCK2qTlnYik/OyIm1qu+kiSpyzobOB54s+z3BxZn5pKyPx/YsGxvCDwFUI6/WMovi7dwjiRJqoN69rC4hKrLZK0TgRszczPgxrIPsBewWfkcBZwPVYIDOAXYnuotxilNSQ5JkqQViYh9gOcy894Out9RETEjImYsWLCgI24pSVKXVbeERWbeCixsFq7tZtm8++VlWbkT6BsRg4A9gRsyc2FmLgJu4J1JEEmSpNbsCHw8IuYBU6iGgpxD1dboVcoMBp4u208DQwDK8XWAF2rjLZyzTGZemJljMnPMwIED2/9pJEnqRjp6Dov1M/OZsv1nYP2y3Vo3yzZ3v/SNhiRJai4zT8rMwZk5lGrSzJsy81DgZuCAUmwicE3Znlb2Kcdvysws8QllFZFhVL1C7+6gx5AkqVvqtEk3y49/tuP1fKMhSZLa6gTguIiYQzVHxcUlfjHQv8SPowxfzcxHgKnAo8B1wDGZubTDay1JUjfSa8VF2tWzETEoM58pQz6eK/HWulk+DezSLH5LB9RTkiR1MZl5C6UdkZlzaWGVj8z8G3BgK+efBpxWvxpKkqRaHd3DorabZfPul4eV1ULGAi+WoSPXA3tERL8y2eYeJSZJkiRJkrqwuvWwiIgrqXpHDIiI+VSrfZwOTI2IScATwEGl+HRgb6o1zV8FPg2QmQsj4pvAPaXcNzKz+USekiRJkiSpi6lbwiIzD27l0G4tlE3gmFauMxmY3I5VkyRJkiRJDa7TJt2UJEmSJElqjQkLSZIkSZLUcExYSJIkSZKkhmPCQpIkSZIkNRwTFpIkSZIkqeGYsJAkSZIkSQ3HhIUkSZIkSWo4JiwkSZIkSVLDMWEhSZIkSZIajgkLSZIkrTaeeuopPvaxjzFixAhGjhzJOeecA8Cpp57KhhtuyOjRoxk9ejTTp08HYN68eay55prL4kcfffSya1155ZVsueWWbLXVVowbN47nn3++U55JktSyXp1dAUmSJKmtevXqxfe//3222WYb/vKXv7Dtttuy++67A/DFL36RL3/5y+84Z5NNNmHmzJlviy1ZsoRjjz2WRx99lAEDBnD88cfzgx/8gFNPPbUjHkOS1Ab2sJAkSdJqY9CgQWyzzTYAvO9972P48OE8/fTTK32dzCQzeeWVV8hMXnrpJTbYYIP2rq4k6V0wYaEua/HixRxwwAFsscUWDB8+nDvuuIOvfOUrbLHFFmy11VZ84hOfYPHixQD89Kc/XdZVdPTo0fTo0eMdb2IkSVJjmTdvHvfffz/bb789AD/4wQ/YaqutOOKII1i0aNGyco8//jhbb701O++8M7fddhsAvXv35vzzz2fLLbdkgw024NFHH2XSpEmd8hySpJaZsFCXdeyxxzJu3Dh+//vf88ADDzB8+HB23313Hn74YR588EE233xzvvOd7wBw6KGHMnPmTGbOnMnll1/OsGHDGD16dCc/gSRJas3LL7/M/vvvz9lnn8373/9+PvvZz/LHP/6RmTNnMmjQIL70pS8BVY+MJ598kvvvv58zzzyTQw45hJdeeok33niD888/n/vvv58//elPbLXVVsvaBZKkxmDCQl3Siy++yK233rrsTckaa6xB37592WOPPejVq5q6ZezYscyfP/8d51555ZVMmDChQ+srSZLa7o033mD//ffn0EMPZb/99gNg/fXXp2fPnvTo0YPPfOYz3H333QD06dOH/v37A7DtttuyySabMGvWrGU9KTfZZBMigoMOOoj/+7//65wHkiS1yISFuqTHH3+cgQMH8ulPf5qtt96aI488kldeeeVtZSZPnsxee+31jnOvuuoqDj744I6qqiRJWgmZyaRJkxg+fDjHHXfcsvgzzzyzbPuXv/wlo0aNAmDBggUsXboUgLlz5zJ79mw23nhjNtxwQx599FEWLFgAwA033MDw4cM78EkkSSviKiHqkpYsWcJ9993Heeedx/bbb8+xxx7L6aefzje/+U0ATjvtNHr16sWhhx76tvPuuusu1lprrWWNHEmS1Fh+97vfcfnll7PlllsuG7757W9/myuvvJKZM2cSEQwdOpT//M//BODWW2/l5JNPpnfv3vTo0YMLLriAddddF4BTTjmFnXbaid69e/PBD36QSy65pLMeS5LUAhMW6pIGDx7M4MGDl03CdcABB3D66acDcMkll3Dttddy4403EhFvO2/KlCn2rpAkqYH9/d//PZn5jvjee+/dYvn999+f/fffv8VjRx99NEcffXS71k+S1H5MWKhL+sAHPsCQIUP4wx/+wIc+9CFuvPFGRowYwXXXXccZZ5zBb3/7W9Zaa623nfPmm28yderUZbOHS5IkSZI6jwkLdVnnnXcehx56KK+//jobb7wxP/nJT/jwhz/Ma6+9xu677w5UE29ecMEFQNVldMiQIWy88cadWW1JklTr1HU6+f4vdu79JakbM2GhLmv06NHMmDHjbbE5c+a0Wn6XXXbhzjvvrHe1JEmSJEltYMJCja2z36qsCt/ESJIkSdK75rKmkiRJkiSp4ZiwkCRJkiRJDceEhSRJkiRJajgmLCRJkiRJUsMxYSFJkrqsiBgSETdHxKMR8UhEHFvi60bEDRExu/ztV+IREedGxJyIeDAitqm51sRSfnZETOysZ5IkqbvolIRFRMyLiIciYmZEzCixlW44SJL0/9u7+yip6vvw4+8PIDaREmIUwV1TYsWiQUGgYBoPQclSYjwolRrNg4hQiz9MwLRpaE5PU8WT4FNjQvNQNQTSUGzEGjhACAS1qZ4SIGSJ1AdE2PxYz8pDFVCMysO3f8yFDsvj6u7cmdn365w9O3PvnTuf/TAz3w+f+713pOPYC/xVSul84GJgUkScD0wFlqeUegPLs/sAnwB6Zz83Ad+FQp0CfBUYAgwGvnqgVpEkSW0jzxkWl6aU+qeUBmX3W1Q4SJIkHU9KqSmltCa7/RrwLFADXAnMzjabDVyV3b4S+GEqWAF0i4iewJ8Cy1JKr6SUXgWWASNL+KdIktTulNMpIS0tHCRJkk5YRPQCLgJ+CZyRUmrKVr0MnJHdrgE2Fz2sMVt2tOXNn+OmiFgdEau3bdvWqvFLktTe5NWwSMDSiPhVRNyULWtp4XAICwRJknQ0EdEFeASYklLaVbwupZQo1CbvWkrp/pTSoJTSoNNPP701dilJUruVV8PikpTSAAqne0yKiKHFK99J4WCBIEmSjiQiTqLQrJiTUvr3bPGWAzM2s99bs+UvAWcVPbw2W3a05ZIkqY3k0rBIKb2U/d4KPErh4lUtLRwkSZKOKSIC+D7wbErpH4tWLQAOfNPHWGB+0fLrs4t+XwzszGaA/gwYERHvzy62OSJbJkmS2kjJGxYRcUpE/P6B2xQG/HW0vHCQJEk6no8CnwMuy76drD4iLgemA3UR8QLw8ew+wGJgI7ABeAD4fwAppVeAacCq7Of2bJkkSWojnXJ4zjOARwsHPOgE/GtKaUlErAJ+HBHjgd8C12TbLwYup1A4vAGMK33IkiSpEqWUngTiKKuHH2H7BEw6yr5mAjNbLzpJknQsJW9YpJQ2Av2OsPx/aGHhIEmSJEmSqlM5fa2pJEmSJEkSYMNCkiRJkiSVIRsWkiRJkiSp7NiwkCRJkiRJZceGhSRJkiRJKjs2LCRJkiRJUtmxYSFJkiRJksqODQtJkiRJklR2bFhIkiRJkqSyY8NCkiRJkiSVHRsWkiRJkiSp7NiwkCRJkiRJZceGhSRJklThevXqxQUXXED//v0ZNGgQAK+88gp1dXX07t2buro6Xn311UMes2rVKjp16sS8efPyCFmSjsuGhSRJklQFHn/8cerr61m9ejUA06dPZ/jw4bzwwgsMHz6c6dOnH9x23759fPnLX2bEiBF5hStJx2XDQpIkSapC8+fPZ+zYsQCMHTuWn/zkJwfXzZgxg6uvvpru3bvnFZ4kHZcNC0mSJKnCRQQjRoxg4MCB3H///QBs2bKFnj17AtCjRw+2bNkCwEsvvcSjjz7KzTffnFu8knQiOuUdgKTqsmPHDiZMmMC6deuICGbOnMl73/teJk6cyOuvv06vXr2YM2cOXbt2zTtUSZKqxpNPPklNTQ1bt26lrq6OPn36HLI+IogIAKZMmcKdd95Jhw4eu5RU3mxYSGpVkydPZuTIkcybN4+3336bN954g7q6Ou655x4+9rGPMXPmTO6++26mTZuWd6iSJFWNmpoaALp3787o0aNZuXIlZ5xxBk1NTfTs2ZOmpqaDp3+sXr2aa6+9FoDt27ezePFiOnXqxFVXXZVb/JJ0JLZVJbWanTt38otf/ILx48cD0LlzZ7p168b69esZOnQoAHV1dTzyyCN5hilJUlXZvXs3r7322sHbS5cupW/fvowaNYrZs2cDMHv2bK688koANm3aRENDAw0NDYwZM4bvfOc7NisklSUbFpJazaZNmzj99NMZN24cF110ERMmTGD37t18+MMfZv78+QA8/PDDbN68OedIJUmqHlu2bOGSSy6hX79+DB48mE9+8pOMHDmSqVOnsmzZMnr37s3Pf/5zpk6dmneobebGG2+ke/fu9O3b97B19957LxHB9u3bAXjuuef4yEc+wsknn8w999xT6lAltYCnhEhqNXv37mXNmjXMmDGDIUOGMHnyZKZPn87MmTP5whe+wLRp0xg1ahSdO3fOO1RJkqrG2Wefzdq1aw9b/oEPfIDly5cf87GzZs1qo6hK64YbbuCWW27h+uuvP2T55s2bWbp0KR/84AcPLjv11FP51re+dci3pkgqTzYsJLWa2tpaamtrGTJkCABjxoxh+vTpTJs2jaVLlwKwfv16Fi1alGeYVWvfvn0MGjSImpoaFi5cmHc4VevGG29k4cKFdO/enXXr1uUdjqR24ILZF+QdAk+PfTrvEI5p6NChNDQ0HLb81ltv5a677jp4OgwUrvPRvXv3qq5H3nzzTYYOHcpbb73F3r17GTNmDLfddhvLly/nS1/6Evv376dLly7MmjWLc845J+9wS8ZaqaCS8uApIZJaTY8ePTjrrLN4/vnnAVi+fDnnn38+W7duBWD//v3ccccdTJw4Mc8wq9Y3v/lNzjvvvLzDqHo33HADS5YsyTsMSdJxzJ8/n5qaGvr165d3KCV38skn89hjj7F27Vrq6+tZsmQJK1as4Oabb2bOnDnU19fz6U9/mjvuuCPvUEvKWqmgkvJgw0JSq5oxYwaf+cxnuPDCC6mvr+crX/kKc+fO5dxzz6VPnz6ceeaZjBs3Lu8wq05jYyOLFi1iwoQJeYdS9YYOHcqpp56adxiSpGN44403+NrXvsbtt9+edyi5iAi6dOkCwJ49e9izZ8/Br7bdtWsXULhY+plnnplnmCVlrVRQaXnwlBBJrap///6sXr36kGWTJ09m8uTJOUXUPkyZMoW77rrr4FXiJUlqz1588UU2bdp0cHZFY2MjAwYMYOXKlfTo0SPn6Epj3759DBw4kA0bNjBp0iSGDBnCgw8+yOWXX8573vMeunbtyooVK/IOs2SslQoqLQ/OsJCkCnfgegoDBw7MOxRJksrCBRdcwNatWw9+fWttbS1r1qxpN80KgI4dO1JfX09jYyMrV65k3bp1fOMb32Dx4sU0NjYybtw4vvjFL+YdZklYKxVUYh6cYSEJKI8Ler0T5X4RsFJ46qmnWLBgAYsXL+bNN99k165dfPazn+VHP/pR3qFJklQS1113HU888QTbt2+ntraW2267jfHjxx9x25dffplBgwaxa9cuOnTowH333cczzzxD165dSxx1aXTr1o1LL72Un/70p6xdu/bgxdE/9alPMXLkyJyjKw1rpYJKzEPFzLCIiJER8XxEbIiI6v0SaUlqoa9//es0NjbS0NDAQw89xGWXXVbWA49UyaxHpPI0d+5cmpqa2LNnD42NjYc1KxoaGjjttNOAwkXCGxsb2bVrFzt27KCxsbHqmhXbtm1jx44dAPzud79j2bJlnHfeeezcuZP169cDHFzWHlgrFVRiHipihkVEdAS+DdQBjcCqiFiQUnom38gkSe1NS47iqbpYj0j5ebZP/v+xPu+5Z/MO4YQ1NTUxduxY9u3bx/79+7nmmmu44ooreOCBB7j66qvp0KED73//+5k5c2beoUrHVBENC2AwsCGltBEgIh4CrgQsECSpyLBhwxg2bFjeYVS1uXPn5h2C8mM9IqkiXHjhhfz6178+bPno0aMZPXp0DhGVD2ulgkrJQ6U0LGqAzUX3G4EhOcUiSZLaJ+sRSbn59sTH8g6BSd+7LO8Q1M5USsPiuCLiJuCm7O7rEfF8nvG8Q6cB29tix3FnW+y1orVZrrkt2mS3Fa7tXts3mO9m2u61reYqNdd/kHcA1awM65F39Tqtkvrh3b1Xq2Ncf9efV1Uy3r67PIQ5uOWfWzGS/FTq+N2ayiEHJ1SPVErD4iXgrKL7tdmyg1JK9wP3lzKo1hYRq1NKg/KOoz0w16VlvkvHXJeOuW6XKq4e8XVqDkvA+zkAAAkCSURBVMAcHGAezAGYA6isHFTKt4SsAnpHxIciojNwLbAg55gkSVL7Yj0iSVIJVcQMi5TS3oi4BfgZ0BGYmVL675zDkiRJ7Yj1iCRJpVURDQuAlNJiYHHecbSxsplC2g6Y69Iy36VjrkvHXLdDFViP+Do1B2AODjAP5gDMAVRQDiKllHcMkiRJkiRJh6iUa1hIkiRJkqR2xIZFiUXEzIjYGhHrjrI+IuJbEbEhIn4TEQNKHWO1iIizIuLxiHgmIv47IiYfYRvz3Uoi4vciYmVErM3yfdsRtjk5Iv4ty/cvI6JX6SOtHhHRMSJ+HRELj7DOXLeSiOgWEfMi4rmIeDYiPtJsvZ8jypW1hWM+OA4Xc3yEiGiIiKcjoj4iVh9hfVW/H8DxOyL+KPv3P/CzKyKmNNum7HNgw6L0ZgEjj7H+E0Dv7Ocm4LsliKla7QX+KqV0PnAxMCkizm+2jfluPW8Bl6WU+gH9gZERcXGzbcYDr6aUzgG+AdxZ4hirzWTg2aOsM9et55vAkpRSH6Afh+fczxHlbRbWFo75jsPFHB8LLk0p9T/K11dW+/sB2vn4nVJ6Pvv37w8MBN4AHm22WdnnwIZFiaWUfgG8coxNrgR+mApWAN0iomdpoqsuKaWmlNKa7PZrFD6kapptZr5bSZbD17O7J2U/zS+ScyUwO7s9DxgeEVGiEKtKRNQCnwQePMom5roVRMT7gKHA9wFSSm+nlHY028zPEeXK2sIxHxyHD3B8PGFV/X5w/D7McODFlNJvmy0v+xzYsCg/NcDmovuNHD7gqoWy6X4XAb9stsp8t6JsCmY9sBVYllI6ar5TSnuBncAHShtl1bgP+Btg/1HWm+vW8SFgG/CDbHrxgxFxSrNt/BxRuWtXr9H2POY7DgOOjwckYGlE/CoibjrC+mp/Pzh+H+paYO4Rlpd9DmxYqOpFRBfgEWBKSmlX3vFUs5TSvmzaWS0wOCL65h1TNYqIK4CtKaVf5R1LO9AJGAB8N6V0EbAbmJpvSJKOpr2P+e19HHZ8PMQlKaUBFKb8T4qIoXkHVGKO35mI6AyMAh7OO5Z3woZF+XkJOKvofm22TO9ARJxEoXCZk1L69yNsYr7bQDbl7nEOP6f6YL4johPwPuB/ShtdVfgoMCoiGoCHgMsi4kfNtjHXraMRaCw6SjmPQgFUzM8Rlbt28Rp1zP8/7XgcdnzMpJReyn5vpXDdgsHNNqn294Pj9//5BLAmpbTlCOvKPgc2LMrPAuD67IqtFwM7U0pNeQdVibLzEb8PPJtS+sejbGa+W0lEnB4R3bLb7wHqgOeabbYAGJvdHgM8llJqfn6tjiOl9LcppdqUUi8KU/weSyl9ttlm5roVpJReBjZHxB9li4YDzzTbzM8Rlbuqf4065jsOg+PjARFxSkT8/oHbwAig+bcIVfX7wfH7ENdx5NNBoAJy0CnvANqbiJgLDANOi4hG4KsULopESul7wGLgcmADhSu5jssn0qrwUeBzwNPZ+ZwAXwE+COa7DfQEZkdERwrN0B+nlBZGxO3A6pTSAgrF5L9ExAYKF4i7Nr9wq4+5bjOfB+ZkUyo3AuMiYiL4OaLyYG0BOOaD4/BRtcMcnAE8ml1LtBPwrymlJe1w7Gr343fWsKoD/rJoWUXlIKqsoShJkiRJkqqAp4RIkiRJkqSyY8NCkiRJkiSVHRsWkiRJkiSp7NiwkCRJkiRJZceGhSRJkiRJKjs2LKQKFhEpIu4tuv/XEfEPrbTvWRExpjX2dZzn+fOIeDYiHm+2vFdENP/O8OaPHRYRC1v4fE9ExKB3EqskSTqc9Yj1iNRWbFhIle0t4M8i4rS8AykWEZ1asPl44C9SSpe2VTySJKlNWY9IahM2LKTKthe4H7i1+YrmRyQi4vXs97CI+I+ImB8RGyNiekR8JiJWRsTTEfGHRbv5eESsjoj1EXFF9viOEXF3RKyKiN9ExF8W7fc/I2IB8MwR4rku2/+6iLgzW/b3wCXA9yPi7qP9kdnRjf+MiDXZz58Ure4aEYsi4vmI+F5EdMgeMyIi/ivb/uGI6NJsnx2zHK3L4josh5Ik6YRYj1iPSG2iJV1HSeXp28BvIuKuFjymH3Ae8AqwEXgwpTQ4IiYDnwemZNv1AgYDfwg8HhHnANcDO1NKfxwRJwNPRcTSbPsBQN+U0qbiJ4uIM4E7gYHAq8DSiLgqpXR7RFwG/HVKafUx4t0K1KWU3oyI3sBc4MA0ysHA+cBvgSUUjvA8Afwd8PGU0u6I+DLwReD2on32B2pSSn2zGLudUOYkSdKRWI9Yj0itzoaFVOFSSrsi4ofAF4DfneDDVqWUmgAi4kXgwAD/NFA8FfLHKaX9wAsRsRHoA4wALiw6WvI+oDfwNrCyeXGQ+WPgiZTStuw55wBDgZ+cYLwnAf8UEf2BfcC5RetWppQ2ZvudS+EIyZsUioanIgKgM/Bfzfa5ETg7ImYAi4pyIEmSWsh6xHpEags2LKTqcB+wBvhB0bK9ZKd9ZdMSOxete6vo9v6i+/s59HMhNXueBATw+ZTSz4pXRMQwYPc7C/+4bgW2UDgS04FCAXC8GJellK472g5TSq9GRD/gT4GJwDXAja0ZtCRJ7Yz1yOExWo9I74LXsJCqQErpFeDHFC4YdUADhSmPAKMoHBVoqT+PiA7ZeaRnA88DPwNujoiTACLi3Ig45Tj7WQl8LCJOi4iOwHXAf7QgjvcBTdnRlc8BHYvWDY6ID2VF0KeAJ4EVwEezKaNExCkRUXwUhChcGKxDSukRCtM1B7QgHkmS1Iz1iPWI1NqcYSFVj3uBW4ruPwDMj4i1FM6lfCdHG/4/hcG9KzAxO2fzQQrnkq6JwvzGbcBVx9pJSqkpIqYCj1M42rAopTS/BXF8B3gkIq7n8L9lFfBPwDnZ/h9NKe2PiBuAudl5rVAoAtYXPa4G+MGBi2IBf9uCeCRJ0pFZj1iPSK0mUmo+e0mSJEmSJClfnhIiSZIkSZLKjg0LSZIkSZJUdmxYSJIkSZKksmPDQpIkSZIklR0bFpIkSZIkqezYsJAkSZIkSWXHhoUkSZIkSSo7NiwkSZIkSVLZ+V8OdpYv/qHrxwAAAABJRU5ErkJggg==\n"},"metadata":{}}]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"# Special thanks to https://github.com/makinacorpus/easydict/blob/master/easydict/__init__.py\n\nclass EasyDict(dict):\n\n    def __init__(self, d=None, **kwargs):\n        if d is None:\n            d = {}\n        if kwargs:\n            d.update(**kwargs)\n        for k, v in d.items():\n            setattr(self, k, v)\n        # Class attributes\n        for k in self.__class__.__dict__.keys():\n            if not (k.startswith('__') and k.endswith('__')) and not k in ('update', 'pop'):\n                setattr(self, k, getattr(self, k))\n\n    def __setattr__(self, name, value):\n        if isinstance(value, (list, tuple)):\n            value = [self.__class__(x)\n                     if isinstance(x, dict) else x for x in value]\n        elif isinstance(value, dict) and not isinstance(value, self.__class__):\n            value = self.__class__(value)\n        super(EasyDict, self).__setattr__(name, value)\n        super(EasyDict, self).__setitem__(name, value)\n\n    __setitem__ = __setattr__\n\n    def update(self, e=None, **f):\n        d = e or dict()\n        d.update(f)\n        for k in d:\n            setattr(self, k, d[k])\n\n    def pop(self, k, d=None):\n        delattr(self, k)\n        return super(EasyDict, self).pop(k, d)","execution_count":15,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"conf = EasyDict()\nconf.sampling_rate = 44100\nconf.duration = 5\nconf.hop_length = 347 # to make time steps 128\nconf.fmin = 20\nconf.fmax = conf.sampling_rate // 2\nconf.n_mels = 128\nconf.n_fft = conf.n_mels * 20\n\nconf.samples = conf.sampling_rate * conf.duration\n\ntrain_curated_path = '../input/train_curated/'\ntrain_noisy_path = '../input/train_noisy/'\ntest_path = '../input/test/'","execution_count":16,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def read_audio(conf, pathname, trim_long_data):\n    y, sr = librosa.load(pathname, sr=conf.sampling_rate)\n    # trim silence\n    if 0 < len(y): # workaround: 0 length causes error\n        y, _ = librosa.effects.trim(y) # trim, top_db=default(60)\n    # make it unified length to conf.samples\n    if len(y) > conf.samples: # long enough\n        if trim_long_data:\n            y = y[0:0+conf.samples]\n    else: # pad blank\n        padding = conf.samples - len(y)    # add padding at both ends\n        offset = padding // 2\n        y = np.pad(y, (offset, conf.samples - len(y) - offset), 'constant')\n    return y\n\ndef audio_to_melspectrogram(conf, audio):\n    spectrogram = librosa.feature.melspectrogram(audio, \n                                                 sr=conf.sampling_rate,\n                                                 n_mels=conf.n_mels,\n                                                 hop_length=conf.hop_length,\n                                                 n_fft=conf.n_fft,\n                                                 fmin=conf.fmin,\n                                                 fmax=conf.fmax)\n    spectrogram = librosa.power_to_db(spectrogram)\n    spectrogram = spectrogram.astype(np.float32)\n    return spectrogram\n\ndef read_as_melspectrogram(conf, pathname, trim_long_data, debug_display=False):\n    x = read_audio(conf, pathname, trim_long_data)\n    mels = audio_to_melspectrogram(conf, x)\n    if debug_display:\n        IPython.display.display(IPython.display.Audio(x, rate=conf.sampling_rate))\n        show_melspectrogram(conf, mels)\n    return mels\n\ndef convert_wav_to_image(df, source):\n    X = []\n    for i, row in tqdm_notebook(df.iterrows()):\n        try:\n            x = read_as_melspectrogram(conf, f'{source[0]}/{str(row.fname)}', trim_long_data=True)\n        except:\n            x = read_as_melspectrogram(conf, f'{source[1]}/{str(row.fname)}', trim_long_data=True)\n\n        #x_color = mono_to_color(x)\n        X.append(x.transpose())\n        #df.loc[i, 'length'] = x.shape[1]\n    return X","execution_count":17,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#For baseline, noisy set is not used.\n#train = pd.concat([train_curated, train_noisy],axis=0)","execution_count":18,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#del train_curated, train_noisy\n\n#gc.collect()","execution_count":19,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\n#X = np.array(convert_wav_to_image(train, source=[train_curated_path, train_noisy_path]))\nX = np.array(convert_wav_to_image(train_curated, source=[train_curated_path]))","execution_count":20,"outputs":[{"output_type":"display_data","data":{"text/plain":"HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"3008c1ed6ec4422090ae337d92a9c7b8"}},"metadata":{}},{"output_type":"stream","text":"\nCPU times: user 6min 13s, sys: 4min 30s, total: 10min 43s\nWall time: 5min 42s\n","name":"stdout"}]},{"metadata":{"trusted":true},"cell_type":"code","source":"Y = train_curated[label_columns].values","execution_count":21,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#del train\n#gc.collect()","execution_count":22,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras import backend as K\nfrom keras.engine.topology import Layer\nfrom keras import initializers, regularizers, constraints, optimizers, layers\nfrom keras.layers import Embedding, Input, Dense, CuDNNGRU,concatenate, Bidirectional, SpatialDropout1D, Conv1D, GlobalAveragePooling1D, GlobalMaxPooling1D, Dropout\nfrom keras.models import Model\nfrom keras.optimizers import Adam\nfrom keras.callbacks import EarlyStopping","execution_count":23,"outputs":[{"output_type":"stream","text":"Using TensorFlow backend.\n","name":"stderr"}]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"class Attention(Layer):\n    def __init__(self, step_dim,\n                 W_regularizer=None, b_regularizer=None,\n                 W_constraint=None, b_constraint=None,\n                 bias=True, **kwargs):\n        self.supports_masking = True\n        self.init = initializers.get('glorot_uniform')\n\n        self.W_regularizer = regularizers.get(W_regularizer)\n        self.b_regularizer = regularizers.get(b_regularizer)\n\n        self.W_constraint = constraints.get(W_constraint)\n        self.b_constraint = constraints.get(b_constraint)\n\n        self.bias = bias\n        self.step_dim = step_dim\n        self.features_dim = 0\n        super(Attention, self).__init__(**kwargs)\n\n    def build(self, input_shape):\n        assert len(input_shape) == 3\n\n        self.W = self.add_weight((input_shape[-1],),\n                                 initializer=self.init,\n                                 name='{}_W'.format(self.name),\n                                 regularizer=self.W_regularizer,\n                                 constraint=self.W_constraint)\n        self.features_dim = input_shape[-1]\n\n        if self.bias:\n            self.b = self.add_weight((input_shape[1],),\n                                     initializer='zero',\n                                     name='{}_b'.format(self.name),\n                                     regularizer=self.b_regularizer,\n                                     constraint=self.b_constraint)\n        else:\n            self.b = None\n\n        self.built = True\n\n    def compute_mask(self, input, input_mask=None):\n        return None\n\n    def call(self, x, mask=None):\n        features_dim = self.features_dim\n        step_dim = self.step_dim\n\n        eij = K.reshape(K.dot(K.reshape(x, (-1, features_dim)),\n                        K.reshape(self.W, (features_dim, 1))), (-1, step_dim))\n\n        if self.bias:\n            eij += self.b\n\n        eij = K.tanh(eij)\n\n        a = K.exp(eij)\n\n        if mask is not None:\n            a *= K.cast(mask, K.floatx())\n\n        a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx())\n\n        a = K.expand_dims(a)\n        weighted_input = x * a\n        return K.sum(weighted_input, axis=1)\n\n    def compute_output_shape(self, input_shape):\n        return input_shape[0],  self.features_dim","execution_count":24,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sequence_input = Input(shape=(636,128), dtype='float32')\nx = CuDNNGRU(128, return_sequences=True)(sequence_input) \natt = Attention(636)(x)\navg_pool = GlobalAveragePooling1D()(x)\nmax_pool = GlobalMaxPooling1D()(x) \n\nx = concatenate([att, avg_pool, max_pool])\n\npreds = Dense(80, activation='softmax')(x)\n\nmodel = Model(sequence_input, preds)\nmodel.summary()","execution_count":25,"outputs":[{"output_type":"stream","text":"WARNING:tensorflow:From /opt/conda/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\nInstructions for updating:\nColocations handled automatically by placer.\n__________________________________________________________________________________________________\nLayer (type)                    Output Shape         Param #     Connected to                     \n==================================================================================================\ninput_1 (InputLayer)            (None, 636, 128)     0                                            \n__________________________________________________________________________________________________\ncu_dnngru_1 (CuDNNGRU)          (None, 636, 128)     99072       input_1[0][0]                    \n__________________________________________________________________________________________________\nattention_1 (Attention)         (None, 128)          764         cu_dnngru_1[0][0]                \n__________________________________________________________________________________________________\nglobal_average_pooling1d_1 (Glo (None, 128)          0           cu_dnngru_1[0][0]                \n__________________________________________________________________________________________________\nglobal_max_pooling1d_1 (GlobalM (None, 128)          0           cu_dnngru_1[0][0]                \n__________________________________________________________________________________________________\nconcatenate_1 (Concatenate)     (None, 384)          0           attention_1[0][0]                \n                                                                 global_average_pooling1d_1[0][0] \n                                                                 global_max_pooling1d_1[0][0]     \n__________________________________________________________________________________________________\ndense_1 (Dense)                 (None, 80)           30800       concatenate_1[0][0]              \n==================================================================================================\nTotal params: 130,636\nTrainable params: 130,636\nNon-trainable params: 0\n__________________________________________________________________________________________________\n","name":"stdout"}]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.compile(loss='categorical_crossentropy',optimizer=Adam(0.005),metrics=['acc'])","execution_count":26,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_train, x_val, y_train, y_val = train_test_split(X, Y, test_size=0.2, random_state=123)","execution_count":28,"outputs":[]},{"metadata":{"_kg_hide-input":false,"trusted":true},"cell_type":"code","source":"del X, Y\ngc.collect()","execution_count":29,"outputs":[{"output_type":"execute_result","execution_count":29,"data":{"text/plain":"10"},"metadata":{}}]},{"metadata":{"_kg_hide-input":false,"trusted":true},"cell_type":"code","source":"assert len(x_train) == len(y_train)\nassert len(x_val) == len(y_val)","execution_count":30,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"es = EarlyStopping(monitor='val_acc', mode='max', verbose=1, patience=3)","execution_count":31,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.fit(np.array(x_train),\n          y_train,\n          batch_size=1024,\n          epochs=100,\n          validation_data=(np.array(x_val), y_val),\n          callbacks = [es])","execution_count":32,"outputs":[{"output_type":"stream","text":"WARNING:tensorflow:From /opt/conda/lib/python3.6/site-packages/tensorflow/python/ops/math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\nInstructions for updating:\nUse tf.cast instead.\nWARNING:tensorflow:From /opt/conda/lib/python3.6/site-packages/tensorflow/python/ops/math_grad.py:102: div (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\nInstructions for updating:\nDeprecated in favor of operator or tf.math.divide.\nTrain on 3976 samples, validate on 994 samples\nEpoch 1/100\n3976/3976 [==============================] - 5s 1ms/step - loss: 5.3089 - acc: 0.0236 - val_loss: 4.9422 - val_acc: 0.0412\nEpoch 2/100\n3976/3976 [==============================] - 3s 753us/step - loss: 4.8055 - acc: 0.0604 - val_loss: 4.6250 - val_acc: 0.0905\nEpoch 3/100\n3976/3976 [==============================] - 3s 750us/step - loss: 4.4840 - acc: 0.1016 - val_loss: 4.4143 - val_acc: 0.0845\nEpoch 4/100\n3976/3976 [==============================] - 3s 762us/step - loss: 4.2439 - acc: 0.1426 - val_loss: 4.2101 - val_acc: 0.1519\nEpoch 5/100\n3976/3976 [==============================] - 3s 757us/step - loss: 4.0277 - acc: 0.1793 - val_loss: 4.0830 - val_acc: 0.1348\nEpoch 6/100\n3976/3976 [==============================] - 3s 762us/step - loss: 3.8401 - acc: 0.1939 - val_loss: 3.9440 - val_acc: 0.1821\nEpoch 7/100\n3976/3976 [==============================] - 3s 745us/step - loss: 3.6918 - acc: 0.2301 - val_loss: 3.8487 - val_acc: 0.1761\nEpoch 8/100\n3976/3976 [==============================] - 3s 746us/step - loss: 3.5450 - acc: 0.2591 - val_loss: 3.7771 - val_acc: 0.1891\nEpoch 9/100\n3976/3976 [==============================] - 3s 767us/step - loss: 3.4421 - acc: 0.2676 - val_loss: 3.6611 - val_acc: 0.2123\nEpoch 10/100\n3976/3976 [==============================] - 3s 755us/step - loss: 3.3250 - acc: 0.3053 - val_loss: 3.5905 - val_acc: 0.2274\nEpoch 11/100\n3976/3976 [==============================] - 3s 754us/step - loss: 3.2466 - acc: 0.3194 - val_loss: 3.6028 - val_acc: 0.2233\nEpoch 12/100\n3976/3976 [==============================] - 3s 759us/step - loss: 3.1732 - acc: 0.3224 - val_loss: 3.5413 - val_acc: 0.2505\nEpoch 13/100\n3976/3976 [==============================] - 3s 764us/step - loss: 3.1102 - acc: 0.3405 - val_loss: 3.4830 - val_acc: 0.2495\nEpoch 14/100\n3976/3976 [==============================] - 3s 751us/step - loss: 3.0119 - acc: 0.3473 - val_loss: 3.4151 - val_acc: 0.2565\nEpoch 15/100\n3976/3976 [==============================] - 3s 759us/step - loss: 2.9204 - acc: 0.3715 - val_loss: 3.3752 - val_acc: 0.2636\nEpoch 16/100\n3976/3976 [==============================] - 3s 759us/step - loss: 2.8675 - acc: 0.3893 - val_loss: 3.3436 - val_acc: 0.2736\nEpoch 17/100\n3976/3976 [==============================] - 3s 754us/step - loss: 2.7998 - acc: 0.4064 - val_loss: 3.3233 - val_acc: 0.2777\nEpoch 18/100\n3976/3976 [==============================] - 3s 755us/step - loss: 2.7561 - acc: 0.4087 - val_loss: 3.2935 - val_acc: 0.2797\nEpoch 19/100\n3976/3976 [==============================] - 3s 759us/step - loss: 2.6771 - acc: 0.4283 - val_loss: 3.2498 - val_acc: 0.2998\nEpoch 20/100\n3976/3976 [==============================] - 3s 758us/step - loss: 2.6358 - acc: 0.4328 - val_loss: 3.2716 - val_acc: 0.2918\nEpoch 21/100\n3976/3976 [==============================] - 3s 755us/step - loss: 2.5930 - acc: 0.4369 - val_loss: 3.2282 - val_acc: 0.2978\nEpoch 22/100\n3976/3976 [==============================] - 3s 757us/step - loss: 2.5396 - acc: 0.4575 - val_loss: 3.1902 - val_acc: 0.3199\nEpoch 23/100\n3976/3976 [==============================] - 3s 754us/step - loss: 2.5054 - acc: 0.4577 - val_loss: 3.2127 - val_acc: 0.3189\nEpoch 24/100\n3976/3976 [==============================] - 3s 750us/step - loss: 2.4809 - acc: 0.4706 - val_loss: 3.1753 - val_acc: 0.3099\nEpoch 25/100\n3976/3976 [==============================] - 3s 749us/step - loss: 2.4519 - acc: 0.4718 - val_loss: 3.1481 - val_acc: 0.3229\nEpoch 26/100\n3976/3976 [==============================] - 3s 761us/step - loss: 2.3930 - acc: 0.4854 - val_loss: 3.1193 - val_acc: 0.3129\nEpoch 27/100\n3976/3976 [==============================] - 3s 766us/step - loss: 2.3530 - acc: 0.4945 - val_loss: 3.1153 - val_acc: 0.3491\nEpoch 28/100\n3976/3976 [==============================] - 3s 771us/step - loss: 2.3040 - acc: 0.4940 - val_loss: 3.0758 - val_acc: 0.3380\nEpoch 29/100\n3976/3976 [==============================] - 3s 757us/step - loss: 2.2587 - acc: 0.5166 - val_loss: 3.0539 - val_acc: 0.3310\nEpoch 30/100\n3976/3976 [==============================] - 3s 756us/step - loss: 2.2227 - acc: 0.5113 - val_loss: 3.0929 - val_acc: 0.3330\nEpoch 00030: early stopping\n","name":"stdout"},{"output_type":"execute_result","execution_count":32,"data":{"text/plain":"<keras.callbacks.History at 0x7fef39e37f98>"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_train_pred = model.predict(np.array(x_train))\ny_val_pred = model.predict(np.array(x_val))","execution_count":33,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_lwlrap = calculate_overall_lwlrap_sklearn(y_train, y_train_pred)\nval_lwlrap = calculate_overall_lwlrap_sklearn(y_val, y_val_pred)\n\nprint(f'Training LWLRAP : {train_lwlrap:.4f}')\nprint(f'Validation LWLRAP : {val_lwlrap:.4f}')","execution_count":34,"outputs":[{"output_type":"stream","text":"Training LWLRAP : 0.6781\nValidation LWLRAP : 0.4999\n","name":"stdout"}]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nX_test = np.array(convert_wav_to_image(test, source=[test_path]))","execution_count":35,"outputs":[{"output_type":"display_data","data":{"text/plain":"HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"0aa9e32b2611441f88dbdef5f3d4ed66"}},"metadata":{}},{"output_type":"stream","text":"\nCPU times: user 1min 27s, sys: 1min 3s, total: 2min 30s\nWall time: 1min 20s\n","name":"stdout"}]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions = model.predict(np.array(X_test))","execution_count":36,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test[label_columns] = predictions","execution_count":37,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.to_csv('submission.csv', index=False)","execution_count":38,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}