{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[],"dockerImageVersionId":28755,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np, pandas as pd, os, tensorflow as tf\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras import layers, models\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.preprocessing import LabelEncoder\n\nTRAIN_DIR, TEST_DIR = '/kaggle/input/datasets/kelbycraft/kwc-freesound-2018-melspec-256/spectrograms/train', '/kaggle/input/datasets/kelbycraft/kwc-freesound-2018-melspec-256/spectrograms/test'\nIMG_H, IMG_W = 256, 384\nBATCH, NUM_CLASSES = 32, 41\n\n# build a dataframe of all training images: filepath + label\nrows = []\nfor lbl in sorted(os.listdir(TRAIN_DIR)):\n    d = f'{TRAIN_DIR}/{lbl}'\n    if os.path.isdir(d):\n        for fn in os.listdir(d):\n            rows.append({'path': f'{d}/{fn}', 'label': lbl, 'fname': fn})\ndf = pd.DataFrame(rows)\nprint('total train images:', len(df))\n\nle = LabelEncoder()\ndf['y'] = le.fit_transform(df['label'])\n\ndef mapk_from_probs(probs, true_idx, k=3):\n    top_k = np.argsort(-probs, axis=1)[:, :k]\n    s = 0.0\n    for i, t in enumerate(true_idx):\n        h = np.where(top_k[i] == t)[0]\n        if len(h): s += 1.0/(h[0]+1)\n    return s/len(true_idx)\n\ndef build_cnn():\n    m = models.Sequential([\n        layers.Input((IMG_H, IMG_W, 3)),\n        layers.Conv2D(32,3,activation='relu',padding='same'), layers.BatchNormalization(), layers.MaxPooling2D(),\n        layers.Conv2D(64,3,activation='relu',padding='same'), layers.BatchNormalization(), layers.MaxPooling2D(),\n        layers.Conv2D(128,3,activation='relu',padding='same'), layers.BatchNormalization(), layers.MaxPooling2D(),\n        layers.Conv2D(128,3,activation='relu',padding='same'), layers.BatchNormalization(), layers.MaxPooling2D(),\n        layers.GlobalAveragePooling2D(),\n        layers.Dropout(0.3), layers.Dense(256, activation='relu'),\n        layers.Dropout(0.3), layers.Dense(NUM_CLASSES, activation='softmax'),\n    ])\n    m.compile('adam','categorical_crossentropy',metrics=['accuracy'])\n    return m\n\n# IMPORTANT: identical fold split to the YAMNet notebook\nskf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)\n\n# test generator (fixed, reused each fold)\ntest_gen = ImageDataGenerator(rescale=1./255).flow_from_dataframe(\n    pd.DataFrame({'path':[f'{TEST_DIR}/all/{f}' for f in sorted(os.listdir(f\"{TEST_DIR}/all\"))]}),\n    x_col='path', y_col=None, target_size=(IMG_H,IMG_W), batch_size=BATCH,\n    class_mode=None, shuffle=False)\ntest_fnames = [os.path.basename(p).replace('.png','.wav') for p in test_gen.filenames]\n\noof_probs = np.zeros((len(df), NUM_CLASSES))\ntest_probs_folds = np.zeros((5, len(test_fnames), NUM_CLASSES))\nfold_scores = []\n\nfor fold, (tr_idx, va_idx) in enumerate(skf.split(df['path'], df['y'])):\n    tr_df, va_df = df.iloc[tr_idx], df.iloc[va_idx]\n\n    tr_gen = ImageDataGenerator(rescale=1./255).flow_from_dataframe(\n        tr_df, x_col='path', y_col='label', target_size=(IMG_H,IMG_W),\n        batch_size=BATCH, class_mode='categorical', classes=list(le.classes_),\n        shuffle=True, seed=42)\n    va_gen = ImageDataGenerator(rescale=1./255).flow_from_dataframe(\n        va_df, x_col='path', y_col='label', target_size=(IMG_H,IMG_W),\n        batch_size=BATCH, class_mode='categorical', classes=list(le.classes_),\n        shuffle=False)\n\n    model = build_cnn()\n    model.fit(tr_gen, validation_data=va_gen, epochs=30,\n              callbacks=[tf.keras.callbacks.EarlyStopping(patience=6, restore_best_weights=True),\n                         tf.keras.callbacks.ReduceLROnPlateau(patience=2, factor=0.5)],\n              verbose=2)\n\n    va_probs = model.predict(va_gen, verbose=0)\n    oof_probs[va_idx] = va_probs\n    test_probs_folds[fold] = model.predict(test_gen, verbose=0)\n    score = mapk_from_probs(va_probs, va_df['y'].values)\n    fold_scores.append(score)\n    model.save(f'/kaggle/working/cnn_256mel_fold{fold}.keras')\n    print(f'fold {fold}: MAP@3 = {score:.4f}')\n\nfold_scores = np.array(fold_scores)\nprint(f'\\nCNN-256mel 5-fold MAP@3: {fold_scores.mean():.4f} ± {fold_scores.std():.4f}')\nprint(f'Overall OOF MAP@3: {mapk_from_probs(oof_probs, df[\"y\"].values):.4f}')\n\nnp.save('/kaggle/working/cnn_oof_probs.npy', oof_probs)\nnp.save('/kaggle/working/cnn_test_probs.npy', test_probs_folds.mean(axis=0))\ndf[['fname','label','y']].to_csv('/kaggle/working/cnn_train_order.csv', index=False)\nprint('artifacts saved')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null}]}