{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport glob\nimport os\nimport gc\nimport time\nfrom scipy.interpolate import interp1d\nimport lightgbm as lgb\nimport xgboost as xgb\nfrom joblib import Parallel, delayed\nfrom tqdm.notebook import tqdm\nfrom scipy.stats import rankdata\n\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.model_selection import GroupKFold\nfrom sklearn.metrics import roc_auc_score, label_ranking_average_precision_score\n\nimport soundfile as sf\n# Librosa Libraries\nimport librosa\nimport librosa.display\nimport IPython.display as ipd\nimport matplotlib.pyplot as plt","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"trainfiles = glob.glob( '../input/rfcx-species-audio-detection/train/*.flac' )\ntestfiles = glob.glob( '../input/rfcx-species-audio-detection/test/*.flac' )\nlen(trainfiles), len(testfiles), trainfiles[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"traint = pd.read_csv( '../input/rfcx-species-audio-detection/train_tp.csv' )\ntrainf = pd.read_csv( '../input/rfcx-species-audio-detection/train_fp.csv' )\ntraint.shape, trainf.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"traint.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"trainf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def extract_fft(fn):\n    data, samplerate = sf.read(fn)\n    data = np.array(data)\n\n    varfft = np.abs( np.fft.fft(data)[:(len(data)//2)] )\n    return np.array( varfft.reshape( (1000,1440) ).mean(axis=1) )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"FT = []\nfor fn in tqdm(traint.recording_id.values):\n    FT.append( extract_fft( '../input/rfcx-species-audio-detection/train/'+fn+'.flac' ) )\nFT = np.stack(FT)\ngc.collect()\n\nFT.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# This loop runs in 7min using cupy(GPU) and 40min on numpy(CPU). ~7x Faster in GPU\n\nFF = []\nfor fn in tqdm(trainf.recording_id.values):\n    FF.append( extract_fft( '../input/rfcx-species-audio-detection/train/'+fn+'.flac' ) )\nFF = np.stack(FF)\ngc.collect()\n\nFF.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Combine True Positives and False Positives\n\nTRAIN = np.vstack( (FT, FF) )\n\ndel FT, FF\ngc.collect()\nTRAIN.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"TEST = []\nfor fn in tqdm(testfiles):\n    TEST.append( extract_fft(fn) )\nTEST = np.stack(TEST)\ngc.collect()\n\nTEST.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tt = traint[['recording_id','species_id']].copy()\ntf = trainf[['recording_id','species_id']].copy()\ntf['species_id'] = -1\n\nTRAIN_TAB = pd.concat( (tt, tf) )\n\nfor i in range(24):\n    TRAIN_TAB['s'+str(i)] = 0\n    TRAIN_TAB.loc[TRAIN_TAB.species_id==i,'s'+str(i)] = 1\n\nTRAIN_TAB.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler\n\nstd = StandardScaler()\nstd.fit( np.vstack((TRAIN,TEST)) )\n\nTRAIN = std.transform(TRAIN)\nTEST  = std.transform(TEST)\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.ensemble import RandomForestClassifier as RF\nsub = pd.DataFrame({'recording_id': [f.split('/')[-1].split('.')[0] for f in testfiles] })\ngkf = GroupKFold(5)\nAUC_scores = []\n\ngroups = TRAIN_TAB['recording_id'].values\nfor tgt in range(24):\n    target = TRAIN_TAB['s'+str(tgt)].values\n\n    ytrain = np.zeros(TRAIN.shape[0])\n    ytest = np.zeros(TEST.shape[0])\n    for ind_train, ind_valid in gkf.split( TRAIN, target, groups ):\n        model = RF(max_depth = 25,n_estimators = 6000,class_weight =\"balanced\",random_state=0,n_jobs=-1)\n        model.fit( TRAIN[ind_train], target[ind_train] )\n        \n        ytrain[ind_valid] = model.predict_proba(TRAIN[ind_valid])[:,1]\n        ytest += model.predict_proba(TEST)[:,1] / 5.\n    AUC_scores.append(roc_auc_score(target, ytrain))\n\n    print( 'Target AUC', tgt, roc_auc_score(target, ytrain) )\n    \n    TRAIN_TAB['y'+str(tgt)] = ytrain\n    sub['s'+str(tgt)] = ytest\nprint(np.mean(AUC_scores)) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"perd = TRAIN_TAB[TRAIN_TAB.columns[26:]]\ntrue = TRAIN_TAB[TRAIN_TAB.columns[2:26]]\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"label_ranking_average_precision_score(true.values,perd)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}