{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport glob\nimport os\nfrom scipy.interpolate import interp1d\nimport gc\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.model_selection import GroupKFold\n\nimport soundfile as sf\n# Librosa Libraries\nimport librosa\nimport librosa.display\nimport IPython.display as ipd\nimport matplotlib.pyplot as plt\nfrom tqdm.notebook import tqdm\n\nfrom sklearn.metrics import roc_auc_score, label_ranking_average_precision_score","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"trainfiles = glob.glob( '../input/rfcx-species-audio-detection/train/*.flac' )\ntestfiles = glob.glob( '../input/rfcx-species-audio-detection/test/*.flac' )\nlen(trainfiles), len(testfiles), trainfiles[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"traint = pd.read_csv( '../input/rfcx-species-audio-detection/train_tp.csv' )\ntraint['t_dif'] = traint['t_max'] - traint['t_min']\ntraint['f_dif'] = traint['f_max'] - traint['f_min']\n\ntrainf = pd.read_csv( '../input/rfcx-species-audio-detection/train_fp.csv' )\ntrainf['t_dif'] = trainf['t_max'] - trainf['t_min']\ntrainf['f_dif'] = trainf['f_max'] - trainf['f_min']\n\ntraint.shape, trainf.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"traint.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"trainf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"trainf.t_min.hist(bins=100)\ntrainf.t_max.hist(bins=100, alpha=0.5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"trainf.f_min.hist(bins=20)\ntrainf.f_max.hist(bins=20, alpha=0.5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"trainf.t_dif.hist(bins=100)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"trainf.f_dif.unique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"trainf.nunique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"traint.nunique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data, samplerate = sf.read(trainfiles[0]) \nprint( data.shape, samplerate )\nlibrosa.display.waveplot(y = data, sr = samplerate, color = \"#B14D\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"traint.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"trainf.describe()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Min and Max frequencies are: 93 and 10687"},{"metadata":{"trusted":true},"cell_type":"code","source":"TRAIN = []\nTARGET = []\nfor i in tqdm(range(traint.shape[0])):\n\n    fn = traint.recording_id.values[i]\n    tmin = traint.t_min.values[i]\n    tmax = traint.t_max.values[i]\n    fmin = traint.f_min.values[i]\n    fmax = traint.f_max.values[i]\n    #print(tmin,tmax, fmin,fmax )\n\n    data, samplerate = sf.read( '../input/rfcx-species-audio-detection/train/'+fn+'.flac')\n    #print( data.shape, samplerate )\n    var_time = np.arange(0,data.shape[0]) / samplerate\n\n    data = data[ np.where( (var_time>=tmin)&(var_time<=tmax) )[0] ]\n\n    varfft = np.abs( np.fft.fft(data)[:(len(data)//2)] )\n    x = np.linspace(0, len(varfft), num=len(varfft), endpoint=True)\n    f1 = interp1d(x, varfft, kind='cubic')\n    x = np.linspace(0, len(varfft), num=1000, endpoint=True)\n    varfft = f1(x)\n    \n    TRAIN.append( varfft )\n    TARGET.append( traint.species_id.values[i] )\n    \nFT = np.stack(TRAIN)\nTARGET = np.array(TARGET)\nFT.shape, len(TARGET)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"np.unique(TARGET, return_counts=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from joblib import Parallel, delayed\n\ndef extract_features( fn ):\n    data, samplerate = sf.read( '../input/rfcx-species-audio-detection/train/'+fn+'.flac')\n\n    varfft = np.abs( np.fft.fft(data)[:(len(data)//2)] )\n    x = np.linspace(0, len(varfft), num=len(varfft), endpoint=True)\n    f1 = interp1d(x, varfft, kind='cubic')\n    x = np.linspace(0, len(varfft), num=1000, endpoint=True)\n    varfft = f1(x)\n    \n    return varfft\n    \nFP = Parallel(n_jobs=4)(delayed(extract_features)(fn) for i in tqdm(trainf.recording_id.values))\nFP = np.stack(FP)\ngc.collect()\nFP.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def extract_features( fn ):\n    data, samplerate = sf.read(fn)\n\n    varfft = np.abs( np.fft.fft(data)[:(len(data)//2)] )\n    x = np.linspace(0, len(varfft), num=len(varfft), endpoint=True)\n    f1 = interp1d(x, varfft, kind='cubic')\n    x = np.linspace(0, len(varfft), num=1000, endpoint=True)\n    varfft = f1(x)\n    \n    return varfft\n    \nTEST = Parallel(n_jobs=4)(delayed(extract_features)(fn) for fn in tqdm(testfiles))\nTEST = np.stack(TEST)\ngc.collect()\nTEST.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"TRAIN = np.vstack( (FT, FP) )\nTRAIN.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tt = traint[['recording_id','species_id']].copy()\ntf = trainf[['recording_id','species_id']].copy()\n\ntf['species_id'] = -1\n\nTRAIN_TAB = pd.concat( (tt, tf) )\n\nfor i in range(24):\n    TRAIN_TAB['s'+str(i)] = 0\n    TRAIN_TAB.loc[TRAIN_TAB.species_id==i,'s'+str(i)] = 1\n\nTRAIN_TAB.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"TRAIN_TAB.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub = pd.DataFrame({'recording_id': [f.split('/')[-1].split('.')[0] for f in testfiles] })\ngkf = GroupKFold(5)\n\ngroups = TRAIN_TAB['recording_id'].values\nfor tgt in range(24):\n    target = TRAIN_TAB['s'+str(tgt)].values\n\n    ytrain = np.zeros(TRAIN.shape[0])\n    ytest = np.zeros(TEST.shape[0])\n    for ind_train, ind_valid in gkf.split( TRAIN, target, groups ):\n        model = LogisticRegression( C=1, max_iter=5000 )\n        model.fit( TRAIN[ind_train], target[ind_train] )\n        \n        ytrain[ind_valid] = model.predict_proba(TRAIN[ind_valid])[:,1]\n        ytest += model.predict_proba(TEST)[:,1] / 5.\n\n    print( 'Target AUC', tgt, roc_auc_score(target, ytrain) )\n    \n    TRAIN_TAB['y'+str(tgt)] = ytrain\n    sub['s'+str(tgt)] = ytest","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}