{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Feature engineering"},{"metadata":{},"cell_type":"markdown","source":"## The feature engineering stage (with vectorization implementation) was added. Vectorization will help increase the calculation speed to get more features."},{"metadata":{},"cell_type":"markdown","source":"## This notebook is based in Giba's Notebook \n\n### [\"Tabular XGboost GPU + FFT GPU + Cuml = FAST\"](https://www.kaggle.com/titericz/0-525-tabular-xgboost-gpu-fft-gpu-cuml-fast)"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport glob\nimport os\nimport gc\nimport time\nfrom scipy.interpolate import interp1d\nimport lightgbm as lgb\nimport xgboost as xgb\nfrom joblib import Parallel, delayed\nfrom tqdm.notebook import tqdm\nfrom scipy.stats import rankdata\nimport IPython.display as ipd  # To play sound in the notebook\n\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.model_selection import GroupKFold\nfrom sklearn.metrics import roc_auc_score, label_ranking_average_precision_score\nimport soundfile as sf\nimport seaborn as sns\n\n# Librosa Libraries\nimport librosa\nimport librosa.display\nimport IPython.display as ipd\nimport matplotlib.pyplot as plt\n\nimport cuml as cm\nimport cupy as cp\nimport pickle\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"trainfiles = glob.glob( '../input/rfcx-species-audio-detection/train/*.flac' )\ntestfiles = glob.glob( '../input/rfcx-species-audio-detection/test/*.flac' )\nlen(trainfiles), len(testfiles), trainfiles[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"traint = pd.read_csv( '../input/rfcx-species-audio-detection/train_tp.csv' )\ntrainf = pd.read_csv( '../input/rfcx-species-audio-detection/train_fp.csv' )\ntraint.shape, trainf.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"traint.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Number of species\nlen(traint['species_id'].unique())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"trainf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"trainf.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Extra information\ntrain_general = pd.concat([traint, trainf])\ntrain_general['t_diff'] = train_general['t_max'] - train_general['t_min']\ntrain_general['f_diff'] = train_general['f_max'] - train_general['f_min']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_general.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Species\nsns.countplot(train_general['species_id'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Frequency domain\ndef figurecpFTT(data,samplerate):\n    # Frequency domain representation\n    data = cp.array(data)\n    fourierTransform = cp.fft.fft(data)/len(data)           # Normalize amplitude\n    fourierTransform = fourierTransform[:len(data)//2] # Exclude sampling frequency\n\n    tpCount     = len(data)\n    values      = cp.arange(int(tpCount/2))\n    timePeriod  = tpCount/samplerate\n    frequencies = cp.asnumpy(values/timePeriod)\n    \n    absFFT = cp.asnumpy(abs(fourierTransform)) \n#     print(frequencies)\n    plt.plot(frequencies,absFFT)\n        ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def frequeciesVec(data,samplerate):\n    tpCount     = 2*len(data)\n    values      = cp.arange(int(tpCount/2))\n    timePeriod  = tpCount/samplerate\n    frequencies = cp.asnumpy(values/timePeriod)\n    return cp.array(frequencies)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#FFT\nfilesound = trainfiles[0]\ndata, samplerate = sf.read(filesound)\nfigurecpFTT(data, samplerate)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Feature engineering using Vectorization Implementation"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Vectorization\n\ndef meanF(x): \n    return x.mean(axis=1)\n\ndef varianceF(x):\n    return x.var(axis=1)\n\ndef skewnessF(x):\n    skw = 3 * (x.mean(axis=1) - x[:,x.shape[1]/2])\n    skw = skw / x.std(axis=1)\n    return skw\n\ndef kurtosisF(x):\n    z = ((x - x.mean(axis=1,keepdims=True))**4).sum(axis=1)\n    n = x.shape[1]\n    s = n*(x.std(axis=1))**4\n    kur = z/s\n    return kur\n\ndef totalpowerF(x):\n    return (x**2).sum(axis=1)\n\ndef rmsF(x):\n    x = x**2\n    return cp.sqrt(x.mean(axis=1))\n\ndef stdF(x):\n    return x.std(axis=1)\n\ndef centroidF(x,frequencies):     \n    n = x * frequencies\n    s = x.sum(axis=1)    \n    centroid = n / s[:,None]\n    return centroid.sum(axis=1)\n\ndef entropyF(x):\n    px = x / (x.sum(axis=1))[:,None]\n    r = px*cp.log2(px)\n    return -r.sum(axis=1)\n\ndef peakF(x):    \n    return x.max(axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def featuresextractionVec(signalFFT):\n    frequecies = frequeciesVec(signalFFT,samplerate).reshape( (1000,1440) )\n    varfft = signalFFT.reshape( (1000,1440) )\n    features = cp.array([meanF(varfft), varianceF(varfft), skewnessF(varfft), kurtosisF(varfft), totalpowerF(varfft), stdF(varfft), rmsF(varfft), entropyF(varfft), peakF(varfft), centroidF(varfft,frequecies) ])\n    L=features.shape[0]*features.shape[1]\n    features = features.reshape(1,L)[0]\n    return features","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def extract_fft(fn):\n    data, samplerate = sf.read(fn)\n    data = cp.array(data)    \n    varfft = cp.abs( cp.fft.fft(data)[:(len(data)//2)] )\n    features = featuresextractionVec(varfft)\n    return features","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"FT = []\nfor fn in tqdm(traint.recording_id.values):\n    FT.append( extract_fft( '../input/rfcx-species-audio-detection/train/'+fn+'.flac') )\nFT = np.stack(FT)\ngc.collect()\n\nFT.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# This loop runs in 7min using cupy(GPU) and 40min on numpy(CPU). ~7x Faster in GPU\n\nFF = []\nfor fn in tqdm(trainf.recording_id.values):\n    FF.append( extract_fft( '../input/rfcx-species-audio-detection/train/'+fn+'.flac' ) )\nFF = np.stack(FF)\ngc.collect()\n\nFF.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Combine True Positives and False Positives\n\nTRAIN = np.vstack( (FT, FF) )\n\n\ndel FT, FF\ngc.collect()\nTRAIN.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"TEST = []\nfor fn in tqdm(testfiles):\n    TEST.append( extract_fft(fn) )\nTEST = np.stack(TEST)\ngc.collect()\n\nTEST.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#To Numpy format\nTRAIN = cp.asnumpy(TRAIN)\nTEST = cp.asnumpy(TEST)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tt = traint[['recording_id','species_id']].copy()\ntf = trainf[['recording_id','species_id']].copy()\ntf['species_id'] = -1\n\nTRAIN_TAB = pd.concat( (tt, tf) )\n\nfor i in range(24):\n    TRAIN_TAB['s'+str(i)] = 0\n    TRAIN_TAB.loc[TRAIN_TAB.species_id==i,'s'+str(i)] = 1\n\nTRAIN_TAB.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def saveFile(data, name):\n    pickle_out = open(name,\"wb\")\n    pickle.dump(data, pickle_out)\n    pickle_out.close() ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Save TRAIN, TEST, TRAIN_TAB\nsaveFile(TRAIN,\"TRAIN.pickle\")\nsaveFile(TEST,\"TEST.pickle\")\nsaveFile(TRAIN_TAB,\"TRAIN_TAB.pickle\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # To Open\n# pickle_in = open(\"TRAIN.pickle\",\"rb\")\n# TRAIN = pickle.load(pickle_in)\n# pickle_in = open(\"TEST.pickle\",\"rb\")\n# TEST = pickle.load(pickle_in)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#1000 random features was selected to avoid a long training time\n\nimport random\n\nrandom.seed(30)\nimp_indx = random.sample(range(0, 10000), 1000) \nTRAIN = TRAIN[:,imp_indx]\nTEST = TEST[:,imp_indx]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"TRAIN.shape, TEST.shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Modeling"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler\n\nstd = StandardScaler()\nstd.fit( np.vstack((TRAIN,TEST)) )\n\nTRAIN = std.transform(TRAIN)\nTEST  = std.transform(TEST)\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"TRAIN_TAB.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub = pd.DataFrame({'recording_id': [f.split('/')[-1].split('.')[0] for f in testfiles] })\ngkf = GroupKFold(5)\n\nSCORE = []\ngroups = TRAIN_TAB['recording_id'].values\nfor tgt in range(0,24):\n    starttime = time.time()\n    target = TRAIN_TAB['s'+str(tgt)].values\n\n    ytrain = np.zeros(TRAIN.shape[0])\n    ytest = np.zeros(TEST.shape[0])\n    for ind_train, ind_valid in gkf.split( TRAIN, target, groups ):\n        \n        # Define 4 models\n        model1 = xgb.XGBClassifier(n_estimators=1000,\n                                   max_depth=6,\n                                   learning_rate=0.09,\n                                   verbosity=0,\n                                   min_child_weight=1,\n                                   objective='binary:logistic',\n                                   subsample=0.95,\n                                   colsample_bytree=0.95,\n                                   random_state=2021,\n                                   tree_method='gpu_hist',\n                                   predictor='gpu_predictor',\n                                   n_jobs=2,\n                                   scale_pos_weight =  np.sum(target==0) / np.sum(target==1),\n                                  )\n#         scale_pos_weight = np.sum(target==0) / np.sum(target==1)\n        model2 = cm.linear_model.LogisticRegression( C=1, max_iter=5000 )\n        model3 = cm.svm.SVC(C=1.0, class_weight='balanced', probability=True, kernel='rbf', gamma='auto')\n        model4 = cm.neighbors.KNeighborsClassifier(n_neighbors=10)\n        \n        # Train using GPUs\n        model1.fit( X=TRAIN[ind_train], y=target[ind_train], eval_set=[(TRAIN[ind_valid], target[ind_valid])], eval_metric='auc', early_stopping_rounds=30, verbose=False )\n        model2.fit( TRAIN[ind_train], target[ind_train] )\n        model3.fit( TRAIN[ind_train], target[ind_train] )\n        model4.fit( TRAIN[ind_train], target[ind_train] )\n        \n        # Predict valid and test sets\n        yvalid1 = model1.predict_proba(TRAIN[ind_valid])[:,1]\n        yvalid2 = model2.predict_proba(TRAIN[ind_valid])[:,1]\n        yvalid3 = model3.predict_proba(TRAIN[ind_valid])[:,1]\n        yvalid4 = model4.predict_proba(TRAIN[ind_valid])[:,1]\n        ytest1 = model1.predict_proba(TEST)[:,1]\n        ytest2 = model2.predict_proba(TEST)[:,1]\n        ytest3 = model3.predict_proba(TEST)[:,1]\n        ytest4 = model4.predict_proba(TEST)[:,1]\n        \n        #Rank predictions\n        SZ = len(ind_valid) + len(ytest1)\n        yvalid1 = rankdata( np.concatenate((yvalid1,ytest1)) )[:len(ind_valid)] / SZ\n        yvalid2 = rankdata( np.concatenate((yvalid2,ytest2)) )[:len(ind_valid)] / SZ\n        yvalid3 = rankdata( np.concatenate((yvalid3,ytest3)) )[:len(ind_valid)] / SZ\n        yvalid4 = rankdata( np.concatenate((yvalid4,ytest4)) )[:len(ind_valid)] / SZ\n        ytest1 = rankdata( np.concatenate((yvalid1,ytest1)) )[len(ind_valid):] / SZ\n        ytest2 = rankdata( np.concatenate((yvalid2,ytest2)) )[len(ind_valid):] / SZ\n        ytest3 = rankdata( np.concatenate((yvalid3,ytest3)) )[len(ind_valid):] / SZ\n        ytest4 = rankdata( np.concatenate((yvalid4,ytest4)) )[len(ind_valid):] / SZ\n        \n        #Weighted average models\n        ytrain[ind_valid] = (0.40*yvalid1+0.20*yvalid2+0.20*yvalid3+0.20*yvalid4) / 4.\n        ytest += (0.40*ytest1+0.20*ytest2+0.20*ytest3+0.20*ytest4) / (4.*5)\n\n    score = roc_auc_score(target, ytrain)\n    print( 'Target AUC', tgt, score, time.time()-starttime )\n    SCORE.append(score)\n    \n    TRAIN_TAB['y'+str(tgt)] = ytrain\n    sub['s'+str(tgt)] = ytest\n\nprint('Overall Score:', np.mean(SCORE) )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub.to_csv('submission_vec.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Thanks for sharing!"}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}