{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Now Kaggle have RAPIDS runing natively in notebooks. Lets take advantage on it ;) ","metadata":{}},{"cell_type":"markdown","source":"# To enable RAPIDS just set Notebook Environment to \"latest\" and enable GPU","metadata":{}},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport glob\nimport os\nimport gc\nimport time\nfrom scipy.interpolate import interp1d\nimport lightgbm as lgb\nimport xgboost as xgb\nfrom joblib import Parallel, delayed\nfrom tqdm.notebook import tqdm\nfrom scipy.stats import rankdata\n\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.model_selection import GroupKFold\nfrom sklearn.metrics import roc_auc_score, label_ranking_average_precision_score\n\nimport soundfile as sf\n# Librosa Libraries\nimport librosa\nimport librosa.display\nimport IPython.display as ipd\nimport matplotlib.pyplot as plt\n\nimport cuml as cm\nimport cupy as cp","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-05-09T18:48:41.082833Z","iopub.execute_input":"2023-05-09T18:48:41.083528Z","iopub.status.idle":"2023-05-09T18:48:41.089573Z","shell.execute_reply.started":"2023-05-09T18:48:41.083494Z","shell.execute_reply":"2023-05-09T18:48:41.088701Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainfiles = glob.glob( '../input/rfcx-species-audio-detection/train/*.flac' )\ntestfiles = glob.glob( '../input/rfcx-species-audio-detection/test/*.flac' )\nlen(trainfiles), len(testfiles), trainfiles[0]","metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","execution":{"iopub.status.busy":"2023-05-09T18:37:42.758816Z","iopub.execute_input":"2023-05-09T18:37:42.759159Z","iopub.status.idle":"2023-05-09T18:37:43.537391Z","shell.execute_reply.started":"2023-05-09T18:37:42.759127Z","shell.execute_reply":"2023-05-09T18:37:43.536268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"traint = pd.read_csv( '../input/rfcx-species-audio-detection/train_tp.csv' )\ntrainf = pd.read_csv( '../input/rfcx-species-audio-detection/train_fp.csv' )\ntraint.shape, trainf.shape","metadata":{"execution":{"iopub.status.busy":"2023-05-09T18:37:43.539295Z","iopub.execute_input":"2023-05-09T18:37:43.539684Z","iopub.status.idle":"2023-05-09T18:37:43.573936Z","shell.execute_reply.started":"2023-05-09T18:37:43.539647Z","shell.execute_reply":"2023-05-09T18:37:43.573118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"traint.head()","metadata":{"execution":{"iopub.status.busy":"2023-05-09T18:37:43.57633Z","iopub.execute_input":"2023-05-09T18:37:43.576883Z","iopub.status.idle":"2023-05-09T18:37:43.604591Z","shell.execute_reply.started":"2023-05-09T18:37:43.576852Z","shell.execute_reply":"2023-05-09T18:37:43.601664Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainf.head()","metadata":{"execution":{"iopub.status.busy":"2023-05-09T18:37:43.606958Z","iopub.execute_input":"2023-05-09T18:37:43.607919Z","iopub.status.idle":"2023-05-09T18:37:43.632679Z","shell.execute_reply.started":"2023-05-09T18:37:43.607862Z","shell.execute_reply":"2023-05-09T18:37:43.631807Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def extract_fft(fn):\n    data, samplerate = sf.read(fn)\n    data = cp.array(data)\n\n    varfft = cp.abs( cp.fft.fft(data)[:(len(data)//2)] )\n    \n    return cp.asnumpy( varfft.reshape( (1000,1440) ).mean(axis=1) )","metadata":{"execution":{"iopub.status.busy":"2023-05-09T18:37:43.634264Z","iopub.execute_input":"2023-05-09T18:37:43.634975Z","iopub.status.idle":"2023-05-09T18:37:43.642318Z","shell.execute_reply.started":"2023-05-09T18:37:43.634941Z","shell.execute_reply":"2023-05-09T18:37:43.640635Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"FT = []\nfor fn in tqdm(traint.recording_id.values):\n    FT.append( extract_fft( '../input/rfcx-species-audio-detection/train/'+fn+'.flac' ) )\nFT = np.stack(FT)\ngc.collect()\n\nFT.shape","metadata":{"execution":{"iopub.status.busy":"2023-05-09T18:48:50.774232Z","iopub.execute_input":"2023-05-09T18:48:50.774566Z","iopub.status.idle":"2023-05-09T18:48:55.006241Z","shell.execute_reply.started":"2023-05-09T18:48:50.774538Z","shell.execute_reply":"2023-05-09T18:48:55.004937Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# This loop runs in 7min using cupy(GPU) and 40min on numpy(CPU). ~7x Faster in GPU\n\nFF = []\nfor fn in tqdm(trainf.recording_id.values):\n    FF.append( extract_fft( '../input/rfcx-species-audio-detection/train/'+fn+'.flac' ) )\nFF = np.stack(FF)\ngc.collect()\n\nFF.shape","metadata":{"execution":{"iopub.status.busy":"2023-05-09T18:49:04.802972Z","iopub.execute_input":"2023-05-09T18:49:04.803831Z","iopub.status.idle":"2023-05-09T18:49:06.211066Z","shell.execute_reply.started":"2023-05-09T18:49:04.803796Z","shell.execute_reply":"2023-05-09T18:49:06.209802Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Combine True Positives and False Positives\n\nTRAIN = np.vstack( (FT, FF) )\n\ndel FT, FF\ngc.collect()\nTRAIN.shape","metadata":{"execution":{"iopub.status.busy":"2023-05-09T18:49:21.679979Z","iopub.execute_input":"2023-05-09T18:49:21.680926Z","iopub.status.idle":"2023-05-09T18:49:21.864997Z","shell.execute_reply.started":"2023-05-09T18:49:21.68088Z","shell.execute_reply":"2023-05-09T18:49:21.864122Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TEST = []\nfor fn in tqdm(testfiles):\n    TEST.append( extract_fft(fn) )\nTEST = np.stack(TEST)\ngc.collect()\n\nTEST.shape","metadata":{"execution":{"iopub.status.busy":"2023-05-09T18:49:27.526505Z","iopub.execute_input":"2023-05-09T18:49:27.526895Z","iopub.status.idle":"2023-05-09T18:49:40.69537Z","shell.execute_reply.started":"2023-05-09T18:49:27.526864Z","shell.execute_reply":"2023-05-09T18:49:40.694001Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tt = traint[['recording_id','species_id']].copy()\ntf = trainf[['recording_id','species_id']].copy()\ntf['species_id'] = -1\n\nTRAIN_TAB = pd.concat( (tt, tf) )\n\nfor i in range(24):\n    TRAIN_TAB['s'+str(i)] = 0\n    TRAIN_TAB.loc[TRAIN_TAB.species_id==i,'s'+str(i)] = 1\n\nTRAIN_TAB.head()","metadata":{"execution":{"iopub.status.busy":"2023-05-09T18:37:57.845854Z","iopub.status.idle":"2023-05-09T18:37:57.847474Z","shell.execute_reply.started":"2023-05-09T18:37:57.847223Z","shell.execute_reply":"2023-05-09T18:37:57.847247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler\n\nstd = StandardScaler()\nstd.fit( np.vstack((TRAIN,TEST)) )\n\nTRAIN = std.transform(TRAIN)\nTEST  = std.transform(TEST)\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-05-09T18:37:57.848855Z","iopub.status.idle":"2023-05-09T18:37:57.849673Z","shell.execute_reply.started":"2023-05-09T18:37:57.849427Z","shell.execute_reply":"2023-05-09T18:37:57.849451Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub = pd.DataFrame({'recording_id': [f.split('/')[-1].split('.')[0] for f in testfiles] })\ngkf = GroupKFold(5)\n\nSCORE = []\ngroups = TRAIN_TAB['recording_id'].values\nfor tgt in range(0,24):\n    starttime = time.time()\n    target = TRAIN_TAB['s'+str(tgt)].values\n\n    ytrain = np.zeros(TRAIN.shape[0])\n    ytest = np.zeros(TEST.shape[0])\n    for ind_train, ind_valid in gkf.split( TRAIN, target, groups ):\n        \n        # Define 4 models\n        model1 = xgb.XGBClassifier(n_estimators=1000,\n                                   max_depth=4,\n                                   learning_rate=0.05,\n                                   verbosity=0,\n                                   objective='binary:logistic',\n                                   subsample=0.95,\n                                   colsample_bytree=0.95,\n                                   random_state=2021,\n                                   tree_method='gpu_hist',\n                                   predictor='gpu_predictor',\n                                   n_jobs=2,\n                                   scale_pos_weight = np.sum(target==0) / np.sum(target==1),\n                                  )\n        model2 = cm.linear_model.LogisticRegression( C=1, max_iter=5000 )\n        model3 = cm.svm.SVC(C=1.0, class_weight='balanced', probability=True, kernel='rbf', gamma='auto')\n        model4 = cm.neighbors.KNeighborsClassifier(n_neighbors=55)\n        \n        # Train using GPUs\n        model1.fit( X=TRAIN[ind_train], y=target[ind_train], eval_set=[(TRAIN[ind_valid], target[ind_valid])], eval_metric='auc', early_stopping_rounds=30, verbose=False )\n        model2.fit( TRAIN[ind_train], target[ind_train] )\n        model3.fit( TRAIN[ind_train], target[ind_train] )\n        model4.fit( TRAIN[ind_train], target[ind_train] )\n        \n        # Predict valid and test sets\n        yvalid1 = model1.predict_proba(TRAIN[ind_valid])[:,1]\n        yvalid2 = model2.predict_proba(TRAIN[ind_valid])[:,1]\n        yvalid3 = model3.predict_proba(TRAIN[ind_valid])[:,1]\n        yvalid4 = model4.predict_proba(TRAIN[ind_valid])[:,1]\n        ytest1 = model1.predict_proba(TEST)[:,1]\n        ytest2 = model2.predict_proba(TEST)[:,1]\n        ytest3 = model3.predict_proba(TEST)[:,1]\n        ytest4 = model4.predict_proba(TEST)[:,1]\n        \n        #Rank predictions\n        SZ = len(ind_valid) + len(ytest1)\n        yvalid1 = rankdata( np.concatenate((yvalid1,ytest1)) )[:len(ind_valid)] / SZ\n        yvalid2 = rankdata( np.concatenate((yvalid2,ytest2)) )[:len(ind_valid)] / SZ\n        yvalid3 = rankdata( np.concatenate((yvalid3,ytest3)) )[:len(ind_valid)] / SZ\n        yvalid4 = rankdata( np.concatenate((yvalid4,ytest4)) )[:len(ind_valid)] / SZ\n        ytest1 = rankdata( np.concatenate((yvalid1,ytest1)) )[len(ind_valid):] / SZ\n        ytest2 = rankdata( np.concatenate((yvalid2,ytest2)) )[len(ind_valid):] / SZ\n        ytest3 = rankdata( np.concatenate((yvalid3,ytest3)) )[len(ind_valid):] / SZ\n        ytest4 = rankdata( np.concatenate((yvalid4,ytest4)) )[len(ind_valid):] / SZ\n        \n        #Weighted average models\n        ytrain[ind_valid] = (0.40*yvalid1+0.20*yvalid2+0.20*yvalid3+0.20*yvalid4) / 4.\n        ytest += (0.40*ytest1+0.20*ytest2+0.20*ytest3+0.20*ytest4) / (4.*5)\n\n    score = roc_auc_score(target, ytrain)\n    print( 'Target AUC', tgt, score, time.time()-starttime )\n    SCORE.append(score)\n    \n    TRAIN_TAB['y'+str(tgt)] = ytrain\n    sub['s'+str(tgt)] = ytest\n\nprint('Overall Score:', np.mean(SCORE) )","metadata":{"execution":{"iopub.status.busy":"2023-05-09T18:37:57.851057Z","iopub.status.idle":"2023-05-09T18:37:57.851925Z","shell.execute_reply.started":"2023-05-09T18:37:57.851666Z","shell.execute_reply":"2023-05-09T18:37:57.851691Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub.head()","metadata":{"execution":{"iopub.status.busy":"2023-05-09T18:37:57.853359Z","iopub.status.idle":"2023-05-09T18:37:57.854167Z","shell.execute_reply.started":"2023-05-09T18:37:57.853898Z","shell.execute_reply":"2023-05-09T18:37:57.853923Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2023-05-09T18:37:57.855513Z","iopub.status.idle":"2023-05-09T18:37:57.856314Z","shell.execute_reply.started":"2023-05-09T18:37:57.856043Z","shell.execute_reply":"2023-05-09T18:37:57.856066Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls","metadata":{"execution":{"iopub.status.busy":"2023-05-09T18:37:57.857673Z","iopub.status.idle":"2023-05-09T18:37:57.858484Z","shell.execute_reply.started":"2023-05-09T18:37:57.858237Z","shell.execute_reply":"2023-05-09T18:37:57.85826Z"},"trusted":true},"execution_count":null,"outputs":[]}]}