{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nimport os\n\nimport soundfile as sf  \nfrom tsfresh.feature_extraction import feature_calculators\nimport librosa\nimport pywt\n\nfrom glob import glob \nfrom joblib import Parallel, delayed\nfrom tqdm import tqdm_notebook\n\nimport gc","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"os.listdir('../input/rfcx-species-audio-detection/')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tp = pd.read_csv('../input/rfcx-species-audio-detection/train_tp.csv')\nfp = pd.read_csv('../input/rfcx-species-audio-detection/train_fp.csv')\ntrain = tp.append(fp)\ndel(tp,fp)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train_paths = glob('../input/rfcx-species-audio-detection/train/*')\ntest_paths = glob('../input/rfcx-species-audio-detection/test/*')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def audio_read(path):\n    data, samplerate = sf.read(path) \n    return data\n\ndef denoise_signal_simple(x, wavelet='db4', level=1):\n    coeff = pywt.wavedec(x, wavelet, mode=\"per\")\n    #univeral threshold\n    uthresh = 10\n    coeff[1:] = (pywt.threshold(i, value=uthresh, mode='hard') for i in coeff[1:])\n    # Reconstruct the signal using the thresholded coefficients\n    return pywt.waverec(coeff, wavelet, mode='per')\n\n\ndef feature_gen(path):\n    X = audio_read(path)\n    z = X - np.median(X,axis=0)\n    sig = z\n    den_sample_simple = denoise_signal_simple(sig)\n    mfcc = librosa.feature.mfcc(sig)\n    mfcc_mean = mfcc.mean(axis=1)\n    percentile_roll50_std_20 = np.percentile(pd.Series(sig).rolling(50).std().dropna().values, 20)\n    \n    return [feature_calculators.number_peaks(den_sample_simple, 2),percentile_roll50_std_20,mfcc_mean[18],mfcc_mean[4]]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n#  28min 23s\nmax_len = len(train_paths)\nX = pd.DataFrame(np.array([Parallel(n_jobs=4)(delayed(feature_gen)(filename) for filename in tqdm_notebook(train_paths[:max_len]))])[0])\nX['recording_id'] = pd.Series(train_paths).apply(lambda x: x.split('/')[-1].split('.')[0])\ntrain = train.merge(X,on='recording_id')\ndel(X,train_paths)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.preprocessing import OneHotEncoder\nOHE = OneHotEncoder(sparse=False)\nOHE.fit(train['species_id'].values.reshape(-1, 1))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y = train['species_id']\ntrain.drop(['recording_id','species_id','t_min','songtype_id','f_min','t_max','f_max'],axis=1,inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from catboost import CatBoostClassifier\nfrom lightgbm import LGBMClassifier\nfrom sklearn.metrics import accuracy_score\ncat = LGBMClassifier(n_estimators=1000)\ncat.fit(train,y)\naccuracy_score(y,cat.predict(train))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub = pd.read_csv('../input/rfcx-species-audio-detection/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(test_paths)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test = pd.DataFrame(np.array([Parallel(n_jobs=4)(delayed(feature_gen)(filename) for filename in tqdm_notebook(test_paths))])[0])\npreds = pd.DataFrame(cat.predict(test))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# test case\n# sub.loc[:,'s0':'s23'] = OHE.transform(train['species_id'].values.reshape(-1, 1))[:sub.shape[0],:]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub.loc[:,'s0':'s23'] = OHE.transform(np.array(preds).flatten().reshape(-1, 1))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}