{"cells":[{"metadata":{},"cell_type":"markdown","source":"- Original notebook: https://www.kaggle.com/titericz/0-525-tabular-xgboost-gpu-fft-gpu-cuml-fast/\n- Changes to the original notebook\n        - Training samples more carefully selected\n        - One model instead of multiple models\n        - Hyperparameter fine tuned\n- This model does not use data in `TFRecord` format. No data in the `tfrecords` folder is used.\n- This model does not use `t_min, f_min, t_max, f_max` to extract relevant sections of the audio.\n- Because the testing data does not have `songtype_id, t_min, f_min, t_max, f_max`, also will not use them as plain XGBoost features"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import copy\nimport gc\nimport glob\nimport numpy as np\nimport pandas as pd\nimport time\n\nimport cupy as cp\nfrom matplotlib import pyplot\nfrom sklearn.model_selection import StratifiedKFold, RandomizedSearchCV\nimport soundfile as sf\nfrom tqdm.notebook import tqdm\nfrom xgboost.sklearn import XGBClassifier\nfrom xgboost import plot_importance","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"trainfiles = sorted(glob.glob( '../input/rfcx-species-audio-detection/train/*.flac' ))\ntestfiles = sorted(glob.glob( '../input/rfcx-species-audio-detection/test/*.flac' ))\nlen(trainfiles), len(testfiles), trainfiles[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"traint = pd.read_csv( '../input/rfcx-species-audio-detection/train_tp.csv' )\ntrainf = pd.read_csv( '../input/rfcx-species-audio-detection/train_fp.csv' )\ntraint.shape, trainf.shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Take a look at the True Positive species labels and False Positives species labels\n\n- This model does not use `t_min, f_min, t_max, f_max` to extract relevant sections of the audio.\n- Because the testing data does not have `songtype_id, t_min, f_min, t_max, f_max`, also will not use them as plain XGBoost features"},{"metadata":{"trusted":true},"cell_type":"code","source":"traint.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"trainf.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Fourier transform function\n\n- The reason to reshape to (1000, 1440) is to reduce the features. Instead of using 1440000 features, we average every 1440 features, decreasing the number of features to 1000.\n- In `varfft = cp.abs( cp.fft.fft(data)[:(len(data)//2)] )` we only get half of the length, because the rest half is just mirroring."},{"metadata":{"trusted":true},"cell_type":"code","source":"pyplot.rcParams[\"figure.figsize\"] = (20,10)\ndata, samplerate = sf.read('../input/rfcx-species-audio-detection/train/a66546dfd.flac')\ndata = cp.array(data)\nvarfft = cp.abs( cp.fft.fft(data)[:(len(data)//2)] )\nreshaped = cp.asnumpy( varfft.reshape( (1000,1440) ).mean(axis=1) )\n\nprint(f\"len(cp.fft.fft(data)): {len(cp.fft.fft(data))}\")\nprint(f\"len(data): {len(data)}\")\nprint(f\"len(data)//2: {len(data)//2}\")\nprint(f\"varfft.shape: {varfft.shape}\")\nprint(f\"reshaped.shape: {reshaped.shape}\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Before reshaping"},{"metadata":{"trusted":true},"cell_type":"code","source":"pyplot.plot(range(0, len(varfft)), cp.asnumpy(varfft))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"After reshaping"},{"metadata":{"trusted":true},"cell_type":"code","source":"pyplot.plot(range(0, len(reshaped)), cp.asnumpy(reshaped))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del data, varfft, reshaped","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def extract_fft(fn):\n    data, samplerate = sf.read(fn)\n    data = cp.array(data)\n    varfft = cp.abs( cp.fft.fft(data)[:(len(data)//2)] )\n    return cp.asnumpy( varfft.reshape( (1000,1440) ).mean(axis=1) )","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### X_train"},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train = []\nfor fn in tqdm(trainfiles):\n    X_train.append( extract_fft(fn) )\nX_train = np.stack(X_train)\ngc.collect()\n\nX_train.shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### X_test"},{"metadata":{"trusted":true},"cell_type":"code","source":"X_test = []\nfor fn in tqdm(testfiles):\n    X_test.append( extract_fft(fn) )\nX_test = np.stack(X_test)\ngc.collect()\n\nX_test.shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### y_train"},{"metadata":{"trusted":true},"cell_type":"code","source":"tt = traint[['recording_id','species_id']].copy()\ntf = trainf[['recording_id','species_id']].copy()\ntt[\"tp_and_fp\"] = \"tp\"\ntf[\"tp_and_fp\"] = \"fp\"\n# The order is True Positive first, False Positive second\ny_train_all_classes = pd.concat( (tt, tf) )\n\nfor i in range(24):\n    y_train_all_classes['s'+str(i)] = 0\n    # Notice that the False Positive labels should be 0\n    # Will correct them in the helper function\n    y_train_all_classes.loc[y_train_all_classes.species_id==i,'s'+str(i)] = 1\n\ny_train_all_classes.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"A helper function"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_unique_tp_and_tn(y_train_all_classes, target_col):\n    \"\"\"\n    Get the recording_id of True Positive and True Negative only\n    \"\"\"\n    df_tp_and_tn = (\n        y_train_all_classes[[\"recording_id\", \"tp_and_fp\", target_col]]\n        # Exclude False Negative ones as they are useless\n        .query(f'tp_and_fp == \"tp\" or {target_col} == 1')\n    )\n    # If they are False Positive, need to correct the label\n    df_tp_and_tn.loc[df_tp_and_tn.tp_and_fp==\"fp\", target_col] = 0\n    df_unique_tp_and_tn = (\n        df_tp_and_tn\n        .groupby(\"recording_id\")\n        .max(target_col)\n    )\n    return df_unique_tp_and_tn\n\n# Example\nget_unique_tp_and_tn(y_train_all_classes, \"s19\").sort_values(\"s19\", ascending=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Turn into dataframe"},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train = pd.DataFrame(X_train)\nrecording_id = [path[44:53] for path in trainfiles]\nX_train = X_train.set_index(pd.Index(recording_id))\n\nX_test = pd.DataFrame(X_test)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Check"},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Train\n\n    specify a submission file skeleton\n    specify a hyperparameter dict template\n    specify a RandomizedSearchCV template\n    for each class\n        extract the corresponding target col\n        specify an edited hyperparameter dict, for the sake of scale_pos_weight\n        specify an edited RandomizedSearchCV, for the sake of scale_pos_weight\n        RandomizedSearchCV fit(X_train, y_train)\n        RandomizedSearchCV.best_estimator_.predict_proba(X_test)\n        append the prediction to the submission file as a new column\n  \nThe following 2 parameters make GPU work for XGBClassifier\n- `tree_method='gpu_hist'`\n- `predictor='gpu_predictor'`"},{"metadata":{"trusted":true},"cell_type":"code","source":"sub = pd.DataFrame({'recording_id': [f.split('/')[-1].split('.')[0] for f in testfiles] })\n\nparams = {\n    \"n_estimators\": [40, 60, 80, 100, 120],\n    \"max_depth\": [1, 2, 3, 5, 8, 13],\n    \"learning_rate\": [0.02, 0.04, 0.08, 0.16, 0.32],\n    \"colsample_bytree\": [0.5, 0.6, 0.7, 0.8, 0.9, 1],\n    \"colsample_bylevel\": [0.5, 0.6, 0.7, 0.8, 0.9, 1],\n    \"colsample_bynode\": [0.5, 0.6, 0.7, 0.8, 0.9, 1],\n    \"subsample\": [0.5, 0.6, 0.7, 0.8, 0.9, 1],\n    \"gamma\": [0, 0.001, 0.005, 0.025, 0.125],\n    \"min_child_weight\": [0, 1, 2, 3, 4, 5],\n    \"max_delta_step\": [0, 1, 2, 3, 4, 5],\n    \"reg_alpha\": [0, 0.001, 0.005, 0.025, 0.125],\n    \"reg_lambda\": [0.9, 1, 1.1, 1.2]\n}\n\nmodel = XGBClassifier(\n    n_jobs=-1,\n    random_state=1,\n    tree_method=\"gpu_hist\",\n    predictor=\"gpu_predictor\"\n)\n\ncv = RandomizedSearchCV(\n    estimator = model, \n    param_distributions = params,\n    n_jobs = -1,\n    n_iter = 32,\n    cv = StratifiedKFold(n_splits=5, random_state=1),\n    return_train_score = False,\n    random_state = 1,\n    refit = True,\n    scoring = \"average_precision\"\n)\n\nfor tgt in range(0,24):\n    # updated each loop\n    starttime = time.time()\n    # extract the corresponding target col\n    y_train = get_unique_tp_and_tn(y_train_all_classes, 's'+str(tgt))\n    # part of the scale_pos_weight can only be specified within the loop\n    params_copy = copy.deepcopy(params)\n    params_copy[\"scale_pos_weight\"] = [4, 8, 16, np.sum(y_train.values==0) / np.sum(y_train.values==1)]\n    cv_copy = copy.deepcopy(cv)\n    cv_copy.param_distributions = params_copy\n    # fit\n    cv_copy.fit(\n        X_train.loc[y_train.index],\n        y_train.values.ravel()\n    )\n    print(\"==================================================\")\n    print(f\"best_score of {tgt}: {cv_copy.best_score_}\")\n    print(f\"best_params of {tgt}: {cv_copy.best_params_}\")\n    # plot_importance(cv_copy.best_estimator_, max_num_features = 20, title = \"Feature importance\" + str(tgt))\n    # pyplot.scatter(range(len(cv_copy.best_estimator_.feature_importances_)), cv_copy.best_estimator_.feature_importances_)\n\n    # predict\n    sub['s'+str(tgt)] = cv_copy.best_estimator_.predict_proba(X_test)[:,1]\n    print(f\"{tgt} time: {time.time()-starttime}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}