{"cells":[{"metadata":{"_uuid":"20e3c1948c0233ee9a367d3994667da843a48685"},"cell_type":"markdown","source":"The aim of this kernel is to test (and share) the effect of augmenting training set, applying it to the initial kernel \"Basic Feature Benchmark with Quantiles\" [https://www.kaggle.com/andrekos/basic-feature-benchmark-with-quantiles] and considering first approach of \"Basic Feature Benchmark\" [https://www.kaggle.com/inversion/basic-feature-benchmark]\n\nFrom the initial approach of dividing the training set in (n/150_00) segments, this solution **doubles the number of segments** under the following conditions:\n*  it avoids creating segments very similar to each other (with more than 50% of overlaping regions): each new segment shares 50% of 2 sequential segments (as in Figure 1 below); \n*  it uses custom split iterator in order to avoid segments in train set which share regions with segments in validation set.\n\nFigure 1: ![](https://i.imgur.com/rovjuNK_d.jpg?maxwidth=640&fidelity=medium)\n\nApparently this augmentation does not change the results significantly (not with the features currently used).\n\nNot completely tested yet. Open to suggestions and contributions.\n"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"848bcfb106ab28a82b0e0a1bcc2cfd3e8dd8b086"},"cell_type":"code","source":"import matplotlib.pyplot as plt\nfrom tqdm import tqdm\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.svm import NuSVR\nfrom sklearn.kernel_ridge import KernelRidge\nfrom sklearn.metrics import mean_absolute_error, make_scorer\nfrom sklearn.model_selection import GridSearchCV\n\nfrom sklearn.model_selection import KFold","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/train.csv', dtype={'acoustic_data': np.int16, 'time_to_failure': np.float64})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"46dd8c6c7769c19ce128852599b0a7128a6bfeb3"},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f7a11d526a85b84a87086415f8856154dc9736db"},"cell_type":"code","source":"# pandas doesn't show us all the decimals\npd.options.display.precision = 15","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6d699ce5d45b00745d8ac685fe082c46ce9a356a"},"cell_type":"code","source":"# much better!\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1772c3a5aa5dbe3d672031eb64e6bac596206588"},"cell_type":"code","source":"# Create a training file with simple derived features\nrows = 150_000\n# shift_step for augmented training set\nshift_step = int(np.floor(rows / 2))\nsegments = int(np.floor(train.shape[0] / rows))\nsegments_augmented = 2*segments - 1\n\nX_train = pd.DataFrame(index=range(segments_augmented), dtype=np.float64,\n                       columns=['ave', 'std', 'max', 'min','q95','q99', 'q05','q01'])\ny_train = pd.DataFrame(index=range(segments_augmented), dtype=np.float64,\n                       columns=['time_to_failure'])\n\n\nfor segment in tqdm(range(segments)):\n    for do_shift in [False,True]:        \n        if(do_shift):\n            shift = shift_step\n            idx = segments + segment            \n            if(segment==segments-1): #last segment would be incomplete for the shifted version\n                continue\n        else:\n            shift = 0\n            idx = segment\n        \n        seg = train.iloc[segment*rows+shift:segment*rows+shift+rows]\n\n        x = seg['acoustic_data'].values\n        y = seg['time_to_failure'].values[-1]\n\n        y_train.loc[idx, 'time_to_failure'] = y\n\n        X_train.loc[idx, 'ave'] = x.mean()\n        X_train.loc[idx, 'std'] = x.std()\n        X_train.loc[idx, 'max'] = x.max()\n        X_train.loc[idx, 'min'] = x.min()\n        X_train.loc[idx, 'q95'] = np.quantile(x,0.95)\n        X_train.loc[idx, 'q99'] = np.quantile(x,0.99)\n        X_train.loc[idx, 'q05'] = np.quantile(x,0.05)\n        X_train.loc[idx, 'q01'] = np.quantile(x,0.01)\n\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ab3bd0bcc19a849d5b42e321b8997a54dd078831"},"cell_type":"code","source":"X_train.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8d7d30d9b05de93e37cb7a0e6d46e6bf5e5935e9"},"cell_type":"code","source":"# check new segments\nX_train.loc[[0,1,2,3,4,0+segments,1+segments,2+segments,3+segments]]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c2f28f7269af8bd5ba9fe1bec70da271bc93d3fe"},"cell_type":"code","source":"scaler = StandardScaler()\nscaler.fit(X_train)\nX_train_scaled = scaler.transform(X_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8ffa9bd3af8946fc677b18add896d6cf235795ca"},"cell_type":"code","source":"nFolds = 3\n# custom kfold to remove segments in train set which share regions with validation set\ncustomKFoldAvoidLeakToValidation = []\nfor train_id, valid_id in KFold(nFolds,shuffle=True).split(X_train):\n    must_remove = []    \n    for v in valid_id:       \n        if(v>=segments):                \n            must_remove.append(v-segments)\n            must_remove.append(v-segments+1)                \n        else:                \n            must_remove.append(v+segments-1)\n            must_remove.append(v+segments)                \n    train_id = [t for t in train_id if t not in (must_remove)]\n    customKFoldAvoidLeakToValidation.append((train_id, valid_id)) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6752bde073e29cd4d44d97b6be5fe8d970521803"},"cell_type":"code","source":"scorer = make_scorer(mean_absolute_error, greater_is_better=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9bb9c43c3a5de20552fbb2a6c7d91e81ddb55e1e","scrolled":false},"cell_type":"code","source":"parameters = [{ 'gamma': [0.6, 0.7, 0.8],\n               'C': [2.35, 2.4, 2.45, 2.5],\n              'nu': [0.85, 0.9, 0.95]}]\n\n##best_params_\n#parameters = [{'C': [2.35], 'gamma': [0.6], 'nu': [0.9]}]\n\nreg1 = GridSearchCV(NuSVR(kernel='rbf', tol=0.01), parameters, cv = customKFoldAvoidLeakToValidation, scoring=scorer)\nreg1.fit(X_train_scaled, y_train.values.flatten())\ny_pred1 = reg1.predict(X_train_scaled)\n\nprint(reg1.best_params_)\nprint(reg1.best_score_)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e1b0cf862b81685f1b670521377712e09f42e85c"},"cell_type":"code","source":"parameters = [{'gamma': [0.06, 0.1, 0.08, 0.09], #np.logspace(-2, 2, 5)\n               'alpha': [0.005, 0.01, 0.05]}]\n\n#best_params_\n#parameters = [{'alpha': [0.05], 'gamma': [0.06]}]\n\nreg2 = GridSearchCV(KernelRidge(kernel='rbf'), parameters, cv = customKFoldAvoidLeakToValidation, scoring=scorer)\nreg2.fit(X_train_scaled, y_train.values.flatten())\ny_pred2 = reg2.predict(X_train_scaled)\n\nprint(reg2.best_params_)\nprint(reg2.best_score_)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"24e30e9c567061e96a129956fa9836412f1afdc8"},"cell_type":"code","source":"plt.tight_layout()\nf = plt.figure(figsize=(12, 6))\nf.add_subplot(1,2, 1)\nplt.scatter(y_train.values.flatten(), y_pred1)\nplt.title('reg1', fontsize=20)\nplt.xlim(0, 20)\nplt.ylim(0, 20)\nplt.xlabel('actual', fontsize=12)\nplt.ylabel('predicted', fontsize=12)\nplt.plot([(0, 0), (20, 20)], [(0, 0), (20, 20)])\nf.add_subplot(1,2, 2)\nplt.scatter(y_train.values.flatten(), y_pred2)\nplt.title('reg2', fontsize=20)\nplt.xlim(0, 20)\nplt.ylim(0, 20)\nplt.xlabel('actual', fontsize=12)\nplt.ylabel('predicted', fontsize=12)\nplt.plot([(0, 0), (20, 20)], [(0, 0), (20, 20)])\nplt.show(block=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a84ad54d37b7fca77d2f9b30e0786721c6d1f727","scrolled":true},"cell_type":"code","source":"score1 = mean_absolute_error(y_train.values.flatten(), y_pred1)\nprint(f'Score1: {score1:0.3f}')\nscore2 = mean_absolute_error(y_train.values.flatten(), y_pred2)\nprint(f'Score2: {score2:0.3f}')\nscore3 = mean_absolute_error(y_train.values.flatten(), (0.5*y_pred1 + 0.5*y_pred2 ) )\nprint(f'Score3: {score3:0.3f}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6e100154ef818874f21916aa61addd1701677fc7"},"cell_type":"code","source":"submission = pd.read_csv('../input/sample_submission.csv', index_col='seg_id')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"74e6d87f69f6c26d5d84595b11e36a184862bcfb"},"cell_type":"code","source":"X_test = pd.DataFrame(columns=X_train.columns, dtype=np.float64, index=submission.index)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8e9c6c4a4f5184eda4c16d0c9c8a0ebd46f11abe"},"cell_type":"code","source":"for seg_id in X_test.index:\n    seg = pd.read_csv('../input/test/' + seg_id + '.csv')\n    \n    x = seg['acoustic_data'].values\n    \n    X_test.loc[seg_id, 'ave'] = x.mean()\n    X_test.loc[seg_id, 'std'] = x.std()\n    X_test.loc[seg_id, 'max'] = x.max()\n    X_test.loc[seg_id, 'min'] = x.min()\n    X_test.loc[seg_id, 'q95'] = np.quantile(x,0.95)\n    X_test.loc[seg_id, 'q99'] = np.quantile(x,0.99)\n    X_test.loc[seg_id, 'q05'] = np.quantile(x,0.05)\n    X_test.loc[seg_id, 'q01'] = np.quantile(x,0.01)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"eca824c317d8aa9f14aba2ed04f6d08e783c1827"},"cell_type":"code","source":"X_test_scaled = scaler.transform(X_test)\npredictions_submit = 0.5*reg1.predict(X_test_scaled) + 0.5*reg2.predict(X_test_scaled) \n#remove non-positive predictions\npredictions_submit = predictions_submit.clip(min=0)\nsubmission['time_to_failure'] = predictions_submit\nsubmission.to_csv('submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8d7fd157e10ac49e9924ecf20d06b94bba39ce2e"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}