{"cells":[{"metadata":{"_uuid":"f92e65033299d13b5d6dc3e5f9ada7c557f529d4"},"cell_type":"markdown","source":"- added Gaussian Process Regression with a generalised RBF kernel to the mix"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nimport os\nprint(os.listdir(\"../input\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"848bcfb106ab28a82b0e0a1bcc2cfd3e8dd8b086"},"cell_type":"code","source":"import matplotlib.pyplot as plt\nfrom tqdm import tqdm\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.svm import NuSVR\nfrom sklearn.kernel_ridge import KernelRidge\nfrom sklearn.metrics import mean_absolute_error, make_scorer\nfrom sklearn.model_selection import GridSearchCV","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/train.csv', dtype={'acoustic_data': np.int16, 'time_to_failure': np.float64})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"46dd8c6c7769c19ce128852599b0a7128a6bfeb3"},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f7a11d526a85b84a87086415f8856154dc9736db"},"cell_type":"code","source":"# pandas doesn't show us all the decimals\npd.options.display.precision = 15","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6d699ce5d45b00745d8ac685fe082c46ce9a356a"},"cell_type":"code","source":"# much better!\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d5dd381f0f0bbd7cc2322a2c48a6784f4669194f"},"cell_type":"code","source":"from sklearn.linear_model import LinearRegression\ndef add_trend_feature(arr, abs_values=False):\n    idx = np.array(range(len(arr)))\n    if abs_values:\n        arr = np.abs(arr)\n    lr = LinearRegression()\n    lr.fit(idx.reshape(-1, 1), arr)\n    return lr.coef_[0]\n\nfrom scipy import stats","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1772c3a5aa5dbe3d672031eb64e6bac596206588"},"cell_type":"code","source":"# Create a training file with simple derived features\n\nrows = 150_000\nsegments = int(np.floor(train.shape[0] / rows))\n\nX_train = pd.DataFrame(index=range(segments), dtype=np.float64,\n                       columns=['ave', 'std', 'max', 'min','q95','q99', 'q05','q01',\n                               'abs_max', 'abs_mean', 'abs_std', 'trend', 'abs_trend', 'iqr', \n                                'q999','q001','ave10'])\ny_train = pd.DataFrame(index=range(segments), dtype=np.float64,\n                       columns=['time_to_failure'])\n\nfor segment in tqdm(range(segments)):\n    seg = train.iloc[segment*rows:segment*rows+rows]\n    x = seg['acoustic_data'].values\n    y = seg['time_to_failure'].values[-1]\n    \n    y_train.loc[segment, 'time_to_failure'] = y\n    \n    X_train.loc[segment, 'ave'] = x.mean()\n    X_train.loc[segment, 'std'] = x.std()\n    X_train.loc[segment, 'max'] = x.max()\n    X_train.loc[segment, 'min'] = x.min()\n    X_train.loc[segment, 'q95'] = np.quantile(x,0.95)\n    X_train.loc[segment, 'q99'] = np.quantile(x,0.99)\n    X_train.loc[segment, 'q05'] = np.quantile(x,0.05)\n    X_train.loc[segment, 'q01'] = np.quantile(x,0.01)\n    \n    X_train.loc[segment, 'abs_max'] = np.abs(x).max()\n    X_train.loc[segment, 'abs_mean'] = np.abs(x).mean()\n    X_train.loc[segment, 'abs_std'] = np.abs(x).std()\n    X_train.loc[segment, 'trend'] = add_trend_feature(x)\n    X_train.loc[segment, 'abs_trend'] = add_trend_feature(x, abs_values=True)\n    \n    X_train.loc[segment, 'iqr'] = np.subtract(*np.percentile(x, [75, 25]))\n    X_train.loc[segment, 'q999'] = np.quantile(x,0.999)\n    X_train.loc[segment, 'q001'] = np.quantile(x,0.001)\n    X_train.loc[segment, 'ave10'] = stats.trim_mean(x, 0.1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ae66efa59d5b1fbf7547433def198b840ddf8bae"},"cell_type":"code","source":"X_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c2f28f7269af8bd5ba9fe1bec70da271bc93d3fe"},"cell_type":"code","source":"scaler = StandardScaler()\nscaler.fit(X_train)\nX_train_scaled = scaler.transform(X_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9bb9c43c3a5de20552fbb2a6c7d91e81ddb55e1e"},"cell_type":"code","source":"scorer = make_scorer(mean_absolute_error, greater_is_better=False)\nparameters = [{ 'gamma': [0.6, 0.7, 0.8],\n               'C': [2.35, 2.4, 2.45, 2.5],\n              'nu': [0.85, 0.9, 0.95]}]\n\nreg1 = GridSearchCV(NuSVR(kernel='rbf', tol=0.01), parameters, cv = 3, scoring=scorer)\nreg1.fit(X_train_scaled, y_train.values.flatten())\ny_pred1 = reg1.predict(X_train_scaled)\n\nprint(reg1.best_params_)\nprint(reg1.best_score_)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e1b0cf862b81685f1b670521377712e09f42e85c"},"cell_type":"code","source":"parameters = [{ 'gamma': [0.06, 0.1, 0.08, 0.09], #np.logspace(-2, 2, 5)\n               'alpha': [0.005, 0.01, 0.05]}]\n\nreg2 = GridSearchCV(KernelRidge(kernel='rbf'), parameters, cv = 3, scoring=scorer)\nreg2.fit(X_train_scaled, y_train.values.flatten())\ny_pred2 = reg2.predict(X_train_scaled)\n\nprint(reg2.best_params_)\nprint(reg2.best_score_)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"24e30e9c567061e96a129956fa9836412f1afdc8"},"cell_type":"code","source":"plt.tight_layout()\nf = plt.figure(figsize=(12, 6))\nf.add_subplot(1,2, 1)\nplt.scatter(y_train.values.flatten(), y_pred1)\nplt.title('reg1', fontsize=20)\nplt.xlim(0, 20)\nplt.ylim(0, 20)\nplt.xlabel('actual', fontsize=12)\nplt.ylabel('predicted', fontsize=12)\nplt.plot([(0, 0), (20, 20)], [(0, 0), (20, 20)])\nf.add_subplot(1,2, 2)\nplt.scatter(y_train.values.flatten(), y_pred2)\nplt.title('reg2', fontsize=20)\nplt.xlim(0, 20)\nplt.ylim(0, 20)\nplt.xlabel('actual', fontsize=12)\nplt.ylabel('predicted', fontsize=12)\nplt.plot([(0, 0), (20, 20)], [(0, 0), (20, 20)])\nplt.show(block=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a84ad54d37b7fca77d2f9b30e0786721c6d1f727"},"cell_type":"code","source":"score1 = mean_absolute_error(y_train.values.flatten(), y_pred1)\nprint(f'Score1: {score1:0.3f}')\nscore2 = mean_absolute_error(y_train.values.flatten(), y_pred2)\nprint(f'Score2: {score2:0.3f}')\nscore3 = mean_absolute_error(y_train.values.flatten(), y_pred1*0.5+y_pred2*0.5)\nprint(f'Score3: {score3:0.3f}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6e100154ef818874f21916aa61addd1701677fc7"},"cell_type":"code","source":"submission = pd.read_csv('../input/sample_submission.csv', index_col='seg_id')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"74e6d87f69f6c26d5d84595b11e36a184862bcfb"},"cell_type":"code","source":"X_test = pd.DataFrame(columns=X_train.columns, dtype=np.float64, index=submission.index)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8e9c6c4a4f5184eda4c16d0c9c8a0ebd46f11abe"},"cell_type":"code","source":"for seg_id in X_test.index:\n    seg = pd.read_csv('../input/test/' + seg_id + '.csv')\n    \n    x = seg['acoustic_data'].values\n    \n    X_test.loc[seg_id, 'ave'] = x.mean()\n    X_test.loc[seg_id, 'std'] = x.std()\n    X_test.loc[seg_id, 'max'] = x.max()\n    X_test.loc[seg_id, 'min'] = x.min()\n    X_test.loc[seg_id, 'q95'] = np.quantile(x,0.95)\n    X_test.loc[seg_id, 'q99'] = np.quantile(x,0.99)\n    X_test.loc[seg_id, 'q05'] = np.quantile(x,0.05)\n    X_test.loc[seg_id, 'q01'] = np.quantile(x,0.01)\n    \n    X_test.loc[seg_id, 'abs_max'] = np.abs(x).max()\n    X_test.loc[seg_id, 'abs_mean'] = np.abs(x).mean()\n    X_test.loc[seg_id, 'abs_std'] = np.abs(x).std()\n    X_test.loc[seg_id, 'trend'] = add_trend_feature(x)\n    X_test.loc[seg_id, 'abs_trend'] = add_trend_feature(x, abs_values=True)\n    \n    X_test.loc[seg_id, 'iqr'] = np.subtract(*np.percentile(x, [75, 25]))\n    X_test.loc[seg_id, 'q999'] = np.quantile(x,0.999)\n    X_test.loc[seg_id, 'q001'] = np.quantile(x,0.001)\n    X_test.loc[seg_id, 'ave10'] = stats.trim_mean(x, 0.1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"eca824c317d8aa9f14aba2ed04f6d08e783c1827"},"cell_type":"code","source":"X_test_scaled = scaler.transform(X_test)\nsubmission['time_to_failure'] = reg1.predict(X_test_scaled)*0.5 + reg2.predict(X_test_scaled)*0.5\nsubmission.to_csv('submission.csv')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}