{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm\n%matplotlib inline\n\nfrom sklearn.preprocessing import StandardScaler # for data scaling\nfrom sklearn.model_selection import GridSearchCV # hyperparameter optimization\nfrom catboost import CatBoostRegressor, Pool #catagorical gradient boosting\nfrom sklearn.svm import NuSVR, SVR\n\nimport os\nIS_LOCAL = False\nif(IS_LOCAL):\n    PATH=\"../input/LANL/\"\nelse:\n    PATH=\"../input/\"\nos.listdir(PATH)\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6c8a46f9e4ac4bb0bb57f45c17682dae5e583ecf"},"cell_type":"code","source":"train = pd.read_csv(PATH+'train.csv', nrows = 6000000, dtype={'acoustic_data': np.int16, 'time_to_failure': np.float32})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6bd54888de62740ee9b9c7c22576aa429d7230e7"},"cell_type":"code","source":"train.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"eff83f03ddf48349127f2404088925e8850c9a7a"},"cell_type":"code","source":"#visualize the dataset\ntrain_ad_sample_df = train['acoustic_data'].values[::100]\ntrain_ttf_sample_df = train['time_to_failure'].values[::100]\n\ndef plot_acc_ttf_data(train_ad_sample_df, train_ttf_sample_df, title = \"acoustic data + ttf\"):\n    fig, ax1 = plt.subplots(figsize = (12, 8))\n    plt.title(title)\n    plt.plot(train_ad_sample_df, color = 'r')\n    ax1.set_ylabel('acoustic data', color='r')\n    plt.legend(['acoustic data'], loc=(0.01, 0.95))\n    ax2 = ax1.twinx()\n    plt.plot(train_ttf_sample_df, color ='b')\n    ax2.set_ylabel('time to failure', color = 'b')\n    plt.legend(['time to faliure'], loc = (0.01, 0.9))\n    plt.grid(True)\n    \nplot_acc_ttf_data(train_ad_sample_df, train_ttf_sample_df)\ndel train_ad_sample_df\ndel train_ttf_sample_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f461028ed23a9a3a3641948c6242c46fc553339c"},"cell_type":"code","source":"train = pd.read_csv(PATH+'train.csv', dtype={'acoustic_data': np.int16, 'time_to_failure': np.float32})\nrows = 150000\nsegments = int(np.floor(train.shape[0] / rows))\nprint(\"Number of segments: \", segments)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e93ca01e9dceb9874bdc8da7d2204f2f1d208e01"},"cell_type":"code","source":"features = ['mean','max','variance','min', 'stdev', 'quantile(0.01)', 'quantile(0.05)', 'quantile(0.95)', 'quantile(0.99)']\n\nX = pd.DataFrame(index=range(segments), dtype=np.float64, columns=features)\nY = pd.DataFrame(index=range(segments), dtype=np.float64, columns=['time_to_failure'])\n\nfor segment in tqdm(range(segments)):\n    seg = train.iloc[segment*rows:segment*rows+rows]\n    \n    x = seg['acoustic_data'].values\n    y = seg['time_to_failure'].values[-1]\n    \n    Y.loc[segment, 'time_to_failure'] = y\n    X.loc[segment, 'mean'] = x.mean()\n    X.loc[segment, 'stdev'] = x.std()\n    X.loc[segment, 'variance'] = np.var(x)\n    X.loc[segment, 'max'] = x.max()\n    X.loc[segment, 'min'] = x.min()\n#     X.loc[segment, 'kur'] = x.kurtosis()\n#     X.loc[segment, 'skew'] = x.skew()\n    X.loc[segment, 'quantile(0.01)'] = np.quantile(x, 0.01)\n    X.loc[segment, 'quantile(0.05)'] = np.quantile(x, 0.05)\n    X.loc[segment, 'quantile(0.95)'] = np.quantile(x, 0.95)\n    X.loc[segment, 'quantile(0.99)'] = np.quantile(x, 0.99)\n    \n    #FFT transform values -\n    \"\"\"\n    from: 'https://www.kaggle.com/gpreda/lanl-earthquake-eda-and-prediction' kernel\n     FFT is useful for sequence data feature extraction\n     other than FFT there is Wavelet transform, which can be used to extract low level features, \n     Wavelet transform is though a lil bit complex in terms of computation\n    \"\"\"\n    \n    z = np.fft.fft(x)\n    realFFT = np.real(z)\n    imagFFT = np.imag(z)\n    X.loc[segment, 'A0'] = abs(z[0])\n    X.loc[segment, 'Rmean'] = realFFT.mean()\n    X.loc[segment, 'Rstd'] = realFFT.std()\n    X.loc[segment, 'Rmax'] = realFFT.max()\n    X.loc[segment, 'Rmin'] = realFFT.min()\n    X.loc[segment, 'Imean'] = imagFFT.mean()\n    X.loc[segment, 'Istd'] = imagFFT.std()\n    X.loc[segment, 'Imax'] = imagFFT.max()\n    X.loc[segment, 'Imin'] = imagFFT.min()\n    \nX.describe().T","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7e1e0fed6ddd69b61051518e667f63bd3cbc133d"},"cell_type":"code","source":"X.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bc9567cb3ba32e269dbe869d805d08cfb42fd72d"},"cell_type":"code","source":"# Scaling the data\nscaler = StandardScaler()\nscaler.fit(X)\nscaled_X = pd.DataFrame(scaler.transform(X), columns = X.columns)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"02378db31e6d6c4450129f8fbf54b2f14682f879"},"cell_type":"code","source":"scaled_X.head(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8c647d6eb9a6e50c02039debfb0a475ce6d9a2ec"},"cell_type":"code","source":"# process the test data\nsubmission = pd.read_csv('../input/sample_submission.csv', index_col='seg_id')\nX_test = pd.DataFrame(columns = X.columns, dtype = np.float64, index = submission.index)\nX_test.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cd8ada087614bc44e4c61f8d1fa8de59212fb59e"},"cell_type":"code","source":"submission.shape, X_test.index.shape","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3a246c8b781e4edccdb5f2c3638e6198d0630f0a"},"cell_type":"markdown","source":"We'll used index of submission file to get the segment id and do all the operations."},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"3a5edac6f0c21cc44df44e82291dd467906386f1"},"cell_type":"code","source":"# process the test data\nfor i, seg_id in enumerate(tqdm(X_test.index)):\n    seg = pd.read_csv('../input/test/' + seg_id + '.csv')\n    \n    x = pd.Series(seg['acoustic_data'].values)\n    z = np.fft.fft(x)\n    realFFT = np.real(z)\n    imagFFT = np.imag(z)\n    \n    X_test.loc[seg_id, 'mean'] = x.mean()\n    X_test.loc[seg_id, 'stdev'] = x.std()\n    X_test.loc[seg_id, 'variance'] = np.var(x)\n    X_test.loc[seg_id, 'max'] = x.max()\n    X_test.loc[seg_id, 'min'] = x.min()\n    X_test.loc[seg_id, 'quantile(0.01)'] = np.quantile(x, 0.01)\n    X_test.loc[seg_id, 'quantile(0.05)'] = np.quantile(x, 0.05)\n    X_test.loc[seg_id, 'quantile(0.95)'] = np.quantile(x, 0.95)\n    X_test.loc[seg_id, 'quantile(0.99)'] = np.quantile(x, 0.99)\n    X_test.loc[seg_id, 'A0'] = abs(z[0])\n    X_test.loc[seg_id, 'Rmean'] = realFFT.mean()\n    X_test.loc[seg_id, 'Rstd'] = realFFT.std()\n    X_test.loc[seg_id, 'Rmax'] = realFFT.max()\n    X_test.loc[seg_id, 'Rmin'] = realFFT.min()\n    X_test.loc[seg_id, 'Imean'] = imagFFT.mean()\n    X_test.loc[seg_id, 'Istd'] = imagFFT.std()\n    X_test.loc[seg_id, 'Imax'] = imagFFT.max()\n    X_test.loc[seg_id, 'Imin'] = imagFFT.min()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d036df4334448a0e3af3bb78a4acb55ad9ddb9d4"},"cell_type":"code","source":"# build a model\nX_test.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6930ba2397ba8e37c5104ff9999bc2a215457eb8"},"cell_type":"code","source":"# Scaling the test data\nscaled_test_x = pd.DataFrame(scaler.transform(X_test), columns = X_test.columns)\nscaled_test_x.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1a256778ba9ec868b01f9320bea849e70c2797f5"},"cell_type":"code","source":"scaled_test_x.tail()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"581aa36b4d98dd28c47bac4cc65f545f4ab3ee14"},"cell_type":"markdown","source":"    this is a supervised learning problem but we need more features\n    feature engineeering\n    we'll use statistical feature learning here in which statistical features such as:\n    mean, variance, standard daviation are used to build new features\n        def gen_features(X):\n            strain = []\n            strain.append(X.mean())\n            strain.append(X.std())\n            strain.append(X.min())\n            strain.append(X.max())\n            strain.append(X.kurtosis()) #tailed data feature\n            strain.append(X.skew()) #skewness\n            strain.append(np.quantile(X,0.01))\n            strain.append(np.quantile(X,0.05))\n            strain.append(np.quantile(X,0.95))\n            strain.append(np.quantile(X,0.99)) #sample distributions of same probabilities\n            return pd.Series(strain)\n            \n    train = pd.read_csv(PATH+'train.csv', iterator=True, chunksize=150_000, dtype={'acoustic_data': np.int16, 'time_to_failure': np.float64})\n    X_train = pd.DataFrame()\n    y_train = pd.Series()\n    for df in train:\n        ch = gen_features(df['acoustic_data'])\n        X_train = X_train.append(ch, ignore_index=True)\n        y_train = y_train.append(pd.Series(df['time_to_failure'].values[-1]))"},{"metadata":{"trusted":true,"_uuid":"2e432eff62fe69942aa89c8818b46eb8f3d3d2a6"},"cell_type":"markdown","source":"### implement CatBoost Model\n    using gradient boosting based on catagorical features\n    catagorical features can't be related to each other, essentially\n    there are other boosting models available such as XGboost\n    \"\"\"\n    gradient Boosting:\n    Step - 1 - computing gradient of loss fucntion we want to optimize for each input object\n    step - 2 - learning the decision tree which predicts gradients of the loss function\n\n    ELI5 Time\n    step - 1 - first model data with simple models and analyze data for errors\n    step - 2 - errors signify data points that are difficult to fit by a simple model\n    step - 3 - in later models, we particularly focus on those hard to fit data to get them right\n    step - 4 - lastly, we combine all the predictors by giving some weighs to each predictor.\n    \"\"\"\n\n# MODEL - 1 CatBoost"},{"metadata":{"trusted":true,"_uuid":"771c7e443c0397d266c0c0edd5dc4de6d3ebca0f"},"cell_type":"code","source":"train_pool = Pool(X, Y)\nm = CatBoostRegressor(iterations = 10000, loss_function = 'MAE', boosting_type = 'Ordered')\nm.fit(X, Y, silent = True)\nm.best_score_","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4185d1789366172c00719c0e6dd409b81bd1fc03"},"cell_type":"code","source":"#predictions\npredictions = np.zeros(len(scaled_test_x))\npredictions += m.predict(scaled_test_x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"73df6d9541a192341c51bae65690716e1e62c8ef"},"cell_type":"code","source":"submission['time_to_failure'] = predictions","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0838480eed331027714d4d320b8c1b753932f861"},"cell_type":"code","source":"submission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5f1d2bd8b49457dd68142819410daeffca0ff073"},"cell_type":"code","source":"submission.to_csv('submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"347e72dfcfafe656a6878886965dc7d84412e145"},"cell_type":"markdown","source":"## MODEL 2 - SVM"},{"metadata":{"trusted":true,"_uuid":"da213dc57c54159a2fefbe9b53f967861a0a2011"},"cell_type":"code","source":"parameters = [{'gamma': [0.001, 0.005, 0.01, 0.02, 0.05, 0.1],\n               'C': [0.1, 0.2, 0.5, 1, 1.5, 2]}]\nreg1 = GridSearchCV(SVR(kernel='rbf', tol=0.01), parameters, cv=5, scoring='neg_mean_absolute_error')\nreg1.fit(scaled_X, Y.values.flatten())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"de51057318da8d3b2e13ff8d3b41141a5ce97ddf"},"cell_type":"code","source":"predictions = reg1.predict(scaled_test_x)\nprint(predictions.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"daea0e8af41417127867dacef6876ad48400198e"},"cell_type":"code","source":"submission['time_to_failure'] = predictions\nsubmission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"69312ab13b114f7ed6250f5f74ac996de3c70a65"},"cell_type":"code","source":"submission.to_csv('submissionSVM.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7d3ad548400fb7f9a26aec05eed0ac4e8cc176be"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"655bf091a7341abaa12c8add0d2e806a0c8189df"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}