{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport lightgbm as lgb\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\n# import torch\n\n# print(os.listdir())\n# print(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## me trying\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"# try:\n#     del train_full\n# except:\n#     pass\n# train_full = pd.read_csv(\"../input/train.csv\", nrows=100000000,dtype={'acoustic_data': np.int16, 'time_to_failure': np.float32})\n# train_full.rename({'acoustic_data': \"signal\", 'time_to_failure': \"quaketime\"}, inplace=True, axis=\"columns\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# try:\n#     del data_loader\n# except:\n#     pass\n# data_loader = preprocessing(nrows=10000000, ask_valid=False)\n# data_loader.plot_train_data()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nclass myscaler:\n    def __init__(self, data):\n        self.shape = data.shape\n        if len(data.shape) == 3:\n            self.diff_stat = []\n            self.scaler(data)\n            self.transform = self._transform\n        elif len(data.shape) == 2:\n            scaler = StandardScaler()\n            scaler.fit(data)\n            self.transform = scaler.transform\n        elif len(data.shape == 1):\n            print(\"dim error\")\n            self.transform = None\n        \n    def scaler(self, data):\n        features = data.shape[-1]\n        for i in range(features):\n            self.diff_stat.append([np.mean(data[...,i]), np.std(data[...,i])])\n        \n    def _transform(self, data):\n        assert data.shape[-1] == self.shape[-1]\n        features = data.shape[-1]\n        for i in range(features):\n             data[...,i] = (data[...,i] - self.diff_stat[i][0])/self.diff_stat[i][1]\n        return data\n                                   \n    def transform(self,data):\n        return self.transform(data)\n        \ndef gen_feature(data):\n        features = []\n        features.append(data.abs().min())\n        features.append(data.abs().max())\n        features.append(data.abs().mean())\n        features.append(data.min())\n        features.append(data.max())\n        features.append(data.mean())\n        features.append(data.kurtosis())\n        features.append(data.skew())\n        for i in np.arange(0.01, 0.1, 0.01):\n            features.append(np.quantile(data, i))\n        for i in np.arange(0.9, 1, 0.01):\n            features.append(np.quantile(data, i))\n        data = data.diff().dropna()\n        features.append(data.abs().min())\n        features.append(data.abs().max())\n        features.append(data.abs().mean())\n        features.append(data.min())\n        features.append(data.max())\n        features.append(data.mean())\n        features.append(data.kurtosis())\n        features.append(data.skew())\n        for i in np.arange(0.01, 0.1, 0.01):\n            features.append(np.quantile(data, i))\n        for i in np.arange(0.9, 1, 0.01):\n            features.append(np.quantile(data, i))\n        return pd.Series(features)\n\ndef rolling_feature(data, window_size = 2000, step = 2000):\n    def feature_small_set(data):\n        features = []\n        features.append(data.min())\n        features.append(data.max())\n        features.append(data.mean())\n        features.append(data.std())\n        data = np.diff(data)\n        features.append(data.min())\n        features.append(data.max())\n        features.append(data.mean())\n        features.append(data.std())\n#         features.append(data.kurtosis())\n#         features.append(data.skew())\n#         for i in np.arange(0.01, 0.1, 0.01):\n#             features.append(np.quantile(data, i))\n        for i in np.arange(0.9, 1, 0.01):\n            features.append(np.quantile(data, i))\n        return features\n    ans = []\n    for i in range(window_size, len(data), step):\n        ans.append(feature_small_set(data[(i-window_size):i]))\n    return np.array(ans)\n    \ndef check_quake(data):\n    return (data.diff(-1).dropna() >= 0).all()\n\n\ndef load_batch(data,labels, batch_size=20, shuffle=True):\n    assert len(data) == len(labels)\n    size = len(data)\n    for i in range(0, size, batch_size):\n        if(i+batch_size > size):\n            yield data[i:size], labels[i:size]\n        else:\n            yield data[i:i+batch_size], labels[i:i + batch_size]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"features_name = [\"abs_min\", \"abs_max\", \"abs_mean\", \"min\", 'max', 'mean', 'kurtosis', 'skew'] + [\"quantile\" + f\"{i:.3}\" for i in np.arange(0.01, 0.1, 0.01)] + [\"quantile\" + f\"{i:.3}\" for i in np.arange(0.9, 1, 0.01)]\n    \nfeatures_name = features_name + [\"diff_\" + i for i in features_name ]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def gen_data(feature_fun = gen_feature, test_size=100, size = None, shuffle=True, start_from = None):\n    train_gen = pd.read_csv(\"../input/train.csv\", chunksize=150_000, dtype={'acoustic_data': np.int16, 'time_to_failure': np.float32})\n    X_train = []\n    y_train = []\n    X_test = []\n    y_test = []\n    import gc\n    if start_from is None:\n        start_from = 0\n    for i, data in enumerate(train_gen):\n        data.rename({'acoustic_data': \"signal\", 'time_to_failure': \"quaketime\"}, inplace=True, axis=\"columns\")\n        if i < start_from:\n            del data\n            continue\n        if size is not None and i >= start_from + size:\n            break\n        if check_quake(data[\"quaketime\"]) == False:\n            print(\"quake at\", i, \"excluded\")\n            try:\n                print(X_train[-1].shape)\n            except IndexError:\n                pass\n        X_train.append(feature_fun(data.signal))\n        y_train.append(data.quaketime.values[-1])\n        del data\n        gc.collect()\n    X_train = np.array(X_train)\n    y_train = np.array(y_train)\n    if shuffle:\n        index = np.arange(0, len(X_train))\n        np.random.shuffle(index)\n        X_train = X_train[index]\n        y_train = y_train[index]\n    X_test = X_train[:test_size]\n    y_test = y_train[:test_size]\n    X_train = X_train[test_size:]\n    y_train = y_train[test_size:]\n    print(X_train.shape, y_train.shape)\n    print(X_test.shape, y_test.shape)\n    return X_train, y_train, X_test, y_test\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train, y_train, X_test, y_test = gen_data(size=2000,test_size=200)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot(y_train)\nplt.plot(y_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from catboost import CatBoostRegressor, Pool\n#data scaling\nfrom sklearn.preprocessing import StandardScaler\n#hyperparameter optimization\nfrom sklearn.model_selection import GridSearchCV\n#support vector machine model\nfrom sklearn.svm import NuSVR, SVR\n#kernel ridge model\nfrom sklearn.kernel_ridge import KernelRidge\nfrom sklearn.ensemble import RandomForestRegressor\n\nscaler1 = StandardScaler()\nscaler1.fit(X_train)\nX_train_scaled = scaler1.transform(X_train)\nX_test = scaler1.transform(X_test)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## catboost"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_pool = Pool(X_train_scaled, y_train)\ncat_models = CatBoostRegressor(iterations=10000, loss_function=\"MAE\", boosting_type=\"Ordered\")\ncat_models.fit(X_train_scaled, y_train, silent=True)\nprint(cat_models.best_score_)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_pool = Pool(X_test, y_test)\npre = cat_models.predict(X_test)\nprint(\"catboost score:\", np.mean(np.abs(y_test - pre)))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## RandomForest"},{"metadata":{"trusted":true},"cell_type":"code","source":"rf = RandomForestRegressor(max_depth=2, n_estimators=3000,max_features=40, criterion=\"mae\")\nrf.fit(X_train_scaled, y_train)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nfrom collections import OrderedDict\nfrom pprint import pprint\nimportant = rf.feature_importances_\nprint(len(important), len(features_name))\npprint(OrderedDict(zip(features_name, important)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"rf score:\", rf.score(X_test, y_test.flatten()))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## SVM"},{"metadata":{"trusted":true},"cell_type":"code","source":"\n# parameters = [{'gamma': [0.001, 0.005, 0.01, 0.02, 0.05, 0.1],\n#                'C': [0.1, 0.2, 0.25, 0.5, 1, 1.5, 2]}]\n#                #'nu': [0.75, 0.8, 0.85, 0.9, 0.95, 0.97]}]\n\n# reg1 = GridSearchCV(SVR(kernel='rbf', tol=0.01), parameters, cv=5, scoring='neg_mean_absolute_error')\n# reg1.fit(X_train_scaled, y_train.flatten())\n# y_pred1 = reg1.predict(X_train_scaled)\n# # {'C': 2, 'gamma': 0.001}\n# print(\"Best CV score: {:.4f}\".format(reg1.best_score_))\n# print(reg1.best_params_)\n\n\nsvm = SVR(kernel=\"rbf\", tol=0.01, C=2, gamma=0.001)\nsvm.fit(X_train_scaled, y_train)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"svm score:\", np.mean(np.abs(y_test - svm.predict(X_test))))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# para = [{#'degree':[3,5,7,9], \n#          \"nu\":[0.3, 0.5, 0.7, 0.9],\n#         \"C\":[0.2, 0.5, 1, 2]}]\n\n\n\n# svm = GridSearchCV(NuSVR(kernel=\"poly\", degree=3,gamma=\"scale\"),para, cv=5, scoring='neg_mean_absolute_error')\n# svm.fit(X_train_scaled, y_train.flatten())\n# y_pred = svm.predict(X_train_scaled)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## lgbm"},{"metadata":{"trusted":true},"cell_type":"code","source":"params = {'num_leaves': 128,\n          'min_child_samples': 79,\n          'objective': 'gamma',\n          'max_depth': -1,\n          'learning_rate': 0.01,\n          \"boosting_type\": \"gbdt\",\n          \"subsample_freq\": 5,\n          \"subsample\": 0.9,\n          \"bagging_seed\": 11,\n          \"metric\": 'mae',\n          \"verbosity\": -1,\n          'reg_alpha': 0.1302650970728192,\n          'reg_lambda': 0.3603427518866501,\n          'colsample_bytree': 0.1\n         }\nlgbm = lgb.LGBMRegressor(**params, n_estimators = 50000, n_jobs = -1)\nlgbm.fit(X_train_scaled, y_train, eval_set=[(X_train_scaled, y_train), (X_test, y_test)], eval_metric='mae',\n                    verbose=10000)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"lgbm score:\", np.mean(np.abs(y_test - lgbm.predict(X_test))))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## predict and plot"},{"metadata":{"trusted":true},"cell_type":"code","source":"# y_pred1 = reg1.predict(X_train_scaled)\n# y_pred2 = cat_models.predict(X_train_scaled)\n# y_pred3 = rf.predict(X_train_scaled)\n# y_pred4 = lgbm.predict(X_train_scaled)\n# y = (y_pred1 + y_pred2 + y_pred3)/3\n# fig, ax = plt.subplots(5, 1, figsize=(14,6))\n# plt.figure(figsize=(8, 6))\n# ax[0].scatter(y_train.flatten(), y_pred1, c='b')\n# ax[0].plot([(0, 0), (20, 20)], [(0, 0), (20, 20)])\n# ax[1].scatter(y_train.flatten(), y_pred2, c='r')\n# ax[1].plot([(0, 0), (20, 20)], [(0, 0), (20, 20)])\n# ax[2].scatter(y_train.flatten(), y_pred3,c=\"g\")\n# ax[2].plot([(0, 0), (20, 20)], [(0, 0), (20, 20)])\n# ax[3].scatter(y_train.flatten(), y, c='g')\n# ax[3].plot([(0, 0), (20, 20)], [(0, 0), (20, 20)])\n# plt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# predict"},{"metadata":{"trusted":true},"cell_type":"code","source":"def predict(models, x, weights = None):\n    ans = []\n    y = 0\n    for i, model in enumerate(models):\n        if weights is None:\n            y += model.predict(x)*(1/len(models))\n        else:\n            y += model.predict(x)*weights[i]\n    return y\nmodels = [cat_models, svm, rf, lgbm]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.read_csv('../input/sample_submission.csv', index_col='seg_id')\nindex = list(submission.index)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test = []\nfor seg_id in index:\n    seg = pd.read_csv('../input/test/' + seg_id + '.csv')\n    test.append(gen_feature(seg['acoustic_data']))\ntest = np.array(test)\ntest = scaler1.transform(test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_pre1 = predict(models, test, [0.3,0.3,0.2,0.2])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Lstm"},{"metadata":{"trusted":true},"cell_type":"code","source":"try:\n    del X_train, y_train, X_test, y_test\nexcept:\n    pass\nX_train, y_train, X_test, y_test = gen_data(feature_fun = rolling_feature,size=2000,test_size=200)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nfrom keras.models import Sequential\nfrom keras.layers import Dense, Conv1D, LSTM, Dropout, BatchNormalization, LeakyReLU, CuDNNGRU\nfrom keras.optimizers import Adam\nfrom keras.models import Model\nfrom keras import regularizers\nimport keras\nfrom keras.utils import plot_model\nfrom sklearn.metrics import mean_absolute_error\nfrom keras.callbacks import EarlyStopping\n\nclass MyLSTM:\n\n    def __init__(self, data, labels, valid_data, valid_labels,input_shape = 149):\n        # return super().__init__(*args, **kwargs)\n        self.nn = self.bulid_lstm(data.shape[1], data.shape[2])\n        cb = EarlyStopping(monitor='val_loss', patience=1, verbose=0, mode='min')\n        self.nn.compile(optimizer=\"adam\", loss=\"MAE\")\n        self.input_shape = (data.shape[1], data.shape[2])\n        self.data = data\n        self.labels = labels\n        self.valid = (valid_data,valid_labels)\n\n    def bulid_lstm(self, input_shape, feature_shape):\n        seq = keras.Input(shape=(input_shape, feature_shape))\n#         x = Conv1D(32, kernel_size=7, strides=2, kernel_regularizer=regularizers.l1(0.01))(seq)\n#         x = Conv1D(8, kernel_size=7, strides=2, padding=\"valid\", kernel_regularizer=regularizers.l1(0.01))(x)\n#         x = BatchNormalization()(x)\n#         x = LeakyReLU()(x)\n#         x = Dropout(rate=0.2)(x)\n#         x = LSTM(units=32, return_sequences=True, activation='tanh')(x)\n#         x = Dropout(rate=0.2)(x)\n#         x = LSTM(units=16, return_sequences=True)(x)\n#         x = Dropout(0.2)(x)\n#         x = LSTM(units=50, return_sequences=True)(x)\n#         x = Dropout(0.2)(x)\n        x = CuDNNGRU(48)(seq)\n#         x = LSTM(units=8)(x)\n#         x = Dropout(rate=0.2)(x)\n        x = Dense(10, activation='relu')(x)\n        x = Dense(units=1)(x)\n        model = Model(inputs=seq, outputs=x)\n        model.summary()\n        # plot_model(model, to_file=\"see.jpg\")\n        # model.compile(optimizer=\"rmsprop\", loss=\"MAE\")\n        return model\n\n    def train_on_batch(self, epochs = 50, batch_size=20, intervals=10):\n        losses = []\n        test_losses = []\n        for epoch in range(epochs):\n            for batch_i, data in enumerate(load_batch(self.data, self.labels, batch_size=batch_size)):\n#                 print(data[0].shape)\n                loss = self.nn.train_on_batch(data[0], data[1],)# validation_data=(self.valid[0], self.valid[1]))\n                losses.append(loss)\n                valid = np.mean(np.abs(self.valid[1] - self.nn.predict(self.valid[0])))\n                test_losses.append(np.mean(valid))\n                if batch_i%intervals == 0 or (batch_i == batch_size-1):\n                    print(f\"[Epoch {epoch}/{epochs}] [Batch {batch_i}] [MAE loss {loss}] [valid loss {valid}]\")\n            plt.plot(losses, label=\"training loss\")\n            plt.plot(test_losses, label=\"validation loss\")\n            plt.legend()\n            plt.show()\n            if epoch > 4:\n                stop = (test_losses[-1] > np.min(test_losses) + 0.1)\n                if stop:\n                    print(f\"stop train at [Epoch {epoch}/{epochs}]\")\n                    break\n\n    def predict(self, test_data):\n        return self.nn.predict(test_data)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lstm = MyLSTM(X_train, y_train, X_test, y_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lstm.train_on_batch(batch_size=50,epochs=50)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot(y_test)\nplt.scatter(list(range(len(y_test))),lstm.predict(X_test))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.read_csv('../input/sample_submission.csv', index_col='seg_id')\nindex = list(submission.index)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test = []\nfor seg_id in index:\n    seg = pd.read_csv('../input/test/' + seg_id + '.csv')\n    test.append(rolling_feature(seg['acoustic_data']))\ntest = np.array(test)\ntest = scaler.transform(test)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_pre2 = lstm.predict(test)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## final"},{"metadata":{"trusted":true},"cell_type":"code","source":"y_pre = 0.6*y_pre1 + 0.4*y_pre","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.time_to_failure = y_pre\nsubmission.to_csv('submission.csv')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}