{"cells":[{"metadata":{},"cell_type":"markdown","source":"I made too much simple LightGBM & XGboosting & NN Blend model.\nhttps://www.kaggle.com/artgor helped me.\n\nI'm beginner, so there may be many strange point.\nPlease give me a advise.\n\n--------------------------------------------------------------\n\nLightGBMとXGboostingとNNの混合によるシンプルな回帰モデルを構築しました。\nhttps://www.kaggle.com/artgor　さんのkernelsを参考にさせていただきました。\n\n日本語で書かれたkernelsがほとんどなかったため\n少しでも初心者の助けになればとこちらのkernelsを作成します。\n\nなお、私自身も初心者ですので理解のできていない点が多くあるかと思います。\nご指摘やアドバイスがあれば是非お願いします。"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nimport matplotlib.pyplot as plt\n%matplotlib inline\nfrom tqdm import tqdm_notebook\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.svm import NuSVR, SVR\nfrom sklearn.metrics import mean_absolute_error\npd.options.display.precision = 15\n\nimport lightgbm as lgb\nimport xgboost as xgb\nimport time\nimport datetime\n\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.model_selection import StratifiedKFold, KFold, RepeatedKFold\nfrom sklearn.metrics import mean_squared_error, mean_absolute_error\nfrom sklearn.linear_model import Ridge, RidgeCV\nimport gc\nfrom catboost import CatBoostRegressor\nimport seaborn as sns\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Loading Data.\n\"time_to_failure\" has slight differences, so read by np.float64.\n\n-------------------\n\nデータの読み込み。\n\"time_to_failure\"はデータの値の差が非常に小さいので、np.float64で読み込む。"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"X = pd.read_csv(\"../input/train.csv\", nrows = 600000000, dtype={'acoustic_data': np.int16, 'time_to_failure': np.float64})","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Making Features.\n\n-----------------------\n\n特徴量の作成。"},{"metadata":{"trusted":true},"cell_type":"code","source":"%time\nimport scipy\n\nrows = 150_000\ntrain = X\nsegments = int(np.floor(train.shape[0] / rows))\n\nX_tr = pd.DataFrame(index=range(segments), dtype=np.float64,\n                       columns=['ave', 'std', 'max', 'min',\n                               'av_change_abs', 'av_change_rate', 'abs_max', 'abs_min',\n                               'std_first_50000', 'std_last_50000', 'std_first_10000', 'std_last_10000',\n                               'avg_first_50000', 'avg_last_50000', 'avg_first_10000', 'avg_last_10000',\n                               'min_first_50000', 'min_last_50000', 'min_first_10000', 'min_last_10000',\n                               'max_first_50000', 'max_last_50000', 'max_first_10000', 'max_last_10000',\n                               'skew','skew_first_10000','skew_last_10000',\n                                'skew_first_50000','skew_last_50000',\n                                'kurt','kurt_first_10000','kurt_last_10000',\n                                'kurt_first_50000','kurt_last_50000'])\ny_tr = pd.DataFrame(index=range(segments), dtype=np.float64,\n                       columns=['time_to_failure'])\n\ntotal_mean = train['acoustic_data'].mean()\ntotal_std = train['acoustic_data'].std()\ntotal_max = train['acoustic_data'].max()\ntotal_min = train['acoustic_data'].min()\ntotal_sum = train['acoustic_data'].sum()\ntotal_abs_max = np.abs(train['acoustic_data']).sum()\n\nfor segment in tqdm_notebook(range(segments)):\n    seg = train.iloc[segment*rows:segment*rows+rows]\n    x = seg['acoustic_data'].values\n    y = seg['time_to_failure'].values[-1]\n    \n    y_tr.loc[segment, 'time_to_failure'] = y\n    X_tr.loc[segment, 'ave'] = x.mean()\n    X_tr.loc[segment, 'std'] = x.std()\n    X_tr.loc[segment, 'max'] = x.max()\n    X_tr.loc[segment, 'min'] = x.min()\n    \n    \n    X_tr.loc[segment, 'av_change_abs'] = np.mean(np.diff(x))\n    X_tr.loc[segment, 'av_change_rate'] = np.mean(np.nonzero((np.diff(x) / x[:-1]))[0])\n    X_tr.loc[segment, 'abs_max'] = np.abs(x).max()\n    X_tr.loc[segment, 'abs_min'] = np.abs(x).min()\n    \n    X_tr.loc[segment, 'std_first_50000'] = x[:50000].std()\n    X_tr.loc[segment, 'std_last_50000'] = x[-50000:].std()\n    X_tr.loc[segment, 'std_first_10000'] = x[:10000].std()\n    X_tr.loc[segment, 'std_last_10000'] = x[-10000:].std()\n    \n    X_tr.loc[segment, 'avg_first_50000'] = x[:50000].mean()\n    X_tr.loc[segment, 'avg_last_50000'] = x[-50000:].mean()\n    X_tr.loc[segment, 'avg_first_10000'] = x[:10000].mean()\n    X_tr.loc[segment, 'avg_last_10000'] = x[-10000:].mean()\n    \n    X_tr.loc[segment, 'min_first_50000'] = x[:50000].min()\n    X_tr.loc[segment, 'min_last_50000'] = x[-50000:].min()\n    X_tr.loc[segment, 'min_first_10000'] = x[:10000].min()\n    X_tr.loc[segment, 'min_last_10000'] = x[-10000:].min()\n    \n    X_tr.loc[segment, 'max_first_50000'] = x[:50000].max()\n    X_tr.loc[segment, 'max_last_50000'] = x[-50000:].max()\n    X_tr.loc[segment, 'max_first_10000'] = x[:10000].max()\n    X_tr.loc[segment, 'max_last_10000'] = x[-10000:].max()\n    \n    X_tr.loc[segment, 'skew'] = scipy.stats.skew(x)\n    X_tr.loc[segment, 'kurt'] = scipy.stats.kurtosis(x)\n        \n    X_tr.loc[segment, 'skew_first_10000'] = scipy.stats.skew(x[:10000])\n    X_tr.loc[segment, 'skew_last_10000'] = scipy.stats.skew(x[:-10000])\n        \n    X_tr.loc[segment, 'kurt_first_10000'] = scipy.stats.kurtosis(x[:10000])\n    X_tr.loc[segment, 'kurt_last_10000'] = scipy.stats.kurtosis(x[:-10000])\n    \n    X_tr.loc[segment, 'skew_first_50000'] = scipy.stats.skew(x[:50000])\n    X_tr.loc[segment, 'skew_last_50000'] = scipy.stats.skew(x[:-50000])\n        \n    X_tr.loc[segment, 'kurt_first_50000'] = scipy.stats.kurtosis(x[:50000])\n    X_tr.loc[segment, 'kurt_last_50000'] = scipy.stats.kurtosis(x[:-50000])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Normalize Features.\n\n------------------------\n\n作成した特徴量の正規化。"},{"metadata":{"trusted":true},"cell_type":"code","source":"scaler = StandardScaler()\nscaler.fit(X_tr)\nX_train_scaled = pd.DataFrame(scaler.transform(X_tr), columns=X_tr.columns)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport numpy as np\nfrom scipy.stats import norm\nfrom sklearn.preprocessing import StandardScaler\nfrom scipy import stats\nimport warnings\nwarnings.filterwarnings('ignore')\n%matplotlib inline\nprint(y_tr['time_to_failure'].describe())\nsns.distplot(y_tr['time_to_failure'], bins=500)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train_scaled.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"var = 'skew_first_10000'\ndata = pd.concat([y_tr['time_to_failure'], X_train_scaled], axis=1)\ndata.plot.scatter(x=var, y='time_to_failure', ylim=(0,20));","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"corrmat = data.corr()\nf, ax = plt.subplots(figsize=(12, 9))\nsns.heatmap(corrmat, vmax=.8, square=True);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"k = 10 #number of variables for heatmap\ncols = corrmat.nlargest(k, 'time_to_failure')['time_to_failure'].index\ncm = np.corrcoef(data[cols].values.T)\nsns.set(font_scale=1.25)\nhm = sns.heatmap(cm, cbar=True, annot=True, square=True, fmt='.2f', annot_kws={'size': 10}, yticklabels=cols.values, xticklabels=cols.values)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.set()\ncols = ['time_to_failure', 'min', 'min_last_50000', 'min_last_10000', 'min_first_50000', 'min_first_10000', 'av_change_rate']\nsns.pairplot(data[cols], size = 2.5)\nplt.show();","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Making data to predict.\n\n--------------------------------\n\n訓練データの作成。"},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.read_csv('../input/sample_submission.csv', index_col='seg_id')\nX_test = pd.DataFrame(columns=X_tr.columns, dtype=np.float64, index=submission.index)\nplt.figure(figsize=(22, 16))\n\nfor i, seg_id in enumerate(tqdm_notebook(X_test.index)):\n    seg = pd.read_csv('../input/test/' + seg_id + '.csv')\n    \n    x = seg['acoustic_data'].values\n    X_test.loc[seg_id, 'ave'] = x.mean()\n    X_test.loc[seg_id, 'std'] = x.std()\n    X_test.loc[seg_id, 'max'] = x.max()\n    X_test.loc[seg_id, 'min'] = x.min()\n        \n    X_test.loc[seg_id, 'av_change_abs'] = np.mean(np.diff(x))\n    X_test.loc[seg_id, 'av_change_rate'] = np.mean(np.nonzero((np.diff(x) / x[:-1]))[0])\n    X_test.loc[seg_id, 'abs_max'] = np.abs(x).max()\n    X_test.loc[seg_id, 'abs_min'] = np.abs(x).min()\n    \n    X_test.loc[seg_id, 'std_first_50000'] = x[:50000].std()\n    X_test.loc[seg_id, 'std_last_50000'] = x[-50000:].std()\n    X_test.loc[seg_id, 'std_first_10000'] = x[:10000].std()\n    X_test.loc[seg_id, 'std_last_10000'] = x[-10000:].std()\n    \n    X_test.loc[seg_id, 'avg_first_50000'] = x[:50000].mean()\n    X_test.loc[seg_id, 'avg_last_50000'] = x[-50000:].mean()\n    X_test.loc[seg_id, 'avg_first_10000'] = x[:10000].mean()\n    X_test.loc[seg_id, 'avg_last_10000'] = x[-10000:].mean()\n    \n    X_test.loc[seg_id, 'min_first_50000'] = x[:50000].min()\n    X_test.loc[seg_id, 'min_last_50000'] = x[-50000:].min()\n    X_test.loc[seg_id, 'min_first_10000'] = x[:10000].min()\n    X_test.loc[seg_id, 'min_last_10000'] = x[-10000:].min()\n    \n    X_test.loc[seg_id, 'max_first_50000'] = x[:50000].max()\n    X_test.loc[seg_id, 'max_last_50000'] = x[-50000:].max()\n    X_test.loc[seg_id, 'max_first_10000'] = x[:10000].max()\n    X_test.loc[seg_id, 'max_last_10000'] = x[-10000:].max()\n    \n    X_test.loc[seg_id, 'skew'] = scipy.stats.skew(x)\n    X_test.loc[seg_id, 'kurt'] = scipy.stats.kurtosis(x)\n        \n    X_test.loc[seg_id, 'skew_first_10000'] = scipy.stats.skew(x[:10000])\n    X_test.loc[seg_id, 'skew_last_10000'] = scipy.stats.skew(x[:-10000])\n        \n    X_test.loc[seg_id, 'kurt_first_10000'] = scipy.stats.kurtosis(x[:10000])\n    X_test.loc[seg_id, 'kurt_last_10000'] = scipy.stats.kurtosis(x[:-10000])\n            \n    X_test.loc[seg_id, 'skew_first_50000'] = scipy.stats.skew(x[:50000])\n    X_test.loc[seg_id, 'skew_last_50000'] = scipy.stats.skew(x[:-50000])\n        \n    X_test.loc[seg_id, 'kurt_first_50000'] = scipy.stats.kurtosis(x[:50000])\n    X_test.loc[seg_id, 'kurt_last_50000'] = scipy.stats.kurtosis(x[:-50000])\n    \nX_test_scaled = pd.DataFrame(scaler.transform(X_test), columns=X_test.columns)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Trainig and Predict by LightGBM.\n\n--------------------------------------\n\nLightGBMによる学習および予測。"},{"metadata":{"trusted":true},"cell_type":"code","source":"model = lgb.LGBMRegressor(n_estimators=10000,learning_rate=0.0005, n_jobs=-1)\nmodel.fit(X_train_scaled,y_tr)\ny_pred_lgb = model.predict(X_test_scaled)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Trainig and Predict by XGboosting.\n\n--------------------------------------\nXGboostingによる学習および予測。"},{"metadata":{"trusted":true},"cell_type":"code","source":"model = xgb.XGBRegressor(n_estimators=10)\nmodel.fit(X_train_scaled,y_tr)\ny_pred_xgb = model.predict(X_test_scaled)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Training and Predict by NN.\n\n-------------------------------\n\nNNによる学習および予測。"},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.layers import Input, Dense\nfrom keras.models import Model\nimport tensorflow as tf\n\ninputs = Input(shape=(34,))\nx = Dense(128, activation='relu')(inputs)\nx = Dense(128, activation='relu')(x)\npredictions = Dense(1)(x)\nmodel_3 = Model(inputs=inputs, outputs=predictions)\nmodel_3.compile(optimizer=tf.train.RMSPropOptimizer(0.001),\n                      loss='mse',\n                    metrics=['accuracy'])\nmodel_3.fit(X_train_scaled, y_tr, epochs=15, verbose=0)\ny_pred_nn = model.predict(X_test_scaled).flatten()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_test_scaled","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Blending.\n\n---------------------------\n\n混合。"},{"metadata":{"trusted":true},"cell_type":"code","source":"sample = pd.read_csv(\"../input/sample_submission.csv\")\nsample['time_to_failure'] = (y_pred_lgb+y_pred_xgb+y_pred_nn)/3\nsample.to_csv('submission.csv',index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Cross Validation.\n\n---------------------------\n\n交差検証。"},{"metadata":{"trusted":true},"cell_type":"code","source":"%time\n\nfrom sklearn.model_selection import StratifiedKFold, ShuffleSplit, cross_val_score\nfrom sklearn.metrics import log_loss, roc_auc_score\nfrom tqdm import tqdm\nfrom statistics import mean\n\nn_estimators = [5,10]\n\nfor n_est in  n_estimators:\n    print(n_est)\n    mae = []\n    cv = ShuffleSplit(n_splits=5,random_state=0)\n    for train, valid in cv.split(X_train_scaled, y_tr):\n        x_train = X_train_scaled.iloc[train]\n        x_valid = X_train_scaled.iloc[valid]\n        y_train = y_tr.iloc[train]\n        y_valid = y_tr.iloc[valid]\n        \n        model_1 = lgb.LGBMRegressor(n_estimators=10000, learning_rate=0.0005,n_jobs=-1,random_state=0)\n        model_1.fit(x_train, y_train)\n        y_pred_lgb = model_1.predict(x_valid)\n        \n        model_2 = xgb.XGBRegressor(n_estimators=n_est, n_jobs=-1,random_state=0)\n        model_2.fit(x_train, y_train)\n        y_pred_xgb = model_2.predict(x_valid)\n        \n        inputs = Input(shape=(34,))\n        x = Dense(128, activation='relu')(inputs)\n        x = Dense(128, activation='relu')(x)\n        predictions = Dense(1)(x)\n        model_3 = Model(inputs=inputs, outputs=predictions)\n        model_3.compile(optimizer=tf.train.RMSPropOptimizer(0.001),\n                      loss='mse',\n                    metrics=['accuracy'])\n        model_3.fit(x_train, y_train, epochs=15, verbose=0)\n        y_pred_nn = model_3.predict(x_valid).flatten()\n        \n        y_pred = (y_pred_lgb+y_pred_xgb+y_pred_nn)/3\n        mae.append(mean_absolute_error(y_valid, y_pred))\n    print(mean(mae))","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}