{"cells":[{"metadata":{},"cell_type":"markdown","source":"I made too much simple NN model.\nhttps://www.kaggle.com/artgor helped me.\n\nI'm beginner, so there may be many strange point.\nPlease give me a advise.\n\n--------------------------------------------------------------\n\nNNによるシンプルな回帰モデルを構築しました。\nhttps://www.kaggle.com/artgor　さんのkernelsを参考にさせていただきました。\n\n日本語で書かれたkernelsがほとんどなかったため\n少しでも初心者の助けになればとこちらのkernelsを作成します。\n\nなお、私自身も初心者ですので理解のできていない点が多くあるかと思います。\nご指摘やアドバイスがあれば是非お願いします。"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nimport matplotlib.pyplot as plt\n%matplotlib inline\nfrom tqdm import tqdm_notebook\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.svm import NuSVR, SVR\nfrom sklearn.metrics import mean_absolute_error\npd.options.display.precision = 15\n\nimport lightgbm as lgb\nimport xgboost as xgb\nimport time\nimport datetime\n\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.model_selection import StratifiedKFold, KFold, RepeatedKFold\nfrom sklearn.metrics import mean_squared_error, mean_absolute_error\nfrom sklearn.linear_model import Ridge, RidgeCV\nimport gc\nfrom catboost import CatBoostRegressor\nimport seaborn as sns\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n","execution_count":1,"outputs":[{"output_type":"stream","text":"['test', 'train.csv', 'sample_submission.csv']\n","name":"stdout"}]},{"metadata":{},"cell_type":"markdown","source":"Loading Data.\n\n-------------------\n\nデータの読み込み。"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"X = pd.read_csv(\"../input/train.csv\", nrows = 600000000, dtype={'acoustic_data': np.int16, 'time_to_failure': np.float64})","execution_count":2,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Making Features.\n\n-----------------------\n\n特徴量の作成。"},{"metadata":{"trusted":true},"cell_type":"code","source":"%time\n\nrows = 150_000\ntrain = X\nsegments = int(np.floor(train.shape[0] / rows))\n\nX_tr = pd.DataFrame(index=range(segments), dtype=np.float64,\n                       columns=['ave', 'std', 'max', 'min',\n                               'av_change_abs', 'av_change_rate', 'abs_max', 'abs_min',\n                               'std_first_50000', 'std_last_50000', 'std_first_10000', 'std_last_10000',\n                               'avg_first_50000', 'avg_last_50000', 'avg_first_10000', 'avg_last_10000',\n                               'min_first_50000', 'min_last_50000', 'min_first_10000', 'min_last_10000',\n                               'max_first_50000', 'max_last_50000', 'max_first_10000', 'max_last_10000'])\ny_tr = pd.DataFrame(index=range(segments), dtype=np.float64,\n                       columns=['time_to_failure'])\n\ntotal_mean = train['acoustic_data'].mean()\ntotal_std = train['acoustic_data'].std()\ntotal_max = train['acoustic_data'].max()\ntotal_min = train['acoustic_data'].min()\ntotal_sum = train['acoustic_data'].sum()\ntotal_abs_max = np.abs(train['acoustic_data']).sum()\n\nfor segment in tqdm_notebook(range(segments)):\n    seg = train.iloc[segment*rows:segment*rows+rows]\n    x = seg['acoustic_data'].values\n    y = seg['time_to_failure'].values[-1]\n    \n    y_tr.loc[segment, 'time_to_failure'] = y\n    X_tr.loc[segment, 'ave'] = x.mean()\n    X_tr.loc[segment, 'std'] = x.std()\n    X_tr.loc[segment, 'max'] = x.max()\n    X_tr.loc[segment, 'min'] = x.min()\n    \n    \n    X_tr.loc[segment, 'av_change_abs'] = np.mean(np.diff(x))\n    X_tr.loc[segment, 'av_change_rate'] = np.mean(np.nonzero((np.diff(x) / x[:-1]))[0])\n    X_tr.loc[segment, 'abs_max'] = np.abs(x).max()\n    X_tr.loc[segment, 'abs_min'] = np.abs(x).min()\n    \n    X_tr.loc[segment, 'std_first_50000'] = x[:50000].std()\n    X_tr.loc[segment, 'std_last_50000'] = x[-50000:].std()\n    X_tr.loc[segment, 'std_first_10000'] = x[:10000].std()\n    X_tr.loc[segment, 'std_last_10000'] = x[-10000:].std()\n    \n    X_tr.loc[segment, 'avg_first_50000'] = x[:50000].mean()\n    X_tr.loc[segment, 'avg_last_50000'] = x[-50000:].mean()\n    X_tr.loc[segment, 'avg_first_10000'] = x[:10000].mean()\n    X_tr.loc[segment, 'avg_last_10000'] = x[-10000:].mean()\n    \n    X_tr.loc[segment, 'min_first_50000'] = x[:50000].min()\n    X_tr.loc[segment, 'min_last_50000'] = x[-50000:].min()\n    X_tr.loc[segment, 'min_first_10000'] = x[:10000].min()\n    X_tr.loc[segment, 'min_last_10000'] = x[-10000:].min()\n    \n    X_tr.loc[segment, 'max_first_50000'] = x[:50000].max()\n    X_tr.loc[segment, 'max_last_50000'] = x[-50000:].max()\n    X_tr.loc[segment, 'max_first_10000'] = x[:10000].max()\n    X_tr.loc[segment, 'max_last_10000'] = x[-10000:].max()","execution_count":3,"outputs":[{"output_type":"stream","text":"CPU times: user 0 ns, sys: 0 ns, total: 0 ns\nWall time: 9.3 µs\n","name":"stdout"},{"output_type":"display_data","data":{"text/plain":"HBox(children=(IntProgress(value=0, max=4000), HTML(value='')))","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"013f70a2183e4b9f87b90c586fbdcc4a"}},"metadata":{}},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:37: RuntimeWarning: divide by zero encountered in true_divide\n/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:37: RuntimeWarning: invalid value encountered in true_divide\n","name":"stderr"},{"output_type":"stream","text":"\n","name":"stdout"}]},{"metadata":{},"cell_type":"markdown","source":"Normalize Features.\n\n------------------------\n\n作成した特徴量の正規化。"},{"metadata":{"trusted":true},"cell_type":"code","source":"scaler = StandardScaler()\nscaler.fit(X_tr)\nX_train_scaled = pd.DataFrame(scaler.transform(X_tr), columns=X_tr.columns)","execution_count":4,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Making data to predict.\n\n--------------------------------\n\n訓練データの作成。"},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.read_csv('../input/sample_submission.csv', index_col='seg_id')\nX_test = pd.DataFrame(columns=X_tr.columns, dtype=np.float64, index=submission.index)\nplt.figure(figsize=(22, 16))\n\nfor i, seg_id in enumerate(tqdm_notebook(X_test.index)):\n    seg = pd.read_csv('../input/test/' + seg_id + '.csv')\n    \n    x = seg['acoustic_data'].values\n    X_test.loc[seg_id, 'ave'] = x.mean()\n    X_test.loc[seg_id, 'std'] = x.std()\n    X_test.loc[seg_id, 'max'] = x.max()\n    X_test.loc[seg_id, 'min'] = x.min()\n        \n    X_test.loc[seg_id, 'av_change_abs'] = np.mean(np.diff(x))\n    X_test.loc[seg_id, 'av_change_rate'] = np.mean(np.nonzero((np.diff(x) / x[:-1]))[0])\n    X_test.loc[seg_id, 'abs_max'] = np.abs(x).max()\n    X_test.loc[seg_id, 'abs_min'] = np.abs(x).min()\n    \n    X_test.loc[seg_id, 'std_first_50000'] = x[:50000].std()\n    X_test.loc[seg_id, 'std_last_50000'] = x[-50000:].std()\n    X_test.loc[seg_id, 'std_first_10000'] = x[:10000].std()\n    X_test.loc[seg_id, 'std_last_10000'] = x[-10000:].std()\n    \n    X_test.loc[seg_id, 'avg_first_50000'] = x[:50000].mean()\n    X_test.loc[seg_id, 'avg_last_50000'] = x[-50000:].mean()\n    X_test.loc[seg_id, 'avg_first_10000'] = x[:10000].mean()\n    X_test.loc[seg_id, 'avg_last_10000'] = x[-10000:].mean()\n    \n    X_test.loc[seg_id, 'min_first_50000'] = x[:50000].min()\n    X_test.loc[seg_id, 'min_last_50000'] = x[-50000:].min()\n    X_test.loc[seg_id, 'min_first_10000'] = x[:10000].min()\n    X_test.loc[seg_id, 'min_last_10000'] = x[-10000:].min()\n    \n    X_test.loc[seg_id, 'max_first_50000'] = x[:50000].max()\n    X_test.loc[seg_id, 'max_last_50000'] = x[-50000:].max()\n    X_test.loc[seg_id, 'max_first_10000'] = x[:10000].max()\n    X_test.loc[seg_id, 'max_last_10000'] = x[-10000:].max()\n    \nX_test_scaled = pd.DataFrame(scaler.transform(X_test), columns=X_test.columns)","execution_count":5,"outputs":[{"output_type":"display_data","data":{"text/plain":"HBox(children=(IntProgress(value=0, max=2624), HTML(value='')))","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"f945266196d244a4bb5bf4e899940fa7"}},"metadata":{}},{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:15: RuntimeWarning: divide by zero encountered in true_divide\n  from ipykernel import kernelapp as app\n/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:15: RuntimeWarning: invalid value encountered in true_divide\n  from ipykernel import kernelapp as app\n","name":"stderr"},{"output_type":"stream","text":"\n","name":"stdout"},{"output_type":"display_data","data":{"text/plain":"<Figure size 1584x1152 with 0 Axes>"},"metadata":{}}]},{"metadata":{},"cell_type":"markdown","source":"Trainig and Predict by NN.\n\n--------------------------------------\n\nNNによる学習および予測。"},{"metadata":{"trusted":true},"cell_type":"code","source":"X_test_scaled.shape","execution_count":6,"outputs":[{"output_type":"execute_result","execution_count":6,"data":{"text/plain":"(2624, 24)"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.layers import Input, Dense\nfrom keras.models import Model\nimport tensorflow as tf\n\n# This returns a tensor\ninputs = Input(shape=(24,))\n\n# a layer instance is callable on a tensor, and returns a tensor\nx = Dense(128, activation='relu')(inputs)\nx = Dense(128, activation='relu')(x)\npredictions = Dense(1)(x)\n\n# This creates a model that includes\n# the Input layer and three Dense layers\nmodel = Model(inputs=inputs, outputs=predictions)\nmodel.compile(optimizer=tf.train.RMSPropOptimizer(0.001),\n              loss='mse',\n              metrics=['accuracy'])\nmodel.fit(X_train_scaled, y_tr,epochs=10)  # starts training","execution_count":7,"outputs":[{"output_type":"stream","text":"Using TensorFlow backend.\n","name":"stderr"},{"output_type":"stream","text":"WARNING:tensorflow:From /opt/conda/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\nInstructions for updating:\nColocations handled automatically by placer.\nWARNING:tensorflow:From /opt/conda/lib/python3.6/site-packages/tensorflow/python/ops/math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\nInstructions for updating:\nUse tf.cast instead.\nEpoch 1/10\n4000/4000 [==============================] - 1s 176us/step - loss: 23.0998 - acc: 0.0000e+00\nEpoch 2/10\n4000/4000 [==============================] - 0s 44us/step - loss: 10.6214 - acc: 0.0000e+00\nEpoch 3/10\n4000/4000 [==============================] - 0s 40us/step - loss: 9.7376 - acc: 0.0000e+00\nEpoch 4/10\n4000/4000 [==============================] - 0s 41us/step - loss: 9.2776 - acc: 0.0000e+00\nEpoch 5/10\n4000/4000 [==============================] - 0s 37us/step - loss: 9.0397 - acc: 0.0000e+00\nEpoch 6/10\n4000/4000 [==============================] - 0s 36us/step - loss: 8.7724 - acc: 0.0000e+00\nEpoch 7/10\n4000/4000 [==============================] - 0s 39us/step - loss: 8.6368 - acc: 0.0000e+00\nEpoch 8/10\n4000/4000 [==============================] - 0s 37us/step - loss: 8.4919 - acc: 0.0000e+00\nEpoch 9/10\n4000/4000 [==============================] - 0s 37us/step - loss: 8.3613 - acc: 0.0000e+00\nEpoch 10/10\n4000/4000 [==============================] - 0s 42us/step - loss: 8.2947 - acc: 0.0000e+00\n","name":"stdout"},{"output_type":"execute_result","execution_count":7,"data":{"text/plain":"<keras.callbacks.History at 0x7fe5df2acdd8>"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_pred_nn = model.predict(X_test_scaled).flatten()","execution_count":8,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Making data to submit.\n\n---------------------------\n\n提出用データの作成。"},{"metadata":{"trusted":true},"cell_type":"code","source":"sample = pd.read_csv(\"../input/sample_submission.csv\")\nsample['time_to_failure'] = y_pred_nn\nsample.to_csv('submission.csv',index=False)","execution_count":9,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Cross Validation.\n\n---------------------------\n\n交差検証。"},{"metadata":{"trusted":true},"cell_type":"code","source":"%time\n\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import log_loss, roc_auc_score\n\nn_estimators = [15,16]\nfunctions = ['relu']\nnumbers = [64,128]\nfor funcs in functions:\n  for number in numbers:\n    for n_est in  n_estimators:\n        print(n_est)\n        print(number)\n        print(funcs)\n        cv = KFold(n_splits=5, shuffle=True,random_state=0)\n        for train, valid in cv.split(X_train_scaled, y_tr):\n            x_train = X_train_scaled.iloc[train]\n            x_valid = X_train_scaled.iloc[valid]\n            y_train = y_tr.iloc[train]\n            y_valid = y_tr.iloc[valid]\n        \n            inputs = Input(shape=(24,))\n            x = Dense(number, activation=funcs)(inputs)\n            x = Dense(number, activation=funcs)(x)\n            predictions = Dense(1)(x)\n            model = Model(inputs=inputs, outputs=predictions)\n            model.compile(optimizer=tf.train.RMSPropOptimizer(0.001),\n                      loss='mse',\n                    metrics=['accuracy'])\n            model.fit(x_train, y_train, epochs=n_est, verbose=0)\n            y_pred_nn = model.predict(x_valid)\n            y_pred = y_pred_nn\n            print(mean_absolute_error(y_valid, y_pred))  ","execution_count":null,"outputs":[{"output_type":"stream","text":"CPU times: user 0 ns, sys: 0 ns, total: 0 ns\nWall time: 10.5 µs\n15\n64\nrelu\n2.3405214526156595\n2.3523465314171337\n2.3016325594056943\n2.31688229776901\n2.291593008028064\n16\n64\nrelu\n2.3444747597632034\n2.2942874239290876\n2.3678300082913015\n2.3267173563425616\n2.195877104222692\n15\n128\nrelu\n2.3052142118814793\n2.3352317691500835\n2.3367280551733756\n2.285286053278569\n2.2310413313773156\n16\n128\nrelu\n2.3019578612367515\n2.309181808223173\n2.299370191638223\n2.250706806736971\n","name":"stdout"}]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}