{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import train_test_split\nimport seaborn as sns\nimport os\nfrom tqdm import tqdm\nimport gc\nimport pickle","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# train and submission"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train = pd.read_csv(\"../input/predict-volcanic-eruptions-ingv-oe/train.csv\")\nsample_submission = pd.read_csv(\"../input/predict-volcanic-eruptions-ingv-oe/sample_submission.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.isna().sum(axis=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.distplot(train['time_to_eruption'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_submission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_submission.shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data files"},{"metadata":{},"cell_type":"markdown","source":"### example1 in train"},{"metadata":{"trusted":true},"cell_type":"code","source":"# take 1000015382.csv for example\ntest_data = pd.read_csv('../input/predict-volcanic-eruptions-ingv-oe/train/1000015382.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('shape of 1000015382.csv', test_data.shape)\ntest_data.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, axs = plt.subplots(2, 5, figsize=(50, 30))\n\nfor i, ax in enumerate(axs.ravel()):\n    ax.plot(test_data['sensor_'+str(i+1)])\n    ax.set_title('sensor_'+str(i+1))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.distplot(test_data['sensor_1'], label='sensor_1')\nsns.distplot(test_data['sensor_2'], label='sensor_2')\nsns.distplot(test_data['sensor_3'], label='sensor_3')\nplt.legend()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train.loc[train['segment_id']==1000015382])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### example2 in test"},{"metadata":{"trusted":true},"cell_type":"code","source":"test_data2 = pd.read_csv('../input/predict-volcanic-eruptions-ingv-oe/test/1001028887.csv')\n\nfig, axs = plt.subplots(2, 5, figsize=(50, 30))\n\nfor i, ax in enumerate(axs.ravel()):\n    ax.plot(test_data2['sensor_'+str(i+1)])\n    ax.set_title('sensor_'+str(i+1))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.distplot(test_data2['sensor_1'], label='sensor_1')\nsns.distplot(test_data2['sensor_4'], label='sensor_4')\nsns.distplot(test_data2['sensor_3'], label='sensor_3')\nplt.legend()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### example 3 in train"},{"metadata":{"trusted":true},"cell_type":"code","source":"test_data3 = pd.read_csv('../input/predict-volcanic-eruptions-ingv-oe/train/1000745424.csv')\n\nfig, axs = plt.subplots(2, 5, figsize=(50, 30))\n\nfor i, ax in enumerate(axs.ravel()):\n    ax.plot(test_data3['sensor_'+str(i+1)])\n    ax.set_title('sensor_'+str(i+1))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.distplot(test_data3['sensor_1'], label='sensor_1')\nsns.distplot(test_data3['sensor_2'], label='sensor_2')\nsns.distplot(test_data3['sensor_3'], label='sensor_3')\nplt.legend()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train.loc[train['segment_id']==1000745424])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"##  the datas in the ten minutes of logs are very different"},{"metadata":{},"cell_type":"markdown","source":"***MODEL***"},{"metadata":{"trusted":true},"cell_type":"code","source":"# generate feature\n# collect mean / std / 5 / 10 / 20 / 40 percentile / min / max / +5000 / +10000 / +20000 self-corr\ndef generate_feature():\n    \n    def helper(path):\n        data = []\n        for file in tqdm(os.listdir(path)):\n            tmp = []\n            file_path = os.path.join(path, file)\n            d = pd.read_csv(file_path)\n            tmp.append(eval(file[:-4]))\n            # mean\n            tmp += d.mean(axis=0).values.astype('float32').tolist()\n            # std\n            tmp += d.std(axis=0).values.astype('float32').tolist()\n            # min\n            tmp += d.min(axis=0).values.astype('float32').tolist()\n            # max\n            tmp += d.max(axis=0).values.astype('float32').tolist()\n            # 5 percentile\n            tmp += d.quantile(0.05, axis=0).values.astype('float32').tolist()\n            # 10 percentile\n            tmp += d.quantile(0.1, axis=0).values.astype('float32').tolist()\n            # 20 percentile\n            tmp += d.quantile(0.2, axis=0).values.astype('float32').tolist()\n            # 40 percentile\n            tmp += d.quantile(0.4, axis=0).values.astype('float32').tolist()\n            # shift\n            for col in d:\n                d[col+'_5000'] = d[col].shift(5000)\n                d[col+'_10000'] = d[col].shift(5000)\n                d[col+'_20000'] = d[col].shift(5000)\n            # +5000 / +10000 / +20000 self-corr\n            for col in d.columns[:10]:\n                col1 = col+'_5000'\n                col2 = col+'_10000'\n                col3 = col+'_20000'\n                tmp1 = d.loc[:, [col, col1]].dropna()\n                tmp2 = d.loc[:, [col, col2]].dropna()\n                tmp3 = d.loc[:, [col, col3]].dropna()\n                tmp += [tmp1[col].corr(tmp1[col1]), tmp2[col].corr(tmp2[col2]), tmp3[col].corr(tmp3[col3])]\n                \n            data.append(tmp)\n        return data\n                   \n    print('train_part: ')\n    train_part_fea = helper('../input/predict-volcanic-eruptions-ingv-oe/train')\n    print('test_part: ')\n    test_part_fea = helper('../input/predict-volcanic-eruptions-ingv-oe/test')\n    \n    return train_part_fea, test_part_fea","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#def na_mark(data, file_has_na_name):\n#    name = set([eval(i[:-4]) for i in file_has_na_name])\n#    data['na_mark'] = 0\n#    data.loc[data['segment_id'].isin(name), 'na_mark'] = 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_part_fea, test_part_fea = generate_feature()\n\nwith open('train_part_fea.pkl', 'wb') as f1:\n    pickle.dump(train_part_fea, f1)\n    \nwith open('test_part_fea.pkl', 'wb') as f2:\n    pickle.dump(test_part_fea, f2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#train_part_fea = pd.read_pickle('../input/ingv-eda-basemodel/train_part_fea.pkl')\n#test_part_fea = pd.read_pickle('../input/ingv-eda-basemodel/test_part_fea.pkl')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"base_colname = ['sensor_'+str(i) for i in range(1, 11)]\nfea_colname = ['segment_id'] + [j + '_mean' for j in base_colname] + [j + '_std' for j in base_colname] + \\\n                [j + '_min' for j in base_colname] + [j + '_max' for j in base_colname] + \\\n                    [j + '_5_quant' for j in base_colname] + [j + '_10_quant' for j in base_colname] + \\\n                        [j + '_20_quant' for j in base_colname] + [j + '_40_quant' for j in base_colname] + \\\n                    [j + i for j in base_colname for i in ['_5000_self_corr', '_10000_self_corr', '_20000_self_corr']]\n\ntrain = pd.merge(train, pd.DataFrame(train_part_fea, columns=fea_colname), on='segment_id', how='left')\nsample_submission = pd.merge(sample_submission, pd.DataFrame(test_part_fea, columns=fea_colname), on='segment_id', how='left')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train, X_val, y_train, y_val = train_test_split(train.drop(['segment_id', 'time_to_eruption'], axis=1).values, \n                                                    train['time_to_eruption'].values, \n                                                    test_size=0.25, random_state=42)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import lightgbm as lgb\n\ntrain_data = lgb.Dataset(X_train, label=y_train)\nval_data = lgb.Dataset(X_val, y_val, reference=train_data,)\n\n#params = {'objective': 'mae', \n#          'num_iterations': ,\n#          'learning_rate': , \n#          'num_leaves': ,\n#          'seed': ,\n#          'metric': 'mae'}\n\nparams = { 'num_leaves': 85,\n          'n_estimators': 6000,\n    'min_data_in_leaf': 10, \n    'objective':'mae',\n    'max_depth': -1,\n    'learning_rate': 0.01,\n    'max_bins': 2048,\n    \"boosting\": \"gbdt\",\n    \"feature_fraction\": 0.91,\n    \"bagging_freq\": 1,\n    \"bagging_fraction\": 0.91,\n    \"bagging_seed\": 42,\n    \"metric\": 'mae',\n    \"lambda_l1\": 0.1,\n    \"verbosity\": -1,\n    \"nthread\": -1,\n    \"random_state\": 42}\n\nmodel = lgb.train(params=params, train_set=train_data, valid_sets=[train_data, val_data], valid_names=['train', 'val'], \n                  early_stopping_rounds=50)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## predict"},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.DataFrame({'segment_id': sample_submission['segment_id'].values, \n    'time_to_eruption': model.predict(sample_submission.iloc[:, 2:].values)})\nsubmission.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}