{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nimport os\n\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nplt.style.use('seaborn-white')\n%matplotlib inline\n\nfrom sklearn.model_selection import train_test_split, GridSearchCV\nimport lightgbm as lgbm\nfrom lightgbm import LGBMRegressor\nfrom xgboost import XGBRegressor\n\nfile_list=[]\nfile_list_train=[]\nfile_list_test=[]\n\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        file_list.append(os.path.join(dirname, filename))\n\nPATH = '/kaggle/input/predict-volcanic-eruptions-ingv-oe/'\n\nfor dirname, _, filenames in os.walk('/kaggle/input/predict-volcanic-eruptions-ingv-oe/train'):\n    for filename in filenames:\n        file_list_train.append(os.path.join(dirname, filename))\n\nfor dirname, _, filenames in os.walk('/kaggle/input/predict-volcanic-eruptions-ingv-oe/test'):\n    for filename in filenames:\n        file_list_test.append(os.path.join(dirname, filename))","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-12-21T14:18:40.618003Z","iopub.execute_input":"2021-12-21T14:18:40.6186Z","iopub.status.idle":"2021-12-21T14:18:46.108175Z","shell.execute_reply.started":"2021-12-21T14:18:40.618467Z","shell.execute_reply":"2021-12-21T14:18:46.107445Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(file_list[0])\nprint(pd.read_csv(file_list[0]))","metadata":{"execution":{"iopub.status.busy":"2021-12-21T14:18:46.110145Z","iopub.execute_input":"2021-12-21T14:18:46.110772Z","iopub.status.idle":"2021-12-21T14:18:46.138333Z","shell.execute_reply.started":"2021-12-21T14:18:46.110723Z","shell.execute_reply":"2021-12-21T14:18:46.137684Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(file_list_train[0])\nprint(pd.read_csv(file_list_train[0]))","metadata":{"execution":{"iopub.status.busy":"2021-12-21T14:18:46.139346Z","iopub.execute_input":"2021-12-21T14:18:46.139675Z","iopub.status.idle":"2021-12-21T14:18:46.250177Z","shell.execute_reply.started":"2021-12-21T14:18:46.139648Z","shell.execute_reply":"2021-12-21T14:18:46.249276Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(file_list[1])\nprint(pd.read_csv(file_list[1]))","metadata":{"execution":{"iopub.status.busy":"2021-12-21T14:18:46.252446Z","iopub.execute_input":"2021-12-21T14:18:46.252758Z","iopub.status.idle":"2021-12-21T14:18:46.268989Z","shell.execute_reply.started":"2021-12-21T14:18:46.252715Z","shell.execute_reply":"2021-12-21T14:18:46.26795Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#file_list_test\nprint(len(file_list_test))\nkeys = list(pd.read_csv(file_list_test[0]).keys())\nprint(keys)\nnanCount = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\nfor index in range(len(file_list_test)):\n    if(index % 200 == 0):\n        print(index)\n    df = pd.read_csv(file_list_test[index])\n    for key in df.keys():\n        if df[key].isna().sum() == 60001:\n            nanCount[keys.index(key)] += 1\nprint(nanCount)\ndata={'sensors': keys, 'count': nanCount}\nmyFrame = pd.DataFrame(data)\nprint(myFrame)\n\n    ","metadata":{"execution":{"iopub.status.busy":"2021-12-21T14:18:46.269927Z","iopub.execute_input":"2021-12-21T14:18:46.270346Z","iopub.status.idle":"2021-12-21T14:25:40.562154Z","shell.execute_reply.started":"2021-12-21T14:18:46.270311Z","shell.execute_reply":"2021-12-21T14:25:40.561314Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"myFrame.plot(figsize =(20, 20), x=\"sensors\", y=\"count\", kind=\"bar\",  rot=5, fontsize=14 )","metadata":{"execution":{"iopub.status.busy":"2021-12-21T14:25:40.563871Z","iopub.execute_input":"2021-12-21T14:25:40.564348Z","iopub.status.idle":"2021-12-21T14:25:40.964754Z","shell.execute_reply.started":"2021-12-21T14:25:40.564312Z","shell.execute_reply":"2021-12-21T14:25:40.963856Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(file_list_test[0])\nprint(pd.read_csv(file_list_test[0]))","metadata":{"execution":{"iopub.status.busy":"2021-12-21T14:25:40.965895Z","iopub.execute_input":"2021-12-21T14:25:40.96612Z","iopub.status.idle":"2021-12-21T14:25:41.045326Z","shell.execute_reply.started":"2021-12-21T14:25:40.966092Z","shell.execute_reply":"2021-12-21T14:25:41.044333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(file_list_test))","metadata":{"execution":{"iopub.status.busy":"2021-12-21T14:25:41.046793Z","iopub.execute_input":"2021-12-21T14:25:41.047158Z","iopub.status.idle":"2021-12-21T14:25:41.052563Z","shell.execute_reply.started":"2021-12-21T14:25:41.047114Z","shell.execute_reply":"2021-12-21T14:25:41.051755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(file_list_train))","metadata":{"execution":{"iopub.status.busy":"2021-12-21T14:25:41.054004Z","iopub.execute_input":"2021-12-21T14:25:41.054318Z","iopub.status.idle":"2021-12-21T14:25:41.063402Z","shell.execute_reply.started":"2021-12-21T14:25:41.054278Z","shell.execute_reply":"2021-12-21T14:25:41.062055Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"files_train = [file.split('/')[-1].split('.')[-2] for file in file_list_train]\nfiles_test = [file.split('/')[-1].split('.')[-2] for file in file_list_test]\nprint(files_train[0:10])\nprint(files_test[0:10])\n","metadata":{"execution":{"iopub.status.busy":"2021-12-21T14:25:41.06793Z","iopub.execute_input":"2021-12-21T14:25:41.068309Z","iopub.status.idle":"2021-12-21T14:25:41.087624Z","shell.execute_reply.started":"2021-12-21T14:25:41.068264Z","shell.execute_reply":"2021-12-21T14:25:41.086661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_set = set(files_test)\ntrain_set = set(files_train)\ninter = test_set.intersection(train_set)\nprint(inter)","metadata":{"execution":{"iopub.status.busy":"2021-12-21T14:25:41.095255Z","iopub.execute_input":"2021-12-21T14:25:41.095498Z","iopub.status.idle":"2021-12-21T14:25:41.102552Z","shell.execute_reply.started":"2021-12-21T14:25:41.09547Z","shell.execute_reply":"2021-12-21T14:25:41.101688Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv(PATH+'train.csv')","metadata":{"execution":{"iopub.status.busy":"2021-12-21T14:25:41.104069Z","iopub.execute_input":"2021-12-21T14:25:41.104299Z","iopub.status.idle":"2021-12-21T14:25:41.335307Z","shell.execute_reply.started":"2021-12-21T14:25:41.104272Z","shell.execute_reply":"2021-12-21T14:25:41.334229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.distplot(train['time_to_eruption'], hist=True, kde=True, bins=100, color='blue', hist_kws={'edgecolor':'black'})","metadata":{"execution":{"iopub.status.busy":"2021-12-21T14:25:41.336609Z","iopub.execute_input":"2021-12-21T14:25:41.336853Z","iopub.status.idle":"2021-12-21T14:25:41.761505Z","shell.execute_reply.started":"2021-12-21T14:25:41.336825Z","shell.execute_reply":"2021-12-21T14:25:41.760657Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['time_to_eruption'].describe()","metadata":{"execution":{"iopub.status.busy":"2021-12-21T14:25:41.762815Z","iopub.execute_input":"2021-12-21T14:25:41.763649Z","iopub.status.idle":"2021-12-21T14:25:41.776933Z","shell.execute_reply.started":"2021-12-21T14:25:41.76361Z","shell.execute_reply":"2021-12-21T14:25:41.775889Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_segment_id = pd.read_csv(PATH+'/train/800654756.csv')\n\ndf_segment_id.plot(figsize=(20, 20), subplots=True, layout=(10, 1), rot = 0,\n                  lw=1, title='segment_id #800654756')","metadata":{"execution":{"iopub.status.busy":"2021-12-21T14:25:41.778519Z","iopub.execute_input":"2021-12-21T14:25:41.779352Z","iopub.status.idle":"2021-12-21T14:25:44.397003Z","shell.execute_reply.started":"2021-12-21T14:25:41.779315Z","shell.execute_reply":"2021-12-21T14:25:44.396269Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(train.sort_values('time_to_eruption', axis=0, ascending=True).iloc[[0, -1], :])\n\nsegment_id_min = 601524801\nsegment_id_max = 1923243961\n\ndf_segment_id_min = pd.read_csv(PATH+'/train/'+str(segment_id_min)+'.csv')\ndf_segment_id_max = pd.read_csv(PATH+'/train/'+str(segment_id_max)+'.csv')","metadata":{"execution":{"iopub.status.busy":"2021-12-21T14:25:44.398615Z","iopub.execute_input":"2021-12-21T14:25:44.399238Z","iopub.status.idle":"2021-12-21T14:25:44.593551Z","shell.execute_reply.started":"2021-12-21T14:25:44.399164Z","shell.execute_reply":"2021-12-21T14:25:44.592679Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_segment_id_min.plot(figsize=(20,20), subplots=True, layout=(10,1), rot=0, lw=1, title='segment_id #601524801 (min)')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-12-21T14:25:44.594766Z","iopub.execute_input":"2021-12-21T14:25:44.595069Z","iopub.status.idle":"2021-12-21T14:25:46.78789Z","shell.execute_reply.started":"2021-12-21T14:25:44.595023Z","shell.execute_reply":"2021-12-21T14:25:46.786928Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_segment_id_max.plot(figsize=(20,20), subplots=True, layout=(10,1), rot=0, lw=1, title='segment_id #1923243961 (min)')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-12-21T14:25:46.789186Z","iopub.execute_input":"2021-12-21T14:25:46.789444Z","iopub.status.idle":"2021-12-21T14:25:49.272816Z","shell.execute_reply.started":"2021-12-21T14:25:46.789414Z","shell.execute_reply":"2021-12-21T14:25:49.272255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def build_features(signal, ts, sensor_id):\n    X = pd.DataFrame()\n    f = np.fft.fft(signal)\n    f_real = np.real(f)\n    X.loc[ts, f'{sensor_id}_sum'] = signal.sum()\n    X.loc[ts, f'{sensor_id}_mean'] = signal.mean()\n    X.loc[ts, f'{sensor_id}_std'] = signal.std()\n    X.loc[ts, f'{sensor_id}_var'] = signal.var()\n    X.loc[ts, f'{sensor_id}_max'] = signal.max()\n    X.loc[ts, f'{sensor_id}_min'] = signal.min()\n    X.loc[ts, f'{sensor_id}_skew'] = signal.skew()\n    X.loc[ts, f'{sensor_id}_mad'] = signal.mad()\n    X.loc[ts, f'{sensor_id}_kurtosis'] = signal.kurtosis()\n    X.loc[ts, f'{sensor_id}_quantile99'] = np.quantile(signal, 0.99)\n    X.loc[ts, f'{sensor_id}_quantile95'] = np.quantile(signal, 0.95)\n    X.loc[ts, f'{sensor_id}_quantile85'] = np.quantile(signal, 0.85)\n    X.loc[ts, f'{sensor_id}_quantile75'] = np.quantile(signal, 0.75)\n    X.loc[ts, f'{sensor_id}_quantile55'] = np.quantile(signal, 0.55)\n    X.loc[ts, f'{sensor_id}_quantile45'] = np.quantile(signal, 0.45)\n    X.loc[ts, f'{sensor_id}_quantile25'] = np.quantile(signal, 0.25)\n    X.loc[ts, f'{sensor_id}_quantile15'] = np.quantile(signal, 0.15)\n    X.loc[ts, f'{sensor_id}_quantile05'] = np.quantile(signal, 0.05)\n    X.loc[ts, f'{sensor_id}_quantile01'] = np.quantile(signal, 0.01)\n    X.loc[ts, f'{sensor_id}_fft_real_mean'] = f_real.mean()\n    X.loc[ts, f'{sensor_id}_fft_real_std'] = f_real.std()\n    X.loc[ts, f'{sensor_id}_fft_real_max'] = f_real.max()\n    X.loc[ts, f'{sensor_id}_fft_real_min'] = f_real.min()\n    \n    return X\n   ","metadata":{"execution":{"iopub.status.busy":"2021-12-21T14:25:49.27389Z","iopub.execute_input":"2021-12-21T14:25:49.274192Z","iopub.status.idle":"2021-12-21T14:25:49.284341Z","shell.execute_reply.started":"2021-12-21T14:25:49.274165Z","shell.execute_reply":"2021-12-21T14:25:49.283675Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from itertools import islice\n\n\ntrain_set = list()\nseg=0\nprint(enumerate(train.segment_id))\n\nfor seg, segment_id in enumerate(train.segment_id):\n    signals = pd.read_csv(PATH+'/train/'+str(segment_id)+'.csv')\n    train_row=[]\n#     print(segment_id)\n    if seg % 200 == 0:\n        print('Processing segment_id={}'.format(seg))\n    \n    for sensor in range(0, 10):\n        sensor_id = f'sensor_{sensor+1}'\n        train_row.append(build_features(signals[sensor_id].fillna(0), segment_id, sensor_id))\n    \n    train_row = pd.concat(train_row, axis=1)\n    train_set.append(train_row)\n    seg+=1\n    \ntrain_set = pd.concat(train_set)\nprint(train_set)","metadata":{"execution":{"iopub.status.busy":"2021-12-21T14:25:49.285524Z","iopub.execute_input":"2021-12-21T14:25:49.28586Z","iopub.status.idle":"2021-12-21T15:03:43.554083Z","shell.execute_reply.started":"2021-12-21T14:25:49.285829Z","shell.execute_reply":"2021-12-21T15:03:43.553257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_set = train_set.reset_index()\ntrain_set = train_set.rename(columns={'index': 'segment_id'})\nprint(train_set)\ntrain_set = pd.merge(train_set, train, on='segment_id')\nprint(train_set)","metadata":{"execution":{"iopub.status.busy":"2021-12-21T15:03:43.555482Z","iopub.execute_input":"2021-12-21T15:03:43.555721Z","iopub.status.idle":"2021-12-21T15:03:43.628884Z","shell.execute_reply.started":"2021-12-21T15:03:43.555693Z","shell.execute_reply":"2021-12-21T15:03:43.628064Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train_set.head(3))","metadata":{"execution":{"iopub.status.busy":"2021-12-21T15:03:43.630377Z","iopub.execute_input":"2021-12-21T15:03:43.630809Z","iopub.status.idle":"2021-12-21T15:03:43.644708Z","shell.execute_reply.started":"2021-12-21T15:03:43.630779Z","shell.execute_reply":"2021-12-21T15:03:43.643813Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_files = []\nfor dirname, _, filenames in os.walk(PATH+'/test/'):\n    for filename in filenames:\n        test_files.append(filename[:-4])\n\ntest = pd.DataFrame(test_files, columns=['segment_id'])","metadata":{"execution":{"iopub.status.busy":"2021-12-21T15:03:43.646077Z","iopub.execute_input":"2021-12-21T15:03:43.646377Z","iopub.status.idle":"2021-12-21T15:03:44.499869Z","shell.execute_reply.started":"2021-12-21T15:03:43.646348Z","shell.execute_reply":"2021-12-21T15:03:44.499278Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_set = list()\nseg=0\n\nfor seg, segment_id in enumerate(test.segment_id):\n    signals = pd.read_csv(PATH+'/test/'+str(segment_id)+'.csv')\n    test_row=[]\n    \n    if seg%200 == 0:\n        print('Processing segment_id={}'.format(seg))\n        \n    for sensor in range(0, 10):\n        sensor_id = f'sensor_{sensor+1}'\n        test_row.append(build_features(signals[sensor_id].fillna(0), segment_id, sensor_id))\n    \n    test_row = pd.concat(test_row, axis=1)\n    test_set.append(test_row)\n    seg+=1\n    \ntest_set = pd.concat(test_set)\nprint(test_set)\n","metadata":{"execution":{"iopub.status.busy":"2021-12-21T15:03:44.50134Z","iopub.execute_input":"2021-12-21T15:03:44.501586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_set=test_set.reset_index()\ntest_set=test_set.rename(columns={'index': 'segment_id'})\nprint(test_set)\ntest_set=pd.merge(test_set, test, on=\"segment_id\")\nprint(test_set)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(test_set.head(3))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = train_set.drop(['segment_id', 'time_to_eruption'], axis=1)\ny = train_set['time_to_eruption']\n\nX_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.2, random_state=42)\n\nprint(X_train.head(3))\nprint('np.shape(X_train) = ', np.shape(X_train))\n\nprint(y_train.head(3))\nprint('np.shape(y_train) = ', np.shape(y_train))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.ensemble import RandomForestRegressor\nmodel = RandomForestRegressor(max_depth=20, random_state=0)\nmodel.fit(X_train, y_train)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = model.predict(X_valid)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import mean_squared_error\nmse = mean_squared_error(y_valid, y_pred)\nfig=plt.figure()\nmulreg = fig.add_subplot(1,1,1)\nmulreg.scatter(y_valid, y_pred, color='r')\nmulreg.set_title('Nonlinear Regression ')\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions = model.predict(test_set.drop(columns=['segment_id'], axis=1))\n\nsubmission = pd.DataFrame()\nsubmission['segment_id'] = test_set['segment_id']\nsubmission['time_to_eruption'] = predictions\nsubmission.to_csv('submission.csv', header=True, index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}