{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Enviroments"},{"metadata":{"trusted":true},"cell_type":"code","source":"import seaborn as sns\nimport matplotlib.pyplot as plt\nplt.style.use('seaborn-white')\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Kaggle Data"},{"metadata":{"trusted":true},"cell_type":"code","source":"PATH = '/kaggle/input/predict-volcanic-eruptions-ingv-oe/'\n\n!echo 'Files: train and test'\ntrain_files = []\ntest_files  = []\n\nfor file in os.listdir(PATH+'/train/'):\n    train_files.append(file)\n    \nfor file in os.listdir(PATH+'/test/'):\n    test_files.append(file)\n    \nprint('Number of train files: {}'.format(len(train_files)))\nprint('Number of test  files: {}'.format(len(test_files )))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":""},{"metadata":{"trusted":true},"cell_type":"code","source":"# Train file\ntrain = pd.read_csv(PATH+'train.csv')\n\n# Submission file\nsample_submission = pd.read_csv(PATH+'sample_submission.csv')\n\ntest_files = []\nfor dirname, _, filenames in os.walk(PATH+'/test/'):\n    for filename in filenames:\n        test_files.append(filename[:-4]) # without .csv extension\n        \ntest = pd.DataFrame(test_files, columns=[\"segment_id\"])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":""},{"metadata":{"trusted":true},"cell_type":"code","source":"test","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# The relation between eruption time"},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.distplot(train['time_to_eruption'], \n             hist=True, \n             kde=True, \n             bins=100, \n             color = 'blue', \n             hist_kws={'edgecolor':'black'})","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"*Deriving the mininum and maximum time of erruption and it's segment id*"},{"metadata":{"trusted":true},"cell_type":"code","source":"display(train.sort_values('time_to_eruption', axis=0, ascending=True).iloc[[0,-1],:])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"segment_id_min =  601524801\nsegment_id_max = 1923243961\ndf_segment_id_min = pd.read_csv(PATH+'/train/'+str(segment_id_min)+'.csv')\ndf_segment_id_max = pd.read_csv(PATH+'/train/'+str(segment_id_max)+'.csv')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Data about the sensors of the  maximum enurption time segment "},{"metadata":{"trusted":true},"cell_type":"code","source":"\ndf_segment_id_min.plot(figsize=(20,20),\n                       subplots=True, \n                       layout=(10,1),\n                       rot=0, \n                       lw=1, \n                       #colormap='jet',\n                       title='601524801 (min)'\n                      )\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_segment_id_min.plot(figsize=(20,20),\n                       subplots=False, \n                       layout=(10,1),\n                       rot=0, \n                       lw=1, \n                       #colormap='jet',\n                       title='601524801 (min)'\n                      )\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Data about the sensors of the  maximum enurption time segment "},{"metadata":{"trusted":true},"cell_type":"code","source":"\ndf_segment_id_max.plot(figsize=(20,20),\n                       subplots=True, \n                       layout=(10,1),\n                       rot=0, \n                       lw=1, \n                       #colormap='jet',\n                       title='1923243961 (min)'\n                      )\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_segment_id_max.plot(figsize=(20,20),\n                       subplots=False, \n                       layout=(10,1),\n                       rot=0, \n                       lw=2, \n                       #colormap='jet',\n                       title='1923243961 (max)'\n                      )\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":""},{"metadata":{},"cell_type":"markdown","source":"This clear function below is taken from Kostiantyn Isaienkov to derive data from the sensors"},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_features(signal, ts, sensor_id):\n    X = pd.DataFrame()\n    f = np.fft.fft(signal)\n    f_real = np.real(f)\n    X.loc[ts, f'{sensor_id}_sum']       = signal.sum()\n    X.loc[ts, f'{sensor_id}_mean']      = signal.mean()\n    X.loc[ts, f'{sensor_id}_std']       = signal.std()\n    X.loc[ts, f'{sensor_id}_var']       = signal.var() \n    X.loc[ts, f'{sensor_id}_max']       = signal.max()\n    X.loc[ts, f'{sensor_id}_min']       = signal.min()\n    X.loc[ts, f'{sensor_id}_skew']      = signal.skew()\n    X.loc[ts, f'{sensor_id}_mad']       = signal.mad()\n    X.loc[ts, f'{sensor_id}_kurtosis']  = signal.kurtosis()\n    X.loc[ts, f'{sensor_id}_quantile99']= np.quantile(signal, 0.99)\n    X.loc[ts, f'{sensor_id}_quantile95']= np.quantile(signal, 0.95)\n    X.loc[ts, f'{sensor_id}_quantile85']= np.quantile(signal, 0.85)\n    X.loc[ts, f'{sensor_id}_quantile75']= np.quantile(signal, 0.75)\n    X.loc[ts, f'{sensor_id}_quantile55']= np.quantile(signal, 0.55)\n    X.loc[ts, f'{sensor_id}_quantile45']= np.quantile(signal, 0.45) \n    X.loc[ts, f'{sensor_id}_quantile25']= np.quantile(signal, 0.25) \n    X.loc[ts, f'{sensor_id}_quantile15']= np.quantile(signal, 0.15) \n    X.loc[ts, f'{sensor_id}_quantile05']= np.quantile(signal, 0.05)\n    X.loc[ts, f'{sensor_id}_quantile01']= np.quantile(signal, 0.01)\n    X.loc[ts, f'{sensor_id}_fft_real_mean']= f_real.mean()\n    X.loc[ts, f'{sensor_id}_fft_real_std'] = f_real.std()\n    X.loc[ts, f'{sensor_id}_fft_real_max'] = f_real.max()\n    X.loc[ts, f'{sensor_id}_fft_real_min'] = f_real.min()\n\n    return X","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Building datasets"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_set = list()\nseg=0\n\nfor seg, segment_id in enumerate(train.segment_id):\n    signals = pd.read_csv(PATH+'/train/'+str(segment_id)+'.csv')\n    train_row = []\n    \n    if seg%200 == 0:\n        print('Processing segment_id={}'.format(seg))\n        \n    for sensor in range(0, 10):\n        sensor_id = f'sensor_{sensor+1}'\n        train_row.append(build_features(signals[sensor_id].fillna(0), segment_id, sensor_id))\n        \n    train_row = pd.concat(train_row, axis=1)\n    train_set.append(train_row)\n    seg+=1\n    \ntrain_set = pd.concat(train_set)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"*  saving the training set with all the trash to be sure"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_set_safe=train_set\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_set = train_set.rename(columns={'index': 'segment_id'})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_set\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X = train_set.drop(['segment_id','level_0'], axis=1)\nY = train.drop(['time_to_eruption'], axis=1)\nX=Y.join(X)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_set=X","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_set = list()\nseg=0\n\nfor seg, segment_id in enumerate(test.segment_id):\n    signals = pd.read_csv(PATH+'/test/'+str(segment_id)+'.csv')\n    test_row = []\n    \n    if seg%200 == 0:\n        print('Processing segment_id={}'.format(seg))\n        \n    for sensor in range(0, 10):\n        sensor_id = f'sensor_{sensor+1}'\n        test_row.append(build_features(signals[sensor_id].fillna(0), segment_id, sensor_id))\n        \n    test_row = pd.concat(test_row, axis=1)\n    test_set.append(test_row)\n    seg+=1\n    \ntest_set = pd.concat(test_set)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_set","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}