{"cells":[{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport math\nimport datetime\nfrom fastai.tabular.all import *\n# from fastai.tabular import *\nfrom fastai.imports import *\nfrom fastai.metrics import error_rate\n#from fastai.callbacks import *\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import Normalizer\nimport scipy.stats as spstats","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"path = '/kaggle/input/predict-volcanic-eruptions-ingv-oe/'\noutput_path = '/kaggle/output/kaggle/working/modles/'","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**The below dataframe consist of the segment ID and the target value i.e. the time left for the volcano to erupt.\nFor each of the segment ID's we have been provided with a csv file with 10 mins of logs of readings belonging to 10 different sensors.**"},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv(path+\"train.csv\")\ntrain","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_submission = pd.read_csv(\"../input/predict-volcanic-eruptions-ingv-oe/sample_submission.csv\")\nsample_submission","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['time_to_eruption'].describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_readings = glob.glob(path+\"train/*\")\nlen(train_readings)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_readings = glob.glob(path+\"test/*\")\nlen(test_readings)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_readings[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sensor_file = pd.read_csv(train_readings[0])\nsensor_file","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"For each segment ID, we have got 10 sensors and 60001 readings from each one of them."},{"metadata":{},"cell_type":"markdown","source":"### Feature Creation"},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_features(df,signal,seg_id,sensor_id):\n    f = np.fft.fft(signal)\n    f_real = np.real(f)\n    df.loc[seg_id, f'{sensor_id}_sum']       = signal.sum()\n    df.loc[seg_id, f'{sensor_id}_mean']      = signal.mean()\n    df.loc[seg_id, f'{sensor_id}_std']       = signal.std()\n    df.loc[seg_id, f'{sensor_id}_var']       = signal.var() \n    df.loc[seg_id, f'{sensor_id}_max']       = signal.max()\n    df.loc[seg_id, f'{sensor_id}_min']       = signal.min()\n    df.loc[seg_id, f'{sensor_id}_skew']      = signal.skew()\n    df.loc[seg_id, f'{sensor_id}_mad']       = signal.mad()\n    df.loc[seg_id, f'{sensor_id}_kurtosis']  = signal.kurtosis()\n    df.loc[seg_id, f'{sensor_id}_quantile99']= np.quantile(signal, 0.99)\n    df.loc[seg_id, f'{sensor_id}_quantile95']= np.quantile(signal, 0.95)\n    df.loc[seg_id, f'{sensor_id}_quantile85']= np.quantile(signal, 0.85)\n    df.loc[seg_id, f'{sensor_id}_quantile75']= np.quantile(signal, 0.75)\n    df.loc[seg_id, f'{sensor_id}_quantile55']= np.quantile(signal, 0.55)\n    df.loc[seg_id, f'{sensor_id}_quantile45']= np.quantile(signal, 0.45) \n    df.loc[seg_id, f'{sensor_id}_quantile25']= np.quantile(signal, 0.25) \n    df.loc[seg_id, f'{sensor_id}_quantile15']= np.quantile(signal, 0.15) \n    df.loc[seg_id, f'{sensor_id}_quantile05']= np.quantile(signal, 0.05)\n    df.loc[seg_id, f'{sensor_id}_quantile01']= np.quantile(signal, 0.01)\n    df.loc[seg_id, f'{sensor_id}_fft_real_mean']= f_real.mean()\n    df.loc[seg_id, f'{sensor_id}_fft_real_std'] = f_real.std()\n    df.loc[seg_id, f'{sensor_id}_fft_real_max'] = f_real.max()\n    df.loc[seg_id, f'{sensor_id}_fft_real_min'] = f_real.min()\n    df.loc[seg_id, f'{sensor_id}_fft_real_median'] = np.median(f_real)\n    df.loc[seg_id, f'{sensor_id}_fft_real_skew'] = spstats.skew(f_real)\n    df.loc[seg_id, f'{sensor_id}_fft_real_kurtosis'] = spstats.kurtosis(f_real)\n    \n    return df","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"####  Create features for Training Data"},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv(path+'train.csv')\ntrain_df = pd.DataFrame()\ntrain_df['segment_id'] = train.segment_id\ntrain_df = train_df.set_index('segment_id')\n\nj=0\nfor seg in train.segment_id:\n    signals = pd.read_csv(path+f'train/{seg}.csv')\n    if j%500 == 0:\n        print(j)\n    for i in range(1, 11):\n        sensor_id = f'sensor_{i}'\n        train_df = create_features(train_df, signals[sensor_id].fillna(0), seg, sensor_id,)\n    j+=1    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.merge(train_df.reset_index(), train, on=['segment_id'], how='left').set_index('segment_id')\ntrain_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = train_df.reset_index()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y = train_df['time_to_eruption']\ntrain_df = train_df.drop(['segment_id'], axis = 1)\ntrain_df","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Create features for Test Data"},{"metadata":{"trusted":true},"cell_type":"code","source":"test = pd.read_csv(path+'sample_submission.csv')\ntest_df = pd.DataFrame()\ntest_df['segment_id'] = test.segment_id\ntest_df = test_df.set_index('segment_id')\n\nj=0\nfor seg in test.segment_id:\n    signals = pd.read_csv(path+f'test/{seg}.csv')\n    if j%500 == 0:\n        print(j)\n    for i in range(1, 11):\n        sensor_id = f'sensor_{i}'\n        test_df = create_features(test_df, signals[sensor_id].fillna(0), seg, sensor_id,)\n    j+=1 ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df = test_df.reset_index()\ntest_set = test_df\ntest_df = test_df.drop(['segment_id'], axis = 1)\ntest_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for i in list(train_df.columns):\n    print(i)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cont_names = list(train_df.columns)\n#removing time to eruption column\ncont_names.pop()\ncont_names","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cat_names = []\nprocs = [Categorify, FillMissing, Normalize]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"splits = RandomSplitter(valid_pct=0.2)(range_of(train_df))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"to = TabularPandas(train_df, procs=[Categorify, FillMissing,Normalize],\n                   cat_names = cat_names,\n                   cont_names = cont_names,\n                   y_names='time_to_eruption',\n                   splits=splits)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dls = to.dataloaders(bs=64)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dls.show_batch()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn = tabular_learner(dls, layers=[200,100], metrics=mae, ps=[0.001,0.01], emb_drop=0.01)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.lr_find(suggestions=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.model_dir='/kaggle/working/' ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.fit_one_cycle(50,0.33113112449646,cbs=SaveModelCallback(monitor='mae', comp=np.less, fname=\"stage-1\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.load('stage-1')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dl = learn.dls.test_dl(test_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds = learn.get_preds(dl=dl)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_preds = []\nfor i in np.array(preds[0]):\n#     print(i[0])\n    test_preds.append(i[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test['time_to_eruption'] = test_preds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Work in Progress"},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}