{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport math\nimport datetime\nfrom fastai.tabular.all import *\n# from fastai.tabular import *\nfrom fastai.imports import *\nfrom fastai.metrics import error_rate\n#from fastai.callbacks import *\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import Normalizer\nimport scipy.stats as spstats","metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport math\nimport datetime\nfrom fastai.tabular.all import *\n# from fastai.tabular import *\nfrom fastai.imports import *\nfrom fastai.metrics import error_rate\n#from fastai.callbacks import *\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import Normalizer\nimport scipy.stats as spstats","metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path = '/kaggle/input/predict-volcanic-eruptions-ingv-oe/'\noutput_path = '/kaggle/output/kaggle/working/modles/'","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**The below dataframe consist of the segment ID and the target value i.e. the time left for the volcano to erupt.\nFor each of the segment ID's we have been provided with a csv file with 10 mins of logs of readings belonging to 10 different sensors.**","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv(path+\"train.csv\")\ntrain","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission = pd.read_csv(\"../input/predict-volcanic-eruptions-ingv-oe/sample_submission.csv\")\nsample_submission","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['time_to_eruption'].describe()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_readings = glob.glob(path+\"train/*\")\nlen(train_readings)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_readings = glob.glob(path+\"test/*\")\nlen(test_readings)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_readings[0]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sensor_file = pd.read_csv(train_readings[0])\nsensor_file","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"For each segment ID, we have got 10 sensors and 60001 readings from each one of them.","metadata":{}},{"cell_type":"markdown","source":"### Feature Creation","metadata":{}},{"cell_type":"code","source":"def create_features(df,signal,seg_id,sensor_id):\n    f = np.fft.fft(signal)\n    f_real = np.real(f)\n    df.loc[seg_id, f'{sensor_id}_sum']       = signal.sum()\n    df.loc[seg_id, f'{sensor_id}_mean']      = signal.mean()\n    df.loc[seg_id, f'{sensor_id}_std']       = signal.std()\n    df.loc[seg_id, f'{sensor_id}_var']       = signal.var() \n    df.loc[seg_id, f'{sensor_id}_max']       = signal.max()\n    df.loc[seg_id, f'{sensor_id}_min']       = signal.min()\n    df.loc[seg_id, f'{sensor_id}_skew']      = signal.skew()\n    df.loc[seg_id, f'{sensor_id}_mad']       = signal.mad()\n    df.loc[seg_id, f'{sensor_id}_kurtosis']  = signal.kurtosis()\n    df.loc[seg_id, f'{sensor_id}_quantile99']= np.quantile(signal, 0.99)\n    df.loc[seg_id, f'{sensor_id}_quantile95']= np.quantile(signal, 0.95)\n    df.loc[seg_id, f'{sensor_id}_quantile85']= np.quantile(signal, 0.85)\n    df.loc[seg_id, f'{sensor_id}_quantile75']= np.quantile(signal, 0.75)\n    df.loc[seg_id, f'{sensor_id}_quantile55']= np.quantile(signal, 0.55)\n    df.loc[seg_id, f'{sensor_id}_quantile45']= np.quantile(signal, 0.45) \n    df.loc[seg_id, f'{sensor_id}_quantile25']= np.quantile(signal, 0.25) \n    df.loc[seg_id, f'{sensor_id}_quantile15']= np.quantile(signal, 0.15) \n    df.loc[seg_id, f'{sensor_id}_quantile05']= np.quantile(signal, 0.05)\n    df.loc[seg_id, f'{sensor_id}_quantile01']= np.quantile(signal, 0.01)\n    df.loc[seg_id, f'{sensor_id}_fft_real_mean']= f_real.mean()\n    df.loc[seg_id, f'{sensor_id}_fft_real_std'] = f_real.std()\n    df.loc[seg_id, f'{sensor_id}_fft_real_max'] = f_real.max()\n    df.loc[seg_id, f'{sensor_id}_fft_real_min'] = f_real.min()\n    df.loc[seg_id, f'{sensor_id}_fft_real_median'] = np.median(f_real)\n    df.loc[seg_id, f'{sensor_id}_fft_real_skew'] = spstats.skew(f_real)\n    df.loc[seg_id, f'{sensor_id}_fft_real_kurtosis'] = spstats.kurtosis(f_real)\n    \n    return df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"####  Create features for Training Data","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv(path+'train.csv')\ntrain_df = pd.DataFrame()\ntrain_df['segment_id'] = train.segment_id\ntrain_df = train_df.set_index('segment_id')\n\nj=0\nfor seg in train.segment_id:\n    signals = pd.read_csv(path+f'train/{seg}.csv')\n    if j%500 == 0:\n        print(j)\n    for i in range(1, 11):\n        sensor_id = f'sensor_{i}'\n        train_df = create_features(train_df, signals[sensor_id].fillna(0), seg, sensor_id,)\n    j+=1    ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.merge(train_df.reset_index(), train, on=['segment_id'], how='left').set_index('segment_id')\ntrain_df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = train_df.reset_index()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y = train_df['time_to_eruption']\ntrain_df = train_df.drop(['segment_id'], axis = 1)\ntrain_df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Create features for Test Data","metadata":{}},{"cell_type":"code","source":"test = pd.read_csv(path+'sample_submission.csv')\ntest_df = pd.DataFrame()\ntest_df['segment_id'] = test.segment_id\ntest_df = test_df.set_index('segment_id')\n\nj=0\nfor seg in test.segment_id:\n    signals = pd.read_csv(path+f'test/{seg}.csv')\n    if j%500 == 0:\n        print(j)\n    for i in range(1, 11):\n        sensor_id = f'sensor_{i}'\n        test_df = create_features(test_df, signals[sensor_id].fillna(0), seg, sensor_id,)\n    j+=1 ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = test_df.reset_index()\ntest_set = test_df\ntest_df = test_df.drop(['segment_id'], axis = 1)\ntest_df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.columns","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in list(train_df.columns):\n    print(i)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cont_names = list(train_df.columns)\n#removing time to eruption column\ncont_names.pop()\ncont_names","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cat_names = []\nprocs = [Categorify, FillMissing, Normalize]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"splits = RandomSplitter(valid_pct=0.2)(range_of(train_df))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"to = TabularPandas(train_df, procs=[Categorify, FillMissing,Normalize],\n                   cat_names = cat_names,\n                   cont_names = cont_names,\n                   y_names='time_to_eruption',\n                   splits=splits)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dls = to.dataloaders(bs=64)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dls.show_batch()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learn = tabular_learner(dls, layers=[200,100], metrics=mae, ps=[0.001,0.01], emb_drop=0.01)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learn.model","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learn.lr_find(suggestions=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learn.model_dir='/kaggle/working/' ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learn.fit_one_cycle(50,0.33113112449646,cbs=SaveModelCallback(monitor='mae', comp=np.less, fname=\"stage-1\"))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learn.load('stage-1')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dl = learn.dls.test_dl(test_df)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = learn.get_preds(dl=dl)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds[0]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_preds = []\nfor i in np.array(preds[0]):\n#     print(i[0])\n    test_preds.append(i[0])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test['time_to_eruption'] = test_preds","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.to_csv('submission.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Work in Progress","metadata":{}},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}