{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import glob\nimport os\nimport pandas as pd \nfrom tqdm import tqdm\nfrom sklearn.metrics import mean_absolute_error,mean_squared_error\nfrom sklearn.ensemble import RandomForestRegressor\nfrom sklearn.model_selection import train_test_split","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Step:1**\n* load the data from train and test which consists of segment_id(i.e. ID code for the data segment. Matches the name of the associated data csv file contains the reding of the sensors)\n"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"#loading the train and test data\ntrain=pd.read_csv(\"/kaggle/input/predict-volcanic-eruptions-ingv-oe/train.csv\")\nsubmission=pd.read_csv(\"/kaggle/input/predict-volcanic-eruptions-ingv-oe/sample_submission.csv\")\n\n#segment id is for the csv files of the sensors\n_1136037770=pd.read_csv(\"/kaggle/input/predict-volcanic-eruptions-ingv-oe/train/1136037770.csv\")\n_1136037770.head()\n_1136037770.shape\n\n#getting the sum of every sensor reading in every csv\ndef col_trans(data):\n    for dat in data.columns.values:\n        data[f'{dat}_sum']=data[dat].sum()\n    data=data.iloc[0:1,10:]\n    return data\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Step 2:**\n* load the csv files from the directory as per the segment_id and perform operations for test and train and creating a new DataFrame and merge it with the existing DataFrame on segment_id feature.\n\n**Note:Uncomment and run the cell below for the first time (Making csv for the data merged)(may take sometime) Comment it after 1st run.**\n\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"#loading data into the df using glob\n#creating a new df for loading the data sum and concatenating it with the previous csv \ntrain_df=pd.DataFrame()\nfor i in tqdm(train[\"segment_id\"]):\n    raw_path=f'/kaggle/input/predict-volcanic-eruptions-ingv-oe/train/{i}.csv'\n    temp_df=pd.read_csv(raw_path)\n    temp_df=temp_df.fillna(0)\n    temp_df[\"segment_id\"]=i\n    temp_df=col_trans(temp_df)\n    train_df=pd.concat([train_df,temp_df],axis=0,ignore_index=True)\n    \n#dropping the column for sum of segment_ids\ntrain_df.drop([\"segment_id_sum\"],axis=1,inplace=True)\ntrain_df.shape\n\n\n#loading data into the df using glob\n#creating a new df for loading the data sum and concatenating it with the previous csv \ntest_df=pd.DataFrame()\nfor i in tqdm(submission[\"segment_id\"]):\n    raw_path=f'/kaggle/input/predict-volcanic-eruptions-ingv-oe/test/{i}.csv'\n    temp_df=pd.read_csv(raw_path)\n    temp_df=temp_df.fillna(0)\n    temp_df[\"segment_id\"]=i\n    temp_df=col_trans(temp_df)\n    test_df=pd.concat([test_df,temp_df],axis=0,ignore_index=True)\n    \n#dropping the column for sum of segment_ids\ntest_df.drop([\"segment_id_sum\"],axis=1,inplace=True)\ntest_df.shape\n\n#merging the train dataframe created with train on segment_id \ntrain=train.merge(train_df,on='segment_id')\n\n#merging the test dataframe created with test on segment_id\nsubmission=submission.merge(test_df,on='segment_id')\n\n#writing the data to csv files so that we wont be going through this step again and again\ntrain.to_csv(\"train_final.csv\",index=False)\nsubmission.to_csv(\"test_final.csv\",index=False)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#loading the files written in csv by merging\ntrain_df_final=pd.read_csv(\"./train_final.csv\")\ntest_df_final=pd.read_csv(\"./test_final.csv\")\ntrain_df_final.shape,test_df_final.shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Step 3:**\n* Split the data into train,validation and test for the modelling and train the model with train set and validate it on validation set to check the performance(mad and mse) and make predictions and make the submission file.  "},{"metadata":{"trusted":true},"cell_type":"code","source":"#splitting the data into train and validation\ntrain_split,val=train_test_split(train_df_final,test_size=.20,random_state=33)\nprint(f'train shape {train_split.shape} and validation shape {val.shape}')\n\n#train_full\ntrain_df_final_y=train_df_final.time_to_eruption\ntrain_df_final_x=train_df_final.drop([\"time_to_eruption\",\"segment_id\"],axis=1)\n\n#train splitted\ntrain_split_y=train_split[\"time_to_eruption\"]\ntrain_split_x=train_split.drop([\"time_to_eruption\",\"segment_id\"],axis=1)\n\n#validation\nval_split_y=val.time_to_eruption\nval_split_x=val.drop([\"time_to_eruption\",\"segment_id\"],axis=1)\n\n#test\ntest_split_x=test_df_final.drop([\"time_to_eruption\",\"segment_id\"],axis=1)\n\n#defining a Random forst regressor\nrfr=RandomForestRegressor()\nrfr.fit(train_split_x,train_split_y)\npredicted=rfr.predict(val_split_x)\n\nprint(f'mean squared error is {mean_squared_error(predicted,val_split_y)}')\nprint(f'mean absolute error is {mean_absolute_error(predicted,val_split_y)}')\nrfr.fit(train_df_final_x,train_df_final_y)\npredicted=rfr.predict(test_split_x)\nsubmission=pd.DataFrame({\"segment_id\":test_df_final[\"segment_id\"],\"time_to_eruption\":predicted})\nsubmission.to_csv(\"submission_rfr.csv\",index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}