{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np \nimport pandas as pd \nimport os\nimport seaborn as sns\nimport matplotlib\nfrom matplotlib import pyplot as plt\nfrom lightgbm import LGBMRegressor\nimport tensorflow as tf\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.feature_selection import RFE \nfrom sklearn.tree import DecisionTreeRegressor\nfrom sklearn.metrics import mean_squared_error as mse\n\n\n\n\nroot_filepath = \"../input/predict-volcanic-eruptions-ingv-oe/\"\nsave_filepath = \"./\"\nos.mkdir(\"./preprocessed\")\nos.mkdir(\"./predictions\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"def get_signals(id, folder=\"train\"):\n    return pd.read_csv(root_filepath + f\"{folder}/{str(id)}.csv\")\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Read the input files and do a little **preprocessing**"},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv(root_filepath+\"train.csv\")\ntest = pd.read_csv(root_filepath+\"sample_submission.csv\").drop(\"time_to_eruption\", axis=1)\nsorted_train = train.sort_values(\"time_to_eruption\")\nplt.hist((train[\"time_to_eruption\"]), bins=100)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Showing the furier transform for the signals of low and hight time eruption examples."},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(25,20))\nfor i in range(20):\n    plt.subplot(10,2,i+1)\n    if i%2==0:\n        plt.plot(np.fft.fftshift(np.real(np.fft.fft(get_signals(list(sorted_train[\"segment_id\"])[12])[f\"sensor_{i//2+1}\"].fillna(0)))))\n        plt.title(f\"{i//2+1} MIN sensor\")\n    else:\n        plt.plot(np.fft.fftshift(np.real(np.fft.fft(get_signals(list(sorted_train[\"segment_id\"])[-12])[f\"sensor_{i//2+1}\"].fillna(0)))))\n        plt.title(f\"{i//2+1} maX sensor\")\n        \n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Showing the same signals without fft"},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(25,20))\nfor i in range(20):\n    plt.subplot(10,2,i+1)\n    if i%2==0:\n        plt.plot(get_signals(list(sorted_train[\"segment_id\"])[12])[f\"sensor_{i//2+1}\"].fillna(0))\n        plt.title(f\"{i//2+1} MIN sensor\")\n    else:\n        plt.plot(get_signals(list(sorted_train[\"segment_id\"])[-12])[f\"sensor_{i//2+1}\"].fillna(0))\n        plt.title(f\"{i//2+1} maX sensor\")\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Lets make the following features:\n    - for every sensor:\n        * 5,10,25,30,60,70,90% quantile\n        * signal mean\n        * signal std\n        * signal variance\n        * skew\n    - for its fft compute:\n        * min\n        * max\n        * mean\n        * std"},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_feature_particle(signal, segment_id, sensor_id):\n    output = pd.DataFrame()\n    signal = signal.fillna(0)\n    furier = np.real(np.fft.fft(signal))\n    output.loc[segment_id, f\"5th_quantile_s{sensor_id}\"] = np.quantile(signal, 0.05)\n    output.loc[segment_id, f\"10th_quantile_s{sensor_id}\"] = np.quantile(signal, 0.1)    \n    output.loc[segment_id, f\"25th_quantile_s{sensor_id}\"] = np.quantile(signal, 0.25)    \n    output.loc[segment_id, f\"30th_quantile_s{sensor_id}\"] = np.quantile(signal, 0.3)    \n    output.loc[segment_id, f\"60th_quantile_s{sensor_id}\"] = np.quantile(signal, 0.6)    \n    output.loc[segment_id, f\"70th_quantile_s{sensor_id}\"] = np.quantile(signal, 0.7)    \n    output.loc[segment_id, f\"90th_quantile_s{sensor_id}\"] = np.quantile(signal, 0.9) \n    output.loc[segment_id, f\"mean_s{sensor_id}\"] = signal.mean()\n    output.loc[segment_id, f\"std_s{sensor_id}\"] = signal.std()\n    output.loc[segment_id, f\"var_s{sensor_id}\"] = signal.var()\n    output.loc[segment_id, f\"skew_s{sensor_id}\"] = signal.skew()\n    output.loc[segment_id, f\"fft_mean_s{sensor_id}\"] = furier.mean()\n    output.loc[segment_id, f\"fft_std_s{sensor_id}\"] = furier.std()\n    output.loc[segment_id, f\"fft_min_s{sensor_id}\"] = furier.min()\n    output.loc[segment_id, f\"fft_max_s{sensor_id}\"] = furier.max() \n    \n    return output\n\n\n\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_na_feat(segment_id, folder=\"train\"):\n    output = pd.DataFrame()\n    data = get_signals(segment_id, folder=folder)\n    for i in range(1,11):\n        output.loc[segment_id, f\"na_percent_s{i}\"] = data[f\"sensor_{i}\"].isna().sum()/len(data[f\"sensor_{i}\"])\n    return output","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def make_features(train, folder=\"train\"):\n    segments = []\n    ci = 0\n\n    for seg in train[\"segment_id\"]:\n        signals = get_signals(seg, folder=folder)\n        segment_row = []\n        if ci % 100 == 0:\n            print(ci)\n        for i in range(1,11):\n            segment_row.append(create_feature_particle(signals[f\"sensor_{i}\"], seg, i))\n        segments.append(pd.concat(segment_row + [create_na_feat(seg, folder=folder)], axis=1))\n        ci += 1\n\n    featured_train = pd.concat(segments, axis=0)\n    featured_train = featured_train.reset_index()\n    featured_train = featured_train.rename(columns={featured_train.columns[0]:\"segment_id\"})\n    return featured_train","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **A cell to preprocess the train and the test set**"},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"featured_train = pd.merge(make_features(train), train, on=\"segment_id\")\nfeatured_train.to_csv(save_filepath + \"preprocessed/featured_train.csv\")\nfeatured_test = make_features(test, folder=\"test\")\nfeatured_test.to_csv(save_filepath + \"preprocessed/featured_test.csv\")\nprint(\"Save successful!\")\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"featured_train = pd.read_csv(\"../input/featured-train/featured_train.csv\")\nfeatured_test = pd.read_csv(\"../input/featured-train/featured_test.csv\")\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"f_train = featured_train.drop([\"segment_id\", \"time_to_eruption\"]+list(featured_train.columns)[-10:], axis=1)\ncorr_matrix = f_train.corrwith(featured_train[\"time_to_eruption\"])\nfig = plt.figure(figsize=(8, 30))\nsns.scatterplot(x=list(corr_matrix), y=corr_matrix.index)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dropped_cols = [i for i in corr_matrix.index if abs(corr_matrix[i])<0.01]\nprint(dropped_cols)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X = featured_train.drop([\"segment_id\", \"time_to_eruption\"], axis=1)\ny = featured_train[\"time_to_eruption\"]\nrfe_test = RFE(estimator=DecisionTreeRegressor(), n_features_to_select=70, step=3)\nrfe_test.fit(X,y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"mask = rfe_test.support_\nremoved = [col for ind, col in zip(mask, list(X.columns)[:-10]) if not ind]\nprint(removed)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train, X_val, y_train, y_val = train_test_split(X, y, random_state=103, test_size=0.2, shuffle=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def rmse(true, pred):\n    return np.sqrt(mse(true,pred))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"params = {\n    'num_leaves': 29,\n    'n_estimators': 289,\n    'max_depth': 8,\n    'min_child_samples': 507,\n    'learning_rate': 0.0812634327662599,\n    'min_data_in_leaf': 13,\n    'bagging_fraction': 0.020521665677937423,\n    'feature_fraction': 0.05776459974779927,\n    'random_state': 101\n}\n\nlgb_model = LGBMRegressor(**params)\nlgb_model.fit(X_train, y_train)\npred = lgb_model.predict(X_val)\n\nprint(rmse(pred, y_val))    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.concat([test,pd.DataFrame(lgb_model.predict(featured_test.drop([\"segment_id\"], axis=1)))], axis=1)\nsubmission.to_csv(\"./submission.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}