{"cells":[{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport warnings\n\nwarnings.filterwarnings(\"ignore\")\nwarnings.simplefilter(action='ignore', category=FutureWarning)\n\nfrom catboost import CatBoostRegressor\nfrom joblib import Parallel, delayed\nfrom tensorflow import keras\n\nfrom sklearn.linear_model import LinearRegression\nfrom sklearn.metrics import mean_absolute_error\nfrom sklearn.model_selection import KFold\nfrom sklearn.preprocessing import StandardScaler","execution_count":1,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"X_train_1 = pd.read_csv(\"../input/lanl-master-s-features-creating-0/train_X_features_865.csv\")\nX_train_2 = pd.read_csv(\"../input/lanl-master-s-features-creating-1/train_X_features_865.csv\")\ny_1 = pd.read_csv(\"../input/lanl-master-s-features-creating-0/train_y.csv\", index_col=False, header=None)\ny_2 = pd.read_csv(\"../input/lanl-master-s-features-creating-1/train_y.csv\", index_col=False, header=None)","execution_count":3,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train = pd.concat([X_train_1, X_train_2], axis=0)\nX_train = X_train.reset_index(drop=True)\nX_train.shape","execution_count":4,"outputs":[{"output_type":"execute_result","execution_count":4,"data":{"text/plain":"(33000, 865)"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train.head()","execution_count":5,"outputs":[{"output_type":"execute_result","execution_count":5,"data":{"text/plain":" FFT_Mag_01q0 ... abs_max_roll_mean_1000\n0 139.890381 ... 5.415\n1 140.598496 ... 5.362\n2 123.395925 ... 5.312\n3 92.046507 ... 5.562\n4 103.474480 ... 5.206\n\n[5 rows x 865 columns]","text/html":"
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
FFT_Mag_01q0FFT_Mag_10q0FFT_Mag_90q0FFT_Mag_99q0FFT_Mag_mean0FFT_Mag_std0FFT_Mag_max0FFT_Phz_mean0FFT_Phz_std0FFT_Mag_01q2500FFT_Mag_10q2500FFT_Mag_90q2500FFT_Mag_99q2500FFT_Mag_mean2500FFT_Mag_std2500FFT_Mag_max2500FFT_Phz_mean2500FFT_Phz_std2500FFT_Mag_01q5000FFT_Mag_10q5000FFT_Mag_90q5000FFT_Mag_99q5000FFT_Mag_mean5000FFT_Mag_std5000FFT_Mag_max5000FFT_Phz_mean5000FFT_Phz_std5000FFT_Mag_01q7500FFT_Mag_10q7500FFT_Mag_90q7500FFT_Mag_99q7500FFT_Mag_mean7500FFT_Mag_std7500FFT_Mag_max7500FFT_Phz_mean7500FFT_Phz_std7500FFT_Mag_01q10000FFT_Mag_10q10000FFT_Mag_90q10000FFT_Mag_99q10000...q01_roll_std_100q05_roll_std_100q95_roll_std_100q99_roll_std_100av_change_abs_roll_std_100av_change_rate_roll_std_100abs_max_roll_std_100ave_roll_mean_100std_roll_mean_100max_roll_mean_100min_roll_mean_100q01_roll_mean_100q05_roll_mean_100q95_roll_mean_100q99_roll_mean_100av_change_abs_roll_mean_100av_change_rate_roll_mean_100abs_max_roll_mean_100ave_roll_std_1000std_roll_std_1000max_roll_std_1000min_roll_std_1000q01_roll_std_1000q05_roll_std_1000q95_roll_std_1000q99_roll_std_1000av_change_abs_roll_std_1000av_change_rate_roll_std_1000abs_max_roll_std_1000ave_roll_mean_1000std_roll_mean_1000max_roll_mean_1000min_roll_mean_1000q01_roll_mean_1000q05_roll_mean_1000q95_roll_mean_1000q99_roll_mean_1000av_change_abs_roll_mean_1000av_change_rate_roll_mean_1000abs_max_roll_mean_1000
0139.890381430.7209623841.6656077039.0624751747.9876671505.80739612964.190508-0.0190640.917755580.9896371788.14175910385.44785917303.4104775576.9755503670.07851021851.886653-0.0187640.904588576.3789902244.23255710573.08562615464.7660906118.3468963304.63247024716.3979780.0214920.912655698.1745782463.55225713396.59683821973.2395907353.6127334529.55652632176.902122-0.0060210.909976304.096958992.7662858185.99787813830.325240...2.3419542.52812512.60070129.647864-4.876389e-0574991.82038764.9161574.4293630.54620213.91-6.313.123.735.125.660.00000274986.64514413.915.2610404.76121137.8348712.5429052.6423552.75396114.85648931.176557-1.579223e-0574576.87658037.8348714.4294680.2379305.4153.3273.8824.0294.8044.941-7.516779e-0774577.6799645.415
1140.598496431.8725383093.8283065914.7129821545.3200421239.73863814530.7915530.0029100.904066371.7798621247.1658148126.08553413429.2081744307.2323462810.56815216980.104876-0.0007150.909822758.4748602403.55230210859.39575415352.2568836500.5552913294.12991019523.499828-0.0125940.903179549.7665371893.11336211412.30897918897.6508806049.6875903897.71099626050.2650720.0061730.900904232.076227898.3832205984.96703711639.375545...2.3202882.4864079.47764522.7103393.857706e-0774941.49834587.3118204.2413090.53169220.71-5.293.083.544.945.32-0.00000374937.94590020.714.6731844.22299040.8831272.5127632.6245812.73431311.22871724.8137333.154961e-0774486.70039440.8831274.2411740.2620435.3623.1393.6363.7924.6814.865-2.791946e-0674484.3686155.362
2123.395925364.1147752052.0513883594.3077161157.415026765.79877911148.510210-0.0290140.904867326.587430974.2396814765.0509897051.4934362682.7435871499.3393429153.525311-0.0235700.915614391.6276931548.2067607269.25893110307.1516704170.4631682252.93324015298.295361-0.0182230.891113581.3022611843.33064611490.30670418905.3023226056.5870803985.53286728425.9653550.0087420.898519297.355997852.6866525616.31747410041.900327...2.3303882.50032310.44570417.8181929.870522e-0775071.73202334.9085664.6166890.4254308.291.643.613.935.295.650.00000475068.6417638.294.5910892.65247117.4291332.5242922.6742612.73155610.60025015.2673575.168715e-0674654.82204117.4291334.6159600.2397395.3123.9324.0294.1805.0045.1022.221477e-0674653.8408605.312
392.046507370.2154072011.1699153849.0506131125.200572817.60473119980.904036-0.0021310.905245216.375637763.1197274205.6307095822.0219942335.2624841325.9961087109.750499-0.0247010.911108370.5477841218.9763746861.74288410345.3917623757.2151912227.18023813184.6541970.0017870.909935488.1929901661.20409112983.88787417999.9871646501.3450394378.75668022646.9787800.0013450.898393254.215327786.9165078551.99382415609.258230...2.2332652.3606846.08621619.4441423.704347e-0575102.81214596.5449084.7029290.45230614.33-2.363.704.015.375.720.00000275104.76068314.333.8068204.07278346.8298652.4526292.5377832.5927508.41831216.4537535.558937e-0674690.71752646.8298654.7011680.2845315.5623.8173.9804.2315.1485.432-3.892617e-0774688.0573885.562
4103.474480342.8809691725.9160002824.536178990.653858601.4849545872.137835-0.0085270.914271204.654237694.9187183593.6580895253.7411652040.4479411142.2677677832.216209-0.0478620.907809285.8390961056.7278135113.0426007870.4009683000.9937341647.53289711748.378766-0.0132150.908274420.3940641199.4227117219.81523411711.8259083878.8264672498.22147616038.562751-0.0047560.907722198.981008651.0990463889.6458586446.635167...2.2329942.3761446.29043713.881533-1.184534e-0675112.06613228.9579984.5809020.3663118.011.713.693.995.165.40-0.00000175110.2125578.013.5028401.82885014.6330822.4887882.5462542.5992997.22939612.682185-1.800481e-0674730.79089214.6330824.5807890.1968015.2063.7624.0714.2374.8825.026-8.926174e-0774730.0133475.206
\n
"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"y = pd.concat([y_1, y_2], axis=0)\ny = y.reset_index(drop=True)\ny.shape","execution_count":6,"outputs":[{"output_type":"execute_result","execution_count":6,"data":{"text/plain":"(33000, 1)"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_train = pd.Series(y[0].values)","execution_count":7,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_test = pd.read_csv(\"../input/lanl-master-s-features-creating-0/test_X_features_10.csv\")","execution_count":8,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"scaler = StandardScaler()\ntrain_columns = X_train.columns\n\nX_train[train_columns] = scaler.fit_transform(X_train[train_columns])\nX_test[train_columns] = scaler.transform(X_test[train_columns])","execution_count":9,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_columns = X_train.columns\nn_fold = 5","execution_count":10,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nfolds = KFold(n_splits=n_fold, shuffle=True, random_state=42)\n\noof = np.zeros(len(X_train))\ntrain_score = []\ncat_predictions = np.zeros(len(X_test))\n\nfor fold_, (trn_idx, val_idx) in enumerate(folds.split(X_train, y_train.values)):\n print(\"fold {}\".format(fold_))\n\n X_tr, X_val = X_train[train_columns].iloc[trn_idx], X_train[train_columns].iloc[val_idx]\n y_tr, y_val = y_train.iloc[trn_idx], y_train.iloc[val_idx]\n\n model = CatBoostRegressor(n_estimators=25000, verbose=-1, objective=\"MAE\", loss_function=\"MAE\", boosting_type=\"Ordered\", task_type=\"GPU\")\n model.fit(X_tr, y_tr, eval_set=[(X_val, y_val)], verbose=2500, early_stopping_rounds=500)\n oof[val_idx] = model.predict(X_val)\n\n cat_predictions += model.predict(X_test[train_columns]) / folds.n_splits\n train_score.append(model.best_score_['learn'][\"MAE\"])\n\ncv_score = mean_absolute_error(y_train, oof)\nprint(f\"After {n_fold}: test_CV = {cv_score:.3f} | train_CV = {np.mean(train_score):.3f} | {cv_score-np.mean(train_score):.3f}\", end=\" \")","execution_count":null,"outputs":[{"output_type":"stream","text":"fold 0\n0:\tlearn: 5.6534239\ttest: 5.7103149\tbest: 5.7103149 (0)\ttotal: 69.5ms\tremaining: 28m 58s\n","name":"stdout"}]},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_model(input_dim=10):\n model = keras.Sequential()\n model.add(keras.layers.Dense(256, activation=\"relu\", input_dim=input_dim))\n model.add(keras.layers.Dropout(0.3))\n model.add(keras.layers.Dense(128, activation=\"relu\"))\n model.add(keras.layers.Dropout(0.3))\n model.add(keras.layers.Dense(96, activation=\"relu\"))\n model.add(keras.layers.Dropout(0.3))\n model.add(keras.layers.Dense(1, activation=\"linear\"))\n\n optimizer = keras.optimizers.Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0, amsgrad=False)\n model.compile(optimizer=optimizer, loss='mae')\n\n return model\n\npatience = 50\ncall_ES = keras.callbacks.EarlyStopping(monitor='val_loss', min_delta=0, patience=patience, verbose=1, mode='auto', baseline=None, restore_best_weights=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nfolds = KFold(n_splits=n_fold, shuffle=True, random_state=42)\n\nNN_oof = np.zeros(len(X_train))\ntrain_score = []\nNN_predictions = np.zeros(len(X_test))\n\nfor fold_, (trn_idx, val_idx) in enumerate(folds.split(X_train, y_train.values)):\n print(\"fold {}\".format(fold_))\n \n X_tr, X_val = X_train[train_columns].iloc[trn_idx], X_train[train_columns].iloc[val_idx]\n y_tr, y_val = y_train.iloc[trn_idx], y_train.iloc[val_idx]\n \n model = create_model(X_train.shape[-1])\n model.fit(X_tr, y_tr, epochs=500, batch_size=32, verbose=0, callbacks=[call_ES,], validation_data=[X_val, y_val])\n \n NN_oof[val_idx] = model.predict(X_val)[:, 0]\n \n NN_predictions += model.predict(X_test[train_columns])[:, 0] / folds.n_splits\n history = model.history.history\n tr_loss = history[\"loss\"]\n val_loss = history[\"val_loss\"]\n print(f\"loss: {tr_loss[-patience]:.3f} | val_loss: {val_loss[-patience]:.3f} | diff: {val_loss[-patience]-tr_loss[-patience]:.3f}\")\n train_score.append(tr_loss[-patience])\n \ncv_score = mean_absolute_error(y_train, NN_oof)\nprint(f\"After {n_fold}: test_CV = {cv_score:.3f} | train_CV = {np.mean(train_score):.3f} | {cv_score-np.mean(train_score):.3f}\", end=\" \")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Scirpus_predictions = pd.read_csv(\"../input/andrews-new-script-plus-a-genetic-program-model/gpI.csv\")\nScirpus_predictions.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.read_csv('../input/LANL-Earthquake-Prediction/sample_submission.csv')\n\nsubmission[\"time_to_failure\"] = (cat_predictions + NN_predictions + Scirpus_predictions.time_to_failure.values) / 3\nsubmission.to_csv('submission.csv', index=False)\nsubmission.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.7.3"}},"nbformat":4,"nbformat_minor":1}