{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Used notebooks\n\n### TPSMAR22 EDA which makes sense:\nhttps://www.kaggle.com/code/ambrosm/tpsmar22-eda-which-makes-sense<br/>\ntaking care of the \"special values\"\n\n### Voting of multiple regressors:\nhttps://www.kaggle.com/code/martynovandrey/tps-mar-22-fe-model-selection<br/>\n\nadd missing datetime values (tic=20min, fill=previous)<br/>\nadd global grouped stats (group=['month', 'day', 'road'], target=congestion, stats=min,max,median)<br/>\nadd 2 PCA features (train on train_df only)<br/>\nfixing outliers after predicting data<br/>\n\nBest regressors:\n- HistGradientBoostingRegressor(max_iter=10875, learning_rate=0.0307, max_leaf_nodes=176, max_depth=18, min_samples_leaf=49, loss='absolute_error')\n- GradientBoostingRegressor(n_estimators=300, loss='absolute_error')\n- CatBoostRegressor(logging_level='Silent', eval_metric='MAE', loss_function='MAE')\n- LGBMRegressor(n_estimators=3900, metric='mae')\n- VotingRegressor with 7 CatBoostRegressor with different random states\n\n### Features selection by Optuna:\nhttps://www.kaggle.com/code/ottpocket/feature-selection-notebook/notebook?scriptVersionId=91697657\n\nadd many rolling window features: \n- HM (hour+minute), DHM (weekdat+hour+minute)\n- roll by 3/5/10 ticks\n- apply statistics: mean,var,median,min,max\n\nselect important features by Optuna<br/>\ntrain by LGBMRegressor\n\n\n### TPS Mar22 - Top 6% Solution - EDA/FE/Blending\nhttps://www.kaggle.com/code/javigallego/tps-mar22-top-6-solution-eda-fe-blending/notebook?scriptVersionId=90196710<br/>\nCyclical Features (hours with sin/cos)<br/>\nCatBoost with \"eval_set\" and optuna<br/>\nXGBoost with GridSearchCV\n\n### TPS-Mar 2022 Simple CatBoost Using Median\nhttps://www.kaggle.com/code/naokisugimura/tps-mar-2022-simple-catboost-using-median/notebook?scriptVersionId=91536158<br/>\nKFold(n_splits=5,shuffle=True) + optuna + CatBoost","metadata":{}},{"cell_type":"markdown","source":"## Notebooks to be used in the part II\n\n### ETNA\nhttps://www.kaggle.com/code/chikovalexander/tps-mar-2022-etna/notebook?scriptVersionId=91575908\n\n### PyTorch LSTM:\nhttps://www.kaggle.com/code/aboriginal3153/tps-mar-22-pytorch-lstm\n\n### TPS_2022_03_pytorch_score(4.923)\nhttps://www.kaggle.com/code/zhangcheche/tps-2022-03-pytorch-score-4-923\n\n### Private LB: Top 2% Google's TFT\nLibrary: Pytorch Forecasting<br/>\nhttps://pytorch-forecasting.readthedocs.io/en/stable/<br/>\nhttps://www.kaggle.com/code/abdulravoofshaik/private-lb-top-2-google-s-tft","metadata":{}},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nfrom matplotlib.ticker import MaxNLocator\nimport os\n\nRANDOM_SEED = 111\n\nnp.random.seed(RANDOM_SEED)\n\nfrom numpy.random import default_rng\nrng = default_rng(RANDOM_SEED)\n\nfrom sklearn.model_selection import KFold, ShuffleSplit, StratifiedKFold, StratifiedShuffleSplit, TimeSeriesSplit\nfrom sklearn.metrics import mean_absolute_error\nfrom sklearn.preprocessing import KBinsDiscretizer, minmax_scale\nfrom sklearn.decomposition import PCA\nfrom sklearn.utils import shuffle\nfrom category_encoders.hashing import HashingEncoder\n\npd.set_option('display.max_columns', None)\n\nimport tensorflow as tf\n#import tensorflow_addons as tfa\ntf.random.set_seed(RANDOM_SEED)\n\nAUTO = tf.data.experimental.AUTOTUNE\n\ntf.__version__","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-24T19:44:57.494493Z","iopub.execute_input":"2022-07-24T19:44:57.494950Z","iopub.status.idle":"2022-07-24T19:45:08.714386Z","shell.execute_reply.started":"2022-07-24T19:44:57.494858Z","shell.execute_reply":"2022-07-24T19:45:08.713355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%script echo skipping\n\n!pip install -q h2o\n\nimport h2o\nfrom h2o.automl import H2OAutoML\nfrom h2o.sklearn import H2OAutoMLRegressor\nh2o.init()","metadata":{"execution":{"iopub.status.busy":"2022-07-24T19:45:08.716358Z","iopub.execute_input":"2022-07-24T19:45:08.717075Z","iopub.status.idle":"2022-07-24T19:45:08.746380Z","shell.execute_reply.started":"2022-07-24T19:45:08.717042Z","shell.execute_reply":"2022-07-24T19:45:08.745330Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv('../input/tabular-playground-series-mar-2022/train.csv')\ntest_df = pd.read_csv('../input/tabular-playground-series-mar-2022/test.csv')\n\ntrain_df['time'] = pd.to_datetime(train_df['time'])\ntest_df['time'] = pd.to_datetime(test_df['time'])\n\ntrain_df.set_index(keys='time', inplace=True)\ntest_df.set_index(keys='time', inplace=True)\n\ntrain_df_last = train_df.index.max()\ntrain_df_index = train_df.index.values\n\ntrain_df.congestion.replace(to_replace=0, value=np.nan, inplace=True)\ntrain_df['road'] = train_df[\"x\"].astype('str') + train_df[\"y\"].astype('str') + train_df[\"direction\"]\nfor road in train_df['road'].unique():\n    train_df.loc[train_df['road']==road, 'congestion'] = train_df.loc[train_df['road']==road, 'congestion'].interpolate()","metadata":{"execution":{"iopub.status.busy":"2022-07-24T19:45:08.749368Z","iopub.execute_input":"2022-07-24T19:45:08.751191Z","iopub.status.idle":"2022-07-24T19:45:30.048666Z","shell.execute_reply.started":"2022-07-24T19:45:08.751135Z","shell.execute_reply":"2022-07-24T19:45:30.047621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#%%script echo skipping\n\n#imputing missing records into both train and test\n\ndate_range = pd.date_range(start=train_df.index.min(), end=train_df.index.max(), freq='20min')\ncols_not_cong = train_df.columns[~train_df.columns.isin(['congestion'])]\n\nfull_df = pd.DataFrame(columns=train_df.columns)\nfor road in train_df[\"road\"].unique():\n    road_df = train_df[train_df.road == road]\n    road_df = road_df.reindex(date_range)\n    #road_df['new_vals'] = road_df.congestion.isna()\n    road_df[cols_not_cong] = road_df[cols_not_cong].fillna(method='ffill')\n    road_df['congestion'] = road_df['congestion'].interpolate()\n    full_df = full_df.append(road_df)\n    \nfull_df['x'] = full_df['x'].astype('int8')\nfull_df['y'] = full_df['y'].astype('int8')\nprint('Add missing train date values: ', train_df.shape[0], full_df.shape[0])\n\nfull_df.sort_index(inplace=True)\ntrain_df = full_df\ntrain_df","metadata":{"execution":{"iopub.status.busy":"2022-07-24T19:45:30.051974Z","iopub.execute_input":"2022-07-24T19:45:30.052766Z","iopub.status.idle":"2022-07-24T19:45:42.314488Z","shell.execute_reply.started":"2022-07-24T19:45:30.052715Z","shell.execute_reply":"2022-07-24T19:45:42.313353Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%script echo skipping\n\n#add percent change\n\n#train_df['cong_pct'] = 0\nfull_df = pd.DataFrame(columns=train_df.columns)\nfor road in train_df[\"road\"].unique():\n    road_df = train_df[train_df.road == road]\n    road_df['cong_pct'] = road_df.congestion.pct_change()\n    full_df = full_df.append(road_df)\n\nfull_df.sort_index(inplace=True)\ntrain_df = full_df\ntrain_df[train_df.road == '00EB']","metadata":{"execution":{"iopub.status.busy":"2022-07-24T19:45:42.315943Z","iopub.execute_input":"2022-07-24T19:45:42.316267Z","iopub.status.idle":"2022-07-24T19:45:42.343959Z","shell.execute_reply.started":"2022-07-24T19:45:42.316236Z","shell.execute_reply":"2022-07-24T19:45:42.342666Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%script echo skipping\n\n#remove pykes of mean +/- std\n\nboundary1 = train_df['cong_pct'].mean() + train_df['cong_pct'].std()*6\nboundary2 = train_df['cong_pct'].mean() - train_df['cong_pct'].std()*6\noutliners = (train_df['cong_pct']>boundary1) | (train_df['cong_pct']<boundary2)\nprint(boundary1, boundary2, train_df[outliners].shape[0])\n\ntrain_df.drop(['cong_pct'], axis=1, inplace=True)\ntrain_df.loc[outliners, 'congestion'] = np.nan\nfor road in train_df['road'].unique():\n    train_df.loc[train_df['road']==road, 'congestion'] = train_df.loc[train_df['road']==road, 'congestion'].interpolate()","metadata":{"execution":{"iopub.status.busy":"2022-07-24T19:45:42.345644Z","iopub.execute_input":"2022-07-24T19:45:42.346013Z","iopub.status.idle":"2022-07-24T19:45:42.392986Z","shell.execute_reply.started":"2022-07-24T19:45:42.345977Z","shell.execute_reply":"2022-07-24T19:45:42.391461Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_df = train_df.append(test_df)\nall_df.drop(['row_id'], axis=1, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-24T19:45:42.395139Z","iopub.execute_input":"2022-07-24T19:45:42.396235Z","iopub.status.idle":"2022-07-24T19:45:42.572540Z","shell.execute_reply.started":"2022-07-24T19:45:42.396191Z","shell.execute_reply":"2022-07-24T19:45:42.571327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%script echo skipping\n\nall_df.congestion.hist(bins=20)\nplt.show()\nall_df.cong_pct.hist(bins=20)\nplt.show()\nall_df.cong_pct.plot()","metadata":{"execution":{"iopub.status.busy":"2022-07-24T19:45:42.574070Z","iopub.execute_input":"2022-07-24T19:45:42.575045Z","iopub.status.idle":"2022-07-24T19:45:42.603022Z","shell.execute_reply.started":"2022-07-24T19:45:42.575010Z","shell.execute_reply":"2022-07-24T19:45:42.601595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['hour'] = train_df.index.hour\ntrain_df['cong_bins'] = pd.cut(train_df.congestion, bins=10, labels=False)\n\ngroup_cong = train_df.groupby(['hour']).cong_bins\ntrain_df.drop(['cong_bins'], axis=1, inplace=True)\n\ngrouped_df = pd.DataFrame(data={'mean':group_cong.mean(), 'std':group_cong.std()*3}, index=train_df['hour'].unique())\ngrouped_df.plot()","metadata":{"execution":{"iopub.status.busy":"2022-07-24T19:45:42.605081Z","iopub.execute_input":"2022-07-24T19:45:42.606127Z","iopub.status.idle":"2022-07-24T19:45:43.185374Z","shell.execute_reply.started":"2022-07-24T19:45:42.606089Z","shell.execute_reply":"2022-07-24T19:45:43.184195Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dir_dict = {'EB': (1, 0), 'NB': (0, 1), 'SB': (0, -1), 'WB': (-1, 0), 'NE': (1, 1), 'SE': (1, -1), 'NW': (-1, 1), 'SW': (-1, -1)}\n\nroadways = all_df[['x', 'y']].drop_duplicates()\nroad_dir = all_df[['x', 'y', 'direction']].drop_duplicates()\n\nplt.figure(figsize=(8, 8))\nplt.scatter(roadways.x, roadways.y)\nplt.gca().set_aspect('equal')\nfor _, x, y, d in road_dir.itertuples():\n    dx, dy = dir_dict[d]\n    dx, dy = dx/4, dy/4\n    plt.plot([x, x+dx], [y, y+dy], color='#ffd700')\nplt.gca().xaxis.set_major_locator(MaxNLocator(integer=True)) # only integer labels\nplt.gca().yaxis.set_major_locator(MaxNLocator(integer=True)) # only integer labels\nplt.xlabel('x')\nplt.ylabel('y')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-24T19:45:43.190320Z","iopub.execute_input":"2022-07-24T19:45:43.191103Z","iopub.status.idle":"2022-07-24T19:45:43.617825Z","shell.execute_reply.started":"2022-07-24T19:45:43.191067Z","shell.execute_reply":"2022-07-24T19:45:43.616265Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Graph directions by roads\n1. 00eb,01eb,02eb\n2. 00eb,01eb,02nb,12nb,22nb,32nb\n3. 00eb,01enb,12enb","metadata":{}},{"cell_type":"code","source":"all_df[\"road\"] = all_df[\"x\"].astype('str') + all_df[\"y\"].astype('str') + all_df[\"direction\"]\nall_df['hour'] = all_df.index.hour\nall_df[\"weekday\"] = all_df.index.to_series().apply(lambda x: x.weekday())\nall_df[\"weekend\"] = all_df[\"weekday\"]>=5\nall_df[\"morning\"] = (all_df[\"hour\"]>=4) & (all_df[\"hour\"]<12)\nall_df[\"midday\"] = (all_df[\"hour\"]>=12) & (all_df[\"hour\"]<18)\nall_df[\"evening\"] = (all_df[\"hour\"]>=18) & (all_df[\"hour\"]<23)\nall_df[\"night\"] = (all_df[\"hour\"]>=23) | (all_df[\"hour\"]<4)\n\ngrouped_cong = all_df.groupby(['road','hour','weekday']).congestion\n                            \ncalc_feat = pd.DataFrame(grouped_cong.mean()).reset_index()\ncalc_feat = calc_feat.rename(columns={'congestion':'cong_mean'})\nall_df = all_df.merge(calc_feat, on=['road','hour','weekday'], how='left').set_index(all_df.index)\n\nall_df['quant5'] = KBinsDiscretizer(n_bins=5, encode='ordinal', strategy='uniform').fit_transform(all_df['cong_mean'].values.reshape(-1, 1)).squeeze()\nall_df['quant3'] = KBinsDiscretizer(n_bins=3, encode='ordinal', strategy='uniform').fit_transform(all_df['cong_mean'].values.reshape(-1, 1)).squeeze()\n\ncalc_feat = pd.DataFrame(grouped_cong.min()).reset_index()\ncalc_feat = calc_feat.rename(columns={'congestion':'cong_min'})\nall_df = all_df.merge(calc_feat, on=['road','hour','weekday'], how='left').set_index(all_df.index)\n\ncalc_feat = pd.DataFrame(grouped_cong.max()).reset_index()\ncalc_feat = calc_feat.rename(columns={'congestion':'cong_max'})\nall_df = all_df.merge(calc_feat, on=['road','hour','weekday'], how='left').set_index(all_df.index)\n\ncalc_feat = pd.DataFrame(grouped_cong.std()).reset_index()\ncalc_feat = calc_feat.rename(columns={'congestion':'cong_std'})\nall_df = all_df.merge(calc_feat, on=['road','hour','weekday'], how='left').set_index(all_df.index)","metadata":{"execution":{"iopub.status.busy":"2022-07-24T19:45:43.623477Z","iopub.execute_input":"2022-07-24T19:45:43.624781Z","iopub.status.idle":"2022-07-24T19:45:54.806220Z","shell.execute_reply.started":"2022-07-24T19:45:43.624714Z","shell.execute_reply":"2022-07-24T19:45:54.805055Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from math import sin, cos, pi, exp\nsin_vals = {\n    'NB': 0.0,\n    'NE': sin(1 * pi/4),\n    'EB': 1.0,\n    'SE': sin(3 * pi/4),\n    'SB': 0.0,\n    'SW': sin(5 * pi/4),    \n    'WB': -1.0,    \n    'NW': sin(7 * pi/4),  \n}\n\ncos_vals = {\n    'NB': 1.0,\n    'NE': cos(1 * pi/4),\n    'EB': 0.0,\n    'SE': cos(3 * pi/4),\n    'SB': -1.0,\n    'SW': cos(5 * pi/4),    \n    'WB': 0.0,    \n    'NW': cos(7 * pi/4),  \n}\n\nall_df['week'] = all_df.index.isocalendar().week\nall_df['minute'] = all_df.index.minute\nall_df['hm'] = all_df[\"hour\"].astype('str') + all_df[\"minute\"].astype('str')\n#all_df['dhm'] = all_df[\"weekday\"].astype('str') + all_df[\"hour\"].astype('str')\n\nall_df['sin'] = all_df['direction'].map(sin_vals)\nall_df['cos'] = all_df['direction'].map(cos_vals)\nall_df['x_cos_hour'] = (all_df.x * all_df.cos * all_df.hour).astype('float64')\nall_df['y_sin_hour'] = (all_df.y * all_df.sin * all_df.hour).astype('float64')\nall_df['hour_s'] = np.sin(np.pi * all_df['hour']/12)    #[0..2pi]\nall_df['hour_c'] = np.cos(np.pi * all_df['hour']/12)    #[0..2pi]\n\nall_df['cong_bins'] = pd.cut(all_df.congestion, bins=10, labels=False)\ngrouped_cong = all_df.groupby(['hour']).cong_bins\ncalc_feat = pd.DataFrame(grouped_cong.mean()).reset_index()\ncalc_feat = calc_feat.rename(columns={'cong_bins':'cong_hbins_mean'})\nall_df = all_df.merge(calc_feat, on=['hour'], how='left').set_index(all_df.index)\nall_df.drop(['cong_bins'], axis=1, inplace=True)\n\ngrouped_cong = all_df.groupby(by=['x','y']).congestion\ncalc_feat = pd.DataFrame(grouped_cong.mean()).reset_index()\ncalc_feat = calc_feat.rename(columns={'congestion':'cong_xy_mean'})\nall_df = all_df.merge(calc_feat, on=['x','y'], how='left').set_index(all_df.index)\n\ngrouped_cong = all_df.groupby(by=['hour']).congestion\ncalc_feat = pd.DataFrame(grouped_cong.mean()).reset_index()\ncalc_feat = calc_feat.rename(columns={'congestion':'cong_h_mean'})\nall_df = all_df.merge(calc_feat, on=['hour'], how='left').set_index(all_df.index)\n\ngrouped_cong = all_df.groupby(by=['hour','direction']).congestion\ncalc_feat = pd.DataFrame(grouped_cong.mean()).reset_index()\ncalc_feat = calc_feat.rename(columns={'congestion':'cong_hd_mean'})\nall_df = all_df.merge(calc_feat, on=['hour','direction'], how='left').set_index(all_df.index)\n\nall_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-24T19:45:54.807913Z","iopub.execute_input":"2022-07-24T19:45:54.808364Z","iopub.status.idle":"2022-07-24T19:46:01.386097Z","shell.execute_reply.started":"2022-07-24T19:45:54.808321Z","shell.execute_reply":"2022-07-24T19:46:01.384726Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_cols = [i for i in all_df.columns if i.startswith('cong')]\ncat_cols = all_df.columns[~all_df.columns.isin(num_cols)].tolist()\n\nfor col in cat_cols:\n    col_val = all_df[col].astype('category').cat.codes\n    if (col_val.nunique()<128):\n        all_df[col] = col_val.astype('int8')\n    elif (col_val.nunique()<256):\n        all_df[col] = col_val.astype('int16')\n    else:\n        all_df[col] = pd.cut(col_val, bins=255, labels=False).astype('int16')\n    \n#scale 0,100 and int8 numerical variables\nfor col in [x for x in num_cols if x != 'congestion']:\n    print(col, all_df[col].min(), all_df[col].max())\n    all_df[col] = minmax_scale(all_df[col], [0,100]).astype('int8')\n\nall_df[cat_cols].nunique()","metadata":{"execution":{"iopub.status.busy":"2022-07-24T19:46:01.387937Z","iopub.execute_input":"2022-07-24T19:46:01.388495Z","iopub.status.idle":"2022-07-24T19:46:02.906142Z","shell.execute_reply.started":"2022-07-24T19:46:01.388449Z","shell.execute_reply":"2022-07-24T19:46:02.904873Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_df.min()<0","metadata":{"execution":{"iopub.status.busy":"2022-07-24T19:46:02.908070Z","iopub.execute_input":"2022-07-24T19:46:02.908699Z","iopub.status.idle":"2022-07-24T19:46:02.947473Z","shell.execute_reply.started":"2022-07-24T19:46:02.908605Z","shell.execute_reply":"2022-07-24T19:46:02.946228Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import seaborn as sns\n\ncorr = np.abs(all_df.corr())\ncorr = corr[corr>0.5].fillna(0)\n\ng = sns.clustermap(corr, annot=True, cmap=\"PiYG\")\ng.fig.set_figwidth(16)\ng.fig.set_figheight(16)","metadata":{"execution":{"iopub.status.busy":"2022-07-24T19:46:02.948589Z","iopub.execute_input":"2022-07-24T19:46:02.948953Z","iopub.status.idle":"2022-07-24T19:46:09.725157Z","shell.execute_reply.started":"2022-07-24T19:46:02.948923Z","shell.execute_reply":"2022-07-24T19:46:09.723954Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%script echo skipping\n\n#OneHotEncoder for categorical variables\n\nfrom sklearn.preprocessing import OrdinalEncoder, MinMaxScaler, StandardScaler, OneHotEncoder, Binarizer, KBinsDiscretizer, QuantileTransformer, PolynomialFeatures\n#from sklearn.decomposition import PCA\n#from sklearn.feature_extraction import FeatureHasher\n\n#fh = FeatureHasher(n_features=5, input_type='string')\n#tmp_enc = fh.fit_transform(all_df[cat_cols])\n#tmp_enc.toarray()\n\nfor col in cat_cols:\n    tmp_vals = all_df[col]\n    all_cats = len(tmp_vals.unique())\n    if all_cats>15:\n        infreq_cats = tmp_vals.value_counts().nsmallest(all_cats-15).index.to_list()\n        tmp_vals[tmp_vals.isin(infreq_cats)] = -1\n    \n    tmp_enc = OneHotEncoder(sparse=False).fit_transform(tmp_vals.values.reshape(-1,1))\n    print(col, tmp_enc.shape[1])\n    all_df[[col+str(x) for x in range(tmp_enc.shape[1])]] = tmp_enc\n    all_df.drop(col, axis=1, inplace=True)\n\n    \n#all_df.drop(['time','x','y'], axis=1, inplace=True)\nprint('columns:', len(all_df.columns))","metadata":{"execution":{"iopub.status.busy":"2022-07-24T19:46:09.726813Z","iopub.execute_input":"2022-07-24T19:46:09.727535Z","iopub.status.idle":"2022-07-24T19:46:09.762614Z","shell.execute_reply.started":"2022-07-24T19:46:09.727488Z","shell.execute_reply":"2022-07-24T19:46:09.761509Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#%%script echo skipping\n\n#best features after optuna+LGBM\nFEATURES = ['x', 'road', 'hour', 'weekend', 'morning', 'cong_mean', 'quant5', 'cong_min', 'cong_std', 'minute', 'hm', 'cos', 'cong_hd_mean']\n\ntrain_df = all_df[all_df.index<=train_df_last]\ntest_df = all_df[all_df.index>train_df_last]\n\nval_times_start = train_df.index[-1] - pd.Timedelta(days=7)\nval_times_end = val_times_start + pd.Timedelta(hours=12)\n\nX_train = train_df.loc[:val_times_start]\nX_val = train_df.loc[val_times_start+pd.Timedelta(minutes=20):val_times_end]\n\ny_train = X_train.congestion\ny_val = X_val.congestion\n\nX_train.drop(['congestion'], axis=1, inplace=True)\nX_val.drop(['congestion'], axis=1, inplace=True)\ntest_df.drop(['congestion'], axis=1, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-24T19:46:09.764484Z","iopub.execute_input":"2022-07-24T19:46:09.764887Z","iopub.status.idle":"2022-07-24T19:46:19.007024Z","shell.execute_reply.started":"2022-07-24T19:46:09.764847Z","shell.execute_reply":"2022-07-24T19:46:19.005830Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%script echo skipping\n\nall_df['dataset_test'] = all_df.index.isin(test_df.index)\nall_df['dataset_val'] = all_df.index.isin(X_val.index)\n\n#all_df.to_csv('all_processed.csv',index=True)\nall_df.to_pickle('all_processed.pkl')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%script echo skipping\n\n#https://medium.com/optuna/using-optuna-to-optimize-tensorflow-hyperparameters-57b6d4d316a2\n\"\"\"\nBest trial:\n  Value:  5.099968603112113\n  Params:  ['x' 'road' 'hour' 'weekend' 'morning' 'cong_mean' 'quant5' 'cong_min' 'cong_std' 'minute' 'hm' 'cos' 'cong_hd_mean']\n\"\"\"\n\n\nimport optuna\nfrom lightgbm import LGBMRegressor\n\nmodel = LGBMRegressor(n_estimators=100, metric='mae')\n\n#X[cat_cols] = X[cat_cols].apply(lambda col: col.cat.codes)\nall_cols = list(X_train.columns)\n\n\ndef objective(trial):\n    FEATURES = []\n    for feat in all_cols:\n        select_feat = trial.suggest_categorical(feat, [True, False])\n        if select_feat:\n            FEATURES.append(feat)\n        \n    model.fit(X_train[FEATURES], y_train)\n    y_pred = model.predict(X_val[FEATURES])\n    score = mean_absolute_error(y_val, y_pred)  \n    print(score)\n    return score\n\n\noptuna.logging.set_verbosity(optuna.logging.WARNING)\nstudy = optuna.create_study(direction=\"minimize\")\nstudy.optimize(objective, n_trials=100)\n\nprint(\"Study statistics: \")\nprint(\"  Number of finished trials: \", len(study.trials))\n\nprint(\"Best trial:\")\ntrial = study.best_trial\n\nprint(\"  Value: \", trial.value)\nprint(\"  Params: \", np.array(all_cols)[list(trial.params.values())])","metadata":{"execution":{"iopub.status.busy":"2022-07-24T19:46:19.008868Z","iopub.execute_input":"2022-07-24T19:46:19.009728Z","iopub.status.idle":"2022-07-24T19:46:19.049384Z","shell.execute_reply.started":"2022-07-24T19:46:19.009680Z","shell.execute_reply":"2022-07-24T19:46:19.047530Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%script echo skipping\n\n#limited: 5.106687889792889\n#all: 5.1165541126341\n\nfrom lightgbm import LGBMRegressor\n\nmodel = LGBMRegressor(n_estimators=500, metric='mae')\nmodel.fit(X_train[FEATURES], y_train)\ny_pred = model.predict(X_val[FEATURES])\nscore = mean_absolute_error(y_val, y_pred)  \nprint(score)","metadata":{"execution":{"iopub.status.busy":"2022-07-24T19:46:19.051746Z","iopub.execute_input":"2022-07-24T19:46:19.052527Z","iopub.status.idle":"2022-07-24T19:46:19.091868Z","shell.execute_reply.started":"2022-07-24T19:46:19.052478Z","shell.execute_reply":"2022-07-24T19:46:19.090287Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%script echo skipping\n\nfrom sklearn.ensemble import HistGradientBoostingRegressor\nfrom sklearn.metrics import mean_absolute_error as mae\nimport optuna\nfrom optuna.samplers import TPESampler\n\n#5.025843432570435 and parameters: {'learning_rate': 0.0006759689483814301, 'max_leaf_nodes': 175, 'max_depth': 16, 'min_samples_leaf': 27}\nall_cols = list(X_train.columns)\n\ndef objective_params(trial):\n    params = {\n        'max_iter': 1000, \n        'learning_rate': trial.suggest_loguniform('learning_rate', 0.0001, 0.3),\n        'max_leaf_nodes': trial.suggest_int(\"max_leaf_nodes\", 10, 500),\n        'max_depth': trial.suggest_int(\"max_depth\", 4, 16),\n        'min_samples_leaf':trial.suggest_int(\"min_samples_leaf\", 5, 100),\n        'categorical_features': [i in cat_cols for i in FEATURES],\n        'loss': 'absolute_error', \n        'random_state': 111\n    }\n\n    model = HistGradientBoostingRegressor(**params)\n    model.fit(X_train[FEATURES], y_train)\n    y_pred = model.predict(X_val[FEATURES])\n    score = mae(y_val, y_pred)\n    print(score)\n    return score\n\n\ndef objective(trial):\n    FEATURES = []\n    for feat in all_cols:\n        select_feat = trial.suggest_categorical(feat, [True, False])\n        if select_feat:\n            FEATURES.append(feat)\n        \n    model = HistGradientBoostingRegressor(categorical_features=[i in cat_cols for i in FEATURES], max_iter=1000, loss='absolute_error', random_state=111)\n    model.fit(X_train[FEATURES], y_train)\n    y_pred = model.predict(X_val[FEATURES])\n    score = mean_absolute_error(y_val, y_pred)  \n    print(score)\n    return score\n\n \n#optuna.logging.set_verbosity(optuna.logging.WARNING)\nstudy = optuna.create_study(direction = 'minimize',sampler = TPESampler(seed=111))\nstudy.optimize(objective, n_trials=50)\nprint(\"Best Score:\",study.best_value)\nprint(\"Best trial\",[x for x in study.best_trial.params if study.best_trial.params[x]])\n\nbest_params = study.best_params","metadata":{"execution":{"iopub.status.busy":"2022-07-24T19:46:19.094826Z","iopub.execute_input":"2022-07-24T19:46:19.095363Z","iopub.status.idle":"2022-07-24T19:46:19.140004Z","shell.execute_reply.started":"2022-07-24T19:46:19.095308Z","shell.execute_reply":"2022-07-24T19:46:19.138491Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## training types\n\n#### Vary by pre-processing (last-week split, all columns)\n\n|nothing|impute 0|impute 0<br/>add missing|impute 0<br/>add missing<br/>remove pykes\n|---|---|---|---\n|5.0792|5.0498|5.0373|5.0587\n\n#### Vary by split types (impute0+add-missing, all columns)\n\n|last week|5 ts split|5 folds|5 shufflefolds\n|---|---|---|---\n|5.0373|6.254|6.196|6.0777\n\n#### Vary by columns (last-week split, impute0+add-missing)\n\n|all cols|select 1|select 2\n|---|---|---\n|5.0373|5.0016|4.9999","metadata":{}},{"cell_type":"code","source":"%%script echo skipping\n\n#4.9989678307719325 - ['y','road','weekday','weekend','midday','evening','night','cong_mean','quant5', 'quant3','cong_std','hm','x_cos_hour','hour_s','cong_hbins_mean','cong_xy_mean']\n#limited features: MAE:  5.02446280445921\n\nfrom sklearn.ensemble import HistGradientBoostingRegressor\n\n#FEATURES = X_train.columns\nFEATURES = ['y','road','weekday','weekend','midday','evening','night','cong_mean','quant5', 'quant3','cong_std','hm','x_cos_hour','hour_s','cong_hbins_mean','cong_xy_mean']\ncatfeats = [i in cat_cols for i in FEATURES]\nprint('categorical features: ', np.array(FEATURES)[catfeats])\n\n#{'learning_rate': 0.11031364518378378, 'max_leaf_nodes': 370, 'max_depth': 13, 'min_samples_leaf': 77, 'max_bins': 216}\n#learning_rate=0.0307, max_leaf_nodes=176, max_depth=18, min_samples_leaf=49\n#learning_rate=0.0006759689483814301, max_leaf_nodes=175, max_depth=16, min_samples_leaf=27\nregressor = HistGradientBoostingRegressor(categorical_features=catfeats, max_iter=1000, loss='absolute_error', random_state=111)\n\n#GradientBoostingRegressor().feature_importances_\n\n#1-MAE:  6.240727628602992\n\"\"\"cols = ['weekend', 'midday', 'night', 'cong_mean', '0direction', '3direction','4direction', '6direction', '7direction', '1road', '2road', '8road', '9road',\n '11road', '1hour', '3hour', '4hour', '5hour', '8hour', '9hour', '10hour','11hour', '12hour', '13hour', '1month', '2month', '3month', '2weekday',\n '3weekday', '3junction', '9junction', '10junction', '11junction']\nX = X[cols]\"\"\"\n\nregressor.fit(X_train[FEATURES], y_train)\ny_pred = regressor.predict(X_val[FEATURES])\ntest_pred = regressor.predict(test_df[FEATURES])\nprint('MAE: ', mean_absolute_error(y_val, y_pred))","metadata":{"execution":{"iopub.status.busy":"2022-07-24T19:46:19.141907Z","iopub.execute_input":"2022-07-24T19:46:19.142395Z","iopub.status.idle":"2022-07-24T19:46:19.184224Z","shell.execute_reply.started":"2022-07-24T19:46:19.142345Z","shell.execute_reply":"2022-07-24T19:46:19.183021Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%script echo skipping\n\n\nfrom sklearn.ensemble import HistGradientBoostingRegressor\n\ntrain_df = all_df[all_df.index<=train_df_last]\ntest_df = all_df[all_df.index>train_df_last]\ny = train_df.congestion\ntrain_df.drop(['congestion'], axis=1, inplace=True)\n\nFEATURES = train_df.columns\ncatfeats = [i in cat_cols for i in FEATURES]\nprint('categorical features: ', np.array(FEATURES)[catfeats])\n\nregressor = HistGradientBoostingRegressor(categorical_features=catfeats, max_iter=1000, loss='absolute_error', random_state=111)\n\nagg_pred = []\nagg_score = []\n\nsplitter = KFold(5, shuffle=True, random_state=111)\nfor i, (train_index, test_index) in enumerate(splitter.split(train_df, y)):\n    X_train, X_val = train_df.iloc[train_index], train_df.iloc[test_index]\n    y_train, y_val = y[train_index], y[test_index]\n    \n    regressor.fit(X_train[FEATURES], y_train)\n    y_pred = regressor.predict(X_val[FEATURES])\n    agg_pred.append(regressor.predict(test_df[FEATURES]))\n    agg_score.append(mean_absolute_error(y_val, y_pred))\n    print(i, 'MAE: ', agg_score[-1])\n    \ntest_pred = np.mean(agg_pred, axis=0)\nprint('mean score: ', np.mean(agg_score))","metadata":{"execution":{"iopub.status.busy":"2022-07-24T19:46:19.186323Z","iopub.execute_input":"2022-07-24T19:46:19.186862Z","iopub.status.idle":"2022-07-24T19:46:19.230328Z","shell.execute_reply.started":"2022-07-24T19:46:19.186777Z","shell.execute_reply":"2022-07-24T19:46:19.228971Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%script echo skipping\n\n#train/validate scores: 6.2444950475757075, 6.565473060477016\n#limited features: 6.23343663295066 5.112278133761486\n\ntrain_frame = h2o.H2OFrame(X_train[FEATURES])\nholdout_frame = h2o.H2OFrame(X_val[FEATURES])\n\nmodel = H2OAutoMLRegressor(seed=RANDOM_SEED, max_runtime_secs=600, nfolds=5, stopping_metric='MAE', sort_metric='MAE', stopping_rounds=10, verbosity='warn')\n\nmodel.fit(x=train_frame, y=y_train.values)\ntrain_pred = np.squeeze(model.predict(train_frame).as_data_frame().values)\nholdout_pred = np.squeeze(model.predict(holdout_frame).as_data_frame().values)\n\nscore_train = mean_absolute_error(train_pred, y_train)\nscore_holdout = mean_absolute_error(holdout_pred, y_val)\nprint(score_train, score_holdout)","metadata":{"execution":{"iopub.status.busy":"2022-07-24T19:46:19.232614Z","iopub.execute_input":"2022-07-24T19:46:19.233241Z","iopub.status.idle":"2022-07-24T19:46:19.272221Z","shell.execute_reply.started":"2022-07-24T19:46:19.233187Z","shell.execute_reply":"2022-07-24T19:46:19.271088Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%script echo skipping\n\n#6.2984373762376045 5.086570997935049\n\ny_col = 'congestion'\n\ntrain_frame = h2o.H2OFrame(X_train[FEATURES+[y_col]])\nholdout_frame = h2o.H2OFrame(X_val[FEATURES+[y_col]])\ntest_frame = h2o.H2OFrame(test_df[FEATURES])\n\n\nmodel = H2OAutoML(seed=RANDOM_SEED, max_runtime_secs=1800, nfolds=0, stopping_metric='MAE', sort_metric='MAE', stopping_rounds=15, verbosity='warn')\n\n#training_frame,\n#validation_frame = NULL,\n#leaderboard_frame = NULL,\n#blending_frame = NULL,\n#fold_column = 'foldid'\n\nmodel.train(y=y_col, training_frame=train_frame, validation_frame=holdout_frame)\ntrain_pred = np.squeeze(model.predict(train_frame).as_data_frame().values)\nholdout_pred = np.squeeze(model.predict(holdout_frame).as_data_frame().values)\ntest_pred = np.squeeze(model.predict(test_frame).as_data_frame().values)\n\nscore_train = mean_absolute_error(train_pred, y_train)\nscore_holdout = mean_absolute_error(holdout_pred, y_val)\nprint(score_train, score_holdout)","metadata":{"execution":{"iopub.status.busy":"2022-07-24T19:46:19.273671Z","iopub.execute_input":"2022-07-24T19:46:19.274045Z","iopub.status.idle":"2022-07-24T19:46:19.316646Z","shell.execute_reply.started":"2022-07-24T19:46:19.274008Z","shell.execute_reply":"2022-07-24T19:46:19.315594Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%script echo skipping\n\n#AttributeError: Can't get attribute 'new_block' on <module 'pandas.core.internals.blocks' from '/opt/conda/lib/python3.7/site-packages/pandas/core/internals/blocks.py'>\n!pip install -U lightautoml\n\nfrom lightautoml.automl.presets.tabular_presets import TabularAutoML, TabularUtilizedAutoML\nfrom lightautoml.tasks import Task\nimport lightautoml as lama\n\n#X[cat_cols] = X[cat_cols].apply(lambda col: col.cat.codes)\n\nall_cols = list(X.columns)\n\nroles = {\n    lama.dataset.roles.DatetimeRole(base_date=True, seasonality=('d'), base_feats=False): 'time',\n    lama.dataset.roles.CategoryRole(dtype=np.float): cat_cols,\n    lama.dataset.roles.NumericRole(dtype=np.float): 'cong_mean'\n}\n\ntask = Task('reg', loss='mse', metric='mse')\nautoml = TabularAutoML(task=task, timeout=300,\n                       reader_params  = {'cv': 5, 'random_state': 42},\n                       general_params = {'use_algos': [['lgb', 'lgb_tuned', 'cb', 'cb_tuned']]})\n\ntrain_pred = automl.fit_predict({'data': X_train[FEATURES], 'target': y_train}, train_features=all_cols, roles=roles, verbose=1)\ntrain_pred = train_pred.data[:, 0]\n\nholdout_pred = automl.predict({'data': X_test[FEATURES]}, features_names=all_cols)\nholdout_pred = holdout_pred.data[:, 0]\n\nscore_train = mean_absolute_error(train_pred, y_train)\nscore_holdout = mean_absolute_error(holdout_pred, y_test)\nprint(score_train, score_holdout)\n\nfast_fi = automl.get_feature_scores('fast')\nfast_fi = fast_fi.set_index('Feature')['Importance']\nprint('Importance:', fast_fi)","metadata":{"execution":{"iopub.status.busy":"2022-07-24T19:46:19.319344Z","iopub.execute_input":"2022-07-24T19:46:19.319702Z","iopub.status.idle":"2022-07-24T19:46:19.368571Z","shell.execute_reply.started":"2022-07-24T19:46:19.319659Z","shell.execute_reply":"2022-07-24T19:46:19.367020Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%script echo skipping\n\n#limited - Best Score: 7.01843357315337\n#all - Best Score: 7.0599229730902975\n\nfrom catboost import CatBoostRegressor\nfrom sklearn.metrics import mean_absolute_error as mae\nimport optuna\nfrom optuna.samplers import TPESampler\noptuna.logging.set_verbosity(optuna.logging.WARNING)\n\n\ndef objective(trial):\n    params = {\n        \"random_state\":trial.suggest_categorical(\"random_state\", [2022]),\n        'learning_rate' : trial.suggest_loguniform('learning_rate', 0.0001, 0.3),\n        'bagging_temperature' :trial.suggest_loguniform('bagging_temperature', 0.01, 100.00),\n        \"n_estimators\": 100,\n        \"max_depth\":trial.suggest_int(\"max_depth\", 4, 16),\n        'random_strength' :trial.suggest_int('random_strength', 0, 100),\n        \"l2_leaf_reg\":trial.suggest_float(\"l2_leaf_reg\",1e-8,3e-5),\n        \"min_child_samples\": trial.suggest_int(\"min_child_samples\", 5, 100),\n        \"max_bin\": trial.suggest_int(\"max_bin\", 200, 500),\n        'od_type': trial.suggest_categorical('od_type', ['IncToDec', 'Iter']),\n        'task_type': 'GPU',\n        'loss_function': 'MAE',\n        'eval_metric': 'MAE'\n    }\n\n    model = CatBoostRegressor(**params)\n    model.fit(\n        X_train, y_train,\n        eval_set=[(X_val, y_val)],\n        early_stopping_rounds=5, verbose=0\n    )\n\n    y_train_pred = model.predict(X_train)\n    y_valid_pred = model.predict(X_val)\n    train_mae = mae(y_train, y_train_pred)\n    valid_mae = mae(y_val, y_valid_pred)\n\n    print(f'MAE of Train: {train_mae}, Validation: {valid_mae}')\n    return valid_mae\n\n\n\nstudy = optuna.create_study(\n    study_name = 'cat_parameter_opt',\n    direction = 'minimize',\n    sampler = TPESampler(seed=42),\n)\nstudy.optimize(objective, n_trials=100)\nprint(\"Best Score:\",study.best_value)\nprint(\"Best trial\",study.best_trial.params)\n\nbest_params = study.best_params\n\nmodel = CatBoostRegressor(**best_params, n_estimators=3000, verbose=100)\nmodel = model.fit(X_train, y_train, eval_set=[(X_val, y_val)], early_stopping_rounds=5)\ntest_pred = model.predict(X_test)\n\nmodel.get_feature_importance()","metadata":{"execution":{"iopub.status.busy":"2022-07-24T19:46:19.371551Z","iopub.execute_input":"2022-07-24T19:46:19.372078Z","iopub.status.idle":"2022-07-24T19:46:19.414229Z","shell.execute_reply.started":"2022-07-24T19:46:19.372025Z","shell.execute_reply":"2022-07-24T19:46:19.413038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%script echo skipping\n\nss = pd.read_csv('/kaggle/input/tabular-playground-series-mar-2022/sample_submission.csv')\nss.congestion = test_pred\nss.to_csv('submission.csv',index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}