{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Delta Prediction\n\nThis notebook demonstrates one of our team's solution methods,\"Delta Prediction.\n\n## First  \nI got a lot of advice on this solution from my teammates [@tomo20180402](https://www.kaggle.com/tomo20180402),[@Masaya](https://www.kaggle.com/irrohas) and [@koji](https://www.kaggle.com/minomonter).  \nThank you for your help in creating this solution  \n\n## Overview\nThis is a model of Light GBM with the objective function of the difference from ground_truth (delta_latDeg,delta_lngDeg).  \nIt was an effective method for downtown and trees, but it did not work well for highways.  \nTherefore, it was very powerful in Public, but not so effective in Private.\n\n### Score  \n* Public : 　6.665 → 5.162　 ▲1.503  \n* Private: 　5.098 → 4.687 　▲0.411  \n* CV　　: 　4.879 → 4.215 　▲0.664\n\n### Some Point  \n* Examples of feature values  \n  1. Location information of baseline_data\n  2. Difference and statistics between previous and next point\n  3. phoneName\n* We applied this process a total of two times in the pipeline.  \n  1. The first one is before post-processing such as kf and phone_mean.  \n  1. The second one is after post-processing.\n* We divided the phone into 5 parts and created a CV. Actually,we wanted to make a CV with \"collectionName\", but it didn't get a good score.  \n  This CV may have caused to overlearning.\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"code","source":"from pathlib import Path\nimport numpy as np\nimport pandas as pd\nimport copy\nimport plotly.express as px\nimport plotly.graph_objects as go\nimport pyproj\nimport json\nimport bisect\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom matplotlib.collections import LineCollection\nfrom matplotlib.colors import ListedColormap, BoundaryNorm\nimport pickle\nimport random\nfrom tqdm.notebook import tqdm\nimport lightgbm as lgb\nimport warnings\nwarnings.simplefilter('ignore')\npd.set_option('display.max_rows',30)\npd.set_option('display.max_columns',None)","metadata":{"_kg_hide-output":true,"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-08-07T07:48:41.965166Z","iopub.execute_input":"2021-08-07T07:48:41.965617Z","iopub.status.idle":"2021-08-07T07:48:45.208436Z","shell.execute_reply.started":"2021-08-07T07:48:41.965526Z","shell.execute_reply":"2021-08-07T07:48:45.207167Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def calc_haversine(lat1, lon1, lat2, lon2):\n    RADIUS = 6_367_000\n    lat1, lon1, lat2, lon2 = map(np.radians, [lat1, lon1, lat2, lon2])\n    dlat = lat2 - lat1\n    dlon = lon2 - lon1\n    a = np.sin(dlat/2)**2 + \\\n        np.cos(lat1) * np.cos(lat2) * np.sin(dlon/2)**2\n    dist = 2 * RADIUS * np.arcsin(a**0.5)\n    return dist\n\ndef percentile50(x):\n    return np.percentile(x, 50)\ndef percentile95(x):\n    return np.percentile(x, 95)\n\ndef get_train_score(df, gt):\n    gt = gt.rename(columns={'latDeg':'latDeg_gt', 'lngDeg':'lngDeg_gt'})\n    df = df.merge(gt, on=['collectionName', 'phoneName', 'millisSinceGpsEpoch'], how='inner')\n    # calc_distance_error\n    df['err'] = calc_haversine(df['latDeg_gt'], df['lngDeg_gt'], df['latDeg'], df['lngDeg'])\n    # calc_evaluate_score\n    df['phone'] = df['collectionName'] + '_' + df['phoneName']\n    res = df.groupby('phone')['err'].agg([percentile50, percentile95])\n    res['p50_p90_mean'] = (res['percentile50'] + res['percentile95']) / 2 \n    score = res['p50_p90_mean'].mean()\n    return score","metadata":{"_kg_hide-output":true,"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-08-07T07:48:45.210437Z","iopub.execute_input":"2021-08-07T07:48:45.210820Z","iopub.status.idle":"2021-08-07T07:48:45.223870Z","shell.execute_reply.started":"2021-08-07T07:48:45.210762Z","shell.execute_reply":"2021-08-07T07:48:45.221946Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def eval_all(df_pred, df_gt):\n    scores = []\n    compared_cols = [\"latDeg_truth\",\"lngDeg_truth\",\"latDeg_pred\",\"lngDeg_pred\"]\n    collections = sorted(df_gt['collectionName'].unique())\n    for collection in collections:\n        df_pred_col = df_pred[df_pred['collectionName'] == collection]\n        df_gt_col = df_gt[df_gt['collectionName'] == collection]\n        \n        score = get_train_score(df_pred_col, df_gt_col)\n        \n        df_merged = pd.merge_asof(df_gt_col.sort_values('millisSinceGpsEpoch'), df_pred_col.sort_values('millisSinceGpsEpoch'), \n                                  on=\"millisSinceGpsEpoch\", by=[\"collectionName\", \"phoneName\"], \n                                  direction='nearest',tolerance=100000, suffixes=('_truth', '_pred'))\n        df_merged = df_merged.sort_values(by=[\"collectionName\", \"phoneName\", \"millisSinceGpsEpoch\"], ignore_index=True)\n\n        haversine = calc_haversine(*df_merged[compared_cols].to_numpy().transpose()).mean()\n        scores.append([collection, haversine, score])\n    \n    score = get_train_score(df_pred, df_gt)\n    df_merged = pd.merge_asof(df_gt.sort_values('millisSinceGpsEpoch'), df_pred.sort_values('millisSinceGpsEpoch'), \n                              on=\"millisSinceGpsEpoch\", by=[\"collectionName\", \"phoneName\"], \n                              direction='nearest',tolerance=100000, suffixes=('_truth', '_pred'))\n    haversine = calc_haversine(*df_merged[compared_cols].to_numpy().transpose()).mean()\n    scores.append(['all', haversine, score])\n    \n    df_scores = pd.DataFrame(scores, columns=['collection', 'haversine', 'score'])\n    return df_scores","metadata":{"_kg_hide-output":true,"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-08-07T07:48:45.226857Z","iopub.execute_input":"2021-08-07T07:48:45.227345Z","iopub.status.idle":"2021-08-07T07:48:45.241016Z","shell.execute_reply.started":"2021-08-07T07:48:45.227295Z","shell.execute_reply":"2021-08-07T07:48:45.240110Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_features(df):\n    \n    for i in range(1,51):\n        df[f\"latDeg_dist_lag{i}\"] = (df[\"latDeg\"] - df.groupby([\"collectionName\",\"phoneName\"])[\"latDeg\"].shift(i)).fillna(0)\n        df[f\"lngDeg_dist_lag{i}\"] = (df[\"lngDeg\"] - df.groupby([\"collectionName\",\"phoneName\"])[\"lngDeg\"].shift(i)).fillna(0)\n        df[f\"latDeg_dist_lead{i}\"] = (df[\"latDeg\"] - df.groupby([\"collectionName\",\"phoneName\"])[\"latDeg\"].shift(-i)).fillna(0)\n        df[f\"lngDeg_dist_lead{i}\"] = (df[\"lngDeg\"] - df.groupby([\"collectionName\",\"phoneName\"])[\"lngDeg\"].shift(-i)).fillna(0)\n\n    for i in range(1,26):\n        df[f\"latDeg_dist_laglead{i}_mean\"] = df[[f\"latDeg_dist_lag{num}\" for num in range(1,i+1)] + [f\"latDeg_dist_lead{num}\" for num in range(1,i+1)]].mean(axis=1)\n        df[f\"lngDeg_dist_laglead{i}_mean\"] = df[[f\"lngDeg_dist_lag{num}\" for num in range(1,i+1)] + [f\"lngDeg_dist_lead{num}\" for num in range(1,i+1)]].mean(axis=1)\n        df[f\"latDeg_dist_laglead{i}_median\"] = df[[f\"latDeg_dist_lag{num}\" for num in range(1,i+1)] + [f\"latDeg_dist_lead{num}\" for num in range(1,i+1)]].median(axis=1)\n        df[f\"lngDeg_dist_laglead{i}_median\"] = df[[f\"lngDeg_dist_lag{num}\" for num in range(1,i+1)] + [f\"lngDeg_dist_lead{num}\" for num in range(1,i+1)]].median(axis=1)\n\n    for i in range(2,51):\n        for col in [\"lag\",\"lead\"]:\n            df[f\"latDeg_dist_{col}{i}_mean\"] = df[[f\"latDeg_dist_{col}{num}\" for num in range(1,i+1)]].mean(axis=1)\n            df[f\"lngDeg_dist_{col}{i}_mean\"] = df[[f\"lngDeg_dist_{col}{num}\" for num in range(1,i+1)]].mean(axis=1)\n            df[f\"latDeg_dist_{col}{i}_median\"] = df[[f\"latDeg_dist_{col}{num}\" for num in range(1,i+1)]].median(axis=1)\n            df[f\"lngDeg_dist_{col}{i}_median\"] = df[[f\"lngDeg_dist_{col}{num}\" for num in range(1,i+1)]].median(axis=1)\n    \n    return df\n\n\ndef get_haversine_kf(df_before, df_after):\n    df_before = df_before.rename(columns={\"latDeg\":\"latDeg_before\", \"lngDeg\":\"lngDeg_before\"})\n    df_after = df_after.rename(columns={\"latDeg\":\"latDeg_after\", \"lngDeg\":\"lngDeg_after\"})\n    df = pd.merge(df_before, df_after, on=['collectionName', 'phoneName', 'millisSinceGpsEpoch', 'phone'])\n        \n    df[\"delta_lat_kf\"] = df[\"latDeg_after\"] - df[\"latDeg_before\"]\n    df[\"delta_lng_kf\"] = df[\"lngDeg_after\"] - df[\"lngDeg_before\"]\n        \n    add_features = [\"delta_lat_kf\", \"delta_lng_kf\"] \n    \n    df = df.fillna(0)\n    df = df[['collectionName', 'phoneName', 'millisSinceGpsEpoch'] + add_features]\n    \n    return df","metadata":{"_kg_hide-input":false,"execution":{"iopub.status.busy":"2021-08-07T07:48:45.242402Z","iopub.execute_input":"2021-08-07T07:48:45.242928Z","iopub.status.idle":"2021-08-07T07:48:45.263199Z","shell.execute_reply.started":"2021-08-07T07:48:45.242892Z","shell.execute_reply":"2021-08-07T07:48:45.262407Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class FeatureGenerator:\n    def __init__(self, tomo_features):\n        self.tomo_features = tomo_features\n        self.bins_lat = None\n        self.bins_lng = None\n        self.original_lat_cols = None\n        self.original_lng_cols = None\n        self.original_phone_cols = None\n\n    def generate(self, df_input, routes, is_train=True):\n        df = df_input[df_input['collectionName'].isin(routes)]\n        if is_train:\n            drop_index = df[df['dist'] > 100].index\n            df = df.drop(drop_index).reset_index(drop=True)\n\n            df = df.rename(columns={'latDeg_basepred': 'latDeg'})\n            df = df.rename(columns={'lngDeg_basepred': 'lngDeg'})\n\n            _, self.bins_lat = pd.cut(df['latDeg'], bins=1000, retbins=True)\n            _, self.bins_lng = pd.cut(df['lngDeg'], bins=1000, retbins=True)\n       \n        df['lat_flg'] = pd.cut(df['latDeg'], self.bins_lat, labels=False)\n        df['lng_flg'] = pd.cut(df['lngDeg'], self.bins_lng, labels=False)\n\n        if is_train:\n            df['lat_flg_str'] = 'lat' + df['lat_flg'].astype(str)\n            df['lng_flg_str'] = 'lng' + df['lng_flg'].astype(str)\n            df_lat_flg_categorized = pd.get_dummies(df['lat_flg_str'])\n            df_lng_flg_categorized = pd.get_dummies(df['lng_flg_str'])\n            df_phone_categorized = pd.get_dummies(df['phoneName'])\n            self.original_lat_cols = df_lat_flg_categorized.columns\n            self.original_lng_cols = df_lng_flg_categorized.columns\n            self.original_phone_cols = df_phone_categorized.columns\n        else:\n            lat_flg_str = 'lat' + df['lat_flg'][~np.isnan(df['lat_flg'])].astype(np.int64).astype(str)\n            lng_flg_str = 'lng' + df['lng_flg'][~np.isnan(df['lng_flg'])].astype(np.int64).astype(str)\n            df_lat_flg_categorized = pd.get_dummies(lat_flg_str)\n            df_lng_flg_categorized = pd.get_dummies(lng_flg_str)\n            df_phone_categorized = pd.get_dummies(df['phoneName'])\n\n            missing_lat_cols = set(self.original_lat_cols) - set(df_lat_flg_categorized.columns)\n            missing_lng_cols = set(self.original_lng_cols) - set(df_lng_flg_categorized.columns)\n            missing_phone_cols = set(self.original_phone_cols) - set(df_phone_categorized.columns)\n            redundant_phone_cols = set(df_phone_categorized.columns) - set(self.original_phone_cols)\n            df_lng_flg_categorized.loc[:, missing_lng_cols] = 0\n            df_lat_flg_categorized.loc[:, missing_lat_cols] = 0\n            df_phone_categorized.loc[:, missing_phone_cols] = 0\n            df_phone_categorized = df_phone_categorized.drop(redundant_phone_cols, axis=1)\n\n            df_lat_flg_categorized = df_lat_flg_categorized.merge(\n                df, left_index=True, right_index=True, how='right').fillna(0)[self.original_lat_cols]\n            df_lng_flg_categorized = df_lng_flg_categorized.merge(\n                df, left_index=True, right_index=True, how='right').fillna(0)[self.original_lng_cols]\n\n        df_list = [\n            df[['collectionName', 'phoneName', 'millisSinceGpsEpoch']],\n            df[['latDeg', 'lngDeg']],\n            df_phone_categorized,\n            df_lat_flg_categorized,\n            df_lng_flg_categorized,\n            df[self.tomo_features],\n        ]\n        if is_train:\n            df_list.append(df[['delta_lat', 'delta_lng', 'fold']])\n\n        df = pd.concat(df_list, axis = 1)\n        df['flg'] = 'train' if is_train else 'test'\n        df = df.fillna(0)\n\n        return df","metadata":{"execution":{"iopub.status.busy":"2021-08-07T07:48:45.264577Z","iopub.execute_input":"2021-08-07T07:48:45.264904Z","iopub.status.idle":"2021-08-07T07:48:45.285758Z","shell.execute_reply.started":"2021-08-07T07:48:45.264875Z","shell.execute_reply":"2021-08-07T07:48:45.284748Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class DeltaPredict:\n    def __init__(self, lgbm_params, df_gt, df_train, df_test, is_second=False, apply_pseudo_label=False,\n                 df_pred_test=None, df_pseudo_label_test=None, early_stopping_rounds=None):\n        self.base_features = ['collectionName', 'phoneName', 'millisSinceGpsEpoch', 'fold', 'delta_lat', 'delta_lng', 'flg']\n\n        self.df_gt = df_gt\n        self.lgbm_params = lgbm_params\n        self.early_stopping_rounds = early_stopping_rounds\n\n        self.apply_pseudo_label = apply_pseudo_label\n        self.df_pred_test = df_pred_test\n        self.df_pseudo_label_test = df_pseudo_label_test\n        \n        if is_second:\n            self.df_train_tomo = get_features(df_train).drop(['latDeg', 'lngDeg', 'phone'], axis=1)\n            self.df_test_tomo = get_features(df_test).drop(['latDeg', 'lngDeg', 'phone'], axis=1)\n        else:\n            self.df_train_tomo = df_train.drop(['latDeg', 'lngDeg', 'phone'], axis=1)\n            self.df_test_tomo = df_test.drop(['latDeg', 'lngDeg', 'phone'], axis=1)\n\n        self.tomo_features = list(set(self.df_train_tomo.columns) - set([\n            'collectionName',\n            'phoneName',\n            'millisSinceGpsEpoch',\n            'heightAboveWgs84EllipsoidM',\n            'latDeg_lag1',\n            'lngDeg_lag1',\n        ]))\n\n        self.routes_dict = {\n            'tree_1': tree_1,\n            'downtown': downtowns\n        }\n        self.feature_generator_dict = {\n            'tree_1': FeatureGenerator(self.tomo_features),\n            'downtown': FeatureGenerator(self.tomo_features),\n        }\n        self.models_dict_dict = {}\n\n    def train_model(self, df_input, routes_name):\n        routes = self.routes_dict[routes_name]\n        df_train = self.feature_generator_dict[routes_name].generate(df_input, routes, is_train=True)\n\n        df_valid = pd.DataFrame()\n        models_dict = {}\n\n        for fold in df_train.fold.unique():\n            X_train = df_train[df_train['fold'] != fold].drop(self.base_features, axis=1)\n            y_train = df_train[df_train['fold'] != fold][['delta_lat', 'delta_lng']]\n\n            X_valid  = df_train[df_train['fold'] == fold].drop(self.base_features, axis=1)\n            y_valid  = df_train[df_train['fold'] == fold][['collectionName', 'phoneName', 'millisSinceGpsEpoch', 'latDeg', 'lngDeg', 'delta_lat','delta_lng']]\n\n            lgb_train_lat = lgb.Dataset(X_train, y_train.delta_lat)\n            lgb_eval_lat = lgb.Dataset(X_valid, y_valid.delta_lat)\n            model_lat = lgb.train(self.lgbm_params,\n                              lgb_train_lat,\n                              valid_sets=[lgb_train_lat, lgb_eval_lat],\n                              early_stopping_rounds=self.early_stopping_rounds,\n                              verbose_eval=-1)\n            delta_lat_valid = model_lat.predict(X_valid)\n\n            lgb_train_lng = lgb.Dataset(X_train, y_train.delta_lng)\n            lgb_eval_lng = lgb.Dataset(X_valid, y_valid.delta_lng)\n            model_lng = lgb.train(self.lgbm_params,\n                              lgb_train_lng,\n                              valid_sets=[lgb_train_lng, lgb_eval_lng],\n                              early_stopping_rounds=self.early_stopping_rounds,\n                              verbose_eval=-1)\n            delta_lng_valid = model_lng.predict(X_valid)\n\n            y_valid['delta_lat'] = delta_lat_valid\n            y_valid['delta_lng'] = delta_lng_valid\n            df_valid = pd.concat([df_valid, y_valid])\n\n            models_dict[fold] = {}\n            models_dict[fold]['lat'] = model_lat\n            models_dict[fold]['lng'] = model_lng\n        return models_dict, df_valid\n\n    def predict_test_data(self, df_input, routes_name):\n        routes = self.routes_dict[routes_name]\n        df_test = self.feature_generator_dict[routes_name].generate(df_input, routes, is_train=False)\n        \n        df_delta_test = df_test[['collectionName', 'phoneName', 'millisSinceGpsEpoch', 'latDeg', 'lngDeg']]\n\n        models_dict = self.models_dict_dict[routes_name]\n\n        for fold in models_dict.keys():\n            X_test = df_test.drop(['collectionName', 'phoneName', 'millisSinceGpsEpoch', 'flg'], axis=1)\n            delta_lat_test_l  = models_dict[fold]['lat'].predict(X_test)\n\n            delta_lng_test_l = models_dict[fold]['lng'].predict(X_test)\n\n            delta_lat_test = delta_lat_test_l\n            delta_lng_test = delta_lng_test_l\n\n            df_delta_test[f'delta_lat_{fold}'] = delta_lat_test\n            df_delta_test[f'delta_lng_{fold}'] = delta_lng_test\n\n        df_delta_test['delta_lat'] = (df_delta_test['delta_lat_fold_1'] + \n                                      df_delta_test['delta_lat_fold_2'] + \n                                      df_delta_test['delta_lat_fold_3'] + \n                                      df_delta_test['delta_lat_fold_4'] + \n                                      df_delta_test['delta_lat_fold_5']\n                                             ) / 5\n        df_delta_test['delta_lng'] = (df_delta_test['delta_lng_fold_1'] +\n                                      df_delta_test['delta_lng_fold_2'] +\n                                      df_delta_test['delta_lng_fold_3'] +\n                                      df_delta_test['delta_lng_fold_4'] +\n                                      df_delta_test['delta_lng_fold_5']\n                                             ) / 5\n\n        df_delta_test = df_delta_test[['collectionName', 'phoneName', 'millisSinceGpsEpoch', 'delta_lat', 'delta_lng']]\n        return df_delta_test\n\n    def train_and_predict(self, df_input):\n        df = df_input[['phone', 'millisSinceGpsEpoch', 'latDeg', 'lngDeg', 'collectionName', 'phoneName']].copy()\n        df_train_epoch = df_input[['collectionName', 'phoneName', 'millisSinceGpsEpoch']]\n\n        if self.apply_pseudo_label:\n            df_all = pd.concat([df, self.df_pred_test], axis=0)\n            df_gt_all = pd.concat([self.df_gt, self.df_pseudo_label_test], axis=0)\n            df_tomo_all = pd.concat([self.df_train_tomo, self.df_test_tomo], axis=0)\n        else:\n            df_all = df\n            df_gt_all = self.df_gt\n            df_tomo_all = self.df_train_tomo\n\n        df_all = df_all.merge(df_gt_all[['collectionName', 'phoneName', 'millisSinceGpsEpoch', 'latDeg', 'lngDeg']],\n                      on=['collectionName', 'phoneName', 'millisSinceGpsEpoch'], \n                      how='left', suffixes=['_basepred', '_truth'])\n        df_all['dist'] = calc_haversine(*df_all[['latDeg_basepred', 'lngDeg_basepred', 'latDeg_truth', 'lngDeg_truth']].values.T)\n        df_all['delta_lat'] = df_all['latDeg_truth'] - df_all['latDeg_basepred']\n        df_all['delta_lng'] = df_all['lngDeg_truth'] - df_all['lngDeg_basepred']\n        df_all = df_all.merge(df_tomo_all, on=['collectionName', 'phoneName', 'millisSinceGpsEpoch'], how='left')\n        df_all['phone'] = df_all['collectionName'] + '_' + df_all['phoneName']\n        random.seed(0)\n        \n        fold_df = pd.read_csv(\"../input/get-cross-validation-set/fold.csv\")[['collectionName', 'phoneName', 'millisSinceGpsEpoch',f'fold__{target_set}']]\n        fold_df = fold_df.rename(columns={f\"fold__{target_set}\":\"fold\"})\n        df_all = df_all.merge(fold_df, on=['collectionName', 'phoneName', 'millisSinceGpsEpoch'], how=\"left\")\n\n        regions = [\n            'downtown', \n            'tree_1', \n        ]\n        valid_list = []\n        for region in tqdm(regions, desc='LightGBM (Training & Inference)'):\n            print(\"***\" + region + \"***\")\n            self.models_dict_dict[region], df_pred_valid_region = self.train_model(df_all, region)\n            valid_list.append(df_pred_valid_region)\n\n        df_all = df_all.rename(columns={'latDeg_basepred': 'latDeg', 'lngDeg_basepred': 'lngDeg'})\n        for df_delta in valid_list:\n            df_tmp = df_all.merge(\n                df_delta, on=['collectionName', 'phoneName', 'millisSinceGpsEpoch', 'latDeg', 'lngDeg'], \n                how='left', suffixes=['_origin', ''])\n            df_tmp['latDeg'] = df_tmp['latDeg'] + df_tmp['delta_lat'].fillna(0)\n            df_tmp['lngDeg'] = df_tmp['lngDeg'] + df_tmp['delta_lng'].fillna(0)\n            df_all = df_tmp[['phone', 'collectionName', 'phoneName', 'millisSinceGpsEpoch', 'latDeg', 'lngDeg']]\n        df_deltapred_train = df_train_epoch.merge(\n            df_all, on=['collectionName', 'phoneName', 'millisSinceGpsEpoch'], how='left')\n                \n        return df_deltapred_train\n\n    def predict(self, df_input):\n        df = df_input[['phone', 'millisSinceGpsEpoch', 'latDeg', 'lngDeg', 'collectionName', 'phoneName']].copy()\n\n        df = df.merge(self.df_test_tomo, on=['collectionName', 'phoneName', 'millisSinceGpsEpoch'], how='left')\n\n        regions = [\n            'downtown',\n            'tree_1', \n                  ]\n        test_list = []\n        for region in tqdm(regions, desc='LightGBM (Inference)'):\n            print(\"***\" + region + \"***\")\n            df_pred_test_region = self.predict_test_data(df, region)\n            test_list.append(df_pred_test_region)\n\n        for df_delta in test_list:\n            df_tmp = df.merge(\n                df_delta, on=['collectionName', 'phoneName', 'millisSinceGpsEpoch'], \n                how='left')\n            df_tmp['latDeg'] = df_tmp['latDeg'] + df_tmp['delta_lat'].fillna(0)\n            df_tmp['lngDeg'] = df_tmp['lngDeg'] + df_tmp['delta_lng'].fillna(0)\n            df = df_tmp[['phone', 'collectionName', 'phoneName', 'millisSinceGpsEpoch', 'latDeg', 'lngDeg']]\n        df['latDeg'] = df['latDeg'].astype(np.float)\n        df['lngDeg'] = df['lngDeg'].astype(np.float)\n        df['millisSinceGpsEpoch'] = df['millisSinceGpsEpoch'].astype(np.int64)\n                \n        return df\n    \n    def add_features(self, df_add, is_train=True):\n        if is_train:\n            assert len(self.df_train_tomo) == len(df_add), f\"mismatch length {len(self.df_train_tomo)} - {len(df_add)}\"\n            self.df_train_tomo = pd.merge(self.df_train_tomo, df_add, on=['collectionName', 'phoneName', 'millisSinceGpsEpoch'])\n        else:\n            assert len(self.df_test_tomo) == len(df_add), f\"mismatch length {len(self.df_test_tomo)} - {len(df_add)}\"\n            self.df_test_tomo = pd.merge(self.df_test_tomo, df_add, on=['collectionName', 'phoneName', 'millisSinceGpsEpoch'])","metadata":{"execution":{"iopub.status.busy":"2021-08-07T07:48:45.288227Z","iopub.execute_input":"2021-08-07T07:48:45.288538Z","iopub.status.idle":"2021-08-07T07:48:45.336114Z","shell.execute_reply.started":"2021-08-07T07:48:45.288509Z","shell.execute_reply":"2021-08-07T07:48:45.334808Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open('../input/region-classification/region_type_train.json') as f:\n    region_type_train = json.load(f)\nwith open('../input/region-classification/region_type_test.json') as f:\n    region_type_test = json.load(f)\n\ndowntowns = [*[key for (key, val) in region_type_train.items() if 'downtown' in val], *[key for (key, val) in region_type_test.items() if 'downtown' in val]]\ntrees = [*[key for (key, val) in region_type_train.items() if 'tree' in val], *[key for (key, val) in region_type_test.items() if 'tree' in val]]\nhighways = [*[key for (key, val) in region_type_train.items() if 'tree' not in val and 'downtown' not in val],\n          *[key for (key, val) in region_type_test.items() if 'tree' not in val and 'downtown' not in val]]\ntree_1 = [*[key for (key, val) in region_type_train.items() if 'tree_1' in val], *[key for (key, val) in region_type_test.items() if 'tree_1' in val]]\ndowntown_tree_1 = downtowns + tree_1","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-08-07T07:48:45.337547Z","iopub.execute_input":"2021-08-07T07:48:45.338230Z","iopub.status.idle":"2021-08-07T07:48:45.369292Z","shell.execute_reply.started":"2021-08-07T07:48:45.338192Z","shell.execute_reply":"2021-08-07T07:48:45.368337Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"datapath = Path(\"../input/google-smartphone-decimeter-challenge/\")\nground_truths = (datapath / \"train\").rglob(\"ground_truth.csv\")\ndf_gt = pd.concat([pd.read_csv(filepath) for filepath in ground_truths], ignore_index=True)\n\ndf_pred_train = pd.read_csv('../input/gnss-ensembled/train_submission_filtered.csv')\ndf_pred_train['collectionName'] = df_pred_train['phone'].apply(lambda x: x.split('_')[0])\ndf_pred_train['phoneName'] = df_pred_train['phone'].apply(lambda x: x.split('_')[1])\n\ndf_pred_test = pd.read_csv('../input/gnss-ensembled/submission_filtered.csv')\ndf_pred_test['collectionName'] = df_pred_test['phone'].apply(lambda x: x.split('_')[0])\ndf_pred_test['phoneName'] = df_pred_test['phone'].apply(lambda x: x.split('_')[1])\n\ndf_train_tomo = pd.read_pickle('../input/210723-get-feature-for-delta-pred-1/train_df.pkl')\ndf_test_tomo = pd.read_pickle('../input/210723-get-feature-for-delta-pred-1/test_df.pkl')\n\ndf_pseudo_label_test = pd.read_csv('../input/210711-delta-pred-3-my-get-mean-over-phones/submission.csv')\ndf_pseudo_label_test['collectionName'] = df_pseudo_label_test['phone'].apply(lambda x: x.split('_')[0])\ndf_pseudo_label_test['phoneName'] = df_pseudo_label_test['phone'].apply(lambda x: x.split('_')[1])","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-08-07T07:48:45.371501Z","iopub.execute_input":"2021-08-07T07:48:45.372104Z","iopub.status.idle":"2021-08-07T07:49:00.793439Z","shell.execute_reply.started":"2021-08-07T07:48:45.372067Z","shell.execute_reply":"2021-08-07T07:49:00.792349Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lgbm_params_1 = {\n    \"objective\":\"regression\",\n    \"metric\":\"rmse\",\n    \"max_depth\":-1,\n    \"learning_rate\":0.1,\n    \"random_state\":2021,\n    \"n_estimators\":10000,\n    \"verbose\":-1,\n}\nlgbm_params_2 = {\n    \"objective\":\"regression\",\n    \"metric\":\"rmse\",\n    \"random_state\":2021,\n}\nlgbm_params_3 = {\n    \"random_state\":2021,\n}\n\ntarget_set = 1","metadata":{"execution":{"iopub.status.busy":"2021-08-07T07:49:00.795160Z","iopub.execute_input":"2021-08-07T07:49:00.795539Z","iopub.status.idle":"2021-08-07T07:49:00.802115Z","shell.execute_reply.started":"2021-08-07T07:49:00.795505Z","shell.execute_reply":"2021-08-07T07:49:00.800640Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training","metadata":{}},{"cell_type":"code","source":"delta_predictor_1 = DeltaPredict(lgbm_params_3, df_gt, df_train_tomo, df_test_tomo, is_second=False)","metadata":{"execution":{"iopub.status.busy":"2021-08-07T07:49:00.803574Z","iopub.execute_input":"2021-08-07T07:49:00.803957Z","iopub.status.idle":"2021-08-07T07:49:02.918846Z","shell.execute_reply.started":"2021-08-07T07:49:00.803924Z","shell.execute_reply":"2021-08-07T07:49:02.918007Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_pred_train_processed = df_pred_train.copy()\ndf_pred_train_processed = delta_predictor_1.train_and_predict(df_pred_train_processed)","metadata":{"execution":{"iopub.status.busy":"2021-08-07T07:49:02.920018Z","iopub.execute_input":"2021-08-07T07:49:02.920504Z","iopub.status.idle":"2021-08-07T07:52:11.312552Z","shell.execute_reply.started":"2021-08-07T07:49:02.920452Z","shell.execute_reply":"2021-08-07T07:52:11.311570Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Before Delta Prediction","metadata":{}},{"cell_type":"code","source":"df_eval = eval_all(df_pred_train, df_gt)\ndf_eval[df_eval['collection'].isin(downtown_tree_1)]","metadata":{"execution":{"iopub.status.busy":"2021-08-07T07:52:11.313941Z","iopub.execute_input":"2021-08-07T07:52:11.314389Z","iopub.status.idle":"2021-08-07T07:52:15.054297Z","shell.execute_reply.started":"2021-08-07T07:52:11.314358Z","shell.execute_reply":"2021-08-07T07:52:15.052991Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## After Delta Prediction","metadata":{}},{"cell_type":"code","source":"df_eval = eval_all(df_pred_train_processed, df_gt)\ndf_eval[df_eval['collection'].isin(downtown_tree_1)]","metadata":{"execution":{"iopub.status.busy":"2021-08-07T07:52:15.056177Z","iopub.execute_input":"2021-08-07T07:52:15.056631Z","iopub.status.idle":"2021-08-07T07:52:18.792352Z","shell.execute_reply.started":"2021-08-07T07:52:15.056581Z","shell.execute_reply":"2021-08-07T07:52:18.791267Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Predict and Submission","metadata":{}},{"cell_type":"code","source":"df_pred_test_processed = df_pred_test.copy()\ndf_pred_test_processed = delta_predictor_1.predict(df_pred_test_processed)","metadata":{"execution":{"iopub.status.busy":"2021-08-07T07:52:22.658327Z","iopub.execute_input":"2021-08-07T07:52:22.658713Z","iopub.status.idle":"2021-08-07T07:52:36.963367Z","shell.execute_reply.started":"2021-08-07T07:52:22.658685Z","shell.execute_reply":"2021-08-07T07:52:36.962357Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub = pd.read_csv('../input/google-smartphone-decimeter-challenge/sample_submission.csv')\nsub = sub.assign(\n    latDeg = df_pred_test_processed.latDeg,\n    lngDeg = df_pred_test_processed.lngDeg\n)\nsub.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2021-08-07T07:52:36.965117Z","iopub.execute_input":"2021-08-07T07:52:36.965894Z","iopub.status.idle":"2021-08-07T07:52:37.838422Z","shell.execute_reply.started":"2021-08-07T07:52:36.965844Z","shell.execute_reply":"2021-08-07T07:52:37.837179Z"},"trusted":true},"execution_count":null,"outputs":[]}]}