{"cells":[{"metadata":{},"cell_type":"markdown","source":"I've released,[my inference kernel here](https://www.kaggle.com/kneroma/lgbm-on-lyft-tabular-data-inference). Here, Now, I'm releasing the training method. To use this kernel, you can just fork it and give your trained lgbms as input to the [inference kernel](https://www.kaggle.com/kneroma/lgbm-on-lyft-tabular-data-inference).\n\nThe training step uses a [parquet version of the official training datatset](https://www.kaggle.com/kneroma/lyft-train-as-parquet).\n\nOk, let's dive in it !","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd, numpy as np\nimport seaborn as sns\nfrom matplotlib import pyplot as plt\nfrom pathlib import Path\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import lightgbm as lgb\nimport gc\nfrom sklearn.model_selection import train_test_split\nimport time\nfrom tqdm.notebook import tqdm\n\nimport re,json","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"DATA_ROOT = Path(\"../input/lyft-train-as-parquet/train\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Reading the data","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_scene_path(scene):\n meta = \"meta_{}_{}.json\".format(*re.search( r\"scenes_(\\d+)_(\\d+)\", scene.stem).groups())\n with open(DATA_ROOT/meta) as f:\n meta = json.load(f)\n frame = DATA_ROOT/meta[\"frames\"][\"results\"][\"filename\"]\n agent = DATA_ROOT/meta[\"agents\"][\"results\"][\"filename\"]\n return (scene, frame, agent)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"SCENES = np.array(list(DATA_ROOT.glob(\"scenes_*.parquet.snappy\")))\nSCENES = SCENES[np.random.permutation(len(SCENES))]\nprint(\"NB SCENES:\", len(SCENES))\nscene = SCENES[0]\nscene","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"get_scene_path(scene)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"reader = pd.read_parquet","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def merge(scenes, frames, agents, shift, verbose=False):\n df = scenes.merge(frames, on = \"scene_db_id\")\n df = df.merge(agents, on=\"frame_db_id\")\n\n shift_cols = [\n \"centroid_x\", \"centroid_y\",\n \"yaw\",\n \"velocity_x\",\"velocity_y\",\n \"nagents\",\"nlights\",\n 'extent_x', 'extent_y','extent_z',\n 'label_probabilities_PERCEPTION_LABEL_UNKNOWN',\n 'label_probabilities_PERCEPTION_LABEL_CAR', \n 'label_probabilities_PERCEPTION_LABEL_CYCLIST', \n 'label_probabilities_PERCEPTION_LABEL_PEDESTRIAN',\n ]\n new_shift_cols = [\"centroid_xs\", \"centroid_ys\"] + shift_cols[2:]\n\n df[new_shift_cols] = df.groupby([\"scene_db_id\", \"track_id\"])[shift_cols].shift(shift)\n nulls = df[[\"centroid_xs\", \"centroid_ys\", \"yaw\", \"velocity_x\",\"velocity_y\"]].isnull().any(1)\n shape0 = df.shape\n df = df[~nulls]\n \n if verbose:\n print(\"SHAPE0:\", shape0)\n print(\"nulls ratio:\", nulls.sum()/shape0[0])\n \n return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def read_all(shift=1, max_len=1e5):\n \"\"\"\n Read parquet files from the SCENES list until the df's size is greater than `max_len`.\n \n If you want better accuracy, you need to increase `max_len`.\n With `max_len=12e6` I got a score of 200.xxx.\n But note that training time increases as max_len increases.\n \"\"\"\n dfs = None\n scenes = SCENES[np.random.permutation(len(SCENES))]\n for scene in scenes:\n SCENES_FILE,FRAMES_FILE,AGENTS_FILE = get_scene_path(scene)\n \n scenes = reader(SCENES_FILE)\n \n frames = reader(FRAMES_FILE)\n frames[\"nagents\"] = frames[\"agent_index_interval_end\"] - frames[\"agent_index_interval_start\"]\n frames[\"nlights\"] = frames[\"traffic_light_faces_index_interval_end\"\n ] - frames[\"traffic_light_faces_index_interval_start\"]\n \n agents = reader(AGENTS_FILE)\n agents.rename(columns = {\"agent_id\": \"agent_db_id\"}, inplace=True)\n \n df = merge(scenes, frames, agents, shift=shift)\n \n dfs = df if dfs is None else dfs.append(df)\n dfs.reset_index(inplace=True, drop=True)\n \n if len(dfs) > max_len:\n break\n \n return dfs","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# LGBM trainer","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def lgbm_trainer(shift=1, root=None, params=None):\n t0 = time.strftime(\"%Y%m%d%H%M%S\")\n T00 = time.time()\n root = \"model_{}\".format(t0) if root is None else str(root)\n params = PARAMS if params is None else params\n \n df = read_all(shift=shift)\n print(\"df.shape:\", df.shape)\n \n df_centroid = df[[\"centroid_x\", \"centroid_y\"]]\n df = df[TRAIN_COLS]\n gc.collect()\n \n train_index, test_index = train_test_split(df.index.values.reshape((-1,1)),\n df.index.values, test_size = .20, random_state=177)[2:]\n print(\"\\n\")\n for suffix in [\"x\", \"y\"]:\n print(\"--> {}\".format(suffix.upper()))\n target_name = \"centroid_\" + suffix\n target = (df_centroid[target_name] - df[target_name+\"s\"])\n \n train_data = lgb.Dataset(df.loc[train_index], label= target.loc[train_index])\n test_data = lgb.Dataset(df.loc[test_index], label= target.loc[test_index])\n\n clf = lgb.train(params,train_data, valid_sets = [train_data, test_data],\n early_stopping_rounds=60, verbose_eval= 40)\n \n clf.save_model(\"models/{}/lgbm_{}_shift_{:02d}.bin\".format(root, suffix, shift))\n print('\\n')\n print(\"elapsed: {:.5f} min\".format((time.time()-T00)/60))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_time(format_=\"%Y-%m-%d %H:%M:%S\"):\n return time.strftime(format_)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_50_shifts(root=None):\n root = root or \"model_{}\".format(time.strftime(\"%Y%m%d%H%M%S\"))\n Path(\"models\").joinpath(root).mkdir(exist_ok=True, parents=True)\n params = PARAMS.copy()\n for shift in tqdm(list(range(50, 0, -1))):\n print('\\n ******************* SHIFT {:02d} {} ***********\\n'.format(shift,get_time()))\n \n if not (shift-1)%5:\n params[\"num_iterations\"] = max(100, params[\"num_iterations\"] - 20)\n# params[\"num_leaves\"] = max(31, params[\"num_leaves\"] - 10)\n \n meta = {\n \"TRAIN_COLS\": TRAIN_COLS,\n \"params\": params,\n \"shift\": shift,\n \"start\": get_time(),\n \"end\": None\n }\n \n \n lgbm_trainer(root=root, shift=shift, params=params)\n meta[\"end\"] = get_time()\n with open(\"models/{}/meta_shift_{:02d}.json\".format(root, shift), \"w\") as f:\n json.dump(meta, f, indent=2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Params","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"**Feel free to tweek these params a little bit**","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Feel free to tweek these params a little bit\n\nPARAMS = {\n 'objective':'regression', \n 'boosting': 'gbdt',\n# 'feature_fraction': 0.5 , \n# 'scale_pos_weight' : 1/40., \n 'num_iterations' : 200,\n 'learning_rate' : 0.15,\n 'max_depth': 31,\n# 'min_data_in_leaf': 64,\n 'num_leaves': 128,\n# 'bagging_freq' : 1,\n# 'bagging_fraction' : 0.8,\n# 'tree_learner': 'voting' ,\n# 'boost_from_average': True,\n 'verbosity' : 0,\n 'num_threads': 2,\n 'metric' : ['mse'],\n 'metric': [ \"l1\", \"rmse\"],\n \"verbosity\": 1,\n# 'reg_alpha': 0.1,\n# 'reg_lambda': 0.3\n }","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Uncomment the columns if you want more\n\nTRAIN_COLS = [\n# 'ego_translation_x', \n# 'ego_translation_y', \n# 'ego_translation_z', \n# 'ego_rotation_xx', \n# 'ego_rotation_xy', \n# 'ego_rotation_xz', \n# 'ego_rotation_yx', \n# 'ego_rotation_yy', \n# 'ego_rotation_yz', \n# 'ego_rotation_zx', \n# 'ego_rotation_zy', \n# 'ego_rotation_zz', \n 'extent_x', \n 'extent_y', \n 'extent_z', \n 'velocity_x', \n 'velocity_y', \n 'label_probabilities_PERCEPTION_LABEL_UNKNOWN', \n 'label_probabilities_PERCEPTION_LABEL_CAR', \n 'label_probabilities_PERCEPTION_LABEL_CYCLIST', \n 'label_probabilities_PERCEPTION_LABEL_PEDESTRIAN', \n 'yaw', \n 'nagents', \n 'nlights', \n 'centroid_xs', \n 'centroid_ys',\n]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"len(TRAIN_COLS):\", len(TRAIN_COLS))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training","execution_count":null},{"metadata":{"scrolled":false,"trusted":true},"cell_type":"code","source":"%%time\n\n# Train 50x2 lgbm models (50 time dimensions X 2 space dimensions)\n# Save it as lgbm_{x or y}_shift_{i:02d}\n# Each model has it's own meta_shift_{i:02d} which contains the model's params\n# You can juts feed the ouputs as inputs to the inference kernel\n# The inference kernel is at https://www.kaggle.com/kneroma/lgbm-on-lyft-tabular-data-inference\n\ntrain_50_shifts(\"lyft_lgbm_model\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"
@Kkiller
","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}