{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Analysis on the Distributions of Features and Targets\n\nAnalyze the distribution of features (except images) and targets for the random 8192 samples.\n\nChange log:\n- v7 - random selected sample"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"from typing import Dict\n\nfrom tempfile import gettempdir\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\nimport torch\nfrom torch import nn, optim\nfrom torch.utils.data import DataLoader\nimport torchvision\nfrom torchvision.models.resnet import resnet50, resnet18, resnet34, resnet101\nfrom tqdm import tqdm\n\nimport l5kit\nfrom l5kit.configs import load_config_data\nfrom l5kit.data import LocalDataManager, ChunkedDataset\nfrom l5kit.dataset import AgentDataset, EgoDataset\nfrom l5kit.rasterization import build_rasterizer\nfrom l5kit.evaluation import write_pred_csv, compute_metrics_csv, read_gt_csv, create_chopped_dataset\nfrom l5kit.evaluation.chop_dataset import MIN_FUTURE_STEPS\nfrom l5kit.evaluation.metrics import neg_multi_log_likelihood, time_displace\nfrom l5kit.geometry import transform_points\nfrom l5kit.visualization import PREDICTED_POINTS_COLOR, TARGET_POINTS_COLOR, draw_trajectory\nfrom prettytable import PrettyTable\nfrom pathlib import Path\n\nimport matplotlib.pyplot as plt\n\nimport os\nimport random\nimport time\n\nfrom IPython.display import display\nfrom tqdm import tqdm_notebook\nimport gc, psutil\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nl5kit.__version__","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# Memory measurement\ndef memory(verbose=True):\n    mem = psutil.virtual_memory()\n    gb = 1024*1024*1024\n    if verbose:\n        print('Physical memory:',\n              '%.2f GB (used),'%((mem.total - mem.available) / gb),\n              '%.2f GB (available)'%((mem.available) / gb), '/',\n              '%.2f GB'%(mem.total / gb))\n    return (mem.total - mem.available) / gb\n\ndef gc_memory(verbose=True):\n    m = gc.collect()\n    if verbose:\n        print('GC:', m, end=' | ')\n        memory()\n\nmemory();","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# folder = '..' \nfolder = '/kaggle'\ntest_run = False","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# --- Lyft configs ---\ncfg = {\n    'format_version': 4,\n    'data_path': f'{folder}/input/lyft-motion-prediction-autonomous-vehicles',\n    'model_params': {\n        'model_architecture': 'resnet34',\n        'history_num_frames': 10,\n        'history_step_size': 1,\n        'history_delta_time': 0.1,\n        'future_num_frames': 50,\n        'future_step_size': 1,\n        'future_delta_time': 0.1,\n    },\n    'raster_params': {\n        'raster_size': [2, 2],  # [224, 224],\n        'pixel_size': [0.5, 0.5],\n        'ego_center': [0.25, 0.5],\n        'map_type': 'py_semantic',\n        'satellite_map_key': 'aerial_map/aerial_map.png',\n        'semantic_map_key': 'semantic_map/semantic_map.pb',\n        'dataset_meta_key': 'meta.json',\n        'filter_agents_threshold': 0.5,\n    },\n    'train_data_loader': {\n        'key': 'scenes/train.zarr',\n        'batch_size': 512 if test_run else 8192,\n        'shuffle': True,  # so that we are not focus on a few scenes\n        'num_workers': 4,  # 4\n    },    \n    'test_data_loader': {\n        'key': 'scenes/test.zarr',\n        'batch_size': 512 if test_run else 8192,\n        'shuffle': True,  # so that we are not focus on a few scenes\n        'num_workers': 4,  # 4\n    },\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n# set env variable for data\nDIR_INPUT = cfg[\"data_path\"]\nos.environ[\"L5KIT_DATA_FOLDER\"] = DIR_INPUT\ndm = LocalDataManager()\n# Build rasterizer\nrasterizer = build_rasterizer(cfg, dm)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n# Train dataset\ntrain_cfg = cfg[\"train_data_loader\"]\ntrain_zarr = ChunkedDataset(dm.require(train_cfg[\"key\"])).open(cached=False)  # try to turn off cache\ntrain_dataset = AgentDataset(cfg, train_zarr, rasterizer)\ntrain_dataloader = DataLoader(train_dataset, shuffle=train_cfg[\"shuffle\"], \n                              batch_size=train_cfg[\"batch_size\"], num_workers=train_cfg[\"num_workers\"])\nprint(train_dataset)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n# Test dataset\ntest_cfg = cfg[\"test_data_loader\"]\ntest_zarr = ChunkedDataset(dm.require(test_cfg[\"key\"])).open(cached=False)  # try to turn off cache\ntest_mask = np.load(f\"{DIR_INPUT}/scenes/mask.npz\")[\"arr_0\"]\ntest_dataset = AgentDataset(cfg, test_zarr, rasterizer, agents_mask=test_mask)\ntest_dataloader = DataLoader(test_dataset, shuffle=test_cfg[\"shuffle\"],\n                             batch_size=test_cfg[\"batch_size\"], num_workers=test_cfg[\"num_workers\"])\nprint(test_dataset)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('train set size:', len(train_dataset))\nprint(' test set size:', len(test_dataset))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Single sample via dataset API"},{"metadata":{"trusted":true},"cell_type":"code","source":"data = train_dataset[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data.keys()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('datakey, shape, type:')\nfor k, v in data.items():\n    if isinstance(v, np.ndarray):\n        print('%30s'%k, v.shape, type(v))\n    else:\n        print('%30s'%k, type(v))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Example of single data\nfor k, v in data.items():\n    if isinstance(v, np.ndarray):\n        print(k)\n        print(' ', v[:3])\n    else:\n        print(k, v)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Distribution of a batch"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ntrain_data = next(iter(train_dataloader))\nmemory();","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ntest_data = next(iter(test_dataloader))\nmemory();","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del train_data['image']\ndel test_data['image']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"metrics = [\n    'target_positions',\n    'target_availabilities',\n    'history_positions',\n    'history_yaws',\n    'history_availabilities',\n    'centroid',\n    'yaw',\n    'extent',\n]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('datakey, shape, type:')\nfor k, v in train_data.items():\n    if isinstance(v, torch.Tensor):\n        print('%30s'%k, v.shape, type(v))\n    else:\n        print('%30s'%k, type(v))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('distinct track_id in the batch (train, test):',\n      pd.Series(train_data['track_id'].numpy()).nunique(), ',',\n      pd.Series(test_data['track_id'].numpy()).nunique())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"nx = 5\nplt_bins = 50\nfigure_width = 15\ndef plot_distribution(databatch, metrics=metrics):\n    for m in metrics:\n        data = databatch[m].numpy()\n        print(m, data.shape)\n        if m.endswith('availabilities'):\n            data = data.reshape(-1)\n        dim = len(data.shape)\n        if dim == 1:\n            plt.figure(figsize=(figure_width*2/5, 2))\n            plt.hist(data, bins=plt_bins)\n            plt.xlabel(m); plt.ylabel('N')\n            plt.grid()\n            plt.title(m)\n        elif dim == 2:\n            n = data.shape[1]\n            ny = np.ceil(n / nx).astype('int')\n            plt.figure(figsize=(figure_width, ny*2))\n            for i in range(n):\n                plt.subplot(ny, nx, i+1)\n                plt.hist(data[:, i], bins=plt_bins)\n                plt.xlabel(m+f' ({i})')\n                plt.grid()\n                if i%nx==0:\n                    plt.ylabel('N')\n            plt.suptitle(m)\n            plt.tight_layout(rect=[0, 0.03, 1, 0.95])\n        elif dim == 3:\n            # availabilities filter\n            if m.startswith('target_'):\n                weights = databatch['target_availabilities'].numpy()\n            elif m.startswith('history_'):\n                weights = databatch['history_availabilities'].numpy()\n            else:\n                weights = None\n            if weights is not None and (weights == 0).all():\n                continue\n            n, nj = data.shape[1:]\n            ny = np.ceil(n / nx).astype('int')\n            plt.figure(figsize=(figure_width, ny*2))\n            for i in range(n):\n                plt.subplot(ny, nx, i+1)\n                w = weights[:, i] if weights is not None else None\n                for j in range(nj):\n                    plt.hist(data[:, i, j], weights=w, bins=plt_bins, alpha=0.7, label=f'{j}')\n                plt.xlabel(m+f' ({i})')\n                plt.grid()\n                if i%nx == 0:\n                    plt.ylabel('N')\n                if i%nx == nx-1:\n                    plt.legend()\n            plt.suptitle(m)\n            plt.tight_layout(rect=[0, 0.03, 1, 0.95])\n        plt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Train set distribution"},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_distribution(train_data)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Test set distribution"},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_distribution(test_data)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Mean values of each metric"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_weights(databatch, m):\n    \"\"\"the availabilities of the frames as the weights\"\"\"\n    shape = databatch[m].shape\n    weights = None\n    if len(shape) == 3:\n        if m.startswith('target_'):\n            weights = databatch['target_availabilities'].numpy()\n        elif m.startswith('history_'):\n            weights = databatch['history_availabilities'].numpy()\n    if weights is not None:\n        weights = np.tile(np.expand_dims(weights, -1), (1, 1, shape[-1]))\n    return weights\n\ndef describe_mean(databatch, metrics=metrics):\n    for m in metrics:\n        data = databatch[m].numpy()\n        weights = get_weights(databatch, m)\n        # use weights to exclude not available points\n        print(m, np.average(data, axis=0, weights=weights))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('train set average')\ndescribe_mean(train_data, metrics=metrics)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('test set mean')\ndescribe_mean(test_data, metrics=[m for m in metrics if not m.startswith('target_')])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We see the target is at a weird biased position where y is always a bit negative (|y| is about 0.1) and x is alway go to about 8. Perhaps, this is due to the small batch."},{"metadata":{},"cell_type":"markdown","source":"# target_positions and history_positoins\n\nNote the stdev include the non-available points."},{"metadata":{"trusted":true},"cell_type":"code","source":"# average across all timesteps\ntrain_mean_target_positions_all = np.average(\n    train_data['target_positions'].numpy(), axis=(0, 1), weights=get_weights(train_data, 'target_positions'))\ntrain_mean_history_positions_all = np.average(\n    train_data['history_positions'].numpy(), axis=(0, 1), weights=get_weights(train_data, 'history_positions'))\ntest_mean_history_positions_all = np.average(\n    test_data['history_positions'].numpy(), axis=(0, 1), weights=get_weights(test_data, 'history_positions'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('== train target_positions ==')\nprint('mean', train_mean_target_positions_all)\nprint('stdev', np.std(train_data['target_positions'].numpy(), axis=(0, 1)))\nprint('== train history_positions ==')\nprint('mean', train_mean_history_positions_all)\nprint('stdev', np.std(train_data['history_positions'].numpy(), axis=(0, 1)))\nprint('== test history_positions ==')\nprint('mean', test_mean_history_positions_all)\nprint('stdev', np.std(test_data['history_positions'].numpy(), axis=(0, 1)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# average per timestep\ntrain_mean_target_positions = np.average(\n    train_data['target_positions'].numpy(), axis=0, weights=get_weights(train_data, 'target_positions'))\ntrain_mean_history_positions = np.average(\n    train_data['history_positions'].numpy(), axis=0, weights=get_weights(train_data, 'history_positions'))\ntest_mean_history_positions = np.average(\n    test_data['history_positions'].numpy(), axis=0, weights=get_weights(test_data, 'history_positions'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(6, 6))\nplt.scatter(*train_mean_target_positions.T, s=10)\nplt.title('train mean target_positions')\nplt.xlabel('x (m)')\nplt.ylabel('y (m)')\nplt.grid()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(8, 8))\nplt.scatter(*train_mean_history_positions.T, s=10)\nplt.title('train mean history_positions')\nplt.xlabel('x (m)')\nplt.ylabel('y (m)')\nplt.grid()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(6, 6))\nplt.scatter(*test_mean_history_positions.T, s=10)\nplt.title('test mean history_positions')\nplt.xlabel('x (m)')\nplt.ylabel('y (m)')\nplt.grid()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# nx = 5\n# def plot_distribution(databatch, metrics=metrics):\n#     for m in metrics:\n#         data = databatch[m].numpy()\n#         print(data.shape)\n#         dim = len(data.shape)\n#         if dim == 1:\n#             plt.hist(data, bins=100)\n#             plt.xlabel(m); plt.ylabel('N')\n#             plt.title(m)\n#         elif dim == 2:\n#             n = data.shape[1]\n#             ny = np.ceil(n / nx).astype('int')\n#             plt.figure(figsize=(12, ny*3))\n#             for i in range(n):\n#                 plt.subplot(ny, nx, i+1).set_title(m)\n#                 plt.hist(data[:, i], bins=100)\n#                 plt.xlabel(m+f' ({i})')\n#                 if i%nx==0:\n#                     plt.ylabel('N')\n#         plt.show()            ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# tr_it = iter(train_dataloader)\n# n_batch = 1\n# for i in tqdm_notebook(range(n_batch)):\n#     data = next(tr_it)\n#     data['target_positions']\n#     data['target_availabilities']\n#     data['history_positions']\n#     data['history_yaws']\n#     data['history_availabilities']\n#     data['centroid']\n#     data['yaw']\n#     data['extent']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# %%time\n# test_data = next(iter(test_dataloader))\n# memory()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}