{"cells":[{"metadata":{},"cell_type":"markdown","source":"Thanks @Rustem Iskuzhin, for pointing out how to install Lyft sdk\n\n\nTo get understanding of how each files are related with each other, I tried to extract a single scene.\nThis way, it is also easier to load it. So some testing on the smaller dataset can be done, and then\none can move on to the complete dataset\n\nPrereq:\n- All required library is installed (specifically: lyft_dataset_sdk)\n- Virtual folder for images, data and lidar created pointing to train_images, train_data and train_lidar\n- All folders required on SAMP_DATA_PATH has been created\n\n\n\nI have uploaded the single scene data as [sampData.zip](https://www.kaggle.com/aknirala/sampdata)"},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install lyft-dataset-sdk -q","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#First import:  \nfrom lyft_dataset_sdk.lyftdataset import LyftDataset  #Assuming you have already installed it\nimport pandas as pd\nimport json","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!mkdir ./sampData\n!mkdir ./sampData/train_data\n!mkdir ./sampData/train_images\n!mkdir ./sampData/train_lidar\n!mkdir ./sampData/train_maps","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ln -s /kaggle/input/3d-object-detection-for-autonomous-vehicles/test_images images\n!ln -s /kaggle/input/3d-object-detection-for-autonomous-vehicles/test_maps maps\n!ln -s /kaggle/input/3d-object-detection-for-autonomous-vehicles/test_lidar lidar","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Some path initialization:\nDATA_PATH = \"/kaggle/input/3d-object-detection-for-autonomous-vehicles/\"\nwLoc = \"./sampData/\"","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":true},"cell_type":"code","source":"#Load all the data\n#lyft_dataset = LyftDataset(data_path=DATA_PATH, json_path=DATA_PATH+'train_data')\nlyft_dataset = LyftDataset(data_path=\".\", json_path=DATA_PATH+'train_data')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#We'll start from scene.json, first scene from this.\n#scene.json:\nf = open(wLoc +\"train_data/\" + \"scene.json\", \"w\")\njson.dump([lyft_dataset.scene[0]], f)\nf.close()\n\n#Next 180s are in:log.json, \n#log.json: just write the one for scene[0][\"map_token\"] (1 entry)\nf = open(wLoc +\"train_data/\" + \"log.json\", \"w\")\njson.dump([lyft_dataset.get(\"log\", lyft_dataset.scene[0][\"log_token\"])], f)\nf.close()\n\n#map.json: this is same for all so copy as it is (we can reduce but it's OK)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#sample.json  Loop from first_sample_token in scene\nS = []\nt = lyft_dataset.scene[0][\"first_sample_token\"]\nwhile t is not None and t != \"\":\n    S.append(lyft_dataset.get(\"sample\", t))\n    t = lyft_dataset.get(\"sample\", t)[\"next\"]\nf = open(wLoc +\"train_data/\" + \"sample.json\", \"w\")\njson.dump(S, f)\nf.close()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#sample_data.json: from samples extracted \n#                    > from their data\n#                       > extract each sample_data\nSD = []\nfor s in S:\n    for sName in s[\"data\"]:\n        SD.append(lyft_dataset.get(\"sample_data\", s[\"data\"][sName]))\nf = open(wLoc +\"train_data/\" + \"sample_data.json\", \"w\")\njson.dump(SD, f)\nf.close()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#sample_annotation.json:create a set of sample tokens and exhaustively check.. and include\n# What is the better way to do it?\nsTokenSet = set()\nfor s in S:\n    sTokenSet.add(s[\"token\"])\n\nSA = []\nfor sa in lyft_dataset.sample_annotation:\n    if sa[\"sample_token\"] in sTokenSet:\n        SA.append(sa)\n\nf = open(wLoc +\"train_data/\" + \"sample_annotation.json\", \"w\")\njson.dump(SA, f)\nf.close()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#instance.json: get instance_token from sample_annotation (need to get Set) and then extract\niSet = set()\nfor sa in SA:\n    iSet.add(sa[\"instance_token\"])\n\nI = []\nfor t in iSet:\n    I.append(lyft_dataset.get(\"instance\", t))\n\nf = open(wLoc +\"train_data/\" + \"instance.json\", \"w\")\njson.dump(I, f)\nf.close()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#ego_pose.json: Extract from sample_data\nepSet = set()\nfor sd in SD:\n    epSet.add(sd[\"ego_pose_token\"])\n\n#this reduced 1260 SDs to 632 ego poses\nEP = []\nfor ep in epSet:\n    EP.append(lyft_dataset.get(\"ego_pose\", ep))\n\nf = open(wLoc +\"train_data/\" + \"ego_pose.json\", \"w\")\njson.dump(EP, f)\nf.close()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#attribute.json: small copy as it is\n#calibrated_sensor.json: small, copy as it is??\n#category.json: copy as it is\n#map.json:copy as it is as it is same for all (If we want we can prune list of 180 tokens to just one log in it)\n# visibility.json: copy as it is\n#sensor.json: copy as it is","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#To copy imags and lidar we would rely on SD\n#You might wanna modify the folder names\ncpCommand = \"\"\nfor sd in SD:\n    if sd[\"filename\"][-4:] == \"jpeg\":\n        cpCommand += \"\\ncp \"+DATA_PATH+\"train_\"+sd[\"filename\"]+ \" \" +wLoc + \"train_images/\"\n    else:\n        cpCommand += \"\\ncp \"+DATA_PATH+\"train_\"+sd[\"filename\"]+ \" \" +wLoc + \"train_lidar/\"\n\n#print(cpCommand) #Copy paste it and run in suitabel folder. Then rename your folders","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Copy train_maps manually.\n\n#For train.csv: Load all and filter thsoe which are needed\ntr = pd.read_csv(DATA_PATH+\"train.csv\")\ntrSamp = tr.loc[tr[\"Id\"].isin([s[\"token\"] for s in S])]\ntrSamp.to_csv(wLoc+\"train.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Create virtual links at wLoc. After stepping in wLoc do\n\"\"\"\n$ln -s train_images images\n$ln -s train_lidar lidar\n$ln -s train_maps maps\n$ln -s train_data data\n\n\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Now lets try to load our sample using lyft sdk\nsamp_lyft_dataset = LyftDataset(data_path=wLoc, json_path=wLoc+'train_data')\n#This will fail as of now, coz files  have not been copied etc.,,.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.8"}},"nbformat":4,"nbformat_minor":1}