{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":8899,"databundleVersionId":46091,"sourceType":"competition"}],"dockerImageVersionId":30664,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Посмотрим что есть в `/kaggle/input/cvpr-2018-autonomous-driving`","metadata":{}},{"cell_type":"markdown","source":"# Этап 1. Разбираемся с данными","metadata":{"execution":{"iopub.status.busy":"2024-02-29T18:40:30.863315Z","iopub.execute_input":"2024-02-29T18:40:30.863658Z","iopub.status.idle":"2024-02-29T18:40:30.869385Z","shell.execute_reply.started":"2024-02-29T18:40:30.863632Z","shell.execute_reply":"2024-02-29T18:40:30.867980Z"}}},{"cell_type":"code","source":"from pathlib import Path\nTMP_DIR = Path('../temp')\nTMP_DIR.mkdir(exist_ok=True)","metadata":{"execution":{"iopub.status.busy":"2024-03-01T02:27:25.395087Z","iopub.execute_input":"2024-03-01T02:27:25.395620Z","iopub.status.idle":"2024-03-01T02:27:25.403399Z","shell.execute_reply.started":"2024-03-01T02:27:25.395577Z","shell.execute_reply":"2024-03-01T02:27:25.401950Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls -lah /kaggle/input/cvpr-2018-autonomous-driving","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2024-03-01T01:02:06.542982Z","iopub.execute_input":"2024-03-01T01:02:06.543394Z","iopub.status.idle":"2024-03-01T01:02:07.702941Z","shell.execute_reply.started":"2024-03-01T01:02:06.543359Z","shell.execute_reply":"2024-03-01T01:02:07.701453Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"---\nTrain:\n- train_video_list.zip -- video lists for training images\n- train_color.zip -- the original training images\n- train_label.zip -- the training images labels\n\nTest:\n- test.zip -- the test set images\n- test_video_list_and_name_mapping.zip -- the mapping from md5 to timestamp and video lists for testing images\n\n---\n\n- sample_submission.csv.zip -- a sample submission file in the correct format\n\n---\n\n- EvaluationScriptsAndExamples.zip  -- evaluation scripts in C# and examples\n- convertVideotoCSV.py - an example script to convert images in a video clip into the CSV format","metadata":{}},{"cell_type":"markdown","source":"Посмотрим что есть в train_video_list.zip:","metadata":{}},{"cell_type":"code","source":"!mkdir -p /kaggle/temp/train_video_list\n!unzip /kaggle/input/cvpr-2018-autonomous-driving/train_video_list.zip -d ../temp/train_video_list","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2024-03-01T02:28:53.954011Z","iopub.execute_input":"2024-03-01T02:28:53.954473Z","iopub.status.idle":"2024-03-01T02:28:56.243494Z","shell.execute_reply.started":"2024-03-01T02:28:53.954425Z","shell.execute_reply":"2024-03-01T02:28:56.241885Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls train_video_list","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2024-03-01T01:02:10.006923Z","iopub.execute_input":"2024-03-01T01:02:10.007449Z","iopub.status.idle":"2024-03-01T01:02:11.113736Z","shell.execute_reply.started":"2024-03-01T01:02:10.007395Z","shell.execute_reply":"2024-03-01T01:02:11.112202Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Посмотрим содержимое первого файлика:","metadata":{}},{"cell_type":"code","source":"!cat ../temp/train_video_list/road01_cam_5_video_10_image_list_train.txt","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2024-03-01T02:29:07.237860Z","iopub.execute_input":"2024-03-01T02:29:07.238352Z","iopub.status.idle":"2024-03-01T02:29:08.383907Z","shell.execute_reply.started":"2024-03-01T02:29:07.238291Z","shell.execute_reply":"2024-03-01T02:29:08.382379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Пример строки:\n`road01_ins\\ColorImage\\Record054\\Camera 5\\170908_072637296_Camera_5.jpg\troad01_ins\\Label\\Record054\\Camera 5\\170908_072637296_Camera_5_instanceIds.png`\n\nВ каждой строке по 2 пути. Предполагаю, что первый из `train_color.zip` а второй из `train_label.zip`\n\nОба архива здоровые (92G и 228M), просто разархивировать не выйдет, надо как-то выкручиваться. Попробуем загрузку с помощью [ZipFile](https://docs.python.org/3/library/zipfile.html)","metadata":{}},{"cell_type":"code","source":"train_color_path = '/kaggle/input/cvpr-2018-autonomous-driving/train_color.zip'\ntrain_label_path = '/kaggle/input/cvpr-2018-autonomous-driving/train_label.zip'","metadata":{"execution":{"iopub.status.busy":"2024-03-01T01:02:12.300771Z","iopub.execute_input":"2024-03-01T01:02:12.301285Z","iopub.status.idle":"2024-03-01T01:02:12.307638Z","shell.execute_reply.started":"2024-03-01T01:02:12.301243Z","shell.execute_reply":"2024-03-01T01:02:12.306098Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from zipfile import ZipFile\nfrom IPython.display import display\n\nwith ZipFile(train_color_path) as myzip:\n    with myzip.open('road01_ins\\ColorImage\\Record054\\Camera 5\\170908_072637296_Camera_5.jpg') as myfile:\n        display(myfile.read())","metadata":{"execution":{"iopub.status.busy":"2024-03-01T01:02:12.309557Z","iopub.execute_input":"2024-03-01T01:02:12.310406Z","iopub.status.idle":"2024-03-01T01:02:13.866805Z","shell.execute_reply.started":"2024-03-01T01:02:12.310364Z","shell.execute_reply":"2024-03-01T01:02:13.861110Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Не получилось :(\n\nПопробуем прочитать список файлов в архиве","metadata":{}},{"cell_type":"code","source":"with ZipFile(train_color_path) as myzip:\n    print(myzip.namelist())","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2024-03-01T01:02:16.095486Z","iopub.execute_input":"2024-03-01T01:02:16.096018Z","iopub.status.idle":"2024-03-01T01:02:17.060692Z","shell.execute_reply.started":"2024-03-01T01:02:16.095971Z","shell.execute_reply":"2024-03-01T01:02:17.057617Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Окей, в пути нет ни инфо о дороге, ни о камере. Просто имя файла и название директории.\n\nПопробуем с этими знаниями опять просто считать одно изображение","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport matplotlib.image as mpimg\n\nwith ZipFile(train_color_path) as myzip:\n    with myzip.open('train_color/' + '170908_072637296_Camera_5.jpg') as myfile:\n        image = mpimg.imread(myfile)\n        plt.imshow(image)\n        ","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-03-01T02:29:41.412656Z","iopub.execute_input":"2024-03-01T02:29:41.413062Z","iopub.status.idle":"2024-03-01T02:29:44.392588Z","shell.execute_reply.started":"2024-03-01T02:29:41.413028Z","shell.execute_reply":"2024-03-01T02:29:44.391729Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with ZipFile(train_label_path) as myzip:\n    with myzip.open('train_label/' + '170908_072637296_Camera_5_instanceIds.png') as myfile:\n        image = mpimg.imread(myfile)\n        plt.imshow(image)","metadata":{"execution":{"iopub.status.busy":"2024-03-01T02:29:47.582810Z","iopub.execute_input":"2024-03-01T02:29:47.583205Z","iopub.status.idle":"2024-03-01T02:29:49.411666Z","shell.execute_reply.started":"2024-03-01T02:29:47.583166Z","shell.execute_reply":"2024-03-01T02:29:49.410445Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Выведем ещё картинок:","metadata":{}},{"cell_type":"code","source":"img_id = '171206_033930650_Camera_5'\n\nwith ZipFile(train_color_path) as myzip:\n    with myzip.open('train_color/' + img_id + '.jpg') as myfile:\n        image = mpimg.imread(myfile)\n        plt.imshow(image)\n        plt.show()\n        \nwith ZipFile(train_label_path) as myzip:\n    with myzip.open('train_label/' + img_id + '_instanceIds.png') as myfile:\n        image = mpimg.imread(myfile)\n        plt.imshow(image)\n        plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-03-01T02:29:51.228272Z","iopub.execute_input":"2024-03-01T02:29:51.228751Z","iopub.status.idle":"2024-03-01T02:29:55.494083Z","shell.execute_reply.started":"2024-03-01T02:29:51.228717Z","shell.execute_reply":"2024-03-01T02:29:55.493169Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Всё сходится.\nТеперь организуем датафрейм в нужном формате:\n\nroad01_ins\\ColorImage\\Record054\\Camera 5\\170908_072637296_Camera_5.jpg    road01_ins\\Label\\Record054\\Camera 5\\170908_072637296_Camera_5_instanceIds.png\n\nroad | camera | video | record | name","metadata":{}},{"cell_type":"code","source":"from typing import List, Tuple\nfrom dataclasses import dataclass\n\n\n@dataclass\nclass VideoListEntity:\n    road: str\n    record: str\n    camera: str\n    name: str\n\ndef line_to_VLE(line: str) -> VideoListEntity:\n    splits = line.split('\\\\')\n    road = splits[0]\n    record = splits[2]\n    camera = splits[3]\n    name = splits[4].split('.')[0]\n    return VideoListEntity(road, record, camera, name)\n        \ndef get_VLE_from_file(path: str) -> List[VideoListEntity]:\n    with open(path) as f:\n        return [line_to_VLE(x) for x in f.readlines()]\n\n\nget_VLE_from_file(str(TMP_DIR) + \"/train_video_list/road01_cam_5_video_10_image_list_train.txt\")","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2024-03-01T02:30:27.324243Z","iopub.execute_input":"2024-03-01T02:30:27.324704Z","iopub.status.idle":"2024-03-01T02:30:27.378571Z","shell.execute_reply.started":"2024-03-01T02:30:27.324672Z","shell.execute_reply":"2024-03-01T02:30:27.377091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nfrom itertools import chain\n\nvles = chain.from_iterable([get_VLE_from_file(str(TMP_DIR) + '/train_video_list/' + filename) for filename in os.listdir(str(TMP_DIR) + '/train_video_list')])\nvles = list(vles)","metadata":{"execution":{"iopub.status.busy":"2024-03-01T02:31:23.432528Z","iopub.execute_input":"2024-03-01T02:31:23.432920Z","iopub.status.idle":"2024-03-01T02:31:23.594963Z","shell.execute_reply.started":"2024-03-01T02:31:23.432888Z","shell.execute_reply":"2024-03-01T02:31:23.593510Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\n\ntrain_video_list = pd.DataFrame([vle.__dict__ for vle in vles ])\n\ntrain_video_list","metadata":{"execution":{"iopub.status.busy":"2024-03-01T02:31:27.255125Z","iopub.execute_input":"2024-03-01T02:31:27.255607Z","iopub.status.idle":"2024-03-01T02:31:27.334226Z","shell.execute_reply.started":"2024-03-01T02:31:27.255560Z","shell.execute_reply":"2024-03-01T02:31:27.332998Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_video_list.info()","metadata":{"execution":{"iopub.status.busy":"2024-03-01T02:31:32.580262Z","iopub.execute_input":"2024-03-01T02:31:32.580798Z","iopub.status.idle":"2024-03-01T02:31:32.616683Z","shell.execute_reply.started":"2024-03-01T02:31:32.580764Z","shell.execute_reply":"2024-03-01T02:31:32.615863Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_video_list['camera'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2024-03-01T02:31:34.395543Z","iopub.execute_input":"2024-03-01T02:31:34.396368Z","iopub.status.idle":"2024-03-01T02:31:34.420529Z","shell.execute_reply.started":"2024-03-01T02:31:34.396316Z","shell.execute_reply":"2024-03-01T02:31:34.419212Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_video_list['road'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2024-03-01T02:31:35.892077Z","iopub.execute_input":"2024-03-01T02:31:35.892544Z","iopub.status.idle":"2024-03-01T02:31:35.912536Z","shell.execute_reply.started":"2024-03-01T02:31:35.892506Z","shell.execute_reply":"2024-03-01T02:31:35.911117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(train_video_list['record'].unique())","metadata":{"execution":{"iopub.status.busy":"2024-03-01T02:31:37.711660Z","iopub.execute_input":"2024-03-01T02:31:37.712934Z","iopub.status.idle":"2024-03-01T02:31:37.731249Z","shell.execute_reply.started":"2024-03-01T02:31:37.712889Z","shell.execute_reply":"2024-03-01T02:31:37.729580Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_video_list['record'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2024-03-01T02:31:39.005264Z","iopub.execute_input":"2024-03-01T02:31:39.005712Z","iopub.status.idle":"2024-03-01T02:31:39.029391Z","shell.execute_reply.started":"2024-03-01T02:31:39.005670Z","shell.execute_reply":"2024-03-01T02:31:39.027898Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Что с `Record007`?","metadata":{}},{"cell_type":"code","source":"def show_img_from_zips(img_id):\n    with ZipFile(train_color_path) as myzip:\n        with myzip.open('train_color/' + img_id + '.jpg') as myfile:\n            image = mpimg.imread(myfile)\n            plt.imshow(image)\n            plt.show()\n\n    with ZipFile(train_label_path) as myzip:\n        with myzip.open('train_label/' + img_id + '_instanceIds.png') as myfile:\n            image = mpimg.imread(myfile)\n            print(image.shape)\n            plt.imshow(image)\n            plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-03-01T01:02:35.640766Z","iopub.execute_input":"2024-03-01T01:02:35.641180Z","iopub.status.idle":"2024-03-01T01:02:35.649350Z","shell.execute_reply.started":"2024-03-01T01:02:35.641145Z","shell.execute_reply":"2024-03-01T01:02:35.648174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_video_list[train_video_list['record'] == 'Record007']","metadata":{"execution":{"iopub.status.busy":"2024-03-01T01:02:38.200234Z","iopub.execute_input":"2024-03-01T01:02:38.200684Z","iopub.status.idle":"2024-03-01T01:02:38.223389Z","shell.execute_reply.started":"2024-03-01T01:02:38.200651Z","shell.execute_reply":"2024-03-01T01:02:38.222052Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for img in train_video_list[train_video_list['record'] == 'Record007']['name']:\n    show_img_from_zips(img)","metadata":{"execution":{"iopub.status.busy":"2024-03-01T01:02:39.494955Z","iopub.execute_input":"2024-03-01T01:02:39.495411Z","iopub.status.idle":"2024-03-01T01:02:56.016268Z","shell.execute_reply.started":"2024-03-01T01:02:39.495373Z","shell.execute_reply":"2024-03-01T01:02:56.015025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Ничего особенного.\n\nРезюме первого этапа:\n1. Картинок много: 42369 шт.\n2. Предположительно размер каждого изображения (2710, 3384) px\n3. Камеры две: предположительно камера 5 слева, камера 6 справа\n4. 48 записей(record). Предположительно в каждой записи данные с двух камер. Есть нечётное количество кадров, при чётном количестве камер.  ","metadata":{}},{"cell_type":"markdown","source":"# Этап 2. Подготавливаем датасет для обучения\n\nСтоит задача сегментации изображения\n\nЛейблы описаны здесь: https://www.kaggle.com/competitions/cvpr-2018-autonomous-driving/data\n\nВ соревновании оцениваются 7 лейблов: car, motorcycle, bicycle, pedestrian, truck, bus, tricycle.\nТочно интересует транспорт:\n- car, 33\n- motorbicycle, 34\n- bicycle, 35\n- truck, 38\n- bus, 39\n- tricycle, 40\n\nИнтересуют пешеходы:\n- person, 36\n- rider, 37 (?)\n\n---\n\nГруппы, они не оцениваются, но могут пригодиться:\n* car_groups, 161\n* motorbicycle_group, 162\n* bicycle_group, 163\n* truck_group, 166\n* bus_group, 167\n* tricycle_group, 168\n* person_group, 164\n* rider_group, 165 (?)\n\n---\n\n\n- All the images are the same size (width, height) of the original images\n- Pixel values indicate both the label and the instance.\n- Each label could contain multiple object instances.\n- int(PixelValue / 1000) is the label (class of object)\n- PixelValue % 1000 is the instance id\n- For example, a pixel value of 33000 means it belongs to label 33 (a car), is instance #0, while the pixel value of 33001 means it also belongs to class 33 (a car) , and is instance #1. These represent two different cars in an image.\n\n---\n\nPixelValue -- ?\n\n---\nЗадача: найти изображения, которые содержат нужные классы. Удобно, если на одно изображении будет сразу несколько классов\n","metadata":{}},{"cell_type":"markdown","source":"Нам придётся много возиться с лейблами, попробуем разархивировать `train_label.zip`","metadata":{}},{"cell_type":"code","source":"!unzip -qq /kaggle/input/cvpr-2018-autonomous-driving/train_label.zip -d ../temp/","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2024-03-01T02:32:16.361751Z","iopub.execute_input":"2024-03-01T02:32:16.362211Z","iopub.status.idle":"2024-03-01T02:32:30.614737Z","shell.execute_reply.started":"2024-03-01T02:32:16.362174Z","shell.execute_reply":"2024-03-01T02:32:30.612787Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from PIL import Image\nimage = Image.open(str(TMP_DIR) + '/train_label/170927_064538580_Camera_6_instanceIds.png', 'r')\n\n# image = mpimg.imread('train_label/170927_064538580_Camera_6_instanceIds.png')\n# plt.imshow(image)","metadata":{"execution":{"iopub.status.busy":"2024-03-01T02:32:33.995638Z","iopub.execute_input":"2024-03-01T02:32:33.996482Z","iopub.status.idle":"2024-03-01T02:32:34.007362Z","shell.execute_reply.started":"2024-03-01T02:32:33.996240Z","shell.execute_reply":"2024-03-01T02:32:34.005654Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"list(filter(lambda x: x//1000 == 33, image.getdata()))","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2024-03-01T02:32:37.444494Z","iopub.execute_input":"2024-03-01T02:32:37.444918Z","iopub.status.idle":"2024-03-01T02:32:39.668723Z","shell.execute_reply.started":"2024-03-01T02:32:37.444887Z","shell.execute_reply":"2024-03-01T02:32:39.667204Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\n\nimg = np.asarray(image)","metadata":{"execution":{"iopub.status.busy":"2024-03-01T02:32:41.725896Z","iopub.execute_input":"2024-03-01T02:32:41.726318Z","iopub.status.idle":"2024-03-01T02:32:41.808714Z","shell.execute_reply.started":"2024-03-01T02:32:41.726267Z","shell.execute_reply":"2024-03-01T02:32:41.807617Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img.shape","metadata":{"execution":{"iopub.status.busy":"2024-03-01T02:32:43.331651Z","iopub.execute_input":"2024-03-01T02:32:43.332058Z","iopub.status.idle":"2024-03-01T02:32:43.341801Z","shell.execute_reply.started":"2024-03-01T02:32:43.332028Z","shell.execute_reply":"2024-03-01T02:32:43.340389Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img // 1000 == 33","metadata":{"execution":{"iopub.status.busy":"2024-03-01T02:32:44.587784Z","iopub.execute_input":"2024-03-01T02:32:44.588519Z","iopub.status.idle":"2024-03-01T02:32:44.627756Z","shell.execute_reply.started":"2024-03-01T02:32:44.588481Z","shell.execute_reply":"2024-03-01T02:32:44.626402Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.unique(img[(img // 1000 == 33)] % 1000)","metadata":{"execution":{"iopub.status.busy":"2024-03-01T02:32:46.276986Z","iopub.execute_input":"2024-03-01T02:32:46.277776Z","iopub.status.idle":"2024-03-01T02:32:46.335862Z","shell.execute_reply.started":"2024-03-01T02:32:46.277726Z","shell.execute_reply":"2024-03-01T02:32:46.334383Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def count_number_and_instances(img, label_id):\n    pixels = img[(img // 1000 == label_id)]\n    number = len(pixels)\n    instances = len(np.unique(pixels % 1000))\n    return number, instances\n\n\ncount_number_and_instances(img, 33)","metadata":{"execution":{"iopub.status.busy":"2024-03-01T02:32:48.547387Z","iopub.execute_input":"2024-03-01T02:32:48.548534Z","iopub.status.idle":"2024-03-01T02:32:48.631377Z","shell.execute_reply.started":"2024-03-01T02:32:48.548494Z","shell.execute_reply":"2024-03-01T02:32:48.630063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lmap = {\n    'car': 33,\n    'motorbicycle': 34,\n    'bicycle': 35,\n    'truck': 38,\n    'bus': 39,\n    'tricycle': 40,\n    'person': 36,\n}\n\n\ndef count_label_stat_for_img(img, lmap):\n    res = {}\n    for l in lmap.keys():\n        res[l+'_number'], res[l+'_instances'] = count_number_and_instances(img, lmap[l])\n    return res\n\ncount_label_stat_for_img(img, lmap)","metadata":{"execution":{"iopub.status.busy":"2024-03-01T02:32:55.961210Z","iopub.execute_input":"2024-03-01T02:32:55.961655Z","iopub.status.idle":"2024-03-01T02:32:56.099827Z","shell.execute_reply.started":"2024-03-01T02:32:55.961620Z","shell.execute_reply":"2024-03-01T02:32:56.098514Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def count_label_stat(name):\n    try:\n        image = Image.open(str(TMP_DIR) + '/train_label/'+ name + '_instanceIds.png', 'r')\n        img = np.asarray(image)\n        return {**count_label_stat_for_img(img, lmap), 'name': name}\n    except Exception as e:\n        print(e)\n        return {'name': name}\n\ncount_label_stat('170927_064538580_Camera_6')","metadata":{"execution":{"iopub.status.busy":"2024-03-01T02:33:02.308093Z","iopub.execute_input":"2024-03-01T02:33:02.309379Z","iopub.status.idle":"2024-03-01T02:33:02.697777Z","shell.execute_reply.started":"2024-03-01T02:33:02.309323Z","shell.execute_reply":"2024-03-01T02:33:02.696908Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"count_label_stat('sdfsdfs')","metadata":{"execution":{"iopub.status.busy":"2024-03-01T02:33:05.140903Z","iopub.execute_input":"2024-03-01T02:33:05.141576Z","iopub.status.idle":"2024-03-01T02:33:05.150179Z","shell.execute_reply.started":"2024-03-01T02:33:05.141541Z","shell.execute_reply":"2024-03-01T02:33:05.149363Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Теперь загрузим в датафрейм информацию о лейблах","metadata":{}},{"cell_type":"code","source":"from tqdm.auto import tqdm\n\ntqdm.pandas(desc=\"count_label_stat\")","metadata":{"execution":{"iopub.status.busy":"2024-03-01T01:03:13.029575Z","iopub.execute_input":"2024-03-01T01:03:13.030398Z","iopub.status.idle":"2024-03-01T01:03:13.236913Z","shell.execute_reply.started":"2024-03-01T01:03:13.030354Z","shell.execute_reply":"2024-03-01T01:03:13.235429Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"stats = pd.DataFrame(list(train_video_list['name'].progress_apply(count_label_stat)))","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2024-03-01T01:02:13.932049Z","iopub.status.idle":"2024-03-01T01:02:13.932709Z","shell.execute_reply.started":"2024-03-01T01:02:13.932388Z","shell.execute_reply":"2024-03-01T01:02:13.932415Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Медленно","metadata":{}},{"cell_type":"code","source":"stats","metadata":{"execution":{"iopub.status.busy":"2024-03-01T01:02:13.936693Z","iopub.status.idle":"2024-03-01T01:02:13.937282Z","shell.execute_reply.started":"2024-03-01T01:02:13.936988Z","shell.execute_reply":"2024-03-01T01:02:13.937014Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import multiprocessing as mp\nimport typing\n\nimport numpy as np\nimport pandas as pd\nfrom tqdm.auto import tqdm\n\n\ndef parallel_apply(\n    df_or_s: typing.Union[pd.DataFrame, pd.Series],\n    func: typing.Callable,\n    split_size: int,\n    n_jobs: int = mp.cpu_count(),\n) -> typing.Union[pd.DataFrame, pd.Series]:\n    with mp.Pool(n_jobs) as pool:\n        split = np.array_split(df_or_s, len(df_or_s)/split_size)\n        lens = list(map(len, split))\n#         print(lens)\n        print(sum(lens))\n\n        ret_list = tqdm(pool.imap(func, split), total=len(split))\n\n        output_df_or_s = pd.concat(ret_list)\n\n    return output_df_or_s","metadata":{"execution":{"iopub.status.busy":"2024-03-01T01:03:17.494369Z","iopub.execute_input":"2024-03-01T01:03:17.494788Z","iopub.status.idle":"2024-03-01T01:03:17.521939Z","shell.execute_reply.started":"2024-03-01T01:03:17.494753Z","shell.execute_reply":"2024-03-01T01:03:17.520670Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def xx(s):\n    return s.apply(count_label_stat)","metadata":{"execution":{"iopub.status.busy":"2024-03-01T01:03:19.534977Z","iopub.execute_input":"2024-03-01T01:03:19.535495Z","iopub.status.idle":"2024-03-01T01:03:19.543833Z","shell.execute_reply.started":"2024-03-01T01:03:19.535455Z","shell.execute_reply":"2024-03-01T01:03:19.542499Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nres = parallel_apply(train_video_list['name'], xx, 10)\nres","metadata":{"execution":{"iopub.status.busy":"2024-03-01T01:03:21.536023Z","iopub.execute_input":"2024-03-01T01:03:21.536484Z","iopub.status.idle":"2024-03-01T02:20:07.173735Z","shell.execute_reply.started":"2024-03-01T01:03:21.536449Z","shell.execute_reply":"2024-03-01T02:20:07.172828Z"},"scrolled":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"res_df = pd.DataFrame(list(res))","metadata":{"execution":{"iopub.status.busy":"2024-03-01T02:22:59.539505Z","iopub.execute_input":"2024-03-01T02:22:59.539886Z","iopub.status.idle":"2024-03-01T02:22:59.959490Z","shell.execute_reply.started":"2024-03-01T02:22:59.539858Z","shell.execute_reply":"2024-03-01T02:22:59.958084Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"res_df.info()","metadata":{"execution":{"iopub.status.busy":"2024-03-01T02:23:11.940680Z","iopub.execute_input":"2024-03-01T02:23:11.941061Z","iopub.status.idle":"2024-03-01T02:23:11.961415Z","shell.execute_reply.started":"2024-03-01T02:23:11.941032Z","shell.execute_reply":"2024-03-01T02:23:11.960146Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_video_list","metadata":{"execution":{"iopub.status.busy":"2024-03-01T02:34:19.199230Z","iopub.execute_input":"2024-03-01T02:34:19.199715Z","iopub.status.idle":"2024-03-01T02:34:19.213705Z","shell.execute_reply.started":"2024-03-01T02:34:19.199676Z","shell.execute_reply":"2024-03-01T02:34:19.212730Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"res_df","metadata":{"execution":{"iopub.status.busy":"2024-03-01T02:35:17.285124Z","iopub.execute_input":"2024-03-01T02:35:17.285666Z","iopub.status.idle":"2024-03-01T02:35:17.340513Z","shell.execute_reply.started":"2024-03-01T02:35:17.285625Z","shell.execute_reply":"2024-03-01T02:35:17.339191Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_info = pd.merge(train_video_list, res_df, on=['name'])\ntrain_info.to_csv('train_info.csv')","metadata":{"execution":{"iopub.status.busy":"2024-03-01T03:02:01.234480Z","iopub.execute_input":"2024-03-01T03:02:01.234895Z","iopub.status.idle":"2024-03-01T03:02:02.078385Z","shell.execute_reply.started":"2024-03-01T03:02:01.234863Z","shell.execute_reply":"2024-03-01T03:02:02.077076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_info[train_info['car_instances'] == 0]","metadata":{"execution":{"iopub.status.busy":"2024-03-01T03:10:50.117041Z","iopub.execute_input":"2024-03-01T03:10:50.117723Z","iopub.status.idle":"2024-03-01T03:10:50.165346Z","shell.execute_reply.started":"2024-03-01T03:10:50.117650Z","shell.execute_reply":"2024-03-01T03:10:50.164102Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_info[train_info['car_instances'].isna()]","metadata":{"execution":{"iopub.status.busy":"2024-03-01T03:10:39.760479Z","iopub.execute_input":"2024-03-01T03:10:39.760939Z","iopub.status.idle":"2024-03-01T03:10:39.793076Z","shell.execute_reply.started":"2024-03-01T03:10:39.760906Z","shell.execute_reply":"2024-03-01T03:10:39.791518Z"},"trusted":true},"execution_count":null,"outputs":[]}]}