{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":13578,"databundleVersionId":588368,"sourceType":"competition"}],"dockerImageVersionId":30698,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# # This Python 3 environment comes with many helpful analytics libraries installed\n# # It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# # For example, here's several helpful packages to load\n\n# import numpy as np # linear algebra\n# import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# # Input data files are available in the read-only \"../input/\" directory\n# # For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\n# import os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\n# # You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# # You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-05-31T17:30:05.821928Z","iopub.execute_input":"2024-05-31T17:30:05.822344Z","iopub.status.idle":"2024-05-31T17:30:05.827126Z","shell.execute_reply.started":"2024-05-31T17:30:05.822292Z","shell.execute_reply":"2024-05-31T17:30:05.826061Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## what we need to know about these data?\n* train_images: is folder with images for training where the img name is in train.csv\n* test_images: is folder with iamges for testing \n* unicode_translation: is spreadsheet where is the translation of labels\n* train.csv: spreadsheet has train img name and labels with location (x_center,y_center,width, height)\n\n## what do we need to do?\n* create model using YOLO \n\n## what do we need to do that?\n* create folder with training img after resize them to (416,416)\n* create folder of labels which has text file for each img has index of labels and locations\n* create ymal file whch has path for [training folder, label foler, labels list]\n\n\n","metadata":{}},{"cell_type":"code","source":"!pip install yolov8","metadata":{"execution":{"iopub.status.busy":"2024-05-31T17:30:05.829554Z","iopub.execute_input":"2024-05-31T17:30:05.829875Z","iopub.status.idle":"2024-05-31T17:30:35.655345Z","shell.execute_reply.started":"2024-05-31T17:30:05.829842Z","shell.execute_reply":"2024-05-31T17:30:35.654327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## import","metadata":{}},{"cell_type":"code","source":"import os \nimport zipfile\nimport shutil\nimport random\nfrom PIL import Image\n\nimport pandas as pd \nimport numpy as np \nimport matplotlib.pyplot as plt \nimport seaborn as sns \n\nimport yaml\nimport tensorflow as tf \nimport torch\n","metadata":{"execution":{"iopub.status.busy":"2024-05-31T17:30:35.656889Z","iopub.execute_input":"2024-05-31T17:30:35.657354Z","iopub.status.idle":"2024-05-31T17:30:51.160384Z","shell.execute_reply.started":"2024-05-31T17:30:35.657319Z","shell.execute_reply":"2024-05-31T17:30:51.159407Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gpus = tf.config.experimental.list_physical_devices('GPU')\nif gpus:    \n    try:\n        for gpu in gpus:\n            tf.config.experimental.set_memory_growth(gpu, True)\n        logical_gpus = tf.config.experimental.list_logical_devices('GPU')\n        print(len(gpus), \"Physical GPUs,\", len(logical_gpus), \"Logical GPUs\")\n    except RuntimeError as e:\n        # Memory growth must be set before GPUs have been initialized\n        print(e)","metadata":{"execution":{"iopub.status.busy":"2024-05-31T17:30:51.161569Z","iopub.execute_input":"2024-05-31T17:30:51.162113Z","iopub.status.idle":"2024-05-31T17:30:51.339501Z","shell.execute_reply.started":"2024-05-31T17:30:51.162087Z","shell.execute_reply":"2024-05-31T17:30:51.338562Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Read the data","metadata":{}},{"cell_type":"code","source":"train_dataset=pd.read_csv('/kaggle/input/kuzushiji-recognition/train.csv')\ntrain_dataset","metadata":{"execution":{"iopub.status.busy":"2024-05-31T17:30:51.341883Z","iopub.execute_input":"2024-05-31T17:30:51.342252Z","iopub.status.idle":"2024-05-31T17:30:51.719513Z","shell.execute_reply.started":"2024-05-31T17:30:51.342226Z","shell.execute_reply":"2024-05-31T17:30:51.718604Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"let's check if we have missing value or duplication ","metadata":{}},{"cell_type":"code","source":"print(train_dataset.isna().sum())\nprint(train_dataset.duplicated().sum())","metadata":{"execution":{"iopub.status.busy":"2024-05-31T17:30:51.720878Z","iopub.execute_input":"2024-05-31T17:30:51.721171Z","iopub.status.idle":"2024-05-31T17:30:51.768785Z","shell.execute_reply.started":"2024-05-31T17:30:51.721146Z","shell.execute_reply":"2024-05-31T17:30:51.767939Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## preparing the data\npreparing the data is finding from train file the labels and coordinations, fix the coordination to fit it in YOLO model ","metadata":{}},{"cell_type":"code","source":"## let's create function seperate the labels with box boundries \ndef seperate_labels(label):\n    label_arr = np.array(label.split(' '))\n    label_arr = label_arr.reshape(-1, 5)\n    return label_arr","metadata":{"execution":{"iopub.status.busy":"2024-05-31T17:30:51.769805Z","iopub.execute_input":"2024-05-31T17:30:51.770102Z","iopub.status.idle":"2024-05-31T17:30:51.774680Z","shell.execute_reply.started":"2024-05-31T17:30:51.770077Z","shell.execute_reply":"2024-05-31T17:30:51.773883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset.labels=train_dataset.labels.map(seperate_labels)\ntrain_dataset['num_labels']= train_dataset.labels.apply(lambda x:x.shape[0])\ntrain_dataset","metadata":{"execution":{"iopub.status.busy":"2024-05-31T17:30:51.775738Z","iopub.execute_input":"2024-05-31T17:30:51.776023Z","iopub.status.idle":"2024-05-31T17:30:53.340959Z","shell.execute_reply.started":"2024-05-31T17:30:51.775979Z","shell.execute_reply":"2024-05-31T17:30:53.340101Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"3605 *0.8","metadata":{"execution":{"iopub.status.busy":"2024-05-31T17:30:53.342302Z","iopub.execute_input":"2024-05-31T17:30:53.342594Z","iopub.status.idle":"2024-05-31T17:30:53.348350Z","shell.execute_reply.started":"2024-05-31T17:30:53.342570Z","shell.execute_reply":"2024-05-31T17:30:53.347458Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset['letter_unicode']= train_dataset.labels.apply(lambda x:np.array([item[0] for item in x]))\ntrain_dataset","metadata":{"execution":{"iopub.status.busy":"2024-05-31T17:30:53.349625Z","iopub.execute_input":"2024-05-31T17:30:53.349971Z","iopub.status.idle":"2024-05-31T17:30:54.448847Z","shell.execute_reply.started":"2024-05-31T17:30:53.349940Z","shell.execute_reply":"2024-05-31T17:30:54.447973Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# finding the most using letter (label) = number of labels \nlabels = np.concatenate(train_dataset.letter_unicode.values)\nunique_label, freq = np.unique(labels, return_counts=True)\nlabels_freq = pd.DataFrame(index= unique_label, data = freq, columns=['freq']).sort_values(by='freq',ascending=False)\nlabels_freq","metadata":{"execution":{"iopub.status.busy":"2024-05-31T17:30:54.449955Z","iopub.execute_input":"2024-05-31T17:30:54.450258Z","iopub.status.idle":"2024-05-31T17:30:54.631759Z","shell.execute_reply.started":"2024-05-31T17:30:54.450234Z","shell.execute_reply":"2024-05-31T17:30:54.630747Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## lets select the most common 100\n\nlabels_list = list(labels_freq.iloc[:100].index) \nlabels_freq.iloc[:100]","metadata":{"execution":{"iopub.status.busy":"2024-05-31T17:30:54.633014Z","iopub.execute_input":"2024-05-31T17:30:54.633281Z","iopub.status.idle":"2024-05-31T17:30:54.696708Z","shell.execute_reply.started":"2024-05-31T17:30:54.633258Z","shell.execute_reply":"2024-05-31T17:30:54.695753Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## visuliztion \nplt.figure(figsize=(10,20))\nsns.barplot(data =labels_freq.iloc[:50], y= labels_freq.iloc[:50].index, x = 'freq', label = 'freq')\n# plt.xticks(rotation=90)\nplt.grid()","metadata":{"execution":{"iopub.status.busy":"2024-05-31T17:30:54.697808Z","iopub.execute_input":"2024-05-31T17:30:54.698102Z","iopub.status.idle":"2024-05-31T17:30:55.496347Z","shell.execute_reply.started":"2024-05-31T17:30:54.698078Z","shell.execute_reply":"2024-05-31T17:30:55.495380Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## create img, label folders \n### unzip files","metadata":{}},{"cell_type":"code","source":"zip_train_path = '/kaggle/input/kuzushiji-recognition/train_images.zip'\nzip_test_path  = '/kaggle/input/kuzushiji-recognition/test_images.zip'\nextract_train  = '/kaggle/working/train' \nextract_test   = '/kaggle/working/test'\nos.makedirs(extract_train, exist_ok=True)\nos.makedirs(extract_test, exist_ok=True)\nwith zipfile.ZipFile(zip_train_path, mode='r')as zip_ref:\n    zip_ref.extractall(extract_train)\nwith zipfile.ZipFile(zip_test_path, mode='r') as zip_ref:\n    zip_ref.extractall(extract_test)","metadata":{"execution":{"iopub.status.busy":"2024-05-31T17:30:55.501500Z","iopub.execute_input":"2024-05-31T17:30:55.501791Z","iopub.status.idle":"2024-05-31T17:31:52.451024Z","shell.execute_reply.started":"2024-05-31T17:30:55.501766Z","shell.execute_reply":"2024-05-31T17:31:52.450196Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## create val file ","metadata":{}},{"cell_type":"code","source":"3605*0.2","metadata":{"execution":{"iopub.status.busy":"2024-05-31T17:31:52.452115Z","iopub.execute_input":"2024-05-31T17:31:52.452383Z","iopub.status.idle":"2024-05-31T17:31:52.458217Z","shell.execute_reply.started":"2024-05-31T17:31:52.452360Z","shell.execute_reply":"2024-05-31T17:31:52.457328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_path = '/kaggle/working/validation'\nos.makedirs(val_path, exist_ok=True)\nall_train_img = os.listdir(extract_train)\nrandom.shuffle(all_train_img)\nval_img_name = all_train_img[:720]\nfor img in val_img_name:\n#     print(img)\n    t_path= os.path.join(extract_train, img)\n    v_path= os.path.join(val_path, img)\n    shutil.move(t_path, val_path)\n","metadata":{"execution":{"iopub.status.busy":"2024-05-31T17:31:52.459354Z","iopub.execute_input":"2024-05-31T17:31:52.459620Z","iopub.status.idle":"2024-05-31T17:31:52.509020Z","shell.execute_reply.started":"2024-05-31T17:31:52.459596Z","shell.execute_reply":"2024-05-31T17:31:52.508123Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(os.listdir('/kaggle/working/train'))","metadata":{"execution":{"iopub.status.busy":"2024-05-31T17:31:52.510081Z","iopub.execute_input":"2024-05-31T17:31:52.510351Z","iopub.status.idle":"2024-05-31T17:31:52.518187Z","shell.execute_reply.started":"2024-05-31T17:31:52.510322Z","shell.execute_reply":"2024-05-31T17:31:52.517353Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(os.listdir('/kaggle/working/validation'))","metadata":{"execution":{"iopub.status.busy":"2024-05-31T17:31:52.519210Z","iopub.execute_input":"2024-05-31T17:31:52.519936Z","iopub.status.idle":"2024-05-31T17:31:52.526776Z","shell.execute_reply.started":"2024-05-31T17:31:52.519911Z","shell.execute_reply":"2024-05-31T17:31:52.525904Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(os.listdir('/kaggle/working/test'))","metadata":{"execution":{"iopub.status.busy":"2024-05-31T17:31:52.527790Z","iopub.execute_input":"2024-05-31T17:31:52.528082Z","iopub.status.idle":"2024-05-31T17:31:52.540426Z","shell.execute_reply.started":"2024-05-31T17:31:52.528059Z","shell.execute_reply":"2024-05-31T17:31:52.539612Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### resize imgs, edit bounding box coordinates ","metadata":{}},{"cell_type":"code","source":"def resize_img(old_img_path, new_img_path,target_size):\n    img = Image.open(old_img_path)\n    original_width, original_height = img.size\n    img= img.resize(target_size,Image.LANCZOS)\n    img.save(new_img_path)\n    return original_width, original_height","metadata":{"execution":{"iopub.status.busy":"2024-05-31T17:31:52.541549Z","iopub.execute_input":"2024-05-31T17:31:52.542246Z","iopub.status.idle":"2024-05-31T17:31:52.547560Z","shell.execute_reply.started":"2024-05-31T17:31:52.542210Z","shell.execute_reply":"2024-05-31T17:31:52.546677Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def resize_edit_box(old_file_path, new_path, data=None, target_size=(640,640) ):\n    os.makedirs(f'{new_path}/images', exist_ok=True)\n    \n    if data is None:\n        for img_name in os.listdir(old_file_path):\n            img_path_old = os.path.join(old_file_path, img_name)\n            img_path_new = os.path.join(f'{new_path}/images', img_name)\n            resize_img(img_path_old, img_path_new, target_size)\n    else:\n        os.makedirs(f'{new_path}/labels', exist_ok= True)\n        for img_name in os.listdir(old_file_path):\n            image_id=img_name[:-4]\n            if set(data[data.image_id==image_id]['letter_unicode'].values[0]) & set(labels_list):\n                img_path_old = os.path.join(old_file_path, img_name)\n                img_path_new = os.path.join(f'{new_path}/images', img_name)\n                original_width, original_height = resize_img(img_path_old, img_path_new, target_size)\n                new_boxes = []\n                scale_factor_width = target_size[0]/original_width\n                scale_factor_height= target_size[1]/original_height\n                boxes = data[data.image_id==image_id]['labels'].values[0]\n                with open (os.path.join(f'{new_path}/labels',f'{image_id}.txt'), 'a') as file:\n                    for box in boxes:\n                        if box[0] in labels_list:\n                            char_index = labels_list.index(box[0])\n                            scaled_x = int((int(box[1])+0.5*int(box[3])) *scale_factor_width )/target_size[0]\n                            scaled_y = int((int(box[2])+0.5*int(box[4])) *scale_factor_height)/target_size[0]\n                            scaled_width = int(int(box[3]) *scale_factor_width )/target_size[1]\n                            scaled_height = int(int(box[4]) *scale_factor_width )/target_size[1]\n                            new_box = [char_index,scaled_x,scaled_y,scaled_width,scaled_height]\n                            new_boxes.append(new_box)\n                            data_boxes = pd.DataFrame(new_boxes)\n                    if len(new_boxes)==0:\n                        print(f'{img_name} need to remove')\n                    file.write(data_boxes.to_string(header=False, index=False))\n            else:\n                print(f'{image_id} does not have labels')\n               \n        \n            \n   \n    \n#     os.makedirs(f'{new_path}/images', exist_ok=True)\n#     target_size = (640,640)\n    \n\n#     ## resizing the img\n#     for img_name in os.listdir(old_file_path):\n#         img_path = os.path.join(old_file_path, img_name)\n#         img = Image.open(img_path)\n#         original_width, original_height = img.size\n#         img= img.resize(target_size,Image.LANCZOS)\n#         img.save(os.path.join(f'{new_path}/images', img_name))\n#         # recalculate the location, change the label with the index in labels list\n#         if data is not None:\n#             os.makedirs(f'{new_path}/labels', exist_ok= True)\n#             image_id=img_name[:-4]            \n#             new_boxes = []\n            \n#             scale_factor_width = target_size[0]/original_width\n#             scale_factor_height= target_size[1]/original_height\n#             boxes = data[data.image_id==image_id]['labels'].values[0]\n#             with open (os.path.join(f'{new_path}/labels',f'{image_id}.txt'), 'a') as file:\n                        \n#                 for box in boxes:\n#                     if box[0] in labels_list:\n#                         char_index = labels_list.index(box[0])\n#                         scaled_x = int((int(box[1])) *scale_factor_width )\n#                         scaled_y = int(int(box[2]) *scale_factor_height)\n#                         scaled_width = int(int(box[3]) *scale_factor_width )\n#                         scaled_height = int(int(box[4]) *scale_factor_width )\n#                         new_box = [char_index,scaled_x,scaled_y,scaled_width,scaled_height]\n#                         new_boxes.append(new_box)\n#                         data_boxes = pd.DataFrame(new_boxes)\n#                 if len(new_boxes)==0:\n#                     print(f'{img_name} need to remove')\n#                 file.write(data_boxes.to_string(header=False, index=False))\n                        \n\n            \n        \n    ","metadata":{"execution":{"iopub.status.busy":"2024-05-31T17:31:52.548850Z","iopub.execute_input":"2024-05-31T17:31:52.549159Z","iopub.status.idle":"2024-05-31T17:31:52.565387Z","shell.execute_reply.started":"2024-05-31T17:31:52.549133Z","shell.execute_reply":"2024-05-31T17:31:52.564467Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nresize_edit_box('/kaggle/working/train','/kaggle/working/scaled_train', train_dataset)","metadata":{"execution":{"iopub.status.busy":"2024-05-31T17:31:52.566376Z","iopub.execute_input":"2024-05-31T17:31:52.566656Z","iopub.status.idle":"2024-05-31T17:41:06.361943Z","shell.execute_reply.started":"2024-05-31T17:31:52.566633Z","shell.execute_reply":"2024-05-31T17:41:06.360948Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nresize_edit_box('/kaggle/working/validation','/kaggle/working/scaled_validation', train_dataset)","metadata":{"execution":{"iopub.status.busy":"2024-05-31T17:41:06.363050Z","iopub.execute_input":"2024-05-31T17:41:06.363364Z","iopub.status.idle":"2024-05-31T17:43:25.222221Z","shell.execute_reply.started":"2024-05-31T17:41:06.363330Z","shell.execute_reply":"2024-05-31T17:43:25.221234Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nresize_edit_box('/kaggle/working/test','/kaggle/working/scaled_test')","metadata":{"execution":{"iopub.status.busy":"2024-05-31T17:43:25.223413Z","iopub.execute_input":"2024-05-31T17:43:25.223724Z","iopub.status.idle":"2024-05-31T17:47:54.872949Z","shell.execute_reply.started":"2024-05-31T17:43:25.223698Z","shell.execute_reply":"2024-05-31T17:47:54.872019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(os.listdir('/kaggle/working/scaled_validation/images')))\nprint(len(os.listdir('/kaggle/working/scaled_validation/labels')))","metadata":{"execution":{"iopub.status.busy":"2024-05-31T17:47:54.874139Z","iopub.execute_input":"2024-05-31T17:47:54.874443Z","iopub.status.idle":"2024-05-31T17:47:54.880934Z","shell.execute_reply.started":"2024-05-31T17:47:54.874417Z","shell.execute_reply":"2024-05-31T17:47:54.880012Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(os.listdir('/kaggle/working/scaled_train/images')))\nprint(len(os.listdir('/kaggle/working/scaled_train/labels')))","metadata":{"execution":{"iopub.status.busy":"2024-05-31T17:47:54.882217Z","iopub.execute_input":"2024-05-31T17:47:54.882815Z","iopub.status.idle":"2024-05-31T17:47:54.894153Z","shell.execute_reply.started":"2024-05-31T17:47:54.882783Z","shell.execute_reply":"2024-05-31T17:47:54.893309Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## create yaml file ","metadata":{}},{"cell_type":"code","source":"\n\n# Define your data dictionary\ndata = {\n    'train': '/kaggle/working/scaled_train/images',\n    'train_labels': '/kaggle/working/scaled_train/labels',\n    'val': '/kaggle/working/scaled_validation/images',\n    'val_labels': '/kaggle/working/scaled_validation/labels',\n    'nc': 100,\n    'names': labels_list\n}\n\n# Specify the file path\nyaml_file_path = '/kaggle/working/dataset.yaml'\n\n# Write the YAML file\nwith open(yaml_file_path, 'w') as file:\n    yaml.dump(data, file, default_flow_style=False)\n\nprint(f'YAML file created at {yaml_file_path}')\n","metadata":{"execution":{"iopub.status.busy":"2024-05-31T17:47:54.895452Z","iopub.execute_input":"2024-05-31T17:47:54.895695Z","iopub.status.idle":"2024-05-31T17:47:54.906862Z","shell.execute_reply.started":"2024-05-31T17:47:54.895673Z","shell.execute_reply":"2024-05-31T17:47:54.906040Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Read and print the YAML file to verify contents\nwith open(yaml_file_path, 'r') as file:\n    content = yaml.safe_load(file)\n    print(content)","metadata":{"execution":{"iopub.status.busy":"2024-05-31T17:47:54.908077Z","iopub.execute_input":"2024-05-31T17:47:54.908402Z","iopub.status.idle":"2024-05-31T17:47:54.925737Z","shell.execute_reply.started":"2024-05-31T17:47:54.908376Z","shell.execute_reply":"2024-05-31T17:47:54.924898Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# yaml_content = '''\n\n# train: '/kaggle/working/scaled_train/images'  \n\n# nc: 100  \n\n# \"names\": \n# [\"U+306B\", \"U+306E\", \"U+3057\", \"U+3066\", \"U+3068\", \"U+3092\", \"U+306F\", \"U+304B\", \"U+308A\", \"U+306A\", \"U+3082\", \"U+3044\", \"U+308B\", \"U+3089\", \"U+305F\", \"U+304F\", \"U+3078\", \"U+307E\", \"U+304D\", \"U+308C\",\n#  \"U+3055\", \"U+3075\", \"U+3064\", \"U+3093\", \"U+304C\", \"U+3046\", \"U+3059\", \"U+3042\", \"U+3084\", \"U+304A\", \"U+3053\", \"U+3088\", \"U+3072\", \"U+3051\", \"U+3070\", \"U+305B\", \"U+305D\", \"U+309D\", \"U+3081\", \"U+3060\",\n#  \"U+4E00\", \"U+3067\", \"U+3069\", \"U+307F\", \"U+30A2\", \"U+306D\", \"U+4EBA\", \"U+3031\", \"U+308F\", \"U+3061\", \"U+30FB\", \"U+4E8B\", \"U+5165\", \"U+308D\", \"U+305A\", \"U+51FA\", \"U+3079\", \"U+4E91\", \"U+3058\", \"U+5B50\",\n#  \"U+53C8\", \"U+307B\", \"U+306C\", \"U+898B\", \"U+3086\", \"U+7269\", \"U+5927\", \"U+6B64\", \"U+3080\", \"U+4E0A\", \"U+65E5\", \"U+3054\", \"U+5176\", \"U+4E09\", \"U+3056\", \"U+65B9\", \"U+4E2D\", \"U+5C0F\", \"U+4E5F\", \"U+4F55\",\n#  \"U+25CB\", \"U+4E8C\", \"U+5973\", \"U+5019\", \"U+5FA1\", \"U+56FD\", \"U+5341\", \"U+3073\", \"U+4ECA\", \"U+30CF\", \"U+6642\", \"U+662F\", \"U+3065\", \"U+300C\", \"U+6C34\", \"U+5408\", \"U+624B\", \"U+30C8\", \"U+6240\", \"U+4E94\"]\n# '''\n\n# yaml_file = '/kaggle/working/kuzushiji.yaml'\n# with open (yaml_file, 'w')as file:\n#     file.write(yaml_content)","metadata":{"execution":{"iopub.status.busy":"2024-05-31T17:47:54.926946Z","iopub.execute_input":"2024-05-31T17:47:54.927704Z","iopub.status.idle":"2024-05-31T17:47:54.933828Z","shell.execute_reply.started":"2024-05-31T17:47:54.927671Z","shell.execute_reply":"2024-05-31T17:47:54.933102Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.environ['WANDB_MODE'] = 'disabled'\n","metadata":{"execution":{"iopub.status.busy":"2024-05-31T17:47:54.934792Z","iopub.execute_input":"2024-05-31T17:47:54.935085Z","iopub.status.idle":"2024-05-31T17:47:54.946207Z","shell.execute_reply.started":"2024-05-31T17:47:54.935062Z","shell.execute_reply":"2024-05-31T17:47:54.945363Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from ultralytics import YOLO\n\n# Load a model\nmodel = YOLO('yolov8n.yaml')  # build a new model from YAML\nmodel = YOLO('yolov8n.pt')  # load a pretrained model (recommended for training)\nmodel = YOLO('yolov8n.yaml').load('yolov8n.pt')  # build from YAML and transfer weights\n\n# Train the model\nresults = model.train(data=yaml_file_path, epochs=30, imgsz=640)","metadata":{"execution":{"iopub.status.busy":"2024-05-31T17:47:54.947300Z","iopub.execute_input":"2024-05-31T17:47:54.948125Z","iopub.status.idle":"2024-05-31T18:37:35.398196Z","shell.execute_reply.started":"2024-05-31T17:47:54.948092Z","shell.execute_reply":"2024-05-31T18:37:35.397059Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Testing & submission","metadata":{}},{"cell_type":"code","source":"submission = pd.read_csv('/kaggle/input/kuzushiji-recognition/sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2024-05-31T18:37:35.402280Z","iopub.execute_input":"2024-05-31T18:37:35.402734Z","iopub.status.idle":"2024-05-31T18:37:35.420357Z","shell.execute_reply.started":"2024-05-31T18:37:35.402697Z","shell.execute_reply":"2024-05-31T18:37:35.419532Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission","metadata":{"execution":{"iopub.status.busy":"2024-05-31T18:37:35.421420Z","iopub.execute_input":"2024-05-31T18:37:35.421674Z","iopub.status.idle":"2024-05-31T18:37:35.441549Z","shell.execute_reply.started":"2024-05-31T18:37:35.421652Z","shell.execute_reply":"2024-05-31T18:37:35.440614Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_path = '/kaggle/working/runs/detect/train/weights/best.pt'\nbest_model = YOLO(model_path)","metadata":{"execution":{"iopub.status.busy":"2024-05-31T18:37:35.442902Z","iopub.execute_input":"2024-05-31T18:37:35.443672Z","iopub.status.idle":"2024-05-31T18:37:35.495310Z","shell.execute_reply.started":"2024-05-31T18:37:35.443638Z","shell.execute_reply":"2024-05-31T18:37:35.494515Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results = []\nfor test_img_path in os.listdir('/kaggle/working/scaled_test/images'):\n    test_img = os.path.join('/kaggle/working/scaled_test/images',test_img_path)\n    result = best_model.predict(test_img, conf=0.5)\n#     submission[submission.image_id==test_img_path]['labels'] = \n    results.append(result)\nresults[0]","metadata":{"execution":{"iopub.status.busy":"2024-05-31T18:37:35.497403Z","iopub.execute_input":"2024-05-31T18:37:35.498017Z","iopub.status.idle":"2024-05-31T18:38:12.774181Z","shell.execute_reply.started":"2024-05-31T18:37:35.497967Z","shell.execute_reply":"2024-05-31T18:38:12.773314Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result[0].path","metadata":{"execution":{"iopub.status.busy":"2024-05-31T18:38:12.775355Z","iopub.execute_input":"2024-05-31T18:38:12.775611Z","iopub.status.idle":"2024-05-31T18:38:12.781635Z","shell.execute_reply.started":"2024-05-31T18:38:12.775587Z","shell.execute_reply":"2024-05-31T18:38:12.780716Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result[0].boxes.cls","metadata":{"execution":{"iopub.status.busy":"2024-05-31T18:38:12.783133Z","iopub.execute_input":"2024-05-31T18:38:12.783823Z","iopub.status.idle":"2024-05-31T18:38:12.798881Z","shell.execute_reply.started":"2024-05-31T18:38:12.783788Z","shell.execute_reply":"2024-05-31T18:38:12.798159Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\n# Prepare results for saving\npredictions = {}\nfor result in results:\n    \n    predictions[result[0].path]={\n        'boxes':result[0].boxes.xyxy,\n        'scores':result[0].boxes.conf,\n        'classes':result[0].boxes.cls\n    }\n\n","metadata":{"execution":{"iopub.status.busy":"2024-05-31T18:38:12.800264Z","iopub.execute_input":"2024-05-31T18:38:12.800601Z","iopub.status.idle":"2024-05-31T18:38:12.837726Z","shell.execute_reply.started":"2024-05-31T18:38:12.800570Z","shell.execute_reply":"2024-05-31T18:38:12.837028Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_sumbission(res):\n    data = []\n    for img_path, details in res.items():\n        boxes = details['boxes'].tolist() if isinstance(details['boxes'], torch.Tensor) else details['boxes']\n        scores = details['scores'].tolist() if isinstance(details['scores'], torch.Tensor) else details['scores']\n        classes = details['classes'].tolist() if isinstance(details['classes'], torch.Tensor) else details['classes']\n        \n        for box, cls in zip(boxes,classes):\n#             label_with_box = {'class':labels_list[int(cls)], 'box':box}\n            label_with_box = [labels_list[int(cls)]]+ box\n            \n            row = {'image_id':img_path.split('/')[-1],\n                     'labels' : label_with_box\n                  }\n            data.append(row)\n    df = pd.DataFrame(data)\n    return df ","metadata":{"execution":{"iopub.status.busy":"2024-05-31T18:38:12.838818Z","iopub.execute_input":"2024-05-31T18:38:12.839162Z","iopub.status.idle":"2024-05-31T18:38:12.847487Z","shell.execute_reply.started":"2024-05-31T18:38:12.839127Z","shell.execute_reply":"2024-05-31T18:38:12.846452Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\n\n# Sample DataFrame\ndata = {'col1': [[1, 2, 3], [4, 5, 6], [7, 8, 9]]}\ndf = pd.DataFrame(data)\ndf","metadata":{"execution":{"iopub.status.busy":"2024-05-31T18:44:17.349377Z","iopub.execute_input":"2024-05-31T18:44:17.349800Z","iopub.status.idle":"2024-05-31T18:44:17.361568Z","shell.execute_reply.started":"2024-05-31T18:44:17.349770Z","shell.execute_reply":"2024-05-31T18:44:17.360471Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def label_to_string(list):\n    return ' '.join([str(elm) for elm in list])","metadata":{"execution":{"iopub.status.busy":"2024-05-31T18:38:12.848795Z","iopub.execute_input":"2024-05-31T18:38:12.849092Z","iopub.status.idle":"2024-05-31T18:38:12.859099Z","shell.execute_reply.started":"2024-05-31T18:38:12.849067Z","shell.execute_reply":"2024-05-31T18:38:12.858302Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['col2']= df.col1.apply(label_to_string)\ndf","metadata":{"execution":{"iopub.status.busy":"2024-05-31T18:45:14.904231Z","iopub.execute_input":"2024-05-31T18:45:14.905176Z","iopub.status.idle":"2024-05-31T18:45:14.915429Z","shell.execute_reply.started":"2024-05-31T18:45:14.905139Z","shell.execute_reply":"2024-05-31T18:45:14.914538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_file = create_sumbission(predictions)\nsubmission_file= submission_file.groupby('image_id')['labels'].apply(lambda x: ' '.join(map(str,x))).reset_index()\nsubmission_file['labels'] = submission_file.labels.str.strip('[')\nsubmission_file['labels'] = submission_file.labels.str.strip(']')\nsubmission_file['labels'] = submission_file.labels.str.replace(',',' ')\nsubmission_file.to_csv('submission.csv', index=False)\nsubmission_file","metadata":{"execution":{"iopub.status.busy":"2024-05-31T18:59:26.697557Z","iopub.execute_input":"2024-05-31T18:59:26.697850Z","iopub.status.idle":"2024-05-31T18:59:27.781055Z","shell.execute_reply.started":"2024-05-31T18:59:26.697823Z","shell.execute_reply":"2024-05-31T18:59:27.780024Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_file.iloc[0,1]","metadata":{"execution":{"iopub.status.busy":"2024-05-31T18:50:13.691015Z","iopub.execute_input":"2024-05-31T18:50:13.691849Z","iopub.status.idle":"2024-05-31T18:50:13.697665Z","shell.execute_reply.started":"2024-05-31T18:50:13.691816Z","shell.execute_reply":"2024-05-31T18:50:13.696702Z"},"trusted":true},"execution_count":null,"outputs":[]}]}