{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport json\nimport pandas as pd\nimport seaborn as sn\nimport matplotlib.pyplot as plt\nfrom pprint import pprint\n\nsn.set()\n\nPATH_DATASET = \"/kaggle/input/herbarium-2022-fgvc9\"\n\nwith open(os.path.join(PATH_DATASET, \"train_metadata.json\")) as fp:\n    train_data = json.load(fp)\n\nwith open(os.path.join(PATH_DATASET, \"test_metadata.json\")) as fp:\n    test_data = json.load(fp)\n\npprint(train_data.keys())\npprint(len(test_data))","metadata":{"execution":{"iopub.status.busy":"2022-03-24T15:30:19.146059Z","iopub.execute_input":"2022-03-24T15:30:19.146398Z","iopub.status.idle":"2022-03-24T15:30:35.207041Z","shell.execute_reply.started":"2022-03-24T15:30:19.146313Z","shell.execute_reply":"2022-03-24T15:30:35.205509Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_annotations = pd.DataFrame(train_data['annotations'])\ndisplay(train_annotations.head(3))\n\naxs = train_annotations[[\"genus_id\", \"institution_id\", \"category_id\"]].hist(bins=100, sharey=True, figsize=(8, 8), grid=True, layout=(3, 1))\n_= [ax.set_yscale('log') for ax in axs[0]]","metadata":{"execution":{"iopub.status.busy":"2022-03-24T15:30:56.472335Z","iopub.execute_input":"2022-03-24T15:30:56.472633Z","iopub.status.idle":"2022-03-24T15:30:59.85345Z","shell.execute_reply.started":"2022-03-24T15:30:56.4726Z","shell.execute_reply":"2022-03-24T15:30:59.852747Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_categories = pd.DataFrame(train_data['categories']).set_index(\"category_id\")\ndisplay(train_categories.head())","metadata":{"execution":{"iopub.status.busy":"2022-03-24T15:31:03.66597Z","iopub.execute_input":"2022-03-24T15:31:03.666659Z","iopub.status.idle":"2022-03-24T15:31:03.702459Z","shell.execute_reply.started":"2022-03-24T15:31:03.666619Z","shell.execute_reply":"2022-03-24T15:31:03.701784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_genera = pd.DataFrame(train_data['genera']).set_index(\"genus_id\")\ndisplay(train_genera.head())","metadata":{"execution":{"iopub.status.busy":"2022-03-24T15:31:06.246747Z","iopub.execute_input":"2022-03-24T15:31:06.247228Z","iopub.status.idle":"2022-03-24T15:31:06.260173Z","shell.execute_reply.started":"2022-03-24T15:31:06.247191Z","shell.execute_reply":"2022-03-24T15:31:06.259137Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_institutions = pd.DataFrame(train_data['institutions']).set_index(\"institution_id\")\ndisplay(train_institutions.head())","metadata":{"execution":{"iopub.status.busy":"2022-03-24T15:31:08.660908Z","iopub.execute_input":"2022-03-24T15:31:08.661545Z","iopub.status.idle":"2022-03-24T15:31:08.67093Z","shell.execute_reply.started":"2022-03-24T15:31:08.66151Z","shell.execute_reply":"2022-03-24T15:31:08.670104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_images = pd.DataFrame(train_data['images']).set_index(\"image_id\")\ndisplay(train_images.head())","metadata":{"execution":{"iopub.status.busy":"2022-03-24T15:31:11.466695Z","iopub.execute_input":"2022-03-24T15:31:11.467184Z","iopub.status.idle":"2022-03-24T15:31:12.544423Z","shell.execute_reply.started":"2022-03-24T15:31:11.467152Z","shell.execute_reply":"2022-03-24T15:31:12.543737Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_distances = pd.DataFrame(train_data['distances'])\ndisplay(train_distances.head())\n\nfig = plt.figure(figsize=(18, 18))\nheat = train_distances.pivot(index=\"genus_id_y\", columns=\"genus_id_x\", values=\"distance\")\n_= sn.heatmap(heat, ax=fig.gca())","metadata":{"execution":{"iopub.status.busy":"2022-03-24T15:31:15.181522Z","iopub.execute_input":"2022-03-24T15:31:15.181815Z","iopub.status.idle":"2022-03-24T15:31:32.729366Z","shell.execute_reply.started":"2022-03-24T15:31:15.181785Z","shell.execute_reply":"2022-03-24T15:31:32.728649Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = pd.merge(train_annotations, train_images, how=\"left\", right_index=True, left_on=\"image_id\")\ndf_train = pd.merge(df_train, train_categories, how=\"left\", right_index=True, left_on=\"category_id\")\ndf_train = pd.merge(df_train, train_institutions, how=\"left\", right_index=True, left_on=\"institution_id\")\n# df_train = pd.merge(df_train, train_genera, how=\"left\", right_index=True, left_on=\"genus_id\")\n\ndisplay(df_train.head())\nprint(f\"training images: {len(df_train)}\")","metadata":{"execution":{"iopub.status.busy":"2022-03-24T15:31:41.010273Z","iopub.execute_input":"2022-03-24T15:31:41.010536Z","iopub.status.idle":"2022-03-24T15:31:41.91541Z","shell.execute_reply.started":"2022-03-24T15:31:41.010505Z","shell.execute_reply":"2022-03-24T15:31:41.91473Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.sample(frac=1)\n\nfig, axarr = plt.subplots(nrows=2, ncols=5, figsize=(12, 6))\nfor i, (_, row) in enumerate(df_train[:10].iterrows()):\n    img_path = os.path.join(PATH_DATASET, \"train_images\", row[\"file_name\"])\n    img = plt.imread(img_path)\n    axarr[i // 5, i % 5].imshow(img)\n#     print(row)\nfig.tight_layout()","metadata":{"execution":{"iopub.status.busy":"2022-03-24T15:31:45.816977Z","iopub.execute_input":"2022-03-24T15:31:45.817266Z","iopub.status.idle":"2022-03-24T15:31:49.250965Z","shell.execute_reply.started":"2022-03-24T15:31:45.817235Z","shell.execute_reply":"2022-03-24T15:31:49.248628Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import glob\nimport numpy as np\nfrom tqdm.auto import tqdm\nfrom joblib import Parallel, delayed\nimport matplotlib.pyplot as plt\n\ndef _color_means(img_path):\n    img = plt.imread(img_path)\n    means = {i: np.mean(img[..., i]) / 255.0 for i in range(3)}\n    std = {i: np.std(img[..., i]) / 255.0 for i in range(3)}\n    return means, std\n\nimages = glob.glob(os.path.join(PATH_DATASET, \"train_images\", \"*\", \"*\", \"*.jpg\"))\n# images += glob.glob(os.path.join(PATH_DATASET, \"test_images\", \"*\", \"*.jpg\"))\nclr_mean_std = Parallel(n_jobs=os.cpu_count())(delayed(_color_means)(fn) for fn in tqdm(images[:15000]))","metadata":{"execution":{"iopub.status.busy":"2022-03-24T15:31:53.090182Z","iopub.execute_input":"2022-03-24T15:31:53.090912Z","iopub.status.idle":"2022-03-24T15:41:56.125264Z","shell.execute_reply.started":"2022-03-24T15:31:53.090872Z","shell.execute_reply":"2022-03-24T15:41:56.124493Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img_color_mean = pd.DataFrame([c[0] for c in clr_mean_std]).describe()\ndisplay(img_color_mean)\nimg_color_std = pd.DataFrame([c[1] for c in clr_mean_std]).describe()\ndisplay(img_color_std)\n\nimg_color_mean = list(img_color_mean.T[\"mean\"])\nimg_color_std = list(img_color_std.T[\"mean\"])\nprint(img_color_mean, img_color_std)","metadata":{"execution":{"iopub.status.busy":"2022-03-24T15:42:05.408401Z","iopub.execute_input":"2022-03-24T15:42:05.408679Z","iopub.status.idle":"2022-03-24T15:42:05.506915Z","shell.execute_reply.started":"2022-03-24T15:42:05.408648Z","shell.execute_reply":"2022-03-24T15:42:05.506223Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install -q effdet \"icevision[all]\" 'lightning-flash[image]'\n# !pip install -q \"pytorch-lightning==1.4.*\"\n!pip uninstall -y wandb","metadata":{"execution":{"iopub.status.busy":"2022-03-24T15:42:12.089294Z","iopub.execute_input":"2022-03-24T15:42:12.089696Z","iopub.status.idle":"2022-03-24T15:43:33.142254Z","shell.execute_reply.started":"2022-03-24T15:42:12.089643Z","shell.execute_reply":"2022-03-24T15:43:33.141369Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip download -q effdet \"icevision[all]\" 'lightning-flash[image]' --dest frozen_packages --prefer-binary\n!rm frozen_packages/torch-*\n!ls -l frozen_packages","metadata":{"execution":{"iopub.status.busy":"2022-03-24T15:43:40.512147Z","iopub.execute_input":"2022-03-24T15:43:40.51243Z","iopub.status.idle":"2022-03-24T15:46:02.318712Z","shell.execute_reply.started":"2022-03-24T15:43:40.5124Z","shell.execute_reply":"2022-03-24T15:46:02.317793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport flash\nfrom flash.core.data.utils import download_data\nfrom flash.image import ImageClassificationData, ImageClassifier","metadata":{"execution":{"iopub.status.busy":"2022-03-24T15:46:19.906641Z","iopub.execute_input":"2022-03-24T15:46:19.906922Z","iopub.status.idle":"2022-03-24T15:46:31.456355Z","shell.execute_reply.started":"2022-03-24T15:46:19.906891Z","shell.execute_reply":"2022-03-24T15:46:31.455612Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from dataclasses import dataclass\nfrom torchvision import transforms as T\nfrom typing import Tuple, Callable\nfrom flash.core.data.io.input_transform import InputTransform\n\n@dataclass\nclass ImageClassificationInputTransform(InputTransform):\n\n    image_size: Tuple[int, int] = (224, 224)\n\n    def input_per_sample_transform(self):\n        return T.Compose([\n            T.ToTensor(),\n            T.Resize(self.image_size),\n            # T.Normalize([0.778, 0.756, 0.709], [0.246, 0.250, 0.253]),\n            T.Normalize(img_color_mean, img_color_std),\n        ])\n\n    def train_input_per_sample_transform(self):\n        return T.Compose([\n            T.ToTensor(),\n            T.Resize(self.image_size),\n            # T.Normalize([0.778, 0.756, 0.709], [0.246, 0.250, 0.253]),\n            T.Normalize(img_color_mean, img_color_std),\n            T.RandomHorizontalFlip(),\n            T.RandomAffine(degrees=10, scale=(0.9, 1.1), translate=(0.1, 0.1)),\n            # T.ColorJitter(),\n            # T.RandomAutocontrast(),\n            # T.RandomPerspective(distortion_scale=0.1),\n        ])\n\n    def target_per_sample_transform(self) -> Callable:\n        return torch.as_tensor","metadata":{"execution":{"iopub.status.busy":"2022-03-24T15:46:49.792446Z","iopub.execute_input":"2022-03-24T15:46:49.793148Z","iopub.status.idle":"2022-03-24T15:46:49.801988Z","shell.execute_reply.started":"2022-03-24T15:46:49.793112Z","shell.execute_reply":"2022-03-24T15:46:49.800761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(df_train)","metadata":{"execution":{"iopub.status.busy":"2022-03-24T15:46:53.279656Z","iopub.execute_input":"2022-03-24T15:46:53.280233Z","iopub.status.idle":"2022-03-24T15:46:53.285473Z","shell.execute_reply.started":"2022-03-24T15:46:53.280193Z","shell.execute_reply":"2022-03-24T15:46:53.284801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"datamodule = ImageClassificationData.from_data_frame(\n    input_field=\"file_name\",\n    target_fields=\"category_id\",\n    # for simplicity take just half of the data\n    train_data_frame=df_train[:len(df_train) // 10],\n    train_images_root=os.path.join(PATH_DATASET, \"train_images\"),\n    train_transform=ImageClassificationInputTransform,\n    batch_size=128,\n    transform_kwargs={\"image_size\": (224, 224)},\n    num_workers=3,\n)","metadata":{"execution":{"iopub.status.busy":"2022-03-24T15:48:13.344515Z","iopub.execute_input":"2022-03-24T15:48:13.345033Z","iopub.status.idle":"2022-03-24T15:50:36.995653Z","shell.execute_reply.started":"2022-03-24T15:48:13.344995Z","shell.execute_reply":"2022-03-24T15:50:36.994897Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torch import nn, optim\nmodel = ImageClassifier(backbone=\"resnet50\", num_classes=datamodule.num_classes,learning_rate=0.001,optimizer=optim.Adam)","metadata":{"execution":{"iopub.status.busy":"2022-03-24T15:50:44.211951Z","iopub.execute_input":"2022-03-24T15:50:44.21222Z","iopub.status.idle":"2022-03-24T15:50:47.237004Z","shell.execute_reply.started":"2022-03-24T15:50:44.212191Z","shell.execute_reply":"2022-03-24T15:50:47.236269Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc\ngc.collect()\ntrainer = flash.Trainer(max_epochs=4, gpus=torch.cuda.device_count())\ntrainer.finetune(model, datamodule=datamodule, strategy=\"freeze\")","metadata":{"execution":{"iopub.status.busy":"2022-03-24T15:51:00.605387Z","iopub.execute_input":"2022-03-24T15:51:00.605898Z","iopub.status.idle":"2022-03-24T17:30:27.761411Z","shell.execute_reply.started":"2022-03-24T15:51:00.605856Z","shell.execute_reply":"2022-03-24T17:30:27.760638Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainer.save_checkpoint(\"fgvc_9.pt\")\n","metadata":{"execution":{"iopub.status.busy":"2022-03-24T17:35:40.665969Z","iopub.execute_input":"2022-03-24T17:35:40.666267Z","iopub.status.idle":"2022-03-24T17:35:40.991284Z","shell.execute_reply.started":"2022-03-24T17:35:40.666236Z","shell.execute_reply":"2022-03-24T17:35:40.990503Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport json\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\nPATH_DATASET = \"/kaggle/input/herbarium-2022-fgvc9\"\n\nwith open(os.path.join(PATH_DATASET, \"test_metadata.json\")) as fp:\n    test_data = json.load(fp)\n\nprint(len(test_data))\ndf_test = pd.DataFrame(test_data).set_index(\"image_id\")\ndisplay(df_test.head())","metadata":{"execution":{"iopub.status.busy":"2022-03-24T17:35:57.117863Z","iopub.execute_input":"2022-03-24T17:35:57.118122Z","iopub.status.idle":"2022-03-24T17:35:57.868131Z","shell.execute_reply.started":"2022-03-24T17:35:57.118092Z","shell.execute_reply":"2022-03-24T17:35:57.867432Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from dataclasses import dataclass\nfrom torchvision import transforms as T\nfrom typing import Tuple, Callable\nfrom flash.core.data.io.input_transform import InputTransform\n\n@dataclass\nclass ImageClassificationInputTransform(InputTransform):\n\n    image_size: Tuple[int, int] = (224, 224)\n\n    def input_per_sample_transform(self):\n        return T.Compose([\n            T.ToTensor(),\n            T.Resize(self.image_size),\n            T.Normalize([0.778, 0.756, 0.709], [0.246, 0.250, 0.253]),\n            # T.Normalize(img_color_mean, img_color_std),\n        ])\n\n    def target_per_sample_transform(self) -> Callable:\n        return torch.as_tensor","metadata":{"execution":{"iopub.status.busy":"2022-03-24T17:36:01.212786Z","iopub.execute_input":"2022-03-24T17:36:01.213407Z","iopub.status.idle":"2022-03-24T17:36:01.220857Z","shell.execute_reply.started":"2022-03-24T17:36:01.213372Z","shell.execute_reply":"2022-03-24T17:36:01.220154Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"datamodule = ImageClassificationData.from_data_frame(\n    input_field=\"file_name\",\n    predict_data_frame=df_test,\n    predict_images_root=os.path.join(PATH_DATASET, \"test_images\"),\n    predict_transform=ImageClassificationInputTransform,\n    batch_size=128,\n    transform_kwargs={\"image_size\": (512, 512)},\n    num_workers=3,\n)","metadata":{"execution":{"iopub.status.busy":"2022-03-24T17:36:18.869833Z","iopub.execute_input":"2022-03-24T17:36:18.870104Z","iopub.status.idle":"2022-03-24T17:48:00.568893Z","shell.execute_reply.started":"2022-03-24T17:36:18.870074Z","shell.execute_reply":"2022-03-24T17:48:00.568122Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions = []\nfor lbs in trainer.predict(model, datamodule=datamodule, output=\"labels\"):\n    # lbs = [torch.argmax(p[\"preds\"].float()).item() for p in preds]\n    predictions += lbs","metadata":{"execution":{"iopub.status.busy":"2022-03-24T17:48:15.428978Z","iopub.execute_input":"2022-03-24T17:48:15.429252Z","iopub.status.idle":"2022-03-24T19:14:39.628013Z","shell.execute_reply.started":"2022-03-24T17:48:15.429221Z","shell.execute_reply":"2022-03-24T19:14:39.627288Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.DataFrame({\"Id\": df_test.index, \"Predicted\": predictions}).set_index(\"Id\")\nsubmission.to_csv(\"submission.csv\")\n! head submission.csv","metadata":{"execution":{"iopub.status.busy":"2022-03-24T19:15:05.440698Z","iopub.execute_input":"2022-03-24T19:15:05.441307Z","iopub.status.idle":"2022-03-24T19:15:06.55395Z","shell.execute_reply.started":"2022-03-24T19:15:05.441265Z","shell.execute_reply":"2022-03-24T19:15:06.5531Z"},"trusted":true},"execution_count":null,"outputs":[]}]}