{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Explore 🕵 provdided data","metadata":{}},{"cell_type":"code","source":"!ls -l /kaggle/input\n\nPATH_DATASET = \"/kaggle/input/happy-whale-and-dolphin\"","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-03-04T17:40:53.313343Z","iopub.execute_input":"2022-03-04T17:40:53.313764Z","iopub.status.idle":"2022-03-04T17:40:54.094000Z","shell.execute_reply.started":"2022-03-04T17:40:53.313663Z","shell.execute_reply":"2022-03-04T17:40:54.093171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\ndf_train = pd.read_csv(os.path.join(PATH_DATASET, \"train.csv\"))\ndisplay(df_train.head())\nprint(f\"Dataset size: {len(df_train)}\")\nprint(f\"Unique ids: {len(df_train['individual_id'].unique())}\")","metadata":{"execution":{"iopub.status.busy":"2022-03-04T17:41:00.936508Z","iopub.execute_input":"2022-03-04T17:41:00.937426Z","iopub.status.idle":"2022-03-04T17:41:01.076115Z","shell.execute_reply.started":"2022-03-04T17:41:00.937375Z","shell.execute_reply":"2022-03-04T17:41:01.075179Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Lets see how top individulas we have in the database...","metadata":{}},{"cell_type":"code","source":"import numpy as np\nfrom pprint import pprint\n\nspecies_individuals = {}\nfor name, dfg in df_train.groupby(\"species\"):\n    species_individuals[name] = dfg[\"individual_id\"].value_counts()\n\nsi_max = max(list(map(len, species_individuals.values())))\nsi = {n: [0] * si_max for n in species_individuals}\nfor n, counts in species_individuals.items():\n    si[n][:len(counts)] = list(np.log(counts))\nsi = pd.DataFrame(si)","metadata":{"execution":{"iopub.status.busy":"2022-03-04T17:41:05.430641Z","iopub.execute_input":"2022-03-04T17:41:05.431609Z","iopub.status.idle":"2022-03-04T17:41:05.524730Z","shell.execute_reply.started":"2022-03-04T17:41:05.431556Z","shell.execute_reply":"2022-03-04T17:41:05.523840Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import seaborn as sn\n\nfig = plt.figure(figsize=(10, 8))\nax = sn.heatmap(si[:500].T, cmap=\"BuGn\", ax=fig.gca())","metadata":{"execution":{"iopub.status.busy":"2022-03-04T17:41:07.486242Z","iopub.execute_input":"2022-03-04T17:41:07.486531Z","iopub.status.idle":"2022-03-04T17:41:09.696844Z","shell.execute_reply.started":"2022-03-04T17:41:07.486501Z","shell.execute_reply":"2022-03-04T17:41:09.695986Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"top_ids = {n: list(inds[:4].index) + ['new_individual']  for n, inds in species_individuals.items()}\ndisplay(pd.DataFrame(top_ids).T)","metadata":{"execution":{"iopub.status.busy":"2022-03-04T17:41:14.582841Z","iopub.execute_input":"2022-03-04T17:41:14.583167Z","iopub.status.idle":"2022-03-04T17:41:14.606668Z","shell.execute_reply.started":"2022-03-04T17:41:14.583136Z","shell.execute_reply":"2022-03-04T17:41:14.605537Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Inference with Lightning⚡Flash\n\n**this is inference for the folowing kernel:** https://www.kaggle.com/jirkaborovec/whale-dolphin-eda-classify-lit-flash","metadata":{}},{"cell_type":"code","source":"!pip install -q 'lightning-flash[image]' --find-links /kaggle/input/whale-dolphin-eda-classify-lit-flash/frozen_packages/ --no-index\n!pip uninstall -y wandb","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-03-04T17:41:19.533360Z","iopub.execute_input":"2022-03-04T17:41:19.533667Z","iopub.status.idle":"2022-03-04T17:41:51.489499Z","shell.execute_reply.started":"2022-03-04T17:41:19.533634Z","shell.execute_reply":"2022-03-04T17:41:51.488665Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport flash\nfrom flash.image import ImageClassificationData, ImageClassifier","metadata":{"execution":{"iopub.status.busy":"2022-03-04T17:41:51.491656Z","iopub.execute_input":"2022-03-04T17:41:51.492158Z","iopub.status.idle":"2022-03-04T17:42:03.640436Z","shell.execute_reply.started":"2022-03-04T17:41:51.492118Z","shell.execute_reply":"2022-03-04T17:42:03.639797Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import glob\n\nimgs = glob.glob(\"/kaggle/input/happy-whale-and-dolphin/test_images/*.jpg\")\ndf_test = pd.DataFrame(map(os.path.basename, imgs), columns=[\"image\"])\ndisplay(df_test.head())\nprint(len(df_test))","metadata":{"execution":{"iopub.status.busy":"2022-03-04T17:42:57.016182Z","iopub.execute_input":"2022-03-04T17:42:57.016801Z","iopub.status.idle":"2022-03-04T17:42:57.734529Z","shell.execute_reply.started":"2022-03-04T17:42:57.016764Z","shell.execute_reply":"2022-03-04T17:42:57.733474Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 1. Load the task ⚙️","metadata":{}},{"cell_type":"code","source":"model = ImageClassifier.load_from_checkpoint(\n    \"/kaggle/input/whale-dolphin-eda-classify-lit-flash/image_classification_model.pt\"\n)\nprint(model.labels)","metadata":{"execution":{"iopub.status.busy":"2022-03-04T17:42:03.641710Z","iopub.execute_input":"2022-03-04T17:42:03.642123Z","iopub.status.idle":"2022-03-04T17:42:26.524010Z","shell.execute_reply.started":"2022-03-04T17:42:03.642087Z","shell.execute_reply":"2022-03-04T17:42:26.523036Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Trainer Args\nGPUS = int(torch.cuda.is_available())  # Set to 1 if GPU is enabled for notebook\ntrainer = flash.Trainer(gpus=GPUS)","metadata":{"execution":{"iopub.status.busy":"2022-03-04T17:42:26.525957Z","iopub.execute_input":"2022-03-04T17:42:26.526463Z","iopub.status.idle":"2022-03-04T17:42:26.541836Z","shell.execute_reply.started":"2022-03-04T17:42:26.526426Z","shell.execute_reply":"2022-03-04T17:42:26.541116Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2. Run predictions 🎉","metadata":{}},{"cell_type":"code","source":"datamodule = ImageClassificationData.from_data_frame(\n    input_field=\"image\",\n    predict_data_frame=df_test,\n    # for simplicity take just fraction of the data\n    # predict_data_frame=df_test[:len(df_test) // 1000],\n    predict_images_root=os.path.join(PATH_DATASET, \"test_images\"),\n    batch_size=32,\n    transform_kwargs={\"image_size\": (300, 300)},\n    num_workers=3,\n)","metadata":{"execution":{"iopub.status.busy":"2022-03-04T17:43:07.031967Z","iopub.execute_input":"2022-03-04T17:43:07.032280Z","iopub.status.idle":"2022-03-04T17:43:07.126404Z","shell.execute_reply.started":"2022-03-04T17:43:07.032246Z","shell.execute_reply":"2022-03-04T17:43:07.125690Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions = []\nfor lbs in trainer.predict(model, datamodule=datamodule, output=\"labels\"):\n    # lbs = [torch.argmax(p[\"preds\"].float()).item() for p in preds]\n    predictions += lbs\n\ndf_test[\"prediction\"] = predictions\ndisplay(df_test.head())","metadata":{"execution":{"iopub.status.busy":"2022-03-04T17:43:13.180912Z","iopub.execute_input":"2022-03-04T17:43:13.181205Z","iopub.status.idle":"2022-03-04T17:43:44.297967Z","shell.execute_reply.started":"2022-03-04T17:43:13.181177Z","shell.execute_reply":"2022-03-04T17:43:44.296692Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Browse some images","metadata":{}},{"cell_type":"code","source":"nb_species = len(df_test[\"prediction\"].unique())\nfig, axarr = plt.subplots(ncols=5, nrows=nb_species, figsize=(12, nb_species * 2))\n\nfor i, (name, dfg) in enumerate(df_test.groupby(\"prediction\")):\n    axarr[i, 0].set_title(name)\n    for j, (_, row) in enumerate(dfg[:5].iterrows()):\n        im_path = os.path.join(PATH_DATASET, \"test_images\", row[\"image\"])\n        img = plt.imread(im_path)\n        axarr[i, j].imshow(img)\n        axarr[i, j].set_axis_off()","metadata":{"execution":{"iopub.status.busy":"2022-03-04T17:45:06.102502Z","iopub.execute_input":"2022-03-04T17:45:06.102814Z","iopub.status.idle":"2022-03-04T17:45:47.483295Z","shell.execute_reply.started":"2022-03-04T17:45:06.102786Z","shell.execute_reply":"2022-03-04T17:45:47.482114Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Static 📥 submission","metadata":{}},{"cell_type":"code","source":"!head ../input/happy-whale-and-dolphin/sample_submission.csv","metadata":{"execution":{"iopub.status.busy":"2022-03-04T17:46:12.969275Z","iopub.execute_input":"2022-03-04T17:46:12.969606Z","iopub.status.idle":"2022-03-04T17:46:13.769672Z","shell.execute_reply.started":"2022-03-04T17:46:12.969571Z","shell.execute_reply":"2022-03-04T17:46:13.768488Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test[\"predictions\"] = [\" \".join(top_ids[lb]) for lb in df_test[\"prediction\"]]\ndf_test[[\"image\",\"predictions\"]].set_index(\"image\").to_csv(\"submission.csv\")\n\n!head submission.csv","metadata":{"execution":{"iopub.status.busy":"2022-03-04T17:50:38.519735Z","iopub.execute_input":"2022-03-04T17:50:38.520114Z","iopub.status.idle":"2022-03-04T17:50:39.339683Z","shell.execute_reply.started":"2022-03-04T17:50:38.520078Z","shell.execute_reply":"2022-03-04T17:50:39.338502Z"},"trusted":true},"execution_count":null,"outputs":[]}]}