{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install -qq --upgrade wandb","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport random\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm\n\nimport wandb\nwandb.login()","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-02-01T17:52:36.701064Z","iopub.execute_input":"2022-02-01T17:52:36.701790Z","iopub.status.idle":"2022-02-01T17:52:36.711959Z","shell.execute_reply.started":"2022-02-01T17:52:36.701751Z","shell.execute_reply":"2022-02-01T17:52:36.710899Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ROOT_PATH = '../input/happy-whale-and-dolphin'\nIMGS_DIR = f'{ROOT_PATH}/train_images'\ndf = pd.read_csv(f'{ROOT_PATH}/train.csv')\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2022-02-01T17:31:51.200061Z","iopub.execute_input":"2022-02-01T17:31:51.200430Z","iopub.status.idle":"2022-02-01T17:31:51.332344Z","shell.execute_reply.started":"2022-02-01T17:31:51.200390Z","shell.execute_reply":"2022-02-01T17:31:51.331492Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## `species`","metadata":{}},{"cell_type":"code","source":"unique_species = df.species.unique()\nprint(unique_species)\nprint('Num of unique species: ', len(unique_species))","metadata":{"execution":{"iopub.status.busy":"2022-02-01T17:32:50.627794Z","iopub.execute_input":"2022-02-01T17:32:50.628786Z","iopub.status.idle":"2022-02-01T17:32:50.639134Z","shell.execute_reply.started":"2022-02-01T17:32:50.628740Z","shell.execute_reply":"2022-02-01T17:32:50.638060Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> Most of the species have `_whale` or `_dolphine` identifiers. \n\n> There are total of 30 unique species. \n\n> `beluga` is whale; what's `globis`?","metadata":{}},{"cell_type":"code","source":"df.species.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-02-01T17:39:30.451434Z","iopub.execute_input":"2022-02-01T17:39:30.451751Z","iopub.status.idle":"2022-02-01T17:39:30.466335Z","shell.execute_reply.started":"2022-02-01T17:39:30.451720Z","shell.execute_reply":"2022-02-01T17:39:30.465295Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> It's gonna be hard to identify few species. ","metadata":{}},{"cell_type":"markdown","source":"## `individual_id`s","metadata":{}},{"cell_type":"code","source":"print('Number of unique individual_ids: :', len(df.individual_id.unique()))","metadata":{"execution":{"iopub.status.busy":"2022-02-01T17:38:33.341118Z","iopub.execute_input":"2022-02-01T17:38:33.341446Z","iopub.status.idle":"2022-02-01T17:38:33.350518Z","shell.execute_reply.started":"2022-02-01T17:38:33.341416Z","shell.execute_reply":"2022-02-01T17:38:33.349785Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.individual_id.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-02-01T17:41:50.411617Z","iopub.execute_input":"2022-02-01T17:41:50.411903Z","iopub.status.idle":"2022-02-01T17:41:50.433843Z","shell.execute_reply.started":"2022-02-01T17:41:50.411873Z","shell.execute_reply":"2022-02-01T17:41:50.432746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> There are species with just one image associated with them. It's gonna be hard to cluster these individual species. It would be worse if they belong to sparse species. ","metadata":{}},{"cell_type":"markdown","source":"## Map `individual_id` to Images\n\nWe will be using [W&B Tables](https://docs.wandb.ai/guides/data-vis) feature to easily build log images belonging to unique `individual_id`. W&B Tables is like a 2D grid (spreadsheet) which supports rich media and interactiveness. For simplicity and memory consideration we will log at max 5 images per unique id. We are also going to log for those ids that has more than 5 images. This is our first look at the data. ","metadata":{}},{"cell_type":"code","source":"# Initialize a W&B run\nrun = wandb.init(project='happywhale_eda')\n\n# Initialize an empty W&B Table\ndata_table = wandb.Table(columns=['individual_id', 'image_1', 'image_2', 'image_3', 'image_4', 'image_5'])\n\nfor unique_id, tmp_df in tqdm(df.groupby('individual_id')):\n    if len(tmp_df) > 5:\n        # Sample 5 images randomly\n        sample_imgs = random.sample(list(tmp_df.image.values), 5)\n        # Add data to the table row-wise\n        data_table.add_data(unique_id,\n                            wandb.Image(f'{IMGS_DIR}/{sample_imgs[0]}'),\n                            wandb.Image(f'{IMGS_DIR}/{sample_imgs[1]}'),\n                            wandb.Image(f'{IMGS_DIR}/{sample_imgs[2]}'),\n                            wandb.Image(f'{IMGS_DIR}/{sample_imgs[3]}'),\n                            wandb.Image(f'{IMGS_DIR}/{sample_imgs[4]}'))\n        \n# Log the table\nwandb.log({'mapping_table': data_table})\n\n# Finish the run\nwandb.finish()","metadata":{"execution":{"iopub.status.busy":"2022-02-01T18:01:13.830888Z","iopub.execute_input":"2022-02-01T18:01:13.831267Z","iopub.status.idle":"2022-02-01T18:17:27.704584Z","shell.execute_reply.started":"2022-02-01T18:01:13.831227Z","shell.execute_reply":"2022-02-01T18:17:27.703321Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Check out this cool Table here: http://wandb.me/happywhale-tables\n\n![img](https://i.imgur.com/txSOmzH.mp4)","metadata":{}},{"cell_type":"markdown","source":"> Images are of varying sizes.\n\n> Images are of whale/dolphin humpback (+dorsal fins, backs, heads and flanks).","metadata":{}}]}