{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Yum or Yuck Butterfly Mimics 2022 – Explore Dataset\n\n**Author:** [Keith Pinson](https://github.com/keithpinson)<br>\n**Date created:** 2022/07/28<br>\n**Version:** 1.0.0001<br>\n**Description:** Explore the dataset<br>\n**Platform:** Kaggle Packages including Tensorflow 2.6.4 with GPU support<br>\n<br>\n","metadata":{"pycharm":{"name":"#%% md\n"}}},{"cell_type":"code","source":"import datetime\n\nprint(\"executed\",datetime.datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\"),\"local time\")","metadata":{"pycharm":{"name":"#%%\n"},"execution":{"iopub.status.busy":"2022-07-31T23:24:46.483492Z","iopub.execute_input":"2022-07-31T23:24:46.484574Z","iopub.status.idle":"2022-07-31T23:24:46.516011Z","shell.execute_reply.started":"2022-07-31T23:24:46.484469Z","shell.execute_reply":"2022-07-31T23:24:46.514770Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"---\n## Set Environment\n---","metadata":{"pycharm":{"name":"#%% md\n"}}},{"cell_type":"code","source":"import os\nimport platform\nimport random\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\nimport seaborn as sns\n\nfrom skimage import io\n","metadata":{"pycharm":{"name":"#%%\n"},"execution":{"iopub.status.busy":"2022-07-31T23:24:46.518103Z","iopub.execute_input":"2022-07-31T23:24:46.518442Z","iopub.status.idle":"2022-07-31T23:24:48.244515Z","shell.execute_reply.started":"2022-07-31T23:24:46.518387Z","shell.execute_reply":"2022-07-31T23:24:48.243449Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### <u>Dataset paths and names</u>","metadata":{"pycharm":{"name":"#%% md\n"}}},{"cell_type":"code","source":"os_system = platform.system()  # 'Windows', 'Linux'\n\nhosted_by = 'Kaggle' if os.environ.get('KAGGLE_URL_BASE') else \\\n            ('Windows' if os.environ.get('WINDIR') else \\\n            'Unknown')\n\nif hosted_by == 'Kaggle':\n    dataset_name = \"yum-or-yuck-butterfly-mimics-2022\"\n\n    # Setting the variables assuming a Kaggle platform\n    base_dir = \"/kaggle\"\n    dataset_dir = os.path.join(base_dir, 'input', dataset_name)\n    data_dir = os.path.join(dataset_dir, 'data', 'butterfly_mimics')\n    working_dir = os.path.join(base_dir, 'working')\n    temp_dir = os.path.join(base_dir, 'temp')\n\ntrain_dir = os.path.join(data_dir, 'images')\ntest_dir = os.path.join(data_dir, 'image_holdouts')\ntrain_csv = os.path.join(data_dir, 'images.csv')\ntest_csv = os.path.join(data_dir, 'image_holdouts.csv')\nsubmit_csv = os.path.join(working_dir, 'submission.csv')\n\nsample_submit_csv = os.path.join(dataset_dir, 'sample_submission.csv')\n\n\nclass_names = ['black', 'monarch', 'pipevine', 'spicebush', 'tiger', 'viceroy']\nclass_count = len(class_names)\n","metadata":{"pycharm":{"name":"#%%\n"},"execution":{"iopub.status.busy":"2022-07-31T23:24:48.246519Z","iopub.execute_input":"2022-07-31T23:24:48.247444Z","iopub.status.idle":"2022-07-31T23:24:48.258087Z","shell.execute_reply.started":"2022-07-31T23:24:48.247382Z","shell.execute_reply":"2022-07-31T23:24:48.256534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"---\n### Check our Dataset Environment\n---\n","metadata":{"pycharm":{"name":"#%% md\n"}}},{"cell_type":"markdown","source":"### First, what does our directory structure look like?\n```\n.\n├── image_holdouts\n│   ├── gna250129d.jpg\n│   ├── goabc6e644.jpg\n│   ├── gpcb27504e.jpg\n╎   ╎\n│   └── zx54a72a62.jpg\n├── images\n│   ├── ggc1e08cbc.jpg\n│   ├── gh150f104b.jpg\n│   ├── gh20ab0d9c.jpg\n╎   ╎\n│   └── zze50f4f4f.jpg\n├── image_holdouts.csv\n└── images.csv\n```","metadata":{"pycharm":{"name":"#%% md\n"}}},{"cell_type":"markdown","source":"### <u>Datasets</u>\n\nNotice that folders and files contain no naming information to indicate the butterfly species. This information is contained in the CSV files. So we will have to read the `'images.csv'` to get the training labels.","metadata":{"pycharm":{"name":"#%% md\n"}}},{"cell_type":"code","source":"# Extract the dataset\ntrain_csv_data = pd.read_csv(train_csv)\ntest_csv_data = pd.read_csv(test_csv)","metadata":{"collapsed":false,"pycharm":{"name":"#%%\n"},"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-07-31T23:24:48.259765Z","iopub.execute_input":"2022-07-31T23:24:48.260560Z","iopub.status.idle":"2022-07-31T23:24:48.304276Z","shell.execute_reply.started":"2022-07-31T23:24:48.260521Z","shell.execute_reply":"2022-07-31T23:24:48.303278Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"---\n### Explore Training Dataset\n---\n","metadata":{"pycharm":{"name":"#%% md\n"}}},{"cell_type":"markdown","source":"### What do the csv files look like?","metadata":{"pycharm":{"name":"#%% md\n"}}},{"cell_type":"code","source":"print(\"From the images.csv file:\")\nprint(\"-------------------------\")\ntrain_csv_data.head()","metadata":{"collapsed":false,"pycharm":{"name":"#%%\n"},"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-07-31T23:24:48.306570Z","iopub.execute_input":"2022-07-31T23:24:48.307192Z","iopub.status.idle":"2022-07-31T23:24:48.329710Z","shell.execute_reply.started":"2022-07-31T23:24:48.307159Z","shell.execute_reply":"2022-07-31T23:24:48.328811Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The `'image'` field will be transformed to the image feature and the `'name'` will be encoded to become the label. The `'stage'` field can be dropped since there are no caterpillars in the dataset. Advanced models may find the `'side'` field useful.","metadata":{"pycharm":{"name":"#%% md\n"}}},{"cell_type":"markdown","source":"### What species of butterflies do we have?","metadata":{"pycharm":{"name":"#%% md\n"}}},{"cell_type":"code","source":"print(train_csv_data['name'].unique())","metadata":{"collapsed":false,"pycharm":{"name":"#%%\n"},"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-07-31T23:24:48.331070Z","iopub.execute_input":"2022-07-31T23:24:48.331689Z","iopub.status.idle":"2022-07-31T23:24:48.344197Z","shell.execute_reply.started":"2022-07-31T23:24:48.331656Z","shell.execute_reply":"2022-07-31T23:24:48.343196Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### How many images of each do we have?","metadata":{"pycharm":{"name":"#%% md\n"}}},{"cell_type":"code","source":"print(\"Total:\\t\\t\", train_csv_data['name'].count())\nprint(\"\")\nprint(train_csv_data['name'].value_counts())","metadata":{"collapsed":false,"pycharm":{"name":"#%%\n"},"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-07-31T23:24:48.345828Z","iopub.execute_input":"2022-07-31T23:24:48.346528Z","iopub.status.idle":"2022-07-31T23:24:48.362048Z","shell.execute_reply.started":"2022-07-31T23:24:48.346487Z","shell.execute_reply":"2022-07-31T23:24:48.359677Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Let's visualize the count","metadata":{"pycharm":{"name":"#%% md\n"}}},{"cell_type":"code","source":"fig = plt.figure(figsize=(16, 5))\nsns.set_theme(font_scale=2,palette=\"Set2\")\nsns.countplot(x=train_csv_data['name'],\n            order=train_csv_data['name'].value_counts().index).set(title='Count of Butterfly by Species')\nplt.xticks(rotation=0);","metadata":{"collapsed":false,"pycharm":{"name":"#%%\n"},"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-07-31T23:24:48.364095Z","iopub.execute_input":"2022-07-31T23:24:48.365022Z","iopub.status.idle":"2022-07-31T23:24:48.672309Z","shell.execute_reply.started":"2022-07-31T23:24:48.364974Z","shell.execute_reply":"2022-07-31T23:24:48.670920Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Let's visualize the size of the image data?","metadata":{"pycharm":{"name":"#%% md\n"}}},{"cell_type":"code","source":"first_file = os.path.join(train_dir, train_csv_data.image[1]+\".jpg\")\n\nimage = io.imread(first_file, as_gray=False)\n\nfigure = plt.figure()\nax = figure.add_axes([0, 0, 1, 1])\n\nax.grid(None)\nax.axis('on')\n\nax.xaxis.labelpad = 20\nax.xaxis.set_label_text(f'Shape: {image.shape}')\nplt.imshow(image,  interpolation_stage='rgb')\nplt.show()\n\n","metadata":{"collapsed":false,"pycharm":{"name":"#%%\n"},"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-07-31T23:24:48.674231Z","iopub.execute_input":"2022-07-31T23:24:48.674688Z","iopub.status.idle":"2022-07-31T23:24:48.983876Z","shell.execute_reply.started":"2022-07-31T23:24:48.674647Z","shell.execute_reply":"2022-07-31T23:24:48.982481Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"All images are 224x224 pixel RGB photos.","metadata":{"pycharm":{"name":"#%% md\n"}}},{"cell_type":"markdown","source":"### What do the images look like?","metadata":{"pycharm":{"name":"#%% md\n"}}},{"cell_type":"code","source":"# Run or re-run this to start at the first image\nj = 0\n","metadata":{"collapsed":false,"pycharm":{"name":"#%%\n"},"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-07-31T23:24:48.985416Z","iopub.execute_input":"2022-07-31T23:24:48.985942Z","iopub.status.idle":"2022-07-31T23:24:48.990482Z","shell.execute_reply.started":"2022-07-31T23:24:48.985910Z","shell.execute_reply":"2022-07-31T23:24:48.989299Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Run this cell over and over to step through the images\n\nrows = 5\ncols = 5\n\nplt.style.use(\"default\")\nfig, axs = plt.subplots(nrows=rows, ncols=cols, figsize=(13, 13))\n\nplt.suptitle(\"Training Dataset of Butterfly Mimics\", fontsize=20)\n\ndata_length = len(train_csv_data)\nmaxloop = data_length//(rows*cols)\n\nif j <= maxloop:\n    for i, ax in enumerate(axs.flat):\n        k = (j*rows*cols) + i\n\n        if k < data_length:\n            file = os.path.join(train_dir, train_csv_data.image[k]+\".jpg\")\n            image = io.imread(file, as_gray=False)\n\n            ax.grid(None)\n            ax.axis('on')\n\n            ax.imshow(image, interpolation_stage='rgb')\n            ax.set(xticks=[], yticks=[], xlabel = train_csv_data.name[k])\n        else:\n            ax.set_visible(False)\n\n    fig.text(.83, .965, f\"page {j+1}\", color='grey', fontsize=14)\n\n    j = j + 1 if j < maxloop else 0\n    plt.show()","metadata":{"collapsed":false,"pycharm":{"name":"#%%\n"},"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-07-31T23:24:48.995457Z","iopub.execute_input":"2022-07-31T23:24:48.996698Z","iopub.status.idle":"2022-07-31T23:24:51.295428Z","shell.execute_reply.started":"2022-07-31T23:24:48.996630Z","shell.execute_reply":"2022-07-31T23:24:51.294089Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Do we have any missing data?","metadata":{"pycharm":{"name":"#%% md\n"}}},{"cell_type":"code","source":"print(\"Missing data from the images.csv file:\")\nprint(\"--------------------------------------\")\ntrain_csv_data.isna().sum()","metadata":{"collapsed":false,"pycharm":{"name":"#%%\n"},"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-07-31T23:24:51.296857Z","iopub.execute_input":"2022-07-31T23:24:51.298019Z","iopub.status.idle":"2022-07-31T23:24:51.310331Z","shell.execute_reply.started":"2022-07-31T23:24:51.297985Z","shell.execute_reply":"2022-07-31T23:24:51.308794Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"---\n### Explore Test Dataset\n---\n","metadata":{"pycharm":{"name":"#%% md\n"}}},{"cell_type":"markdown","source":"### What do the test csv files look like?","metadata":{"pycharm":{"name":"#%% md\n"}}},{"cell_type":"code","source":"print(\"From the image_holdouts.csv file:\")\nprint(\"---------------------------------\")\n\ntest_csv_data.head()","metadata":{"pycharm":{"name":"#%%\n"},"execution":{"iopub.status.busy":"2022-07-31T23:24:51.311993Z","iopub.execute_input":"2022-07-31T23:24:51.312499Z","iopub.status.idle":"2022-07-31T23:24:51.326074Z","shell.execute_reply.started":"2022-07-31T23:24:51.312451Z","shell.execute_reply":"2022-07-31T23:24:51.325102Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### What do the test images look like?","metadata":{"pycharm":{"name":"#%% md\n"}}},{"cell_type":"code","source":"# Run and then re-run this cell to start with the first image\nj = 0","metadata":{"pycharm":{"name":"#%%\n"},"execution":{"iopub.status.busy":"2022-07-31T23:24:51.327723Z","iopub.execute_input":"2022-07-31T23:24:51.328389Z","iopub.status.idle":"2022-07-31T23:24:51.335162Z","shell.execute_reply.started":"2022-07-31T23:24:51.328348Z","shell.execute_reply":"2022-07-31T23:24:51.334124Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nfig, ax = plt.subplots(5, 5, figsize=(15, 15))\n\nmaxloop = int(len(test_csv_data)/25)+1\n\nif j < maxloop:\n    for i, axi in enumerate(ax.flat):\n        k = (j*25) + i\n\n        if k < len(test_csv_data):\n            file = os.path.join(test_dir, test_csv_data.image[k]+\".jpg\")\n            image = io.imread(file, as_gray=False)\n\n            axi.grid(None)\n            axi.axis('on')\n\n            axi.imshow(image, interpolation_stage='rgb')\n            axi.set(xticks=[], yticks=[], xlabel = test_csv_data.image[k])\n        else:\n            axi.set_visible(False)\n\n    j = j + 1\n    plt.show()","metadata":{"pycharm":{"name":"#%%\n"},"execution":{"iopub.status.busy":"2022-07-31T23:24:51.336832Z","iopub.execute_input":"2022-07-31T23:24:51.337527Z","iopub.status.idle":"2022-07-31T23:24:53.557161Z","shell.execute_reply.started":"2022-07-31T23:24:51.337485Z","shell.execute_reply":"2022-07-31T23:24:53.555771Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"---\n### Explore Submissions\n---","metadata":{"pycharm":{"name":"#%% md\n"}}},{"cell_type":"markdown","source":"### What should a submission look like?","metadata":{"pycharm":{"name":"#%% md\n"}}},{"cell_type":"code","source":"print(\"Example Submission\")\nprint(\"------------------\")\nsubmit_ds = pd.read_csv(sample_submit_csv)\nsubmit_ds.head()","metadata":{"pycharm":{"name":"#%%\n"},"execution":{"iopub.status.busy":"2022-07-31T23:24:53.558710Z","iopub.execute_input":"2022-07-31T23:24:53.559277Z","iopub.status.idle":"2022-07-31T23:24:53.577360Z","shell.execute_reply.started":"2022-07-31T23:24:53.559243Z","shell.execute_reply":"2022-07-31T23:24:53.576473Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### &nbsp;\n### That's it. Everything looks good!","metadata":{"pycharm":{"name":"#%% md\n"}}},{"cell_type":"code","source":"","metadata":{"pycharm":{"name":"#%%\n"}},"execution_count":null,"outputs":[]}]}