{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Importing Modules","metadata":{}},{"cell_type":"code","source":"# Standard imports\nimport os\nimport pandas as pd\nimport numpy as np\nimport plotly.express as px\nimport plotly.graph_objs as go\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom tqdm import trange\nfrom colorama import Fore\nfrom glob import glob\nimport json\nfrom pprint import pprint\nimport time\nimport cv2\nfrom enum import Enum\nfrom IPython.display import display\nimport random\nimport inspect\n\n# For Data preparation\nfrom sklearn.preprocessing import *\nfrom sklearn.model_selection import *\nfrom sklearn.metrics import *\n\n# Regression Models\nfrom sklearn.linear_model import LinearRegression, Ridge, ElasticNet\n\nfrom sklearn.tree import DecisionTreeRegressor\nfrom sklearn.ensemble import ExtraTreesRegressor, RandomForestRegressor, VotingRegressor\nfrom sklearn.ensemble import AdaBoostRegressor, GradientBoostingRegressor, StackingRegressor\nfrom sklearn.neighbors import KNeighborsRegressor\n\nfrom catboost import CatBoostRegressor\nfrom xgboost import XGBRegressor\nfrom lightgbm import LGBMRegressor\n\n# For building models\nimport torch\nfrom torch.utils.data import DataLoader, Dataset\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torchvision import datasets, transforms, models\n\n# Tensorflow modules\nimport tensorflow as tf\nfrom tensorflow.keras.applications import *\nfrom tensorflow.keras.layers import *\nfrom tensorflow.keras.optimizers import *\nfrom tensorflow.keras.losses import *\nfrom tensorflow.keras.callbacks import *\nfrom tensorflow.keras.metrics import *\nfrom tensorflow.keras.utils import plot_model\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.models import *\n\n\n# For Transformer\nimport transformers\nfrom transformers import AutoTokenizer, BertModel\nfrom transformers import AdamW\nfrom transformers import get_linear_schedule_with_warmup\n\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n# To ignore tensorflow warnings\nos.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'\n\n\nprint(\n    f\"GPU is available : {tf.test.is_gpu_available(cuda_only=False, min_cuda_compute_capability=None)}\")","metadata":{"execution":{"iopub.status.busy":"2022-02-13T04:27:23.332156Z","iopub.execute_input":"2022-02-13T04:27:23.332920Z","iopub.status.idle":"2022-02-13T04:27:35.774789Z","shell.execute_reply.started":"2022-02-13T04:27:23.332831Z","shell.execute_reply":"2022-02-13T04:27:35.773652Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Config","metadata":{}},{"cell_type":"code","source":"class Config(Enum):\n    '''\n    It basically contains all the path location and other stuffs\n    '''\n\n    def __str__(self):\n        return self.value\n\n    TRAIN_CSV = \"../input/happy-whale-and-dolphin/train.csv\"\n    TEST_CSV = \"../input/happy-whale-and-dolphin/sample_submission.csv\"\n    TRAIN_DIR = \"../input/happy-whale-and-dolphin/train_images\"\n    TEST_DIR = \"../input/happy-whale-and-dolphin/test_images\"\n\n\ndef setSeed(seed):\n    \"\"\"\n    Setting the seed of all the random function to maintain reproducibility\n    \"\"\"\n    np.random.seed(seed)\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    os.environ['TF_CUDNN_DETERMINISTIC'] = str(seed)\n    tf.random.set_seed(seed)\n    print('SEEDITIZATION DONE !')\n\nsetSeed(42)","metadata":{"execution":{"iopub.status.busy":"2022-02-13T04:28:33.186285Z","iopub.execute_input":"2022-02-13T04:28:33.186521Z","iopub.status.idle":"2022-02-13T04:28:33.195073Z","shell.execute_reply.started":"2022-02-13T04:28:33.186497Z","shell.execute_reply":"2022-02-13T04:28:33.193745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Helper functions","metadata":{}},{"cell_type":"code","source":"def giveHistogram(df: \"data File\", col_name: str, bins=None, dark=False):\n    \"\"\"\n    To create histogram plots\n    \"\"\"\n    fig = px.histogram(df, x=col_name, color = col_name, template=\"plotly_dark\" if dark else \"ggplot2\",\n                       nbins=bins if bins != None else 1 + int(np.log2(len(df))))\n    fig.update_layout(\n        title_text=f\"Distribution of {col_name}\",\n        title_x=0.5,\n    )\n    fig.update_xaxes(categoryorder = 'total descending')\n    fig.show()\n\n\ndef widthAndHeightDist(df: \"data_file\", col_name: \"col name that contains the img path\", dark=False):\n    \"\"\"\n    Give Histogram distribution of image width and height\n    \"\"\"\n    widths = []\n    heights = []\n    bins = 1 + int(np.log2(len(df)))\n    total_images = list(df[col_name].values)\n    for idx in trange(len(total_images), desc=\"Collecting widths and heights...\", bar_format=\"{l_bar}%s{bar:50}%s{r_bar}\" % (Fore.CYAN, Fore.RESET), position=0, leave=True):\n        cur_path = total_images[idx]\n        h, w, _ = cv2.imread(cur_path).shape\n        widths.append(w)\n        heights.append(h)\n\n    figW = px.histogram(widths, nbins=bins,\n                        template=\"plotly_dark\" if dark else \"ggplot2\")\n    figW.update_layout(title='Distribution of Image Widths', title_x=0.5)\n    figW.show()\n\n    figH = px.histogram(heights, nbins=bins,\n                        template=\"plotly_dark\" if dark else \"ggplot2\")\n    figH.update_layout(title='Distribution of Image Heights', title_x=0.5)\n    figH.show()\n\n\ndef buildGridImages(df: \"data_file\", img_path_col_name: str, label_col_name: str, nrows=5, ncols=4, img_size=512):\n    \"\"\"\n    To build an image grid\n    \"\"\"\n\n    df = df.sample(nrows*ncols)\n    paths = df[img_path_col_name].values\n    labels = df[label_col_name].values\n\n    text_color = (255, 255, 255)\n    box_color = (0, 0, 0)\n\n    plt.figure(figsize=(20, 12))\n    for i in range(nrows * ncols):\n        plt.subplot(nrows, ncols, i+1)\n        img = cv2.imread(paths[i])\n        img = cv2.resize(img, (img_size, img_size))\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n\n        plt.axis(\"off\")\n        plt.title(str(labels[i]))\n        plt.imshow(img)\n\n    plt.tight_layout()\n    plt.show()\n\n\ndef create_folds(data, target=\"label\", regression=True, num_splits=5):\n    \"\"\"\n    Helper function to create folds\n    \"\"\"\n    data[\"kfold\"] = -1\n    data = data.sample(frac=1).reset_index(drop=True)\n    kf = StratifiedKFold(n_splits=num_splits)\n\n    if regression:\n        # Applying Sturg's rule to calculate the no. of bins for target\n        num_bins = int(1 + np.log2(len(data)))\n\n        data.loc[:, \"bins\"] = pd.cut(data[target], bins=num_bins, labels=False)\n        for f, (t_, v_) in enumerate(kf.split(X=data, y=data.bins.values)):\n            data.loc[v_, 'kfold'] = f\n        data = data.drop([\"bins\"], axis=1)\n    else:\n        for f, (t_, v_) in enumerate(kf.split(X=data, y=data[target].values)):\n            data.loc[v_, 'kfold'] = f\n\n    return ","metadata":{"execution":{"iopub.status.busy":"2022-02-13T04:56:08.231901Z","iopub.execute_input":"2022-02-13T04:56:08.232159Z","iopub.status.idle":"2022-02-13T04:56:08.260188Z","shell.execute_reply.started":"2022-02-13T04:56:08.232131Z","shell.execute_reply":"2022-02-13T04:56:08.259105Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Looking inside the data","metadata":{}},{"cell_type":"code","source":"data_df = pd.read_csv(Config.TRAIN_CSV.value)\ntest_df = pd.read_csv(Config.TEST_CSV.value)\n\ndata_df['path'] = data_df['image'].apply(lambda x: f\"{Config.TRAIN_DIR.value}/{x}\")\ntest_df['path'] = test_df['image'].apply(lambda x: f\"{Config.TEST_DIR.value}/{x}\")\n\ndata_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-02-13T04:31:08.933408Z","iopub.execute_input":"2022-02-13T04:31:08.934401Z","iopub.status.idle":"2022-02-13T04:31:09.174982Z","shell.execute_reply.started":"2022-02-13T04:31:08.934336Z","shell.execute_reply":"2022-02-13T04:31:09.174154Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Extracting whale or dolphin\ndata_df['animal_kind'] = data_df['species'].apply(lambda x: x.split(\"_\")[-1])\ndata_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-02-13T04:34:34.060716Z","iopub.execute_input":"2022-02-13T04:34:34.061133Z","iopub.status.idle":"2022-02-13T04:34:34.091136Z","shell.execute_reply.started":"2022-02-13T04:34:34.061100Z","shell.execute_reply":"2022-02-13T04:34:34.090340Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-02-13T04:32:46.717762Z","iopub.execute_input":"2022-02-13T04:32:46.718068Z","iopub.status.idle":"2022-02-13T04:32:46.727635Z","shell.execute_reply.started":"2022-02-13T04:32:46.718039Z","shell.execute_reply":"2022-02-13T04:32:46.726765Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Distribution of whale and dolphin in the dataset","metadata":{}},{"cell_type":"code","source":"giveHistogram(df = data_df, col_name = 'animal_kind', bins=None, dark=1)","metadata":{"execution":{"iopub.status.busy":"2022-02-13T04:56:12.543787Z","iopub.execute_input":"2022-02-13T04:56:12.544208Z","iopub.status.idle":"2022-02-13T04:56:13.622391Z","shell.execute_reply.started":"2022-02-13T04:56:12.544167Z","shell.execute_reply":"2022-02-13T04:56:13.621480Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Distribution of the species","metadata":{}},{"cell_type":"code","source":"giveHistogram(df = data_df, col_name = 'species', bins=None, dark=1)","metadata":{"execution":{"iopub.status.busy":"2022-02-13T04:56:18.540229Z","iopub.execute_input":"2022-02-13T04:56:18.540666Z","iopub.status.idle":"2022-02-13T04:56:19.252211Z","shell.execute_reply.started":"2022-02-13T04:56:18.540628Z","shell.execute_reply":"2022-02-13T04:56:19.251381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Lets look at some of the images","metadata":{}},{"cell_type":"code","source":"whales = data_df.loc[data_df['animal_kind'] == 'whale']\ndolphin = data_df.loc[data_df['animal_kind'] == 'dolphin']\nbeluga = data_df.loc[data_df['animal_kind'] == 'beluga']\ndolpin = data_df.loc[data_df['animal_kind'] == 'dolpin']\nglobis = data_df.loc[data_df['animal_kind'] == 'globis']","metadata":{"execution":{"iopub.status.busy":"2022-02-13T04:42:18.597111Z","iopub.execute_input":"2022-02-13T04:42:18.598230Z","iopub.status.idle":"2022-02-13T04:42:18.637883Z","shell.execute_reply.started":"2022-02-13T04:42:18.598173Z","shell.execute_reply":"2022-02-13T04:42:18.636331Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Whale 🐳","metadata":{}},{"cell_type":"code","source":"buildGridImages(df = whales, img_path_col_name = 'path', label_col_name = 'species', nrows=5, ncols=4, img_size=512)","metadata":{"execution":{"iopub.status.busy":"2022-02-13T04:42:57.712528Z","iopub.execute_input":"2022-02-13T04:42:57.712796Z","iopub.status.idle":"2022-02-13T04:43:00.853563Z","shell.execute_reply.started":"2022-02-13T04:42:57.712746Z","shell.execute_reply":"2022-02-13T04:43:00.852071Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Dolphin 🐬","metadata":{}},{"cell_type":"code","source":"buildGridImages(df = dolphin, img_path_col_name = 'path', label_col_name = 'species', nrows=5, ncols=4, img_size=512)","metadata":{"execution":{"iopub.status.busy":"2022-02-13T04:43:57.122415Z","iopub.execute_input":"2022-02-13T04:43:57.122701Z","iopub.status.idle":"2022-02-13T04:44:00.626237Z","shell.execute_reply.started":"2022-02-13T04:43:57.122673Z","shell.execute_reply":"2022-02-13T04:44:00.625312Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Beluga","metadata":{}},{"cell_type":"code","source":"buildGridImages(df = beluga, img_path_col_name = 'path', label_col_name = 'species', nrows=5, ncols=4, img_size=512)","metadata":{"execution":{"iopub.status.busy":"2022-02-13T04:44:26.839170Z","iopub.execute_input":"2022-02-13T04:44:26.839611Z","iopub.status.idle":"2022-02-13T04:44:29.042802Z","shell.execute_reply.started":"2022-02-13T04:44:26.839574Z","shell.execute_reply":"2022-02-13T04:44:29.042227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Dolpin 🐬❓","metadata":{}},{"cell_type":"code","source":"buildGridImages(df = dolpin, img_path_col_name = 'path', label_col_name = 'species', nrows=5, ncols=4, img_size=512)","metadata":{"execution":{"iopub.status.busy":"2022-02-13T04:45:17.190282Z","iopub.execute_input":"2022-02-13T04:45:17.190726Z","iopub.status.idle":"2022-02-13T04:45:20.049350Z","shell.execute_reply.started":"2022-02-13T04:45:17.190691Z","shell.execute_reply":"2022-02-13T04:45:20.048391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### globis","metadata":{}},{"cell_type":"code","source":"buildGridImages(df = globis, img_path_col_name = 'path', label_col_name = 'species', nrows=5, ncols=4, img_size=512)","metadata":{"execution":{"iopub.status.busy":"2022-02-13T04:45:48.348859Z","iopub.execute_input":"2022-02-13T04:45:48.349100Z","iopub.status.idle":"2022-02-13T04:45:50.971082Z","shell.execute_reply.started":"2022-02-13T04:45:48.349074Z","shell.execute_reply":"2022-02-13T04:45:50.969725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##### *OMG 😝 Spelling mistake dolphin -> dolpin 🤣😂*","metadata":{}},{"cell_type":"markdown","source":"### Distribution of Widths and Heights","metadata":{}},{"cell_type":"code","source":"widthAndHeightDist(df = data_df, col_name = 'path', dark=1)","metadata":{},"execution_count":null,"outputs":[]}]}