{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# # This Python 3 environment comes with many helpful analytics libraries installed\n# # It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# # For example, here's several helpful packages to load\n\n# import numpy as np # linear algebra\n# import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# # Input data files are available in the read-only \"../input/\" directory\n# # For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\n# import os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\n# # You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# # You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport random\nimport pandas as pd\nfrom skimage import io\nfrom joblib import Parallel, delayed\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.model_selection import StratifiedKFold, GroupKFold\n\nimport numpy as np","metadata":{"execution":{"iopub.status.busy":"2022-02-27T14:57:41.869859Z","iopub.execute_input":"2022-02-27T14:57:41.870319Z","iopub.status.idle":"2022-02-27T14:57:43.23372Z","shell.execute_reply.started":"2022-02-27T14:57:41.870272Z","shell.execute_reply":"2022-02-27T14:57:43.233084Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def seed_all(seed_value):\n    random.seed(seed_value) # Python\n    np.random.seed(seed_value) # cpu vars\n#     torch.manual_seed(seed_value) # cpu  vars    \n#     if torch.cuda.is_available(): \n#         torch.cuda.manual_seed(seed_value)\n#         torch.cuda.manual_seed_all(seed_value) # gpu vars\n#     if torch.backends.cudnn.is_available:\n#         torch.backends.cudnn.deterministic = True\n#         torch.backends.cudnn.benchmark = False\n    print('# SEEDING DONE')\nseed_all(42)","metadata":{"execution":{"iopub.status.busy":"2022-02-27T14:57:43.235751Z","iopub.execute_input":"2022-02-27T14:57:43.236219Z","iopub.status.idle":"2022-02-27T14:57:43.24322Z","shell.execute_reply.started":"2022-02-27T14:57:43.236175Z","shell.execute_reply":"2022-02-27T14:57:43.242557Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"base_path = '/kaggle/input/happy-whale-and-dolphin'","metadata":{"execution":{"iopub.status.busy":"2022-02-27T14:57:43.244657Z","iopub.execute_input":"2022-02-27T14:57:43.245026Z","iopub.status.idle":"2022-02-27T14:57:43.253846Z","shell.execute_reply.started":"2022-02-27T14:57:43.244992Z","shell.execute_reply":"2022-02-27T14:57:43.253045Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv(f'{base_path}/train.csv')\n# df['image_path'] = base_path+'/train_images/'+df['image']\n# df['split'] = 'Train'\n\ntest_df = pd.read_csv(f'{base_path}/sample_submission.csv')\n# test_df['image_path'] = base_path+'/test_images/'+test_df['image']\n# test_df['split'] = 'Test'\n\nprint('Train Images: {:,} | Test Images: {:,}'.format(len(df), len(test_df)))","metadata":{"execution":{"iopub.status.busy":"2022-02-27T14:57:43.255754Z","iopub.execute_input":"2022-02-27T14:57:43.256095Z","iopub.status.idle":"2022-02-27T14:57:43.411774Z","shell.execute_reply.started":"2022-02-27T14:57:43.256063Z","shell.execute_reply":"2022-02-27T14:57:43.410914Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head()","metadata":{"execution":{"iopub.status.busy":"2022-02-27T14:57:43.413221Z","iopub.execute_input":"2022-02-27T14:57:43.413729Z","iopub.status.idle":"2022-02-27T14:57:43.433385Z","shell.execute_reply.started":"2022-02-27T14:57:43.413685Z","shell.execute_reply":"2022-02-27T14:57:43.432506Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.species.unique()","metadata":{"execution":{"iopub.status.busy":"2022-02-27T14:57:43.436406Z","iopub.execute_input":"2022-02-27T14:57:43.436646Z","iopub.status.idle":"2022-02-27T14:57:43.453841Z","shell.execute_reply.started":"2022-02-27T14:57:43.436618Z","shell.execute_reply":"2022-02-27T14:57:43.453232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(list(df.individual_id.unique()))","metadata":{"execution":{"iopub.status.busy":"2022-02-27T14:57:43.455493Z","iopub.execute_input":"2022-02-27T14:57:43.456081Z","iopub.status.idle":"2022-02-27T14:57:43.469083Z","shell.execute_reply.started":"2022-02-27T14:57:43.456029Z","shell.execute_reply":"2022-02-27T14:57:43.46837Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n_classes_ori = len(list(df.species.unique()))","metadata":{"execution":{"iopub.status.busy":"2022-02-27T14:57:43.47009Z","iopub.execute_input":"2022-02-27T14:57:43.470301Z","iopub.status.idle":"2022-02-27T14:57:43.484936Z","shell.execute_reply.started":"2022-02-27T14:57:43.470275Z","shell.execute_reply":"2022-02-27T14:57:43.484004Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Fix Meta Data\nFolowing cells,\n* Converts `beluga`, `globis` to `whales` for 2class label.\n* Fixes Duplicate Labels.","metadata":{}},{"cell_type":"code","source":"# convert beluga, globis to whales\ndf.loc[df.species.str.contains('beluga'), 'species'] = 'beluga_whale'\ndf.loc[df.species.str.contains('globis'), 'species'] = 'short_finned_pilot_whale'\ndf.loc[df.species.str.contains('pilot_whale'), 'species'] = 'short_finned_pilot_whale'\ndf['class'] = df.species.map(lambda x: 'whale' if 'whale' in x else 'dolphin')\n\n# fix duplicate labels\n# https://www.kaggle.com/c/happy-whale-and-dolphin/discussion/304633\ndf['species'] = df['species'].str.replace('bottlenose_dolpin','bottlenose_dolphin')\ndf['species'] = df['species'].str.replace('kiler_whale','killer_whale')","metadata":{"execution":{"iopub.status.busy":"2022-02-27T14:57:43.486535Z","iopub.execute_input":"2022-02-27T14:57:43.486826Z","iopub.status.idle":"2022-02-27T14:57:43.701744Z","shell.execute_reply.started":"2022-02-27T14:57:43.48679Z","shell.execute_reply":"2022-02-27T14:57:43.701137Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head()","metadata":{"execution":{"iopub.status.busy":"2022-02-27T14:57:43.703675Z","iopub.execute_input":"2022-02-27T14:57:43.704356Z","iopub.status.idle":"2022-02-27T14:57:43.715821Z","shell.execute_reply.started":"2022-02-27T14:57:43.704318Z","shell.execute_reply":"2022-02-27T14:57:43.715119Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n_classes = len(list(df.species.unique()))","metadata":{"execution":{"iopub.status.busy":"2022-02-27T14:57:43.71889Z","iopub.execute_input":"2022-02-27T14:57:43.719202Z","iopub.status.idle":"2022-02-27T14:57:43.727407Z","shell.execute_reply.started":"2022-02-27T14:57:43.71917Z","shell.execute_reply":"2022-02-27T14:57:43.726589Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f'Num original classes: {n_classes_ori}')\nprint(f'Num classes after fix: {n_classes}')","metadata":{"execution":{"iopub.status.busy":"2022-02-27T14:57:43.729485Z","iopub.execute_input":"2022-02-27T14:57:43.729696Z","iopub.status.idle":"2022-02-27T14:57:43.740582Z","shell.execute_reply.started":"2022-02-27T14:57:43.729671Z","shell.execute_reply":"2022-02-27T14:57:43.739794Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Encode labels","metadata":{}},{"cell_type":"code","source":"classes_encoder = LabelEncoder()\ndf['class_encode'] = classes_encoder.fit_transform(df['class'])\nclasses_list = list(df['class'].unique())\nclasses_encode_list = classes_encoder.transform(classes_list)\nclasses_unsorted_dict = {encode : name for encode, name in zip(classes_encode_list, classes_list)}\nclasses_dict = {}\nfor key, value in sorted(classes_unsorted_dict.items()):\n    classes_dict[int(key)] = value\n\nspecies_encoder = LabelEncoder()\ndf['species_encode'] = species_encoder.fit_transform(df['species'])\nspecies_list = list(df['species'].unique())\nspecies_encode_list = species_encoder.transform(species_list)\nspecies_unsorted_dict = {encode : name for encode, name in zip(species_encode_list, species_list)}\nspecies_dict = {}\nfor key, value in sorted(species_unsorted_dict.items()):\n    species_dict[int(key)] = value\n\nindividual_encoder = LabelEncoder()\ndf['individual_id_encode'] = individual_encoder.fit_transform(df['individual_id'])\nindividual_list = list(df['individual_id'].unique())\nindividual_encode_list = individual_encoder.transform(individual_list)\nindividual_unsorted_dict = {encode : name for encode, name in zip(individual_encode_list, individual_list)}\nindividual_dict = {}\nfor key, value in sorted(individual_unsorted_dict.items()):\n    individual_dict[int(key)] = value","metadata":{"execution":{"iopub.status.busy":"2022-02-27T14:57:43.741503Z","iopub.execute_input":"2022-02-27T14:57:43.74188Z","iopub.status.idle":"2022-02-27T14:57:46.438333Z","shell.execute_reply.started":"2022-02-27T14:57:43.741851Z","shell.execute_reply":"2022-02-27T14:57:46.437206Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head()","metadata":{"execution":{"iopub.status.busy":"2022-02-27T14:57:46.439731Z","iopub.execute_input":"2022-02-27T14:57:46.439968Z","iopub.status.idle":"2022-02-27T14:57:46.451824Z","shell.execute_reply.started":"2022-02-27T14:57:46.439937Z","shell.execute_reply":"2022-02-27T14:57:46.451198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import json\n\ndef save_json(file, data):\n    assert file.split('.')[-1] == 'json'\n    with open(file, 'w') as f:\n        json.dump(data, f)\n        \nsave_json('/kaggle/working/classes.json', classes_dict)\nsave_json('/kaggle/working/species.json', species_dict)\nsave_json('/kaggle/working/individual_ids.json', individual_dict)","metadata":{"execution":{"iopub.status.busy":"2022-02-27T14:57:46.452727Z","iopub.execute_input":"2022-02-27T14:57:46.453241Z","iopub.status.idle":"2022-02-27T14:57:46.50414Z","shell.execute_reply.started":"2022-02-27T14:57:46.453211Z","shell.execute_reply":"2022-02-27T14:57:46.503495Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Get shape information","metadata":{}},{"cell_type":"code","source":"%%time\ndef train_process(i):\n    im = io.imread(f'../input/happy-whale-and-dolphin/train_images/' + df.iloc[i].image)\n    shape = list(im.shape)\n    return shape if len(shape) == 3 else shape + [1]\n    \ndf['shape'] = Parallel(n_jobs=4)(delayed(train_process)(i) for i in range(len(df)))\ndf[['d0', 'd1', 'd2']] = pd.DataFrame(df['shape'].to_list())\ndf.drop(columns='shape', inplace=True)\ndf.to_csv('/kaggle/working/train_finetune.csv')","metadata":{"execution":{"iopub.status.busy":"2022-02-27T14:59:35.155929Z","iopub.execute_input":"2022-02-27T14:59:35.156627Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ndef test_process(i):\n    im = io.imread('../input/happy-whale-and-dolphin/test_images/' + test_df.iloc[i].image)\n    shape = list(im.shape)\n    return shape if len(shape) == 3 else shape + [1]\n    \ntest_df['shape'] = Parallel(n_jobs=4)(delayed(test_process)(i) for i in range(len(test_df)))\ntest_df[['d0', 'd1', 'd2']] = pd.DataFrame(test_df['shape'].to_list())\ntest_df.drop(columns='shape', inplace=True)\ntest_df.to_csv('/kaggle/working/test_finetune.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Split 5 & 10 folds","metadata":{}},{"cell_type":"code","source":"df_class_5f = df.copy()\nskf = StratifiedKFold(n_splits=5)\n\nfor fold, ( _, val_) in enumerate(skf.split(X=df_class_5f, y=df_class_5f.class_encode)):\n    df_class_5f.loc[val_ , \"fold\"] = fold\n    \ndf_class_5f.to_csv('/kaggle/working/train_class_5fold.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_class_10f = df.copy()\nskf = StratifiedKFold(n_splits=10)\n\nfor fold, ( _, val_) in enumerate(skf.split(X=df_class_10f, y=df_class_10f.class_encode)):\n    df_class_10f.loc[val_ , \"fold\"] = fold\n    \ndf_class_10f.to_csv('/kaggle/working/train_class_10fold.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_class_10f.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_species_5f = df.copy()\nskf = StratifiedKFold(n_splits=5)\n\nfor fold, ( _, val_) in enumerate(skf.split(X=df_species_5f, y=df_species_5f.species_encode)):\n    df_species_5f.loc[val_ , \"fold\"] = fold\n    \ndf_species_5f.to_csv('/kaggle/working/train_species_5fold.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_species_10f = df.copy()\nskf = StratifiedKFold(n_splits=10)\n\nfor fold, ( _, val_) in enumerate(skf.split(X=df_species_10f, y=df_species_10f.species_encode)):\n    df_species_10f.loc[val_ , \"fold\"] = fold\n    \ndf_species_10f.to_csv('/kaggle/working/train_species_10fold.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_species_10f.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_individual_5f = df.copy()\nskf = StratifiedKFold(n_splits=5)\n\nfor fold, ( _, val_) in enumerate(skf.split(X=df_individual_5f, y=df_individual_5f.individual_id_encode)):\n    df_individual_5f.loc[val_ , \"fold\"] = fold\n    \ndf_individual_5f.to_csv('/kaggle/working/train_individual_5fold.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_individual_10f = df.copy()\nskf = StratifiedKFold(n_splits=10)\n\nfor fold, ( _, val_) in enumerate(skf.split(X=df_individual_10f, y=df_individual_10f.individual_id_encode)):\n    df_individual_10f.loc[val_ , \"fold\"] = fold\n    \ndf_individual_10f.to_csv('/kaggle/working/train_individual_10fold.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_individual_10f.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}