{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"### TL;DR:\nThis is the train example for test species dataset, it is not full train, only 1 epoch for 1 fold because kaggle limitations.\nMy predictors trained 5 epoch and 5 folds, also batch size=32. Average val accuracy 98.6%, my train log:\n\n```\nFold 0 resutls\nepoch     train_loss  valid_loss  accuracy  time\n0         0.767579    0.238040    0.937200  11:17\n1         0.634035    0.160043    0.960419  11:16\n2         0.505939    0.099926    0.974919  11:18\n3         0.440220    0.063862    0.985108  11:20\n4         0.424336    0.055490    0.987362  11:21\n\nFold 1 results\nepoch     train_loss  valid_loss  accuracy  time\n0         0.749576    0.311206    0.919075  13:05\n1         0.626596    0.168583    0.956794  19:06\n2         0.527211    0.093829    0.976389  18:50\n3         0.434782    0.061494    0.985108  16:42\n4         0.426031    0.058528    0.985990  12:47\n\nFold 2 results\nepoch     train_loss  valid_loss  accuracy  time\n0         0.781070    0.232682    0.937690  11:22\n1         0.622824    0.142476    0.964730  12:09\n2         0.518737    0.078674    0.978642  16:59\n3         0.462086    0.062647    0.984129  11:21\n4         0.423454    0.055090    0.986382  12:28\n\nFold 3 results\nepoch     train_loss  valid_loss  accuracy  time\n0         0.760659    0.247128    0.929061  17:27\n1         0.630934    0.138731    0.966392  19:21\n2         0.539373    0.091751    0.978248  19:05\n3         0.445625    0.061304    0.985009  19:03\n4         0.420296    0.055081    0.987164  18:49\n\nFold 4 results\nepoch     train_loss  valid_loss  accuracy  time\n0         0.769067    0.217799    0.941211  11:38\n1         0.644752    0.174513    0.953655  17:56\n2         0.508525    0.090482    0.975407  18:48\n3         0.453215    0.067261    0.985009  19:09\n4         0.426319    0.062320    0.985205  19:12\n```\n","metadata":{}},{"cell_type":"code","source":"!pip install timm","metadata":{"execution":{"iopub.status.busy":"2022-02-05T21:41:41.766124Z","iopub.execute_input":"2022-02-05T21:41:41.766406Z","iopub.status.idle":"2022-02-05T21:41:51.443618Z","shell.execute_reply.started":"2022-02-05T21:41:41.766372Z","shell.execute_reply":"2022-02-05T21:41:51.442819Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport sys\nimport gc\n\nimport numpy as np \nimport pandas as pd \nfrom timm import create_model\nfrom fastai.vision.all import *","metadata":{"execution":{"iopub.status.busy":"2022-02-05T21:41:51.445701Z","iopub.execute_input":"2022-02-05T21:41:51.44596Z","iopub.status.idle":"2022-02-05T21:41:54.761391Z","shell.execute_reply.started":"2022-02-05T21:41:51.445923Z","shell.execute_reply":"2022-02-05T21:41:54.760528Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"root_dir = '../input/happy-whale-and-dolphin/'\ntrain_dir = os.path.join(root_dir, 'train_images')\ntest_dir = os.path.join(root_dir, 'test_images')\ntrain_csv = os.path.join(root_dir, 'train.csv')\ntest_csv = os.path.join(root_dir, 'sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2022-02-05T21:41:54.76497Z","iopub.execute_input":"2022-02-05T21:41:54.765212Z","iopub.status.idle":"2022-02-05T21:41:54.769935Z","shell.execute_reply.started":"2022-02-05T21:41:54.765182Z","shell.execute_reply":"2022-02-05T21:41:54.769215Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Read data and fix errors in species based on discussions:","metadata":{}},{"cell_type":"code","source":"test_df = pd.read_csv(test_csv)\ntest_df['path'] = test_df.image.apply(lambda x: os.path.join(test_dir, x))","metadata":{"execution":{"iopub.status.busy":"2022-02-05T21:41:54.77204Z","iopub.execute_input":"2022-02-05T21:41:54.7725Z","iopub.status.idle":"2022-02-05T21:41:54.897199Z","shell.execute_reply.started":"2022-02-05T21:41:54.772457Z","shell.execute_reply":"2022-02-05T21:41:54.896455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"seed = 0xDEAD\nBATCH_SIZE=16","metadata":{"execution":{"iopub.status.busy":"2022-02-05T21:41:54.898429Z","iopub.execute_input":"2022-02-05T21:41:54.898684Z","iopub.status.idle":"2022-02-05T21:41:54.902993Z","shell.execute_reply.started":"2022-02-05T21:41:54.89865Z","shell.execute_reply":"2022-02-05T21:41:54.902178Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv(train_csv)\ntrain_df.species.replace({\"globis\": \"short_finned_pilot_whale\",\n                          \"pilot_whale\": \"short_finned_pilot_whale\",\n                          \"kiler_whale\": \"killer_whale\",\n                          \"bottlenose_dolpin\": \"bottlenose_dolphin\"}, inplace=True)\nsorted(train_df.species.unique()), len(train_df.species.unique())","metadata":{"execution":{"iopub.status.busy":"2022-02-05T21:41:54.904361Z","iopub.execute_input":"2022-02-05T21:41:54.904896Z","iopub.status.idle":"2022-02-05T21:41:55.010005Z","shell.execute_reply.started":"2022-02-05T21:41:54.904853Z","shell.execute_reply":"2022-02-05T21:41:55.009346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Generate images paths and classes labels:","metadata":{}},{"cell_type":"code","source":"class_mapper = {x: y for x, y in zip(train_df.species.unique(), range(len(train_df.species.unique())))}\ntrain_df['path'] = train_df.image.apply(lambda x: os.path.join(train_dir, x))\ntrain_df['label'] = train_df.species.apply(lambda x: class_mapper[x])\nprint(class_mapper)\ntrain_df.head()\n","metadata":{"execution":{"iopub.status.busy":"2022-02-05T21:41:55.011332Z","iopub.execute_input":"2022-02-05T21:41:55.011589Z","iopub.status.idle":"2022-02-05T21:41:55.151192Z","shell.execute_reply.started":"2022-02-05T21:41:55.011554Z","shell.execute_reply":"2022-02-05T21:41:55.150505Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Image.open(train_df['path'][1])","metadata":{"execution":{"iopub.status.busy":"2022-02-05T21:35:58.016101Z","iopub.execute_input":"2022-02-05T21:35:58.016594Z","iopub.status.idle":"2022-02-05T21:36:02.487594Z","shell.execute_reply.started":"2022-02-05T21:35:58.016555Z","shell.execute_reply":"2022-02-05T21:36:02.486793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Get folds for cross-validation:","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import KFold\nfrom sklearn.model_selection import StratifiedKFold\n\ntrain_df['fold'] = -1\n\nN_FOLDS = 5\nstrat_kfold = StratifiedKFold(n_splits=N_FOLDS, random_state=seed, shuffle=True)\nfor i, (_, train_index) in enumerate(strat_kfold.split(train_df.index, train_df['label'])):\n    train_df.iloc[train_index, -1] = i\n    \ntrain_df['fold'] = train_df['fold'].astype('int')\n\ntrain_df.fold.value_counts().plot.bar()\nfor i in range(N_FOLDS):\n    for cls in train_df.species.unique():\n        print(f'fold {i} class {cls} num samples {len(train_df[(train_df.fold == i) & (train_df.species==cls)])}')","metadata":{"execution":{"iopub.status.busy":"2022-02-05T21:41:55.197182Z","iopub.execute_input":"2022-02-05T21:41:55.197697Z","iopub.status.idle":"2022-02-05T21:41:56.393958Z","shell.execute_reply.started":"2022-02-05T21:41:55.197669Z","shell.execute_reply":"2022-02-05T21:41:56.393301Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training","metadata":{}},{"cell_type":"code","source":"def get_data(fold):\n    train_df_f = train_df.copy()\n    train_df_f['is_valid'] = (train_df_f['fold'] == fold)\n    \n    dls = ImageDataLoaders.from_df(train_df_f, #pass in train DataFrame\n                               valid_col='is_valid', \n                               seed=seed, #seed\n                               fn_col='path', #filename/path is in the second column of the DataFrame\n                               label_col='label', #label is in the first column of the DataFrame\n                               y_block=CategoryBlock, #The type of target\n                               bs=BATCH_SIZE, #pass in batch size\n                               num_workers=8,\n                               item_tfms=Resize(224), #pass in item_tfms\n                               batch_tfms=setup_aug_tfms([Brightness(), Contrast(), Hue(), Saturation()])) #pass in batch_tfms\n    \n    return dls","metadata":{"execution":{"iopub.status.busy":"2022-02-05T21:41:58.379713Z","iopub.execute_input":"2022-02-05T21:41:58.379977Z","iopub.status.idle":"2022-02-05T21:41:58.386244Z","shell.execute_reply.started":"2022-02-05T21:41:58.379947Z","shell.execute_reply":"2022-02-05T21:41:58.385032Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = get_data(0)\ndata.show_batch(max_n=9)","metadata":{"execution":{"iopub.status.busy":"2022-02-05T21:27:52.625319Z","iopub.execute_input":"2022-02-05T21:27:52.625566Z","iopub.status.idle":"2022-02-05T21:28:03.240477Z","shell.execute_reply.started":"2022-02-05T21:27:52.625537Z","shell.execute_reply":"2022-02-05T21:28:03.239705Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_learner(fold_num):\n    data = get_data(fold_num)\n    model = create_model('swin_large_patch4_window7_224', pretrained=True, num_classes=data.c)\n    learn = Learner(data, model, loss_func=CrossEntropyLossFlat(), metrics=accuracy).to_fp16()\n    \n    return learn, data","metadata":{"execution":{"iopub.status.busy":"2022-02-05T21:42:00.617762Z","iopub.execute_input":"2022-02-05T21:42:00.618028Z","iopub.status.idle":"2022-02-05T21:42:00.623213Z","shell.execute_reply.started":"2022-02-05T21:42:00.617997Z","shell.execute_reply":"2022-02-05T21:42:00.622552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learn, _ = get_learner(0)\nlearn.lr_find()","metadata":{"execution":{"iopub.status.busy":"2022-02-05T21:30:09.710319Z","iopub.execute_input":"2022-02-05T21:30:09.711087Z","iopub.status.idle":"2022-02-05T21:34:01.985205Z","shell.execute_reply.started":"2022-02-05T21:30:09.711045Z","shell.execute_reply":"2022-02-05T21:34:01.983577Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Only one fold and one epoch, kaggle time limitation:","metadata":{"execution":{"iopub.status.busy":"2022-02-04T14:29:59.508123Z","iopub.execute_input":"2022-02-04T14:29:59.508824Z","iopub.status.idle":"2022-02-04T14:30:00.204653Z","shell.execute_reply.started":"2022-02-04T14:29:59.508773Z","shell.execute_reply":"2022-02-04T14:30:00.203867Z"}}},{"cell_type":"code","source":"all_preds = []\nfor i in range(N_FOLDS):\n    print(f'Fold {i} results')\n    \n    learn, dls = get_learner(fold_num=i)\n    learn.fit_one_cycle(1, 1e-4, cbs=[MixUp()]) \n    learn.recorder.plot_loss()\n\n    test_dl = dls.test_dl(test_df)\n    preds, _ = learn.tta(dl=test_dl, n=4, beta=0)\n    all_preds.append(preds)\n    \n    del learn\n    torch.cuda.empty_cache()\n    gc.collect()\n    break","metadata":{"execution":{"iopub.status.busy":"2022-02-05T21:42:10.414246Z","iopub.execute_input":"2022-02-05T21:42:10.41482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df['label'] = all_preds[0].argmax(axis=1)  # change it for full training\nlabel_mapper = {v: k for k, v in class_mapper.items()}\ntest_df['species'] = test_df['label'].apply(lambda x: label_mapper[x])\nplt.xticks(rotation='vertical')\ntest_df.species.hist(bins=26)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Save result to csv:","metadata":{}},{"cell_type":"code","source":"test_df = test_df.drop('predictions', axis=1)\ntest_df.to_csv('test_classes.csv', index=False)","metadata":{},"execution_count":null,"outputs":[]}]}