{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport tensorflow_addons as tfa\nimport h5py\nfrom sklearn.metrics import f1_score\n\nfrom sklearn.model_selection import train_test_split\nfrom fastai.vision.all import *","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CFG:\n    \n    seed=42\n    crop_size=510\n    img_size=510\n    label_type=MultiCategoryBlock\n    bs = 32\n    classes = ['complex', 'frog_eye_leaf_spot', 'healthy', 'powdery_mildew', 'rust', 'scab']\n    \n    train_df_path = Path('../input/plant-pathology-2021-fgvc8/train.csv')\n    image_path = Path('../input/resized-plant2021/img_sz_640')\n    dup_path = Path('../input/pp2021-duplicates-revealing/duplicates.csv')\n    \n    item_tfms=RandomResizedCrop(crop_size)\n    batch_tfms=[*aug_transforms(mult=2.0, flip_vert=True, size=img_size),\n                Normalize.from_stats(*imagenet_stats)\n    ]\n    cutmix = CutMix(1.)\n\nset_seed(CFG.seed, reproducible=True)\n\ndef remove_dup(df, dup):\n    index = []\n    for i in dup[2]:\n        index.append(df.loc[df['image'] == i].index[0])\n    df = df.drop(index, axis=0).reset_index(drop=True)\n    \n    return df\n\ndef get_x(x):\n    return CFG.image_path/x['image']\n\ndef get_y(y):\n    return y['labels'].split(' ')\n\ndef lv_dataset(df,n_toshow=9):\n    db = DataBlock(blocks=(ImageBlock, CFG.label_type(vocab=CFG.classes)),\n                   splitter=RandomSplitter(seed=CFG.seed),\n                   get_x=get_x,\n                   get_y=get_y,\n                   item_tfms=CFG.item_tfms,\n                   batch_tfms=CFG.batch_tfms\n                  )\n\n    dls = db.dataloaders(df, bs=CFG.bs)\n    dls.show_batch(max_n=n_toshow)\n    \n    return dls\n\ndef get_BestThreshs(preds, targs):\n    scores = []\n    thresholds = np.arange(0.01, 1., 0.01)\n    \n    for threshold in thresholds:\n        metric = tfa.metrics.F1Score(num_classes=len(CFG.classes),\n                                     threshold=threshold\n                                    )\n        metric.update_state(targs, preds)\n        scores.append(metric.result().numpy())\n    \n    df = pd.DataFrame(columns=CFG.classes, index=pd.Index(thresholds, name='threshold'), data=scores)\n    \n    thresholds = []\n    for cls in CFG.classes:\n        thresholds.append(df[cls].idxmax())\n    \n    for cls, th in zip(CFG.classes, thresholds):\n        print(cls + ' >>>> ', th)\n        \n    return thresholds\n\ndef save_thresholds(thresholds):\n    with h5py.File('thresholds.h5', 'w') as h:\n        h.create_dataset('thresholds', data=thresholds)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv(CFG.train_df_path)\ndup = pd.read_csv(CFG.dup_path, names=[1,2])\ndf = remove_dup(df, dup)\ntrain_df, val_df = train_test_split(df, test_size=0.1, shuffle=True, random_state=32)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dls = lv_dataset(train_df,20)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learn = cnn_learner(dls, resnet50, metrics=[accuracy_multi]).to_fp16()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"min_lr, lr = learn.lr_find()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learn.fine_tune(7, lr, freeze_epochs=2)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_dls = learn.dls.test_dl(val_df, with_labels=True)\npredictions = 0\nlb = 0\nfor m in ['../input/trained-models1/alexnet.pkl', '../input/trained-models1/resnet50.pkl']:\n    learn = load_learner(m, cpu=False).to_fp32()\n    preds, y = learn.get_preds(dl=val_dls)\n    predictions += preds\n    lb = y\n\npredictions /= 2\nthresholds = get_BestThreshs(predictions, lb)\nsave_thresholds(thresholds)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"save_thresholds(thresholds)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learn.export(f'resnet50.pkl')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}