{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Using https://www.kaggle.com/dragonzhang/fastai-cell-tile-prototyping-training ! Thanks for the great kernel.","metadata":{}},{"cell_type":"code","source":"!pip install /kaggle/input/iterative-stratification/iterative-stratification-master/","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sys\npackage_path = '../input/efficientnet-pytorch/EfficientNet-PyTorch/EfficientNet-PyTorch-master'\nsys.path.append(package_path)\n\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom fastai.vision.all import *\nimport pickle\nimport os","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path = Path('../input/hpa-cell-tiles-sample-balanced-dataset')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv(path/'cell_df.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(df)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels = [str(i) for i in range(19)]\nfor x in labels: df[x] = df['image_labels'].apply(lambda r: int(x in r.split('|')))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Change below to `frac=1` to run on the whole training sample","metadata":{}},{"cell_type":"code","source":"#dfs = df.sample(frac=0.1, random_state=42)\n\n#dfs = df.sample(frac=1, random_state=42)\n\n#let try less data for efficientnetb5\n\ndfs = df.sample(frac=1, random_state=42)\n\n\ndfs = dfs.reset_index(drop=True)\nlen(dfs)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"unique_counts = {}\nfor lbl in labels:\n    unique_counts[lbl] = len(dfs[dfs.image_labels == lbl])\n\nfull_counts = {}\nfor lbl in labels:\n    count = 0\n    for row_label in dfs['image_labels']:\n        if lbl in row_label.split('|'): count += 1\n    full_counts[lbl] = count\n    \ncounts = list(zip(full_counts.keys(), full_counts.values(), unique_counts.values()))\ncounts = np.array(sorted(counts, key=lambda x:-x[1]))\ncounts = pd.DataFrame(counts, columns=['label', 'full_count', 'unique_count'])\ncounts.set_index('label').T\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(dfs)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Using multilabel stratification for the train-validation split.\n\nThere is some leakage in the code below (cells belonging to the same image should be in the same split). However, when I fixed that, I got a lower score... coincidence? ","metadata":{}},{"cell_type":"code","source":"nfold = 5\nseed = 42\n\ny = dfs[labels].values\nX = dfs[['image_id', 'cell_id']].values\n\ndfs['fold'] = np.nan\n\nfrom iterstrat.ml_stratifiers import MultilabelStratifiedKFold\nmskf = MultilabelStratifiedKFold(n_splits=nfold, random_state=seed)\nfor i, (_, test_index) in enumerate(mskf.split(X, y)):\n    dfs.iloc[test_index, -1] = i\n    \ndfs['fold'] = dfs['fold'].astype('int')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfs['is_valid'] = False\ndfs['is_valid'][dfs['fold'] == 0] = True","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfs.is_valid.value_counts()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_x(r): return path/'cells'/(r['image_id']+'_'+str(r['cell_id'])+'.jpg')\nimg = get_x(dfs.loc[12])\nimg = PILImage.create(img)\nimg.show();","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_y(r): return r['image_labels'].split('|')\nget_y(dfs.loc[12])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_stats = ([0.07237246, 0.04476176, 0.07661699], [0.17179589, 0.10284516, 0.14199627])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import albumentations","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class AlbumentationsTransform(DisplayedTransform):\n    split_idx,order=0,2\n    def __init__(self, train_aug): store_attr()\n    \n    def encodes(self, img: PILImage):\n        aug_img = self.train_aug(image=np.array(img))['image']\n        return PILImage.create(aug_img)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_train_aug(): return albumentations.Compose([\n            #albumentations.HueSaturationValue(\n                #hue_shift_limit=0.2, \n                #sat_shift_limit=0.2, \n                #val_shift_limit=0.2, \n                #p=0.5\n            #),\n            albumentations.CoarseDropout(p=0.5),\n            albumentations.RandomContrast(p = 0.6)\n])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"item_tfms = [RandomResizedCrop(224, min_scale=0.75, ratio=(1.,1.)),AlbumentationsTransform(get_train_aug()) ]\nbatch_tfms = [*aug_transforms(flip_vert=True, size=128, max_warp=0),  Normalize.from_stats(*sample_stats)]\nbs=256","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dblock = DataBlock(blocks=(ImageBlock, MultiCategoryBlock(vocab=labels)),\n                splitter=ColSplitter(col='is_valid'),\n                get_x=get_x,\n                get_y=get_y,\n                item_tfms=item_tfms,\n                batch_tfms=batch_tfms\n                )\ndls = dblock.dataloaders(dfs, bs=bs)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dls.show_batch(nrows=3, ncols=3)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}