{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"from shutil import copyfile\n\ncopyfile(src=\"../input/import-file/inceptionresnetv2.py\", dst=\"../working/pretrainedmodels.py\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import glob\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\n\n# Any results you write to the current directory are saved as output.\nimport argparse\nfrom itertools import islice\nimport json\nfrom pathlib import Path\nimport shutil\nimport warnings\nfrom typing import Dict\nimport os\nimport sys\nfrom collections import OrderedDict\nimport math\nimport random\nfrom typing import Callable, List\nfrom datetime import datetime\nimport json\nimport glob\nfrom multiprocessing.pool import ThreadPool\nimport gc\n\nimport torch\nfrom torch import nn, cuda\nfrom torch.nn import functional as F\nimport torch.utils.model_zoo as model_zoo\nfrom torch.utils.data import DataLoader\nfrom torch.utils.data import Dataset\nfrom torchvision.transforms import (\n            ToTensor, Normalize, Compose, Resize, CenterCrop, RandomCrop,\n                RandomHorizontalFlip)\n\nimport tqdm\nfrom PIL import Image\nimport cv2\ncv2.setNumThreads(0)\nfrom pretrainedmodels import *\n\ndef seed_everything(seed=1234):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n                               \nseed_everything(7)\n\nDATA_ROOT = Path('/kaggle/input/imet-2019-fgvc6')\nN_CLASSES = 1103\n\ntest_transform = Compose([\n    RandomCrop(320, pad_if_needed=True),\n    RandomHorizontalFlip(),\n    ])\n\n\ntensor_transform = Compose([\n    ToTensor(),\n    Normalize(mean=[0.5949, 0.5611, 0.5185], std=[0.2900, 0.2844, 0.2811]),\n    ])\n\nclass TTADataset:\n    def __init__(self, root: Path, df: pd.DataFrame,\n                 image_transform: Callable, tta: int):\n        self._root = root\n        self._df = df\n        self._image_transform = image_transform\n        self._tta = tta\n\n    def __len__(self):\n        return len(self._df) * self._tta\n\n    def __getitem__(self, idx):\n        item = self._df.iloc[idx % len(self._df)]\n        image = load_transform_image(item, self._root, self._image_transform)\n        return image, item.id\n\n\ndef load_transform_image(\n        item, root: Path, image_transform: Callable, debug: bool = False):\n    image = load_image(item, root)\n    image = image_transform(image)\n    if debug:\n        image.save('_debug.png')\n    return tensor_transform(image)\n\n\ndef train_load_transform_image(\n        item, root: Path, image_transform: Callable, debug: bool = False):\n    image = load_image(item, root)\n    image = image_transform(image)\n    if debug:\n        image.save('_debug.png')\n    return train_tensor_transform(image)\n\n\ndef load_image(item, root: Path) -> Image.Image:\n    image = cv2.imread(str(root / f'{item.id}.png'))\n    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n    return Image.fromarray(image)\n\n\ndef get_ids(root: Path) -> List[str]:\n    return sorted({p.name.split('_')[0] for p in root.glob('*.png')})\n\ndef load_model(model: nn.Module, path: Path) -> Dict:\n    state = torch.load(str(path))\n    model.load_state_dict(state['model'])\n    print('Loaded model from epoch {epoch}, step {step:,}'.format(**state))\n    return state\n\ndef mean_df(df: pd.DataFrame) -> pd.DataFrame:\n    return df.groupby(level=0).mean()\n\ndef get_classes(item):\n    return ' '.join(cls for cls, is_present in item.items() if is_present)\n\ndef binarize_prediction(probabilities, threshold: float, argsorted=None,\n                        min_labels=1, max_labels=10):\n    \"\"\" Return matrix of 0/1 predictions, same shape as probabilities.\n    \"\"\"\n    assert probabilities.shape[1] == N_CLASSES\n    if argsorted is None:\n        argsorted = probabilities.argsort(axis=1)\n    max_mask = _make_mask(argsorted, max_labels)\n    min_mask = _make_mask(argsorted, min_labels)\n    \n    prob_mask = []\n    for prob in probabilities:\n        prob_mask.append(prob > prob.max()/7)\n        \n    prob_mask = np.array(prob_mask, dtype=np.int)\n    \n    return (max_mask & prob_mask) | min_mask\n\n\ndef _make_mask(argsorted, top_n: int):\n    mask = np.zeros_like(argsorted, dtype=np.uint8)\n    col_indices = argsorted[:, -top_n:].reshape(-1)\n    row_indices = [i // top_n for i in range(len(col_indices))]\n    mask[row_indices, col_indices] = 1\n    return mask\n\nargs = {\n    'batch_size':16,\n    'tta':2,\n    'use_cuda':1,\n    'workers':1,\n    'threshold':0.1,\n    'max_labels':10,\n    'output':'/kaggle/working/submission.csv',\n}\n\ndef create_model(model):\n    feature_dim = model.last_linear.in_features\n    class AvgPool(nn.Module):\n        def forward(self, x):\n            # print (x.size())\n            return F.avg_pool2d(x, x.shape[2:])\n    model.avg_pool = AvgPool()\n    model.avgpool = AvgPool()\n    model.last_linear = nn.Linear(feature_dim, N_CLASSES)\n    model = torch.nn.DataParallel(model)\n    model = model.cuda()\n    return model\n    \ndef test(model, loader, model_path, multi=False, half=False):\n    load_model(model, model_path / 'best-model.pt')\n    df = predict(model, loader, use_cuda=args['use_cuda'], half=half)\n    return df\n    \ndef predict(model, loader, use_cuda: bool, half=False):\n    model.eval()\n    all_outputs, all_ids = [], []\n    with torch.no_grad():\n        for inputs, ids in loader:\n            inputs = inputs.cuda()\n            outputs = torch.sigmoid(model(inputs))\n            # outputs = model(inputs)\n            all_outputs.append(outputs.detach().cpu().numpy())\n            all_ids.extend(ids)\n    df = pd.DataFrame(\n        data=np.concatenate(all_outputs),\n        index=all_ids,\n        columns=map(str, range(N_CLASSES)))\n    df = mean_df(df)\n    return df\n\nimport string\ndef randomString2(stringLength=8):\n    \"\"\"Generate a random string of fixed length \"\"\"\n    letters= string.ascii_lowercase\n    return ''.join(random.sample(letters,stringLength))\n\n\ntest_root = DATA_ROOT /'test'\ntest_df = pd.read_csv(DATA_ROOT / 'sample_submission.csv')\n# df = pd.concat([df]*5, ignore_index=True)\n# df['new_id'] = [randomString2() for i in range(len(df))]\nloader = DataLoader(\n        dataset=TTADataset(test_root, test_df, test_transform, tta=args['tta']),\n        shuffle=False,\n        batch_size=args['batch_size'],\n        num_workers=args['workers'],\n    )\n\n\nimport gc\n\ndfs = []\nmodel = inceptionresnetv2(pretrained=False)\nmodel = create_model(model)\n#for i in range(5,10):\nfor i in range(1):\n    df = test(model, loader, Path(f'/kaggle/input/import-file/'), multi=True)\n    gc.collect()\n    dfs.append(df)\ndf = pd.concat(dfs)\ndf = mean_df(df)\nout_path = '05_30_inres2.h5'\n#df.to_hdf(out_path, 'prob', index_label='id')\nprint(f'Saved predictions to {out_path}')\npred = df.values\n\ndf[:] = binarize_prediction(pred, threshold=args['threshold'], max_labels=args['max_labels'])\ndf = df.apply(get_classes, axis=1)\ndf.name = 'attribute_ids'\ndf.to_csv(args['output'], header=True)\nprint(f'Saved submission.csv to' + args['output'])","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}