{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nimport torch\nimport torch.nn as nn\nimport multiprocessing\nfrom tqdm import tqdm\nfrom sklearn.preprocessing import LabelEncoder","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"IN_KERNEL = os.environ.get('KAGGLE_WORKING_DIR') is not None\nMIN_SAMPLES_PER_CLASS = 100\nBATCH_SIZE = 64\nNUM_WORKERS = multiprocessing.cpu_count()\nNUM_TOP_PREDICTS = 1\nENABLE_FAST_SKIP = False","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/landmark-recognition-2020/train.csv')\ntest = pd.read_csv('../input/landmark-recognition-2020/sample_submission.csv')\ntrain_dir = '../input/landmark-recognition-2020/train/'\ntest_dir = '../input/landmark-recognition-2020/test/'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import time\nimport sys\nsys.path.append('../input/landmarkrecognition2020')\nfrom datasets.datasets import *\nimport models","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_data(train, test, train_dir, test_dir):\n    counts = train.landmark_id.value_counts()\n    selected_classes = counts[counts >= MIN_SAMPLES_PER_CLASS].index\n    num_classes = selected_classes.shape[0]\n    print('classes with at least N samples:', num_classes)\n\n    train_mask = train.landmark_id.isin(selected_classes)\n    train = train.loc[train_mask].copy()\n    print('train_df', train.shape)\n    print('test_df', test.shape)\n\n    # filter non-existing test images\n    exists = lambda img: os.path.exists(f'{test_dir}/{img[0]}/{img[1]}/{img[2]}/{img}.jpg')\n    test_mask = test.id.apply(exists)\n    test = test.loc[test_mask].copy()\n    print('test_df after filtering', test.shape)\n\n    label_encoder = LabelEncoder()\n    label_encoder.fit(train.landmark_id.values)\n    print('found classes', len(label_encoder.classes_))\n    assert len(label_encoder.classes_) == num_classes\n\n    train.landmark_id = label_encoder.transform(train.landmark_id)\n\n    train_dataset = ImageDataset(train, train_dir, mode='train')\n    test_dataset = ImageDataset(test, test_dir, mode='test')\n\n    train_loader = DataLoader(train_dataset, batch_size=BATCH_SIZE,\n                              shuffle=False, num_workers=4, drop_last=True)\n\n    test_loader = DataLoader(test_dataset, batch_size=BATCH_SIZE,\n                             shuffle=False, num_workers=NUM_WORKERS)\n\n    return train_loader, test_loader, label_encoder, num_classes","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def inference(data_loader, model):\n    model.eval()\n\n    activation = nn.Softmax(dim=1)\n    all_predicts, all_confs, all_targets = [], [], []\n\n    with torch.no_grad():\n        for i, data in enumerate(tqdm(data_loader)):\n            if data_loader.dataset.mode != 'test':\n                input_, target = data['image'], data['target']\n            else:\n                input_, target = data['image'], None\n\n            output = model(input_.cuda())\n            output = activation(output)\n\n            confs, predicts = torch.topk(output, NUM_TOP_PREDICTS)\n            all_confs.append(confs)\n            all_predicts.append(predicts)\n\n            if target is not None:\n                all_targets.append(target)\n            \n#             if (i+1)%10 == 0:\n#                 print('Have processed images is ', (i+1)*BATCH_SIZE)\n\n    predicts = torch.cat(all_predicts)\n    confs = torch.cat(all_confs)\n    targets = torch.cat(all_targets) if len(all_targets) else None\n\n    return predicts, confs, targets","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def generate_submission(test_loader, model, label_encoder):\n    sample_sub = pd.read_csv('../input/landmark-recognition-2020/sample_submission.csv')\n\n    predicts_gpu, confs_gpu, _ = inference(test_loader, model)\n    predicts, confs = predicts_gpu.cpu().numpy(), confs_gpu.cpu().numpy()\n\n    labels = [label_encoder.inverse_transform(pred) for pred in predicts]\n    print('labels')\n    print(np.array(labels))\n    print('confs')\n    print(np.array(confs))\n\n    sub = test_loader.dataset.df\n    def concat(label: np.ndarray, conf: np.ndarray) -> str:\n        return ' '.join([f'{L} {c}' for L, c in zip(label, conf)])\n    sub['landmarks'] = [concat(label, conf) for label, conf in zip(labels, confs)]\n\n    sample_sub = sample_sub.set_index('id')\n    sub = sub.set_index('id')\n    sample_sub.update(sub)\n\n    sample_sub.to_csv('submission.csv')\n    print(sub)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if __name__ == '__main__':\n    global_start_time = time.time()\n    train_loader, test_loader, label_encoder, num_classes = load_data(train, test, train_dir, test_dir)\n    arch = 'resnet50'\n    model_path = '../input/modelspath/resnet50_best.pth.tar'\n\n#     if ENABLE_FAST_SKIP and test.id[0] == \"00084cdf8f600d00\":\n    # This is a run on the public data, skip it to speed up submission run on private data.\n    print(\"Skipping run on public test set.\")\n    sample_sub = pd.read_csv('../input/landmark-recognition-2020/sample_submission.csv')\n    sample_sub.to_csv('submission.csv')\n#     else:\n#         #create model\n#         print(\"==> creating model '{}'\".format(arch))\n#         model = models.__dict__[arch](num_classes = num_classes)\n#         model.cuda()\n\n#         #original saved file with DataParallel or without DataParallel\n#         state_dict = torch.load(model_path)['state_dict']\n#         #create new OrderedDict that does not contain 'module.'\n#         from collections import OrderedDict\n#         new_state_dict = OrderedDict()\n#         for k,v in state_dict.items():\n#             if 'module' in k:\n#                 k = k[7:] # remove 'module.'\n#             new_state_dict[k] = v\n#         #load params\n#         model.load_state_dict(new_state_dict)\n#         model.eval()\n\n#         print('inference mode')\n#         generate_submission(test_loader, model, label_encoder)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}