{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import gc\nimport os\nimport sys\nimport time\nimport random\nimport logging\nimport datetime as dt\nimport cv2\n\nimport numpy as np\nimport pandas as pd\n\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nimport torch.utils.data as data\nimport torch.nn.functional as F\nimport torchvision as vision\nfrom torch.utils.data import Dataset, DataLoader\n\nfrom torch.optim.lr_scheduler import CosineAnnealingLR\n\nfrom pathlib import Path\nfrom PIL import Image\nfrom contextlib import contextmanager\n\nfrom joblib import Parallel, delayed\nfrom tqdm import tqdm\nfrom fastprogress import master_bar, progress_bar\n\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import fbeta_score","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"labels = pd.read_csv(\"../input/imet-2019-fgvc6/labels.csv\")\ntrain = pd.read_csv(\"../input/imet-2019-fgvc6/train.csv\")  #109237\ntest = pd.read_csv(\"../input/imet-2019-fgvc6/sample_submission.csv\")\n\n#parameters\nnum_epochs = 200\nbatch_size = 200\nlearning_rate = 0.0001\n\ndevice = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_transforms = {'train':vision.transforms.Compose([\n        vision.transforms.RandomResizedCrop(332),\n        vision.transforms.RandomHorizontalFlip(),\n        vision.transforms.ToTensor(),\n        vision.transforms.Normalize(\n            [0.485, 0.456, 0.406], \n            [0.229, 0.224, 0.225])\n    ]),\n    'val': vision.transforms.Compose([\n        vision.transforms.Resize(500),\n        vision.transforms.RandomResizedCrop(332),\n        vision.transforms.ToTensor(),\n        vision.transforms.Normalize(\n            [0.485, 0.456, 0.406], \n            [0.229, 0.224, 0.225])\n    ])\n}\ndata_transforms[\"test\"] = data_transforms[\"val\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class IMetDataset(Dataset):  \n    def __init__(self, df, images_dir, transforms = None):\n        self.df = df\n        self.images_dir = images_dir\n        self.transforms = transforms\n    \n    def __len__(self):\n        return self.df.shape[0]\n    \n    def __getitem__(self, idx):\n        cur_idx_row = self.df.iloc[idx]\n        img_id = cur_idx_row['id']\n        img_name = img_id + \".png\"\n        img_path = os.path.join(self.images_dir, img_name)\n        \n        img = cv2.imread(img_path)\n        img = Image.fromarray(img)\n        \n        if self.transforms is not None:\n            img = self.transforms(img)\n        \n        labels_list = cur_idx_row['attribute_ids'].split(' ')\n\n        FILLVAL = 1.0\n        label = torch.zeros((1103,), dtype=torch.float32)\n\n        for i in labels_list:\n\n            label[int(i)] = FILLVAL\n\n        return img, label\n    \nclass IMetLoadData(Dataset):\n    def __init__(self, train, labels):\n        self.train = train\n        self.labels = labels\n    \n    def __len__(self):\n        return self.train.shape[0]\n    \n    def __getitem__(self, idx):\n        return self.train[idx], self.labels[idx]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Classifier(nn.Module):\n    def __init__(self):\n        super(Classifier, self).__init__()\n        \n    def forward(self, x):\n        return x\n\nclass Densenet201(nn.Module):\n    def __init__(self, pretrained: Path):\n        super(Densenet201, self).__init__()\n        self.densenet201 = vision.models.densenet201()\n        self.densenet201.load_state_dict(torch.load(pretrained))\n        self.densenet201.classifier = Classifier()\n        \n        dense = nn.Sequential(*list(self.densenet201.children())[:-1])\n        for param in dense.parameters():\n            param.requires_grad = False\n        \n    def forward(self, x):\n        return self.densenet201(x)\n    \nclass MultiLayer(nn.Module):\n    def __init__(self):\n        super(MultiLayer, self).__init__()\n        self.linear1 = nn.Linear(1920, 1920)\n        self.relu = nn.ReLU()\n        self.linear2 = nn.Linear(1920, 1103)\n        self.dropout = nn.Dropout(0.5)\n        self.sigmoid = nn.Sigmoid()\n        \n    def forward(self, x):\n        x = self.relu(self.linear1(x))\n        x = self.dropout(x)\n        return self.sigmoid(self.linear2(x))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset = IMetDataset(train, \"../input/imet-2019-fgvc6/train\", \n                            transforms = data_transforms['train'])\ntrain_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=False, \n                          num_workers=2, pin_memory=True)\ntest_dataset = IMetDataset(test, \"../input/imet-2019-fgvc6/test\", \n                            transforms = data_transforms['test'])\ntest_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False, \n                          num_workers=2, pin_memory=True)\n\ndef get_feature_vector(df, loader):\n    matrix = torch.zeros((df.shape[0], 1920)).to(device)\n    preds = torch.zeros((df.shape[0], 1103)).to(device)\n    model = Densenet201(\"../input/densenet201/densenet201.pth\")\n    model.to(device)\n    batch = loader.batch_size\n    for i, (i_batch,labels) in tqdm(enumerate(loader)):\n        i_batch = i_batch.to(device)\n        labels = labels.to(device)\n        pred = model(i_batch).detach()\n        matrix[i * batch:(i + 1) * batch] = pred\n        preds[i * batch:(i + 1) * batch] = labels\n    return matrix, preds\n\ntrain_tensor, train_labels = get_feature_vector(train, train_loader)\ntest_tensor, test_labels = get_feature_vector(test, test_loader)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_preds = np.zeros((len(train_tensor), 1103))\nfold = KFold(n_splits = 5, random_state = 10)\nloss_fn = nn.BCELoss(reduction=\"mean\").to(device)\nmodel = MultiLayer()\nmodel.to(device)\ntag = dt.datetime.now().strftime(\"%Y-%m-%d-%H-%M-%S\")\npath = Path(f\"bin/{tag}\")\npath.mkdir(exist_ok=True, parents=True)\n\nfor fold_num, (trn_idx, val_idx) in enumerate(fold.split(train_tensor)):\n    X_train, X_val = train_tensor[trn_idx, :], train_tensor[val_idx, :]\n    Y_train, Y_val = train_labels[trn_idx, :], train_labels[val_idx, :]\n    \n    X_train.to(device)\n    Y_train.to(device)\n    train_dataset = IMetLoadData(X_train, Y_train)\n    train_loader = data.DataLoader(train_dataset, batch_size=batch_size, shuffle=False)\n    val_dataset = IMetLoadData(X_val, Y_val)\n    val_loader = data.DataLoader(val_dataset, batch_size=batch_size, shuffle=False)\n    \n    \n        \n    optimizer = optim.Adam(params=model.parameters(), lr=learning_rate)\n    scheduler = CosineAnnealingLR(optimizer, T_max=num_epochs)\n    best_score = np.inf\n    \n    for epoch in range(num_epochs):\n        model.train()\n        avg_loss = 0.0\n        \n        for i_batch, y_batch in train_loader:\n            y_pred = model(i_batch)\n            loss = loss_fn(y_pred, y_batch)\n            optimizer.zero_grad()\n            loss.backward()\n            optimizer.step()\n            avg_loss += loss.item() / len(train_loader)\n        \n        model.eval()\n        valid_preds = np.zeros((len(val_loader.dataset), 1103))\n        avg_val_loss = 0.0\n        for i, (i_batch, y_batch) in enumerate(val_loader):\n            with torch.no_grad():\n                y_pred = model(i_batch).detach()\n                avg_val_loss += loss_fn(y_pred, y_batch).item() / len(val_loader)\n                valid_preds[i * batch_size:(i + 1) * batch_size] = y_pred.cpu().numpy()\n                \n        scheduler.step()\n        \n        print(\"=========================================\")\n        print(f\"Epoch {epoch + 1} / {num_epochs}  Fold {fold_num + 1} / 5\")\n        print(\"=========================================\")\n        print(f\"avg_loss: {avg_loss:.8f}\")\n        print(f\"avg_val_loss: {avg_val_loss:.8f}\")\n            \n        if best_score > avg_val_loss:\n            torch.save(model.state_dict(), path / f\"best{fold_num}.pth\")\n            best_score = avg_val_loss\n        \n    model.load_state_dict(torch.load(path / f\"best{fold_num}.pth\"))\n    model.eval()\n    valid_preds = np.zeros((len(val_loader.dataset), 1103))\n    avg_val_loss = 0.0\n    for i, (i_batch, y_batch) in enumerate(val_loader):\n        with torch.no_grad():\n            y_pred = model(i_batch).detach()\n            avg_val_loss += loss_fn(y_pred, y_batch).item() / len(val_loader)\n            valid_preds[i * batch_size:(i + 1) * batch_size] = y_pred.cpu().numpy()\n    print(f\"Best Validation Loss: {avg_val_loss:.8f}\")\n    \n    train_preds[val_idx] = valid_preds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def threshold_search(y_pred, y_true):\n    score = []\n    candidates = np.arange(0, 1.0, 0.01)\n    for th in progress_bar(candidates):\n        yp = (y_pred > th).astype(int)\n        score.append(fbeta_score(y_pred=yp, y_true=y_true, beta=2, average=\"samples\"))\n    score = np.array(score)\n    pm = score.argmax()\n    best_th, best_score = candidates[pm], score[pm]\n    return best_th, best_score\n\nbest_threshold, best_score = threshold_search(train_preds, train_labels.cpu().numpy())\nprint(f\"best_threshold = {best_threshold}\")\nprint(f\"best_score = {best_score}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_dataset = IMetLoadData(test_tensor, test_labels)\ntest_loader = data.DataLoader(test_dataset, batch_size=batch_size, shuffle=False)\npreds = np.zeros((test_tensor.size(0), 1103))\n\nfor pth in path.iterdir():\n    model.load_state_dict(torch.load(pth))\n    model.to(device)\n    model.eval()\n    temp = np.zeros_like(preds)\n    for i, (i_batch, labels) in enumerate(test_loader):\n        with torch.no_grad():\n            y_pred = model(i_batch).detach()\n            temp[i * batch_size:(i + 1) * batch_size] = y_pred.cpu().numpy()\n    preds += temp / 5\n\npreds = (preds > best_threshold).astype(int)\nprediction = []\nfor i in range(preds.shape[0]):\n    pred1 = np.argwhere(preds[i] == 1.0).reshape(-1).tolist()\n    pred_str = \" \".join(list(map(str, pred1)))\n    prediction.append(pred_str)\n\ntest.attribute_ids = prediction\ntest.to_csv(\"submission.csv\", index=False)\ntest.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}