{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install timm","metadata":{"execution":{"iopub.status.busy":"2022-08-04T09:38:35.768700Z","iopub.execute_input":"2022-08-04T09:38:35.769587Z","iopub.status.idle":"2022-08-04T09:38:49.698995Z","shell.execute_reply.started":"2022-08-04T09:38:35.769454Z","shell.execute_reply":"2022-08-04T09:38:49.697462Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np \nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport cv2 as cv\nimport random\nimport warnings\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\nimport os\nfrom sklearn.metrics import confusion_matrix\nfrom sklearn.model_selection import train_test_split\nimport timm\nfrom tqdm import tqdm\nimport albumentations as A\nimport gc\nfrom torchvision import models","metadata":{"execution":{"iopub.status.busy":"2022-08-04T09:38:49.702652Z","iopub.execute_input":"2022-08-04T09:38:49.704729Z","iopub.status.idle":"2022-08-04T09:39:01.160278Z","shell.execute_reply.started":"2022-08-04T09:38:49.704678Z","shell.execute_reply":"2022-08-04T09:39:01.158746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SEED = 123\nrandom.seed(SEED)\nnp.random.seed(SEED)\ntorch.manual_seed(SEED)\ntorch.cuda.manual_seed(SEED)\ntorch.cuda.manual_seed_all(SEED)\ntorch.backends.cudnn.deterministic = True\ntorch.backends.cudnn.benchmark = False\n\nwarnings.filterwarnings('ignore')\ndevice = 'cuda' if torch.cuda.is_available() else 'cpu'\nprint(f'\\n Device : {device.upper()}')","metadata":{"execution":{"iopub.status.busy":"2022-08-04T09:39:01.164719Z","iopub.execute_input":"2022-08-04T09:39:01.165272Z","iopub.status.idle":"2022-08-04T09:39:01.263861Z","shell.execute_reply.started":"2022-08-04T09:39:01.165237Z","shell.execute_reply":"2022-08-04T09:39:01.262622Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class AptosDataset(Dataset):\n    def __init__(self, X, y, img_dir, transforms, resize = (512, 512)):\n        self.img_dir = img_dir\n        self.resize = resize\n        self.transforms = transforms\n        self.X = X\n        self.y = y\n            \n    def __len__(self):\n        return self.y.shape[0]\n    \n    def __getitem__(self, idx):\n        img_name = self.X[idx] + '.png'\n        img_path = os.path.join(self.img_dir, img_name)\n        img_vector = cv.imread(img_path)\n        if self.transforms:\n            transformed = self.transforms(image = img_vector)\n            img_vector = transformed['image']\n        if self.resize:\n            img_vector = cv.resize(img_vector, self.resize)\n            img_vector = img_vector.reshape((3, self.resize[0], self.resize[1]))\n        img_vector = torch.tensor(img_vector)\n        target = self.y[idx]\n        target = torch.tensor(target)\n        \n        return img_vector, target\n        \n    def show(self, idx):\n        img_vector, target = self.__getitem__(idx)\n        plt.imshow(img_vector.detach().numpy())\n        plt.title(f'{target.detach().numpy()}')\n        plt.show()\n        \n    def get_class_weights(self):\n        class_weights = class_weight.compute_class_weight(class_weight='balanced', classes=np.array([0, 1, 2, 3, 4]),\n                                                          y=self.y.values)\n        class_weights = torch.tensor(class_weights, dtype=torch.float)\n        return class_weights\n","metadata":{"execution":{"iopub.status.busy":"2022-08-04T09:39:01.267648Z","iopub.execute_input":"2022-08-04T09:39:01.268579Z","iopub.status.idle":"2022-08-04T09:39:01.310845Z","shell.execute_reply.started":"2022-08-04T09:39:01.268534Z","shell.execute_reply":"2022-08-04T09:39:01.307864Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def Cmatrix(rater_a, rater_b, min_rating=None, max_rating=None):\n    assert(len(rater_a) == len(rater_b))\n    if min_rating is None:\n        min_rating = min(rater_a + rater_b)\n    if max_rating is None:\n        max_rating = max(rater_a + rater_b)\n    num_ratings = int(max_rating - min_rating + 1)\n    conf_mat = [[0 for i in range(num_ratings)]\n                for j in range(num_ratings)]\n    for a, b in zip(rater_a, rater_b):\n        conf_mat[a - min_rating][b - min_rating] += 1\n    return conf_mat\n\ndef histogram(ratings, min_rating=None, max_rating=None):\n    if min_rating is None:\n        min_rating = min(ratings)\n    if max_rating is None:\n        max_rating = max(ratings)\n    num_ratings = int(max_rating - min_rating + 1)\n    hist_ratings = [0 for x in range(num_ratings)]\n    for r in ratings:\n        hist_ratings[r - min_rating] += 1\n    return hist_ratings\n\ndef quadratic_weighted_kappa(y, y_pred):\n    rater_a = y\n    rater_b = y_pred\n    min_rating=None\n    max_rating=None\n    rater_a = np.array(rater_a, dtype=int)\n    rater_b = np.array(rater_b, dtype=int)\n    assert(len(rater_a) == len(rater_b))\n    if min_rating is None:\n        min_rating = min(min(rater_a), min(rater_b))\n    if max_rating is None:\n        max_rating = max(max(rater_a), max(rater_b))\n    conf_mat = Cmatrix(rater_a, rater_b,\n                                min_rating, max_rating)\n    num_ratings = len(conf_mat)\n    num_scored_items = float(len(rater_a))\n\n    hist_rater_a = histogram(rater_a, min_rating, max_rating)\n    hist_rater_b = histogram(rater_b, min_rating, max_rating)\n\n    numerator = 0.0\n    denominator = 0.0\n\n    for i in range(num_ratings):\n        for j in range(num_ratings):\n            expected_count = (hist_rater_a[i] * hist_rater_b[j]\n                              / num_scored_items)\n            try:\n                d = pow(i - j, 2.0) / pow(num_ratings - 1, 2.0)\n                numerator += d * conf_mat[i][j] / num_scored_items\n                denominator += d * expected_count / num_scored_items\n            except:\n                return 0\n\n    return (1.0 - numerator / denominator)","metadata":{"execution":{"iopub.status.busy":"2022-08-04T09:39:01.314676Z","iopub.execute_input":"2022-08-04T09:39:01.315117Z","iopub.status.idle":"2022-08-04T09:39:01.353303Z","shell.execute_reply.started":"2022-08-04T09:39:01.315076Z","shell.execute_reply":"2022-08-04T09:39:01.346267Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TRAIN_PATH = '../input/aptos2019-blindness-detection/train.csv'\nTEST_PATH = '../input/aptos2019-blindness-detection/test.csv'\nSAMPLE_SUB_PATH = '../input/aptos2019-blindness-detection/sample_submission.csv'\n\nTRAIN_IMG = '../input/aptos2019-blindness-detection/train_images'\nTEST_IMG = '../input/aptos2019-blindness-detection/test_images'\n\nLEARNING_RATE = 1e-4\nN_EPOCHS = 5\nBATCH_SIZE = 4\nIMG_DIM = 512","metadata":{"execution":{"iopub.status.busy":"2022-08-04T09:39:01.356480Z","iopub.execute_input":"2022-08-04T09:39:01.357711Z","iopub.status.idle":"2022-08-04T09:39:01.373597Z","shell.execute_reply.started":"2022-08-04T09:39:01.357657Z","shell.execute_reply":"2022-08-04T09:39:01.369249Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv(TRAIN_PATH)\nX = df['id_code'].values\ny = df['diagnosis'].values\n\nX_train, X_val, y_train, y_val = train_test_split(X, y, test_size = 0.2, stratify = y)","metadata":{"execution":{"iopub.status.busy":"2022-08-04T09:39:01.386613Z","iopub.execute_input":"2022-08-04T09:39:01.393385Z","iopub.status.idle":"2022-08-04T09:39:01.442024Z","shell.execute_reply.started":"2022-08-04T09:39:01.393300Z","shell.execute_reply":"2022-08-04T09:39:01.440376Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train.shape","metadata":{"execution":{"iopub.status.busy":"2022-08-04T09:39:01.449935Z","iopub.execute_input":"2022-08-04T09:39:01.453783Z","iopub.status.idle":"2022-08-04T09:39:01.482887Z","shell.execute_reply.started":"2022-08-04T09:39:01.453722Z","shell.execute_reply":"2022-08-04T09:39:01.479989Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transforms = A.Compose([\n                        A.RandomBrightnessContrast(),\n                        A.HueSaturationValue(),\n                       ]\n                      )\n\ntrain_dataset = AptosDataset(X_train, y_train, TRAIN_IMG, transforms, resize = (IMG_DIM, IMG_DIM))\nvalid_dataset = AptosDataset(X_val, y_val,TRAIN_IMG, transforms, resize = (IMG_DIM, IMG_DIM))\n\ntrain_loader = DataLoader(train_dataset, batch_size = BATCH_SIZE)\nvalid_loader = DataLoader(valid_dataset, batch_size = BATCH_SIZE)\n\nmodel = models.efficientnet_b0(pretrained = True)\nmodel.classifier = nn.Sequential(\n                         nn.Linear(in_features=1280, out_features=512, bias=True),\n                         nn.Linear(in_features=512, out_features=5, bias=True),\n                         nn.Softmax()\n                                )\nmodel.to(device)\n# model.load_state_dict(torch.load('../input/hued-sat/brightness.bin')) \nerror = nn.CrossEntropyLoss()\noptimizer = torch.optim.Adam(model.parameters(), lr = LEARNING_RATE)\nscheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=1e-5)","metadata":{"execution":{"iopub.status.busy":"2022-08-04T09:39:01.487232Z","iopub.execute_input":"2022-08-04T09:39:01.490707Z","iopub.status.idle":"2022-08-04T09:39:12.723081Z","shell.execute_reply.started":"2022-08-04T09:39:01.490658Z","shell.execute_reply":"2022-08-04T09:39:12.721280Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"best = 100\nfor i in range(1, N_EPOCHS + 1):\n    train_loss = []\n    val_loss = []\n    kappa = []\n#     scheduler.step()\n    model.train()\n    for idx, (X_batch, y_batch) in enumerate(tqdm(train_loader)):\n        X_batch = X_batch.to(device)\n        y_batch = y_batch.to(device)\n        optimizer.zero_grad()\n        output = model(X_batch.float())\n        loss = error(output, y_batch)\n        train_loss.append(loss.item())\n        loss.backward()\n        optimizer.step()\n        \n        del X_batch, y_batch, output\n        gc.collect()\n        torch.cuda.empty_cache()\n        \n    model.eval()\n    for idx, (X_batch, y_batch) in enumerate(tqdm(valid_loader)):\n        X_batch = X_batch.to(device)\n        y_batch = y_batch.to(device)\n        output = model(X_batch.float())\n        loss = error(output, y_batch)\n        val_loss.append(loss.item())\n        output = output.to('cpu').detach().numpy()\n        output = np.argmax(output, axis = 1)\n        output = np.round(output)\n        y_batch = y_batch.to('cpu').detach().numpy()\n        kappa.append(quadratic_weighted_kappa(y_batch, output))\n        \n        del X_batch, y_batch, output\n        gc.collect()\n        torch.cuda.empty_cache()\n        \n    if np.mean(val_loss) < best:\n        best = np.mean(val_loss)\n        torch.save(model.state_dict(), 'model.bin')\n        print('Saved Model!!!')\n    print(f'Epoch {i} :')\n    print(f'Average training loss : {np.mean(train_loss)}')\n    print(f'Average validation loss : {np.mean(val_loss)}')\n    print(f'Average Quadratic Kappa : {np.mean(kappa)}')\n    print('_'* 50)","metadata":{"execution":{"iopub.status.busy":"2022-08-04T09:39:12.730989Z","iopub.execute_input":"2022-08-04T09:39:12.731423Z","iopub.status.idle":"2022-08-04T10:52:32.093490Z","shell.execute_reply.started":"2022-08-04T09:39:12.731383Z","shell.execute_reply":"2022-08-04T10:52:32.092134Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Efficientnet:\n- CV - 0.57269\n- PUBLIC - 0.343965\n- PRIVATE - 0.526175\n\n#### Aug:\n- CV - 0.55289\n- PUBLIC - \n- PRIVATE -","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}