{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Overview\n\n* Plant Pathology 2021 Competition\n* Use pretrained PyTorch ResNet model\n* Multi-label classification\n* Model exported as ONNX  ","metadata":{}},{"cell_type":"markdown","source":"## Imports","metadata":{}},{"cell_type":"code","source":"from typing import List, Dict\n\nimport random\nimport os\n\nimport numpy as np\nimport pandas as pd\nimport PIL\n\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\n\nimport torchvision\nimport torch.onnx\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torchvision import transforms as T\n\nimport skimage.io as io\nfrom tqdm.notebook import tqdm","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Configuration","metadata":{}},{"cell_type":"code","source":"import torch\nprint(torch.__version__)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import seaborn as sns\nimport matplotlib.pyplot as plt\n\nplt.rc('font', size=15)\nplt.rc('axes', titlesize=18)  \nplt.rc('xtick', labelsize=10)  \nplt.rc('ytick', labelsize=10)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Config: \n    \"\"\"\n    \"\"\"\n    DEVICE = 'cuda' if torch.cuda.is_available() else 'cpu'\n    INPUT_PATH = '../input/plant-pathology-2021-fgvc8'\n    OUTPUT_PATH = './'\n    N_EPOCH = 30\n    BATCH_SIZE = 64\n    TEST_SIZE = 0.2\n    RANDOM_STATE = 42\n    SAMPLE_FRAC = 1.0\n    IMG_SIZE = 224\n    LEARNING_RATE = 0.000001\n    TRAIN_DATA_FILE = os.path.join(INPUT_PATH, 'train.csv')\n    MODEL_ONNX_FILE = os.path.join(OUTPUT_PATH, f'plant2021_{DEVICE}.onnx')\n    INPUT_MODEL_FILE = os.path.join(INPUT_PATH, f'plant2021_{DEVICE}.pth') \n    OUTPUT_MODEL_FILE = os.path.join(OUTPUT_PATH, f'plant2021_{DEVICE}.pth')\n    CLASS_THRESHOLD = 0.4\n    CLASSES = [\n        'rust', \n        'complex', \n        'healthy', \n        'powdery_mildew', \n        'scab', \n        'frog_eye_leaf_spot'\n    ]\n    N_CLASSES = len(CLASSES)\n    \n    folders = dict({\n        'data': INPUT_PATH,\n        'train': '../input/resized-plant2021/img_sz_256',\n        'val': '../input/resized-plant2021/img_sz_256',\n        'test':  os.path.join(INPUT_PATH, 'train_images')\n    })\n    \n    @staticmethod\n    def set_seed():\n        torch.manual_seed(Config.RANDOM_STATE)\n        random.seed(Config.RANDOM_STATE)\n        np.random.seed(Config.RANDOM_STATE)\n        \nConfig.set_seed()        ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f'Using {Config.DEVICE} device.')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def to_numpy(tensor):\n    \"\"\"Auxiliary function to convert tensors into numpy arrays\n    \"\"\"\n    return tensor.detach().cpu().numpy() if tensor.requires_grad else tensor.cpu().numpy()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data Preparation","metadata":{}},{"cell_type":"code","source":"def read_image_labels():\n    \"\"\"\n    \"\"\"\n    df = pd.read_csv(Config.TRAIN_DATA_FILE).set_index('image')\n    return df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img_labels = read_image_labels().sample(\n    frac=Config.SAMPLE_FRAC, \n    random_state=Config.RANDOM_STATE\n)\n\nimg_labels.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Label distribution","metadata":{}},{"cell_type":"code","source":"def get_image_infos(img_labels):\n    \"\"\"\n    \"\"\"\n    df = img_labels.reset_index().groupby(by='labels').count().reset_index()\n    df.columns = ['disease', 'count']\n    \n    df['%'] = np.round((df['count'] / img_labels.shape[0]), 2) * 100\n    df = df.set_index('disease').sort_values(by='count', ascending=False)\n\n    return df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"get_image_infos(img_labels)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_image_counts(img_labels):\n    fig, ax = plt.subplots(figsize=(18, 7))\n    sns.set_style(\"whitegrid\")\n    palette = sns.color_palette(\"Blues_r\", 12)\n\n    sns.countplot(\n        x='labels', \n        palette=palette,\n        data=img_labels,\n        order=img_labels['labels'].value_counts().index,\n    );\n\n    plt.ylabel(\"# of observations\", size=20);\n    plt.xlabel(\"Class names\", size=20)\n\n    plt.xticks(rotation=45)\n    \n    fig.tight_layout()\n    plt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_image_counts(img_labels)  ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img_labels.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## One hot encoding","metadata":{}},{"cell_type":"code","source":"def get_single_labels(unique_labels) -> List[str]:\n    \"\"\"Splitting multi-labels and returning a list of classes\"\"\"\n    single_labels = []\n    \n    for label in unique_labels:\n        single_labels += label.split()\n        \n    single_labels = set(single_labels)\n    return list(single_labels)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_one_hot_encoded_labels(dataset_df) -> pd.DataFrame:\n    \"\"\"\n    \"\"\"\n    df = dataset_df.copy()\n    \n    unique_labels = df.labels.unique()\n    column_names = get_single_labels(unique_labels)\n    \n    df[column_names] = 0        \n    \n    # one-hot-encoding\n    for label in unique_labels:                \n        label_indices = df[df['labels'] == label].index\n        splited_labels = label.split()\n        df.loc[label_indices, splited_labels] = 1\n    \n    return df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"one_hot_encoded_labels = get_one_hot_encoded_labels(img_labels)\none_hot_encoded_labels.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Visualization of images","metadata":{}},{"cell_type":"code","source":"def get_image(image_id, kind='train'):\n    \"\"\"Loads an image from file\n    \"\"\"\n    fname = os.path.join(Config.folders[kind], image_id)\n    return PIL.Image.open(fname)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def visualize_images(image_ids, labels, nrows=1, ncols=4, kind='train', image_transform=None):\n    \"\"\"\n    \"\"\"\n    fig, axes = plt.subplots(nrows=nrows, ncols=ncols, figsize=(20, 8))\n    for image_id, label, ax in zip(image_ids, labels, axes.flatten()):\n        \n        fname = os.path.join(Config.folders[kind], image_id)\n        image = np.array(PIL.Image.open(fname))\n        \n        if image_transform:\n            image = transform = A.Compose(\n                [t for t in image_transform.transforms if not isinstance(t, (\n                    A.Normalize, \n                    ToTensorV2\n                ))])(image=image)['image']\n        \n        io.imshow(image, ax=ax)\n        \n        ax.set_title(f\"Class: {label}\", fontsize=12)\n        ax.get_xaxis().set_visible(False)\n        ax.get_yaxis().set_visible(False)\n        \n        del image\n        \n    plt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"visualize_images(img_labels.index, img_labels.labels, nrows=2, ncols=4)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Augmentation pipeline","metadata":{}},{"cell_type":"code","source":"train_transform = A.Compose([\n    A.Rotate(\n        always_apply=False, \n        p=0.1, \n        limit=(-68, 178), \n        interpolation=1, \n        border_mode=0, \n        value=(0, 0, 0), \n        mask_value=None\n    ),\n    A.RandomShadow(\n        num_shadows_lower=1, \n        num_shadows_upper=1, \n        shadow_dimension=3, \n        shadow_roi=(0, 0.6, 1, 1), \n        p=0.4\n    ),\n    A.ShiftScaleRotate(\n        shift_limit=0.05, \n        scale_limit=0.05, \n        rotate_limit=15, \n        p=0.6\n    ),\n    A.RandomFog(\n        fog_coef_lower=0.2, \n        fog_coef_upper=0.2, \n        alpha_coef=0.2, \n        p=0.3\n    ),\n    A.RGBShift(\n        r_shift_limit=15, \n        g_shift_limit=15, \n        b_shift_limit=15, \n        p=0.3\n    ),\n    A.RandomBrightnessContrast(\n        p=0.3\n    ),\n    A.GaussNoise(\n        var_limit=(50, 70),  \n        always_apply=False, \n        p=0.3\n    ),\n    A.Resize(\n        height=Config.IMG_SIZE,\n        width=Config.IMG_SIZE,\n    ),\n    A.CoarseDropout(\n        max_holes=5, \n        max_height=5, \n        max_width=5, \n        min_holes=3, \n        min_height=5, \n        min_width=5,\n        always_apply=False, \n        p=0.2\n    ),\n    A.Normalize(\n        mean=(0.485, 0.456, 0.406), \n        std=(0.229, 0.224, 0.225)\n    ),\n    ToTensorV2(),\n])\n\nval_transform = A.Compose([\n    A.Resize(\n        height=Config.IMG_SIZE,\n        width=Config.IMG_SIZE,\n    ),\n    A.Normalize(\n        mean=(0.485, 0.456, 0.406), \n        std=(0.229, 0.224, 0.225)\n    ),\n    ToTensorV2(),\n])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"images = img_labels.sample(n=5)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"visualize_images(\n    images.index, \n    images.labels, \n    nrows=1,\n    ncols=5,\n    image_transform=train_transform,\n    kind='train'\n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"visualize_images(\n    images.index, \n    images.labels, \n    nrows=1,\n    ncols=5,\n    image_transform=val_transform,\n    kind='test'\n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Database","metadata":{}},{"cell_type":"code","source":"from scipy.stats import bernoulli\nfrom torch.utils.data import Dataset\n\nclass PlantDataset(Dataset):\n    \"\"\"\n    \"\"\"\n    def __init__(self, \n                 image_ids, \n                 targets,\n                 transform=None, \n                 target_transform=None, \n                 kind='train'):\n        self.image_ids = image_ids\n        self.targets = targets\n        self.transform = transform\n        self.target_transform = target_transform\n        self.kind = kind\n    \n    def __len__(self):\n        return len(self.image_ids)\n    \n    def __getitem__(self, idx):\n        # load and transform image\n        img = np.array(get_image(self.image_ids.iloc[idx], kind=self.kind))\n        \n        if self.transform:\n            img = self.transform(image=img)['image']\n        \n        # get image target \n        target = self.targets[idx]\n        if self.target_transform:\n            target = self.target_transform(target)\n        \n        return img, target","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\nX_train, X_vaild, y_train, y_vaild = train_test_split(\n    pd.Series(img_labels.index), \n    np.array(one_hot_encoded_labels[Config.CLASSES]),  \n    test_size=Config.TEST_SIZE, \n    random_state=Config.RANDOM_STATE\n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_set = PlantDataset(X_train, y_train, transform=train_transform, kind='train')\nval_set = PlantDataset(X_vaild, y_vaild, transform=val_transform, kind='val')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f'Train size: {len(train_set)}')\nprint(f'Validation size: {len(val_set)}')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torch.utils.data import DataLoader\nfrom torch.nn import BatchNorm2d\n\ntrain_loader = DataLoader(train_set, batch_size=Config.BATCH_SIZE, shuffle=True)\nvalid_loader = DataLoader(val_set, batch_size=Config.BATCH_SIZE, shuffle=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Create ResNet model (pretrained)","metadata":{}},{"cell_type":"code","source":"def load_model(model, load_path=Config.INPUT_MODEL_FILE):\n    model.load_state_dict(torch.load(load_path))\n    model.eval()\n    \ndef save_weights(model, save_path=Config.OUTPUT_MODEL_FILE):\n    torch.save(model.state_dict(), save_path)\n\ndef create_model(pretrained=True):\n    model = torchvision.models.resnet50(pretrained=pretrained).to(Config.DEVICE)\n    \n    for param in model.layer1.parameters():\n        param.requires_grad = False\n        \n    for param in model.layer2.parameters():\n        param.requires_grad = False  \n        \n    for param in model.layer3.parameters():\n        param.requires_grad = False \n    \n    model.fc = torch.nn.Sequential(\n        torch.nn.Linear(\n            in_features=model.fc.in_features,\n            out_features=Config.N_CLASSES\n        ),\n        torch.nn.Sigmoid()\n    ).to(Config.DEVICE)\n    \n    return model","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = create_model(pretrained=True).to(Config.DEVICE);","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Train model","metadata":{}},{"cell_type":"code","source":"class MetricMonitor:\n    def __init__(self):\n        self.reset()\n\n    def reset(self):\n        self.losses = []\n        self.accuracies = []\n        self.scores = []\n        self.metrics = dict({\n            'loss': self.losses,\n            'acc': self.accuracies,\n            'f1': self.scores\n        })\n\n    def update(self, metric_name, value):\n        self.metrics[metric_name] += [value]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import f1_score, accuracy_score\n\ndef get_metrics(\n    y_pred_proba, \n    y_test, \n    threshold=Config.CLASS_THRESHOLD,\n    labels=Config.CLASSES) -> None:\n    \"\"\"\n    \"\"\"\n    y_pred = np.where(y_pred_proba > threshold, 1, 0)\n\n    y1 = y_pred.round().astype(np.float)\n    y2 = y_test.round().astype(np.float)\n    \n    f1 = f1_score(y1, y2, average='micro')\n    acc = accuracy_score(y1, y2, normalize=True)\n\n    return acc, f1","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def training_loop(\n    dataloader, \n    model, \n    loss_fn, \n    optimizer, \n    epoch, \n    monitor = MetricMonitor(), \n    is_train=True\n) -> None:\n    \"\"\"\n    \"\"\"\n    size = len(dataloader.dataset)\n    \n    loss_val = 0\n    accuracy = 0\n    f1score = 0\n    \n    if is_train:\n        model.train()\n    else:\n        model.eval()\n    \n    stream = tqdm(dataloader)\n    for batch, (X, y) in enumerate(stream, start=1):\n        X = X.to(Config.DEVICE)\n        y = y.to(Config.DEVICE)\n        \n        # compute prediction and loss\n        pred_prob = model(X)\n        loss = loss_fn(pred_prob, y)\n    \n        if is_train:\n            # backpropagation\n            optimizer.zero_grad()\n            loss.backward()\n            optimizer.step()\n\n        \n        loss_val += loss.item()\n        acc, f1 = get_metrics(to_numpy(pred_prob), to_numpy(y))\n        \n        accuracy += acc \n        f1score += f1\n\n        phase = 'Train' if is_train else 'Val'\n        stream.set_description(\n            f'Epoch {epoch:3d}/{Config.N_EPOCH} - {phase} - Loss: {loss_val/batch:.4f}, ' + \n            f'Acc: {accuracy/batch:.4f}, F1: {f1score/batch:.4f}'\n        )\n\n    monitor.update('loss', loss_val/batch)\n    monitor.update('acc', accuracy/batch)\n    monitor.update('f1', f1score/batch) ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_monitor = MetricMonitor()\ntest_monitor = MetricMonitor()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# initialize the loss function\nloss_fn = nn.MultiLabelSoftMarginLoss()\n\noptimizer = torch.optim.Adam(\n    model.parameters(),\n    lr=Config.LEARNING_RATE\n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nfor epoch in range(1, Config.N_EPOCH + 1):\n    # training loop\n    training_loop(\n        train_loader, \n        model, \n        loss_fn, \n        optimizer, \n        epoch, \n        train_monitor,\n        is_train=True\n    )\n    \n    # validation loop\n    training_loop(\n        valid_loader, \n        model, \n        loss_fn, \n        optimizer, \n        epoch, \n        test_monitor,\n        is_train=False\n    )","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Plot metrics","metadata":{}},{"cell_type":"code","source":"from matplotlib.ticker import MaxNLocator \n\ndef plot_result(\n    train_losses, \n    test_losses, \n    train_accuracies, \n    test_accuracies, \n    train_scores,\n    test_scores\n) -> None:\n    \n    epochs = range(1, len(train_losses) + 1)\n    fig, ax = plt.subplots(nrows=1, ncols=3, figsize=(22, 5))\n    \n    # plot loss values\n    ax[0].plot(epochs, train_losses, label='Training loss', marker ='o')\n    ax[0].plot(epochs, test_losses, label='Validation loss', marker ='o')\n    ax[0].legend(frameon=False, fontsize=14)\n    \n    ax[0].get_xaxis().set_major_locator(MaxNLocator(integer=True))\n    ax[0].set_title('Loss', fontsize=18)\n    ax[0].set_xlabel('Epoch', fontsize=14) \n    ax[0].set_ylabel('Loss', fontsize=14)  \n    \n    # plot accuracies \n    ax[1].plot(epochs, train_accuracies, label='Training Accuracy', marker ='o')\n    ax[1].plot(epochs, test_accuracies, label='Validation accuracy', marker ='o')\n    ax[1].legend(frameon=False, fontsize=14)\n    \n    ax[1].get_xaxis().set_major_locator(MaxNLocator(integer=True))\n    ax[1].set_title('Accuracy', fontsize=18)\n    ax[1].set_xlabel('Epoch', fontsize=14) \n    ax[1].set_ylabel('Accuracy', fontsize=14)\n    \n    ax[2].plot(epochs, train_scores, label='Training F1-Score', marker ='o')\n    ax[2].plot(epochs, test_scores, label='Validation F1-Score', marker ='o')\n    ax[2].legend(frameon=False, fontsize=14)\n    \n    ax[2].get_xaxis().set_major_locator(MaxNLocator(integer=True))\n    ax[2].set_title('F1-Score', fontsize=18)\n    ax[2].set_xlabel('Epoch', fontsize=14) \n    ax[2].set_ylabel('F1-Score', fontsize=14) \n        \n    plt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_result(\n    train_monitor.losses, \n    test_monitor.losses,\n    train_monitor.accuracies, \n    test_monitor.accuracies, \n    train_monitor.scores,\n    test_monitor.scores\n)    ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Export to ONNX","metadata":{}},{"cell_type":"code","source":"def export_model(model):\n    dummy_input = torch.randn([\n        Config.BATCH_SIZE, \n        3, \n        Config.IMG_SIZE, \n        Config.IMG_SIZE\n    ]).to(Config.DEVICE)\n    dummy_output = model(dummy_input)\n\n    # Export the model\n    torch.onnx.export(\n        model,               \n        dummy_input,                        \n        Config.MODEL_ONNX_FILE,   \n        export_params=True,        \n        opset_version=10,          # the ONNX version to export the model to\n        do_constant_folding=True,   \n        input_names = ['input'],   # the model's input names\n        output_names = ['output'], # the model's output names\n        dynamic_axes=\n        {\n            'input': { 0: 'batch_size'},    # variable lenght axes\n            'output': { 0: 'batch_size'}\n        }\n    )","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"export_model(model) # export model as ONNX\nsave_weights(model)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Confusion matrix","metadata":{}},{"cell_type":"code","source":"batch = Config.BATCH_SIZE\n\ny_true = np.empty(shape=(0, 6), dtype=np.int)\ny_pred_proba = np.empty(shape=(0, 6), dtype=np.int)\n\nstream = tqdm(valid_loader)\nfor batch, (X, y) in enumerate(stream, start=1):\n    X = X.to(Config.DEVICE)\n    y = to_numpy(y.to(Config.DEVICE))\n    pred = to_numpy(model(X))\n    \n    y_true = np.vstack((y_true, y))\n    y_pred_proba = np.vstack((y_pred_proba, pred))\n    ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import multilabel_confusion_matrix\n\ndef plot_confusion_matrix(\n    y_test, \n    y_pred_proba, \n    threshold=Config.CLASS_THRESHOLD, \n    label_names=Config.CLASSES\n)-> None:\n    \"\"\"\n    \"\"\"\n    y_pred = np.where(y_pred_proba > threshold, 1, 0)\n    c_matrices = multilabel_confusion_matrix(y_test, y_pred)\n    \n    cmap = plt.get_cmap('Blues')\n    fig, axes = plt.subplots(nrows=2, ncols=3, figsize=(15, 8))\n\n    for cm, label, ax in zip(c_matrices, label_names, axes.flatten()):\n        sns.heatmap(cm, annot=True, fmt='g', ax=ax, cmap=cmap);\n\n        ax.set_xlabel('Predicted labels');\n        ax.set_ylabel('True labels'); \n        ax.set_title(f'{label}');\n\n    plt.tight_layout()    \n    plt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_confusion_matrix(y_true, y_pred_proba)    ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Scores","metadata":{}},{"cell_type":"code","source":"y_pred = np.where(y_pred_proba > 0.3, 1, 0)\naccuracy, f1 = get_metrics(y_pred, y_true)\n\npd.DataFrame({\n    'name': ['F1', 'Accuracy'],\n    'sorce': [f1, accuracy]\n}).set_index('name')\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}