{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Install WanDB","metadata":{}},{"cell_type":"code","source":"!pip install --upgrade -q wandb","metadata":{"execution":{"iopub.status.busy":"2022-04-19T05:14:00.794801Z","iopub.execute_input":"2022-04-19T05:14:00.795062Z","iopub.status.idle":"2022-04-19T05:14:11.064146Z","shell.execute_reply.started":"2022-04-19T05:14:00.795035Z","shell.execute_reply":"2022-04-19T05:14:11.063443Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from kaggle_secrets import UserSecretsClient\nimport wandb\nfrom wandb.keras import WandbCallback\n\nimport tensorflow as tf\nprint(tf.__version__)\nfrom tensorflow.keras import layers\nfrom tensorflow.keras import models\nimport tensorflow_addons as tfa\n\nimport os\nimport json\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n%matplotlib inline\n\nfrom sklearn.model_selection import train_test_split","metadata":{"execution":{"iopub.status.busy":"2022-04-19T05:14:11.066307Z","iopub.execute_input":"2022-04-19T05:14:11.066563Z","iopub.status.idle":"2022-04-19T05:14:11.078015Z","shell.execute_reply.started":"2022-04-19T05:14:11.066534Z","shell.execute_reply":"2022-04-19T05:14:11.076968Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Add Secret Kagle and Login into WanDB","metadata":{}},{"cell_type":"code","source":"user_secrets = UserSecretsClient()\nsecret_value_0 = user_secrets.get_secret(\"wandb_api\")\n\nwandb.login(key=wandb_api)\n# Or using\n# ! wandb login $secret_value_0","metadata":{"execution":{"iopub.status.busy":"2022-04-19T05:14:11.079682Z","iopub.execute_input":"2022-04-19T05:14:11.079974Z","iopub.status.idle":"2022-04-19T05:14:11.404924Z","shell.execute_reply.started":"2022-04-19T05:14:11.079938Z","shell.execute_reply":"2022-04-19T05:14:11.4041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def seed_everything():\n    os.environ['TF_CUDNN_DETERMINISTIC'] = '1' \n    np.random.seed(hash(\"improves reproducibility\") % 2**32 - 1)\n    tf.random.set_seed(hash(\"by removing stochasticity\") % 2**32 - 1)\n    \nseed_everything()","metadata":{"execution":{"iopub.status.busy":"2022-04-19T05:14:11.40657Z","iopub.execute_input":"2022-04-19T05:14:11.407036Z","iopub.status.idle":"2022-04-19T05:14:11.413181Z","shell.execute_reply.started":"2022-04-19T05:14:11.407001Z","shell.execute_reply":"2022-04-19T05:14:11.412367Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TRAIN_PATH = '../input/resized-plant2021/img_sz_256/'\nAUTOTUNE = tf.data.experimental.AUTOTUNE\n# Store your hyperparameters as a dictionary, because you can later directly log this config dict to W&B.\nCONFIG = dict (\n    num_labels = 6,\n    train_val_split = 0.2,\n    img_width = 224,\n    img_height = 224,\n    batch_size = 64,\n    epochs = 10,\n    learning_rate = 0.001,\n    architecture = \"CNN\",\n    infra = \"Kaggle\",\n    competition = 'plant-pathology',\n    _wandb_kernel = 'ayut'\n)","metadata":{"execution":{"iopub.status.busy":"2022-04-19T05:14:11.414283Z","iopub.execute_input":"2022-04-19T05:14:11.414918Z","iopub.status.idle":"2022-04-19T05:14:11.424348Z","shell.execute_reply.started":"2022-04-19T05:14:11.414873Z","shell.execute_reply":"2022-04-19T05:14:11.423673Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 4. Build input pipeline\n\n# Encode competition-provided labels \nlabel_to_id = {\n    'healthy': 0,\n    'scab': 1,\n    'frog_eye_leaf_spot': 2,\n    'rust': 3,\n    'complex': 4,\n    'powdery_mildew': 5\n}\nid_to_label = {value:key for key, value in label_to_id.items()} \nid_to_label","metadata":{"execution":{"iopub.status.busy":"2022-04-19T05:14:11.425536Z","iopub.execute_input":"2022-04-19T05:14:11.425958Z","iopub.status.idle":"2022-04-19T05:14:11.440457Z","shell.execute_reply.started":"2022-04-19T05:14:11.425918Z","shell.execute_reply":"2022-04-19T05:14:11.439721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Helper fu\ndef make_path(row):\n    return TRAIN_PATH+row.image\n\ndef parse_labels(row):\n    label_list = row.labels.split()\n    labels = []\n    for label in label_list:\n        labels.append(str(label_to_id[label]))\n    \n    return ' '.join(labels)\n\n# Read train.csv file\ndf = pd.read_csv('../input/plant-pathology-2021-fgvc8/train.csv')\n# Get absolute path\ndf['image'] = df.apply(lambda row: make_path(row), axis=1)\n# Parse labels\ndf['labels'] = df.apply(lambda row: parse_labels(row), axis=1)\n\n# Look at the dataframe\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2022-04-19T05:14:11.441448Z","iopub.execute_input":"2022-04-19T05:14:11.441678Z","iopub.status.idle":"2022-04-19T05:14:12.257263Z","shell.execute_reply.started":"2022-04-19T05:14:11.441651Z","shell.execute_reply":"2022-04-19T05:14:12.25672Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 5. Training and validation split\n\ntrain_df, valid_df = train_test_split(df, test_size=CONFIG['train_val_split'])\nprint(f'Number of train images: {len(train_df)} and validation images: {len(valid_df)}')","metadata":{"execution":{"iopub.status.busy":"2022-04-19T05:14:12.258451Z","iopub.execute_input":"2022-04-19T05:14:12.258849Z","iopub.status.idle":"2022-04-19T05:14:12.269591Z","shell.execute_reply.started":"2022-04-19T05:14:12.258818Z","shell.execute_reply":"2022-04-19T05:14:12.268706Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 6. Helper functions for input pipeline\n\n@tf.function\ndef decode_image(image):\n    # Convert the compressed string to a 3D uint8 tensor\n    image = tf.image.decode_jpeg(image, channels=3)\n    \n    # Normalize image\n    image = tf.image.convert_image_dtype(image, dtype=tf.float32)\n    \n    # Resize the image to the desired size\n    return image\n\n@tf.function\ndef load_image(df_dict):\n    # Load image\n    image = tf.io.read_file(df_dict['image'])\n    image = decode_image(image)\n    \n    # Resize image\n    image = tf.image.resize(image, (CONFIG['img_height'], CONFIG['img_width']))\n    \n    # Parse label\n    label = tf.strings.split(df_dict['labels'], sep='')\n    label = tf.strings.to_number(label, out_type=tf.int32)\n    label = tf.reduce_sum(tf.one_hot(indices=label, depth=CONFIG['num_labels']), axis=0)\n    \n    return image, label","metadata":{"execution":{"iopub.status.busy":"2022-04-19T05:14:12.2709Z","iopub.execute_input":"2022-04-19T05:14:12.272328Z","iopub.status.idle":"2022-04-19T05:14:12.282808Z","shell.execute_reply.started":"2022-04-19T05:14:12.272282Z","shell.execute_reply":"2022-04-19T05:14:12.282169Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 7. Build data loaders\n\nAUTOTUNE = tf.data.AUTOTUNE\n\ntrainloader = tf.data.Dataset.from_tensor_slices(dict(train_df))\nvalidloader = tf.data.Dataset.from_tensor_slices(dict(valid_df))\n\ntrainloader = (\n    trainloader\n    .shuffle(1024)\n    .map(load_image, num_parallel_calls=AUTOTUNE)\n    .batch(CONFIG['batch_size'])\n    .prefetch(AUTOTUNE)\n)\n\nvalidloader = (\n    validloader\n    .map(load_image, num_parallel_calls=AUTOTUNE)\n    .batch(CONFIG['batch_size'])\n    .prefetch(AUTOTUNE)\n)","metadata":{"execution":{"iopub.status.busy":"2022-04-19T05:14:12.286264Z","iopub.execute_input":"2022-04-19T05:14:12.286783Z","iopub.status.idle":"2022-04-19T05:14:12.436136Z","shell.execute_reply.started":"2022-04-19T05:14:12.286684Z","shell.execute_reply":"2022-04-19T05:14:12.435515Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Data loader sanity check\n\ndef show_batch(image_batch, label_batch):\n    plt.figure(figsize=(20,20))\n    for n in range(25):\n        ax = plt.subplot(5,5,n+1)\n        plt.imshow(image_batch[n])\n        plt.title(' '.join([id_to_label[i] for i, label in enumerate(label_batch[n].numpy()) if label==1.]))\n        plt.axis('off')\n\nimage_batch, label_batch = next(iter(trainloader))\nshow_batch(image_batch, label_batch)","metadata":{"execution":{"iopub.status.busy":"2022-04-19T05:27:24.345081Z","iopub.execute_input":"2022-04-19T05:27:24.346248Z","iopub.status.idle":"2022-04-19T05:27:24.510803Z","shell.execute_reply.started":"2022-04-19T05:27:24.346193Z","shell.execute_reply":"2022-04-19T05:27:24.509545Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 8. Define model: EfficientNetB0 trained on ImageNet as backbone\n\ndef get_model():\n    base_model = tf.keras.applications.EfficientNetB0(include_top=False, weights='imagenet')\n    base_model.trainabe = True\n\n    inputs = layers.Input((CONFIG['img_height'], CONFIG['img_width'], 3))\n    x = base_model(inputs, training=True)\n    x = layers.GlobalAveragePooling2D()(x)\n    x = layers.Dropout(0.5)(x)\n    outputs = layers.Dense(len(label_to_id), activation='sigmoid')(x)\n    \n    return models.Model(inputs, outputs)\n\n# Model sanity check\ntf.keras.backend.clear_session()\nmodel = get_model()\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2022-04-19T05:22:54.866701Z","iopub.execute_input":"2022-04-19T05:22:54.867111Z","iopub.status.idle":"2022-04-19T05:23:00.214284Z","shell.execute_reply.started":"2022-04-19T05:22:54.867077Z","shell.execute_reply":"2022-04-19T05:23:00.213374Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Initialize model\ntf.keras.backend.clear_session()\nmodel = get_model()\n\n# Compile model\noptimizer = tf.keras.optimizers.Adam(learning_rate=CONFIG['learning_rate'])\nmodel.compile(optimizer, \n              loss=tfa.losses.SigmoidFocalCrossEntropy(), \n              metrics=[tf.keras.metrics.AUC(multi_label=True), tfa.metrics.F1Score(num_classes=6, average='micro')])","metadata":{"execution":{"iopub.status.busy":"2022-04-19T05:23:02.539929Z","iopub.execute_input":"2022-04-19T05:23:02.540567Z","iopub.status.idle":"2022-04-19T05:23:04.984469Z","shell.execute_reply.started":"2022-04-19T05:23:02.540525Z","shell.execute_reply":"2022-04-19T05:23:04.98366Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Update CONFIG dict with the name of the model.\nCONFIG['model_name'] = 'efficientnetb0'\nprint('Training configuration: ', CONFIG)\n\n# Initialize W&B run\nrun = wandb.init(project='plant-pathology', \n                 config=CONFIG,\n                 group='EfficientNet', \n                 job_type='train')","metadata":{"execution":{"iopub.status.busy":"2022-04-19T05:23:10.804143Z","iopub.execute_input":"2022-04-19T05:23:10.804874Z","iopub.status.idle":"2022-04-19T05:23:20.293779Z","shell.execute_reply.started":"2022-04-19T05:23:10.80483Z","shell.execute_reply":"2022-04-19T05:23:20.292913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CONFIG = dict (\n    num_labels = 6,\n    train_val_split = 0.2,\n    img_width = 224,\n    img_height = 224,\n    batch_size = 64,\n    epochs = 10,\n    learning_rate = 0.001,\n    architecture = \"CNN\",\n    infra = \"Kaggle\",\n    model_name = \"efficientnetb0\"\n)","metadata":{"execution":{"iopub.status.busy":"2022-04-19T05:23:39.155667Z","iopub.execute_input":"2022-04-19T05:23:39.15604Z","iopub.status.idle":"2022-04-19T05:23:39.162404Z","shell.execute_reply.started":"2022-04-19T05:23:39.156005Z","shell.execute_reply":"2022-04-19T05:23:39.16149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Add \"type\" and \"kaggle_competition\" to `wandb.config` directly\nwandb.config.type = 'baseline'\nwandb.config.kaggle_competition = 'Plant Pathology 2021 - FGVC8'","metadata":{"execution":{"iopub.status.busy":"2022-04-19T05:14:12.572545Z","iopub.status.idle":"2022-04-19T05:14:12.572856Z","shell.execute_reply.started":"2022-04-19T05:14:12.572689Z","shell.execute_reply":"2022-04-19T05:14:12.572705Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"earlystopper = tf.keras.callbacks.EarlyStopping(\n    monitor='val_loss', patience=3, verbose=0, mode='min',\n    restore_best_weights=True\n)\n\n# Train\nmodel.fit(trainloader, \n          epochs=CONFIG['epochs'],\n          validation_data=validloader,\n          callbacks=[WandbCallback(),\n                     earlystopper])\n\n# Close W&B run\nrun.finish()","metadata":{"execution":{"iopub.status.busy":"2022-04-19T05:14:12.574011Z","iopub.status.idle":"2022-04-19T05:14:12.574359Z","shell.execute_reply.started":"2022-04-19T05:14:12.574188Z","shell.execute_reply":"2022-04-19T05:14:12.574211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Reference","metadata":{}},{"cell_type":"markdown","source":"https://www.kaggle.com/code/ayuraj/experiment-tracking-with-weights-and-biases/","metadata":{}}]}