{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-04-05T13:20:49.377520Z","iopub.execute_input":"2023-04-05T13:20:49.378148Z","iopub.status.idle":"2023-04-05T13:21:00.183576Z","shell.execute_reply.started":"2023-04-05T13:20:49.378074Z","shell.execute_reply":"2023-04-05T13:21:00.182451Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pip install vit-keras","metadata":{"execution":{"iopub.status.busy":"2023-04-05T13:21:00.188736Z","iopub.execute_input":"2023-04-05T13:21:00.189577Z","iopub.status.idle":"2023-04-05T13:21:13.172968Z","shell.execute_reply.started":"2023-04-05T13:21:00.189531Z","shell.execute_reply":"2023-04-05T13:21:13.171722Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport pandas as pd\nimport numpy as np\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import classification_report\nfrom transformers import ViTModel, TFAutoModel, AutoTokenizer\nfrom tensorflow.keras.applications import EfficientNetB3, ResNet50\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.layers import Dense, Dropout, Input, GlobalAveragePooling2D\nfrom tensorflow.keras.optimizers import Adam\nfrom vit_keras import visualize, vit","metadata":{"execution":{"iopub.status.busy":"2023-04-05T13:21:13.175194Z","iopub.execute_input":"2023-04-05T13:21:13.175684Z","iopub.status.idle":"2023-04-05T13:21:13.185042Z","shell.execute_reply.started":"2023-04-05T13:21:13.175632Z","shell.execute_reply":"2023-04-05T13:21:13.183966Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Set seed for reproducibility\nnp.random.seed(42)\ntf.random.set_seed(42)\n\n# Load and preprocess data\ntrain_df = pd.read_csv('/kaggle/input/cassava-leaf-disease-classification/train.csv')\ntrain_df['label'] = train_df['label'].astype(str)\ntrain_df['path'] = train_df['image_id'].apply(lambda x: '/kaggle/input/cassava-leaf-disease-classification/train_images/' + x)\ntrain_df = train_df[['path', 'label']]\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-04-05T13:21:13.188660Z","iopub.execute_input":"2023-04-05T13:21:13.189061Z","iopub.status.idle":"2023-04-05T13:21:13.583675Z","shell.execute_reply.started":"2023-04-05T13:21:13.189034Z","shell.execute_reply":"2023-04-05T13:21:13.582480Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Split data into train and validation sets\ntrain_set, val_set = train_test_split(train_df, test_size=0.2, random_state=42)\n\n# Define image data generator with augmentation\ndatagen_train = ImageDataGenerator(\n    rotation_range=90,\n    zoom_range=0.2,\n    width_shift_range=0.1,\n    height_shift_range=0.1,\n    horizontal_flip=True,\n    vertical_flip=True,\n    preprocessing_function=tf.keras.applications.resnet50.preprocess_input\n)\n\ndatagen_val = ImageDataGenerator(\n    preprocessing_function=tf.keras.applications.resnet50.preprocess_input\n)","metadata":{"execution":{"iopub.status.busy":"2023-04-05T13:21:13.585494Z","iopub.execute_input":"2023-04-05T13:21:13.585886Z","iopub.status.idle":"2023-04-05T13:21:13.599965Z","shell.execute_reply.started":"2023-04-05T13:21:13.585845Z","shell.execute_reply":"2023-04-05T13:21:13.598713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Set image input size and batch size\nimg_size = (224, 224)\nbatch_size = 16\n\n# Create train and validation generators\ntrain_gen = datagen_train.flow_from_dataframe(\n    train_set,\n    x_col='path',\n    y_col='label',\n    target_size=img_size,\n    batch_size=batch_size,\n    class_mode='sparse',\n    shuffle=True,\n    seed=42\n)","metadata":{"execution":{"iopub.status.busy":"2023-04-05T13:21:13.601902Z","iopub.execute_input":"2023-04-05T13:21:13.602599Z","iopub.status.idle":"2023-04-05T13:21:13.787866Z","shell.execute_reply.started":"2023-04-05T13:21:13.602553Z","shell.execute_reply":"2023-04-05T13:21:13.786863Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_gen = datagen_val.flow_from_dataframe(\n    val_set,\n    x_col='path',\n    y_col='label',\n    target_size=img_size,\n    batch_size=batch_size,\n    class_mode='sparse',\n    shuffle=False\n)\n","metadata":{"execution":{"iopub.status.busy":"2023-04-05T13:21:13.789366Z","iopub.execute_input":"2023-04-05T13:21:13.790015Z","iopub.status.idle":"2023-04-05T13:21:13.956686Z","shell.execute_reply.started":"2023-04-05T13:21:13.789951Z","shell.execute_reply":"2023-04-05T13:21:13.955667Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define CNN model\ncnn_model = keras.models.Sequential([\n    keras.applications.ResNet50(\n        include_top=False,\n        input_shape=img_size + (3,),\n        pooling='avg',\n        weights='imagenet'\n    ),\n    keras.layers.Dense(512, activation='relu'),\n    keras.layers.Dropout(0.5),\n    keras.layers.Dense(5, activation='softmax')\n])","metadata":{"execution":{"iopub.status.busy":"2023-04-05T13:21:13.958204Z","iopub.execute_input":"2023-04-05T13:21:13.958725Z","iopub.status.idle":"2023-04-05T13:21:16.744741Z","shell.execute_reply.started":"2023-04-05T13:21:13.958684Z","shell.execute_reply":"2023-04-05T13:21:16.743722Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define ViT model\ndef create_vit_model(img_size):\n    input_layer = layers.Input(shape=img_size + (3,))\n    x = layers.Rescaling(1./255)(input_layer)\n    x = layers.RandomFlip(\"horizontal_and_vertical\")(x)\n    x = layers.RandomRotation(0.1)(x)\n    x = layers.RandomZoom(0.1)(x)\n    x = layers.RandomCrop(img_size)(x)\n    x = layers.experimental.preprocessing.Normalization()(x)\n    x = layers.experimental.preprocessing.Resizing(224, 224, interpolation=\"bilinear\")(x)\n\n    pretrained = ViTModel.from_pretrained('google/vit-base-patch16-224')\n    x = pretrained(x)[0]\n    x = layers.Dropout(0.5)(x)\n    x = layers.Dense(5, activation='softmax')(x)\n\n    model = keras.Model(inputs=input_layer, outputs=x)\n\n    return model","metadata":{"execution":{"iopub.status.busy":"2023-04-05T13:21:16.746270Z","iopub.execute_input":"2023-04-05T13:21:16.746656Z","iopub.status.idle":"2023-04-05T13:21:16.757893Z","shell.execute_reply.started":"2023-04-05T13:21:16.746617Z","shell.execute_reply":"2023-04-05T13:21:16.756745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from vit_keras import vit\n\ndef create_vit_model(img_size):\n    input_layer = layers.Input(shape=img_size + (3,))\n    # normalize image pixel values\n    x = layers.experimental.preprocessing.Normalization()(input_layer)\n    # resize image to expected shape\n    x = layers.experimental.preprocessing.Resizing(224, 224, interpolation=\"bilinear\")(x)\n    # apply ViT model to the input\n    vit_model = vit.vit_b16(\n        image_size=224,\n        activation=\"softmax\",\n        pretrained=True,\n        include_top=True,\n        pretrained_top=False,\n        classes=5,\n    )\n    vit_output = vit_model(x)\n    # create and return the model\n    return models.Model(input_layer, vit_output)\n","metadata":{"execution":{"iopub.status.busy":"2023-04-05T13:21:16.762878Z","iopub.execute_input":"2023-04-05T13:21:16.763149Z","iopub.status.idle":"2023-04-05T13:21:16.772459Z","shell.execute_reply.started":"2023-04-05T13:21:16.763122Z","shell.execute_reply":"2023-04-05T13:21:16.771449Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras import models\n","metadata":{"execution":{"iopub.status.busy":"2023-04-05T13:21:16.774148Z","iopub.execute_input":"2023-04-05T13:21:16.774572Z","iopub.status.idle":"2023-04-05T13:21:16.783745Z","shell.execute_reply.started":"2023-04-05T13:21:16.774532Z","shell.execute_reply":"2023-04-05T13:21:16.782674Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create an ensemble of ViT and CNN models\nvit = create_vit_model(img_size)\ncnn = cnn_model\ninput_layer = Input(shape=img_size + (3,))\nvit_output = vit(input_layer)\ncnn_output = cnn(input_layer)\nmerged = layers.concatenate([vit_output, cnn_output])\nensemble_model = Model(inputs=input_layer, outputs=merged)","metadata":{"execution":{"iopub.status.busy":"2023-04-05T13:21:16.786941Z","iopub.execute_input":"2023-04-05T13:21:16.787850Z","iopub.status.idle":"2023-04-05T13:21:48.716305Z","shell.execute_reply.started":"2023-04-05T13:21:16.787813Z","shell.execute_reply":"2023-04-05T13:21:48.715296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Compile ensemble model\noptimizer = keras.optimizers.Adam(lr=0.0001)\nloss = 'sparse_categorical_crossentropy'\nmetrics = ['accuracy']\nvit.compile(optimizer=optimizer, loss=loss, metrics=metrics)\ncnn.compile(optimizer=optimizer, loss=loss, metrics=metrics)\n\n# Combine models into an ensemble\ncombined_input = layers.Input(shape=img_size + (3,))\nvit_output = vit(combined_input)\ncnn_output = cnn(combined_input)\ncombined_output = layers.Average\n","metadata":{"execution":{"iopub.status.busy":"2023-04-05T13:21:48.717650Z","iopub.execute_input":"2023-04-05T13:21:48.718771Z","iopub.status.idle":"2023-04-05T13:21:50.928162Z","shell.execute_reply.started":"2023-04-05T13:21:48.718732Z","shell.execute_reply":"2023-04-05T13:21:50.927091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Compile ensemble model\nensemble_model.compile(\n    loss='sparse_categorical_crossentropy',\n    optimizer=keras.optimizers.Adam(lr=1e-4),\n    metrics=['accuracy']\n)","metadata":{"execution":{"iopub.status.busy":"2023-04-05T13:21:50.929863Z","iopub.execute_input":"2023-04-05T13:21:50.930281Z","iopub.status.idle":"2023-04-05T13:21:50.953043Z","shell.execute_reply.started":"2023-04-05T13:21:50.930242Z","shell.execute_reply":"2023-04-05T13:21:50.951990Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define callbacks for early stopping and saving best model\nearly_stopping = EarlyStopping(\n    monitor='val_loss',\n    patience=5,\n    restore_best_weights=True,\n    mode='min'\n)\n\ncheckpoint = ModelCheckpoint(\n    '/kaggle/working/best_model.h5',\n    save_best_only=True,\n    monitor='val_loss',\n    mode='min'\n)","metadata":{"execution":{"iopub.status.busy":"2023-04-05T13:21:50.954707Z","iopub.execute_input":"2023-04-05T13:21:50.955076Z","iopub.status.idle":"2023-04-05T13:21:50.961026Z","shell.execute_reply.started":"2023-04-05T13:21:50.955038Z","shell.execute_reply":"2023-04-05T13:21:50.959626Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Train ensemble model\nhistory = ensemble_model.fit(\n    train_gen,\n    epochs=10,\n    validation_data=val_gen,\n    callbacks=[early_stopping, checkpoint]\n)","metadata":{"execution":{"iopub.status.busy":"2023-04-05T13:21:50.962969Z","iopub.execute_input":"2023-04-05T13:21:50.963983Z","iopub.status.idle":"2023-04-05T15:24:35.009875Z","shell.execute_reply.started":"2023-04-05T13:21:50.963952Z","shell.execute_reply":"2023-04-05T15:24:35.008739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load best model and predict on test set\nbest_model = keras.models.load_model('/kaggle/working/best_model.h5')\ntest_df = pd.read_csv('/kaggle/input/cassava-leaf-disease-classification/sample_submission.csv')\ntest_df['path'] = test_df['image_id'].apply(lambda x: '/kaggle/input/cassava-leaf-disease-classification/test_images/' + x)\ntest_gen = datagen_val.flow_from_dataframe(\n    test_df,\n    x_col='path',\n    y_col=None,\n    target_size=img_size,\n    batch_size=batch_size,\n    class_mode=None,\n    shuffle=False\n)\ntest_pred = np.argmax(best_model.predict(test_gen), axis=1)","metadata":{"execution":{"iopub.status.busy":"2023-04-05T15:24:35.011412Z","iopub.execute_input":"2023-04-05T15:24:35.012131Z","iopub.status.idle":"2023-04-05T15:24:53.128186Z","shell.execute_reply.started":"2023-04-05T15:24:35.012088Z","shell.execute_reply":"2023-04-05T15:24:53.127027Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Save predictions to CSV\ntest_df['label'] = test_pred.astype(int)\ntest_df[['image_id', 'label']].to_csv('/kaggle/working/submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2023-04-05T15:24:53.130442Z","iopub.execute_input":"2023-04-05T15:24:53.131629Z","iopub.status.idle":"2023-04-05T15:24:53.170189Z","shell.execute_reply.started":"2023-04-05T15:24:53.131588Z","shell.execute_reply":"2023-04-05T15:24:53.169289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Evaluate the ensemble model on the validation set\nval_loss, val_acc = ensemble_model.evaluate(val_gen, verbose=1)\nprint(f'Validation accuracy: {val_acc:.4f}')\n\n# Make predictions on the validation set\nval_preds = ensemble_model.predict(val_gen)\nval_pred_labels = np.argmax(val_preds, axis=1)\n\n# Print classification report\nprint(classification_report(val_gen.labels, val_pred_labels))","metadata":{"execution":{"iopub.status.busy":"2023-04-05T15:24:53.172177Z","iopub.execute_input":"2023-04-05T15:24:53.173284Z","iopub.status.idle":"2023-04-05T15:27:38.427139Z","shell.execute_reply.started":"2023-04-05T15:24:53.173246Z","shell.execute_reply":"2023-04-05T15:27:38.425904Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}