{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"# Ulrich - B0, B2, B4\n# Piyush - B1, B3, B5\n# Kai - B6, B7\n# Ming - metadata","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nimport tensorflow as tf\nfrom tqdm import tqdm\nimport csv\nimport PIL\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session\n\nprint(\"Tensorflow version \" + tf.__version__)\nAUTO = tf.data.experimental.AUTOTUNE","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Detect hardware, return appropriate distribution strategy\ntry:\n    # TPU detection. No parameters necessary if TPU_NAME environment variable is\n    # set: this is always the case on Kaggle.\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Running on TPU ', tpu.master())\nexcept ValueError as e:\n    tpu = None\n    print(e)\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    # Default distribution strategy in Tensorflow. Works on CPU and single GPU.\n    strategy = tf.distribute.get_strategy()\n\nREPLICAS = strategy.num_replicas_in_sync\nprint(\"REPLICAS: \", REPLICAS)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"pip install -U efficientnet","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_folder = '../input/melanoma-foldered-collection/kaggle/working/224x224-dataset-melanoma'\ntest_folder = '../input/melanoma-test/kaggle/working/224x224-test'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"nb_mel = len(os.listdir(os.path.join(train_folder,'melanoma')))\nnb_other = len(os.listdir(os.path.join(train_folder,'other')))\nprint(f'{nb_mel} melanoma training samples')\nprint(f'{nb_other} other training samples')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import random\n\nseed_nr = 10\nrandom.seed(seed_nr)\n\ndef add_unique_idx(seq,i):\n    idx = random.randint(0,nb_other-1)\n    if not idx in seq:\n        seq[i] = idx\n    else:\n        add_unique_idx(seq,i)\n\nrandom_sequence = np.zeros(nb_mel,dtype='int32')\nfor i in range(nb_mel):\n    add_unique_idx(random_sequence,i)\nvalues, counts = np.unique(random_sequence,return_counts=True)\nassert(len(values)==nb_mel) # nb_mel unique indices","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"custom_train_folder = '/kaggle/working/training_subset'\n\nmelanoma_folder = '/kaggle/working/training_subset/melanoma'\nif not os.path.exists(melanoma_folder):\n    os.makedirs(melanoma_folder)\n\nother_subset_folder = '/kaggle/working/training_subset/other'\nif not os.path.exists(other_subset_folder):\n    os.makedirs(other_subset_folder)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from shutil import copyfile\n\n# Transfer files\n# Melanoma\nmelanoma_src = os.path.join(train_folder,'melanoma')\nfor file in tqdm(os.listdir(melanoma_src)):\n    dest = os.path.join(melanoma_folder,file)\n    src = os.path.join(melanoma_src,file)\n    copyfile(src,dest)\n    \n# Other\nother_src = os.path.join(train_folder,'other')\nfor idx, file in tqdm(enumerate(os.listdir(other_src))):\n    if idx in random_sequence:\n        dest = os.path.join(other_subset_folder,file)\n        src = os.path.join(other_src,file)\n        copyfile(src,dest)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"assert(len(os.listdir(os.path.join(melanoma_folder)))==nb_mel)\nassert(len(os.listdir(os.path.join(other_subset_folder)))==nb_mel)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import efficientnet.tfkeras as efn \n\nWIDTH = 224\nHEIGHT = 224\nCHANNELS = 3\ninput_shape = (WIDTH,HEIGHT,CHANNELS)\n# TODO\n# Try EfficientNetB0 up until EfficientNetB7 with varying pooling parameter as well.\n# Running all combinations is not necessary, we just want to explore some configurations.\nwith strategy.scope():\n    eff_model = efn.EfficientNetB0(include_top=False, weights='imagenet', input_shape=input_shape, pooling=None) # TODO: try pooling = 'avg' and pooling = 'max'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(eff_model.summary())\n# There is no average or max global pooling at the end, probably not ideal but this is only a demo.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# TODO: find first layer from last block\n# in case of EfficientNetB0, it is block7a_expand_conv (first layer from last block 7a)\n# All layers starting from this layer are finetuned and are thus trainable, all previous layers' knowledge will be transferred and are thus untrainable\nwith strategy.scope():\n    first_trainable_layer = 'block7a_expand_conv'\n    reached = False\n    for idx, layer in enumerate(eff_model.layers):\n        if layer.name == first_trainable_layer:\n            reached = True\n        if not reached:\n            layer.trainable = False\n        #print(layer.name)\n        #print(layer.trainable)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.models import Model\nfrom tensorflow.keras.layers import Dense, Flatten\nfrom tensorflow.keras.metrics import Accuracy, Precision, Recall, AUC\nfrom tensorflow.keras.optimizers import Adam, Adamax, RMSprop, SGD\n\nwith strategy.scope():\n    # 1 output melanoma probability, 1 output other probability (sum = 1)\n    final_layer = Dense(2,activation='softmax',name='final_layer')(Flatten()(eff_model.output))\n    eff_finetuning = Model(eff_model.input,final_layer)\n    print(eff_finetuning.summary())\n    acc = Accuracy()\n    pre = Precision()\n    rec = Recall()\n    auc = AUC()\n\n    lr = 0.0001\n    opt = Adam(learning_rate=lr)\n\n    eff_finetuning.compile(optimizer=opt,loss='categorical_crossentropy',metrics=[acc,pre,rec,auc])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# TODO: inspect amount of trainable parameters in your model\n# 1.254.834 trainable parameters in EfficientNetB0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.preprocessing.image import ImageDataGenerator","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Augmentation arguments will be added in the future. We first start exploring non-augmented models (runs faster).\ntrain_generator = ImageDataGenerator(rescale=1./255,validation_split=0.0)\n#test_generator = ImageDataGenerator(rescale=1./255)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size = 128*REPLICAS\ntrain_flow = train_generator.flow_from_directory(custom_train_folder,target_size=(WIDTH,HEIGHT),batch_size=batch_size,class_mode='categorical')\n#test_flow = test_generator.flow_from_directory(test_folder,target_size=(WIDTH,HEIGHT),batch_size=batch_size,class_mode='categorical')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# TODO: change epochs\n# 50 epochs should be enough for first impression\nepochs = 50\n# TODO\n# Check final loss and metrics after training\neff_finetuning.fit(train_flow,epochs=epochs,verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# TODO\n# Save model (change name)\nmodel_name = 'effnet_b0_nopool.hdf5'\neff_finetuning.save_weights(model_name)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_csv = '../input/siim-isic-melanoma-classification/test.csv'\nwith open(test_csv,'r') as test_file:\n    reader = csv.reader(test_file)\n    header = next(reader)\n    with open('effnet_b0_nopool_subm.csv', 'w') as subm_file:\n        writer = csv.writer(subm_file,delimiter=',')\n        writer.writerow(['image_name','target'])\n        for row in tqdm(reader):\n            img_name = os.path.join(test_folder,row[0]+'.jpg')\n            img_input = np.array([np.asarray(PIL.Image.open(img_name))])\n            prob = eff_finetuning.predict(img_input)[0][0]\n            writer.writerow([row[0],prob])\n        subm_file.close()\n    test_file.close()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# TODO\n# Report the following in the 'evaluation' channel on Discord :)\n\n# 2. Chosen model (b0 - b7)\n# 3. No global pooling / max global pooling / avg global pooling?\n# 4. Name of your first trainable layer (just to make sure the highest level features are finetuned)\n# 5. Amount of trainable parameters\n# 6. Epochs (50 is fine)\n# 7. Final loss, accuracy, precision, recall, auc\n# 8. HDF5 file of your model weights\n# 9. Submission file\n# 10. Submission score\n\n# Now take a coffee","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## **Code below is not important!**","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"with open('submission.csv', 'w') as subm_file:\n        writer = csv.writer(subm_file,delimiter=',')\n        writer.writerow(['image_name','target'])\n        predictions = eff_finetuning.predict(test_flow)\n        for pred in tqdm(predictions):\n            prob = pred[0] # or pred[1]\n            name = ???\n            writer.writerow([name,prob])\n        subm_file.close()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"# Output labels are mainly [0,1], so 'other' class has index 1 and 'melanoma' class index 0\ntest = next(train_flow)\nprint(test)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}