{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Import dataset"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"!pip install \"../input/keras-application/Keras_Applications-1.0.8-py3-none-any.whl\"\n!pip install \"../input/efficientnet111/efficientnet-1.1.1-py3-none-any.whl\"\n!pip install \"../input/pycocotools/pycocotools-2.0-cp37-cp37m-linux_x86_64.whl\"\n!pip install \"../input/hpapytorchzoozip/pytorch_zoo-master\"\n!pip install \"../input/hpacellsegmentatormaster/HPA-Cell-Segmentation-master\"\n!pip install \"../input/tfexplainforoffline/tf_explain-0.2.1-py3-none-any.whl\"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# import packages"},{"metadata":{"trusted":true},"cell_type":"code","source":"import os, glob\nimport tensorflow as tf\ngpus = tf.config.experimental.list_physical_devices('GPU')\nfor gpu in gpus:\n    tf.config.experimental.set_memory_growth(gpu, True)\n# tf.compat.v1.disable_eager_execution()\nimport random\nfrom sklearn.model_selection import train_test_split\nimport cv2\nimport numpy as np\nimport pandas as pd\nimport multiprocessing\nfrom copy import deepcopy\nfrom sklearn.metrics import precision_recall_curve, auc\nimport keras\nimport keras.backend as K\nfrom keras.optimizers import Adam\nfrom keras.callbacks import Callback\n# please note, that locally I've trained a keras.efficientnet model, but using tensorflow.keras.applications.EfficientNetB0 should lead to the same results\nfrom efficientnet.keras import EfficientNetB0\nfrom keras.layers import Dense, Flatten\nfrom keras.models import Model, load_model\nfrom keras.utils import Sequence\nfrom albumentations import Compose, VerticalFlip, HorizontalFlip, Rotate, GridDistortion\nimport matplotlib.pyplot as plt\nfrom IPython.display import Image, display\nfrom numpy.random import seed\nseed(10)\nfrom tensorflow.python.framework import ops\nimport gc\nfrom numba import cuda \nimport hpacellseg.cellsegmentator as cellsegmentator\nfrom hpacellseg.utils import label_cell, label_nuclei\nfrom tqdm.auto import tqdm\nimport base64\nimport numpy as np\nfrom pycocotools import _mask as coco_mask\nimport typing as t\nimport zlib\nimport warnings\nfrom tf_explain.core.integrated_gradients import IntegratedGradients\nwarnings.filterwarnings('ignore')\n\ntf.random.set_seed(10)\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Preview the datasets"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv('../input/hpa-single-cell-image-classification/train.csv')\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## one-hot encocding class names"},{"metadata":{"trusted":true},"cell_type":"code","source":"specified_class_names = \"\"\"0. Nucleoplasm\n1. Nuclear membrane\n2. Nucleoli\n3. Nucleoli fibrillar center\n4. Nuclear speckles\n5. Nuclear bodies\n6. Endoplasmic reticulum\n7. Golgi apparatus\n8. Intermediate filaments\n9. Actin filaments \n10. Microtubules\n11. Mitotic spindle\n12. Centrosome\n13. Plasma membrane\n14. Mitochondria\n15. Aggresome\n16. Cytosol\n17. Vesicles and punctate cytosolic patterns\n18. Negative\"\"\"\n\nclass_names = [item.split('. ')[1] for item in specified_class_names.split('\\n')]\nclass_names","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['Label'] = train_df['Label'].map(lambda x: set(map(int, x.split('|'))))\nfor class_idx, class_name in enumerate(class_names):\n    train_df[class_name] = train_df['Label'].map(lambda x: 1 if class_idx in x else 0)\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# dictionary for fast access to ohe vectors\nid_2_ohe_vector = {img: vec for img, vec in zip(train_df['ID'], train_df.iloc[:, 2:-1].values)}","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## find images with unique label combinations, these images will be put into training dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"label_combinations = train_df['Label'].map(lambda x: str(sorted(list(x))))\nprint(\"There {} images with unique label combinations \".format(sum(label_combinations.value_counts()==1)))\nlabel_combinations_counts = label_combinations.value_counts()\nunique_label_combination = label_combinations_counts.index[label_combinations_counts==1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_ids_unique_label_comb = train_df['ID'][train_df['Label'].map(lambda x:str(sorted(list(x))) in unique_label_combination)]\nnon_unique_label_comb_bool_idx = train_df['Label'].map(lambda x:str(sorted(list(x))) not in unique_label_combination)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## create train and validation sets"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_ids, val_ids = train_test_split(train_df['ID'][non_unique_label_comb_bool_idx].values,\n                        test_size = 0.2,\n                        stratify = label_combinations[non_unique_label_comb_bool_idx],\n                        random_state = 42)\ntrain_ids = np.concatenate((train_ids, train_ids_unique_label_comb))","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}