{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.read_csv(\"../input/hpa-single-cell-image-classification/sample_submission.csv\")\nsubmission.to_csv(\"./submission.csv\", index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"try:\n    import hpacellseg.cellsegmentator as cellsegmentator\n    from hpacellseg.utils import label_cell\nexcept:\n    !pip install -q \"/kaggle/input/pycocotools/pycocotools-2.0-cp37-cp37m-linux_x86_64.whl\"\n    !pip install -q \"/kaggle/input/hpapytorchzoozip/pytorch_zoo-master\"\n    !pip install -q \"/kaggle/input/hpacellsegmentatormaster/HPA-Cell-Segmentation-master\"\n    import hpacellseg.cellsegmentator as cellsegmentator\n    from hpacellseg.utils import label_cell","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def fill_targets(row):\n    labels = np.array(row.Label.split(\"|\")).astype(np.int)\n    for num in labels:\n        name = label_names[int(num)]\n        row.loc[name] = 1\n    return row\n\ntrain_labels = pd.read_csv(\"../input/hpa-single-cell-image-classification/train.csv\")\nlabel_names = {\n    0: \"Nucleoplasm\",\n    1: \"Nuclear membrane\",\n    2: \"Nucleoli\",\n    3: \"Nucleoli fibrillar center\",\n    4: \"Nuclear speckles\",\n    5: \"Nuclear bodies\",\n    6: \"Endoplasmic reticulum\",\n    7: \"Golgi apparatus\",\n    8: \"Intermediate filaments\",\n    9: \"Actin filaments\",\n    10: \"Microtubules\",\n    11: \"Mitotic spindle\",\n    12: \"Centrosome\",\n    13: \"Plasma membrane\",\n    14: \"Mitochondria\",\n    15: \"Aggresome\",\n    16: \"Cytosol\",\n    17: \"Vesicles and punctate cytosolic patterns\",\n    18: \"Negative\"\n}\n\nfor key in label_names.keys():\n    train_labels[label_names[key]] = 0\n\ntrain_labels = train_labels.apply(fill_targets, axis=1)\ntrain_labels","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import seaborn as sns\nimport matplotlib.pyplot as plt\n\ntarget_counts = train_labels.drop([\"ID\", \"Label\"],axis=1).sum(axis=0).sort_values(ascending=True)\nplt.figure(figsize=(10,5))\ng = sns.barplot(x=target_counts.index.values, y=target_counts.values, order=target_counts.index, color=\"seagreen\")\ng.set_yscale(\"log\")\n\nprint(target_counts.values)\n\nylabels = [format(label, '.0f') for label in g.get_yticks()]\n# print(ylabels)\ng.set_yticklabels(ylabels)\n\ng.set_xticklabels(g.get_xticklabels(), weight='bold', rotation=90)\ng.set_yticklabels(g.get_yticklabels(), weight='bold')\nplt.ylabel(\"Número de Amostras\", weight='bold')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import RepeatedKFold\nsplitter = RepeatedKFold(n_splits=5, n_repeats=1, random_state=1234567)\n\npartitions = []\n\nfor train_idx, valid_idx in splitter.split(train_labels.index.values):\n    partition = {}\n    partition[\"train\"] = train_labels.values[train_idx]\n    partition[\"validation\"] = train_labels.values[valid_idx]\n    partitions.append(partition)\n    \npartition = partitions[0]\n\ntrain = pd.DataFrame(partition[\"train\"]) \ntrain.columns = train_labels.columns\ntrain.to_csv(\"./train_minor.csv\", index=False)\n\nvalid = pd.DataFrame(partition[\"validation\"]) \nvalid.columns = train_labels.columns\nvalid.to_csv(\"./valid.csv\", index=False)\n\nprint(train.shape)\nprint(valid.shape)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd \ndf_train = pd.read_csv('./train_minor.csv')\ndf_val = pd.read_csv('./valid.csv')\ndf_train['Image'] = df_train['ID'] + '_green.png'\ndf_val['Image'] = df_val['ID'] + '_green.png'","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels = list(df_val.columns)\nlabels = labels[2:-1]\nlabels","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Determinando os pesos\nimport numpy as np\ncount = np.sum(df_train[labels])\nmax_val = float(max(count))\nclass_weights = {idx: max_val/count[idx] for idx in range(len(labels))}    \nclass_weights ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\n\ngenerator = tf.keras.preprocessing.image.ImageDataGenerator(preprocessing_function=tf.keras.applications.densenet.preprocess_input)\n\ntrain_generator = generator.flow_from_dataframe(\n    df_train, directory='../input/hpa-single-cell-image-classification/train', x_col='Image', y_col=labels,\n    target_size=(1024, 1024), class_mode='raw', batch_size=2, shuffle=False)\n\nval_generator = generator.flow_from_dataframe(\n    df_val, directory='../input/hpa-single-cell-image-classification/train', x_col='Image', y_col=labels,\n    target_size=(1024, 1024), class_mode='raw', batch_size=2, shuffle=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"densenet = tf.keras.applications.DenseNet201(weights='imagenet', include_top=False)\n\nfor layer in densenet.layers:\n  layer.trainable = True\n\nmodel = tf.keras.models.Sequential([\n      densenet,\n      tf.keras.layers.GlobalAveragePooling2D(),\n      tf.keras.layers.Dense(19, activation='sigmoid')\n])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"otimizador = tf.keras.optimizers.Adam(learning_rate=0.0001)\n\nmodel.compile(loss='binary_crossentropy', optimizer=otimizador)\nmodel.summary()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mc = tf.keras.callbacks.ModelCheckpoint('./densenet201_weight_green_1024.hdf5', save_best_only=True, verbose=1)\nes = tf.keras.callbacks.EarlyStopping(monitor='val_loss', mode='min', restore_best_weights=True, verbose=1, patience=10)\nlr = tf.keras.callbacks.ReduceLROnPlateau(monitor='val_loss', mode='min', patience=5, verbose=1)\n\nmodel.fit(train_generator, epochs=50, validation_data=val_generator, callbacks=[mc, es, lr], class_weight=class_weights)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = tf.keras.models.load_model('./densenet201_weight_green_1024.hdf5')\n\nmodel.summary()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred = model.predict(val_generator, verbose=1)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_true = []\ni = 0\nfor batch, label in val_generator:\n  i += batch.shape[0]\n  for l in label:\n    y_true.append(l)\n  if i == 7269: break","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn import metrics\n\ndef compute_metrics_(data_gt, data_pd, num_classes):\n    acc_score = []\n    precision = []\n    recall = []\n    f1_score = []\n    \n    result = []\n    real = []\n\n    for i in range(num_classes):\n        # roc_auc.append(metrics.roc_auc_score(data_gt[:, i], data_pd[:, i]))\n        acc_score.append(metrics.accuracy_score(data_gt[:, i], data_pd[:, i].round()))\n        precision.append(metrics.precision_score(data_gt[:, i], data_pd[:, i].round()))\n        recall.append(metrics.recall_score(data_gt[:, i], data_pd[:, i].round()))\n        f1_score.append(metrics.f1_score(data_gt[:, i], data_pd[:, i].round()))\n\n        result.append(data_pd[:, i].round())\n        real.append(data_gt[:, i].round())\n        \n    acc_score = np.array(acc_score)\n    precision = np.array(precision)\n    recall = np.array(recall)\n    f1_score = np.array(f1_score)\n\n    print(len(result))\n    return acc_score, precision, recall, f1_score","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\n\nvalues = compute_metrics_(np.array(y_true), pred, 19)\n\nprint('acc_score')\nfor i in range(len(values[0])):\n  print(labels[i], '->', values[0][i])\nprint('média ->', np.mean(values[0]))\n\nprint('\\n'*2)\n\nprint('precision')\nfor i in range(len(values[1])):\n  print(labels[i], '->', values[1][i])\nprint('média ->', np.mean(values[1]))\n\nprint('\\n'*2)\n\nprint('recall')\nfor i in range(len(values[2])):\n  print(labels[i], '->', values[2][i])\nprint('média ->', np.mean(values[2]))\n\nprint('\\n'*2)\n\nprint('f1_score')\nfor i in range(len(values[3])):\n  print(labels[i], '->', values[3][i])\nprint('média ->', np.mean(values[3]))\n\nprint('\\n'*2)","metadata":{},"execution_count":null,"outputs":[]}]}