{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Try and get keras plot to work\n!pip install -q pydot\n!pip install -q pydotplus\n!apt-get install -q graphviz\n\nprint(\"\\n... OTHER IMPORTS STARTING ...\\n\")\nprint(\"\\n\\tVERSION INFORMATION\")\n\n# Machine Learning and Data Science Imports\nimport tensorflow_addons as tfa; print(f\"\\t\\t– TENSORFLOW ADDONS VERSION: {tfa.__version__}\");\nimport tensorflow as tf; print(f\"\\t\\t– TENSORFLOW VERSION: {tf.__version__}\");\nimport pandas as pd; pd.options.mode.chained_assignment = None;\nimport numpy as np; print(f\"\\t\\t– NUMPY VERSION: {np.__version__}\");\nimport scipy; print(f\"\\t\\t– SCIPY VERSION: {scipy.__version__}\");\n\n# Built In Imports\nfrom collections import Counter\nfrom datetime import datetime\nimport multiprocessing\nfrom glob import glob\nimport warnings\nimport requests\nimport imageio\nimport IPython\nimport urllib\nimport zipfile\nimport pickle\nimport random\nimport shutil\nimport string\nimport math\nimport tqdm\nimport time\nimport gzip\nimport io\nimport os\nimport gc\nimport re\n\n# Visualization Imports\nfrom matplotlib.colors import ListedColormap\nimport matplotlib.patches as patches\nimport plotly.graph_objects as go\nimport matplotlib.pyplot as plt\nfrom tqdm.notebook import tqdm\nimport plotly.express as px\nimport seaborn as sns\nfrom PIL import Image\nimport matplotlib; print(f\"\\t\\t– MATPLOTLIB VERSION: {matplotlib.__version__}\");\nimport plotly\nimport PIL\nimport cv2\nimport ast\n\n# PRESETS\nLBL_NAMES = [\"Nucleoplasm\", \"Nuclear Membrane\", \"Nucleoli\", \"Nucleoli Fibrillar Center\", \"Nuclear Speckles\", \"Nuclear Bodies\", \"Endoplasmic Reticulum\", \"Golgi Apparatus\", \"Intermediate Filaments\", \"Actin Filaments\", \"Microtubules\", \"Mitotic Spindle\", \"Centrosome\", \"Plasma Membrane\", \"Mitochondria\", \"Aggresome\", \"Cytosol\", \"Vesicles\", \"Negative\"]\nINT_2_STR = {x:LBL_NAMES[x] for x in np.arange(19)}\nINT_2_STR_LOWER = {k:v.lower().replace(\" \", \"_\") for k,v in INT_2_STR.items()}\nSTR_2_INT_LOWER = {v:k for k,v in INT_2_STR_LOWER.items()}\nSTR_2_INT = {v:k for k,v in INT_2_STR.items()}\nFIG_FONT = dict(family=\"Helvetica, Arial\", size=14, color=\"#7f7f7f\")\nLABEL_COLORS = [px.colors.label_rgb(px.colors.convert_to_RGB_255(x)) for x in sns.color_palette(\"Spectral\", len(LBL_NAMES))]\nLABEL_COL_MAP = {str(i):x for i,x in enumerate(LABEL_COLORS)}\n\nprint(\"\\n\\n... IMPORTS COMPLETE ...\\n\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Define the path to the root data directory\nROOT_DIR = \"/kaggle/input\"\n\n# Define the path to the competition data directory\nCOMP_DIR = os.path.join(ROOT_DIR, \"hpa-single-cell-image-classification\")\n\n# Define path to the filtered TP IDs for each class\nPKL_DIR = os.path.join(ROOT_DIR, \"hpa-rule-based-single-cell-filtering\")\n\n# Define the paths to the training tiles for the cell-wise classification dataset\nRED_TILE_DIR = os.path.join(ROOT_DIR, \"human-protein-atlas-red-cell-tile-dataset\")\nGREEN_TILE_DIR = os.path.join(ROOT_DIR, \"human-protein-atlas-green-cell-tile-dataset\")\nBLUE_TILE_DIR = os.path.join(ROOT_DIR, \"human-protein-atlas-blue-cell-tile-dataset\")\nYELLOW_TILE_DIR = os.path.join(ROOT_DIR, \"human-protein-atlas-yellow-cell-tile-dataset\")\n\n# Define the paths to the training and testing tfrecord and \n# image folders respectively for the competition data\nTRAIN_IMG_DIR = os.path.join(COMP_DIR, \"train\")\nTRAIN_TFREC_DIR = os.path.join(COMP_DIR, \"train_tfrecords\")\nTEST_IMG_DIR = os.path.join(COMP_DIR, \"test\")\nTEST_TFREC_DIR = os.path.join(COMP_DIR, \"test_tfrecords\")\n\n# Capture all the relevant full image paths for the competition dataset\nTRAIN_IMG_PATHS = sorted([os.path.join(TRAIN_IMG_DIR, f_name) for f_name in os.listdir(TRAIN_IMG_DIR)])\nTEST_IMG_PATHS = sorted([os.path.join(TEST_IMG_DIR, f_name) for f_name in os.listdir(TEST_IMG_DIR)])\nprint(f\"\\n... Recall that 4 training images compose one example (R,G,B,Y) ...\")\nprint(f\"... \\t– i.e. The first 4 training files are:\")\nfor path in [x.rsplit('/',1)[1] for x in TRAIN_IMG_PATHS[:4]]: print(f\"... \\t\\t– {path}\")\nprint(f\"\\n... The number of training images is {len(TRAIN_IMG_PATHS)} i.e. {len(TRAIN_IMG_PATHS)//4} 4-channel images ...\")\nprint(f\"... The number of testing images is {len(TEST_IMG_PATHS)} i.e. {len(TEST_IMG_PATHS)//4} 4-channel images ...\")\n\n# Capture all the relevant full tfrec paths\nTRAIN_TFREC_PATHS = sorted([os.path.join(TRAIN_TFREC_DIR, f_name) for f_name in os.listdir(TRAIN_TFREC_DIR)])\nTEST_TFREC_PATHS = sorted([os.path.join(TEST_TFREC_DIR, f_name) for f_name in os.listdir(TEST_TFREC_DIR)])\nprint(f\"\\n... The number of training tfrecord files is {len(TRAIN_TFREC_PATHS)} ...\")\nprint(f\"... The number of testing tfrecord files is {len(TEST_TFREC_PATHS)} ...\\n\")\n\n# Random Useful Info\nORIGINAL_DIST_MAP = {0: 37472, 1: 4845, 2: 12672, 3: 12882, 4: 17527, 5: 15337, 6: 10198, 7: 18825, 8: 11194, 9: 5322, 10: 7789, 11: 10, 12: 13952, 13: 21168, 14: 27494, 15: 2275, 16: 22738, 17: 5619, 18: 952}\n\n# Define paths to the relevant csv files\nTRAIN_CSV = os.path.join(ROOT_DIR, \"hpa-train-data-with-additional-metadata/updated_train.csv\")\n\nprint(\"\\n... Loading massive train dataframe ...\\n\")\n# Create the relevant dataframe objects\ntrain_df = pd.read_csv(TRAIN_CSV)\n# train_df.mask_rles = train_df.mask_rles.apply(lambda x: ast.literal_eval(x))\n# train_df.mask_bboxes = train_df.mask_bboxes.apply(lambda x: ast.literal_eval(x))\n    \nprint(\"\\n\\nTRAIN DATAFRAME\\n\\n\")\ndisplay(train_df.head(3))","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}