{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n#for dirname, _, filenames in os.walk('/kaggle/input'):\n#    for filename in filenames:\n#        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session\n\nimport pandas as pd","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install \"../input/pycocotools/pycocotools-2.0-cp37-cp37m-linux_x86_64.whl\"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install \"../input/hpacellsegmentatormaster/HPA-Cell-Segmentation-master\"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# image folders\nTEST_IMGS_FOLDER = '../input/hpa-single-cell-image-classification/test/'\nTRAIN_IMGS_FOLDER = '../input/hpa-single-cell-image-classification/train/'","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"First, want to use the singly labeled images only for the initial model, thus we want to ensure each batch contains a relatively uniform distribution of classes (i.e. the number of pictures from each class is similar to each other) (reference: https://stats.stackexchange.com/questions/221139/distribution-of-classes-in-neural-network-batches), so we build a dictionary where the keys are the classes and the values are the name of the images. This can be done using train.csv.","metadata":{}},{"cell_type":"code","source":"train_csv = pd.read_csv('../input/hpa-single-cell-image-classification/train.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_csv","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# keep the rows where there is only 1 label by checking for the symbol '|'\nsingle_label_train = train_csv[~train_csv['Label'].str.contains('\\|')].copy() # .copy() gets rid of the warning below\n# A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead\nsingle_label_train['Label'] = single_label_train['Label'].astype('int')\nsingle_label_train = single_label_train.sort_values(by = 'Label')\nsingle_label_train","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# shows the distribution of labels\n# some labels are very rare\n# 分支A：手动检查label 15, 18, 11 在多label场景内是否占大多数，否则dataset非常unbalanced\nsingle_label_train['Label'].value_counts()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 分支A\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}