{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# python lang sepecific\nfrom path import Path\n\n# Data manipulation\nimport pandas\nimport numpy as np\nimport cv2\n\n# Data Visualization\nimport matplotlib.pyplot as plt\nimport seaborn as sns","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ad73ac71c243a731432a63953e3b2315000e79c1"},"cell_type":"markdown","source":"# Data format\n### Independent Variable: \n- Four file per sample - Each file is different filter on the subcellular protein patterns represented by the sample\n- Four filters:\n    - Green : Protein of interest plus 3 cellular landmarks\n    - Blue : nucleus \n    - Red : microtubules \n    - Yellow :  endoplasmic reticulum\nFormat :`[filename]_[filter color].png`\n### Dependent Variable: \n- Number of classes :  `28`\n- Green filter should be use for prediction, filters are used as references."},{"metadata":{"trusted":true,"_uuid":"51f8c9bbc2bb24c2b9730c54a7f3d28f33654574"},"cell_type":"code","source":"DATA = Path('../input')\nDATA.listdir()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4e9eb1bf4d078a7b96f554d02010f99b6fabd815"},"cell_type":"code","source":"train_dir = DATA/'train'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ca7dceefda8dd60b18574f94a23501ec909832c7"},"cell_type":"code","source":"train_fn = DATA/'train.csv'\ntrain_labels = pandas.read_csv(train_fn)\ntrain_labels.head()\nprint(\"Number of sample:\", train_labels.shape[0])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"884902e6285717f351ffbe5444ac666410f266ef"},"cell_type":"markdown","source":"## Plot a Sample"},{"metadata":{"trusted":true,"_uuid":"1bf197b6b5df872ecc168d961e5b4c04d60e8c6d"},"cell_type":"code","source":"label_id2name = {\n    0:  \"Nucleoplasm\",  \n    1:  \"Nuclear membrane\",   \n    2:  \"Nucleoli\",   \n    3:  \"Nucleoli fibrillar center\",   \n    4:  \"Nuclear speckles\",\n    5:  \"Nuclear bodies\",   \n    6:  \"Endoplasmic reticulum\",   \n    7:  \"Golgi apparatus\",   \n    8:  \"Peroxisomes\",   \n    9:  \"Endosomes\",   \n    10:  \"Lysosomes\",   \n    11:  \"Intermediate filaments\",   \n    12:  \"Actin filaments\",   \n    13:  \"Focal adhesion sites\",   \n    14:  \"Microtubules\",   \n    15:  \"Microtubule ends\",   \n    16:  \"Cytokinetic bridge\",   \n    17:  \"Mitotic spindle\",   \n    18:  \"Microtubule organizing center\",   \n    19:  \"Centrosome\",   \n    20:  \"Lipid droplets\",   \n    21:  \"Plasma membrane\",   \n    22:  \"Cell junctions\",   \n    23:  \"Mitochondria\",   \n    24:  \"Aggresome\",   \n    25:  \"Cytosol\",   \n    26:  \"Cytoplasmic bodies\",   \n    27:  \"Rods & rings\"\n}\n\nlabel_name2id = {v: k for k, v in label_id2name.items()}\n\ndef fill_targets(row):\n    row.Target = np.array(row.Target.split(\" \")).astype(np.int)\n    for num in row.Target:\n        name = label_id2name[int(num)]\n        row.loc[name] = 1\n    return row","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"13724d5a0d8f9b39d2e3640a1ecd3ceb9e6cfb8a"},"cell_type":"code","source":"def show_img(im, figsize=None, ax=None, title=None):\n    if not ax: fig, ax = plt.subplots(figsize=figsize)\n    im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB)\n    if title:  ax.set_title(title)\n    ax.imshow(im)\n    ax.get_xaxis().set_visible(False)\n    ax.get_yaxis().set_visible(False)\n    return ax","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"244c055b1edd4cc57ffa156efe228819190a6605"},"cell_type":"code","source":"# sample id\nId = train_labels['Id'].iloc[100]\nId","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2186f7df12626d0ec129d2dd47af6fd518f028d8"},"cell_type":"code","source":"label = train_labels[train_labels['Id'] == Id]\nlabel_id = label['Target'].values[0][0]\nlabel_name = label_id2name[int(label_id)]\nlabel_name","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dd2938fcb53b1872ca92137e4af3d7b450dbbf82"},"cell_type":"code","source":"fig, axes = plt.subplots(2, 2, figsize=(12, 12))\nfor i, ax in enumerate(axes.flat):\n    filters = ['green', 'red', 'blue', 'yellow']\n    img = cv2.imread(train_dir/f'{Id}_{filters[i]}.png')\n    show_img(img, ax=ax, title=f'{label_name}/ filter: {filters[i]}')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6171aa27d79f973d410adade47929a4f83e49d5d"},"cell_type":"markdown","source":"## Class Distribution"},{"metadata":{"trusted":true,"_uuid":"0a39b67e81d8725bb9e04a81d5f500ff283585fa"},"cell_type":"code","source":"for key in label_id2name:\n    train_labels[label_id2name[key]] = 0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"eaa462f37c1f35782f8f417d76f38b7da6960186"},"cell_type":"code","source":"train_labels = train_labels.apply(fill_targets, axis=1)\ntrain_labels.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8c37876c37a8bfef4244b0c8d2336480a31603a1"},"cell_type":"code","source":"target_counts = train_labels.drop([\"Id\", \"Target\"], axis=1).sum(axis=0).sort_values(ascending=False)\nplt.figure(figsize=(15, 15))\nsns.barplot(y=target_counts.index.values, x=target_counts.values, order=target_counts.index)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"173c31a3ffec46df2ead1dfc36a14294d3453d35"},"cell_type":"markdown","source":"## How many targets are most common ?"},{"metadata":{"trusted":true,"_uuid":"3c254d8a55aa589f16dbff7e7e7192f7b0e69ab9"},"cell_type":"code","source":"if \"number_of_targets\" in train_labels:\n    train_labels = train_labels.drop([\"number_of_targets\"], axis=1)\ntrain_labels[\"number_of_targets\"] = train_labels.drop([\"Id\", \"Target\"], axis=1).sum(axis=1)\ncount_prec = np.round(100 * train_labels[\"number_of_targets\"].value_counts() / train_labels.shape[0], 2)\nplt.figure(figsize=(20, 5))\nsns.barplot(x=count_prec.index.values, y=count_prec.values, palette=\"Reds\")\nplt.xlabel(\"# target per image\")\nplt.ylabel(\"% of samples\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"39e872d15cdd6b94f1aaf80c3b27ecfb581ab8be"},"cell_type":"markdown","source":"## Corellation between target"},{"metadata":{"trusted":true,"_uuid":"3dac820bfa672448823033a3d443abfce3cb185a"},"cell_type":"code","source":"plt.figure(figsize=(15, 15))\nsns.heatmap(train_labels[train_labels.number_of_targets > 1].drop(\n    [\"Id\", \"Target\", \"number_of_targets\"], axis=1\n).corr(), cmap=\"RdYlBu\", vmin=-1, vmax=1)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"79b8e6b1205fd3c33e93a25b4993dcf6c55e2e16"},"cell_type":"markdown","source":"# Plot a image of specific target"},{"metadata":{"trusted":true,"_uuid":"852dfd81263c327c3390f8c9cb34c4aa64e0af0e"},"cell_type":"code","source":"def load_image(basepath, image_id):\n    images = np.zeros((4, 512, 512))\n    filters = ['green', 'red', 'blue', 'yellow']\n    for i, f in enumerate(filters):\n        images[i,:,:] = cv2.imread(basepath/f'{image_id}_{f}.png', cv2.IMREAD_GRAYSCALE) # cv2 return 3 same gray scale as 3 channel\n    return images\n    \ndef plot_image_row(image, subax, title):\n    subax[0].imshow(image[0], cmap='Greens')\n    subax[0].set_title(title)\n    subax[1].imshow(image[1], cmap='Reds')\n    subax[1].set_title('stained microtubules')\n    subax[2].imshow(image[2], cmap='Blues')\n    subax[2].set_title('stained nucles')\n    subax[3].imshow(image[3], cmap='Oranges')\n    subax[3].set_title('stained endoplasmatic reticulum')\n    return subax\n\ndef make_title(sample_id):\n    file_targets = train_labels.loc[train_labels.Id==sample_id, \"Target\"].values[0]\n    title = \" - \"\n    for n in file_targets:\n        title += label_id2name[int(n)] + ' - '\n    return title","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7563a70e0edcc113cc9d27e08c0c089114360d5b"},"cell_type":"code","source":"your_choice = [\"Lysosomes\", \"Endosomes\"]\ntarget_list = [label_name2id[name] for name in your_choice]\ntarget_list","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6ecafd31bc53e7c76201bb38ec6449ef16425d0a"},"cell_type":"code","source":"def check_subset(targets):\n        return np.where(set(target_list).issuperset(set(targets)), 1, 0)\n    \ntrain_labels[\"check_col\"] = train_labels.Target.apply(\n            lambda l: check_subset(l)\n        )\n\nimage_id = train_labels[train_labels.check_col==1].Id.values\nimg = load_image(train_dir, image_id[0])\nimg.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1eb74c11c89eb0f7e3a82d0710cc0e741898e0b9"},"cell_type":"code","source":"fig, ax = plt.subplots(2,4, figsize=(20,5*2))\nfor i, ax in enumerate(ax):\n    img = load_image(train_dir, image_id[i])\n    plot_image_row(img, ax, make_title(image_id[i]))\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4ecd288034d27402ff0c62083ce362edc92cc34d"},"cell_type":"code","source":"from fastai.conv_learner import *\nfrom fastai.imports import *\nfrom fastai.transforms import *\nfrom fastai.dataset import *\nfrom sklearn.metrics import fbeta_score\nimport warnings\n\ndef f2(preds, targs, start=0.17, end=0.24, step=0.01):\n    with warnings.catch_warnings():\n        warnings.simplefilter(\"ignore\")\n        return max([fbeta_score(targs, (preds>th), 2, average='samples')\n                    for th in np.arange(start,end,step)])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7db50127898d2ff2c49fa5000981500a6996872d"},"cell_type":"code","source":"PATH = './'\nINPUT = Path('../input')\nTRAIN = INPUT/'train'\nTEST = INPUT/'test'\nLABELS = INPUT/'train.csv'\nSAMPLE = INPUT/'sample_submission.csv'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"af6011a212c8b234a71a3753af3a7227101b3244"},"cell_type":"code","source":"metrics=[f2]\nf_model = resnet18","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"34ad6fccd48172903deb0d4c9c47abc3a1e4e2c3"},"cell_type":"code","source":"def get_data(sz):\n    tfms = tfms_from_model(f_model, sz)\n    return ImageClassifierData.from_csv(PATH, 'train-jpg', label_csv, tfms=tfms,\n                    suffix='.jpg', val_idxs=val_idxs, test_name='test-jpg-v2')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"347707cf12f170df2019549d7c36140cd125b951"},"cell_type":"code","source":"train_names = list({f[:36] for f in os.listdir(TRAIN)})\ntest_names = list({f[:36] for f in os.listdir(TEST)})\ntr_n, val_n = train_test_split(train_names, test_size=0.1, random_state=42)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6578a6aa225cc1b8b57c69b0e69d22d147b39b23"},"cell_type":"code","source":"len(tr_n), len(val_n), len(test_names)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"925303a07361d19ab9fc6c5c432fda2fa380254c"},"cell_type":"code","source":"def make_grb(path, id):\n    filters = ['red', 'green', 'blue']\n    flags = cv2.IMREAD_GRAYSCALE\n    img = [cv2.imread(str(path/f'{id}_{f}.png'), flags).astype(np.float32)/255\n          for f in filters]\n    return np.stack(img, axis=-1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5d6a7d8ed41862c4ec8740898329074dd2c96432"},"cell_type":"code","source":"# img = [cv2.imread(TRAIN/f'{Id}_{f}.png', cv2.IMREAD_GRAYSCALE)/255 for f in filters]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7b2fa279a5568f8927d17997d8a66abcbeaed16d"},"cell_type":"code","source":"class pdFilesDataset(FilesDataset):\n    def __init__(self, fnames, path, transform):\n        self.labels = pd.read_csv(LABELS).set_index('Id')\n        self.labels['Target'] = [[int(i) for i in s.split()] for s in self.labels['Target']]\n        super().__init__(fnames, transform, path)\n    \n    def get_x(self, i):\n        return make_grb(self.path, self.fnames[i])\n    \n    def get_y(self, i):\n        labels = self.labels.loc[self.fnames[i]]['Target']\n        return np.eye(len(label_id2name), dtype=np.float)[labels].sum(axis=0)\n        \n    @property\n    def is_multi(self): return True # is multilabel classification\n    \n    @property\n    def is_reg(self): return False\n    \n    def get_c(self): return len(label_id2name)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0553a22e7777004548df021ccaaa5599fc376436"},"cell_type":"code","source":"def get_data(sz, bs):\n    stats = A([0.0808, 0.0530, 0.0550], [0.394, 0.321, 0.327])\n    tfms = tfms_from_stats(stats, sz, crop_type=CropType.NO, tfm_y=TfmType.NO)\n    ds = ImageData.get_ds(pdFilesDataset, (tr_n[:-(len(tr_n)%bs)],TRAIN), \n                (val_n,TRAIN), tfms, test=(test_names,TEST))\n    md = ImageData(PATH, ds, bs, num_workers=1, classes=None)\n    return md","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e8b67b35eee6c30d8b752586a58bd9d5a3e2e3c1"},"cell_type":"code","source":"bs = 16\nsz = 256\nmd = get_data(sz,bs)\n\nx,y = next(iter(md.trn_dl))\nx.shape, y.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a02376c46aa51df452e5d6a32b42430256dc9177"},"cell_type":"code","source":"def display_imgs(x):\n    columns = 4\n    bs = x.shape[0]\n    rows = min((bs+3)//4,4)\n    fig=plt.figure(figsize=(columns*4, rows*4))\n    for i in range(rows):\n        for j in range(columns):\n            idx = i+j*columns\n            fig.add_subplot(rows, columns, idx+1)\n            plt.axis('off')\n            plt.imshow((x[idx,:,:,:3]*255).astype(np.int))\n    plt.show()\n\ndisplay_imgs(np.asarray(md.trn_ds.denorm(x)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4d2a089605e5a7026e8b8492b7748a1be8a4dab6"},"cell_type":"code","source":"plt.imshow(md.val_ds.denorm(to_np(x))[1]);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d55080c3e5af4c72f97fb10743c9148151ba8702"},"cell_type":"code","source":"learn = ConvLearner.pretrained(f_model, md, metrics=metrics)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"07746296c221a6e28add175bcb5eb52a1b4b188d"},"cell_type":"code","source":"dir(learn)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c5b6c023c2c4ae82f4bc3ed120d19d9ce212283b"},"cell_type":"code","source":"learn.crit","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a7b2d6f16636d930f3c9a8fcd53cf579761278ec"},"cell_type":"code","source":"learn.lr_find()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3017321060560f1200caccae996ff8dacd335da2"},"cell_type":"code","source":"learn.sched.plot()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4f8b6e6d583be1afe42d5d92690597f54ffc3d89"},"cell_type":"code","source":"lr = 2e-2\nlearn.fit(lr, 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"30a0d879b8ad7f0a84423ee797261c60c8dcb14a"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}