{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Visualizing batch effects with t-SNE.\n\nI tried to visualize inherent batch effects of the data with t-SNE. Since each image is a 512\\*512\\*6 = 1572864-dimensional vector, separately using all of the pixels as an element of a vector makes t-SNE computation too intensive. To overcome it, I simply took average of the pixel intensities for each of the six channels given for an image, therefore handled a six-dimensional vector for each image.\n\nAnother thing to note is that as the objective of this kernel is to visualize the batch effects, I only exploited positive and negative control-images to exclude siRNA-related effects."},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nfrom PIL import Image as image\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\nimport tqdm\n\n!pip install opentsne\nfrom openTSNE.sklearn import TSNE\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_controls = pd.read_csv('../input/train_controls.csv')\ntest_controls = pd.read_csv('../input/test_controls.csv')\ntrain_controls['cell_line'] = [v[0] for v in train_controls.id_code.str.split('-')]\ntest_controls['cell_line'] = [v[0] for v in test_controls.id_code.str.split('-')]\n\ntrain_controls.shape, test_controls.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_controls.head(3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_controls.head(3)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Let's extract IDs of control siRNAs that are present in all of the plates."},{"metadata":{"trusted":true},"cell_type":"code","source":"positive_sirnas = train_controls[train_controls.well_type == 'positive_control'].sirna.unique()\nnegative_sirna = train_controls[train_controls.well_type == 'negative_control'].sirna.unique()[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"negative_sirna","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"positive_sirnas","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(positive_sirnas)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_image_file(experiment, plate, well, channel, site=1, train=True):\n    train_path = f'../input/train/{experiment}/Plate{plate}/{well}_s{site}_w{channel}.png'\n    test_path = f'../input/test/{experiment}/Plate{plate}/{well}_s{site}_w{channel}.png'\n    if os.path.exists(train_path):\n        return train_path\n    else:\n        return test_path\n    \ndef get_image_nparray(experiment, plate, well, channel, site=1, train=True):\n    img = np.array(image.open(get_image_file(experiment, plate, well, channel, site, train)))\n    return img.reshape(-1)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"data = []\nfor i, row in tqdm.tqdm(train_controls.iterrows(), total=len(train_controls)):\n    v = []\n    for channel in range(1, 7):\n        # Take means of pixel intensities.\n        v.append(get_image_nparray(experiment=row.experiment, plate=row.plate, well=row.well, channel=channel).mean())\n    data.append(v)\ndata = np.array(data)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data.shape","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":false,"_kg_hide-input":false,"trusted":true},"cell_type":"code","source":"embedding = TSNE().fit_transform(data)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Add column `cell_line` to the metadata."},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_controls.head(3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_controls.cell_line.unique()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Visualize!"},{"metadata":{},"cell_type":"markdown","source":"# Visualizing batch effects for cell line RPE.\n\nClear distinction between RPE-01~03 and RPE-04~07 is shown."},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = plt.figure(figsize=(10, 10))\nax = fig.add_subplot(111)\nprint('There are %d experiments.' % len(train_controls[train_controls.cell_line.values == 'RPE'].experiment.unique()))\nfor exp in train_controls.experiment.unique():\n    mask = (train_controls.experiment.values == exp) & (train_controls.cell_line.values == 'RPE')\n    if sum(mask) != 0:\n        ax.scatter(embedding[mask, 0], embedding[mask, 1], label=exp, s=6)\n\nax.legend();","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Visualizing batch effects for cell line HEPG2.\n\nInterestingly, again, HEPG2-01~03, HEP-04, and HEPG2-05~07 are clustered toghther."},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = plt.figure(figsize=(10, 10))\nax = fig.add_subplot(111)\nprint('There are %d experiments.' % len(train_controls[train_controls.cell_line.values == 'HEPG2'].experiment.unique()))\n\nfor exp in train_controls.experiment.unique():\n    mask = (train_controls.experiment.values == exp) & (train_controls.cell_line.values == 'HEPG2')\n    if sum(mask) != 0:\n        ax.scatter(embedding[mask, 0], embedding[mask, 1], label=exp, s=6)\n\nax.legend();","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Visualizing batch effects for cell line HUVEC.\n\nAs there are 16 different experiments for HUVEC cell lines, it is difficult to visualize with discrete color maps.\n\nInstead, it is displayed with a continuous color map such that mapped color gets darker as experiment IDs change from 01 to 16.\n\nAgain we can notice that the experiments with closer experiment IDs are clustered together."},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = plt.figure(figsize=(10, 10))\nax = fig.add_subplot(111)\n\nmask = (train_controls.cell_line.values == 'HUVEC')\nprint('There are %d experiments.' % len(train_controls[mask].experiment.unique()))\nax.scatter(embedding[mask, 0], embedding[mask, 1], c=[float(v[1]) / 16 for v in train_controls[mask].experiment.str.split('-')], cmap=plt.cm.Blues, s=4);","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Visualizing batch effects for cell line U2OS.\n\nThere are three U2OS cell line experiments. We can notice that each experiment can be more or less separated in t-SNE space, but there were no apparent inter-experiment clustering."},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = plt.figure(figsize=(10, 10))\nax = fig.add_subplot(111)\nprint('There are %d experiments.' % len(train_controls[train_controls.cell_line.values == 'U2OS'].experiment.unique()))\n\nfor exp in train_controls.experiment.unique():\n    mask = (train_controls.experiment.values == exp) & (train_controls.cell_line.values == 'U2OS')\n    if sum(mask) != 0:\n        ax.scatter(embedding[mask, 0], embedding[mask, 1], label=exp, s=4)\n\nax.legend();","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Cell line-based coloring."},{"metadata":{"trusted":true},"cell_type":"code","source":"print('There are %d experiments.' % len(train_controls.experiment.unique()))\n\nfig = plt.figure(figsize=(10, 10))\nax = fig.add_subplot(111)\nfor cl in train_controls.cell_line.unique():\n    mask = (train_controls.cell_line.values == cl)\n    ax.scatter(embedding[mask, 0], embedding[mask, 1], label=cl, s=4)\n\nax.legend()\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# siRNA-based coloring.\n\nWe hope that images to be clustered according to the treated siRNAs. Unfortunately, but as expected, we could not identify any distinct clusters that reflects the type of treated siRNA.\n\nFor the sake of visualization, only ten siRNAs are shown."},{"metadata":{"trusted":true},"cell_type":"code","source":"print('There are %d experiments.' % len(train_controls.experiment.unique()))\n\nfig = plt.figure(figsize=(10, 10))\nax = fig.add_subplot(111)\nfor si in train_controls.sirna.unique()[:10]:\n    mask = (train_controls.sirna.values == si)\n    ax.scatter(embedding[mask, 0], embedding[mask, 1], label=si, s=4)\n\nax.legend();","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Marking negative controls.\n\nPerhaps one of the intuitive way to envision the effect of batch effect (or plate effect) removal is to make images without any siRNA treatment clustered in the latent space where the images are distributed.\n\nBefore that, let's see how the negative control-images are distributed without any correction for batch effect.\n\nIt was expected that they suffer from severe experiment-wise batch effects, but also, "},{"metadata":{"trusted":true},"cell_type":"code","source":"print('There are %d experiments.' % len(train_controls.experiment.unique()))\n\nfig = plt.figure(figsize=(10, 10))\nax = fig.add_subplot(111)\n\nmask = (train_controls.sirna.values == 1138)\nax.scatter(embedding[:, 0], embedding[:, 1], alpha=0.11, color='grey', s=3)\nax.scatter(embedding[mask, 0], embedding[mask, 1], label=1138, s=8, color='red')\n\nax.legend();","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Visualizing test controls along with train controls.\n\nSeeing which test experiments has similar batch effects with which train experiments may give us some insight about how to deal with batch effects in test dataset."},{"metadata":{"trusted":true},"cell_type":"code","source":"test_controls['cell_line'] = [v[0] for v in test_controls.id_code.str.split('-')]\ncontrols = pd.concat([train_controls, test_controls])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"controls.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data = []\nfor i, row in tqdm.tqdm(controls.iterrows(), total=len(controls)):\n    v = []\n    for channel in range(1, 7):\n        # Take means of pixel intensities.\n        v.append(get_image_nparray(experiment=row.experiment, plate=row.plate, well=row.well, channel=channel).mean())\n    data.append(v)\n\ndata = np.array(data)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"embedding = TSNE().fit_transform(data)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# [With test controls] Visualizing batch effects for cell line RPE."},{"metadata":{"trusted":true},"cell_type":"code","source":"CELL_LINE = 'RPE'\n\nfig = plt.figure(figsize=(10, 10))\nax = fig.add_subplot(111)\n\nunique_exps = controls[controls.cell_line.values == CELL_LINE].experiment.unique()\nprint('There are %d experiments.' % len(unique_exps))\n\nfor exp in unique_exps:\n    mask = (controls.experiment.values == exp) & (controls.cell_line.values == CELL_LINE)\n    if sum(mask) != 0:\n        ax.scatter(embedding[mask, 0], embedding[mask, 1], label=exp, s=6)\n\nax.legend();","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"CELL_LINE = 'HEPG2'\n\nfig = plt.figure(figsize=(10, 10))\nax = fig.add_subplot(111)\n\nunique_exps = controls[controls.cell_line.values == CELL_LINE].experiment.unique()\nprint('There are %d experiments.' % len(unique_exps))\n\nfor exp in unique_exps:\n    mask = (controls.experiment.values == exp) & (controls.cell_line.values == CELL_LINE)\n    if sum(mask) != 0:\n        ax.scatter(embedding[mask, 0], embedding[mask, 1], label=exp, s=6)\n\nax.legend();","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# [With test controls] Visualizing batch effects for cell line HUVEC."},{"metadata":{"trusted":true},"cell_type":"code","source":"CELL_LINE = 'HUVEC'\n\nfig = plt.figure(figsize=(10, 10))\nax = fig.add_subplot(111)\n\nunique_exps = controls[controls.cell_line.values == CELL_LINE].experiment.unique()\nprint('There are %d experiments.' % len(unique_exps))\n\nfor exp in unique_exps:\n    mask = (controls.experiment.values == exp) & (controls.cell_line.values == CELL_LINE)\n    if sum(mask) != 0:\n        ax.scatter(embedding[mask, 0], embedding[mask, 1], label=exp, s=6)\n\nax.legend();","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# [With test controls] Visualizing batch effects for cell line U2OS."},{"metadata":{"trusted":true},"cell_type":"code","source":"CELL_LINE = 'U2OS'\n\nfig = plt.figure(figsize=(10, 10))\nax = fig.add_subplot(111)\n\nunique_exps = controls[controls.cell_line.values == CELL_LINE].experiment.unique()\nprint('There are %d experiments.' % len(unique_exps))\n\nfor exp in unique_exps:\n    mask = (controls.experiment.values == exp) & (controls.cell_line.values == CELL_LINE)\n    if sum(mask) != 0:\n        ax.scatter(embedding[mask, 0], embedding[mask, 1], label=exp, s=6)\n\nax.legend();","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}