{"cells":[{"metadata":{},"cell_type":"markdown","source":"In this competition we are asked to predict siRNAs (one way of genetic perturbations). \n\nThe siRNA was applied repeatedly to multiple cell lines for a total of 51 batches. \n\nIn each batch there are 4 plates. \n\nIn each plate there are 308 wells.\n\nIn each well microscopic images were taken at 2 sites and across 6 imaging channels."},{"metadata":{},"cell_type":"markdown","source":"## Libraries"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.\nimport sys\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nimport cv2\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom mpl_toolkits.mplot3d import Axes3D\nfrom matplotlib.cm import get_cmap\nfrom sklearn.decomposition import PCA\nfrom sklearn.manifold import TSNE\nfrom sklearn.model_selection import train_test_split\n\nfrom scipy.stats import norm\n\nimport keras\nfrom keras import layers\nfrom keras.models import Model\nfrom keras import metrics\nfrom keras import backend as K   # 'generic' backend so ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Load data"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train_control = pd.read_csv('../input/train_controls.csv')\ntest_control = pd.read_csv('../input/test_controls.csv')\ntrain = pd.read_csv('../input/train.csv')\ntest = pd.read_csv('../input/test.csv')\npixel_stats = pd.read_csv('../input/pixel_stats.csv')\nsample_submission = pd.read_csv('../input/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Information about data"},{"metadata":{"trusted":true},"cell_type":"code","source":"dfs = [train_control, test_control, train, test, pixel_stats, sample_submission]\nnames = ['train control', 'test control', 'train', 'test', 'pixel stats', 'sample submission']\n\n# display info about a DataFrame\ndef dispDF(df, name):\n    print(\"========== \" + name + \" ==========\")\n    print(\"SHAPE ----------------------\")\n    print(df.shape)\n    print('')\n    print(\"HEAD ----------------------\")\n    print(df.head(7))\n    print('')\n    print(\"DATA TYPE ----------------\")\n    print(df.dtypes)\n    print('')\n    print(\"NAN counts ----------------\")\n    print(df.isna().sum())\n    print('')\n#     print(\"UNIQUES -------------------\")\n#     print(df.nunique())\n#     print('')\n    print(\"======================================\")\n    \npd.set_option('display.expand_frame_repr', False)\nfor df, name in zip(dfs, names):\n    dispDF(df, name)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Use RXRX for visualization\nI am grateful to [Nanashi's great kernel: Quick Visualization + EDA](https://www.kaggle.com/jesucristo/quick-visualization-eda/data). It works great, but not always for some reasons. So for now I load images from given folders and visualize them via cv2."},{"metadata":{"trusted":true},"cell_type":"code","source":"# !git clone https://github.com/recursionpharma/rxrx1-utils\n# print ('rxrx1-utils cloned!')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# !ls","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# sys.path.append('rxrx1-utils')\n# import rxrx.io as rio","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # randomly plot N (RGB) images\n# N = 15\n# np.random.seed(1220)\n# r = np.random.choice(train.shape[0], N)\n\n# fig, ax = plt.subplots(int(N/5), 5, figsize=(24, 18))\n# ax = ax.flatten()\n# for i in range(N):\n#     t = rio.load_site_as_rgb('train', train.loc[r[i], 'experiment'],\n#                              train.loc[r[i], 'plate'], \n#                              train.loc[r[i], 'well'], 1)\n#     ax[i].imshow(t)\n#     ax[i].axis('off')\n#     ax[i].set_title(train.loc[r[i], 'id_code'])\n# plt.tight_layout()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# image loader\ndef image_loader(train, row, site, channel, npix):\n    experiment = train.loc[row, 'experiment']\n    plate = train.loc[row, 'plate']\n    well = train.loc[row, 'well']\n    img = cv2.imread('../input/train/' + experiment +\n                    '/Plate' + str(plate) + '/' + well +\n                    '_s' + str(site) + '_w' + str(channel) + '.png')\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) \n    img = cv2.resize(img, (npix, npix)) # resize to npix x npix (for now)\n    return img","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# randomly plot N (RGB) images\nN = 30\nnp.random.seed(1220)\nr = np.random.choice(train.shape[0], N)\n\nfig, ax = plt.subplots(int(N/5), 5, figsize=(24, 18))\nax = ax.flatten()\nfor i in range(N):\n    img = image_loader(train, r[i], np.random.randint(1, 2), \n                       np.random.randint(1, 6), 256)\n    ax[i].imshow(img)\n    ax[i].axis('off')\n    ax[i].set_title(train.loc[r[i], 'id_code'])\nplt.tight_layout()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Cool! There is one weird guy (HEPG2-01_1_123) but others look stunning!"},{"metadata":{},"cell_type":"markdown","source":"## Visualizing targets"},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(1, 3, figsize=(12, 8))\nax = ax.flatten()\n\nsns.distplot(train['sirna'], kde=False, norm_hist=False, color='k', ax=ax[0])\nax[0].set_title('train')\nsns.distplot(train_control['sirna'], kde=False, norm_hist=False, color='k', ax=ax[1])\nax[1].set_title('train_control')\nsns.distplot(test_control['sirna'], kde=False, norm_hist=False, color='k', ax=ax[2])\nax[2].set_title('test_control')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Clustering images by image statistics\nWe have 'pixel_stats.csv', which summarizes stats for all the training images. Do we see clusters using this file?"},{"metadata":{"trusted":true},"cell_type":"code","source":"# mean, std, median, min, max for each site and channel\nstats = np.zeros((train.shape[0], 60))\ncolumns = ['mean', 'std', 'median', 'min', 'max']\nfor i, d in enumerate(train['id_code']):\n    temp_stats = pixel_stats.loc[pixel_stats['id_code'] == d, columns]\n    stats[i, :] = np.reshape(temp_stats.values, (60, ))\n\nprint(\"stats shape: \" + str(np.shape(stats)))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### PCA"},{"metadata":{"trusted":true},"cell_type":"code","source":"# use a subset of the data\nuniques = np.unique(train['sirna'].values)\ntrainX, valX, trainy, valy = train_test_split(stats, train['sirna'].values, test_size=0.9, random_state=1220)\n\nX_decomposed = PCA(n_components=2).fit_transform(trainX)\n\nfig, ax = plt.subplots(1, 1, figsize=(12, 8))\ncmap = sns.color_palette(\"husl\", len(uniques))\n\nfor i, u in enumerate(uniques):\n    marker = \"$\" + str(u) + \"$\"\n    idx = trainy == u\n    ax.scatter(X_decomposed[idx, 0], X_decomposed[idx, 1],\n              marker=marker, color=cmap[i])\nax.set_title(\"PCA\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### TSNE"},{"metadata":{"trusted":true},"cell_type":"code","source":"X_decomposed = TSNE(n_components=2).fit_transform(trainX)\n\nfig, ax = plt.subplots(1, 1, figsize=(12, 8))\n\nfor i, u in enumerate(uniques):\n    marker = \"$\" + str(u) + \"$\"\n    idx = trainy == u\n    ax.scatter(X_decomposed[idx, 0], X_decomposed[idx, 1],\n              marker=marker, color=cmap[i])\nax.set_title(\"TSNE\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Ah ... it looks bad. It may be too ambitious to separate data by simple image statistics."}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}