{"cells":[{"metadata":{"id":"lRAgy0gRPf8i","colab_type":"text"},"cell_type":"markdown","source":""},{"metadata":{"id":"6e_i9-8ewAOv","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"#imports\n\nimport pandas as pd\nimport json\nimport os\nimport sys\nimport tensorflow as tf\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport seaborn as sns\nfrom sklearn.metrics import accuracy_score\nimport numpy as np\nfrom sklearn.model_selection import train_test_split\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"id":"4EIyz23f7-wK","colab_type":"code","outputId":"f5c600d1-d87d-4cfd-f7c7-30dd52593bbe","colab":{"base_uri":"https://localhost:8080/","height":279},"trusted":true},"cell_type":"code","source":"!git clone https://github.com/recursionpharma/rxrx1-utils\nprint ('rxrx1-utils cloned!')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nsys.path.append('rxrx1-utils')\n    \nfrom rxrx.main import main\nimport rxrx.io as rio","execution_count":null,"outputs":[]},{"metadata":{"id":"2UcWw6FsPKBk","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":36},"outputId":"84c0a3be-4220-4e32-88db-c0713b189fdd","trusted":true},"cell_type":"code","source":"t = rio.load_site('train', 'RPE-05', 3, 'D19', 1)\n\nt.shape","execution_count":null,"outputs":[]},{"metadata":{"id":"kEDycBaOPJ62","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":821},"outputId":"35372940-a921-42bb-cb78-80e59aeeddd4","trusted":true},"cell_type":"code","source":"fig, axes = plt.subplots(2, 3, figsize=(16, 16))\n\nfor i, ax in enumerate(axes.flatten()):\n    ax.axis('off')\n    ax.set_title('channel {}'.format(i + 1))\n    _ = ax.imshow(t[:, :, i], cmap='gray')\n\n    ","execution_count":null,"outputs":[]},{"metadata":{"id":"ayAf-WyGYa1C","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":36},"outputId":"47a1b456-4b05-4c6b-82a0-a4685aaf53c8","trusted":true},"cell_type":"code","source":"x = rio.convert_tensor_to_rgb(t)\n\nx.shape","execution_count":null,"outputs":[]},{"metadata":{"id":"txoZnn7XZt7Q","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":486},"outputId":"b58850d9-b875-4600-d9be-94a01ed07bf3","trusted":true},"cell_type":"code","source":"plt.figure(figsize=(8, 8))\nplt.axis('off')\n\n_ = plt.imshow(x)","execution_count":null,"outputs":[]},{"metadata":{"id":"4_0SdW3taEjf","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":394},"outputId":"dc51b02e-0f79-4062-d234-54a0c5161bf8","trusted":true},"cell_type":"code","source":"\n\nmd = rio.combine_metadata()\n\n\nmd.head(10)\n\n","execution_count":null,"outputs":[]},{"metadata":{"id":"KnjofFF_0zcU","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":36},"outputId":"4be919bb-f356-4cd0-9893-ab9680857b36","trusted":true},"cell_type":"code","source":"md.shape","execution_count":null,"outputs":[]},{"metadata":{"id":"QY2D2LnyuLCb","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":167},"outputId":"06170604-a552-4d87-eec8-ac344e13b8e9","trusted":true},"cell_type":"code","source":"md.index","execution_count":null,"outputs":[]},{"metadata":{"id":"N1ZbPTD0uLPv","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":260},"outputId":"31c2053b-5e1d-44e8-af86-022853a17933","trusted":true},"cell_type":"code","source":"md.info()","execution_count":null,"outputs":[]},{"metadata":{"id":"4rufnbFk7lth","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":306},"outputId":"a7b5a7cc-8d58-4301-9dce-c92826ca3986","trusted":true},"cell_type":"code","source":"plt.figure(figsize= (5,10))\nplt.subplot(311)\nplt.title('cell_type')\nplt.tight_layout()\nsns.countplot(y =md['cell_type'])","execution_count":null,"outputs":[]},{"metadata":{"id":"Oh-uG9hM760_","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":167},"outputId":"574965df-9aac-40ab-90a6-526e0e3be530","trusted":true},"cell_type":"code","source":"plt.subplot(312)\nplt.title('dataset')\nplt.tight_layout()\nsns.countplot(y = md['dataset'])","execution_count":null,"outputs":[]},{"metadata":{"id":"zQTkWqoG8TWm","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":167},"outputId":"85d8a36e-baf1-40c2-845f-38cca5e0c08b","trusted":true},"cell_type":"code","source":"plt.subplot(312)\nplt.title('plate')\nplt.tight_layout()\nsns.countplot(y = md['plate'])","execution_count":null,"outputs":[]},{"metadata":{"id":"iFV8sGpP82vD","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":167},"outputId":"c0c79d23-554e-4d37-f174-1de4e2ade2f3","trusted":true},"cell_type":"code","source":"plt.subplot(312)\nplt.title(\"site\")\nplt.tight_layout()\nsns.countplot(y = md['site'])","execution_count":null,"outputs":[]},{"metadata":{"id":"Jj-QSWvR9DY3","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":167},"outputId":"21162ea2-7468-44e2-ce12-240488b322bb","trusted":true},"cell_type":"code","source":"plt.subplot(312)\nplt.title('well_type')\nplt.tight_layout()\nsns.countplot(y = md['well_type'])","execution_count":null,"outputs":[]},{"metadata":{"id":"2XjhWyaC9PTP","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":765},"outputId":"3277b4dd-5140-4673-e119-4af60432bfe3","trusted":true},"cell_type":"code","source":"#unique values\nfor i in md.columns:\n    print (\">> \",i,\"\\t\", md[i].unique())","execution_count":null,"outputs":[]},{"metadata":{"id":"tm3zERs_WYBm","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":185},"outputId":"49abb7e5-e007-4b84-b0eb-b9659c49a29e","trusted":true},"cell_type":"code","source":"#Missing values\nmissing_count = md.isnull().sum()\nmissing_count","execution_count":null,"outputs":[]},{"metadata":{"id":"u6WUzzQ0zk1o","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":237},"outputId":"95123764-1423-468a-f583-526047894d20","trusted":true},"cell_type":"code","source":"#fill in missing values\nmd = md.fillna(0)\nmd.head()","execution_count":null,"outputs":[]},{"metadata":{"id":"tff-VGQB0Tms","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":36},"outputId":"8faa2ba4-9e9c-4b7e-fda7-20157b267f44","trusted":true},"cell_type":"code","source":"#split into train and test\ntrain_df = md[md[\"dataset\"] == \"train\"]\ntest_df = md[md[\"dataset\"] == \"test\"]\ntrain_df.shape, test_df.shape","execution_count":null,"outputs":[]},{"metadata":{"id":"XQMEF-uQRLLW","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":424},"outputId":"07dd8cf0-0ff6-4dbb-9f45-64336011756c","trusted":true},"cell_type":"code","source":"#siRNA distribution for train and test sets\nplt.figure(figsize=(16,6))\nplt.title(\"Distribution of siRNA in the train and test set\")\nsns.distplot(train_df.sirna,color=\"green\", kde=True,bins='auto', label='train')\nsns.distplot(test_df.sirna,color=\"blue\", kde=True, bins='auto', label='test')\nplt.legend()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"id":"oq1k80dqSKr-","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":370},"outputId":"114c0ac2-1c50-4479-d04d-9206d39aaa48","trusted":true},"cell_type":"code","source":"feat1 = 'sirna'\nfig = plt.subplots(figsize=(15, 5))\n\n# train data\nplt.subplot(1, 2, 1)\nsns.kdeplot(train_df[feat1][train_df['site'] == 1], shade=False, color=\"b\", label = 'site 1')\nsns.kdeplot(train_df[feat1][train_df['site'] == 2], shade=False, color=\"r\", label = 'site 2')\nplt.title(feat1)\nplt.xlabel('Feature Values')\nplt.ylabel('Probability')\n\n# test data\nplt.subplot(1, 2, 2)\nsns.kdeplot(test_df[feat1][test_df['site'] == 1], shade=False, color=\"b\", label = 'site 1')\nsns.kdeplot(test_df[feat1][test_df['site'] == 2], shade=False, color=\"r\", label = 'site 2')\nplt.title(feat1)\nplt.xlabel('Feature Values')\nplt.ylabel('Probability')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"id":"jmMvlW1te0np","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":280},"outputId":"5d9593f0-fbeb-4625-bc3e-b1ef37710f82","trusted":true},"cell_type":"code","source":"\ntrain_df['category'] = train_df['experiment'].apply(lambda x: x.split('-')[0])\ntest_df['category'] = test_df['experiment'].apply(lambda x: x.split('-')[0])\n\ntrain_target_df = pd.get_dummies(train_df['sirna'])\n\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"id":"KUAUZI0dhwr1","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":36},"outputId":"4291b2bb-6193-4229-9104-29d2a83f12fd","trusted":true},"cell_type":"code","source":"train_df.shape, test_df.shape, train_target_df.shape\n","execution_count":null,"outputs":[]},{"metadata":{"id":"9rSNlPt-c7Z2","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":36},"outputId":"abc07b57-2567-455e-da29-cbe4748f3766","trusted":true},"cell_type":"code","source":"train_df.to_csv(\"train_df.csv\")\ntest_df.to_csv(\"test_df.csv\")\nprint(\"done\")","execution_count":null,"outputs":[]},{"metadata":{"id":"YxNYcNpqic05","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"# Pixel stats\ndf_pix = pd.read_csv(\"../input/pixel_stats.csv\")\ndf_pix.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Flatten the pixels\ndf_pix['idx'] = df_pix.groupby('id_code').cumcount()\ndf_pix = df_pix.pivot(index='id_code',columns='idx')[['mean','std', 'median','min','max' ]]\ndf_pix.columns = df_pix.columns.get_level_values(0)\ndf_pix.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Only train on treatment well_type\ndf_pix=df_pix.reset_index()\nmd=md[md.well_type=='treatment']\nmd=md.reset_index()\n\nmd.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df=md[['id_code','sirna', 'dataset','well_type']].merge(df_pix, on='id_code', how='left')\ndf.head()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_training = df.loc[df.dataset=='train']\ndf_test = df.loc[df.dataset=='test']\ndf_training.shape, df_test.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ndf_train=df_training.drop(['dataset', 'well_type'], axis=1)\ndf_test=df_test.drop(['dataset','well_type'], axis=1)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#df_train=df_train.drop(['index'], axis=1)\n#df_test=df_test.drop(['index'], axis=1)\n#df_train.head(), df_test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#create validation set\ntrain, test = train_test_split(df_train, test_size=0.1)\ntrain.shape, test.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cols=[]\nfor i in range(len(train.columns[2:])):\n    cols.append(train.columns[i+2]+str(i))\ntrain.columns=['id_code','sirna']+cols\ntest.columns=['id_code','sirna']+cols\ntrain.head(100)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X = train[train.columns[2:]].copy()\ny = train.sirna.values.astype(int)\nX_test = test[test.columns[2:]].copy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.neighbors import KNeighborsClassifier \nknn = KNeighborsClassifier(n_neighbors = 7).fit(X, y) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_train = knn.predict(X)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"accuracy_score(y, pred_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X.shape, test.shape, test[test.columns[2:]].shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test['pred']=knn.predict(test[test.columns[2:]])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X.shape, y","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_sub=pd.read_csv(\"../input/sample_submission.csv\")\ndf_sub.head()\n\ndf_submission=df_sub.drop(['sirna'], axis=1).merge(test[['id_code','pred']], on='id_code', how='left')\ndf_submission.columns=['id_code','sirna']\ndf_submission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_submission.to_csv('test_submission.csv',index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"colab":{"name":"Recursion Cellular Image Classification EDA","version":"0.3.2","provenance":[],"collapsed_sections":[]},"kernelspec":{"name":"python3","display_name":"Python 3"},"accelerator":"TPU"},"nbformat":4,"nbformat_minor":1}