{"cells":[{"metadata":{},"cell_type":"markdown","source":"**Code for preprocessing and transfer learning from ResNet50 is available here:**\n\nhttps://github.com/evagian/Kaggle-Recursion-Cellular-Image-Classification"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"def balanced_subsample(x,y,subsample_size=1.0):\n\n    class_xs = []\n    min_elems = None\n\n    for yi in np.unique(y):\n        elems = x[(y == yi)]\n        class_xs.append((yi, elems))\n        if min_elems == None or elems.shape[0] < min_elems:\n            min_elems = elems.shape[0]\n\n    use_elems = min_elems\n    if subsample_size < 1:\n        use_elems = int(min_elems*subsample_size)\n\n    xs = []\n    ys = []\n\n    for ci,this_xs in class_xs:\n        if len(this_xs) > use_elems:\n            this_xs.reindex(np.random.permutation(this_xs.index))\n\n        x_ = this_xs[:use_elems]\n        y_ = np.empty(use_elems)\n        y_.fill(ci)\n\n        xs.append(x_)\n        ys.append(y_)\n\n    xs = pd.concat(xs)\n    ys = pd.Series(data=np.concatenate(ys), name='sirna')\n\n    return xs,ys","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data = pd.read_csv(\"/kaggle/input/recursion-cellular-image-classification/train.csv\")\n# Preview the first 5 lines of the loaded data\nprint(data.head())\nprint(data.shape)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Splitting train and validation set using balanced subsampling**"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Split train set\nxstrain,ystrain = balanced_subsample(data.drop('sirna', axis=1),data['sirna'],subsample_size=1/3)\n\nbalanced_sample_train = pd.concat([xstrain,ystrain], axis=1)\nprint(balanced_sample_train.head())\nprint(balanced_sample_train.shape)\nbalanced_sample_train = balanced_sample_train.dropna()\n\nbalanced_sample_train = balanced_sample_train.astype({\"plate\": int, \"sirna\": int})\n\nbalanced_sample_train.to_csv('/kaggle/input/recursion-cellular-image-classification/input/balanced_sample_train.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Split test set\ncommon = data.merge(balanced_sample_train,on=['id_code','sirna'])\nprint(common)\ntest_data = data[(~data.id_code.isin(common.id_code))&(~data.sirna.isin(common.sirna))]\n\nprint(test_data.head())\nprint(test_data.shape)\n\nxstest,ystest = balanced_subsample(test_data.drop('sirna', axis=1),test_data['sirna'],subsample_size=1/6)\n\nbalanced_sample_test = pd.concat([xstest,ystest], axis=1)\nprint(balanced_sample_test.head())\nprint(balanced_sample_test.shape)\n\nbalanced_sample_test = balanced_sample_test.dropna()\nbalanced_sample_test = balanced_sample_test.astype({\"plate\": int, \"sirna\": int})\n\nbalanced_sample_test.to_csv('/kaggle/input/recursion-cellular-image-classification/input/balanced_sample_test.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}