{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":14420,"databundleVersionId":868327,"sourceType":"competition"},{"sourceId":4326728,"sourceType":"datasetVersion","datasetId":2548025}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport imageio\nimport h5py\n%matplotlib inline","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-25T20:44:14.299177Z","iopub.execute_input":"2025-11-25T20:44:14.29951Z","iopub.status.idle":"2025-11-25T20:44:17.574617Z","shell.execute_reply.started":"2025-11-25T20:44:14.299479Z","shell.execute_reply":"2025-11-25T20:44:17.573858Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import sys\n\npath = '../input/mlcoursechapter3/chapter3'\n# data_path = '../input/mlcoursechapter3/chapter3/dataset'\ndata_path = '../input/recursion-cellular-image-classification'\nsys.path.append(path)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-25T21:20:28.908687Z","iopub.execute_input":"2025-11-25T21:20:28.908994Z","iopub.status.idle":"2025-11-25T21:20:28.912956Z","shell.execute_reply.started":"2025-11-25T21:20:28.908973Z","shell.execute_reply":"2025-11-25T21:20:28.912331Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = pd.read_csv(f'{data_path}/train.csv')\nprint('Dimesions of traning set：{0}'.format(train.shape))\ntrain.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-25T21:20:31.165334Z","iopub.execute_input":"2025-11-25T21:20:31.165979Z","iopub.status.idle":"2025-11-25T21:20:31.208434Z","shell.execute_reply.started":"2025-11-25T21:20:31.165953Z","shell.execute_reply":"2025-11-25T21:20:31.207663Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train['dataset'] ='train'\ntrain['well_type']='treatment'\ntrain['cell_type']=[train['experiment'][i].partition('-')[0] for i in range(train.shape[0])]\ntrain.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-25T20:44:29.564556Z","iopub.execute_input":"2025-11-25T20:44:29.56526Z","iopub.status.idle":"2025-11-25T20:44:29.714681Z","shell.execute_reply.started":"2025-11-25T20:44:29.565237Z","shell.execute_reply":"2025-11-25T20:44:29.714104Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_controls = pd.read_csv(f'{data_path}/train_controls.csv')\nprint('Dimensions of traning control set'.format(train_controls.shape))\ntrain_controls.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-25T21:21:35.287312Z","iopub.execute_input":"2025-11-25T21:21:35.287609Z","iopub.status.idle":"2025-11-25T21:21:35.30608Z","shell.execute_reply.started":"2025-11-25T21:21:35.287587Z","shell.execute_reply":"2025-11-25T21:21:35.305396Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_controls['dataset'] ='train_controls'\n\ntrain_controls['cell_type']=[train_controls['experiment'][i].partition('-')[0] for i in range(train_controls.shape[0])]\ntrain_controls.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-25T20:44:35.062528Z","iopub.execute_input":"2025-11-25T20:44:35.063016Z","iopub.status.idle":"2025-11-25T20:44:35.088979Z","shell.execute_reply.started":"2025-11-25T20:44:35.062995Z","shell.execute_reply":"2025-11-25T20:44:35.088252Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test = pd.read_csv(f'{data_path}/test.csv')\nprint('Test set dimensions'.format(test.shape))\ntest.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-25T21:22:04.450351Z","iopub.execute_input":"2025-11-25T21:22:04.450909Z","iopub.status.idle":"2025-11-25T21:22:04.473953Z","shell.execute_reply.started":"2025-11-25T21:22:04.450885Z","shell.execute_reply":"2025-11-25T21:22:04.473192Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test['dataset'] = 'test'\ntest['well_type'] = 'unknow'\ntest['cell_type'] = [test['experiment'][i].partition('-')[0] for i in range(test.shape[0])]\ntest['sirna'] = 'unknow'\ntest.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-25T21:22:12.803894Z","iopub.execute_input":"2025-11-25T21:22:12.804495Z","iopub.status.idle":"2025-11-25T21:22:12.891467Z","shell.execute_reply.started":"2025-11-25T21:22:12.804473Z","shell.execute_reply":"2025-11-25T21:22:12.890834Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_controls = pd.read_csv(f'{data_path}/test_controls.csv')\nprint('Test control set dimensions'.format(test_controls.shape))\ntest_controls.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-25T21:22:33.923554Z","iopub.execute_input":"2025-11-25T21:22:33.924179Z","iopub.status.idle":"2025-11-25T21:22:33.938757Z","shell.execute_reply.started":"2025-11-25T21:22:33.924131Z","shell.execute_reply":"2025-11-25T21:22:33.938173Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_controls['dataset'] = 'test_controls'\ntest_controls['cell_type'] = [test_controls['experiment'][i].partition('-')[0] for i in range(test_controls.shape[0])]\ntest_controls.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-25T20:44:50.500619Z","iopub.execute_input":"2025-11-25T20:44:50.501276Z","iopub.status.idle":"2025-11-25T20:44:50.519569Z","shell.execute_reply.started":"2025-11-25T20:44:50.501254Z","shell.execute_reply":"2025-11-25T20:44:50.518983Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"frames=[train,train_controls,test,test_controls]\ncombined = pd.concat(frames,sort=False)\ncombined.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-25T20:44:52.789757Z","iopub.execute_input":"2025-11-25T20:44:52.790352Z","iopub.status.idle":"2025-11-25T20:44:52.804781Z","shell.execute_reply.started":"2025-11-25T20:44:52.790329Z","shell.execute_reply":"2025-11-25T20:44:52.804035Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"combined.isnull().sum()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-25T20:44:54.805386Z","iopub.execute_input":"2025-11-25T20:44:54.805885Z","iopub.status.idle":"2025-11-25T20:44:54.832337Z","shell.execute_reply.started":"2025-11-25T20:44:54.805861Z","shell.execute_reply":"2025-11-25T20:44:54.831722Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for col in ['train','test','train_controls','test_controls']:\n    x=combined[combined.dataset==col]['cell_type'].value_counts().index\n    y=combined[combined.dataset==col]['cell_type'].value_counts()\n    plt.bar(x,y,label=col,alpha=0.7)\n    plt.legend()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-25T20:44:56.806275Z","iopub.execute_input":"2025-11-25T20:44:56.806872Z","iopub.status.idle":"2025-11-25T20:44:57.103925Z","shell.execute_reply.started":"2025-11-25T20:44:56.806852Z","shell.execute_reply":"2025-11-25T20:44:57.103303Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for col in ['train','test','train_controls','test_controls']:\n    labels=combined[combined.dataset==col]['sirna'].value_counts()\n    print(\"\\n{0}number of tags：{1}，The top 5 tags with the most repitions are：\\n{2}\"\n          .format(col,len(labels),labels.head(5)))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-25T21:24:09.479594Z","iopub.execute_input":"2025-11-25T21:24:09.479898Z","iopub.status.idle":"2025-11-25T21:24:09.518077Z","shell.execute_reply.started":"2025-11-25T21:24:09.479877Z","shell.execute_reply":"2025-11-25T21:24:09.517448Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"set1=set(list(combined[combined.dataset=='train']['sirna'].unique())).intersection \\\n(set(list(combined[combined.dataset=='train_controls']['sirna'].unique())))\nprint('The number of duplicate labels is:'.format(len(set1)))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-25T21:24:48.849417Z","iopub.execute_input":"2025-11-25T21:24:48.849703Z","iopub.status.idle":"2025-11-25T21:24:48.871792Z","shell.execute_reply.started":"2025-11-25T21:24:48.849682Z","shell.execute_reply":"2025-11-25T21:24:48.871202Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"set2=set(list(combined[combined.dataset=='train_controls']['sirna'].unique())).intersection \\\n(set(list(combined[combined.dataset=='test_controls']['sirna'].unique())))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-25T21:25:01.664899Z","iopub.execute_input":"2025-11-25T21:25:01.665751Z","iopub.status.idle":"2025-11-25T21:25:01.680304Z","shell.execute_reply.started":"2025-11-25T21:25:01.665721Z","shell.execute_reply":"2025-11-25T21:25:01.679498Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"all_train = combined[(combined.dataset == 'train')]\nU2OS_train = all_train[all_train['experiment'].str.contains('U2OS')]\nU2OS_train = U2OS_train[['id_code', 'sirna']]\nU2OS_train = U2OS_train.reset_index(drop=True)\nprint(U2OS_train.shape)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-25T20:45:11.844459Z","iopub.execute_input":"2025-11-25T20:45:11.845022Z","iopub.status.idle":"2025-11-25T20:45:11.86836Z","shell.execute_reply.started":"2025-11-25T20:45:11.844998Z","shell.execute_reply":"2025-11-25T20:45:11.867773Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# %ls /kaggle/input/recursion-cellular-image-classification/train/U2OS-01/Plate1/D05_s1_w1.png","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-25T20:45:15.742815Z","iopub.execute_input":"2025-11-25T20:45:15.743393Z","iopub.status.idle":"2025-11-25T20:45:15.746767Z","shell.execute_reply.started":"2025-11-25T20:45:15.743365Z","shell.execute_reply":"2025-11-25T20:45:15.746127Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def load_image(basepath, id_code, site):\n    images = np.zeros(shape=(6,512,512))\n    path = id_code.partition('_')[0] + '/Plate' + id_code.partition('_')[2][0] \\\n    + '/'+ id_code.rpartition('_')[2]\n    images[0,:,:] = imageio.imread(basepath + path + site + '_w1' + '.png')\n    images[1,:,:] = imageio.imread(basepath + path + site + '_w2' + \".png\")\n    images[2,:,:] = imageio.imread(basepath + path + site + '_w3' + \".png\")\n    images[3,:,:] = imageio.imread(basepath + path + site + '_w4' + \".png\")\n    images[4,:,:] = imageio.imread(basepath + path + site + '_w5' + \".png\")\n    images[5,:,:] = imageio.imread(basepath + path + site + '_w6' + \".png\")\n    return images\nfig, ax = plt.subplots(2,3,figsize=(18,10))\nimages = load_image(f'{data_path}/train/', U2OS_train['id_code'][40], '_s1')\nax[0][0].imshow(images[0], cmap=\"Blues\")\nax[0][1].imshow(images[1], cmap=\"Greens\")\nax[0][2].imshow(images[2], cmap=\"hot\")\nax[1][0].imshow(images[3], cmap=\"viridis\")\nax[1][1].imshow(images[4], cmap=\"gist_heat\")\nax[1][2].imshow(images[5], cmap=\"pink\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-25T20:45:17.781479Z","iopub.execute_input":"2025-11-25T20:45:17.782278Z","iopub.status.idle":"2025-11-25T20:45:19.09474Z","shell.execute_reply.started":"2025-11-25T20:45:17.782253Z","shell.execute_reply":"2025-11-25T20:45:19.093824Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from keras import Sequential\nfrom keras.layers import Dense,Activation,Conv2D,MaxPooling2D,Flatten,Dropout\n\nmodel=Sequential(name='VGG-16')\n#BLOCK1\nmodel.add(Conv2D(filters=64,kernel_size=(3,3),activation='relu',\n                padding='same',name='block1_conv1',input_shape=(224,224,3)))\nmodel.add(Conv2D(filters=64,kernel_size=(3,3),activation='relu',\n                padding='same',name='block1_conv2'))\nmodel.add(MaxPooling2D(pool_size=(2,2),strides=(2,2),name='block1_pool'))\n# BLOCK2\nmodel.add(Conv2D(filters = 128, kernel_size = (3, 3), activation = 'relu', \n                 padding = 'same', name = 'block2_conv1'))\nmodel.add(Conv2D(filters = 128, kernel_size = (3, 3), activation = 'relu', \n                 padding = 'same', name = 'block2_conv2'))\nmodel.add(MaxPooling2D(pool_size = (2, 2), strides = (2, 2), name = 'block2_pool'))\n# BLOCK3\nmodel.add(Conv2D(filters = 256, kernel_size = (3, 3), activation = 'relu', \n                 padding = 'same', name = 'block3_conv1'))\nmodel.add(Conv2D(filters = 256, kernel_size = (3, 3), activation = 'relu', \n                 padding = 'same', name = 'block3_conv2'))\nmodel.add(Conv2D(filters = 256, kernel_size = (3, 3), activation = 'relu', \n                 padding = 'same', name = 'block3_conv3'))\nmodel.add(MaxPooling2D(pool_size = (2, 2), strides = (2, 2), name = 'block3_pool'))\n# BLOCK4\nmodel.add(Conv2D(filters = 512, kernel_size = (3, 3), activation = 'relu', \n                 padding = 'same', name = 'block4_conv1'))\nmodel.add(Conv2D(filters = 512, kernel_size = (3, 3), activation = 'relu', \n                 padding = 'same', name = 'block4_conv2'))\nmodel.add(Conv2D(filters = 512, kernel_size = (3, 3), activation = 'relu', \n                 padding = 'same', name = 'block4_conv3'))\nmodel.add(MaxPooling2D(pool_size = (2, 2), strides = (2, 2), name = 'block4_pool'))\n# BLOCK5\nmodel.add(Conv2D(filters = 512, kernel_size = (3, 3), activation = 'relu', \n                 padding = 'same', name = 'block5_conv1'))\nmodel.add(Conv2D(filters = 512, kernel_size = (3, 3), activation = 'relu', \n                 padding = 'same', name = 'block5_conv2'))\nmodel.add(Conv2D(filters = 512, kernel_size = (3, 3), activation = 'relu', \n                 padding = 'same', name = 'block5_conv3'))\nmodel.add(MaxPooling2D(pool_size = (2, 2), strides = (2, 2), name = 'block5_pool'))\nmodel.add(Flatten())\n#FC1\nmodel.add(Dense(4096, activation = 'relu', name = 'fc1'))\nmodel.add(Dropout(0.5))\n#FC2\nmodel.add(Dense(4096, activation = 'relu', name = 'fc2'))\nmodel.add(Dropout(0.5))\n#Softmax\nmodel.add(Dense(1108, activation = 'softmax', name = 'prediction'))\nmodel.summary()\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-25T20:45:25.992332Z","iopub.execute_input":"2025-11-25T20:45:25.992822Z","iopub.status.idle":"2025-11-25T20:45:41.802622Z","shell.execute_reply.started":"2025-11-25T20:45:25.992799Z","shell.execute_reply":"2025-11-25T20:45:41.801898Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# os.path.join(base_path, dataset, experiment, \"Plate{}\".format(plate),\n#                         \"{}_s{}_w{}.png\".format(address, site, channel))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-03T01:15:03.911783Z","iopub.execute_input":"2024-12-03T01:15:03.912417Z","iopub.status.idle":"2024-12-03T01:15:03.917733Z","shell.execute_reply.started":"2024-12-03T01:15:03.912367Z","shell.execute_reply":"2024-12-03T01:15:03.916343Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\n\nimport numpy as np\nfrom skimage.io import imread\nimport pandas as pd\n\nimport tensorflow as tf\n\nDEFAULT_BASE_PATH = '.'\nDEFAULT_METADATA_BASE_PATH = os.path.join(DEFAULT_BASE_PATH, 'metadata')\nDEFAULT_IMAGES_BASE_PATH = os.path.join(DEFAULT_BASE_PATH, 'dataset')\nDEFAULT_CHANNELS = (1, 2, 3, 4, 5, 6)\nRGB_MAP = {\n    1: {\n        'rgb': np.array([19, 0, 249]),\n        'range': [0, 51]\n    },\n    2: {\n        'rgb': np.array([42, 255, 31]),\n        'range': [0, 107]\n    },\n    3: {\n        'rgb': np.array([255, 0, 25]),\n        'range': [0, 64]\n    },\n    4: {\n        'rgb': np.array([45, 255, 252]),\n        'range': [0, 191]\n    },\n    5: {\n        'rgb': np.array([250, 0, 253]),\n        'range': [0, 89]\n    },\n    6: {\n        'rgb': np.array([254, 255, 40]),\n        'range': [0, 191]\n    }\n}\n\n\ndef load_image(image_path):\n    with tf.io.gfile.GFile(image_path, 'rb') as f:\n        return imread(f) \n        #, format='png'\n\n\ndef load_images_as_tensor(image_paths, dtype=np.uint8):\n    n_channels = len(image_paths)\n\n    data = np.ndarray(shape=(512, 512, n_channels), dtype=dtype)\n\n    for ix, img_path in enumerate(image_paths):\n        data[:, :, ix] = load_image(img_path)\n\n    return data\n\n\ndef convert_tensor_to_rgb(t, channels=DEFAULT_CHANNELS, vmax=255, rgb_map=RGB_MAP):\n    \"\"\"\n    Converts and returns the image data as RGB image\n\n    Parameters\n    ----------\n    t : np.ndarray\n        original image data\n    channels : list of int\n        channels to include\n    vmax : int\n        the max value used for scaling\n    rgb_map : dict\n        the color mapping for each channel\n        See rxrx.io.RGB_MAP to see what the defaults are.\n\n    Returns\n    -------\n    np.ndarray the image data of the site as RGB channels\n    \"\"\"\n    colored_channels = []\n    for i, channel in enumerate(channels):\n        x = (t[:, :, i] / vmax) / \\\n            ((rgb_map[channel]['range'][1] - rgb_map[channel]['range'][0]) / 255) + \\\n            rgb_map[channel]['range'][0] / 255\n        x = np.where(x > 1., 1., x)\n        x_rgb = np.array(\n            np.outer(x, rgb_map[channel]['rgb']).reshape(512, 512, 3),\n            dtype=int)\n        colored_channels.append(x_rgb)\n    im = np.array(np.array(colored_channels).sum(axis=0), dtype=int)\n    im = np.where(im > 255, 255, im)\n    return im\n\n\ndef image_path(dataset,\n               experiment,\n               plate,\n               address,\n               site,\n               channel,\n               base_path=DEFAULT_IMAGES_BASE_PATH):\n    \"\"\"\n    Returns the path of a channel image.\n\n    Parameters\n    ----------\n    dataset : str\n        what subset of the data: train, test\n    experiment : str\n        experiment name\n    plate : int\n        plate number\n    address : str\n        plate address\n    site : int\n        site number\n    channel : int\n        channel number\n    base_path : str\n        the base path of the raw images\n\n    Returns\n    -------\n    str the path of image\n    \"\"\"\n    return os.path.join(base_path, dataset, experiment, \"Plate{}\".format(plate),\n                        \"{}_s{}_w{}.png\".format(address, site, channel))\n\n\ndef load_site(dataset,\n              experiment,\n              plate,\n              well,\n              site,\n              channels=DEFAULT_CHANNELS,\n              base_path=DEFAULT_IMAGES_BASE_PATH):\n    \"\"\"\n    Returns the image data of a site\n\n    Parameters\n    ----------\n    dataset : str\n        what subset of the data: train, test\n    experiment : str\n        experiment name\n    plate : int\n        plate number\n    address : str\n        plate address\n    site : int\n        site number\n    channels : list of int\n        channels to include\n    base_path : str\n        the base path of the raw images\n\n    Returns\n    -------\n    np.ndarray the image data of the site\n    \"\"\"\n    channel_paths = [\n        image_path(\n            dataset, experiment, plate, well, site, c, base_path=base_path)\n        for c in channels\n    ]\n    return load_images_as_tensor(channel_paths)\n\n\ndef load_site_as_rgb(dataset,\n                     experiment,\n                     plate,\n                     well,\n                     site,\n                     channels=DEFAULT_CHANNELS,\n                     base_path=DEFAULT_IMAGES_BASE_PATH,\n                     rgb_map=RGB_MAP):\n    \"\"\"\n    Loads and returns the image data as RGB image\n\n    Parameters\n    ----------\n    dataset : str\n        what subset of the data: train, test\n    experiment : str\n        experiment name\n    plate : int\n        plate number\n    address : str\n        plate address\n    site : int\n        site number\n    channels : list of int\n        channels to include\n    base_path : str\n        the base path of the raw images\n    rgb_map : dict\n        the color mapping for each channel\n        See rxrx.io.RGB_MAP to see what the defaults are.\n\n    Returns\n    -------\n    np.ndarray the image data of the site as RGB channels\n    \"\"\"\n    x = load_site(dataset, experiment, plate, well, site, channels, base_path)\n    return convert_tensor_to_rgb(x, channels, rgb_map=rgb_map)\n\n\ndef _tf_read_csv(path):\n    with tf.io.gfile.GFile(path, 'rb') as f:\n        return pd.read_csv(f)\n\n\ndef _load_dataset(base_path, dataset, include_controls=True):\n    df = _tf_read_csv(os.path.join(base_path, dataset + '.csv'))\n    if include_controls:\n        controls = _tf_read_csv(\n            os.path.join(base_path, dataset + '_controls.csv'))\n        df['well_type'] = 'treatment'\n        df = pd.concat([controls, df], sort=True)\n    df['cell_type'] = df.experiment.str.split(\"-\").apply(lambda a: a[0])\n    df['dataset'] = dataset\n    dfs = []\n    for site in (1, 2):\n        df = df.copy()\n        df['site'] = site\n        dfs.append(df)\n    res = pd.concat(dfs).sort_values(\n        by=['id_code', 'site']).set_index('id_code')\n    return res\n\n\ndef combine_metadata(base_path=DEFAULT_METADATA_BASE_PATH,\n                     include_controls=True):\n    \"\"\"\n    Combines all metadata files into a single dataframe and\n    expands it to include sites, not just wells.\n\n    Note, that the dtype of sirna is a float due to the missing\n    test values but it should be treated as an int.\n\n    Parameters\n    ----------\n    base_path : str\n        where the metadata files from Kaggle live\n    include_controls : bool\n        indicate if you want the controls included in the dataframe\n\n    Returns\n    -------\n    pandas.DataFrame the combined metadata\n    \"\"\"\n    df = pd.concat(\n        [\n            _load_dataset(\n                base_path, dataset, include_controls=include_controls)\n            for dataset in ['test', 'train']\n        ],\n        sort=True)\n    return df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-25T20:45:56.447306Z","iopub.execute_input":"2025-11-25T20:45:56.44824Z","iopub.status.idle":"2025-11-25T20:45:56.696126Z","shell.execute_reply.started":"2025-11-25T20:45:56.448214Z","shell.execute_reply":"2025-11-25T20:45:56.695489Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import sys\nsys.path.append('../input/mlcoursechapter3/chapter3/rxrx1utils')\nimport rxrx.io as rio\ncell_image = load_site_as_rgb('train', 'HUVEC-01', 3, 'K09', 2, base_path=data_path)\nplt.figure(figsize=(5, 5))\nplt.imshow(cell_image)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-25T21:26:08.11089Z","iopub.execute_input":"2025-11-25T21:26:08.111492Z","iopub.status.idle":"2025-11-25T21:26:08.524035Z","shell.execute_reply.started":"2025-11-25T21:26:08.111469Z","shell.execute_reply":"2025-11-25T21:26:08.523116Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import tensorflow as tf\nresized = tf.image.resize(cell_image,(224,224))\nresized = np.asarray(resized, dtype='uint8')\nplt.imshow(resized)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-25T20:46:15.415336Z","iopub.execute_input":"2025-11-25T20:46:15.416026Z","iopub.status.idle":"2025-11-25T20:46:15.633241Z","shell.execute_reply.started":"2025-11-25T20:46:15.416003Z","shell.execute_reply":"2025-11-25T20:46:15.632493Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\nsirna_code = pd.get_dummies(U2OS_train, columns = ['sirna'])\ntrain_data_x = U2OS_train\ntrain_data_y = sirna_code.drop(['id_code'], axis=1)\nprint(train_data_x.shape)\nprint(train_data_y.shape)\ntrain_data_y.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-25T20:46:20.063913Z","iopub.execute_input":"2025-11-25T20:46:20.064434Z","iopub.status.idle":"2025-11-25T20:46:20.087974Z","shell.execute_reply.started":"2025-11-25T20:46:20.064411Z","shell.execute_reply":"2025-11-25T20:46:20.087291Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#P3.22 \nfrom sklearn.model_selection import train_test_split\nx_train_id, x_val_id, Y_train, Y_val = train_test_split(\ntrain_data_x, train_data_y, test_size = .33, random_state=0)\nprint('Feature Dimensions of Training Set： {0}，label dimensions： {1}'.format(x_train_id.shape,Y_train.shape))\nprint('Feature Dimensions of the Validation Set： {0}，label dimensions： {1}'.format(x_val_id.shape,Y_val.shape))\nx_train_id.reset_index(drop=True, inplace=True)\nY_train.reset_index(drop=True, inplace=True)\nx_val_id.reset_index(drop=True, inplace=True)\nY_val.reset_index(drop=True, inplace=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-25T21:27:39.494111Z","iopub.execute_input":"2025-11-25T21:27:39.494979Z","iopub.status.idle":"2025-11-25T21:27:39.509699Z","shell.execute_reply.started":"2025-11-25T21:27:39.494954Z","shell.execute_reply":"2025-11-25T21:27:39.508894Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"classes = pd.concat([x_train_id['sirna'],x_val_id['sirna']],axis=0).to_list()\nfor i in range(len(classes)):\n    classes[i]=classes[i].encode(encoding='utf-8')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-25T20:46:32.442378Z","iopub.execute_input":"2025-11-25T20:46:32.44266Z","iopub.status.idle":"2025-11-25T20:46:32.448067Z","shell.execute_reply.started":"2025-11-25T20:46:32.44264Z","shell.execute_reply":"2025-11-25T20:46:32.447306Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%mkdir ./cell","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-25T20:46:36.260406Z","iopub.execute_input":"2025-11-25T20:46:36.261064Z","iopub.status.idle":"2025-11-25T20:46:36.40251Z","shell.execute_reply.started":"2025-11-25T20:46:36.261033Z","shell.execute_reply":"2025-11-25T20:46:36.401483Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%time\ndef make_dataset(x_data,y_data,x_data_id,Y_data):\n    i = 0\n    for code in x_data_id['id_code']:\n        cell_type = code.partition('_')[0] \n        plate = int(code.partition('_')[2][0]) \n        well = code.rpartition('_')[2] \n\n        img = load_site_as_rgb('train', cell_type, plate, well, 1, base_path=data_path)\n\n        resized = tf.image.resize(img,(224,224)) \n        resized = np.asarray(resized, dtype='uint8')\n        filename = './cell/'+code+'.jpg'\n\n        plt.imsave(filename,resized) \n        x_data[i] = resized \n        y_data[i]=Y_data.loc[i].ravel()\n        i += 1\nx_train_shape = (x_train_id.shape[0],224,224,3) \nx_val_shape = (x_val_id.shape[0],224,224,3)\nwith h5py.File('cell.h5','w') as f:\n\n    x_train = f.create_dataset(\"x_train\", x_train_shape ,'i1')  \n    y_train = f.create_dataset(\"y_train\", Y_train.shape ,'i1') \n    x_val = f.create_dataset(\"x_val\", x_val_shape ,'i1') \n    y_val = f.create_dataset(\"y_val\", Y_val.shape ,'i1') \n\n    classes = f.create_dataset('classes',data = classes)\n    make_dataset(x_train,y_train,x_train_id,Y_train)\n    print('x_train')\n    make_dataset(x_val,y_val,x_val_id,Y_val) \n    print('x_val')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-25T20:46:39.75727Z","iopub.execute_input":"2025-11-25T20:46:39.757569Z","iopub.status.idle":"2025-11-25T20:56:30.126779Z","shell.execute_reply.started":"2025-11-25T20:46:39.757545Z","shell.execute_reply":"2025-11-25T20:56:30.125937Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from keras.applications.vgg16 import VGG16\nfrom keras.preprocessing import image\nfrom keras.applications.vgg16 import preprocess_input\nmodel = VGG16(weights='imagenet', include_top=False)\ndef VGG16_extract_features(img):\n    x = np.expand_dims(img, axis=0)\n    features = model.predict(x)\n    return features\nfeatures = VGG16_extract_features(resized)\nfeatures.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-25T21:01:47.867805Z","iopub.execute_input":"2025-11-25T21:01:47.868632Z","iopub.status.idle":"2025-11-25T21:01:50.207716Z","shell.execute_reply.started":"2025-11-25T21:01:47.868607Z","shell.execute_reply":"2025-11-25T21:01:50.207085Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def load_dataset():\n    with h5py.File('cell.h5','r') as f:\n        x_train = f['x_train'][:]\n        y_train = f['y_train'][:] \n        x_val = f['x_val'][:] \n        y_val = f['y_val'][:] \n        classes = f['classes'][:] \n        return x_train, y_train, x_val, y_val, classes\nX_train, Y_train, X_val, Y_val, classes = load_dataset()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-25T21:01:58.343051Z","iopub.execute_input":"2025-11-25T21:01:58.343692Z","iopub.status.idle":"2025-11-25T21:01:58.635473Z","shell.execute_reply.started":"2025-11-25T21:01:58.343665Z","shell.execute_reply":"2025-11-25T21:01:58.634854Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%time\nm = X_train.shape[0]\nx_train = np.zeros((m,7,7,512))\nfor i in range(m):\n    x_train[i] = VGG16_extract_features(X_train[i])\nprint(x_train.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-25T21:02:03.71421Z","iopub.execute_input":"2025-11-25T21:02:03.714496Z","iopub.status.idle":"2025-11-25T21:04:28.520999Z","shell.execute_reply.started":"2025-11-25T21:02:03.714476Z","shell.execute_reply":"2025-11-25T21:04:28.520314Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%time\n#P3.28\nm = X_val.shape[0]\nx_val = np.zeros((m,7,7,512))\nfor i in range(m):\n    x_val[i] = VGG16_extract_features(X_val[i])\nprint(x_val.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-25T21:11:15.4255Z","iopub.execute_input":"2025-11-25T21:11:15.426104Z","iopub.status.idle":"2025-11-25T21:12:27.314896Z","shell.execute_reply.started":"2025-11-25T21:11:15.426084Z","shell.execute_reply":"2025-11-25T21:12:27.314327Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\nwith h5py.File('cell_features.h5','w') as f:\n    x_train = f.create_dataset(\"x_train\", data=x_train) \n    y_train = f.create_dataset(\"y_train\", data=Y_train) \n    x_val = f.create_dataset(\"x_val\", data=x_val) \n    y_val = f.create_dataset(\"y_val\", data=Y_val) \n    classes = f.create_dataset('classes',data = classes) ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-25T21:13:14.583006Z","iopub.execute_input":"2025-11-25T21:13:14.583787Z","iopub.status.idle":"2025-11-25T21:13:14.988737Z","shell.execute_reply.started":"2025-11-25T21:13:14.583758Z","shell.execute_reply":"2025-11-25T21:13:14.988157Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#P3.30\nimport h5py\ndef load_VGG16_dataset():\n    with h5py.File('cell_features.h5','r') as f:\n        x_train = f['x_train'][:]\n        y_train = f['y_train'][:]\n        x_val = f['x_val'][:] \n        y_val = f['y_val'][:] \n        classes = f['classes'][:]\n        return x_train, y_train, x_val, y_val, classes\nX_train, Y_train, X_val, Y_val, classes = load_VGG16_dataset()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-25T21:13:20.114119Z","iopub.execute_input":"2025-11-25T21:13:20.114431Z","iopub.status.idle":"2025-11-25T21:13:20.523701Z","shell.execute_reply.started":"2025-11-25T21:13:20.11441Z","shell.execute_reply":"2025-11-25T21:13:20.523101Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from keras import Sequential\nfrom keras.layers import Dense, Conv2D, Flatten, Dropout\nmodel = Sequential(name = 'VGG16-Cell-Transfer')\nmodel.add(Conv2D(filters = 64, kernel_size = (1, 1), activation = 'relu',\npadding = 'same', input_shape = (7, 7, 512)))\n\nmodel.add(Flatten())\nmodel.add(Dense(2048, activation = 'relu', name = 'fc1'))\nmodel.add(Dropout(0.5))\nmodel.add(Dense(2048, activation = 'relu', name = 'fc2'))\nmodel.add(Dropout(0.5))\nmodel.add(Dense(1108, activation = 'softmax', name = 'prediction'))\nmodel.summary()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-25T21:13:25.524935Z","iopub.execute_input":"2025-11-25T21:13:25.526171Z","iopub.status.idle":"2025-11-25T21:13:25.605137Z","shell.execute_reply.started":"2025-11-25T21:13:25.526105Z","shell.execute_reply":"2025-11-25T21:13:25.604518Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model.compile(optimizer='RMSProp',loss='categorical_crossentropy',metrics=['accuracy'])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-25T21:13:35.415901Z","iopub.execute_input":"2025-11-25T21:13:35.416638Z","iopub.status.idle":"2025-11-25T21:13:35.42953Z","shell.execute_reply.started":"2025-11-25T21:13:35.416614Z","shell.execute_reply":"2025-11-25T21:13:35.428938Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"epochs = 10\nbatch_size = 32\nhistory = model.fit(X_train, Y_train, epochs=epochs, batch_size=batch_size,validation_data=(X_val,Y_val))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-25T21:14:05.851241Z","iopub.execute_input":"2025-11-25T21:14:05.851919Z","iopub.status.idle":"2025-11-25T21:14:16.849412Z","shell.execute_reply.started":"2025-11-25T21:14:05.851894Z","shell.execute_reply":"2025-11-25T21:14:16.848843Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n%matplotlib inline\nx = range(1, len(history.history['accuracy'])+1)\nplt.plot(x, history.history['accuracy'])\nplt.plot(x, history.history['val_accuracy'])\nplt.title('Model accuracy')\nplt.ylabel('Accuracy')\nplt.xlabel('Epoch')\nplt.xticks(x)\nplt.legend(['Train', 'Val'], loc='upper left')\nplt.show()\nplt.plot(x, history.history['loss'])\nplt.plot(x, history.history['val_loss'])\nplt.title('Model loss')\nplt.ylabel('Loss')\nplt.xlabel('Epoch')\nplt.xticks(x)\nplt.legend(['Train', 'Val'], loc='lower left')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-25T21:14:29.501216Z","iopub.execute_input":"2025-11-25T21:14:29.501511Z","iopub.status.idle":"2025-11-25T21:14:29.833866Z","shell.execute_reply.started":"2025-11-25T21:14:29.50149Z","shell.execute_reply":"2025-11-25T21:14:29.833274Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import h5py\ndef load_dataset():\n    with h5py.File('cell.h5','r') as f:\n        x_train = f['x_train'][:] \n        y_train = f['y_train'][:]\n        x_val = f['x_val'][:] \n        y_val = f['y_val'][:] \n        classes = f['classes'][:]\n        return x_train, y_train, x_val, y_val, classes\nX_train, Y_train, X_val, Y_val, classes = load_dataset()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-25T21:14:39.579319Z","iopub.execute_input":"2025-11-25T21:14:39.579604Z","iopub.status.idle":"2025-11-25T21:14:39.934049Z","shell.execute_reply.started":"2025-11-25T21:14:39.579585Z","shell.execute_reply":"2025-11-25T21:14:39.933478Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\nX_train, Y_train, X_val, Y_val, classes = load_dataset()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-25T21:14:42.874864Z","iopub.execute_input":"2025-11-25T21:14:42.87511Z","iopub.status.idle":"2025-11-25T21:14:43.193253Z","shell.execute_reply.started":"2025-11-25T21:14:42.875094Z","shell.execute_reply":"2025-11-25T21:14:43.192651Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from keras import Sequential\nfrom keras.layers import Dense, Conv2D, Flatten, Dropout","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-25T21:14:45.571667Z","iopub.execute_input":"2025-11-25T21:14:45.572409Z","iopub.status.idle":"2025-11-25T21:14:45.576052Z","shell.execute_reply.started":"2025-11-25T21:14:45.572386Z","shell.execute_reply":"2025-11-25T21:14:45.575187Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from tensorflow.keras.applications.resnet50 import ResNet50\nResNet50_model = ResNet50(include_top=True, weights=None, classes=1108)\nResNet50_model.compile(optimizer='RMSProp',\nloss='categorical_crossentropy',\nmetrics=['accuracy'])\nepochs = 5\nbatch_size = 32\nhistory = ResNet50_model.fit(X_train, Y_train, epochs=epochs, batch_size=batch_size,validation_data=(X_val,Y_val))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-25T21:14:50.063336Z","iopub.execute_input":"2025-11-25T21:14:50.063611Z","iopub.status.idle":"2025-11-25T21:16:50.41773Z","shell.execute_reply.started":"2025-11-25T21:14:50.063591Z","shell.execute_reply":"2025-11-25T21:16:50.417111Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model.save('VGG16_Transfer_model.h5') \nResNet50_model.save('ResNet50_model.h5')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-25T21:17:03.962797Z","iopub.execute_input":"2025-11-25T21:17:03.963415Z","iopub.status.idle":"2025-11-25T21:17:04.824817Z","shell.execute_reply.started":"2025-11-25T21:17:03.963391Z","shell.execute_reply":"2025-11-25T21:17:04.824206Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from keras.models import load_model\nfrom keras.preprocessing import image\nfrom tensorflow.keras.applications.resnet50 import preprocess_input\nimport pandas as pd\nimport numpy as np\nResNet50_model = load_model('ResNet50_model.h5')\nimg_path = './cell/U2OS-01_2_B12.jpg'\nimg = image.load_img(img_path, target_size=(224, 224))\nplt.imshow(img)\ntrain = pd.read_csv(f\"{data_path}/train.csv\")\nprint('The real label is:'.format(train[train.id_code == 'U2OS-01_2_B12']['sirna'].values))\nx = image.img_to_array(img)\nx = np.expand_dims(x, axis=0)\nx = preprocess_input(x)\npred = ResNet50_model.predict(x) \npred = pred.ravel()\nindex = np.argmax(pred)\npred = np.zeros(1108)\npred[index] = 1\nprint('The model predicted value is：'.format(pred))\nfor i, y in enumerate(Y_train):\n    if (y == pred).all():\n        print('The model predicts the label as： {0}'.format(classes[i].decode('utf-8')))\n        break","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-25T21:34:56.401093Z","iopub.execute_input":"2025-11-25T21:34:56.401822Z","iopub.status.idle":"2025-11-25T21:35:00.388227Z","shell.execute_reply.started":"2025-11-25T21:34:56.401797Z","shell.execute_reply":"2025-11-25T21:35:00.387477Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}