{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":14420,"databundleVersionId":868327,"sourceType":"competition"}],"dockerImageVersionId":30664,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\n\nfrom tqdm import tqdm\nimport PIL\nimport cv2\nfrom PIL import Image, ImageOps\n\nfrom keras.models import Sequential, load_model\nfrom keras.layers import (Activation, Dropout, Flatten, Dense, Input, Conv2D, GlobalAveragePooling2D)\nfrom keras.applications.densenet import DenseNet121\nimport keras\nfrom keras.models import Model\n\nSIZE = 224\nNUM_CLASSES = 1108\n\ntrain_csv = pd.read_csv(\"../input/recursion-cellular-image-classification/train.csv\")\ntest_csv = pd.read_csv(\"../input/recursion-cellular-image-classification/test.csv\")\n# sub = pd.read_csv(\"../input/recursion-cellular-keras-densenet/submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2024-03-25T04:00:29.574841Z","iopub.execute_input":"2024-03-25T04:00:29.575903Z","iopub.status.idle":"2024-03-25T04:00:29.728589Z","shell.execute_reply.started":"2024-03-25T04:00:29.575726Z","shell.execute_reply":"2024-03-25T04:00:29.727158Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_csv.head()","metadata":{"execution":{"iopub.status.busy":"2024-03-25T04:04:43.761528Z","iopub.execute_input":"2024-03-25T04:04:43.762528Z","iopub.status.idle":"2024-03-25T04:04:43.776768Z","shell.execute_reply.started":"2024-03-25T04:04:43.762486Z","shell.execute_reply":"2024-03-25T04:04:43.775498Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_csv.shape","metadata":{"execution":{"iopub.status.busy":"2024-03-25T04:00:29.756259Z","iopub.execute_input":"2024-03-25T04:00:29.756711Z","iopub.status.idle":"2024-03-25T04:00:29.766286Z","shell.execute_reply.started":"2024-03-25T04:00:29.756679Z","shell.execute_reply":"2024-03-25T04:00:29.764982Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_csv.info()","metadata":{"execution":{"iopub.status.busy":"2024-03-25T04:00:29.769152Z","iopub.execute_input":"2024-03-25T04:00:29.769549Z","iopub.status.idle":"2024-03-25T04:00:29.802076Z","shell.execute_reply.started":"2024-03-25T04:00:29.769516Z","shell.execute_reply":"2024-03-25T04:00:29.800685Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 'sirna_'を空文字に置換して、sirnaカラムを更新します。\ntrain_csv['sirna'] = train_csv['sirna'].str.replace('sirna_', '')\ntrain_csv['sirna'] = train_csv['sirna'].astype(int)","metadata":{"execution":{"iopub.status.busy":"2024-03-25T04:00:29.803687Z","iopub.execute_input":"2024-03-25T04:00:29.804473Z","iopub.status.idle":"2024-03-25T04:00:29.834431Z","shell.execute_reply.started":"2024-03-25T04:00:29.804438Z","shell.execute_reply":"2024-03-25T04:00:29.833228Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_csv.head()","metadata":{"execution":{"iopub.status.busy":"2024-03-25T04:00:29.835675Z","iopub.execute_input":"2024-03-25T04:00:29.836065Z","iopub.status.idle":"2024-03-25T04:00:29.848917Z","shell.execute_reply.started":"2024-03-25T04:00:29.836036Z","shell.execute_reply":"2024-03-25T04:00:29.847918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_csv.info()","metadata":{"execution":{"iopub.status.busy":"2024-03-25T04:00:29.850881Z","iopub.execute_input":"2024-03-25T04:00:29.851317Z","iopub.status.idle":"2024-03-25T04:00:29.875149Z","shell.execute_reply.started":"2024-03-25T04:00:29.851281Z","shell.execute_reply":"2024-03-25T04:00:29.873940Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # NumPy 配列の全要素をリストとして変換\n# unique_list = list(unique_values_sirna)\n# unique_list.sort()\n# print(unique_list[:100])","metadata":{"execution":{"iopub.status.busy":"2024-03-25T04:00:29.876428Z","iopub.execute_input":"2024-03-25T04:00:29.877405Z","iopub.status.idle":"2024-03-25T04:00:29.882054Z","shell.execute_reply.started":"2024-03-25T04:00:29.877371Z","shell.execute_reply":"2024-03-25T04:00:29.880947Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_csv.describe().round(2)","metadata":{"execution":{"iopub.status.busy":"2024-03-25T04:00:29.883061Z","iopub.execute_input":"2024-03-25T04:00:29.883406Z","iopub.status.idle":"2024-03-25T04:00:29.914745Z","shell.execute_reply.started":"2024-03-25T04:00:29.883378Z","shell.execute_reply":"2024-03-25T04:00:29.913661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 'sirna' カラムのユニーク値0~10までのカウント数を出力\nunique_counts_sirna_0_to_10 = train_csv['sirna'].value_counts().sort_index().loc[0:10]\nprint(unique_counts_sirna_0_to_10)","metadata":{"execution":{"iopub.status.busy":"2024-03-25T04:00:29.918402Z","iopub.execute_input":"2024-03-25T04:00:29.918743Z","iopub.status.idle":"2024-03-25T04:00:29.930918Z","shell.execute_reply.started":"2024-03-25T04:00:29.918716Z","shell.execute_reply":"2024-03-25T04:00:29.929222Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 追加するデータを辞書形式で作成\nnew_row = {\n    'id_code': 'NA',\n    'experiment': 'NA',\n    'plate': 4,\n    'well': 'NA',\n    'sirna': 4\n}\n\n# 辞書をデータフレームに変換して追加\ntrain_csv = pd.concat([train_csv, pd.DataFrame([new_row])], ignore_index=True)","metadata":{"execution":{"iopub.status.busy":"2024-03-25T04:00:29.932770Z","iopub.execute_input":"2024-03-25T04:00:29.933232Z","iopub.status.idle":"2024-03-25T04:00:29.945124Z","shell.execute_reply.started":"2024-03-25T04:00:29.933189Z","shell.execute_reply":"2024-03-25T04:00:29.943906Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 'sirna' カラムのユニーク値0~10までのカウント数を出力\nunique_counts_sirna_0_to_10 = train_csv['sirna'].value_counts().sort_index().loc[0:10]\nprint(unique_counts_sirna_0_to_10)","metadata":{"execution":{"iopub.status.busy":"2024-03-25T04:00:29.948211Z","iopub.execute_input":"2024-03-25T04:00:29.948817Z","iopub.status.idle":"2024-03-25T04:00:29.958582Z","shell.execute_reply.started":"2024-03-25T04:00:29.948766Z","shell.execute_reply":"2024-03-25T04:00:29.957430Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # sirna の値が 0 のものに対して、plate カラムから得られる値を確認\n# plate_values_for_sirna_0 = train_csv[train_csv['sirna'] == 0]['plate']\n# print(plate_values_for_sirna_0)","metadata":{"execution":{"iopub.status.busy":"2024-03-25T04:00:29.959640Z","iopub.execute_input":"2024-03-25T04:00:29.959977Z","iopub.status.idle":"2024-03-25T04:00:29.965871Z","shell.execute_reply.started":"2024-03-25T04:00:29.959946Z","shell.execute_reply":"2024-03-25T04:00:29.964964Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.stack([train_csv.plate.values[train_csv.sirna == i] \nfor i in range(10)]).transpose()","metadata":{"execution":{"iopub.status.busy":"2024-03-25T04:00:29.966980Z","iopub.execute_input":"2024-03-25T04:00:29.967363Z","iopub.status.idle":"2024-03-25T04:00:29.985266Z","shell.execute_reply.started":"2024-03-25T04:00:29.967332Z","shell.execute_reply":"2024-03-25T04:00:29.983954Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# you will see the same output here for each sirna number\ntrain_csv.loc[train_csv.sirna==0,'plate'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2024-03-25T04:00:29.986719Z","iopub.execute_input":"2024-03-25T04:00:29.987804Z","iopub.status.idle":"2024-03-25T04:00:29.998281Z","shell.execute_reply.started":"2024-03-25T04:00:29.987759Z","shell.execute_reply":"2024-03-25T04:00:29.996996Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plate_groups = np.zeros((1108,4), int)\nfor sirna in range(1108):\n    grp = train_csv.loc[train_csv.sirna==sirna,:].plate.value_counts().index.values\n    if len(grp) == 3:  # grpの要素が3つある場合のみ処理\n        plate_groups[sirna, 0:3] = grp\n        plate_groups[sirna, 3] = 10 - grp.sum()  # 残りの1つのプレート番号を計算\n    elif len(grp) > 0:  # grpが空でないが、要素が3つではない場合\n        print(f\"sirna: {sirna} has unusual grp: {grp}\")\n    # grpが空の場合は何もしない（または必要に応じて記録を残す）\n\nplate_groups[:10,:]    ","metadata":{"execution":{"iopub.status.busy":"2024-03-25T04:00:30.000019Z","iopub.execute_input":"2024-03-25T04:00:30.000493Z","iopub.status.idle":"2024-03-25T04:00:30.883395Z","shell.execute_reply.started":"2024-03-25T04:00:30.000453Z","shell.execute_reply":"2024-03-25T04:00:30.882180Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plate_groups = np.zeros((1108,4), int)\nfor sirna in range(1108):\n    grp = train_csv.loc[train_csv.sirna==sirna,:].plate.value_counts().index.values\n    if len(grp) != 4:\n        print(f\"sirna: {sirna}, grp: {grp}\")  # 問題のあるsirnaとgrpの内容を表示\n    else:\n        plate_groups[sirna, :] = grp","metadata":{"execution":{"iopub.status.busy":"2024-03-25T04:00:30.884798Z","iopub.execute_input":"2024-03-25T04:00:30.885834Z","iopub.status.idle":"2024-03-25T04:00:31.894362Z","shell.execute_reply.started":"2024-03-25T04:00:30.885799Z","shell.execute_reply":"2024-03-25T04:00:31.893347Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# all_test_expを定義\nall_test_exp = test_csv.experiment.unique()\n\n# all_test_expの各要素を出力\nfor idx in range(len(all_test_exp)):\n    print(f\"Index {idx}: {all_test_exp[idx]}\")","metadata":{"execution":{"iopub.status.busy":"2024-03-25T04:00:31.896104Z","iopub.execute_input":"2024-03-25T04:00:31.896584Z","iopub.status.idle":"2024-03-25T04:00:31.905023Z","shell.execute_reply.started":"2024-03-25T04:00:31.896542Z","shell.execute_reply":"2024-03-25T04:00:31.904038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# all_test_exp = test_csv.experiment.unique()\n\n# group_plate_probs = np.zeros((len(all_test_exp),4))\n# for idx in range(len(all_test_exp)):\n#     preds = test_csv.loc[test_csv.experiment == all_test_exp[idx],'sirna'].values\n#     pp_mult = np.zeros((len(preds),1108))\n#     pp_mult[range(len(preds)),preds] = 1\n    \n#     sub_test = test_csv.loc[test_csv.experiment == all_test_exp[idx],:]\n#     assert len(pp_mult) == len(sub_test)\n    \n#     for j in range(4):\n#         mask = np.repeat(plate_groups[np.newaxis, :, j], len(pp_mult), axis=0) == \\\n#                np.repeat(sub_test.plate.values[:, np.newaxis], 1108, axis=1)\n        \n#         group_plate_probs[idx,j] = np.array(pp_mult)[mask].sum()/len(pp_mult)","metadata":{"execution":{"iopub.status.busy":"2024-03-25T04:00:31.906730Z","iopub.execute_input":"2024-03-25T04:00:31.907081Z","iopub.status.idle":"2024-03-25T04:00:31.916233Z","shell.execute_reply.started":"2024-03-25T04:00:31.907043Z","shell.execute_reply":"2024-03-25T04:00:31.914521Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(test_csv.columns.tolist())","metadata":{"execution":{"iopub.status.busy":"2024-03-25T04:00:31.917830Z","iopub.execute_input":"2024-03-25T04:00:31.918420Z","iopub.status.idle":"2024-03-25T04:00:31.925961Z","shell.execute_reply.started":"2024-03-25T04:00:31.918386Z","shell.execute_reply":"2024-03-25T04:00:31.924869Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train_csv.columns.tolist())","metadata":{"execution":{"iopub.status.busy":"2024-03-25T04:00:31.927330Z","iopub.execute_input":"2024-03-25T04:00:31.927752Z","iopub.status.idle":"2024-03-25T04:00:31.935806Z","shell.execute_reply.started":"2024-03-25T04:00:31.927720Z","shell.execute_reply":"2024-03-25T04:00:31.934806Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_model(input_shape,n_out):\n    input_tensor = Input(shape=input_shape)\n    base_model = DenseNet121(include_top=False,\n                   weights=None,\n                   input_tensor=input_tensor)\n    x = GlobalAveragePooling2D()(base_model.output)\n    x = Dense(1024, activation='relu')(x)\n \n    final_output = Dense(n_out, activation='softmax', name='final_output')(x)\n    model = Model(input_tensor, final_output)\n    \n    return model","metadata":{"execution":{"iopub.status.busy":"2024-03-25T04:00:31.936863Z","iopub.execute_input":"2024-03-25T04:00:31.937286Z","iopub.status.idle":"2024-03-25T04:00:31.945886Z","shell.execute_reply.started":"2024-03-25T04:00:31.937232Z","shell.execute_reply":"2024-03-25T04:00:31.944907Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras.layers import Input, Dense, GlobalAveragePooling2D\nfrom keras.models import Model\nfrom keras.applications.densenet import DenseNet121\n\ndef create_model(input_shape, n_out):\n    input_tensor = Input(shape=input_shape)\n    base_model = DenseNet121(include_top=False, weights=None, input_tensor=input_tensor)\n    x = GlobalAveragePooling2D()(base_model.output)\n    x = Dense(1024, activation='relu')(x)\n    final_output = Dense(n_out, activation='softmax', name='final_output')(x)\n    model = Model(input_tensor, final_output)\n    return model\n\n# テスト用の入力形状と出力数を定義\ninput_shape = (224, 224, 3)  # 例えば224x224のRGB画像\nn_out = 10  # 出力クラス数\n\n# モデルを作成\nmodel = create_model(input_shape, n_out)\n\n# モデルの概要を出力\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2024-03-25T04:00:31.947698Z","iopub.execute_input":"2024-03-25T04:00:31.948045Z","iopub.status.idle":"2024-03-25T04:00:34.300991Z","shell.execute_reply.started":"2024-03-25T04:00:31.948007Z","shell.execute_reply":"2024-03-25T04:00:34.299925Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = create_model(input_shape=(SIZE,SIZE,3),n_out=NUM_CLASSES)","metadata":{"execution":{"iopub.status.busy":"2024-03-25T04:00:34.302676Z","iopub.execute_input":"2024-03-25T04:00:34.303097Z","iopub.status.idle":"2024-03-25T04:00:35.903108Z","shell.execute_reply.started":"2024-03-25T04:00:34.303059Z","shell.execute_reply":"2024-03-25T04:00:35.901961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# model.load_weights('../input/recursion-cellular-keras-densenet/Densenet121.h5')","metadata":{"execution":{"iopub.status.busy":"2024-03-25T04:00:35.904375Z","iopub.execute_input":"2024-03-25T04:00:35.904710Z","iopub.status.idle":"2024-03-25T04:00:35.909684Z","shell.execute_reply.started":"2024-03-25T04:00:35.904683Z","shell.execute_reply":"2024-03-25T04:00:35.908575Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# predicted = []\n# for i, name in tqdm(enumerate(test_csv['id_code'])):\n#     path1 = os.path.join('../input/recursion-cellular-image-classification/test.csv', name+'_s1.jpeg')\n#     image1 = cv2.imread(path1)\n#     score_predict1 = model.predict((image1[np.newaxis])/255)\n    \n#     path2 = os.path.join('../input/recursion-cellular-image-classification-224-jpg/test/test/', name+'_s2.jpeg')\n#     image2 = cv2.imread(path2)\n#     score_predict2 = model.predict((image2[np.newaxis])/255)\n    \n#     predicted.append(0.5*(score_predict1 + score_predict2))\n#     #predicted.append(score_predict1)","metadata":{"execution":{"iopub.status.busy":"2024-03-26T03:18:46.565800Z","iopub.execute_input":"2024-03-26T03:18:46.566297Z","iopub.status.idle":"2024-03-26T03:18:46.596828Z","shell.execute_reply.started":"2024-03-26T03:18:46.566252Z","shell.execute_reply":"2024-03-26T03:18:46.595385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}