{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":91249,"databundleVersionId":11294684,"sourceType":"competition"},{"sourceId":11920757,"sourceType":"datasetVersion","datasetId":7447556,"isSourceIdPinned":true},{"sourceId":11920762,"sourceType":"datasetVersion","datasetId":7447564,"isSourceIdPinned":true},{"sourceId":11920769,"sourceType":"datasetVersion","datasetId":7447573,"isSourceIdPinned":true},{"sourceId":11920774,"sourceType":"datasetVersion","datasetId":7447583,"isSourceIdPinned":true},{"sourceId":243015959,"sourceType":"kernelVersion"}],"dockerImageVersionId":31012,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 1. References\n- {Notebook} [Parse Data](https://www.kaggle.com/code/andrewjdarley/parse-data)\n- {Discussion} [More Motor Annotations](https://www.kaggle.com/competitions/byu-locating-bacterial-flagellar-motors-2025/discussion/569921)\n- {Discussion} [CryoET Dataset with Pixel Anomalies Corrected](https://www.kaggle.com/competitions/byu-locating-bacterial-flagellar-motors-2025/discussion/575028)\n- {Notebook} [BYU - Download CroyoET Dataset (1/4)](https://www.kaggle.com/code/sunghoshim/byu-download-croyoet-dataset-1-4)","metadata":{}},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nfrom PIL import Image\nimport yaml\nimport json\nfrom pathlib import Path\nfrom tqdm.notebook import tqdm\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.cluster import DBSCAN","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-05-28T11:26:52.027688Z","iopub.execute_input":"2025-05-28T11:26:52.028031Z","iopub.status.idle":"2025-05-28T11:26:55.658840Z","shell.execute_reply.started":"2025-05-28T11:26:52.027983Z","shell.execute_reply":"2025-05-28T11:26:55.657827Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 2. Params","metadata":{}},{"cell_type":"code","source":"MOTOR_SIZE = 500  # angstroms\nTRUST = 4\n\nUSE_CRYOET = True\n\nDESC = \"safe motor v6. M_SZ 500, no CryoET filter, bg\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-28T11:27:50.787796Z","iopub.execute_input":"2025-05-28T11:27:50.788119Z","iopub.status.idle":"2025-05-28T11:27:50.793255Z","shell.execute_reply.started":"2025-05-28T11:27:50.788098Z","shell.execute_reply":"2025-05-28T11:27:50.792197Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 3. Labels","metadata":{}},{"cell_type":"markdown","source":"## 3.1. BYU","metadata":{}},{"cell_type":"code","source":"# Load the labels CSV\ndf_label_byu = pd.read_csv(\"/kaggle/input/byu-locating-bacterial-flagellar-motors-2025/train_labels.csv\")\nprint(df_label_byu.shape)\nprint(f\"{df_label_byu['tomo_id'].nunique()} tomos\")\nnum_tomo_with_motor_byu = df_label_byu.loc[df_label_byu['Number of motors'] >= 1, 'tomo_id'].nunique()\nprint(f\"{num_tomo_with_motor_byu} tomos with motors\")\nprint(f\"- {df_label_byu.loc[df_label_byu['Number of motors'] == 1, 'tomo_id'].nunique():3d} (+4) tomos with one motor\")\nprint(f\"- {df_label_byu.loc[df_label_byu['Number of motors'] > 1, 'tomo_id'].nunique():3d} (-4) tomos with multiple motors\")\n\ndf_label_byu['jpgs_dir'] = '/kaggle/input/byu-locating-bacterial-flagellar-motors-2025/train'\ndf_label_byu['box_size'] = round(MOTOR_SIZE / df_label_byu['Voxel spacing'], 1)\n# df_label_byu['trust'] = (TRUST_SIZE / df_label_byu['Voxel spacing']).astype(int).clip(upper=MAX_TRUST)\ndf_label_byu.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-28T11:27:53.768569Z","iopub.execute_input":"2025-05-28T11:27:53.768867Z","iopub.status.idle":"2025-05-28T11:27:53.838039Z","shell.execute_reply.started":"2025-05-28T11:27:53.768844Z","shell.execute_reply":"2025-05-28T11:27:53.837202Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_label_byu['Voxel spacing'].describe()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-28T11:28:01.719545Z","iopub.execute_input":"2025-05-28T11:28:01.719831Z","iopub.status.idle":"2025-05-28T11:28:01.732419Z","shell.execute_reply.started":"2025-05-28T11:28:01.719811Z","shell.execute_reply":"2025-05-28T11:28:01.731263Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_label_byu['box_size'].describe()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-28T11:28:02.591801Z","iopub.execute_input":"2025-05-28T11:28:02.592088Z","iopub.status.idle":"2025-05-28T11:28:02.601743Z","shell.execute_reply.started":"2025-05-28T11:28:02.592062Z","shell.execute_reply":"2025-05-28T11:28:02.600673Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# motor 없는 거 제외\ndf_byu = df_label_byu[df_label_byu['Motor axis 0'] != -1.0].copy()\ndf_byu.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-28T11:28:27.449991Z","iopub.execute_input":"2025-05-28T11:28:27.450325Z","iopub.status.idle":"2025-05-28T11:28:27.461919Z","shell.execute_reply.started":"2025-05-28T11:28:27.450302Z","shell.execute_reply":"2025-05-28T11:28:27.460773Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"ser_num_motor = df_byu.groupby('tomo_id').size()\nser_num_motor","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-28T11:28:33.130738Z","iopub.execute_input":"2025-05-28T11:28:33.131343Z","iopub.status.idle":"2025-05-28T11:28:33.141991Z","shell.execute_reply.started":"2025-05-28T11:28:33.131290Z","shell.execute_reply":"2025-05-28T11:28:33.140931Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_byu['num_motor'] = ser_num_motor[df_byu['tomo_id']].values\ndf_byu.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-28T11:28:42.093136Z","iopub.execute_input":"2025-05-28T11:28:42.093669Z","iopub.status.idle":"2025-05-28T11:28:42.114678Z","shell.execute_reply.started":"2025-05-28T11:28:42.093622Z","shell.execute_reply":"2025-05-28T11:28:42.113614Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_byu['num_motor'].value_counts()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-28T11:28:46.263673Z","iopub.execute_input":"2025-05-28T11:28:46.264583Z","iopub.status.idle":"2025-05-28T11:28:46.276812Z","shell.execute_reply.started":"2025-05-28T11:28:46.264546Z","shell.execute_reply":"2025-05-28T11:28:46.275979Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 'Number of motors' 이상한 motor\ndf_byu[df_byu['Number of motors'] != df_byu['num_motor']]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-28T11:29:06.862936Z","iopub.execute_input":"2025-05-28T11:29:06.863224Z","iopub.status.idle":"2025-05-28T11:29:06.881407Z","shell.execute_reply.started":"2025-05-28T11:29:06.863203Z","shell.execute_reply":"2025-05-28T11:29:06.880428Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_byu['Number of motors'] = df_byu['num_motor']\ndf_byu = df_byu.drop('num_motor', axis=1)\ndf_byu['dataset'] = 'byu'\nprint(df_byu.shape)\ndf_byu.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-28T11:29:14.886008Z","iopub.execute_input":"2025-05-28T11:29:14.886365Z","iopub.status.idle":"2025-05-28T11:29:14.905655Z","shell.execute_reply.started":"2025-05-28T11:29:14.886320Z","shell.execute_reply":"2025-05-28T11:29:14.904712Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_sorted = df_byu.sort_values(['tomo_id', 'Motor axis 0'])\ndf_sorted.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-28T11:29:41.474014Z","iopub.execute_input":"2025-05-28T11:29:41.474491Z","iopub.status.idle":"2025-05-28T11:29:41.491833Z","shell.execute_reply.started":"2025-05-28T11:29:41.474463Z","shell.execute_reply":"2025-05-28T11:29:41.491065Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_sorted['z_diff_prev'] = df_sorted.groupby(['tomo_id'])['Motor axis 0'].diff().abs()\ndf_sorted['z_diff_next'] = df_sorted.groupby(['tomo_id'])['Motor axis 0'].diff(-1).abs()\ndf_sorted.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-28T11:35:00.142867Z","iopub.execute_input":"2025-05-28T11:35:00.143180Z","iopub.status.idle":"2025-05-28T11:35:00.165568Z","shell.execute_reply.started":"2025-05-28T11:35:00.143156Z","shell.execute_reply":"2025-05-28T11:35:00.164492Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_sorted[((df_sorted['z_diff_prev'].isna()) & (df_sorted['z_diff_next'].isna()))].shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-28T11:35:05.990082Z","iopub.execute_input":"2025-05-28T11:35:05.990409Z","iopub.status.idle":"2025-05-28T11:35:05.997852Z","shell.execute_reply.started":"2025-05-28T11:35:05.990383Z","shell.execute_reply":"2025-05-28T11:35:05.996978Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_sorted[((df_sorted['z_diff_prev'].isna()) & (df_sorted['z_diff_next'] >= 10))].shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-28T11:35:06.740297Z","iopub.execute_input":"2025-05-28T11:35:06.740879Z","iopub.status.idle":"2025-05-28T11:35:06.749108Z","shell.execute_reply.started":"2025-05-28T11:35:06.740854Z","shell.execute_reply":"2025-05-28T11:35:06.747914Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_sorted[(df_sorted['z_diff_prev'] >= 10) & df_sorted['z_diff_next'].isna()].shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-28T11:35:08.529894Z","iopub.execute_input":"2025-05-28T11:35:08.530216Z","iopub.status.idle":"2025-05-28T11:35:08.537550Z","shell.execute_reply.started":"2025-05-28T11:35:08.530191Z","shell.execute_reply":"2025-05-28T11:35:08.536453Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_sorted[(df_sorted['z_diff_prev'] >= 10) & (df_sorted['z_diff_next'] >= 10)].shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-28T11:35:13.548446Z","iopub.execute_input":"2025-05-28T11:35:13.548726Z","iopub.status.idle":"2025-05-28T11:35:13.555494Z","shell.execute_reply.started":"2025-05-28T11:35:13.548705Z","shell.execute_reply":"2025-05-28T11:35:13.554698Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"317 + 32 + 30 + 10","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-28T11:35:18.177423Z","iopub.execute_input":"2025-05-28T11:35:18.178193Z","iopub.status.idle":"2025-05-28T11:35:18.183085Z","shell.execute_reply.started":"2025-05-28T11:35:18.178160Z","shell.execute_reply":"2025-05-28T11:35:18.182260Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"condition = (df_sorted['z_diff_prev'] < 10) | (df_sorted['z_diff_next'] < 10)\n\ndf_byu_final = df_sorted[~condition].copy()\ndf_byu_final = df_byu_final.drop(['z_diff_prev', 'z_diff_next'], axis=1)\nprint(df_byu_final.shape)\ndf_byu_final.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-28T11:36:05.810102Z","iopub.execute_input":"2025-05-28T11:36:05.810492Z","iopub.status.idle":"2025-05-28T11:36:05.830869Z","shell.execute_reply.started":"2025-05-28T11:36:05.810462Z","shell.execute_reply":"2025-05-28T11:36:05.830014Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_byu_final['Voxel spacing'].hist()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-28T11:36:14.198972Z","iopub.execute_input":"2025-05-28T11:36:14.199302Z","iopub.status.idle":"2025-05-28T11:36:14.642907Z","shell.execute_reply.started":"2025-05-28T11:36:14.199259Z","shell.execute_reply":"2025-05-28T11:36:14.642136Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_byu_final['Voxel spacing'].value_counts()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-28T11:36:18.485917Z","iopub.execute_input":"2025-05-28T11:36:18.486305Z","iopub.status.idle":"2025-05-28T11:36:18.496472Z","shell.execute_reply.started":"2025-05-28T11:36:18.486278Z","shell.execute_reply":"2025-05-28T11:36:18.495342Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_byu_final['Array shape (axis 1)'].describe()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-28T11:36:20.923533Z","iopub.execute_input":"2025-05-28T11:36:20.923812Z","iopub.status.idle":"2025-05-28T11:36:20.933130Z","shell.execute_reply.started":"2025-05-28T11:36:20.923791Z","shell.execute_reply":"2025-05-28T11:36:20.932298Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_byu_final['voxel_spacing_int'] = df_byu_final['Voxel spacing'].astype(int)\ndf_byu_final['voxel_spacing_int'].value_counts()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-28T11:36:23.040930Z","iopub.execute_input":"2025-05-28T11:36:23.041228Z","iopub.status.idle":"2025-05-28T11:36:23.049755Z","shell.execute_reply.started":"2025-05-28T11:36:23.041207Z","shell.execute_reply":"2025-05-28T11:36:23.048981Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_byu_final['box_size'].describe()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-28T11:36:25.331505Z","iopub.execute_input":"2025-05-28T11:36:25.331783Z","iopub.status.idle":"2025-05-28T11:36:25.341176Z","shell.execute_reply.started":"2025-05-28T11:36:25.331763Z","shell.execute_reply":"2025-05-28T11:36:25.340130Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 3.2. CryoET","metadata":{}},{"cell_type":"code","source":"df_cryoet_list = []\ndf_temp = pd.read_csv(\"/kaggle/input/byu-croyoet-trust4-jpg-14/labels.csv\")\ndf_temp['jpgs_dir'] = '/kaggle/input/byu-croyoet-trust4-jpg-14/jpgs'\ndf_cryoet_list.append(df_temp)\ndf_temp.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-28T11:36:28.745596Z","iopub.execute_input":"2025-05-28T11:36:28.745891Z","iopub.status.idle":"2025-05-28T11:36:28.772486Z","shell.execute_reply.started":"2025-05-28T11:36:28.745868Z","shell.execute_reply":"2025-05-28T11:36:28.771523Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_temp = pd.read_csv(\"/kaggle/input/byu-croyoet-trust4-jpg-24/labels.csv\")\ndf_temp['jpgs_dir'] = '/kaggle/input/byu-croyoet-trust4-jpg-24/jpgs'\ndf_cryoet_list.append(df_temp)\n\ndf_temp = pd.read_csv(\"/kaggle/input/byu-croyoet-trust4-jpg-34/labels.csv\")\ndf_temp['jpgs_dir'] = '/kaggle/input/byu-croyoet-trust4-jpg-34/jpgs'\ndf_cryoet_list.append(df_temp)\n\ndf_temp = pd.read_csv(\"/kaggle/input/byu-croyoet-trust4-jpg-44/labels.csv\")\ndf_temp['jpgs_dir'] = '/kaggle/input/byu-croyoet-trust4-jpg-44/jpgs'\ndf_cryoet_list.append(df_temp)\n\nlen(df_cryoet_list)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-28T11:36:30.455144Z","iopub.execute_input":"2025-05-28T11:36:30.455444Z","iopub.status.idle":"2025-05-28T11:36:30.509584Z","shell.execute_reply.started":"2025-05-28T11:36:30.455423Z","shell.execute_reply":"2025-05-28T11:36:30.508866Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_cryoet = pd.concat(df_cryoet_list).reset_index(drop=True)\ndf_cryoet['box_size'] = round(MOTOR_SIZE / df_cryoet['Voxel spacing'], 1)\ndf_cryoet['row_id'] = range(len(df_cryoet))\ndf_cryoet['dataset'] = 'cryoet'\nprint(df_cryoet.shape)\ndf_cryoet.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-28T11:36:52.526736Z","iopub.execute_input":"2025-05-28T11:36:52.527031Z","iopub.status.idle":"2025-05-28T11:36:52.545136Z","shell.execute_reply.started":"2025-05-28T11:36:52.527007Z","shell.execute_reply":"2025-05-28T11:36:52.544192Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_cryoet['Voxel spacing'].hist()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Voxel spacing 너무 작은거 제외 (홀수 버전)\n# condition = (df_cryoet['Voxel spacing'] > 10)\n# df_cryoet = df_cryoet[condition]\n# print(df_cryoet.shape)\n# df_cryoet['Voxel spacing'].hist()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-28T11:37:15.345389Z","iopub.execute_input":"2025-05-28T11:37:15.345677Z","iopub.status.idle":"2025-05-28T11:37:15.525880Z","shell.execute_reply.started":"2025-05-28T11:37:15.345655Z","shell.execute_reply":"2025-05-28T11:37:15.525183Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_cryoet['Voxel spacing'].value_counts()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-28T11:37:44.404540Z","iopub.execute_input":"2025-05-28T11:37:44.404833Z","iopub.status.idle":"2025-05-28T11:37:44.412657Z","shell.execute_reply.started":"2025-05-28T11:37:44.404810Z","shell.execute_reply":"2025-05-28T11:37:44.411776Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(df_cryoet.shape)\nprint(f\"{df_cryoet['tomo_id'].nunique()} tomos\")\nnum_tomo_with_motor_cryoet = (df_cryoet.groupby('tomo_id').size()>=1).sum()\nprint(f\"{num_tomo_with_motor_cryoet} tomos with motors\")\nprint(f\"- {(df_cryoet.groupby('tomo_id').size()==1).sum():4d} tomos with one motor\")\nprint(f\"- {(df_cryoet.groupby('tomo_id').size()>1).sum():4d} tomos with multiple motors\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-28T11:37:47.581540Z","iopub.execute_input":"2025-05-28T11:37:47.581826Z","iopub.status.idle":"2025-05-28T11:37:47.595952Z","shell.execute_reply.started":"2025-05-28T11:37:47.581804Z","shell.execute_reply":"2025-05-28T11:37:47.595038Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# df_cryoet['Voxel spacing'].hist(bins=20)\ndf_cryoet['Voxel spacing'].hist()\ndf_cryoet['Voxel spacing'].describe()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-28T11:37:52.563575Z","iopub.execute_input":"2025-05-28T11:37:52.564186Z","iopub.status.idle":"2025-05-28T11:37:52.746790Z","shell.execute_reply.started":"2025-05-28T11:37:52.564160Z","shell.execute_reply":"2025-05-28T11:37:52.745532Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_cryoet['voxel_spacing_int'] = df_cryoet['Voxel spacing'].astype(int)\ndf_cryoet['voxel_spacing_int'].value_counts()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-28T11:38:03.177371Z","iopub.execute_input":"2025-05-28T11:38:03.177648Z","iopub.status.idle":"2025-05-28T11:38:03.185640Z","shell.execute_reply.started":"2025-05-28T11:38:03.177630Z","shell.execute_reply":"2025-05-28T11:38:03.184746Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_cryoet['box_size'].hist(bins=30)\ndf_cryoet['box_size'].describe()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-28T11:38:09.874138Z","iopub.execute_input":"2025-05-28T11:38:09.874496Z","iopub.status.idle":"2025-05-28T11:38:10.076606Z","shell.execute_reply.started":"2025-05-28T11:38:09.874471Z","shell.execute_reply":"2025-05-28T11:38:10.075598Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_cryoet['Array shape (axis 1)'].hist()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-28T11:38:18.501440Z","iopub.execute_input":"2025-05-28T11:38:18.501735Z","iopub.status.idle":"2025-05-28T11:38:18.680314Z","shell.execute_reply.started":"2025-05-28T11:38:18.501713Z","shell.execute_reply":"2025-05-28T11:38:18.679415Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_cryoet['Array shape (axis 2)'].hist()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-28T11:38:19.235659Z","iopub.execute_input":"2025-05-28T11:38:19.236227Z","iopub.status.idle":"2025-05-28T11:38:19.409733Z","shell.execute_reply.started":"2025-05-28T11:38:19.236201Z","shell.execute_reply":"2025-05-28T11:38:19.408891Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 4. Output folder","metadata":{}},{"cell_type":"code","source":"path_output = Path().cwd()\npath_output","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-28T11:38:21.864527Z","iopub.execute_input":"2025-05-28T11:38:21.864823Z","iopub.status.idle":"2025-05-28T11:38:21.870570Z","shell.execute_reply.started":"2025-05-28T11:38:21.864800Z","shell.execute_reply":"2025-05-28T11:38:21.869543Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"path_images_train = path_output / \"images\" / \"train\"\npath_images_val = path_output / \"images\" / \"val\"\npath_labels_train = path_output / \"labels\" / \"train\"\npath_labels_val = path_output / \"labels\" / \"val\"\n\nfor path in [path_images_train, path_images_val, path_labels_train, path_labels_val]:\n    path.mkdir(parents=True, exist_ok=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-28T11:38:23.792797Z","iopub.execute_input":"2025-05-28T11:38:23.793138Z","iopub.status.idle":"2025-05-28T11:38:23.798952Z","shell.execute_reply.started":"2025-05-28T11:38:23.793115Z","shell.execute_reply":"2025-05-28T11:38:23.798108Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 5. Util Functions","metadata":{}},{"cell_type":"code","source":"# Image processing functions\ndef normalize_slice(slice_data):\n    \"\"\"\n    Normalize slice data using 2nd and 98th percentiles\n    \"\"\"\n    # Calculate percentiles\n    p2 = np.percentile(slice_data, 2)\n    p98 = np.percentile(slice_data, 98)\n    \n    # Clip the data to the percentile range\n    clipped_data = np.clip(slice_data, p2, p98)\n    \n    # Normalize to [0, 255] range\n    normalized = 255 * (clipped_data - p2) / (p98 - p2)\n    \n    return np.uint8(normalized)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-28T11:38:26.613908Z","iopub.execute_input":"2025-05-28T11:38:26.614228Z","iopub.status.idle":"2025-05-28T11:38:26.620759Z","shell.execute_reply.started":"2025-05-28T11:38:26.614201Z","shell.execute_reply":"2025-05-28T11:38:26.619384Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def process_tomogram_set(df, tomogram_ids, path_images_dir, path_labels_dir, set_name):\n    df_motors = df[df['tomo_id'].isin(tomogram_ids)]\n    print(f\"[{set_name}] {len(df_motors)} motors in {df_motors['tomo_id'].nunique()} tomos\")\n\n    processed_slices = 0\n    for idx, row in tqdm(df_motors.iterrows(), total=len(df_motors), desc=set_name):\n        path_jpgs = Path(row['jpgs_dir'])\n        tomo_id = row['tomo_id']\n        cz = int(row['Motor axis 0'])\n        cy = int(row['Motor axis 1'])\n        cx = int(row['Motor axis 2'])\n        z_max = int(row['Array shape (axis 0)'])\n        box_size = row['box_size']\n        trust = TRUST\n\n        z_min = max(0, cz - trust)\n        z_max = min(z_max - 1, cz + trust)\n\n        for z in range(z_min, z_max + 1):\n            slice_filename = f\"slice_{z:04d}.jpg\"\n            path_src = path_jpgs / tomo_id / slice_filename\n    \n            img = Image.open(path_src)    \n            np_img = np.array(img)\n            np_normalized = normalize_slice(np_img)\n        \n            # Create destination filename (with unique identifier)\n            dest_filename = f\"{tomo_id}_z{z:04d}.jpg\"\n            dest_path = path_images_dir / dest_filename\n\n            if dest_path.is_file():\n                print(f\"{tomo_id}_z{z:04d}.jpg already exists\")\n            else:\n                # Save the normalized image\n                Image.fromarray(np_normalized).save(dest_path)\n            \n            # Get image dimensions\n            img_width, img_height = img.size\n        \n            # Create YOLO format label\n            # YOLO format: <class> <x_center> <y_center> <width> <height>\n            # Values are normalized to [0, 1]\n            x_center_norm = cx / img_width\n            y_center_norm = cy / img_height\n            box_width_norm = box_size / img_width\n            box_height_norm = box_size / img_height        \n        \n            # Write label file\n            label_path = path_labels_dir / dest_filename.replace('.jpg', '.txt')\n            with open(label_path, 'a') as f:\n                f.write(f\"0 {x_center_norm} {y_center_norm} {box_width_norm} {box_height_norm}\\n\")\n\n            processed_slices += 1\n\n    return processed_slices","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-28T11:38:29.734541Z","iopub.execute_input":"2025-05-28T11:38:29.735479Z","iopub.status.idle":"2025-05-28T11:38:29.748628Z","shell.execute_reply.started":"2025-05-28T11:38:29.735442Z","shell.execute_reply":"2025-05-28T11:38:29.747630Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 6. BYU data","metadata":{}},{"cell_type":"markdown","source":"## 6.1. split","metadata":{}},{"cell_type":"code","source":"one_motor_tomos = df_byu_final.loc[df_byu_final['Number of motors'] == 1, 'tomo_id'].unique()\nprint(f\"Found {len(one_motor_tomos)} unique tomograms with 1 motor\")\n\nmulti_motor_tomos = df_byu_final.loc[df_byu_final['Number of motors'] > 1, 'tomo_id'].unique()\nprint(f\"Found {len(multi_motor_tomos)} unique tomograms with multiple motors\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-28T11:38:32.104032Z","iopub.execute_input":"2025-05-28T11:38:32.104428Z","iopub.status.idle":"2025-05-28T11:38:32.112770Z","shell.execute_reply.started":"2025-05-28T11:38:32.104401Z","shell.execute_reply":"2025-05-28T11:38:32.111527Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"ser_temp = df_byu_final.groupby('tomo_id')['voxel_spacing_int'].mean()\nser_temp = ser_temp[ser_temp.index.isin(one_motor_tomos)]\nser_temp","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-28T11:38:33.595831Z","iopub.execute_input":"2025-05-28T11:38:33.596092Z","iopub.status.idle":"2025-05-28T11:38:33.605329Z","shell.execute_reply.started":"2025-05-28T11:38:33.596072Z","shell.execute_reply":"2025-05-28T11:38:33.604509Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"ser_temp.value_counts()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-28T11:38:35.859863Z","iopub.execute_input":"2025-05-28T11:38:35.860175Z","iopub.status.idle":"2025-05-28T11:38:35.867736Z","shell.execute_reply.started":"2025-05-28T11:38:35.860154Z","shell.execute_reply":"2025-05-28T11:38:35.866313Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"ser_train, ser_test = train_test_split(ser_temp, test_size=0.3, random_state=42, stratify=ser_temp)\nser_train.shape, ser_test.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-28T11:38:37.556387Z","iopub.execute_input":"2025-05-28T11:38:37.556703Z","iopub.status.idle":"2025-05-28T11:38:37.567059Z","shell.execute_reply.started":"2025-05-28T11:38:37.556677Z","shell.execute_reply":"2025-05-28T11:38:37.566065Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"ser_test.value_counts()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-28T11:38:38.935302Z","iopub.execute_input":"2025-05-28T11:38:38.935846Z","iopub.status.idle":"2025-05-28T11:38:38.942697Z","shell.execute_reply.started":"2025-05-28T11:38:38.935822Z","shell.execute_reply":"2025-05-28T11:38:38.941639Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"ser_val, ser_fbscore = train_test_split(ser_test, test_size=0.3, random_state=42, stratify=ser_test)\nser_val.shape, ser_fbscore.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-28T11:38:40.324640Z","iopub.execute_input":"2025-05-28T11:38:40.324920Z","iopub.status.idle":"2025-05-28T11:38:40.333710Z","shell.execute_reply.started":"2025-05-28T11:38:40.324897Z","shell.execute_reply":"2025-05-28T11:38:40.332665Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_tomos = ser_train.index.tolist() + multi_motor_tomos.tolist()\nval_tomos = ser_val.index.tolist()\ntest_tomos = ser_fbscore.index.tolist()\nlen(train_tomos), len(val_tomos), len(test_tomos)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-28T11:38:42.814898Z","iopub.execute_input":"2025-05-28T11:38:42.815681Z","iopub.status.idle":"2025-05-28T11:38:42.822182Z","shell.execute_reply.started":"2025-05-28T11:38:42.815641Z","shell.execute_reply":"2025-05-28T11:38:42.821393Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"num_tomos_byu_train = len(train_tomos)\nnum_tomos_byu_val = len(val_tomos)\nnum_tomos_byu_test = len(test_tomos) \nnum_tomos_byu_train, num_tomos_byu_val, num_tomos_byu_test","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-28T11:38:44.982916Z","iopub.execute_input":"2025-05-28T11:38:44.983221Z","iopub.status.idle":"2025-05-28T11:38:44.989407Z","shell.execute_reply.started":"2025-05-28T11:38:44.983198Z","shell.execute_reply":"2025-05-28T11:38:44.988306Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_byu_final['split'] = 'train'\ndf_byu_final.loc[df_byu_final['tomo_id'].isin(val_tomos), 'split'] = 'val'\ndf_byu_final.loc[df_byu_final['tomo_id'].isin(test_tomos), 'split'] = 'test'\ndf_byu_final.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-28T11:38:47.758180Z","iopub.execute_input":"2025-05-28T11:38:47.758913Z","iopub.status.idle":"2025-05-28T11:38:47.782320Z","shell.execute_reply.started":"2025-05-28T11:38:47.758883Z","shell.execute_reply":"2025-05-28T11:38:47.781225Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_byu_final['split'].value_counts()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-28T11:41:05.719113Z","iopub.execute_input":"2025-05-28T11:41:05.719768Z","iopub.status.idle":"2025-05-28T11:41:05.730411Z","shell.execute_reply.started":"2025-05-28T11:41:05.719736Z","shell.execute_reply":"2025-05-28T11:41:05.728882Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 6.2. Save images and labels","metadata":{}},{"cell_type":"code","source":"num_imgs_byu_train = process_tomogram_set(df_byu_final, train_tomos, path_images_train, path_labels_train, \"train(byu)\")\nnum_imgs_byu_train","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-27T13:33:45.153890Z","iopub.execute_input":"2025-05-27T13:33:45.154185Z","iopub.status.idle":"2025-05-27T13:33:48.638584Z","shell.execute_reply.started":"2025-05-27T13:33:45.154162Z","shell.execute_reply":"2025-05-27T13:33:48.637231Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"num_imgs_byu_val = process_tomogram_set(df_byu_final, val_tomos, path_images_val, path_labels_val, \"val(byu)\")\nnum_imgs_byu_val","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-19T00:18:33.911481Z","iopub.execute_input":"2025-05-19T00:18:33.911880Z","iopub.status.idle":"2025-05-19T00:19:05.593036Z","shell.execute_reply.started":"2025-05-19T00:18:33.911858Z","shell.execute_reply":"2025-05-19T00:19:05.591968Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 7. CryoET data","metadata":{}},{"cell_type":"markdown","source":"## 7.1. split","metadata":{}},{"cell_type":"code","source":"tomos = df_cryoet['tomo_id'].unique()\nprint(f\"Found {len(tomos)} unique tomograms\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-28T11:41:10.650924Z","iopub.execute_input":"2025-05-28T11:41:10.651228Z","iopub.status.idle":"2025-05-28T11:41:10.656990Z","shell.execute_reply.started":"2025-05-28T11:41:10.651206Z","shell.execute_reply":"2025-05-28T11:41:10.656048Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_cryoet.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-28T11:41:12.107069Z","iopub.execute_input":"2025-05-28T11:41:12.107391Z","iopub.status.idle":"2025-05-28T11:41:12.113434Z","shell.execute_reply.started":"2025-05-28T11:41:12.107367Z","shell.execute_reply":"2025-05-28T11:41:12.112500Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_cryoet['voxel_spacing_int'].value_counts()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-28T11:49:02.153780Z","iopub.execute_input":"2025-05-28T11:49:02.154104Z","iopub.status.idle":"2025-05-28T11:49:02.162019Z","shell.execute_reply.started":"2025-05-28T11:49:02.154078Z","shell.execute_reply":"2025-05-28T11:49:02.160984Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 짝수 버전용 (no CryoET filter)\ndf_cryoet.loc[df_cryoet['voxel_spacing_int'] == 9, 'voxel_spacing_int'] = 10  # 9 인게 너무 적어서 안 나뉨. 10이랑 합침\ndf_cryoet['voxel_spacing_int'].value_counts()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 짝수 버전용 (no CryoET filter)\ndf_train, df_test = train_test_split(df_cryoet, test_size=0.15, random_state=42, stratify=df_cryoet['voxel_spacing_int'])\ndf_train.shape, df_test.shape","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 홀수 버전용 (CryoET filter)\n# df_train, df_test = train_test_split(df_cryoet, test_size=0.20, random_state=42, stratify=df_cryoet['voxel_spacing_int'])\n# print(df_train.shape, df_test.shape)\n# df_cryoet['voxel_spacing_int'].value_counts()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-28T11:52:20.154448Z","iopub.execute_input":"2025-05-28T11:52:20.155503Z","iopub.status.idle":"2025-05-28T11:52:20.166309Z","shell.execute_reply.started":"2025-05-28T11:52:20.155456Z","shell.execute_reply":"2025-05-28T11:52:20.165293Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# df_fbscore.shape 를 30개 정도로 맞추고 싶음\ndf_val, df_fbscore = train_test_split(df_test, test_size=0.2, random_state=42, stratify=df_test['voxel_spacing_int'])\ndf_val.shape, df_fbscore.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-28T11:52:23.160656Z","iopub.execute_input":"2025-05-28T11:52:23.161065Z","iopub.status.idle":"2025-05-28T11:52:23.172765Z","shell.execute_reply.started":"2025-05-28T11:52:23.161029Z","shell.execute_reply":"2025-05-28T11:52:23.171896Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_tomos = df_train['tomo_id'].unique()\nval_tomos = df_val['tomo_id'].unique()\ntest_tomos = df_fbscore['tomo_id'].unique()\nnum_tomos_cryoet_train, num_tomos_cryoet_val, num_tomos_cryoet_test = len(train_tomos), len(val_tomos), len(test_tomos)\nnum_tomos_cryoet_train, num_tomos_cryoet_val, num_tomos_cryoet_test","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-28T11:52:26.335591Z","iopub.execute_input":"2025-05-28T11:52:26.335874Z","iopub.status.idle":"2025-05-28T11:52:26.343614Z","shell.execute_reply.started":"2025-05-28T11:52:26.335852Z","shell.execute_reply":"2025-05-28T11:52:26.342774Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_cryoet['split'] = 'train'\ndf_cryoet.loc[df_cryoet['tomo_id'].isin(val_tomos), 'split'] = 'val'\ndf_cryoet.loc[df_cryoet['tomo_id'].isin(test_tomos), 'split'] = 'test'\ndf_cryoet.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-28T11:52:28.421172Z","iopub.execute_input":"2025-05-28T11:52:28.421527Z","iopub.status.idle":"2025-05-28T11:52:28.437982Z","shell.execute_reply.started":"2025-05-28T11:52:28.421504Z","shell.execute_reply":"2025-05-28T11:52:28.437136Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_cryoet['split'].value_counts()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-28T11:52:32.095790Z","iopub.execute_input":"2025-05-28T11:52:32.096386Z","iopub.status.idle":"2025-05-28T11:52:32.104865Z","shell.execute_reply.started":"2025-05-28T11:52:32.096342Z","shell.execute_reply":"2025-05-28T11:52:32.103990Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 7.2. Save images and labels","metadata":{}},{"cell_type":"code","source":"num_imgs_cryoet_train = 0\nif USE_CRYOET:\n    num_imgs_cryoet_train = process_tomogram_set(df_cryoet, train_tomos, path_images_train, path_labels_train, \"train(cryoet)\")\nnum_imgs_cryoet_train","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-28T11:52:35.117561Z","iopub.execute_input":"2025-05-28T11:52:35.118580Z","iopub.status.idle":"2025-05-28T11:52:38.291195Z","shell.execute_reply.started":"2025-05-28T11:52:35.118546Z","shell.execute_reply":"2025-05-28T11:52:38.289842Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"num_imgs_cryoet_val = 0\nif USE_CRYOET:\n    num_imgs_cryoet_val = process_tomogram_set(df_cryoet, val_tomos, path_images_val, path_labels_val, \"val(cryoet)\")\nnum_imgs_cryoet_val","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-19T00:20:08.990016Z","iopub.execute_input":"2025-05-19T00:20:08.990396Z","iopub.status.idle":"2025-05-19T00:21:07.228855Z","shell.execute_reply.started":"2025-05-19T00:20:08.990373Z","shell.execute_reply":"2025-05-19T00:21:07.227789Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 8. yaml","metadata":{}},{"cell_type":"code","source":"# Create YAML configuration file for YOLO\nyaml_content = {\n    'path': '/kaggle/input/byu-cryoet-yolo-dataset',  # dataset name to be made\n    'train': 'images/train',\n    'val': 'images/val',\n    'names': {0: 'motor'}\n}\n\nwith open('dataset.yaml', 'w') as f:\n    yaml.dump(yaml_content, f, default_flow_style=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-27T13:38:16.610941Z","iopub.execute_input":"2025-05-27T13:38:16.611228Z","iopub.status.idle":"2025-05-27T13:38:16.617794Z","shell.execute_reply.started":"2025-05-27T13:38:16.611208Z","shell.execute_reply":"2025-05-27T13:38:16.616839Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 9. Background","metadata":{}},{"cell_type":"code","source":"df_raw = pd.read_csv('/kaggle/input/byu-inference-for-bg/df_raw.csv')\nprint(df_raw.shape)\ndf_raw.head()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_submission_df(eps=10, min_samples=3, score_th=4.0):\n    cz_list = []\n    cy_list = []\n    cx_list = []\n    bw_list = []\n    bh_list = []\n    conf_list = []\n    tomo_id_list = []\n    \n    for i, tomo_id in enumerate(df_raw['tomo_id'].unique()):\n        df_tomo = df_raw[df_raw['tomo_id'] == tomo_id].copy()\n        db = DBSCAN(eps=eps, min_samples=min_samples).fit(df_tomo[['Motor axis 0', 'Motor axis 1', 'Motor axis 2']])\n        df_tomo['cluster'] = db.labels_\n    \n        for cluster in df_tomo['cluster'].unique():\n            if cluster == -1:\n                continue\n            idx = df_tomo.loc[df_tomo['cluster'] == cluster, 'conf'].idxmax()\n            row = df_tomo.loc[idx]\n            # print(f\"Pick ({row['Motor axis 0']}, {row['Motor axis 1']}, {row['Motor axis 2']})\")\n            cz_list.append(row['Motor axis 0'])\n            cy_list.append(row['Motor axis 1'])\n            cx_list.append(row['Motor axis 2'])\n            bw_list.append(row['bw'])\n            bh_list.append(row['bh'])\n            conf_list.append(row['conf'])\n            tomo_id_list.append(tomo_id)\n\n    return pd.DataFrame({\n        'Motor axis 0': cz_list,\n        'Motor axis 1': cy_list,\n        'Motor axis 2': cx_list,\n        'bw': bw_list,\n        'bh': bh_list,\n        'tomo_id': tomo_id_list,\n        'conf': conf_list,\n    })","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_bg = get_submission_df()\nprint(df_bg.shape)\ndf_bg","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_bg.to_csv('df_bg', index=False)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"path_train = Path('/kaggle/input/byu-locating-bacterial-flagellar-motors-2025/train')","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"path_images_train","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"num_imgs_bg = 0\n\nfor idx, row in df_bg.iterrows():\n    tomo_id = row['tomo_id']\n    z = row['Motor axis 0']\n    path_img = path_train / tomo_id / f\"slice_{z:04d}.jpg\"\n\n    img = Image.open(path_img)    \n    np_img = np.array(img)\n    np_normalized = normalize_slice(np_img)\n\n    # Create destination filename (with unique identifier)\n    dest_filename = f\"_bg_{tomo_id}_z{z:04d}.jpg\"\n    dest_path = path_images_train / dest_filename\n    print(f'[{idx:03d}] {dest_filename}')\n\n    if dest_path.is_file():\n        print(f\"_bg_{tomo_id}_z{z:04d}.jpg already exists\")\n    else:\n        # Save the normalized image\n        Image.fromarray(np_normalized).save(dest_path)        \n        num_imgs_bg += 1\n\nnum_imgs_bg","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 10. Dataset Info","metadata":{}},{"cell_type":"code","source":"df_dataset = pd.concat([df_byu_final, df_cryoet])\ndf_dataset","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-27T13:38:19.463260Z","iopub.execute_input":"2025-05-27T13:38:19.463592Z","iopub.status.idle":"2025-05-27T13:38:19.487194Z","shell.execute_reply.started":"2025-05-27T13:38:19.463566Z","shell.execute_reply":"2025-05-27T13:38:19.486203Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_dataset.to_csv('df_dataset.csv', index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-20T15:59:34.452882Z","iopub.execute_input":"2025-05-20T15:59:34.453218Z","iopub.status.idle":"2025-05-20T15:59:34.484624Z","shell.execute_reply.started":"2025-05-20T15:59:34.453195Z","shell.execute_reply":"2025-05-20T15:59:34.483853Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"dataset_info = {\n    'desc': DESC,\n    'MOTOR_SIZE' : MOTOR_SIZE,\n    'TRUST' : TRUST,\n    'USE_CRYOET' : USE_CRYOET,\n    'num_imgs_train': num_imgs_byu_train + num_imgs_cryoet_train,\n    'num_imgs_val': num_imgs_byu_val + num_imgs_cryoet_val,\n    'num_imgs_bg': num_imgs_bg,\n    \"byu\": {\n        'num_imgs_byu_train': num_imgs_byu_train,\n        'num_imgs_byu_val': num_imgs_byu_val,\n        'num_tomos_byu_train': num_tomos_byu_train,\n        'num_tomos_byu_val': num_tomos_byu_val,\n        'num_tomos_byu_test': num_tomos_byu_test,\n    },\n    'cryoet': {\n        'num_imgs_cryoet_train': num_imgs_cryoet_train,\n        'num_imgs_cryoet_val': num_imgs_cryoet_val,\n        'num_tomos_cryoet_train': num_tomos_cryoet_train,\n        'num_tomos_cryoet_val': num_tomos_cryoet_val,\n        'num_tomos_cryoet_test': num_tomos_cryoet_test,\n    },\n}\n\nwith open('dataset_info.json', 'w', encoding='utf-8') as f:\n    json.dump(dataset_info, f, ensure_ascii=False, indent=4)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-15T15:22:46.037429Z","iopub.execute_input":"2025-05-15T15:22:46.037973Z","iopub.status.idle":"2025-05-15T15:22:46.056234Z","shell.execute_reply.started":"2025-05-15T15:22:46.037838Z","shell.execute_reply":"2025-05-15T15:22:46.054342Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"dataset_info","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 11. Check output","metadata":{}},{"cell_type":"code","source":"!apt install tree","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-15T15:23:03.744109Z","iopub.execute_input":"2025-05-15T15:23:03.744447Z","iopub.status.idle":"2025-05-15T15:23:12.488969Z","shell.execute_reply.started":"2025-05-15T15:23:03.744422Z","shell.execute_reply":"2025-05-15T15:23:12.487636Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!tree --filelimit 10","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-15T15:23:12.491078Z","iopub.execute_input":"2025-05-15T15:23:12.491421Z","iopub.status.idle":"2025-05-15T15:23:12.621966Z","shell.execute_reply.started":"2025-05-15T15:23:12.491392Z","shell.execute_reply":"2025-05-15T15:23:12.620439Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!cat dataset.yaml","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-14T15:02:09.185599Z","iopub.execute_input":"2025-05-14T15:02:09.185973Z","iopub.status.idle":"2025-05-14T15:02:09.31461Z","shell.execute_reply.started":"2025-05-14T15:02:09.185917Z","shell.execute_reply":"2025-05-14T15:02:09.313333Z"}},"outputs":[],"execution_count":null}]}