{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84969,"databundleVersionId":10033515,"sourceType":"competition"},{"sourceId":9890675,"sourceType":"datasetVersion","datasetId":6074268},{"sourceId":9890681,"sourceType":"datasetVersion","datasetId":6040928}],"dockerImageVersionId":30787,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"#!pip download connected-components-3d\n#!pip download zarr\n#!pip download numcodecs\n\ntry:\n    import zarr\nexcept: \n    !cp -r '/kaggle/input/hengck-czii-cryo-et-02/wheel_file' '/kaggle/working/'\n    !pip install /kaggle/working/wheel_file/asciitree-0.3.3/asciitree-0.3.3\n    !pip install --no-index --find-links=/kaggle/working/wheel_file zarr\n    !pip install --no-index --find-links=/kaggle/working/wheel_file connected-components-3d\n\nprint('PIP INSTALL OK!!!')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-13T09:56:53.554337Z","iopub.execute_input":"2024-11-13T09:56:53.555397Z","iopub.status.idle":"2024-11-13T09:56:53.566219Z","shell.execute_reply.started":"2024-11-13T09:56:53.555347Z","shell.execute_reply":"2024-11-13T09:56:53.564716Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from datetime import datetime\n\nimport pandas as pd\nimport pytz\nprint('LOGGING TIME OF START:',  datetime.strftime(datetime.now(pytz.timezone('Asia/Singapore')), \"%Y-%m-%d %H:%M:%S\"))\n\nimport sys\nsys.path.append('/kaggle/input/hengck-czii-cryo-et-02')\n\nfrom czii_helper import *\nfrom dataset import *\nfrom model import *\nimport numpy as np\nfrom scipy.optimize import linear_sum_assignment\nimport glob\n#import cc3d\nimport cv2\n\nimport matplotlib.pyplot as plt\nfrom mpl_toolkits.mplot3d import Axes3D\n\nprint('IMPORT OK!!!')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2024-11-13T09:56:53.568732Z","iopub.execute_input":"2024-11-13T09:56:53.569248Z","iopub.status.idle":"2024-11-13T09:56:53.586582Z","shell.execute_reply.started":"2024-11-13T09:56:53.569195Z","shell.execute_reply":"2024-11-13T09:56:53.585164Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"DATA_KAGGLE_DIR = '/kaggle/input/czii-cryo-et-object-identification'\n\nMODE='local'\n\n \nif MODE == 'local':\n    valid_dir = f'{DATA_KAGGLE_DIR}/train'\n    #valid_id = ['TS_5_4', 'TS_99_9', ]\n    valid_id = glob.glob(f'{valid_dir}/static/ExperimentRuns/*')\n    valid_id = [f.split('/')[-1] for f in valid_id]\n\nif MODE == 'submit':\n    valid_dir = f'{DATA_KAGGLE_DIR}/test'\n    valid_id = glob.glob(f'{valid_dir}/static/ExperimentRuns/*')\n    valid_id = [f.split('/')[-1] for f in valid_id]\n\nprint('valid_id:', len(valid_id), valid_id)\n\nprint('MODE:', MODE)\nprint('SETTING OK!!!')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-13T09:56:53.588491Z","iopub.execute_input":"2024-11-13T09:56:53.588916Z","iopub.status.idle":"2024-11-13T09:56:53.665969Z","shell.execute_reply.started":"2024-11-13T09:56:53.588877Z","shell.execute_reply":"2024-11-13T09:56:53.664560Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":" ","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\nsubmit_df=[]\nfor id in valid_id:\n    for n in PARTICLE_NAME:\n        print(id,n)\n        D,H,W = 184, 630, 630\n        xyz=np.random.uniform(0,1,(100,3))\n        xyz = xyz*[[W,H,0.5*D]]+[[0,0,0.25*D]]\n        xyz = xyz*10\n        submit_df.append(\n            pd.DataFrame({'experiment': id, 'particle_type': n, 'x': xyz[:, 0], 'y': xyz[:, 1], 'z': xyz[:, 2]})\n        )\nsubmit_df = pd.concat(submit_df)\nsubmit_df.insert(loc=0, column='id', value=np.arange(len(submit_df)))\n \n    \nprint('submit_df', submit_df.shape)\nprint(submit_df)\nsubmit_df.to_csv('submission.csv', index=False)\n\nprint('MODE:', MODE)\nprint('SUBMIT OK!!!')\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-13T09:56:53.667353Z","iopub.execute_input":"2024-11-13T09:56:53.667696Z","iopub.status.idle":"2024-11-13T09:56:53.734503Z","shell.execute_reply.started":"2024-11-13T09:56:53.667662Z","shell.execute_reply":"2024-11-13T09:56:53.733163Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if 1:\n    if MODE=='local':\n        pd.set_option('display.max_columns', 500)\n        submit_df=pd.read_csv(\n           'submission.csv'\n            # '/kaggle/input/hengck-czii-cryo-et-weights-01/submission.csv'\n        )\n        gb, lb_score = compute_lb(submit_df, f'{valid_dir}/overlay')\n        print('lb_score:',lb_score)\n        print(gb)\n        print('')\n\n\n        #--------------------------------------------\n        #visualisation\n\n        fig = plt.figure(figsize=(18, 8))\n\n        id = valid_id[1]\n        truth = read_one_truth(id,overlay_dir=f'{valid_dir}/overlay')\n\n        submit_df=pd.read_csv('submission.csv')\n        id_df = submit_df[submit_df['experiment']==id]\n        for p in PARTICLE:\n            p = dotdict(p)\n            xyz_truth = truth[p.name]\n            xyz_predict = id_df[id_df['particle_type']==p.name][['x','y','z']].values\n            hit, fp, miss, metric = do_one_eval(xyz_truth, xyz_predict, p.radius)\n            print(id, p.name)\n            print('\\t num truth   :',len(xyz_truth) )\n            print('\\t num predict :',len(xyz_predict) )\n            print('\\t num hit  :',len(hit[0]) )\n            print('\\t num fp   :',len(fp) )\n            print('\\t num miss :',len(miss) )\n            ax = fig.add_subplot(2, 3, p.label, projection='3d')\n\n            if 0:\n                pt = xyz_predict\n                ax.scatter(pt[:, 0], pt[:, 1], pt[:, 2], alpha=0.25, color='b', label='predict')\n                pt = xyz_truth\n                ax.scatter(pt[:, 0], pt[:, 1], pt[:, 2], s=160, alpha=0.25, color='r', label='truth')\n                ax.scatter(pt[:, 0], pt[:, 1], pt[:, 2], s=160, facecolors='none', edgecolors='r')\n            if 1:\n                if hit[0]:\n                    pt = xyz_predict[hit[0]]\n                    ax.scatter(pt[:, 0], pt[:, 1], pt[:, 2], alpha=0.5, color='r', label='predict')\n                    pt = xyz_truth[hit[1]]\n                    ax.scatter(pt[:, 0], pt[:, 1], pt[:, 2], s=80, facecolors='none', edgecolors='r', label='truth')\n                if fp:\n                    pt = xyz_predict[fp]\n                    ax.scatter(pt[:, 0], pt[:, 1], pt[:, 2], alpha=0.5, color='k', label='fp')\n                if miss:\n                    pt = xyz_truth[miss]\n                    ax.scatter(pt[:, 0], pt[:, 1], pt[:, 2], s=160, alpha=0.5, facecolors='none', edgecolors='k', label='miss')\n            ax.legend()\n            ax.set_title(\n                f'{id}:{p.name} ({p.difficulty})\\npredict={metric[0]}, truth={metric[1]}, hit={metric[2]}, miss={metric[3]}, fp={metric[4]}')\n\n        plt.tight_layout()\n        plt.show()\n        zz=0\n\n'''\n\n xyz=np.random.uniform(0,1,(800,3))\nlb_score: 0.007004517259076899\n         particle_type     P    T  hit  miss    fp  precision    recall  \\\n0         apo-ferritin  5600  375    1   374  5599   0.000179  0.002667   \n1         beta-amylase  5600   87    0    87  5600   0.000000  0.000000   \n2   beta-galactosidase  5600  112    1   111  5599   0.000179  0.008929   \n3             ribosome  5600  331   11   320  5589   0.001964  0.033233   \n4        thyroglobulin  5600  251    6   245  5594   0.001071  0.023904   \n5  virus-like-particle  5600  113    2   111  5598   0.000357  0.017699   \n\n\n\n xyz=np.random.uniform(0,1,(500,3))\n['TS_86_3', 'TS_6_6', 'TS_6_4', 'TS_5_4', 'TS_73_6', 'TS_99_9', 'TS_69_2']\nlb_score: 0.006996786960526867\n         particle_type     P    T  hit  miss    fp  precision    recall  \\\n0         apo-ferritin  3500  375    0   375  3500   0.000000  0.000000   \n1         beta-amylase  3500   87    0    87  3500   0.000000  0.000000   \n2   beta-galactosidase  3500  112    1   111  3499   0.000286  0.008929   \n3             ribosome  3500  331    7   324  3493   0.002000  0.021148   \n4        thyroglobulin  3500  251    5   246  3495   0.001429  0.019920   \n5  virus-like-particle  3500  113    2   111  3498   0.000571  0.017699  \n\n\n\n\n xyz=np.random.uniform(0,1,(200,3))\nlb_score: 0.0042559912546522236\n         particle_type     P    T  hit  miss    fp  precision    recall  \\\n0         apo-ferritin  1400  375    0   375  1400   0.000000  0.000000   \n1         beta-amylase  1400   87    0    87  1400   0.000000  0.000000   \n2   beta-galactosidase  1400  112    0   112  1400   0.000000  0.000000   \n3             ribosome  1400  331    3   328  1397   0.002143  0.009063   \n4        thyroglobulin  1400  251    1   250  1399   0.000714  0.003984   \n5  virus-like-particle  1400  113    3   110  1397   0.002143  0.026549   \n\n\n xyz=np.random.uniform(0,1,(100,3))\nlb_score: 0.0017358351892922466\n         particle_type    P    T  hit  miss   fp  precision    recall  \\\n0         apo-ferritin  700  375    1   374  699   0.001429  0.002667   \n1         beta-amylase  700   87    0    87  700   0.000000  0.000000   \n2   beta-galactosidase  700  112    0   112  700   0.000000  0.000000   \n3             ribosome  700  331    1   330  699   0.001429  0.003021   \n4        thyroglobulin  700  251    0   251  700   0.000000  0.000000   \n5  virus-like-particle  700  113    1   112  699   0.001429  0.008850   \n'''","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-13T09:56:53.736516Z","iopub.execute_input":"2024-11-13T09:56:53.736877Z","iopub.status.idle":"2024-11-13T09:56:55.700563Z","shell.execute_reply.started":"2024-11-13T09:56:53.736839Z","shell.execute_reply":"2024-11-13T09:56:55.699364Z"}},"outputs":[],"execution_count":null}]}