{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.14"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":84969,"databundleVersionId":10033515,"sourceType":"competition"},{"sourceId":10696465,"sourceType":"datasetVersion","datasetId":6628234},{"sourceId":221517018,"sourceType":"kernelVersion"}],"dockerImageVersionId":30787,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true},"papermill":{"default_parameters":{},"duration":759.810296,"end_time":"2025-02-06T12:17:16.035815","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2025-02-06T12:04:36.225519","version":"2.6.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This notebook can be used to obtain our 11th place solution to the CZII - CryoET competition. It does not match the solution exactly; we simplified the code a bit which leads to small differences. It is set up to be able to be submitted as-is. Training code is also included, but a pretrained model is used by default.\n\nFor a writeup of the solution, see https://www.kaggle.com/competitions/czii-cryo-et-object-identification/discussion/561837","metadata":{"editable":true,"papermill":{"duration":13.215024,"end_time":"2025-02-06T12:04:51.947003","exception":false,"start_time":"2025-02-06T12:04:38.731979","status":"completed"},"slideshow":{"slide_type":""},"tags":[]}},{"cell_type":"code","source":"import os\ndeps_path = '/kaggle/usr/lib/cz_packages/'\n!pip install --no-index --find-links {deps_path} --requirement {deps_path}/requirements.txt\nimport sys\nsys.path.append('/kaggle/input/my-cet-library/')\nimport cz_support as czs\nimport numpy as np\nimport time\nimport copy\nimport sys\nimport matplotlib.pyplot as plt\nimport cz_classification","metadata":{"editable":true,"papermill":{"duration":13.215024,"end_time":"2025-02-06T12:04:51.947003","exception":false,"start_time":"2025-02-06T12:04:38.731979","status":"completed"},"slideshow":{"slide_type":""},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-02-08T16:49:09.279456Z","iopub.execute_input":"2025-02-08T16:49:09.279798Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Load data\nloaded_data_synthetic, loaded_data_train, loaded_data_test = czs.load_all_data()\n\n# Load pretrained model, or train it here\nuse_pretrained_model = True\nif use_pretrained_model:\n    model =czs.dill_load(czs.model_loc()+'pretrained_model.pickle')\nelse:\n    # Training takes about 12 hours on an RTX4090. It will not run in the Kaggle environment (VRAM too limited).\n    model = cz_classification.ClassificationModel()\n    model.train_real(loaded_data_train)\n\n# Model tweaks based on leaderboard fine tuning\nmodel.classifiers[0].trained_threshold[0] = 1. * model.classifiers[0].trained_threshold[0]\nmodel.classifiers[2].trained_threshold[0] = 1 * model.classifiers[2].trained_threshold[0]\nmodel.classifiers[3].trained_threshold[0] = 1.1 * model.classifiers[3].trained_threshold[0]\nmodel.classifiers[4].trained_threshold[0] = 0.95 * model.classifiers[4].trained_threshold[0]\nmodel.classifiers[5].trained_threshold[0] = 1. * model.classifiers[5].trained_threshold[0]\nmodel.unet_model.heat_map_to_locations[3].radius_kmeans = 12\nmodel.unet_model.heat_map_to_locations[3].xy_scaling_kmeans = 0.8\nmodel.unet_model.heat_map_to_locations[3].cubicity_threshold = 0.2\nmodel.unet_model.heat_map_to_locations[3].threshold = 0.1\nmodel.unet_model.heat_map_to_locations[3].radius_dilate = 3\n\n# Run the actual inference\nmodel.run_in_parallel = True\npredicted_data = model.infer(loaded_data_test)\n\n# Postprocessing tweaks based on leaderboard fine tuning\nfor d in predicted_data:\n    # Add a small offset to all Z predictions\n    for lab in d.labels:\n        lab.zyx[:,0] = lab.zyx[:,0]+1\n    # Additional decision boundary for 2D neural network for beta-galactosidase, and remove really big clusters\n    lab = d.labels[2]\n    lab.select_indices(lab.notes['p_2d_neural']>-0.000114848*lab.notes['original_cluster_size']+0.50529)\n    lab.select_indices(lab.notes['original_cluster_size']<3500)\n\n# Report in-sample results if not in sumbission\nif not os.getenv('KAGGLE_IS_COMPETITION_RERUN'):\n    print(czs.evaluate_predictions(copy.deepcopy(predicted_data), loaded_data_train[:3]))\n\n# Write submission file\nczs.write_submission_file(predicted_data)","metadata":{"editable":true,"papermill":{"duration":0.014388,"end_time":"2025-02-06T12:05:29.750369","exception":false,"start_time":"2025-02-06T12:05:29.735981","status":"completed"},"slideshow":{"slide_type":""},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}