{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84969,"databundleVersionId":10033515,"sourceType":"competition"}],"dockerImageVersionId":30787,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# CZII Prepare Image Data for UNET","metadata":{}},{"cell_type":"markdown","source":"https://www.kaggle.com/code/stpeteishii/czii-zarr-image-view\n\nhttps://www.kaggle.com/code/stpeteishii/czii-data-3d-view\n\nhttps://www.kaggle.com/code/stpeteishii/czii-static-image-and-overlay-image","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport numpy as np\nimport os\nimport cv2","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-10T03:54:50.290765Z","iopub.execute_input":"2024-11-10T03:54:50.291615Z","iopub.status.idle":"2024-11-10T03:54:50.295574Z","shell.execute_reply.started":"2024-11-10T03:54:50.291577Z","shell.execute_reply":"2024-11-10T03:54:50.294683Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!rm -rf mask\n!rm -rf image\n!rm -rf timage\n\n!mkdir image\n!mkdir mask\n!mkdir timage","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-10T03:59:26.192129Z","iopub.execute_input":"2024-11-10T03:59:26.192968Z","iopub.status.idle":"2024-11-10T03:59:28.187171Z","shell.execute_reply.started":"2024-11-10T03:59:26.192923Z","shell.execute_reply":"2024-11-10T03:59:28.18605Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"dir0=\"/kaggle/input/czii-cryo-et-object-identification/train/overlay/ExperimentRuns/\"\nnames=os.listdir(dir0)\nTSpaths=[]\nfor name in names:\n    TSpaths+=[os.path.join(dir0,name)]\nprint(TSpaths)\n\nPATHS=[]\nfor TSpath in TSpaths:\n    paths=[]\n    for dirname, _, filenames in os.walk(TSpath):\n        for filename in filenames:\n            paths+=[(os.path.join(dirname, filename))]\n    PATHS+=[paths]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-10T03:54:52.301263Z","iopub.execute_input":"2024-11-10T03:54:52.303109Z","iopub.status.idle":"2024-11-10T03:54:52.310486Z","shell.execute_reply.started":"2024-11-10T03:54:52.30307Z","shell.execute_reply":"2024-11-10T03:54:52.309491Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import json\n\ndef load_from_json(file_path):\n    with open(file_path, 'r', encoding='utf-8') as f:\n        data = json.load(f)\n    return data\n\nfrom collections import defaultdict\ncnt = defaultdict(int)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-10T03:54:52.329351Z","iopub.execute_input":"2024-11-10T03:54:52.329647Z","iopub.status.idle":"2024-11-10T03:54:52.334419Z","shell.execute_reply.started":"2024-11-10T03:54:52.329616Z","shell.execute_reply":"2024-11-10T03:54:52.333519Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"LL=[]\nfor paths in PATHS:\n    L=[]\n    for i,path in enumerate(paths):\n        data=load_from_json(path)\n        cnt[i]=data['pickable_object_name']\n        X=[]\n        Y=[]\n        Z=[]\n        Z2=[]\n        for datai in data['points']:\n            xyz=datai['location']\n            X+=[xyz['x']]\n            Y+=[xyz['y']]\n            Z+=[xyz['z']]\n            Z2+=[xyz['z']//50]\n        L+=[(i,X,Y,Z,Z2)]\n    LL+=[L]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-10T03:54:52.346086Z","iopub.execute_input":"2024-11-10T03:54:52.346365Z","iopub.status.idle":"2024-11-10T03:54:52.431689Z","shell.execute_reply.started":"2024-11-10T03:54:52.346334Z","shell.execute_reply":"2024-11-10T03:54:52.430753Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"---","metadata":{}},{"cell_type":"markdown","source":"# create mask images","metadata":{}},{"cell_type":"markdown","source":"---","metadata":{}},{"cell_type":"code","source":"def z_filtered(L,k,j):\n    \n    fig = plt.figure(figsize=(8,8))\n    ax = fig.add_subplot(111)\n    \n    for data in L:\n        i, X, Y, Z, Z2 = data \n        filtered_indices = [idx for idx, z in enumerate(Z2) if z==j]\n        \n        # Apply the filter to X, Y, and Z2\n        filtered_X = [X[idx] for idx in filtered_indices]\n        filtered_Y = [Y[idx] for idx in filtered_indices]\n        filtered_Z2 = [Z2[idx] for idx in filtered_indices]\n        \n        # Plot the filtered data\n        scatter = ax.scatter(filtered_X, filtered_Y, c=[i] * len(filtered_X),\n                             vmin=0, vmax=5, cmap='viridis')\n    \n    ax.set_xlabel('X Label')\n    ax.set_ylabel('Y Label')\n    #cbar = fig.colorbar(scatter, ax=ax)\n    #cbar.set_label('Label')\n    plt.axis('off')\n    plt.savefig(f'mask/{k}_{j:02}.png', bbox_inches='tight', pad_inches=0)\n    plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-10T03:54:52.433193Z","iopub.execute_input":"2024-11-10T03:54:52.433523Z","iopub.status.idle":"2024-11-10T03:54:52.442553Z","shell.execute_reply.started":"2024-11-10T03:54:52.433488Z","shell.execute_reply":"2024-11-10T03:54:52.441613Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for k,L in enumerate(LL):\n    for j in range(35):\n        print(k,j)\n        z_filtered(L,k,j)\n        print()\n        print('-----------------------------'*2)\n        print()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-10T03:54:52.44388Z","iopub.execute_input":"2024-11-10T03:54:52.444238Z","iopub.status.idle":"2024-11-10T03:54:59.882556Z","shell.execute_reply.started":"2024-11-10T03:54:52.444204Z","shell.execute_reply":"2024-11-10T03:54:59.881607Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"---","metadata":{}},{"cell_type":"markdown","source":"# create train images","metadata":{}},{"cell_type":"markdown","source":"---","metadata":{}},{"cell_type":"code","source":"!pip install zarr","metadata":{"trusted":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-11-10T03:54:59.883777Z","iopub.execute_input":"2024-11-10T03:54:59.884125Z","iopub.status.idle":"2024-11-10T03:55:11.329933Z","shell.execute_reply.started":"2024-11-10T03:54:59.884073Z","shell.execute_reply":"2024-11-10T03:55:11.328987Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport numpy as np\nimport os\nimport zarr\nfrom PIL import Image","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-10T03:55:11.331697Z","iopub.execute_input":"2024-11-10T03:55:11.332141Z","iopub.status.idle":"2024-11-10T03:55:11.338296Z","shell.execute_reply.started":"2024-11-10T03:55:11.332093Z","shell.execute_reply":"2024-11-10T03:55:11.337275Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"dir1='/kaggle/input/czii-cryo-et-object-identification/train/static/ExperimentRuns'\n\nnames=os.listdir(dir1)\nSpaths=[]\nfor name in names:\n    Spaths+=[os.path.join(dir1,name)]\nprint(Spaths)\n\nZARRS=[]\nfor sp in Spaths:\n    files=os.listdir(os.path.join(sp,'VoxelSpacing10.000'))\n    ZARR=[]\n    for file in files:\n        ZARR+=[os.path.join(sp,'VoxelSpacing10.000',file)]\n    ZARRS+=[ZARR]\n\nfor k,ZARR in enumerate(ZARRS):\n    for j,diri in enumerate(ZARR[0:1]):\n        print(ZARR[j].split('/')[-1]) \n        data = zarr.open(diri, mode='r') \n        fig = plt.figure(figsize=(12,18))\n    \n        for i in range(35):\n            ax = plt.subplot(7, 5, i + 1)\n            plt.axis('off')\n            image=data[2][i]\n            min_val, max_val = image.min(), image.max()\n            simage = ((image - min_val) / (max_val - min_val) * 255).astype('uint8')\n            plt.imshow(simage)\n            cv2.imwrite(f'image/{k}_{i:02}.png', simage)\n        plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-10T03:55:11.339428Z","iopub.execute_input":"2024-11-10T03:55:11.339708Z","iopub.status.idle":"2024-11-10T03:55:11.348742Z","shell.execute_reply.started":"2024-11-10T03:55:11.339676Z","shell.execute_reply":"2024-11-10T03:55:11.347721Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"---","metadata":{}},{"cell_type":"markdown","source":"# create test images","metadata":{}},{"cell_type":"markdown","source":"---","metadata":{}},{"cell_type":"code","source":"dir2='/kaggle/input/czii-cryo-et-object-identification/test/static/ExperimentRuns'\n\nnames=os.listdir(dir2)\ntSpaths=[]\nfor name in names:\n    tSpaths+=[os.path.join(dir2,name)]\nprint(tSpaths)\n\ntZARRS=[]\nfor sp in tSpaths:\n    files=os.listdir(os.path.join(sp,'VoxelSpacing10.000'))\n    tZARR=[]\n    for file in files:\n        tZARR+=[os.path.join(sp,'VoxelSpacing10.000',file)]\n    tZARRS+=[tZARR]\n\nfor k,tZARR in enumerate(tZARRS):\n    for j,diri in enumerate(tZARR[0:1]):\n        print(tZARR[j].split('/')[-1]) \n        data = zarr.open(diri, mode='r') \n        fig = plt.figure(figsize=(12,18))\n    \n        for i in range(35):\n            ax = plt.subplot(7, 5, i + 1)\n            plt.axis('off')\n            image=data[2][i]\n            min_val, max_val = image.min(), image.max()\n            simage = ((image - min_val) / (max_val - min_val) * 255).astype('uint8')\n            plt.imshow(simage)\n            cv2.imwrite(f'timage/{k}_{i:02}.png', simage)\n        plt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}