{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Data Wrangling","metadata":{}},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport random","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-10-29T19:38:55.633564Z","iopub.execute_input":"2022-10-29T19:38:55.634032Z","iopub.status.idle":"2022-10-29T19:38:55.640130Z","shell.execute_reply.started":"2022-10-29T19:38:55.633984Z","shell.execute_reply":"2022-10-29T19:38:55.638711Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data1 = pd.read_csv(\"../input/sartorius-cell-instance-segmentation/train.csv\")\ndata1_images = \"../input/sartorius-cell-instance-segmentation/train\"","metadata":{"execution":{"iopub.status.busy":"2022-10-29T19:38:55.659291Z","iopub.execute_input":"2022-10-29T19:38:55.659679Z","iopub.status.idle":"2022-10-29T19:38:56.007508Z","shell.execute_reply.started":"2022-10-29T19:38:55.659648Z","shell.execute_reply":"2022-10-29T19:38:56.006166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data1.head()","metadata":{"execution":{"iopub.status.busy":"2022-10-29T19:38:56.009819Z","iopub.execute_input":"2022-10-29T19:38:56.010247Z","iopub.status.idle":"2022-10-29T19:38:56.026319Z","shell.execute_reply.started":"2022-10-29T19:38:56.010209Z","shell.execute_reply":"2022-10-29T19:38:56.024989Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def transform1(d, path):\n    this = d.groupby(\"id\")[\"cell_type\"].unique()\n    arr = np.empty(shape=(len(this),2),dtype=\"object\")\n    for i in range(len(this)):\n        arr[i,1] = this[i][0]\n        arr[i,0] = path + \"/\" + this.index[i] + \".png\"\n    return pd.DataFrame(arr, columns=(\"path\",\"class\"))","metadata":{"execution":{"iopub.status.busy":"2022-10-29T19:38:56.028380Z","iopub.execute_input":"2022-10-29T19:38:56.029156Z","iopub.status.idle":"2022-10-29T19:38:56.038913Z","shell.execute_reply.started":"2022-10-29T19:38:56.028919Z","shell.execute_reply":"2022-10-29T19:38:56.037421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data1 = transform1(data1, data1_images)\ndata1","metadata":{"execution":{"iopub.status.busy":"2022-10-29T19:38:56.043012Z","iopub.execute_input":"2022-10-29T19:38:56.044166Z","iopub.status.idle":"2022-10-29T19:38:56.117109Z","shell.execute_reply.started":"2022-10-29T19:38:56.044111Z","shell.execute_reply":"2022-10-29T19:38:56.115835Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"semi_dir = \"../input/sartorius-cell-instance-segmentation/train_semi_supervised\"","metadata":{"execution":{"iopub.status.busy":"2022-10-29T19:38:56.120956Z","iopub.execute_input":"2022-10-29T19:38:56.121336Z","iopub.status.idle":"2022-10-29T19:38:56.129101Z","shell.execute_reply.started":"2022-10-29T19:38:56.121303Z","shell.execute_reply":"2022-10-29T19:38:56.127732Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def transform2(path):\n    liss = os.listdir(path)\n    arr = np.empty(shape=(len(liss),2),dtype=\"object\")\n    for i in range(len(liss)):\n        arr[i,0] = os.path.join(path,liss[i])\n        v = liss[i].split(\"[\")[0]\n        if v==\"astros\":\n            v=\"astro\"\n        arr[i,1] = v\n    return pd.DataFrame(arr, columns=(\"path\",\"class\"))","metadata":{"execution":{"iopub.status.busy":"2022-10-29T19:38:56.130564Z","iopub.execute_input":"2022-10-29T19:38:56.130922Z","iopub.status.idle":"2022-10-29T19:38:56.142513Z","shell.execute_reply.started":"2022-10-29T19:38:56.130889Z","shell.execute_reply":"2022-10-29T19:38:56.141256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data2 = transform2(semi_dir)\ndata2","metadata":{"execution":{"iopub.status.busy":"2022-10-29T19:38:56.143812Z","iopub.execute_input":"2022-10-29T19:38:56.144226Z","iopub.status.idle":"2022-10-29T19:38:56.174187Z","shell.execute_reply.started":"2022-10-29T19:38:56.144122Z","shell.execute_reply":"2022-10-29T19:38:56.172650Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = pd.concat([data1,data2]).reset_index(drop=True)\ndata","metadata":{"execution":{"iopub.status.busy":"2022-10-29T19:38:56.175889Z","iopub.execute_input":"2022-10-29T19:38:56.176569Z","iopub.status.idle":"2022-10-29T19:38:56.194144Z","shell.execute_reply.started":"2022-10-29T19:38:56.176524Z","shell.execute_reply":"2022-10-29T19:38:56.192816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Exploratory Analysis","metadata":{}},{"cell_type":"markdown","source":"### Distribution of the Target Variable","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(20,8))\nsns.countplot(x='class', data=data, palette=\"cool_r\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-10-29T19:38:56.195412Z","iopub.execute_input":"2022-10-29T19:38:56.195898Z","iopub.status.idle":"2022-10-29T19:38:56.412328Z","shell.execute_reply.started":"2022-10-29T19:38:56.195861Z","shell.execute_reply":"2022-10-29T19:38:56.410906Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### The dataset is relatively imbalanced","metadata":{}},{"cell_type":"markdown","source":"### Images & Histograms","metadata":{}},{"cell_type":"code","source":"def plot_samples(data=data, k=10):\n    n = data.shape[0]\n    for i in range(k):\n        ind = random.randint(0,n-1)\n        row = data.iloc[ind,:]\n        img = plt.imread(row[\"path\"])\n        clss = row[\"class\"]\n        img_hist, img_bins = np.histogram(img, bins=256, range=(0,1))\n        fig, axes = plt.subplots(nrows=1, ncols=2, figsize=(20,10))\n        plt.suptitle(f\"Class: {clss}\", fontsize=25)\n        axes[0].imshow(img, cmap='gray')\n        axes[1].plot(img_bins[0:-1], img_hist)\n        axes[0].set_title(\"Image\")\n        axes[1].set_title(\"Histogram\")\n        plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-10-29T19:40:04.658857Z","iopub.execute_input":"2022-10-29T19:40:04.659388Z","iopub.status.idle":"2022-10-29T19:40:04.669791Z","shell.execute_reply.started":"2022-10-29T19:40:04.659349Z","shell.execute_reply":"2022-10-29T19:40:04.668204Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_samples()","metadata":{"execution":{"iopub.status.busy":"2022-10-29T19:40:05.123915Z","iopub.execute_input":"2022-10-29T19:40:05.124330Z","iopub.status.idle":"2022-10-29T19:40:10.183766Z","shell.execute_reply.started":"2022-10-29T19:40:05.124297Z","shell.execute_reply":"2022-10-29T19:40:10.182123Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}