{ "cells": [ { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "db15e186-8bfd-373a-3919-26471e072173" }, "outputs": [], "source": [ "# This Python 3 environment comes with many helpful analytics libraries installed\n", "# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n", "# For example, here's several helpful packages to load in \n", "import dicom\n", "import os\n", "import numpy as np # linear algebra\n", "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n", "import matplotlib.pyplot as plt\n", "\n", "# Input data files are available in the \"../input/\" directory.\n", "# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n", "\n", "#from subprocess import check_output\n", "#print(check_output([\"ls\", \"../input\"]).decode(\"utf8\"))\n", "\n", "# Any results you write to the current directory are saved as output.\n", "data_dir = '../input/sample_images/'\n", "patients = os.listdir(data_dir)\n", "labels_df = pd.read_csv('../input/stage1_labels.csv',index_col=0)\n", "labels_df.head()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "d97a6ba6-30ff-dffc-687e-3989c7bc0966" }, "outputs": [], "source": [ "import cv2\n", "import math\n", "\n", "def chunks(l, n):\n", " # Credit: Ned Batchelder\n", " # Link: http://stackoverflow.com/questions/312443/how-do-you-split-a-list-into-evenly-sized-chunks\n", " \"\"\"Yield successive n-sized chunks from l.\"\"\"\n", " for i in range(0, len(l), n):\n", " yield l[i:i + n]\n", "\n", "def mean(l):\n", " return sum(l) / len(l)\n", "\n", "IMG_PX_SIZE=150\n", "HM_SLICES = 20\n", "\n", "def process_data(patient,labels_df,img_px_size=50, hm_slices=20, visualize=False):\n", " label = labels_df.get_value(patient,'cancer')\n", " path = data_dir + patient\n", " slices = [dicom.read_file(path +'/'+s) for s in os.listdir(path)]\n", " slices.sort(key = lambda x: int(x.ImagePositionPatient[2]))\n", " new_slices = []\n", " slices = [cv2.resize(np.array(each_slice.pixel_array),(IMG_PX_SIZE,IMG_PX_SIZE)) for each_slice in slices]\n", " chunk_sizes = math.ceil(len(slices) / HM_SLICES)\n", " for slice_chunk in chunks(slices, chunk_sizes):\n", " slice_chunk = list(map(mean, zip(*slice_chunk)))\n", " new_slices.append(slice_chunk)\n", " \n", " if len(new_slices) == HM_SLICES-1:\n", " new_slices.append(new_slices[-1])\n", "\n", " if len(new_slices) == HM_SLICES-2:\n", " new_slices.append(new_slices[-1])\n", " new_slices.append(new_slices[-1])\n", "\n", " if len(new_slices) == HM_SLICES+2:\n", " new_val = list(map(mean, zip(*[new_slices[HM_SLICES-1],new_slices[HM_SLICES],])))\n", " del new_slices[HM_SLICES]\n", " new_slices[HM_SLICES-1] = new_val\n", " \n", " if len(new_slices) == HM_SLICES+1:\n", " new_val = list(map(mean, zip(*[new_slices[HM_SLICES-1],new_slices[HM_SLICES],])))\n", " del new_slices[HM_SLICES]\n", " new_slices[HM_SLICES-1] = new_val\n", " \n", " print(len(slices), len(new_slices))\n", " #print(len(slices),slices[0].pixel_array.shape)\n", " #print(slices[0])\n", " if visualize:\n", " fig = plt.figure()\n", " for num,each_slice in enumerate(new_slices):\n", " y = fig.add_subplot(4,5,num+1)\n", " #new_image = cv2.resize(np.array(each_slice.pixel_array),(IMG_PX_SIZE,IMG_PX_SIZE))\n", " y.imshow(each_slice)\n", " plt.show()\n", " if label == 1: label=np.array([0,1])\n", " elif label == 0: label=np.array([1,0])\n", " \n", " return np.array(new_slices),label\n", "\n", "much_data = []\n", "for num,patient in enumerate(patients):\n", " if num % 100 == 0:\n", " print(num)\n", " try:\n", " img_data,label = process_data(patient,labels_df,img_px_size=IMG_PX_SIZE, hm_slices=HM_SLICES)\n", " #print(img_data.shape,label)\n", " much_data.append([img_data,label])\n", " except KeyError as e:\n", " print('This is unlabeled data!')\n", "\n", "#np.save('muchdata-{}-{}-{}.npy'.format(IMG_PX_SIZE,IMG_PX_SIZE,HM_SLICES), much_data)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "3cfe871f-f933-556d-6b9c-e2efa7aeffe8" }, "outputs": [], "source": [ "len(patients)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "420ea1be-c16c-cd93-82c9-6c1d9fa3c295" }, "outputs": [], "source": [ "import tensorflow as tf\n", "n_classes = 2\n", "batch_size = 10\n", "\n", "x = tf.placeholder('float')\n", "y = tf.placeholder('float')\n", "\n", "keep_rate = 0.8\n", "def conv3d(x, W):\n", " return tf.nn.conv3d(x, W, strides=[1,1,1,1,1], padding='SAME')\n", "\n", "def maxpool3d(x):\n", " # size of window movement of window as you slide about\n", " return tf.nn.max_pool3d(x, ksize=[1,2,2,2,1], strides=[1,2,2,2,1], padding='SAME')\n", "def convolutional_neural_network(x):\n", " # # 5 x 5 x 5 patches, 1 channel, 32 features to compute.\n", " weights = {'W_conv1':tf.Variable(tf.random_normal([3,3,3,1,32])),\n", " # 5 x 5 x 5 patches, 32 channels, 64 features to compute.\n", " 'W_conv2':tf.Variable(tf.random_normal([3,3,3,32,64])),\n", " # 64 features\n", " 'W_fc':tf.Variable(tf.random_normal([54080,1024])),\n", " 'out':tf.Variable(tf.random_normal([1024, n_classes]))}\n", "\n", " biases = {'b_conv1':tf.Variable(tf.random_normal([32])),\n", " 'b_conv2':tf.Variable(tf.random_normal([64])),\n", " 'b_fc':tf.Variable(tf.random_normal([1024])),\n", " 'out':tf.Variable(tf.random_normal([n_classes]))}\n", "\n", " # image X image Y image Z\n", " x = tf.reshape(x, shape=[-1, IMG_PX_SIZE, IMG_PX_SIZE, HM_SLICES, 1])\n", "\n", " conv1 = tf.nn.relu(conv3d(x, weights['W_conv1']) + biases['b_conv1'])\n", " conv1 = maxpool3d(conv1)\n", "\n", "\n", " conv2 = tf.nn.relu(conv3d(conv1, weights['W_conv2']) + biases['b_conv2'])\n", " conv2 = maxpool3d(conv2)\n", "\n", " fc = tf.reshape(conv2,[-1, 54080])\n", " fc = tf.nn.relu(tf.matmul(fc, weights['W_fc'])+biases['b_fc'])\n", " fc = tf.nn.dropout(fc, keep_rate)\n", "\n", " output = tf.matmul(fc, weights['out'])+biases['out']\n", "\n", " return output" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "e23af6eb-5aa8-350b-fe35-64e1aee2b7dc" }, "outputs": [], "source": [ "#much_data = np.load('muchdata-50-50-20.npy')\n", "# If you are working with the basic sample data, use maybe 2 instead of 100 here... you don't have enough data to really do this\n", "train_data = much_data[:-2]\n", "validation_data = much_data[-2:]\n", "\n", "\n", "def train_neural_network(x):\n", " prediction = convolutional_neural_network(x)\n", " cost = tf.reduce_mean( tf.nn.softmax_cross_entropy_with_logits(logits=prediction,labels=y) )\n", " optimizer = tf.train.AdamOptimizer(learning_rate=1e-5).minimize(cost)\n", " \n", " hm_epochs = 10\n", " with tf.Session() as sess:\n", " sess.run(tf.initialize_all_variables())\n", " \n", " successful_runs = 0\n", " total_runs = 0\n", " \n", " for epoch in range(hm_epochs):\n", " for data in train_data:\n", " total_runs += 1\n", " try:\n", " epoch_loss = 0\n", " X = data[0]\n", " Y = data[1]\n", " #print(Y)\n", " _, c = sess.run([optimizer, cost], feed_dict={x: X, y: Y})\n", " epoch_loss += c\n", " successful_run += 1\n", " except Exception as e:\n", " pass\n", " #print(str(e))\n", " \n", " print('Epoch', epoch+1, 'completed out of',hm_epochs,'loss:',epoch_loss)\n", "\n", " correct = tf.equal(tf.argmax(prediction, 1), tf.argmax(y, 1))\n", " accuracy = tf.reduce_mean(tf.cast(correct, 'float'))\n", "\n", " print('Accuracy:',accuracy.eval({x:[i[0] for i in validation_data], y:[i[1] for i in validation_data]}))\n", " \n", " print('Done. Finishing accuracy:')\n", " print('Accuracy:',accuracy.eval({x:[i[0] for i in validation_data], y:[i[1] for i in validation_data]}))\n", " \n", " print('fitment percent:',successful_runs/total_runs)\n", "\n", "\n", "train_neural_network(x)" ] } ], "metadata": { "_change_revision": 0, "_is_fork": false, "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.6.0" } }, "nbformat": 4, "nbformat_minor": 0 }