{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"# Cancer Detection project, WIP - Rory Gold\n# Aim is to gain familiarity with the use of CNNs with tensorflow for image classification\n\n# CNN structure is based off advice from online Data Science posts and Kaggle Deep Learning microcourse","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# Import libraries\nimport os\n\nimport numpy as np\nimport pandas as pd\nimport random\nimport math \nimport time\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.preprocessing import OneHotEncoder\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.metrics import mean_absolute_error\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.model_selection import RandomizedSearchCV\n\nfrom tensorflow import keras\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPooling2D, AveragePooling2D\nfrom sklearn.model_selection import train_test_split\nfrom tensorflow.python import keras\n\nfrom keras.callbacks import ModelCheckpoint\nfrom keras.callbacks import EarlyStopping\n\nfrom tensorflow.keras.wrappers.scikit_learn import KerasClassifier\nfrom tensorflow.python.keras.constraints import maxnorm\nfrom tensorflow.keras.optimizers import Adam\n\nfrom os.path import join\nfrom tensorflow.keras.preprocessing.image import load_img, img_to_array\nfrom tensorflow.keras.applications.resnet50 import preprocess_input\n# print('Stage complete')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Define Functions\ndef get_filepaths(dirname):\n    img_paths = []\n    for _,_,filenames in os.walk(dirname):\n        for filename in filenames:\n            img_paths.append(os.path.join(dirname, filename))\n    return img_paths, filenames\n\ndef get_file_ids(filenames):\n    file_ids = []\n    splt_fstop = '.'\n    for filename in filenames:\n        file_ids.append(filename.partition(splt_fstop)[0])\n    return file_ids\n\ndef pandas_to_array(data):\n    data_array = data.to_numpy()\n    return data_array\n\ndef OHE_encode_target(target):\n    OHE_target = pd.get_dummies(target)\n    OHE_target.columns = ['No_Cancer', 'Cancer']\n    return OHE_target\n\ndef balance_sample_classes(train_labels, img_paths, img_range):\n    train_labels = train_labels.iloc[img_range]\n    number_samples = len(train_labels)\n    number_cancer = sum(train_labels.label)\n    class_balance = number_cancer/number_samples\n    half_samples = number_samples/2\n\n    noncancer_index = train_labels.index[train_labels.label==0]\n    cancer_index = train_labels.index[train_labels.label==1]\n\n    balance_difference = abs(number_cancer-half_samples)\n    # If less cancer labels, need to reduce non cancer labels, drop difference number of non cancer\n    if number_cancer < half_samples:\n        noncancer_index = noncancer_index#[0:(len(noncancer_index)-int(2*balance_difference))]\n    if number_cancer > half_samples:\n        cancer_index = cancer_index#[0:(len(cancer_index)-int(2*balance_difference))]\n    \n    # Above code commented out as experimenting with 0 class balancing in the hope that randomized dataset will give representation\n    # of spread of data\n        \n    balanced_index = np.concatenate([noncancer_index, cancer_index])\n    balanced_index_sorted = np.sort(balanced_index)\n\n    train_labels_balanced = train_labels.loc[balanced_index_sorted]\n    train_labels = train_labels_balanced.reset_index(drop=True)\n\n    img_paths_balanced = []\n    for index in balanced_index_sorted:\n        img_paths_balanced.append(img_paths[index])\n    img_paths = img_paths_balanced\n    return train_labels, img_paths\n\nimage_size=96\ndef read_and_prep_images_ver3(img_paths, img_height=image_size, img_width=image_size):\n    img_arrays = []\n    for img_path in img_paths:\n        img = load_img(img_path, target_size=(img_height, img_width))\n        img_arrays.append(img_to_array(img))\n        img.close\n    img_array = np.array(img_arrays)\n    \n    # Scale images between 0 and 1\n    output = img_array/255\n    return output\n\ndef make_range_listing(list_range):\n    index_list = []\n    for index in list_range:\n        index_list.append(index)\n    return index_list\n\ndef shuffle_listing(listing):\n    shuffled_listing = []\n    for index in index_list:\n        shuffled_listing.append(listing[index])\n    return shuffled_listing\n\n# Create function to generate train batch data for a specific image range\ndef generate_train_data(img_range, img_paths_train, train_file_ids): \n    # Insert labels for each file\n    train_file = '/kaggle/input/histopathologic-cancer-detection/train_labels.csv'\n    train_labels = pd.read_csv(train_file)\n\n    # Sort train_labels by train image listing order\n    train_labels['id_cat'] = pd.Categorical(train_labels.id, categories = train_file_ids, ordered= True)\n    train_labels = train_labels.sort_values('id_cat')\n    train_labels = train_labels.reset_index(drop=True)\n    del train_labels['id_cat']\n\n    train_labels, img_paths_train = balance_sample_classes(train_labels, img_paths_train, img_range)\n    train_data = read_and_prep_images_ver3(img_paths_train)\n\n    train_Y = OHE_encode_target(train_labels.label)\n    train_Y = pandas_to_array(train_Y)\n    return train_data, train_Y\n\n#print('Stage complete')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Set random seed\nnp.random.seed(1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Choose images to work with\ndirname_train = '/kaggle/input/histopathologic-cancer-detection/train/'\ndirname_test = '/kaggle/input/histopathologic-cancer-detection/test/'\n\nimg_paths_train, filenames_train = get_filepaths(dirname_train)\ntrain_file_ids = get_file_ids(filenames_train)\n#print('Stage complete')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Insert code to randomize listing - simply randomized listing entries\nlist_range = range(0,len(img_paths_train))\nindex_list = make_range_listing(list_range)\nrandom.shuffle(index_list)\n\n# Shuffle path listings\nimg_paths_train = shuffle_listing(img_paths_train)\nfilenames_train = shuffle_listing(filenames_train)\ntrain_file_ids = shuffle_listing(train_file_ids)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def Create_Model(image_size, num_classes):\n    img_width = image_size\n    img_height = image_size\n    model = Sequential()\n    model.add(Conv2D(200, kernel_size=(3,3), strides=2, activation='relu', input_shape=(img_height, img_width, 3)))\n    model.add(AveragePooling2D())\n    model.add(Dropout(0.3))\n    model.add(Conv2D(200, kernel_size=(3,3), strides=2, activation='relu'))\n    model.add(AveragePooling2D())\n    model.add(Dropout(0.3))\n    model.add(Conv2D(200, kernel_size=(3,3), strides=2, activation='relu'))\n    model.add(AveragePooling2D())\n    model.add(Dropout(0.3))\n    model.add(Flatten())\n    model.add(Dense(128, activation='relu'))\n    model.add(Dense(num_classes, activation='softmax'))\n    return model\n\ndef train_model(model, train_data, train_Y, callbacks_list):\n    model.fit(train_data, train_Y, batch_size=128, epochs=150, validation_split=0.1, callbacks = callbacks_list)\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Create model\nimage_size = 96\nnum_classes = 2\n\n# Define early stopping parameters\nfilepath=\"weights.best.hdf5\"\ncheckpoint = ModelCheckpoint(filepath, monitor='val_accuracy', verbose=1, save_best_only=True, mode='max')\nes = EarlyStopping(monitor = 'val_accuracy', mode = 'max', verbose = 1, patience=10)\ncallbacks_list = [checkpoint, es]\nmodel = Create_Model(image_size, num_classes)\n\n# Compile model\nmodel.compile(loss=keras.losses.categorical_crossentropy,optimizer='adam',metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Fit model with early stopping\n# Define img_range\nimg_range = range(0,35000)\n\n# Generate training data\ntrain_data, train_Y = generate_train_data(img_range, img_paths_train, train_file_ids)\n#print('Stage Complete')\nmodel = train_model(model, train_data, train_Y, callbacks_list)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Clear training data to clear RAM for next stage\ndel train_data, train_Y","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Generate validation data\nimg_range_val = range(210000, 220025)\nvalidation_data, validation_Y = generate_train_data(img_range_val, img_paths_train, train_file_ids)\n\n# Evaluate model so far\nmodel.load_weights(\"weights.best.hdf5\")\nmodel.compile(loss = 'binary_crossentropy', optimizer = 'adam', metrics = ['accuracy'])\nscores = model.evaluate(validation_data, validation_Y)\n\n# Delete validation data and show scores\ndel validation_data, validation_Y\nprint('My model scored', scores)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}