{"cells":[{"metadata":{"_uuid":"6539b00d53e42730275acff519a86124d2269c3f"},"cell_type":"markdown","source":"# First Attempt\nThis is my first attempt to do image recognition. We begin the kernel with importing important stuffs"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\n\n# Image resizing\nfrom scipy.misc import imread, imresize\n%matplotlib inline\n\n# For more readable dictionary print\nimport pprint\n\n# Ignore Warning\nimport warnings \nwarnings.filterwarnings('ignore')\n\nimport os\nprint(os.listdir(\"../input\"))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"95d992620eb7a017525657f12393fa3116622cf0"},"cell_type":"markdown","source":"Transform the training dataframe to multiple class and print the size of training and test."},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"%%time\nprotein_part = [\"Nucleoplasm\", \"Nuclear membrane\", \"Nucleoli\", \"Nucleoli fibrillar center\",\n                \"Nuclear speckles\", \"Nuclear bodies\", \"Endoplasmic reticulum\", \"Golgi apparatus\",\n                \"Peroxisomes\", \"Endosomes\", \"Lysosomes\", \"Intermediate filaments\", \"Actin filaments\",\n                \"Focal adhesion sites\", \"Microtubules\", \"Microtubule ends\", \"Cytokinetic bridge\", \n                \"Mitotic spindle\", \"Microtubule organizing center\", \"Centrosome\", \"Lipid droplets\",\n                \"Plasma membrane\", \"Cell junctions\", \"Mitochondria\", \"Aggresome\", \"Cytosol\", \"Cytoplasmic bodies\",\n                \"Rods & rings\"]\n\ntrain = pd.read_csv(\"../input/train.csv\")\ntrain[\"list\"] = train[\"Target\"].apply(lambda x: x.split(\" \"))\nprint(\"Train Shape: \", train.shape)\n\nfor i in protein_part:\n    train[i] = 0\n\nfor i in train.index:\n    for k in train.loc[i, \"list\"]:\n        train.loc[i, protein_part[int(k)]] = 1\n\nsample = pd.read_csv(\"../input/sample_submission.csv\")\nprint(sample.head())\n\nprint(\"Total Train Image: \",len(os.listdir(\"../input/train\")))\nprint(\"Total Test Image: \",len(os.listdir(\"../input/test\")))\n\nprint(\"List of Protein Type:\")\nfor num, i in enumerate(protein_part):\n    print(num, i)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9298c7d5cf5adebab7f12b1f215b42f3b66d5a9c"},"cell_type":"markdown","source":"Printing the occurence of each protein part. Some protein part frequently appear compared to the others. This is classified as imbalance class. Declaring `class_weight` to fought imbalances. (i don't know whether this is a good way or not)"},{"metadata":{"trusted":true,"_uuid":"a343f6483b2389f361d4e7335878d86ef403cfec"},"cell_type":"code","source":"arr = []\nfor i in protein_part:\n    arr.append({\"Protein\": i, \"Occurences\": train[i].sum(), \"Proportion\": train[i].sum()/len(train) })\narr = pd.DataFrame(arr)\narr.set_index(\"Protein\", inplace=True)\narr[\"Reciproc\"] = 1 / arr[\"Proportion\"]\n\nclass_weight = {i: arr.iloc[i,2] for i in range(len(protein_part))}\ndel arr","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"bbebf18d0ec6e6e94697581783bfa6b73ba73258"},"cell_type":"markdown","source":"Create helper function to load image associated with id and transform it to a matrix with shape of `IMAGE_DIMENSION` "},{"metadata":{"trusted":true,"_uuid":"fa2223c4ac4c1bc819fabcd4c043af53e8f0419b"},"cell_type":"code","source":"from sklearn.preprocessing import MinMaxScaler\nIMAGE_DIMENSION = 128\ndef transform2arrayTest(image_id, dimension=IMAGE_DIMENSION):\n    mms = MinMaxScaler()\n    final = np.zeros([dimension,dimension,4])\n    image_list = [\"_red.png\", \"_blue.png\", \"_green.png\", \"_yellow.png\"]\n    for num, i in enumerate(image_list):\n        img = imread(\"../input/test/\" + image_id + i)\n        img = imresize(img, (dimension, dimension))\n        mms.fit(img)\n        img = mms.transform(img)\n        final[:,:,num] = img\n    return final\n\ndef transform2array(image_id, dimension=IMAGE_DIMENSION):\n    mms = MinMaxScaler()\n    final = np.zeros([dimension,dimension,4])\n    image_list = [\"_red.png\", \"_blue.png\", \"_green.png\", \"_yellow.png\"]\n    for num, i in enumerate(image_list):\n        img = imread(\"../input/train/\" + image_id + i)\n        img = imresize(img, (dimension, dimension))\n        mms.fit(img)\n        img = mms.transform(img)\n        final[:,:,num] = img\n    return final","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"731827a0ab3ebcd89337513e28ec10d7f3dc970f"},"cell_type":"markdown","source":"Import keras and declare F1 scoring because apparently keras does not have one."},{"metadata":{"trusted":true,"_uuid":"d881e120154f0730d6ff5d12367d16a8b349058e"},"cell_type":"code","source":"%%time\n\nfrom keras.models import Sequential, load_model\nfrom keras.layers import Conv2D, MaxPooling2D\nfrom keras.layers import Activation, Dropout, Flatten, Dense\nfrom keras.metrics import categorical_accuracy\nfrom keras import backend as K\n\n# From https://www.kaggle.com/guglielmocamporese/macro-f1-score-keras#\nimport tensorflow as tf\n\ndef f1(y_true, y_pred):\n    y_pred = K.round(y_pred)\n    tp = K.sum(K.cast(y_true*y_pred, 'float'), axis=0)\n    # tn = K.sum(K.cast((1-y_true)*(1-y_pred), 'float'), axis=0)\n    fp = K.sum(K.cast((1-y_true)*y_pred, 'float'), axis=0)\n    fn = K.sum(K.cast(y_true*(1-y_pred), 'float'), axis=0)\n\n    p = tp / (tp + fp + K.epsilon())\n    r = tp / (tp + fn + K.epsilon())\n\n    f1 = 2*p*r / (p+r+K.epsilon())\n    f1 = tf.where(tf.is_nan(f1), tf.zeros_like(f1), f1)\n    return K.mean(f1)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c53e18226825d56dcc9cdd559f3c4973b935e4b3"},"cell_type":"markdown","source":"Declaring neural architecture, this is based on [this](https://blog.keras.io/building-powerful-image-classification-models-using-very-little-data.html)"},{"metadata":{"trusted":true,"_uuid":"d881e120154f0730d6ff5d12367d16a8b349058e"},"cell_type":"code","source":"def baseline_model():\n    model = Sequential()\n    model.add(Conv2D(32, (3, 3), input_shape=(128, 128, 4)))\n    model.add(Activation('relu'))\n    model.add(MaxPooling2D(pool_size=(2, 2)))\n\n    model.add(Conv2D(32, (3, 3)))\n    model.add(Activation('relu'))\n    model.add(MaxPooling2D(pool_size=(2, 2)))\n\n    model.add(Conv2D(64, (3, 3)))\n    model.add(Activation('relu'))\n    model.add(MaxPooling2D(pool_size=(2, 2)))\n\n    model.add(Flatten())  # this converts our 3D feature maps to 1D feature vectors\n    model.add(Dense(64))\n    model.add(Activation('relu'))\n    model.add(Dropout(0.5))\n    model.add(Dense(46))\n    model.add(Activation('relu'))\n    model.add(Dropout(0.5))\n    model.add(Dense(28))\n    model.add(Activation('sigmoid'))\n\n    model.compile(optimizer='adam',\n                  loss='categorical_crossentropy',\n                  metrics=[categorical_accuracy, f1])\n    return model","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"06a282f1272051e9d13c04444998a826315e42bd"},"cell_type":"markdown","source":"Because the data is pretty large, we preparing data in batches in attempt to reduce memory usage."},{"metadata":{"trusted":true,"_uuid":"4b2bb18aa7b485a08317aa2e51acda7e517f3636"},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nimport time\n\ndef prepare_data(start, end):\n    X = np.zeros([end-start, 128, 128, 4])\n    for num, i in enumerate(train.Id.loc[start:end-1]):\n        X[num, :, :, :] = transform2array(i)\n    Y = train[list(train.columns[3:])].loc[start:end-1]\n    return train_test_split(X, Y, test_size=0.2)\n    \n\ndef massive_print(num, num0):\n    print(\"TRAINING PART {0:d} OF {1:d} || ITERATION NUMBER {2:d} OF {3:d}\".format(num[0],num[1], num0[0], num0[1]))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d43a05e7ac580db3c6cc4563c93cfabb1d74a566"},"cell_type":"markdown","source":"Iteration declared here. "},{"metadata":{"trusted":true,"_uuid":"4d2bdd7e329bb2b7b52f8a5dd8860233226a06e6"},"cell_type":"code","source":"%%time\n\niteration = 18\ndivision = 10\npartial_limit = np.array(np.percentile(train.index, np.linspace(0,100,division+1)), dtype=\"int64\")\n\nmodel = baseline_model()\n\naccuracy_arr, error_arr, f1_arr = [], [], []\n\nfor j in range(iteration):\n    for i in range(division):\n        massive_print((i, division),(j, iteration-1))\n        start_time = time.time()\n        X_tr, X_te, Y_tr, Y_te =  prepare_data(partial_limit[i], partial_limit[i+1])\n        print(\"Composing data require {0:2.2f} seconds\".format(time.time() - start_time))\n\n        start_time = time.time()\n        model.fit(X_tr, Y_tr, epochs=20, batch_size=200, class_weight=class_weight,verbose=0)\n        scores = model.evaluate(X_te, Y_te, verbose=0)\n        accuracy_arr.append(scores[1])\n        error_arr.append(scores[0])\n        f1_arr.append(scores[2])\n        print(\"Baseline Score: %.2f%%\" % (scores[1]*100))\n        print(\"Elapsed training time: \", time.time() - start_time)\n        print(\" \")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e4d67c10decb8b2524fe56f6b689b1ade534d12e"},"cell_type":"markdown","source":"Delete training variables to release some memories. Plotting accuracy, loss, and F1 score."},{"metadata":{"trusted":true,"_uuid":"168e7f80ca8eed12edc8a9ce5a8799fb692c9ee7"},"cell_type":"code","source":"del X_tr\ndel X_te\ndel Y_tr\ndel Y_te\n\nplt.figure(figsize=(13,4))\nplt.plot(accuracy_arr)\nplt.ylabel(\"Accuracy\")\nplt.xlabel(\"Iteration\")\nplt.grid()\nplt.title(\"Accuracy Plot over Iteration\")\n\nplt.figure(figsize=(13,4))\nplt.plot(error_arr)\nplt.ylabel(\"Loss\")\nplt.xlabel(\"Iteration\")\nplt.grid()\nplt.title(\"Loss Plot over Iteration\")\n\nplt.figure(figsize=(13,4))\nplt.plot(f1_arr)\nplt.ylabel(\"F1 Macro\")\nplt.xlabel(\"Iteration\")\nplt.grid()\nplt.title(\"F1 Plot over Iteration\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5e4ed3c74058038664f8b59f5ba786cf6f10619d"},"cell_type":"markdown","source":"Helper function for preparing test set. These are the steps\n1. Neural network predict training set so we could get the probability threshold for each label. Handled by `train_predict` and `find_threshold.\n2. Neural network predict test set so we get the probility of each sample \n3. Neural network use probability threshold on to turn probability estimation to exact prediction. Handled by `rigid_prediction`\n4. Turn the exact prediction to string like the original training set. Handled by `prediction2string`\n5. Submit the result\n\nThe process of reading image from test set takes a lot of memory, this process need to batched. Helper function handles `batch_preparation`\n"},{"metadata":{"trusted":true,"_uuid":"168e7f80ca8eed12edc8a9ce5a8799fb692c9ee7"},"cell_type":"code","source":"from sklearn.metrics import roc_curve\n\nlen(os.listdir(\"../input/test/\"))\ntest = pd.read_csv(\"../input/sample_submission.csv\")\n\ndef batch_preparation(start, end):\n    final = np.zeros([end-start, 128, 128, 4])\n    for num, i in enumerate(test.Id.loc[start:end-1]):\n        final[num, :, :, :] = transform2arrayTest(i)\n    return final\n\ndef rigid_prediction(model, test_array, prediction_threshold):\n    prediction = model.predict(test_array)\n    final = np.zeros(prediction.shape)\n    for i in range(prediction.shape[0]):\n        for j in range(prediction.shape[1]):\n            if prediction[i, j] >= prediction_threshold[j]:\n                final[i, j] = 1\n            else:\n                final[i, j] = 0\n    return final\n\ndef prediction2string(array):\n    super_container = []\n    \n    for i in range(array.shape[0]):\n        prediction = np.array(array[i, :], dtype=\"int64\")\n        container = []\n        \n        for num, i in enumerate(prediction):\n            if i == 1:\n                container.append(num)\n                \n        string_container = str(container)[1:-1].replace(\",\", \"\")\n        super_container.append(string_container)\n    return super_container\n\ndef train_predict(train, model, division=10):\n    \n    partial_limit = np.array(np.percentile(train.index, np.linspace(0,100,division + 1)), dtype=\"int64\")\n    total_prediction = np.zeros(train[train.columns[3:]].shape)\n    \n    for k in range(division):\n        start, end = partial_limit[k], partial_limit[k+1]\n        X = np.zeros([end-start, 128, 128, 4])\n        \n        for num, i in enumerate(train.Id.loc[start:end-1]):\n            X[num, :, :, :] = transform2array(i)\n            \n        total_prediction[start:end, :] = model.predict(X)\n    return total_prediction\n\ndef find_threshold(train, train_prediction):\n    cols = train.columns[3:]\n    prediction_threshold = np.zeros(len(cols))\n    report = {}\n    \n    for num, i in enumerate(cols):    \n        y_true = train[i]\n        y_pred = train_prediction[:,num]\n        staging_value = 0\n        \n        for k in np.linspace(0,1,300):\n            rigid = np.array([1 if p > k else 0 for p in y_pred])\n            tpr, fpr, _ = roc_curve(y_true, rigid)\n            \n            if tpr[1]/fpr[1] > staging_value:\n                staging_value = tpr[1]/fpr[1]\n                prediction_threshold[num] = k\n                \n        report[cols[num]] = staging_value\n    pprint.pprint(report)\n    return prediction_threshold\n\ndef toRigidPrediction(y_pred, threshold):\n    return np.array([1 if k > threshold else 0 for k in y_pred])\n\n# Testing Unit for find_threshold function and random prediction baseline\na = (train.shape[0], train.shape[1]-3)\ndummy_train_prediction = np.random.random(a)\nfind_threshold(train, dummy_train_prediction)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c880112f553cbe458d753c6b57cf322a0aec5a7e"},"cell_type":"markdown","source":"Finding Threshold for test set rigid prediction "},{"metadata":{"trusted":true,"_uuid":"05cef643e297e6b3da1573b9d063a60901359a1b"},"cell_type":"code","source":"%%time\ntrain_prediction = train_predict(train, model)\nthreshold_prediction = find_threshold(train, train_prediction)\nprint(threshold_prediction)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"dc72f3844dc6ef56a634282946915e2b30b73628"},"cell_type":"markdown","source":"Predicting test set, reporting protein part occurences, and submit the result."},{"metadata":{"trusted":true,"_uuid":"05cef643e297e6b3da1573b9d063a60901359a1b","scrolled":false},"cell_type":"code","source":"%%time\nlimit = np.array(np.percentile(test.index, range(0, 100, division)), dtype=\"int64\")\nlimit = np.concatenate([limit, [len(test.index)]])\ntest_counter = np.zeros((len(test), len(protein_part)))\n\nfor i in range(division):\n    print(\"Start:\", limit[i],\"End:\",  limit[i+1])\n    test_batch = batch_preparation(limit[i], limit[i+1])\n    \n    rigid = rigid_prediction(model, test_batch, threshold_prediction)\n    test_counter[limit[i]:limit[i+1],:] = rigid\n    \n    container = prediction2string(rigid)\n    test.Predicted.loc[limit[i]:limit[i+1]-1] = container\n\narr = []\nfor i in range(len(protein_part)):\n    arr.append({\"Protein\": protein_part[i], \"Occurences\": test_counter[:,i].sum(), \"Proportion\": test_counter[:,i].sum()/len(test) })\narr = pd.DataFrame(arr)\narr.set_index(\"Protein\", inplace=True)\narr[\"Reciproc\"] = 1 / arr[\"Proportion\"]\nprint(arr)\n\nprint(test.head())\ntest.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}