{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport math\nimport sys\nfrom scipy import misc\nimport tensorflow as tf\nfrom skimage import img_as_float\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.callbacks import ReduceLROnPlateau,ModelCheckpoint\nfrom keras.layers import Dense,Dropout,Conv2D,MaxPooling2D,BatchNormalization,Input,UpSampling2D,Concatenate,Activation,Add,Flatten,Concatenate,Lambda,Reshape,AveragePooling2D\nfrom keras.models import Model\nfrom keras.models import load_model\nfrom keras.models import model_from_json\nfrom keras.optimizers import Adam\nimport keras.metrics\nimport matplotlib.pyplot as plt\nfrom keras import backend as K\n#from keras import Callback\nfrom sklearn.model_selection import train_test_split","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"config = tf.ConfigProto()\njit_level = tf.OptimizerOptions.ON_1\nconfig.graph_options.optimizer_options.global_jit_level = jit_level\nsess = tf.Session(config=config)\ntf.keras.backend.set_session(sess)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d3c42fc6e38f059c6d5bc5b581d0fe3003c615de"},"cell_type":"code","source":"name_label_dict = {\n0:  'Nucleoplasm',\n1:  'Nuclear membrane',\n2:  'Nucleoli',\n3:  'Nucleoli fibrillar center',\n4:  'Nuclear speckles',\n5:  'Nuclear bodies',\n6:  'Endoplasmic reticulum',\n7:  'Golgi apparatus',\n8:  'Peroxisomes',\n9:  'Endosomes',\n10:  'Lysosomes',\n11:  'Intermediate filaments',\n12:  'Actin filaments',\n13:  'Focal adhesion sites',   \n14:  'Microtubules',\n15:  'Microtubule ends',  \n16:  'Cytokinetic bridge',   \n17:  'Mitotic spindle',\n18:  'Microtubule organizing center',  \n19:  'Centrosome',\n20:  'Lipid droplets',\n21:  'Plasma membrane',   \n22:  'Cell junctions', \n23:  'Mitochondria',\n24:  'Aggresome',\n25:  'Cytosol',\n26:  'Cytoplasmic bodies',   \n27:  'Rods & rings' }","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ac86de615cb4354c309f8b31df6557959fd229ed","_kg_hide-input":true},"cell_type":"code","source":"PATH = '../input/'\nTRAIN = '../input/dadada/tdd/train/'\nTEST = '../input/human-protein-atlas-image-classification/test/'\nLABEL = '../input/ce-unet-1/nntrain.csv'\nSAMPLE = '../input/human-protein-atlas-image-classification/sample_submission.csv'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0c96a053c9d6ec4127211aaa079703db598658e9","_kg_hide-input":true},"cell_type":"code","source":"label = pd.read_csv(LABEL)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"df53c06c8d927d89bc1d2a069e5adca402e0a81e","_kg_hide-input":true},"cell_type":"code","source":"train_names = list(set('_'.join((f.split('_'))[:-1]) for f in os.listdir(TRAIN)))\ntest_names = list(set('_'.join((f.split('_'))[:-1]) for f in os.listdir(TEST)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"35d6a740c65d25d07dee992bdf2aa65ffe6c44fe"},"cell_type":"code","source":"train_data, valid_data = train_test_split(train_names, test_size=0.1, random_state=42)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"961cd138751f37c0092fca68d521a641ac8db9a7"},"cell_type":"code","source":"def open_rgby(path,pid): #a function that reads RGBY image\n    colors = ['red','green','blue','yellow']\n    tmp = np.zeros(shape=(512,512,4))\n    for i,color in enumerate(colors) :\n        tmp[:,:,i] = img_as_float(misc.imread(os.path.join(path, str(pid)+'_'+color+'.png')))\n    return tmp","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a944f017d4fd3b51ca4168905eae772db1c39a08"},"cell_type":"code","source":"def generator(train,label,batch_size) :\n    batch_features = np.zeros((batch_size,512,512,4))\n    batch_labels = np.zeros((batch_size,28))\n    while True :\n        for i in range(batch_size) :\n            idx= np.random.choice(len(train),1)[0]\n            batch_features[i,:,:,:] = open_rgby(TRAIN,train[idx])\n            temp = set([int(i) for i in (label.loc[label['Id']==train[idx]]['Target'].values[0]).split()])\n            batch_labels[i,:] =  np.array([1 if i in temp else 0 for i in range(28)])\n        \n        out = {'main':batch_labels,'encoder':batch_features}\n        yield batch_features,out","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f4787f1a63b4da87337452219a8b6bbfd1ce275d"},"cell_type":"code","source":"def progressbar(name,i,n):    \n    sys.stdout.write('\\r')\n    sys.stdout.write(name+\": [%-20s] %d%% %d/%d\" % ('=' * int(20. * (i + 1) / n - 1) + '>',\n                                                    int(100. * (i + 1) / n) , i,n))\n    sys.stdout.flush()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e24985f78b7caedcf5302f69d1711c724f8032bc"},"cell_type":"code","source":"def padding(tensor,h,w,r):\n    return tf.pad(tensor, tf.constant([[0,0],[h,h], [w,w],[r,r]]), \"CONSTANT\")\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cc27a40324212b5ad4d17b5a50c272cdff48a7c2"},"cell_type":"markdown","source":"inp = Input(shape=(512,512,4))\nx = Conv2D(64,(3,3),padding='same')(inp)#256\nx = BatchNormalization()(x)\nx = Activation('relu')(x)\n\nenc1 = MaxPooling2D((2,2),padding='same')(x)\nx = Conv2D(32,(3,3),padding='same')(enc1)#128\nx = BatchNormalization()(x)\nx = Activation('relu')(x)\n\nenc2 = MaxPooling2D((2,2),padding='same')(x)\nx = Conv2D(16,(3,3),padding='same')(enc2)#64\nx = BatchNormalization()(x)\nx = Activation('relu')(x)\n\nenc3 = MaxPooling2D((2,2),padding='same')(x)\nx = Conv2D(16,(3,3),padding='same')(enc3)\nx = BatchNormalization()(x)\nx = Activation('relu')(x)\n\nx = UpSampling2D((2,2))(x)\nx = Conv2D(32,(3,3),padding='same')(x)\nx = BatchNormalization()(x)\nx = Activation('relu')(x)\n\nx = UpSampling2D((2,2))(x)\nx = Conv2D(64,(3,3),padding='same')(x)\nx = BatchNormalization()(x)\nx = Activation('relu')(x)\n\nx = UpSampling2D((2,2))(x)\ndecoder = Conv2D(4,(3,3),activation='sigmoid',padding='same',name='encoder')(x)\n\npadded0 = Lambda(lambda t: padding(t,0,0,30))(inp)\npadded1 = Lambda(lambda t: padding(t,128,128,0))(enc1) #256,256,64\npadded2 = Lambda(lambda t: padding(t,192,192,16))(enc2) #128,128,32\npadded3 = Lambda(lambda t: padding(t,224,224,24))(enc3) #64,64,16\n\nadd = Concatenate()([padded0,padded1,padded2,padded3])\navg = AveragePooling2D((64,64))(add)\nflat = (Flatten())(avg)\ndense1 = (Dense(128, activation='relu'))(flat)\ndense2 = (Dense(64, activation='relu'))(dense1)\nout = (Dense(28, activation='sigmoid',name='main'))(dense2)\n\nmodel = Model(inputs=inp,outputs=[out,decoder])"},{"metadata":{"trusted":true,"_uuid":"81ff09d0f3b81d8214ee415fed51219879b5ae5a"},"cell_type":"markdown","source":"model.compile(optimizer=Adam(),\n              loss={'main':'binary_crossentropy','encoder':'mse'},\n              loss_weights={'main':1,'encoder':0.4},\n              metrics=['acc'])"},{"metadata":{"_uuid":"5c605b4baceac46ee76dc9c058dbc056a3fe965c"},"cell_type":"markdown","source":"model = load_model('../input/c_Unet_pc/c_Unet_save.h5',\n                   custom_objects={'padding': padding})"},{"metadata":{"trusted":true,"_uuid":"9fadc1ec8161ba3177520c9d648aa2138bd619df","scrolled":false,"_kg_hide-input":true},"cell_type":"code","source":"model = load_model('../input/ce-unet-2/ce_Unet_save28.h5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b2cb53215a388f4f43760e890f401cdc2a1c2c17","_kg_hide-output":true},"cell_type":"code","source":"epoch = 4\nhistory = model.fit_generator(generator(train_data,label,4), \n                                  samples_per_epoch=len(train_data)//4, epochs=epoch,\n                                  validation_data=generator(valid_data,label,4),\n                                  validation_steps=len(valid_data)//4,\n                                  callbacks=[ModelCheckpoint('mymodel_valid.h5',monitor='val_loss',save_best_only=True),\n                                  ReduceLROnPlateau(monitor='val_loss',patience=3)])\nmodel.save('ce_Unet_save32.h5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ce1d53c95172326a5b26396ecd2d149a5f8f5f31","_kg_hide-output":true},"cell_type":"code","source":"model.save('ce_Unet_save32.h5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b633a6f1595143aeb07847e7194733afbe33fe35"},"cell_type":"code","source":"plt.plot(history.history['main_loss'])\nplt.plot(history.history['val_main_loss'])\nplt.savefig(\"ce_Unet_loss_history.png\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d06143d5407f0253e46f6ff94454804f034428f2"},"cell_type":"markdown","source":"batch_features = np.zeros((1,512,512,4))\no = []\nm=[]\ntrain_len = len(train_names)\npred = np.zeros((train_len,28))\nfor k in range(train_len):\n    batch_features[:,:,:,:] = open_rgby(TRAIN,train_names[k])\n    pred[k],l = model.predict(batch_features)\n    o.append(train_names[k])\n    m.append(label.loc[label['Id']==o[k],'Target'].values[0])\n    progressbar('thres find',k,train_len)"},{"metadata":{"trusted":true,"_uuid":"0dc6d47bc373a326b44eee28f8646eb622f59564"},"cell_type":"markdown","source":"pred = (pred).T"},{"metadata":{"trusted":true,"_uuid":"e249d30bde6418e7aabb59cecd2fea1ac88e0f46"},"cell_type":"markdown","source":"pdf = pd.DataFrame(data = {'Id':o,'Target':m,\n                          0:pred[0],1:pred[1],2:pred[2],3:pred[3],\n                          4:pred[4],5:pred[5],6:pred[6],7:pred[7],\n                          8:pred[8],11:pred[11],9:pred[9],10:pred[10],\n                          12:pred[12],13:pred[13],14:pred[14],15:pred[15],\n                          16:pred[16],17:pred[17],18:pred[18],19:pred[19],\n                          20:pred[20],21:pred[21],22:pred[22],23:pred[24],\n                          24:pred[24],25:pred[25],26:pred[26],27:pred[27]\n                          })\npdf = pdf.sort_values(by = ['Id'])"},{"metadata":{"trusted":true,"_uuid":"b58b02102807607f750a1ccdbdb84f3de46d8a04"},"cell_type":"markdown","source":"cal=np.zeros((28,2))\nfor index, row in pdf.iterrows():\n    t  = row['Target'].split()\n    for e in t:\n        cal[int(e)][1]+=row[int(e)]\n        cal[int(e)][0]+=1"},{"metadata":{"trusted":true,"_uuid":"5804160691e36316e622cadadb8f683d03eedf3c"},"cell_type":"markdown","source":"cal"},{"metadata":{"_uuid":"c3e7a8599cf373f2978a523d54240cd90297a6cb"},"cell_type":"markdown","source":"thres = [e[1]/e[0]*0.98 for e in cal]\nprint(thres)"},{"metadata":{"trusted":true,"_uuid":"478a87a4fc33d132a345661849532df9fdbea4a8"},"cell_type":"markdown","source":"thres = [0.7332285743126011, 0.5805582254419689, 0.6830990579013652, 0.33133608226415084, 0.5892030431602163, \n         0.33554810233834226, 0.30124813610105283, 0.48676254640934774, 0.05069314378334587, 0.07186778056655506,\n         0.07550342306674565, 0.27888409512220735, 0.13545893820158397, 0.16009892984130975, 0.7817713042352804, \n         0.0065349138485544245, 0.04680300266810713, 0.22093092964865113, 0.2273163021918054, 0.30589077523285696,\n         0.07060120899301446, 0.4474626562126171, 0.14693263627129444, 0.002407802767594962, 0.22767899205469075, \n         0.5782695626512265, 0.09819474722240953, 0.10218797941007664]"},{"metadata":{"trusted":true,"_uuid":"6abd9dee7c89d5473bf4bbf170b134a912329ae1"},"cell_type":"markdown","source":"out = []\nbatch_features = np.zeros((1,512,512,4))\ntest_len = len(test_names)\npred = np.zeros((test_len,28))\nfor k in range(test_len):\n    batch_features[:,:,:,:] = open_rgby(TEST,test_names[k])\n    pred[k],l = model.predict(batch_features)\n    \n    outli=''\n    for j in range(28):\n        if(pred[k][j]>thres[j]): \n            outli+=(str(j)+\" \")\n    out.append(outli.strip())\n    progressbar('test predict',k,test_len)"},{"metadata":{"trusted":true,"_uuid":"2a270eb9e57390a5c2b761d52a3f5a019df4f38a"},"cell_type":"markdown","source":"df = pd.DataFrame(data = {'Id':test_names,'Predicted':out})\ndf = df.sort_values(by = ['Id'])\ndf.head()"},{"metadata":{"_kg_hide-output":true,"trusted":true,"_uuid":"b73945db5219c3236b0430a59e9cf39f9624dd35"},"cell_type":"markdown","source":"df.to_csv('c_unet_32'+\".csv\", header=True, index=False)"},{"metadata":{"trusted":true,"_uuid":"746e2fb515aa71ee3b9153e9edb93a042a4b7d9b"},"cell_type":"markdown","source":"pred = (pred).T"},{"metadata":{"trusted":true,"_uuid":"71bf686e2eb8651287194f1b3f9e29bbb88123c2"},"cell_type":"markdown","source":"pdf = pd.DataFrame(data = {'Id':test_names,\n                          0:pred[0],1:pred[1],2:pred[2],3:pred[3],\n                          4:pred[4],5:pred[5],6:pred[6],7:pred[7],\n                          8:pred[8],11:pred[11],9:pred[9],10:pred[10],\n                          12:pred[12],13:pred[13],14:pred[14],15:pred[15],\n                          16:pred[16],17:pred[17],18:pred[18],19:pred[19],\n                          20:pred[20],21:pred[21],22:pred[22],23:pred[24],\n                          24:pred[24],25:pred[25],26:pred[26],27:pred[27]\n                          })\npdf = pdf.sort_values(by = ['Id'])\npdf.head()"},{"metadata":{"_kg_hide-output":true,"trusted":true,"_uuid":"908c4fba8a4d317a5e899e44ef1a0f75df837a55"},"cell_type":"markdown","source":"pdf.to_csv('c_unet_32_raw'+\".csv\", header=True, index=False)"},{"metadata":{"trusted":true,"_uuid":"baf5d93a006d5972a4095fbf03aaba030a675e7d"},"cell_type":"markdown","source":"pdf.mean()"}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}