{"cells":[{"metadata":{"_uuid":"c6ee79d7-230a-43b4-8916-581460b4a1f0","_cell_guid":"b2e6f620-c04e-431d-9996-312ec14c31aa","trusted":true},"cell_type":"code","source":"!pip install -q focal_loss\n# import os\nimport numpy as np\nimport pandas as pd\nfrom keras.models import load_model, Sequential\nfrom keras.optimizers import RMSprop\nfrom keras.layers import Dense\nfrom keras.callbacks import EarlyStopping\nfrom pylab import histogram, plot, show, interp, xlabel, ylabel\nfrom cv2 import resize, imread, cvtColor, inpaint, threshold, morphologyEx, INTER_AREA, MORPH_BLACKHAT, \\\n\tgetStructuringElement, \\\n\tTHRESH_BINARY, INPAINT_TELEA, COLOR_BGR2GRAY\nimport random\nfrom focal_loss import BinaryFocalLoss\n# from keras.losses import BinaryCrossentropy\nfrom pickle import load, dump","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5a0ca691-4279-4b33-b279-c1cebcb70072","_cell_guid":"9912940d-eb81-4386-befc-8ebd3b0b8f4a","trusted":true},"cell_type":"code","source":"\ncfg = {\n\t'IM_SIZE': (288, 192),\n\t'NBR_BINS': 256,\n\t'EPOCH_MAIN': 100,\n\t'EPOCH_RANGE': 95,\n\t'NEURON_MAIN': 100,\n\t'NEURON_RANGE': 60,\n\t'LAYER_MAIN': 7,\n\t'LAYER_RANGE': 5,\n\t# 'LOSS': BinaryCrossentropy( label_smoothing=0.7 ),\n\t'LOSS': BinaryFocalLoss( gamma=3, label_smoothing=0.7 ),\n\t'CALLBACK': EarlyStopping( monitor='loss', patience=5 ),\n\t'TEST_DATA_PERCENTAGE': 10,\n\t'ITERATIONS': 5,\n\t'USE_OLD_DATA': False,\n\t'SAVE_NEW_DATA': False,\n\t# 'OFFLINE': True\n\t'OFFLINE': False\n}\n\nprint( 'Imported Successfully!' )","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e1287088-662d-4cd2-a4b2-4ac19e579eb1","_cell_guid":"c820c147-6906-4c7b-a5af-a6769c11d962","trusted":true,"scrolled":true},"cell_type":"code","source":"\n\nf = \"../input/siim-isic-melanoma-classification/\"\nfimtr = \"../input/melanoma-merged-external-data-512x512-jpeg/512x512-dataset-melanoma/512x512-dataset-melanoma/\"\nfimte = \"../input/melanoma-merged-external-data-512x512-jpeg/512x512-test/512x512-test/\"\ntraindata = pd.read_csv( f+\"train.csv\" )\ntestdata = pd.read_csv( f+\"test.csv\" )\ntrainData = { 'image_name': [], 'sex': [], 'age_approx': [], 'anatom_site_general_challenge': [] }\ntestData = { 'image_name': [], 'sex': [], 'age_approx': [], 'anatom_site_general_challenge': [] }\ntrainData = { k: list( traindata[k] ) for k in [\"image_name\", \"sex\", \"age_approx\", \"anatom_site_general_challenge\"] }\ntestData = { k: list( testdata[k] ) for k in [\"image_name\", \"sex\", \"age_approx\", \"anatom_site_general_challenge\"] }\ny = list( traindata['target'] )\ndel traindata\ndel testdata\nprint( 'Defined main variables successfully!' )\nrm = []\nold = cfg['USE_OLD_DATA']\nsave = cfg['SAVE_NEW_DATA']\n\nif cfg['OFFLINE']:\n\tpics = ['ISIC_0015719', 'ISIC_0052212', 'ISIC_0068279', 'ISIC_0074268', 'ISIC_0074311']\n\tx = { k: [] for k in trainData.keys() }\n\tytmp = []\n\n\tfor pic in pics:\n\t\ti = trainData['image_name'].index( pic )\n\t\t[x[k].append( trainData[k][i] ) for k in trainData.keys()]\n\t\tytmp.append( y[i] )\n\n\ttrainData = x\n\ty = ytmp","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4a513bf5-f458-4d8e-9520-c905367124f5","_cell_guid":"d8d4d4a8-d945-4bdc-87cc-035f6c67652a","trusted":true,"scrolled":true},"cell_type":"code","source":"\nuniqueanatom = list( set( trainData['anatom_site_general_challenge'] ) )\nuniquesex = unique = list( set( trainData['sex'] ) )\ntry:\n    uniqueanatom.remove( 'nan' )\nexcept ValueError:\n    pass\ntry:\n    uniquesex.remove( 'nan' )\nexcept ValueError:\n    pass\n\ndef encode( d ):\n    global uniqueanatom, uniquesex\n    rd = d\n    for i in range( len( d['anatom_site_general_challenge'] ) ):\n        if d['anatom_site_general_challenge'][i] in uniqueanatom:\n            rd['anatom_site_general_challenge'][i] = uniqueanatom.index( d['anatom_site_general_challenge'][i] )\n        else:\n            rd['anatom_site_general_challenge'][i] = 0\n\n    for i in range( len( d['sex'] ) ):\n        if d['sex'][i] in uniquesex:\n            rd['sex'][i] = uniquesex.index( d['sex'][i] )\n        else:\n            rd['sex'][i] = 0\n    return rd\n\n\ndef rmrubbish( trainData ):\n\tprint( 'rmrubbish():\\n\\tInput Length:', len( trainData ), end=', ' )\n\trd = trainData\n\trm = []\n\n\tfor k in rd.keys():\n\t\tfor i in range( len( rd[k] ) ):\n\t\t\tif rd[k][i] == 0 or not rd[k][i]:\n\t\t\t\trm.append( i )\n\trm = list( set( rm ) )\n\td = { k: [] for k in rd.keys() }\n\tfor k in rd.keys():\n\t\tfor i in range( len( rd[k] ) ):\n\t\t\tif i not in rm:\n\t\t\t\td[k].append( rd[k][i] )\n\t# for i in range( len( trainData[ 'sex' ] ) ) :\n\t#     if trainData[ 'sex' ][ i ] == 0 :\n\t#         for k in rd.keys():\n\t#             rd[k].pop( i )\n\t#         y.pop( i )\n\t# for i in range( len( trainData[ 'anatom_site_general_challenge' ] ) ) :\n\t#     if trainData[ 'anatom_site_general_challenge' ][ i ] == 0 :\n\t#         rd.pop( i )\n\t#         y.pop( i )\n\tprint( '\\tOutput:', len( rd ) )\n\treturn rd\n\n\ndef histeq( im, nbr_bins=256 ):\n\t\"\"\" Histogram equalization of a grayscale image. \"\"\"  # get image histogram\n\timhist, bins = histogram( im, nbr_bins )\n\tcdf = imhist.cumsum()  # cumulative distribution function\n\tcdf = 255*cdf/cdf[-1]  # normalize\n\t# use linear interpolation of cdf to find new pixel values\n\treturn interp( im, bins[:-1], cdf )\n\n\ndef rmhair( image ):\n\t# (200,133), (128, 85), (192, 128), (500, 300)\n\tsrc = resize( imread( image, 0 ), cfg['IM_SIZE'], interpolation=INTER_AREA )\n\trmhim = inpaint( src, threshold( morphologyEx(\n\t\t# cvtColor( src, COLOR_BGR2GRAY ),\n\t\tsrc,\n\t\tMORPH_BLACKHAT,\n\t\tgetStructuringElement( 1, (17, 17) ) ),\n\t\t10, 255, THRESH_BINARY )[1], 1, INPAINT_TELEA )\n\n\treturn rmhim\n\n\ndef preprocess( ind, fim, r=True ):\n\tprint( 'Preprocessing' )\n\n\tD = encode( ind )\n\n\tif r:\n\t\tD = rmrubbish( D )\n\n\timages = []\n\tfor image in D['image_name']:\n\t\tim = list(\n\t\t\tnp.array( rmhair( fim+image+'.jpg' ) ).flatten().astype( np.float32 ) )\n\t\t# for i in range( len( im ) ):\n\t\t# \tim[i] = int( im[i] )\n\t\timages.append( im )\n\td = []\n\n\tfor i in range( len( D['image_name'] ) ):\n\t\td.append( np.array(images[i]+[int(D['sex'][i]), int(D['age_approx'][i]),\n\t\t                     int(D['anatom_site_general_challenge'][i])]).astype( np.uint8 ))\n\t\timages[i] = None\n\n\tif save:\n\t\ttry:\n\t\t\tdump( d, open( 'data.pickle', 'wb' ) )\n\t\t\tprint( 'Saved data!' )\n\t\texcept:\n\t\t\tprint( 'Couldn\\'t save data' )\n\treturn d\n\n\nprint( 'Defined functions successfully!\\n' )","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"457cc8f3-7056-4c5c-be2f-082c5c607fd3","_cell_guid":"201134c5-33f9-43c1-90b5-a1e4f963331c","trusted":true,"scrolled":true},"cell_type":"code","source":"\ntry:\n\tif old:\n\t\ttraindata = load( open( 'traindata.pickle', 'rb' ) )\n\telse:\n\t\traise FileNotFoundError\nexcept:\n\tif old:\n\t\ttry:\n\t\t\t# !wget https://www.kaggleusercontent.com/kf/39175984/eyJhbGciOiJkaXIiLCJlbmMiOiJBMTI4Q0JDLUhTMjU2In0..K93-kw_CIB2KZPdId_3kWw.WK2EnB2_ReqgtoHizmTzsDQRvWf0pjFCaD54_jwoFOjQgMWSnjIBZYu0S2u0dMtZFy6TvgTm5T3hIcpC4YoLzc9RYnKa6Hk5KwIUNGwg8FoUxXK2cTLq9N_Y0NWo4zWFXirzmDQxSqktPyWcjrbt412TEHMwIWQ7iig4asW7gr7sMBpg_OHAQHGosfiGNcwGlopXk1qvN-g_64cDZqLqWiw27ryy6-ewCH0VUhiA97ZLF_eCoFf7aJFyo9D1KutZqRMgY3mijKp3hWsdYGJrLPgoI-QVwpvUyHwv6IAOcybahMNEi6jegTlzXX45Maoo5bBO-HdpTA315aJG3CIdKpdAYYEYrkNZs9cLjdFenARc4wGHzeoj6Gac8BnvIajwpmnFaw699wywYyqGbcxhKgJoCbVm7pXnTrZVXSv8HozhW8yZ33TOm4fYQZyNzJ810pS9qObJtMt87yhSA2RI91ZjzkmTCtZ2ByFi6tYSmbPJukyWP7lgZGIKtbh4P7Jt1wvp0JXKniXF6dvlqslKEzH9DmYh8qek80LMMSuFmd_31gATTgH7su6CGJLaHio3DGIUT2C9QR9YrGGr7bmKYPL889g0aapml6cdL5g3hQZ7v_uRvbcbD-3MSVfnWYQOt4DncVILrAtuqP7ogJjHLQ.Amz1cTQyanbCelCEXFmDag/traindata.pickle\n\t\t\ttraindata = load( open( 'traindata.pickle', 'rb' ) )\n\t\texcept:\n\t\t\ttraindata = preprocess( trainData, fimtr )\n\telse:\n\t\ttraindata = preprocess( trainData, fimtr )\n\nprint( 'Defined traindata sucessfully!\\n' )","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3fc304a0-a9d5-4064-aff3-d57c1a6e4f8e","_cell_guid":"ccb5e6f6-76b7-462d-8400-9e0fb9dcb131","trusted":true,"scrolled":true},"cell_type":"code","source":"l = len( y )\ns = int( (cfg['TEST_DATA_PERCENTAGE']/100)*l )+1\n\nx_train = []\nfor i in range( len( traindata )-s ):\n\tx_train.append( traindata[i] )\n\ttraindata[i] = None\ny_train = []\nfor i in range( len( y )-s ):\n\ty_train.append( y[i] )\n\ty[i] = None\nprint( 'Defined x_train and y_train successfully!\\n' )","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"13f22d64-ac72-4c20-84ba-4e509c9992af","_cell_guid":"85f35d55-f5da-40ec-8f5a-97962c8bb1b6","trusted":true,"scrolled":true},"cell_type":"code","source":"\nx_test = []\nfor i in range( s ):\n\tx_test.append( traindata[-i-1] )\n\ttraindata[-i-1] = None\ny_test = []\nfor i in range( s ):\n\ty_test.append( y[-i-1] )\n\ty[-i-1] = None\n\nassert len( y_train )+len( y_test ) == l\ndel l\nprint( 'Defined x_test and y_test successfully!\\n' )","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5391cd0d-07f6-4a86-8745-6529804d1d90","_cell_guid":"2d054076-0143-4385-8ed6-33ca8f79f3e7","trusted":true,"scrolled":true},"cell_type":"code","source":"\nx_train = np.array( x_train ).astype( np.float32 )\ny_train = np.array( y_train )\nx_test = np.array( x_test ).astype( np.float32 )\ny_test = np.array( y_test )\n\ntry:\n\tmain_model = load_model( 'melanoma.h5' )\n\tacc = main_model.evaluate( x_test, y_test, verbose=0 )[1]\n\tprint( 'Loaded accuracy:', acc )\n\thistory = None\n\tepochs = None\n\tneurons = None\nexcept:\n\tmain_model = None\n\tacc = 0\n\thistory = None\n\tepochs = None\n\tneurons = None\nneuronsL, layersL, accL = ([], [], [])\n\n#\n# import pylab as plb\n#\n# plb.imshow( x_train[2][:-3].reshape( (133, 200) ) )\n# plb.show()\n\n# if cfg['useG/T-PU']:\n# Detect hardware, return appropriate distribution strategy\nimport tensorflow as tf\n\ntry:\n\t# TPU detection. No parameters necessary if TPU_NAME environment\n\t# variable is set. On Kaggle this is always the case.\n\ttpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n\tprint( 'Running on TPU ', tpu.master() )\nexcept ValueError:\n\ttpu = None\n\nif tpu:\n\ttf.config.experimental_connect_to_cluster( tpu )\n\ttf.tpu.experimental.initialize_tpu_system( tpu )\n\tstrategy = tf.distribute.experimental.TPUStrategy( tpu )\nelse:\n\t# default distribution strategy in Tensorflow. Works on CPU and single GPU.\n\tstrategy = tf.distribute.get_strategy()\n\nprint( \"REPLICAS: \", strategy.num_replicas_in_sync )","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"72092300-9cb5-4720-8f87-edc25393abae","_cell_guid":"e083df19-8903-4168-8248-bea084f6df1e","trusted":true,"scrolled":true},"cell_type":"code","source":"\nstrat = True\nfor i in range( cfg['ITERATIONS'] ):\n    # if cfg['useG/T-PU']:\n    print( 'Model '+str( i )+' :' )\n    mneurons = random.randint( cfg['NEURON_MAIN']-cfg['NEURON_RANGE'], cfg['NEURON_MAIN']+cfg['NEURON_RANGE'] )\n    mepochs = random.randint( cfg['EPOCH_MAIN']-cfg['EPOCH_RANGE'], cfg['EPOCH_MAIN']+cfg['EPOCH_RANGE'] )\n    mlayers = random.randint( cfg['LAYER_MAIN']-cfg['LAYER_RANGE'], cfg['LAYER_MAIN']+cfg['LAYER_RANGE'] )\n    print( '\\tNeurons:', mneurons )\n    print( '\\tLayers:', mlayers )\n\n    # if cfg['useG/T-PU']:\n    if strat:\n        try:\n            with strategy.scope():\n                model = Sequential()\n                model.add( Conv2D( input_shape=(128, 128), filters=64, kernel_size=(3, 3), padding=\"same\", activation=\"relu\" ) )\n                model.add( Conv2D( filters=64, kernel_size=(3, 3), padding=\"same\", activation=\"relu\" ) )\n                model.add( MaxPool2D( pool_size=(2, 2), strides=(2, 2) ) )\n                model.add( Conv2D( filters=128, kernel_size=(3, 3), padding=\"same\", activation=\"relu\" ) )\n                model.add( Conv2D( filters=128, kernel_size=(3, 3), padding=\"same\", activation=\"relu\" ) )\n                model.add( MaxPool2D( pool_size=(2, 2), strides=(2, 2) ) )\n                model.add( Conv2D( filters=256, kernel_size=(3, 3), padding=\"same\", activation=\"relu\" ) )\n                model.add( Conv2D( filters=256, kernel_size=(3, 3), padding=\"same\", activation=\"relu\" ) )\n                model.add( Conv2D( filters=256, kernel_size=(3, 3), padding=\"same\", activation=\"relu\" ) )\n                model.add( MaxPool2D( pool_size=(2, 2), strides=(2, 2) ) )\n                model.add( Conv2D( filters=512, kernel_size=(3, 3), padding=\"same\", activation=\"relu\" ) )\n                model.add( Conv2D( filters=512, kernel_size=(3, 3), padding=\"same\", activation=\"relu\" ) )\n                model.add( Conv2D( filters=512, kernel_size=(3, 3), padding=\"same\", activation=\"relu\" ) )\n                model.add( MaxPool2D( pool_size=(2, 2), strides=(2, 2) ) )\n                model.add( Conv2D( filters=512, kernel_size=(3, 3), padding=\"same\", activation=\"relu\" ) )\n                model.add( Conv2D( filters=512, kernel_size=(3, 3), padding=\"same\", activation=\"relu\" ) )\n                model.add( Conv2D( filters=512, kernel_size=(3, 3), padding=\"same\", activation=\"relu\" ) )\n                model.add( MaxPool2D( pool_size=(2, 2), strides=(2, 2) ) )\n\n                model.compile( optimizer=RMSprop( lr=0.002 ), loss=cfg['LOSS'],\n                               metrics=[\"accuracy\"] )\n                print( '\\tStrategy:', strat )\n        except:\n            strat = False\n    else:\n        layers = [Dense( mneurons, activation=\"relu\" )]+[Dense( mneurons, activation=\"relu\" )]*mlayers+[\n            Dense( 1, activation=\"sigmoid\" )]\n        model = Sequential( layers )\n        model.compile( optimizer=RMSprop( lr=0.002 ), loss=cfg['LOSS'],\n                       metrics=[\"accuracy\"] )\n        print( '\\tStrategy:', strat )\n\n    try:\n        hist = model.fit( x_train, y_train, epochs=mepochs, callbacks=[cfg['CALLBACK']], verbose=0 ).history\n    except KeyboardInterrupt:\n        break\n\n    macc = model.evaluate( x_test, y_test, verbose=0 )[1]\n    print( '\\tMacc: ', macc )\n    try:\n        neuronsL, layersL, accL = load( open( 'lists.pickle', 'rb' ) )\n        neuronsL.append( mneurons )\n        layersL.append( mlayers )\n        accL.append( macc )\n        dump( (neuronsL, layersL, accL), open( 'lists.pickle', 'wb' ) )\n        print( '\\tSaved:', neuronsL[-1], layersL[-1], accL[-1] )\n        neuronsL, layersL, accL = ([], [], [])\n    except:\n        neuronsL.append( mneurons )\n        layersL.append( mlayers )\n        accL.append( macc )\n        dump( (neuronsL, layersL, accL), open( 'lists.pickle', 'wb' ) )\n        print( '\\tFirst Saved:', neuronsL[-1], layersL[-1], accL[-1] )\n        neuronsL, layersL, accL = ([], [], [])\n    if macc > acc:\n        try:\n            model.save( \"melanoma.h5\" )\n            main_model = model\n            print( 'Saved model!' )\n        except:\n            print( 'Model Not saved' )\n        acc = macc\n        epochs = mepochs\n        neurons = mneurons\n        history = hist\nx_train, y_train, x_test, y_test = (None, None, None, None)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9523e300-eb1a-4352-bb1c-ff798c329c44","_cell_guid":"7bc96a58-4ad0-46f0-8cc0-6fe03c88a5ec","trusted":true,"scrolled":true},"cell_type":"code","source":"\nif history:\n\tplot( range( len( history['accuracy'] ) ), history['accuracy'], 'r-' )\n\txlabel( 'Epochs' )\n\tylabel( 'Accuracy' )\n\tshow()\n\ntry:\n\tneuronsL, layersL, accL = load( open( 'lists.pickle', 'rb' ) )\n\tfor i in range( len( accL ) ):\n\t\tb = 1-accL[i]+0.3\n\t\tr = accL[i]\n\t\tplot( layersL[i], neuronsL[i], '.', color=(r, 0, b) )\n\t\tprint( 'Neurons:', layersL[i], '; Epochs:', neuronsL[i], '; Acc:', accL[i] )\n\txlabel( 'Layers' )\n\tylabel( 'Neurons' )\n\tshow()\nexcept:\n\tprint( 'Couldn\\'t plot network data' )\n\nprint( 'Successfuly trained and saved model!\\n' )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"!wget 'https://www.kaggleusercontent.com/kf/39822866/eyJhbGciOiJkaXIiLCJlbmMiOiJBMTI4Q0JDLUhTMjU2In0..32KTzOldgMygZ6SMLxfTow._yP8dD8YDBYQwZQrETlQQkX9Kh7-w-Z7CM6RfK-6mbKkPIsiTcSbVVfEop7BjRwzjZSwHEOqcQ8f2Crr3EfcUF_-lsHm2hsnbdEdVNB8S_BeR6XaiXHI8NR0d2xbyCSv3iHeNJlkgpboILcWKH84J7hVeb34Cc_-RtjTI3WYS4U97YO9dugSvyX7jegXTwYzDivU5viMi4aFnYm2IlYknCHlWSmvncP0-7xoydeGdL0jVALTSGgevXIwVGfyukxvrpwLOSZmqiu8KNAB0H4IKZ_CLBIPfOV4gtiSftl8OuV6c_STOli4dKGqmSkdNsZtJJhV_IdU97TRFZhz-DVonqSnRKf92BOH073BIo6dXU-3xRsJG6x0wwvw2G2wMKYOSddG0puW3mFGZct37VAhC9PxD0iI23dg5VEwPfm-JiWcQwQDrqnaBD8CqYghVS-Fml8zmskeRUJrp2xCXAhkiTcnQWra1ymKueA-bbKu8VLVbVIGZj05PcQhvmdLHjJgiIg4Q4Soj-xpXmhn36lVDvyCcLJG6IbYMUoWlovgf4WnOUKd6vQ4BBYET4AjuRiwevuG6vYDKE6xopgGMhy4KHQrBSihZBI--5AHVQTfLp3cqKPdP7Gs5vn9QJzPCDWxyuBeqGkb3rePFCJvwMW7MP3S9X6tTU6pZ6Tmtbu63ds.BA0SNZO0Em-3KbA8x18Lnw/melanoma.h5'","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"28bbe0cf-5130-49d8-a0e7-7add7f7f7508","_cell_guid":"b6054c4e-8af4-4297-b308-f97aaa9e90d3","trusted":true,"scrolled":true},"cell_type":"code","source":"model = load_model('melanoma.h5')\ndata = trainData\npredictiondata = np.array( preprocess( data, fimtr, False ) ).astype( int )\n\npredicted = list( main_model.predict( predictiondata ) )\n\npredictions = []\n\nfor i in range( len( predicted ) ):\n\tpredictions.append( predicted[i][0] )\n\ndf = pd.DataFrame( data={ 'image_name': data['image_name'], 'target': predictions } )\ndf.to_csv( 'submission.csv', sep=',', index=False )","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}