{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np\nimport pandas as pd\nimport os\nimport matplotlib\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm_notebook\n%matplotlib inline \nimport cv2 as cv\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nfrom keras.models import Sequential \nfrom keras.layers import Conv2D,MaxPool2D,DepthwiseConv2D,Flatten,Dense,Dropout,AveragePooling2D,BatchNormalization\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.optimizers import Adam\nfrom keras.utils import to_categorical\nfrom tensorflow.keras import regularizers","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.preprocessing.image import ImageDataGenerator\nimg_height=512\nimg_width=512\nbatch_size=128\ndirectory=\"../input//dl-hack-track-1-cv/train/\"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"datagen=ImageDataGenerator(\n    rescale=1.0/255,\n#     featurewise_center=True,\n#     featurewise_std_normalization=True,\n#     rotation_range=20,\n#     width_shift_range=0.2,\n#     height_shift_range=0.2,\n    horizontal_flip=False,\n    data_format=\"channels_last\",\n    validation_split=0.2,\n    dtype=tf.float32\n    \n    \n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data=datagen.flow_from_directory(\n    \"../input//dl-hack-track-1-cv/train/\",\n    target_size=(512,512),\n    batch_size=batch_size,\n    color_mode=\"rgb\",\n    class_mode='categorical',\n    shuffle=True,\n    subset='training'\n    \n)\nval_data=datagen.flow_from_directory(\n    \"../input//dl-hack-track-1-cv/train/\",\n    target_size=(512,512),\n    batch_size=batch_size,\n    color_mode=\"rgb\",\n    class_mode='categorical',\n    shuffle=True,\n    subset='validation'\n    \n)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# model= keras.Sequential()\n# # input shape is shape of images\n# model.add(keras.Input(shape=(512,512,3)))\n# model.add(DepthwiseConv2D(3,(2,2)))\n# model.add(BatchNormalization())\n# model.add(Conv2D(32,(3,3),activation = 'relu',padding='same')) \n# model.add(BatchNormalization())\n# model.add(MaxPool2D((2, 2)))\n# model.add(Conv2D(48, (3, 3), activation='relu', padding='same'))\n# model.add(Conv2D(64, (3, 3), activation='relu', padding='same'))\n# model.add(MaxPool2D((2, 2)))\n# model.add(Conv2D(80, (3, 3), activation='relu', padding='same'))\n# model.add(Conv2D(128, (3, 3), activation='relu', padding='same'))\n# model.add(MaxPool2D((2, 2)))\n# model.add(Flatten())\n# model.add(Dense(64, activation='relu'))\n# model.add(Dense(16,activation=\"relu\"))\n# model.add(Dense(2,activation=\"sigmoid\"))\n# model.compile(\n#         optimizer=tf.keras.optimizers.Adam(0.00005),\n#             loss='sparse_categorical_crossentropy',\n#             metrics=['accuracy']\n#     )\n# model.summary()\n# #===========================================================\n\n# batches=1\n# q=True\n# for x_batch, y_batch in train_data:\n#         if q:\n#             model.fit(x_batch, y_batch)\n#             q=False\n#         else:\n#             model.evaluate(x_batch,y_batch)\n#             q=True\n#         batches += 1\n        \n#         if batches >= 70:\n#             # we need to break the loop by hand because\n#             # the generator loops indefinitely\n#             break","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def inflow(i):\n    x=MaxPool2D((2, 2))(i)\n    x=DepthwiseConv2D((3,3),padding=\"same\")(x)\n    x=BatchNormalization()(x)\n    #x=Conv2D(3,(3,3),padding=\"same\",activation=\"relu\")(x)\n    return x\ndef outflow(x):\n    x=Flatten()(x)\n    x=Dense(64,activation='relu')(x)\n    x=Dense(16,activation='relu')(x)\n    x=Dense(2,activation=\"sigmoid\")(x)\n    return x\n\nvgg19 =keras.applications.VGG16(weights='imagenet',input_shape=(256,256,3), include_top=False) \n# for layer in vgg19.layers:\n#     layer.trainable=False\n\n    \ndef midflow(x):\n    x=Conv2D(filters=64,kernel_size=(3,3),padding=\"same\", activation=\"relu\")(x)\n    x=Conv2D(filters=64,kernel_size=(3,3),padding=\"same\", activation=\"relu\")(x)\n    x=MaxPool2D(pool_size=(2,2),strides=(2,2))(x)\n    \n    #========================================================================\n    y=Conv2D(filters=64,kernel_size=(1,1), padding=\"same\", activation=\"relu\")(x)\n    y=DepthwiseConv2D(kernel_size=(2,2),strides=(2,2), padding=\"same\", activation=\"relu\")(y)\n\n    #=========================================================================\n    x=Conv2D(filters=128, kernel_size=(3,3), padding=\"same\", activation=\"relu\")(x)\n    x=DepthwiseConv2D(kernel_size=(3,3), padding=\"same\", activation=\"relu\")(x)\n    x=MaxPool2D(pool_size=(2,2),strides=(2,2))(x)\n    x=layers.Concatenate()([x,y])\n\n    x=Conv2D(filters=256, kernel_size=(3,3), padding=\"same\", activation=\"relu\")(x)\n    x=Conv2D(filters=256, kernel_size=(3,3), padding=\"same\", activation=\"relu\")(x)\n    x=Conv2D(filters=256, kernel_size=(3,3), padding=\"same\", activation=\"relu\")(x)\n    x=MaxPool2D(pool_size=(2,2),strides=(2,2))(x)\n    \n    #============================================================================\\\n    \n    y=Conv2D(filters=64, kernel_size=(3,3), padding=\"same\", activation=\"relu\")(y)\n    y=Conv2D(filters=64, kernel_size=(3,3), padding=\"same\", activation=\"relu\")(y)\n    y=DepthwiseConv2D(kernel_size=(2,2),strides=(2,2), padding=\"same\", activation=\"relu\")(y)\n    y=MaxPool2D(pool_size=(2,2),strides=(2,2))(y)\n    \n    \n    z=Conv2D(filters=256,kernel_size=(1,1), padding=\"same\", activation=\"relu\")(x)\n    z=Conv2D(filters=256,kernel_size=(1,1), padding=\"same\", activation=\"relu\")(z)\n    z=DepthwiseConv2D(kernel_size=(2,2),strides=(2,2), padding=\"same\", activation=\"relu\")(z)\n    \n    #=============================================================================\n    x=Conv2D(filters=576, kernel_size=(3,3), padding=\"same\", activation=\"relu\")(x)\n    x=Conv2D(filters=576, kernel_size=(3,3), padding=\"same\", activation=\"relu\")(x)\n    x=Conv2D(filters=576,kernel_size=(3,3), padding=\"same\", activation=\"relu\")(x)\n    x=MaxPool2D(pool_size=(2,2),strides=(2,2))(x)\n    \n    x=layers.Concatenate()([x,y,z])\n    x=Conv2D(filters=896, kernel_size=(3,3), activation=\"relu\")(x)\n    x=Conv2D(filters=896, kernel_size=(2,2), padding=\"same\", activation=\"relu\")(x)\n    x=DepthwiseConv2D(kernel_size=(3,3), padding=\"same\", activation=\"relu\")(x)\n    x=MaxPool2D(pool_size=(2,2),strides=(2,2))(x)\n    return x    \n\ninputs=keras.Input(shape=(512,512,3))\ny=inflow(inputs)\ny=midflow(y)\noutputs=outflow(y)\nModel= keras.Model(inputs,outputs)\nModel.summary()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Model.compile(\n       optimizer=tf.keras.optimizers.Adam(0.0000005),\n       loss='categorical_crossentropy',\n       metrics=['accuracy']\n)\nModel.load_weights(\"transfer.h5\")\nModel.fit(train_data,epochs=1\n          ,validation_data=val_data\n         )\nModel.save(\"transfer.h5\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<pre>\nModel.compile(\n       optimizer=tf.keras.optimizers.Adam(0.00005),\n       loss='categorical_crossentropy',\n       metrics=['accuracy']\n)\n#Model.load_weights(\"transfer.h5\")\nModel.fit(train_data,epochs=3\n          ,validation_data=val_data\n         )\nModel.save(\"transfer.h5\")\n\nEpoch 1/3\n219/219 [==============================] - 1196s 5s/step - loss: 0.5817 - accuracy: 0.6411 - val_loss: 0.6919 - val_accuracy: 0.5164\nEpoch 2/3\n219/219 [==============================] - 1223s 6s/step - loss: 0.2059 - accuracy: 0.9193 - val_loss: 0.2684 - val_accuracy: 0.8927\nEpoch 3/3\n219/219 [==============================] - 1177s 5s/step - loss: 0.1251 - accuracy: 0.9529 - val_loss: 0.1380 - val_accuracy: 0.9453\n</pre>\n","metadata":{}},{"cell_type":"code","source":"Model.compile(\n   optimizer=tf.keras.optimizers.Adam(0.0000001),\n   loss='categorical_crossentropy',\n   metrics=['accuracy']\n\n)\nModel.load_weights(\"transfer.h5\")\nModel.fit(train_data,epochs=1,validation_data=val_data)\nModel.save(\"transfer.h5\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\nModel: \"model\"\n_________________________________________________________________\nLayer (type)                 Output Shape              Param #   \n=================================================================\ninput_2 (InputLayer)         [(None, 512, 512, 3)]     0         \n_________________________________________________________________\ndepthwise_conv2d (DepthwiseC (None, 256, 256, 3)       15        \n_________________________________________________________________\nbatch_normalization (BatchNo (None, 256, 256, 3)       12        \n_________________________________________________________________\nVgg-16\n_________________________________________________________________\nflatten (Flatten)            (None, 32768)             0         \n_________________________________________________________________\ndense (Dense)                (None, 64)                2097216   \n_________________________________________________________________\ndense_1 (Dense)              (None, 16)                1040      \n_________________________________________________________________\ndense_2 (Dense)              (None, 2)                 34        \n=================================================================\nTotal params: 16,813,005\nTrainable params: 16,812,999\nNon-trainable params: 6\n\n\n\nModel.compile(\n       optimizer=tf.keras.optimizers.Adam(0.000003),\n       loss='categorical_crossentropy',\n       metrics=['accuracy']\n)\n#Model.load_weights(\"transfer.h5\")\nModel.fit(train_data,epochs=1,validation_data=val_data)\nModel.save(\"transfer.h5\")\n\n\n\n\"\"\"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def classify(Id):\n    x=predictwithId(Id)\n    if x>0.995:\n        return 0.99\n    if x<0.0025:\n        return 0.005\n    return float(x)\n    \n\ndef predictwithId(Id):\n    path='../input/dl-hack-track-1-cv/test/'+str(Id)+\".png\"\n    image = tf.keras.preprocessing.image.load_img(path,\n                                              grayscale=False, \n                                              color_mode=\"rgb\", \n                                              target_size=None,\n                                              interpolation=\"nearest\")\n    input_arr = keras.preprocessing.image.img_to_array(image)\n    input_arr = np.array(input_arr)/255  # Convert single image to a batch.\n    result=Model.predict(np.reshape(input_arr,(1,512,512,3)))[:,1]\n    return result\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path='../input/dl-hack-track-1-cv/train/fake/'+str(4)+\".png\"\nimage = tf.keras.preprocessing.image.load_img(path,\n                                              grayscale=False, \n                                              color_mode=\"rgb\", \n                                              target_size=None,\n                                              interpolation=\"nearest\")\ninput_arr = keras.preprocessing.image.img_to_array(image)\ninput_arr = np.array(input_arr)/255  # Convert single image to a batch.\nresult=Model.predict(np.reshape(input_arr,(1,512,512,3)))[:,1]\nfloat(result)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission=pd.read_csv(\"../input/dl-hack-track-1-cv/sample_submission.csv\")\nsubmission['p_real']=submission['id'].apply(classify)\nprint(len(submission))\nsubmission.set_index('id').to_csv('sample_submission.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Model.save(\"transfer.h5\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path='../input/dl-hack-track-1-cv/test/'+str(0)+\".png\"\nimage = tf.keras.preprocessing.image.load_img(path,\n                                              grayscale=False, \n                                              color_mode=\"rgb\", \n                                              target_size=None,\n                                              interpolation=\"nearest\")\ninput_arr = keras.preprocessing.image.img_to_array(image)\ninput_arr = np.array(input_arr)/255  # Convert single image to a batch.\ninput_arr = np.reshape(input_arr,(1,512,512,3))/255\nmodel2 = tf.keras.models.load_model(\"m2.h5\")\nblocks = [7]\noutputs = [model2.layers[i].output for i in blocks]\nmodel2 = keras.Model( inputs= model2.inputs, outputs = outputs)\nfeature_map = model2.predict(input_arr)\n\nfor i,fmap in zip(blocks,feature_map):\n    fig = plt.figure(figsize=(20,15))\n    #https://stackoverflow.com/a/12444777\n    fig.suptitle(\"BLOCK_{}\".format(i) , fontsize=20)\n    for i in range(1,features.shape[3]+1):\n        plt.subplot(8,8,i)\n        plt.imshow(fmap[:,:,i-1] , cmap='gray')\n        #print(fmap[:,:,i].shape)\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndef inflow(i):\n    x=Conv2D(3,3,padding=\"same\")(i)\n    x=DepthwiseConv2D((2,2),strides=(2,2))(x)\n    return x\nvgg_model=keras.applications.VGG16(weights='imagenet', include_top=False, input_shape=(256,256,3))\nmodel_aug=Sequential()\nmodel_aug.add(Conv2D(3,3,padding=\"same\",input_shape=(512,512,3)))\nmodel_aug.add(DepthwiseConv2D((2,2),strides=(2,2)))\nmodel_aug.add(vgg_model)\ntop_model=Sequential()\n\n\ntop_model.add(Flatten(input_shape=(2, 2, 512)))\n#model_aug.add(Dropout(0.3))\ntop_model.add(Dense(64, activation='relu'))\n\ntop_model.add(Dense(1, activation='sigmoid'))\n\nmodel_aug.add(top_model)\n\nfor layer in model_aug.layers[3].layers[:17]:\n    layer.trainable=False\nmodel      ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}