{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-05-15T05:53:02.649207Z","iopub.execute_input":"2022-05-15T05:53:02.649946Z","iopub.status.idle":"2022-05-15T05:53:03.976470Z","shell.execute_reply.started":"2022-05-15T05:53:02.649844Z","shell.execute_reply":"2022-05-15T05:53:03.966181Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\n","metadata":{"execution":{"iopub.status.busy":"2022-05-15T05:53:08.281077Z","iopub.execute_input":"2022-05-15T05:53:08.281800Z","iopub.status.idle":"2022-05-15T05:53:13.354077Z","shell.execute_reply.started":"2022-05-15T05:53:08.281759Z","shell.execute_reply":"2022-05-15T05:53:13.353289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nfrom PIL import Image\nimport os\nimport time\nimport glob\nfrom PIL import *\nimport skimage\nimport cv2\nimport matplotlib.pyplot as plt\n\n\n# from google.colab.patches import cv2_imshow\n# IMAGE_PATH = '../input/draper-satellite-image-chronology/train_sm/train_sm/set107_*.jpeg'\n\nWIDTH = 100\nHEIGHT = 100\nSEQUENCE = np.array([])\nBASIC_SEQUENCE = np.array([])\nNEXT_SEQUENCE = np.array([])\nNUMBER = 0\n\ndef image_initialize(image):\n    picture = Image.open(image)\n    picture = picture.resize((WIDTH, HEIGHT), Image.ANTIALIAS)\n    picture = picture.convert('L')\n    data = np.array(picture.getdata()).reshape(WIDTH, HEIGHT, 1)\n#     cv2_imshow(data)\n    plt.figure(figsize=(5,5))\n    plt.imshow(data)\n    return data\n\nfor file in glob.glob('../input/draper-satellite-image-chronology/train_sm/train_sm/set107_*.jpeg'):\n    image_initialize(file)","metadata":{"execution":{"iopub.status.busy":"2022-05-15T05:53:15.851280Z","iopub.execute_input":"2022-05-15T05:53:15.851874Z","iopub.status.idle":"2022-05-15T05:53:18.188141Z","shell.execute_reply.started":"2022-05-15T05:53:15.851832Z","shell.execute_reply":"2022-05-15T05:53:18.187277Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nfrom PIL import Image\nimport os\nimport time\nimport glob\nfrom PIL import *\nimport skimage\n\n\nWIDTH = 100\nHEIGHT = 100\nSEQUENCE = np.array([])\nBASIC_SEQUENCE = np.array([])\nNEXT_SEQUENCE = np.array([])\nNUMBER = 0\n\ndef image_initialize(image):\n    picture = Image.open(image)\n    picture = picture.resize((WIDTH, HEIGHT), Image.ANTIALIAS)\n    picture = picture.convert('L')\n#     picture.save('./temp.png')  \n    data = np.array(picture.getdata()).reshape(WIDTH, HEIGHT, 1)\n    return data\n\nfor file in glob.glob(\"../input/draper-satellite-image-chronology/train_sm/train_sm/set107_*.jpeg\"):\n    image_array = (image_initialize(file))/255\n    SEQUENCE = np.append(SEQUENCE, image_array)\n    NUMBER += 1\n    print(SEQUENCE[0])\n    print(NUMBER)\n    print(time.strftime('%Y-%m-%d %H:%M:%S', time.localtime()))\n\nSEQUENCE = SEQUENCE.reshape(NUMBER, WIDTH * HEIGHT)\n\nnp.savez('./sequence_array.npz', sequence_array=SEQUENCE)\nprint('Data saved.')\nprint(time.strftime('%Y-%m-%d %H:%M:%S', time.localtime()))","metadata":{"execution":{"iopub.status.busy":"2022-05-15T05:53:19.939326Z","iopub.execute_input":"2022-05-15T05:53:19.939862Z","iopub.status.idle":"2022-05-15T05:53:20.771068Z","shell.execute_reply.started":"2022-05-15T05:53:19.939821Z","shell.execute_reply":"2022-05-15T05:53:20.770167Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def ssim_loss(y_true, y_pred):\n  return tf.reduce_mean(tf.image.ssim(y_true, y_pred, 1.0))","metadata":{"execution":{"iopub.status.busy":"2022-05-15T05:53:22.347246Z","iopub.execute_input":"2022-05-15T05:53:22.348180Z","iopub.status.idle":"2022-05-15T05:53:22.353662Z","shell.execute_reply.started":"2022-05-15T05:53:22.348124Z","shell.execute_reply":"2022-05-15T05:53:22.352378Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def psnr(y_true,y_pred):\n    max_pixel = 1.0\n    return (10.0 * K.log((max_pixel ** 2) / (K.mean(K.square(y_pred - y_true), axis=-1)))) / 2.303","metadata":{"execution":{"iopub.status.busy":"2022-05-15T05:53:24.212983Z","iopub.execute_input":"2022-05-15T05:53:24.213247Z","iopub.status.idle":"2022-05-15T05:53:24.218082Z","shell.execute_reply.started":"2022-05-15T05:53:24.213215Z","shell.execute_reply":"2022-05-15T05:53:24.217184Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport tensorflow as tf\nfrom PIL import Image\nimport os\n# from keras.models import Sequential\n# from keras.layers.convolutional import Conv3D\n# from keras.layers.convolutional_recurrent import ConvLSTM2D\n# from keras.layers import BatchNormalization\nimport matplotlib.pyplot as plt\n\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nfrom tensorflow.keras.layers import Conv3D,ConvLSTM2D,BatchNormalization,MaxPooling2D, Dense, Flatten,GRU,Conv2D,Reshape,Permute,Lambda,Input,SimpleRNN\nfrom tensorflow.keras.models import Sequential, load_model\nfrom tensorflow.keras import backend as K\nimport pylab as pl\nimport time\n# from keras.utils import multi_gpu_model\nfrom keras import optimizers\n# from skimage.metrics import structural_similarity as ssim\n# import imutils\nimport cv2\n# from google.colab.patches import cv2_imshow\n\ndef custom_layer(tensor):\n    tensor=tensor[:,None]\n    return tensor\n\n\n#Loading array\nSEQUENCE = np.load('./sequence_array.npz')['sequence_array']  # load array\nn_samples = len(SEQUENCE)\n\nWIDTH = 100\nHEIGHT = 100\nn_frames = 1\n\n# step =1\nSEQUENCE = SEQUENCE.reshape(n_samples, WIDTH, HEIGHT, 1)\nBASIC_SEQUENCE = np.zeros((n_samples-n_frames, n_frames, WIDTH, HEIGHT, 1))\nNEXT_SEQUENCE = np.zeros((n_samples-n_frames, n_frames, WIDTH, HEIGHT, 1))\n\nfor i in range(n_frames):\n    BASIC_SEQUENCE[:, i, :, :, :] = SEQUENCE[i:i+n_samples-n_frames]\n    NEXT_SEQUENCE[:, i, :, :, :] = SEQUENCE[i+1:i+n_samples-n_frames+1]","metadata":{"execution":{"iopub.status.busy":"2022-05-15T05:53:25.406294Z","iopub.execute_input":"2022-05-15T05:53:25.407038Z","iopub.status.idle":"2022-05-15T05:53:25.425364Z","shell.execute_reply.started":"2022-05-15T05:53:25.406997Z","shell.execute_reply":"2022-05-15T05:53:25.424509Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\n\nclass TransformerBlock(layers.Layer):\n    def __init__(self, embed_dim, num_heads, ff_dim):\n        super(TransformerBlock, self).__init__()\n        self.att = layers.MultiHeadAttention(num_heads=num_heads, key_dim=embed_dim)\n        self.ffn = keras.Sequential(\n            [layers.Dense(ff_dim, activation=\"relu\"), layers.Dense(embed_dim),]\n        )\n        self.layernorm1 = layers.LayerNormalization(epsilon=1e-3)\n        self.layernorm2 = layers.LayerNormalization(epsilon=1e-3)\n\n    def call(self, inputs, training):\n        attn_output = self.att(inputs, inputs)\n        # attn_output = self.dropout1(attn_output, training=training)\n        out1 = self.layernorm1(inputs + attn_output)\n        ffn_output = self.ffn(out1)\n        # ffn_output = self.dropout2(ffn_output, training=training)\n        return self.layernorm2(out1 + ffn_output)\n","metadata":{"execution":{"iopub.status.busy":"2022-05-15T05:53:28.078893Z","iopub.execute_input":"2022-05-15T05:53:28.079708Z","iopub.status.idle":"2022-05-15T05:53:28.089126Z","shell.execute_reply.started":"2022-05-15T05:53:28.079654Z","shell.execute_reply":"2022-05-15T05:53:28.088172Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model=Sequential()\nmodel.add(Conv2D(filters=32, kernel_size=(3, 3),input_shape=(None, WIDTH, HEIGHT, 1), padding=\"same\"))\nmodel.add(Reshape((10000,32)))\nmodel.add(TransformerBlock(embed_dim=32, num_heads=2, ff_dim=32))\nmodel.add(BatchNormalization())\nmodel.add(Reshape((100,100,32)))\nmodel.add(Lambda(custom_layer))\nmodel.add(Conv3D(filters=1, kernel_size=(3, 3, 3), activation=\"relu\", padding='same', data_format=\"channels_last\"))\nmodel.compile(loss=ssim_loss, optimizer='adadelta',metrics=[tf.keras.metrics.MeanSquaredError(),tf.keras.metrics.RootMeanSquaredError(),ssim_loss,psnr])\n","metadata":{"execution":{"iopub.status.busy":"2022-05-15T05:53:29.679810Z","iopub.execute_input":"2022-05-15T05:53:29.680823Z","iopub.status.idle":"2022-05-15T05:53:32.235825Z","shell.execute_reply.started":"2022-05-15T05:53:29.680767Z","shell.execute_reply":"2022-05-15T05:53:32.235080Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.summary(line_length=100)","metadata":{"execution":{"iopub.status.busy":"2022-05-15T05:53:33.617677Z","iopub.execute_input":"2022-05-15T05:53:33.618578Z","iopub.status.idle":"2022-05-15T05:53:33.633339Z","shell.execute_reply.started":"2022-05-15T05:53:33.618521Z","shell.execute_reply":"2022-05-15T05:53:33.632601Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainingfraction = 1.0\ntrain_size = round(n_samples * trainingfraction)\n\n### Train the network\n# model.fit(BASIC_SEQUENCE[:train_size], NEXT_SEQUENCE[:train_size],  epochs=10, validation_split=0.05)\nt_start = time.time()\nH=model.fit(BASIC_SEQUENCE[:train_size], NEXT_SEQUENCE[:train_size], verbose=2, epochs=10, batch_size=2)\nt_end = time.time()\n\nprint('Training Elapsed time:')\nprint(t_end - t_start)\n\n### Take an example from the test set and predict the next steps\nindex = 3\n\nnum_test_frames = 0 ### Number of frames to predict\n\n\ntrain_pred = BASIC_SEQUENCE[index][:n_frames-num_test_frames:, ::, ::, ::]  ##(track)\nfor j in range(n_frames):\n    t_start = time.time()\n    new_pos = model.predict(train_pred[np.newaxis, :, :, :, :])\n    t_end = time.time()\n    print('Testing Elapsed time:')\n    print(t_end - t_start)\n    new = new_pos[:, -1, :, :, :]\n    train_pred = np.concatenate((train_pred, new), axis=0)\n\n### Compare predictions to the truth\ntruth = BASIC_SEQUENCE[index][:, :, :, :]\n\nfor i in range(n_frames):\n    fig = plt.figure(figsize=(10, 5))\n    ax = fig.add_subplot(122)\n    ### In left panel show original then predicted frames\n    # ax.text(1, 3, 'Predictions', fontsize=20, color='w')\n    plt.text(1, 3, 'Predictions', fontsize=20)\n    pred = train_pred[i, ::, ::, 0]\n    plt.imshow(pred, cmap='binary')\n    plt.savefig('%i_Prediction.png' % (i + 1))\n\nfor i in range(n_frames):   \n    fig = plt.figure(figsize=(10, 5))\n    ax = fig.add_subplot(122)\n    plt.text(1, 3, 'Ground truth', fontsize=20)\n    gt = truth[i, ::, ::, 0]\n    plt.imshow(gt, cmap='binary')\n    plt.savefig('%i_GroundTruth.png' % (i + 1))","metadata":{"execution":{"iopub.status.busy":"2022-05-15T05:53:35.016127Z","iopub.execute_input":"2022-05-15T05:53:35.016657Z","iopub.status.idle":"2022-05-15T05:53:46.488960Z","shell.execute_reply.started":"2022-05-15T05:53:35.016617Z","shell.execute_reply":"2022-05-15T05:53:46.488245Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import math\n\ndef MSE(img1, img2):\n        squared_diff = (img2-img1) ** 2\n        summed = np.sum(squared_diff)\n        num_pix = img1.shape[0] * img1.shape[1] #img1 and 2 should have same shape\n        err = summed / num_pix\n        return err\n\ndef RMSE(img1, img2):\n        squared_diff = (img2-img1) ** 2\n        summed = np.sum(squared_diff)\n        num_pix = img1.shape[0] * img1.shape[1] #img1 and 2 should have same shape\n        err = summed / num_pix\n        err=math.sqrt(err)\n        return err","metadata":{"execution":{"iopub.status.busy":"2022-05-15T05:53:46.922983Z","iopub.execute_input":"2022-05-15T05:53:46.923756Z","iopub.status.idle":"2022-05-15T05:53:46.932164Z","shell.execute_reply.started":"2022-05-15T05:53:46.923711Z","shell.execute_reply":"2022-05-15T05:53:46.931387Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from skimage.metrics import structural_similarity as ssim\n#Load Data\nimageA = cv2.imread('./1_Prediction.png')\nimageB = cv2.imread('./1_GroundTruth.png')\n\n# compute the Structural Similarity Index (SSIM) between the two\n# images, ensuring that the difference image is returned\n(score, diff) = ssim(imageA, imageB, full=True, multichannel=True)\nprint(\"SSIM: {}\".format(score))\n\nmse_score=MSE(imageA,imageB)\n\nrmse_score=RMSE(imageA,imageB)\n\nprint(\"MSE score: {}\".format(mse_score))\nprint(\"RMSE score: {}\".format(rmse_score))\n","metadata":{"execution":{"iopub.status.busy":"2022-05-15T05:53:52.559041Z","iopub.execute_input":"2022-05-15T05:53:52.559317Z","iopub.status.idle":"2022-05-15T05:53:53.104526Z","shell.execute_reply.started":"2022-05-15T05:53:52.559286Z","shell.execute_reply":"2022-05-15T05:53:53.102344Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(H.history['ssim_loss'])\nplt.plot(H.history['mean_squared_error'])\nplt.plot(H.history['root_mean_squared_error'])\nplt.title('Metrics of the Model')\nplt.ylabel('Metrics')\nplt.xlabel('Epochs')\nplt.legend(['ssim_loss','mean_squared_error','root_mean_squared_error'])\n# plt.rcParams[\"figure.figsize\"] = (10,10)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-15T05:53:55.765117Z","iopub.execute_input":"2022-05-15T05:53:55.765579Z","iopub.status.idle":"2022-05-15T05:53:55.990025Z","shell.execute_reply.started":"2022-05-15T05:53:55.765538Z","shell.execute_reply":"2022-05-15T05:53:55.989298Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(H.history['ssim_loss'])\nplt.ylabel('Metrics')\nplt.xlabel('Epochs')\nplt.title('SSIM LOSS')\nplt.legend(['ssim_loss'])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-15T05:54:02.411965Z","iopub.execute_input":"2022-05-15T05:54:02.412252Z","iopub.status.idle":"2022-05-15T05:54:02.606020Z","shell.execute_reply.started":"2022-05-15T05:54:02.412221Z","shell.execute_reply":"2022-05-15T05:54:02.605299Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(H.history['mean_squared_error'])\nplt.ylabel('Metrics')\nplt.xlabel('Epochs')\nplt.title('Mean Squared Error')\nplt.legend(['mean_squared_error'])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-15T05:54:07.381051Z","iopub.execute_input":"2022-05-15T05:54:07.381739Z","iopub.status.idle":"2022-05-15T05:54:07.580261Z","shell.execute_reply.started":"2022-05-15T05:54:07.381697Z","shell.execute_reply":"2022-05-15T05:54:07.579501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(H.history['root_mean_squared_error'])\nplt.ylabel('Metrics')\nplt.xlabel('Epochs')\nplt.title('Root Mean Squared Error')\nplt.legend(['root_mean_squared_error'])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-15T05:54:08.591472Z","iopub.execute_input":"2022-05-15T05:54:08.592170Z","iopub.status.idle":"2022-05-15T05:54:08.785169Z","shell.execute_reply.started":"2022-05-15T05:54:08.592131Z","shell.execute_reply":"2022-05-15T05:54:08.784338Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}