{"cells":[{"metadata":{},"cell_type":"markdown","source":"I strongly beg you to use google to understand GAN models and how they are coded in keras\n\nI am using keras and I have used training data as 28 folders downloaded seperately instead of single big file.\n\nThis code is working but needed few changes\n\n* It works only on AWS, It can also be used on other platforms but modifications are needed\n* * replace the bucket name with your ubucket name from aws\n* * I did not find code to unzip the files. I manually unzipped from my desktop and uploaded them in my bucket\n* * you can find code to unzip files at https://www.kaggle.com/c/deepfake-detection-challenge/discussion/121695\n* I have used Sagemaker and ml.p3.2xlarge instance type and you should contact customer service to do that\n* * It takes lot of time on other instances and it took me probably 2 minutes for 1 video and as tha means 1600*2 for 1600 video and it's ~20 hours\n* improvements can be made to tune hyperameters, accuracy function\n* When I mention Images I mean videos.\n* My unzipped videos are in a folder and the folder is in an s3 bucket and by deafault I used only 10 videos from that folder. You can just change that from for sample in list(tes)[:10]: to for sample in list(tes): and you can train on all videos\n\nThe main thing I want to show through this code that for generating the images by using generative model I am using the fake videos as initial images and training on them. Normally generative model takes latent points as input initially\n\nAfter training on all this videos, we can use the weights generated by descriminitory model to predict the videos as real or fake on test data as the descriminatory model will be intelligent enough after training"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","collapsed":true,"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":false},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        pass#print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n!pip install mtcnn\n!pip install tqdm\n!pip install smart_open\nfrom __future__ import print_function, division\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nfrom keras.datasets import mnist\nfrom keras.layers import Input, Dense, Reshape, Flatten, Dropout\nfrom keras.layers import LSTM\nfrom keras.layers import TimeDistributed\nfrom keras.layers import BatchNormalization, Activation, ZeroPadding2D\nfrom keras.layers import LeakyReLU\nfrom keras.models import Sequential, Model\nfrom keras.optimizers import Adam\nfrom mtcnn import MTCNN\nfrom smart_open import smart_open\nimport matplotlib.pyplot as plt\nimport cv2\nfrom io import BytesIO\nimport numpy as np\nimport boto3\nimport pandas as pd\nfrom sagemaker import get_execution_role\nfrom boto.s3.connection import S3Connection\nfrom matplotlib import pyplot as plt\nfrom PIL import Image\nfrom tqdm.notebook import tqdm\nimport time\n\nimport sys\n\nimport numpy as np\nprint('second')\nclass GAN():\n    def __init__(self):\n        self.img_rows = 16\n        self.img_cols = 16\n        self.channels = 3\n        self.total=299\n        self.img_shape = (self.total,self.img_rows, self.img_cols, self.channels)\n        self.fakeimg_dim=(299,16,16,3)\n        self.latent_dim = self.fakeimg_dim\n        \n\n        # Build and compile the discriminator\n        self.discriminator = self.build_discriminator()\n        self.discriminator.compile(loss='binary_crossentropy',optimizer='Adam'\n            ,metrics=['accuracy'])\n\n        # Build the generator\n        self.generator = self.build_generator()\n        # The generator takes noise as input and generates imgs\n        z = Input(shape=(self.fakeimg_dim))\n        img = self.generator(z)\n\n        print('cf v')\n        \n        print(img.shape)\n        # For the combined model we will only train the generator\n        self.discriminator.trainable = False\n        self.discriminator.compile(loss='binary_crossentropy',optimizer='Adam',metrics=['accuracy'])\n\n\n\n        # The discriminator takes generated images as input and determines validity\n        validity = self.discriminator(img)\n\n        # The combined model  (stacked generator and discriminator)\n        # Trains the generator to fool the discriminator\n        self.combined = Model(z, validity)\n        self.combined.compile(loss='binary_crossentropy',optimizer='Adam')\n\n\n\n    def build_generator(self):\n        model = Sequential()\n\n        model.add(Dense(12, input_shape=self.fakeimg_dim))\n        model.add(LeakyReLU(alpha=0.2))\n        model.add(BatchNormalization(momentum=0.8))\n        model.add(Dense(24))\n        model.add(LeakyReLU(alpha=0.2))\n        model.add(BatchNormalization(momentum=0.8))\n        model.add(Dense(3))\n        model.add(LeakyReLU(alpha=0.2))\n        model.add(BatchNormalization(momentum=0.8))\n        #model.add(Dense(np.prod(self.img_shape), activation='tanh'))\n        #model.add(Reshape(self.img_shape))\n\n        model.summary()\n\n        noise = Input(shape=(self.fakeimg_dim))\n        img = model(noise)\n        \n        \n        return Model(noise, img)\n\n    def build_discriminator(self):\n\n        model = Sequential()\n        # define CNN model\n        # define LSTM model\n\n        model.add(TimeDistributed(Dense(36),input_shape=self.img_shape))\n        model.add(TimeDistributed(Flatten()))\n\n        model.add(LeakyReLU(alpha=0.2))\n\n        model.add(LSTM(24))\n        model.add(Dense(24))\n        model.add(LeakyReLU(alpha=0.2))\n        model.add(Dense(12))\n        model.add(LeakyReLU(alpha=0.2))\n        model.add(Dense(1, activation='sigmoid'))\n        model.summary()\n        img = Input(shape=(self.img_shape))\n        \n        \n        opt = Adam()\n        return Model(img, model(img))\n\n    def train(self, epochs, batch_size=128, sample_interval=50):\n        def plot_faces(images, figsize=(10.8/2, 19.2/2)):\n                    shape = images[0].shape\n                    try:\n                        print('lenimages')\n                        print(len(images))\n                        print(shape)\n                    except:\n                        pass\n                    images = images[np.linspace(0, len(images)-1, 16).astype(int)]\n                    im_plot = []\n                    for i in range(0, 16, 4):\n                        im_plot.append(np.concatenate(images[i:i+4], axis=0))\n                    im_plot = np.concatenate(im_plot, axis=1)\n\n                    fig, ax = plt.subplots(1, 1, figsize=figsize)\n                    try:\n                        print('implotshape')\n                        print(im_plot.shape)\n                    except:\n                        pass\n                    ax.imshow(im_plot)\n                    ax.xaxis.set_visible(False)\n                    ax.yaxis.set_visible(False)\n\n                    ax.grid(False)\n                    fig.tight_layout()\n        def timer(detector, detect_fn, images, *args):\n                    start = time.time()\n                    faces = detect_fn(detector, images, *args)\n                    elapsed = time.time() - start\n                    return faces, elapsed  \n        def sample_images(self, epoch):\n                r, c = 5, 5\n                noise = np.random.normal(0, 1, (r * c, self.latent_dim))\n                gen_imgs = self.generator.predict(noise)\n\n                # Rescale images 0 - 1\n                gen_imgs = 0.5 * gen_imgs + 0.5\n\n                fig, axs = plt.subplots(r, c)\n                cnt = 0\n                for i in range(r):\n                    for j in range(c):\n                        axs[i,j].imshow(gen_imgs[cnt, :,:,0],cmap='gray')\n                        axs[i,j].axis('off')\n                        cnt += 1\n                fig.savefig(\"/nm%d.png\" % epoch)\n                plt.close()\n        def detect_mtcnn(detector, images):\n                    nonlocal faces\n                    faces = []\n                    nonlocal faces2\n                    faces2 = []\n                    oldface=[]\n                    error=[]\n                    final=[]\n                    for image in images:\n                        boxes = detector.detect_faces(image)\n                        try:\n                            box = boxes[0]['box']\n                            face = image[box[1]:box[3]+box[1], box[0]:box[2]+box[0]]\n                            oldface=face\n                            faces.append(face)\n                        except:\n                            try:\n                                faces.append(oldface)\n                            except:\n                                pass\n            \n                    return faces\n        \"\"\"\"The below code is working but difficult to understand as I had written it very badly. \n        I can explain it better here. I am having unzipped video files in a folder and the folder is in s3 bucket.\n        I am initally reading json file from that folder in the bucket and from that json file I am reading a real,fake,real,\n        fake,real,fake .... of all the videos I am sure the video files can be read in many ways but I am just reading that\n        way so that the discriminator and the generator will be reading the similar videos when training. \n        I believe the weights will be robust when the training is not done in order. \n        By not in order I mean when a fake video is sent for generator to train, disriminator should use any other video \n        instead of it's real video.\"\"\"\n        total=[]\n        final=[]\n        import boto3\n        s3 = boto3.resource('s3')\n        my_bucket = s3.Bucket(yourbucketname)\n        li=[]\n        \n        for object_summary in my_bucket.objects.all():\n            l=object_summary.key\n            li.append(l)\n            import zipfile\n\n        x=li\n        #print(x)\n        client = boto3.client('s3')\n        dire2=[]\n        for dire in x: \n            #my_bucket = yourbucketname\n            #print(dire)\n            #resi=client.list_objects(Bucket=my_bucket,Prefix=str(dire).split('/')[0]+'/')\n            #for x in resi.get('Contents', [])[1:]:\n            if str(dire).endswith('.json'):\n                print(dire)\n                break\n            else:\n                continue\n            \n        x1=dict()\n        x=dict()\n        x1['Key']=dire\n        #print(li)\n        my_bucket = yourbucketname\n        df=pd.read_json(smart_open('s3://'+str(my_bucket)+'/'+x1['Key'])).T\n        df=df.iloc[:,[1]].reset_index()\n        df=df.rename(columns={'index':'fake'})\n        tes=[]\n        for a,b in zip(df['fake'],df['original']):\n            if a in [x.split('/')[-1] for x in li[1:]] and b in [x.split('/')[-1] for x in li[1:]]:\n                tes.append(a)\n                tes.append(b)\n        tes=pd.Series(tes).fillna(method='ffill')\n        i=-1\n        print(tes)\n        for sample in list(tes)[:10]:\n            i=i+1\n            if i%2==0:\n                final=[]\n                try:\n                    del old\n\n                except:\n                    pass\n            reader = cv2.VideoCapture('http://'+str(my_bucket)+'.s3.amazonaws.com/'+li[0].split('/')[0]+'/'+sample)\n            images_540_960 = []\n            for i in tqdm(range(int(reader.get(cv2.CAP_PROP_FRAME_COUNT)))):\n                _, image = reader.read()\n                image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n                images_540_960.append(cv2.resize(image, (960, 540)))\n            reader.release()\n            images_540_960 = np.stack(images_540_960)\n\n            try:\n                diff=images_540_960 -old\n            except:\n                old=images_540_960 \n            detector = MTCNN()\n            times_mtcnn = []\n            print('Detecting faces in 540x960 frames', end='')\n            try:\n                        faces2, elapsed2 = timer(detector, detect_mtcnn, old)\n                        final.append(faces2)\n            except:\n                        faces, elapsed = timer(detector, detect_mtcnn, images_540_960)\n                        final.append(faces)\n        final=[[cv2.resize(face, (16, 16)) for face in x] for x in final]\n        final=[np.array(x) for x in final]\n        try:\n            print('printing real faces')\n            plot_faces(np.stack([cv2.resize(face, (16, 16)) for face in final]))\n        except:\n            plot_faces(np.stack([cv2.resize(face, (16, 16)) for face in diff]))\n\n        if final[0].shape[0]>self.total:\n            length=final[0].shape[0]-self.total\n            np.append(final[0],np.zeros(length,16,16,3))\n        else:\n            final[0]=final[0][:self.total,:,:,:]\n        if final[1].shape[0]>self.total:\n            length=final[1].shape[0]-self.total\n            np.append(final[0],np.zeros(length,16,16,3))\n        else:\n            final[1]=final[1][:self.total,:,:,:]\n        try:\n            X_train.extend(final)\n        except:\n            X_train= final\n\n        X_train_num=[X_train[x] for x in range(len(X_train)) if x%2==0]\n        lat_train_num=[X_train[x] for x in range(len(X_train)) if x%2!=0]\n        X_train=np.array(X_train_num)\n        lat_train=np.array(lat_train_num)\n        valid = np.ones((batch_size, 1))\n        fake = np.zeros((batch_size, 1))\n        for epoch in range(epochs):\n            idx = np.random.randint(0, X_train.shape[0], batch_size)\n            imgs = X_train[idx]\n            idx = np.random.randint(0, lat_train.shape[0], batch_size)\n            global noise\n            noise = lat_train[idx]\n            #noise = np.random.normal(0, 1, (batch_size, self.latent_dim))\n            # Generate a batch of new images\n            print('dcsdc')\n            gen_imgs = self.generator.predict(noise)\n            try:\n                model_json = self.generator.to_json()\n                with open(\"model.json\", \"w\") as json_file:\n                    json_file.write(model_json)\n                # serialize weights to HDF5\n                self.generator.save_weights(\"model.h5\")\n            except Exception as e:\n                print(e)\n            \n            # Train the discriminator\n            print('dcs')\n            print(gen_imgs.shape)\n            print(imgs.shape)\n            d_loss_real = self.discriminator.train_on_batch(imgs, valid)\n            d_loss_fake = self.discriminator.train_on_batch(gen_imgs, fake)\n            try:\n                model_json =self.discriminator.to_json()\n                with open(\"model3.json\", \"w\") as json_file:\n                    json_file.write(model_json)\n                # serialize weights to HDF5\n                self.discriminator.save_weights(\"model3.h5\")\n            except Exception as e:\n                print(e)\n            d_loss = 0.5 * np.add(d_loss_real, d_loss_fake)\n\n            # ---------------------\n            #  Train Generator\n            # ---------------------\n\n            #noise = np.random.normal(0, 1, (batch_size, self.latent_dim))\n            idx = np.random.randint(0, lat_train.shape[0], batch_size)\n            noise = lat_train[idx]\n\n            # Train the generator (to have the discriminator label samples as valid)\n            g_loss = self.combined.train_on_batch(noise, valid)\n            \n           \n        \n            \n            try:\n                model_json2 = self.combined.to_json()\n                with open(\"model2.json\", \"w\") as json_file:\n                    json_file.write(model_json2)\n                # serialize weights to HDF5\n                self.combined.save_weights(\"model2.h5\")\n            except Exception as e:\n                print(e)\n\n            # Plot the progress\n            print (\"%d [D loss: %f, acc.: %f] [G loss: %f]\" % (epoch, d_loss[0], d_loss[1], g_loss))\n\n            # If at save interval => save generated image samples\n            if epoch >80:\n                self.save_imgs(epoch)\n                \n    def save_imgs(self, epoch):\n        r, c = 5, 5\n        images = noise\n\n        # Rescale images 0 - 1\n        #images = 0.5 * images + 0.5\n\n        #im_plot = []\n        #for i in range(0, 16, 4):\n            #im_plot.append(np.concatenate(images[i:i+4], axis=0))\n        #im_plot = np.concatenate(im_plot, axis=1)\n\n        fig, ax = plt.subplots(1, 1, figsize=(10.8/2, 19.2/2))\n        print('imagesshape')\n        print(images.shape)\n        ax.imshow(images[0,0,:,:,0],cmap='gray')\n        ax.xaxis.set_visible(False)\n        ax.yaxis.set_visible(False)\n        ax.grid(False)\n        fig.savefig(\"output/mnist_%d.png\" % epoch)\n        #plt.close()\n\n\n\n\nif __name__ == '__main__':\n    gan = GAN()\n    gan.train(epochs=100, batch_size=3, sample_interval=2)\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}