{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"from __future__ import absolute_import, division, print_function\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport cv2\nimport random\nfrom IPython.display import HTML\nimport pandas as pd\nimport numpy as np\nimport base64\nimport math\nfrom __future__ import absolute_import, division, print_function\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport cv2\nimport random\nimport math\nimport tensorflow as tf\nfrom tensorflow import keras\nimport pickle\nimport time\nimport os\nimport itertools\nfrom keras.models import load_model\nimport gc\nfrom keras.datasets import fashion_mnist\nimport matplotlib.pyplot as plt\nfrom keras.utils import np_utils\nfrom keras.models import Sequential\nfrom keras.layers import Dense,Conv2D,Flatten,Dropout,BatchNormalization,MaxPooling2D\nlabels = [i[1][1] for i in pd.read_csv(\"../input/train.csv\").iterrows() if i[1][1] != 'new_whale']\nlabel_set = set(labels)\nlabel_converter = {n[1]:n[0] for n in enumerate(label_set)}\nlabel_converter_back = {n[0]:n[1] for n in enumerate(label_set)}\nimg_train_labels = np.array([label_converter[label] for label in labels])\ntest = [i for i in pd.read_csv(\"../input/train.csv\").iterrows() if i[1]['Id'] != 'new_whale']\nnum_labels = len(label_set)\ndef gen(start,end, batch_size): \n        for n in range((end-start)//batch_size):\n                images = []\n                labels = []\n                for _ in test[start:end][n*batch_size: batch_size+batch_size*n]: \n                        img , label = _[1][0] , _[1][1]\n                        label =  label_converter[label]\n                        image = cv2.resize(cv2.imread(\"../input/train/\"+img,0), (992,512)) \n                        images.append(image)\n                        labels.append(label)\n                yield np.expand_dims(np.array(images),axis=3) , np_utils.to_categorical(labels,num_labels)\ndef gen_rand(num):\n    images = []\n    labels = []\n    for i in range(num):\n        randindx = random.randint(1,len(test)-1)\n        img = test[randindx][1]['Image']\n        label = test[randindx][1]['Id']\n        label =  label_converter[label]\n        image = cv2.resize(cv2.imread(\"../input/train/\"+img,0), (992,512))\n        images.append(image)\n        labels.append(label)\n    yield np.expand_dims(np.array(images),axis=3) , np_utils.to_categorical(labels,num_labels)\n  \ndef gen_rand_batch(num,batch_size):\n    for i in range(num):\n        yield next(gen_rand(batch_size))\ntest_imgs= os.listdir('../input/test')\n\ndef gen_test(start, stop):\n    num = stop - start\n    images = []\n    img_name = []\n    test_imgs= os.listdir('../input/test')[start:stop]\n    for i in range(num):\n        img = test_imgs[i]\n        img_name.append(img)\n        image =  cv2.resize(cv2.imread(\"../input/test/\"+img,0), (992,512))\n        images.append(image)\n    yield np.expand_dims(np.array(images),axis=3) , img_name\n  \n\ndef gen_test_batch(num,batch_size):\n    for i in range(num/batch): \n        yield next(gen_test(i*batch_size,(i+1)*(batch_size)))\n        \ndef format_predictions(pred):\n        p = np.argsort(pred)[::-1][:5]\n        return ' '.join([(lambda x: label_converter_back[x])(x) for x in p])","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"model = Sequential([ \n    Conv2D(64 , kernel_size=(5,5), strides=(5,5),input_shape=(512,992,1)),\n    MaxPooling2D(pool_size=(2, 2)),\n    BatchNormalization(),\n    Conv2D(32 , kernel_size=(4,4), strides=(4,4)),\n    MaxPooling2D(pool_size=(2, 2)),\n    BatchNormalization(),\n    Conv2D(16 , kernel_size=(3,3), strides=(3,3)),\n    MaxPooling2D(pool_size=(2, 2)),\n    BatchNormalization(),\n    Flatten(),\n    Dense(100,activation='relu'),\n    Dense(1152,activation='softsign'),\n    Dense(num_labels, activation='softmax')\n])\n\n\nmodel.compile(loss='categorical_crossentropy',optimizer='Nadam', metrics=['categorical_accuracy'])\n#model.summary()\n\ntrain_num = 1500\nbatch = 100\n#test_batch = next(gen_rand(100))\nstart = time.time()\nodel = load_model('whale_identifier.h5')\nfor i in range(2):\n    model.save('whale_identifier.h5') \n    model.fit_generator(gen_rand_batch(train_num,batch), validation_data=None,steps_per_epoch=train_num//batch)\nend = time.time()\nprint(end - start)\n#model.fit_generator(gen_rand_batch(train_num,batch), validation_data=None,steps_per_epoch=train_num//batch)\n#model.save('whale_identifier.h5') \n\n#del model \nmodel = load_model('whale_identifier.h5')\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"46f2461d545e605b7a584a01be6428049ba40177"},"cell_type":"code","source":"\n# tot//batch -- > 7960//40  for optimal full batch run\ndef model_to_kaggle(num,batch):\n    #os.system('touch eggs.csv')\n    g = np.empty(shape=(0, 512, 992, 1))\n    files = []\n    batch_predictions = lambda P :[format_predictions(p) for p in P]\n    with open('submission.csv', 'w', newline='') as csvfile:\n        fieldnames = ['Image','Id']\n        header_writer = csv.DictWriter(csvfile, fieldnames=[\"Image\", \"Id\"])\n        header_writer.writeheader()\n        writer = csv.writer(csvfile, quoting=csv.QUOTE_MINIMAL)\n        for i in range(num//batch):         \n            for load in gen_test(i*batch,i*batch+batch):\n                g = np.empty(shape=(0, 512, 992, 1))\n                g = load[0]\n                preds = batch_predictions(model.predict(g))\n                for z in zip(load[1] , preds):\n                    writer.writerow([z[0]] + [z[1]])\n\n        pass\nstart = time.time()\nmodel_to_kaggle(7960,40)\nend = time.time()\nprint('Ran in {} [sec]'.format(end-start))\npd.read_csv('submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6ac523fa52149b0433e728a4b14c49d13a5824ec"},"cell_type":"code","source":"def create_download_link(df, title = \"Download CSV file\", filename = \"submission.csv\"):  \n    csv = df.to_csv(index=False)\n    b64 = base64.b64encode(csv.encode())\n    payload = b64.decode()\n    html = '<a download=\"{filename}\" href=\"data:text/csv;base64,{payload}\" target=\"_blank\">{title}</a>'\n    html = html.format(payload=payload,title=title,filename=filename)\n    return HTML(html)\n\ncreate_download_link(pd.read_csv('submission.csv'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"18c337ae5ff7f06e3c4f1492872341e721f108e3"},"cell_type":"code","source":"def create_download_link(df, title = \"Download CSV file\", filename = \"submission.csv\"):  \n    csv = df.to_csv(index=False)\n    b64 = base64.b64encode(csv.encode())\n    payload = b64.decode()\n    html = '<a download=\"{filename}\" href=\"data:text/csv;base64,{payload}\" target=\"_blank\">{title}</a>'\n    html = html.format(payload=payload,title=title,filename=filename)\n    return HTML(html)\n\ncreate_download_link(d)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6b51e6700b4a4a1bfc1b90a92560006b1ea5d81a"},"cell_type":"code","source":"\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"08fd3abb6331a614833a9bba400acad8bf7027bc"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"28537ce23864f01cba02781f13e88c2bb901d711"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"375a63b8efa41c643066a20c29a52fbe568c70d2"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d5ec51b2721ea13d9f7a5d71b543e4c06e254943"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"520cd0e1b5468c53e5d607af37e70b43c02c3714"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8871dd70e5399134b90e09ba63cda27dbc7c81a4"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5a7a2f971fb05480472d311b77b7e5d8108103af"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6edc4d0a22ea54ee7e61baefb3c575152174ab94"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b987e8fac76962f0b4a372e4ad3de900b32346f4"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3a580d1b7a6c2a801eab92806969ddc39123e32a"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8969c855cf2cccbfd782f2044dfef05392c75a33"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}