{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nimport cv2\nimport tensorflow as tf\nimport matplotlib.pyplot as plt\nfrom PIL import Image\nimport albumentations as A\n\nfrom sklearn.metrics import f1_score, classification_report\nimport pickle\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.preprocessing import OneHotEncoder, LabelEncoder\nfrom numpy import array\nfrom random import shuffle, seed","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","id":"dcO2IxLbiqQY","execution":{"iopub.status.busy":"2021-07-06T12:37:38.724584Z","iopub.execute_input":"2021-07-06T12:37:38.724978Z","iopub.status.idle":"2021-07-06T12:37:46.281979Z","shell.execute_reply.started":"2021-07-06T12:37:38.724902Z","shell.execute_reply":"2021-07-06T12:37:46.281131Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"list_paths = []\nfor subdir, dirs, files in os.walk(\"../input\"):\n    for file in files:\n        filepath = subdir + os.sep + file\n        list_paths.append(filepath)\n        \nlist_train = [filepath for filepath in list_paths if \"train/\" in filepath]\nseed(420)\nshuffle(list_train)\nlist_test = [filepath for filepath in list_paths if \"test/\" in filepath]","metadata":{"id":"sBo5sya-iqQc","execution":{"iopub.status.busy":"2021-07-06T12:37:46.283468Z","iopub.execute_input":"2021-07-06T12:37:46.283793Z","iopub.status.idle":"2021-07-06T12:37:48.955043Z","shell.execute_reply.started":"2021-07-06T12:37:46.283755Z","shell.execute_reply":"2021-07-06T12:37:48.954027Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"list_train[:5]","metadata":{"id":"jBtM1tHSiqQd","outputId":"a065480b-8b49-46da-fca4-0634b3b4dfcf","execution":{"iopub.status.busy":"2021-07-06T12:37:48.960336Z","iopub.execute_input":"2021-07-06T12:37:48.962541Z","iopub.status.idle":"2021-07-06T12:37:48.974782Z","shell.execute_reply.started":"2021-07-06T12:37:48.962450Z","shell.execute_reply":"2021-07-06T12:37:48.974026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_class_from_path(filepath):\n    return os.path.dirname(filepath).split(os.sep)[-1]","metadata":{"id":"6oa-5IQtiqQf","execution":{"iopub.status.busy":"2021-07-06T12:37:48.979306Z","iopub.execute_input":"2021-07-06T12:37:48.981671Z","iopub.status.idle":"2021-07-06T12:37:48.987547Z","shell.execute_reply.started":"2021-07-06T12:37:48.981633Z","shell.execute_reply":"2021-07-06T12:37:48.986804Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels = [get_class_from_path(filepath) for filepath in list_train]","metadata":{"id":"ThwZsJumiqQf","execution":{"iopub.status.busy":"2021-07-06T12:37:48.991521Z","iopub.execute_input":"2021-07-06T12:37:48.993993Z","iopub.status.idle":"2021-07-06T12:37:49.009484Z","shell.execute_reply.started":"2021-07-06T12:37:48.993956Z","shell.execute_reply":"2021-07-06T12:37:49.008617Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels[:5]","metadata":{"id":"HD66I9YLiqQf","outputId":"122acab9-fd91-41b9-dcea-4a7388e9335e","execution":{"iopub.status.busy":"2021-07-06T12:37:49.014067Z","iopub.execute_input":"2021-07-06T12:37:49.016220Z","iopub.status.idle":"2021-07-06T12:37:49.025528Z","shell.execute_reply.started":"2021-07-06T12:37:49.016161Z","shell.execute_reply":"2021-07-06T12:37:49.024535Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = pd.DataFrame(labels, columns=['class'])\ntrain_data['path'] = list_train\ntrain_data.head()","metadata":{"id":"l9eW-fUviqQg","outputId":"52944f46-c1f0-4718-96e3-bac1fe179949","execution":{"iopub.status.busy":"2021-07-06T12:37:49.030431Z","iopub.execute_input":"2021-07-06T12:37:49.033174Z","iopub.status.idle":"2021-07-06T12:37:49.065904Z","shell.execute_reply.started":"2021-07-06T12:37:49.033134Z","shell.execute_reply":"2021-07-06T12:37:49.065209Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def open_image(img_path):\n#     image = cv2.imread(img_path)\n# #     array_img = np.array(image)\n#     return image\n\n# def crop(img):\n#     width, height = img.size  # Get dimensions\n\n#     left = (width - 112) / 2\n#     top = (height - 112) / 2\n#     right = (width + 112) / 2\n#     bottom = (height + 112) / 2\n\n#     return img.crop((left, top, right, bottom))\n\n\n# def gamma_correction(array_img, gamma):\n#     invGamma = 1.0 / gamma\n#     table = np.array([((i / 255.0) ** invGamma) * 255 for i in np.arange(0, 256)]).astype(\"uint8\")\n\n#     return cv2.LUT(array_img, table)\n\n\n# def jpg_compression(array, quality):\n#     img = Image.fromarray(array)\n#     img.save('img.jpg', \"JPEG\", quality=quality)\n#     return cv2.cvtColor(cv2.imread('img.jpg'), cv2.COLOR_BGR2RGB)\n\n\n# def resizing(array_img, factor):\n#     h, w, ch = array_img.shape\n#     return cv2.resize(array_img, (int(factor * w), int(factor * h)), interpolation=cv2.INTER_CUBIC)","metadata":{"id":"KNCnfdNHiqQh","execution":{"iopub.status.busy":"2021-07-06T12:37:49.071365Z","iopub.execute_input":"2021-07-06T12:37:49.073673Z","iopub.status.idle":"2021-07-06T12:37:49.080023Z","shell.execute_reply.started":"2021-07-06T12:37:49.073602Z","shell.execute_reply":"2021-07-06T12:37:49.078976Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data['class'].value_counts().sort_values().plot(kind='bar')","metadata":{"id":"MORnNa2JiqQi","outputId":"873a62bf-4d96-4dbb-e0dc-8060e83078cf","execution":{"iopub.status.busy":"2021-07-06T12:37:49.085676Z","iopub.execute_input":"2021-07-06T12:37:49.088609Z","iopub.status.idle":"2021-07-06T12:37:49.352927Z","shell.execute_reply.started":"2021-07-06T12:37:49.088573Z","shell.execute_reply":"2021-07-06T12:37:49.352048Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = train_data['path']\ny = train_data['class']","metadata":{"id":"UmClv0eQiqQi","execution":{"iopub.status.busy":"2021-07-06T12:37:49.354279Z","iopub.execute_input":"2021-07-06T12:37:49.354608Z","iopub.status.idle":"2021-07-06T12:37:49.360930Z","shell.execute_reply.started":"2021-07-06T12:37:49.354574Z","shell.execute_reply":"2021-07-06T12:37:49.360237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"splitter = StratifiedKFold(n_splits=5, random_state=42, shuffle=True)\nsplits = list(splitter.split(X=X, y=y))","metadata":{"id":"M60WPl4SiqQj","execution":{"iopub.status.busy":"2021-07-06T12:37:49.363170Z","iopub.execute_input":"2021-07-06T12:37:49.363414Z","iopub.status.idle":"2021-07-06T12:37:49.379204Z","shell.execute_reply.started":"2021-07-06T12:37:49.363392Z","shell.execute_reply":"2021-07-06T12:37:49.378345Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y.shape","metadata":{"id":"yP5qjuQTiqQj","outputId":"01cc4295-17a6-4876-fc64-8c17c4dd6257","execution":{"iopub.status.busy":"2021-07-06T12:37:49.382089Z","iopub.execute_input":"2021-07-06T12:37:49.382391Z","iopub.status.idle":"2021-07-06T12:37:49.389170Z","shell.execute_reply.started":"2021-07-06T12:37:49.382362Z","shell.execute_reply":"2021-07-06T12:37:49.388393Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y = pd.get_dummies(y)\ny.head()","metadata":{"id":"YCiaU66DiqQk","outputId":"4c00d237-6325-486b-df7a-999707a4c04e","execution":{"iopub.status.busy":"2021-07-06T12:37:49.390536Z","iopub.execute_input":"2021-07-06T12:37:49.390967Z","iopub.status.idle":"2021-07-06T12:37:49.409408Z","shell.execute_reply.started":"2021-07-06T12:37:49.390907Z","shell.execute_reply":"2021-07-06T12:37:49.408489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dict_map = {\"0\": \"HTC-1-M7\", \n            \"1\": \"LG-Nexus-5x\", \n            \"2\": \"Motorola-Droid-Maxx\", \n            \"3\": \"Motorola-Nexus-6\", \n            \"4\": \"Motorola-X\", \n            \"5\": \"Samsung-Galaxy-Note3\",\n            \"6\": \"Samsung-Galaxy-S4\",\n            \"7\": \"Sony-NEX-7\",\n            \"8\": \"iPhone-4s\",\n            \"9\": \"iPhone-6\"}","metadata":{"id":"l7n6ZreuiqQk","execution":{"iopub.status.busy":"2021-07-06T12:37:49.410762Z","iopub.execute_input":"2021-07-06T12:37:49.411105Z","iopub.status.idle":"2021-07-06T12:37:49.415533Z","shell.execute_reply.started":"2021-07-06T12:37:49.411057Z","shell.execute_reply":"2021-07-06T12:37:49.414719Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"id":"5-w86rRNiqQk","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def open_image(img_path):\n    image = cv2.imread(img_path)\n#     array_img = np.array(image)\n    return image","metadata":{"id":"2CCRFsD5iqQl","execution":{"iopub.status.busy":"2021-07-06T12:37:49.417029Z","iopub.execute_input":"2021-07-06T12:37:49.417714Z","iopub.status.idle":"2021-07-06T12:37:49.424878Z","shell.execute_reply.started":"2021-07-06T12:37:49.417677Z","shell.execute_reply":"2021-07-06T12:37:49.424199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def preprocess(images):\n    return (images / 127.5) - 1.0","metadata":{"id":"bT_Ov_EWiqQl","execution":{"iopub.status.busy":"2021-07-06T12:37:49.427918Z","iopub.execute_input":"2021-07-06T12:37:49.428238Z","iopub.status.idle":"2021-07-06T12:37:49.433808Z","shell.execute_reply.started":"2021-07-06T12:37:49.428208Z","shell.execute_reply":"2021-07-06T12:37:49.432947Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SHAPE = 512\n\ntrain_augmentations = A.Compose([#A.RGBShift(),\n                                 A.RandomCrop(height=SHAPE,width=SHAPE),\n                                 A.RandomGamma(gamma_limit=(80, 120), p=0.8),\n                                #  A.Blur(),\n                                #  A.GaussNoise(),\n                                 A.JpegCompression(quality_lower=70, quality_upper=90, p=0.9),\n                                 A.GridDistortion(interpolation=cv2.INTER_CUBIC, p=0.9)\n                                 ])\n\nteste_augmentations = A.Compose([#A.RGBShift(),\n                                 A.CenterCrop(height=SHAPE,width=SHAPE)\n#                                  A.RandomGamma(),\n                                #  A.Blur(),\n                                #  A.GaussNoise(),\n#                                  A.JpegCompression(quality_lower=70, quality_upper=100, p=0.5),\n#                                  A.GridDistortion(interpolation=cv2.INTER_CUBIC, p=0.5)                                 \n                                 ])","metadata":{"id":"vNXtb6FUiqQl","execution":{"iopub.status.busy":"2021-07-06T12:37:49.434843Z","iopub.execute_input":"2021-07-06T12:37:49.435303Z","iopub.status.idle":"2021-07-06T12:37:49.443971Z","shell.execute_reply.started":"2021-07-06T12:37:49.435270Z","shell.execute_reply":"2021-07-06T12:37:49.443245Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"id":"iKvJdL3XiqQm"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CameraDataset(tf.keras.utils.Sequence):\n    def __init__(self, X_set, y_set, batch_size, augmenter=None, test=False, *args, **kwargs):\n        \n        self.batch_size = batch_size\n        self.x_set = X_set\n        self.y_set = y_set\n        self.test = test\n        self.augmenter = augmenter\n        \n    def __len__(self):\n        return int(len(self.x_set) / self.batch_size)\n    \n    \n    def __getitem__(self, index):\n        X = self.x_set[index * self.batch_size : (index + 1) * self.batch_size]        \n        y = self.y_set[index * self.batch_size : (index + 1) * self.batch_size]\n        \n        X = [(self.augmenter(image=open_image(x))['image']) for x in X]\n        \n        if self.test:\n            return np.array(X)\n        \n        return np.array(X), y.values","metadata":{"id":"3ppMFlmiiqQm","execution":{"iopub.status.busy":"2021-07-06T12:37:49.445119Z","iopub.execute_input":"2021-07-06T12:37:49.445608Z","iopub.status.idle":"2021-07-06T12:37:49.457989Z","shell.execute_reply.started":"2021-07-06T12:37:49.445532Z","shell.execute_reply":"2021-07-06T12:37:49.457253Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset = CameraDataset(X, y, batch_size=8, augmenter=train_augmentations)","metadata":{"id":"7_5V1Nj6iqQn","execution":{"iopub.status.busy":"2021-07-06T12:37:49.459021Z","iopub.execute_input":"2021-07-06T12:37:49.459682Z","iopub.status.idle":"2021-07-06T12:37:49.465910Z","shell.execute_reply.started":"2021-07-06T12:37:49.459649Z","shell.execute_reply":"2021-07-06T12:37:49.465220Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_set, y_set = train_dataset.__getitem__(50)","metadata":{"id":"JI3nF2RHiqQo","execution":{"iopub.status.busy":"2021-07-06T12:37:49.468086Z","iopub.execute_input":"2021-07-06T12:37:49.468647Z","iopub.status.idle":"2021-07-06T12:37:51.077590Z","shell.execute_reply.started":"2021-07-06T12:37:49.468614Z","shell.execute_reply":"2021-07-06T12:37:51.076754Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_set[0].shape","metadata":{"id":"6hF83IR9iqQo","outputId":"adc9a688-94d4-4b9c-f7f6-41eef75e2789","execution":{"iopub.status.busy":"2021-07-06T12:37:51.079969Z","iopub.execute_input":"2021-07-06T12:37:51.080532Z","iopub.status.idle":"2021-07-06T12:37:51.086000Z","shell.execute_reply.started":"2021-07-06T12:37:51.080492Z","shell.execute_reply":"2021-07-06T12:37:51.085233Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_set[0]","metadata":{"id":"-sekgQXoiqQo","outputId":"f9bca4ec-040c-48ff-e44f-1cc712bb6714","execution":{"iopub.status.busy":"2021-07-06T12:37:51.087334Z","iopub.execute_input":"2021-07-06T12:37:51.087790Z","iopub.status.idle":"2021-07-06T12:37:51.099640Z","shell.execute_reply.started":"2021-07-06T12:37:51.087755Z","shell.execute_reply":"2021-07-06T12:37:51.098791Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(x_set[3])","metadata":{"id":"jRosfbPXiqQp","outputId":"7bc65ed1-044b-4d8b-b9a4-6cad6e6bf325","execution":{"iopub.status.busy":"2021-07-06T12:37:51.104565Z","iopub.execute_input":"2021-07-06T12:37:51.104821Z","iopub.status.idle":"2021-07-06T12:37:51.270937Z","shell.execute_reply.started":"2021-07-06T12:37:51.104798Z","shell.execute_reply":"2021-07-06T12:37:51.270163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def build_model():\n#     inputs =tf.keras.layers.Input(shape=(112,112,3))    \n#     model = tf.keras.applications.EfficientNetB0(include_top=False, input_tensor=inputs, weights=\"imagenet\", classes=10)\n#     model.trainable = True    \n#     x = tf.keras.layers.GlobalAveragePooling2D(name=\"avg_pool\")(model.output)\n#     x = tf.keras.layers.BatchNormalization()(x)\n#     top_dropout_rate = 0.5\n#     x = tf.keras.layers.Dropout(top_dropout_rate)(x)\n#     x = tf.keras.layers.Dense(512, activation='relu')(x)\n#     x = tf.keras.layers.Dropout(top_dropout_rate)(x)\n#     x = tf.keras.layers.BatchNormalization()(x)\n#     x = tf.keras.layers.Dropout(top_dropout_rate)(x)\n#     x = tf.keras.layers.Dense(512, activation='relu')(x)\n#     x = tf.keras.layers.BatchNormalization()(x)\n#     x = tf.keras.layers.Flatten()(x)    \n#     outputs = tf.keras.layers.Dense(10, activation='softmax')(x)    \n#     model =tf.keras.Model(inputs=inputs, outputs=outputs)    \n#     model.compile(optimizer=tf.keras.optimizers.Adam(lr=1e-4), loss='categorical_crossentropy', metrics=['accuracy'])\n#     model.summary()\n    #-------------------------------------------------------------------\n    base_model = tf.keras.applications.DenseNet201(weights='imagenet', include_top=False, input_shape=[SHAPE, SHAPE, 3])\n    base_model.trainable = True\n    inputs =tf.keras.layers.Input(shape=(SHAPE,SHAPE,3))\n    x = tf.keras.applications.densenet.preprocess_input(inputs)\n    x = base_model(x, training=True)\n    x = tf.keras.layers.GlobalAveragePooling2D()(x)\n    x = tf.keras.layers.Dense(64, activation='relu')(x)\n    x = tf.keras.layers.Dropout(0.3)(x)\n    x = tf.keras.layers.Dense(32, activation='relu')(x)\n    x = tf.keras.layers.Dropout(0.3)(x)\n    outputs = tf.keras.layers.Dense(10, activation='softmax')(x)\n\n    model =tf.keras.Model(inputs=inputs, outputs=outputs)\n\n    model.compile(optimizer=tf.keras.optimizers.Adam(lr=1e-4), loss='categorical_crossentropy', metrics=['accuracy'])\n\n    model.summary() \n    return model","metadata":{"id":"Aywspy-liqQp","execution":{"iopub.status.busy":"2021-07-06T12:37:51.272669Z","iopub.execute_input":"2021-07-06T12:37:51.273237Z","iopub.status.idle":"2021-07-06T12:37:51.282909Z","shell.execute_reply.started":"2021-07-06T12:37:51.273200Z","shell.execute_reply":"2021-07-06T12:37:51.282131Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# model = build_model()","metadata":{"id":"eHphXy6viqQq","execution":{"iopub.status.busy":"2021-07-06T12:37:51.283998Z","iopub.execute_input":"2021-07-06T12:37:51.284479Z","iopub.status.idle":"2021-07-06T12:37:51.292003Z","shell.execute_reply.started":"2021-07-06T12:37:51.284441Z","shell.execute_reply":"2021-07-06T12:37:51.291270Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def get_callbacks_list(fold):\n    \n#     path = f\"output/fold{fold}/\"\n    \n#     checkpoint = tf.keras.callbacks.ModelCheckpoint(filepath=path + \"weights.h5\", monitor=\"val_accuracy\", save_best_only=True, mode='max')\n    \n#     reduce_lr = tf.keras.callbacks.ReduceLROnPlateau(monitor=\"val_accuracy\", factor=0.9, patience=2, min_lr=1e-6, mode=\"max\", verbose=True)\n    \n#     early_stopping = tf.keras.callbacks.EarlyStopping(monitor=\"val_accuracy\", patience=5, mode=\"max\", verbose=True)\n    \n    \n#     return [checkpoint, reduce_lr, early_stopping]","metadata":{"id":"KfW25Su1iqQq","execution":{"iopub.status.busy":"2021-07-06T12:37:51.293351Z","iopub.execute_input":"2021-07-06T12:37:51.293726Z","iopub.status.idle":"2021-07-06T12:37:51.304574Z","shell.execute_reply.started":"2021-07-06T12:37:51.293689Z","shell.execute_reply":"2021-07-06T12:37:51.303642Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# val_f1 = []\n# import gc\n\n\n# for n_fold, (t_idx, v_idx) in enumerate(splits):\n#     #if n_fold < 1: treinar por só uma época\n# #     X_train = [list_train[x] for x in t_idx]\n# #     y_train = [y[x] for x in t_idx]\n# #     X_val = [list_train[x] for x in v_idx]\n# #     y_val = [y[x] for x in v_idx]\n    \n#     X_train = X[X.index.isin(t_idx)]\n#     y_train = y[y.index.isin(t_idx)]\n#     X_val = X[X.index.isin(v_idx)]\n#     y_val = y[y.index.isin(v_idx)]\n    \n\n#     train_dataset = CameraDataset(X_train, y_train, batch_size=32,augmenter=train_augmentations)\n#     val_dataset = CameraDataset(X_val, y_val, batch_size=32,augmenter=teste_augmentations)\n    \n#     callbacks_list = get_callbacks_list(n_fold)\n    \n#     model = build_model()\n    \n#     history = model.fit(train_dataset, validation_data=val_dataset, epochs=1, callbacks=callbacks_list)\n    \n#     model.load_weights(f\"output/fold{n_fold}/weights.h5\")\n    \n#     x_test = CameraDataset(X_val, y_val, batch_size=32, test=True, augmenter=teste_augmentations)\n    \n#     y_pred = model.predict(x_test)\n    \n#     del model\n#     gc.collect()\n#     f1score = f1_score(y_val.values.argmax(axis=1), y_pred.argmax(axis=1), average='micro')\n#     print(classification_report(y_val.values.argmax(axis=1), y_pred.argmax(axis=1), digits=3))\n    \n#     val_f1.append(f1score)\n    \n# print(f\"Final f1 score: {np.mean(val_f1)}\")","metadata":{"id":"WBy5rxbUiqQq","execution":{"iopub.status.busy":"2021-07-06T12:37:51.307779Z","iopub.execute_input":"2021-07-06T12:37:51.308117Z","iopub.status.idle":"2021-07-06T12:37:51.313982Z","shell.execute_reply.started":"2021-07-06T12:37:51.308089Z","shell.execute_reply":"2021-07-06T12:37:51.313209Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42, shuffle=True, stratify=y)\n\ntrain_dataset = CameraDataset(X_train, y_train, batch_size=10,augmenter=train_augmentations)\nval_dataset = CameraDataset(X_test, y_test, batch_size=10,augmenter=teste_augmentations)","metadata":{"id":"4xJs9aO3oNh4","execution":{"iopub.status.busy":"2021-07-06T12:37:51.316102Z","iopub.execute_input":"2021-07-06T12:37:51.316345Z","iopub.status.idle":"2021-07-06T12:37:51.376206Z","shell.execute_reply.started":"2021-07-06T12:37:51.316322Z","shell.execute_reply":"2021-07-06T12:37:51.375480Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = build_model()","metadata":{"id":"iWk54FjwO_OB","execution":{"iopub.status.busy":"2021-07-06T12:37:51.377372Z","iopub.execute_input":"2021-07-06T12:37:51.377722Z","iopub.status.idle":"2021-07-06T12:37:59.535671Z","shell.execute_reply.started":"2021-07-06T12:37:51.377687Z","shell.execute_reply":"2021-07-06T12:37:59.534784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"file_path = \"output/weights.best.hdf5\"\nn = 0\n\ncheckpoint = tf.keras.callbacks.ModelCheckpoint(file_path, monitor=\"val_accuracy\", save_best_only=True, mode='max')\n\nreduce_lr = tf.keras.callbacks.ReduceLROnPlateau(monitor=\"val_accuracy\", factor=0.9, patience=2, min_lr=1e-6, mode=\"max\", verbose=True)\n\nearly_stopping = tf.keras.callbacks.EarlyStopping(monitor=\"val_accuracy\", patience=5, mode=\"max\", verbose=True)\n\ncallbacks_list = [checkpoint, reduce_lr, early_stopping]\n\n# model = tf.keras.models.load_model(file_path)\n\nhistory = model.fit(train_dataset, validation_data=val_dataset, epochs=10, callbacks=callbacks_list)\n\nwhile(n<2):\n  print(n)\n  if n>0:\n    model = tf.keras.models.load_model(file_path)\n\n  n += 1\n\n  history = model.fit(train_dataset, validation_data=val_dataset, epochs=10, batch_size=10, callbacks=callbacks_list)","metadata":{"id":"IE3mrj0uqGHw","outputId":"790eea60-c159-42d3-bbd3-b0aab1eb5505","execution":{"iopub.status.busy":"2021-07-06T12:37:59.536951Z","iopub.execute_input":"2021-07-06T12:37:59.537288Z","iopub.status.idle":"2021-07-06T17:40:31.040851Z","shell.execute_reply.started":"2021-07-06T12:37:59.537250Z","shell.execute_reply":"2021-07-06T17:40:31.035315Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission = pd.read_csv('../input/sp-society-camera-model-identification/sample_submission.csv')\nsample_submission.head()","metadata":{"id":"q7rIVBen5L4r","outputId":"8ccd3617-78ff-4935-c148-21b98986cd1c","execution":{"iopub.status.busy":"2021-07-06T17:40:31.050832Z","iopub.execute_input":"2021-07-06T17:40:31.052215Z","iopub.status.idle":"2021-07-06T17:40:31.132400Z","shell.execute_reply.started":"2021-07-06T17:40:31.052177Z","shell.execute_reply":"2021-07-06T17:40:31.131611Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test = '../input/sp-society-camera-model-identification/test/test/' + sample_submission['fname']","metadata":{"id":"I71hhaDd54V1","execution":{"iopub.status.busy":"2021-07-06T17:40:31.133580Z","iopub.execute_input":"2021-07-06T17:40:31.133916Z","iopub.status.idle":"2021-07-06T17:40:31.475818Z","shell.execute_reply.started":"2021-07-06T17:40:31.133882Z","shell.execute_reply":"2021-07-06T17:40:31.475049Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test[:5]","metadata":{"id":"u7VUDzqAuKVJ","outputId":"a6385d4b-75a5-426f-9c84-115532de7d3f","execution":{"iopub.status.busy":"2021-07-06T17:40:31.477102Z","iopub.execute_input":"2021-07-06T17:40:31.477439Z","iopub.status.idle":"2021-07-06T17:40:31.485009Z","shell.execute_reply.started":"2021-07-06T17:40:31.477404Z","shell.execute_reply":"2021-07-06T17:40:31.484096Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_and_resize(filepath):\n    im_array = np.array(Image.open((filepath)), dtype=\"uint8\")\n    pil_im = Image.fromarray(im_array)\n    new_array = np.array(pil_im)\n    return new_array","metadata":{"id":"t6icVzxS6rG4","execution":{"iopub.status.busy":"2021-07-06T17:40:31.486664Z","iopub.execute_input":"2021-07-06T17:40:31.487380Z","iopub.status.idle":"2021-07-06T17:40:31.492598Z","shell.execute_reply.started":"2021-07-06T17:40:31.487343Z","shell.execute_reply":"2021-07-06T17:40:31.491769Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def crop(img):\n    width, height = img.size  # Get dimensions\n\n    left = (width - 224) / 2\n    top = (height - 224) / 2\n    right = (width + 224) / 2\n    bottom = (height + 224) / 2\n\n    return img.crop((left, top, right, bottom))","metadata":{"id":"JnXIbHOC6wK1","execution":{"iopub.status.busy":"2021-07-06T17:40:31.493735Z","iopub.execute_input":"2021-07-06T17:40:31.494251Z","iopub.status.idle":"2021-07-06T17:40:31.501797Z","shell.execute_reply.started":"2021-07-06T17:40:31.494217Z","shell.execute_reply":"2021-07-06T17:40:31.500976Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"file_path = \"output/weights.best.hdf5\"\nmodel = tf.keras.models.load_model(file_path)\nX_test = np.array([read_and_resize(filepath) for filepath in X_test])\npred_mean = model.predict(X_test)","metadata":{"id":"ighrzBmHqaFC","execution":{"iopub.status.busy":"2021-07-06T17:40:31.503134Z","iopub.execute_input":"2021-07-06T17:40:31.503554Z","iopub.status.idle":"2021-07-06T17:42:35.075195Z","shell.execute_reply.started":"2021-07-06T17:40:31.503519Z","shell.execute_reply":"2021-07-06T17:42:35.073303Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels_test=[]\nfor item in pred_mean.argmax(axis=1):\n    labels_test.append(dict_map[str(item)])","metadata":{"id":"p-JoHZcTqVfS","execution":{"iopub.status.busy":"2021-07-06T17:42:35.079699Z","iopub.execute_input":"2021-07-06T17:42:35.079959Z","iopub.status.idle":"2021-07-06T17:42:35.090460Z","shell.execute_reply.started":"2021-07-06T17:42:35.079933Z","shell.execute_reply":"2021-07-06T17:42:35.089725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_mean.argmax(axis=1)[:10]","metadata":{"id":"xoTqrRQUlZGN","outputId":"fd12fbaa-d0f2-46ae-980c-cc27b4163df7","execution":{"iopub.status.busy":"2021-07-06T17:42:35.091880Z","iopub.execute_input":"2021-07-06T17:42:35.092469Z","iopub.status.idle":"2021-07-06T17:42:35.100416Z","shell.execute_reply.started":"2021-07-06T17:42:35.092433Z","shell.execute_reply":"2021-07-06T17:42:35.099370Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels_test[:10]","metadata":{"id":"gQh7XRbEl3Ma","outputId":"f2976cfe-8404-49f0-943a-e9dcc3d31cac","execution":{"iopub.status.busy":"2021-07-06T17:42:35.101652Z","iopub.execute_input":"2021-07-06T17:42:35.102246Z","iopub.status.idle":"2021-07-06T17:42:35.107800Z","shell.execute_reply.started":"2021-07-06T17:42:35.102210Z","shell.execute_reply":"2021-07-06T17:42:35.106927Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission['camera'] = labels_test\nsample_submission.head()","metadata":{"id":"W16SuUPFmAk7","outputId":"5190573d-bed5-44b0-9087-96f896512150","execution":{"iopub.status.busy":"2021-07-06T17:42:35.109233Z","iopub.execute_input":"2021-07-06T17:42:35.109750Z","iopub.status.idle":"2021-07-06T17:42:35.129946Z","shell.execute_reply.started":"2021-07-06T17:42:35.109716Z","shell.execute_reply":"2021-07-06T17:42:35.129227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"  sample_submission['camera'].value_counts()","metadata":{"id":"HO4DFP6UsBaR","outputId":"af05b6d6-4723-4118-989c-47e4a659221e","execution":{"iopub.status.busy":"2021-07-06T17:42:35.131099Z","iopub.execute_input":"2021-07-06T17:42:35.131498Z","iopub.status.idle":"2021-07-06T17:42:35.146903Z","shell.execute_reply.started":"2021-07-06T17:42:35.131460Z","shell.execute_reply":"2021-07-06T17:42:35.146145Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission.to_csv(\"submission.csv\", index=False)","metadata":{"id":"mcSR_8gpfx5G","execution":{"iopub.status.busy":"2021-07-06T17:42:35.148168Z","iopub.execute_input":"2021-07-06T17:42:35.148709Z","iopub.status.idle":"2021-07-06T17:42:36.289267Z","shell.execute_reply.started":"2021-07-06T17:42:35.148664Z","shell.execute_reply":"2021-07-06T17:42:36.288410Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission['camera'].value_counts().sort_values().plot(kind='bar')","metadata":{"id":"KXk1itbSfxvj","outputId":"99efa709-af89-459c-e71f-103118036407","execution":{"iopub.status.busy":"2021-07-06T17:42:36.292547Z","iopub.execute_input":"2021-07-06T17:42:36.292804Z","iopub.status.idle":"2021-07-06T17:42:36.506700Z","shell.execute_reply.started":"2021-07-06T17:42:36.292779Z","shell.execute_reply":"2021-07-06T17:42:36.505972Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"id":"w-Y1XMuc5m5U"},"execution_count":null,"outputs":[]}]}