{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt \nfrom tensorflow.keras import layers, models, losses, optimizers\nimport tensorflow as tf \n# import xgboost as xgb \nfrom sklearn import preprocessing, ensemble, linear_model \nimport os, sys, pickle \nfrom pylab import *\n# import lightgbm as lgb \n# from sklearn.metrics import roc_curve, auc, accuracy_score\n# from sklearn.pipeline import Pipeline \n# from sklearn.impute import SimpleImputer \n# from sklearn.model_selection import train_test_split \nimport time, json, io, gc \nfrom PIL import Image \n\ndef resize_image(img_arr):\n    new_img = Image.fromarray(img_arr).resize(size=(224, 224))\n    ret_arr = np.array(new_img)\n    return ret_arr \n\ndef jpg_img_resize(path):\n    img = Image.open(path).resize(size=(224, 224))\n    arr = np.array(img)\n    return arr\n\n\ndata_path = '/kaggle/input/siim-isic-melanoma-classification/'\ntfrec_loc = data_path+'tfrecords/'\n# dicom_loc = data_path+'train/'\n\n# train_data = pd.read_csv(data_path+'train.csv')\n# x = os.stat(data_path+'train.csv').st_size/1e6\n# print(x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tf_rec_files = [[file for file in files if 'train' in file] \\\n                for _, _, files in os.walk(tfrec_loc)][0]\n# count = 0\n# for file in tf_rec_files:\n#     count += os.stat(tfrec_loc+file).st_size/1e6\n\n# print(count/1000)\n# print(os.stat(tfrec_loc+tf_rec_files[0]).st_size/1e6)\n# tf_dataset1 = tf.data.TFRecordDataset(data_path+'tfrecords/train00-2071.tfrec')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"image_desc = {\n    'image': tf.io.FixedLenFeature([], tf.string), \n    'image_name': tf.io.FixedLenFeature([], tf.string), \n    'target': tf.io.FixedLenFeature([], tf.int64),\n}\n\ndef parse_img_func(example):\n    return tf.io.parse_single_example(example, image_desc)\n\ndef transform_rec(tfrec):\n    dataset = tf.data.TFRecordDataset(tfrec_loc+tfrec)\n#     print(sys.getsizeof(dataset))\n    parsed_set = dataset.map(parse_img_func)\n    img_arrays = [np.array(Image.open(io.BytesIO(i['image'].numpy()))) for i in parsed_set]\n    img_arrays = np.array(list(map(resize_image, img_arrays)))\n    img_names = [i['image_name'].numpy() for i in parsed_set]\n    targets = [i['target'].numpy() for i in parsed_set]\n    return img_arrays, img_names, targets\nstart = time.time()\nimg_arrays, img_names, targets = transform_rec(tf_rec_files[0])\nnext_arrays, next_names, next_targets = transform_rec(tf_rec_files[1]) \nthird_arrays, third_names, third_targets = transform_rec(tf_rec_files[2])\nend = time.time()\nprint(end-start)\n# next_arrays, next_names, next_targets = transform_rec(tf_rec_files[1]) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"comb_img_arrays = np.concatenate((img_arrays, next_arrays, third_arrays))\ncomb_img_names = np.concatenate((img_names, next_names, third_names))\ncomb_target_arrays = np.concatenate((targets, next_targets, third_targets))\n\ntargets = np.c_[comb_target_arrays]\nimgs = tf.cast(comb_img_arrays, tf.float32)\n# data_dict = {'imgs': comb_img_arrays, \n#             'img_names': comb_img_names, \n#             'targets': comb_target_arrays}\n\n# data_file = open(\"data_pick_2.pkl\", \"wb\")\n# pickle.dump(data_dict, data_file)\n# data_file.close()\n\n# del img_arrays\n# del next_arrays\n# del next_names \n# del next_targets \n# del comb_img_arrays\n# del comb_target_arrays\n# gc.collect()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"comb_img_arrays[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# detect and init the TPU\ntpu = tf.distribute.cluster_resolver.TPUClusterResolver()\ntf.config.experimental_connect_to_cluster(tpu)\ntf.tpu.experimental.initialize_tpu_system(tpu)\n\n# instantiate a distribution strategy\ntpu_strategy = tf.distribute.experimental.TPUStrategy(tpu)\n\n# instantiating the model in the strategy scope creates the model on the TPU\nwith tpu_strategy.scope():\n    print(\"Starting up\")\n    pic_mod = models.Sequential([\n        layers.Conv2D(96, (3,3), activation='relu', input_shape=(224, 224, 3)), \n        layers.MaxPooling2D((3,3)), \n        layers.Conv2D(256, (3,3), activation='relu'),\n        layers.MaxPooling2D((3,3)), \n        layers.Conv2D(384, (3,3), activation='relu'),\n    #     layers.Conv2D(384, (3,3), activation='relu'),\n        layers.Conv2D(256, (3,3), activation='relu'),\n        layers.Flatten(), \n        layers.Dense(128, activation='relu'), \n        layers.Dense(1, activation='sigmoid')\n    ])\n    sgd = optimizers.Adam(lr=0.5)\n    pic_mod.compile(optimizer=sgd, loss='binary_crossentropy', \n                    metrics=[tf.keras.metrics.AUC()])\n    print(\"Ready to train\")\n    time.sleep(4)\nhist = pic_mod.fit(imgs, targets, batch_size=218, epochs=10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}