{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-input":false,"_kg_hide-output":false},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\n\nimport pandas as pd \nimport numpy as np \nimport tensorflow as tf\nfrom tensorflow.keras import layers, models\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport json\n\nimport pathlib\nfrom PIL import Image\n\nfrom sklearn.model_selection import train_test_split\nfrom keras.preprocessing.image import ImageDataGenerator, array_to_img, img_to_array, load_img\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"input_dir=\"/kaggle/input/cassava-leaf-disease-classification\"\nos.listdir(input_dir)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# classification labels\n\nfile_path = input_dir + \"/label_num_to_disease_map.json\"\nwith open(file_path,\"r\") as fp:\n    json_data = json.load(fp)\nclass_name=list(json_data.values())\n\nprint(json_data)  #list\nprint(class_name)  #list","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# train_data labels\n\ntrain=pd.read_csv(input_dir + \"/train.csv\")\nprint(train.head(5))\n\n#### test sample\ntrain = train.head(1000)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# sample image and label \n\nsample_image = train.sample(1)\nind = int(sample_image.index.values)\nsample_id = sample_image.loc[ind,\"image_id\"]\nsample_label = sample_image.loc[ind,\"label\"]\n\nimage = plt.imread(input_dir+\"/train_images/\"+sample_id)\nplt.imshow(image)\nplt.xlabel(class_name[sample_label])\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# show some sample images and labels\n\nplt.figure(figsize=(9,9))\nfor i in range(9):\n    plt.subplot(3,3,i+1)\n    image=plt.imread(input_dir+\"/train_images/\"+train.loc[i,\"image_id\"])\n    plt.imshow(image)\n    plt.xticks([])\n    plt.yticks([])\n    plt.grid(False)\n    plt.xlabel(class_name[train.loc[i,\"label\"]])\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# simple eda show statistics characters\nsns.countplot(x='label',data=train)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **data processing**"},{"metadata":{},"cell_type":"markdown","source":"method 01： numpy array -> dataframe   FAIL"},{"metadata":{"trusted":true},"cell_type":"code","source":"# 将image转换会np.array形式  \n\n# 一个sample的array\nimage = tf.keras.preprocessing.image.load_img(input_dir+\"/train_images/\"+sample_id)\ninput_arr=tf.keras.preprocessing.image.img_to_array(image)\nprint(input_arr.shape)\n\n\n###### the data size is too large, thus oom, sample!!! \ntrain = train.head(1000)  # later train=train.sample(2000)\n\nimg_list=[]\nlabel_list=[]\nfor m,n in zip(train.image_id, train.label):\n    # id = m, label = n\n    image = tf.keras.preprocessing.image.load_img(input_dir+\"/train_images/\"+m)\n    input_arr=tf.keras.preprocessing.image.img_to_array(image)\n    #img_list.append(np.array([input_arr]))\n    img_list.append(input_arr)\n    label_list.append(n)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# process data into df\ndf = pd.concat([pd.Series(img_list),pd.Series(label_list)], axis=1)\ndf.columns=[\"img_data\",\"label\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.head(1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X= df[\"img_data\"]\ny= df[\"label\"].astype('str')\n\n## preprocessing  resize&rescale\nX=np.stack(X/255.0)  #normalize \nX=X.reshape(-1,600,800,1)  \n\n\n### validation split\nX_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.1, random_state=42)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"###### ImageDataGenerator：Generate batches of tensor image data with real-time data augmentation.\nfrom keras.preprocessing.image import ImageDataGenerator, array_to_img, img_to_array, load_img\n\n\ndatagen = ImageDataGenerator(featurewise_center=False, #set input mean to 0 over the dataset\n                             samplewise_center=False,  #Set each sample mean to 0\n                             featurewise_std_normalization=False,  #Divide inputs by std of the dataset, feature-wise\n                             samplewise_std_normalization=False, #Divide each input by its std\n                             zca_whitening=False,  #Apply ZCA whitening\n                             rotation_range=0,  #Degree range for random rotations\n                             zoom_range=0,  #Range for random zoom\n                             width_shift_range=0.05,  #randomly shift images horizontally (fraction of total width)\n                             height_shift_range=0.05,  #randomly shift images vertically (fraction of total height)  \n                             horizontal_flip=False,  #Randomly flip inputs horizontally\n                             vertical_flip=False  #Randomly flip inputs vertically\n                            )\n\ndatagen.fit(X_train)\n\n\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"\n\n\n\n\n# **##############################################**"},{"metadata":{},"cell_type":"markdown","source":" **method 02： tensorflow frame   success**"},{"metadata":{"trusted":true},"cell_type":"code","source":"BATCH_SIZE = 16\nSTEPS_PER_EPOCH = len(train)*0.8 / BATCH_SIZE\nVALIDATION_STEPS = len(train)*0.2 / BATCH_SIZE\nEPOCHS = 10  #10\nTARGET_SIZE = 224","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.dtypes","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.label = train.label.astype('str')\ntrain_generator = ImageDataGenerator(validation_split = 0.2,\n                                     preprocessing_function = None,\n                                     zoom_range = 0.2,\n                                     cval = 0.2,\n                                     horizontal_flip = True,\n                                     vertical_flip = True,\n                                     fill_mode = 'nearest',\n                                     shear_range = 0.2,\n                                     height_shift_range = 0.2,\n                                     width_shift_range = 0.2) \\\n    .flow_from_dataframe(train,\n                         directory = os.path.join(input_dir, \"train_images\"),\n                         subset = \"training\",\n                         x_col = \"image_id\",\n                         y_col = \"label\",\n                         target_size = (TARGET_SIZE, TARGET_SIZE),\n                         batch_size = BATCH_SIZE,\n                         class_mode = \"sparse\")\n\nvalidation_generator = ImageDataGenerator(validation_split = 0.2) \\\n    .flow_from_dataframe(train,\n                         directory = os.path.join(input_dir, \"train_images\"),\n                         subset = \"validation\",\n                         x_col = \"image_id\",\n                         y_col = \"label\",\n                         target_size = (TARGET_SIZE, TARGET_SIZE),\n                         batch_size = BATCH_SIZE,\n                         class_mode = \"sparse\")\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# model "},{"metadata":{"trusted":true},"cell_type":"code","source":"# model.fit(datagen.flow(x_train,y_train,batch_size=32),steps_per_epoch=len(x_train)/32, epochs=epochs)\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"##### efficientnetb0\nefn = tf.keras.applications.EfficientNetB0(input_shape = (TARGET_SIZE, TARGET_SIZE, 3),include_top=False,weights='imagenet')\n\nefn_model = tf.keras.Sequential([efn,\n                            tf.keras.layers.GlobalAveragePooling2D(),\n                            tf.keras.layers.Dense(len(class_name),activation = 'softmax')])\n\nefn_model.compile(optimizer=tf.keras.optimizers.Adam(),\n             loss='sparse_categorical_crossentropy',\n             metrics=[\"accuracy\"])\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"early_stop = tf.keras.callbacks.EarlyStopping(monitor = 'val_loss', min_delta = 0.001, \n                           patience = 5, mode = 'min', verbose = 1,\n                           restore_best_weights = True)\n\n# efn_history = efn_model.fit(datagen.flow(X_train,y_train,batch_size=32),steps_per_epoch=len(X_train)/32, epochs=epochs, validation_data = (X_val,y_val))\nefn_history = efn_model.fit(train_generator,batch_size=32, epochs=EPOCHS, \n                            validation_data = validation_generator,\n                           callbacks = [early_stop])\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_acc = efn_history.history['accuracy']\nval_acc = efn_history.history['val_accuracy']\n\nprint(train_acc)\nprint(val_acc)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"##### mobile_net \n\nmobile_net = tf.keras.applications.MobileNetV2(input_shape = (TARGET_SIZE, TARGET_SIZE, 3),include_top=False,weights ='imagenet')\n#mobile_net.trainable = False\n\nmon_model = tf.keras.Sequential([mobile_net,\n                           tf.keras.layers.GlobalAveragePooling2D(),\n                           tf.keras.layers.Dense(len(class_name),activation = 'softmax')])\n\nmon_model.compile(optimizer=tf.keras.optimizers.Adam(),\n             loss='sparse_categorical_crossentropy',\n             metrics=[\"accuracy\"])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"early_stop = tf.keras.callbacks.EarlyStopping(monitor = 'val_loss', min_delta = 0.001, \n                           patience = 5, mode = 'min', verbose = 1,\n                           restore_best_weights = True)\n\n\n# mon_history = mon_model.fit(datagen.flow(X_train,y_train,batch_size=32),steps_per_epoch=len(X_train)/32, epochs=epochs, validation_data = (X_val,y_val))\nmon_history = mon_model.fit(train_generator,batch_size=32, epochs=EPOCHS, \n                            validation_data = validation_generator,\n                           callbacks = [early_stop])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_acc = mon_history.history['accuracy']\nval_acc = mon_history.history['val_accuracy']\nprint(train_acc)\nprint(val_acc)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### cnn\n\nmodel=models.Sequential()\nmodel.add(layers.Conv2D(32,(3,3),activation='relu', input_shape=(600,800,3)))\n\n## model.add(layers.Conv2D(filters = 32, kernel_size = (5,5), padding = 'same',activation='relu'))\n\nmodel.add(layers.MaxPooling2D((2,2)))\n\n## model.add(Dropout(0.1))\n\nmodel.add(layers.Conv2D(64,(3,3),activation='relu'))\nmodel.add(layers.MaxPooling2D((2,2)))\n\nmodel.add(layers.Flatten())\nmodel.add(layers.Dense(64, activation='relu'))\nmodel.add(layers.Dense(class_name))\n\n\nmodel.complie(optimizer='adam',\n             loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n             metrics=['accuracy'])\n\nhistory = model.fit(x_train,y_train,epochs=10,validation_data=(x_val,y_val))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"base_learning_rate = 0.0001\nmodel.compile(loss=tf.keras.losses.BinaryCrossentropy(from_logits=True),\n              optimizer = tf.keras.optimizers.RMSprop(lr=base_learning_rate/10),\n              metrics=['accuracy'])##修改了optimizer\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n### early stop\nearly_stop = EarlyStopping(monitor = 'val_loss', min_delta = 0.001, \n                           patience = 5, mode = 'min', verbose = 1,\n                           restore_best_weights = True)\n\n# Set a learning rate annealer\nlearning_rate_reduction = ReduceLROnPlateau(monitor='val_acc', \n                                            patience=1, \n                                            verbose=1, \n                                            factor=0.5, \n                                            min_lr=0.0001)\n\n\n# Fit the model\nhistory = model.fit_generator(datagen.flow(X_train,y_train, batch_size=batch_size),\n                              epochs = epochs, validation_data = (X_val,y_val),\n                              verbose = 2, steps_per_epoch=X_train.shape[0] // batch_size\n                              , callbacks=[learning_rate_reduction])\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"##### visualize result\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.fit(datagen.flow(x_train,y_train,batch_size=32),steps_per_epoch=len(x_train)/32, epochs=epochs)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# 将image转换会np.array形式 方法01\n\n\n\nsample_image = train.sample(1)\nind = int(sample_image.index.values)\nsample_id = sample_image.loc[ind,\"image_id\"]\nsample_label = sample_image.loc[ind,\"label\"]\n\n\nimage=Image.open(input_dir+\"/train_images/\"+sample_id)\nprint((np.array(image)).shape)  \n\n# normalize \nx = image/255.0  \n\n# reshape\nx= x.reshape(64,64,3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# 将image转换会np.array形式   方法02\nimage = tf.keras.preprocessing.image.load_img(input_dir+\"/train_images/\"+sample_id)\ninput_arr=tf.keras.preprocessing.image.img_to_array(image)\nprint(input_arr.shape)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# validation split\n\nbatch_size = 32\nimg_height=180\nimg_width=180\n\ntrain_ds = tf.keras.preprocessing.image_dataset_from_directory(\n    data_path,validation_split=0.2,subset='training',seed=123,image_size=(img_height, img_width),batch_size=batch_size)\n\nval_ds = tf.keras.preprocessing.image_dataset_from_directory(\n    data_path,validation_split=0.2,subset='validation',seed=123,image_size=(img_height, img_width),batch_size=batch_size)\n\n### 或者把image全转成array,变成dataframe,然后train_test_split\n\n### 或者 ImageDataGenerator(rescale,validation_split).flow_from_directory(source),再zip打包x,y","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\n\nimport pathlib\ndata_dir = pathlib.Path(input_dir+\"/train_images\")  #其实就是path\n# 遍历所有文件的路径\n# for i in data_dir.iterdir():\n#     print(i)\n\n#train_ds = tf.keras.preprocessing.image_dataset_from_directory()\n\nfrom keras.preprocessing.image import ImageDataGenerator, array_to_img, img_to_array, load_img\n\n\n\n###### ImageDataGenerator：Generate batches of tensor image data with real-time data augmentation.\n\n\n# datagen = ImageDataGenerator(rotation_range=44,rescale=1./255,wid_shift_range=0.4,height_shift_range=0.8,\n#                             shear_range=0.7,zoom_range=0.3,horizontal_flip=True,vertical_flip=True,\n#                              fill_mode='nearest')\n\ndatagen = ImageDataGenerator(featurewise_center=False, #set input mean to 0 over the dataset\n                             samplewie_center=False,  #Set each sample mean to 0\n                             featurewise_std_normalization=False,  #Divide inputs by std of the dataset, feature-wise\n                             samplewise_std_normalization=False, #Divide each input by its std\n                             zca_whitening=False,  #Apply ZCA whitening\n                             rotation_range=0,  #Degree range for random rotations\n                             zoom_range=0,  #Range for random zoom\n                             width_shift_range=0.05,  #randomly shift images horizontally (fraction of total width)\n                             height_shift_range=0.05,  #randomly shift images vertically (fraction of total height)  \n                             horizontal_flip=False,  #Randomly flip inputs horizontally\n                             vertical_flip=False  #Randomly flip inputs vertically\n                            )\n# compute quantities required for featurewise normalization\n\ndatagen.fit(x_train) \n\nmodel.fit(datagen.flow(x_train,y_train,batch_size=32),steps_per_epoch=len(x_train)/32, epochs=epochs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## ## preprocessing reshape,rescale\n\n\n# model = models.Sequential()\n# model.add(layers.Conv2D(32,(3,3),activation='relu',input_shape(32,32,3)))\n\nimg_size=100\nresize_and_rescale = tf.keras.Sequential([\n    layers.experimental.preprocessing.Resizing(img_size,img_size),  # reshape\n    layers.experimental.preprocessing.Rescaling(1./255)  #rescale pixel values\n])\n\nsample = resize_and_rescale(sample_image)\nimage = plt.imread(input_dir+\"/train_images/\"+sample_id)\nplt.imshow(image)\nplt.xlabel(class_name[sample_label])\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### data augmentation\n\ndata_augmentation = tf.keras.Sequential(layers.experimental.preprocessing.RamdomFlip(\"horizontal_and_vertical\"),\n                                       layers.experimental.preprocessing.RandomRotation(0.2))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}