{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport tensorflow as tf\nimport tensorflow.keras as keras\nfrom sklearn.pipeline import Pipeline\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nimport os\nimport time\nimport shutil\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm \nimport math\n%matplotlib inline","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"images_train_path=\"../input/fungiclef2022/DF20-300px/DF20_300\"\nimages_test_path=\"../input/fungiclef2022/DF20-300px/DF20_300\"\n\ntrain_data_path = \"../input/fungi2022-train-test/train_fungi2022/train_fungi2022\"\ntest_data_path= \"../input/fungi2022-train-test/test_fungi2022/test_fungi2022\"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport shutil\nimport zipfile\n\nclass ZipDir:\n    def __init__(self, path_file_zipped, name_file_zip, path_contain_zip_file):\n        self.source_path = path_file_zipped\n        self.name_file_zip = name_file_zip\n        self.path_contain = path_contain_zip_file\n    def __zipdir(self, ziph):\n        id=0\n        for root, dirs, files in tqdm(os.walk(self.source_path)):\n            n = len(files)\n            for file in files:\n                id+=1\n                #print(f\"{id*100/n}%\")\n                ziph.write(os.path.join(root, file), \n                           os.path.relpath(os.path.join(root, file), \n                                           os.path.join(self.source_path, '..')\n                                          )\n                          )\n        #print(id)\n    def Zip(self, mode=\"w\"):\n        with zipfile.ZipFile(f'{self.name_file_zip}.zip', mode, zipfile.ZIP_DEFLATED) as zipf:\n            self.__zipdir(zipf)\n            \nclass tool_work_with_file:\n    def __init__(self):\n        pass\n    def create_dir(self,path):\n        try:\n            if os.path.exists(path) == False:\n                os.mkdir(path)\n                print(\"Folder is created successfully.\")\n            else:\n                print(\"Folder has existed.\")\n        except:\n            print(\"Fail to create folder.\")\n    def extrac_zip_file(self,path,target_path):\n        print(os.path.exists(path))\n        print(os.path.exists(target_path))\n        try:\n            with zipfile.ZipFile(path,mode=\"w\") as z:\n                z.extractall(target_path)\n                print(\"Extracted all\")\n        except Exception as e:\n            print(e)\n            print(\"Invalid file\")\n    def move_file_to_other_folder(self, name_files, curr_files_path, move_files_path ):\n        d = pd.DataFrame(name_files)\n        d.columns=[\"num\"]\n        n=d.reset_index()[\"class\"].unique().__len__()\n        print(f\"Num folder:{n}\")\n        #count = 0\n        for i in tqdm(name_files.index.values.tolist()):\n            dir, file = i\n            #count+=1\n            file=file+\".jpg\"\n            dir_training_data_path=os.path.join(move_files_path,dir) \n            if os.path.exists(dir_training_data_path) == False:\n                os.mkdir(dir_training_data_path)\n            shutil.copy(os.path.join(curr_files_path,file),os.path.join(dir_training_data_path,file))\n            #print(f\"{count*100/n}%\")\n        print(\"Copy file to folder\")\n    def clean_dir(self,path):\n        for root, dirs, files in os.walk(path):\n            for file in files:\n                os.remove(os.path.join(root,file))\n        print(\"Done!\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.pipeline import Pipeline\nfrom sklearn.neighbors import KNeighborsClassifier","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Process Data","metadata":{}},{"cell_type":"code","source":"def splite_data(df,values_count_df, cols, rate):\n    data_demo = pd.DataFrame(columns=cols)\n    df = df[cols]\n    classes = df[\"class\"].unique().tolist()\n    #print(classes)\n    for class_ in tqdm(classes):\n        n_sample = values_count_df.loc[class_][0]*rate\n        data_demo = data_demo.append(df[df[\"class\"].isin([class_])].sample(math.ceil(n_sample)), ignore_index=True)\n    return data_demo","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#val_data = pd.read_csv(\"../input/fungiclef2022/DF20-val_metadata.csv\")\n#val_data[[\"class\"]].dropna()[\"class\"].value_counts()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load dataframe keep info of data\ntrain_data = pd.read_csv(\"../input/fungiclef2022/DF20-train_metadata.csv\")\nval_data = pd.read_csv(\"../input/fungiclef2022/DF20-val_metadata.csv\")\nensential_cols = [\"class\",\"ImageUniqueID\"]\n\n# Filter quanlifed data\nvalues_count_train = pd.DataFrame(train_data[ensential_cols].dropna().groupby(\"class\").count())\nvalues_count_val = pd.DataFrame(val_data[ensential_cols].dropna().groupby(\"class\").count())\n\n#values_count = values_count[values_count.ImageUniqueID>1000]\n#qualified_class = values_count.index.values\n\n#train_data = train_data[train_data[\"class\"].isin(qualified_class)]\n#val_data = val_data[val_data[\"class\"].isin(qualified_class)]\n\n#train_data = train_data[ensential_cols]\n#val_data = val_data[ensential_cols]\n\n#train_data = train_data.groupby(\"class\").sample(500)\ntrain_data = splite_data(train_data[ensential_cols].dropna(),values_count_train,ensential_cols,0.5)\nval_data = splite_data(val_data[ensential_cols].dropna(),values_count_val,ensential_cols,0.5)\n\n#train_data = train_data.reset_index()\n#train_data.drop(\"index\", axis=1, inplace=True)\n\n#Create folder to store that images which is arranged in subfolder\n\ntools = tool_work_with_file()\ncreate_new_data=1\nif create_new_data:\n    for i in [(\"./train\",train_data),(\"./test\",val_data)]:\n        path,data = i\n        if os.path.exists(path):\n            tools.clean_dir(path)\n        else:\n            tools.create_dir(path)\n        tools.move_file_to_other_folder(data.groupby(\"class\")[\"ImageUniqueID\"].value_counts(), \n                                        \"../input/fungiclef2022/DF20-300px/DF20_300\", \n                                        path)\n        ZipDir(path,path.split(\"/\")[-1],\"./\").Zip()\n\n    train_data.to_csv(\"train_info.csv\")\n    val_data.to_csv(\"val_info.csv\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train_data = pd.read_csv(\"../input/fungiclef2022/DF20-train_metadata.csv\")\n#d = pd.DataFrame(train_data[ensential_cols].dropna().groupby(\"class\")[\"ImageUniqueID\"].value_counts())\n#d.columns=[\"num\"]\n#d.reset_index()[\"class\"].unique().__len__()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#check = train_data.groupby(\"class\").count()\n#check","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#check.sum()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#values_count.loc[\"Agaricomycetes\"].values[0]\n#demo = train_data[ensential_cols]\n#demo[demo[\"class\"].isin([\"Agaricomycetes\"])]\n#data_demo = pd.DataFrame(columns=[\"class\",\"ImageUniqueID\"])\n#data_demo = data_demo.append(demo[demo[\"class\"].isin([\"Agaricomycetes\"])].sample(600), ignore_index=True)\n#data_demo[\"class\"].unique().tolist()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Load data for training and evaluating\n'''\nIMG_HEIGHT, IMG_WIDTH = 128, 128\nbatch_size_train=900\nprint(\"Loading data for training\")\ndata_train = tf.keras.preprocessing.image_dataset_from_directory(\n    train_data_path,\n    batch_size=batch_size_train,\n    image_size=(IMG_HEIGHT, IMG_WIDTH))\nprint(\"Done\")\nprint(\"Loading data for evaluation\")\ndata_test = tf.keras.preprocessing.image_dataset_from_directory(\n    test_data_path,\n    batch_size=300,\n    image_size=(IMG_HEIGHT, IMG_WIDTH),\n    seed=1)\nprint(\"Done\")\n'''","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Build Model","metadata":{}},{"cell_type":"code","source":"'''\nclass My_Model():\n    def __init__(self, feature_extract_model):\n        self.fem = feature_extract_model\n        \n    def preprocess_data(self,X):\n        return self.fem.fit(X/255.0)\n    \n    def fit(self, classification_model, X,y):\n        X = self.preprocess_data(X)\n        classification_model.fit(X,y)\n        return classification_model\n    \n    def predict(self,classification_model, X):\n        X = self.preprocess_data(X)\n        return classification_model.predict(X)\n'''","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\ndef build_keras_model(input_dims):\n    input_data = keras.Input(shape=input_dims, name=\"input\")\n    base_model = tf.keras.applications.InceptionResNetV2(include_top=False, weights=\"imagenet\", input_shape=(IMG_HEIGHT, IMG_WIDTH, 3))\n    base_model.trainable = False\n    \n    x = base_model(input_data)\n    features_output = keras.layers.Flatten()(x)\n    \n    model = keras.Model( inputs=input_data,\n                         outputs=features_output, \n                         name=\"main_model\")\n\n    model.compile( optimizer=\"adam\", loss=tf.keras.losses.SparseCategoricalCrossentropy(), metrics=['accuracy'])\n    return model\n\nclass Keras2Sklearn_Model:\n    def __init__(self, build_model, input_dims):\n        strategy = tf.distribute.MirroredStrategy()\n        with strategy.scope():\n            self.model = build_model(input_dims)\n        \n    def plot_model(self):\n        return self.model\n    \n    def fit(self, X,*arg):\n        tensor_data = tf.convert_to_tensor(X)\n        output = self.model(tensor_data)\n        return output.numpy()\n    \n    def transform(self,X):\n        return self.fit(X)\n\nclass KNN:\n    def __init__(self, feature_extract_model, n_neighbors=5):\n        self.__fem = feature_extract_model\n        self.__k=n_neighbors\n    \n    def __cosine(self,matrix,vector):\n        dot_product = np.sum(np.multiply(matrix,vector),axis=1)\n        module_vector = np.sqrt(np.sum(np.power(vector,2),axis=1))\n        module_vector_in_matrix = np.sqrt(np.sum(np.power(matrix,2),axis=1))\n        print(dot_product.shape)\n        print((module_vector_in_matrix*module_vector).shape)\n        similarity = np.abs(dot_product/(module_vector_in_matrix*module_vector))\n        return similarity\n\n    def __calculate_distance(self,list_vector,vector_A,method=\"norm2\"):\n        if method==\"norm2\":\n            return np.linalg.norm(list_vector-vector_A,axis=1)\n        elif method==\"cosine\":\n            return self.__cosine(list_vector,vector_A)\n    \n    def __preprocess_data(self,X):\n        return self.__fem.fit(X/255.0)\n    \n    def fit(self,data_X_y):\n        print(f\"fitting\")\n        self.__data_train=data_X_y\n        self.__n_batch = int(266315/batch_size_train)+1\n        print(f\"Batch: {self.__n_batch}\")\n        \n    def __predict_one_point(self,raw_data_point,**kargs):\n        list_k_distance = pd.DataFrame(columns=[\"distance\",'label'])\n        print(f\"Predict one point - batch-{kargs['id_batch']} point-{kargs['id_point']}\")\n        count=0\n        method=\"norm2\"\n        ascending = True\n        s = time.time()\n        s_loop = time.time()\n        for data_train_batch_i in self.__data_train.as_numpy_iterator():\n            e = time.time()\n            print(f\"\\tLoad data from generation: {e-s}s\")\n            \n            s = time.time()\n            X,y = data_train_batch_i\n            e = time.time()\n            print(f\"\\t\\tArchive data x,y: {e-s}s\")\n\n            count+=1\n            \n            s = time.time()\n            X = self.__fem.fit(X/255.0)\n            vector = self.__fem.fit(np.array([raw_data_point])/255.0)\n            e = time.time()\n            print(f\"\\t\\tFeature extract: {e-s}s\")\n            \n            s = time.time()\n            if method==\"consine\":\n                ascending = False\n            distance = self.__calculate_distance(X,vector,method)\n            e = time.time()\n            print(f\"\\t\\tCalculate distance: {e-s}s\")\n            \n            s = time.time()\n            data = pd.DataFrame({\"distance\":distance,\"label\":y})\n            data = data.sort_values(\"distance\",ascending=ascending).head(self.__k)\n            e = time.time()\n            print(f\"\\t\\tFind local k-nearest points: {e-s}s\")\n            \n            s = time.time()\n            list_k_distance = list_k_distance.append(data, ignore_index=True)\n            list_k_distance = list_k_distance.sort_values(\"distance\",ascending=ascending).head(self.__k)\n            e = time.time()\n            print(f\"\\t\\tFind global k-nearest points: {e-s}s\")\n            #print(f\"Data:\\n{data}\")\n            #print(f\"\\nlist_k_distance:\\n{list_k_distance}\")\n            print(f\"\\t{count*100/self.__n_batch}%\")\n            #print(\"-\"*20)\n            s = time.time()\n            \n        try:\n            e_loop = time.time()\n            print(f\"Predict one point in: {e_loop-s_loop}s\")\n            return list_k_distance.groupby(\"label\").count().sort_values(\"distance\").tail(1).index.to_list()[0]\n        except Exception as e:\n            print(e)\n            print(f\"Error bacth {kargs['id_batch']} list_k_distance: {list_k_distance}\")\n            raise e\n    \n    def evaluate(self,data_X):\n        prediction = []\n        y_true=[]\n        id=0\n        for data_test_batch_i in data_X.as_numpy_iterator():\n            X,y = data_test_batch_i\n            y_true+=y.tolist()\n            id+=1\n            id_point=0\n            for data_test_point in X:\n                id_point+=1\n                prediction.append((self.__predict_one_point(data_test_point,id_batch=id,id_point=id_point)))\n        return (prediction,y_true)\n    \n    def predict(self, data_point):\n        return self.__predict_one_point(data_point)\n    \n    def set_k(self,k):\n        self.__k=k\n    \n    def get_k(self):\n        return self.__k\n    \n#model = KNN( Keras2Sklearn_Model(build_keras_model, (IMG_HEIGHT, IMG_WIDTH, 3)), n_neighbors=50 )'''","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#model.fit(data_train)\n#predict,y_true = model.evaluate(data_test)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#predict","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#y_true","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#from sklearn.metrics import accuracy_score\n#accuracy_score(y_true, predict)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nhistory ={}\nfor i in range(1,51):\n    model.set_k(i)\n    predict,y_true = model.evaluate(data_test)\n    history.update({i:accuracy_score(y_true, predict)})\n'''","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#plt.plot(list(history.keys()),list(history.values()))\n#plt.show()\n#plt.savefig('k_acc.png')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}