{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n#for dirname, _, filenames in os.walk('/kaggle/input'):\n# for filename in filenames:\n# print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Importing and Setting up"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import torch\nprint(torch.__version__)\nprint(torch.cuda.is_available())\n\nimport fastai\nprint(fastai.__version__)\n\nfrom fastai.vision.all import *\n\nfrom sklearn import metrics as skm\nimport logging, sys\n\nfrom scipy import stats\nimport psutil\n\nimport re\nimport itertools\n\npd.options.mode.chained_assignment = None\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pip install gputil","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import GPUtil\ngpus = GPUtil.getGPUs()\ntry:\n gpu = gpus[0]\n gpu_available = True\nexcept:\n print(\"no gpu detected\")\n gpu_available = False","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_size(bytes, suffix=\"B\"):\n \"\"\"\n Scale bytes to its proper format\n e.g:\n 1253656 => '1.20MB'\n 1253656678 => '1.17GB'\n \"\"\"\n factor = 1024\n for unit in [\"\", \"K\", \"M\", \"G\", \"T\", \"P\"]:\n if bytes < factor:\n return f\"{bytes:.2f}{unit}{suffix}\"\n bytes /= factor","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import string\n \n# initializing size of string \nN = 7\n \n# using random.choices() \n# generating random strings\ndef randomstr():\n return ''.join(random.choices(string.ascii_uppercase + string.digits, k = N)) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# p = psutil.Process().cpu_percent(interval=1)\n# cpupcnt = p.cpu_percent() / psutil.cpu_count()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# This functions pulls up hardware details like CPU and memory usage\ndef get_hardware_info():\n cputpcnt = psutil.cpu_percent()\n CPU_data = {\"CPU Percent\":cputpcnt}\n GPU_data = {}\n if gpu_available:\n for gpu in gpus:\n gpu = {\"GPU Load\":gpu.load, \"GPU Memory Free\":gpu.memoryFree, \"GPU Memory Used\":gpu.memoryUsed, \"GPU Total\":gpu.memoryTotal}\n GPU_data = {**GPU_data, **gpu}\n svm = psutil.virtual_memory()\n vmds = [get_size(svu) for svu in [svm.total, svm.available, svm.used, svm.free]]\n vMem_data = {\"vMem percent\":svm.percent, \"vMem total\":vmds[0], \"vMem available\":vmds[1], \"vMem used\":vmds[2], \"vMem free=\":vmds[3]}\n fulldata = {**CPU_data, **GPU_data ,**vMem_data}\n hardware_log = \"\"\n for x, y in fulldata.items(): hardware_log+= f\"{x}={y}, \"\n return hardware_log","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## If GPU runs out of memory\nimport torch, gc\ngc.collect()\ntorch.cuda.empty_cache()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Setting up Complete\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Exploring the Data\n\nAfter looking going through the dataset, the structure seems to be as follows:\n```\nnybg2020 - test - metadata.json\n | \\- images - 000 - 0.jpg\n | | |- 1.jpg\n | | \\- ...\n | \\- ... \n | \n \\- train - metadata.json\n \\- images - 000 - 00 - 437000.jpg\n | | - ...\n | |- 01\n | \\- ...\n |- 001 - 00\n | \\- ...\n \\- ...\n```\n\nLet's now pull up the metadata json files from train and test, and check their contents."},{"metadata":{"trusted":true},"cell_type":"code","source":"train_metadata_path = r\"../input/herbarium-2020-fgvc7/nybg2020/train/metadata.json\"\ntest_metadata_path = r\"../input/herbarium-2020-fgvc7/nybg2020/test/metadata.json\"\ntrain_path = Path('/kaggle/input/herbarium-2020-fgvc7/nybg2020/train/')\ntest_path = Path('/kaggle/input/herbarium-2020-fgvc7/nybg2020/test/')\nwith open(train_metadata_path, encoding=\"utf8\", errors='ignore') as f:\n train_metadata = json.load(f)\nwith open(test_metadata_path, encoding=\"utf8\", errors='ignore') as f:\n test_metadata = json.load(f)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(train_metadata['images']), len(train_metadata['annotations'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"total_images = len(train_metadata['images'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_metadata['annotations'][0], train_metadata['images'][0]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The structure of the metadata is as follows. The number of elements of annotations and images are the same. The labels (independent variable) is in annotations, while the directery of the image is in images.\n\n```\nmetadata = {annotations:[{'category_id': 11524, \n 'id': 818566, \n 'image_id': 818566, \n 'region_id': 1}, ...],\n categories: [{'family': 'Orchidaceae',\n 'genus': 'Aa',\n 'id': 0,\n 'name': 'Aa mathewsii (Rchb.f.) Schltr.'}, ...]\n images: [{'file_name': 'images/156/72/354106.jpg',\n 'height': 1000,\n 'id': 354106,\n 'license': 1,\n 'width': 661},...]}\n \n```"},{"metadata":{"trusted":true},"cell_type":"code","source":"# y = 0\n# for x in range(len(train_metadata[\"annotations\"])):\n# if not train_metadata[\"annotations\"][x][\"image_id\"] == train_metadata[\"images\"][x][\"id\"]:\n# print(\"The ordering is not the same!\")\n# break\n# else: y+=1\n# print(\"Ordering is the same for %d images.\" %y)\n# print(y==total_images)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"path = r'/kaggle/input/herbarium-2020-fgvc7/nybg2020/train/images/000/'\nsubpath = r'/kaggle/input/herbarium-2020-fgvc7/nybg2020/train/images/010/05/'\ntest_path = Path('/kaggle/input/herbarium-2020-fgvc7/nybg2020/test/')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n# imagelist = []\n# subdir_imagelist = []\n\n# # creates a list of all image file names in all subdirectories in int format\n# for files in os.listdir(path): \n# subdir = os.path.join(path, files)\n# if os.path.isdir(subdir):\n# imgs = os.scandir(subdir)\n# imagelist += [int(x.name.split('.')[0]) for x in list(imgs)]\n \n# # creates a list of all image file names in int format\n# if os.path.isdir(subpath):\n# subdir_imgs = os.scandir(subpath)\n# subdir_imagelist += [int(x.name.split('.')[0]) for x in list(subdir_imgs)]\n \n# #imagelist, subdir_imagelist","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#for entry in train_metadata['annotations']:\n# if entry['image_id'] in imagelist:\n# print(entry['category_id'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# creating a dataframe from train_metadata annotations and imagesges, because it is easier to search\ntrain_images_df = pd.DataFrame(train_metadata[\"images\"])\ntrain_annotations_df = pd.DataFrame(train_metadata[\"annotations\"])\ndataset = train_images_df.merge(train_annotations_df, how=\"outer\", on=\"id\")[[\"id\", \"image_id\", \"category_id\", \"file_name\"]]\ndataset[\"img_in_cat\"] = dataset.groupby(\"category_id\")[\"category_id\"].transform(len)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"total_category_list = train_annotations_df[\"category_id\"].unique()\ntotal_category_list.sort()\ntotal_category_list = list(total_category_list)\n# creating a list with the indices of all the images we want in the subset\n# random_species_subset = random.sample(range(total_species), 10000)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"total_species = len(train_metadata[\"categories\"])\nprint(\"Total number of species is %d, Total number of images is %d\" %(total_species, total_images))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Metadata has been imported. DataFrame with all values has been created.\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Statistical Inference"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Get a dataframe images per species vs number of species\ndef get_stats(ds):\n df = ds.loc[:,[\"id\", \"category_id\"]]\n df[\"num_imgs\"] = df.groupby(\"category_id\").transform(len)\n sample = df.groupby(\"category_id\").sample(1)\n sample.reset_index()\n freq_list = sample[\"num_imgs\"].to_list()\n sample[\"num_cats\"]=sample.groupby(\"num_imgs\")[\"category_id\"].transform(len)\n sample = sample.groupby(\"num_imgs\").sample(1)\n sample = sample.sort_values(\"num_imgs\")\n return sample.reset_index().loc[:, [\"num_imgs\", \"num_cats\"]], freq_list","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def desc_extract(percent=1, df=dataset):\n img_num_by_pcnt = int(df.shape[0]*percent/100)\n species_num_by_pcnt = int(df.shape[0]*percent/100)\n sdataset = df.sample(img_num_by_pcnt)\n num_cats_in_pcnt = len(sdataset[\"category_id\"].unique())\n pcnt_species = num_cats_in_pcnt/img_num_by_pcnt\n hist_chart, freq_list = get_stats(sdataset)\n return stats.describe(freq_list), hist_chart, freq_list, [pcnt_species, len(freq_list), num_cats_in_pcnt]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def print_stats(pcnt=100, print_desc=True, dtf=dataset):\n full_desc, full_hc, full_fl, full_list = desc_extract(pcnt, dtf)\n low10 = np.quantile(full_fl, 0.10)\n top10 = np.quantile(full_fl, 0.90)\n q0 = np.quantile(full_fl, 0.0)\n q1 = np.quantile(full_fl, 0.25)\n q2 = np.quantile(full_fl, 0.5)\n q3 = np.quantile(full_fl, 0.75)\n q4 = np.quantile(full_fl, 1)\n full_mean = np.mean(full_fl)\n line1 = \"1/4 species have between %d and %d images \\n1/4 species have between %d and %d images \\n\"%(q0, q1, q1, q2)\n line2 = \"1/4 species have between %d and %d images \\n1/4 species have between %d and %d images \\n\"%(q2, q3, q3, q4)\n line3 = f\"%10 species have between 1 and {low10} images and 10% have between {top10} and {q4} images \\n\"\n line4 = \"On average, each specie has %d images \\n\"%full_mean\n line5 = \"Median number of images is %d \\n\"%q3\n fullprint = line1+line2+line3+line4+line5\n if print_desc:\n print(fullprint)\n return [low10, q1, q2, q3, top10, q4, full_mean], fullprint","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"# desclist = []\n# for x in [0.1, 0.5, 1, 5, 10, 50, 100]:\n# st, hc, _, y = desc_extract(percent=x)\n# desclist.append(st)\n# normalized_stats = {\"normalized mean\":st.mean*100/x, \"normalized variance\":st.variance/((x/100)**2)}\n# print(\"\\n Percent is: \", x, \" Some stats: \",y, \"\\n Description: \", st, \"\\n stats normalized to 100%\", normalized_stats, \"\\n\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# print_stats(100)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Plot number of images per species vs number of species\ndef plot_bar(min_num=0, num_to_check=total_images, train_metadata=train_metadata):\n tr_img_df = pd.DataFrame(random.sample(train_metadata[\"images\"], num_to_check))\n tr_anno_df = pd.DataFrame(random.sample(train_metadata[\"annotations\"], num_to_check))\n tr_anno_df[\"len_rows\"] = tr_anno_df.groupby(\"category_id\")[\"category_id\"].transform(len)\n sampledlist = tr_anno_df.groupby(\"category_id\")[\"id\"].apply(lambda s: s.sample(1)).to_list()\n #tr_anno_df = tr_anno_df.sort_values(\"len_rows\")\n lenandcat = tr_anno_df.groupby(\"len_rows\")\n unq = lenandcat[\"category_id\"].unique()\n frequency_series = unq.apply(func = lambda s: len(s))\n graph_x = list(frequency_series.axes[0])[min_num:]\n graph_y = list(frequency_series)[min_num:]\n graph_data = pd.DataFrame(zip(graph_x, graph_y), columns=[\"Number of Images\", \"Number of Species\"])\n graph_data.plot(x=\"Number of Images\", y=\"Number of Species\", kind=\"bar\")\n return graph_data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# The data for the entire dataset is saved as frequency_table\n# graph_data.to_csv(\"frequency_table.csv\")\n# pd.read_csv(\"frequency_table.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# The following function takes int min_num and num_to_check, and train_metadata\n# It returns a list of categories where each category has at least min_num images. It'll only go through the first\n# num_to_check images for this.\n\ndef find_species(min_num=10, num_to_check=total_images, df=dataset):\n set_of_cats = df[df[\"img_in_cat\"]>min_num]['category_id']\n set_of_cats = set(set_of_cats.to_list())\n return list(set_of_cats)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Efficienct code for fetching images\ndef image_fetcher(min_images = 10, min_species = 100, max_img_per_species = -1, dataset=dataset, exclude_categories=[]):\n #This function will output a dataframe which has a minimum of species_min species and \n #minimum of min_img images.\n # First, check if we are collecting all species and/or all images:\n all_species = True if (min_species==total_species or min_species<0) else False\n \n # Firstly, select all categories with minimum number of images = min_images\n # If min_images = 0, this is the set of all categories\n if min_images == 0 or all_species:\n category_list = total_category_list\n else:\n category_list = find_species(min_images)\n if len(category_list)0:\n df.loc[: ,(\"cat_id\")] = df.loc[: ,(\"category_id\")]\n df = df.groupby(\"cat_id\")[[\"id\",\"image_id\", \"file_name\", \"category_id\"]].apply(lambda s: s if s.shape[0]<=max_img_per_species else s.sample(max_img_per_species)) \n df = df.reset_index()\n else:\n # if all species are selected, we ignore min_images restriction\n # we select max_img_per_species or lower from each species\n #df = dataset.groupby(\"cat_id\").apply(lambda s: s if s.shape[0]<=max_img_per_species else s.sample(max_img_per_species))\n df = dataset.groupby(\"category_id\")[[\"image_id\", \"file_name\"]].apply(lambda s: s if s.shape[0]<=max_img_per_species else s.sample(max_img_per_species))\n df = df.reset_index()\n selected_images = df.shape[0]\n num_species = len(df[\"category_id\"].unique())\n return {\"dataframe\": df, \"num images\":selected_images, \"num species\":num_species}\n\n ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# out = image_fetcher(0, 30, -1)\n# print_stats(100, True, out[\"dataframe\"])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# print_stats(100)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"q0imgs = image_fetcher(0, 30, 2, dataset)[\"dataframe\"]\ncatsq = list(q0imgs[\"category_id\"].unique())\nq1imgs = image_fetcher(0, 70, 4, dataset, catsq)[\"dataframe\"]\ncatsq += q1imgs[\"category_id\"].to_list()\nq2imgs = image_fetcher(3, 100, 9, dataset, catsq)[\"dataframe\"]\ncatsq += q2imgs[\"category_id\"].to_list()\nq3imgs = image_fetcher(8, 100, 27, dataset, catsq)[\"dataframe\"]\ncatsq += q3imgs[\"category_id\"].to_list()\nq4imgs = image_fetcher(26, 100, 1500, dataset, catsq)[\"dataframe\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.concat([q0imgs, q1imgs, q2imgs, q3imgs, q4imgs]).loc[:, (\"image_id\", \"id\", \"file_name\", \"category_id\")]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_num_cats = len(train_df[\"category_id\"].unique())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(f\"A small representative subset has been created. \\nIt has {train_num_cats} species and {train_df.shape[0]} images \")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Time to create models and run experiments!\n\n# Creating the DataLoader\n\nWe need to extract a subset of images to train the model, as training on the entire set will by extremely time consuming. \nThe test set's metadata only contains the images dictionary. Thus, the \"images\" dictionary of the metadata.json is going to be the input.\nTo train the model, we need to identify a subset of images, and feed in both the images and annotation dictionaries of that subset. \n\nThere are probably multiple ways of doing this with fastaiv2 library. However, we are going to take a straightforward approach here. This involves creating a new pandas dataframe which contains 3 columns, \"image-id\", \"image-path\" and \"category-id\". \n\n"},{"metadata":{},"cell_type":"markdown","source":"## The subset dataframe\nWe will create a dataframe of ~100 species and all the corresponding images. We'll try to restrict the images to ~10k"},{"metadata":{"trusted":true},"cell_type":"code","source":"# fetch a set of images with min 250 species and min 10000 images\n# all of these species have at least 2 image samples\n# out = image_fetcher(0, 3000, 5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# out[\"num images\"], out[\"num species\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# train_df = out[\"dataframe\"]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Functions of the DataLoader\n\n`get_x` takes in rows from the dataframe and returns the file path of the corresponding image\n\n`get_y` takes in rows from the dataframe and returns the label\n\n`splitter` takes in the entire dataframe and returns two lists; one with the indices of training datapoints and the second with the indices of valid datapoints"},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_labeller(df):\n prop_valid = 0.2\n sampledlist = df.groupby(\"category_id\")[\"image_id\"].apply(lambda s: s.sample(1)).to_list()\n def is_training(x) :\n if x[\"image_id\"] in sampledlist: return True\n else: return random.random() > prop_valid\n df[\"in_training\"] = df.apply(is_training, axis=1)\n return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_to_train = train_labeller(train_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_to_train = image_fetcher(0, -1, 5)[\"dataframe\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_to_train = train_labeller(df_to_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_to_train","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ndef get_x(r): return (train_path/r[\"file_name\"])\n\ndef get_y(r): return r[\"category_id\"]\n\ndef data_splitter(df):\n # This splitter will select at least one of each species and add it to the training set\n # it'll split the remaining images into training and valid based on the value of prop_valid\n # prop_valid = percentage of images to be placed in valid set\n train_idx = df.query('in_training').index.tolist() \n valid_idx = np.setdiff1d(np.array(range(df.shape[0])), np.array(train_idx))\n \n ## Debug\n valid_cats = set([df[\"category_id\"].iloc[i] for i in valid_idx])\n train_cats = set([df[\"category_id\"].iloc[i] for i in train_idx])\n if not valid_cats.issubset(train_cats):\n raise Exception(\"There is a category in valid which is not present in train\")\n \n return train_idx, list(valid_idx)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # get one row from the dataframe and show all the details\n# one_row = train_df.iloc[5]\n# imgpath = get_x(one_row)\n# cat_id = get_y(one_row)\n# sampleimg = Image.open(imgpath)\n# print(cat_id)\n# show_image(sampleimg)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Automating Experimentation\nThere are too many experiments to run. The following code makes that easy."},{"metadata":{"trusted":true},"cell_type":"code","source":"# To quick iteracte over diffrent learners, with different batch sizes, aug transforms, etc\n# AIM: To set up a loop which will iterate over different things I want to try, calculate the results and print them out\n\nclass trainExperiments():\n default_nn = resnet18\n default_dataset = df_to_train\n default_cbs = []\n default_metrics = F1Score(average=\"macro\")\n default_bs = 64\n default_item_tfms = [Resize(600)]\n default_batch_tfms = [*aug_transforms(size=600, min_scale=0.75)]\n experiment_records = \"ExpRecords\"\n \n def __init__(self, experiment_num, run_params = {}):\n self.cbs = []\n self.learner = None\n self.dataset = run_params[\"dataset\"] if \"dataset\" in run_params.keys() else self.default_dataset\n self.cbs = run_params[\"cbs\"] if \"cbs\" in run_params.keys() else self.default_cbs\n self.net = run_params[\"nn\"] if \"nn\" in run_params.keys() else self.default_nn\n self.metrics = run_params[\"metrics\"] if \"metrics\" in run_params.keys() else self.default_metrics\n self.bs = run_params[\"bs\"] if \"bs\" in run_params.keys() else self.default_bs\n self.item_tfms = run_params[\"item_tfms\"] if \"item_tfms\" in run_params.keys() else self.default_item_tfms\n self.batch_tfms = run_params[\"batch_tfms\"] if \"batch_tfms\" in run_params.keys() else self.default_batch_tfms\n self.exp_num = experiment_num\n self.log_name = \"Experiment\"+str(self.exp_num)\n self.text_log = None\n self.iter = 0\n self.record_in_one_file = True\n self.record_file_name = self.experiment_records\n \n def log_after_epoch(self, *args, **kwargs):\n print(\"One epoch complete\")\n pass\n# with open(self.log_name+\"log.txt\", \"a+\") as log_file:\n# log_file.write(get_hardware_info())\n# log_file.write(\"--------------\\n\")\n \n def epoch_writer_wrap(self): \n return Callback(after_epoch=self.log_after_epoch)\n \n def log_initiate(self, train_settings):\n fields = ['Experiment ID', 'Train Loop', 'Architecture', 'Dataset Size', 'Number of Species', 'Method', 'Epochs', 'Learning Rate', 'Batch Size', 'Item Tfms', 'Batch Tfms', 'Callbacks']\n self.log_name = \"Experiment \"+str(self.exp_num)\n self.csvlog = CSVLogger(fname=self.log_name, append=True)\n self.cbs.append(self.csvlog)\n self.cbs.append(self.epoch_writer_wrap())\n self.record_file_name = self.record_file_name if self.record_in_one_file else self.log_name + \"log\"\n if not os.path.isfile(self.record_file_name+\".csv\"):\n with open(self.record_file_name+\".csv\", \"w\") as log_file:\n csvwriter = csv.writer(log_file)\n csvwriter.writerow(fields)\n \n def log_train_loop(self, train_setup):\n fpl = [self.exp_num, self.iter, self.net.__name__, self.dataset.shape[0], len(self.dataset[\"category_id\"].unique()), train_setup[\"method\"], train_setup[\"n\"], train_setup[\"lr\"], self.bs]\n str_fpl = [str(y) for y in fpl]\n fpl_none = [None]*9\n item_tfms_list = re.findall(r'\\w+\\s+--\\s+.+}:', str(self.item_tfms))\n batch_tfms_list = re.findall(r'\\w+\\s+--\\s+.+}:', str(self.batch_tfms))\n first_record = True\n with open(self.record_file_name+\".csv\", \"a+\") as log_file:\n csvwriter = csv.writer(log_file)\n if(len(item_tfms_list)==0 and len(batch_tfms_list)==0):\n csvwriter.writerow(ftr_fpl+[None, None, str(self.cbs)])\n else:\n for i in itertools.zip_longest(item_tfms_list, batch_tfms_list):\n if first_record:\n first_record = False\n csvrow = str_fpl+[i[0], i[1], str(self.cbs)]\n else: csvrow = fpl_none+[i[0], i[1], str(self.cbs)]\n csvwriter.writerow(csvrow)\n \n def get_dls(self):\n dblock = DataBlock(blocks=(ImageBlock, CategoryBlock),\n get_x=get_x,\n get_y=get_y,\n item_tfms = self.item_tfms,\n splitter = data_splitter,\n batch_tfms=self.batch_tfms)\n return dblock.dataloaders(self.dataset, batch_size=self.bs)\n \n def update_learner_dls(self, train_setup):\n self.iter += 1\n append = train_setup[\"append\"] if \"append\" in train_setup.keys() else False\n if \"bs\" in train_setup.keys(): self.bs = train_setup[\"bs\"]\n if \"item_tfms\" in train_setup.keys():\n if append:\n self.item_tfms += train_setup[\"item_tfms\"] \n else: \n self.item_tfms = train_setup[\"item_tfms\"]\n if \"batch_tfms\" in train_setup.keys():\n if append:\n self.batch_tfms += train_setup[\"batch_tfms\"] \n else: \n self.batch_tfms = train_setup[\"batch_tfms\"]\n self.learner.dls = self.get_dls()\n \n def create_learner(self):\n self.learner = cnn_learner(dls=self.get_dls(), arch=self.net, metrics=self.metrics)\n self.learner.add_cbs(self.cbs)\n \n def train_loop(self, train_setup={}):\n # fit, one_batch, fit_one_cycle, fine_tune\n if(len(train_setup)==0): return None\n self.log_train_loop(train_setup)\n n_epoch = train_setup[\"n\"]\n lr = train_setup[\"lr\"]\n if \"freeze\" in train_setup.keys(): self.learner.freeze()\n if train_setup[\"method\"] == \"fit\":\n self.learner.fit(n_epoch, lr)\n if train_setup[\"method\"] == \"all_batches\":\n self.learner.all_batches()\n if train_setup[\"method\"] == \"fit_one_cycle\":\n self.learner.fit_one_cycle(n_epoch, lr)\n if train_setup[\"method\"] == \"fine_tune\":\n self.learner.fine_tune(n_epoch, lr)\n if \"freeze\" in train_setup.keys(): self.learner.unfreeze()\n \n def run_experiment(self, train_settings=[{}]):\n self.log_initiate(train_settings)\n self.get_dls()\n self.create_learner()\n for train_setup in train_settings:\n self.update_learner_dls(train_setup)\n self.train_loop(train_setup)\n self.learner.remove_cbs(self.cbs)\n \n def continue_experiment(self, train_settings=[{}]):\n self.learner.add_cbs(self.cbs)\n if self.learner is None:\n raise Exception(\"Run an experiment first\")\n else:\n for train_setup in train_settings:\n self.update_learner_dls(train_setup)\n self.train_loop(train_setup)\n self.learner.remove_cbs(self.cbs)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"structure = [{\"method\":\"fit_one_cycle\", \"n\":9, \"lr\": 0.02, \"append\": False, \"change_dls\": False, \"item_tfms\":[], \"batch_tfms\":[], \"bs\":32}, {}]"},{"metadata":{"trusted":true},"cell_type":"code","source":"learning_program = [ \n [{\"nn\":resnet34, \"dataset\":df_to_train, \"bs\":60, \"item_tfms\":[Resize(600)], \"batch_tfms\":[*aug_transforms(size=224, min_scale=0.1, max_rotate=90)]}, \n [ \n {\"method\":\"fine_tune\", \"lr\":0.003, \"n\":7, \"bs\":60}\n ]\n ]\n]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Starting the training process!\")\nds1 = learning_program[0]\nle = trainExperiments(randomstr(), ds1[0])\nle.run_experiment(ds1[1])\nle.learner.save('finalout')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# for x in learning_program:\n# le = trainExperiments(randomstr(), x[0])\n# le.run_experiment(x[1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"le.learner.show_training_loop()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# le.learner.lr_find()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"break","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# del le.learner\n# gc.collect()\n# torch.cuda.empty_cache()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gc.collect()\ntorch.cuda.empty_cache()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import shutil\nshutil.make_archive('./expzip','zip','/kaggle/working')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n# [{\"nn\":resnet34, \"dataset\":df_to_train, \"bs\":64, \"item_tfms\":[Resize(600)], \"batch_tfms\":[*aug_transforms(size=256, min_scale=0.1, max_rotate=90)]}, \n# [ \n# {\"method\":\"fine_tune\", \"lr\":0.003, \"n\":12, \"bs\":64}\n# ]\n# ]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}