{"cells": [{"metadata": {"_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5", "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19", "trusted": true}, "cell_type": "code", "source": "# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \nimport datetime\ndate_depart=datetime.datetime.now()\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport weakref\nimport warnings\n\nwarnings.simplefilter(\"ignore\")\nimport logging\nlogging.basicConfig(filename='python.log',level=logging.DEBUG)\nlogging.captureWarnings(True)\nconsole = logging.StreamHandler()\nconsole.setLevel(logging.INFO)\nformatter = logging.Formatter('%(name)-12s: %(levelname)-8s %(message)s')\n# tell the handler to use this format\nconsole.setFormatter(formatter)\nlogging.getLogger('').addHandler(console)\n\nfrom functools import partial\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\nimport cv2\nimport matplotlib.pyplot as plt\nimport imageio\nimport imgaug\nfrom imgaug import augmenters as iaa\nfrom imgaug import parameters as iap\nimport keras\nfrom keras.models import Sequential, Model\nfrom keras.layers import Dense, Flatten, Activation, Dropout, GlobalAveragePooling2D\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras import optimizers, applications\nfrom keras.callbacks import ModelCheckpoint, LearningRateScheduler, TensorBoard, EarlyStopping\nfrom keras import backend as K \nfrom concurrent import futures\nimport os\nimport json\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.", "execution_count": null, "outputs": []}, {"metadata": {"trusted": true, "_kg_hide-input": true}, "cell_type": "code", "source": "#!ls -Rlh ../input", "execution_count": null, "outputs": []}, {"metadata": {"trusted": true}, "cell_type": "code", "source": "duree_max=datetime.timedelta(hours=7,minutes=30)\n\nfichier_modele_base=\"inaturalist\" \ntrain_batch_size=64\nfull_train_batch_size=4\nval_batch_size=4\nepochs=int(1e8)\n\nload_keras_weights=False\ndmax=1024\ndmin=600\ndcrop=512\ndate_limite= date_depart+duree_max\n", "execution_count": null, "outputs": []}, {"metadata": {"_cell_guid": "79c7e3d0-c299-4dcb-8224-4455121ee9b0", "_uuid": "d629ff2d2480ee46fbb7e2d37f6b5fab8052498a", "trusted": true}, "cell_type": "code", "source": "\n\nann_file = '../input/inaturalist-2019-fgvc6/train2019.json'\nwith open(ann_file) as data_file:\n train_anns = json.load(data_file)\ndef get_file_list(ann_file):\n with open(ann_file) as data_file:\n train_anns = json.load(data_file)\n\n train_anns_df = pd.DataFrame(train_anns['annotations'])[['image_id','category_id']]\n train_img_df = pd.DataFrame(train_anns['images'])[['id', 'file_name']].rename(columns={'id':'image_id'})\n train_anns_df [train_anns_df.image_id.duplicated()]\n\n df_train=pd.merge(train_img_df,train_anns_df)[['category_id', 'file_name']]\n return df_train\n\n\ndf_train=get_file_list(ann_file)\ndf_train", "execution_count": null, "outputs": []}, {"metadata": {"trusted": true}, "cell_type": "code", "source": "df_val=get_file_list('../input/inaturalist-2019-fgvc6/val2019.json')\ndf_val", "execution_count": null, "outputs": []}, {"metadata": {"trusted": true}, "cell_type": "code", "source": "train_anns.keys()", "execution_count": null, "outputs": []}, {"metadata": {"trusted": true}, "cell_type": "code", "source": "len(df_train.category_id.unique())", "execution_count": null, "outputs": []}, {"metadata": {"trusted": true}, "cell_type": "code", "source": "df_valid=get_file_list('../input/inaturalist-2019-fgvc6/train2019.json')\ndf_valid", "execution_count": null, "outputs": []}, {"metadata": {"trusted": true}, "cell_type": "code", "source": "classes=len(df_valid.category_id.unique())\nprint(\"random accuracy:\",1/classes)\nclasses", "execution_count": null, "outputs": []}, {"metadata": {"trusted": true}, "cell_type": "code", "source": "df_train[\"catstr\"]=df_train.category_id.astype(\"str\")\n", "execution_count": null, "outputs": []}, {"metadata": {"trusted": true}, "cell_type": "code", "source": "p=next(df_train.sample(n=5).itertuples())\np.category_id\np.file_name", "execution_count": null, "outputs": []}, {"metadata": {"trusted": true}, "cell_type": "code", "source": "plt.figure(figsize=(25,25))\nfor n,tu in enumerate(df_train.sample(n=5).itertuples()):\n cat=tu.category_id\n im=tu.file_name\n plt.subplot(1,5,n+1)\n im=os.path.join(\"../input/inaturalist-2019-fgvc6/train_val2019/\",im)\n plt.axis(\"off\")\n plt.title(cat)\n plt.imshow(imageio.imread(im)) ", "execution_count": null, "outputs": []}, {"metadata": {"trusted": true}, "cell_type": "code", "source": "augmenters=[iaa.Sometimes(0.5, \n [iaa.Affine(scale=(0.99,1.05),\n translate_percent=(0,0.05), \n rotate=iap.Normal(0,3),\n shear=iap.Normal(0,3),\n order=3)]),\n iaa.Sometimes(0.3,[iaa.PiecewiseAffine(scale=(0,0.02))]),\n \n \n \n iaa.Sometimes(0.1,\n [iaa.GaussianBlur(sigma=(0, 0.5)) ]),\n iaa.Sometimes(0.1,\n [iaa.AverageBlur(k=(1, 2))]),\n iaa.AdditiveGaussianNoise(scale=(0, 0.05*255)),\n iaa.AddElementwise((-5, 5)),\n \n iaa.Sometimes(0.1,\n [iaa.Superpixels(p_replace=(0.05, 0.8), n_segments=(16, 128))]\n ),\n \n \n \n \n iaa.Sometimes(0.3,\n [iaa.ElasticTransformation(alpha=(0, 5.0), sigma=(0.1,0.6) )]\n ),\n \n iaa.SaltAndPepper(p=(0.005,0.1)),\n iaa.Sometimes(0.4,\n [iaa.CoarseDropout(p=(0, 0.3),size_percent=(0.02, 0.5))]),\n\n \n \n]\n\naugmenters=[iaa.Sometimes(0.8,[iaa.Sequential(augmenters)])]", "execution_count": null, "outputs": []}, {"metadata": {"trusted": true}, "cell_type": "code", "source": "all(os.path.isfile(os.path.join(\"../input/inaturalist-2019-fgvc6/train_val2019/\",f)) for f in df_train.sample(n=6000).file_name)", "execution_count": null, "outputs": []}, {"metadata": {"trusted": true}, "cell_type": "code", "source": "tu=next(df_train.itertuples())\ntu.file_name\ntu.category_id\n\ndf=df_train.copy()\ndf[\"req\"]=None\n", "execution_count": null, "outputs": []}, {"metadata": {"trusted": true}, "cell_type": "code", "source": "\ndef imgaug_batch_gen(df,batch=16,executor=None,dmax=1024,dmin=512):\n if executor is None:\n executor=futures.ThreadPoolExecutor(max_workers=2)\n prefetch=int(batch*2.1+1) \n df_len=len(df)\n def load_resize(f,dmax=dmax,dmin=dmin):\n img=imageio.imread(f,pilmode=\"RGB\")\n resmin_img=np.min( img.shape[:2] )\n resmin=np.clip(resmin_img,dmin,dmax)\n r=resmin_img/resmin\n \n if r>1.2 or r<1.0:\n img=cv2.resize(img,None,fx=1.05/r, fy=1.05/r, interpolation = cv2.INTER_CUBIC)\n return img\n\n\n while True:\n \n df=df[[\"category_id\",\"file_name\"]].sample(frac=1).reset_index(drop=True)\n df[\"req\"]=None\n i=0\n while i <(df_len-batch):\n df[df.req.notnull()].loc[:i]=None\n for j in range(prefetch):\n try:\n if df.loc[j+i,\"req\"] is None:\n f=os.path.join(\"../input/inaturalist-2019-fgvc6/train_val2019/\",df.loc[j+i,\"file_name\"])\n df.loc[j+i,\"req\"]=executor.submit(load_resize ,f) \n except KeyError:\n logging.exception(\"imgaug_batch_gen\")\n\n df_batch=df.loc[i:i+batch-1]\n imgs=[req.result() for req in df_batch.req]\n resmin=np.min(np.array([im.shape[:2] for im in imgs]))\n resmin=np.clip(resmin,dmin,dmax)\n for j in range(batch):\n x,y,_=imgs[j].shape\n img_resmin=min(x,y)\n r=img_resmin/resmin\n \n if r>1.2 or r<1.0:\n imgs[j]=cv2.resize(imgs[j],None,fx=1.05/r, fy=1.05/r, interpolation = cv2.INTER_CUBIC)\n \n categories=df_batch.category_id.values.astype(\"int32\") \n \n yield imgaug.imgaug.Batch(images=imgs,data=categories) \n df.loc[i:i+batch-1,\"req\"]=None\n i+=batch\n \n\n\ndef batch_gen(df,batch=16,augmenters=[],executor=None,dmax=1024,dmin=512,dcrop=512):\n if executor is None:\n executor=futures.ThreadPoolExecutor(max_workers=2)\n \n dmin=max(dmin,dcrop)\n dmax=max(dmin,dmax)\n gen =imgaug_batch_gen(df,batch=batch,executor=executor,dmax=dmax,dmin=dmin)\n aug=iaa.Sequential(augmenters+[iaa.CropToFixedSize(dmin,dmin),iaa.PadToFixedSize(dmin,dmin)])\n #aug_pool=aug.pool(processes=-1,maxtasksperchild=8)\n #gen=aug_pool.imap_batches_unordered(gen, chunksize=1)\n \n def aug_closure_gen(gen=gen):\n b=next(gen)\n fut=executor.submit(aug.augment_batches,[b],background=False)\n for b in gen:\n imgs=list(fut.result())[0].images_aug\n fut=executor.submit(aug.augment_batches,[b],background=False) \n #b=list(aug.augment_batches([b],background=False))[0]\n\n \n imgs=[im[None,...] for im in imgs]\n X=np.concatenate(imgs).astype(\"float32\")/256\n Y=b.data[...,None] \n yield X,Y\n \n aug_closure=aug_closure_gen(gen)\n\n return aug_closure\n \n \n \n \n \n \n \naug=iaa.Sequential(augmenters)\n#aug_pool=aug.pool(processes=None, maxtasksperchild=None)\n\ngen=batch_gen(df_train,batch=16,augmenters=augmenters)\nX,Y=next(gen)\ndel gen\nb_len=X.shape[0]\ncols=4\nrows=b_len//cols\nif b_len%cols!=0:\n rows=rows+1\n \nplt.figure(figsize=(25,25))\nX.shape[0]\nfor n in range(b_len):\n \n cat=Y[n][0]\n\n plt.subplot(rows,cols,n+1)\n \n plt.axis(\"off\")\n plt.title(cat)\n plt.imshow(X[n]) \n \n \n", "execution_count": null, "outputs": []}, {"metadata": {"trusted": true}, "cell_type": "code", "source": "pretrained_model=keras.applications.Xception(include_top=False, weights=None,pooling=None)\npretrained_model.load_weights(\"../input/xception/xception_weights_tf_dim_ordering_tf_kernels_notop.h5\")\nfichier_modele=f\"{fichier_modele_base}_{pretrained_model.name}.h5\" \nfichier_modele", "execution_count": null, "outputs": []}, {"metadata": {"trusted": true}, "cell_type": "code", "source": "def mean_anwer_guess(y_true, y_pred):\n n=K.get_variable_shape(y_pred)[1]\n y_pred=K.cast(y_pred,\"int64\")\n preds=K.gather(y_true,y_pred)\n return K.mean(preds)\n\n\ndef mean_prandom_ratio(y_true, y_pred):\n n=K.get_variable_shape(y_pred)[1]\n y_pred=K.cast(y_pred,\"int64\")\n preds=K.gather(y_true,y_pred)\n return K.mean(preds*n)\nimport functools\ndef get_sparse_topn__categorical_accuracy(k):\n func=functools.partial(keras.metrics.sparse_top_k_categorical_accuracy,k=k)\n func.__name__=f\"sparse_top_{k}_categorical_accuracy\"\n return func", "execution_count": null, "outputs": []}, {"metadata": {"trusted": true}, "cell_type": "code", "source": "out_regulariser=keras.regularizers.l1_l2(l1=0.01, l2=0.05)\n\nimage_input=keras.Input(shape=(None,None,3), name=\"image_input\", dtype=\"float32\")\nbottleneck1=pretrained_model(image_input)\nbottleneck=keras.layers.Conv2D(filters=600,\n kernel_size=3,padding=\"same\", \n kernel_initializer=keras.initializers.Orthogonal(),\n activation=\"selu\",\n activity_regularizer=keras.regularizers.l1_l2(l1=0.01, l2=0.05),\n strides=2\n \n )(bottleneck1)\n\n\npool1=(keras.layers.GlobalMaxPool2D()(bottleneck1))\n\npool=(keras.layers.GlobalMaxPool2D()(bottleneck))\npool=keras.layers.Concatenate()([pool,pool1])\npool=keras.layers.AlphaDropout(0.3)(pool)\npre_out=keras.layers.Dense(1200 ,\n name=\"pre_out\",\n activation=\"selu\",\n kernel_regularizer=out_regulariser\n )(pool)\nout=keras.layers.Dense(classes,\n activation=\"softmax\"\n ,name=\"out\",\n kernel_initializer=keras.initializers.Orthogonal(),\n kernel_regularizer=out_regulariser)(pre_out)\n\nmodel=keras.Model(inputs=image_input,outputs=out)\npretrained_model.trainable=False\noptimizer=keras.optimizers.Adam(clipnorm=5. , clipvalue=5.,amsgrad=False,lr=0.0005)\nmodel.compile(optimizer,\n loss=\"sparse_categorical_crossentropy\",\n metrics=[\"sparse_categorical_accuracy\",\"sparse_categorical_crossentropy\",mean_anwer_guess,get_sparse_topn__categorical_accuracy(2),\n get_sparse_topn__categorical_accuracy(5),get_sparse_topn__categorical_accuracy(10)]\n )\nmodel.summary()", "execution_count": null, "outputs": []}, {"metadata": {"trusted": true}, "cell_type": "code", "source": "class termination_date(keras.callbacks.Callback ):\n def __init__(self,end_date):\n self.end_date=end_date\n def on_epoch_end(self, batch, logs=None):\n if datetime.datetime.now()>self.end_date:\n self.model.stop_training = True\n logging.info(\"end date\")\n \n \nclass logcallback(keras.callbacks.Callback):\n def __init__(self,logger=None):\n if logger is None:\n logger=logging.getLogger('traincallback')\n\n self.logger=logger\n def on_train_begin(self, logs={}):\n self.logger.info(\"training start: %s\",self.model.name)\n \n\n def on_batch_end(self, batch, logs={}):\n met=\"\"\n for k,v in logs.items():\n met+=f\"{k}: {str(v)} \"\n self.logger.debug(\"batch: %s - %s\",batch,met)\n \n def on_epoch_end(self, epoch, logs=None):\n met=\"\"\n for k,v in logs.items():\n met+=f\"{k}: {str(v)} \"\n self.logger.info(\"epoch: %s - %s\",epoch,met)\n def on_train_end(self, logs={}):\n self.logger.info(\"training end: %s\",self.model.name)\n \n \n \n \n\n", "execution_count": null, "outputs": []}, {"metadata": {"trusted": true}, "cell_type": "code", "source": "callbacks=[\n keras.callbacks.ReduceLROnPlateau(monitor='val_sparse_categorical_accuracy',\n patience=10,\n min_delta=0.0005,\n factor=0.6,\n #min_lr=1e-6,\n verbose=1,\n cooldown=5\n\n ),\n keras.callbacks.ModelCheckpoint(monitor='val_sparse_categorical_accuracy',\n filepath=fichier_modele,\n verbose=1,\n save_best_only=True,\n period=20),\n keras.callbacks.EarlyStopping(monitor='val_sparse_categorical_accuracy',\n patience=20,\n \n verbose=1,\n restore_best_weights=True\n\n ),\n keras.callbacks.CSVLogger(\"train.csv\", separator=',', append=True),\n termination_date(date_limite-datetime.timedelta(minutes=30)),\n keras.callbacks.BaseLogger(),\n logcallback()\n \n\n ]\n", "execution_count": null, "outputs": []}, {"metadata": {"trusted": true}, "cell_type": "code", "source": "", "execution_count": null, "outputs": []}, {"metadata": {"trusted": true}, "cell_type": "code", "source": "if load_keras_weights:\n\n for fp in glob.glob(f\"../input/**/{fichier_modele}\",recursive=True):\n try:\n model.load_weights(fp, by_name=True, skip_mismatch=True)\n logging.info(\"loaded weights:\",fb)\n except Exception as e:\n print(type(e),e)\n logging.exception(\"exception loading: %s %s\",fp,e)\n if os.path.exists(fichier_modele):\n model.load_weights(fichier_modele, by_name=True, skip_mismatch=True)\n logging.info(\"loaded weights:\",fb)", "execution_count": null, "outputs": []}, {"metadata": {"trusted": true}, "cell_type": "code", "source": "\nval_gen=batch_gen(df_val,batch=val_batch_size,dmax=dcrop,dmin=dcrop,dcrop=dcrop)", "execution_count": null, "outputs": []}, {"metadata": {"trusted": true}, "cell_type": "code", "source": "uptime=datetime.datetime.now()-date_depart\nlogging.info(\"pre train start %s\",uptime)", "execution_count": null, "outputs": []}, {"metadata": {"trusted": true}, "cell_type": "code", "source": "hist_pre=model.fit_generator(batch_gen(df_train,batch=train_batch_size,augmenters=augmenters,dmax=dmax,dmin=dmin,dcrop=dcrop),\n steps_per_epoch=1280/train_batch_size, \n epochs=150,\n verbose=1,\n validation_data=val_gen,\n validation_steps=300/val_batch_size,\n callbacks= callbacks+ [ keras.callbacks.EarlyStopping(monitor='val_sparse_categorical_accuracy',\n patience=20,\n \n verbose=1,\n restore_best_weights=True\n\n )]\n \n \n )\nmodel.save(fichier_modele)\n\nlogging.info(\"pre train end %s\",datetime.datetime.now()-date_depart)\nlogging.info(\"remaining time %s\",datetime.timedelta(hours=9)+date_depart-datetime.datetime.now())", "execution_count": null, "outputs": []}, {"metadata": {"trusted": true}, "cell_type": "code", "source": "pretrained_model.trainable=True\noptimizer=keras.optimizers.Adam(clipnorm=5. , clipvalue=5.,amsgrad=True)\nmodel.compile(optimizer,\n loss=\"sparse_categorical_crossentropy\",\n metrics=model.metrics\n )\nmodel.summary()", "execution_count": null, "outputs": []}, {"metadata": {"trusted": true}, "cell_type": "code", "source": "uptime=datetime.datetime.now()-date_depart\nlogging.info(\"main train %s\",uptime)", "execution_count": null, "outputs": []}, {"metadata": {"trusted": true}, "cell_type": "code", "source": "hist=model.fit_generator(batch_gen(df_train,batch=full_train_batch_size,augmenters=augmenters,dmax=dmax,dmin=dmin,dcrop=dcrop),\n steps_per_epoch=1280/full_train_batch_size, epochs=epochs, verbose=1,\n validation_data=val_gen,\n validation_steps=400/val_batch_size,\n callbacks= callbacks \n )\nmodel.save(fichier_modele)\nprint(datetime.datetime.now()-date_depart)\n ", "execution_count": null, "outputs": []}, {"metadata": {"trusted": true}, "cell_type": "code", "source": "uptime=datetime.datetime.now()-date_depart\nlogging.info(\"post train %s\",uptime)\nlogging.info(\"remaining time %s\",datetime.timedelta(hours=9)+date_depart-datetime.datetime.now())", "execution_count": null, "outputs": []}, {"metadata": {"trusted": true}, "cell_type": "code", "source": "if (datetime.datetime.now()-date_depart)