{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import sys","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sys.path.append('/kaggle/input/pyfiles')\nfrom util import *\nfrom train import *\nfrom triplet_loss import *\nfrom initialize_training import *\nfrom evaluation import *","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Train Triplet Individual","metadata":{}},{"cell_type":"code","source":"parent_dir = \"/kaggle/working/\"\nloss = \"hard_triplet\"\nshuffle = False\nlearning_rate = 0.0001\nepochs = 5\ntrain_data_path = \"../input/whalecsv/path-data-train_tri.csv\"\nvalid_data_path = \"../input/whalecsv/path-data-valid_tri.csv\"\ntrain_images_path = \"../input/happy-whale-and-dolphin/train_images\"\ndf_train = pd.read_csv(train_data_path).head(5000)\ndf_valid = pd.read_csv(valid_data_path).head(1000)\ninitialize_training_triplet(parent_dir,loss,shuffle,learning_rate,epochs,df_train,df_valid,train_images_path)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Train Species Embedding","metadata":{}},{"cell_type":"code","source":"parent_dir = \"/kaggle/working/softmax\"\nloss = \"softmax\"\nlearning_rate = 0.001\nto_train = \"species\"\nembedding_size = 30\nepochs = 5\ntrain_data_path = \"../input/whalecsv/path-data-train_emb.csv\"\nvalid_data_path = \"../input/whalecsv/path-data-valid_emb.csv\"\ntrain_images_path = \"../input/happy-whale-and-dolphin/train_images\"\ndf_train = pd.read_csv(train_data_path).head(1000)\ndf_valid = pd.read_csv(valid_data_path).head(50)\ninitialize_training(parent_dir,to_train,loss,embedding_size,learning_rate,epochs,df_train,df_valid,train_images_path)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Train Individual Embedding","metadata":{}},{"cell_type":"code","source":"parent_dir = \"/kaggle/working/individual\"\nloss = \"softmax\"\nlearning_rate = 0.001\nto_train = \"individual_id\"\nembedding_size = 30\nepochs = 2\ntrain_data_path = \"../input/whalecsv/path-data-train_emb.csv\"\nvalid_data_path = \"../input/whalecsv/path-data-valid_emb.csv\"\ntrain_images_path = \"../input/happy-whale-and-dolphin/train_images\"\ndf_train = pd.read_csv(train_data_path).head(30000)\ndf_valid = pd.read_csv(valid_data_path).head(50)\ninitialize_training(parent_dir,to_train,loss,embedding_size,learning_rate,epochs,df_train,df_valid,train_images_path)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Species / Embedding Evaluation","metadata":{}},{"cell_type":"code","source":"parent_dir = \"/kaggle/working/\"\nrow_name = \"species\" # \"species or \"individual_id\"\nto_train = row_name\nloss = \"center\"\nembedding_size = 30\ntrain_data_path = \"../input/whalecsv/path-data-train_emb.csv\"\nvalid_data_path = \"../input/whalecsv/path-data-valid_emb.csv\"\ntrain_images_path = \"../input/happy-whale-and-dolphin/train_images\"\nweights_path = \"../input/species-center/final\"\nsave_file_train = \"/kaggle/working/path-data-train_emb.csv\"\nsave_file_valid = \"/kaggle/working/path-data-train_emb.csv\"\n\ncreate_evaluation_emb(parent_dir,to_train,loss,embedding_size,train_data_path,valid_data_path,train_images_path,weights_path,save_file_train,save_file_valid)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data_path = \"../input/species-center/embedding_train_species_center.csv\"\nvalid_data_path = \"../input/species-center/embedding_valid_species_center.csv\"\nrow_name = \"individual_id\"\nget_accuracies(train_data_path,valid_data_path,row_name)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Individual / Embedding Evaluation","metadata":{}},{"cell_type":"code","source":"parent_dir = \"/kaggle/working/\"\nrow_name = \"individual_id\" # \"species or \"individual_id\"\nto_train = row_name\nloss = \"softmax\"\nembedding_size = 30\ntrain_data_path = \"../input/whalecsv/path-data-train_emb.csv\"\nvalid_data_path = \"../input/whalecsv/path-data-valid_emb.csv\"\ntrain_images_path = \"../input/happy-whale-and-dolphin/train_images\"\nweights_path = \"../input/individual-softmax-files/final\"\nsave_file_train = \"/kaggle/working/path-data-train_emb.csv\"\nsave_file_valid = \"/kaggle/working/path-data-train_emb.csv\"\n\ncreate_evaluation_emb(parent_dir,to_train,loss,embedding_size,train_data_path,valid_data_path,train_images_path,weights_path,save_file_train,save_file_valid)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data_path = \"../input/individual-softmax-files/embedding_train_individual_softmax.csv\"\nvalid_data_path = \"../input/individual-softmax-files/embedding_valid_individual_softmax.csv\"\nrow_name = \"individual_id\"\nget_accuracies(train_data_path,valid_data_path,row_name)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Triplet Evaluation","metadata":{}},{"cell_type":"code","source":"train_data_path = \"../input/whalecsv/path-data-train_emb.csv\"\nvalid_data_path = \"../input/whalecsv/path-data-valid_emb.csv\"\n\ntrain_images_path = \"../input/happy-whale-and-dolphin/train_images\"\nweights_path = \"../input/triweigths/final\"\n\nsave_file_train = \"/kaggle/working/path-data-train_emb.csv\"\nsave_file_valid = \"/kaggle/working/path-data-train_emb.csv\"\n\ncreate_evaluation_triplet(train_data_path,valid_data_path,train_images_path,weights_path,save_file_train,save_file_valid)\nprint(\"Done\")\nget_accuracies(save_file_train,save_file_valid,\"individual_id\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Create DataFrames","metadata":{}},{"cell_type":"code","source":"df = pd.read_csv('/kaggle/input/happy-whale-and-dolphin/train.csv')\nsave_path = \"/kaggle/working/\"\nprepare_df(df,save_path)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Statistics Plots","metadata":{}},{"cell_type":"markdown","source":"## Plot Dataset Distributions","metadata":{}},{"cell_type":"code","source":"df_train = pd.read_csv(\"../input/whalecsv/path-data-train_emb.csv\")\ndf_valid = pd.read_csv(\"../input/whalecsv/path-data-valid_emb.csv\")\n\nspecies_train = df_train[\"species\"]\nspecies_valid = df_valid[\"species\"]\nindividuals_train = df_train[\"individual_id\"]\nindividuals_valid = df_valid[\"individual_id\"]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.rcParams['figure.figsize'] = [6, 3]\nplt.hist(species_train,bins = 30,label =\"training\",color =\"red\", histtype='step', stacked=True, fill=False)\nplt.hist(species_valid,bins = 30,label =\"validation\",color =\"green\", histtype='step', stacked=True, fill=False)\nplt.legend()\nplt.tight_layout()\nplt.savefig(\"/kaggle/working/species\")\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.rcParams['figure.figsize'] = [6, 3]\nplt.hist(individuals_train,bins=15587, label =\"training\",color =\"red\", histtype='step', stacked=True, fill=False)\nplt.hist(individuals_valid,bins=15587, label =\"validation\",color = \"green\", histtype='step', stacked=True, fill=False)\nplt.legend()\nplt.ylim([0,350])\nprint(\"Plotting...\")\nplt.savefig(\"/kaggle/working/individuals\")\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Plot Images","metadata":{}},{"cell_type":"code","source":"df_train = pd.read_csv(\"../input/whalecsv/path-data-train_emb.csv\").head(40)\ndf_valid = pd.read_csv(\"../input/whalecsv/path-data-valid_emb.csv\").head(10)\nstep =\"class\"\n# create tensorflow datasets from pandas dataframes\nds_train = create_dataset(df_train,\"../input/happy-whale-and-dolphin/train_images\",)\nds_valid = create_dataset(df_valid,\"../input/happy-whale-and-dolphin/train_images\",)\n# preprocess datasets\nds_train_aug = preprocess_color(ds_train,\"train\",False)\nds_train_norm = preprocess_color(ds_train,\"\",False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for image, label in ds_train_aug.take(1):\n    f, ax = plt.subplots(1,4) \n    \n# use the created array to output your multiple images. In this case I have stacked 4 images vertically\n    ax[0].imshow(image[5],cmap='gray')\n    ax[1].imshow(image[6],cmap='gray')\n    ax[2].imshow(image[3],cmap='gray')\n    ax[3].imshow(image[4],cmap='gray')\n    plt.savefig(\"/kaggle/working/imgs_augmented\", dpi=1200)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for image, label, in ds_train_norm.take(1):\n    f, ax = plt.subplots(1,4) \n    \n# use the created array to output your multiple images. In this case I have stacked 4 images vertically\n    ax[0].imshow(image[5],cmap='gray')\n    ax[1].imshow(image[6],cmap='gray')\n    ax[2].imshow(image[3],cmap='gray')\n    ax[3].imshow(image[4],cmap='gray')\n    plt.savefig(\"/kaggle/working/imgs_normal\", dpi=1200)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}