{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport warnings\nwarnings.filterwarnings(action='ignore')\n\nimport pandas as pd\nimport librosa\nimport numpy as np\n\nfrom sklearn.utils import shuffle\nfrom PIL import Image\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\n\nimport tensorflow as tf\n# Global vars\nRANDOM_SEED = 1337\nSAMPLE_RATE = 32000\nSIGNAL_LENGTH = 5 # seconds\nSPEC_SHAPE = (48, 128) # height x width\nFMIN = 500\nFMAX = 12500","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load metadata file\ntrain = pd.read_csv('../input/birdclef-2021/train_metadata.csv',)\n\n# Limit the number of training samples and classes\n\n\n# Second, assume that birds with the most training samples are also the most common\n# A species needs at least 200 recordings with a rating above 4 to be considered common\nbirds_count = {}\nfor bird_species, count in zip(train.primary_label.unique(), \n                               train.groupby('primary_label')['primary_label'].count().values):\n    birds_count[bird_species] = count\nmost_represented_birds = [key for key,value in birds_count.items()] \n\nTRAIN = train.query('primary_label in @most_represented_birds')\nLABELS = sorted(TRAIN.primary_label.unique())\n\n# Let's see how many species and samples we have left\nprint('NUMBER OF SPECIES IN TRAIN DATA:', len(LABELS))\nprint('NUMBER OF SAMPLES IN TRAIN DATA:', len(TRAIN))\nprint('LABELS:', most_represented_birds)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TRAIN_SPECS = []\nfor folder in os.listdir(\"../input/saving-melspecs/melspectrogram_dataset/\"):\n    for files in os.listdir(\"../input/saving-melspecs/melspectrogram_dataset/\"+folder):\n        TRAIN_SPECS.append(\"../input/saving-melspecs/melspectrogram_dataset/\"+folder+\"/\"+files)\n#print(TRAIN_SPECS)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(15, 7))\nfor i in range(12):\n    spec = Image.open(TRAIN_SPECS[i])\n    plt.subplot(3, 4, i + 1)\n    plt.title(TRAIN_SPECS[i].split(os.sep)[-1])\n    plt.imshow(spec, origin='lower')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_specs, train_labels = [], []\nwith tqdm(total=len(TRAIN_SPECS)) as pbar:\n    i =0\n    for path in TRAIN_SPECS:\n        pbar.update(1)\n\n        # Open image\n        spec = Image.open(path)\n\n        # Convert to numpy array\n        spec = np.array(spec, dtype='float32')\n        \n        # Normalize between 0.0 and 1.0\n        # and exclude samples with nan \n        spec -= spec.min()\n        spec /= spec.max()\n        if not spec.max() == 1.0 or not spec.min() == 0.0:\n            continue\n\n        # Add channel axis to 2D array\n        spec = np.expand_dims(spec, -1)\n\n        # Add new dimension for batch size\n        spec = np.expand_dims(spec, 0)\n\n        # Add to train data\n        if len(train_specs) == 0:\n            train_specs = spec\n        else:\n            train_specs = np.vstack((train_specs, spec))\n\n        # Add to label data\n        target = np.zeros((len(LABELS)), dtype='float32')\n        bird = path.split(os.sep)[-2]\n        target[LABELS.index(bird)] = 1.0\n        if len(train_labels) == 0:\n            train_labels = target\n        else:\n            train_labels = np.vstack((train_labels, target))\n        i+=1\n        if i%1000==0:\n            trainSpecsfileName = \"../working/\"+ str(i) + \"_train_specs.npy\"\n            np.save(trainSpecsfileName, train_specs)\n            \n            trainLabelfileName = \"../working/\"+ str(i) + \"_train_label.npy\"\n            np.save(trainLabelfileName, train_labels)\n            train_specs = []\n            train_labels = []\n\n            ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.save(\"../working/Last_train_specs.npy\", train_specs)\nnp.save(\"../working/Last_train_labels.npy\", train_labels)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_labels.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_specs.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Converting all these indivdual NPY files into One","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport os","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"listFiles = []\nfor file in os.listdir(\"./\"):\n\n    if file[-9:] == \"specs.npy\":\n        try:\n            listFiles.append(int(file[:-16]))\n        except:\n            print(file)\n\nlistFiles.sort()\nprint(listFiles)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os.path\nfrom os import path\n\nfor i in listFiles:\n    pathForSpecs = \"./\" + str(i)+\"_train_specs.npy\"\n    pathForLabel = \"./\" + str(i)+\"_train_label.npy\"\n    \n    if not path.exists(pathForSpecs) or not path.exists(pathForLabel):\n        print(i)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(listFiles)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_specs_0To50k = []\ntrain_specs_0To50k = np.load(\"./1000_train_specs.npy\")\n\nfor i in range(1,50):\n    print(listFiles[i])\n    trainSpecsCurrentArray = np.load(\"./\" + str(listFiles[i])+\"_train_specs.npy\")\n    arrSpecs = np.concatenate((train_specs_0To50k,trainSpecsCurrentArray), axis=0)\n    train_specs_0To50k = arrSpecs\n\ntrain_specs_50To100k = np.load(\"./\" + str(listFiles[50])+\"_train_specs.npy\")\nfor i in range(51,100):\n    print(listFiles[i])\n    trainSpecsCurrentArray = np.load(\"./\" + str(listFiles[i])+\"_train_specs.npy\")\n    arrSpecs = np.concatenate((train_specs_50To100k,trainSpecsCurrentArray), axis=0)\n    train_specs_50To100k = arrSpecs\n\ntrain_specs_100To150k = np.load(\"./\" + str(listFiles[100])+\"_train_specs.npy\")\nfor i in range(101,150):\n    print(listFiles[i])\n    trainSpecsCurrentArray = np.load(\"./\" + str(listFiles[i])+\"_train_specs.npy\")\n    arrSpecs = np.concatenate((train_specs_100To150k,trainSpecsCurrentArray), axis=0)\n    train_specs_100To150k = arrSpecs\n    #train_specs_100To150k.shape\n    \ntrain_specs_150To172k = np.load(\"./\" + str(listFiles[150])+\"_train_specs.npy\")\nfor i in range(151,len(listFiles)):\n    print(listFiles[i])\n    trainSpecsCurrentArray = np.load(\"./\" + str(listFiles[i])+\"_train_specs.npy\")\n    arrSpecs = np.concatenate((train_specs_150To172k,trainSpecsCurrentArray), axis=0)\n    train_specs_150To172k = arrSpecs","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\n#Save 50 iteration x 4 files\nnp.save(\"./train_specs_0To50k.npy\",train_specs_0To50k)\nnp.save(\"./train_specs_50To100k.npy\",train_specs_50To100k)\nnp.save(\"./train_specs_100To150k.npy\",train_specs_100To150k)\nnp.save(\"./train_specs_150To172k.npy\",train_specs_150To172k)\ntrain_specs_0To50k = []\ntrain_specs_50To100k = []\ntrain_specs_100To150k = []\ntrain_specs_150To172k = []\n'''","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_specs_0To100k =  np.concatenate((train_specs_0To50k,train_specs_50To100k), axis=0)\ntrain_specs_0To50k = []\ntrain_specs_50To100k = []\ntrain_specs_100To170k =  np.concatenate((train_specs_100To150k,train_specs_150To172k), axis=0)\ntrain_specs_100To150k = []\ntrain_specs_150To172k = []\nprint(train_specs_0To100k.shape,train_specs_100To170k.shape)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\n#Save 100K iteration x 2 files\nnp.save(\"./train_specs_0To100k.npy\",train_specs_0To100k)\nnp.save(\"./train_specs_100To170k.npy\",train_specs_100To170k)\nprint(train_specs_0To100k.shape,train_specs_100To170k.shape)\ntrain_specs_0To100k = []\ntrain_specs_100To170k = []\n\n'''","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_specs_0To170k =  np.concatenate((train_specs_0To100k,train_specs_100To170k), axis=0)\nprint(train_specs_0To170k.shape)\ntrain_specs_0To100k = []\ntrain_specs_100To170k = []","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.save(\"./train_specs_0To170k.npy\",train_specs_0To170k)\ntrain_specs_0To170k = []","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"file_size = os.path.getsize('./train_specs_0To170k.npy')\nprint(\"File Size is :\", file_size, \"bytes\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_label_0To50k = []\ntrain_label_0To50k = np.load(\"./1000_train_label.npy\")\n\nfor i in range(1,50):\n    print(listFiles[i])\n    trainlabelCurrentArray = np.load(\"./\" + str(listFiles[i])+\"_train_label.npy\")\n    arrSpecs = np.concatenate((train_label_0To50k,trainlabelCurrentArray), axis=0)\n    train_label_0To50k = arrSpecs\n\ntrain_label_50To100k = np.load(\"./\" + str(listFiles[50])+\"_train_label.npy\")\nfor i in range(51,100):\n    print(listFiles[i])\n    trainlabelCurrentArray = np.load(\"./\" + str(listFiles[i])+\"_train_label.npy\")\n    arrSpecs = np.concatenate((train_label_50To100k,trainlabelCurrentArray), axis=0)\n    train_label_50To100k = arrSpecs\n\ntrain_label_100To150k = np.load(\"./\" + str(listFiles[100])+\"_train_label.npy\")\nfor i in range(101,150):\n    print(listFiles[i])\n    trainlabelCurrentArray = np.load(\"./\" + str(listFiles[i])+\"_train_label.npy\")\n    arrSpecs = np.concatenate((train_label_100To150k,trainlabelCurrentArray), axis=0)\n    train_label_100To150k = arrSpecs\n    \ntrain_label_150To172k = np.load(\"./\" + str(listFiles[150])+\"_train_label.npy\")\nfor i in range(151,len(listFiles)):\n    print(listFiles[i])\n    trainlabelCurrentArray = np.load(\"./\" + str(listFiles[i])+\"_train_label.npy\")\n    arrSpecs = np.concatenate((train_label_150To172k,trainlabelCurrentArray), axis=0)\n    train_label_150To172k = arrSpecs","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train_label_0To50k.shape)\nprint(train_label_50To100k.shape)\nprint(train_label_100To150k.shape)\nprint(train_label_150To172k.shape)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\n#Save 50 iteration x 4 files\nnp.save(\"./train_label_0To50k.npy\",train_label_0To50k)\nnp.save(\"./train_label_50To100k.npy\",train_label_50To100k)\nnp.save(\"./train_label_100To150k.npy\",train_label_100To150k)\nnp.save(\"./train_label_150To172k.npy\",train_label_150To172k)\ntrain_label_0To50k = []\ntrain_label_50To100k = []\ntrain_label_100To150k = []\ntrain_label_150To172k = []\n'''","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_label_0To100k =  np.concatenate((train_label_0To50k,train_label_50To100k), axis=0)\ntrain_label_0To50k = []\ntrain_label_50To100k = []\ntrain_label_100To170k =  np.concatenate((train_label_100To150k,train_label_150To172k), axis=0)\ntrain_label_100To150k = []\ntrain_label_150To172k = []\nprint(len(train_label_0To100k),len(train_label_100To170k))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''#Save 100K iteration x 2 files\nnp.save(\"./train_label_0To100k.npy\",train_label_0To100k)\nnp.save(\"./train_label_100To170k.npy\",train_label_100To170k)\nprint(train_label_0To100k.shape,train_label_100To170k.shape)\ntrain_label_0To100k = []\ntrain_label_100To170k = []'''","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_label_0To170k =  np.concatenate((train_label_0To100k,train_label_100To170k), axis=0)\ntrain_label_0To100k = []\ntrain_label_100To170k = []","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.save(\"./train_label_0To170k.npy\",train_label_0To170k)\ntrain_label_0To170k.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"file_size = os.path.getsize('./train_label_0To170k.npy')\nprint(\"File Size is :\", file_size, \"bytes\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Delete All Individual NPY files","metadata":{}},{"cell_type":"code","source":"for i in listFiles:\n    pathForSpecs = \"./\" + str(i)+\"_train_specs.npy\"\n    pathForLabel = \"./\" + str(i)+\"_train_label.npy\"\n    \n    os.remove(pathForSpecs)\n    os.remove(pathForLabel)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Test for Concatenate","metadata":{}},{"cell_type":"code","source":"'''fisrtArraySpecs = np.load(\"./1000_train_specs.npy\")\nsecondArraySpecs = np.load(\"./2000_train_specs.npy\")\n\nprint(fisrtArraySpecs[:4].shape,secondArraySpecs[:4].shape)\narr = np.concatenate((fisrtArraySpecs[:4], secondArraySpecs[:4]), axis=0)\nprint(arr.shape)'''","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''from PIL import Image as im\n#data = im.fromarray(array)\n\nplt.figure(figsize=(15, 7))\nfor i in range(8):\n    newArray = (arr[i] * 255).astype(np.uint8)\n    newArray = np.squeeze(newArray, axis=2)\n    spec = Image.fromarray(newArray)\n    plt.subplot(2, 4, i + 1)\n    #plt.title(TRAIN_SPECS[i].split(os.sep)[-1])\n    plt.imshow(spec, origin='lower')'''","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''from PIL import Image as im\n#data = im.fromarray(array)\n\nplt.figure(figsize=(15, 7))\nfor i in range(4):\n    newArray = (fisrtArraySpecs[i] * 255).astype(np.uint8)\n    newArray = np.squeeze(newArray, axis=2)\n    spec = Image.fromarray(newArray)\n    plt.subplot(2, 4, i + 1)\n    #plt.title(TRAIN_SPECS[i].split(os.sep)[-1])\n    plt.imshow(spec, origin='lower')'''","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''from PIL import Image as im\n#data = im.fromarray(array)\n\nplt.figure(figsize=(15, 7))\nfor i in range(4):\n    newArray = (secondArraySpecs[i] * 255).astype(np.uint8)\n    newArray = np.squeeze(newArray, axis=2)\n    spec = Image.fromarray(newArray)\n    plt.subplot(2, 4, i + 1)\n    #plt.title(TRAIN_SPECS[i].split(os.sep)[-1])\n    plt.imshow(spec, origin='lower')'''","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}