{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport json, codecs\nfrom collections import Counter","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-03-28T13:53:51.925961Z","iopub.execute_input":"2022-03-28T13:53:51.926455Z","iopub.status.idle":"2022-03-28T13:53:51.931023Z","shell.execute_reply.started":"2022-03-28T13:53:51.926402Z","shell.execute_reply":"2022-03-28T13:53:51.930340Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"It's a simple verification of iwildcam2022_test_information.json","metadata":{}},{"cell_type":"code","source":"def extractor(lis, nam):\n    length = len(lis)\n    result = ['' for i in range(length)]\n    for i in range(length):\n        result[i] = lis[i][nam]\n    return result\n\ndef deletor(lis, inde):    \n    if len(inde)>0:\n        inde.sort()\n        lis_new = lis.copy()\n        for i in range(len(inde)):\n            del lis_new[inde[i]-i]\n        return lis_new\n    else:\n        return lis\n    \ndef ploter(seq_all, num):\n    import matplotlib.pyplot as plt\n    import matplotlib.image as imgplt\n    \n    seq = seq_all[num]['image']\n    file_name = extractor(seq, 'file_name')\n    \n    plt.figure(figsize=(30,30))\n    for i in range(len(file_name)):            \n        x = imgplt.imread('../train/train/' + file_name[i])\n        plt.subplot(3,4,i+1)\n        plt.imshow(x)","metadata":{"execution":{"iopub.status.busy":"2022-03-28T13:54:54.099686Z","iopub.execute_input":"2022-03-28T13:54:54.099979Z","iopub.status.idle":"2022-03-28T13:54:54.109661Z","shell.execute_reply.started":"2022-03-28T13:54:54.099950Z","shell.execute_reply":"2022-03-28T13:54:54.109021Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with codecs.open(\"../input/iwildcam2022-fgvc9/metadata/metadata/iwildcam2022_train_annotations.json\", 'r',\n                 encoding='utf-8', errors='ignore') as f:\n    train_meta = json.load(f)\n               \nfile_name = extractor(train_meta['images'], 'id')\nseq_name = extractor(train_meta['images'], 'seq_id')\n\nfile_name2 = extractor(train_meta['annotations'], 'image_id')\ncate_name = extractor(train_meta['annotations'], 'category_id')","metadata":{"execution":{"iopub.status.busy":"2022-03-28T14:01:04.621539Z","iopub.execute_input":"2022-03-28T14:01:04.621851Z","iopub.status.idle":"2022-03-28T14:01:06.849501Z","shell.execute_reply.started":"2022-03-28T14:01:04.621819Z","shell.execute_reply":"2022-03-28T14:01:06.848531Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Reading data","metadata":{}},{"cell_type":"code","source":"seq_name_format = dict(Counter(seq_name))\nseq_all = ['' for i in range(len(seq_name_format))]\n\ni = 0\nseq_loc = list()\nseq_loc_empty = list()\nseq_loc.append(-1)\nseq_loc_temp = seq_loc.copy()\n\nfor key,value in seq_name_format.items():\n\n    seq_loc = [j+seq_loc_temp[-1] for j in range(1,value+1)]\n    image = seq_loc.copy()\n    categories = seq_loc.copy()\n    is_seq_empty = 1\n    empty_frame_loc = list()\n    \n    for j in range(1,value+1):\n               \n        loc = j+seq_loc_temp[-1]\n        \n        seq_frame_num = train_meta['images'][loc]['seq_frame_num']\n        category = cate_name[file_name2.index(file_name[loc])]\n        if category == 0:\n            empty_frame_loc.append(seq_frame_num)\n        else:\n            is_seq_empty = 0\n            categories[seq_frame_num] = category\n            image[seq_frame_num] = train_meta['images'][loc]\n        \n    if is_seq_empty == 1:\n        seq_loc_empty.append(i)\n    else:\n        categories = deletor(categories, empty_frame_loc)\n        image = deletor(image, empty_frame_loc)\n       \n    # seq_loc = [i+seq_loc_temp[-1] for i in range(1,value+1)]\n    # image = [train_meta['images'][i+seq_loc_temp[-1]] for i in range(1,value+1)]\n    # category = [cate_name[file_name2.index(file_name[i+seq_loc_temp[-1]])] for i in range(1,value+1)]\n\n    seq_loc_temp = seq_loc.copy()\n    seq_all[i] = {'seq_id':key,'image':image,'category_id':categories, 'is_irr':False}\n    i += 1\n    if i%1000 ==0:\n        print(i)\n\nseq_all = deletor(seq_all, seq_loc_empty)","metadata":{"execution":{"iopub.status.busy":"2022-03-28T14:01:38.939006Z","iopub.execute_input":"2022-03-28T14:01:38.939302Z","iopub.status.idle":"2022-03-28T14:14:06.255852Z","shell.execute_reply.started":"2022-03-28T14:01:38.939273Z","shell.execute_reply":"2022-03-28T14:14:06.254756Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Dealing with sequences. I found some totally empty, others partly empty, but the majority is intact. I reckon empty pictures valuless, therefore, they have been removed from the re-organised variable: seq_all ","metadata":{}},{"cell_type":"code","source":"categories = extractor(seq_all, 'category_id')\nirr_cate_loc = list()\n\nfor i in range(len(categories)):\n    category = categories[i]\n    x_s = category[0]\n    is_irr = 0\n    for x in category:\n        if x != x_s:\n            is_irr = 1\n            seq_all[i]['is_irr'] = True\n            break\n    if is_irr == 1:\n        irr_cate_loc.append(i)","metadata":{"execution":{"iopub.status.busy":"2022-03-28T14:14:38.683146Z","iopub.execute_input":"2022-03-28T14:14:38.683620Z","iopub.status.idle":"2022-03-28T14:14:38.721943Z","shell.execute_reply.started":"2022-03-28T14:14:38.683569Z","shell.execute_reply":"2022-03-28T14:14:38.720994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Another stuff intrigue: Some sequences contains more than one species， which means it cannot be described as only one category. So there is another tag in seq_all: 'is_irr', that is, is it irrgular. Luckily, most sequences are regular","metadata":{}},{"cell_type":"code","source":"#with open('../metadata/metadata/seq_all.json', 'w') as f:\n#   json.dump(seq_all, f)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"seq_all saved for further analysis, and I deem the data structure of seq_all.json superior as iwildcam2022_train_annotations.json","metadata":{}}]}