{"cells":[{"metadata":{},"cell_type":"markdown","source":"## Import Libraries"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport os\nimport pandas as pd\nimport shutil\nfrom pathlib import Path\nfrom shutil import unpack_archive\nfrom subprocess import check_output\nprint(os.listdir(\"../input/recursion-cellular-image-classification\"))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Unpack zip files"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"path = '../input/recursion-cellular-image-classification' # specify directory where the competition zip files were downloaded\nfileList = os.listdir(path) # get file list in the path directory\n# list files\nprint(fileList)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# unpack zip files in path directory\nfor i in fileList:\n    if \"zip\" in i:\n        unpack_archive(path + i, path, 'zip')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fileList = os.listdir(path) # get updated file list in path directory\n# list files/folders\nprint(fileList)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# populate train.csv into a pandas DataFrame\ntrain = pd.read_csv(path + '/train.csv')\nprint(train.shape) # print DataFrame shape\ntrain.head() # print first 5 records of the DataFrame","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# populate test.csv into a pandas DataFrame\ntest = pd.read_csv(path + '/test.csv')\nprint(test.shape) # print DataFrame shape\ntest.head() # print first 5 records of the DataFrame","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Update directory structore"},{"metadata":{"trusted":true},"cell_type":"code","source":"# create DataFrame containing a sorted list of sirna from the train DataFrame\nimgClasses = train.sirna.sort_values(axis=0)\n# create Dataframe containing a unique list of sirna from the train DataFrame\nimgClassesUnique = imgClasses.unique()\nprint(imgClassesUnique,len(imgClassesUnique))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"shutil.rmtree(work_path + \"train\")\nshutil.rmtree(work_path + \"valid\")\nshutil.rmtree(work_path + \"test\")\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# create train, valid, and test directories\n# create train, valid, and test directories\nwork_path = '../working/'\nshutil.os.mkdir(work_path + \"train\")\nshutil.os.mkdir(work_path + \"valid\")\nshutil.os.mkdir(work_path + \"test\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(os.listdir(work_path+'train'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# create directories for each unique species in the leafClassesUnique Dataframe in the train and valid directories previous created\nfor i in imgClassesUnique:\n    shutil.os.mkdir(work_path + 'train/' + str(i))\n    shutil.os.mkdir(work_path + 'valid/' + str(i))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(os.listdir(path))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Create the modified Test csv file appropriate for the work\ndftest = pd.read_csv(path+'/test.csv');\ndftest = dftest['id_code'];\ndictest = {};\nfor fold1 in os.listdir(path+'/test'):\n    for fold2 in os.listdir(path+'/test/'+fold1):\n        for image in os.listdir(path+'/test/'+fold1+'/'+fold2):\n            dictest[str(fold1)+'_'+fold2[5:]+'_'+image[0:3]] = str(fold1)+'/'+fold2+'/'+image;\ntestData = pd.DataFrame(list(dictest.items()), columns=['id_code','foldPath']);\ntestData.to_csv(r'../working/testData.csv', index = None, header=True);\n#print(testData.tail())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# move the test images from the images directory to the test directory\nfor i in testData.iloc[:,1]: # test image labels are identified in the id column of the test DataFrame\n    shutil.copy(path + \"/test/\"+ i, work_path + \"test\")\n    data_file = Path(work_path + 'test/'+i.split('/')[-1])\n    data_file.rename(work_path + 'test/'+ i.split('/')[0]+'_'+str(i.split('/')[1][5:])+'_'+i.split('/')[-1][0:3] + '.png')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#print(os.listdir(work_path+'/test'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Create the modified Train csv file appropriate for the work\npath = '../input/recursion-cellular-image-classification'\ndftrain = pd.read_csv(path+'/train.csv');\ndftrain = dftrain[['id_code','sirna']];\ndic = {};\nfor fold1 in os.listdir(path+'/train'):\n    for fold2 in os.listdir(path+'/train/'+fold1):\n        for image in os.listdir(path+'/train/'+fold1+'/'+fold2):\n            dic[str(fold1)+'_'+fold2[5:]+'_'+image[0:3]] = str(fold1)+'/'+fold2+'/'+image;\ndf = pd.DataFrame(list(dic.items()), columns=['id_code','Item']);\ndftraindf = pd.merge(df, dftrain);\ntrainData = dftraindf[['Item','sirna']];\ntrainData.to_csv(r'../working/trainData.csv', index = None, header=True);\nprint(trainData.head())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for index, row in trainData.iterrows(): # test image labels are identified in the id column of the test DataFrame\n    leave_path = path + '/train/'+ row[0]\n    dest_path = work_path + 'train/' + str(row[1])\n    #print(leave_path, dest_path)\n    shutil.copy(leave_path, dest_path)\n    data_file = Path(work_path + 'train/' +str(row[1])+'/'+row[0].split('/')[-1])\n    #work_path + 'train/' +str(row[1])+'/'+row[0].split('/')[-1]\n    data_file.rename(work_path + 'train/' +str(row[1])+'/'+row[0].split('/')[0]+'_'+str(row[0].split('/')[1][5:])+'_'+row[0].split('/')[-1][0:3] + '.png')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#print(os.listdir(work_path+'/train/11'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"for name in dir():\n    if not name.startswith('_'):\n        del globals()[name]\"\"\"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}