{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Peek"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport tensorflow\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\nimport time\nimport gc\nstart_time = time.time()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Import Modules"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"random_seed = 17025\nnp.random.seed(random_seed)\nfrom tensorflow.keras.models import load_model\nfrom sklearn.metrics import confusion_matrix\nfrom skimage.transform import resize\nimport matplotlib.pyplot as plt","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Read Data"},{"metadata":{"trusted":true},"cell_type":"code","source":"#Load train and test data\ntrain = pd.read_csv('../input/bengaliai-cv19/train.csv')\ntest = pd.read_csv('../input/bengaliai-cv19/test.csv')\ngmap = pd.read_csv('../input/bengaliai-cv19/class_map.csv')\n\npq_paths = {'Train_0':'../input/bengaliai-cv19/train_image_data_0.parquet',\n            'Train_1':'../input/bengaliai-cv19/train_image_data_1.parquet',\n            'Train_2':'../input/bengaliai-cv19/train_image_data_2.parquet',\n            'Train_3':'../input/bengaliai-cv19/train_image_data_3.parquet',\n            'Test_0':'../input/bengaliai-cv19/test_image_data_0.parquet',\n            'Test_1':'../input/bengaliai-cv19/test_image_data_1.parquet',\n            'Test_2':'../input/bengaliai-cv19/test_image_data_2.parquet',\n            'Test_3':'../input/bengaliai-cv19/test_image_data_3.parquet'}\n\ntarget_cols = ['grapheme_root','vowel_diacritic','consonant_diacritic']","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Pre-processing"},{"metadata":{"trusted":true},"cell_type":"code","source":"def values_from_pqt(name, batch_size):\n    \n    pqt = pd.read_parquet(pq_paths[name])\n    \n    PQT_COLUMNS = pqt.columns[1:]\n    concurrent_n = batch_size//4\n    \n    for i in range(0, pqt.shape[0], batch_size):\n        \n        accumulator_x = np.array([], dtype='float16')\n        \n        for n in range(i, i+batch_size, concurrent_n):\n            print('PQT %s begin %i end %i' %(name, i, n+concurrent_n))\n            try:\n                value = 1 - (pqt[PQT_COLUMNS][n:n+concurrent_n].values.reshape(concurrent_n, 137, 236, 1))/255\n                value = resize(value, (concurrent_n, 80, 80, 1))\n            except:\n                value = pqt[PQT_COLUMNS][n:].values\n                if value.shape[0]:\n                    value = 1 - (value.reshape(value.shape[0], 137, 236, 1))/255\n                    value = resize(value, (value.shape[0], 80, 80, 1))\n                else:\n                    break\n            try:\n                accumulator_x = np.concatenate([accumulator_x, value.astype('float16')], axis=0)\n            except:\n                accumulator_x = value.astype('float16')\n        try:\n            yield (np.array(accumulator_x).astype('float16'), pqt.image_id[i:i+batch_size].values.tolist())\n        except:\n            yield (np.array(accumulator_x).astype('float16'), pqt.image_id[i:].values.tolist())\n    del pqt\n    gc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Part 3: Prediction"},{"metadata":{"trusted":true},"cell_type":"code","source":"prefix = \"../input/three-part-model-training-kernel-only-part-2-3/\" #Viable to change\n\ntargets = {'grapheme_root': load_model(prefix+'multi_out_cnn_model_root.h5'),\n           'vowel_diacritic': load_model(prefix+'multi_out_cnn_model_vowel.h5'),\n           'consonant_diacritic': load_model(prefix+'multi_out_cnn_model_consonant.h5')\n          }\n\ntarget_cols = ['consonant_diacritic','grapheme_root','vowel_diacritic'] #Arrange in this order in sample_submission\n\npredictions = {'row_id':[], 'target':{\n    'consonant_diacritic':[],\n    'grapheme_root':[],\n    'vowel_diacritic':[]}\n              }\n\nprint(\"Start reading data:\")\n\nfor i in range(4):\n    for test_x, ids in values_from_pqt(\"Test_{}\".format(i), 10000):\n        print(\"Currently Predicting: Test_{}\".format(i))\n        #For X values\n        for component in target_cols:\n            print(\"For %s:\" %component)\n            preds = np.argmax(targets[component].predict(test_x), axis=1)\n            try:\n                predictions['target'][component] = np.concatenate([\n                    predictions['target'][component], preds],\n                    axis=0\n                )\n            except:\n                predictions['target'][component] = preds\n        #For Y values\n        try:\n            predictions['row_id'] = np.concatenate([\n                predictions['row_id'], ids\n            ])\n        except:\n            predictions['row_id'] = ids\n        print(\"#\"*72)\nids = []\ntgt = []\nfor i in range(len(predictions['row_id'])):\n    for col in target_cols:\n        ids.append(predictions['row_id'][i]+'_'+col)\n        tgt.append(int(predictions['target'][col][i]))\n\nsubmission = pd.DataFrame()\nsubmission['row_id'] = ids\nsubmission['target'] = tgt\nsubmission.to_csv('submission.csv', index=False)\nprint(submission)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# End"},{"metadata":{"trusted":true},"cell_type":"code","source":"end_time = time.time()\ntotal_time = end_time - start_time\nhours = total_time//3600\nminutes = (total_time%3600)//60\nseconds = (total_time%60)\nprint(\"Total Time spent is: %i hours, %i minutes, and %i seconds\" %((hours, minutes, seconds)))","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}