{"cells":[{"metadata":{},"cell_type":"markdown","source":"**1_data****"},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport os\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_dir = '../input/bengaliai-cv19/'\nfiles_train = [f'train_image_data_{fid}.parquet' for fid in range(4)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"files_train","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"F = os.path.join(data_dir, files_train[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"F","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train0 = pd.read_parquet(F)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train0.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train0.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"idx = np.random.randint(len(train0))\nimg = train0.iloc[idx, 1:].values.astype(np.uint8)\n\nplt.imshow(255 - img.reshape(137,236), cmap ='gray')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train = pd.read_csv('../input/bengaliai-cv19/train.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize = (10,20))\ndf_train['grapheme_root'].value_counts().sort_index().plot.barh()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize = (10,10))\ndf_train['vowel_diacritic'].value_counts().sort_index().plot.barh()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize = (10,10))\ndf_train['consonant_diacritic'].value_counts().sort_index().plot.barh()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pip install iterative-stratification","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from iterstrat.ml_stratifiers import MultilabelStratifiedKFold","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train['id'] = df_train['image_id'].apply(lambda x: int(x.split('_')[1]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X = df_train[['id', 'grapheme_root', 'vowel_diacritic', 'consonant_diacritic']].values[:,0]\ny = df_train[['id', 'grapheme_root', 'vowel_diacritic', 'consonant_diacritic']].values[:,1:]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"mskf = MultilabelStratifiedKFold(n_splits=6, random_state=42)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train['fold'] = -1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for i, (trn_idx, vld_idx) in enumerate(mskf.split(X, y)):\n    df_train.loc[vld_idx, 'fold'] = i","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train['fold'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"os.makedirs('../outputs')\n!mkdir ../outputs/train_images\n!ls ../outputs","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train.to_csv('../outputs/df_folds.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import joblib\nfrom tqdm import tqdm","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for fname in files_train : \n    F = os.path.join(data_dir, files_train[0])\n    df_train = pd.read_parquet(F)\n    img_ids = df_train['image_id'].values\n    img_array = df_train.iloc[:, 1:].values\n    for idx in tqdm(range(len(df_train))):\n        img_id = img_ids[idx]\n        img = img_array[idx]\n        joblib.dump(img, f'../outputs/train_images/{img_id}.pkl')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}