{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport json\nimport numpy as np\nimport pandas as pd\nfrom tqdm import tqdm","metadata":{"execution":{"iopub.status.busy":"2022-08-08T04:40:10.945980Z","iopub.execute_input":"2022-08-08T04:40:10.947622Z","iopub.status.idle":"2022-08-08T04:40:10.971473Z","shell.execute_reply.started":"2022-08-08T04:40:10.946202Z","shell.execute_reply":"2022-08-08T04:40:10.970816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import wandb\nfrom kaggle_secrets import UserSecretsClient\nuser_secrets = UserSecretsClient()\nsecret_value_0 = user_secrets.get_secret(\"wandb_api\")\nwandb.login(key=secret_value_0)","metadata":{"execution":{"iopub.status.busy":"2022-08-08T04:40:10.972807Z","iopub.execute_input":"2022-08-08T04:40:10.973141Z","iopub.status.idle":"2022-08-08T04:40:14.074037Z","shell.execute_reply.started":"2022-08-08T04:40:10.973112Z","shell.execute_reply":"2022-08-08T04:40:14.073104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ROOT_DIR = '../input/semi-inat-2020/'\n# print(os.listdir(ROOT_DIR))\n\nANNOTATION_DIR = ROOT_DIR+'annotation/annotation/'\n# print(os.listdir(ANNOTATION_DIR))\n\nTRAINVAL_LABELLED_DIR = ROOT_DIR+'trainval_images/'\n# print(os.listdir(TRAINVAL_LABELLED_DIR)) # Has dirs\n\nTRAIN_UNLABELLED_INCLASS_DIR = ROOT_DIR+'u_train_in/'\n# print(os.listdir(TRAIN_UNLABELLED_INCLASS_DIR)) # Images\n\nTRAIN_UNLABELLED_OUTCLASS_DIR = ROOT_DIR+'u_train_out/'\n# print(os.listdir(TRAIN_UNLABELLED_OUTCLASS_DIR)) # Images\n\nTEST_DIR = ROOT_DIR+'test/'\n# print(os.listdir(TEST_DIR)) # Images","metadata":{"execution":{"iopub.status.busy":"2022-08-08T06:10:00.537035Z","iopub.execute_input":"2022-08-08T06:10:00.537301Z","iopub.status.idle":"2022-08-08T06:10:00.542298Z","shell.execute_reply.started":"2022-08-08T06:10:00.537274Z","shell.execute_reply":"2022-08-08T06:10:00.541420Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Log Train Labelled Dataset","metadata":{}},{"cell_type":"code","source":"file = ANNOTATION_DIR + 'anno_l_train.json'\n\n# load data using Python JSON module\nwith open(file,'r') as f:\n    data = json.loads(f.read())\n# Flatten data\ndata_annotations = pd.json_normalize(data, record_path=['annotations'])\ndata_images = pd.json_normalize(data, record_path=['images'])","metadata":{"execution":{"iopub.status.busy":"2022-05-02T08:38:28.929623Z","iopub.execute_input":"2022-05-02T08:38:28.930161Z","iopub.status.idle":"2022-05-02T08:38:29.104892Z","shell.execute_reply.started":"2022-05-02T08:38:28.930112Z","shell.execute_reply":"2022-05-02T08:38:29.103876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"training_labelled = pd.concat([data_annotations , data_images.drop(['id'], axis = 1)], axis = 1)\ntraining_labelled.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-02T08:38:30.352546Z","iopub.execute_input":"2022-05-02T08:38:30.353012Z","iopub.status.idle":"2022-05-02T08:38:30.373918Z","shell.execute_reply.started":"2022-05-02T08:38:30.352970Z","shell.execute_reply":"2022-05-02T08:38:30.372972Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"run = wandb.init(project='ssl-study-data')\n\ndata_artifact = wandb.Artifact(name='train', type='labelled-dataset')\ndata_at = wandb.Table(columns=['image_id', 'image', 'label', 'width', 'height'])\n\nfor i, tmp_df in tqdm(training_labelled.iterrows()):\n    assert os.path.isfile(TRAINVAL_LABELLED_DIR+tmp_df.file_name)\n    data_at.add_data(\n        int(tmp_df.image_id),\n        wandb.Image(TRAINVAL_LABELLED_DIR+tmp_df.file_name),\n        int(tmp_df.category_id),\n        int(tmp_df.width),\n        int(tmp_df.height)\n    )\n    \ndata_artifact.add(data_at, 'train-labelled')\nrun.log_artifact(data_artifact)\nwandb.finish()","metadata":{"execution":{"iopub.status.busy":"2022-05-02T08:26:59.463382Z","iopub.execute_input":"2022-05-02T08:26:59.464009Z","iopub.status.idle":"2022-05-02T08:30:12.418504Z","shell.execute_reply.started":"2022-05-02T08:26:59.463943Z","shell.execute_reply":"2022-05-02T08:30:12.417375Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Log Val Labelled Dataset","metadata":{}},{"cell_type":"code","source":"file = ANNOTATION_DIR + 'anno_val.json'\n\n# load data using Python JSON module\nwith open(file,'r') as f:\n    data = json.loads(f.read())\n# Flatten data\ndata_annotations = pd.json_normalize(data, record_path=['annotations'])\ndata_images = pd.json_normalize(data, record_path=['images'])","metadata":{"execution":{"iopub.status.busy":"2022-05-02T08:38:55.424440Z","iopub.execute_input":"2022-05-02T08:38:55.424774Z","iopub.status.idle":"2022-05-02T08:38:55.487872Z","shell.execute_reply.started":"2022-05-02T08:38:55.424746Z","shell.execute_reply":"2022-05-02T08:38:55.487339Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_labelled = pd.concat([data_annotations , data_images.drop(['id'], axis = 1)], axis = 1)\nval_labelled.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-02T08:50:14.621710Z","iopub.execute_input":"2022-05-02T08:50:14.622015Z","iopub.status.idle":"2022-05-02T08:50:14.635887Z","shell.execute_reply.started":"2022-05-02T08:50:14.621985Z","shell.execute_reply":"2022-05-02T08:50:14.634939Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"run = wandb.init(project='ssl-study-data')\n\ndata_artifact = wandb.Artifact(name='val', type='labelled-dataset')\ndata_at = wandb.Table(columns=['image_id', 'image', 'label', 'width', 'height'])\n\nfor i, tmp_df in tqdm(val_labelled.iterrows()):\n    assert os.path.isfile(TRAINVAL_LABELLED_DIR+tmp_df.file_name)\n    data_at.add_data(\n        int(tmp_df.image_id),\n        wandb.Image(TRAINVAL_LABELLED_DIR+tmp_df.file_name),\n        int(tmp_df.category_id),\n        int(tmp_df.width),\n        int(tmp_df.height)\n    )\n    \ndata_artifact.add(data_at, 'val-labelled')\nrun.log_artifact(data_artifact)\nwandb.finish()","metadata":{"execution":{"iopub.status.busy":"2022-05-02T09:08:38.508423Z","iopub.execute_input":"2022-05-02T09:08:38.508713Z","iopub.status.idle":"2022-05-02T09:10:18.313506Z","shell.execute_reply.started":"2022-05-02T09:08:38.508678Z","shell.execute_reply":"2022-05-02T09:10:18.312962Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Log Unlabelled Inclass Dataset","metadata":{}},{"cell_type":"code","source":"file = ANNOTATION_DIR + 'anno_u_train_in.json'\n\n# load data using Python JSON module\nwith open(file,'r') as f:\n    data = json.loads(f.read())\n# Flatten data\ndata_annotations = pd.json_normalize(data, record_path=['annotations'])\ndata_images = pd.json_normalize(data, record_path=['images'])","metadata":{"execution":{"iopub.status.busy":"2022-05-02T09:31:09.673140Z","iopub.execute_input":"2022-05-02T09:31:09.673444Z","iopub.status.idle":"2022-05-02T09:31:10.315748Z","shell.execute_reply.started":"2022-05-02T09:31:09.673411Z","shell.execute_reply":"2022-05-02T09:31:10.314531Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inclass_unlabelled = pd.concat([data_annotations , data_images.drop(['id'], axis = 1)], axis = 1)\ninclass_unlabelled.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-02T09:31:37.684300Z","iopub.execute_input":"2022-05-02T09:31:37.684584Z","iopub.status.idle":"2022-05-02T09:31:37.701579Z","shell.execute_reply.started":"2022-05-02T09:31:37.684557Z","shell.execute_reply":"2022-05-02T09:31:37.700455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"run = wandb.init(project='ssl-study-data')\n\ndata_artifact = wandb.Artifact(name='in-class', type='unlabelled-dataset')\ndata_at = wandb.Table(columns=['image_id', 'image', 'width', 'height'])\n\nfor i, tmp_df in tqdm(inclass_unlabelled.iterrows()):\n    assert os.path.isfile(TRAIN_UNLABELLED_INCLASS_DIR+tmp_df.file_name)\n    data_at.add_data(\n        int(tmp_df.image_id),\n        wandb.Image(TRAIN_UNLABELLED_INCLASS_DIR+tmp_df.file_name),\n        int(tmp_df.width),\n        int(tmp_df.height)\n    )\n    \ndata_artifact.add(data_at, 'inclass-unlabelled')\nrun.log_artifact(data_artifact)\nwandb.finish()","metadata":{"execution":{"iopub.status.busy":"2022-05-02T09:40:17.656399Z","iopub.execute_input":"2022-05-02T09:40:17.656676Z","iopub.status.idle":"2022-05-02T10:03:07.068086Z","shell.execute_reply.started":"2022-05-02T09:40:17.656649Z","shell.execute_reply":"2022-05-02T10:03:07.066580Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Log Unlabelled Out of Class Dataset","metadata":{}},{"cell_type":"code","source":"file = ANNOTATION_DIR + 'anno_u_train_out.json'\n\n# load data using Python JSON module\nwith open(file,'r') as f:\n    data = json.loads(f.read())\n# Flatten data\ndata_annotations = pd.json_normalize(data, record_path=['annotations'])\ndata_images = pd.json_normalize(data, record_path=['images'])","metadata":{"execution":{"iopub.status.busy":"2022-05-02T10:09:15.995019Z","iopub.execute_input":"2022-05-02T10:09:15.995396Z","iopub.status.idle":"2022-05-02T10:09:20.288433Z","shell.execute_reply.started":"2022-05-02T10:09:15.995359Z","shell.execute_reply":"2022-05-02T10:09:20.286916Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"outclass_unlabelled = pd.concat([data_annotations , data_images.drop(['id'], axis = 1)], axis = 1)\noutclass_unlabelled.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-02T10:09:42.693562Z","iopub.execute_input":"2022-05-02T10:09:42.694207Z","iopub.status.idle":"2022-05-02T10:09:42.721916Z","shell.execute_reply.started":"2022-05-02T10:09:42.694146Z","shell.execute_reply":"2022-05-02T10:09:42.720923Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"run = wandb.init(project='ssl-study-data')\n\ndata_artifact = wandb.Artifact(name='out-class', type='unlabelled-dataset')\ndata_at = wandb.Table(columns=['image_id', 'image', 'width', 'height'])\n\nfor i, tmp_df in tqdm(outclass_unlabelled.iterrows()):\n    assert os.path.isfile(TRAIN_UNLABELLED_OUTCLASS_DIR+tmp_df.file_name)\n    data_at.add_data(\n        int(tmp_df.image_id),\n        wandb.Image(TRAIN_UNLABELLED_OUTCLASS_DIR+tmp_df.file_name),\n        int(tmp_df.width),\n        int(tmp_df.height)\n    )\n    \ndata_artifact.add(data_at, 'outclass-unlabelled')\nrun.log_artifact(data_artifact)\nwandb.finish()","metadata":{"execution":{"iopub.status.busy":"2022-05-02T10:11:51.057824Z","iopub.execute_input":"2022-05-02T10:11:51.058152Z","iopub.status.idle":"2022-05-02T12:02:27.679991Z","shell.execute_reply.started":"2022-05-02T10:11:51.058118Z","shell.execute_reply":"2022-05-02T12:02:27.676610Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Log test dataset","metadata":{}},{"cell_type":"code","source":"!wget https://raw.githubusercontent.com/cvl-umass/semi-inat-2020/master/solution.csv","metadata":{"execution":{"iopub.status.busy":"2022-08-08T04:41:48.715989Z","iopub.execute_input":"2022-08-08T04:41:48.716271Z","iopub.status.idle":"2022-08-08T04:41:50.199197Z","shell.execute_reply.started":"2022-08-08T04:41:48.716225Z","shell.execute_reply":"2022-08-08T04:41:50.198449Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = pd.read_csv(\"solution.csv\")\ntest_df.alignhead()","metadata":{"execution":{"iopub.status.busy":"2022-08-08T04:42:27.840909Z","iopub.execute_input":"2022-08-08T04:42:27.841218Z","iopub.status.idle":"2022-08-08T04:42:27.865642Z","shell.execute_reply.started":"2022-08-08T04:42:27.841176Z","shell.execute_reply":"2022-08-08T04:42:27.864995Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"file = ANNOTATION_DIR + 'anno_test.json'\n\n# load data using Python JSON module\nwith open(file,'r') as f:\n    data = json.loads(f.read())\n\n# Flatten data\ndata_annotations = pd.json_normalize(data, record_path=['annotations'])\ndata_images = pd.json_normalize(data, record_path=['images'])","metadata":{"execution":{"iopub.status.busy":"2022-08-08T05:32:59.435428Z","iopub.execute_input":"2022-08-08T05:32:59.435785Z","iopub.status.idle":"2022-08-08T05:32:59.569511Z","shell.execute_reply.started":"2022-08-08T05:32:59.435748Z","shell.execute_reply":"2022-08-08T05:32:59.568846Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_labelled = pd.concat([data_annotations , data_images.drop(['id'], axis = 1), test_df], axis = 1)\ntest_labelled.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-08T05:37:20.180476Z","iopub.execute_input":"2022-08-08T05:37:20.180774Z","iopub.status.idle":"2022-08-08T05:37:20.197365Z","shell.execute_reply.started":"2022-08-08T05:37:20.180744Z","shell.execute_reply":"2022-08-08T05:37:20.196422Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"run = wandb.init(project='ssl-study-data')\n\ndata_artifact = wandb.Artifact(name='test', type='labelled-dataset')\ndata_at = wandb.Table(columns=['image_id', 'image', 'label', 'width', 'height'])\n\nfor i, tmp_df in tqdm(test_labelled.iterrows()):\n    assert os.path.isfile(TEST_DIR+tmp_df.file_name)\n    data_at.add_data(\n        int(tmp_df.Id),\n        wandb.Image(TEST_DIR+tmp_df.file_name),\n        int(tmp_df.Category),\n        int(tmp_df.width),\n        int(tmp_df.height)\n    )\n    \ndata_artifact.add(data_at, 'test-labelled')\nrun.log_artifact(data_artifact)\nwandb.finish()","metadata":{"execution":{"iopub.status.busy":"2022-08-08T06:12:59.903954Z","iopub.execute_input":"2022-08-08T06:12:59.904225Z","iopub.status.idle":"2022-08-08T06:19:29.427750Z","shell.execute_reply.started":"2022-08-08T06:12:59.904196Z","shell.execute_reply":"2022-08-08T06:19:29.426650Z"},"trusted":true},"execution_count":null,"outputs":[]}]}