{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"
\nEDA and Dataset Versioning
using W&B
\n
","metadata":{}},{"cell_type":"markdown","source":"\nSome key points to note in the original data:\n\n - Not all the images have a corresponding caption
\n - Some images captions available in many languages
\n
\n
\n\nThis dataset contains 300px b64_bytes for all the images (no downloading required) along with a list of captions in all the available languages","metadata":{}},{"cell_type":"markdown","source":"# Install Required Libraries
","metadata":{}},{"cell_type":"code","source":"!pip install --upgrade wandb","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-11-01T21:42:02.203013Z","iopub.execute_input":"2021-11-01T21:42:02.203347Z","iopub.status.idle":"2021-11-01T21:42:14.319757Z","shell.execute_reply.started":"2021-11-01T21:42:02.203264Z","shell.execute_reply":"2021-11-01T21:42:14.318824Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Import Required Libraries 📚
","metadata":{}},{"cell_type":"code","source":"import os\n\nimport json\n\nimport base64\nfrom io import BytesIO\nfrom PIL import Image\nfrom IPython.display import display\n\nimport pickle","metadata":{"execution":{"iopub.status.busy":"2021-11-01T21:42:14.321539Z","iopub.execute_input":"2021-11-01T21:42:14.321794Z","iopub.status.idle":"2021-11-01T21:42:14.326851Z","shell.execute_reply.started":"2021-11-01T21:42:14.321768Z","shell.execute_reply":"2021-11-01T21:42:14.325772Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"![](\"https://camo.githubusercontent.com/dd842f7b0be57140e68b2ab9cb007992acd131c48284eaf6b1aca758bfea358b/68747470733a2f2f692e696d6775722e636f6d2f52557469567a482e706e67\")
\n\n\n We will be using Weights and Biases for:\n\n - Visualizing the dataset
\n - Dataset versioning
\n
\nW&B Dashboard: wandb.ai/dchanda/Wikipedia\n
","metadata":{}},{"cell_type":"code","source":"import wandb\n\ntry:\n from kaggle_secrets import UserSecretsClient\n user_secrets = UserSecretsClient()\n api_key = user_secrets.get_secret(\"wandb_api\")\n wandb.login(key=api_key)\n anony = None\nexcept:\n anony = \"must\"\n print('If you want to use your W&B account, go to Add-ons -> Secrets and provide your W&B access token. Use the Label name as wandb_api. \\nGet your W&B access token from here: https://wandb.ai/authorize')","metadata":{"execution":{"iopub.status.busy":"2021-11-01T21:42:14.328475Z","iopub.execute_input":"2021-11-01T21:42:14.328898Z","iopub.status.idle":"2021-11-01T21:42:15.465677Z","shell.execute_reply.started":"2021-11-01T21:42:14.328844Z","shell.execute_reply":"2021-11-01T21:42:15.464982Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Read Data
","metadata":{}},{"cell_type":"markdown","source":"Directory Structure\n\n\n wikipediasmall/\n \n - part-00000-48a6f07e-bb86-4735-aac7-883349f41a28-c000.json
\n - part-00001-48a6f07e-bb86-4735-aac7-883349f41a28-c000.json
\n - part-00002-48a6f07e-bb86-4735-aac7-883349f41a28-c000.json
\n
\nThe files have been downloaded from here","metadata":{}},{"cell_type":"code","source":"filenames = os.listdir(\"../input/wikipediasmall\")\n\njson_content = []\n\nfor file in filenames:\n filename = os.path.join(\"../input/wikipediasmall\", file)\n with open(filename, 'rb') as f:\n for line in f:\n if line:\n obj = json.loads(line)\n content = {}\n content['caption_title_and_reference_description'] = []\n for element in obj['wit_features']:\n if element.get('caption_title_and_reference_description'):\n content['caption_title_and_reference_description'].append(element['caption_title_and_reference_description'])\n\n content['b64_bytes'] = obj['b64_bytes']\n\n # Check if captions or images are empty\n if len(content['caption_title_and_reference_description']) > 0 and content['b64_bytes'] != \"\":\n json_content.append(content)","metadata":{"execution":{"iopub.status.busy":"2021-11-01T21:42:15.467740Z","iopub.execute_input":"2021-11-01T21:42:15.467992Z","iopub.status.idle":"2021-11-01T21:43:09.845432Z","shell.execute_reply.started":"2021-11-01T21:42:15.467938Z","shell.execute_reply":"2021-11-01T21:43:09.844054Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"json_content[0].keys()","metadata":{"execution":{"iopub.status.busy":"2021-11-01T21:43:09.848048Z","iopub.execute_input":"2021-11-01T21:43:09.848354Z","iopub.status.idle":"2021-11-01T21:43:09.860241Z","shell.execute_reply.started":"2021-11-01T21:43:09.848314Z","shell.execute_reply":"2021-11-01T21:43:09.859370Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(json_content)","metadata":{"execution":{"iopub.status.busy":"2021-11-01T21:43:09.861651Z","iopub.execute_input":"2021-11-01T21:43:09.862502Z","iopub.status.idle":"2021-11-01T21:43:09.872752Z","shell.execute_reply.started":"2021-11-01T21:43:09.862465Z","shell.execute_reply":"2021-11-01T21:43:09.872139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Visualize 1 Example
","metadata":{}},{"cell_type":"code","source":"index = 100\n\nout = base64.b64decode(json_content[index]['b64_bytes'])\nimg = Image.open(BytesIO(out)).convert(\"RGB\")\nprint(json_content[index]['caption_title_and_reference_description'])\ndisplay(img)","metadata":{"execution":{"iopub.status.busy":"2021-11-01T21:43:09.875236Z","iopub.execute_input":"2021-11-01T21:43:09.875650Z","iopub.status.idle":"2021-11-01T21:43:09.940896Z","shell.execute_reply.started":"2021-11-01T21:43:09.875620Z","shell.execute_reply":"2021-11-01T21:43:09.940288Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Visualize Entire Dataset
","metadata":{}},{"cell_type":"code","source":"run = wandb.init(project='Wikipedia',\n job_type='Visualization',\n anonymous='must')","metadata":{"execution":{"iopub.status.busy":"2021-11-01T21:43:09.941853Z","iopub.execute_input":"2021-11-01T21:43:09.942140Z","iopub.status.idle":"2021-11-01T21:43:16.954762Z","shell.execute_reply.started":"2021-11-01T21:43:09.942111Z","shell.execute_reply":"2021-11-01T21:43:16.953962Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preview_table = wandb.Table(columns=['Image', 'Captions'])\nfor content in json_content[:1000]:\n out = base64.b64decode(content['b64_bytes'])\n img = Image.open(BytesIO(out)).convert(\"RGB\")\n preview_table.add_data(wandb.Image(img), \n content['caption_title_and_reference_description'])\n\nwandb.log({'Visualization': preview_table})\nrun.finish()","metadata":{"execution":{"iopub.status.busy":"2021-11-01T21:43:16.956435Z","iopub.execute_input":"2021-11-01T21:43:16.956683Z","iopub.status.idle":"2021-11-01T21:44:16.290105Z","shell.execute_reply.started":"2021-11-01T21:43:16.956652Z","shell.execute_reply":"2021-11-01T21:44:16.289155Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"View the Complete Table Here ⮕","metadata":{}},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"# Dataset Versioning
","metadata":{}},{"cell_type":"code","source":"with open(\"data.pkl\", \"wb\") as fp:\n pickle.dump(json_content, fp)","metadata":{"execution":{"iopub.status.busy":"2021-11-01T21:44:16.291573Z","iopub.execute_input":"2021-11-01T21:44:16.291774Z","iopub.status.idle":"2021-11-01T21:44:18.518182Z","shell.execute_reply.started":"2021-11-01T21:44:16.291748Z","shell.execute_reply":"2021-11-01T21:44:18.517262Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"run = wandb.init(project=\"Wikipedia\", \n job_type=\"data\",\n anonymous=\"must\")\nprocessed_data = wandb.Artifact(\"Wiki-data\", type=\"dataset\",\n description=\"Processed data from file 00000 and 00001\",\n metadata={\"source\": \"analytics.wikimedia.org/published/datasets/one-off/caption_competition/training/joined/\",\n \"sizes\": len(json_content)})\nprocessed_data.add_file(\"data.pkl\")\nrun.log_artifact(processed_data)\nrun.finish()","metadata":{"execution":{"iopub.status.busy":"2021-11-01T21:44:18.519492Z","iopub.execute_input":"2021-11-01T21:44:18.520312Z","iopub.status.idle":"2021-11-01T21:44:52.501415Z","shell.execute_reply.started":"2021-11-01T21:44:18.520275Z","shell.execute_reply":"2021-11-01T21:44:52.500569Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Download Data
","metadata":{}},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"run = wandb.init(project=\"Wikipedia\", \n anonymous=\"must\")\nartifact = run.use_artifact('dchanda/Wikipedia/Wiki-data:v0', type='dataset')\nartifact_dir = artifact.download()\nrun.finish()","metadata":{"execution":{"iopub.status.busy":"2021-11-01T21:44:52.502834Z","iopub.execute_input":"2021-11-01T21:44:52.503230Z","iopub.status.idle":"2021-11-01T21:45:04.697867Z","shell.execute_reply.started":"2021-11-01T21:44:52.503192Z","shell.execute_reply":"2021-11-01T21:45:04.696965Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for file in os.listdir(artifact_dir):\n filepath = os.path.join(artifact_dir, file)\n with open(filepath, \"rb\") as fp:\n contents = pickle.load(fp)\n\nindex = 100\n\nout = base64.b64decode(contents[index]['b64_bytes'])\nimg = Image.open(BytesIO(out)).convert(\"RGB\")\nprint(contents[index]['caption_title_and_reference_description'])\ndisplay(img)","metadata":{"execution":{"iopub.status.busy":"2021-11-01T21:45:04.699273Z","iopub.execute_input":"2021-11-01T21:45:04.699528Z","iopub.status.idle":"2021-11-01T21:45:06.548870Z","shell.execute_reply.started":"2021-11-01T21:45:04.699497Z","shell.execute_reply":"2021-11-01T21:45:06.548118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}}]}