{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-01-10T21:01:35.856789Z","iopub.execute_input":"2022-01-10T21:01:35.857474Z","iopub.status.idle":"2022-01-10T21:01:35.890175Z","shell.execute_reply.started":"2022-01-10T21:01:35.857365Z","shell.execute_reply":"2022-01-10T21:01:35.889203Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.environ[\"KMP_SETTINGS\"] = \"false\"\nos.environ[\"TF_CPP_MIN_LOG_LEVEL\"] = \"3\"\nimport tensorflow as tf\nimport numpy as np\nfrom zipfile import ZipFile","metadata":{"execution":{"iopub.status.busy":"2022-01-10T21:01:35.892396Z","iopub.execute_input":"2022-01-10T21:01:35.892627Z","iopub.status.idle":"2022-01-10T21:01:41.021513Z","shell.execute_reply.started":"2022-01-10T21:01:35.892592Z","shell.execute_reply":"2022-01-10T21:01:41.020894Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DS_PATH = '/kaggle/input/painter-by-numbers'\nds_path_of = lambda x : os.path.join(DS_PATH, x)\ndf = pd.read_csv(ds_path_of('all_data_info.csv'))","metadata":{"execution":{"iopub.status.busy":"2022-01-10T21:01:41.02265Z","iopub.execute_input":"2022-01-10T21:01:41.023507Z","iopub.status.idle":"2022-01-10T21:01:41.59467Z","shell.execute_reply.started":"2022-01-10T21:01:41.023463Z","shell.execute_reply":"2022-01-10T21:01:41.593891Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = df.sample(frac=1, random_state=27)\ntrain_df = df[df['in_train'] == True]\ntest_df = df[df['in_train'] == False]\ntrain_filenames = train_df['new_filename']\ntest_filenames = test_df['new_filename']\ntrain_labels = train_df['artist'].astype(\"category\").cat.codes \ntest_labels = test_df['artist'].astype(\"category\").cat.codes ","metadata":{"execution":{"iopub.status.busy":"2022-01-10T21:01:41.597206Z","iopub.execute_input":"2022-01-10T21:01:41.597538Z","iopub.status.idle":"2022-01-10T21:01:41.679569Z","shell.execute_reply.started":"2022-01-10T21:01:41.597496Z","shell.execute_reply":"2022-01-10T21:01:41.67869Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IMG_WIDTH = 224\nIMG_HEIGHT = 224\nNUM_EXAMPLES = len(df)\nEXAMPLE_SIZE = (IMG_WIDTH * IMG_HEIGHT * 3) + 8\nSHARD_SIZE = 150 * int(2**20)\nEXAMPLES_PER_SHARD = SHARD_SIZE // EXAMPLE_SIZE\nTRAIN_EXAMPLES = len(test_filenames)\nTEST_EXAMPLES = len(train_filenames)\n\nTRAIN_SHARDS = TRAIN_EXAMPLES // EXAMPLES_PER_SHARD\nTEST_SHARDS = TEST_EXAMPLES // EXAMPLES_PER_SHARD\n\nNUM_CLASSES = max(train_labels)\n\nprint(f'Num train examples: {TRAIN_EXAMPLES}')\nprint(f'Num test examples: {TEST_EXAMPLES}')\nprint(f'Num classes: {NUM_CLASSES}')\nprint(f'Num examples per shard is {EXAMPLES_PER_SHARD}')\nprint(f'Dataset size is: {(EXAMPLE_SIZE * NUM_EXAMPLES)/(2**30):.0f} GB')","metadata":{"execution":{"iopub.status.busy":"2022-01-10T21:15:59.633413Z","iopub.execute_input":"2022-01-10T21:15:59.634302Z","iopub.status.idle":"2022-01-10T21:15:59.656847Z","shell.execute_reply.started":"2022-01-10T21:15:59.634226Z","shell.execute_reply":"2022-01-10T21:15:59.655762Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from multiprocessing import Process, Semaphore, cpu_count\n\ndef _bytes_feature(value):\n  \"\"\"Returns a bytes_list from a string / byte.\"\"\"\n  if isinstance(value, type(tf.constant(0))):\n    value = value.numpy() # BytesList won't unpack a string from an EagerTensor.\n  return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))\n\ndef _int64_feature(value):\n  \"\"\"Returns an int64_list from a bool / enum / int / uint.\"\"\"\n  return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))\n\ndef write_shard(shard, zippath, output_filename, num_zeros, n_shards, shard_size, subdir, filenames, labels):\n    with ZipFile(zippath) as input_zip:\n        shard_filename = f'{output_filename}.{str(shard+1).zfill(num_zeros)}-of-{n_shards}'\n        start = shard_size * shard\n        end = shard_size * (shard+1)\n        try:\n            with tf.io.TFRecordWriter(shard_filename) as writer:\n                for filename, label in zip(filenames[start:end], labels[start:end]):\n                    img = tf.io.decode_image(input_zip.read(subdir + filename), channels=3, expand_animations=False)\n                    img = tf.image.resize(img, (IMG_HEIGHT, IMG_WIDTH))\n                    img = tf.image.convert_image_dtype(img, tf.uint8)\n                    img_bytes = tf.io.encode_jpeg(img)\n\n                    example = tf.train.Example(features=tf.train.Features(feature={\n                        'label' : _int64_feature(label),\n                        'image' : _bytes_feature(img_bytes)\n                    }))\n\n                    writer.write(example.SerializeToString())\n        except Exception as e:\n            print(e)\n            \ndef worker(shard, zippath, output_filename, num_zeros, n_shards, shard_size, subdir, filenames, labels, sema):\n    write_shard(shard, zippath, output_filename, num_zeros, n_shards, shard_size, subdir, filenames, labels)\n    sema.release()\n                    \ndef write_dataset(filenames, labels, input_file, subdir, output_filename, shard_size):\n    n_shards = len(filenames) // shard_size\n    num_zeros = len(str(n_shards))\n    done = 0\n    zippath = ds_path_of(input_file)\n    sema = Semaphore(cpu_count())\n    processes = []\n    for shard in range(n_shards):\n        sema.acquire()\n        print(f'\\rShard {shard+1}/{n_shards} started',end=\"\")\n        proc = Process(target=worker, args=(shard, zippath, output_filename, num_zeros, n_shards, shard_size, subdir, filenames, labels, sema))\n        processes.append(proc)\n        proc.start()\n        \n    [p.join() for p in processes]\n    print('Done')\n            ","metadata":{"execution":{"iopub.status.busy":"2022-01-10T21:01:41.703569Z","iopub.execute_input":"2022-01-10T21:01:41.703794Z","iopub.status.idle":"2022-01-10T21:01:41.722229Z","shell.execute_reply.started":"2022-01-10T21:01:41.703765Z","shell.execute_reply":"2022-01-10T21:01:41.721324Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"save_path = '/kaggle/working/tfrecords/'\n\nif not os.path.exists(save_path):\n    os.makedirs(save_path)","metadata":{"execution":{"iopub.status.busy":"2022-01-10T21:19:31.548733Z","iopub.execute_input":"2022-01-10T21:19:31.549141Z","iopub.status.idle":"2022-01-10T21:19:31.554269Z","shell.execute_reply.started":"2022-01-10T21:19:31.549103Z","shell.execute_reply":"2022-01-10T21:19:31.553581Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"write_dataset(train_filenames, train_labels, 'train.zip', 'train/', os.path.join(save_path,'train.tfrecord'), EXAMPLES_PER_SHARD)","metadata":{"execution":{"iopub.status.busy":"2022-01-10T21:01:41.739799Z","iopub.execute_input":"2022-01-10T21:01:41.740031Z","iopub.status.idle":"2022-01-10T21:12:54.358957Z","shell.execute_reply.started":"2022-01-10T21:01:41.739999Z","shell.execute_reply":"2022-01-10T21:12:54.358072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"write_dataset(test_filenames, test_labels, 'test.zip', 'test/', os.path.join(save_path,'test.tfrecord'), EXAMPLES_PER_SHARD)","metadata":{"execution":{"iopub.status.busy":"2022-01-10T21:12:54.361759Z","iopub.execute_input":"2022-01-10T21:12:54.362128Z","iopub.status.idle":"2022-01-10T21:15:59.627912Z","shell.execute_reply.started":"2022-01-10T21:12:54.362087Z","shell.execute_reply":"2022-01-10T21:15:59.626822Z"},"trusted":true},"execution_count":null,"outputs":[]}]}