{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"
\n\n

ULTRA MNIST

\n
CREATED BY: DARIEN SCHETTLER
\n\n
\n\n---\n\n
\n\n
\n 🛑   WARNING:

THIS IS A WORK IN PROGRESS
\n
\n\n\n
\n 👏   IF YOU FORK THIS OR FIND THIS HELPFUL   👏

PLEASE UPVOTE!

This was a lot of work for me and while it may seem silly, it makes me feel appreciated when others like my work. 😅\n
\n\n\n","metadata":{"papermill":{"duration":0.120561,"end_time":"2021-11-06T21:15:09.611563","exception":false,"start_time":"2021-11-06T21:15:09.491002","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"

\n\n

\n\n

TABLE OF CONTENTS

\n\n---\n\n

0    IMPORTS

\n\n---\n\n

1    BACKGROUND INFORMATION

\n\n---\n\n

2    SETUP

\n\n---\n\n

3    HELPER FUNCTIONS

\n\n---\n\n

4    DATASET EXPLORATION & PREPROCESSING

\n\n---\n\n

5    BASELINE

\n\n---","metadata":{"papermill":{"duration":0.085591,"end_time":"2021-11-06T21:15:09.78303","exception":false,"start_time":"2021-11-06T21:15:09.697439","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"
\n\n\n\n

0  IMPORTS    ⤒

","metadata":{"papermill":{"duration":0.050527,"end_time":"2021-11-06T21:15:09.894476","exception":false,"start_time":"2021-11-06T21:15:09.843949","status":"completed"},"tags":[]}},{"cell_type":"code","source":"print(\"\\n... IMPORTS STARTING ...\\n\")\n\nprint(\"\\n... PIP/APT INSTALLS AND DOWNLOADS/ZIP STARTING ...\")\n!pip install imutils\nimport imutils\nprint(\"... PIP/APT INSTALLS COMPLETE ...\\n\")\n\nprint(\"\\n\\tVERSION INFORMATION\")\n# Machine Learning and Data Science Imports\nimport tensorflow as tf; print(f\"\\t\\t– TENSORFLOW VERSION: {tf.__version__}\");\nimport tensorflow_hub as tfhub; print(f\"\\t\\t– TENSORFLOW HUB VERSION: {tfhub.__version__}\");\nimport tensorflow_addons as tfa; print(f\"\\t\\t– TENSORFLOW ADDONS VERSION: {tfa.__version__}\");\nimport pandas as pd; pd.options.mode.chained_assignment = None;\nimport numpy as np; print(f\"\\t\\t– NUMPY VERSION: {np.__version__}\");\nimport sklearn; print(f\"\\t\\t– SKLEARN VERSION: {sklearn.__version__}\");\nfrom sklearn.preprocessing import RobustScaler, PolynomialFeatures\nfrom pandarallel import pandarallel; pandarallel.initialize();\nfrom sklearn.model_selection import GroupKFold, StratifiedKFold\n\n# # RAPIDS\n# import cudf, cupy, cuml\n# from cuml.neighbors import NearestNeighbors\n# from cuml.manifold import TSNE, UMAP\n\n# Built In Imports\nfrom kaggle_datasets import KaggleDatasets\nfrom collections import Counter\nfrom datetime import datetime\nfrom glob import glob\nimport warnings\nimport requests\nimport hashlib\nimport imageio\nimport IPython\nimport sklearn\nimport urllib\nimport zipfile\nimport pickle\nimport random\nimport shutil\nimport string\nimport json\nimport math\nimport time\nimport gzip\nimport ast\nimport sys\nimport io\nimport os\nimport gc\nimport re\n\n# Visualization Imports\nfrom matplotlib.colors import ListedColormap\nimport matplotlib.patches as patches\nimport plotly.graph_objects as go\nimport matplotlib.pyplot as plt\nfrom tqdm.notebook import tqdm; tqdm.pandas();\nimport plotly.express as px\nimport seaborn as sns\nfrom PIL import Image, ImageEnhance\nimport matplotlib; print(f\"\\t\\t– MATPLOTLIB VERSION: {matplotlib.__version__}\");\nfrom matplotlib import animation, rc; rc('animation', html='jshtml')\nimport plotly\nimport PIL\nimport cv2\n\nimport plotly.io as pio\nprint(pio.renderers)\n\ndef seed_it_all(seed=7):\n \"\"\" Attempt to be Reproducible \"\"\"\n os.environ['PYTHONHASHSEED'] = str(seed)\n random.seed(seed)\n np.random.seed(seed)\n tf.random.set_seed(seed)\n\n \nprint(\"\\n\\n... IMPORTS COMPLETE ...\\n\")","metadata":{"papermill":{"duration":162.144149,"end_time":"2021-11-06T21:17:52.087371","exception":false,"start_time":"2021-11-06T21:15:09.943222","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-03-10T15:21:20.46279Z","iopub.execute_input":"2022-03-10T15:21:20.463081Z","iopub.status.idle":"2022-03-10T15:21:30.484521Z","shell.execute_reply.started":"2022-03-10T15:21:20.463048Z","shell.execute_reply":"2022-03-10T15:21:30.483223Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n\n\n\n

1  BACKGROUND INFORMATION    ⤒

\n\n---\n","metadata":{"papermill":{"duration":0.05019,"end_time":"2021-11-06T21:17:52.231372","exception":false,"start_time":"2021-11-06T21:17:52.181182","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"

1.1 BASIC COMPETITION INFORMATION

\n\n---\n\n
PRIMARY TASK DESCRIPTION\n\nTBD\n---\n\n
CONTEXT\n\nTBD\n\n---\n\n
MORE BACKGROUND INFORMATION\n\nTBD","metadata":{"papermill":{"duration":0.053372,"end_time":"2021-11-06T21:17:52.337029","exception":false,"start_time":"2021-11-06T21:17:52.283657","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"

1.2 COMPETITION EVALUATION

\n\n---\n\n
GENERAL EVALUATION INFORMATION\n\nTBD\n\n
IS THIS A CODE COMPETITION?\n\nNO!\n","metadata":{"papermill":{"duration":0.052259,"end_time":"2021-11-06T21:17:52.443218","exception":false,"start_time":"2021-11-06T21:17:52.390959","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"

1.3 DATASET OVERVIEW

\n\n---\n\n
GENERAL INFORMATION\n\nTBD\n\n
DISCOVERED INFORMATION [TENTATIVE]\n\nTBD","metadata":{"papermill":{"duration":0.052579,"end_time":"2021-11-06T21:17:52.548406","exception":false,"start_time":"2021-11-06T21:17:52.495827","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"
\n\n\n\n

2  SETUP    ⤒

\n\n---\n","metadata":{"papermill":{"duration":0.052944,"end_time":"2021-11-06T21:17:52.65284","exception":false,"start_time":"2021-11-06T21:17:52.599896","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"

2.1 ACCELERATOR DETECTION

\n\n---\n\nIn order to use **`TPU`**, we use **`TPUClusterResolver`** for the initialization which is necessary to connect to the remote cluster and initialize cloud TPUs. Let's go over two important points\n\n1. When using TPU on Kaggle, you don't need to specify arguments for **`TPUClusterResolver`**\n2. However, on **G**oogle **C**ompute **E**ngine (**GCE**), you will need to do the following:\n\n
\n\n```python\n# The name you gave to the TPU to use\nTPU_WORKER = 'my-tpu-name'\n\n# or you can also specify the grpc path directly\n# TPU_WORKER = 'grpc://xxx.xxx.xxx.xxx:8470'\n\n# The zone you chose when you created the TPU to use on GCP.\nZONE = 'us-east1-b'\n\n# The name of the GCP project where you created the TPU to use on GCP.\nPROJECT = 'my-tpu-project'\n\ntpu = tf.distribute.cluster_resolver.TPUClusterResolver(tpu=TPU_WORKER, zone=ZONE, project=PROJECT)\n```\n\n
\n 🛑   WARNING:

- Although the Tensorflow documentation says it is the project name that should be provided for the argument `project`, it is actually the Project ID, that you should provide. This can be found on the GCP project dashboard page.
\n
\n\n
\n 📖   REFERENCES:

\n - Guide - Use TPUs
\n - Doc - TPUClusterResolver
\n\n
","metadata":{"papermill":{"duration":0.053821,"end_time":"2021-11-06T21:17:52.761303","exception":false,"start_time":"2021-11-06T21:17:52.707482","status":"completed"},"tags":[]}},{"cell_type":"code","source":"print(f\"\\n... ACCELERATOR SETUP STARTING ...\\n\")\n\n# Detect hardware, return appropriate distribution strategy\ntry:\n # TPU detection. No parameters necessary if TPU_NAME environment variable is set. On Kaggle this is always the case.\n TPU = tf.distribute.cluster_resolver.TPUClusterResolver() \nexcept ValueError:\n TPU = None\n\nif TPU:\n print(f\"\\n... RUNNING ON TPU - {TPU.master()}...\")\n tf.config.experimental_connect_to_cluster(TPU)\n tf.tpu.experimental.initialize_tpu_system(TPU)\n strategy = tf.distribute.experimental.TPUStrategy(TPU)\nelse:\n print(f\"\\n... RUNNING ON CPU/GPU ...\")\n # Yield the default distribution strategy in Tensorflow\n # --> Works on CPU and single GPU.\n strategy = tf.distribute.get_strategy() \n\n# What Is a Replica?\n# --> A single Cloud TPU device consists of FOUR chips, each of which has TWO TPU cores. \n# --> Therefore, for efficient utilization of Cloud TPU, a program should make use of each of the EIGHT (4x2) cores. \n# --> Each replica is essentially a copy of the training graph that is run on each core and \n# trains a mini-batch containing 1/8th of the overall batch size\nN_REPLICAS = strategy.num_replicas_in_sync\n \nprint(f\"... # OF REPLICAS: {N_REPLICAS} ...\\n\")\n\nprint(f\"\\n... ACCELERATOR SETUP COMPLTED ...\\n\")","metadata":{"papermill":{"duration":0.07574,"end_time":"2021-11-06T21:17:52.892074","exception":false,"start_time":"2021-11-06T21:17:52.816334","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-03-10T15:21:18.995966Z","iopub.execute_input":"2022-03-10T15:21:18.996225Z","iopub.status.idle":"2022-03-10T15:21:19.01654Z","shell.execute_reply.started":"2022-03-10T15:21:18.996196Z","shell.execute_reply":"2022-03-10T15:21:19.015626Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

2.2 COMPETITION DATA ACCESS

\n\n---\n\nTPUs read data must be read directly from **G**oogle **C**loud **S**torage **(GCS)**. Kaggle provides a utility library – **`KaggleDatasets`** – which has a utility function **`.get_gcs_path`** that will allow us to access the location of our input datasets within **GCS**.

\n\n
\n 📌   TIPS:

- If you have multiple datasets attached to the notebook, you should pass the name of a specific dataset to the `get_gcs_path()` function. In our case, the name of the dataset is the name of the directory the dataset is mounted within.

\n
","metadata":{"papermill":{"duration":0.053551,"end_time":"2021-11-06T21:17:52.999073","exception":false,"start_time":"2021-11-06T21:17:52.945522","status":"completed"},"tags":[]}},{"cell_type":"code","source":"print(\"\\n... DATA ACCESS SETUP STARTED ...\\n\")\n\nif TPU:\n # Google Cloud Dataset path to training and validation images\n DATA_DIR = KaggleDatasets().get_gcs_path('ultra-mnist')\n save_locally = tf.saved_model.SaveOptions(experimental_io_device='/job:localhost')\n load_locally = tf.saved_model.LoadOptions(experimental_io_device='/job:localhost')\nelse:\n # Local path to training and validation images\n DATA_DIR = \"/kaggle/input/ultra-mnist\"\n save_locally = None\n load_locally = None\n\nprint(f\"\\n... DATA DIRECTORY PATH IS:\\n\\t--> {DATA_DIR}\")\n\nprint(f\"\\n... IMMEDIATE CONTENTS OF DATA DIRECTORY IS:\")\nfor file in tf.io.gfile.glob(os.path.join(DATA_DIR, \"*\")): print(f\"\\t--> {file}\")\n\nprint(\"\\n\\n... DATA ACCESS SETUP COMPLETED ...\\n\")","metadata":{"papermill":{"duration":0.0797,"end_time":"2021-11-06T21:17:53.133972","exception":false,"start_time":"2021-11-06T21:17:53.054272","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-03-10T15:21:19.019513Z","iopub.execute_input":"2022-03-10T15:21:19.020463Z","iopub.status.idle":"2022-03-10T15:21:19.057137Z","shell.execute_reply.started":"2022-03-10T15:21:19.020414Z","shell.execute_reply":"2022-03-10T15:21:19.056217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

2.3 LEVERAGING XLA OPTIMIZATIONS

\n\n---\n\n\n**XLA** (Accelerated Linear Algebra) is a domain-specific compiler for linear algebra that can accelerate TensorFlow models with potentially no source code changes. **The results are improvements in speed and memory usage**.\n\n
\n\nWhen a TensorFlow program is run, all of the operations are executed individually by the TensorFlow executor. Each TensorFlow operation has a precompiled GPU/TPU kernel implementation that the executor dispatches to.\n\nXLA provides us with an alternative mode of running models: it compiles the TensorFlow graph into a sequence of computation kernels generated specifically for the given model. Because these kernels are unique to the model, they can exploit model-specific information for optimization.

\n\n
\n 🛑   WARNING:

- XLA can not currently compile functions where dimensions are not inferrable: that is, if it's not possible to infer the dimensions of all tensors without running the entire computation
\n
\n\n
\n 📌   NOTE:

- XLA compilation is only applied to code that is compiled into a graph (in TF2 that's only a code inside tf.function).
- The jit_compile API has must-compile semantics, i.e. either the entire function is compiled with XLA, or an errors.InvalidArgumentError exception is thrown)\n
\n\n
\n 📖   REFERENCE:

- XLA: Optimizing Compiler for Machine Learning
\n
","metadata":{"papermill":{"duration":0.051494,"end_time":"2021-11-06T21:17:53.236017","exception":false,"start_time":"2021-11-06T21:17:53.184523","status":"completed"},"tags":[]}},{"cell_type":"code","source":"print(f\"\\n... XLA OPTIMIZATIONS STARTING ...\\n\")\n\nprint(f\"\\n... CONFIGURE JIT (JUST IN TIME) COMPILATION ...\\n\")\n# enable XLA optmizations (10% speedup when using @tf.function calls)\ntf.config.optimizer.set_jit(True)\n\nprint(f\"\\n... XLA OPTIMIZATIONS COMPLETED ...\\n\")","metadata":{"papermill":{"duration":0.128128,"end_time":"2021-11-06T21:17:53.442803","exception":false,"start_time":"2021-11-06T21:17:53.314675","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-03-10T15:21:19.058438Z","iopub.execute_input":"2022-03-10T15:21:19.058772Z","iopub.status.idle":"2022-03-10T15:21:19.073696Z","shell.execute_reply.started":"2022-03-10T15:21:19.058729Z","shell.execute_reply":"2022-03-10T15:21:19.072739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

2.4 BASIC DATA DEFINITIONS & INITIALIZATIONS

\n\n---\n","metadata":{"papermill":{"duration":0.090507,"end_time":"2021-11-06T21:17:53.624612","exception":false,"start_time":"2021-11-06T21:17:53.534105","status":"completed"},"tags":[]}},{"cell_type":"code","source":"print(\"\\n... BASIC DATA SETUP STARTING ...\\n\\n\")\n\nprint(\"\\n... TRAIN DATAFRAME ...\\n\")\nTRAIN_CSV = os.path.join(DATA_DIR, \"train.csv\")\ntrain_df = pd.read_csv(TRAIN_CSV)\ntrain_df[\"f_path\"] = DATA_DIR+\"/train/\"+train_df.id+\".jpeg\"\ndisplay(train_df)\n\nprint(\"\\n... SAMPLE SUBMISSION DATAFRAME ...\\n\")\nSS_CSV = os.path.join(DATA_DIR, \"sample_submission.csv\")\nss_df = pd.read_csv(SS_CSV)\nss_df[\"f_path\"] = DATA_DIR+\"/test/\"+ss_df.id+\".jpeg\"\ndisplay(ss_df)","metadata":{"execution":{"iopub.status.busy":"2022-03-10T15:21:19.074764Z","iopub.execute_input":"2022-03-10T15:21:19.075761Z","iopub.status.idle":"2022-03-10T15:21:19.212979Z","shell.execute_reply.started":"2022-03-10T15:21:19.075722Z","shell.execute_reply":"2022-03-10T15:21:19.211782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n\n\n\n\n\n

\n 3  HELPER FUNCTION & CLASSES    ⤒\n

\n\n---","metadata":{"papermill":{"duration":0.054893,"end_time":"2021-11-06T21:17:54.695576","exception":false,"start_time":"2021-11-06T21:17:54.640683","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def flatten_l_o_l(nested_list):\n \"\"\" Flatten a list of lists \"\"\"\n nested_list = [x if type(x) is list else [x,] for x in nested_list]\n return [item for sublist in nested_list for item in sublist]\n\ndef sector_cleaner_2nd(img):\n img=img/255\n for i in range(4):\n for j in range(4):\n border = \\\n img[1000*i:1000*(i+1),min(1000*j,3999)].sum() +\\\n img[1000*i:1000*(i+1),1000*(j+1)-1].sum() +\\\n img[min(1000*i,3999),1000*j:1000*(j+1)].sum() +\\\n img[1000*(i+1)-1,1000*j:1000*(j+1)].sum()\n if border>=2000:\n img[1000*i:1000*(i+1),1000*j:1000*(j+1)]=np.abs(img[1000*i:1000*(i+1),1000*j:1000*(j+1)]-1)\n \n return (img*255).astype(np.uint8)\n\ndef path_to_crops(f_path, tmp_size=(4000,4000), resize_to=(28,28), pad_black=(2,2), \n blur_kernel=(7,7), fill_kernel=(11,11), thresh_val=50, \n inter_down=cv2.INTER_AREA, c_area_max=0.000001, use_sec_clean=True,\n return_sec=False, save_to_disk=True, save_dir=\"/kaggle/working/28x28_crops\"):\n \n if pad_black is not None:\n resize_to = (resize_to[0]-pad_black[0]*2, resize_to[1]-pad_black[1]*2)\n \n # Read Image\n orig_img = cv2.imread(f_path, flags=cv2.IMREAD_GRAYSCALE)\n \n sec_clean_img = sector_cleaner_2nd(orig_img.copy())\n \n # Resize To Smaller For Performance Improvement \n sec_clean_img = cv2.resize(sec_clean_img, tmp_size, interpolation=inter_down)\n \n # Create a copy to use to find the bounding boxes for the numbers\n img = sec_clean_img.copy()\n \n # Some intermediate initialization\n resized_area = tmp_size[0]*tmp_size[1]\n cnt_area_thresh = c_area_max*resized_area\n \n # Blur the image\n img = cv2.GaussianBlur(img, blur_kernel, 0)\n \n # Threshold the image\n img = cv2.threshold(img, thresh_val, 255, cv2.THRESH_BINARY)[1]\n \n # Fill in contours that are near to each other\n # img = cv2.morphologyEx(img, cv2.MORPH_CLOSE, cv2.getStructuringElement(cv2.MORPH_ELLIPSE, fill_kernel));\n \n # find contours in the image and return bounding boxes\n cnts = cv2.findContours(img.copy(), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)\n bboxes = [cv2.boundingRect(cnt) for cnt in imutils.grab_contours(cnts) if cv2.contourArea(cnt)>cnt_area_thresh]\n crops = [cv2.copyMakeBorder(cv2.resize(sec_clean_img[box[1]:box[1]+box[3], box[0]:box[0]+box[2]], resize_to, interpolation=inter_down), top=pad_black[1], bottom=pad_black[1], left=pad_black[0], right=pad_black[1], borderType=cv2.BORDER_CONSTANT, value=0) for box in bboxes]\n \n # print(len(cnts))\n if save_to_disk:\n for i, crop in enumerate(crops):\n cv2.imwrite(os.path.join(save_dir, f\"crop_{(i+1):03}of{len(crops):03}_{f_path.rsplit('/', 1)[-1]}\"), crop)\n else:\n if return_sec:\n return sec_clean_img, crops\n else:\n return crops\n \ndef plot_df_row(row):\n _sec_clean, _crops = path_to_crops(row.f_path, save_to_disk=False, return_sec=True)\n _rows = int(np.ceil((len(_crops)+2)/4))\n\n print(\"\\n... DETECTED DIGITS ...\\n\")\n plt.figure(figsize=(20,6*_rows))\n\n plt.subplot(_rows,4,1)\n plt.imshow(cv2.resize(cv2.imread(row.f_path), (4000,4000), interpolation=cv2.INTER_AREA), cmap=\"gray\")\n plt.title(f\"ORIGINAL IMAGE - SUM={row.digit_sum}\", fontweight=\"bold\")\n plt.axis(False)\n \n plt.subplot(_rows,4,2)\n plt.imshow(_sec_clean, cmap=\"gray\")\n plt.title(f\"SEC CLEAN IMAGE - SUM={row.digit_sum}\", fontweight=\"bold\")\n plt.axis(False)\n\n for i, _crop in enumerate(_crops):\n plt.subplot(_rows,4,i+3)\n plt.imshow(_crop, cmap=\"gray\") \n plt.title(f\"\\nDETECTED DIGIT #{i+1}\", fontweight=\"bold\")\n plt.axis(False)\n plt.tight_layout()\n plt.show() \n \ndef get_tfgan_pred(crops, return_sum=False, combo=\"max\", do_combo=True):\n try:\n pred_1 = tfgan(tf.cast(tf.expand_dims(tf.stack(crops, axis=0), axis=-1), tf.float32))\n\n if do_combo:\n pred_2 = tfgan(255.- tf.cast(tf.expand_dims(tf.stack(crops, axis=0), axis=-1), tf.float32))\n if combo==\"max\":\n preds = tf.argmax(tf.math.reduce_max(tf.stack([pred_1, pred_2]), axis=0), axis=-1)\n elif combo==\"mean\":\n preds = tf.argmax(tf.math.reduce_mean(tf.stack([pred_1, pred_2]), axis=0), axis=-1)\n else:\n preds=tf.argmax(pred_1, axis=-1)\n if return_sum:\n return tf.math.reduce_sum(preds).numpy()\n else:\n return preds.numpy()\n except:\n return 1","metadata":{"papermill":{"duration":0.098071,"end_time":"2021-11-06T21:17:54.848036","exception":false,"start_time":"2021-11-06T21:17:54.749965","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-03-10T15:21:19.214371Z","iopub.execute_input":"2022-03-10T15:21:19.214616Z","iopub.status.idle":"2022-03-10T15:21:19.251257Z","shell.execute_reply.started":"2022-03-10T15:21:19.214584Z","shell.execute_reply":"2022-03-10T15:21:19.250139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n\n\n\n\n\n

\n 5  MODEL BASELINE    ⤒\n

\n\n---\n\nTBD","metadata":{"execution":{"iopub.status.busy":"2022-03-09T16:12:49.194824Z","iopub.execute_input":"2022-03-09T16:12:49.195617Z","iopub.status.idle":"2022-03-09T16:12:49.202167Z","shell.execute_reply.started":"2022-03-09T16:12:49.195577Z","shell.execute_reply":"2022-03-09T16:12:49.200859Z"}}},{"cell_type":"code","source":"os.makedirs('/kaggle/working/28x28_crops', exist_ok=True)\n_ = ss_df.f_path.progress_apply(lambda x: path_to_crops(x))","metadata":{"execution":{"iopub.status.busy":"2022-03-10T13:56:31.702768Z","iopub.execute_input":"2022-03-10T13:56:31.703031Z","iopub.status.idle":"2022-03-10T15:20:03.53868Z","shell.execute_reply.started":"2022-03-10T13:56:31.703004Z","shell.execute_reply":"2022-03-10T15:20:03.537448Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# tfgan = tfhub.KerasLayer(\"https://tfhub.dev/tensorflow/tfgan/eval/mnist/logits/1\")","metadata":{"execution":{"iopub.status.busy":"2022-03-10T01:17:14.515862Z","iopub.execute_input":"2022-03-10T01:17:14.516568Z","iopub.status.idle":"2022-03-10T01:17:17.640996Z","shell.execute_reply.started":"2022-03-10T01:17:14.51653Z","shell.execute_reply":"2022-03-10T01:17:17.640278Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# demo_row = train_df.iloc[3]\n\n# plot_df_row(demo_row)\n\n# _crops = path_to_crops(demo_row.f_path, save_to_disk=False)\n# print(get_tfgan_pred(_crops, do_combo=False))","metadata":{"execution":{"iopub.status.busy":"2022-03-10T02:04:09.054993Z","iopub.execute_input":"2022-03-10T02:04:09.055569Z","iopub.status.idle":"2022-03-10T02:04:12.79705Z","shell.execute_reply.started":"2022-03-10T02:04:09.055525Z","shell.execute_reply":"2022-03-10T02:04:12.796257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# sub_train_df = train_df.sample(100).reset_index(drop=True)\n\n# sub_train_df[\"pred_sum\"] = sub_train_df.f_path.progress_apply(lambda x: get_tfgan_pred(path_to_crops(x, save_to_disk=False), return_sum=True, do_combo=False).numpy())\n# sub_train_df[\"pred_sum\"] = sub_train_df[\"pred_sum\"].apply(lambda x: x if x<=27 else 27)\n# display(sub_train_df)\n# (sub_train_df[\"pred_sum\"]==sub_train_df[\"digit_sum\"]).sum()/100","metadata":{"execution":{"iopub.status.busy":"2022-03-10T02:01:28.038866Z","iopub.execute_input":"2022-03-10T02:01:28.03912Z","iopub.status.idle":"2022-03-10T02:01:49.83081Z","shell.execute_reply.started":"2022-03-10T02:01:28.039091Z","shell.execute_reply":"2022-03-10T02:01:49.830121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ss_df[\"digit_sum\"] = ss_df.f_path.progress_apply(lambda x: get_tfgan_pred(path_to_crops(x, save_to_disk=False), return_sum=True, do_combo=False))\n# ss_df[\"digit_sum\"] = ss_df[\"digit_sum\"].apply(lambda x: x if x<=27 else 27)\n# ss_df[[\"id\", \"digit_sum\"]].to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2022-03-10T02:04:48.277722Z","iopub.execute_input":"2022-03-10T02:04:48.278328Z","iopub.status.idle":"2022-03-10T03:27:53.075165Z","shell.execute_reply.started":"2022-03-10T02:04:48.278276Z","shell.execute_reply":"2022-03-10T03:27:53.074461Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}