{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":38760,"databundleVersionId":4493939,"sourceType":"competition"}],"dockerImageVersionId":30301,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"**Otto Recommender System:** Converting train.jsonl file to TFRecord file format. The idea is to use TFRecord file format further and learn TensorFlow Recommenders (TFRS) while implementing this usecase  \n\nFollowing resources gave me good understanding of WHAT, WHY and How of TFRecord file format. \n\n* https://www.tensorflow.org/tutorials/load_data/tfrecord\n* https://keras.io/examples/keras_recipes/creating_tfrecords/\n* https://www.kaggle.com/code/ryanholbrook/tfrecords-basics/notebook\n\nAs I start to learn TensorFlow, your remarks / comments for sure can help in improving my knowledge ","metadata":{}},{"cell_type":"code","source":"import numpy as np \nimport pandas as pd \nimport os\nimport zipfile\nimport json\nfrom tqdm import tqdm\nimport tensorflow as tf","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-11-30T15:18:35.832999Z","iopub.execute_input":"2022-11-30T15:18:35.833530Z","iopub.status.idle":"2022-11-30T15:18:41.428298Z","shell.execute_reply.started":"2022-11-30T15:18:35.833424Z","shell.execute_reply":"2022-11-30T15:18:41.427174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"INPUT_DIR = '../input/otto-recommender-system'\nTRAIN_FILE = f'{INPUT_DIR}/train.jsonl'\nTEST_FILE = f'{INPUT_DIR}/test.jsonl'","metadata":{"execution":{"iopub.status.busy":"2022-11-30T15:18:41.430426Z","iopub.execute_input":"2022-11-30T15:18:41.431269Z","iopub.status.idle":"2022-11-30T15:18:41.437509Z","shell.execute_reply.started":"2022-11-30T15:18:41.431222Z","shell.execute_reply":"2022-11-30T15:18:41.436246Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def _int64_feature(value):\n    # Returns an int64_list from a bool / enum / int / uint\n    return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))\n\ndef serialize_example(session, aid, ts, typ):\n    # Creates a tf.train.Example message ready to be written to a file.\n    # Create a dictionary mapping the feature name to the tf.train.Example-compatible data type.\n    feature = {\n          'session': _int64_feature(session),\n          'aid': _int64_feature(aid),\n          'ts': _int64_feature(ts),\n          'typ': _int64_feature(typ),\n      }\n    # Create a Features message using tf.train.Example.\n    example_proto = tf.train.Example(features=tf.train.Features(feature=feature))\n    return example_proto.SerializeToString()","metadata":{"execution":{"iopub.status.busy":"2022-11-30T15:18:41.439116Z","iopub.execute_input":"2022-11-30T15:18:41.439559Z","iopub.status.idle":"2022-11-30T15:18:41.457023Z","shell.execute_reply.started":"2022-11-30T15:18:41.439515Z","shell.execute_reply":"2022-11-30T15:18:41.456108Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def preprocess(df):\n    initial_data, features={}, {}\n    session_list, item_A_list, ts_list = [], [], []\n    for index in df.index:\n        session_id = str(df['session'][index])\n        session_list.append(session_id)\n        item_A_list.append(str(df['events'][index][0]['aid']))\n        ts_list.append(df['events'][index][0]['ts'])\n        click_list, cart_list, order_list = [], [], []\n        for event in df['events'][index]:\n            if event['type']=='clicks':\n                click_list.append(str(event['aid']))\n            elif event['type']=='carts':\n                cart_list.append(str(event['aid']))\n            else:\n                order_list.append(str(event['aid']))\n        initial_data[session_id] = {'clicks':click_list, 'carts':cart_list, 'orders':order_list}\n    features={'session_list':session_list, 'item_A_list':item_A_list, 'ts_list':ts_list}\n    return features, initial_data\n\ndef features_list(train_df):\n    session = []\n    aid = []\n    ts = []\n    typ = []\n    codes = {'clicks':1, 'carts':2, 'orders':3}\n\n    for index in train_df.index:\n        session_id = train_df['session'][index]\n        for event in train_df['events'][index]:\n            session.append(session_id)\n            aid.append(event['aid'])\n            ts.append(event['ts'])\n            typ.append(codes[event['type']])\n    return session, aid, ts, typ\n\ndef write_tfrecord(file_num, session, aid, ts, typ):\n    filename = f'train_chunk_{file_num}.tfrecord'\n    options = tf.io.TFRecordOptions(compression_type='ZLIB')\n    \n    with tf.io.TFRecordWriter(filename, options=options) as writer:\n        for i in range(len(session)):\n            example = serialize_example(session[i], aid[i], ts[i], typ[i])\n            writer.write(example)\n    zip_tfrecord(filename)\n    return\n\ndef zip_tfrecord(filename):\n    OUTPUT_DIR = '/kaggle/working/'\n    zip_file = f'{OUTPUT_DIR}/otto_tfrecord.zip'\n    filename = f'{OUTPUT_DIR}/{filename}'\n    \n    with zipfile.ZipFile(zip_file, 'a') as zf:\n        zf.write(filename)\n    os.remove(filename)\n    return","metadata":{"execution":{"iopub.status.busy":"2022-11-30T15:18:41.459075Z","iopub.execute_input":"2022-11-30T15:18:41.460108Z","iopub.status.idle":"2022-11-30T15:18:41.471601Z","shell.execute_reply.started":"2022-11-30T15:18:41.460059Z","shell.execute_reply":"2022-11-30T15:18:41.470429Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nnum_lines = sum(1 for line in open(TRAIN_FILE))\nprint('Number of lines: ', num_lines)\n\nchunk_size=100000\nnum_chunks= int(np.ceil(num_lines / chunk_size))\nprint('Number of chunks: ',num_chunks)","metadata":{"execution":{"iopub.status.busy":"2022-11-30T15:18:41.473203Z","iopub.execute_input":"2022-11-30T15:18:41.473592Z","iopub.status.idle":"2022-11-30T15:20:49.655683Z","shell.execute_reply.started":"2022-11-30T15:18:41.473548Z","shell.execute_reply":"2022-11-30T15:20:49.653980Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nchunks = pd.read_json(TRAIN_FILE, lines=True, chunksize=chunk_size)\n\nfor e, chunk in tqdm(enumerate(chunks)):\n    print('Processing chunk # : ', e)\n    session, aid, ts, typ = features_list(chunk)\n    write_tfrecord(e, session, aid, ts, typ)","metadata":{"execution":{"iopub.status.busy":"2022-11-30T15:20:49.657689Z","iopub.execute_input":"2022-11-30T15:20:49.658131Z","iopub.status.idle":"2022-11-30T15:20:53.696316Z","shell.execute_reply.started":"2022-11-30T15:20:49.658086Z","shell.execute_reply":"2022-11-30T15:20:53.694975Z"},"trusted":true},"execution_count":null,"outputs":[]}]}