{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"The original [notebook](https://www.kaggle.com/code/radek1/eda-an-overview-of-the-full-dataset) is done by Radek. I experimented to learn the techniques.","metadata":{}},{"cell_type":"markdown","source":"### import utils and otto dataset","metadata":{}},{"cell_type":"code","source":"# make sure fastkaggle is install and imported\nimport os\n\ntry: import fastkaggle\nexcept ModuleNotFoundError:\n    os.system(\"pip install -Uq fastkaggle\")\n\nfrom fastkaggle import *\n\n# use fastdebug.utils \nif iskaggle: os.system(\"pip install nbdev snoop\")\n\nif iskaggle:\n    path = \"../input/fastdebugutils0\"\n    import sys\n    sys.path\n    sys.path.insert(1, path)\n    import utils as fu\n    from utils import *\nelse: \n    from fastdebug.utils import *\n    import fastdebug.utils as fu\n\n\n\n# download (if necessary and return the path of the dataset)\nhome = \"/Users/Natsume/Documents/fastdebug/kaggle_datasets/\"\ncomp = 'otto-recommender-system' # https://www.kaggle.com/competitions/paddy-disease-classification/submissions\npath = download_kaggle_dataset(comp, local_folder=home)#, install='fastai \"timm>=0.6.2.dev0\"')\n# path = setup_comp(comp, install='fastai \"timm>=0.6.2.dev0\"')","metadata":{"execution":{"iopub.status.busy":"2022-11-14T03:53:15.451664Z","iopub.execute_input":"2022-11-14T03:53:15.452178Z","iopub.status.idle":"2022-11-14T03:53:26.586641Z","shell.execute_reply.started":"2022-11-14T03:53:15.452141Z","shell.execute_reply":"2022-11-14T03:53:26.585452Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom matplotlib import pyplot as plt","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-11-14T03:53:26.589161Z","iopub.execute_input":"2022-11-14T03:53:26.590311Z","iopub.status.idle":"2022-11-14T03:53:26.595912Z","shell.execute_reply.started":"2022-11-14T03:53:26.590270Z","shell.execute_reply":"2022-11-14T03:53:26.594641Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let us read in the `train` and `test` datasets.","metadata":{}},{"cell_type":"markdown","source":"### rd: how to read parquet file with `pd.read_parquet`","metadata":{}},{"cell_type":"code","source":"train = pd.read_parquet('../input/otto-full-optimized-memory-footprint/train.parquet')","metadata":{"execution":{"iopub.status.busy":"2022-11-14T03:53:26.597660Z","iopub.execute_input":"2022-11-14T03:53:26.598147Z","iopub.status.idle":"2022-11-14T03:53:44.953761Z","shell.execute_reply.started":"2022-11-14T03:53:26.598099Z","shell.execute_reply":"2022-11-14T03:53:44.952604Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = pd.read_parquet('../input/otto-full-optimized-memory-footprint/test.parquet')","metadata":{"execution":{"iopub.status.busy":"2022-11-14T04:03:26.377688Z","iopub.execute_input":"2022-11-14T04:03:26.378215Z","iopub.status.idle":"2022-11-14T04:03:26.636313Z","shell.execute_reply.started":"2022-11-14T04:03:26.378174Z","shell.execute_reply":"2022-11-14T04:03:26.635203Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let us also read in the pickle file that will allow us to decipher the `type` information that has been encoded as integers to conserve memory.","metadata":{}},{"cell_type":"code","source":"!pip install pickle5\n\nimport pickle5 as pickle\n\nwith open('../input/otto-full-optimized-memory-footprint/id2type.pkl', \"rb\") as fh:\n    id2type = pickle.load(fh)","metadata":{"execution":{"iopub.status.busy":"2022-11-14T03:53:44.954842Z","iopub.execute_input":"2022-11-14T03:53:44.955369Z","iopub.status.idle":"2022-11-14T03:53:56.678857Z","shell.execute_reply.started":"2022-11-14T03:53:44.955318Z","shell.execute_reply":"2022-11-14T03:53:56.677588Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"id2type","metadata":{"execution":{"iopub.status.busy":"2022-11-14T03:53:56.682598Z","iopub.execute_input":"2022-11-14T03:53:56.683389Z","iopub.status.idle":"2022-11-14T03:53:56.691578Z","shell.execute_reply.started":"2022-11-14T03:53:56.683340Z","shell.execute_reply":"2022-11-14T03:53:56.690369Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open('../input/otto-full-optimized-memory-footprint/type2id.pkl', \"rb\") as fh:\n    type2id = pickle.load(fh)","metadata":{"execution":{"iopub.status.busy":"2022-11-14T03:53:56.693020Z","iopub.execute_input":"2022-11-14T03:53:56.693422Z","iopub.status.idle":"2022-11-14T03:53:56.705602Z","shell.execute_reply.started":"2022-11-14T03:53:56.693389Z","shell.execute_reply":"2022-11-14T03:53:56.704344Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"type2id","metadata":{"execution":{"iopub.status.busy":"2022-11-14T03:53:56.707277Z","iopub.execute_input":"2022-11-14T03:53:56.707628Z","iopub.status.idle":"2022-11-14T03:53:56.718224Z","shell.execute_reply.started":"2022-11-14T03:53:56.707598Z","shell.execute_reply":"2022-11-14T03:53:56.717395Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.shape, test.shape","metadata":{"execution":{"iopub.status.busy":"2022-11-14T03:53:56.719515Z","iopub.execute_input":"2022-11-14T03:53:56.719856Z","iopub.status.idle":"2022-11-14T03:53:56.732915Z","shell.execute_reply.started":"2022-11-14T03:53:56.719825Z","shell.execute_reply":"2022-11-14T03:53:56.732063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The `train` dataset contains 216716096 datapoints with `test` containing only 6928123.\n\nProportion of `test` to `train`:","metadata":{}},{"cell_type":"code","source":"test.shape[0]/train.shape[0]","metadata":{"execution":{"iopub.status.busy":"2022-11-14T03:53:56.734492Z","iopub.execute_input":"2022-11-14T03:53:56.735166Z","iopub.status.idle":"2022-11-14T03:53:56.746899Z","shell.execute_reply.started":"2022-11-14T03:53:56.735130Z","shell.execute_reply":"2022-11-14T03:53:56.746047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The size of the test set is ~3.1% of the train set. This can give us an idea of how lightweight the inference is likely to be compared to training.","metadata":{}},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2022-11-14T03:53:56.748486Z","iopub.execute_input":"2022-11-14T03:53:56.748802Z","iopub.status.idle":"2022-11-14T03:53:56.765813Z","shell.execute_reply.started":"2022-11-14T03:53:56.748774Z","shell.execute_reply":"2022-11-14T03:53:56.764593Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### rd: pd - unique - how to find all the unique values","metadata":{}},{"cell_type":"markdown","source":"How many session are there in `train` and `test`?","metadata":{}},{"cell_type":"code","source":"train.session.unique()\ntrain.session.unique().shape","metadata":{"execution":{"iopub.status.busy":"2022-11-14T03:53:56.767133Z","iopub.execute_input":"2022-11-14T03:53:56.767636Z","iopub.status.idle":"2022-11-14T03:54:00.175115Z","shell.execute_reply.started":"2022-11-14T03:53:56.767605Z","shell.execute_reply":"2022-11-14T03:54:00.173749Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.session.unique().shape[0], test.session.unique().shape[0]","metadata":{"execution":{"iopub.status.busy":"2022-11-14T03:54:00.177015Z","iopub.execute_input":"2022-11-14T03:54:00.177391Z","iopub.status.idle":"2022-11-14T03:54:01.952527Z","shell.execute_reply.started":"2022-11-14T03:54:00.177360Z","shell.execute_reply":"2022-11-14T03:54:01.950899Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.session.max() + 1","metadata":{"execution":{"iopub.status.busy":"2022-11-14T03:54:01.955063Z","iopub.execute_input":"2022-11-14T03:54:01.955610Z","iopub.status.idle":"2022-11-14T03:54:02.161363Z","shell.execute_reply.started":"2022-11-14T03:54:01.955562Z","shell.execute_reply":"2022-11-14T03:54:02.160276Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.session.unique().shape[0]/train.session.unique().shape[0]","metadata":{"execution":{"iopub.status.busy":"2022-11-14T03:54:02.168281Z","iopub.execute_input":"2022-11-14T03:54:02.169328Z","iopub.status.idle":"2022-11-14T03:54:03.951013Z","shell.execute_reply.started":"2022-11-14T03:54:02.169261Z","shell.execute_reply":"2022-11-14T03:54:03.949772Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Seems the sessions in the test set are much shorter! Let's confirm this.","metadata":{}},{"cell_type":"code","source":"test.head()","metadata":{"execution":{"iopub.status.busy":"2022-11-14T03:54:03.952480Z","iopub.execute_input":"2022-11-14T03:54:03.952825Z","iopub.status.idle":"2022-11-14T03:54:03.965019Z","shell.execute_reply.started":"2022-11-14T03:54:03.952792Z","shell.execute_reply":"2022-11-14T03:54:03.963709Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### rd: pd - group - how to use group","metadata":{}},{"cell_type":"code","source":"test.groupby('session')['aid'].count()\ntrain.groupby('session')['aid'].count()","metadata":{"execution":{"iopub.status.busy":"2022-11-14T03:54:03.966470Z","iopub.execute_input":"2022-11-14T03:54:03.967642Z","iopub.status.idle":"2022-11-14T03:54:13.887134Z","shell.execute_reply.started":"2022-11-14T03:54:03.967596Z","shell.execute_reply":"2022-11-14T03:54:13.885758Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.groupby('session')['aid'].count().apply(np.log1p)","metadata":{"execution":{"iopub.status.busy":"2022-11-14T03:54:13.889258Z","iopub.execute_input":"2022-11-14T03:54:13.889644Z","iopub.status.idle":"2022-11-14T03:54:14.349730Z","shell.execute_reply.started":"2022-11-14T03:54:13.889599Z","shell.execute_reply":"2022-11-14T03:54:14.348496Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.groupby('session')['aid'].count().apply(np.log1p).hist()","metadata":{"execution":{"iopub.status.busy":"2022-11-14T03:54:14.351753Z","iopub.execute_input":"2022-11-14T03:54:14.352134Z","iopub.status.idle":"2022-11-14T03:54:15.126108Z","shell.execute_reply.started":"2022-11-14T03:54:14.352102Z","shell.execute_reply":"2022-11-14T03:54:15.124789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.groupby('session')['aid'].count().apply(np.log1p).hist()","metadata":{"execution":{"iopub.status.busy":"2022-11-14T03:54:15.127817Z","iopub.execute_input":"2022-11-14T03:54:15.128355Z","iopub.status.idle":"2022-11-14T03:54:25.306582Z","shell.execute_reply.started":"2022-11-14T03:54:15.128306Z","shell.execute_reply":"2022-11-14T03:54:25.304949Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### rd: np - log1p - return natural log and also be super accurate in floating point","metadata":{}},{"cell_type":"code","source":"np.info(np.log1p) # see the notes for its actual usage here","metadata":{"execution":{"iopub.status.busy":"2022-11-14T03:54:25.308169Z","iopub.execute_input":"2022-11-14T03:54:25.308556Z","iopub.status.idle":"2022-11-14T03:54:25.314868Z","shell.execute_reply.started":"2022-11-14T03:54:25.308518Z","shell.execute_reply":"2022-11-14T03:54:25.313566Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### rd: recsys - otto - are train and test sessions have time intersection? how to convert timestamp to date? question","metadata":{}},{"cell_type":"markdown","source":"I have raised a [question](https://www.kaggle.com/code/radek1/eda-an-overview-of-the-full-dataset/comments#2024482) on why `/1000` in converting timestamp","metadata":{}},{"cell_type":"markdown","source":"There might be something at play here. Could the organizers have thrown us a curve ball and the train and test data are not from the same distribution?\n\nLet's quickly look at timestamps.","metadata":{}},{"cell_type":"code","source":"import datetime\n\ndatetime.datetime.fromtimestamp(train.ts.min()/1000), datetime.datetime.fromtimestamp(train.ts.max()/1000)\ndatetime.datetime.fromtimestamp(train.ts.min()), datetime.datetime.fromtimestamp(train.ts.max())","metadata":{"execution":{"iopub.status.busy":"2022-11-14T03:54:25.316706Z","iopub.execute_input":"2022-11-14T03:54:25.317206Z","iopub.status.idle":"2022-11-14T03:54:26.105899Z","shell.execute_reply.started":"2022-11-14T03:54:25.317158Z","shell.execute_reply":"2022-11-14T03:54:26.104832Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import datetime\n\ndatetime.datetime.fromtimestamp(test.ts.min()/1000), datetime.datetime.fromtimestamp(test.ts.max()/1000)\ndatetime.datetime.fromtimestamp(test.ts.min()), datetime.datetime.fromtimestamp(test.ts.max())","metadata":{"execution":{"iopub.status.busy":"2022-11-14T03:54:26.107193Z","iopub.execute_input":"2022-11-14T03:54:26.107540Z","iopub.status.idle":"2022-11-14T03:54:26.148302Z","shell.execute_reply.started":"2022-11-14T03:54:26.107509Z","shell.execute_reply":"2022-11-14T03:54:26.146945Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Looks like we have temporally split data. The problem is that the data doesn't come from the same period. In most geographies the beginning of September is the start of the school year!\n\nThat is the period where people are coming back from vacation, commerce resumes after a slowdown during the vacation season.\n\nThe organizers are not making this easy for us 🙂","metadata":{}},{"cell_type":"markdown","source":"### rd: recsys - otto - any new items in test not in train? len - set - tolist","metadata":{}},{"cell_type":"markdown","source":"Let's see if there are any new items in the test set that were not see in train.","metadata":{}},{"cell_type":"code","source":"len(set(test.aid.tolist()) - set(train.aid.tolist()))","metadata":{"execution":{"iopub.status.busy":"2022-11-14T03:54:26.149875Z","iopub.execute_input":"2022-11-14T03:54:26.150279Z","iopub.status.idle":"2022-11-14T03:55:19.126196Z","shell.execute_reply.started":"2022-11-14T03:54:26.150244Z","shell.execute_reply":"2022-11-14T03:55:19.125260Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### rd: recsys - otto - do sessions in train and test have different length distributions? groupby - count - describe","metadata":{}},{"cell_type":"markdown","source":"So at least we have that going for us, no new items in the test set! 😊\n\nI just scanned the forums really quickly and seems we have an answer as to why the session length differs between `train` and `test`!\n\nFirst, let's look at the data more closely.","metadata":{}},{"cell_type":"code","source":"train.groupby('session')['aid'].count().describe()","metadata":{"execution":{"iopub.status.busy":"2022-11-14T03:55:19.128134Z","iopub.execute_input":"2022-11-14T03:55:19.128585Z","iopub.status.idle":"2022-11-14T03:55:27.979553Z","shell.execute_reply.started":"2022-11-14T03:55:19.128541Z","shell.execute_reply":"2022-11-14T03:55:27.978346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.groupby('session')['aid'].count().describe()","metadata":{"execution":{"iopub.status.busy":"2022-11-14T03:55:27.981487Z","iopub.execute_input":"2022-11-14T03:55:27.981910Z","iopub.status.idle":"2022-11-14T03:55:28.465899Z","shell.execute_reply.started":"2022-11-14T03:55:27.981873Z","shell.execute_reply":"2022-11-14T03:55:28.464574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### rd: recsys - otto - what is a session with detailed answers? why is train sessions and test session are different in time length and total items\n\nAnd here is the [key piece of information on the forums](https://www.kaggle.com/competitions/otto-recommender-system/discussion/363554#2015486).\n\nApparently, a session are all actions by a user in the tracking period. So naturally, if the tracking period is shorter, the sessions will also be shorter.\n\nMaybe there is nothing amiss happening here.","metadata":{}},{"cell_type":"markdown","source":"### rd: recsys - otto - does sessions or users intersect between train and test","metadata":{}},{"cell_type":"code","source":"train.session.max(), test.session.min()","metadata":{"execution":{"iopub.status.busy":"2022-11-14T03:55:28.467226Z","iopub.execute_input":"2022-11-14T03:55:28.467620Z","iopub.status.idle":"2022-11-14T03:55:28.674649Z","shell.execute_reply.started":"2022-11-14T03:55:28.467553Z","shell.execute_reply":"2022-11-14T03:55:28.673499Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"An we see that the `session_ids` are not overlapping between `train` and `test` so it will be impossible to map the users (even if we have seen them before in train). We have to assume each session is from a different user.\n\nNow that I know a bit more about the dataset, I can't wait to start playing around with it. This is shaping up to be a very interesting problem! 😊","metadata":{}},{"cell_type":"markdown","source":"### TBA","metadata":{}},{"cell_type":"code","source":"train.head()\ntest.head()","metadata":{"execution":{"iopub.status.busy":"2022-11-14T03:55:28.675954Z","iopub.execute_input":"2022-11-14T03:55:28.676357Z","iopub.status.idle":"2022-11-14T03:55:28.694966Z","shell.execute_reply.started":"2022-11-14T03:55:28.676322Z","shell.execute_reply":"2022-11-14T03:55:28.693476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# the number of items\nlen(train.session.unique())\nlen(test.session.unique())","metadata":{"execution":{"iopub.status.busy":"2022-11-14T03:55:28.697066Z","iopub.execute_input":"2022-11-14T03:55:28.697944Z","iopub.status.idle":"2022-11-14T03:55:30.487276Z","shell.execute_reply.started":"2022-11-14T03:55:28.697895Z","shell.execute_reply":"2022-11-14T03:55:30.486138Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# the number of items\nlen(train.aid.unique())\nlen(test.aid.unique())","metadata":{"execution":{"iopub.status.busy":"2022-11-14T03:55:30.488701Z","iopub.execute_input":"2022-11-14T03:55:30.489184Z","iopub.status.idle":"2022-11-14T03:55:41.309642Z","shell.execute_reply.started":"2022-11-14T03:55:30.489140Z","shell.execute_reply":"2022-11-14T03:55:41.306606Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#events\ntrain.shape[0]\ntest.shape[0]","metadata":{"execution":{"iopub.status.busy":"2022-11-14T03:55:41.312761Z","iopub.execute_input":"2022-11-14T03:55:41.313280Z","iopub.status.idle":"2022-11-14T03:55:41.335656Z","shell.execute_reply.started":"2022-11-14T03:55:41.313233Z","shell.execute_reply":"2022-11-14T03:55:41.330982Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#clicks\ntrain.loc[train.type == 0].shape\ntest.loc[test.type == 0].shape","metadata":{"execution":{"iopub.status.busy":"2022-11-14T03:55:41.340228Z","iopub.execute_input":"2022-11-14T03:55:41.341022Z","iopub.status.idle":"2022-11-14T03:55:56.373871Z","shell.execute_reply.started":"2022-11-14T03:55:41.340956Z","shell.execute_reply":"2022-11-14T03:55:56.372532Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#carts\ntrain.loc[train.type == 1].shape\ntest.loc[test.type == 1].shape","metadata":{"execution":{"iopub.status.busy":"2022-11-14T03:55:56.375959Z","iopub.execute_input":"2022-11-14T03:55:56.376749Z","iopub.status.idle":"2022-11-14T03:55:57.777241Z","shell.execute_reply.started":"2022-11-14T03:55:56.376709Z","shell.execute_reply":"2022-11-14T03:55:57.775922Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#orders\tDensity\ntrain.loc[train.type == 2].shape\ntest.loc[test.type == 2].shape","metadata":{"execution":{"iopub.status.busy":"2022-11-14T03:55:57.778883Z","iopub.execute_input":"2022-11-14T03:55:57.779323Z","iopub.status.idle":"2022-11-14T03:55:58.376216Z","shell.execute_reply.started":"2022-11-14T03:55:57.779286Z","shell.execute_reply":"2022-11-14T03:55:58.374841Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Density\n# train.loc[train.type == 2].shape\n# test.loc[test.type == 2].shape","metadata":{"execution":{"iopub.status.busy":"2022-11-14T03:55:58.377735Z","iopub.execute_input":"2022-11-14T03:55:58.378296Z","iopub.status.idle":"2022-11-14T03:55:58.383913Z","shell.execute_reply.started":"2022-11-14T03:55:58.378243Z","shell.execute_reply":"2022-11-14T03:55:58.382429Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Train and test: events per session\ntrain.groupby(\"session\")[\"type\"].count().describe()\ntest.groupby(\"session\")[\"type\"].count().describe()","metadata":{"execution":{"iopub.status.busy":"2022-11-14T03:55:58.385754Z","iopub.execute_input":"2022-11-14T03:55:58.386274Z","iopub.status.idle":"2022-11-14T03:56:08.422463Z","shell.execute_reply.started":"2022-11-14T03:55:58.386219Z","shell.execute_reply":"2022-11-14T03:56:08.421126Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### convert int to uint8, how to tell whether info will get lost or not?","metadata":{}},{"cell_type":"code","source":"train.ts[0].dtype\ntrain.type[0].dtype\ntrain.memory_usage()\ntest.memory_usage()","metadata":{"execution":{"iopub.status.busy":"2022-11-14T04:25:00.999927Z","iopub.execute_input":"2022-11-14T04:25:01.000412Z","iopub.status.idle":"2022-11-14T04:25:01.022903Z","shell.execute_reply.started":"2022-11-14T04:25:01.000375Z","shell.execute_reply":"2022-11-14T04:25:01.022060Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import datetime\n\ndatetime.datetime.fromtimestamp(train.ts.min()/1000), datetime.datetime.fromtimestamp(train.ts.max()/1000)\ndatetime.datetime.fromtimestamp(train.ts.min()), datetime.datetime.fromtimestamp(train.ts.max())","metadata":{"execution":{"iopub.status.busy":"2022-11-14T03:56:08.450221Z","iopub.execute_input":"2022-11-14T03:56:08.451222Z","iopub.status.idle":"2022-11-14T03:56:09.236431Z","shell.execute_reply.started":"2022-11-14T03:56:08.451160Z","shell.execute_reply":"2022-11-14T03:56:09.235194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import datetime\n\ndatetime.datetime.fromtimestamp(test.ts.min()/1000), datetime.datetime.fromtimestamp(test.ts.max()/1000)\ndatetime.datetime.fromtimestamp(test.ts.min()), datetime.datetime.fromtimestamp(test.ts.max())","metadata":{"execution":{"iopub.status.busy":"2022-11-14T03:56:09.238336Z","iopub.execute_input":"2022-11-14T03:56:09.239383Z","iopub.status.idle":"2022-11-14T03:56:09.278755Z","shell.execute_reply.started":"2022-11-14T03:56:09.239345Z","shell.execute_reply":"2022-11-14T03:56:09.277141Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2022-11-14T04:04:51.422340Z","iopub.execute_input":"2022-11-14T04:04:51.422770Z","iopub.status.idle":"2022-11-14T04:04:51.475500Z","shell.execute_reply.started":"2022-11-14T04:04:51.422726Z","shell.execute_reply":"2022-11-14T04:04:51.474337Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_exp = test.copy()\ntest_exp.ts = test_exp.ts.astype(np.uint8)\ntest_exp.memory_usage()\ntest.memory_usage()","metadata":{"execution":{"iopub.status.busy":"2022-11-14T04:05:38.708791Z","iopub.execute_input":"2022-11-14T04:05:38.709199Z","iopub.status.idle":"2022-11-14T04:05:38.728596Z","shell.execute_reply.started":"2022-11-14T04:05:38.709167Z","shell.execute_reply":"2022-11-14T04:05:38.727235Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_exp.ts[0]\ntest.ts[0]","metadata":{"execution":{"iopub.status.busy":"2022-11-14T04:06:18.465588Z","iopub.execute_input":"2022-11-14T04:06:18.466426Z","iopub.status.idle":"2022-11-14T04:06:18.476024Z","shell.execute_reply.started":"2022-11-14T04:06:18.466387Z","shell.execute_reply":"2022-11-14T04:06:18.474733Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_exp = test.copy()\ntest_exp.ts = test_exp.ts/1000\ntest_exp.ts = test_exp.ts.astype(np.uint8)\ntest_exp.memory_usage()\ntest.memory_usage()","metadata":{"execution":{"iopub.status.busy":"2022-11-14T04:29:18.425603Z","iopub.execute_input":"2022-11-14T04:29:18.426143Z","iopub.status.idle":"2022-11-14T04:29:18.568894Z","shell.execute_reply.started":"2022-11-14T04:29:18.426101Z","shell.execute_reply":"2022-11-14T04:29:18.567624Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_exp.ts[0]\ntest.ts[0]","metadata":{"execution":{"iopub.status.busy":"2022-11-14T04:29:21.143090Z","iopub.execute_input":"2022-11-14T04:29:21.143487Z","iopub.status.idle":"2022-11-14T04:29:21.154641Z","shell.execute_reply.started":"2022-11-14T04:29:21.143458Z","shell.execute_reply":"2022-11-14T04:29:21.153239Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import datetime\n\ndatetime.datetime.fromtimestamp(train.ts.min()/1000), datetime.datetime.fromtimestamp(train.ts.max()/1000)\ndatetime.datetime.fromtimestamp(train.ts.min()), datetime.datetime.fromtimestamp(train.ts.max())","metadata":{"execution":{"iopub.status.busy":"2022-11-14T04:40:14.027781Z","iopub.execute_input":"2022-11-14T04:40:14.028303Z","iopub.status.idle":"2022-11-14T04:40:14.830113Z","shell.execute_reply.started":"2022-11-14T04:40:14.028263Z","shell.execute_reply":"2022-11-14T04:40:14.828781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import datetime\ndatetime.datetime.fromtimestamp(test_exp.ts.max()/1000000), datetime.datetime.fromtimestamp(test.ts.max()/1000000)\ndatetime.datetime.fromtimestamp(test_exp.ts.max()/1000), datetime.datetime.fromtimestamp(test.ts.max()/1000)\ndatetime.datetime.fromtimestamp(test_exp.ts.max()), datetime.datetime.fromtimestamp(test.ts.max())","metadata":{"execution":{"iopub.status.busy":"2022-11-14T04:40:58.897682Z","iopub.execute_input":"2022-11-14T04:40:58.898883Z","iopub.status.idle":"2022-11-14T04:40:58.952801Z","shell.execute_reply.started":"2022-11-14T04:40:58.898838Z","shell.execute_reply":"2022-11-14T04:40:58.951553Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### If convert to uint8 can save RAM and not lose info, then why not convert other columns to uint8?","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}