{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install nb_black","metadata":{"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%load_ext lab_black\n\nimport os\nimport json\nfrom datetime import datetime\nfrom multiprocessing import Pool\n\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport cudf, cupy\n\nfrom pathlib import Path\nfrom tqdm import tqdm\n\nplt.style.use(\"ggplot\")\ndata_path = Path(\"../input/otto-recommender-system\")\ncache_path = Path(\"../input/otto-full-optimized-memory-footprint\")\nassert data_path.exists()\nassert cache_path.exists()\n\n\nclass Config:\n    chunk_size = 10_000\n    n_threds = 22\n    train = data_path / \"train.jsonl\"\n    test = data_path / \"test.jsonl\"\n    submission = data_path / \"sample_submission.csv\"\n    test_cache = cache_path / \"test.parquet\"\n    train_cache = cache_path / \"train.parquet\"\n\n\nprint(\"Using RAPIDS version\", cudf.__version__)","metadata":{"execution":{"iopub.status.busy":"2022-11-17T10:23:59.865204Z","iopub.execute_input":"2022-11-17T10:23:59.865866Z","iopub.status.idle":"2022-11-17T10:24:01.303800Z","shell.execute_reply.started":"2022-11-17T10:23:59.865784Z","shell.execute_reply":"2022-11-17T10:24:01.302699Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import builtins\nimport types\n\n\ndef imports():\n    for name, val in globals().items():\n        # module imports\n        if isinstance(val, types.ModuleType):\n            yield name, val\n\n            # functions / callables\n        if hasattr(val, \"__call__\"):\n            yield name, val\n\n\n\"\"\"\nusage: If @noglobal decorator is specified, the function throws an exception if global variables are used in it.\n\nref: https://gist.github.com/raven38/4e4c3c7a179283c441f575d6e375510c\n\"\"\"\n\n\ndef noglobal(f):\n    return types.FunctionType(\n        f.__code__, dict(imports()), f.__name__, f.__defaults__, f.__closure__\n    )","metadata":{"_kg_hide-input":true,"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2022-11-17T10:24:01.309663Z","iopub.execute_input":"2022-11-17T10:24:01.310071Z","iopub.status.idle":"2022-11-17T10:24:01.327520Z","shell.execute_reply.started":"2022-11-17T10:24:01.310034Z","shell.execute_reply":"2022-11-17T10:24:01.326480Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntrain = cudf.read_parquet(Config.train_cache)\nsessions = train.session.unique()\nsample = cupy.random.choice(sessions, len(sessions) // 100, replace=False)\ntrain = train.loc[train.session.isin(sample)]\n#train[\"ts\"] = cudf.to_datetime(train[\"ts\"] / 1000, unit=\"s\")\n\nprint(\"We are using random 1/10 of users. Truncated train data has shape\", train.shape)\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2022-11-17T10:24:01.330666Z","iopub.execute_input":"2022-11-17T10:24:01.331736Z","iopub.status.idle":"2022-11-17T10:24:04.999745Z","shell.execute_reply.started":"2022-11-17T10:24:01.331701Z","shell.execute_reply":"2022-11-17T10:24:04.998697Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntrain = train.sort_values(\"ts\")","metadata":{"execution":{"iopub.status.busy":"2022-11-17T10:24:05.003057Z","iopub.execute_input":"2022-11-17T10:24:05.003463Z","iopub.status.idle":"2022-11-17T10:24:05.063135Z","shell.execute_reply.started":"2022-11-17T10:24:05.003435Z","shell.execute_reply":"2022-11-17T10:24:05.061997Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = train.groupby([\"session\", \"aid\"]).first()","metadata":{"execution":{"iopub.status.busy":"2022-11-17T10:24:05.064380Z","iopub.execute_input":"2022-11-17T10:24:05.065237Z","iopub.status.idle":"2022-11-17T10:24:05.173143Z","shell.execute_reply.started":"2022-11-17T10:24:05.065155Z","shell.execute_reply":"2022-11-17T10:24:05.172248Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inconsistent = df[df[\"type\"] != 0].reset_index()","metadata":{"execution":{"iopub.status.busy":"2022-11-17T10:24:05.174370Z","iopub.execute_input":"2022-11-17T10:24:05.174804Z","iopub.status.idle":"2022-11-17T10:24:05.209364Z","shell.execute_reply.started":"2022-11-17T10:24:05.174768Z","shell.execute_reply":"2022-11-17T10:24:05.208484Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = train.groupby([\"session\", \"aid\"]).first()\ninconsistent = df[df[\"type\"] != 0].reset_index()\ninconsistent.to_parquet(\"inconsistent.parquet\", index=False)","metadata":{"execution":{"iopub.status.busy":"2022-11-17T10:24:05.210660Z","iopub.execute_input":"2022-11-17T10:24:05.211205Z","iopub.status.idle":"2022-11-17T10:24:05.326803Z","shell.execute_reply.started":"2022-11-17T10:24:05.211169Z","shell.execute_reply":"2022-11-17T10:24:05.325907Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"first = (\n    train.groupby([\"session\", \"aid\"]).first().reset_index().groupby(\"session\").first()\n)","metadata":{"execution":{"iopub.status.busy":"2022-11-17T10:24:05.328045Z","iopub.execute_input":"2022-11-17T10:24:05.328467Z","iopub.status.idle":"2022-11-17T10:24:05.372861Z","shell.execute_reply.started":"2022-11-17T10:24:05.328430Z","shell.execute_reply":"2022-11-17T10:24:05.372058Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inconsistent = first.merge(\n    inconsistent, on=\"session\", suffixes=[\"_first\", \"_inconsistent\"]\n)","metadata":{"execution":{"iopub.status.busy":"2022-11-17T10:24:05.374093Z","iopub.execute_input":"2022-11-17T10:24:05.374515Z","iopub.status.idle":"2022-11-17T10:24:05.385305Z","shell.execute_reply.started":"2022-11-17T10:24:05.374479Z","shell.execute_reply":"2022-11-17T10:24:05.384275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(inconsistent) / len(train.groupby([\"session\", \"aid\"]).first())","metadata":{"execution":{"iopub.status.busy":"2022-11-17T10:24:05.386770Z","iopub.execute_input":"2022-11-17T10:24:05.387377Z","iopub.status.idle":"2022-11-17T10:24:05.421870Z","shell.execute_reply.started":"2022-11-17T10:24:05.387342Z","shell.execute_reply":"2022-11-17T10:24:05.420928Z"},"trusted":true},"execution_count":null,"outputs":[]}]}