{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":11000,"databundleVersionId":875412,"sourceType":"competition"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import polars as pl\npl.__version__","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-12-15T10:40:09.684871Z","iopub.execute_input":"2023-12-15T10:40:09.685813Z","iopub.status.idle":"2023-12-15T10:40:09.693121Z","shell.execute_reply.started":"2023-12-15T10:40:09.685780Z","shell.execute_reply":"2023-12-15T10:40:09.691380Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pl.scan_csv('/kaggle/input/LANL-Earthquake-Prediction/train.csv', n_rows=10_000_000)","metadata":{"execution":{"iopub.status.busy":"2023-12-15T10:40:09.877777Z","iopub.execute_input":"2023-12-15T10:40:09.878132Z","iopub.status.idle":"2023-12-15T10:40:09.885119Z","shell.execute_reply.started":"2023-12-15T10:40:09.878107Z","shell.execute_reply":"2023-12-15T10:40:09.883717Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"other = pl.LazyFrame({'acoustic_data': [12, 10, 15, 21, 22]})","metadata":{"execution":{"iopub.status.busy":"2023-12-15T10:40:10.239344Z","iopub.execute_input":"2023-12-15T10:40:10.239709Z","iopub.status.idle":"2023-12-15T10:40:10.244829Z","shell.execute_reply.started":"2023-12-15T10:40:10.239683Z","shell.execute_reply":"2023-12-15T10:40:10.243962Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Splitting, then collecting","metadata":{}},{"cell_type":"code","source":"%%time\nsemi, anti = pl.collect_all(\n    [\n        df.join(other, on='acoustic_data', how='semi'),\n        df.join(other, on='acoustic_data', how='anti')\n    ]\n)","metadata":{"execution":{"iopub.status.busy":"2023-12-15T10:40:12.175723Z","iopub.execute_input":"2023-12-15T10:40:12.176138Z","iopub.status.idle":"2023-12-15T10:40:13.004774Z","shell.execute_reply.started":"2023-12-15T10:40:12.176109Z","shell.execute_reply":"2023-12-15T10:40:13.003522Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nsemi, anti = pl.collect_all(\n    [\n        df.join(other, on='acoustic_data', how='semi'),\n        df.join(other, on='acoustic_data', how='anti')\n    ]\n)","metadata":{"execution":{"iopub.status.busy":"2023-12-15T10:39:41.898667Z","iopub.execute_input":"2023-12-15T10:39:41.899048Z","iopub.status.idle":"2023-12-15T10:39:42.706232Z","shell.execute_reply.started":"2023-12-15T10:39:41.899022Z","shell.execute_reply":"2023-12-15T10:39:42.704675Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nsemi, anti = pl.collect_all(\n    [\n        df.join(other, on='acoustic_data', how='semi'),\n        df.join(other, on='acoustic_data', how='anti')\n    ]\n)","metadata":{"execution":{"iopub.status.busy":"2023-12-15T10:39:09.185306Z","iopub.execute_input":"2023-12-15T10:39:09.186518Z","iopub.status.idle":"2023-12-15T10:39:10.002810Z","shell.execute_reply.started":"2023-12-15T10:39:09.186476Z","shell.execute_reply":"2023-12-15T10:39:10.001764Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"other","metadata":{}},{"cell_type":"code","source":"%%time\nsemi, anti = df.join(other, on='acoustic_data', how='semi').collect(), df.join(other, on='acoustic_data', how='anti').collect()","metadata":{"execution":{"iopub.status.busy":"2023-12-15T10:39:09.185306Z","iopub.execute_input":"2023-12-15T10:39:09.186518Z","iopub.status.idle":"2023-12-15T10:39:10.002810Z","shell.execute_reply.started":"2023-12-15T10:39:09.186476Z","shell.execute_reply":"2023-12-15T10:39:10.001764Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nsemi, anti = df.join(other, on='acoustic_data', how='semi').collect(), df.join(other, on='acoustic_data', how='anti').collect()","metadata":{"execution":{"iopub.status.busy":"2023-12-15T10:39:09.185306Z","iopub.execute_input":"2023-12-15T10:39:09.186518Z","iopub.status.idle":"2023-12-15T10:39:10.002810Z","shell.execute_reply.started":"2023-12-15T10:39:09.186476Z","shell.execute_reply":"2023-12-15T10:39:10.001764Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nsemi, anti = df.join(other, on='acoustic_data', how='semi').collect(), df.join(other, on='acoustic_data', how='anti').collect()","metadata":{"execution":{"iopub.status.busy":"2023-12-15T10:39:09.185306Z","iopub.execute_input":"2023-12-15T10:39:09.186518Z","iopub.status.idle":"2023-12-15T10:39:10.002810Z","shell.execute_reply.started":"2023-12-15T10:39:09.186476Z","shell.execute_reply":"2023-12-15T10:39:10.001764Z"}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Collecting, then splitting","metadata":{}},{"cell_type":"code","source":"%%time\ndf_collected = df.collect()\nother_collected = other.collect()\nsemi, anti = (\n    df_collected.join(other_collected, on='acoustic_data', how='semi'),\n    df_collected.join(other_collected, on='acoustic_data', how='anti')\n)","metadata":{"execution":{"iopub.status.busy":"2023-12-15T10:39:10.003913Z","iopub.execute_input":"2023-12-15T10:39:10.004222Z","iopub.status.idle":"2023-12-15T10:39:10.674941Z","shell.execute_reply.started":"2023-12-15T10:39:10.004199Z","shell.execute_reply":"2023-12-15T10:39:10.674044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ndf_collected = df.collect()\nother_collected = other.collect()\nsemi, anti = (\n    df_collected.join(other_collected, on='acoustic_data', how='semi'),\n    df_collected.join(other_collected, on='acoustic_data', how='anti')\n)","metadata":{"execution":{"iopub.status.busy":"2023-12-15T10:39:10.676797Z","iopub.execute_input":"2023-12-15T10:39:10.677732Z","iopub.status.idle":"2023-12-15T10:39:11.219024Z","shell.execute_reply.started":"2023-12-15T10:39:10.677697Z","shell.execute_reply":"2023-12-15T10:39:11.218178Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ndf_collected = df.collect()\nother_collected = other.collect()\nsemi, anti = (\n    df_collected.join(other_collected, on='acoustic_data', how='semi'),\n    df_collected.join(other_collected, on='acoustic_data', how='anti')\n)","metadata":{"execution":{"iopub.status.busy":"2023-12-15T10:40:27.525426Z","iopub.execute_input":"2023-12-15T10:40:27.525785Z","iopub.status.idle":"2023-12-15T10:40:28.045346Z","shell.execute_reply.started":"2023-12-15T10:40:27.525760Z","shell.execute_reply":"2023-12-15T10:40:28.044430Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}