{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Leakage Estimation v2\n\nリークしてる行（＝`id`完全一致）がどの程度存在するかを調べる。","metadata":{}},{"cell_type":"code","source":"from math import nan, isnan\n\nimport pandas as pd\nfrom tqdm import tqdm","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-10T04:52:59.389544Z","iopub.execute_input":"2022-07-10T04:52:59.390421Z","iopub.status.idle":"2022-07-10T04:52:59.421608Z","shell.execute_reply.started":"2022-07-10T04:52:59.390284Z","shell.execute_reply":"2022-07-10T04:52:59.420640Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv(\n    \"../input/4sq-leakage-train-data/train.csv/train.csv\",\n    usecols=[\"id\", \"point_of_interest\"]\n)\ntrain","metadata":{"execution":{"iopub.status.busy":"2022-07-10T04:53:20.770330Z","iopub.execute_input":"2022-07-10T04:53:20.771425Z","iopub.status.idle":"2022-07-10T04:53:24.357149Z","shell.execute_reply.started":"2022-07-10T04:53:20.771367Z","shell.execute_reply":"2022-07-10T04:53:24.356034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.drop_duplicates(subset=[\"id\"], inplace=True)\ntrain","metadata":{"execution":{"iopub.status.busy":"2022-07-10T04:53:24.359220Z","iopub.execute_input":"2022-07-10T04:53:24.359575Z","iopub.status.idle":"2022-07-10T04:53:24.834102Z","shell.execute_reply.started":"2022-07-10T04:53:24.359513Z","shell.execute_reply":"2022-07-10T04:53:24.832986Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = pd.read_csv(\"../input/foursquare-location-matching/test.csv\", usecols=[\"id\"])\ntest = test.merge(train, on=(\"id\"), how=\"left\")\ntest","metadata":{"execution":{"iopub.status.busy":"2022-07-10T04:53:24.835765Z","iopub.execute_input":"2022-07-10T04:53:24.836432Z","iopub.status.idle":"2022-07-10T04:53:25.435722Z","shell.execute_reply.started":"2022-07-10T04:53:24.836390Z","shell.execute_reply":"2022-07-10T04:53:25.434603Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"non_leakage_ratio = sum(test[\"point_of_interest\"].isna()) / len(test)\nnon_leakage_ratio","metadata":{"execution":{"iopub.status.busy":"2022-07-10T04:53:55.997206Z","iopub.execute_input":"2022-07-10T04:53:55.997653Z","iopub.status.idle":"2022-07-10T04:53:56.006709Z","shell.execute_reply.started":"2022-07-10T04:53:55.997619Z","shell.execute_reply":"2022-07-10T04:53:56.005846Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.DataFrame({\"id\": test[\"id\"], \"matches\": [nan] * len(test)})\nsubmission","metadata":{"execution":{"iopub.status.busy":"2022-07-10T04:53:25.447646Z","iopub.execute_input":"2022-07-10T04:53:25.448515Z","iopub.status.idle":"2022-07-10T04:53:25.465122Z","shell.execute_reply.started":"2022-07-10T04:53:25.448473Z","shell.execute_reply":"2022-07-10T04:53:25.463742Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"idxs = submission.index < int(len(submission) * non_leakage_ratio)\nsubmission.loc[idxs, \"matches\"] = submission.loc[idxs, \"id\"]\nsubmission","metadata":{"execution":{"iopub.status.busy":"2022-07-10T04:53:25.467086Z","iopub.execute_input":"2022-07-10T04:53:25.467569Z","iopub.status.idle":"2022-07-10T04:53:25.482510Z","shell.execute_reply.started":"2022-07-10T04:53:25.467535Z","shell.execute_reply":"2022-07-10T04:53:25.481261Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv(\"submission.csv\", index=False)\npd.read_csv(\"submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-07-10T04:53:25.638535Z","iopub.execute_input":"2022-07-10T04:53:25.639153Z","iopub.status.idle":"2022-07-10T04:53:25.656377Z","shell.execute_reply.started":"2022-07-10T04:53:25.639112Z","shell.execute_reply":"2022-07-10T04:53:25.655136Z"},"trusted":true},"execution_count":null,"outputs":[]}]}