{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Leakage Estimation\n\nリークしてる行（＝(`id`, `longitude`, `latitude`)の3つが完全一致）がどの程度存在するかを調べる。","metadata":{}},{"cell_type":"code","source":"from math import nan, isnan\n\nimport pandas as pd\nfrom tqdm import tqdm","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-09T04:24:37.503418Z","iopub.execute_input":"2022-07-09T04:24:37.504447Z","iopub.status.idle":"2022-07-09T04:24:37.531639Z","shell.execute_reply.started":"2022-07-09T04:24:37.504345Z","shell.execute_reply":"2022-07-09T04:24:37.530236Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv(\n    \"../input/4sq-leakage-train-data/train.csv/train.csv\",\n    usecols=[\"name\", \"latitude\", \"longitude\", \"point_of_interest\"]\n)\ntrain","metadata":{"execution":{"iopub.status.busy":"2022-07-09T04:24:37.534070Z","iopub.execute_input":"2022-07-09T04:24:37.534907Z","iopub.status.idle":"2022-07-09T04:24:44.815570Z","shell.execute_reply.started":"2022-07-09T04:24:37.534858Z","shell.execute_reply":"2022-07-09T04:24:44.813861Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.drop_duplicates(subset=[\"name\", \"latitude\", \"longitude\"], inplace=True)\ntrain","metadata":{"execution":{"iopub.status.busy":"2022-07-09T04:24:44.817518Z","iopub.execute_input":"2022-07-09T04:24:44.818037Z","iopub.status.idle":"2022-07-09T04:24:46.301823Z","shell.execute_reply.started":"2022-07-09T04:24:44.817988Z","shell.execute_reply":"2022-07-09T04:24:46.300605Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = pd.read_csv(\"../input/foursquare-location-matching/test.csv\", usecols=[\"name\", \"id\", \"latitude\", \"longitude\"])\ntest = test.merge(train, on=(\"name\", \"latitude\", \"longitude\"), how=\"left\")\ntest","metadata":{"execution":{"iopub.status.busy":"2022-07-09T04:24:46.306234Z","iopub.execute_input":"2022-07-09T04:24:46.307739Z","iopub.status.idle":"2022-07-09T04:24:48.512876Z","shell.execute_reply.started":"2022-07-09T04:24:46.307622Z","shell.execute_reply":"2022-07-09T04:24:48.511837Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"leakage_ratio = sum(test[\"point_of_interest\"].isna()) / len(test)\nleakage_ratio","metadata":{"execution":{"iopub.status.busy":"2022-07-09T04:25:58.050453Z","iopub.execute_input":"2022-07-09T04:25:58.050879Z","iopub.status.idle":"2022-07-09T04:25:58.061616Z","shell.execute_reply.started":"2022-07-09T04:25:58.050845Z","shell.execute_reply":"2022-07-09T04:25:58.059364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.DataFrame({\"id\": test[\"id\"], \"matches\": [nan] * len(test)})\nsubmission","metadata":{"execution":{"iopub.status.busy":"2022-07-09T04:32:05.245227Z","iopub.execute_input":"2022-07-09T04:32:05.245592Z","iopub.status.idle":"2022-07-09T04:32:05.258571Z","shell.execute_reply.started":"2022-07-09T04:32:05.245559Z","shell.execute_reply":"2022-07-09T04:32:05.256920Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"idxs = submission.index < int(len(submission) * leakage_ratio)\nsubmission.loc[idxs, \"matches\"] = submission.loc[idxs, \"id\"]\nsubmission","metadata":{"execution":{"iopub.status.busy":"2022-07-09T04:32:05.790202Z","iopub.execute_input":"2022-07-09T04:32:05.790594Z","iopub.status.idle":"2022-07-09T04:32:05.804120Z","shell.execute_reply.started":"2022-07-09T04:32:05.790561Z","shell.execute_reply":"2022-07-09T04:32:05.802688Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv(\"submission.csv\", index=False)\npd.read_csv(\"submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-07-09T04:32:11.618660Z","iopub.execute_input":"2022-07-09T04:32:11.619070Z","iopub.status.idle":"2022-07-09T04:32:11.636221Z","shell.execute_reply.started":"2022-07-09T04:32:11.619035Z","shell.execute_reply":"2022-07-09T04:32:11.635084Z"},"trusted":true},"execution_count":null,"outputs":[]}]}