{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"! rm -rf /tmp/4sq\n! cp -r /kaggle/input/4sq-clone-source-code-repository/4sq /tmp/4sq\n! cd /tmp/4sq; git log --oneline | head -1","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-07-07T17:51:01.794716Z","iopub.execute_input":"2022-07-07T17:51:01.795022Z","iopub.status.idle":"2022-07-07T17:51:04.644671Z","shell.execute_reply.started":"2022-07-07T17:51:01.794928Z","shell.execute_reply":"2022-07-07T17:51:04.643761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Run All Modules","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\nplt.style.use(\"ggplot\")","metadata":{"execution":{"iopub.status.busy":"2022-07-07T17:51:09.475863Z","iopub.execute_input":"2022-07-07T17:51:09.476678Z","iopub.status.idle":"2022-07-07T17:51:10.02492Z","shell.execute_reply.started":"2022-07-07T17:51:09.476628Z","shell.execute_reply":"2022-07-07T17:51:10.024207Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CFG:\n    DEBUG = False\n    K_NEIGHBORS_COMB = 18\n    K_NEIGHBORS_TEXT = 5\n    K_NEIGHBORS_LOC = 7\n    THRESHOLD = 0.5\n    BLOCKERS = {\"combination\", \"location\", \"text\"}\n    OUTPUT_PATH = \"/tmp/4sq/output\"\n    CHECKPOINT_PATH = \"/kaggle/input/4sq-trained-weights/model_ditto_all_feat_all_n25_seed1234_full_all-MiniLM-L6-v2_ep10_id0.pt\"\n    MODEL = \"/kaggle/input/4sq-pretrained-transformer-models/all-MiniLM-L6-v2\"","metadata":{"execution":{"iopub.status.busy":"2022-07-07T17:52:41.855056Z","iopub.execute_input":"2022-07-07T17:52:41.855354Z","iopub.status.idle":"2022-07-07T17:52:41.861194Z","shell.execute_reply.started":"2022-07-07T17:52:41.855319Z","shell.execute_reply":"2022-07-07T17:52:41.86017Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%bash\ncat << 'EOF' > /tmp/run.bash\nPIP_DEP_PATH='/kaggle/input/4sq-build-runtime-environment/pip_deps'\necho ${PIP_DEP_PATH}\npip install ${PIP_DEP_PATH}/* -f ./ --no-index --no-deps --find-links=\"${PIP_DEP_PATH}\"\n\nEOF\nchmod +x /tmp/run.bash","metadata":{"execution":{"iopub.status.busy":"2022-07-07T17:51:10.026445Z","iopub.execute_input":"2022-07-07T17:51:10.026754Z","iopub.status.idle":"2022-07-07T17:51:10.052367Z","shell.execute_reply.started":"2022-07-07T17:51:10.026704Z","shell.execute_reply":"2022-07-07T17:51:10.05153Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!/tmp/run.bash","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-07-07T17:51:10.054629Z","iopub.execute_input":"2022-07-07T17:51:10.054955Z","iopub.status.idle":"2022-07-07T17:51:30.063842Z","shell.execute_reply.started":"2022-07-07T17:51:10.054913Z","shell.execute_reply":"2022-07-07T17:51:30.062969Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Make H3","metadata":{}},{"cell_type":"code","source":"%set_env INPUT_PATH=/kaggle/input/foursquare-location-matching/test.csv\n%set_env OUTPUT_PATH={CFG.OUTPUT_PATH}","metadata":{"execution":{"iopub.status.busy":"2022-07-07T17:51:30.065915Z","iopub.execute_input":"2022-07-07T17:51:30.066293Z","iopub.status.idle":"2022-07-07T17:51:30.074301Z","shell.execute_reply.started":"2022-07-07T17:51:30.06625Z","shell.execute_reply":"2022-07-07T17:51:30.072534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!echo $OUTPUT_PATH","metadata":{"execution":{"iopub.status.busy":"2022-07-07T17:51:30.075553Z","iopub.execute_input":"2022-07-07T17:51:30.076144Z","iopub.status.idle":"2022-07-07T17:51:30.7693Z","shell.execute_reply.started":"2022-07-07T17:51:30.076107Z","shell.execute_reply":"2022-07-07T17:51:30.768447Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%bash\n\n(cat << 'EOF'\ncd /tmp/4sq\n\nCUDA_VISIBLE_DEVICES=0 python make_h3.py --output_path \"${OUTPUT_PATH}\" \"${INPUT_PATH}\"\n\nEOF\n) > /tmp/run.bash\nchmod +x /tmp/run.bash","metadata":{"execution":{"iopub.status.busy":"2022-07-07T17:51:30.771779Z","iopub.execute_input":"2022-07-07T17:51:30.771997Z","iopub.status.idle":"2022-07-07T17:51:30.793302Z","shell.execute_reply.started":"2022-07-07T17:51:30.771971Z","shell.execute_reply":"2022-07-07T17:51:30.792309Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n!/tmp/run.bash","metadata":{"execution":{"iopub.status.busy":"2022-07-07T17:51:30.795609Z","iopub.execute_input":"2022-07-07T17:51:30.795915Z","iopub.status.idle":"2022-07-07T17:51:32.163143Z","shell.execute_reply.started":"2022-07-07T17:51:30.795876Z","shell.execute_reply":"2022-07-07T17:51:32.162227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if CFG.DEBUG:\n    print(pd.read_csv(f\"{CFG.OUTPUT_PATH}/h3_res1.csv.gz\", compression=\"gzip\"))","metadata":{"execution":{"iopub.status.busy":"2022-07-07T17:51:32.164963Z","iopub.execute_input":"2022-07-07T17:51:32.165249Z","iopub.status.idle":"2022-07-07T17:51:32.18155Z","shell.execute_reply.started":"2022-07-07T17:51:32.165207Z","shell.execute_reply":"2022-07-07T17:51:32.18075Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Preprocess","metadata":{}},{"cell_type":"code","source":"%set_env INPUT_PATH=/kaggle/input/foursquare-location-matching/test.csv\n%set_env H3_PATH={CFG.OUTPUT_PATH}/h3_res1.csv.gz\n%set_env OUTPUT_PATH={CFG.OUTPUT_PATH}","metadata":{"execution":{"iopub.status.busy":"2022-07-07T17:51:32.182888Z","iopub.execute_input":"2022-07-07T17:51:32.183357Z","iopub.status.idle":"2022-07-07T17:51:32.192855Z","shell.execute_reply.started":"2022-07-07T17:51:32.183318Z","shell.execute_reply":"2022-07-07T17:51:32.191631Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%bash\n\n(cat << 'EOF'\ncd /tmp/4sq\n\nCUDA_VISIBLE_DEVICES=0 python preprocess.py --output_path \"${OUTPUT_PATH}\" \"${INPUT_PATH}\" \"${H3_PATH}\"\n\nEOF\n) > /tmp/run.bash\nchmod +x /tmp/run.bash","metadata":{"execution":{"iopub.status.busy":"2022-07-07T17:51:32.194327Z","iopub.execute_input":"2022-07-07T17:51:32.194909Z","iopub.status.idle":"2022-07-07T17:51:32.21564Z","shell.execute_reply.started":"2022-07-07T17:51:32.19487Z","shell.execute_reply":"2022-07-07T17:51:32.214819Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n!/tmp/run.bash","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-07-07T17:51:32.217444Z","iopub.execute_input":"2022-07-07T17:51:32.218019Z","iopub.status.idle":"2022-07-07T17:51:33.441075Z","shell.execute_reply.started":"2022-07-07T17:51:32.217981Z","shell.execute_reply":"2022-07-07T17:51:33.439718Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if CFG.DEBUG:\n    preprocessed_df = pd.read_csv(f\"{CFG.OUTPUT_PATH}/preprocessed.csv.gz\", compression=\"gzip\")\n    print(preprocessed_df.head(1).T)","metadata":{"execution":{"iopub.status.busy":"2022-07-07T17:51:33.444197Z","iopub.execute_input":"2022-07-07T17:51:33.444425Z","iopub.status.idle":"2022-07-07T17:51:33.459702Z","shell.execute_reply.started":"2022-07-07T17:51:33.444397Z","shell.execute_reply":"2022-07-07T17:51:33.458893Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Calc Embeddings","metadata":{}},{"cell_type":"markdown","source":"## Calc Embeddings","metadata":{}},{"cell_type":"code","source":"%set_env INPUT_PATH={CFG.OUTPUT_PATH}/preprocessed.csv.gz\n%set_env OUTPUT_PATH={CFG.OUTPUT_PATH}\n%set_env MODEL={CFG.MODEL}","metadata":{"execution":{"iopub.status.busy":"2022-07-07T17:51:33.46118Z","iopub.execute_input":"2022-07-07T17:51:33.461706Z","iopub.status.idle":"2022-07-07T17:51:33.470385Z","shell.execute_reply.started":"2022-07-07T17:51:33.46166Z","shell.execute_reply":"2022-07-07T17:51:33.469397Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%bash\n\n(cat << 'EOF'\ncd /tmp/4sq\n\nCUDA_VISIBLE_DEVICES=0 python calc_embeddings.py --column \"name\" --output_path \"${OUTPUT_PATH}\" \"${INPUT_PATH}\" \"${MODEL}\"\n\nEOF\n) > /tmp/run.bash\nchmod +x /tmp/run.bash","metadata":{"execution":{"iopub.status.busy":"2022-07-07T17:51:33.472261Z","iopub.execute_input":"2022-07-07T17:51:33.473647Z","iopub.status.idle":"2022-07-07T17:51:33.495999Z","shell.execute_reply.started":"2022-07-07T17:51:33.473608Z","shell.execute_reply":"2022-07-07T17:51:33.495093Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n!/tmp/run.bash","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-07-07T17:51:33.498037Z","iopub.execute_input":"2022-07-07T17:51:33.498335Z","iopub.status.idle":"2022-07-07T17:51:51.154012Z","shell.execute_reply.started":"2022-07-07T17:51:33.4983Z","shell.execute_reply":"2022-07-07T17:51:51.153124Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%bash\n\n(cat << 'EOF'\ncd /tmp/4sq\n\nCUDA_VISIBLE_DEVICES=0 python calc_embeddings.py --column \"categories\" --output_path \"${OUTPUT_PATH}\" \"${INPUT_PATH}\" \"${MODEL}\"\n\nEOF\n) > /tmp/run.bash\nchmod +x /tmp/run.bash","metadata":{"execution":{"iopub.status.busy":"2022-07-07T17:51:51.155936Z","iopub.execute_input":"2022-07-07T17:51:51.156222Z","iopub.status.idle":"2022-07-07T17:51:51.178716Z","shell.execute_reply.started":"2022-07-07T17:51:51.156181Z","shell.execute_reply":"2022-07-07T17:51:51.177692Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n!/tmp/run.bash","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-07-07T17:51:51.18144Z","iopub.execute_input":"2022-07-07T17:51:51.181681Z","iopub.status.idle":"2022-07-07T17:51:58.714784Z","shell.execute_reply.started":"2022-07-07T17:51:51.181653Z","shell.execute_reply":"2022-07-07T17:51:58.71396Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if CFG.DEBUG:\n    from sentence_transformers import SentenceTransformer\n    model = SentenceTransformer(CFG.MODEL)\n    n_dims = model.get_sentence_embedding_dimension()\n    n_items = len(preprocessed_df)\n    print(n_items, n_dims)\n    print(np.memmap(f\"{CFG.OUTPUT_PATH}/embeddings_name.mmp\", mode=\"r\", dtype=\"float32\", shape=(n_items, n_dims)))\n    print(np.memmap(f\"{CFG.OUTPUT_PATH}/embeddings_categories.mmp\", mode=\"r\", dtype=\"float32\", shape=(n_items, n_dims)))","metadata":{"execution":{"iopub.status.busy":"2022-07-07T17:51:58.716777Z","iopub.execute_input":"2022-07-07T17:51:58.717071Z","iopub.status.idle":"2022-07-07T17:52:01.568478Z","shell.execute_reply.started":"2022-07-07T17:51:58.71703Z","shell.execute_reply":"2022-07-07T17:52:01.567737Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Blocking","metadata":{}},{"cell_type":"code","source":"%set_env INPUT_PATH={CFG.OUTPUT_PATH}/preprocessed.csv.gz\n%set_env OUTPUT_PATH={CFG.OUTPUT_PATH}\n%set_env EMBEDDINGS_PATH={CFG.OUTPUT_PATH}\n%set_env K_NEIGHBORS_COMB={CFG.K_NEIGHBORS_COMB}\n%set_env K_NEIGHBORS_LOC={CFG.K_NEIGHBORS_LOC}\n%set_env K_NEIGHBORS_TEXT={CFG.K_NEIGHBORS_TEXT}","metadata":{"execution":{"iopub.status.busy":"2022-07-07T17:52:20.734072Z","iopub.execute_input":"2022-07-07T17:52:20.73434Z","iopub.status.idle":"2022-07-07T17:52:20.749432Z","shell.execute_reply.started":"2022-07-07T17:52:20.73431Z","shell.execute_reply":"2022-07-07T17:52:20.748555Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### combination","metadata":{}},{"cell_type":"code","source":"%%bash\n\n(cat << 'EOF'\ncd /tmp/4sq\n\nCUDA_VISIBLE_DEVICES=0 python blocking.py \"${INPUT_PATH}\" \\\n    --output_path \"${OUTPUT_PATH}\" --embeddings_path \"${EMBEDDINGS_PATH}\" \\\n    --k_neighbor \"${K_NEIGHBORS_COMB}\" --monitor --normalize --debug_iter 0\n\nEOF\n) > /tmp/run.bash\nchmod +x /tmp/run.bash","metadata":{"execution":{"iopub.status.busy":"2022-07-07T17:52:32.226882Z","iopub.execute_input":"2022-07-07T17:52:32.227145Z","iopub.status.idle":"2022-07-07T17:52:32.264939Z","shell.execute_reply.started":"2022-07-07T17:52:32.227116Z","shell.execute_reply":"2022-07-07T17:52:32.264086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nif \"combination\" in CFG.BLOCKERS:\n    !/tmp/run.bash","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-07-07T17:52:49.960188Z","iopub.execute_input":"2022-07-07T17:52:49.960504Z","iopub.status.idle":"2022-07-07T17:52:52.743989Z","shell.execute_reply.started":"2022-07-07T17:52:49.960467Z","shell.execute_reply":"2022-07-07T17:52:52.743044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nif CFG.DEBUG and \"combination\" in CFG.BLOCKERS:\n    stat_df = pd.read_csv(f\"{CFG.OUTPUT_PATH}/stat_combination.csv.gz\", compression=\"gzip\")\n    ax = sns.histplot(data=stat_df, x=\"weight\", log_scale=(True, False), bins=50)\n    plt.show()\n    \n    preds_df = pd.read_csv(f\"{CFG.OUTPUT_PATH}/preds_combination.csv.gz\", compression=\"gzip\")\n    print(preds_df.head(1).T)","metadata":{"execution":{"iopub.status.busy":"2022-07-07T17:52:59.844335Z","iopub.execute_input":"2022-07-07T17:52:59.844643Z","iopub.status.idle":"2022-07-07T17:53:00.660143Z","shell.execute_reply.started":"2022-07-07T17:52:59.844606Z","shell.execute_reply":"2022-07-07T17:53:00.659292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Location","metadata":{}},{"cell_type":"code","source":"%%bash\n\n(cat << 'EOF'\ncd /tmp/4sq\n\nCUDA_VISIBLE_DEVICES=0 python blocking.py \"${INPUT_PATH}\" \\\n    --blocker_type \"location\" \\\n    --output_path \"${OUTPUT_PATH}\" --embeddings_path \"${EMBEDDINGS_PATH}\" \\\n    --k_neighbor \"${K_NEIGHBORS_LOC}\" --monitor --normalize --debug_iter 0\n\nEOF\n) > /tmp/run.bash\nchmod +x /tmp/run.bash","metadata":{"execution":{"iopub.status.busy":"2022-07-07T17:53:07.867915Z","iopub.execute_input":"2022-07-07T17:53:07.868194Z","iopub.status.idle":"2022-07-07T17:53:07.904568Z","shell.execute_reply.started":"2022-07-07T17:53:07.868161Z","shell.execute_reply":"2022-07-07T17:53:07.903647Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nif \"location\" in CFG.BLOCKERS:\n    !/tmp/run.bash","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-07-07T17:53:08.930554Z","iopub.execute_input":"2022-07-07T17:53:08.931287Z","iopub.status.idle":"2022-07-07T17:53:10.918059Z","shell.execute_reply.started":"2022-07-07T17:53:08.931248Z","shell.execute_reply":"2022-07-07T17:53:10.917182Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nif CFG.DEBUG and \"location\" in CFG.BLOCKERS:\n    stat_df = pd.read_csv(f\"{CFG.OUTPUT_PATH}/stat_location.csv.gz\", compression=\"gzip\")\n    ax = sns.histplot(data=stat_df, x=\"weight\", log_scale=(True, False), bins=50)\n    plt.show()    \n    \n    preds_df = pd.read_csv(f\"{CFG.OUTPUT_PATH}/preds_location.csv.gz\", compression=\"gzip\")\n    print(preds_df.head(1).T)","metadata":{"execution":{"iopub.status.busy":"2022-07-07T17:53:21.338096Z","iopub.execute_input":"2022-07-07T17:53:21.338393Z","iopub.status.idle":"2022-07-07T17:53:21.768415Z","shell.execute_reply.started":"2022-07-07T17:53:21.338358Z","shell.execute_reply":"2022-07-07T17:53:21.767396Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Text","metadata":{}},{"cell_type":"code","source":"%%bash\n\n(cat << 'EOF'\ncd /tmp/4sq\n\nCUDA_VISIBLE_DEVICES=0 python blocking.py \"${INPUT_PATH}\" \\\n    --blocker_type \"text\" \\\n    --output_path \"${OUTPUT_PATH}\" --embeddings_path \"${EMBEDDINGS_PATH}\" \\\n    --k_neighbor \"${K_NEIGHBORS_TEXT}\" --monitor --normalize --debug_iter 0\n\nEOF\n) > /tmp/run.bash\nchmod +x /tmp/run.bash","metadata":{"execution":{"iopub.status.busy":"2022-07-07T17:53:26.831756Z","iopub.execute_input":"2022-07-07T17:53:26.832015Z","iopub.status.idle":"2022-07-07T17:53:26.868665Z","shell.execute_reply.started":"2022-07-07T17:53:26.831987Z","shell.execute_reply":"2022-07-07T17:53:26.867624Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nif \"text\" in CFG.BLOCKERS:\n    !/tmp/run.bash","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-07-07T17:53:27.549608Z","iopub.execute_input":"2022-07-07T17:53:27.550405Z","iopub.status.idle":"2022-07-07T17:53:29.877196Z","shell.execute_reply.started":"2022-07-07T17:53:27.550346Z","shell.execute_reply":"2022-07-07T17:53:29.876339Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nif CFG.DEBUG and \"text\" in CFG.BLOCKERS:\n    stat_df = pd.read_csv(f\"{CFG.OUTPUT_PATH}/stat_text.csv.gz\", compression=\"gzip\")\n    ax = sns.histplot(data=stat_df, x=\"weight\", log_scale=(True, False), bins=50)\n    plt.show()    \n    \n    preds_df = pd.read_csv(f\"{CFG.OUTPUT_PATH}/preds_text.csv.gz\", compression=\"gzip\")\n    print(preds_df.head(1).T)","metadata":{"execution":{"iopub.status.busy":"2022-07-07T17:53:32.620414Z","iopub.execute_input":"2022-07-07T17:53:32.621072Z","iopub.status.idle":"2022-07-07T17:53:33.038156Z","shell.execute_reply.started":"2022-07-07T17:53:32.621028Z","shell.execute_reply":"2022-07-07T17:53:33.037015Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Concat Preds","metadata":{}},{"cell_type":"code","source":"%set_env PREDS_PATH_COMB={CFG.OUTPUT_PATH}/preds_combination.csv.gz\n%set_env PREDS_PATH_LOC={CFG.OUTPUT_PATH}/preds_location.csv.gz\n%set_env PREDS_PATH_TEXT={CFG.OUTPUT_PATH}/preds_text.csv.gz\n%set_env OUTPUT_PATH={CFG.OUTPUT_PATH}","metadata":{"execution":{"iopub.status.busy":"2022-07-07T17:55:17.3505Z","iopub.execute_input":"2022-07-07T17:55:17.351077Z","iopub.status.idle":"2022-07-07T17:55:17.361908Z","shell.execute_reply.started":"2022-07-07T17:55:17.351038Z","shell.execute_reply":"2022-07-07T17:55:17.361191Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%bash\n\n(cat << 'EOF'\ncd /tmp/4sq\n\ngit checkout main\ngit pull --rebase\ngit log --oneline | head -1\n\nCUDA_VISIBLE_DEVICES=0 python concat_preds.py \\\n    --preds_paths \"${PREDS_PATH_COMB}\" \\\n    --preds_paths \"${PREDS_PATH_LOC}\" \\\n    --preds_paths \"${PREDS_PATH_TEXT}\" \\\n    --output_path \"${OUTPUT_PATH}\"\n\nEOF\n) > /tmp/run.bash\nchmod +x /tmp/run.bash","metadata":{"execution":{"iopub.status.busy":"2022-07-07T17:55:40.6169Z","iopub.execute_input":"2022-07-07T17:55:40.617183Z","iopub.status.idle":"2022-07-07T17:55:40.651637Z","shell.execute_reply.started":"2022-07-07T17:55:40.617151Z","shell.execute_reply":"2022-07-07T17:55:40.650716Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n!/tmp/run.bash","metadata":{"execution":{"iopub.status.busy":"2022-07-07T17:55:47.647132Z","iopub.execute_input":"2022-07-07T17:55:47.647425Z","iopub.status.idle":"2022-07-07T17:56:09.089202Z","shell.execute_reply.started":"2022-07-07T17:55:47.647393Z","shell.execute_reply":"2022-07-07T17:56:09.088323Z"},"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Make Ditto Input","metadata":{}},{"cell_type":"code","source":"%set_env INPUT_PATH=/kaggle/input/foursquare-location-matching/test.csv\n%set_env PREDS_PATH={CFG.OUTPUT_PATH}/union_preds.csv.gz\n%set_env OUTPUT_PATH={CFG.OUTPUT_PATH}","metadata":{"execution":{"iopub.status.busy":"2022-07-07T17:56:16.557666Z","iopub.execute_input":"2022-07-07T17:56:16.558261Z","iopub.status.idle":"2022-07-07T17:56:16.567314Z","shell.execute_reply.started":"2022-07-07T17:56:16.558222Z","shell.execute_reply":"2022-07-07T17:56:16.566447Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%bash\n\n(cat << 'EOF'\ncd /tmp/4sq\n\nCUDA_VISIBLE_DEVICES=0 python make_ditto_input.py \"${INPUT_PATH}\" \"${PREDS_PATH}\" \\\n     --output_path \"${OUTPUT_PATH}\"\n\nEOF\n) > /tmp/run.bash\nchmod +x /tmp/run.bash","metadata":{"execution":{"iopub.status.busy":"2022-07-07T17:56:17.797929Z","iopub.execute_input":"2022-07-07T17:56:17.798202Z","iopub.status.idle":"2022-07-07T17:56:17.832637Z","shell.execute_reply.started":"2022-07-07T17:56:17.79817Z","shell.execute_reply":"2022-07-07T17:56:17.831604Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n!/tmp/run.bash","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-07-07T17:56:18.865621Z","iopub.execute_input":"2022-07-07T17:56:18.865955Z","iopub.status.idle":"2022-07-07T17:56:20.158884Z","shell.execute_reply.started":"2022-07-07T17:56:18.865919Z","shell.execute_reply":"2022-07-07T17:56:20.15791Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nif CFG.DEBUG:\n    print(pd.read_csv(f\"{CFG.OUTPUT_PATH}/ditto.csv.gz\", compression=\"gzip\").head(1).T)","metadata":{"execution":{"iopub.status.busy":"2022-07-07T17:56:21.044994Z","iopub.execute_input":"2022-07-07T17:56:21.045292Z","iopub.status.idle":"2022-07-07T17:56:21.060284Z","shell.execute_reply.started":"2022-07-07T17:56:21.045255Z","shell.execute_reply":"2022-07-07T17:56:21.059151Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Maching","metadata":{}},{"cell_type":"code","source":"%set_env DITTO_PATH={CFG.OUTPUT_PATH}/ditto.csv.gz\n%set_env CHECKPOINT_PATH={CFG.CHECKPOINT_PATH}\n%set_env MODEL={CFG.MODEL}","metadata":{"execution":{"iopub.status.busy":"2022-07-07T17:56:23.92766Z","iopub.execute_input":"2022-07-07T17:56:23.928214Z","iopub.status.idle":"2022-07-07T17:56:23.935545Z","shell.execute_reply.started":"2022-07-07T17:56:23.928175Z","shell.execute_reply":"2022-07-07T17:56:23.934577Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%bash\n\n(cat << 'EOF'\ncd /tmp/4sq\n\nCUDA_VISIBLE_DEVICES=0 python matcher.py \"${DITTO_PATH}\" \"${CHECKPOINT_PATH}\" \\\n    --output_path=\"${OUTPUT_PATH}\" --fp16 --lm \"${MODEL}\" --use_gpu\n\nEOF\n) > /tmp/run.bash\nchmod +x /tmp/run.bash","metadata":{"execution":{"iopub.status.busy":"2022-07-07T17:56:24.373557Z","iopub.execute_input":"2022-07-07T17:56:24.374276Z","iopub.status.idle":"2022-07-07T17:56:24.408937Z","shell.execute_reply.started":"2022-07-07T17:56:24.374237Z","shell.execute_reply":"2022-07-07T17:56:24.40781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n!/tmp/run.bash","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-07-07T17:56:24.967323Z","iopub.execute_input":"2022-07-07T17:56:24.967626Z","iopub.status.idle":"2022-07-07T17:56:36.429543Z","shell.execute_reply.started":"2022-07-07T17:56:24.967589Z","shell.execute_reply":"2022-07-07T17:56:36.428675Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Make Submission","metadata":{}},{"cell_type":"code","source":"%set_env MATCHED_PATH={CFG.OUTPUT_PATH}/matched.csv.gz\n%set_env OUTPUT_PATH=/kaggle/working\n%set_env THRESHOLD={CFG.THRESHOLD}","metadata":{"execution":{"iopub.status.busy":"2022-07-07T17:56:36.431978Z","iopub.execute_input":"2022-07-07T17:56:36.432376Z","iopub.status.idle":"2022-07-07T17:56:36.442109Z","shell.execute_reply.started":"2022-07-07T17:56:36.432331Z","shell.execute_reply":"2022-07-07T17:56:36.441211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%bash\n\n(cat << 'EOF'\ncd /tmp/4sq\n\nCUDA_VISIBLE_DEVICES=0 python make_submission.py \"${MATCHED_PATH}\" \\\n    --output_path=\"${OUTPUT_PATH}\" --threshold \"${THRESHOLD}\"\n\nEOF\n) > /tmp/run.bash\nchmod +x /tmp/run.bash","metadata":{"execution":{"iopub.status.busy":"2022-07-07T17:56:36.443321Z","iopub.execute_input":"2022-07-07T17:56:36.443779Z","iopub.status.idle":"2022-07-07T17:56:36.481679Z","shell.execute_reply.started":"2022-07-07T17:56:36.443743Z","shell.execute_reply":"2022-07-07T17:56:36.480569Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n!/tmp/run.bash","metadata":{"execution":{"iopub.status.busy":"2022-07-07T17:56:36.484332Z","iopub.execute_input":"2022-07-07T17:56:36.484984Z","iopub.status.idle":"2022-07-07T17:56:37.886084Z","shell.execute_reply.started":"2022-07-07T17:56:36.484941Z","shell.execute_reply":"2022-07-07T17:56:37.885222Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nif CFG.DEBUG:\n    print(pd.read_csv(f\"/kaggle/working/submission.csv\"))","metadata":{"execution":{"iopub.status.busy":"2022-07-07T17:56:37.887895Z","iopub.execute_input":"2022-07-07T17:56:37.888409Z","iopub.status.idle":"2022-07-07T17:56:37.902424Z","shell.execute_reply.started":"2022-07-07T17:56:37.888365Z","shell.execute_reply":"2022-07-07T17:56:37.901585Z"},"trusted":true},"execution_count":null,"outputs":[]}]}