{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"! rm -rf /tmp/4sq\n! cp -r /kaggle/input/4sq-clone-source-code-repository/4sq /tmp/4sq\n! cd /tmp/4sq; git log --oneline | head -1","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-07-07T16:32:26.849762Z","iopub.execute_input":"2022-07-07T16:32:26.850362Z","iopub.status.idle":"2022-07-07T16:32:29.531878Z","shell.execute_reply.started":"2022-07-07T16:32:26.85026Z","shell.execute_reply":"2022-07-07T16:32:29.530877Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Run All Modules","metadata":{}},{"cell_type":"code","source":"class CFG:\n    DEBUG = False\n    K_NEIGHBORS = 15\n    THRESHOLD = 0.5\n    BLOCKERS = {\"combination\"}\n    OUTPUT_PATH = \"/tmp/4sq/output\"\n    CHECKPOINT_PATH = \"/kaggle/input/4sq-trained-weights/model_ditto_all_feat_all_n25_seed1234_full_all-MiniLM-L6-v2_ep10_id0.pt\"\n    MODEL = \"/kaggle/input/4sq-pretrained-transformer-models/all-MiniLM-L6-v2\"","metadata":{"execution":{"iopub.status.busy":"2022-07-07T16:42:13.717182Z","iopub.execute_input":"2022-07-07T16:42:13.717677Z","iopub.status.idle":"2022-07-07T16:42:13.722799Z","shell.execute_reply.started":"2022-07-07T16:42:13.717632Z","shell.execute_reply":"2022-07-07T16:42:13.722018Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\nplt.style.use(\"ggplot\")","metadata":{"execution":{"iopub.status.busy":"2022-07-07T16:32:45.999594Z","iopub.execute_input":"2022-07-07T16:32:46.000133Z","iopub.status.idle":"2022-07-07T16:32:46.666675Z","shell.execute_reply.started":"2022-07-07T16:32:46.000102Z","shell.execute_reply":"2022-07-07T16:32:46.665896Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%bash\ncat << 'EOF' > /tmp/run.bash\nPIP_DEP_PATH='/kaggle/input/4sq-build-runtime-environment/pip_deps'\necho ${PIP_DEP_PATH}\npip install ${PIP_DEP_PATH}/* -f ./ --no-index --no-deps --find-links=\"${PIP_DEP_PATH}\"\n\nEOF\nchmod +x /tmp/run.bash","metadata":{"execution":{"iopub.status.busy":"2022-07-07T16:32:46.671273Z","iopub.execute_input":"2022-07-07T16:32:46.673561Z","iopub.status.idle":"2022-07-07T16:32:46.705276Z","shell.execute_reply.started":"2022-07-07T16:32:46.673505Z","shell.execute_reply":"2022-07-07T16:32:46.704027Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!/tmp/run.bash","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-07-07T16:32:47.121562Z","iopub.execute_input":"2022-07-07T16:32:47.121915Z","iopub.status.idle":"2022-07-07T16:33:05.390873Z","shell.execute_reply.started":"2022-07-07T16:32:47.121875Z","shell.execute_reply":"2022-07-07T16:33:05.390023Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Make H3","metadata":{}},{"cell_type":"code","source":"%set_env INPUT_PATH=/kaggle/input/foursquare-location-matching/test.csv\n%set_env OUTPUT_PATH={CFG.OUTPUT_PATH}","metadata":{"execution":{"iopub.status.busy":"2022-07-07T16:33:05.39276Z","iopub.execute_input":"2022-07-07T16:33:05.392991Z","iopub.status.idle":"2022-07-07T16:33:05.401294Z","shell.execute_reply.started":"2022-07-07T16:33:05.392963Z","shell.execute_reply":"2022-07-07T16:33:05.399799Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!echo $OUTPUT_PATH","metadata":{"execution":{"iopub.status.busy":"2022-07-07T16:33:05.402351Z","iopub.execute_input":"2022-07-07T16:33:05.403657Z","iopub.status.idle":"2022-07-07T16:33:06.207288Z","shell.execute_reply.started":"2022-07-07T16:33:05.403607Z","shell.execute_reply":"2022-07-07T16:33:06.206519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%bash\n\n(cat << 'EOF'\ncd /tmp/4sq\n\nCUDA_VISIBLE_DEVICES=0 python make_h3.py --output_path \"${OUTPUT_PATH}\" \"${INPUT_PATH}\"\n\nEOF\n) > /tmp/run.bash\nchmod +x /tmp/run.bash","metadata":{"execution":{"iopub.status.busy":"2022-07-07T16:33:06.210476Z","iopub.execute_input":"2022-07-07T16:33:06.210733Z","iopub.status.idle":"2022-07-07T16:33:06.231249Z","shell.execute_reply.started":"2022-07-07T16:33:06.210705Z","shell.execute_reply":"2022-07-07T16:33:06.230201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n!/tmp/run.bash","metadata":{"execution":{"iopub.status.busy":"2022-07-07T16:33:06.233554Z","iopub.execute_input":"2022-07-07T16:33:06.233772Z","iopub.status.idle":"2022-07-07T16:33:07.572085Z","shell.execute_reply.started":"2022-07-07T16:33:06.233747Z","shell.execute_reply":"2022-07-07T16:33:07.571186Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if CFG.DEBUG:\n    print(pd.read_csv(f\"{CFG.OUTPUT_PATH}/h3_res1.csv.gz\", compression=\"gzip\"))","metadata":{"execution":{"iopub.status.busy":"2022-07-07T16:33:07.573772Z","iopub.execute_input":"2022-07-07T16:33:07.573986Z","iopub.status.idle":"2022-07-07T16:33:07.589948Z","shell.execute_reply.started":"2022-07-07T16:33:07.573959Z","shell.execute_reply":"2022-07-07T16:33:07.589181Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Preprocess","metadata":{}},{"cell_type":"code","source":"%set_env INPUT_PATH=/kaggle/input/foursquare-location-matching/test.csv\n%set_env H3_PATH={CFG.OUTPUT_PATH}/h3_res1.csv.gz\n%set_env OUTPUT_PATH={CFG.OUTPUT_PATH}","metadata":{"execution":{"iopub.status.busy":"2022-07-07T16:33:07.591579Z","iopub.execute_input":"2022-07-07T16:33:07.591945Z","iopub.status.idle":"2022-07-07T16:33:07.60244Z","shell.execute_reply.started":"2022-07-07T16:33:07.591908Z","shell.execute_reply":"2022-07-07T16:33:07.601544Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%bash\n\n(cat << 'EOF'\ncd /tmp/4sq\n\nCUDA_VISIBLE_DEVICES=0 python preprocess.py --output_path \"${OUTPUT_PATH}\" \"${INPUT_PATH}\" \"${H3_PATH}\"\n\nEOF\n) > /tmp/run.bash\nchmod +x /tmp/run.bash","metadata":{"execution":{"iopub.status.busy":"2022-07-07T16:33:07.604244Z","iopub.execute_input":"2022-07-07T16:33:07.604675Z","iopub.status.idle":"2022-07-07T16:33:07.626105Z","shell.execute_reply.started":"2022-07-07T16:33:07.604636Z","shell.execute_reply":"2022-07-07T16:33:07.625007Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n!/tmp/run.bash","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-07-07T16:33:07.628134Z","iopub.execute_input":"2022-07-07T16:33:07.628435Z","iopub.status.idle":"2022-07-07T16:33:08.823262Z","shell.execute_reply.started":"2022-07-07T16:33:07.628396Z","shell.execute_reply":"2022-07-07T16:33:08.821696Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if CFG.DEBUG:\n    preprocessed_df = pd.read_csv(f\"{CFG.OUTPUT_PATH}/preprocessed.csv.gz\", compression=\"gzip\")\n    print(preprocessed_df.head(1).T)","metadata":{"execution":{"iopub.status.busy":"2022-07-07T16:33:08.826741Z","iopub.execute_input":"2022-07-07T16:33:08.82697Z","iopub.status.idle":"2022-07-07T16:33:08.840962Z","shell.execute_reply.started":"2022-07-07T16:33:08.826942Z","shell.execute_reply":"2022-07-07T16:33:08.8402Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Calc Embeddings","metadata":{}},{"cell_type":"markdown","source":"## Calc Embeddings","metadata":{}},{"cell_type":"code","source":"%set_env INPUT_PATH={CFG.OUTPUT_PATH}/preprocessed.csv.gz\n%set_env OUTPUT_PATH={CFG.OUTPUT_PATH}\n%set_env MODEL={CFG.MODEL}","metadata":{"execution":{"iopub.status.busy":"2022-07-07T16:33:08.842227Z","iopub.execute_input":"2022-07-07T16:33:08.842706Z","iopub.status.idle":"2022-07-07T16:33:08.850936Z","shell.execute_reply.started":"2022-07-07T16:33:08.842669Z","shell.execute_reply":"2022-07-07T16:33:08.849948Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%bash\n\n(cat << 'EOF'\ncd /tmp/4sq\n\nCUDA_VISIBLE_DEVICES=0 python calc_embeddings.py --column \"name\" --output_path \"${OUTPUT_PATH}\" \"${INPUT_PATH}\" \"${MODEL}\"\n\nEOF\n) > /tmp/run.bash\nchmod +x /tmp/run.bash","metadata":{"execution":{"iopub.status.busy":"2022-07-07T16:33:08.852479Z","iopub.execute_input":"2022-07-07T16:33:08.853055Z","iopub.status.idle":"2022-07-07T16:33:08.874676Z","shell.execute_reply.started":"2022-07-07T16:33:08.853017Z","shell.execute_reply":"2022-07-07T16:33:08.87386Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n!/tmp/run.bash","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-07-07T16:33:08.876387Z","iopub.execute_input":"2022-07-07T16:33:08.876902Z","iopub.status.idle":"2022-07-07T16:33:26.373109Z","shell.execute_reply.started":"2022-07-07T16:33:08.876863Z","shell.execute_reply":"2022-07-07T16:33:26.372223Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%bash\n\n(cat << 'EOF'\ncd /tmp/4sq\n\nCUDA_VISIBLE_DEVICES=0 python calc_embeddings.py --column \"categories\" --output_path \"${OUTPUT_PATH}\" \"${INPUT_PATH}\" \"${MODEL}\"\n\nEOF\n) > /tmp/run.bash\nchmod +x /tmp/run.bash","metadata":{"execution":{"iopub.status.busy":"2022-07-07T16:33:26.374979Z","iopub.execute_input":"2022-07-07T16:33:26.375258Z","iopub.status.idle":"2022-07-07T16:33:26.396843Z","shell.execute_reply.started":"2022-07-07T16:33:26.375219Z","shell.execute_reply":"2022-07-07T16:33:26.395985Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n!/tmp/run.bash","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-07-07T16:33:26.398692Z","iopub.execute_input":"2022-07-07T16:33:26.398958Z","iopub.status.idle":"2022-07-07T16:33:33.72329Z","shell.execute_reply.started":"2022-07-07T16:33:26.398922Z","shell.execute_reply":"2022-07-07T16:33:33.722409Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if CFG.DEBUG:\n    from sentence_transformers import SentenceTransformer\n    model = SentenceTransformer(CFG.MODEL)\n    n_dims = model.get_sentence_embedding_dimension()\n    n_items = len(preprocessed_df)\n    print(n_items, n_dims)\n    print(np.memmap(f\"{CFG.OUTPUT_PATH}/embeddings_name.mmp\", mode=\"r\", dtype=\"float32\", shape=(n_items, n_dims)))\n    print(np.memmap(f\"{CFG.OUTPUT_PATH}/embeddings_categories.mmp\", mode=\"r\", dtype=\"float32\", shape=(n_items, n_dims)))","metadata":{"execution":{"iopub.status.busy":"2022-07-07T16:33:33.725552Z","iopub.execute_input":"2022-07-07T16:33:33.726087Z","iopub.status.idle":"2022-07-07T16:33:36.249719Z","shell.execute_reply.started":"2022-07-07T16:33:33.726044Z","shell.execute_reply":"2022-07-07T16:33:36.248911Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Blocking","metadata":{}},{"cell_type":"code","source":"%set_env INPUT_PATH={CFG.OUTPUT_PATH}/preprocessed.csv.gz\n%set_env OUTPUT_PATH={CFG.OUTPUT_PATH}\n%set_env EMBEDDINGS_PATH={CFG.OUTPUT_PATH}\n%set_env K_NEIGHBORS={CFG.K_NEIGHBORS}","metadata":{"execution":{"iopub.status.busy":"2022-07-07T16:34:47.521956Z","iopub.execute_input":"2022-07-07T16:34:47.522232Z","iopub.status.idle":"2022-07-07T16:34:47.534647Z","shell.execute_reply.started":"2022-07-07T16:34:47.522202Z","shell.execute_reply":"2022-07-07T16:34:47.533892Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### combination","metadata":{}},{"cell_type":"code","source":"%%bash\n\n(cat << 'EOF'\ncd /tmp/4sq\n\nCUDA_VISIBLE_DEVICES=0 python blocking.py \"${INPUT_PATH}\" \\\n    --output_path \"${OUTPUT_PATH}\" --embeddings_path \"${EMBEDDINGS_PATH}\" \\\n    --k_neighbor \"${K_NEIGHBORS}\" --monitor --normalize --debug_iter 0\n\nEOF\n) > /tmp/run.bash\nchmod +x /tmp/run.bash","metadata":{"execution":{"iopub.status.busy":"2022-07-07T16:34:48.731835Z","iopub.execute_input":"2022-07-07T16:34:48.7321Z","iopub.status.idle":"2022-07-07T16:34:48.766327Z","shell.execute_reply.started":"2022-07-07T16:34:48.732071Z","shell.execute_reply":"2022-07-07T16:34:48.765458Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nif \"combination\" in CFG.BLOCKERS:\n    !/tmp/run.bash","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-07-07T16:34:48.915825Z","iopub.execute_input":"2022-07-07T16:34:48.916414Z","iopub.status.idle":"2022-07-07T16:34:51.245124Z","shell.execute_reply.started":"2022-07-07T16:34:48.916368Z","shell.execute_reply":"2022-07-07T16:34:51.244184Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nif CFG.DEBUG and \"combination\" in CFG.BLOCKERS:\n    stat_df = pd.read_csv(f\"{CFG.OUTPUT_PATH}/stat_combination.csv.gz\", compression=\"gzip\")\n    ax = sns.histplot(data=stat_df, x=\"weight\", log_scale=(True, False), bins=50)\n    plt.show()\n    \n    preds_df = pd.read_csv(f\"{CFG.OUTPUT_PATH}/preds_combination.csv.gz\", compression=\"gzip\")\n    print(preds_df.head(1).T)","metadata":{"execution":{"iopub.status.busy":"2022-07-07T16:36:55.751589Z","iopub.execute_input":"2022-07-07T16:36:55.752097Z","iopub.status.idle":"2022-07-07T16:36:56.141035Z","shell.execute_reply.started":"2022-07-07T16:36:55.752059Z","shell.execute_reply":"2022-07-07T16:36:56.140364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Location","metadata":{}},{"cell_type":"code","source":"%%bash\n\n(cat << 'EOF'\ncd /tmp/4sq\n\nCUDA_VISIBLE_DEVICES=0 python blocking.py \"${INPUT_PATH}\" \\\n    --blocker_type \"location\" \\\n    --output_path \"${OUTPUT_PATH}\" --embeddings_path \"${EMBEDDINGS_PATH}\" \\\n    --k_neighbor \"${K_NEIGHBORS}\" --monitor --normalize --debug_iter 0\n\nEOF\n) > /tmp/run.bash\nchmod +x /tmp/run.bash","metadata":{"execution":{"iopub.status.busy":"2022-07-07T15:08:22.544312Z","iopub.execute_input":"2022-07-07T15:08:22.544573Z","iopub.status.idle":"2022-07-07T15:08:22.583678Z","shell.execute_reply.started":"2022-07-07T15:08:22.544544Z","shell.execute_reply":"2022-07-07T15:08:22.582414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nif \"location\" in CFG.BLOCKERS:\n    !/tmp/run.bash","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-07-07T16:37:14.718318Z","iopub.execute_input":"2022-07-07T16:37:14.718592Z","iopub.status.idle":"2022-07-07T16:37:14.726831Z","shell.execute_reply.started":"2022-07-07T16:37:14.718561Z","shell.execute_reply":"2022-07-07T16:37:14.726032Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nif CFG.DEBUG and \"location\" in CFG.BLOCKERS:\n    stat_df = pd.read_csv(f\"{CFG.OUTPUT_PATH}/stat_location.csv.gz\", compression=\"gzip\")\n    ax = sns.histplot(data=stat_df, x=\"weight\", log_scale=(True, False), bins=50)\n    plt.show()    \n    \n    preds_df = pd.read_csv(f\"{CFG.OUTPUT_PATH}/preds_location.csv.gz\", compression=\"gzip\")\n    print(preds_df.head(1).T)","metadata":{"execution":{"iopub.status.busy":"2022-07-07T15:09:35.640985Z","iopub.execute_input":"2022-07-07T15:09:35.641346Z","iopub.status.idle":"2022-07-07T15:09:36.273916Z","shell.execute_reply.started":"2022-07-07T15:09:35.641268Z","shell.execute_reply":"2022-07-07T15:09:36.273229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Text","metadata":{}},{"cell_type":"code","source":"%%bash\n\n(cat << 'EOF'\ncd /tmp/4sq\n\nCUDA_VISIBLE_DEVICES=0 python blocking.py \"${INPUT_PATH}\" \\\n    --blocker_type \"text\" \\\n    --output_path \"${OUTPUT_PATH}\" --embeddings_path \"${EMBEDDINGS_PATH}\" \\\n    --k_neighbor \"${K_NEIGHBORS}\" --monitor --normalize --debug_iter 0\n\nEOF\n) > /tmp/run.bash\nchmod +x /tmp/run.bash","metadata":{"execution":{"iopub.status.busy":"2022-07-07T15:09:24.759605Z","iopub.execute_input":"2022-07-07T15:09:24.759864Z","iopub.status.idle":"2022-07-07T15:09:24.798934Z","shell.execute_reply.started":"2022-07-07T15:09:24.759837Z","shell.execute_reply":"2022-07-07T15:09:24.798046Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nif \"text\" in CFG.BLOCKERS:\n    !/tmp/run.bash","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-07-07T15:09:25.316275Z","iopub.execute_input":"2022-07-07T15:09:25.317025Z","iopub.status.idle":"2022-07-07T15:09:27.844859Z","shell.execute_reply.started":"2022-07-07T15:09:25.316988Z","shell.execute_reply":"2022-07-07T15:09:27.843999Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nif CFG.DEBUG and \"text\" in CFG.BLOCKERS:\n    stat_df = pd.read_csv(f\"{CFG.OUTPUT_PATH}/stat_text.csv.gz\", compression=\"gzip\")\n    ax = sns.histplot(data=stat_df, x=\"weight\", log_scale=(True, False), bins=50)\n    plt.show()    \n    \n    preds_df = pd.read_csv(f\"{CFG.OUTPUT_PATH}/preds_text.csv.gz\", compression=\"gzip\")\n    print(preds_df.head(1).T)","metadata":{"execution":{"iopub.status.busy":"2022-07-07T16:38:31.393999Z","iopub.execute_input":"2022-07-07T16:38:31.394725Z","iopub.status.idle":"2022-07-07T16:38:31.401493Z","shell.execute_reply.started":"2022-07-07T16:38:31.394686Z","shell.execute_reply":"2022-07-07T16:38:31.400556Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Make Ditto Input","metadata":{}},{"cell_type":"code","source":"%set_env INPUT_PATH=/kaggle/input/foursquare-location-matching/test.csv\n%set_env PREDS_PATH={CFG.OUTPUT_PATH}/preds_combination.csv.gz\n%set_env OUTPUT_PATH={CFG.OUTPUT_PATH}","metadata":{"execution":{"iopub.status.busy":"2022-07-07T16:39:08.236423Z","iopub.execute_input":"2022-07-07T16:39:08.237023Z","iopub.status.idle":"2022-07-07T16:39:08.244358Z","shell.execute_reply.started":"2022-07-07T16:39:08.236984Z","shell.execute_reply":"2022-07-07T16:39:08.243542Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%bash\n\n(cat << 'EOF'\ncd /tmp/4sq\n\nCUDA_VISIBLE_DEVICES=0 python make_ditto_input.py \"${INPUT_PATH}\" \"${PREDS_PATH}\" \\\n     --output_path \"${OUTPUT_PATH}\"\n\nEOF\n) > /tmp/run.bash\nchmod +x /tmp/run.bash","metadata":{"execution":{"iopub.status.busy":"2022-07-07T16:39:08.788077Z","iopub.execute_input":"2022-07-07T16:39:08.788933Z","iopub.status.idle":"2022-07-07T16:39:08.834422Z","shell.execute_reply.started":"2022-07-07T16:39:08.788883Z","shell.execute_reply":"2022-07-07T16:39:08.833364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n!/tmp/run.bash","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-07-07T16:39:09.636305Z","iopub.execute_input":"2022-07-07T16:39:09.636621Z","iopub.status.idle":"2022-07-07T16:39:10.953257Z","shell.execute_reply.started":"2022-07-07T16:39:09.636582Z","shell.execute_reply":"2022-07-07T16:39:10.952349Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nif CFG.DEBUG:\n    print(pd.read_csv(f\"{CFG.OUTPUT_PATH}/ditto.csv.gz\", compression=\"gzip\").head(1).T)","metadata":{"execution":{"iopub.status.busy":"2022-07-07T16:39:10.955253Z","iopub.execute_input":"2022-07-07T16:39:10.955857Z","iopub.status.idle":"2022-07-07T16:39:10.970926Z","shell.execute_reply.started":"2022-07-07T16:39:10.955812Z","shell.execute_reply":"2022-07-07T16:39:10.969628Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Maching","metadata":{}},{"cell_type":"code","source":"%set_env DITTO_PATH={CFG.OUTPUT_PATH}/ditto.csv.gz\n%set_env CHECKPOINT_PATH={CFG.CHECKPOINT_PATH}\n%set_env MODEL={CFG.MODEL}","metadata":{"execution":{"iopub.status.busy":"2022-07-07T16:39:11.694245Z","iopub.execute_input":"2022-07-07T16:39:11.694529Z","iopub.status.idle":"2022-07-07T16:39:11.702707Z","shell.execute_reply.started":"2022-07-07T16:39:11.694482Z","shell.execute_reply":"2022-07-07T16:39:11.701831Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%bash\n\n(cat << 'EOF'\ncd /tmp/4sq\n\nCUDA_VISIBLE_DEVICES=0 python matcher.py \"${DITTO_PATH}\" \"${CHECKPOINT_PATH}\" \\\n    --output_path=\"${OUTPUT_PATH}\" --fp16 --lm \"${MODEL}\" --use_gpu\n\nEOF\n) > /tmp/run.bash\nchmod +x /tmp/run.bash","metadata":{"execution":{"iopub.status.busy":"2022-07-07T16:39:12.506079Z","iopub.execute_input":"2022-07-07T16:39:12.506637Z","iopub.status.idle":"2022-07-07T16:39:12.541491Z","shell.execute_reply.started":"2022-07-07T16:39:12.506595Z","shell.execute_reply":"2022-07-07T16:39:12.540445Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n!/tmp/run.bash","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-07-07T16:39:13.149669Z","iopub.execute_input":"2022-07-07T16:39:13.150469Z","iopub.status.idle":"2022-07-07T16:39:24.49953Z","shell.execute_reply.started":"2022-07-07T16:39:13.150431Z","shell.execute_reply":"2022-07-07T16:39:24.498655Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Make Submission","metadata":{}},{"cell_type":"code","source":"%set_env MATCHED_PATH={CFG.OUTPUT_PATH}/matched.csv.gz\n%set_env OUTPUT_PATH=/kaggle/working\n%set_env THRESHOLD={CFG.THRESHOLD}","metadata":{"execution":{"iopub.status.busy":"2022-07-07T16:42:23.176853Z","iopub.execute_input":"2022-07-07T16:42:23.17712Z","iopub.status.idle":"2022-07-07T16:42:23.184766Z","shell.execute_reply.started":"2022-07-07T16:42:23.177091Z","shell.execute_reply":"2022-07-07T16:42:23.184011Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%bash\n\n(cat << 'EOF'\ncd /tmp/4sq\n\nCUDA_VISIBLE_DEVICES=0 python make_submission.py \"${MATCHED_PATH}\" \\\n    --output_path=\"${OUTPUT_PATH}\" --threshold \"${THRESHOLD}\"\n\nEOF\n) > /tmp/run.bash\nchmod +x /tmp/run.bash","metadata":{"execution":{"iopub.status.busy":"2022-07-07T16:42:24.009846Z","iopub.execute_input":"2022-07-07T16:42:24.010804Z","iopub.status.idle":"2022-07-07T16:42:24.046427Z","shell.execute_reply.started":"2022-07-07T16:42:24.01075Z","shell.execute_reply":"2022-07-07T16:42:24.045312Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n!/tmp/run.bash","metadata":{"execution":{"iopub.status.busy":"2022-07-07T16:42:24.495321Z","iopub.execute_input":"2022-07-07T16:42:24.496161Z","iopub.status.idle":"2022-07-07T16:42:45.731773Z","shell.execute_reply.started":"2022-07-07T16:42:24.496119Z","shell.execute_reply":"2022-07-07T16:42:45.730824Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nif CFG.DEBUG:\n    print(pd.read_csv(f\"/kaggle/working/submission.csv\"))","metadata":{"execution":{"iopub.status.busy":"2022-07-07T16:43:05.95917Z","iopub.execute_input":"2022-07-07T16:43:05.959892Z","iopub.status.idle":"2022-07-07T16:43:05.971463Z","shell.execute_reply.started":"2022-07-07T16:43:05.959854Z","shell.execute_reply":"2022-07-07T16:43:05.970494Z"},"trusted":true},"execution_count":null,"outputs":[]}]}