{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Example of a simple way to merge `submission.csv`.\nIf you know of a better way, please let me know!","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport os\nimport gc\n\nDATA_DIR = \"../input/open-problems-multimodal\"\nSUMPLE_SUBMISSION = os.path.join(DATA_DIR,\"sample_submission.csv\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# \"csv file location for marging\": \"technology \"\n\ncsvs = {\"../input/open-problems-multimodal/sample_submission.csv\": \"multi\",\n        \"../input/open-problems-multimodal/sample_submission.csv\": \"multi\",\n        \"../input/open-problems-multimodal/sample_submission.csv\": \"cite\",\n        \"../input/open-problems-multimodal/sample_submission.csv\": \"cite\",}","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cite_len = 48663*140 #6,812,820\ncite_count = 0\ncite = np.zeros((48663, 140), dtype=np.float32)\n\nmulti_len = 65744180 - 48663*140 #58,931,360\nmulti_count = 0\nmulti = np.zeros((16780, 3512), dtype=np.float32)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#standerized values row-wise(by cell_id), and get sum of multiple csv\nfor i in csvs:\n    df = pd.read_csv(i, index_col='row_id', squeeze=True)\n    if csvs[i] == \"cite\":\n        cite_pre = df[:cite_len].values.reshape([48663, 140])\n        cite_pre -= cite_pre.mean(axis=1).reshape(-1, 1)\n        cite_pre /= cite_pre.std(axis=1).reshape(-1, 1)\n        cite += cite_pre\n        cite_count += 1\n        del cite_pre\n\n    elif csvs[i] == \"multi\":\n        multi_pre = df[cite_len:].values.reshape([16780, 3512])\n        multi_pre -= multi_pre.mean(axis=1).reshape(-1, 1)\n        multi_pre /= multi_pre.std(axis=1).reshape(-1, 1)\n        multi += multi_pre\n        multi_count += 1\n        del multi_pre\n\n    else:\n        print(\"err!\")\n\n    gc.collect()\n\n# This process does not affect the score\ncite = cite/cite_count\nmulti = multi/multi_count ","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.read_csv(SUMPLE_SUBMISSION, index_col='row_id', squeeze=True)\n\nsubmission.iloc[:cite_len] = cite.ravel()\nsubmission.iloc[cite_len:] = multi.ravel()\nsubmission.to_csv(\"merged_submission.csv\")","metadata":{},"execution_count":null,"outputs":[]}]}