{"cells":[{"metadata":{"_cell_guid":"1a420afb-b558-43cb-850d-36790d90560f","_uuid":"f5c3aa76c6919843ee6020419530098e59dfba02"},"cell_type":"markdown","source":"This kernel creates mapping between click_id  from current test.csv and click_id from test.csv uploaded initially (and re-uploaded officially as test_supplement.csv), which has complete data for time range 2017-Nov-10 12:00 - 2017-Nov-10 23:00 (Chinese local time).\n\nThe mapping can be useful to predict on test_supplement.csv data directly.\n\nOld test data taken from [here](https://www.kaggle.com/tkm2261/old-test-data-on-talkingdata-adtracking), thanks tkm2261."},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","collapsed":true,"trusted":true},"cell_type":"code","source":"\nold_file_path = '../input/old-test-data-on-talkingdata-adtracking/test.csv'\nfile_path = '../input/talkingdata-adtracking-fraud-detection/test.csv'\noutput_file_path = 'mapping.csv'\n\n\ndef _split(line):\n    line = line.strip()\n    index = line.index(',')\n    last_index = line.rindex(',')\n    click_id = line[:index]\n    payload = line[index:]\n    time = line[last_index:]\n    return click_id, payload, time\n\n\ndef _read_same_time(lines, unprocessed_line):\n    click_id, payload, group_time = _split(unprocessed_line)\n    click_id_dict = {payload: [click_id]}\n    while True:\n        unprocessed_line = lines.readline()\n        if not unprocessed_line:\n            return unprocessed_line, click_id_dict, group_time\n        click_id, payload, click_time = _split(unprocessed_line)\n        if group_time == click_time:\n            if payload in click_id_dict:\n                click_id_dict[payload].append(click_id)\n            else:\n                click_id_dict[payload] = [click_id]\n        else:\n            return unprocessed_line, click_id_dict, group_time\n\n\ndef _find_time(lines, group_time, unprocessed_line):\n    if unprocessed_line:\n        click_id, payload, time = _split(unprocessed_line)\n        if group_time == time:\n            return unprocessed_line\n    while True:\n        unprocessed_line = lines.readline()\n        click_id, payload, time = _split(unprocessed_line)\n        if group_time == time:\n            return unprocessed_line\n\n\ndef _save(output, test_click_id_dict, old_test_click_id_dict):\n    for payload, click_ids in test_click_id_dict.items():\n        old_click_ids = old_test_click_id_dict[payload]\n        if len(old_click_ids) != len(click_ids):\n            print('Number of ids mismatch for \"{}\", test ids = {}, old test ids = {}'.format(payload, click_ids,\n                                                                                             old_click_ids))\n        for i in range(len(click_ids)):\n            output.write('{},{}\\n'.format(click_ids[i], old_click_ids[i]))\n\n\nwith open(file_path, \"r\", encoding=\"utf-8\") as test:\n    with open(old_file_path, \"r\", encoding=\"utf-8\") as old_test:\n        with open(output_file_path, \"w\", encoding=\"utf-8\") as output:\n            output.write('click_id,old_click_id\\n')\n            test.readline()  # skip header\n            old_test.readline()  # skip header\n            old_test_unprocessed_line = old_test.readline()\n            test_unprocessed_line = test.readline()\n            while test_unprocessed_line != '':\n                test_unprocessed_line, test_click_id_dict, click_time = _read_same_time(test, test_unprocessed_line)\n                old_test_unprocessed_line = _find_time(old_test, click_time, old_test_unprocessed_line)\n                old_test_unprocessed_line, old_test_click_id_dict, _ = _read_same_time(old_test,\n                                                                                       old_test_unprocessed_line)\n                _save(output, test_click_id_dict, old_test_click_id_dict)\n        pass\n","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"import pandas as pd\nmapping = pd.read_csv('../input/mapping-between-test-supplement-csv-and-test-csv/mapping.csv', dtype={'click_id': 'int32','old_click_id': 'int32'}, engine='c',\n                na_filter=False,memory_map=True)\n","execution_count":1,"outputs":[]},{"metadata":{"_cell_guid":"17b8e430-d8dc-435e-8438-1320cb76e7d8","_uuid":"651693ad26db62dc2f792cf8325d391b5ee32a79","collapsed":true,"trusted":true},"cell_type":"code","source":"print('click id min {}'.format(mapping.click_id.min()))\nprint('click id max {}'.format(mapping.click_id.max()))\nprint('click id count {}'.format(mapping.click_id.count()))\nprint('click id unique count {}'.format(mapping.click_id.unique().shape[0]))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"fbe8e7a1-da4c-41ba-900c-9841919aa61b","_uuid":"4fff1849c3e5c502bc37d0d4d450a6330920f4fe","collapsed":true,"trusted":true},"cell_type":"code","source":"print('old click id min {}'.format(mapping.old_click_id.min()))\nprint('old click id max {}'.format(mapping.old_click_id.max()))\nprint('old click id count {}'.format(mapping.old_click_id.count()))\nprint('old click id unique count {}'.format(mapping.old_click_id.unique().shape[0]))","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}