{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Read Train/Test data - HELP ME!\nI am trying to load train/test data but it seems incorrect. Please help me where is my wrong point.\n\nIn my understand, train/test data should be splited/merged by \"phoneName\" and \"millisSinceGpsEpoch\". To confirm it, I tryed to execute the code below but it seems incorrect.\n\nbaseline_locations_train.csv has 131342 lines but the data I created has 130339. **PLEASE HELP ME!**\n\nI know it should talk in Discussion but it can't attach any files, images. sry...","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport pathlib\nfrom tqdm.notebook import tqdm\n\n# check train data \ntrainfile = pd.read_csv(\"../input/google-smartphone-decimeter-challenge/baseline_locations_train.csv\")\nprint(\"true data num\",len(trainfile)) # num. of train data -> 131342\n\n#############################################################\n# read train data\nINPUT = '../input/google-smartphone-decimeter-challenge'\np = pathlib.Path(INPUT)\ntrain_files = list(p.glob('train/*/*/*_derived.csv'))\ntrain_data = []\nprint(\"data loading\")\nfor filename in tqdm(train_files):\n    train_data.append(pd.read_csv(filename))\n    \ntrain_data = pd.concat(train_data)\n\nprint(len(train_data)) # num. of raw train data -> 3834542\n\n# split by phone name\nphone_names = train_data[\"phoneName\"].value_counts().index\nphone_separated_data = []\nfor phone_name in phone_names:\n    tmp = train_data[train_data[\"phoneName\"]==phone_name]\n    phone_separated_data.append(tmp)\n\n# merge by millisSinceGpsEpoch(actually, I only counted unique values of \"millisSinceGpsEpoch\")\ndata_num = 0\nfor data in phone_separated_data:\n    data_num = data_num + len(data[\"millisSinceGpsEpoch\"].unique())\nprint(\"data num\", data_num) # num. of train data -> 130339; it should be same as \"num. of train data\"= 131342 but not same. Why?? Where is my wrong point??\n\nprint(len(train_data[\"millisSinceGpsEpoch\"].unique())) # num. of only merged by \"millisSinceGpsEpoch\". Of course its incorrect. ->128906","metadata":{"execution":{"iopub.status.busy":"2021-06-14T14:18:34.032424Z","iopub.execute_input":"2021-06-14T14:18:34.032888Z","iopub.status.idle":"2021-06-14T14:18:52.175370Z","shell.execute_reply.started":"2021-06-14T14:18:34.032851Z","shell.execute_reply":"2021-06-14T14:18:52.174405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Also, test data(and sample_submission)","metadata":{}},{"cell_type":"code","source":"# check train data \ntestfile = pd.read_csv(\"../input/google-smartphone-decimeter-challenge/baseline_locations_test.csv\")\nsubmission = pd.read_csv(\"../input/google-smartphone-decimeter-challenge/sample_submission.csv\")\n\nprint(\"true data num\",len(testfile)) # num. of test data -> 91486\nprint(\"true data num\",len(submission)) # num. of test data -> 91486\n\n\n#############################################################\n# read train data\nINPUT = '../input/google-smartphone-decimeter-challenge'\np = pathlib.Path(INPUT)\ntest_files = list(p.glob('test/*/*/*_derived.csv'))\ntest_data = []\nprint(\"data loading\")\nfor filename in tqdm(test_files):\n    test_data.append(pd.read_csv(filename))\n    \ntest_data = pd.concat(test_data)\n\nprint(len(test_data)) # num. of raw test data -> 2523199\n\n# split by phone name\nphone_names = test_data[\"phoneName\"].value_counts().index\nphone_separated_data = []\nfor phone_name in phone_names:\n    tmp = test_data[test_data[\"phoneName\"]==phone_name]\n    phone_separated_data.append(tmp)\n\n# merge by millisSinceGpsEpoch(actually, I only counted unique values of \"millisSinceGpsEpoch\")\ndata_num = 0\nfor data in phone_separated_data:\n    data_num = data_num + len(data[\"millisSinceGpsEpoch\"].unique())\nprint(\"data num\", data_num) # num. of test data -> 90421; it should be same as \"num. of test data\"= 91486 but not same. Why?? Where is my wrong point??\n\nprint(len(test_data[\"millisSinceGpsEpoch\"].unique())) # num. of only merged by \"millisSinceGpsEpoch\". Of course its incorrect. ->86943","metadata":{"execution":{"iopub.status.busy":"2021-06-14T14:18:52.267819Z","iopub.execute_input":"2021-06-14T14:18:52.268435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}