{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n        \nimport dask.dataframe as dd\nimport seaborn as sns\nimport matplotlib as mpl\nimport matplotlib.pyplot as plt\nfrom statistics import mean\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-18T13:19:59.089871Z","iopub.execute_input":"2022-07-18T13:19:59.091223Z","iopub.status.idle":"2022-07-18T13:20:00.809019Z","shell.execute_reply.started":"2022-07-18T13:19:59.091054Z","shell.execute_reply":"2022-07-18T13:20:00.807442Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# intro","metadata":{}},{"cell_type":"markdown","source":"This notebook was created to check the status of test data that could not be checked in [this notebook](https://www.kaggle.com/code/imnaho/amexcompetition-just-prepareing-training-data/) .   \nThe main content is to create the training data described in [this notebook](https://www.kaggle.com/code/imnaho/amexcompetition-just-prepareing-training-data/), so if you are interested, please have a look.\n\nこのノートブックは[こちらのノートブック](https://www.kaggle.com/code/imnaho/amexcompetition-just-prepareing-training-data/)で確認しきれなかったtestデータの状態を確認するために作成したものです。  \nメインの内容は[こちらのノートブック](https://www.kaggle.com/code/imnaho/amexcompetition-just-prepareing-training-data/)で記載している、学習データを作成をするところなので、ご興味があれば、是非ご覧ください。","metadata":{}},{"cell_type":"markdown","source":"# Read test Data","metadata":{}},{"cell_type":"code","source":"#read\ntest_data_path = '../input/amex-default-prediction/test_data.csv'\ntest_data = dd.read_csv(test_data_path, storage_options={'anon': True}, assume_missing=True)\n\n#convert\nname_function = lambda x: f\"data-{x}.parquet\"\ntest_data.to_parquet('./test_data_parquet/', name_function=name_function)","metadata":{"execution":{"iopub.status.busy":"2022-07-18T13:20:00.811576Z","iopub.execute_input":"2022-07-18T13:20:00.812705Z","iopub.status.idle":"2022-07-18T13:28:00.551318Z","shell.execute_reply.started":"2022-07-18T13:20:00.812663Z","shell.execute_reply":"2022-07-18T13:28:00.550039Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(test_data.shape)\ntest_data","metadata":{"execution":{"iopub.status.busy":"2022-07-18T13:28:00.552886Z","iopub.execute_input":"2022-07-18T13:28:00.553286Z","iopub.status.idle":"2022-07-18T13:28:00.770239Z","shell.execute_reply.started":"2022-07-18T13:28:00.553251Z","shell.execute_reply":"2022-07-18T13:28:00.769309Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# check test data","metadata":{}},{"cell_type":"code","source":"cnt=0\nfor i in range(528):\n    test_data_parquet_path = './test_data_parquet/data-'+str(i)+ '.parquet'\n    test_data_parquet = dd.read_parquet(test_data_parquet_path)\n    test_parquet_data=test_data_parquet.compute()    \n    a=test_parquet_data[test_parquet_data['S_2'].isnull()]\n    cnt=cnt+len(a)\nprint(cnt)","metadata":{"execution":{"iopub.status.busy":"2022-07-18T13:28:00.772374Z","iopub.execute_input":"2022-07-18T13:28:00.772867Z","iopub.status.idle":"2022-07-18T13:31:40.391500Z","shell.execute_reply.started":"2022-07-18T13:28:00.772834Z","shell.execute_reply":"2022-07-18T13:31:40.390364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"null_rate={}\ncnt=0\nfor i in range(528):\n    test_data_parquet_path = './test_data_parquet/data-'+str(i)+ '.parquet'\n    test_data_parquet = dd.read_parquet(test_data_parquet_path)\n    test_parquet_data=test_data_parquet.compute()\n    for col in test_parquet_data.columns.tolist():\n        if i ==0:\n            null_rate[col]=test_parquet_data[col].isnull().sum()/len(test_parquet_data)\n        else:\n            null_rate[col]=(null_rate[col]+(test_parquet_data[col].isnull().sum()/len(test_parquet_data)))/2\n    dic2 = sorted(null_rate.items(), key=lambda x:x[1])\ndic2","metadata":{"execution":{"iopub.status.busy":"2022-07-18T13:31:40.392913Z","iopub.execute_input":"2022-07-18T13:31:40.393543Z","iopub.status.idle":"2022-07-18T13:35:44.090617Z","shell.execute_reply.started":"2022-07-18T13:31:40.393510Z","shell.execute_reply":"2022-07-18T13:35:44.089312Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rate=None\nkey_list=[]\ngroup_list=[]\nfor key,value in dic2:\n    if rate==value:\n        key_list.append(key)\n    else:\n        rate=value\n        group_list.append(key_list)\n        key_list=[key]\ngroup_list_2=[x for x in group_list if len(x)>2]\nprint('the number of group is :',len(group_list_2))\n\nfor i in range(len(group_list_2)):\n    print(len(group_list_2[i]))","metadata":{"execution":{"iopub.status.busy":"2022-07-18T13:35:44.092420Z","iopub.execute_input":"2022-07-18T13:35:44.093901Z","iopub.status.idle":"2022-07-18T13:35:44.102576Z","shell.execute_reply.started":"2022-07-18T13:35:44.093852Z","shell.execute_reply":"2022-07-18T13:35:44.101267Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"remove_list=[]\nt_list=[]\n\nfor i in range(528):\n    test_data_parquet_path = './test_data_parquet/data-'+str(i)+ '.parquet'\n    test_data_parquet = dd.read_parquet(test_data_parquet_path)\n    test_parquet_data=test_data_parquet.compute()\n\n    for key_list in group_list_2:\n        #print('list lengh is :',len(key_list))\n        a=test_parquet_data[key_list[0]]\n        a=a[a.isnull()].index.tolist()\n        for key in key_list:\n            b=test_parquet_data[key]\n            b=b[b.isnull()].index.tolist()\n            if a == b:\n                pass\n            else:\n                remove_list.append(key)\n\n                remove_list=pd.unique(remove_list).tolist()\nt_list=pd.unique(t_list).tolist()\nprint('remove_list is :',remove_list)","metadata":{"execution":{"iopub.status.busy":"2022-07-18T13:35:44.104523Z","iopub.execute_input":"2022-07-18T13:35:44.105109Z","iopub.status.idle":"2022-07-18T13:37:51.722576Z","shell.execute_reply.started":"2022-07-18T13:35:44.105051Z","shell.execute_reply":"2022-07-18T13:37:51.721742Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"i=0\n\nfor i in range(7):\n    for key in remove_list:\n        if key in group_list_2[i]:\n            group_list_2[i].remove(key)\n    i=i+1\n\ngroup_list_2=[x for x in group_list_2 if len(x)>2]\nfor i in range(6):\n    print(len(group_list_2[i]))","metadata":{"execution":{"iopub.status.busy":"2022-07-18T13:37:51.723911Z","iopub.execute_input":"2022-07-18T13:37:51.724438Z","iopub.status.idle":"2022-07-18T13:37:51.730740Z","shell.execute_reply.started":"2022-07-18T13:37:51.724406Z","shell.execute_reply":"2022-07-18T13:37:51.729831Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(5):\n    print(group_list_2[i+1])","metadata":{"execution":{"iopub.status.busy":"2022-07-18T13:37:51.731876Z","iopub.execute_input":"2022-07-18T13:37:51.732667Z","iopub.status.idle":"2022-07-18T13:37:51.743947Z","shell.execute_reply.started":"2022-07-18T13:37:51.732637Z","shell.execute_reply":"2022-07-18T13:37:51.742814Z"},"trusted":true},"execution_count":null,"outputs":[]}]}