{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# ====================================================\n# Library\n# ====================================================\nimport gc\nimport warnings\nwarnings.filterwarnings('ignore')\nimport scipy as sp\nimport numpy as np\nimport pandas as pd\npd.set_option('display.max_rows', 500)\npd.set_option('display.max_columns', 500)\npd.set_option('display.width', 1000)\nfrom tqdm.auto import tqdm\nfrom scipy import stats","metadata":{"id":"8IcxJd_WOjhF","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# For each dataset (train, pub, pri), calculate 'miss_next_payment'","metadata":{"id":"kb8xfxjpVgfU"}},{"cell_type":"code","source":"def process(load_file, save_file):\n    print(\".\")\n    test = pd.read_parquet(load_file, columns=[\"customer_ID\", \"S_2\", \"D_39\"])\n    print(\"loaded\")\n\n    x = test.groupby(\"customer_ID\")[\"D_39\"].diff(-1)\n    test[\"D_39_lag\"] = x\n    print(\".\")\n    test['S_2'] = pd.to_datetime(test['S_2']).dt.date\n    print(\".\")\n    t = test.groupby(\"customer_ID\")[\"S_2\"].diff(-1)\n    print(\"+\")\n    t[t.notna()] = t[t.notna()] / pd.Timedelta(days=1)\n\n    test[\"S_2_lag\"] = t\n    test[\"miss_next_payment\"] = 0\n    test.loc[ (test[\"D_39_lag\"] <= -28) | ((test[\"D_39_lag\"] < -14) & (test[\"S_2_lag\"] >= test[\"D_39_lag\"]) & (test[\"D_39\"] - test[\"D_39_lag\"] >= 28)), [\"miss_next_payment\"] ] = 1\n\n    test.loc[test[\"D_39\"] - test[\"D_39_lag\"] == 0, [\"miss_next_payment\"]] = -1\n\n    test = test.drop(columns=[\"D_39\", \"D_39_lag\", \"S_2_lag\"])\n\n    test.to_parquet(save_file)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"process('../input/amex-fe-01-split-test-into-public-and-private/private_test.parquet', 'private_test_days_over.parquet')\nprocess('../input/amex-fe-01-split-test-into-public-and-private/public_test.parquet', 'public_test_days_over.parquet')\nprocess('../input/amex-data-integer-dtypes-parquet-format/train.parquet', 'train_days_over.parquet')","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}