{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"%%time\nimport pandas as pd\nimport numpy as np\n\n# import matplotlib.pyplot as plt\n# import seaborn as sns\nimport itertools\nimport gc\nimport pickle\nimport os\nfrom time import time\n\nimport pathlib\nprint(\"Pasta atual:\", pathlib.Path().absolute())\n\n# from catboost import CatBoostRegressor, Pool, cv\n# from catboost.utils import get_gpu_device_count\n\n# nome do modelo e do .csv\nfrom datetime import datetime as dtime\ntname = str(dtime.now())[:-7]\n\n\n# %%time\npath='../input/kddbr-2020/'\ndef load_year(y):\n    dtypes = pickle.load(open(\n        f'../input/kdd2020-cpr/{y}_dtypes.pkl', 'rb'\n    ))\n    del dtypes['date']\n    df = pd.read_csv(\n        f'../input/kdd2020-cpr/{y}.csv',\n        dtype=dtypes, parse_dates=['date'], index_col='id'\n    )\n    return df\n\nYEARS = [2018]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"%%time\nbase = [load_year(year) for year in YEARS]\nbase = pd.concat(base)\n\ncolumns_size = None\nrows_size = None\nif (columns_size != None or rows_size != None):\n    inputs = list( base.columns[ base.columns.str.contains('input')][:columns_size])\n    inputs.sort()\n    output = list(base.columns[ base.columns.str.contains('output')] )\n    cols = list( base[ inputs + output  ].columns )\n    cols.append('id')\n    cols.append('date')\n    base = base[cols].copy()[:rows_size]\n\n# %%time\ninput_columns = base.columns[base.columns.str.contains('input') ]\noutput_columns = base.columns[base.columns.str.contains('output') ]\n\nprint(F'Inputs: {input_columns.shape} Outputs: {output_columns.shape}')\n# input_columns, output_columns\n\n\n# %%time\ndef create_features(df):\n    df['input_month'] = df.date.dt.month\n    df['input_year'] = df.date.dt.year\n    df['input_day'] = df.date.dt.day\n    df['input_dt_sin_quarter']     = np.sin(2*np.pi*df.date.dt.quarter/4)\n    df['input_dt_sin_day_of_week'] = np.sin(2*np.pi*df.date.dt.dayofweek/6)\n    df['input_dt_sin_day_of_year'] = np.sin(2*np.pi*df.date.dt.dayofyear/365)\n    df['input_dt_sin_day']         = np.sin(2*np.pi*df.date.dt.day/30)\n    df['input_dt_sin_month']       = np.sin(2*np.pi*df.date.dt.month/12)\n    return df\n\ndef is_weekend(num):\n    return num > 5\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ndef date_expand(df, pipeline = True):\n    df['date'] = pd.to_datetime( df.date )\n    dt = df['date'].dt\n    df['week'] = dt.week\n    df['weekday'] = dt.weekday + 1\n    df['weekday_sin'] = np.sin(2*np.pi*df.date.dt.weekday/7)\n    \n    df['weekofyear'] = dt.weekofyear\n    df['weekofyear_sin'] = np.sin(2*np.pi*df.date.dt.weekofyear/52)\n\n    df['weekend'] = dt.weekday.apply(is_weekend)\n    return df if pipeline else None\n\ncreate_features(base);\ndate_expand(base);\ninput_columns = base.columns[base.columns.str.contains('input') ]\noutput_columns = base.columns[base.columns.str.contains('output') ]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n# necessário pois o MultiOutputRegresso espera que não haja NaN nos dados\n# Obviamente, 6358 NÃO está presente em 2018\nCUSTOM_NA = 6358\n\n\n# %%time\nX = base[input_columns[:columns_size] ].fillna(CUSTOM_NA).values\nY = base[output_columns].fillna(0).values\n\nfrom sklearn.model_selection import train_test_split\n# x_train, x_val, y_train, y_val = train_test_split(X, Y, test_size=0.0, random_state=42)\n\n\nfrom xgboost import XGBRegressor\nfrom sklearn.multioutput import MultiOutputRegressor\n\nmodel = XGBRegressor(\n    n_estimators=7, random_state=0,\n    learning_rate=.1, max_depth=1, colsample_bytree=.8, colsample_bynode=.9,\n    tree_method='gpu_hist', sampling_method='gradient_based',\n    gpu_id=0, verbosity=1, missing=CUSTOM_NA\n)\n\nclf = MultiOutputRegressor(model)\nclf.fit(X[:3],Y[:3])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nQTD = None\n#Loading test file\n# _df = load_year(2019)\n\n# _df = create_features(_df)\ninput_columns = _df.columns[_df.columns.str.contains('input') ]\n_df = _df[input_columns].copy()\n_df.fillna(CUSTOM_NA, inplace=True)\n\n# %%time\npred = clf.predict(_df.values[:QTD])\npred_sub = pd.DataFrame(pred)\npred_sub.columns = output_columns\npred_sub['id'] = _df.index[:QTD]\n\n\n# %%time\nsubmission = []\nfor i, row in pred_sub.iterrows():\n    for column, value in zip(output_columns, row.values):\n        _id = \"{}_{}\".format(int(row.id), column)\n        submission.append([_id, value])\n    break\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del _df, test\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.DataFrame(submission)\nsubmission.columns = ['id', 'value']\nsubmission.to_csv('{}.csv'.format(tname), index=False)\n\ntry:\n    os.makedirs('clfs/xgb', exist_ok=True)\n    pickle.dump(clf, open('clfs/xgb/{}.pkl'.format(tname), 'wb') )\nexcept:\n    print('erro no pickle')\n","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}