# This script offers separate parquet files of the 8th and 9th to be used as trainind and testing sets respectively, 
# The goal is to recreate approximatively the conditions of the contest, shifted by a day. It is meant to speed up quick tuning of models.
# The data has been truncated to show only the range of the features appearing in the test data, and the 9th has its timerange reduced as well.
#
# Use pd.read_parquet(file_name).reset_index(drop=True) to read them. It is MUCH faster than compressed csv.
# The reset_index is meant to save space, turning an INT64 index into a Range index.


import numpy as np
import pandas as pd
import os
import gc

columns = ['ip','click_time']     #ip is just a placeholder here.
types = {
        'ip':np.int32
        }

with open('../input/train.csv','rb') as File:
    data=(pd
            .read_csv(File,
                        parse_dates=['click_time'],
                        infer_datetime_format=True,
                        index_col='click_time',
                        usecols=columns,
                        dtype=types,
                        engine='c',
                        sep=',')
            .tz_localize('UTC')
            .tz_convert('Asia/Shanghai')
            .assign(index= lambda x: range(len(x)))
            )

end=data.index[-1]
dt=pd.Timedelta('1 days')
dt_sec=dt-pd.Timedelta('1 seconds')
last_day_index=data.loc[end-dt_sec:end,'index']
ante_day_index=data.loc[end-dt-dt_sec:end-dt,'index']

ante_day_skip=range(1,ante_day_index.min())
ante_day_nrows=ante_day_index.max()-ante_day_index.min()

last_day_skip=range(1,last_day_index.min())
last_day_nrows=last_day_index.max()-last_day_index.min()

del(data,last_day_index,ante_day_index,end,dt,dt_sec)
gc.collect()

columns = ['ip','app','os','device','channel','click_time','is_attributed']     
types = {
        'ip':np.uint32,
        'app': np.uint16,
        'os': np.uint16,
        'device': np.uint16,
        'channel':np.uint16,
        'is_attributed':'bool'
        }

with open('../input/train.csv','rb') as File:
    (pd.read_csv(File,
                parse_dates=['click_time'],
                index_col='click_time',
                infer_datetime_format=True,
                skiprows=ante_day_skip,
                nrows=ante_day_nrows,
                usecols=columns,
                dtype=types,
                engine='c',
                sep=',')
        .tz_localize('UTC')
        .tz_convert('Asia/Shanghai')
        .reset_index()
        .loc[lambda x: x.ip<126420,:]
        .loc[lambda x: x.app<521,:]
        .loc[lambda x: x.channel<498,:]
        .loc[lambda x: x.os<604,:]
        .loc[lambda x: x.device<3031,:]
        .to_parquet('ante_day.pqt',version='2.0')
    )

with open('../input/train.csv','rb') as File:
        (pd.read_csv(File,
                    parse_dates=['click_time'],
                    index_col='click_time',
                    infer_datetime_format=True,
                    skiprows=last_day_skip,
                    nrows=last_day_nrows,
                    usecols=columns,
                    dtype=types,
                    engine='c',
                    sep=',')
            .tz_localize('UTC')
            .tz_convert('Asia/Shanghai')
            .reset_index()
            .loc[lambda x: x.ip<126420,:]
            .loc[lambda x: x.app<521,:]
            .loc[lambda x: x.channel<498,:]
            .loc[lambda x: x.os<604,:]
            .loc[lambda x: x.device<3031,:]
            .loc[lambda x: x.click_time.dt.hour.isin([12,13,17,18,21,22]),:]
            .to_parquet('last_day.pqt',version='2.0')
        )