{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":31254,"databundleVersionId":3103714,"sourceType":"competition"},{"sourceId":2378330,"sourceType":"datasetVersion","datasetId":492658}],"dockerImageVersionId":30787,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# Please turn on GPU\nimport sys\n\n!cp ../input/rapids/rapids.21.06 /opt/conda/envs/rapids.tar.gz\n!cd /opt/conda/envs/ && tar -xzvf rapids.tar.gz > /dev/null\n!rm /opt/conda/envs/rapids.tar.gz\n\nsys.path += [\"/opt/conda/envs/rapids/lib/python3.7/site-packages\"]\nsys.path += [\"/opt/conda/envs/rapids/lib/python3.7\"]\nsys.path += [\"/opt/conda/envs/rapids/lib\"]\n!cp /opt/conda/envs/rapids/lib/libxgboost.so /opt/conda/lib/","metadata":{"execution":{"iopub.status.busy":"2024-10-30T08:36:15.612655Z","iopub.execute_input":"2024-10-30T08:36:15.613071Z","iopub.status.idle":"2024-10-30T08:37:47.332295Z","shell.execute_reply.started":"2024-10-30T08:36:15.613022Z","shell.execute_reply":"2024-10-30T08:37:47.331129Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import cudf\nimport gc\nimport lightgbm as lgb\nimport os\nimport numpy as np\nimport pandas as pd\nfrom tqdm import tqdm","metadata":{"execution":{"iopub.status.busy":"2024-10-30T08:37:47.334613Z","iopub.execute_input":"2024-10-30T08:37:47.334955Z","iopub.status.idle":"2024-10-30T08:37:51.906847Z","shell.execute_reply.started":"2024-10-30T08:37:47.334920Z","shell.execute_reply":"2024-10-30T08:37:51.905600Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"cuDF Version: \", cudf.__version__)","metadata":{"execution":{"iopub.status.busy":"2024-10-30T08:37:51.908776Z","iopub.execute_input":"2024-10-30T08:37:51.909799Z","iopub.status.idle":"2024-10-30T08:37:51.915421Z","shell.execute_reply.started":"2024-10-30T08:37:51.909750Z","shell.execute_reply":"2024-10-30T08:37:51.914462Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time \npreprocess_dir ='../input/h-and-m-personalized-fashion-recommendations/'\ntransactions = cudf.read_csv(os.path.join(preprocess_dir, 'transactions_train.csv'))\narticles = cudf.read_csv(os.path.join(preprocess_dir, 'articles.csv'))\ncustomers = cudf.read_csv(os.path.join(preprocess_dir, 'customers.csv'))","metadata":{"execution":{"iopub.status.busy":"2024-10-30T08:37:51.918157Z","iopub.execute_input":"2024-10-30T08:37:51.918844Z","iopub.status.idle":"2024-10-30T08:38:01.226485Z","shell.execute_reply.started":"2024-10-30T08:37:51.918800Z","shell.execute_reply":"2024-10-30T08:38:01.225240Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n'''articles = articles.dropna(subset=['detail_desc'])\ncustomers['FN'] = customers['FN'].fillna(0.)\ncustomers['Active'] = customers['Active'].fillna(0.)\n#customers['club_member_status'] = customers['club_member_status'].fillna('None')\ncustomers['club_member_status'] = customers['club_member_status'].fillna(0.)\n# 使用众数填充 'age' 列的缺失值\ncustomers['age'] = customers['age'] / 10\ncustomers['age'] = customers['age'].astype(int)'''\ncustomers['FN'] = customers['FN'].fillna(0.)\ncustomers['Active'] = customers['Active'].fillna(0.)\ncustomers['club_member_status'] = customers['club_member_status'].fillna('None')\ncustomers['age'] = customers['age'] / 10\ncustomers['age'] = customers['age'].astype(int)\ncustomers['fashion_news_frequency'] = customers['fashion_news_frequency'].str.lower().fillna('none')\narticles = articles.dropna(subset=['detail_desc'])","metadata":{"execution":{"iopub.status.busy":"2024-10-30T08:38:01.227612Z","iopub.execute_input":"2024-10-30T08:38:01.227944Z","iopub.status.idle":"2024-10-30T08:38:01.359457Z","shell.execute_reply.started":"2024-10-30T08:38:01.227910Z","shell.execute_reply":"2024-10-30T08:38:01.358323Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntransactions['t_dat'] = cudf.to_datetime(transactions['t_dat'])","metadata":{"execution":{"iopub.status.busy":"2024-10-30T08:38:01.360986Z","iopub.execute_input":"2024-10-30T08:38:01.361320Z","iopub.status.idle":"2024-10-30T08:38:01.688391Z","shell.execute_reply.started":"2024-10-30T08:38:01.361284Z","shell.execute_reply":"2024-10-30T08:38:01.687416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ncustomers = customers.merge(\n    transactions.groupby('customer_id')['sales_channel_id'].mean().to_frame().reset_index().rename(columns={'sales_channel_id':'sales_channel_habit'}),\n    on='customer_id',\n    how='left',\n)\ncustomers['sales_channel_habit'] = (customers['sales_channel_habit'] > 1.5).astype(int)\ncustomers['sales_channel_habit'].fillna(0.,inplace=True)","metadata":{"execution":{"iopub.status.busy":"2024-10-30T08:38:01.689840Z","iopub.execute_input":"2024-10-30T08:38:01.690688Z","iopub.status.idle":"2024-10-30T08:38:01.929696Z","shell.execute_reply.started":"2024-10-30T08:38:01.690635Z","shell.execute_reply":"2024-10-30T08:38:01.928556Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''# 计算 age 列的众数\nage_mode = customers['age'].mode().to_pandas().iloc[0]\n\n# 使用众数填充空值\ncustomers['age'] = customers['age'].fillna(age_mode)'''","metadata":{"execution":{"iopub.status.busy":"2024-10-30T08:38:01.930976Z","iopub.execute_input":"2024-10-30T08:38:01.931500Z","iopub.status.idle":"2024-10-30T08:38:01.938859Z","shell.execute_reply.started":"2024-10-30T08:38:01.931437Z","shell.execute_reply":"2024-10-30T08:38:01.937993Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''# 检查空值并打印结果\ndef check_null_values(df, name):\n    print(f\"\\nChecking null values in {name}:\")\n    # 查找空值的列\n    null_columns = df.isna().any()\n    null_columns = null_columns[null_columns].index.to_arrow().to_pylist()\n    \n    # 输出有空值的列\n    if len(null_columns) > 0:\n        print(f\"Columns with null values: {null_columns}\")\n        \n        # 输出有空值的列的唯一值\n        for col in null_columns:\n            unique_values = df[col].unique().sort_values()\n            print(unique_values)\n    else:\n        print(\"No columns with null values found.\")\n\n# 检查 transactions 表\ncheck_null_values(transactions, 'transactions')\n\n# 检查 articles 表\ncheck_null_values(articles, 'articles')\n\n# 检查 customers 表\ncheck_null_values(customers, 'customers')'''","metadata":{"execution":{"iopub.status.busy":"2024-10-30T08:38:01.940271Z","iopub.execute_input":"2024-10-30T08:38:01.940765Z","iopub.status.idle":"2024-10-30T08:38:01.950848Z","shell.execute_reply.started":"2024-10-30T08:38:01.940721Z","shell.execute_reply":"2024-10-30T08:38:01.949745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import cupy as cp\ndef set_gpu(gpu_id):\n    os.environ[\"CUDA_VISIBLE_DEVICES\"] = str(gpu_id)\n    cp.cuda.Device(gpu_id).use()","metadata":{"execution":{"iopub.status.busy":"2024-10-30T08:38:01.954646Z","iopub.execute_input":"2024-10-30T08:38:01.954951Z","iopub.status.idle":"2024-10-30T08:38:01.964213Z","shell.execute_reply.started":"2024-10-30T08:38:01.954918Z","shell.execute_reply":"2024-10-30T08:38:01.963414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def past_purchase_feature(df,transactions):\n    transactions.loc[:, 'count'] = 1\n    \n    time_elapsed_last_purchase = transactions['t_dat'].max()-transactions[['customer_id','article_id','t_dat']].groupby(['customer_id','article_id'])['t_dat'].max()\n    time_elapsed_last_purchase = time_elapsed_last_purchase.dt.days\n    df = df.merge(time_elapsed_last_purchase,on=['article_id','customer_id'],how='left')\n    df = df.rename(columns={'t_dat':'time_elapsed_last_purchase'})\n    print(type(df))\n    df['time_elapsed_last_purchase']=df['time_elapsed_last_purchase'].fillna(1e6)\n    print(type(df))\n    time_elapsed_first_release = transactions[['customer_id','article_id','t_dat']].groupby(['customer_id','article_id'])['t_dat'].min()-cudf.to_datetime('2018-09-01')\n    time_elapsed_first_release = time_elapsed_first_release.dt.days\n    df = df.merge(time_elapsed_first_release,on=['article_id','customer_id'],how='left')\n    df = df.rename(columns={'t_dat':'time_elapsed_first_release'})\n    df['time_elapsed_first_release']=df['time_elapsed_first_release'].fillna(1e6)\n    \n    past_purchase_prob = transactions[['customer_id','article_id','count']].groupby(['customer_id','article_id'])['count'].count().reset_index()\n    norm = transactions[['customer_id','article_id']].groupby('customer_id').count().reset_index().rename(columns={'article_id':'norm'})\n    past_purchase_prob = past_purchase_prob.merge(norm,on='customer_id')\n    past_purchase_prob['count'] = past_purchase_prob['count'] / past_purchase_prob['norm']\n    past_purchase_prob.drop(columns=['norm'],inplace=True)\n    df = df.merge(past_purchase_prob,on=['article_id','customer_id'],how='left')\n    df = df.rename(columns={'count':'past_purchase_prob'})\n    df['past_purchase_prob']=df['past_purchase_prob'].fillna(0.)\n    \n    total_purchase = transactions[['article_id','count']].groupby('article_id')['count'].count().reset_index().rename(columns={'count':'total_purchase'})\n    norm = transactions['count'].sum()\n    total_purchase['total_purchase'] = total_purchase['total_purchase'] / norm\n    df = df.merge(total_purchase,on='article_id',how='left')\n    df['total_purchase']=df['total_purchase'].fillna(0.)\n    \n    number_of_purchase = transactions[['customer_id','count']].groupby('customer_id')['count'].count().reset_index().rename(columns={'count':'number_of_purchase'})\n    df = df.merge(number_of_purchase,on='customer_id',how='left')\n    df['number_of_purchase']=df['number_of_purchase'].fillna(0.)\n    \n    repeated_purchase = transactions[['customer_id','article_id','count']].groupby(['customer_id','article_id'])['count'].count().reset_index().rename(columns={'count':'repeated_purchase'})\n    df = df.merge(repeated_purchase,on=['customer_id','article_id'],how='left')\n    \n    min_dat_purchase = transactions.groupby(['article_id'])['t_dat'].min()\n    max_dat_purchase = transactions.groupby(['article_id'])['t_dat'].max()\n    sale_duration = (max_dat_purchase - min_dat_purchase).to_frame().reset_index().rename(columns={'t_dat':'duration'})\n    sale_duration['duration'] = sale_duration['duration'].dt.days\n    sale_count = transactions.groupby(['article_id'])['t_dat'].count().to_frame().reset_index().rename(columns={'t_dat':'count'})\n    sale_rate = sale_duration.merge(sale_count,on='article_id')\n    sale_rate = sale_rate.loc[sale_rate['duration']!=0]\n    sale_rate['sale_rate'] = sale_rate['count'] / sale_rate['duration']\n    df = df.merge(sale_rate[['article_id','sale_rate']],on='article_id',how='left')\n\n    return df\n    \ndef article_feature_prob_vector(df,transactions,articles,article_features,postfix='_prob',customer_group_name='customer_id'):\n    transactions.loc[:, 'count'] = 1\n    \n    df = df#.to_pandas()\n    if customer_group_name != 'customer_id':\n        df = df.merge(customers[['customer_id',customer_group_name]],on='customer_id',how='left')\n        transactions = transactions.merge(customers[['customer_id',customer_group_name]],on='customer_id',how='left')\n    \n    for article_feature in article_features:\n        transactions = transactions.merge(articles[['article_id',article_feature]],on='article_id',how='left')\n        norm = transactions.groupby([customer_group_name])['count'].count().reset_index()\n        norm.rename(columns={'count':'norm'},inplace=True)\n        count = transactions.groupby([customer_group_name,article_feature])['count'].count().reset_index()\n        count = count.merge(norm,on=customer_group_name)\n        count['count'] = count['count'] / count['norm']\n        count = count.rename(columns={'count':article_feature+postfix})\n        count = count[[customer_group_name,article_feature,article_feature+postfix]]\n        del(norm)\n        df = df.merge(articles[['article_id',article_feature]],on='article_id',how='left')\n        \n        print(article_feature)\n        #df = df.merge(count,on=[customer_group_name,article_feature],how='left')\n        # 将 count 转换为 Pandas DataFrame（如果 count 也是 CuDF DataFrame）\n        #count = count\n\n        # 在 Pandas 中进行合并\n        df = df.merge(count, on=[customer_group_name, article_feature], how='left')\n\n        # 清理内存\n        del  count\n        gc.collect()\n\n        print(article_feature)\n        print(\"done!!!\")\n    #set_gpu(1)\n    #df = cudf.from_pandas(df)\n    return df\n\ndef customer_feature_prob_vector(df,transactions,customers,customer_features,postfix='_prob'):\n    transactions.loc[:, 'count'] = 1\n    \n    for customer_feature in customer_features:\n        transactions = transactions.merge(customers[['customer_id',customer_feature]],on='customer_id',how='left')\n        norm = transactions.groupby(['article_id'])['count'].count().reset_index()\n        norm.rename(columns={'count':'norm'},inplace=True)\n        count = transactions.groupby(['article_id',customer_feature])['count'].count().reset_index()\n        count = count.merge(norm,on='article_id')\n        count['count'] = count['count'] / count['norm']\n        count = count.rename(columns={'count':customer_feature+postfix})\n        count = count[['article_id',customer_feature,customer_feature+postfix]]\n        del(norm)\n        df = df.merge(customers[['customer_id',customer_feature]],on='customer_id',how='left')\n        df = df.merge(count,on=['article_id',customer_feature],how='left')\n    return df\n\ndef construct_feature_df(\n        df,transactions,\n        article_features,\n        articles,\n        customer_features,\n        customers,\n        general_features=['article_id','customer_id'],\n    ):\n    df = article_feature_prob_vector(df,transactions,articles,article_features)\n    df = customer_feature_prob_vector(df,transactions,customers,customer_features)\n    df = past_purchase_feature(df,transactions)\n    df = df[\n            general_features+[f for f in df.columns if '_prob' in f] + \n            ['total_purchase','time_elapsed_last_purchase','past_purchase_prob','number_of_purchase','time_elapsed_first_release','repeated_purchase']\n        ]\n    return df\n\ndef construct_candidate_dict(transactions_3w):\n    purchase_dict_3w = {}\n    for i,x in enumerate(zip(transactions_3w['customer_id'], transactions_3w['article_id'])):\n        cust_id, art_id = x\n        if cust_id not in purchase_dict_3w:\n            purchase_dict_3w[cust_id] = {}\n        if art_id not in purchase_dict_3w[cust_id]:\n            purchase_dict_3w[cust_id][art_id] = 0\n        purchase_dict_3w[cust_id][art_id] += 1\n    return purchase_dict_3w\n\ndef construct_candidate_df(\n        test_df,transactions,\n        nweek=8,\n        n_popular_item=90,\n        n_total_item=None,\n    ):\n    \n    recent_transactions = {}\n    purchase_dict = {}\n    for i in range(1,nweek+1):\n        recent_transactions[i] = transactions[(transactions.t_dat>transactions.t_dat.max()-i*pd.Timedelta(7,unit='day'))&(transactions.t_dat<=transactions.t_dat.max()-(i-1)*pd.Timedelta(7,unit='day'))]\n        purchase_dict[i] = construct_candidate_dict(recent_transactions[i])\n    \n    if 1 in recent_transactions:\n        most_popular_items_1w_all = list((recent_transactions[1]['article_id'].value_counts()).index)[:n_popular_item]\n    else:\n        most_popular_items_1w_all = list(transactions['article_id'].value_counts().index.to_arrow().to_pylist())[:n_popular_item]\n    \n    pred_df = pd.DataFrame()\n    pred_df['customer_id'] = test_df['customer_id'].unique()\n    \n    prediction_list = []\n    \n    for i, cust_id in enumerate(pred_df['customer_id']):\n        s = []\n        total_purchase_dict = {}\n        \n        for i,purchase_dict_week in purchase_dict.items():\n            if cust_id in purchase_dict_week:\n                l = sorted((purchase_dict_week[cust_id]).items(), key=lambda x: x[1], reverse=True)\n                l = [y[0] for y in l]\n                for aid in l:\n                    if aid not in total_purchase_dict:\n                        total_purchase_dict[aid] = 1\n                    else:\n                        total_purchase_dict[aid] += 1\n\n        for aid in most_popular_items_1w_all[:n_popular_item]:\n            if aid not in total_purchase_dict:\n                total_purchase_dict[aid] = 1\n            else:\n                total_purchase_dict[aid] += 1\n\n        if n_total_item is not None:\n            total_purchase_dict = {k: v for k, v in sorted(total_purchase_dict.items(), key=lambda item: item[1], reverse=True)}\n            s = list(total_purchase_dict.keys())[:n_total_item]\n        else:\n            s = list(total_purchase_dict.keys())\n        \n        prediction_list.append(s)\n        \n    pred_df['article_id'] = prediction_list\n    \n    return pred_df\n\ndef construct_test_df(test_df,transactions,article_features,articles,customer_features,customers,how='outer',n_popular_item=90):\n    test_df = construct_candidate_df(test_df,transactions,n_popular_item=n_popular_item).explode(['article_id']).reset_index(drop=True)\n    '''# 在此处插入代码\n        \n    print(\"test_df:\")\n    print(test_df)\n\n    print(\"\\nData types of each column:\")\n    print(test_df.dtypes)\n\n    print(\"\\nNull values in each column:\")\n    null_info = test_df.isna().sum()\n    #print(null_info)'''\n    test_df['article_id'] = test_df['article_id'].astype(int)\n    print(\"\\nData types of each column:\")\n    print(test_df.dtypes)\n    #test_df = cudf.from_pandas(test_df)\n    test_df = construct_feature_df(test_df,transactions,article_features,articles,customer_features,customers,general_features=['article_id','customer_id'])\n    test_df = test_df.fillna(0.)\n    test_df['article_id'] = test_df['article_id'].astype(int)\n    test_df = test_df.sort_values(['customer_id','article_id']).reset_index(drop=True)\n    return test_df\n\ndef construct_gt_df(test_transactions):\n    gt_df = test_transactions.groupby('customer_id')['article_id'].agg(lambda x: x.tolist()).reset_index()\n    gt_df.columns = ['customer_id','ground_truth']\n    return gt_df\n    \ndef construct_dataset(\n        transactions,\n        articles,customers,\n        trn_start_time='2020-08-31',trn_end_time='2020-09-08',\n        val_start_time='2020-09-08',val_end_time='2020-09-15',\n        test_start_time='2020-09-08',test_end_time='2020-09-15',\n        article_features=[\n            'product_type_name','product_group_name',\n            'graphical_appearance_name','colour_group_name',\n            'perceived_colour_value_name','perceived_colour_master_name',\n            'department_name', 'index_name',\n            'index_group_name','section_name',\n            'garment_group_name',\n        ],\n        customer_features=[\n            'FN','Active','club_member_status','age','fashion_news_frequency',#'sales_channel_habit',\n        ],\n    ):\n    \n    trn_start_time = cudf.to_datetime(trn_start_time)\n    trn_end_time = cudf.to_datetime(trn_end_time)\n    val_start_time = cudf.to_datetime(val_start_time)\n    val_end_time = cudf.to_datetime(val_end_time)\n    test_start_time = cudf.to_datetime(test_start_time)\n    test_end_time = cudf.to_datetime(test_end_time)\n    \n    trn_transactions = transactions[(transactions.t_dat > trn_start_time) & (transactions.t_dat <= trn_end_time)]\n    val_transactions = transactions[(transactions.t_dat > val_start_time) & (transactions.t_dat <= val_end_time)]\n    test_transactions = transactions[(transactions.t_dat > test_start_time) & (transactions.t_dat <= test_end_time)]\n    gt_df = construct_gt_df(test_transactions)\n    \n    trn_df = construct_test_df(val_transactions,trn_transactions,article_features,articles,customer_features,customers,how='left',n_popular_item=80)\n    print(\"construct_test_d1fDONE!!\")\n    pos_label = val_transactions[['article_id','customer_id']].groupby(['article_id','customer_id']).size().to_frame('label')\n    pos_label['label'] = pos_label['label'].apply(lambda x: x if x <=20. else 20.)\n    trn_df = trn_df.merge(pos_label,on=['article_id','customer_id'],how='left')\n    trn_df['label']=trn_df['label'].fillna(0.)\n    \n    trn_df = trn_df.merge(trn_df.groupby('customer_id').size().to_frame().rename(columns={0:'group_size'}),on='customer_id')\n    test_df = construct_test_df(test_transactions,val_transactions,article_features,articles,customer_features,customers,how='left',n_popular_item=80)\n    \n    return trn_df.reset_index(drop=True),test_df.reset_index(drop=True),gt_df.reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2024-10-30T08:38:01.965513Z","iopub.execute_input":"2024-10-30T08:38:01.965832Z","iopub.status.idle":"2024-10-30T08:38:02.022705Z","shell.execute_reply.started":"2024-10-30T08:38:01.965787Z","shell.execute_reply":"2024-10-30T08:38:02.021833Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nlabel = 'label'\ntag = ''\nt1,t2,t3,t4 = '2020-05-17','2020-07-17','2020-09-01','2020-09-07'\n\n#base_dir = 'storage/output/220325_dataset_'+'_'.join([t1,t2,t3,t4])+tag+'/'","metadata":{"execution":{"iopub.status.busy":"2024-10-30T08:38:02.023798Z","iopub.execute_input":"2024-10-30T08:38:02.024179Z","iopub.status.idle":"2024-10-30T08:38:02.037439Z","shell.execute_reply.started":"2024-10-30T08:38:02.024134Z","shell.execute_reply":"2024-10-30T08:38:02.036545Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n'''trn_df,test_df,gt_df = construct_dataset(\n    transactions,\n    articles,customers,\n    trn_start_time=t1,trn_end_time=t2,\n    val_start_time=t2,val_end_time=t3,\n    test_start_time=t3,test_end_time=t4,\n)'''\n# 抽取 10% 的样本\nsampled_transactions = transactions.sample(frac=0.25, random_state=42).to_pandas()\n\n# 调用构造数据集的函数\ntrn_df, test_df, gt_df = construct_dataset(\n    sampled_transactions,  # 使用抽样后的数据\n    articles.to_pandas(),\n    customers.to_pandas(),\n    trn_start_time=t1,\n    trn_end_time=t2,\n    val_start_time=t2,\n    val_end_time=t3,\n    test_start_time=t3,\n    test_end_time=t4,\n)\n#os.makedirs(base_dir,exist_ok=True)","metadata":{"execution":{"iopub.status.busy":"2024-10-30T08:38:02.038495Z","iopub.execute_input":"2024-10-30T08:38:02.038772Z","iopub.status.idle":"2024-10-30T08:48:41.370132Z","shell.execute_reply.started":"2024-10-30T08:38:02.038741Z","shell.execute_reply":"2024-10-30T08:48:41.369117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trn_df.to_csv('trn_df.csv',index=False,chunksize=5e5)\ntest_df.to_csv('test_df.csv',index=False)\ngt_df.to_csv('gt_df.csv',index=False)","metadata":{"execution":{"iopub.status.busy":"2024-10-30T08:48:41.371649Z","iopub.execute_input":"2024-10-30T08:48:41.372032Z","iopub.status.idle":"2024-10-30T08:59:42.631594Z","shell.execute_reply.started":"2024-10-30T08:48:41.371997Z","shell.execute_reply":"2024-10-30T08:59:42.630567Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trn_df.shape","metadata":{"execution":{"iopub.status.busy":"2024-10-30T08:59:42.632963Z","iopub.execute_input":"2024-10-30T08:59:42.633338Z","iopub.status.idle":"2024-10-30T08:59:42.639487Z","shell.execute_reply.started":"2024-10-30T08:59:42.633290Z","shell.execute_reply":"2024-10-30T08:59:42.638639Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.shape","metadata":{"execution":{"iopub.status.busy":"2024-10-30T08:59:42.640605Z","iopub.execute_input":"2024-10-30T08:59:42.640884Z","iopub.status.idle":"2024-10-30T08:59:42.652224Z","shell.execute_reply.started":"2024-10-30T08:59:42.640853Z","shell.execute_reply":"2024-10-30T08:59:42.651327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gt_df.shape","metadata":{"execution":{"iopub.status.busy":"2024-10-30T08:59:42.653418Z","iopub.execute_input":"2024-10-30T08:59:42.653823Z","iopub.status.idle":"2024-10-30T08:59:42.662992Z","shell.execute_reply.started":"2024-10-30T08:59:42.653778Z","shell.execute_reply":"2024-10-30T08:59:42.662111Z"},"trusted":true},"execution_count":null,"outputs":[]}]}