{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# モジュールをインポート","metadata":{}},{"cell_type":"code","source":"!pip install japanize_matplotlib","metadata":{"execution":{"iopub.status.busy":"2023-07-20T17:53:46.156704Z","iopub.execute_input":"2023-07-20T17:53:46.157195Z","iopub.status.idle":"2023-07-20T17:54:01.221405Z","shell.execute_reply.started":"2023-07-20T17:53:46.157157Z","shell.execute_reply":"2023-07-20T17:54:01.220183Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import lightgbm as lgb\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport gc\nimport time\nimport warnings\n\nfrom contextlib import contextmanager\nimport japanize_matplotlib\nfrom matplotlib import pyplot as plt\nfrom lightgbm import LGBMClassifier\nfrom sklearn.model_selection import KFold, StratifiedKFold, train_test_split\nfrom sklearn.metrics import roc_auc_score, roc_curve, confusion_matrix, precision_score, recall_score\nfrom datetime import timedelta\nfrom dateutil.relativedelta import relativedelta\n\n\npd.options.display.max_columns = None\npd.options.display.max_rows = 100\npd.options.display.float_format = '{:.2f}'.format\n\nwarnings.simplefilter(action='ignore', category=FutureWarning)\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-07-20T17:54:01.224512Z","iopub.execute_input":"2023-07-20T17:54:01.224980Z","iopub.status.idle":"2023-07-20T17:54:03.082749Z","shell.execute_reply.started":"2023-07-20T17:54:01.224937Z","shell.execute_reply":"2023-07-20T17:54:03.081971Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# データインプット","metadata":{}},{"cell_type":"code","source":"transactions = pd.read_csv(\"../input/h-and-m-personalized-fashion-recommendations/transactions_train.csv\")\ncustomers = pd.read_csv(\"../input/h-and-m-personalized-fashion-recommendations/customers.csv\")\narticles = pd.read_csv(\"../input/h-and-m-personalized-fashion-recommendations/articles.csv\")","metadata":{"execution":{"iopub.status.busy":"2023-07-20T17:54:03.084008Z","iopub.execute_input":"2023-07-20T17:54:03.084310Z","iopub.status.idle":"2023-07-20T17:55:25.857174Z","shell.execute_reply.started":"2023-07-20T17:54:03.084283Z","shell.execute_reply":"2023-07-20T17:55:25.856084Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 関数","metadata":{}},{"cell_type":"code","source":"pre_start_date=\"2018-12-01\"\npre_end_date=\"2019-03-01\"\npos_start_date=\"2019-03-01\"\npos_end_date=\"2019-06-01\"\n\nval_pre_start_date=\"2019-03-01\"\nval_pre_end_date=\"2019-06-01\"\nval_pos_start_date=\"2019-06-01\"\nval_pos_end_date=\"2019-09-01\"","metadata":{"execution":{"iopub.status.busy":"2023-07-20T17:55:25.859899Z","iopub.execute_input":"2023-07-20T17:55:25.860239Z","iopub.status.idle":"2023-07-20T17:55:25.865475Z","shell.execute_reply.started":"2023-07-20T17:55:25.860213Z","shell.execute_reply":"2023-07-20T17:55:25.864587Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 基本","metadata":{}},{"cell_type":"code","source":"#Baseとなる顧客ID 基本的にここにどんどんLeft joinしていく方針\ndef create_target_customers(transactions,pre_start_date,pre_end_date): #最終的にはstart_ym, end_ymを引数に指定\n    transactions['t_dat_dt'] = pd.to_datetime(transactions['t_dat'])\n    start_ym = pd.to_datetime(pre_start_date) \n    end_ym = pd.to_datetime(pre_end_date)\n    transaction_target_ym = transactions.query('t_dat_dt >= @start_ym & t_dat_dt < @end_ym').assign(target_customer = 1)\n    target_customers = transaction_target_ym[['customer_id','target_customer']].drop_duplicates()\n    return target_customers\n\n#ただcustomersの情報を取得するだけ\ndef create_customers_feature(customers): \n    return customers\n","metadata":{"execution":{"iopub.status.busy":"2023-07-20T17:55:25.866750Z","iopub.execute_input":"2023-07-20T17:55:25.867378Z","iopub.status.idle":"2023-07-20T17:55:25.880561Z","shell.execute_reply.started":"2023-07-20T17:55:25.867344Z","shell.execute_reply":"2023-07-20T17:55:25.879684Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 目的変数作成","metadata":{}},{"cell_type":"code","source":"#目的変数を作成するための関数(create_target_customers()で作成したデータフレームを用いる)\ndef create_target_variable(transactions,target_customers,pos_start_date,pos_end_date):\n    #観察期間での購入フラグ作成\n    transactions['t_dat_dt'] = pd.to_datetime(transactions['t_dat'])\n    pos_start_ym = pd.to_datetime(pos_start_date) #ここはあとで関数の引数で指定できるようにする\n    pos_end_ym = pd.to_datetime(pos_end_date)\n    pos_transaction_target_ym = transactions.query('t_dat_dt >= @pos_start_ym & t_dat_dt < @pos_end_ym').assign(pos_target_customer = 0)\n    pos_target_customers = pos_transaction_target_ym[['customer_id','pos_target_customer']].drop_duplicates()\n    \n    #目的変数の作成\n    target_customers=pd.merge(target_customers, pos_target_customers, how=\"left\", on=\"customer_id\")\n    target_customers[\"pos_target_customer\"]=target_customers[\"pos_target_customer\"].fillna(1).astype(\"int\")\n    target_customers = target_customers[['customer_id','pos_target_customer']].drop_duplicates().rename(columns={\"pos_target_customer\":\"target\"})\n\n    return target_customers\n","metadata":{"execution":{"iopub.status.busy":"2023-07-20T17:55:25.882015Z","iopub.execute_input":"2023-07-20T17:55:25.882929Z","iopub.status.idle":"2023-07-20T17:55:25.892199Z","shell.execute_reply.started":"2023-07-20T17:55:25.882903Z","shell.execute_reply":"2023-07-20T17:55:25.891174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 特徴量作成","metadata":{}},{"cell_type":"code","source":"def ymset(pre_start_date,pre_end_date):\n    transactions['t_dat_dt'] = pd.to_datetime(transactions['t_dat'])\n    start_ym = pd.to_datetime(pre_start_date) \n    end_ym = pd.to_datetime(pre_end_date)\n    transaction_ym = transactions.query('t_dat_dt >= @start_ym & t_dat_dt < @end_ym').assign(target_customer = 1)\n    return transaction_ym\n\n#購入額sum\ndef sales(transaction_ym):\n    transaction_sum=transaction_ym.groupby([\"customer_id\"])[\"price\"].sum().reset_index()\n    transaction_sum=transaction_sum.rename(columns= {'price':'sales'})\n    return transaction_sum\n\n#購入額\ndef salesag(transaction_ym):\n    transaction_salesag=transaction_ym.groupby([\"customer_id\"])[\"price\"].agg(meansales=(\"mean\"),maxsales=(\"max\"),minsales=(\"min\"),mediansales=(\"median\")).reset_index()\n    return transaction_salesag\n\n#購入回数3カ月\ndef scount(transaction_ym):\n    transaction_count=transaction_ym.groupby([\"customer_id\"])[\"target_customer\"].sum().reset_index()\n    transaction_count=transaction_count.rename(columns= {'target_customer':'count'})\n    return transaction_count\n\n#購入回数5カ月\ndef pcount(pre_end_date):\n    transactions['t_dat_dt'] = pd.to_datetime(transactions['t_dat'])\n    end_ym = pd.to_datetime(pre_end_date)\n    start_ym = end_ym-relativedelta(months=5)\n    transaction_target_ym = transactions.query('t_dat_dt >= @start_ym & t_dat_dt < @end_ym').assign(target_customer = 1)\n    transaction_target_count=transaction_target_ym.groupby([\"customer_id\"])[\"target_customer\"].sum().reset_index()\n    transaction_target_count=transaction_target_count.rename(columns= {'target_customer':'count'})\n    return transaction_target_count\n\n#経過日数\ndef days(transaction_ym,pre_end_date):\n    end_ym = pd.to_datetime(pre_end_date)\n    transaction_ym[\"days\"]=end_ym-transaction_ym[\"t_dat_dt\"]\n    transaction_ym[\"days_int\"] = (transaction_ym[\"days\"] / timedelta(days=1))\n    transaction_days=transaction_ym[['customer_id','days_int']].groupby('customer_id').min().reset_index()\n    return transaction_days\n\n#index_group_name\ndef index(transaction_ym):\n    articles_index=articles.loc[:, [\"article_id\", \"index_group_name\"]]\n    transaction_articles_index = pd.merge(transaction_ym, articles_index, how='left', on='article_id')\n    transaction_dummies_index=pd.get_dummies(transaction_articles_index,columns=['index_group_name'])\n    transaction_dummies_g=transaction_dummies_index.groupby(['customer_id'])[['index_group_name_Baby/Children', 'index_group_name_Divided','index_group_name_Ladieswear','index_group_name_Menswear','index_group_name_Sport']].max().reset_index()\n    return transaction_dummies_g","metadata":{"execution":{"iopub.status.busy":"2023-07-20T17:55:25.893336Z","iopub.execute_input":"2023-07-20T17:55:25.893614Z","iopub.status.idle":"2023-07-20T17:55:25.908060Z","shell.execute_reply.started":"2023-07-20T17:55:25.893591Z","shell.execute_reply":"2023-07-20T17:55:25.907223Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## モデル作成","metadata":{}},{"cell_type":"code","source":"\n@contextmanager\ndef timer(title):\n    t0 = time.time()\n    yield\n    print(\"{} - done in {:.0f}s\".format(title, time.time() - t0))\n\n# ダミー変数の作成(one-hot encording)\ndef one_hot_encoder(df, nan_as_category = True):\n    original_columns = list(df.columns)\n    categorical_columns = [col for col in df.columns if df[col].dtype == 'object']\n    df = pd.get_dummies(df, columns= categorical_columns, dummy_na= nan_as_category)\n    new_columns = [c for c in df.columns if c not in original_columns]\n    return df, new_columns\n\n#入力するデータフレームの欲しい変数\n\n# LightGBMを使用して、KFoldまたはStratified KFoldを用いたGBDT（Gradient Boosting Decision Tree）を実行する。\ndef make_lightgbm_model(df, num_folds, stratified = False, debug= True):\n    # トレーニング/バリデーションデータとテストデータに分割する。\n    train_df = df[df[\"target\"].notnull()]\n    test_df = df[df[\"target\"].isnull()]\n    print(\"Starting LightGBM. Train shape: {}, test shape: {}\".format(train_df.shape, test_df.shape))\n    del df\n    gc.collect()\n    # クロスバリデーション\n    ## stratifiedの値で層化KFoldかKFoldのどちらのモデルを使うか決める。\n    if stratified:\n        folds = StratifiedKFold(n_splits= num_folds, shuffle=True, random_state=1001)\n    else:\n        folds = KFold(n_splits= num_folds, shuffle=True, random_state=1001)\n    # 結果を格納するための配列やデータフレームを作成する。\n    oof_preds = np.zeros(train_df.shape[0])\n    sub_preds = np.zeros(test_df.shape[0])\n    feature_importance_df = pd.DataFrame()\n    tmp_feats = [f for f in train_df.columns if f not in [\"target\",'customer_id','postal_code','index']]\n    categorical_feature = [col for col in tmp_feats if train_df[col].dtype == 'object']\n    feats = [f for f in tmp_feats if f not in categorical_feature]\n    #print(categorical_feature)\n    \n    X_train, X_valid, y_train, y_valid = train_test_split(train_df[feats], train_df[\"target\"], test_size=0.20, random_state=42)\n\n    \n    \n    # ベイズ最適化によって見つかったLightGBMのパラメータ:後ほど決める\n    clf = LGBMClassifier(\n        nthread=4,\n        n_estimators=10000,\n        learning_rate=0.02,\n        num_leaves=34,\n        colsample_bytree=0.9497036,\n        subsample=0.8715623,\n        max_depth=8,\n        reg_alpha=0.041545473,\n        reg_lambda=0.0735294,\n        min_split_gain=0.0222415,\n        min_child_weight=39.3259775,\n        silent=-1,\n        verbose=-1, )\n    \n    verbose_eval=0\n    clf.fit(X_train, y_train, \n            eval_metric='auc',\n            eval_set=[(X_valid, y_valid)],\n            callbacks=[lgb.early_stopping(stopping_rounds=200, \n                        verbose=True), # early_stopping用コールバック関数\n                    lgb.log_evaluation(verbose_eval)] # コマンドライン出力用コールバック関数\n            )\n    \n    #特徴量の重要度を可視化する\n    fold_importance_df = pd.DataFrame()\n    fold_importance_df[\"feature\"] = feats\n    fold_importance_df[\"importance\"] = clf.feature_importances_\n    feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)\n    \n    display_importances(feature_importance_df)\n\n    \n    return clf\n\n# Display/plot feature importance\ndef display_importances(feature_importance_df_):\n    cols = feature_importance_df_[[\"feature\", \"importance\"]].groupby(\"feature\").mean().sort_values(by=\"importance\", ascending=False)[:40].index\n    best_features = feature_importance_df_.loc[feature_importance_df_.feature.isin(cols)]\n    plt.figure(figsize=(8, 10))\n    sns.barplot(x=\"importance\", y=\"feature\", data=best_features.sort_values(by=\"importance\", ascending=False))\n    plt.title('LightGBM Features (avg over folds)')\n    plt.tight_layout()\n    plt.savefig('lgbm_importances01.png')\n\n\n","metadata":{"execution":{"iopub.status.busy":"2023-07-20T17:55:25.909696Z","iopub.execute_input":"2023-07-20T17:55:25.910160Z","iopub.status.idle":"2023-07-20T17:55:25.928383Z","shell.execute_reply.started":"2023-07-20T17:55:25.910134Z","shell.execute_reply":"2023-07-20T17:55:25.927505Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":" ","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def predict_validation_and_calculate_AUC(model=None,valid_df=None):\n    \n        \n    tmp_feats = [f for f in valid_df.columns if f not in [\"target\",'customer_id','postal_code','index']]\n    categorical_feature = [col for col in tmp_feats if valid_df[col].dtype == 'object']\n    feats = [f for f in tmp_feats if f not in categorical_feature]\n    \n        \n    valid_X = valid_df[feats]\n    valid_y = valid_df[\"target\"]\n    \n    val_preds_proba = model.predict_proba(valid_X, num_iteration=model.best_iteration_)[:, 1]\n    \n    \n    #検証データをモデルで推論した場合のAUCのスコア\n    val_AUC_score = roc_auc_score(valid_y, val_preds_proba)\n    print(f'[accuracy]検証データ:{val_AUC_score:.6f}')\n    \n   \n    threshold = 0.5 #閾値を設定\n    val_preds = (val_preds_proba >= threshold).astype(int)\n    \n    #混同行列\n    cm = confusion_matrix(valid_y, val_preds,labels=[1, 0])\n    sns.heatmap(cm, square=True, cbar=True, annot=True, cmap='Blues')\n    plt.xlabel(\"Pre\", fontsize=13)\n    plt.ylabel(\"GT\", fontsize=13)\n    \n\n    cal_impact_df = valid_df.copy()\n    cal_impact_df[\"確率\"]=val_preds_proba\n    \n    return cal_impact_df\n    \n    #その他、F値などを計算して戻り値で返しても良い。","metadata":{"execution":{"iopub.status.busy":"2023-07-20T17:55:25.929363Z","iopub.execute_input":"2023-07-20T17:55:25.930048Z","iopub.status.idle":"2023-07-20T17:55:25.944441Z","shell.execute_reply.started":"2023-07-20T17:55:25.930024Z","shell.execute_reply":"2023-07-20T17:55:25.943854Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## インパクト算出","metadata":{}},{"cell_type":"code","source":"def calculate_impact(valid_df,threshold_start=0.2,threshold_end=1.0,discount_rate=0.1):\n    \n    #結果をまとめるリストの作成\n    ans_list=[]\n    for i in np.arange(threshold_start,threshold_end,0.1):\n        threshold=i#閾値を設定\n        tmp_valid_df=valid_df.copy()\n        tmp_valid_df[\"pred\"] = (tmp_valid_df[\"確率\"] >= threshold).astype(int)\n        tmp_valid_df=tmp_valid_df.sort_values(\"確率\",ascending=False)\n        \n        ##precision,recallを計算する\n        precision=precision_score(tmp_valid_df[\"target\"],tmp_valid_df[\"pred\"])\n        recall=recall_score(tmp_valid_df[\"target\"],tmp_valid_df[\"pred\"])\n        \n        #施策対象者合計数\n        campaign_targets_count = tmp_valid_df[tmp_valid_df[\"pred\"] == 1].shape[0]\n        #実際の休眠顧客数\n        dormant_customer_count = tmp_valid_df[(tmp_valid_df[\"pred\"] == 1) & (tmp_valid_df[\"target\"] ==1)].shape[0]\n        \n        #送付対象者の中で実際に休眠顧客の人(A)\n        ##送付対象者の中の実際の休眠顧客の期待平均単価\n        expected_unit_price_A = tmp_valid_df[(tmp_valid_df[\"pred\"] == 1) & (tmp_valid_df[\"target\"] == 1)][\"sales\"].mean()\n        \n        ##送付対象者の中で実際の休眠顧客の人から期待できる売上の合計\n        ###ただし、送付対象者の中で実際の休眠顧客の中で施策に反応する人の割合をreactionで表す(今回は0.01)\n        reaction=0.05\n        sales_predict = (dormant_customer_count*reaction)*expected_unit_price_A\n        \n        #送付対象者への施策のコストを計算する(A+C)\n        \n        ##1人辺りのコストはクーポンで割引かれる割引率、cost_per_unit%(基準は10%)で決まる\n        cost_per_unit=discount_rate\n        \n        ##送付対象者の中で実際の休眠顧客の人からの予測されるコスト(Aのコスト:(施策に反応して実際に買いに来てくれた人)*(買いに来た人のクーポンで割引きされた値段))\n        cost_a = (dormant_customer_count*reaction)*(expected_unit_price_A*cost_per_unit)\n        \n        ##送付対象者の中で実際は休眠顧客でなかった人からの予測されるコスト(Cのコスト:(休眠顧客でなく買いに来てくれた人)*(買いに来た人のクーポンで割引きされた値段))\n        ###送付対象者の中で実際は休眠顧客でなかった人の期待平均単価\n        expected_unit_price_C = tmp_valid_df[(tmp_valid_df[\"pred\"] == 1) & (tmp_valid_df[\"target\"] == 0)][\"sales\"].mean()\n        cost_c = (campaign_targets_count-dormant_customer_count)*(expected_unit_price_C*cost_per_unit)\n        \n        #コストの合計\n        cost = cost_a + cost_c\n        \n        #期待できるインパクトを計算する(売上-コスト)\n        impact = sales_predict - cost\n        \n        #計算結果をまとめる\n        ans_list.append([threshold,precision,recall,campaign_targets_count,dormant_customer_count,expected_unit_price_A,\n                    sales_predict,discount_rate,cost,impact])\n        \n        del precision, recall, campaign_targets_count, dormant_customer_count, expected_unit_price_A, expected_unit_price_C, sales_predict, cost_per_unit, cost, cost_a, cost_c, impact, reaction, threshold\n        \n    df = pd.DataFrame(ans_list,columns =['閾値','precision',\"recall\",\"施策対象者数合計\",\"実際の休眠顧客数\",\"休眠顧客の期待平均単価\",\n                                        \"期待売上合計\",\"1件辺りのコスト(百分率)\",\"コスト合計\",\"期待インパクト\"])\n    #max_impact = df[\"期待インパクト\"].max()\n    #print(\"最大のインパクトになるのは、以下のレコードの時である\")\n    #df.query(\"期待インパクト == @max_impact\").head()\n    return df","metadata":{"execution":{"iopub.status.busy":"2023-07-20T17:55:25.947152Z","iopub.execute_input":"2023-07-20T17:55:25.947905Z","iopub.status.idle":"2023-07-20T17:55:25.962050Z","shell.execute_reply.started":"2023-07-20T17:55:25.947881Z","shell.execute_reply":"2023-07-20T17:55:25.960890Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 実行関数","metadata":{}},{"cell_type":"code","source":"## データ前処理の部分を関数化する?","metadata":{"execution":{"iopub.status.busy":"2023-07-20T17:55:25.963142Z","iopub.execute_input":"2023-07-20T17:55:25.963424Z","iopub.status.idle":"2023-07-20T17:55:25.976082Z","shell.execute_reply.started":"2023-07-20T17:55:25.963393Z","shell.execute_reply":"2023-07-20T17:55:25.975374Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def main():\n    #モデルを学習するためのデータ前処理\n    ##Baseとなる顧客IDにleftjoinしていく\n    target_customers = create_target_customers(transactions,\n                                               pre_start_date,\n                                               pre_end_date)\n    print(\"target_customers:\", target_customers.shape)\n    \n    ##目的変数と顧客IDの存在するデータフレームを作成する\n    target_variables = create_target_variable(transactions,\n                                             target_customers,\n                                             pos_start_date,\n                                             pos_end_date)\n    print(\"target_variables:\",target_variables.shape)\n    \n    ##目的変数の存在するデータフレームをBaseとなる顧客IDにleft joinする\n    target_customers = pd.merge(target_customers, target_variables, how='left', on='customer_id')\n    del target_variables\n    gc.collect()\n    \n    ##customersの情報を付与する\n    customers_feature = create_customers_feature(customers=customers)\n    print(\"customers_feature:\", customers_feature.shape)\n    target_customers = pd.merge(target_customers, customers_feature, how='left', on='customer_id')\n    del customers_feature\n    gc.collect()\n    \n    ##特徴量をleft joinする\n    print(\"target_customers:\", target_customers.shape)\n    transaction_ym = ymset(pre_start_date,pre_end_date)\n    \n    ##購入額合計\n    transaction_sum = sales(transaction_ym)\n    target_customers = pd.merge(target_customers, transaction_sum, how='left', on='customer_id')\n    print(\"transaction_sum:\", transaction_sum.shape)\n    del transaction_sum\n    gc.collect()\n    ##購入額\n    transaction_salesag = salesag(transaction_ym)\n    target_customers = pd.merge(target_customers, transaction_salesag, how='left', on='customer_id')\n    print(\"transaction_salesag:\", transaction_salesag.shape)\n    del transaction_salesag\n    gc.collect()       \n    ##購入回数3カ月\n    #transaction_count = scount(transaction_ym)\n    #target_customers = pd.merge(target_customers, transaction_count, how='left', on='customer_id')\n    #print(\"transaction_count:\", transaction_count.shape)\n    #del transaction_count\n    #gc.collect()\n    ##購入回数5カ月\n    transaction_count = pcount(pre_end_date)\n    target_customers = pd.merge(target_customers, transaction_count, how='left', on='customer_id')\n    print(\"target_customers:\", target_customers.shape)\n    del transaction_count\n    gc.collect()  \n    ##経過日数\n    transaction_days = days(transaction_ym,pre_end_date)\n    target_customers = pd.merge(target_customers, transaction_days, how='left', on='customer_id')\n    print(\"transaction_days:\", transaction_days.shape)\n    del transaction_days\n    gc.collect()\n    ##index_group_name\n    transaction_dummies_g = index(transaction_ym)\n    target_customers = pd.merge(target_customers, transaction_dummies_g, how='left', on='customer_id')\n    print(\"transaction_dummies_g:\", transaction_dummies_g.shape)\n    del transaction_dummies_g\n    gc.collect()\n    \n    print(\"target_customers:\", target_customers.shape)\n    del transaction_ym\n        \n    #モデルを作成する\n    ##数値型やbool型の変数以外をダミー変数化する\n    #target_customers=one_hot_encoder(df=target_customers, nan_as_category = True)\n    with timer(\"Run LightGBM with HoldOut\"):\n        model = make_lightgbm_model(df=target_customers,num_folds=3,stratified=False,debug=True)\n    #feature_importanceを可視化する\n    #display_importances(feat_importance)\n    \n    \n    \n    \n    \n    \n    #上で作成したモデルの精度検証をするためのデータ前処理\n    ##Baseとなる顧客IDにleftjoinしていく\n    val_target_customers = create_target_customers(transactions,\n                                               val_pre_start_date,\n                                               val_pre_end_date)\n    print(\"val_target_customers:\", val_target_customers.shape)\n    \n    ##目的変数と顧客IDの存在するデータフレームを作成する\n    val_target_variables = create_target_variable(transactions,\n                                             val_target_customers,\n                                             val_pos_start_date,\n                                             val_pos_end_date)\n    print(\"val_target_variables:\",val_target_variables.shape)\n    \n    ##目的変数の存在するデータフレームをBaseとなる顧客IDにleft joinする\n    val_target_customers = pd.merge(val_target_customers, val_target_variables, how='left', on='customer_id')\n    del val_target_variables\n    gc.collect()\n    \n    ##customersの情報を付与する\n    val_customers_feature = create_customers_feature(customers=customers)\n    print(\"val_customers_feature:\", val_customers_feature.shape)\n    val_target_customers = pd.merge(val_target_customers, val_customers_feature, how='left', on='customer_id')\n    del val_customers_feature\n    gc.collect()\n    \n    ##特徴量をleft joinする\n    print(\"val_target_customers:\", val_target_customers.shape)\n    val_transaction_ym = ymset(val_pre_start_date,val_pre_end_date)\n    \n    ##購入額合計\n    val_transaction_sum = sales(val_transaction_ym)\n    val_target_customers = pd.merge(val_target_customers, val_transaction_sum, how='left', on='customer_id')\n    print(\"val_transaction_sum:\", val_transaction_sum.shape)\n    del val_transaction_sum\n    gc.collect()\n    ##購入額\n    val_transaction_salesag = salesag(val_transaction_ym)\n    val_target_customers = pd.merge(val_target_customers, val_transaction_salesag, how='left', on='customer_id')\n    print(\"val_transaction_salesag:\", val_transaction_salesag.shape)\n    del val_transaction_salesag\n    gc.collect()       \n    ##購入回数5カ月\n    val_transaction_count = pcount(val_pre_end_date)\n    val_target_customers = pd.merge(val_target_customers, val_transaction_count, how='left', on='customer_id')\n    print(\"val_target_customers:\", val_target_customers.shape)\n    del val_transaction_count\n    gc.collect()  \n    ##経過日数\n    val_transaction_days = days(val_transaction_ym,val_pre_end_date)\n    val_target_customers = pd.merge(val_target_customers, val_transaction_days, how='left', on='customer_id')\n    print(\"val_transaction_days:\", val_transaction_days.shape)\n    del val_transaction_days\n    gc.collect()\n    ##index_group_name\n    val_transaction_dummies_g = index(val_transaction_ym)\n    val_target_customers = pd.merge(val_target_customers, val_transaction_dummies_g, how='left', on='customer_id')\n    print(\"val_transaction_dummies_g:\", val_transaction_dummies_g.shape)\n    del val_transaction_dummies_g\n    gc.collect()\n    \n    print(\"val_target_customers:\", val_target_customers.shape)\n    del val_transaction_ym\n    \n    \n    \n    \n    #validationデータをモデルで予測して精度を計算する\n    #特にAUCの値と近藤行列の可視化を行う\n    with timer(\"Estimate and calculate\"):\n        cal_impact_df=predict_validation_and_calculate_AUC(model=model,valid_df=val_target_customers)\n    \n    \n    #インパクトを算出する\n    impact_df=calculate_impact(valid_df=cal_impact_df,threshold_start=0.2,threshold_end=1.0,discount_rate=0.10)\n\n    return impact_df","metadata":{"execution":{"iopub.status.busy":"2023-07-20T17:55:25.977288Z","iopub.execute_input":"2023-07-20T17:55:25.977760Z","iopub.status.idle":"2023-07-20T17:55:26.002042Z","shell.execute_reply.started":"2023-07-20T17:55:25.977735Z","shell.execute_reply":"2023-07-20T17:55:26.001207Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 実行結果","metadata":{}},{"cell_type":"code","source":"sample=main()\nsample.head()","metadata":{"execution":{"iopub.status.busy":"2023-07-20T17:55:26.003029Z","iopub.execute_input":"2023-07-20T17:55:26.003345Z","iopub.status.idle":"2023-07-20T17:59:22.049590Z","shell.execute_reply.started":"2023-07-20T17:55:26.003316Z","shell.execute_reply":"2023-07-20T17:59:22.048592Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample","metadata":{"execution":{"iopub.status.busy":"2023-07-20T17:59:22.051304Z","iopub.execute_input":"2023-07-20T17:59:22.053471Z","iopub.status.idle":"2023-07-20T17:59:22.065057Z","shell.execute_reply.started":"2023-07-20T17:59:22.053444Z","shell.execute_reply":"2023-07-20T17:59:22.064419Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tmp2=sample[[\"閾値\",\"施策対象者数合計\",\"実際の休眠顧客数\",\"実際の休眠顧客数\",\"休眠顧客の期待平均単価\",\"期待売上合計\",\"1件辺りのコスト(百分率)\",\"コスト合計\",\"期待インパクト\"]]\ntmp2.set_index(\"閾値\")","metadata":{"execution":{"iopub.status.busy":"2023-07-20T18:03:02.239516Z","iopub.execute_input":"2023-07-20T18:03:02.240067Z","iopub.status.idle":"2023-07-20T18:03:02.259922Z","shell.execute_reply.started":"2023-07-20T18:03:02.240014Z","shell.execute_reply":"2023-07-20T18:03:02.258695Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample[[\"閾値\",\"期待売上合計\",\"コスト合計\",\"期待インパクト\"]]","metadata":{"execution":{"iopub.status.busy":"2023-07-20T17:59:22.065943Z","iopub.execute_input":"2023-07-20T17:59:22.066605Z","iopub.status.idle":"2023-07-20T17:59:22.087314Z","shell.execute_reply.started":"2023-07-20T17:59:22.066580Z","shell.execute_reply":"2023-07-20T17:59:22.086485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nfig, ax = plt.subplots()\n\nax.plot(sample[\"閾値\"], sample[\"期待売上合計\"], label = 'profit')\nax.plot(sample[\"閾値\"], sample[\"コスト合計\"], label = \"cost\")\nax.plot(sample[\"閾値\"], sample[\"期待インパクト\"], label = 'impact')\n\nax.set_xlabel('threshold', fontsize = 18, fontname = 'MS Gothic')\nax.set_ylabel('amount', fontsize = 18, fontname = 'MS Gothic')\n\n#ax.set_title('Expected value', fontsize = 18, fontname = 'MS Gothic')\n\n#ax.text(0, -0.8, 'テキスト',  fontsize = 14, fontname = 'MS Gothic')\n\nax.legend(prop = {'family' : 'MS Gothic'})\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-07-20T17:59:22.088357Z","iopub.execute_input":"2023-07-20T17:59:22.088611Z","iopub.status.idle":"2023-07-20T17:59:22.418374Z","shell.execute_reply.started":"2023-07-20T17:59:22.088589Z","shell.execute_reply":"2023-07-20T17:59:22.417751Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}