{"cells":[{"metadata":{"trusted":true,"_uuid":"b7dfa144d205cf05efc022d713edacdf098fb742"},"cell_type":"code","source":"#New to Kaggle and this is my first public kernel. Hope you find it interesting or helpful! This kernel provides visualizations on an individual card's journey. You can fork this to explore the training set in search of patterns.\n\n#The final cell provides the only output, a plot of every transaction made by a specified card. The columns on the left are subsector_id, merchant_category_id, and merchant_id, respectively. Each point is a transaction and if you hover over one in an interactive notebook (doesn't work in the kernel output, you'll have to fork to see) you can see the transaction details.\n\n#To use the forked notebook, edit the last cell. You can call the function on any card or row number in the train set.\n\n# Change the below to true before running this kernel. Then run the kernel end-to-end. Then you can pick cards to explore in the last cell, just rerunning the last cell\ninteractive = False","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nif interactive:\n    %matplotlib notebook\nimport matplotlib as mpl\nmpl.rcParams['figure.dpi'] = 60\n\nimport matplotlib.pyplot as plt\nfrom matplotlib import patches\n\nimport datetime\nfrom dateutil.relativedelta import relativedelta\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/train.csv')\ntest = pd.read_csv('../input/test.csv')\nmerchants = pd.read_csv('../input/merchants.csv')\ntrans = pd.read_csv('../input/historical_transactions.csv')\nnew_trans = pd.read_csv('../input/new_merchant_transactions.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"_uuid":"615c3102b60db46a4168734c75b501d692f09ce8"},"cell_type":"code","source":"def get_journey_data(cx_id):\n    # select customer\n    if type(cx_id) is int:\n        cx = train.iloc[cx_id]\n        cx_id = cx['card_id']\n    else:\n        cx = train[train['card_id'] == cx_id].iloc[0]\n        \n    # select transactions\n    cx_hist_trans = trans[trans['card_id']==cx_id]\n    cx_new_trans = new_trans[new_trans['card_id']==cx_id]\n\n    # designate transaction source and merge transaction lists\n    cx_hist_trans = cx_hist_trans.assign(Source='Historical')\n    cx_new_trans = cx_new_trans.assign(Source='New')\n    cx_trans = pd.concat((cx_hist_trans, cx_new_trans), axis=0).reset_index()\n\n    # decompose purchase date\n    trans_date = pd.DataFrame(index=cx_trans.index)\n    trans_date['Timestamp'] = pd.to_datetime(cx_trans['purchase_date'], format='%Y-%m-%d %H:%M:%S')\n    dates, times, years, months, days = zip(*[(d.date(), d.time(), d.year, d.month, d.day) for d in trans_date['Timestamp']])\n    trans_date = trans_date.assign(Date=dates, Time=times, Year=years, Month=months, Day=days)\n    cx_trans = pd.concat((cx_trans, trans_date), axis=1)\n    \n    return cx, clean_trans(cx_trans)\n\ndef clean_trans(trans):\n    trans['merchant_id'] = trans['merchant_id'].where(~trans['merchant_id'].isna(), 'Missing')\n    trans['merchant_category_id'] = trans['merchant_category_id'].where(~trans['merchant_category_id'].isna(), 'Missing')\n    trans['subsector_id'] = trans['subsector_id'].where(~trans['subsector_id'].isna(), 'Missing')\n    return trans","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5bb06929d921b056114d9c8fb97386cc726081bb"},"cell_type":"code","source":"def print_head(cx, cx_trans):\n    print('{:30}'.format('Card:'), cx['card_id'])\n    print('{:30}'.format('Active Date:'), cx['first_active_month'])\n    print('{:30}'.format('Target:'), cx['target'])\n    print('')\n    print('{:30}'.format('# of Transactions:'), len(cx_trans))\n    print('{:30}'.format('# of New Merchants:'), sum(cx_trans['Source']=='New'))\n    print('{:30}'.format('% Authorized:'), str(int(10000*sum(cx_trans['authorized_flag']=='Y')/len(cx_trans))/100.) + '%')\n    print('{:30}'.format('Total Spend:'), sum(cx_trans['purchase_amount']))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"_uuid":"ad17cc76e49fcd8c72e0fd68f3974cf307a7b721"},"cell_type":"code","source":"#outstanding:\n# 1. hover annotate\n# 2. spread/swarm nearby points\n\nmerchant_width = 40\ngroup_width = 15\nkey_date_offset = 0.8\n\ninteractive = False\n\npoints_with_annotation = []\n\ndef plot_journey(cx, trans):    \n    date_range = get_date_range(trans)\n    merchant_info = analyze_merchants(trans)\n    \n    fig, axes = create_figure(merchant_info, date_range, x_offset=-(merchant_width+2*group_width), y_offset=-key_date_offset)\n    \n    draw_swimlanes(merchant_info, title_size=-merchant_width)\n    draw_groups(axes, merchant_info, title_size=group_width, x_offset=-merchant_width)\n    \n    draw_start_line()\n    draw_first_active_month(cx, date_range['min'], -key_date_offset)\n    draw_lag_ref_month(trans, date_range['min'], -key_date_offset)\n    \n    draw_transactions(axes, trans, merchant_info, date_range['min'])\n    \n    fig.canvas.mpl_connect(\"motion_notify_event\", on_move)\n    \ndef draw_lag_ref_month(trans, ref_date, y_offset, pad=0.3, c=\"#666666\", a=0.75):\n    dates = [(x['Date'] + relativedelta(months=-x['month_lag'])).replace(day=1) for _, x in trans.iterrows()]\n    date = last_day_of_month(max(set(dates), key=dates.count))\n    \n    x = (date-ref_date).days + 1\n    plt.annotate('Lag Reference', (x, y_offset+pad), verticalalignment='top', horizontalalignment='center')\n    plt.axvline(x, color=c, alpha=a)\n    \ndef draw_first_active_month(cx, ref_date, y_offset, pad=0.3, c=\"#eeeeee\", a=0.75):\n    active_date = datetime.datetime.strptime(cx['first_active_month'], '%Y-%m')\n    active_date_end = last_day_of_month(active_date)\n    active_start = (active_date.date()-ref_date).days\n    active_end = (active_date_end.date()-ref_date).days\n    \n    if active_end > 0:\n        if active_start < 0:\n            active_start = 0\n        plt.annotate('First Active Month', ((active_start+active_end)/2, y_offset+pad), verticalalignment='top', horizontalalignment='center')\n        plt.axvspan(active_start, active_end, color=c, alpha=a)\n    \ndef draw_start_line(c=\"#eeeeee\", a=0.75):\n    plt.axvline(0, color=c, alpha=a)\n\ndef draw_transactions(ax, trans, info, ref_date, hist_c='b', new_c='r', size=8, tooltip_offset=0.1):\n    for _, t in trans.iterrows():\n        x = (t['Date'] - ref_date).days\n        \n        y = info['merchant_pos'][stringify_merchant(t['subsector_id'], t['merchant_category_id'], t['merchant_id'])]\n        \n        c = hist_c\n        if t['Source'] == 'New':\n            c = new_c\n            \n        point, = ax.plot(x, y, 'o', markersize=size, color=c)\n        \n        y_step = 0\n        if y < 3:\n            y_step = 3\n        annotation = ax.annotate(get_annotation(t),\n        xy=(x, y), xycoords='data',\n        xytext=(x + tooltip_offset, y + tooltip_offset + y_step), textcoords='data',\n            horizontalalignment=\"right\",\n            bbox=dict(boxstyle=\"round\", facecolor=\"w\", edgecolor=\"0.5\", alpha=0.9)\n        )\n        annotation.set_visible(False)\n        points_with_annotation.append([point, annotation])\n\ndef draw_swimlanes(info, title_size, pad=3, c='#eeefff'):\n    i = 0\n    for key in info['merchant_pos']:\n        i = i+1\n        pos = info['merchant_pos'][key]\n        size = info['merchant_size'][key]\n        title, _, _ = de_stringify_merchant(key)\n        \n        if i % 2 == 0:\n            plt.axhspan(pos-size/2, pos+size/2, color=c)\n        plt.annotate(title, (title_size+pad, pos), verticalalignment='center')\n\ndef create_figure(info, date_range, x_offset, y_offset):\n    if interactive:\n        %matplotlib notebook\n    f = plt.figure()\n    ax = plt.axes()\n    \n    # dimensions\n    height = sum(info['merchant_size'].values())\n    width = (date_range['max'] - date_range['min']).days\n    \n    # chart size\n    f.set_size_inches(23, 0.5*(height - y_offset))\n    f.tight_layout(pad=2)\n    \n    # y axis\n    ax.set_ylim(y_offset, height)\n    ax.invert_yaxis()\n    ax.set_yticks([])\n    \n    # x axis\n    ax.xaxis.tick_top()\n    ax.set_xlim(x_offset, width)\n    x_ticks = get_ticks(date_range)\n    ax.set_xticks(x_ticks['Ticks'])\n    ax.set_xticklabels(x_ticks['Labels'])\n    \n    return f, ax\n\ndef last_day_of_month(date):\n    if date.month == 12:\n        return date.replace(day=31)\n    return date.replace(month=date.month+1, day=1) - datetime.timedelta(days=1)\n\ndef get_date_range(trans):\n    return {'min': min(trans['Date']), 'max': max(trans['Date'])}\n\ndef get_ticks(date_range):\n    dates = [date_range['min'].replace(day=15)]\n    while dates[-1] < date_range['max']:\n        dates.append((dates[-1] + relativedelta(months=1)).replace(day=15))\n    if date_range['min'].day > 15:\n        del dates[0]\n        \n    ticks = [(d - date_range['min']).days for d in dates]\n    labels = [d.strftime('%b') + ' ' + str(d.year) for d in dates]\n    return pd.DataFrame({'Ticks': ticks, 'Labels': labels})\n            \ndef analyze_merchants(trans):\n    groups = {}\n    merchant_pos = {}\n    merchant_size = {}\n    \n    subsectors = get_sorted_merchant_group(trans, 'subsector_id')\n    for subsector in subsectors:\n        groups[subsector] = {}\n        sector_trans = trans[trans['subsector_id']==subsector]\n        merchant_categories = get_sorted_merchant_group(sector_trans, 'merchant_category_id')\n        \n        for merchant_category in merchant_categories:\n            groups[subsector][merchant_category] = []\n            category_trans = sector_trans[sector_trans['merchant_category_id']==merchant_category]\n            merchants = get_sorted_merchant_group(category_trans, 'merchant_id')\n            \n            for merchant in merchants:\n                groups[subsector][merchant_category].append(merchant)\n                merchant_trans = category_trans[category_trans['merchant_id']==merchant]\n                \n                merch_str = stringify_merchant(subsector, merchant_category, merchant)\n                merchant_size[merch_str] = calc_merchant_size(merchant_trans)\n                merchant_pos[merch_str] = sum(merchant_size.values()) - merchant_size[merch_str]/2.\n    \n    return {'groups': groups, 'merchant_pos': merchant_pos, 'merchant_size': merchant_size}\n    \ndef calc_merchant_size(trans):\n    return 1.\n\ndef stringify_merchant(subsector, category, merchant):\n    return str(subsector) + '.' + str(category) + '.' + str(merchant)\n\ndef de_stringify_merchant(merchant_string):\n    subsector, category, merchant = merchant_string.split('.')\n    return merchant, category, subsector\n            \ndef get_sorted_merchant_group(cx_trans, group):\n    return list(cx_trans[group].value_counts().index)\n\ndef draw_groups(ax, info, title_size, x_offset, x_pad=1, y_pad=0.1, sub_fill='#aaafff', cat_fill='#cccfff'):\n    plt.axvspan(x_offset-2*title_size, x_offset-x_pad, color='#ffffff')\n    \n    keys = np.array([de_stringify_merchant(x) for x in info['merchant_pos'].keys()])\n    subsectors = np.unique(keys[:, 2])\n    for subsector in subsectors:\n        sub_keys = (keys[:, 2] == subsector)\n        sub_pos = np.array(list(info['merchant_pos'].values()))[sub_keys]\n        sub_size = np.array(list(info['merchant_size'].values()))[sub_keys]\n        \n        x = x_offset - 2 * title_size + x_pad\n        y = sub_pos[0] - sub_size[0]/2 + y_pad\n        width = title_size - x_pad\n        height = sum(sub_size) - y_pad\n        \n        ax.add_patch(patches.Rectangle((x ,y), width, height, color=sub_fill))\n        plt.annotate(subsector, (x+width/2, y+height/2), verticalalignment='center', horizontalalignment='center')\n        \n        categories = np.unique(keys[sub_keys, 1])\n        for category in categories:\n            cat_keys = (keys[:, 1] == category) & sub_keys\n            cat_pos = np.array(list(info['merchant_pos'].values()))[cat_keys]\n            cat_size = np.array(list(info['merchant_size'].values()))[cat_keys]     \n        \n            x = x_offset - title_size + x_pad\n            y = cat_pos[0] - cat_size[0]/2 + y_pad\n            width = title_size - x_pad\n            height = sum(cat_size) - y_pad\n\n            ax.add_patch(patches.Rectangle((x ,y), width, height, color=cat_fill))\n            plt.annotate(category, (x+width/2, y+height/2), verticalalignment='center', horizontalalignment='center')    \n            \ndef on_move(event):\n    visibility_changed = False\n    for point, annotation in points_with_annotation:\n        should_be_visible = (point.contains(event)[0] == True)\n\n        if should_be_visible != annotation.get_visible():\n            visibility_changed = True\n            annotation.set_visible(should_be_visible)\n\n    if visibility_changed:        \n        plt.draw()\n        \ndef get_annotation(transaction):\n    string = ''\n    cats = {\n        'purchase_date': 'Date',\n        'purchase_amount': 'Purchase Amount',\n        'installments': 'Installments',\n        'authorized_flag': 'Approved',\n        'city_id': 'City',\n        'state_id': 'State',\n        'category_1': 'Category 1',\n        'category_2': 'Category 2',\n        'category_3': 'Category 3',\n    }\n    for c in cats:\n        string += c + ': ' + str(transaction[c]) + '\\n'\n    \n    return string[:-1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"63ad3bca084c70be5c13768b7be81e5d40314106"},"cell_type":"code","source":"def get_journey(customer_id):\n    cx, cx_trans = get_journey_data(customer_id)\n    print_head(cx, cx_trans)\n    plot_journey(cx, cx_trans)  ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5bcbdf85784092c0f8f5e9c5e984675a5d54af64"},"cell_type":"code","source":"# this is just a placeholder so I can submit this kernel\ndf = pd.DataFrame({'card_id': test['card_id'], 'target': np.zeros(test['card_id'].shape)})\ndf.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"_uuid":"acd7d5d903c807d3a1f016bc78580cd174f6962f"},"cell_type":"code","source":"# Enter the card_id you would like to explore or an integer to select cards by row\n#get_journey(2) EXAMPLE\n#get_journey('C_ID_d639edf6cd') EXAMPLE\n\n# you can edit this\nget_journey(1)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}