{"cells":[{"metadata":{"collapsed":true,"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":false},"cell_type":"code","source":"import pickle, csv, os\nimport numpy as np\nfrom tqdm import tqdm_notebook, tqdm\nimport matplotlib.pyplot as plt\nfrom collections import defaultdict\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":false,"collapsed":true},"cell_type":"code","source":"print(os.listdir('../input/'))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"d130571c-fc31-4760-a636-c7b75733929e","_uuid":"aa6f00d07991b8727d0111532ffa3b375d34b971","trusted":false,"collapsed":true},"cell_type":"code","source":"row_count = 184903890\npos_count = 0\nneg_count = 0\n\nips = defaultdict(int)\napps = defaultdict(int)\ndevs = defaultdict(int)\noss = defaultdict(int)\nchannels = defaultdict(int)\n\nips_attributed = defaultdict(int)\napps_attributed = defaultdict(int)\ndevs_attributed = defaultdict(int)\noss_attributed = defaultdict(int)\nchannels_attributed = defaultdict(int)\n\nwith open('../input/train.csv') as f:\n    fcsv = csv.reader(f)\n    header = True\n    for row in tqdm_notebook(fcsv, unit='rows', total=184903890+1):\n        # Skip the header row\n        if header:\n            header = False\n            continue\n        # Count positive and negative rows\n        if row[-1] == '0':\n            neg_count += 1\n        elif row[-1] == '1':\n            pos_count += 1\n            \n        # Maintains count for each ip, os, device etc.        \n        ip, app, dev, os, channel, ct, at, is_at = row\n        ips[ip] += 1\n        apps[app] += 1\n        devs[dev] += 1\n        channels[channel] += 1\n        oss[os] += 1\n        \n        # Also store the count how many times each ip, os... appear in\n        # a positive sample\n        if is_at == '1':\n            ips_attributed[ip] += 1\n            apps_attributed[app] += 1\n            devs_attributed[dev] += 1\n            channels_attributed[channel] += 1\n            oss_attributed[os] += 1","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"b823c0be-63d7-47b5-98e0-bc34d111e82f","_uuid":"b23fe3f9dba21fb7c1ed23a69122a4f988b358f1","trusted":false,"collapsed":true},"cell_type":"code","source":"print(pos_count)\nprint(neg_count)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"121fa28f-3b13-476b-8586-48cd4c5bff2d","_uuid":"6cd48b10d026f67e3ff03a0b9223e4bc6be17787","trusted":false,"collapsed":true},"cell_type":"code","source":"print('There were {} ips, {} devices, {} apps, {} channels and {} oss'.format(\n    len(ips),\n    len(devs),\n    len(apps),\n    len(channels),\n    len(oss),\n))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"890f916b-40ba-4464-af7f-bb93dff05aba","_uuid":"b420efa0aff884ea031efb6343b5ed8b5af25c44","trusted":false,"collapsed":true},"cell_type":"code","source":"fallback_prob = 456846/184447044\nprint('Fallback Prob.: ', fallback_prob)","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"_cell_guid":"84972c87-d20c-406b-b9b0-24e62f65d8da","_uuid":"10f0ba8880582919a171c5d758befec1c36997d0","trusted":false},"cell_type":"code","source":"ips_prob = defaultdict(lambda: fallback_prob, {k: ips_attributed[k]/ips[k] for k in ips.keys()})\ndevs_prob = defaultdict(lambda: fallback_prob, {k: devs_attributed[k]/devs[k] for k in devs.keys()})\nchannels_prob = defaultdict(lambda: fallback_prob, {k: channels_attributed[k]/channels[k] for k in channels.keys()})\napps_prob = defaultdict(lambda: fallback_prob, {k: apps_attributed[k]/apps[k] for k in apps.keys()})\noss_prob = defaultdict(lambda: fallback_prob, {k: oss_attributed[k]/oss[k] for k in oss.keys()})","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"d2816d25-a629-415b-b133-3d389907e555","_uuid":"45cc396803743803c23b01d052d708c5dcc04551","trusted":false,"collapsed":true},"cell_type":"code","source":"probs = np.zeros((184903890,1))\ny_true = np.zeros((184903890,1), dtype=np.uint8)\n\ni = 0\nwith open('../input/train.csv') as f:\n    fcsv = csv.reader(f)\n    header = True\n    for row in tqdm_notebook(fcsv, unit='rows', unit_divisor=1000, total=(184903890+1)):\n        if header:\n            header = False\n            continue\n            # collected_rows.append(row)\n        ip, app, dev, os, channel, ct, at, is_at = row\n        probs[i] = ips_prob[ip]*apps_prob[app]*devs_prob[dev]*oss_prob[os]*channels_prob[channel]\n        y_true[i] = np.uint8(is_at)\n        i += 1","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"_cell_guid":"c6e338b5-b6a2-464d-8edf-fc6cc8128524","_uuid":"f1cb317fd6cd671855dc3a40adc003860fb766a0","trusted":false},"cell_type":"code","source":"pos_probs = probs[np.where(y_true==1)]\nneg_probs = probs[np.where(y_true==0)]","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"_cell_guid":"c070480f-02cf-4ce9-9632-ffeaece363d0","_uuid":"c487974537b7ac6243908e355cec42df53692786","trusted":false},"cell_type":"code","source":"safety_offset = 1e-19","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"29d6cc96-5b4b-4c9b-b5ef-7f685a6c3566","_uuid":"027b74e47201a0848365364eccb674343c153664"},"cell_type":"markdown","source":"### Let's see how the modeled probabilities are distributed for (non/)attributed samples"},{"metadata":{"collapsed":true,"_cell_guid":"e23a1428-2e90-46db-a664-6df46ead2318","_uuid":"d27cd53d3d03008f536e4ae3cc9e0b314936d1f9","trusted":false},"cell_type":"code","source":"def plot_hist(start = 0, bins=100):\n    density_neg, bins, _ = plt.hist(np.log10(safety_offset + neg_probs[start:(start + 10000)]), bins=bins, normed=True, alpha=1.0)\n    density_pos, bins, _ = plt.hist(np.log10(safety_offset + pos_probs[start:(start + 10000)]), bins=bins, normed=True, alpha=0.8)\n    _ = plt.legend(['Negative', 'Positive'])\n    return density_pos, density_neg, bins","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"7207f67c-e555-43b7-a692-4baaa25e4168","_uuid":"022a826c982d080375e9f568735e58387c58090c","trusted":false,"collapsed":true},"cell_type":"code","source":"density_pos, density_neg, bins = plot_hist(0)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"f899635c-598c-4bb8-83bf-a9e786e8238e","_uuid":"85f9fb5c26c4514336d891ce4ae3bed79197617a","trusted":false,"collapsed":true},"cell_type":"code","source":"density_pos, density_neg, bins = plot_hist(10000, bins)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"0ee52f27-8ff2-4b0d-983e-f347c3afe47f","_uuid":"f3fba62ec8e2bd0e040bea6d2f4cdbe4616cda18","trusted":false,"collapsed":true},"cell_type":"code","source":"density_pos, density_neg, bins = plot_hist(100000, bins)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"dbe55b77-b57b-4116-8f66-d79a3413d770","_uuid":"ce5808d4f6297114153fde9d7d5f3154ece48794","trusted":false,"collapsed":true},"cell_type":"code","source":"## VERIFY: PMF sums are 1.0\nprint(np.sum(density_pos*(bins[1:] - bins[:-1])))\nprint(np.sum(density_neg*(bins[1:] - bins[:-1])))","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"_cell_guid":"ce88b4dd-3eee-4933-990d-cb57bfe3668e","_uuid":"d53fc20c722cbfd4ca0f7b8209ec9b105695fa6a","trusted":false},"cell_type":"code","source":"pps = density_pos / (density_pos + density_neg)\n# ranged_probs = {bins for i in range(density_pos.shape[0])}","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"_cell_guid":"912c19c7-4fec-435b-a182-91d8dd2c7ff3","_uuid":"5545e43d6df513b526d1f3a0c2f1c7a3251a3c0f","trusted":false},"cell_type":"code","source":"def query_prob_to_bin_prob(query_prob):\n    return pps[np.argmax((np.log10(query_prob + safety_offset) - bins[:-1]) < 0) - 1]    ","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"f42c1f76-b2ed-4280-af03-535b359d6a15","_uuid":"b0cf158e00da3dbc7ae033063a5ba5dd98b03a5c","trusted":false,"collapsed":true},"cell_type":"code","source":"query_prob_to_bin_prob(1e-14)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"9913f31e-06dc-44a6-8f01-543f728522eb","_uuid":"92e321840e1da9bfad39ddafb51021ce079804fa","trusted":false,"collapsed":true},"cell_type":"code","source":"probs_test = np.zeros((18790469,1))\ntest_ids = np.zeros((18790469,1))\ni = 0\nwith open('../input/test.csv') as f:\n    fcsv = csv.reader(f)\n    header = True\n    for row in tqdm_notebook(fcsv, unit='rows', total=(18790469+1)):\n        if header:\n            header = False\n            continue\n            # collected_rows.append(row)\n        cid, ip, app, dev, os, channel, ct = row\n        probs_test[i] = query_prob_to_bin_prob(ips_prob[ip]*apps_prob[app]*devs_prob[dev]*oss_prob[os]*channels_prob[channel])\n        test_ids[i] = int(cid)\n        i += 1","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"9bfa0555-9cf8-4bcc-8d24-038ded424dee","_uuid":"d0656be2636b12e3f9bcb453117e7d2852497139","trusted":false,"collapsed":true},"cell_type":"code","source":"with open('./mean_field_sub1.csv', 'w') as f:\n    f.write('click_id,is_attributed\\n')\n    for i in tqdm(range(test_ids.shape[0])):\n        f.write('%d,%0.6f\\n' % (test_ids[i, 0], probs_test[i, 0]))","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"_cell_guid":"29e01a02-3110-4713-bcd2-95c2b810b130","_uuid":"77b87c99bbd5017b8a0d8ab075ff868b62f06a78","trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"}},"nbformat":4,"nbformat_minor":1}