{"cells":[{"cell_type":"markdown","metadata":{"_cell_guid":"b232bfdf-542d-2a68-1308-1b7da31fb5ab"},"source":"Exploring the Users Click Dilemma\n=============================="},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"2449f1d5-5d2f-d964-3933-215ca76bb948"},"outputs":[],"source":"from sklearn.linear_model import PassiveAggressiveRegressor\nimport time; start_time = time.time()\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n%matplotlib inline\nimport numpy as np\nimport pandas as pd\nimport sqlite3\n\nreg = PassiveAggressiveRegressor(warm_start=True, random_state=123)\nchunksize_ = 50000\nmc_y_label = []\nmc_y_pred = []\nid_test = []\ny_pred = []\n\ndf_train = pd.read_csv('../input/clicks_train.csv', low_memory=False, iterator=True, chunksize=chunksize_)\n\n#stage the promo table\npromo = pd.read_csv('../input/promoted_content.csv', usecols=['ad_id', 'document_id']) #['ad_id', 'document_id', 'campaign_id', 'advertiser_id']\ndocc = pd.read_csv('../input/documents_categories.csv', usecols=['document_id','confidence_level'])\ndocs = pd.read_csv('../input/documents_topics.csv', usecols=['document_id','confidence_level']) \nc = sqlite3.connect(':memory:')\npromo = pd.merge(promo, docc, how='left', on='document_id')\npromo = pd.merge(promo, docs, how='left', on='document_id')\npromo.columns = ['ad_id', 'document_id', 'c1', 'c2']\npromo.to_sql('promo',c)\npromo = pd.read_sql(\"Select ad_id, SUM(c1) AS sc1, SUM(c2) AS sc2 FROM promo GROUP BY ad_id\", c)\nc.close()\n\nprint(\"\\nTrain\\n\" + \"=\"*50)\nfor df in df_train:\n    df = pd.merge(df, promo, how='left', on='ad_id')\n    df = df.fillna(0.0)\n    col = [x for x in df.columns if x not in ['display_id','ad_id','clicked']]\n    reg = reg.partial_fit(df[col], df['clicked'])\n    mc_y_label += list(df['clicked'].values)\n    mc_y_pred += list(reg.predict(df[col]))\n    break\ndf_train = ''\n\ndf_test = pd.read_csv('../input/clicks_test.csv', low_memory=False, iterator=True, chunksize=chunksize_)\nprint(\"\\nTest\\n\" + \"=\"*50)\nfor df in df_test:\n    df = pd.merge(df, promo, how='left', on='ad_id')\n    df = df.fillna(0.0)\n    col = [x for x in df.columns if x not in ['display_id','ad_id','clicked']]\n    if len(id_test)>0:\n        id_test = pd.concat((id_test, df[['display_id','ad_id']]), axis=0, ignore_index=True)\n    else:\n        id_test = df[['display_id','ad_id']]\n    y_pred += list(reg.predict(df[col]))\n    break\ndf_test = ''\n\ndfs = pd.concat((id_test, pd.DataFrame(y_pred)), axis=1, ignore_index=True)\ndfs.columns = ['display_id','ad_id','clicked']\ndfs = dfs[dfs['clicked']>0.0]\ndfs = dfs.sort_values(by=['display_id','clicked'], ascending=[True, False])\ndfs = dfs.reset_index(drop=True)\na = dfs[['display_id','ad_id']].groupby('display_id')['ad_id'].agg(lambda col: ' '.join(map(str,col)))\na = pd.DataFrame(a)\na.colums = ['display_id','ad_id']\na.to_csv('Submission.csv')\nprint(len(a))\nprint(\"Whala!!! : %s minutes ---\" % round(((time.time() - start_time)/60),2))"},{"cell_type":"markdown","metadata":{"_cell_guid":"ca5055c8-b350-65cd-0c30-d761eb293f9f"},"source":"The Cost Function\n=================\n[ Mean Average Precision @12][1]\n\n\n  [1]: https://github.com/benhamner/Metrics/blob/master/Python/ml_metrics/average_precision.py"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"efc78480-61bc-3dea-fc24-f36c5c7b030a"},"outputs":[],"source":"def apk(actual, predicted, k=12):\n    if len(predicted)>k:\n        predicted = predicted[:k]\n    score = 0.0\n    num_hits = 0.0\n    for i,p in enumerate(predicted):\n        if p in actual and p not in predicted[:i]:\n            num_hits += 1.0\n            score += num_hits / (i+1.0)\n    if not actual:\n        return 0.0\n    return score / min(len(actual), k)\n\ndef mapk(actual, predicted, k=12):\n    return np.mean([apk(a,p,k) for a,p in zip(actual, predicted)])"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"50736e34-ed90-9c9a-ebe8-ea5ede9bbf13"},"outputs":[],"source":"df_test = pd.read_csv('../input/clicks_train.csv', low_memory=False, iterator=True, chunksize=chunksize_)\nprint(\"\\nScore\\n\" + \"=\"*50)\nfor df in df_test:\n    df = pd.merge(df, promo, how='left', on='ad_id')\n    df = df.fillna(0.0)\n    col = [x for x in df.columns if x not in ['display_id','ad_id','clicked']]\n    id_test = pd.concat((id_test, df[['display_id','ad_id']]), axis=0, ignore_index=True)\n    mc_y_label = list(df['clicked'].values)\n    y_pred = list(reg.predict(df[col]))\n    break\ndf_test = ''\n\ndfs = pd.concat((id_test, pd.DataFrame(y_pred)), axis=1, ignore_index=True)\ndfs.columns = ['display_id','ad_id','clicked']\ndfs = dfs[dfs['clicked']>0.0]\ndfs = dfs.sort_values(by=['display_id','clicked'], ascending=[True, False])\ndfs = dfs.reset_index(drop=True)\na = dfs[['display_id','ad_id']].groupby('display_id')['ad_id'].agg(lambda col: ' '.join(map(str,col)))\na = pd.DataFrame(a)\na.colums = ['display_id','ad_id']\n\ndfs = pd.concat((id_test, pd.DataFrame(mc_y_label)), axis=1, ignore_index=True)\ndfs.columns = ['display_id','ad_id','clicked']\ndfs = dfs[dfs['clicked']>0.0]\ndfs = dfs.sort_values(by=['display_id','clicked'], ascending=[True, False])\ndfs = dfs.reset_index(drop=True)\nb = dfs[['display_id','ad_id']].groupby('display_id')['ad_id'].agg(lambda col: ' '.join(map(str,col)))\nb = pd.DataFrame(a)\nb.colums = ['display_id','ad_id']\n\nprint(mapk(b['ad_id'], a['ad_id']))"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"25320b5b-4db6-48e1-d263-533c34395918"},"outputs":[],"source":"#pd.read_csv('../input/sample_submission.csv').to_csv('sample_submission.csv')\n#z = zipfile.ZipFile('sample_submission.csv.zip', \"w\", zipfile.ZIP_DEFLATED)\n#z.write('sample_submission.csv')\n#z.close()\n#os.remove('sample_submission.csv')"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"deac162b-5371-9bad-fa8a-068ec45212b8"},"outputs":[],"source":"#docm = pd.read_csv('../input/documents_meta.csv') #['document_id', 'source_id', 'publisher_id', 'publish_time']\n#events = pd.read_csv('../input/events.csv') #['display_id', 'uuid', 'document_id', 'timestamp', 'platform', 'geo_location'] \n#pagev = pd.read_csv('../input/page_views_sample.csv') #['uuid', 'document_id', 'timestamp', 'platform', 'geo_location', 'traffic_source'] \n#doce = pd.read_csv('../input/documents_entities.csv') #['document_id', 'entity_id', 'confidence_level'] "},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"7b778c6c-8a6f-0e22-eb50-c6eebee16660"},"outputs":[],"source":""}],"metadata":{"_change_revision":0,"_is_fork":false,"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.5.2"}},"nbformat":4,"nbformat_minor":0}