{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import IPython\npath = '/kaggle/input/kddcup2012-track2/'\n\nzipPath = path + 'track2.zip'\n!unzip $zipPath\n\n\nIPython.display.clear_output()\n\n","metadata":{"execution":{"iopub.status.busy":"2023-09-04T16:20:30.580965Z","iopub.execute_input":"2023-09-04T16:20:30.581345Z","iopub.status.idle":"2023-09-04T16:23:08.272831Z","shell.execute_reply.started":"2023-09-04T16:20:30.581309Z","shell.execute_reply":"2023-09-04T16:23:08.271533Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"zipPath = path + 'test.zip'\n!unzip $zipPath\n\nIPython.display.clear_output()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"files = {\n    0: '/kaggle/working/track2/training.txt',\n    1: '/kaggle/working/track2/purchasedkeywordid_tokensid.txt',\n    2: '/kaggle/working/track2/descriptionid_tokensid.txt',\n    3: '/kaggle/working/track2/userid_profile.txt',\n    4: '/kaggle/working/track2/queryid_tokensid.txt',\n    5: '/kaggle/working/track2/titleid_tokensid.txt',\n    6: '/kaggle/working/test.txt'\n}","metadata":{"execution":{"iopub.status.busy":"2023-09-05T07:19:15.533592Z","iopub.execute_input":"2023-09-05T07:19:15.534010Z","iopub.status.idle":"2023-09-05T07:19:15.540823Z","shell.execute_reply.started":"2023-09-05T07:19:15.533976Z","shell.execute_reply":"2023-09-05T07:19:15.539613Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm -d file_click*.txt","metadata":{"execution":{"iopub.status.busy":"2023-09-05T07:18:55.025263Z","iopub.execute_input":"2023-09-05T07:18:55.025775Z","iopub.status.idle":"2023-09-05T07:18:56.180297Z","shell.execute_reply.started":"2023-09-05T07:18:55.025735Z","shell.execute_reply":"2023-09-05T07:18:56.178913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nfpath = files[0]\n!awk '$1<1' $fpath| head -n 7000000 >  file_click_0.txt","metadata":{"execution":{"iopub.status.busy":"2023-09-04T17:20:47.604750Z","iopub.execute_input":"2023-09-04T17:20:47.605268Z","iopub.status.idle":"2023-09-04T17:20:56.909317Z","shell.execute_reply.started":"2023-09-04T17:20:47.605227Z","shell.execute_reply":"2023-09-04T17:20:56.907586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n!awk '$1>0' $fpath >  file_click_1.txt","metadata":{"execution":{"iopub.status.busy":"2023-09-04T17:24:20.265553Z","iopub.execute_input":"2023-09-04T17:24:20.267091Z","iopub.status.idle":"2023-09-04T17:26:36.467967Z","shell.execute_reply.started":"2023-09-04T17:24:20.267043Z","shell.execute_reply":"2023-09-04T17:26:36.466006Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!wc -l file_click_0.txt","metadata":{"execution":{"iopub.status.busy":"2023-09-04T17:21:48.636952Z","iopub.execute_input":"2023-09-04T17:21:48.637356Z","iopub.status.idle":"2023-09-04T17:21:50.013393Z","shell.execute_reply.started":"2023-09-04T17:21:48.637318Z","shell.execute_reply":"2023-09-04T17:21:50.011637Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!tail -10 file_click_0.txt|column -t","metadata":{"execution":{"iopub.status.busy":"2023-09-04T17:22:35.273199Z","iopub.execute_input":"2023-09-04T17:22:35.273619Z","iopub.status.idle":"2023-09-04T17:22:36.391371Z","shell.execute_reply.started":"2023-09-04T17:22:35.273585Z","shell.execute_reply":"2023-09-04T17:22:36.389129Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nimport pandas as pd\n\n# user data\nuser = pd.read_csv(files[3], sep='\\t', header=None)\nuser.columns = ['UserId', 'Gender', 'AgeGroup']\n\n# query data\nquery = pd.read_csv(files[4], sep='\\t', header=None)\nquery.columns = ['QueryId', 'Query']\n\n# Ad Description\ndesc = pd.read_csv(files[2], sep='\\t', header=None)\ndesc.columns = ['DescriptionId', 'Description']\n\n# Ad Title\ntitle = pd.read_csv(files[5], sep='\\t', header=None)\ntitle.columns = ['TitleId', 'Title']\n\n# purchased keyword \nkeyword = pd.read_csv(files[1], sep='\\t', header=None)\nkeyword.columns = ['KeywordId', 'Keyword']","metadata":{"execution":{"iopub.status.busy":"2023-09-05T07:19:22.688675Z","iopub.execute_input":"2023-09-05T07:19:22.689477Z","iopub.status.idle":"2023-09-05T07:20:14.042413Z","shell.execute_reply.started":"2023-09-05T07:19:22.689436Z","shell.execute_reply":"2023-09-05T07:20:14.041115Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntrain_cols = ['Click', 'Impression', 'DisplayURL', 'AdId', \n              'AdvertiserId', 'Depth', 'Position',  'QueryId', 'KeywordId', \n              'TitleId',  'DescriptionId',  'UserId'\n             ]","metadata":{"execution":{"iopub.status.busy":"2023-09-04T17:51:03.200698Z","iopub.execute_input":"2023-09-04T17:51:03.201162Z","iopub.status.idle":"2023-09-04T17:51:03.208625Z","shell.execute_reply.started":"2023-09-04T17:51:03.201128Z","shell.execute_reply":"2023-09-04T17:51:03.207336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nimport numpy as np\ndef cosine_similarity(A,B):\n    return np.dot(A,B)/(np.linalg.norm(A)*np.linalg.norm(B))\n\ncosine_similarity_vectorized = np.vectorize(cosine_similarity)\n\ndef get_score(x1, x2):\n    split1 = set(x1.split('|'))\n    split2 = set(x2.split('|'))\n    total_set = split1.union(split2)\n    v1 = np.zeros((len(total_set),))\n    v2 = np.zeros((len(total_set),))\n    for i, j in enumerate(total_set):\n        if j in split1:\n            v1[i]=1\n        if j in split2:\n            v2[i]=1\n    split1, split2, total_set = {}, {}, {}\n    return cosine_similarity(v1,v2)\n\nget_score_vectorized = np.vectorize(get_score)\n\ndef similarity(x1, x2):\n    return len(set(x1.split('|')).intersection(set(x2.split('|'))))\nsimilarity_vectorized = np.vectorize(similarity)\n\ndef divide(a,b):\n    return a/b\ndef length(a):\n    return len(a.split('|'))","metadata":{"execution":{"iopub.status.busy":"2023-09-05T07:20:31.076504Z","iopub.execute_input":"2023-09-05T07:20:31.076956Z","iopub.status.idle":"2023-09-05T07:20:31.090942Z","shell.execute_reply.started":"2023-09-05T07:20:31.076910Z","shell.execute_reply":"2023-09-05T07:20:31.089451Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ndef join_data(filename,cols, train=True):\n    \n    df = pd.read_csv(filename, sep='\\t', header=None,names= cols)\n    df['Id'] = df.index\n    df = pd.merge(df, user, how='left', on = 'UserId')\n    df = pd.merge(df, query, on = 'QueryId')\n    df = pd.merge(df, desc, on = 'DescriptionId')\n    df = pd.merge(df, title, on = 'TitleId')\n    df = pd.merge(df, keyword, on = 'KeywordId')\n    \n    drop_cols = [ 'UserId', 'QueryId', 'DescriptionId', \n                  'TitleId', 'KeywordId'\n                ]\n    df.drop(drop_cols, axis=1,inplace=True)\n    \n    df['Query_len'] = np.vectorize(length)(df['Query'])\n    df['Desc_len'] = np.vectorize(length)(df['Description'])\n    df['Title_len'] = np.vectorize(length)(df['Title'])\n    df['Keyword_len'] = np.vectorize(length)(df['Keyword'])\n    \n    df['Query_Desc_cos_sim'] = get_score_vectorized(df['Query'],df['Description'])\n    df['Query_Title_cos_sim']= get_score_vectorized(df['Query'],df['Title'])\n    df['Query_Keyword_cos_sim'] = get_score_vectorized(df['Query'],df['Keyword'])\n    df['Desc_Title_cos_sim'] = get_score_vectorized(df['Title'],df['Description'])\n    df['Desc_Keyword_cos_sim'] = get_score_vectorized(df['Keyword'],df['Description'])\n    df['Title_Keyword_cos_sim'] = get_score_vectorized(df['Title'],df['Keyword'])\n    \n    df['Query_Desc_sim'] = similarity_vectorized(df['Query'],df['Description'])\n    df['Query_Title_sim']= similarity_vectorized(df['Query'],df['Title'])\n    df['Query_Keyword_sim']= similarity_vectorized(df['Query'],df['Keyword'])\n    df['Desc_Title_sim'] = similarity_vectorized(df['Title'],df['Description'])\n    df['Desc_Keyword_sim'] = similarity_vectorized(df['Keyword'],df['Description'])\n    df['Title_Keyword_sim'] = similarity_vectorized(df['Title'],df['Keyword'])\n    drop_cols = [ 'Query', 'Description', \n                  'Title', 'Keyword'\n                ]\n    df.drop(drop_cols, axis=1,inplace=True)\n    if train:\n        df['CTR'] = np.vectorize(divide)(df['Click'], df['Impression'])\n    \n    return df.sort_values('Id').reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2023-09-05T07:20:37.174265Z","iopub.execute_input":"2023-09-05T07:20:37.174716Z","iopub.status.idle":"2023-09-05T07:20:37.195756Z","shell.execute_reply.started":"2023-09-05T07:20:37.174680Z","shell.execute_reply":"2023-09-05T07:20:37.194508Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\ndef join_data(filename,cols, train=True):\n    \n    df = pd.read_csv(filename, sep='\\t', header=None,names= cols)\n    df['Id'] = df.index\n\n    df = pd.merge(df, user, how='left', on = 'UserId')\n    df = pd.merge(df, query, on = 'QueryId')\n    df = pd.merge(df, desc, on = 'DescriptionId')\n    df = pd.merge(df, title, on = 'TitleId')\n    df = pd.merge(df, keyword, on = 'KeywordId')\n\n    print(\"===============Step 01 is completed===============\")\n\n    drop_cols = [ 'UserId', 'QueryId', 'DescriptionId', \n                  'TitleId', 'KeywordId'\n                ]\n    df.drop(drop_cols, axis=1,inplace=True)\n\n    print(\"===============Step 02 is completed===============\")\n    \n    df['Query_len'] = np.vectorize(length)(df['Query'])\n\n    print(\"===============Step 03 is completed===============\")\n\n    df['Desc_len'] = np.vectorize(length)(df['Description'])\n\n    print(\"===============Step 04 is completed===============\")\n\n    df['Title_len'] = np.vectorize(length)(df['Title'])\n\n    print(\"===============Step 05 is completed===============\")\n\n    df['Keyword_len'] = np.vectorize(length)(df['Keyword'])\n\n    print(\"===============Step 06 is completed===============\")\n    \n    df['Query_Desc_cos_sim'] = get_score_vectorized(df['Query'],df['Description'])\n\n    print(\"===============Step 07 is completed===============\")\n\n    df['Query_Title_cos_sim']= get_score_vectorized(df['Query'],df['Title'])\n\n    print(\"===============Step 08 is completed===============\")\n\n    df['Query_Keyword_cos_sim'] = get_score_vectorized(df['Query'],df['Keyword'])\n\n    print(\"===============Step 09 is completed===============\")\n\n    df['Desc_Title_cos_sim'] = get_score_vectorized(df['Title'],df['Description'])\n\n    print(\"===============Step 10 is completed===============\")\n\n    df['Desc_Keyword_cos_sim'] = get_score_vectorized(df['Keyword'],df['Description'])\n\n    print(\"===============Step 11 is completed===============\")\n\n    df['Title_Keyword_cos_sim'] = get_score_vectorized(df['Title'],df['Keyword'])\n\n    print(\"===============Step 12 is completed===============\")\n    \n    df['Query_Desc_sim'] = similarity_vectorized(df['Query'],df['Description'])\n\n    print(\"===============Step 13 is completed===============\")\n\n    df['Query_Title_sim']= similarity_vectorized(df['Query'],df['Title'])\n\n    print(\"===============Step 14 is completed===============\")\n\n    df['Query_Keyword_sim']= similarity_vectorized(df['Query'],df['Keyword'])\n\n    print(\"===============Step 15 is completed===============\")\n\n    df['Desc_Title_sim'] = similarity_vectorized(df['Title'],df['Description'])\n\n    print(\"===============Step 16 is completed===============\")\n\n    df['Desc_Keyword_sim'] = similarity_vectorized(df['Keyword'],df['Description'])\n\n    print(\"===============Step 17 is completed===============\")\n\n    df['Title_Keyword_sim'] = similarity_vectorized(df['Title'],df['Keyword'])\n\n    print(\"===============Step 18 is completed===============\")\n\n    drop_cols = [ 'Query', 'Description', \n                  'Title', 'Keyword'\n                ]\n    df.drop(drop_cols, axis=1,inplace=True)\n\n    print(\"===============Step 19 is completed===============\")\n\n    if train:\n        df['CTR'] = np.vectorize(divide)(df['Click'], df['Impression'])\n    \n    print(\"===============Step 20 is completed===============\")\n    \n    return df.sort_values('Id').reset_index(drop=True)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nfile0, file1 = 'file_click_0.txt', 'file_click_1.txt'\ntrain_df_0 =  join_data(file0,train_cols)\ntrain_df_1 =  join_data(file1,train_cols)","metadata":{"execution":{"iopub.status.busy":"2023-09-04T18:20:59.932225Z","iopub.execute_input":"2023-09-04T18:20:59.933124Z","iopub.status.idle":"2023-09-04T19:07:14.883815Z","shell.execute_reply.started":"2023-09-04T18:20:59.933082Z","shell.execute_reply":"2023-09-04T19:07:14.879732Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntrain_df = pd.concat([train_df_0, train_df_1]).reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2023-09-04T19:07:14.892230Z","iopub.execute_input":"2023-09-04T19:07:14.893391Z","iopub.status.idle":"2023-09-04T19:07:26.087244Z","shell.execute_reply.started":"2023-09-04T19:07:14.893279Z","shell.execute_reply":"2023-09-04T19:07:26.085756Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.to_csv('train.csv',index=False)","metadata":{"execution":{"iopub.status.busy":"2023-09-04T19:07:26.089019Z","iopub.execute_input":"2023-09-04T19:07:26.089388Z","iopub.status.idle":"2023-09-04T19:12:48.070728Z","shell.execute_reply.started":"2023-09-04T19:07:26.089355Z","shell.execute_reply":"2023-09-04T19:12:48.069348Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntest_cols = [ 'DisplayURL', 'AdId', \n              'AdvertiserId', 'Depth', 'Position',  'QueryId', 'KeywordId', \n              'TitleId',  'DescriptionId',  'UserId'\n             ]\ntest_df = join_data('test.txt', test_cols, False)","metadata":{"execution":{"iopub.status.busy":"2023-09-05T07:20:53.464565Z","iopub.execute_input":"2023-09-05T07:20:53.464961Z","iopub.status.idle":"2023-09-05T08:27:54.281279Z","shell.execute_reply.started":"2023-09-05T07:20:53.464930Z","shell.execute_reply":"2023-09-05T08:27:54.278917Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm -rd track2 *.txt","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.to_csv('test.csv',index=False)","metadata":{"execution":{"iopub.status.busy":"2023-09-05T08:27:54.285707Z","iopub.execute_input":"2023-09-05T08:27:54.286239Z","iopub.status.idle":"2023-09-05T08:35:13.596885Z","shell.execute_reply.started":"2023-09-05T08:27:54.286192Z","shell.execute_reply":"2023-09-05T08:35:13.595480Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-09-05T09:11:30.373296Z","iopub.execute_input":"2023-09-05T09:11:30.373828Z","iopub.status.idle":"2023-09-05T09:11:30.408985Z","shell.execute_reply.started":"2023-09-05T09:11:30.373776Z","shell.execute_reply":"2023-09-05T09:11:30.407800Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!head -5 test.txt|column -t","metadata":{"execution":{"iopub.status.busy":"2023-09-05T09:59:39.976996Z","iopub.execute_input":"2023-09-05T09:59:39.977526Z","iopub.status.idle":"2023-09-05T09:59:41.096171Z","shell.execute_reply.started":"2023-09-05T09:59:39.977482Z","shell.execute_reply":"2023-09-05T09:59:41.094746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mv track2/training.txt .\n!rm -rd track2","metadata":{},"execution_count":null,"outputs":[]}]}