{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":false,"collapsed":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport random\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\nimport findspark\nimport os\nprint(os.listdir(\"../input\"))\n#import spark\n#findspark.init()\n\n# Any results you write to the current directory are saved as output.\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport numpy as np\nfrom sklearn import tree\nfrom sklearn.metrics import accuracy_score\nimport datetime\n\nplt.rc(\"font\", size=14)\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.cross_validation import train_test_split\nimport seaborn as sns\nsns.set(style=\"white\")\nsns.set(style=\"whitegrid\", color_codes=True)\nfrom sklearn.neural_network import MLPClassifier\nfrom sklearn.preprocessing import MinMaxScaler","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":false,"collapsed":true},"cell_type":"code","source":"df=pd.read_csv(\"../input/train.csv\", nrows=10000000)\ndf.head()\n","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"_uuid":"7569d72878cf00204396b989ac0a4f02df04c075","_cell_guid":"2685708a-9d6e-4b18-8616-d81adc781bab","trusted":false},"cell_type":"code","source":"import random\nfilename = \"../input/train.csv\"\nn = 1000000 #number of records in file (excludes header)\ns = 100000 #desired sample size\nskip = random.sample(range(1,n+1),n-s) #the 0-indexed header will not be included in the skip list\ndftest = pd.read_csv(filename, skiprows=skip, nrows = 1000000)\n# Better sampling techniques are certainly required. This is a just for my trial and learning...","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ac0e8f3ac99ebe52fc2e830755cd772eae4afe44","_cell_guid":"5686a0f7-24bd-4168-970e-d960bded6295","trusted":false,"collapsed":true},"cell_type":"code","source":"\n#dftest=pd.read_csv(\"../input/test.csv\", nrows=10000000)\ndftest.head()\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7ed12e279b4037feb32f3563d6f5f51226f62fd9","_cell_guid":"d38cdcee-013b-4766-bbb3-50171fa26a59","trusted":false,"collapsed":true},"cell_type":"code","source":"#make wider graphs\nsns.set(rc={'figure.figsize':(20,5)});\nplt.figure(figsize=(20,5));\nsns.countplot(x='is_attributed', data=df);\n\n# We see that we have very few conversions of the app.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"66573221535b713b864409635ad3ee58e2478ea0","_cell_guid":"342081e2-5e0a-4365-8d14-75a1fa21b533","trusted":false,"collapsed":true},"cell_type":"code","source":"sns.countplot(x='os', data=df);","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f5f87e3cfb12fd0e7ad6efe4bbc12718f7a4365a","_cell_guid":"ad31564f-0e48-40ee-b5bd-33ce736fa034","trusted":false,"collapsed":true},"cell_type":"code","source":"sns.countplot(x=\"device\", data=df) ; ","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f7965a9b7473ef869c9b69a478f59a970a6fbd0d","_cell_guid":"668a6faa-a6a7-44ca-93e1-b07fdb2f118b","trusted":false,"collapsed":true},"cell_type":"code","source":"sns.countplot(x=\"app\", data=df) ; ","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ca1aa6794774db7e04e64b7aada9430c99854623","_cell_guid":"2f4a215a-9b9f-4528-be74-ef4b538e6538","trusted":false,"collapsed":true},"cell_type":"code","source":"df[[\"app\",\"is_attributed\"]].groupby([\"app\"]).count().plot() # Frequency of apps  ","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"36d4faf0dda0fea170fd42650c56e96c72acb827","_cell_guid":"19126eb1-8f80-4bd9-b117-d956a9658ab0","trusted":false,"collapsed":true},"cell_type":"code","source":"df[[\"app\",\"is_attributed\"]].groupby([\"is_attributed\"]).count() # among 10 million clicks, we have 18,717 downloads.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f6a2b941d0981bb30386b174502339766c7ebb63","_cell_guid":"15086e7c-e823-4873-a9b0-a0414398b987","trusted":false,"collapsed":true},"cell_type":"code","source":"df[[\"os\",\"is_attributed\"]].groupby([\"os\"], as_index=False).count().plot()\n# Most used os to least used OS :\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e60c9f60270cd97cc3d7c1aba3c6a275f5c940ef","_cell_guid":"706c7cc3-c57b-4f7b-a04a-662b8e84869b","trusted":false,"collapsed":true},"cell_type":"code","source":"df[[\"app\",\"is_attributed\"]].groupby([\"app\"], as_index=False).count().sort_values(\"is_attributed\", ascending = False)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1a25c4a02711a77a3bcebb5a309dd11f437c1e7b","_cell_guid":"444a4b50-c75d-4614-827d-1b204cea49b9","trusted":false,"collapsed":true},"cell_type":"code","source":"df[[\"app\",\"is_attributed\"]].groupby([\"app\"], as_index=False).mean().sort_values(\"is_attributed\", ascending = False)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4b4ff0164e3a088ea05fcf5d68d3688d441837fa","_cell_guid":"beb18f69-4b9d-4c9c-826a-01ee82f6ff2e","trusted":false,"collapsed":true},"cell_type":"code","source":"df[[\"device\",\"is_attributed\"]].groupby([\"device\"], as_index=False).count().sort_values(\"is_attributed\", ascending = False)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a3bc4af3793e31f5591a931ae48f79869aae6988","_cell_guid":"53f46c9a-bee1-4d3e-a2a7-4ce707149242","trusted":false,"collapsed":true},"cell_type":"code","source":"df[[\"device\",\"is_attributed\"]].groupby([\"device\"], as_index=False).count().plot()","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"_uuid":"929d72fc5c76c580c775fc79671bc57495947e45","_cell_guid":"ad98ffff-c824-4f4b-8252-ce829186df5f","trusted":false},"cell_type":"code","source":"df['click_time'] = pd.to_datetime(df['click_time'])\ndf['attributed_time'] = pd.to_datetime(df['attributed_time'])\ndftest['click_time'] = pd.to_datetime(dftest['click_time'])\ndf['hr']=df['click_time'].dt.hour\ndftest['hr']=dftest['click_time'].dt.hour\n#dftest['attributed_time'] = pd.to_datetime(dftest['attributed_time'])\n#datetime.hour\n# dt.round('H')   \n\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e9a0ab8ea6163439a57243176b90d5ed4b3a97d5","_cell_guid":"94e33106-5a0b-4ac5-b9c2-5f0fd9490af3","trusted":false,"collapsed":true},"cell_type":"code","source":"df[['hr','is_attributed']].groupby(['hr'], as_index=True).count().plot()\nplt.title('HOURLY CLICK FREQUENCY');\nplt.ylabel('Number of Clicks');\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"776d819ce3aea66338a271e32af0cce6e8d043e7","_cell_guid":"36145af8-edaf-44fa-91c0-0836778d049c","trusted":false,"collapsed":true},"cell_type":"code","source":"dftest.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0d120870e4092f924d9c6cfced169156ac483938","_cell_guid":"38f81916-72ad-4784-804f-74834f2f4ab3","trusted":false,"collapsed":true},"cell_type":"code","source":"x = df.drop(['is_attributed','click_time','ip','attributed_time'], axis=1)\ny = df['is_attributed']\ndftestx = dftest.drop(['click_time','ip','attributed_time','is_attributed'], axis = 1)\nytest = dftest['is_attributed']\nx.head();\ndftestx.head()","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"_uuid":"32e351395f124973d77a9f3f914c44a8ddbbd5d7","_cell_guid":"d155f965-623a-47f7-ab17-1639b8d5da1b","trusted":false},"cell_type":"code","source":"scaler = MinMaxScaler()\nscaler.fit(x)\nxscaled = scaler.transform(x)\nscaler.fit(dftestx)\n\ndftestxscaled = scaler.transform(dftestx)\n# The have become numpy arrays\n#x.head()\n#dftestxscaled.head()\npd.DataFrame(xscaled)\npd.DataFrame(dftestxscaled);\n#xscaled.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1459d1fa9bd3033a19bcefd0a421d8ac31d18215","_cell_guid":"6802171c-5bf8-458f-ab14-1afd07e3e887","trusted":false,"collapsed":true},"cell_type":"code","source":"nn = MLPClassifier(hidden_layer_sizes=(5,5,5,5,5), activation='logistic', max_iter=10000000, solver='lbfgs')\nnn.fit(x, y)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ba354abdbcbd660d305fd5beb44e6510475d5384","_cell_guid":"dd0e7da8-9949-4139-a583-cb4780a7a4b4","trusted":false,"collapsed":true},"cell_type":"code","source":"print(accuracy_score (ytest, nn.predict(dftestx)))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3ea964018f402f8ece986c644e67bfe02d254253","_cell_guid":"5f60fb5d-351d-4a6f-897f-1694480ed38c","trusted":false,"collapsed":true},"cell_type":"code","source":"print(\" Thanks for listening! :)\")\n","execution_count":null,"outputs":[]}],"metadata":{"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"}},"nbformat":4,"nbformat_minor":1}