{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import warnings\nwarnings.filterwarnings(\"ignore\")\n\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport networkx as nx\nfrom networkx.algorithms.traversal.depth_first_search import dfs_tree","metadata":{"execution":{"iopub.status.busy":"2022-11-09T22:39:56.521295Z","iopub.execute_input":"2022-11-09T22:39:56.521800Z","iopub.status.idle":"2022-11-09T22:39:57.364577Z","shell.execute_reply.started":"2022-11-09T22:39:56.521698Z","shell.execute_reply":"2022-11-09T22:39:57.363121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!sudo apt-get install -y graphviz graphviz-dev\n!pip install pygraphviz","metadata":{"execution":{"iopub.status.busy":"2022-11-09T22:40:06.698893Z","iopub.execute_input":"2022-11-09T22:40:06.699353Z","iopub.status.idle":"2022-11-09T22:40:35.013908Z","shell.execute_reply.started":"2022-11-09T22:40:06.699309Z","shell.execute_reply":"2022-11-09T22:40:35.012295Z"},"_kg_hide-output":true,"scrolled":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv('../input/otto-recommender-kernel-statistics/otto-recommender-system_kernels.csv')\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2022-11-09T22:42:20.827646Z","iopub.execute_input":"2022-11-09T22:42:20.828368Z","iopub.status.idle":"2022-11-09T22:42:20.853789Z","shell.execute_reply.started":"2022-11-09T22:42:20.828315Z","shell.execute_reply":"2022-11-09T22:42:20.852550Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Top-20 kernel creators ","metadata":{}},{"cell_type":"code","source":"df['KernelAuthor'].value_counts().head(20)","metadata":{"execution":{"iopub.status.busy":"2022-11-09T22:42:25.344772Z","iopub.execute_input":"2022-11-09T22:42:25.345928Z","iopub.status.idle":"2022-11-09T22:42:25.364963Z","shell.execute_reply.started":"2022-11-09T22:42:25.345867Z","shell.execute_reply":"2022-11-09T22:42:25.363407Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Perfomance Tiers vs kernel creators ","metadata":{}},{"cell_type":"code","source":"df['KernelAuthorPerformanceTier'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-11-09T22:42:29.416225Z","iopub.execute_input":"2022-11-09T22:42:29.416722Z","iopub.status.idle":"2022-11-09T22:42:29.426928Z","shell.execute_reply.started":"2022-11-09T22:42:29.416684Z","shell.execute_reply":"2022-11-09T22:42:29.425685Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Kernels medals ","metadata":{}},{"cell_type":"code","source":"df['medal'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-11-09T22:42:31.714651Z","iopub.execute_input":"2022-11-09T22:42:31.715553Z","iopub.status.idle":"2022-11-09T22:42:31.725375Z","shell.execute_reply.started":"2022-11-09T22:42:31.715506Z","shell.execute_reply":"2022-11-09T22:42:31.724129Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Medals for kernels creators (top-20)","metadata":{}},{"cell_type":"code","source":"df.groupby('KernelAuthor')['medal'].value_counts()\\\n                .unstack()[['GOLD', 'SILVER', 'BRONZE']]\\\n                .fillna(0).sort_values(['GOLD', 'SILVER', 'BRONZE'], ascending = False).head(20)","metadata":{"execution":{"iopub.status.busy":"2022-11-09T22:42:33.045856Z","iopub.execute_input":"2022-11-09T22:42:33.046380Z","iopub.status.idle":"2022-11-09T22:42:33.088542Z","shell.execute_reply.started":"2022-11-09T22:42:33.046334Z","shell.execute_reply":"2022-11-09T22:42:33.087441Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Kernels vs Language ","metadata":{}},{"cell_type":"code","source":"df['languageName'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-11-09T22:42:36.503006Z","iopub.execute_input":"2022-11-09T22:42:36.503509Z","iopub.status.idle":"2022-11-09T22:42:36.513767Z","shell.execute_reply.started":"2022-11-09T22:42:36.503469Z","shell.execute_reply":"2022-11-09T22:42:36.512407Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# How many kernels are the forks? ","metadata":{}},{"cell_type":"code","source":"df['isFork'].fillna(0).mean()","metadata":{"execution":{"iopub.status.busy":"2022-11-09T22:42:38.811747Z","iopub.execute_input":"2022-11-09T22:42:38.812249Z","iopub.status.idle":"2022-11-09T22:42:38.822163Z","shell.execute_reply.started":"2022-11-09T22:42:38.812210Z","shell.execute_reply":"2022-11-09T22:42:38.820969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# How many lines are changed in forks?","metadata":{}},{"cell_type":"code","source":"df['ForkLinesChanged'].describe()","metadata":{"execution":{"iopub.status.busy":"2022-11-09T22:42:40.926470Z","iopub.execute_input":"2022-11-09T22:42:40.926999Z","iopub.status.idle":"2022-11-09T22:42:40.942408Z","shell.execute_reply.started":"2022-11-09T22:42:40.926956Z","shell.execute_reply":"2022-11-09T22:42:40.941277Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Use GPU or not? ","metadata":{}},{"cell_type":"code","source":"df['isGpuEnabled'].fillna(0).mean()","metadata":{"execution":{"iopub.status.busy":"2022-11-09T22:42:43.080650Z","iopub.execute_input":"2022-11-09T22:42:43.081308Z","iopub.status.idle":"2022-11-09T22:42:43.092370Z","shell.execute_reply.started":"2022-11-09T22:42:43.081257Z","shell.execute_reply":"2022-11-09T22:42:43.090774Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Upvotes vs views ","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize = (10, 10))\nplt.scatter(df['totalVotes'].values, df['totalViews'].values)\nplt.xlabel('totalVotes')\nplt.ylabel('totalViews')\nplt.grid(True)\nplt.show();","metadata":{"execution":{"iopub.status.busy":"2022-11-09T22:42:43.946896Z","iopub.execute_input":"2022-11-09T22:42:43.947413Z","iopub.status.idle":"2022-11-09T22:42:44.210940Z","shell.execute_reply.started":"2022-11-09T22:42:43.947371Z","shell.execute_reply":"2022-11-09T22:42:44.210011Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize = (10, 10))\nplt.scatter(df['totalVotes'].values, df['bestPublicScore'].values)\nplt.xlabel('totalVotes')\nplt.ylabel('bestPublicScore')\nplt.ylim([0.45, 0.6])\nplt.grid(True)\nplt.show();","metadata":{"execution":{"iopub.status.busy":"2022-11-09T22:42:46.128227Z","iopub.execute_input":"2022-11-09T22:42:46.128947Z","iopub.status.idle":"2022-11-09T22:42:46.334870Z","shell.execute_reply.started":"2022-11-09T22:42:46.128877Z","shell.execute_reply":"2022-11-09T22:42:46.333497Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Most commented kernels ","metadata":{}},{"cell_type":"code","source":"df[['title', 'totalComments']].sort_values('totalComments', ascending = False).head(20).reset_index(drop = True)","metadata":{"execution":{"iopub.status.busy":"2022-11-09T22:42:49.588763Z","iopub.execute_input":"2022-11-09T22:42:49.590421Z","iopub.status.idle":"2022-11-09T22:42:49.606704Z","shell.execute_reply.started":"2022-11-09T22:42:49.590358Z","shell.execute_reply":"2022-11-09T22:42:49.605665Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Fork graph","metadata":{}},{"cell_type":"code","source":"forks = df[['title', 'totalVotes', 'id', 'forkParent']].values\nvotes_dict = {i:v for i,v in zip(forks[:, 2], forks[:, 1])}","metadata":{"execution":{"iopub.status.busy":"2022-11-09T22:42:52.116403Z","iopub.execute_input":"2022-11-09T22:42:52.116906Z","iopub.status.idle":"2022-11-09T22:42:52.125513Z","shell.execute_reply.started":"2022-11-09T22:42:52.116869Z","shell.execute_reply":"2022-11-09T22:42:52.124343Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"G = nx.DiGraph()\nfor title, votes, k_id, parent_id in forks:\n    G.add_node(k_id, title = title, votes = votes)\n    \nfor title, votes, k_id, parent_id in forks:\n    if parent_id != -1:\n        G.add_edge(parent_id, k_id)","metadata":{"execution":{"iopub.status.busy":"2022-11-09T22:42:52.496332Z","iopub.execute_input":"2022-11-09T22:42:52.497650Z","iopub.status.idle":"2022-11-09T22:42:52.505155Z","shell.execute_reply.started":"2022-11-09T22:42:52.497596Z","shell.execute_reply":"2022-11-09T22:42:52.503770Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"connected_components = []\nfor cc in nx.connected_components(G.to_undirected()):\n    if len(cc) == 1:\n        G.remove_node(list(cc)[0])\n    else:\n        connected_components.append(cc)","metadata":{"execution":{"iopub.status.busy":"2022-11-09T22:42:52.786602Z","iopub.execute_input":"2022-11-09T22:42:52.787161Z","iopub.status.idle":"2022-11-09T22:42:52.794821Z","shell.execute_reply.started":"2022-11-09T22:42:52.787113Z","shell.execute_reply":"2022-11-09T22:42:52.793621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize = (30, 8))\npos = nx.nx_agraph.graphviz_layout(G, prog=\"dot\")\nnx.draw(G, pos = pos, node_size=10000)\ntext = nx.draw_networkx_labels(G, pos)\nfor _, t in text.items():\n    t.set_rotation('vertical')\n    t.set_color('white')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-11-09T22:42:54.738730Z","iopub.execute_input":"2022-11-09T22:42:54.740268Z","iopub.status.idle":"2022-11-09T22:42:57.055966Z","shell.execute_reply.started":"2022-11-09T22:42:54.740203Z","shell.execute_reply":"2022-11-09T22:42:57.054724Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nodes_subtree_size = {}\nnodes_cum_votes = {}\nfor node in G.nodes():\n    subtree = dfs_tree(G, node)\n    nodes_subtree_size[node] = len(subtree.nodes()) - 1\n    nodes_cum_votes[node] = sum([votes_dict[n] if n in votes_dict else 0 for n in subtree.nodes()])\n        ","metadata":{"execution":{"iopub.status.busy":"2022-11-09T22:42:58.701507Z","iopub.execute_input":"2022-11-09T22:42:58.701924Z","iopub.status.idle":"2022-11-09T22:42:58.709784Z","shell.execute_reply.started":"2022-11-09T22:42:58.701892Z","shell.execute_reply":"2022-11-09T22:42:58.708488Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"forks = df[['title', 'KernelAuthor', 'id', 'forkParent']]\nforks['TotalForkedKernels'] = forks['id'].map(nodes_subtree_size).fillna(0)\nforks['CumulatedVotes'] = forks['id'].map(nodes_cum_votes).fillna(0)","metadata":{"execution":{"iopub.status.busy":"2022-11-09T22:43:01.667755Z","iopub.execute_input":"2022-11-09T22:43:01.668307Z","iopub.status.idle":"2022-11-09T22:43:01.683916Z","shell.execute_reply.started":"2022-11-09T22:43:01.668259Z","shell.execute_reply":"2022-11-09T22:43:01.682190Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Most forks kernels (top-20)","metadata":{}},{"cell_type":"code","source":"forks.sort_values('TotalForkedKernels', ascending = False).reset_index(drop = True).head(20)","metadata":{"execution":{"iopub.status.busy":"2022-11-09T22:43:03.758243Z","iopub.execute_input":"2022-11-09T22:43:03.758776Z","iopub.status.idle":"2022-11-09T22:43:03.786225Z","shell.execute_reply.started":"2022-11-09T22:43:03.758733Z","shell.execute_reply":"2022-11-09T22:43:03.784859Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Most cumulative votes kernels (top-20) ","metadata":{}},{"cell_type":"code","source":"forks.sort_values('CumulatedVotes', ascending = False).reset_index(drop = True).head(20)","metadata":{"execution":{"iopub.status.busy":"2022-11-09T22:43:05.542127Z","iopub.execute_input":"2022-11-09T22:43:05.542660Z","iopub.status.idle":"2022-11-09T22:43:05.567741Z","shell.execute_reply.started":"2022-11-09T22:43:05.542620Z","shell.execute_reply":"2022-11-09T22:43:05.566338Z"},"trusted":true},"execution_count":null,"outputs":[]}]}