{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install dask","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-02-07T14:53:48.490761Z","iopub.execute_input":"2023-02-07T14:53:48.491242Z","iopub.status.idle":"2023-02-07T14:54:01.699446Z","shell.execute_reply.started":"2023-02-07T14:53:48.491208Z","shell.execute_reply":"2023-02-07T14:54:01.697918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom dask import dataframe as dd\nimport time\nimport seaborn as sns","metadata":{"execution":{"iopub.status.busy":"2023-02-07T14:54:01.702235Z","iopub.execute_input":"2023-02-07T14:54:01.702757Z","iopub.status.idle":"2023-02-07T14:54:01.710205Z","shell.execute_reply.started":"2023-02-07T14:54:01.702714Z","shell.execute_reply":"2023-02-07T14:54:01.708981Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"start = time.time()\ndask_df = dd.read_csv('/kaggle/input/predict-student-performance-from-game-play/train.csv')\nend = time.time()\nprint(\"Read csv with dask: \",(end-start),\"sec\")","metadata":{"execution":{"iopub.status.busy":"2023-02-07T14:54:01.711693Z","iopub.execute_input":"2023-02-07T14:54:01.712200Z","iopub.status.idle":"2023-02-07T14:54:01.753110Z","shell.execute_reply.started":"2023-02-07T14:54:01.712145Z","shell.execute_reply":"2023-02-07T14:54:01.752093Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n# Load the CSV file into a Dask DataFrame\n#df = dd.read_csv('path/to/csvfile.csv')\n\n# Convert the Dask DataFrame to a Parquet file\ndask_df.to_parquet('/kaggle/working/parquetfile.parquet')","metadata":{"execution":{"iopub.status.busy":"2023-02-07T14:54:01.755810Z","iopub.execute_input":"2023-02-07T14:54:01.756657Z","iopub.status.idle":"2023-02-07T14:54:44.005629Z","shell.execute_reply.started":"2023-02-07T14:54:01.756616Z","shell.execute_reply":"2023-02-07T14:54:44.004596Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dask_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-02-07T14:54:44.007087Z","iopub.execute_input":"2023-02-07T14:54:44.007661Z","iopub.status.idle":"2023-02-07T14:54:45.264581Z","shell.execute_reply.started":"2023-02-07T14:54:44.007624Z","shell.execute_reply":"2023-02-07T14:54:45.263552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dask_df.shape","metadata":{"execution":{"iopub.status.busy":"2023-02-07T14:54:45.266354Z","iopub.execute_input":"2023-02-07T14:54:45.271302Z","iopub.status.idle":"2023-02-07T14:54:45.289564Z","shell.execute_reply.started":"2023-02-07T14:54:45.271239Z","shell.execute_reply":"2023-02-07T14:54:45.287925Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dask_df.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2023-02-07T14:54:45.291527Z","iopub.execute_input":"2023-02-07T14:54:45.292313Z","iopub.status.idle":"2023-02-07T14:54:45.317712Z","shell.execute_reply.started":"2023-02-07T14:54:45.292272Z","shell.execute_reply":"2023-02-07T14:54:45.316619Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nplt.figure(figsize=(10,8))\nplt.rcParams.update({'font.size':20})\ndask_df.dtypes.value_counts().plot.pie(explode=[0.1,0.1,0.1],autopct='%1.2f%%',shadow=True)\nplt.title(\"Type sof Data distribution\",color='blue',pad=20);\nplt.axis('off');","metadata":{"execution":{"iopub.status.busy":"2023-02-07T14:54:45.319776Z","iopub.execute_input":"2023-02-07T14:54:45.321344Z","iopub.status.idle":"2023-02-07T14:54:45.484061Z","shell.execute_reply.started":"2023-02-07T14:54:45.321286Z","shell.execute_reply":"2023-02-07T14:54:45.482513Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dask_df.columns","metadata":{"execution":{"iopub.status.busy":"2023-02-07T14:54:45.486050Z","iopub.execute_input":"2023-02-07T14:54:45.487558Z","iopub.status.idle":"2023-02-07T14:54:45.496049Z","shell.execute_reply.started":"2023-02-07T14:54:45.487493Z","shell.execute_reply":"2023-02-07T14:54:45.494911Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sub = pd.read_csv('/kaggle/input/predict-student-performance-from-game-play/sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2023-02-07T14:54:45.504357Z","iopub.execute_input":"2023-02-07T14:54:45.505395Z","iopub.status.idle":"2023-02-07T14:54:45.514683Z","shell.execute_reply.started":"2023-02-07T14:54:45.505350Z","shell.execute_reply":"2023-02-07T14:54:45.513401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_label = pd.read_csv('/kaggle/input/predict-student-performance-from-game-play/train_labels.csv')","metadata":{"execution":{"iopub.status.busy":"2023-02-07T14:54:45.517429Z","iopub.execute_input":"2023-02-07T14:54:45.519159Z","iopub.status.idle":"2023-02-07T14:54:45.760703Z","shell.execute_reply.started":"2023-02-07T14:54:45.519096Z","shell.execute_reply":"2023-02-07T14:54:45.759588Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test = pd.read_csv('/kaggle/input/predict-student-performance-from-game-play/test.csv')","metadata":{"execution":{"iopub.status.busy":"2023-02-07T14:54:45.762477Z","iopub.execute_input":"2023-02-07T14:54:45.763213Z","iopub.status.idle":"2023-02-07T14:54:45.785507Z","shell.execute_reply.started":"2023-02-07T14:54:45.763173Z","shell.execute_reply":"2023-02-07T14:54:45.784454Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_label.columns,df_test.columns,df_sub.columns","metadata":{"execution":{"iopub.status.busy":"2023-02-07T14:54:45.787293Z","iopub.execute_input":"2023-02-07T14:54:45.788051Z","iopub.status.idle":"2023-02-07T14:54:45.800479Z","shell.execute_reply.started":"2023-02-07T14:54:45.788006Z","shell.execute_reply":"2023-02-07T14:54:45.798569Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Read the csv file in chunks\nchunks = pd.read_csv('/kaggle/input/predict-student-performance-from-game-play/train.csv', chunksize=10000)\n\n# Initialize the event_counts dataframe\nevent_count = pd.DataFrame()\n\n# Loop through each chunk and concatenate the result\nfor chunk in chunks:\n    chunk_event_count = chunk['event_name'].value_counts().reset_index()\n    chunk_event_count.columns = ['event_name', 'count']\n    event_count = pd.concat([event_count, chunk_event_count])\n\n# Aggregate the counts for each event\nevent_count = event_count.groupby('event_name').sum().reset_index()\n\n# Plot the bar plot using seaborn\nsns.set_style(\"darkgrid\")\nplt.figure(figsize=(10, 6))\nsns.barplot(x='event_name', y='count', data=event_count, palette='Blues')\nplt.xlabel(\"Event Name\", fontsize=12)\nplt.ylabel(\"Count\", fontsize=12)\nplt.title(\"Count the number of occurrences of each event\", fontsize=16)\nplt.xticks(rotation=90)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-02-07T14:54:45.802143Z","iopub.execute_input":"2023-02-07T14:54:45.802688Z","iopub.status.idle":"2023-02-07T14:55:26.695605Z","shell.execute_reply.started":"2023-02-07T14:54:45.802651Z","shell.execute_reply":"2023-02-07T14:55:26.694038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import seaborn as sns\n# Correlation matrix\ncorr= dask_df.corr()\n#print(corr)\n# Heatmap\nfig, ax = plt.subplots(figsize=(30,15))\nax = sns.heatmap(corr,annot=True, linewidths=.5, fmt='.1f', cmap='YlGnBu')","metadata":{"execution":{"iopub.status.busy":"2023-02-07T14:55:26.697698Z","iopub.execute_input":"2023-02-07T14:55:26.698405Z","iopub.status.idle":"2023-02-07T14:56:37.911270Z","shell.execute_reply.started":"2023-02-07T14:55:26.698244Z","shell.execute_reply":"2023-02-07T14:56:37.908013Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dask_df.columns","metadata":{"execution":{"iopub.status.busy":"2023-02-07T14:56:37.913946Z","iopub.execute_input":"2023-02-07T14:56:37.914781Z","iopub.status.idle":"2023-02-07T14:56:37.925653Z","shell.execute_reply.started":"2023-02-07T14:56:37.914714Z","shell.execute_reply":"2023-02-07T14:56:37.924438Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Read the csv file in chunks\nchunks = pd.read_csv('/kaggle/input/predict-student-performance-from-game-play/train.csv', chunksize=10000)","metadata":{"execution":{"iopub.status.busy":"2023-02-07T14:56:37.927666Z","iopub.execute_input":"2023-02-07T14:56:37.929310Z","iopub.status.idle":"2023-02-07T14:56:37.948717Z","shell.execute_reply.started":"2023-02-07T14:56:37.929244Z","shell.execute_reply":"2023-02-07T14:56:37.946922Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.DataFrame()","metadata":{"execution":{"iopub.status.busy":"2023-02-07T14:56:37.950413Z","iopub.execute_input":"2023-02-07T14:56:37.950797Z","iopub.status.idle":"2023-02-07T14:56:37.959904Z","shell.execute_reply.started":"2023-02-07T14:56:37.950764Z","shell.execute_reply":"2023-02-07T14:56:37.957947Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for chunk in chunks:  \n   \n    df = pd.concat([chunk])\n","metadata":{"execution":{"iopub.status.busy":"2023-02-07T14:56:37.962350Z","iopub.execute_input":"2023-02-07T14:56:37.962830Z","iopub.status.idle":"2023-02-07T14:57:17.490018Z","shell.execute_reply.started":"2023-02-07T14:56:37.962797Z","shell.execute_reply":"2023-02-07T14:57:17.488700Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head()","metadata":{"execution":{"iopub.status.busy":"2023-02-07T14:57:17.491393Z","iopub.execute_input":"2023-02-07T14:57:17.491761Z","iopub.status.idle":"2023-02-07T14:57:17.520219Z","shell.execute_reply.started":"2023-02-07T14:57:17.491728Z","shell.execute_reply":"2023-02-07T14:57:17.518617Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.columns","metadata":{"execution":{"iopub.status.busy":"2023-02-07T14:57:17.522249Z","iopub.execute_input":"2023-02-07T14:57:17.522982Z","iopub.status.idle":"2023-02-07T14:57:17.537550Z","shell.execute_reply.started":"2023-02-07T14:57:17.522942Z","shell.execute_reply":"2023-02-07T14:57:17.536111Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.info()","metadata":{"execution":{"iopub.status.busy":"2023-02-07T14:57:17.539646Z","iopub.execute_input":"2023-02-07T14:57:17.540552Z","iopub.status.idle":"2023-02-07T14:57:17.566647Z","shell.execute_reply.started":"2023-02-07T14:57:17.540508Z","shell.execute_reply":"2023-02-07T14:57:17.565677Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2023-02-07T14:57:17.568555Z","iopub.execute_input":"2023-02-07T14:57:17.569240Z","iopub.status.idle":"2023-02-07T14:57:17.582841Z","shell.execute_reply.started":"2023-02-07T14:57:17.569201Z","shell.execute_reply":"2023-02-07T14:57:17.581505Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Lets identify the categorical feature\n\ndf_cat = df.select_dtypes(include='O').keys()\ndf_cat","metadata":{"execution":{"iopub.status.busy":"2023-02-07T14:57:17.584548Z","iopub.execute_input":"2023-02-07T14:57:17.585248Z","iopub.status.idle":"2023-02-07T14:57:17.596870Z","shell.execute_reply.started":"2023-02-07T14:57:17.585209Z","shell.execute_reply":"2023-02-07T14:57:17.595442Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for cat in df_cat:\n    print(f'{cat}:{df[cat].unique()}')","metadata":{"execution":{"iopub.status.busy":"2023-02-07T14:57:17.599454Z","iopub.execute_input":"2023-02-07T14:57:17.600336Z","iopub.status.idle":"2023-02-07T14:57:17.618469Z","shell.execute_reply.started":"2023-02-07T14:57:17.600285Z","shell.execute_reply":"2023-02-07T14:57:17.616880Z"},"_kg_hide-output":true,"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"name = df['name'].value_counts().reset_index()\nname.columns = ['name', 'count']\n\nplt.figure(figsize=(10, 6))\nsns.barplot(x='name', y='count', data=name, palette='Blues')\nplt.xlabel(\"Name\", fontsize=12)\nplt.ylabel(\"Count\", fontsize=12)\nplt.title(\"Count of occurrences of each event\", fontsize=15)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-02-07T14:57:17.620784Z","iopub.execute_input":"2023-02-07T14:57:17.622061Z","iopub.status.idle":"2023-02-07T14:57:17.824616Z","shell.execute_reply.started":"2023-02-07T14:57:17.621809Z","shell.execute_reply":"2023-02-07T14:57:17.823346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"level_group = df['level_group'].value_counts().reset_index()\nlevel_group.columns = ['level_group', 'count']\n\nplt.figure(figsize=(10, 6))\nsns.barplot(x='level_group', y='count', data=level_group, palette='Blues')\nplt.xlabel(\"level_group\", fontsize=12)\nplt.ylabel(\"Count\", fontsize=12)\nplt.title(\"Count of occurrences of each event\", fontsize=15)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-02-07T14:57:17.826335Z","iopub.execute_input":"2023-02-07T14:57:17.827289Z","iopub.status.idle":"2023-02-07T14:57:18.000715Z","shell.execute_reply.started":"2023-02-07T14:57:17.827236Z","shell.execute_reply":"2023-02-07T14:57:17.999325Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from wordcloud import WordCloud\n","metadata":{"execution":{"iopub.status.busy":"2023-02-07T14:57:18.007496Z","iopub.execute_input":"2023-02-07T14:57:18.008160Z","iopub.status.idle":"2023-02-07T14:57:18.012648Z","shell.execute_reply.started":"2023-02-07T14:57:18.008121Z","shell.execute_reply":"2023-02-07T14:57:18.011618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['fqid'].isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2023-02-07T14:57:18.014174Z","iopub.execute_input":"2023-02-07T14:57:18.014811Z","iopub.status.idle":"2023-02-07T14:57:18.030346Z","shell.execute_reply.started":"2023-02-07T14:57:18.014772Z","shell.execute_reply":"2023-02-07T14:57:18.029262Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['fqid'] =df['fqid'].fillna('unknown')","metadata":{"execution":{"iopub.status.busy":"2023-02-07T14:57:18.032283Z","iopub.execute_input":"2023-02-07T14:57:18.033104Z","iopub.status.idle":"2023-02-07T14:57:18.043445Z","shell.execute_reply.started":"2023-02-07T14:57:18.032976Z","shell.execute_reply":"2023-02-07T14:57:18.042118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"text = \" \".join(df['fqid'].astype(str))\nwordcloud = WordCloud().generate(text)\nplt.figure(figsize=(15,8))\nplt.imshow(wordcloud, interpolation='bilinear')\nplt.axis(\"off\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-02-07T14:57:18.045402Z","iopub.execute_input":"2023-02-07T14:57:18.046006Z","iopub.status.idle":"2023-02-07T14:57:18.590063Z","shell.execute_reply.started":"2023-02-07T14:57:18.045966Z","shell.execute_reply":"2023-02-07T14:57:18.589062Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['room_fqid'] =df['room_fqid'].fillna('unknown')","metadata":{"execution":{"iopub.status.busy":"2023-02-07T14:57:18.592101Z","iopub.execute_input":"2023-02-07T14:57:18.592888Z","iopub.status.idle":"2023-02-07T14:57:18.601278Z","shell.execute_reply.started":"2023-02-07T14:57:18.592829Z","shell.execute_reply":"2023-02-07T14:57:18.599680Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"text = \" \".join(df['room_fqid'].astype(str))\nwordcloud = WordCloud().generate(text)\nplt.figure(figsize=(15,8))\nplt.imshow(wordcloud, interpolation='bilinear')\nplt.axis(\"off\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-02-07T14:57:18.603248Z","iopub.execute_input":"2023-02-07T14:57:18.604201Z","iopub.status.idle":"2023-02-07T14:57:19.045134Z","shell.execute_reply.started":"2023-02-07T14:57:18.604125Z","shell.execute_reply":"2023-02-07T14:57:19.043965Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"text = \" \".join(df['text'].astype(str))\nwordcloud = WordCloud().generate(text)\nplt.figure(figsize=(15,8))\nplt.imshow(wordcloud, interpolation='bilinear')\nplt.axis(\"off\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-02-07T14:57:19.046714Z","iopub.execute_input":"2023-02-07T14:57:19.047373Z","iopub.status.idle":"2023-02-07T14:57:19.638947Z","shell.execute_reply.started":"2023-02-07T14:57:19.047333Z","shell.execute_reply":"2023-02-07T14:57:19.637933Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.columns","metadata":{"execution":{"iopub.status.busy":"2023-02-07T14:57:19.640591Z","iopub.execute_input":"2023-02-07T14:57:19.641224Z","iopub.status.idle":"2023-02-07T14:57:19.650896Z","shell.execute_reply.started":"2023-02-07T14:57:19.641187Z","shell.execute_reply":"2023-02-07T14:57:19.648983Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10,8))\nsns.boxplot(x='level', y='level_group', data=df)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-02-07T14:57:19.653346Z","iopub.execute_input":"2023-02-07T14:57:19.654713Z","iopub.status.idle":"2023-02-07T14:57:19.880934Z","shell.execute_reply.started":"2023-02-07T14:57:19.654645Z","shell.execute_reply":"2023-02-07T14:57:19.879069Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}