{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "## Logs Analytics Dashboard\n", "This dashboard can be used for performing visual analytics on the server log files. You can interact with any chart in the dashboard, like so:\n", "\n", "1. Click on a bar in any bar chart to filter the whole dataset by that bar's value\n", "2. Click on a slice of any pie chart to filter the whole dataset by that pie slices value\n", "\n", "Fun things to try:\n", "1. Select any specific day in the `Daily Events` bar chart (for e.g. why do first and last days have fewer events?)\n", "2. Find out which product queries resulted in server error (click on `Server Error` slice of `Events By Status` pie chart)\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import numpy as np\n", "import pandas as pd\n", "\n", "import ipywidgets as widgets\n", "import bqplot as bq\n", "import bqplot.pyplot as plt" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "def get_status_code(x):\n", " \"\"\"\n", " map integer http status to string status code\n", " \"\"\"\n", " if x >= 200 and x < 300:\n", " return 'SUCCESS'\n", " elif x >= 300 and x < 400:\n", " return 'REDIRECT'\n", " elif x >= 400 and x < 500:\n", " return 'CLIENT ERROR'\n", " elif x >= 500:\n", " return 'SERVER ERROR'" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "def get_events_by_hour(log_data_slice):\n", " \"\"\"\n", " get event counts by hour\n", " \"\"\"\n", " return log_data_slice['status_code']\\\n", " .groupby(lambda x: x.hour)\\\n", " .count()\\\n", " .reindex(np.arange(24))\\\n", " .fillna(0)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# load data into pandas and do some munging\n", "log_data = pd.read_csv('access.log', sep=' ', header=None)\n", "log_data.drop([1, 2], axis=1, inplace=True)\n", "log_data.columns = ['ip_address', 'timestamp', 'request', 'status', 'col1', 'url', 'agent', 'col2']\n", "\n", "log_data.index = pd.to_datetime(log_data['timestamp'].str.replace('[\\[\\]]', ''), \n", " format='%d/%b/%Y:%H:%M:%S')\n", "log_data.drop('timestamp', axis=1, inplace=True)\n", "\n", "# add extra columns for easy querying\n", "log_data['status_code'] = log_data['status'].map(get_status_code)\n", "log_data['category'] = log_data['request'].str.extract('categoryId=(.*)&', expand=False)\n", "log_data['product'] = log_data['request'].str.extract('productId=(.*)&', expand=False)\n", "log_data['hour'] = log_data.index.map(lambda x: x.hour)\n", "log_data['day'] = log_data.index.strftime('%Y-%m-%d')" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "all_days = log_data['day'].value_counts().index\n", "all_hours = np.arange(24)\n", "all_categories = list(log_data['category'].value_counts().index)\n", "all_products = list(log_data['product'].value_counts().index)\n", "all_status_codes = list(log_data['status_code'].value_counts().index)\n", "\n", "category_colors = dict(zip(all_categories, bq.CATEGORY10))\n", "status_label_colors = dict([('SUCCESS', '#006d2c'), \n", " ('CLIENT ERROR', '#fc8d59'), \n", " ('SERVER ERROR', '#a63603')])" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "scrolled": true }, "outputs": [], "source": [ "# daily events bar chart\n", "daily_events_fig = plt.figure(title='Daily Events', \n", " animation_duration=1000,\n", " layout=widgets.Layout(width='900px', height='500px'))\n", "\n", "plt.scales(scales={'x': bq.DateScale()})\n", "\n", "# intsel = BrushIntervalSelector(scale=, marks=[hist])\n", "bar_axes_options = {'x': {'grid_lines': 'none'},\n", " 'y': {'tick_format': ','}}\n", "common_bar_options = dict(interactions={'click': 'select'},\n", " selected_style={'stroke': 'Red',\n", " 'stroke-width': 4},\n", " axes_options=bar_axes_options)\n", "daily_events_bar = plt.bar(pd.to_datetime(all_days), [],\n", " colors=['dodgerblue'], \n", " opacities=[.8] * len(all_days),\n", " **common_bar_options)\n", "\n", "num_days = len(all_days)\n", "filtered_daily_events_bar = plt.bar(pd.to_datetime(all_days),\n", " np.zeros(num_days),\n", " colors=['lightgreen'], \n", " opacities=[.8] * num_days,\n", " axes_options=bar_axes_options)\n", "\n", "# hourly events line chart\n", "hourly_events_fig = plt.figure(title='Hourly Events',\n", " animation_duration=1000,\n", " layout=widgets.Layout(width='600px', height='500px'))\n", "\n", "plt.scales(scales={'x': bq.OrdinalScale()})\n", "hourly_events_bar = plt.bar(all_hours,[],\n", " colors=['goldenrod'],\n", " opacities=[.8] * 24,\n", " padding=.2,\n", " **common_bar_options)\n", "\n", "filtered_hourly_events_bar = plt.bar(all_hours, [],\n", " colors=['lightgreen'],\n", " opacities=[.8] * 24,\n", " padding=.2,\n", " axes_options=bar_axes_options)\n", "\n", "products_fig = plt.figure(title='Events By Product', \n", " animation_duration=1000,\n", " fig_margin=dict(top=60, bottom=20, left=100, right=40),\n", " layout=widgets.Layout(width='550px', height='500px'))\n", "products_bar = plt.bar(all_products, [],\n", " colors=['salmon'],\n", " orientation='horizontal',\n", " opacities=[.8] * len(all_products),\n", " padding=.2,\n", " **common_bar_options)\n", "\n", "common_pie_args = dict(display_labels='outside',\n", " interactions={'click': 'select'},\n", " selected_style={'stroke': 'white',\n", " 'stroke-width': 3},\n", " inner_radius=80,\n", " apply_clip=False)\n", "# categories pie chart\n", "categories_fig = plt.figure(title='Events By Category', animation_duration=1000,\n", " layout=widgets.Layout(width='550px', height='500px'))\n", "categories_pie = plt.pie([], labels=all_categories, **common_pie_args)\n", "\n", "# status codes pie chart\n", "status_codes_fig = plt.figure(title='Events By Status', animation_duration=1000,\n", " layout=widgets.Layout(width='550px', height='500px'))\n", "status_codes_pie = plt.pie([], labels=all_status_codes, **common_pie_args)\n", "\n", "# buttons for updating and resetting filters\n", "update_btn = widgets.Button(description='Update', button_style='Success')\n", "reset_btn = widgets.Button(description='Reset', button_style='Success')" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "fields = ['day', 'hour', 'product', 'category', 'status_code']\n", "field_vals = [all_days, all_hours, all_products, all_categories, all_status_codes]\n", "plots = [daily_events_bar, hourly_events_bar, products_bar, categories_pie, status_codes_pie]" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "def generate_filters():\n", " filters_dict = {}\n", " for field, field_val, plot in zip(fields, field_vals, plots):\n", " if plot.selected:\n", " selected_vals = [field_val[i] for i in plot.selected]\n", " filters_dict[field] = selected_vals\n", " return filters_dict" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "def apply_filters(df, filters):\n", " filtered_df = df\n", " \n", " for k, v in filters.items():\n", " if v and len(v) > 0:\n", " filtered_df = filtered_df[filtered_df[k].isin(v)]\n", " return filtered_df" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "def update_plots(*args):\n", " global filters, filtered_log_data \n", " filters = generate_filters() \n", " filtered_log_data = apply_filters(log_data, filters)\n", " \n", " daily_events = filtered_log_data\\\n", " .resample('D')\\\n", " .count()['status_code']\\\n", " .reindex(pd.to_datetime(all_days))\\\n", " .fillna(0)\n", " daily_events_bar.y = daily_events\n", " \n", " hourly_events_bar.y = get_events_by_hour(filtered_log_data) \n", " \n", " products_bar.y = filtered_log_data['product']\\\n", " .value_counts()\\\n", " .reindex(all_products)\\\n", " .fillna(0)\n", " \n", " events_by_category = filtered_log_data['category'].value_counts()\n", " \n", " with categories_pie.hold_sync():\n", " categories_pie.labels = list(events_by_category.index)\n", " categories_pie.sizes = events_by_category\n", " categories_pie.colors = [category_colors[d] for d in categories_pie.labels]\n", " \n", " events_by_status_code = filtered_log_data['status_code'].value_counts()\n", " \n", " with status_codes_pie.hold_sync():\n", " status_codes_pie.labels = list(events_by_status_code.index)\n", " status_codes_pie.sizes = events_by_status_code.values\n", " status_codes_pie.colors = [status_label_colors[d] for d in status_codes_pie.labels]" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "def reset_filters(*args):\n", " for plot in plots:\n", " plot.selected = None\n", " update_plots(None)\n", " \n", "update_btn.on_click(lambda btn: update_plots(None))\n", "reset_btn.on_click(lambda btn: reset_filters())\n", "btns_layout = widgets.VBox([update_btn, reset_btn], layout=widgets.Layout(overflow_x='hidden'))" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "scrolled": true }, "outputs": [], "source": [ "update_plots(None)\n", "\n", "parent_plots = widgets.HBox([daily_events_fig, hourly_events_fig])\n", "child_plots = widgets.HBox([products_fig, categories_fig, status_codes_fig])\n", "widgets.VBox([widgets.HBox([parent_plots, btns_layout]),child_plots])" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.2" } }, "nbformat": 4, "nbformat_minor": 1 }