Debian Codegraph @

bqplot/examples/Applications/Logs Analytics/Logs Dashboard.ipynb

raw
1 {
2 "cells": [
3 {
4 "cell_type": "markdown",
5 "metadata": {},
6 "source": [
7 "## Logs Analytics Dashboard\n",
8 "This dashboard can be used for performing visual analytics on the server log files. You can interact with any chart in the dashboard, like so:\n",
9 "\n",
10 "1. Click on a bar in any bar chart to filter the whole dataset by that bar's value\n",
11 "2. Click on a slice of any pie chart to filter the whole dataset by that pie slices value\n",
12 "\n",
13 "Fun things to try:\n",
14 "1. Select any specific day in the `Daily Events` bar chart (for e.g. why do first and last days have fewer events?)\n",
15 "2. Find out which product queries resulted in server error (click on `Server Error` slice of `Events By Status` pie chart)\n"
16 ]
17 },
18 {
19 "cell_type": "code",
20 "execution_count": null,
21 "metadata": {},
22 "outputs": [],
23 "source": [
24 "import numpy as np\n",
25 "import pandas as pd\n",
26 "\n",
27 "import ipywidgets as widgets\n",
28 "import bqplot as bq\n",
29 "import bqplot.pyplot as plt"
30 ]
31 },
32 {
33 "cell_type": "code",
34 "execution_count": null,
35 "metadata": {},
36 "outputs": [],
37 "source": [
38 "def get_status_code(x):\n",
39 " \"\"\"\n",
40 " map integer http status to string status code\n",
41 " \"\"\"\n",
42 " if x >= 200 and x < 300:\n",
43 " return 'SUCCESS'\n",
44 " elif x >= 300 and x < 400:\n",
45 " return 'REDIRECT'\n",
46 " elif x >= 400 and x < 500:\n",
47 " return 'CLIENT ERROR'\n",
48 " elif x >= 500:\n",
49 " return 'SERVER ERROR'"
50 ]
51 },
52 {
53 "cell_type": "code",
54 "execution_count": null,
55 "metadata": {},
56 "outputs": [],
57 "source": [
58 "def get_events_by_hour(log_data_slice):\n",
59 " \"\"\"\n",
60 " get event counts by hour\n",
61 " \"\"\"\n",
62 " return log_data_slice['status_code']\\\n",
63 " .groupby(lambda x: x.hour)\\\n",
64 " .count()\\\n",
65 " .reindex(np.arange(24))\\\n",
66 " .fillna(0)"
67 ]
68 },
69 {
70 "cell_type": "code",
71 "execution_count": null,
72 "metadata": {},
73 "outputs": [],
74 "source": [
75 "# load data into pandas and do some munging\n",
76 "log_data = pd.read_csv('access.log', sep=' ', header=None)\n",
77 "log_data.drop([1, 2], axis=1, inplace=True)\n",
78 "log_data.columns = ['ip_address', 'timestamp', 'request', 'status', 'col1', 'url', 'agent', 'col2']\n",
79 "\n",
80 "log_data.index = pd.to_datetime(log_data['timestamp'].str.replace('[\\[\\]]', ''), \n",
81 " format='%d/%b/%Y:%H:%M:%S')\n",
82 "log_data.drop('timestamp', axis=1, inplace=True)\n",
83 "\n",
84 "# add extra columns for easy querying\n",
85 "log_data['status_code'] = log_data['status'].map(get_status_code)\n",
86 "log_data['category'] = log_data['request'].str.extract('categoryId=(.*)&', expand=False)\n",
87 "log_data['product'] = log_data['request'].str.extract('productId=(.*)&', expand=False)\n",
88 "log_data['hour'] = log_data.index.map(lambda x: x.hour)\n",
89 "log_data['day'] = log_data.index.strftime('%Y-%m-%d')"
90 ]
91 },
92 {
93 "cell_type": "code",
94 "execution_count": null,
95 "metadata": {},
96 "outputs": [],
97 "source": [
98 "all_days = log_data['day'].value_counts().index\n",
99 "all_hours = np.arange(24)\n",
100 "all_categories = list(log_data['category'].value_counts().index)\n",
101 "all_products = list(log_data['product'].value_counts().index)\n",
102 "all_status_codes = list(log_data['status_code'].value_counts().index)\n",
103 "\n",
104 "category_colors = dict(zip(all_categories, bq.CATEGORY10))\n",
105 "status_label_colors = dict([('SUCCESS', '#006d2c'), \n",
106 " ('CLIENT ERROR', '#fc8d59'), \n",
107 " ('SERVER ERROR', '#a63603')])"
108 ]
109 },
110 {
111 "cell_type": "code",
112 "execution_count": null,
113 "metadata": {
114 "scrolled": true
115 },
116 "outputs": [],
117 "source": [
118 "# daily events bar chart\n",
119 "daily_events_fig = plt.figure(title='Daily Events', \n",
120 " animation_duration=1000,\n",
121 " layout=widgets.Layout(width='900px', height='500px'))\n",
122 "\n",
123 "plt.scales(scales={'x': bq.DateScale()})\n",
124 "\n",
125 "# intsel = BrushIntervalSelector(scale=, marks=[hist])\n",
126 "bar_axes_options = {'x': {'grid_lines': 'none'},\n",
127 " 'y': {'tick_format': ','}}\n",
128 "common_bar_options = dict(interactions={'click': 'select'},\n",
129 " selected_style={'stroke': 'Red',\n",
130 " 'stroke-width': 4},\n",
131 " axes_options=bar_axes_options)\n",
132 "daily_events_bar = plt.bar(pd.to_datetime(all_days), [],\n",
133 " colors=['dodgerblue'], \n",
134 " opacities=[.8] * len(all_days),\n",
135 " **common_bar_options)\n",
136 "\n",
137 "num_days = len(all_days)\n",
138 "filtered_daily_events_bar = plt.bar(pd.to_datetime(all_days),\n",
139 " np.zeros(num_days),\n",
140 " colors=['lightgreen'], \n",
141 " opacities=[.8] * num_days,\n",
142 " axes_options=bar_axes_options)\n",
143 "\n",
144 "# hourly events line chart\n",
145 "hourly_events_fig = plt.figure(title='Hourly Events',\n",
146 " animation_duration=1000,\n",
147 " layout=widgets.Layout(width='600px', height='500px'))\n",
148 "\n",
149 "plt.scales(scales={'x': bq.OrdinalScale()})\n",
150 "hourly_events_bar = plt.bar(all_hours,[],\n",
151 " colors=['goldenrod'],\n",
152 " opacities=[.8] * 24,\n",
153 " padding=.2,\n",
154 " **common_bar_options)\n",
155 "\n",
156 "filtered_hourly_events_bar = plt.bar(all_hours, [],\n",
157 " colors=['lightgreen'],\n",
158 " opacities=[.8] * 24,\n",
159 " padding=.2,\n",
160 " axes_options=bar_axes_options)\n",
161 "\n",
162 "products_fig = plt.figure(title='Events By Product', \n",
163 " animation_duration=1000,\n",
164 " fig_margin=dict(top=60, bottom=20, left=100, right=40),\n",
165 " layout=widgets.Layout(width='550px', height='500px'))\n",
166 "products_bar = plt.bar(all_products, [],\n",
167 " colors=['salmon'],\n",
168 " orientation='horizontal',\n",
169 " opacities=[.8] * len(all_products),\n",
170 " padding=.2,\n",
171 " **common_bar_options)\n",
172 "\n",
173 "common_pie_args = dict(display_labels='outside',\n",
174 " interactions={'click': 'select'},\n",
175 " selected_style={'stroke': 'white',\n",
176 " 'stroke-width': 3},\n",
177 " inner_radius=80,\n",
178 " apply_clip=False)\n",
179 "# categories pie chart\n",
180 "categories_fig = plt.figure(title='Events By Category', animation_duration=1000,\n",
181 " layout=widgets.Layout(width='550px', height='500px'))\n",
182 "categories_pie = plt.pie([], labels=all_categories, **common_pie_args)\n",
183 "\n",
184 "# status codes pie chart\n",
185 "status_codes_fig = plt.figure(title='Events By Status', animation_duration=1000,\n",
186 " layout=widgets.Layout(width='550px', height='500px'))\n",
187 "status_codes_pie = plt.pie([], labels=all_status_codes, **common_pie_args)\n",
188 "\n",
189 "# buttons for updating and resetting filters\n",
190 "update_btn = widgets.Button(description='Update', button_style='Success')\n",
191 "reset_btn = widgets.Button(description='Reset', button_style='Success')"
192 ]
193 },
194 {
195 "cell_type": "code",
196 "execution_count": null,
197 "metadata": {},
198 "outputs": [],
199 "source": [
200 "fields = ['day', 'hour', 'product', 'category', 'status_code']\n",
201 "field_vals = [all_days, all_hours, all_products, all_categories, all_status_codes]\n",
202 "plots = [daily_events_bar, hourly_events_bar, products_bar, categories_pie, status_codes_pie]"
203 ]
204 },
205 {
206 "cell_type": "code",
207 "execution_count": null,
208 "metadata": {},
209 "outputs": [],
210 "source": [
211 "def generate_filters():\n",
212 " filters_dict = {}\n",
213 " for field, field_val, plot in zip(fields, field_vals, plots):\n",
214 " if plot.selected:\n",
215 " selected_vals = [field_val[i] for i in plot.selected]\n",
216 " filters_dict[field] = selected_vals\n",
217 " return filters_dict"
218 ]
219 },
220 {
221 "cell_type": "code",
222 "execution_count": null,
223 "metadata": {},
224 "outputs": [],
225 "source": [
226 "def apply_filters(df, filters):\n",
227 " filtered_df = df\n",
228 " \n",
229 " for k, v in filters.items():\n",
230 " if v and len(v) > 0:\n",
231 " filtered_df = filtered_df[filtered_df[k].isin(v)]\n",
232 " return filtered_df"
233 ]
234 },
235 {
236 "cell_type": "code",
237 "execution_count": null,
238 "metadata": {},
239 "outputs": [],
240 "source": [
241 "def update_plots(*args):\n",
242 " global filters, filtered_log_data \n",
243 " filters = generate_filters() \n",
244 " filtered_log_data = apply_filters(log_data, filters)\n",
245 " \n",
246 " daily_events = filtered_log_data\\\n",
247 " .resample('D')\\\n",
248 " .count()['status_code']\\\n",
249 " .reindex(pd.to_datetime(all_days))\\\n",
250 " .fillna(0)\n",
251 " daily_events_bar.y = daily_events\n",
252 " \n",
253 " hourly_events_bar.y = get_events_by_hour(filtered_log_data) \n",
254 " \n",
255 " products_bar.y = filtered_log_data['product']\\\n",
256 " .value_counts()\\\n",
257 " .reindex(all_products)\\\n",
258 " .fillna(0)\n",
259 " \n",
260 " events_by_category = filtered_log_data['category'].value_counts()\n",
261 " \n",
262 " with categories_pie.hold_sync():\n",
263 " categories_pie.labels = list(events_by_category.index)\n",
264 " categories_pie.sizes = events_by_category\n",
265 " categories_pie.colors = [category_colors[d] for d in categories_pie.labels]\n",
266 " \n",
267 " events_by_status_code = filtered_log_data['status_code'].value_counts()\n",
268 " \n",
269 " with status_codes_pie.hold_sync():\n",
270 " status_codes_pie.labels = list(events_by_status_code.index)\n",
271 " status_codes_pie.sizes = events_by_status_code.values\n",
272 " status_codes_pie.colors = [status_label_colors[d] for d in status_codes_pie.labels]"
273 ]
274 },
275 {
276 "cell_type": "code",
277 "execution_count": null,
278 "metadata": {},
279 "outputs": [],
280 "source": [
281 "def reset_filters(*args):\n",
282 " for plot in plots:\n",
283 " plot.selected = None\n",
284 " update_plots(None)\n",
285 " \n",
286 "update_btn.on_click(lambda btn: update_plots(None))\n",
287 "reset_btn.on_click(lambda btn: reset_filters())\n",
288 "btns_layout = widgets.VBox([update_btn, reset_btn], layout=widgets.Layout(overflow_x='hidden'))"
289 ]
290 },
291 {
292 "cell_type": "code",
293 "execution_count": null,
294 "metadata": {
295 "scrolled": true
296 },
297 "outputs": [],
298 "source": [
299 "update_plots(None)\n",
300 "\n",
301 "parent_plots = widgets.HBox([daily_events_fig, hourly_events_fig])\n",
302 "child_plots = widgets.HBox([products_fig, categories_fig, status_codes_fig])\n",
303 "widgets.VBox([widgets.HBox([parent_plots, btns_layout]),child_plots])"
304 ]
305 }
306 ],
307 "metadata": {
308 "kernelspec": {
309 "display_name": "Python 3",
310 "language": "python",
311 "name": "python3"
312 },
313 "language_info": {
314 "codemirror_mode": {
315 "name": "ipython",
316 "version": 3
317 },
318 "file_extension": ".py",
319 "mimetype": "text/x-python",
320 "name": "python",
321 "nbconvert_exporter": "python",
322 "pygments_lexer": "ipython3",
323 "version": "3.8.2"
324 }
325 },
326 "nbformat": 4,
327 "nbformat_minor": 1
328 }