Debian Codegraph @

ipydatagrid/datagrid.py

raw
1 # Copyright (c) NumFOCUS.
2 # Distributed under the terms of the Modified BSD License.
3
4 import datetime
5 import decimal
6 import warnings
7 from collections.abc import Iterator
8 from copy import deepcopy
9 from math import floor
10
11 import numpy as np
12 import pandas as pd
13 from bqplot.traits import array_from_json, array_to_json
14 from ipywidgets import CallbackDispatcher, DOMWidget, widget_serialization
15 from traitlets import (
16 Bool,
17 Dict,
18 Enum,
19 Instance,
20 Int,
21 List,
22 Unicode,
23 default,
24 observe,
25 validate,
26 )
27
28 from ._frontend import module_name, module_version
29 from .cellrenderer import BarRenderer, CellRenderer, TextRenderer
30
31
32 class SelectionIterator(Iterator):
33 def __init__(self, selections):
34 self._rect_index = 0
35 self._cell_index = 0
36 self._selections = selections
37
38 def __next__(self):
39 if self._rect_index >= len(self._selections):
40 raise StopIteration
41
42 rect = self._selections[self._rect_index]
43 row_col = self._index_to_row_col(rect, self._cell_index)
44 self._cell_index += 1
45
46 if row_col is None:
47 self._rect_index += 1
48 self._cell_index = 0
49 return self.__next__()
50 elif self._cell_in_previous_selected_rects(row_col):
51 return self.__next__()
52 else:
53 return row_col
54
55 @staticmethod
56 def _index_to_row_col(rect, index):
57 num_rows = rect["r2"] - rect["r1"] + 1
58 num_cols = rect["c2"] - rect["c1"] + 1
59 if index > (num_rows * num_cols - 1):
60 return None
61
62 return {
63 "r": rect["r1"] + floor(index / num_cols),
64 "c": rect["c1"] + index % num_cols,
65 }
66
67 def _cell_in_previous_selected_rects(self, cell):
68 return any(
69 self._cell_in_rect(cell, self._selections[i])
70 for i in range(0, self._rect_index)
71 )
72
73 @staticmethod
74 def _cell_in_rect(cell, rect):
75 return (
76 rect["r1"] <= cell["r"] <= rect["r2"]
77 and rect["c1"] <= cell["c"] <= rect["c2"]
78 )
79
80
81 class SelectionHelper:
82 """A Helper Class for processing selections. Provides an iterator
83 to traverse selected cells.
84 """
85
86 def __init__(self, data, selections, selection_mode, **kwargs):
87 super().__init__(**kwargs)
88 self._data = data
89 self._selections = selections
90 self._selection_mode = selection_mode
91 self._num_columns = -1
92 self._num_rows = -1
93
94 def __iter__(self):
95 selections = [
96 self._transform_rect_for_selection_mode(rect)
97 for rect in self._selections
98 ]
99 return SelectionIterator(selections)
100
101 def __len__(self):
102 return sum(1 for _ in self)
103
104 def all(self):
105 """
106 Returns all selected cells as a list. Each cell is
107 represented as a dictionary
108 with keys 'r': row and 'c': column
109 """
110 return list(self)
111
112 def all_values(self):
113 """
114 Returns values for all selected cells as a list.
115 """
116 return [
117 DataGrid._get_cell_value_by_numerical_index(
118 self._data, cell["c"], cell["r"]
119 )
120 for cell in self
121 ]
122
123 def _transform_rect_for_selection_mode(self, rect):
124 selection_mode = self._selection_mode
125 if selection_mode == "row":
126 return {
127 "r1": rect["r1"],
128 "c1": 0,
129 "r2": rect["r2"],
130 "c2": self._get_num_columns() - 1,
131 }
132 elif selection_mode == "column":
133 return {
134 "r1": 0,
135 "c1": rect["c1"],
136 "r2": self._get_num_rows() - 1,
137 "c2": rect["c2"],
138 }
139 else:
140 return rect
141
142 def _get_num_columns(self):
143 if self._num_columns != -1:
144 return self._num_columns
145
146 data = self._data
147 primary_keys = (
148 []
149 if "primaryKey" not in data["schema"]
150 else data["schema"]["primaryKey"]
151 )
152 col_headers = [
153 field["name"]
154 for field in data["schema"]["fields"]
155 if field["name"] not in primary_keys
156 ]
157 self._num_columns = len(col_headers)
158 return self._num_columns
159
160 def _get_num_rows(self):
161 if self._num_rows != -1:
162 return self._num_rows
163
164 data = self._data
165 self._num_rows = 0 if "data" not in data else len(data["data"])
166 return self._num_rows
167
168
169 def _data_to_json(x, _):
170 if isinstance(x, dict):
171 return {str(k): _data_to_json(v, _) for k, v in x.items()}
172 if isinstance(x, np.ndarray):
173 return _data_to_json(x.tolist(), _)
174 if isinstance(x, (list, tuple)):
175 return [_data_to_json(v, _) for v in x]
176 if isinstance(x, int):
177 return x
178 if isinstance(x, float):
179 if np.isnan(x):
180 return "$NaN$"
181 if np.isposinf(x):
182 return "$Infinity$"
183 if np.isneginf(x):
184 return "$NegInfinity$"
185 return x
186 if isinstance(x, decimal.Decimal):
187 return str(x)
188 if isinstance(x, (datetime.datetime, datetime.date)):
189 return x.isoformat()
190 if x is pd.NaT:
191 return "$NaT$"
192 if pd.isna(x):
193 return "$NaN$"
194 return str(x)
195
196
197 def _data_serialization_impl(data, _):
198 if not data:
199 return {}
200
201 serialized_data = {}
202 for column, value in data["data"].items():
203 arr = value.to_numpy()
204 if arr.size == 0:
205 serialized_data[str(column)] = {
206 "value": [],
207 "dtype": str(arr.dtype),
208 "shape": arr.shape,
209 "type": None,
210 }
211 continue
212 try:
213 serialized_data[str(column)] = array_to_json(arr)
214 except ValueError:
215 # Column is most likely heterogeneous, sending the column raw
216 serialized_data[str(column)] = {
217 "value": _data_to_json(arr, _),
218 "type": "raw",
219 }
220
221 return {
222 "data": serialized_data,
223 "schema": data["schema"],
224 "fields": _data_to_json(data["fields"], _),
225 }
226
227
228 def _data_deserialization_impl(data, _): # noqa: U101
229 if not data:
230 return {}
231
232 deserialized_data = {}
233 for column, value in data["data"].items():
234 deserialized_data[column] = array_from_json(value.to_numpy())
235
236 return {
237 "data": deserialized_data,
238 "schema": data["schema"],
239 "fields": data["fields"],
240 }
241
242
243 _data_serialization = {
244 "from_json": _data_deserialization_impl,
245 "to_json": _data_serialization_impl,
246 }
247
248
249 def _widgets_dict_to_json(x, obj):
250 return {
251 str(k): widget_serialization["to_json"](v, obj) for k, v in x.items()
252 }
253
254
255 _widgets_dict_serialization = {
256 "from_json": widget_serialization["from_json"],
257 "to_json": _widgets_dict_to_json,
258 }
259
260
261 class DataGrid(DOMWidget):
262 """A Grid Widget with filter, sort and selection capabilities.
263
264 Attributes
265 ----------
266 base_row_size : int (default: 20)
267 Default row height
268 base_column_size : int (default: 64)
269 Default column width
270 base_row_header_size : int (default: 64)
271 Default row header width
272 base_column_header_size : int (default: 20)
273 Default column header height
274 header_visibility : {'all', 'row', 'column', 'none'} (default: 'all')
275 Header visibility mode
276 'all': both row and column headers visible
277 'row': only row headers visible
278 'column': only column headers visible
279 'none': neither row and column headers visible
280 dataframe : pandas dataframe
281 Data to display on Data Grid.
282 renderers : dict
283 Custom renderers to use for cell rendering. Keys of dictionary specify
284 column name, and value specifies the renderer
285 default_renderer : CellRenderer (default: TextRenderer)
286 Default renderer to use for cell rendering
287 header_renderer : CellRenderer (default: TextRenderer)
288 Renderer to use for header cell rendering
289 corner_renderer : CellRenderer (default: TextRenderer)
290 Renderer to use for corner header cell rendering
291 selection_mode : {'row', 'column', 'cell', 'none'} (default: 'none')
292 Selection mode used when user clicks on grid or makes selections
293 programmatically.
294 'row': Selecting a cell will select all the cells on the same row
295 'column': Selecting a cell will select all the cells on the same column
296 'cell': Individual cell selection
297 'none': Selection disabled
298 selections : list of dict
299 List of all selections. Selections are represented as rectangular
300 regions. Rectangles are defined as dictionaries with keys:
301 'r1': start row, 'c1': start column, 'r2': end row, 'c2': end column.
302 Start of rectangle is top-left corner and end is bottom-right corner
303 editable : boolean (default: false)
304 Boolean indicating whether cell grid can be directly edited
305 column_widths : Dict of strings to int (default: {})
306 Dict to specify custom column sizes
307 The keys (strings) indicate the names of the columns
308 The values (integers) indicate the widths
309 auto_fit_columns : Bool (default: False)
310 Specify whether column width should automatically be
311 determined by the grid
312 auto_fit_params : Dict. Specify column auto fit parameters.
313 Supported parameters:
314 1) area: where to resize column widths - 'row-header',
315 'body' or 'all' (default)
316 2) padding: add padding to resized column widths (15 pixels by default)
317 3) numCols: cap the number of columns to be resized (None)
318 grid_style : Dict of {propertyName: string | VegaExpr | Dict}
319 Dict to specify global grid styles.
320 The keys (strings) indicate the styling property
321 The values (css color properties or Vega Expression) indicate the values
322 See below for all supported styling properties
323 index_name : str (default: "key")
324 String to specify the index column name. **Only set when the grid
325 is constructed and is not an observable traitlet**
326 horizontal_stripes : bool (default: False)
327 Enable themed coloring of alternate grid rows
328 vertical_stripes : bool (default: False)
329 Enable themed coloring of alternate grid columns
330
331 Accessors (not observable traitlets)
332 ---------
333 selected_cells : list of dict
334 List of selected cells. Each cell is represented as a dictionary
335 with keys 'r': row and 'c': column
336 selected_cell_values : list
337 List of values for all selected cells.
338 selected_cell_iterator : iterator
339 An iterator to traverse selected cells one by one.
340
341 Supported styling properties:
342 void_color : color of the area where the grid is not painted
343 on the canvas
344 background_color : background color for all body cells
345 row_background_color : row-wise background color (can take
346 a string or Vega Expression)
347 column_background_color : column-wise background color (can take a
348 string of Vega Expression)
349 grid_line_color : color of both vertical and horizontal grid lines
350 vertical_grid_line_color : vertical grid line color
351 horizontal_grid_line_color : horizontal grid line color
352 header_background_color : background color for all non-body cells
353 (index and columns)
354 header_grid_line_color : grid line color for all non-body
355 cells (index and columns)
356 header_vertical_grid_line_color : vertical grid line color
357 for all non-body cells
358 header_horizontal_grid_line_color : horizontal grid line color
359 for all non-body cells
360 selection_fill_color : fill color of selected area
361 selection_border_color : border color of selected area
362 header_selection_fill_color : fill color of headers intersecting with
363 selected area at column or row
364 header_selection_border_color : border color of headers
365 intersecting with selected area at column or row
366 cursor_fill_color : fill color of cursor
367 cursor_border_color : border color of cursor
368 scroll_shadow : Dict of color parameters for scroll shadow (vertical and
369 horizontal). Takes three parameters:
370 size : size of shadow in pixels
371 color1 : gradient color 1
372 color2 : gradient color 2
373 color3 : gradient color 3
374 """
375
376 _model_name = Unicode("DataGridModel").tag(sync=True)
377 _model_module = Unicode(module_name).tag(sync=True)
378 _model_module_version = Unicode(module_version).tag(sync=True)
379 _view_name = Unicode("DataGridView").tag(sync=True)
380 _view_module = Unicode(module_name).tag(sync=True)
381 _view_module_version = Unicode(module_version).tag(sync=True)
382
383 base_row_size = Int(20).tag(sync=True)
384 base_column_size = Int(64).tag(sync=True)
385 base_row_header_size = Int(64).tag(sync=True)
386 base_column_header_size = Int(20).tag(sync=True)
387
388 header_visibility = Enum(
389 default_value="all", values=["all", "row", "column", "none"]
390 ).tag(sync=True)
391
392 _transforms = List(Dict()).tag(sync=True, **widget_serialization)
393 _visible_rows = List(Int()).tag(sync=True)
394 _data = Dict().tag(sync=True, **_data_serialization)
395
396 renderers = Dict(Instance(CellRenderer)).tag(
397 sync=True, **_widgets_dict_serialization
398 )
399 default_renderer = Instance(CellRenderer).tag(
400 sync=True, **widget_serialization
401 )
402 header_renderer = Instance(CellRenderer, allow_none=True).tag(
403 sync=True, **widget_serialization
404 )
405 corner_renderer = Instance(CellRenderer, allow_none=True).tag(
406 sync=True, **widget_serialization
407 )
408 selection_mode = Enum(
409 default_value="none", values=["row", "column", "cell", "none"]
410 ).tag(sync=True)
411 selections = List(Dict()).tag(sync=True)
412 editable = Bool(False).tag(sync=True)
413 column_widths = Dict({}).tag(sync=True, to_json=_data_to_json)
414 grid_style = Dict(allow_none=True).tag(
415 sync=True, **_widgets_dict_serialization
416 )
417 auto_fit_columns = Bool(False).tag(sync=True)
418 auto_fit_params = Dict(
419 {"area": "all", "padding": 30, "numCols": None}, allow_none=False
420 ).tag(sync=True)
421 horizontal_stripes = Bool(False).tag(sync=True)
422 vertical_stripes = Bool(False).tag(sync=True)
423
424 def __init__(self, dataframe, index_name=None, **kwargs):
425 # Setting default index name if not explicitly
426 # set by the user.
427 self._index_name = index_name
428 self.data = dataframe
429 super().__init__(**kwargs)
430 self._cell_click_handlers = CallbackDispatcher()
431 self._cell_change_handlers = CallbackDispatcher()
432 self.on_msg(self.__handle_custom_msg)
433 self._set_renderer_defaults()
434
435 def __handle_custom_msg(self, _, content, buffers): # noqa: U101,U100
436 if content["event_type"] == "cell-changed":
437 row = content["row"]
438 column = self._column_index_to_name(
439 self._data, content["column_index"]
440 )
441 value = content["value"]
442 # update data on kernel
443 self._data["data"].loc[row, column] = value
444 # notify python listeners
445 self._cell_change_handlers(
446 {
447 "row": row,
448 "column": column,
449 "column_index": content["column_index"],
450 "value": value,
451 }
452 )
453 elif content["event_type"] == "cell-click":
454 # notify python listeners
455 self._cell_click_handlers(
456 {
457 "region": content["region"],
458 "column": content["column"],
459 "column_index": content["column_index"],
460 "row": content["row"],
461 "primary_key_row": content["primary_key_row"],
462 "cell_value": content["cell_value"],
463 }
464 )
465
466 @property
467 def data(self):
468 trimmed_primary_key = self._data["schema"]["primaryKey"][:-1]
469 if "data" in self._data:
470 df = pd.DataFrame(self._data["data"])
471 else:
472 df = pd.DataFrame(
473 {value["name"]: [] for value in self._data["schema"]["fields"]}
474 )
475 final_df = df.set_index(trimmed_primary_key)
476 final_df = final_df[final_df.columns[:-1]]
477 final_df.drop(columns=["ipydguuid"], inplace=True, errors="ignore")
478 return final_df
479
480 @data.setter
481 def data(self, dataframe):
482 # Reference for the original frame column and index names
483 # This is used to when returning the view data model
484 self.__dataframe_reference_index_names = dataframe.index.names
485 self.__dataframe_reference_columns = dataframe.columns
486 dataframe = dataframe.copy()
487
488 # Primary key used
489 index_key = self.get_dataframe_index(dataframe)
490
491 self._data = self.generate_data_object(
492 dataframe, "ipydguuid", index_key
493 )
494
495 @staticmethod
496 def generate_data_object(dataframe, guid_key="ipydguuid", index_name="key"):
497 dataframe[guid_key] = pd.RangeIndex(0, dataframe.shape[0])
498
499 # Renaming default index name from 'index' to 'key' on
500 # single index DataFrames. This allows users to use
501 # 'index' as a column name. If 'key' exists, we add _x
502 # suffix to id, where { x | 0 <= x < inf }
503 if not isinstance(dataframe.index, pd.MultiIndex):
504 if index_name in dataframe.columns:
505 index = 0
506 new_index_name = f"{index_name}_{index}"
507 while new_index_name in dataframe.columns:
508 index += 1
509 new_index_name = f"{index_name}_{index}"
510 dataframe = dataframe.rename_axis(new_index_name)
511 else:
512 dataframe = dataframe.rename_axis(index_name)
513
514 schema = pd.io.json.build_table_schema(dataframe)
515 reset_index_dataframe = dataframe.reset_index()
516 data = reset_index_dataframe
517
518 # Check for multiple primary keys
519 key = reset_index_dataframe.columns[: dataframe.index.nlevels].tolist()
520
521 num_index_levels = len(key) if isinstance(key, list) else 1
522
523 # Check for nested columns in schema, if so, we need to update the
524 # schema to represent the actual column name values
525 if isinstance(schema["fields"][-1]["name"], tuple):
526 num_column_levels = len(dataframe.columns.levels)
527 primary_key = key.copy()
528
529 for i in range(num_index_levels):
530 new_name = [""] * num_column_levels
531 new_name[0] = schema["fields"][i]["name"]
532 schema["fields"][i]["name"] = tuple(new_name)
533 primary_key[i] = tuple(new_name)
534
535 schema["primaryKey"] = primary_key
536 uuid_pk = list(key[-1])
537 uuid_pk[0] = guid_key
538 schema["primaryKey"].append(tuple(uuid_pk))
539
540 else:
541 schema["primaryKey"] = key
542 schema["primaryKey"].append(guid_key)
543
544 schema["primaryKeyUuid"] = guid_key
545
546 return {
547 "data": data,
548 "schema": schema,
549 "fields": [{field["name"]: None} for field in schema["fields"]],
550 }
551
552 def get_dataframe_index(self, dataframe):
553 """Returns a primary key to be used in ipydatagrid's
554 view of the passed DataFrame"""
555
556 # Passed index_name takes highest priority
557 if self._index_name is not None:
558 return self._index_name
559
560 # Dataframe with names index used by default
561 if dataframe.index.name is not None:
562 return dataframe.index.name
563
564 # If no index_name param, nor named-index DataFrame
565 # have been passed, revert to default "key"
566 return "key"
567
568 def get_cell_value(self, column_name, primary_key_value):
569 """Gets the value for a single or multiple cells by column name and
570 index name.
571
572 Tuples should be used to index into multi-index columns."""
573 row_indices = self._get_row_index_of_primary_key(primary_key_value)
574
575 if isinstance(column_name, list):
576 column_name = tuple(column_name)
577
578 return [self._data["data"][column_name][row] for row in row_indices]
579
580 def set_cell_value(self, column_name, primary_key_value, new_value):
581 """Sets the value for a single cell by column name and primary key.
582
583 Note: This method returns a boolean to indicate if the operation
584 was successful.
585 """
586 row_indices = self._get_row_index_of_primary_key(primary_key_value)
587 # Bail early if key could not be found
588 if not row_indices:
589 return False
590
591 if isinstance(column_name, list):
592 column_name = tuple(column_name)
593
594 # Iterate over all indices
595 outcome = True
596 for row_index in row_indices:
597 has_column = column_name in self._data["data"]
598 if has_column and row_index is not None:
599 self._data["data"].loc[row_index, column_name] = new_value
600 self._notify_cell_change(row_index, column_name, new_value)
601 else:
602 outcome = False
603 return outcome
604
605 def set_row_value(self, primary_key_value, new_value):
606 """Sets the value for a row by and primary key.
607
608 Note: This method returns a boolean to indicate if the operation
609 was successful.
610 """
611 row_indices = self._get_row_index_of_primary_key(primary_key_value)
612 # Bail early if key could not be found
613 if not row_indices:
614 return False
615
616 # Iterate over all indices
617 for row_index in row_indices:
618 column_index = 0
619 column = DataGrid._column_index_to_name(self._data, column_index)
620 while column is not None:
621 self._data["data"].loc[row_index, column] = new_value[
622 column_index
623 ]
624
625 column_index = column_index + 1
626 column = DataGrid._column_index_to_name(
627 self._data, column_index
628 )
629
630 self._notify_row_change(row_index, new_value)
631 return True
632
633 def get_cell_value_by_index(self, column_name, row_index):
634 """Gets the value for a single cell by column name and row index."""
635 return self._data["data"][column_name][row_index]
636
637 def set_cell_value_by_index(self, column_name, row_index, new_value):
638 """Sets the value for a single cell by column name and row index.
639
640 Note: This method returns a boolean to indicate if the operation
641 was successful.
642 """
643 has_column = column_name in self._data["data"]
644 if has_column and 0 <= row_index < len(self._data["data"][column_name]):
645 self._data["data"].loc[row_index, column_name] = new_value
646 self._notify_cell_change(row_index, column_name, new_value)
647 return True
648 return False
649
650 def _notify_cell_change(self, row, column, value):
651 column_index = self._column_name_to_index(column)
652 # notify python listeners
653 self._cell_change_handlers(
654 {
655 "row": row,
656 "column": column,
657 "column_index": column_index,
658 "value": value,
659 }
660 )
661 # notify front-end
662 self.comm.send(
663 data={
664 "method": "custom",
665 "content": {
666 "event_type": "cell-changed",
667 "row": row,
668 "column": column,
669 "column_index": column_index,
670 "value": value,
671 },
672 }
673 )
674
675 def _notify_row_change(self, row, value):
676 # notify front-end
677 self.comm.send(
678 data={
679 "method": "custom",
680 "content": {
681 "event_type": "row-changed",
682 "row": row,
683 "value": value,
684 },
685 }
686 )
687
688 def get_visible_data(self):
689 """Returns a dataframe of the current View."""
690 data = deepcopy(self._data)
691 if self._visible_rows:
692 data["data"] = data["data"].reindex(self._visible_rows)
693
694 at = self._data["schema"]["primaryKey"]
695 return_df = pd.DataFrame(data["data"]).set_index(at)
696 return_df.index = return_df.index.droplevel(return_df.index.nlevels - 1)
697 return_df.index.names = self.__dataframe_reference_index_names
698 return_df.columns = self.__dataframe_reference_columns
699 return return_df
700
701 def transform(self, transforms):
702 """Apply a list of transformation to this DataGrid."""
703 # TODO: Validate this input, or let it fail on view side?
704 self._transforms = transforms
705
706 def revert(self):
707 """Revert all transformations."""
708 self._transforms = []
709
710 @default("default_renderer")
711 def _default_renderer(self):
712 return TextRenderer()
713
714 def clear_selection(self):
715 """Clears all selections."""
716 self.selections.clear()
717 self.send_state("selections")
718
719 def select(self, row1, column1, row2=None, column2=None, clear_mode="none"):
720 """
721 Select an individual cell or rectangular cell region.
722 Parameters
723 ----------
724 row1 : int
725 Row index for individual cell selection or
726 start row index for rectangular region selection.
727 column1 : int
728 Column index for individual cell selection or
729 start column index for rectangular region selection.
730 row2 : int or None, optional (default: None)
731 End row index for rectangular region selection.
732 column2 : int or None, optional (default: None)
733 End column index for rectangular region selection.
734 clear_mode : string, optional, {'all', 'current', 'none'}
735 (default: 'none')
736 Clear mode to use when there are pre-existing selections.
737 'all' removes all pre-existing selections
738 'current' removes last pre-existing selection
739 'none' keeps pre-existing selections
740 """
741 if row2 is None or column2 is None:
742 row2, column2 = row1, column1
743
744 if clear_mode == "all":
745 self.selections.clear()
746 elif clear_mode == "current" and len(self.selections) > 0:
747 self.selections.pop()
748
749 self.selections.append(
750 {
751 "r1": min(row1, row2),
752 "c1": min(column1, column2),
753 "r2": max(row1, row2),
754 "c2": max(column1, column2),
755 }
756 )
757 self.send_state("selections")
758
759 @property
760 def selected_cells(self):
761 """
762 List of selected cells. Each cell is represented as a dictionary
763 with keys 'r': row and 'c': column
764 """
765 return SelectionHelper(
766 self._data, self.selections, self.selection_mode
767 ).all()
768
769 @property
770 def selected_visible_cell_iterator(self):
771 """
772 An iterator to traverse selected visible cells one by one.
773 """
774 # Copy of the front-end data model
775 view_data = self.get_visible_data()
776
777 # Get primary key from dataframe
778 index_key = self.get_dataframe_index(view_data)
779
780 # Serielize to JSON table schema
781 view_data_object = self.generate_data_object(
782 view_data, "ipydguuid", index_key
783 )
784 return SelectionHelper(
785 view_data_object, self.selections, self.selection_mode
786 )
787
788 @property
789 def selected_cell_values(self):
790 """
791 List of values for all selected cells.
792 """
793 return self.selected_visible_cell_iterator.all_values()
794
795 @property
796 def selected_cell_iterator(self):
797 """
798 An iterator to traverse selected cells one by one.
799 """
800 return SelectionHelper(self._data, self.selections, self.selection_mode)
801
802 @validate("selections")
803 def _validate_selections(self, proposal):
804 selections = proposal["value"]
805
806 for rectangle in selections:
807 r1 = min(rectangle["r1"], rectangle["r2"])
808 c1 = min(rectangle["c1"], rectangle["c2"])
809 r2 = max(rectangle["r1"], rectangle["r2"])
810 c2 = max(rectangle["c1"], rectangle["c2"])
811 rectangle["r1"] = r1
812 rectangle["c1"] = c1
813 rectangle["r2"] = r2
814 rectangle["c2"] = c2
815
816 return selections
817
818 @validate("editable")
819 def _validate_editable(self, proposal):
820 value = proposal["value"]
821 if value and self.selection_mode == "none":
822 self.selection_mode = "cell"
823 return value
824
825 @validate("_transforms")
826 def _validate_transforms(self, proposal):
827 transforms = proposal["value"]
828 field_len = len(self._data["schema"]["fields"])
829 for transform in transforms:
830 if "columnIndex" in transform:
831 warnings.warn(
832 "Applying transforms on columnIndex is deprecated, "
833 "please provide the column name instead",
834 DeprecationWarning,
835 stacklevel=4,
836 )
837
838 if "column" not in transform:
839 transform["column"] = self._data["schema"]["fields"][
840 transform["columnIndex"]
841 ]["name"]
842
843 if transform["columnIndex"] > field_len:
844 raise ValueError("Column index is out of bounds.")
845
846 return transforms
847
848 @validate("_data")
849 def _validate_data(self, proposal):
850 table_schema = proposal["value"]
851 column_list = [f["name"] for f in table_schema["schema"]["fields"]]
852 if len(column_list) != len(set(column_list)):
853 msg = "The dataframe must not contain duplicate column names."
854 raise ValueError(msg)
855 return table_schema
856
857 def on_cell_change(self, callback, remove=False):
858 """Register a callback to execute when a cell value changed.
859
860 The callback will be called with one argument, the dictionary
861 containing cell information with keys
862 "row", "column", "column_index", "value".
863
864 Parameters
865 ----------
866 remove: bool (optional)
867 Set to true to remove the callback from the list of callbacks.
868 """
869 self._cell_change_handlers.register_callback(callback, remove=remove)
870
871 def on_cell_click(self, callback, remove=False):
872 """Register a callback to execute when a cell is clicked.
873
874 The callback will be called with one argument, the dictionary
875 containing cell information with following keys:
876 "region", "column", "column_index", "row", "primary_key_row",
877 "cell_value"
878
879 Parameters
880 ----------
881 remove: bool (optional)
882 Set to true to remove the callback from the list of callbacks.
883 """
884 self._cell_click_handlers.register_callback(callback, remove=remove)
885
886 @staticmethod
887 def _column_index_to_name(data, column_index):
888 if "schema" not in data or "fields" not in data["schema"]:
889 return None
890 col_headers = DataGrid._get_col_headers(data)
891 return (
892 None
893 if len(col_headers) <= column_index
894 else col_headers[column_index]
895 )
896
897 @staticmethod
898 def _get_col_headers(data):
899 primary_keys = (
900 []
901 if "primaryKey" not in data["schema"]
902 else data["schema"]["primaryKey"]
903 )
904 col_headers = [
905 field["name"]
906 for field in data["schema"]["fields"]
907 if field["name"] not in primary_keys
908 ]
909 return col_headers
910
911 def _column_name_to_index(self, column_name):
912 if "schema" not in self._data or "fields" not in self._data["schema"]:
913 return None
914 col_headers = self._get_col_headers(self._data)
915 try:
916 return col_headers.index(column_name)
917 except ValueError:
918 pass
919
920 def _get_row_index_of_primary_key(self, value):
921 value = value if isinstance(value, list) else [value]
922 schema = self._data["schema"]
923 key = schema["primaryKey"][:-1] # Omitting ipydguuid
924 if len(value) != len(key):
925 raise ValueError(
926 "The provided primary key value must be the same length "
927 "as the primary key."
928 )
929
930 df = self._data["data"]
931 return pd.RangeIndex(len(df))[
932 (df[key] == value).all(axis="columns")
933 ].to_list()
934
935 @staticmethod
936 def _get_cell_value_by_numerical_index(data, column_index, row_index):
937 """Gets the value for a single cell by column index and row index."""
938 # TODO This is really not efficient, we should speed it up
939 column = DataGrid._column_index_to_name(data, column_index)
940 if column is None:
941 return None
942 return data["data"].loc[row_index, column]
943
944 def _set_renderer_defaults(self):
945 # Set sensible default values for renderers that are not completely
946 # specified, such as missing a min or max value.
947
948 data = None # Only read data once, and only if necessary.
949
950 for name, renderer in self.renderers.items():
951 if isinstance(renderer, BarRenderer):
952 from bqplot import DateScale, LinearScale, Scale
953
954 if renderer.bar_value is None:
955 # If BarRenderer.bar_value is not specified, create an
956 # appropriate Scale based on the column data type.
957 col_schema = next(
958 filter(
959 lambda x: x["name"] == name,
960 self._data["schema"]["fields"],
961 )
962 )
963 is_date = col_schema["type"] in ("date", "time", "datetime")
964 if is_date:
965 renderer.bar_value = DateScale()
966 else:
967 renderer.bar_value = LinearScale()
968
969 scale = renderer.bar_value
970 if (
971 isinstance(scale, Scale)
972 and scale.has_trait("min")
973 and scale.has_trait("max")
974 and (scale.min is None or scale.max is None)
975 ):
976 # Set min and/or max from column data.
977 if data is None:
978 data = self.data # Only want to get the data once
979 column_data = data[name]
980 is_date = isinstance(scale, DateScale)
981 if scale.min is None:
982 min = column_data.min()
983 scale.min = min if is_date else float(min)
984 if scale.max is None:
985 max = column_data.max()
986 scale.max = max if is_date else float(max)
987
988
989 class StreamingDataGrid(DataGrid):
990 """A blazingly fast Grid Widget.
991 This widget needs a live kernel for working
992 (does not work when embedded in static HTML)
993 """
994
995 _model_name = Unicode("StreamingDataGridModel").tag(sync=True)
996 _view_name = Unicode("StreamingDataGridView").tag(sync=True)
997
998 _row_count = Int(0).tag(sync=True)
999 _debounce_delay = Int(160).tag(sync=True)
1000
1001 def __init__(self, *args, debounce_delay=160, **kwargs):
1002 super().__init__(*args, **kwargs)
1003
1004 self._debounce_delay = debounce_delay
1005 self._transformed_data = None
1006
1007 self.on_msg(self._handle_comm_msg)
1008
1009 @property
1010 def data(self):
1011 return super().data
1012
1013 @data.setter
1014 def data(self, dataframe):
1015 self.__dataframe_reference_index_names = dataframe.index.names
1016 self.__dataframe_reference_columns = dataframe.columns
1017 # Not making a copy in the streaming grid
1018 self.__dataframe_reference = dataframe
1019
1020 # TODO support column indices being integers
1021 dataframe.columns = dataframe.columns.map(str)
1022
1023 # Primary key used
1024 index_key = self.get_dataframe_index(dataframe)
1025
1026 self._data_object = self.generate_data_object(
1027 dataframe, "ipydguuid", index_key
1028 )
1029
1030 self._row_count = len(self._data_object["data"])
1031
1032 self._data = {
1033 "data": {},
1034 "schema": self._data_object["schema"],
1035 "fields": self._data_object["fields"],
1036 }
1037
1038 def tick(self):
1039 """Notify that the underlying dataframe has changed."""
1040 self.send({"event_type": "tick"})
1041
1042 def _apply_frontend_transforms(self, frontend_transforms, dataframe):
1043 for transform in frontend_transforms:
1044 if transform["type"] == "sort":
1045 column = transform["column"]
1046 desc = transform.get("desc", False)
1047 dataframe = dataframe.sort_values(
1048 by=[column], ascending=not desc
1049 )
1050 elif transform["type"] == "filter":
1051 column = transform["column"]
1052 operator = transform["operator"]
1053 value = transform["value"]
1054 if operator == "<":
1055 dataframe = dataframe[dataframe[column] < value]
1056 elif operator == ">":
1057 dataframe = dataframe[dataframe[column] > value]
1058 elif operator == "=":
1059 dataframe = dataframe[dataframe[column] == value]
1060 elif operator == "<=":
1061 dataframe = dataframe[dataframe[column] <= value]
1062 elif operator == ">=":
1063 dataframe = dataframe[dataframe[column] >= value]
1064 elif operator == "!=":
1065 dataframe = dataframe[dataframe[column] != value]
1066 elif operator == "empty":
1067 dataframe = dataframe[dataframe[column].isna()]
1068 elif operator == "notempty":
1069 dataframe = dataframe[dataframe[column].notna()]
1070 elif operator == "in":
1071 value = pd.Series(value, dtype=dataframe[column].dtype)
1072 dataframe = dataframe[dataframe[column].isin(value)]
1073 elif operator == "between":
1074 dataframe = dataframe[
1075 dataframe[column].between(value[0], value[1])
1076 ]
1077 elif operator == "startswith":
1078 dataframe = dataframe[
1079 dataframe[column].str.startswith(value)
1080 ]
1081 elif operator == "endswith":
1082 dataframe = dataframe[dataframe[column].str.endswith(value)]
1083 elif operator == "stringContains":
1084 dataframe = dataframe[dataframe[column].str.contains(value)]
1085 elif operator == "contains":
1086 dataframe = dataframe[dataframe[column].str.contains(value)]
1087 elif operator == "!contains":
1088 dataframe = dataframe[
1089 not dataframe[column].str.contains(value)
1090 ]
1091 elif operator == "isOnSameDay":
1092 value = pd.to_datetime(value).date()
1093 dataframe = dataframe[
1094 pd.to_datetime(dataframe[column]).dt.date == value
1095 ]
1096 else:
1097 raise RuntimeError(
1098 f"Unrecognised filter operator '{operator}'"
1099 )
1100
1101 return dataframe
1102
1103 def _handle_comm_msg(self, _, content, _buffs):
1104 event_type = content.get("type", "")
1105
1106 if event_type == "data-request":
1107 r1 = content.get("r1")
1108 r2 = content.get("r2")
1109 c1 = content.get("c1")
1110 c2 = content.get("c2")
1111
1112 # Filter/sort whole dataset before selecting rows/cols of interest
1113 if self._transformed_data is not None:
1114 # Use existing transformed data.
1115 value = self._transformed_data
1116 else:
1117 value = self.__dataframe_reference
1118
1119 value = value.iloc[r1 : r2 + 1, c1 : c2 + 1]
1120
1121 serialized, buffers = self._serialize_helper(value)
1122
1123 answer = {
1124 "event_type": "data-reply",
1125 "value": serialized,
1126 "r1": r1,
1127 "r2": r2,
1128 "c1": c1,
1129 "c2": c2,
1130 }
1131
1132 self.send(answer, buffers)
1133
1134 elif event_type == "unique-values-request":
1135 column = content.get("column")
1136 original = self.__dataframe_reference[column].drop_duplicates()
1137 serialized, buffers = self._serialize_helper(pd.DataFrame(original))
1138
1139 answer = {
1140 "event_type": "unique-values-reply",
1141 "column": column,
1142 "value": serialized,
1143 }
1144 self.send(answer, buffers)
1145
1146 @observe("_transforms")
1147 def _on_transforms_changed(self, change):
1148 # Transforms is an array of dicts.
1149 frontend_transforms = change["new"]
1150
1151 self._transformed_data = None
1152 data = self.__dataframe_reference
1153
1154 if frontend_transforms:
1155 self._transformed_data = self._apply_frontend_transforms(
1156 frontend_transforms, data
1157 )
1158 data = self._transformed_data
1159
1160 self._row_count = len(data) # Sync to frontend.
1161
1162 # Should only request a tick if the transforms have changed.
1163 self.tick()
1164
1165 def _serialize_helper(self, dataframe):
1166 # Primary key used
1167 index_key = self.get_dataframe_index(dataframe)
1168
1169 serialized = _data_serialization_impl(
1170 self.generate_data_object(dataframe, "ipydguuid", index_key), None
1171 )
1172
1173 # Extract all buffers
1174 buffers = []
1175 for column in serialized["data"].keys():
1176 col = serialized["data"][column]
1177 if not isinstance(col, list) and col["type"] != "raw":
1178 buffers.append(col["value"])
1179 col["value"] = len(buffers) - 1
1180
1181 return serialized, buffers