| 1 |
# coding=utf-8 |
| 2 |
# Author: Andreas Christian Mueller <[email protected]> |
| 3 |
# |
| 4 |
# (c) 2012 |
| 5 |
# Modified by: Paul Nechifor <[email protected]> |
| 6 |
# |
| 7 |
# License: MIT |
| 8 |
|
| 9 |
from __future__ import division |
| 10 |
|
| 11 |
import warnings |
| 12 |
from random import Random |
| 13 |
import io |
| 14 |
import os |
| 15 |
import re |
| 16 |
import base64 |
| 17 |
import sys |
| 18 |
import colorsys |
| 19 |
import matplotlib |
| 20 |
import numpy as np |
| 21 |
from operator import itemgetter |
| 22 |
from xml.sax import saxutils |
| 23 |
|
| 24 |
from PIL import Image |
| 25 |
from PIL import ImageColor |
| 26 |
from PIL import ImageDraw |
| 27 |
from PIL import ImageFilter |
| 28 |
from PIL import ImageFont |
| 29 |
|
| 30 |
from .query_integral_image import query_integral_image |
| 31 |
from .tokenization import unigrams_and_bigrams, process_tokens |
| 32 |
|
| 33 |
FILE = os.path.dirname(__file__) |
| 34 |
FONT_PATH = os.environ.get('FONT_PATH', os.path.join(FILE, 'DroidSansMono.ttf')) |
| 35 |
STOPWORDS = set(map(str.strip, open(os.path.join(FILE, 'stopwords')).readlines())) |
| 36 |
|
| 37 |
|
| 38 |
class IntegralOccupancyMap(object): |
| 39 |
def __init__(self, height, width, mask): |
| 40 |
self.height = height |
| 41 |
self.width = width |
| 42 |
if mask is not None: |
| 43 |
# the order of the cumsum's is important for speed ?! |
| 44 |
self.integral = np.cumsum(np.cumsum(255 * mask, axis=1), |
| 45 |
axis=0).astype(np.uint32) |
| 46 |
else: |
| 47 |
self.integral = np.zeros((height, width), dtype=np.uint32) |
| 48 |
|
| 49 |
def sample_position(self, size_x, size_y, random_state): |
| 50 |
return query_integral_image(self.integral, size_x, size_y, |
| 51 |
random_state) |
| 52 |
|
| 53 |
def update(self, img_array, pos_x, pos_y): |
| 54 |
partial_integral = np.cumsum(np.cumsum(img_array[pos_x:, pos_y:], |
| 55 |
axis=1), axis=0) |
| 56 |
# paste recomputed part into old image |
| 57 |
# if x or y is zero it is a bit annoying |
| 58 |
if pos_x > 0: |
| 59 |
if pos_y > 0: |
| 60 |
partial_integral += (self.integral[pos_x - 1, pos_y:] |
| 61 |
- self.integral[pos_x - 1, pos_y - 1]) |
| 62 |
else: |
| 63 |
partial_integral += self.integral[pos_x - 1, pos_y:] |
| 64 |
if pos_y > 0: |
| 65 |
partial_integral += self.integral[pos_x:, pos_y - 1][:, np.newaxis] |
| 66 |
|
| 67 |
self.integral[pos_x:, pos_y:] = partial_integral |
| 68 |
|
| 69 |
|
| 70 |
def random_color_func(word=None, font_size=None, position=None, |
| 71 |
orientation=None, font_path=None, random_state=None): |
| 72 |
"""Random hue color generation. |
| 73 |
|
| 74 |
Default coloring method. This just picks a random hue with value 80% and |
| 75 |
lumination 50%. |
| 76 |
|
| 77 |
Parameters |
| 78 |
---------- |
| 79 |
word, font_size, position, orientation : ignored. |
| 80 |
|
| 81 |
random_state : random.Random object or None, (default=None) |
| 82 |
If a random object is given, this is used for generating random |
| 83 |
numbers. |
| 84 |
|
| 85 |
""" |
| 86 |
if random_state is None: |
| 87 |
random_state = Random() |
| 88 |
return "hsl(%d, 80%%, 50%%)" % random_state.randint(0, 255) |
| 89 |
|
| 90 |
|
| 91 |
class colormap_color_func(object): |
| 92 |
"""Color func created from matplotlib colormap. |
| 93 |
|
| 94 |
Parameters |
| 95 |
---------- |
| 96 |
colormap : string or matplotlib colormap |
| 97 |
Colormap to sample from |
| 98 |
|
| 99 |
Example |
| 100 |
------- |
| 101 |
>>> WordCloud(color_func=colormap_color_func("magma")) |
| 102 |
|
| 103 |
""" |
| 104 |
def __init__(self, colormap): |
| 105 |
import matplotlib.pyplot as plt |
| 106 |
self.colormap = plt.get_cmap(colormap) |
| 107 |
|
| 108 |
def __call__(self, word, font_size, position, orientation, |
| 109 |
random_state=None, **kwargs): |
| 110 |
if random_state is None: |
| 111 |
random_state = Random() |
| 112 |
r, g, b, _ = np.maximum(0, 255 * np.array(self.colormap( |
| 113 |
random_state.uniform(0, 1)))) |
| 114 |
return "rgb({:.0f}, {:.0f}, {:.0f})".format(r, g, b) |
| 115 |
|
| 116 |
|
| 117 |
def get_single_color_func(color): |
| 118 |
"""Create a color function which returns a single hue and saturation with. |
| 119 |
different values (HSV). Accepted values are color strings as usable by |
| 120 |
PIL/Pillow. |
| 121 |
|
| 122 |
>>> color_func1 = get_single_color_func('deepskyblue') |
| 123 |
>>> color_func2 = get_single_color_func('#00b4d2') |
| 124 |
""" |
| 125 |
old_r, old_g, old_b = ImageColor.getrgb(color) |
| 126 |
rgb_max = 255. |
| 127 |
h, s, v = colorsys.rgb_to_hsv(old_r / rgb_max, old_g / rgb_max, |
| 128 |
old_b / rgb_max) |
| 129 |
|
| 130 |
def single_color_func(word=None, font_size=None, position=None, |
| 131 |
orientation=None, font_path=None, random_state=None): |
| 132 |
"""Random color generation. |
| 133 |
|
| 134 |
Additional coloring method. It picks a random value with hue and |
| 135 |
saturation based on the color given to the generating function. |
| 136 |
|
| 137 |
Parameters |
| 138 |
---------- |
| 139 |
word, font_size, position, orientation : ignored. |
| 140 |
|
| 141 |
random_state : random.Random object or None, (default=None) |
| 142 |
If a random object is given, this is used for generating random |
| 143 |
numbers. |
| 144 |
|
| 145 |
""" |
| 146 |
if random_state is None: |
| 147 |
random_state = Random() |
| 148 |
r, g, b = colorsys.hsv_to_rgb(h, s, random_state.uniform(0.2, 1)) |
| 149 |
return 'rgb({:.0f}, {:.0f}, {:.0f})'.format(r * rgb_max, g * rgb_max, |
| 150 |
b * rgb_max) |
| 151 |
return single_color_func |
| 152 |
|
| 153 |
|
| 154 |
class WordCloud(object): |
| 155 |
r"""Word cloud object for generating and drawing. |
| 156 |
|
| 157 |
Parameters |
| 158 |
---------- |
| 159 |
font_path : string |
| 160 |
Font path to the font that will be used (OTF or TTF). |
| 161 |
Defaults to DroidSansMono path on a Linux machine. If you are on |
| 162 |
another OS or don't have this font, you need to adjust this path. |
| 163 |
|
| 164 |
width : int (default=400) |
| 165 |
Width of the canvas. |
| 166 |
|
| 167 |
height : int (default=200) |
| 168 |
Height of the canvas. |
| 169 |
|
| 170 |
prefer_horizontal : float (default=0.90) |
| 171 |
The ratio of times to try horizontal fitting as opposed to vertical. |
| 172 |
If prefer_horizontal < 1, the algorithm will try rotating the word |
| 173 |
if it doesn't fit. (There is currently no built-in way to get only |
| 174 |
vertical words.) |
| 175 |
|
| 176 |
mask : nd-array or None (default=None) |
| 177 |
If not None, gives a binary mask on where to draw words. If mask is not |
| 178 |
None, width and height will be ignored and the shape of mask will be |
| 179 |
used instead. All white (#FF or #FFFFFF) entries will be considerd |
| 180 |
"masked out" while other entries will be free to draw on. [This |
| 181 |
changed in the most recent version!] |
| 182 |
|
| 183 |
contour_width: float (default=0) |
| 184 |
If mask is not None and contour_width > 0, draw the mask contour. |
| 185 |
|
| 186 |
contour_color: color value (default="black") |
| 187 |
Mask contour color. |
| 188 |
|
| 189 |
scale : float (default=1) |
| 190 |
Scaling between computation and drawing. For large word-cloud images, |
| 191 |
using scale instead of larger canvas size is significantly faster, but |
| 192 |
might lead to a coarser fit for the words. |
| 193 |
|
| 194 |
min_font_size : int (default=4) |
| 195 |
Smallest font size to use. Will stop when there is no more room in this |
| 196 |
size. |
| 197 |
|
| 198 |
font_step : int (default=1) |
| 199 |
Step size for the font. font_step > 1 might speed up computation but |
| 200 |
give a worse fit. |
| 201 |
|
| 202 |
max_words : number (default=200) |
| 203 |
The maximum number of words. |
| 204 |
|
| 205 |
stopwords : set of strings or None |
| 206 |
The words that will be eliminated. If None, the build-in STOPWORDS |
| 207 |
list will be used. Ignored if using generate_from_frequencies. |
| 208 |
|
| 209 |
background_color : color value (default="black") |
| 210 |
Background color for the word cloud image. |
| 211 |
|
| 212 |
max_font_size : int or None (default=None) |
| 213 |
Maximum font size for the largest word. If None, height of the image is |
| 214 |
used. |
| 215 |
|
| 216 |
mode : string (default="RGB") |
| 217 |
Transparent background will be generated when mode is "RGBA" and |
| 218 |
background_color is None. |
| 219 |
|
| 220 |
relative_scaling : float (default='auto') |
| 221 |
Importance of relative word frequencies for font-size. With |
| 222 |
relative_scaling=0, only word-ranks are considered. With |
| 223 |
relative_scaling=1, a word that is twice as frequent will have twice |
| 224 |
the size. If you want to consider the word frequencies and not only |
| 225 |
their rank, relative_scaling around .5 often looks good. |
| 226 |
If 'auto' it will be set to 0.5 unless repeat is true, in which |
| 227 |
case it will be set to 0. |
| 228 |
|
| 229 |
.. versionchanged: 2.0 |
| 230 |
Default is now 'auto'. |
| 231 |
|
| 232 |
color_func : callable, default=None |
| 233 |
Callable with parameters word, font_size, position, orientation, |
| 234 |
font_path, random_state that returns a PIL color for each word. |
| 235 |
Overwrites "colormap". |
| 236 |
See colormap for specifying a matplotlib colormap instead. |
| 237 |
To create a word cloud with a single color, use |
| 238 |
``color_func=lambda *args, **kwargs: "white"``. |
| 239 |
The single color can also be specified using RGB code. For example |
| 240 |
``color_func=lambda *args, **kwargs: (255,0,0)`` sets color to red. |
| 241 |
|
| 242 |
regexp : string or None (optional) |
| 243 |
Regular expression to split the input text into tokens in process_text. |
| 244 |
If None is specified, ``r"\w[\w']+"`` is used. Ignored if using |
| 245 |
generate_from_frequencies. |
| 246 |
|
| 247 |
collocations : bool, default=True |
| 248 |
Whether to include collocations (bigrams) of two words. Ignored if using |
| 249 |
generate_from_frequencies. |
| 250 |
|
| 251 |
|
| 252 |
.. versionadded: 2.0 |
| 253 |
|
| 254 |
colormap : string or matplotlib colormap, default="viridis" |
| 255 |
Matplotlib colormap to randomly draw colors from for each word. |
| 256 |
Ignored if "color_func" is specified. |
| 257 |
|
| 258 |
.. versionadded: 2.0 |
| 259 |
|
| 260 |
normalize_plurals : bool, default=True |
| 261 |
Whether to remove trailing 's' from words. If True and a word |
| 262 |
appears with and without a trailing 's', the one with trailing 's' |
| 263 |
is removed and its counts are added to the version without |
| 264 |
trailing 's' -- unless the word ends with 'ss'. Ignored if using |
| 265 |
generate_from_frequencies. |
| 266 |
|
| 267 |
repeat : bool, default=False |
| 268 |
Whether to repeat words and phrases until max_words or min_font_size |
| 269 |
is reached. |
| 270 |
|
| 271 |
include_numbers : bool, default=False |
| 272 |
Whether to include numbers as phrases or not. |
| 273 |
|
| 274 |
min_word_length : int, default=0 |
| 275 |
Minimum number of letters a word must have to be included. |
| 276 |
|
| 277 |
collocation_threshold: int, default=30 |
| 278 |
Bigrams must have a Dunning likelihood collocation score greater than this |
| 279 |
parameter to be counted as bigrams. Default of 30 is arbitrary. |
| 280 |
|
| 281 |
See Manning, C.D., Manning, C.D. and Schütze, H., 1999. Foundations of |
| 282 |
Statistical Natural Language Processing. MIT press, p. 162 |
| 283 |
https://nlp.stanford.edu/fsnlp/promo/colloc.pdf#page=22 |
| 284 |
|
| 285 |
Attributes |
| 286 |
---------- |
| 287 |
``words_`` : dict of string to float |
| 288 |
Word tokens with associated frequency. |
| 289 |
|
| 290 |
.. versionchanged: 2.0 |
| 291 |
``words_`` is now a dictionary |
| 292 |
|
| 293 |
``layout_`` : list of tuples ((string, float), int, (int, int), int, color)) |
| 294 |
Encodes the fitted word cloud. For each word, it encodes the string, |
| 295 |
normalized frequency, font size, position, orientation, and color. |
| 296 |
The frequencies are normalized by the most commonly occurring word. |
| 297 |
The color is in the format of 'rgb(R, G, B).' |
| 298 |
|
| 299 |
Notes |
| 300 |
----- |
| 301 |
Larger canvases make the code significantly slower. If you need a |
| 302 |
large word cloud, try a lower canvas size, and set the scale parameter. |
| 303 |
|
| 304 |
The algorithm might give more weight to the ranking of the words |
| 305 |
than their actual frequencies, depending on the ``max_font_size`` and the |
| 306 |
scaling heuristic. |
| 307 |
""" |
| 308 |
|
| 309 |
def __init__(self, font_path=None, width=400, height=200, margin=2, |
| 310 |
ranks_only=None, prefer_horizontal=.9, mask=None, scale=1, |
| 311 |
color_func=None, max_words=200, min_font_size=4, |
| 312 |
stopwords=None, random_state=None, background_color='black', |
| 313 |
max_font_size=None, font_step=1, mode="RGB", |
| 314 |
relative_scaling='auto', regexp=None, collocations=True, |
| 315 |
colormap=None, normalize_plurals=True, contour_width=0, |
| 316 |
contour_color='black', repeat=False, |
| 317 |
include_numbers=False, min_word_length=0, collocation_threshold=30): |
| 318 |
if font_path is None: |
| 319 |
font_path = FONT_PATH |
| 320 |
if color_func is None and colormap is None: |
| 321 |
version = matplotlib.__version__ |
| 322 |
if version[0] < "2" and version[2] < "5": |
| 323 |
colormap = "hsv" |
| 324 |
else: |
| 325 |
colormap = "viridis" |
| 326 |
self.colormap = colormap |
| 327 |
self.collocations = collocations |
| 328 |
self.font_path = font_path |
| 329 |
self.width = width |
| 330 |
self.height = height |
| 331 |
self.margin = margin |
| 332 |
self.prefer_horizontal = prefer_horizontal |
| 333 |
self.mask = mask |
| 334 |
self.contour_color = contour_color |
| 335 |
self.contour_width = contour_width |
| 336 |
self.scale = scale |
| 337 |
self.color_func = color_func or colormap_color_func(colormap) |
| 338 |
self.max_words = max_words |
| 339 |
self.stopwords = stopwords if stopwords is not None else STOPWORDS |
| 340 |
self.min_font_size = min_font_size |
| 341 |
self.font_step = font_step |
| 342 |
self.regexp = regexp |
| 343 |
if isinstance(random_state, int): |
| 344 |
random_state = Random(random_state) |
| 345 |
self.random_state = random_state |
| 346 |
self.background_color = background_color |
| 347 |
self.max_font_size = max_font_size |
| 348 |
self.mode = mode |
| 349 |
|
| 350 |
if relative_scaling == "auto": |
| 351 |
if repeat: |
| 352 |
relative_scaling = 0 |
| 353 |
else: |
| 354 |
relative_scaling = .5 |
| 355 |
|
| 356 |
if relative_scaling < 0 or relative_scaling > 1: |
| 357 |
raise ValueError("relative_scaling needs to be " |
| 358 |
"between 0 and 1, got %f." % relative_scaling) |
| 359 |
self.relative_scaling = relative_scaling |
| 360 |
if ranks_only is not None: |
| 361 |
warnings.warn("ranks_only is deprecated and will be removed as" |
| 362 |
" it had no effect. Look into relative_scaling.", |
| 363 |
DeprecationWarning) |
| 364 |
self.normalize_plurals = normalize_plurals |
| 365 |
self.repeat = repeat |
| 366 |
self.include_numbers = include_numbers |
| 367 |
self.min_word_length = min_word_length |
| 368 |
self.collocation_threshold = collocation_threshold |
| 369 |
|
| 370 |
# Override the width and height if there is a mask |
| 371 |
if mask is not None: |
| 372 |
self.width = mask.shape[1] |
| 373 |
self.height = mask.shape[0] |
| 374 |
|
| 375 |
def fit_words(self, frequencies): |
| 376 |
"""Create a word_cloud from words and frequencies. |
| 377 |
|
| 378 |
Alias to generate_from_frequencies. |
| 379 |
|
| 380 |
Parameters |
| 381 |
---------- |
| 382 |
frequencies : dict from string to float |
| 383 |
A contains words and associated frequency. |
| 384 |
|
| 385 |
Returns |
| 386 |
------- |
| 387 |
self |
| 388 |
""" |
| 389 |
return self.generate_from_frequencies(frequencies) |
| 390 |
|
| 391 |
def generate_from_frequencies(self, frequencies, max_font_size=None): # noqa: C901 |
| 392 |
"""Create a word_cloud from words and frequencies. |
| 393 |
|
| 394 |
Parameters |
| 395 |
---------- |
| 396 |
frequencies : dict from string to float |
| 397 |
A contains words and associated frequency. |
| 398 |
|
| 399 |
max_font_size : int |
| 400 |
Use this font-size instead of self.max_font_size |
| 401 |
|
| 402 |
Returns |
| 403 |
------- |
| 404 |
self |
| 405 |
|
| 406 |
""" |
| 407 |
# make sure frequencies are sorted and normalized |
| 408 |
frequencies = sorted(frequencies.items(), key=itemgetter(1), reverse=True) |
| 409 |
if len(frequencies) <= 0: |
| 410 |
raise ValueError("We need at least 1 word to plot a word cloud, " |
| 411 |
"got %d." % len(frequencies)) |
| 412 |
frequencies = frequencies[:self.max_words] |
| 413 |
|
| 414 |
# largest entry will be 1 |
| 415 |
max_frequency = float(frequencies[0][1]) |
| 416 |
|
| 417 |
frequencies = [(word, freq / max_frequency) |
| 418 |
for word, freq in frequencies] |
| 419 |
|
| 420 |
if self.random_state is not None: |
| 421 |
random_state = self.random_state |
| 422 |
else: |
| 423 |
random_state = Random() |
| 424 |
|
| 425 |
if self.mask is not None: |
| 426 |
boolean_mask = self._get_bolean_mask(self.mask) |
| 427 |
width = self.mask.shape[1] |
| 428 |
height = self.mask.shape[0] |
| 429 |
else: |
| 430 |
boolean_mask = None |
| 431 |
height, width = self.height, self.width |
| 432 |
occupancy = IntegralOccupancyMap(height, width, boolean_mask) |
| 433 |
|
| 434 |
# create image |
| 435 |
img_grey = Image.new("L", (width, height)) |
| 436 |
draw = ImageDraw.Draw(img_grey) |
| 437 |
img_array = np.asarray(img_grey) |
| 438 |
font_sizes, positions, orientations, colors = [], [], [], [] |
| 439 |
|
| 440 |
last_freq = 1. |
| 441 |
|
| 442 |
if max_font_size is None: |
| 443 |
# if not provided use default font_size |
| 444 |
max_font_size = self.max_font_size |
| 445 |
|
| 446 |
if max_font_size is None: |
| 447 |
# figure out a good font size by trying to draw with |
| 448 |
# just the first two words |
| 449 |
if len(frequencies) == 1: |
| 450 |
# we only have one word. We make it big! |
| 451 |
font_size = self.height |
| 452 |
else: |
| 453 |
self.generate_from_frequencies(dict(frequencies[:2]), |
| 454 |
max_font_size=self.height) |
| 455 |
# find font sizes |
| 456 |
sizes = [x[1] for x in self.layout_] |
| 457 |
try: |
| 458 |
font_size = int(2 * sizes[0] * sizes[1] |
| 459 |
/ (sizes[0] + sizes[1])) |
| 460 |
# quick fix for if self.layout_ contains less than 2 values |
| 461 |
# on very small images it can be empty |
| 462 |
except IndexError: |
| 463 |
try: |
| 464 |
font_size = sizes[0] |
| 465 |
except IndexError: |
| 466 |
raise ValueError( |
| 467 |
"Couldn't find space to draw. Either the Canvas size" |
| 468 |
" is too small or too much of the image is masked " |
| 469 |
"out.") |
| 470 |
else: |
| 471 |
font_size = max_font_size |
| 472 |
|
| 473 |
# we set self.words_ here because we called generate_from_frequencies |
| 474 |
# above... hurray for good design? |
| 475 |
self.words_ = dict(frequencies) |
| 476 |
|
| 477 |
if self.repeat and len(frequencies) < self.max_words: |
| 478 |
# pad frequencies with repeating words. |
| 479 |
times_extend = int(np.ceil(self.max_words / len(frequencies))) - 1 |
| 480 |
# get smallest frequency |
| 481 |
frequencies_org = list(frequencies) |
| 482 |
downweight = frequencies[-1][1] |
| 483 |
for i in range(times_extend): |
| 484 |
frequencies.extend([(word, freq * downweight ** (i + 1)) |
| 485 |
for word, freq in frequencies_org]) |
| 486 |
|
| 487 |
# start drawing grey image |
| 488 |
for word, freq in frequencies: |
| 489 |
if freq == 0: |
| 490 |
continue |
| 491 |
# select the font size |
| 492 |
rs = self.relative_scaling |
| 493 |
if rs != 0: |
| 494 |
font_size = int(round((rs * (freq / float(last_freq)) |
| 495 |
+ (1 - rs)) * font_size)) |
| 496 |
if random_state.random() < self.prefer_horizontal: |
| 497 |
orientation = None |
| 498 |
else: |
| 499 |
orientation = Image.ROTATE_90 |
| 500 |
tried_other_orientation = False |
| 501 |
while True: |
| 502 |
if font_size < self.min_font_size: |
| 503 |
# font-size went too small |
| 504 |
break |
| 505 |
# try to find a position |
| 506 |
font = ImageFont.truetype(self.font_path, font_size) |
| 507 |
# transpose font optionally |
| 508 |
transposed_font = ImageFont.TransposedFont( |
| 509 |
font, orientation=orientation) |
| 510 |
# get size of resulting text |
| 511 |
box_size = draw.textbbox((0, 0), word, font=transposed_font, anchor="lt") |
| 512 |
# find possible places using integral image: |
| 513 |
result = occupancy.sample_position(box_size[3] + self.margin, |
| 514 |
box_size[2] + self.margin, |
| 515 |
random_state) |
| 516 |
if result is not None: |
| 517 |
# Found a place |
| 518 |
break |
| 519 |
# if we didn't find a place, make font smaller |
| 520 |
# but first try to rotate! |
| 521 |
if not tried_other_orientation and self.prefer_horizontal < 1: |
| 522 |
orientation = (Image.ROTATE_90 if orientation is None else |
| 523 |
Image.ROTATE_90) |
| 524 |
tried_other_orientation = True |
| 525 |
else: |
| 526 |
font_size -= self.font_step |
| 527 |
orientation = None |
| 528 |
|
| 529 |
if font_size < self.min_font_size: |
| 530 |
# we were unable to draw any more |
| 531 |
break |
| 532 |
|
| 533 |
x, y = np.array(result) + self.margin // 2 |
| 534 |
# actually draw the text |
| 535 |
draw.text((y, x), word, fill="white", font=transposed_font) |
| 536 |
positions.append((x, y)) |
| 537 |
orientations.append(orientation) |
| 538 |
font_sizes.append(font_size) |
| 539 |
colors.append(self.color_func(word, font_size=font_size, |
| 540 |
position=(x, y), |
| 541 |
orientation=orientation, |
| 542 |
random_state=random_state, |
| 543 |
font_path=self.font_path)) |
| 544 |
# recompute integral image |
| 545 |
if self.mask is None: |
| 546 |
img_array = np.asarray(img_grey) |
| 547 |
else: |
| 548 |
img_array = np.asarray(img_grey) + boolean_mask |
| 549 |
# recompute bottom right |
| 550 |
# the order of the cumsum's is important for speed ?! |
| 551 |
occupancy.update(img_array, x, y) |
| 552 |
last_freq = freq |
| 553 |
|
| 554 |
self.layout_ = list(zip(frequencies, font_sizes, positions, |
| 555 |
orientations, colors)) |
| 556 |
return self |
| 557 |
|
| 558 |
def process_text(self, text): |
| 559 |
"""Splits a long text into words, eliminates the stopwords. |
| 560 |
|
| 561 |
Parameters |
| 562 |
---------- |
| 563 |
text : string |
| 564 |
The text to be processed. |
| 565 |
|
| 566 |
Returns |
| 567 |
------- |
| 568 |
words : dict (string, int) |
| 569 |
Word tokens with associated frequency. |
| 570 |
|
| 571 |
..versionchanged:: 1.2.2 |
| 572 |
Changed return type from list of tuples to dict. |
| 573 |
|
| 574 |
Notes |
| 575 |
----- |
| 576 |
There are better ways to do word tokenization, but I don't want to |
| 577 |
include all those things. |
| 578 |
""" |
| 579 |
|
| 580 |
flags = (re.UNICODE if sys.version < '3' and type(text) is unicode # noqa: F821 |
| 581 |
else 0) |
| 582 |
pattern = r"\w[\w']*" if self.min_word_length <= 1 else r"\w[\w']+" |
| 583 |
regexp = self.regexp if self.regexp is not None else pattern |
| 584 |
|
| 585 |
words = re.findall(regexp, text, flags) |
| 586 |
# remove 's |
| 587 |
words = [word[:-2] if word.lower().endswith("'s") else word |
| 588 |
for word in words] |
| 589 |
# remove numbers |
| 590 |
if not self.include_numbers: |
| 591 |
words = [word for word in words if not word.isdigit()] |
| 592 |
# remove short words |
| 593 |
if self.min_word_length: |
| 594 |
words = [word for word in words if len(word) >= self.min_word_length] |
| 595 |
|
| 596 |
stopwords = set([i.lower() for i in self.stopwords]) |
| 597 |
if self.collocations: |
| 598 |
word_counts = unigrams_and_bigrams(words, stopwords, self.normalize_plurals, self.collocation_threshold) |
| 599 |
else: |
| 600 |
# remove stopwords |
| 601 |
words = [word for word in words if word.lower() not in stopwords] |
| 602 |
word_counts, _ = process_tokens(words, self.normalize_plurals) |
| 603 |
|
| 604 |
return word_counts |
| 605 |
|
| 606 |
def generate_from_text(self, text): |
| 607 |
"""Generate wordcloud from text. |
| 608 |
|
| 609 |
The input "text" is expected to be a natural text. If you pass a sorted |
| 610 |
list of words, words will appear in your output twice. To remove this |
| 611 |
duplication, set ``collocations=False``. |
| 612 |
|
| 613 |
Calls process_text and generate_from_frequencies. |
| 614 |
|
| 615 |
..versionchanged:: 1.2.2 |
| 616 |
Argument of generate_from_frequencies() is not return of |
| 617 |
process_text() any more. |
| 618 |
|
| 619 |
Returns |
| 620 |
------- |
| 621 |
self |
| 622 |
""" |
| 623 |
words = self.process_text(text) |
| 624 |
self.generate_from_frequencies(words) |
| 625 |
return self |
| 626 |
|
| 627 |
def generate(self, text): |
| 628 |
"""Generate wordcloud from text. |
| 629 |
|
| 630 |
The input "text" is expected to be a natural text. If you pass a sorted |
| 631 |
list of words, words will appear in your output twice. To remove this |
| 632 |
duplication, set ``collocations=False``. |
| 633 |
|
| 634 |
Alias to generate_from_text. |
| 635 |
|
| 636 |
Calls process_text and generate_from_frequencies. |
| 637 |
|
| 638 |
Returns |
| 639 |
------- |
| 640 |
self |
| 641 |
""" |
| 642 |
return self.generate_from_text(text) |
| 643 |
|
| 644 |
def _check_generated(self): |
| 645 |
"""Check if ``layout_`` was computed, otherwise raise error.""" |
| 646 |
if not hasattr(self, "layout_"): |
| 647 |
raise ValueError("WordCloud has not been calculated, call generate" |
| 648 |
" first.") |
| 649 |
|
| 650 |
def to_image(self): |
| 651 |
self._check_generated() |
| 652 |
if self.mask is not None: |
| 653 |
width = self.mask.shape[1] |
| 654 |
height = self.mask.shape[0] |
| 655 |
else: |
| 656 |
height, width = self.height, self.width |
| 657 |
|
| 658 |
img = Image.new(self.mode, (int(width * self.scale), |
| 659 |
int(height * self.scale)), |
| 660 |
self.background_color) |
| 661 |
draw = ImageDraw.Draw(img) |
| 662 |
for (word, count), font_size, position, orientation, color in self.layout_: |
| 663 |
font = ImageFont.truetype(self.font_path, |
| 664 |
int(font_size * self.scale)) |
| 665 |
transposed_font = ImageFont.TransposedFont( |
| 666 |
font, orientation=orientation) |
| 667 |
pos = (int(position[1] * self.scale), |
| 668 |
int(position[0] * self.scale)) |
| 669 |
draw.text(pos, word, fill=color, font=transposed_font) |
| 670 |
|
| 671 |
return self._draw_contour(img=img) |
| 672 |
|
| 673 |
def recolor(self, random_state=None, color_func=None, colormap=None): |
| 674 |
"""Recolor existing layout. |
| 675 |
|
| 676 |
Applying a new coloring is much faster than generating the whole |
| 677 |
wordcloud. |
| 678 |
|
| 679 |
Parameters |
| 680 |
---------- |
| 681 |
random_state : RandomState, int, or None, default=None |
| 682 |
If not None, a fixed random state is used. If an int is given, this |
| 683 |
is used as seed for a random.Random state. |
| 684 |
|
| 685 |
color_func : function or None, default=None |
| 686 |
Function to generate new color from word count, font size, position |
| 687 |
and orientation. If None, self.color_func is used. |
| 688 |
|
| 689 |
colormap : string or matplotlib colormap, default=None |
| 690 |
Use this colormap to generate new colors. Ignored if color_func |
| 691 |
is specified. If None, self.color_func (or self.color_map) is used. |
| 692 |
|
| 693 |
Returns |
| 694 |
------- |
| 695 |
self |
| 696 |
""" |
| 697 |
if isinstance(random_state, int): |
| 698 |
random_state = Random(random_state) |
| 699 |
self._check_generated() |
| 700 |
|
| 701 |
if color_func is None: |
| 702 |
if colormap is None: |
| 703 |
color_func = self.color_func |
| 704 |
else: |
| 705 |
color_func = colormap_color_func(colormap) |
| 706 |
self.layout_ = [(word_freq, font_size, position, orientation, |
| 707 |
color_func(word=word_freq[0], font_size=font_size, |
| 708 |
position=position, orientation=orientation, |
| 709 |
random_state=random_state, |
| 710 |
font_path=self.font_path)) |
| 711 |
for word_freq, font_size, position, orientation, _ |
| 712 |
in self.layout_] |
| 713 |
return self |
| 714 |
|
| 715 |
def to_file(self, filename): |
| 716 |
"""Export to image file. |
| 717 |
|
| 718 |
Parameters |
| 719 |
---------- |
| 720 |
filename : string |
| 721 |
Location to write to. |
| 722 |
|
| 723 |
Returns |
| 724 |
------- |
| 725 |
self |
| 726 |
""" |
| 727 |
|
| 728 |
img = self.to_image() |
| 729 |
img.save(filename, optimize=True) |
| 730 |
return self |
| 731 |
|
| 732 |
def to_array(self): |
| 733 |
"""Convert to numpy array. |
| 734 |
|
| 735 |
Returns |
| 736 |
------- |
| 737 |
image : nd-array size (width, height, 3) |
| 738 |
Word cloud image as numpy matrix. |
| 739 |
""" |
| 740 |
return np.array(self.to_image()) |
| 741 |
|
| 742 |
def __array__(self): |
| 743 |
"""Convert to numpy array. |
| 744 |
|
| 745 |
Returns |
| 746 |
------- |
| 747 |
image : nd-array size (width, height, 3) |
| 748 |
Word cloud image as numpy matrix. |
| 749 |
""" |
| 750 |
return self.to_array() |
| 751 |
|
| 752 |
def to_svg(self, embed_font=False, optimize_embedded_font=True, embed_image=False): |
| 753 |
"""Export to SVG. |
| 754 |
|
| 755 |
Font is assumed to be available to the SVG reader. Otherwise, text |
| 756 |
coordinates may produce artifacts when rendered with replacement font. |
| 757 |
It is also possible to include a subset of the original font in WOFF |
| 758 |
format using ``embed_font`` (requires `fontTools`). |
| 759 |
|
| 760 |
Note that some renderers do not handle glyphs the same way, and may |
| 761 |
differ from ``to_image`` result. In particular, Complex Text Layout may |
| 762 |
not be supported. In this typesetting, the shape or positioning of a |
| 763 |
grapheme depends on its relation to other graphemes. |
| 764 |
|
| 765 |
Pillow, since version 4.2.0, supports CTL using ``libraqm``. However, |
| 766 |
due to dependencies, this feature is not always enabled. Hence, the |
| 767 |
same rendering differences may appear in ``to_image``. As this |
| 768 |
rasterized output is used to compute the layout, this also affects the |
| 769 |
layout generation. Use ``PIL.features.check`` to test availability of |
| 770 |
``raqm``. |
| 771 |
|
| 772 |
Consistant rendering is therefore expected if both Pillow and the SVG |
| 773 |
renderer have the same support of CTL. |
| 774 |
|
| 775 |
Contour drawing is not supported. |
| 776 |
|
| 777 |
Parameters |
| 778 |
---------- |
| 779 |
embed_font : bool, default=False |
| 780 |
Whether to include font inside resulting SVG file. |
| 781 |
|
| 782 |
optimize_embedded_font : bool, default=True |
| 783 |
Whether to be aggressive when embedding a font, to reduce size. In |
| 784 |
particular, hinting tables are dropped, which may introduce slight |
| 785 |
changes to character shapes (w.r.t. `to_image` baseline). |
| 786 |
|
| 787 |
embed_image : bool, default=False |
| 788 |
Whether to include rasterized image inside resulting SVG file. |
| 789 |
Useful for debugging. |
| 790 |
|
| 791 |
Returns |
| 792 |
------- |
| 793 |
content : string |
| 794 |
Word cloud image as SVG string |
| 795 |
""" |
| 796 |
|
| 797 |
# TODO should add option to specify URL for font (i.e. WOFF file) |
| 798 |
|
| 799 |
# Make sure layout is generated |
| 800 |
self._check_generated() |
| 801 |
|
| 802 |
# Get output size, in pixels |
| 803 |
if self.mask is not None: |
| 804 |
width = self.mask.shape[1] |
| 805 |
height = self.mask.shape[0] |
| 806 |
else: |
| 807 |
height, width = self.height, self.width |
| 808 |
|
| 809 |
# Get max font size |
| 810 |
if self.max_font_size is None: |
| 811 |
max_font_size = max(w[1] for w in self.layout_) |
| 812 |
else: |
| 813 |
max_font_size = self.max_font_size |
| 814 |
|
| 815 |
# Text buffer |
| 816 |
result = [] |
| 817 |
|
| 818 |
# Get font information |
| 819 |
font = ImageFont.truetype(self.font_path, int(max_font_size * self.scale)) |
| 820 |
raw_font_family, raw_font_style = font.getname() |
| 821 |
# TODO properly escape/quote this name? |
| 822 |
font_family = repr(raw_font_family) |
| 823 |
# TODO better support for uncommon font styles/weights? |
| 824 |
raw_font_style = raw_font_style.lower() |
| 825 |
if 'bold' in raw_font_style: |
| 826 |
font_weight = 'bold' |
| 827 |
else: |
| 828 |
font_weight = 'normal' |
| 829 |
if 'italic' in raw_font_style: |
| 830 |
font_style = 'italic' |
| 831 |
elif 'oblique' in raw_font_style: |
| 832 |
font_style = 'oblique' |
| 833 |
else: |
| 834 |
font_style = 'normal' |
| 835 |
|
| 836 |
# Add header |
| 837 |
result.append( |
| 838 |
'<svg' |
| 839 |
' xmlns="http://www.w3.org/2000/svg"' |
| 840 |
' width="{}"' |
| 841 |
' height="{}"' |
| 842 |
'>' |
| 843 |
.format( |
| 844 |
width * self.scale, |
| 845 |
height * self.scale |
| 846 |
) |
| 847 |
) |
| 848 |
|
| 849 |
# Embed font, if requested |
| 850 |
if embed_font: |
| 851 |
|
| 852 |
# Import here, to avoid hard dependency on fonttools |
| 853 |
import fontTools |
| 854 |
import fontTools.subset |
| 855 |
|
| 856 |
# Subset options |
| 857 |
options = fontTools.subset.Options( |
| 858 |
|
| 859 |
# Small impact on character shapes, but reduce size a lot |
| 860 |
hinting=not optimize_embedded_font, |
| 861 |
|
| 862 |
# On small subsets, can improve size |
| 863 |
desubroutinize=optimize_embedded_font, |
| 864 |
|
| 865 |
# Try to be lenient |
| 866 |
ignore_missing_glyphs=True, |
| 867 |
) |
| 868 |
|
| 869 |
# Load and subset font |
| 870 |
ttf = fontTools.subset.load_font(self.font_path, options) |
| 871 |
subsetter = fontTools.subset.Subsetter(options) |
| 872 |
characters = {c for item in self.layout_ for c in item[0][0]} |
| 873 |
text = ''.join(characters) |
| 874 |
subsetter.populate(text=text) |
| 875 |
subsetter.subset(ttf) |
| 876 |
|
| 877 |
# Export as WOFF |
| 878 |
# TODO is there a better method, i.e. directly export to WOFF? |
| 879 |
buffer = io.BytesIO() |
| 880 |
ttf.saveXML(buffer) |
| 881 |
buffer.seek(0) |
| 882 |
woff = fontTools.ttLib.TTFont(flavor='woff') |
| 883 |
woff.importXML(buffer) |
| 884 |
|
| 885 |
# Create stylesheet with embedded font face |
| 886 |
buffer = io.BytesIO() |
| 887 |
woff.save(buffer) |
| 888 |
data = base64.b64encode(buffer.getbuffer()).decode('ascii') |
| 889 |
url = 'data:application/font-woff;charset=utf-8;base64,' + data |
| 890 |
result.append( |
| 891 |
'<style>' |
| 892 |
'@font-face{{' |
| 893 |
'font-family:{};' |
| 894 |
'font-weight:{};' |
| 895 |
'font-style:{};' |
| 896 |
'src:url("{}")format("woff");' |
| 897 |
'}}' |
| 898 |
'</style>' |
| 899 |
.format( |
| 900 |
font_family, |
| 901 |
font_weight, |
| 902 |
font_style, |
| 903 |
url |
| 904 |
) |
| 905 |
) |
| 906 |
|
| 907 |
# Select global style |
| 908 |
result.append( |
| 909 |
'<style>' |
| 910 |
'text{{' |
| 911 |
'font-family:{};' |
| 912 |
'font-weight:{};' |
| 913 |
'font-style:{};' |
| 914 |
'}}' |
| 915 |
'</style>' |
| 916 |
.format( |
| 917 |
font_family, |
| 918 |
font_weight, |
| 919 |
font_style |
| 920 |
) |
| 921 |
) |
| 922 |
|
| 923 |
# Add background |
| 924 |
if self.background_color is not None: |
| 925 |
result.append( |
| 926 |
'<rect' |
| 927 |
' width="100%"' |
| 928 |
' height="100%"' |
| 929 |
' style="fill:{}"' |
| 930 |
'>' |
| 931 |
'</rect>' |
| 932 |
.format(self.background_color) |
| 933 |
) |
| 934 |
|
| 935 |
# Embed image, useful for debug purpose |
| 936 |
if embed_image: |
| 937 |
image = self.to_image() |
| 938 |
data = io.BytesIO() |
| 939 |
image.save(data, format='JPEG') |
| 940 |
data = base64.b64encode(data.getbuffer()).decode('ascii') |
| 941 |
result.append( |
| 942 |
'<image' |
| 943 |
' width="100%"' |
| 944 |
' height="100%"' |
| 945 |
' href="data:image/jpg;base64,{}"' |
| 946 |
'/>' |
| 947 |
.format(data) |
| 948 |
) |
| 949 |
|
| 950 |
# For each word in layout |
| 951 |
for (word, count), font_size, (y, x), orientation, color in self.layout_: |
| 952 |
x *= self.scale |
| 953 |
y *= self.scale |
| 954 |
|
| 955 |
# Get text metrics |
| 956 |
font = ImageFont.truetype(self.font_path, int(font_size * self.scale)) |
| 957 |
(size_x, size_y), (offset_x, offset_y) = font.font.getsize(word) |
| 958 |
ascent, descent = font.getmetrics() |
| 959 |
|
| 960 |
# Compute text bounding box |
| 961 |
min_x = -offset_x |
| 962 |
max_x = size_x - offset_x |
| 963 |
max_y = ascent - offset_y |
| 964 |
|
| 965 |
# Compute text attributes |
| 966 |
attributes = {} |
| 967 |
if orientation == Image.ROTATE_90: |
| 968 |
x += max_y |
| 969 |
y += max_x - min_x |
| 970 |
transform = 'translate({},{}) rotate(-90)'.format(x, y) |
| 971 |
else: |
| 972 |
x += min_x |
| 973 |
y += max_y |
| 974 |
transform = 'translate({},{})'.format(x, y) |
| 975 |
|
| 976 |
# Create node |
| 977 |
attributes = ' '.join('{}="{}"'.format(k, v) for k, v in attributes.items()) |
| 978 |
result.append( |
| 979 |
'<text' |
| 980 |
' transform="{}"' |
| 981 |
' font-size="{}"' |
| 982 |
' style="fill:{}"' |
| 983 |
'>' |
| 984 |
'{}' |
| 985 |
'</text>' |
| 986 |
.format( |
| 987 |
transform, |
| 988 |
font_size * self.scale, |
| 989 |
color, |
| 990 |
saxutils.escape(word) |
| 991 |
) |
| 992 |
) |
| 993 |
|
| 994 |
# TODO draw contour |
| 995 |
|
| 996 |
# Complete SVG file |
| 997 |
result.append('</svg>') |
| 998 |
return '\n'.join(result) |
| 999 |
|
| 1000 |
def _get_bolean_mask(self, mask): |
| 1001 |
"""Cast to two dimensional boolean mask.""" |
| 1002 |
if mask.dtype.kind == 'f': |
| 1003 |
warnings.warn("mask image should be unsigned byte between 0" |
| 1004 |
" and 255. Got a float array") |
| 1005 |
if mask.ndim == 2: |
| 1006 |
boolean_mask = mask == 255 |
| 1007 |
elif mask.ndim == 3: |
| 1008 |
# if all channels are white, mask out |
| 1009 |
boolean_mask = np.all(mask[:, :, :3] == 255, axis=-1) |
| 1010 |
else: |
| 1011 |
raise ValueError("Got mask of invalid shape: %s" % str(mask.shape)) |
| 1012 |
return boolean_mask |
| 1013 |
|
| 1014 |
def _draw_contour(self, img): |
| 1015 |
"""Draw mask contour on a pillow image.""" |
| 1016 |
if self.mask is None or self.contour_width == 0: |
| 1017 |
return img |
| 1018 |
|
| 1019 |
mask = self._get_bolean_mask(self.mask) * 255 |
| 1020 |
contour = Image.fromarray(mask.astype(np.uint8)) |
| 1021 |
contour = contour.resize(img.size) |
| 1022 |
contour = contour.filter(ImageFilter.FIND_EDGES) |
| 1023 |
contour = np.array(contour) |
| 1024 |
|
| 1025 |
# make sure borders are not drawn before changing width |
| 1026 |
contour[[0, -1], :] = 0 |
| 1027 |
contour[:, [0, -1]] = 0 |
| 1028 |
|
| 1029 |
# use gaussian to change width, divide by 10 to give more resolution |
| 1030 |
radius = self.contour_width / 10 |
| 1031 |
contour = Image.fromarray(contour) |
| 1032 |
contour = contour.filter(ImageFilter.GaussianBlur(radius=radius)) |
| 1033 |
contour = np.array(contour) > 0 |
| 1034 |
contour = np.dstack((contour, contour, contour)) |
| 1035 |
|
| 1036 |
# color the contour |
| 1037 |
ret = np.array(img) * np.invert(contour) |
| 1038 |
if self.contour_color != 'black': |
| 1039 |
color = Image.new(img.mode, img.size, self.contour_color) |
| 1040 |
ret += np.array(color) * contour |
| 1041 |
|
| 1042 |
return Image.fromarray(ret) |