Debian Codegraph @

tests/test_webpages/Tutorial — Webstruct 0.6 documentation.html

raw
1
2
3 <!DOCTYPE html>
4 <!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
5 <!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
6 <head>
7 <meta charset="utf-8">
8
9 <meta name="viewport" content="width=device-width, initial-scale=1.0">
10
11 <title>Tutorial &mdash; Webstruct 0.6 documentation</title>
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27 <link rel="stylesheet" href="_static/css/theme.css" type="text/css" />
28 <link rel="stylesheet" href="_static/pygments.css" type="text/css" />
29 <link rel="index" title="Index" href="genindex.html" />
30 <link rel="search" title="Search" href="search.html" />
31 <link rel="next" title="Reference" href="ref/index.html" />
32 <link rel="prev" title="Webstruct" href="intro.html" />
33
34
35 <script src="_static/js/modernizr.min.js"></script>
36
37
38 <!-- RTD Extra Head -->
39
40 <!--
41 Always link to the latest version, as canonical.
42 http://docs.readthedocs.org/en/latest/canonical.html
43 -->
44 <link rel="canonical" href="https://webstruct.readthedocs.io/en/latest/tutorial.html" />
45
46 <link rel="stylesheet" href="https://media.readthedocs.org/css/readthedocs-doc-embed.css" type="text/css" />
47
48 <script type="text/javascript" src="_static/readthedocs-data.js"></script>
49
50 <!-- Add page-specific data, which must exist in the page js, not global -->
51 <script type="text/javascript">
52 READTHEDOCS_DATA['page'] = 'tutorial'
53 READTHEDOCS_DATA['source_suffix'] = '.rst'
54 </script>
55
56 <script type="text/javascript" src="https://media.readthedocs.org/javascript/readthedocs-analytics.js"></script>
57
58 <!-- end RTD <extrahead> -->
59 </head>
60
61 <body class="wy-body-for-nav">
62
63
64 <div class="wy-grid-for-nav">
65
66
67 <nav data-toggle="wy-nav-shift" class="wy-nav-side">
68 <div class="wy-side-scroll">
69 <div class="wy-side-nav-search">
70
71
72
73 <a href="index.html" class="icon icon-home"> Webstruct
74
75
76
77 </a>
78
79
80
81
82
83 <div class="version">
84 latest
85 </div>
86
87
88
89
90 <div role="search">
91 <form id="rtd-search-form" class="wy-form" action="search.html" method="get">
92 <input type="text" name="q" placeholder="Search docs" />
93 <input type="hidden" name="check_keywords" value="yes" />
94 <input type="hidden" name="area" value="default" />
95 </form>
96 </div>
97
98
99 </div>
100
101 <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
102
103
104
105
106
107
108 <ul class="current">
109 <li class="toctree-l1"><a class="reference internal" href="intro.html">Webstruct</a></li>
110 <li class="toctree-l1 current"><a class="current reference internal" href="#">Tutorial</a><ul>
111 <li class="toctree-l2"><a class="reference internal" href="#get-annotated-data">Get annotated data</a></li>
112 <li class="toctree-l2"><a class="reference internal" href="#from-html-to-tokens">From HTML to Tokens</a></li>
113 <li class="toctree-l2"><a class="reference internal" href="#feature-extraction">Feature Extraction</a></li>
114 <li class="toctree-l2"><a class="reference internal" href="#using-a-sequence-labelling-toolkit">Using a Sequence Labelling Toolkit</a><ul>
115 <li class="toctree-l3"><a class="reference internal" href="#defining-a-model">Defining a Model</a></li>
116 <li class="toctree-l3"><a class="reference internal" href="#training">Training</a></li>
117 </ul>
118 </li>
119 <li class="toctree-l2"><a class="reference internal" href="#named-entity-recognition">Named Entity Recognition</a></li>
120 <li class="toctree-l2"><a class="reference internal" href="#entity-grouping">Entity Grouping</a></li>
121 <li class="toctree-l2"><a class="reference internal" href="#model-development">Model Development</a></li>
122 </ul>
123 </li>
124 <li class="toctree-l1"><a class="reference internal" href="ref/index.html">Reference</a></li>
125 <li class="toctree-l1"><a class="reference internal" href="changes.html">Changes</a></li>
126 </ul>
127
128
129
130 </div>
131 </div>
132 </nav>
133
134 <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
135
136
137 <nav class="wy-nav-top" aria-label="top navigation">
138
139 <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
140 <a href="index.html">Webstruct</a>
141
142 </nav>
143
144
145 <div class="wy-nav-content">
146
147 <div class="rst-content">
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165 <div role="navigation" aria-label="breadcrumbs navigation">
166
167 <ul class="wy-breadcrumbs">
168
169 <li><a href="index.html">Docs</a> &raquo;</li>
170
171 <li>Tutorial</li>
172
173
174 <li class="wy-breadcrumbs-aside">
175
176
177
178 <a href="https://github.com/scrapinghub/webstruct/blob/master/docs/tutorial.rst" class="fa fa-github"> Edit on GitHub</a>
179
180
181
182 </li>
183
184 </ul>
185
186
187 <hr/>
188 </div>
189 <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
190 <div itemprop="articleBody">
191
192 <div class="section" id="tutorial">
193 <h1>Tutorial<a class="headerlink" href="#tutorial" title="Permalink to this headline">¶</a></h1>
194 <p>This tutorial assumes you are familiar with machine learning.</p>
195 <div class="section" id="get-annotated-data">
196 <h2>Get annotated data<a class="headerlink" href="#get-annotated-data" title="Permalink to this headline">¶</a></h2>
197 <p>First, you need the training/development data. We suggest to use
198 <a class="reference external" href="https://github.com/xtannier/WebAnnotator">WebAnnotator</a> Firefox extension to annotate HTML pages.</p>
199 <p>Recommended WebAnnotator options:</p>
200 <img alt="_images/wa-options.png" src="_images/wa-options.png" />
201 <p>Pro tip - enable WebAnnotator toolbar buttons:</p>
202 <img alt="_images/wa-buttons.png" src="_images/wa-buttons.png" />
203 <p>Follow WebAnnotator <a class="reference external" href="http://perso.limsi.fr/xtannier/en/WebAnnotator/">manual</a>
204 to define named entities and annotate some web pages (nested WebAnnotator
205 entities are not supported). Use “Save as..” menu item or “Save as”
206 toolbar button to save the results; don’t use “Export as”.</p>
207 <p>After that you can load annotated webpages as lxml trees:</p>
208 <div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">webstruct</span>
209 <span class="n">trees</span> <span class="o">=</span> <span class="n">webstruct</span><span class="o">.</span><span class="n">load_trees</span><span class="p">(</span><span class="s2">&quot;train/*.html&quot;</span><span class="p">,</span> <span class="n">webstruct</span><span class="o">.</span><span class="n">WebAnnotatorLoader</span><span class="p">())</span>
210 </pre></div>
211 </div>
212 <p>See <a class="reference internal" href="ref/loaders.html#html-loaders"><span class="std std-ref">HTML Loaders</span></a> for more info.
213 <a class="reference external" href="http://gate.ac.uk/">GATE</a> annotation format is also supported.</p>
214 </div>
215 <div class="section" id="from-html-to-tokens">
216 <h2>From HTML to Tokens<a class="headerlink" href="#from-html-to-tokens" title="Permalink to this headline">¶</a></h2>
217 <p>To convert HTML trees to a format suitable for sequence prediction algorithm
218 (like CRF, MEMM or Structured Perceptron) the following approach is used:</p>
219 <ol class="arabic simple">
220 <li>Text is extracted from HTML and split into tokens.</li>
221 <li>For each token a special <a class="reference internal" href="ref/features.html#webstruct.html_tokenizer.HtmlToken" title="webstruct.html_tokenizer.HtmlToken"><code class="xref py py-class docutils literal notranslate"><span class="pre">HtmlToken</span></code></a> instance is created. It
222 contains information not only about the text token itself, but also about
223 its position in HTML tree.</li>
224 </ol>
225 <p>A single HTML page corresponds to a single input sequence
226 (a list of HtmlTokens). For training/testing data
227 (where webpages are already annotated) there is also a list of labels for
228 each webpage, a label per HtmlToken.</p>
229 <p>To transform HTML trees into labels and HTML tokens
230 use <a class="reference internal" href="ref/features.html#webstruct.html_tokenizer.HtmlTokenizer" title="webstruct.html_tokenizer.HtmlTokenizer"><code class="xref py py-class docutils literal notranslate"><span class="pre">HtmlTokenizer</span></code></a>.</p>
231 <div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">html_tokenizer</span> <span class="o">=</span> <span class="n">webstruct</span><span class="o">.</span><span class="n">HtmlTokenizer</span><span class="p">()</span>
232 <span class="n">X</span><span class="p">,</span> <span class="n">y</span> <span class="o">=</span> <span class="n">html_tokenizer</span><span class="o">.</span><span class="n">tokenize</span><span class="p">(</span><span class="n">trees</span><span class="p">)</span>
233 </pre></div>
234 </div>
235 <p>Input trees should be loaded by one of the WebStruct loaders.
236 For consistency, for each tree (even if it is loaded from raw unannotated html)
237 <a class="reference internal" href="ref/features.html#webstruct.html_tokenizer.HtmlTokenizer" title="webstruct.html_tokenizer.HtmlTokenizer"><code class="xref py py-class docutils literal notranslate"><span class="pre">HtmlTokenizer</span></code></a> extracts two arrays: a list of <a class="reference internal" href="ref/features.html#webstruct.html_tokenizer.HtmlToken" title="webstruct.html_tokenizer.HtmlToken"><code class="xref py py-class docutils literal notranslate"><span class="pre">HtmlToken</span></code></a>
238 instances and a list of tags encoded using <a class="reference external" href="http://en.wikipedia.org/wiki/Inside_Outside_Beginning">IOB2</a> encoding
239 (also known as BIO encoding). So in our example <code class="docutils literal notranslate"><span class="pre">X</span></code> is a list of
240 lists of <a class="reference internal" href="ref/features.html#webstruct.html_tokenizer.HtmlToken" title="webstruct.html_tokenizer.HtmlToken"><code class="xref py py-class docutils literal notranslate"><span class="pre">HtmlToken</span></code></a> instances, and <code class="docutils literal notranslate"><span class="pre">y</span></code> is a list of lists
241 of strings.</p>
242 </div>
243 <div class="section" id="feature-extraction">
244 <h2>Feature Extraction<a class="headerlink" href="#feature-extraction" title="Permalink to this headline">¶</a></h2>
245 <p>For supervised machine learning algorithms to work we need to extract
246 <a class="reference external" href="http://en.wikipedia.org/wiki/Features_%28pattern_recognition%29">features</a>.</p>
247 <p>In WebStruct feature vectors are Python dicts
248 <code class="docutils literal notranslate"><span class="pre">{&quot;feature_name&quot;:</span> <span class="pre">&quot;feature_value&quot;}</span></code>; a dict is computed for
249 each HTML token. How to convert these dicts into representation required
250 by a sequence labelling toolkit depends on a toolkit used; we will cover
251 that later.</p>
252 <p>To compute feature dicts we’ll use <a class="reference internal" href="ref/features.html#webstruct.feature_extraction.HtmlFeatureExtractor" title="webstruct.feature_extraction.HtmlFeatureExtractor"><code class="xref py py-class docutils literal notranslate"><span class="pre">HtmlFeatureExtractor</span></code></a>.</p>
253 <p>First, define your feature functions. A feature function should take
254 an <a class="reference internal" href="ref/features.html#webstruct.html_tokenizer.HtmlToken" title="webstruct.html_tokenizer.HtmlToken"><code class="xref py py-class docutils literal notranslate"><span class="pre">HtmlToken</span></code></a> instance and return a feature dict;
255 feature dicts from individual feature functions will be merged
256 into the final feature dict for a token. Feature functions can ask questions
257 about token itself, its neighbours (in the same HTML element),
258 its position in HTML.</p>
259 <div class="admonition note">
260 <p class="first admonition-title">Note</p>
261 <p class="last">WebStruct supports other kind of feature functions that work on multiple
262 tokens; we don’t cover them in this tutorial.</p>
263 </div>
264 <p>There are predefined feature functions in <a class="reference internal" href="ref/features.html#module-webstruct.features" title="webstruct.features"><code class="xref py py-mod docutils literal notranslate"><span class="pre">webstruct.features</span></code></a>,
265 but for this tutorial let’s create some functions ourselves:</p>
266 <div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">token_identity</span><span class="p">(</span><span class="n">html_token</span><span class="p">):</span>
267 <span class="k">return</span> <span class="p">{</span><span class="s1">&#39;token&#39;</span><span class="p">:</span> <span class="n">html_token</span><span class="o">.</span><span class="n">token</span><span class="p">}</span>
268
269 <span class="k">def</span> <span class="nf">token_isupper</span><span class="p">(</span><span class="n">html_token</span><span class="p">):</span>
270 <span class="k">return</span> <span class="p">{</span><span class="s1">&#39;isupper&#39;</span><span class="p">:</span> <span class="n">html_token</span><span class="o">.</span><span class="n">token</span><span class="o">.</span><span class="n">isupper</span><span class="p">()}</span>
271
272 <span class="k">def</span> <span class="nf">parent_tag</span><span class="p">(</span><span class="n">html_token</span><span class="p">):</span>
273 <span class="k">return</span> <span class="p">{</span><span class="s1">&#39;parent_tag&#39;</span><span class="p">:</span> <span class="n">html_token</span><span class="o">.</span><span class="n">parent</span><span class="o">.</span><span class="n">tag</span><span class="p">}</span>
274
275 <span class="k">def</span> <span class="nf">border_at_left</span><span class="p">(</span><span class="n">html_token</span><span class="p">):</span>
276 <span class="k">return</span> <span class="p">{</span><span class="s1">&#39;border_at_left&#39;</span><span class="p">:</span> <span class="n">html_token</span><span class="o">.</span><span class="n">index</span> <span class="o">==</span> <span class="mi">0</span><span class="p">}</span>
277 </pre></div>
278 </div>
279 <p>Next, create <a class="reference internal" href="ref/features.html#webstruct.feature_extraction.HtmlFeatureExtractor" title="webstruct.feature_extraction.HtmlFeatureExtractor"><code class="xref py py-class docutils literal notranslate"><span class="pre">HtmlFeatureExtractor</span></code></a>:</p>
280 <div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">feature_extractor</span> <span class="o">=</span> <span class="n">HtmlFeatureExtractor</span><span class="p">(</span>
281 <span class="n">token_features</span> <span class="o">=</span> <span class="p">[</span>
282 <span class="n">token_identity</span><span class="p">,</span>
283 <span class="n">token_isupper</span><span class="p">,</span>
284 <span class="n">parent_tag</span><span class="p">,</span>
285 <span class="n">border_at_left</span>
286 <span class="p">]</span>
287 <span class="p">)</span>
288 </pre></div>
289 </div>
290 <p>and use it to extract feature dicts:</p>
291 <div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">features</span> <span class="o">=</span> <span class="n">feature_extractor</span><span class="o">.</span><span class="n">fit_transform</span><span class="p">(</span><span class="n">X</span><span class="p">)</span>
292 </pre></div>
293 </div>
294 <p>See <a class="reference internal" href="ref/features.html#feature-extraction"><span class="std std-ref">Feature Extraction</span></a> for more info about HTML tokenization and
295 feature extraction.</p>
296 </div>
297 <div class="section" id="using-a-sequence-labelling-toolkit">
298 <h2>Using a Sequence Labelling Toolkit<a class="headerlink" href="#using-a-sequence-labelling-toolkit" title="Permalink to this headline">¶</a></h2>
299 <p>WebStruct doesn’t provide a CRF or Structured Perceptron implementation;
300 learning and prediction is supposed to be handled by an external
301 sequence labelling toolkit like <a class="reference external" href="http://www.chokkan.org/software/crfsuite/">CRFSuite</a>, <a class="reference external" href="http://wapiti.limsi.fr">Wapiti</a> or <a class="reference external" href="https://github.com/larsmans/seqlearn">seqlearn</a>.</p>
302 <p>Once feature dicts are extracted from HTML you should convert them to
303 a format required by your sequence labelling tooklit and use this toolkit
304 to train a model and do the prediction. For example, you may use
305 DictVectorizer from scikit-learn to convert feature dicts
306 into <a class="reference external" href="https://github.com/larsmans/seqlearn">seqlearn</a> input format.</p>
307 <p>We’ll use <a class="reference external" href="http://www.chokkan.org/software/crfsuite/">CRFSuite</a> in this tutorial.</p>
308 <p>WebStruct provides some helpers for CRFSuite sequence labelling toolkit.
309 To use CRFSuite with WebStruct, you need</p>
310 <ul class="simple">
311 <li>sklearn-crfsuite package (which depends on python-crfsuite and sklearn)</li>
312 </ul>
313 <div class="section" id="defining-a-model">
314 <h3>Defining a Model<a class="headerlink" href="#defining-a-model" title="Permalink to this headline">¶</a></h3>
315 <p>Basic way to define CRF model is the following:</p>
316 <div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">model</span> <span class="o">=</span> <span class="n">webstruct</span><span class="o">.</span><span class="n">create_crfsuite_pipeline</span><span class="p">(</span>
317 <span class="n">token_features</span><span class="o">=</span><span class="p">[</span><span class="n">token_identity</span><span class="p">,</span> <span class="n">token_isupper</span><span class="p">,</span> <span class="n">parent_tag</span><span class="p">,</span> <span class="n">border_at_left</span><span class="p">],</span>
318 <span class="n">verbose</span><span class="o">=</span><span class="kc">True</span>
319 <span class="p">)</span>
320 </pre></div>
321 </div>
322 <p>First <a class="reference internal" href="ref/crfsuite.html#webstruct.crfsuite.create_crfsuite_pipeline" title="webstruct.crfsuite.create_crfsuite_pipeline"><code class="xref py py-func docutils literal notranslate"><span class="pre">create_crfsuite_pipeline()</span></code></a> argument is a list of feature functions which will be used for training.
323 <code class="docutils literal notranslate"><span class="pre">verbose</span></code> is a boolean parameter enabling verbose output of various training information;
324 check sklearn-crfsuite <a class="reference external" href="https://sklearn-crfsuite.readthedocs.io/en/latest/api.html#sklearn_crfsuite.CRF">API reference</a>
325 for available options.</p>
326 <p>Under the hood <a class="reference internal" href="ref/crfsuite.html#webstruct.crfsuite.create_crfsuite_pipeline" title="webstruct.crfsuite.create_crfsuite_pipeline"><code class="xref py py-func docutils literal notranslate"><span class="pre">create_crfsuite_pipeline()</span></code></a> creates a
327 <code class="docutils literal notranslate"><span class="pre">sklearn.pipeline.Pipeline</span></code> with an <a class="reference internal" href="ref/features.html#webstruct.feature_extraction.HtmlFeatureExtractor" title="webstruct.feature_extraction.HtmlFeatureExtractor"><code class="xref py py-class docutils literal notranslate"><span class="pre">HtmlFeatureExtractor</span></code></a> instance
328 followed by <code class="xref py py-class docutils literal notranslate"><span class="pre">sklearn_crfsuite.CRF</span></code> instance. The example above is just a shortcut
329 for the following:</p>
330 <div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">model</span> <span class="o">=</span> <span class="n">Pipeline</span><span class="p">([</span>
331 <span class="p">(</span><span class="s1">&#39;fe&#39;</span><span class="p">,</span> <span class="n">HtmlFeatureExtractor</span><span class="p">(</span>
332 <span class="n">token_features</span> <span class="o">=</span> <span class="p">[</span>
333 <span class="n">token_identity</span><span class="p">,</span>
334 <span class="n">token_isupper</span><span class="p">,</span>
335 <span class="n">parent_tag</span><span class="p">,</span>
336 <span class="n">border_at_left</span><span class="p">,</span>
337 <span class="p">]</span>
338 <span class="p">)),</span>
339 <span class="p">(</span><span class="s1">&#39;crf&#39;</span><span class="p">,</span> <span class="n">sklearn_crfsuite</span><span class="o">.</span><span class="n">CRF</span><span class="p">(</span>
340 <span class="n">verbose</span><span class="o">=</span><span class="kc">True</span>
341 <span class="p">)),</span>
342 <span class="p">])</span>
343 </pre></div>
344 </div>
345 </div>
346 <div class="section" id="training">
347 <h3>Training<a class="headerlink" href="#training" title="Permalink to this headline">¶</a></h3>
348 <p>To train a model use its <code class="docutils literal notranslate"><span class="pre">fit</span></code> method:</p>
349 <div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">model</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">X</span><span class="p">,</span> <span class="n">y</span><span class="p">)</span>
350 </pre></div>
351 </div>
352 <p><code class="docutils literal notranslate"><span class="pre">X</span></code> and <code class="docutils literal notranslate"><span class="pre">y</span></code> are return values of <a class="reference internal" href="ref/features.html#webstruct.html_tokenizer.HtmlTokenizer.tokenize" title="webstruct.html_tokenizer.HtmlTokenizer.tokenize"><code class="xref py py-meth docutils literal notranslate"><span class="pre">HtmlTokenizer.tokenize()</span></code></a>
353 (a list of lists of <a class="reference internal" href="ref/features.html#webstruct.html_tokenizer.HtmlToken" title="webstruct.html_tokenizer.HtmlToken"><code class="xref py py-class docutils literal notranslate"><span class="pre">HtmlToken</span></code></a> instances and a list of
354 lists of string IOB labels).</p>
355 <p>If you use <code class="xref py py-class docutils literal notranslate"><span class="pre">sklearn_crfsuite.CRF</span></code> directly then train it using
356 <code class="xref py py-meth docutils literal notranslate"><span class="pre">CRF.fit()</span></code> method. It accepts 2 lists: a list of lists of
357 feature dicts, and a list of lists of tags:</p>
358 <div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">model</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">features</span><span class="p">,</span> <span class="n">y</span><span class="p">)</span>
359 </pre></div>
360 </div>
361 </div>
362 </div>
363 <div class="section" id="named-entity-recognition">
364 <h2>Named Entity Recognition<a class="headerlink" href="#named-entity-recognition" title="Permalink to this headline">¶</a></h2>
365 <p>Once you got a trained model you can use it to extract entities
366 from unseen (unannotated) webpages. First, get some binary HTML data:</p>
367 <div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">urllib2</span>
368 <span class="gp">&gt;&gt;&gt; </span><span class="n">html</span> <span class="o">=</span> <span class="n">urllib2</span><span class="o">.</span><span class="n">urlopen</span><span class="p">(</span><span class="s2">&quot;http://scrapinghub.com/contact&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">read</span><span class="p">()</span>
369 </pre></div>
370 </div>
371 <p>Then create a <a class="reference internal" href="ref/model.html#webstruct.model.NER" title="webstruct.model.NER"><code class="xref py py-class docutils literal notranslate"><span class="pre">NER</span></code></a> instance initialized with a trained model:</p>
372 <div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">ner</span> <span class="o">=</span> <span class="n">webstruct</span><span class="o">.</span><span class="n">NER</span><span class="p">(</span><span class="n">model</span><span class="p">)</span>
373 </pre></div>
374 </div>
375 <p>The <code class="docutils literal notranslate"><span class="pre">model</span></code> must provide a <code class="docutils literal notranslate"><span class="pre">predict</span></code> method that extracts features
376 from HTML tokens and predicts labels for these tokens. A pipeline created with
377 <a class="reference internal" href="ref/crfsuite.html#webstruct.crfsuite.create_crfsuite_pipeline" title="webstruct.crfsuite.create_crfsuite_pipeline"><code class="xref py py-func docutils literal notranslate"><span class="pre">create_crfsuite_pipeline()</span></code></a> function fits this definition.</p>
378 <p>Finally, use <a class="reference internal" href="ref/model.html#webstruct.model.NER.extract" title="webstruct.model.NER.extract"><code class="xref py py-meth docutils literal notranslate"><span class="pre">NER.extract()</span></code></a> method to extract entities:</p>
379 <div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">ner</span><span class="o">.</span><span class="n">extract</span><span class="p">(</span><span class="n">html</span><span class="p">)</span>
380 <span class="go">[(&#39;Scrapinghub&#39;, &#39;ORG&#39;), ..., (&#39;Iturriaga 3429 ap. 1&#39;, &#39;STREET&#39;), ...]</span>
381 </pre></div>
382 </div>
383 <p>Generally, the steps are:</p>
384 <ol class="arabic simple">
385 <li>Load data using <a class="reference internal" href="ref/loaders.html#webstruct.loaders.HtmlLoader" title="webstruct.loaders.HtmlLoader"><code class="xref py py-class docutils literal notranslate"><span class="pre">HtmlLoader</span></code></a> loader. If a custom HTML cleaner
386 was used for loading training data make sure to apply it here as well.</li>
387 <li>Use the same <code class="docutils literal notranslate"><span class="pre">html_tokenizer</span></code> as used for training to extract HTML tokens
388 from loaded trees. All labels would be “O” when using <a class="reference internal" href="ref/loaders.html#webstruct.loaders.HtmlLoader" title="webstruct.loaders.HtmlLoader"><code class="xref py py-class docutils literal notranslate"><span class="pre">HtmlLoader</span></code></a>
389 loader - <code class="docutils literal notranslate"><span class="pre">y</span></code> can be discarded.</li>
390 <li>Use the same <code class="docutils literal notranslate"><span class="pre">feature_extractor</span></code> as used for training to extract
391 features.</li>
392 <li>Run <code class="docutils literal notranslate"><span class="pre">your_crf.predict()</span></code> method (e.g. <code class="xref py py-meth docutils literal notranslate"><span class="pre">CRF.predict()</span></code>)
393 on features extracted in (3) to get the prediction - a list of IOB2-encoded
394 tags for each input document.</li>
395 <li>Build entities from input tokens based on predicted tags
396 (check <a class="reference internal" href="ref/misc.html#webstruct.sequence_encoding.IobEncoder.group" title="webstruct.sequence_encoding.IobEncoder.group"><code class="xref py py-meth docutils literal notranslate"><span class="pre">IobEncoder.group()</span></code></a> and <a class="reference internal" href="ref/misc.html#webstruct.utils.smart_join" title="webstruct.utils.smart_join"><code class="xref py py-func docutils literal notranslate"><span class="pre">smart_join()</span></code></a>).</li>
397 <li>Split entities into groups (optional). One way to do it is to use
398 <a class="reference internal" href="ref/grouping.html#module-webstruct.grouping" title="webstruct.grouping"><code class="xref py py-mod docutils literal notranslate"><span class="pre">webstruct.grouping</span></code></a>.</li>
399 </ol>
400 <p><a class="reference internal" href="ref/model.html#webstruct.model.NER" title="webstruct.model.NER"><code class="xref py py-class docutils literal notranslate"><span class="pre">NER</span></code></a> helper class combines HTML loading, HTML tokenization,
401 feature extraction, CRF model, entity building and grouping.</p>
402 </div>
403 <div class="section" id="entity-grouping">
404 <h2>Entity Grouping<a class="headerlink" href="#entity-grouping" title="Permalink to this headline">¶</a></h2>
405 <p>Detecting entities on their own is not always enough; in many cases
406 what is wanted is to find the relationship between them. For example,
407 “<strong>street_name/STREET city_name/CITY zipcode_number/ZIPCODE</strong>
408 form an address”, or “<strong>phone/TEL</strong> is a phone of <strong>person/PER</strong>”.</p>
409 <p>The first approximation is to say that all entities from a single webpage
410 are related. For example, if we have extracted some <strong>organizaion/ORG</strong> and some
411 <strong>phone/TEL</strong> from a single webpage we may assume that the phone
412 is a contact phone of the organization.</p>
413 <p>Sometimes there are several “entity groups” on a webpage. If a page
414 contains contact phones of several persons or several business locations
415 it is better to split all entities into groups of related
416 entities - “person name + his/her phone(s)” or “address”.</p>
417 <p>WebStruct provides an <a class="reference internal" href="ref/grouping.html#grouping-algorithm"><span class="std std-ref">unsupervised algorithm</span></a>
418 for extracting such entity groups. Algorithm prefers to build
419 large groups without entities of duplicate types; if a split is needed
420 algorithm tries to split at points where distance between entities is larger.</p>
421 <p>Use <a class="reference internal" href="ref/model.html#webstruct.model.NER.extract_groups" title="webstruct.model.NER.extract_groups"><code class="xref py py-meth docutils literal notranslate"><span class="pre">NER.extract_groups()</span></code></a> to extract groups of entities:</p>
422 <div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">ner</span><span class="o">.</span><span class="n">extract_groups</span><span class="p">(</span><span class="n">html</span><span class="p">)</span>
423 <span class="go">[[...], ... [(&#39;Iturriaga 3429 ap. 1&#39;, &#39;STREET&#39;), (&#39;Montevideo&#39;, &#39;CITY&#39;), ...]]</span>
424 </pre></div>
425 </div>
426 <p>Sometimes it is better to allow some entity types to appear
427 multuple times in a group. For example, a person (PER entity) may have
428 several contact phones and faxes (TEL and FAX entities) - we should penalize
429 groups with multiple PERs, but multiple TELs and FAXes are fine.
430 Use <code class="docutils literal notranslate"><span class="pre">dont_penalize</span></code> argument if you want to allow some entity types
431 to appear multiple times in a group:</p>
432 <div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">ner</span><span class="o">.</span><span class="n">extract_groups</span><span class="p">(</span><span class="n">html</span><span class="p">,</span> <span class="n">dont_penalize</span><span class="o">=</span><span class="p">{</span><span class="s1">&#39;TEL&#39;</span><span class="p">,</span> <span class="s1">&#39;FAX&#39;</span><span class="p">})</span>
433 </pre></div>
434 </div>
435 <p>The simple algorithm WebStruct provides is by no means a general solution
436 to relation detection, but give it a try - maybe it is enough for your task.</p>
437 </div>
438 <div class="section" id="model-development">
439 <h2>Model Development<a class="headerlink" href="#model-development" title="Permalink to this headline">¶</a></h2>
440 <p>To develop the model you need to choose the learning algorithm,
441 features, hyperparameters, etc. To do that you need scoring metrics,
442 cross-validation utilities and tools for debugging what classifier learned.
443 WebStruct helps in the following way:</p>
444 <ol class="arabic">
445 <li><p class="first">Pipeline created by <a class="reference internal" href="ref/crfsuite.html#webstruct.crfsuite.create_crfsuite_pipeline" title="webstruct.crfsuite.create_crfsuite_pipeline"><code class="xref py py-func docutils literal notranslate"><span class="pre">create_crfsuite_pipeline()</span></code></a> is compatible with
446 <a class="reference external" href="http://scikit-learn.org/stable/modules/cross_validation.html">cross-validation</a> and <a class="reference external" href="http://scikit-learn.org/stable/modules/grid_search.html">grid search</a> utilities from scikit-learn;
447 use them to select model parameters and check the quality.</p>
448 <p>One limitation of <a class="reference internal" href="ref/crfsuite.html#webstruct.crfsuite.create_crfsuite_pipeline" title="webstruct.crfsuite.create_crfsuite_pipeline"><code class="xref py py-func docutils literal notranslate"><span class="pre">create_crfsuite_pipeline()</span></code></a> is that <code class="docutils literal notranslate"><span class="pre">n_jobs</span></code>
449 in scikit-learn functions and classes should be 1, but other than that
450 WebStruct objects should work fine with scikit-learn. Just keep in mind
451 that for WebStruct an “observation” is a document, not an individual token,
452 and a “label” is a sequence of labels for a document, not an individual
453 IOB tag.</p>
454 </li>
455 <li><p class="first">There is <a class="reference internal" href="ref/metrics.html#module-webstruct.metrics" title="webstruct.metrics"><code class="xref py py-mod docutils literal notranslate"><span class="pre">webstruct.metrics</span></code></a> module with a couple of metrics useful
456 for sequence classification.</p>
457 </li>
458 </ol>
459 <p>To debug what CRFSuite learned you could use <a class="reference external" href="https://github.com/TeamHG-Memex/eli5">eli5</a> library. With eli5 it would be two calls to
460 <code class="xref py py-func docutils literal notranslate"><span class="pre">eli5.explain_weights()</span></code> and <code class="xref py py-func docutils literal notranslate"><span class="pre">eli5.format_as_html()</span></code> with <code class="xref py py-class docutils literal notranslate"><span class="pre">sklearn_crfsuite.CRF</span></code> instance as argument.
461 As a result you will get transitions and feature weights.</p>
462 </div>
463 </div>
464
465
466 </div>
467
468 </div>
469 <footer>
470
471 <div class="rst-footer-buttons" role="navigation" aria-label="footer navigation">
472
473 <a href="ref/index.html" class="btn btn-neutral float-right" title="Reference" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right"></span></a>
474
475
476 <a href="intro.html" class="btn btn-neutral" title="Webstruct" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left"></span> Previous</a>
477
478 </div>
479
480
481 <hr/>
482
483 <div role="contentinfo">
484 <p>
485 &copy; Copyright 2014-2017, Scrapinghub Inc..
486
487 <span class="commit">
488 Revision <code>9e461566</code>.
489 </span>
490
491
492 </p>
493 </div>
494 Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
495
496 </footer>
497
498 </div>
499 </div>
500
501 </section>
502
503 </div>
504
505
506 <div class="rst-versions" data-toggle="rst-versions" role="note" aria-label="versions">
507 <span class="rst-current-version" data-toggle="rst-current-version">
508 <span class="fa fa-book"> Read the Docs</span>
509 v: latest
510 <span class="fa fa-caret-down"></span>
511 </span>
512 <div class="rst-other-versions">
513 <dl>
514 <dt>Versions</dt>
515
516 <dd><a href="/en/latest/">latest</a></dd>
517
518 <dd><a href="/en/stable/">stable</a></dd>
519
520 <dd><a href="/en/0.6/">0.6</a></dd>
521
522 <dd><a href="/en/0.5/">0.5</a></dd>
523
524 <dd><a href="/en/0.4.1/">0.4.1</a></dd>
525
526 <dd><a href="/en/0.4/">0.4</a></dd>
527
528 <dd><a href="/en/0.3/">0.3</a></dd>
529
530 <dd><a href="/en/0.2/">0.2</a></dd>
531
532 </dl>
533 <dl>
534 <dt>Downloads</dt>
535
536 <dd><a href="//readthedocs.org/projects/webstruct/downloads/pdf/latest/">pdf</a></dd>
537
538 <dd><a href="//readthedocs.org/projects/webstruct/downloads/htmlzip/latest/">htmlzip</a></dd>
539
540 <dd><a href="//readthedocs.org/projects/webstruct/downloads/epub/latest/">epub</a></dd>
541
542 </dl>
543 <dl>
544 <dt>On Read the Docs</dt>
545 <dd>
546 <a href="//readthedocs.org/projects/webstruct/?fromdocs=webstruct">Project Home</a>
547 </dd>
548 <dd>
549 <a href="//readthedocs.org/builds/webstruct/?fromdocs=webstruct">Builds</a>
550 </dd>
551 </dl>
552 <hr/>
553 Free document hosting provided by <a href="http://www.readthedocs.org">Read the Docs</a>.
554
555 </div>
556 </div>
557
558
559
560
561
562 <script type="text/javascript">
563 var DOCUMENTATION_OPTIONS = {
564 URL_ROOT:'./',
565 VERSION:'0.6',
566 LANGUAGE:'en',
567 COLLAPSE_INDEX:false,
568 FILE_SUFFIX:'.html',
569 HAS_SOURCE: true,
570 SOURCELINK_SUFFIX: '.txt'
571 };
572 </script>
573 <script type="text/javascript" src="https://media.readthedocs.org/javascript/jquery/jquery-2.0.3.min.js"></script>
574 <script type="text/javascript" src="https://media.readthedocs.org/javascript/jquery/jquery-migrate-1.2.1.min.js"></script>
575 <script type="text/javascript" src="https://media.readthedocs.org/javascript/underscore.js"></script>
576 <script type="text/javascript" src="https://media.readthedocs.org/javascript/doctools.js"></script>
577 <script type="text/javascript" src="https://media.readthedocs.org/javascript/readthedocs-doc-embed.js"></script>
578
579
580
581 <script type="text/javascript" src="_static/js/theme.js"></script>
582
583 <script type="text/javascript">
584 jQuery(function () {
585 SphinxRtdTheme.Navigation.enable(true);
586 });
587 </script>
588
589 </body>
590 </html>