| 1 |
|
| 2 |
|
| 3 |
<!DOCTYPE html> |
| 4 |
<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]--> |
| 5 |
<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]--> |
| 6 |
<head> |
| 7 |
<meta charset="utf-8"> |
| 8 |
|
| 9 |
<meta name="viewport" content="width=device-width, initial-scale=1.0"> |
| 10 |
|
| 11 |
<title>Tutorial — Webstruct 0.6 documentation</title> |
| 12 |
|
| 13 |
|
| 14 |
|
| 15 |
|
| 16 |
|
| 17 |
|
| 18 |
|
| 19 |
|
| 20 |
|
| 21 |
|
| 22 |
|
| 23 |
|
| 24 |
|
| 25 |
|
| 26 |
|
| 27 |
<link rel="stylesheet" href="_static/css/theme.css" type="text/css" /> |
| 28 |
<link rel="stylesheet" href="_static/pygments.css" type="text/css" /> |
| 29 |
<link rel="index" title="Index" href="genindex.html" /> |
| 30 |
<link rel="search" title="Search" href="search.html" /> |
| 31 |
<link rel="next" title="Reference" href="ref/index.html" /> |
| 32 |
<link rel="prev" title="Webstruct" href="intro.html" /> |
| 33 |
|
| 34 |
|
| 35 |
<script src="_static/js/modernizr.min.js"></script> |
| 36 |
|
| 37 |
|
| 38 |
<!-- RTD Extra Head --> |
| 39 |
|
| 40 |
<!-- |
| 41 |
Always link to the latest version, as canonical. |
| 42 |
http://docs.readthedocs.org/en/latest/canonical.html |
| 43 |
--> |
| 44 |
<link rel="canonical" href="https://webstruct.readthedocs.io/en/latest/tutorial.html" /> |
| 45 |
|
| 46 |
<link rel="stylesheet" href="https://media.readthedocs.org/css/readthedocs-doc-embed.css" type="text/css" /> |
| 47 |
|
| 48 |
<script type="text/javascript" src="_static/readthedocs-data.js"></script> |
| 49 |
|
| 50 |
<!-- Add page-specific data, which must exist in the page js, not global --> |
| 51 |
<script type="text/javascript"> |
| 52 |
READTHEDOCS_DATA['page'] = 'tutorial' |
| 53 |
READTHEDOCS_DATA['source_suffix'] = '.rst' |
| 54 |
</script> |
| 55 |
|
| 56 |
<script type="text/javascript" src="https://media.readthedocs.org/javascript/readthedocs-analytics.js"></script> |
| 57 |
|
| 58 |
<!-- end RTD <extrahead> --> |
| 59 |
</head> |
| 60 |
|
| 61 |
<body class="wy-body-for-nav"> |
| 62 |
|
| 63 |
|
| 64 |
<div class="wy-grid-for-nav"> |
| 65 |
|
| 66 |
|
| 67 |
<nav data-toggle="wy-nav-shift" class="wy-nav-side"> |
| 68 |
<div class="wy-side-scroll"> |
| 69 |
<div class="wy-side-nav-search"> |
| 70 |
|
| 71 |
|
| 72 |
|
| 73 |
<a href="index.html" class="icon icon-home"> Webstruct |
| 74 |
|
| 75 |
|
| 76 |
|
| 77 |
</a> |
| 78 |
|
| 79 |
|
| 80 |
|
| 81 |
|
| 82 |
|
| 83 |
<div class="version"> |
| 84 |
latest |
| 85 |
</div> |
| 86 |
|
| 87 |
|
| 88 |
|
| 89 |
|
| 90 |
<div role="search"> |
| 91 |
<form id="rtd-search-form" class="wy-form" action="search.html" method="get"> |
| 92 |
<input type="text" name="q" placeholder="Search docs" /> |
| 93 |
<input type="hidden" name="check_keywords" value="yes" /> |
| 94 |
<input type="hidden" name="area" value="default" /> |
| 95 |
</form> |
| 96 |
</div> |
| 97 |
|
| 98 |
|
| 99 |
</div> |
| 100 |
|
| 101 |
<div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation"> |
| 102 |
|
| 103 |
|
| 104 |
|
| 105 |
|
| 106 |
|
| 107 |
|
| 108 |
<ul class="current"> |
| 109 |
<li class="toctree-l1"><a class="reference internal" href="intro.html">Webstruct</a></li> |
| 110 |
<li class="toctree-l1 current"><a class="current reference internal" href="#">Tutorial</a><ul> |
| 111 |
<li class="toctree-l2"><a class="reference internal" href="#get-annotated-data">Get annotated data</a></li> |
| 112 |
<li class="toctree-l2"><a class="reference internal" href="#from-html-to-tokens">From HTML to Tokens</a></li> |
| 113 |
<li class="toctree-l2"><a class="reference internal" href="#feature-extraction">Feature Extraction</a></li> |
| 114 |
<li class="toctree-l2"><a class="reference internal" href="#using-a-sequence-labelling-toolkit">Using a Sequence Labelling Toolkit</a><ul> |
| 115 |
<li class="toctree-l3"><a class="reference internal" href="#defining-a-model">Defining a Model</a></li> |
| 116 |
<li class="toctree-l3"><a class="reference internal" href="#training">Training</a></li> |
| 117 |
</ul> |
| 118 |
</li> |
| 119 |
<li class="toctree-l2"><a class="reference internal" href="#named-entity-recognition">Named Entity Recognition</a></li> |
| 120 |
<li class="toctree-l2"><a class="reference internal" href="#entity-grouping">Entity Grouping</a></li> |
| 121 |
<li class="toctree-l2"><a class="reference internal" href="#model-development">Model Development</a></li> |
| 122 |
</ul> |
| 123 |
</li> |
| 124 |
<li class="toctree-l1"><a class="reference internal" href="ref/index.html">Reference</a></li> |
| 125 |
<li class="toctree-l1"><a class="reference internal" href="changes.html">Changes</a></li> |
| 126 |
</ul> |
| 127 |
|
| 128 |
|
| 129 |
|
| 130 |
</div> |
| 131 |
</div> |
| 132 |
</nav> |
| 133 |
|
| 134 |
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap"> |
| 135 |
|
| 136 |
|
| 137 |
<nav class="wy-nav-top" aria-label="top navigation"> |
| 138 |
|
| 139 |
<i data-toggle="wy-nav-top" class="fa fa-bars"></i> |
| 140 |
<a href="index.html">Webstruct</a> |
| 141 |
|
| 142 |
</nav> |
| 143 |
|
| 144 |
|
| 145 |
<div class="wy-nav-content"> |
| 146 |
|
| 147 |
<div class="rst-content"> |
| 148 |
|
| 149 |
|
| 150 |
|
| 151 |
|
| 152 |
|
| 153 |
|
| 154 |
|
| 155 |
|
| 156 |
|
| 157 |
|
| 158 |
|
| 159 |
|
| 160 |
|
| 161 |
|
| 162 |
|
| 163 |
|
| 164 |
|
| 165 |
<div role="navigation" aria-label="breadcrumbs navigation"> |
| 166 |
|
| 167 |
<ul class="wy-breadcrumbs"> |
| 168 |
|
| 169 |
<li><a href="index.html">Docs</a> »</li> |
| 170 |
|
| 171 |
<li>Tutorial</li> |
| 172 |
|
| 173 |
|
| 174 |
<li class="wy-breadcrumbs-aside"> |
| 175 |
|
| 176 |
|
| 177 |
|
| 178 |
<a href="https://github.com/scrapinghub/webstruct/blob/master/docs/tutorial.rst" class="fa fa-github"> Edit on GitHub</a> |
| 179 |
|
| 180 |
|
| 181 |
|
| 182 |
</li> |
| 183 |
|
| 184 |
</ul> |
| 185 |
|
| 186 |
|
| 187 |
<hr/> |
| 188 |
</div> |
| 189 |
<div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article"> |
| 190 |
<div itemprop="articleBody"> |
| 191 |
|
| 192 |
<div class="section" id="tutorial"> |
| 193 |
<h1>Tutorial<a class="headerlink" href="#tutorial" title="Permalink to this headline">¶</a></h1> |
| 194 |
<p>This tutorial assumes you are familiar with machine learning.</p> |
| 195 |
<div class="section" id="get-annotated-data"> |
| 196 |
<h2>Get annotated data<a class="headerlink" href="#get-annotated-data" title="Permalink to this headline">¶</a></h2> |
| 197 |
<p>First, you need the training/development data. We suggest to use |
| 198 |
<a class="reference external" href="https://github.com/xtannier/WebAnnotator">WebAnnotator</a> Firefox extension to annotate HTML pages.</p> |
| 199 |
<p>Recommended WebAnnotator options:</p> |
| 200 |
<img alt="_images/wa-options.png" src="_images/wa-options.png" /> |
| 201 |
<p>Pro tip - enable WebAnnotator toolbar buttons:</p> |
| 202 |
<img alt="_images/wa-buttons.png" src="_images/wa-buttons.png" /> |
| 203 |
<p>Follow WebAnnotator <a class="reference external" href="http://perso.limsi.fr/xtannier/en/WebAnnotator/">manual</a> |
| 204 |
to define named entities and annotate some web pages (nested WebAnnotator |
| 205 |
entities are not supported). Use “Save as..” menu item or “Save as” |
| 206 |
toolbar button to save the results; don’t use “Export as”.</p> |
| 207 |
<p>After that you can load annotated webpages as lxml trees:</p> |
| 208 |
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">webstruct</span> |
| 209 |
<span class="n">trees</span> <span class="o">=</span> <span class="n">webstruct</span><span class="o">.</span><span class="n">load_trees</span><span class="p">(</span><span class="s2">"train/*.html"</span><span class="p">,</span> <span class="n">webstruct</span><span class="o">.</span><span class="n">WebAnnotatorLoader</span><span class="p">())</span> |
| 210 |
</pre></div> |
| 211 |
</div> |
| 212 |
<p>See <a class="reference internal" href="ref/loaders.html#html-loaders"><span class="std std-ref">HTML Loaders</span></a> for more info. |
| 213 |
<a class="reference external" href="http://gate.ac.uk/">GATE</a> annotation format is also supported.</p> |
| 214 |
</div> |
| 215 |
<div class="section" id="from-html-to-tokens"> |
| 216 |
<h2>From HTML to Tokens<a class="headerlink" href="#from-html-to-tokens" title="Permalink to this headline">¶</a></h2> |
| 217 |
<p>To convert HTML trees to a format suitable for sequence prediction algorithm |
| 218 |
(like CRF, MEMM or Structured Perceptron) the following approach is used:</p> |
| 219 |
<ol class="arabic simple"> |
| 220 |
<li>Text is extracted from HTML and split into tokens.</li> |
| 221 |
<li>For each token a special <a class="reference internal" href="ref/features.html#webstruct.html_tokenizer.HtmlToken" title="webstruct.html_tokenizer.HtmlToken"><code class="xref py py-class docutils literal notranslate"><span class="pre">HtmlToken</span></code></a> instance is created. It |
| 222 |
contains information not only about the text token itself, but also about |
| 223 |
its position in HTML tree.</li> |
| 224 |
</ol> |
| 225 |
<p>A single HTML page corresponds to a single input sequence |
| 226 |
(a list of HtmlTokens). For training/testing data |
| 227 |
(where webpages are already annotated) there is also a list of labels for |
| 228 |
each webpage, a label per HtmlToken.</p> |
| 229 |
<p>To transform HTML trees into labels and HTML tokens |
| 230 |
use <a class="reference internal" href="ref/features.html#webstruct.html_tokenizer.HtmlTokenizer" title="webstruct.html_tokenizer.HtmlTokenizer"><code class="xref py py-class docutils literal notranslate"><span class="pre">HtmlTokenizer</span></code></a>.</p> |
| 231 |
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">html_tokenizer</span> <span class="o">=</span> <span class="n">webstruct</span><span class="o">.</span><span class="n">HtmlTokenizer</span><span class="p">()</span> |
| 232 |
<span class="n">X</span><span class="p">,</span> <span class="n">y</span> <span class="o">=</span> <span class="n">html_tokenizer</span><span class="o">.</span><span class="n">tokenize</span><span class="p">(</span><span class="n">trees</span><span class="p">)</span> |
| 233 |
</pre></div> |
| 234 |
</div> |
| 235 |
<p>Input trees should be loaded by one of the WebStruct loaders. |
| 236 |
For consistency, for each tree (even if it is loaded from raw unannotated html) |
| 237 |
<a class="reference internal" href="ref/features.html#webstruct.html_tokenizer.HtmlTokenizer" title="webstruct.html_tokenizer.HtmlTokenizer"><code class="xref py py-class docutils literal notranslate"><span class="pre">HtmlTokenizer</span></code></a> extracts two arrays: a list of <a class="reference internal" href="ref/features.html#webstruct.html_tokenizer.HtmlToken" title="webstruct.html_tokenizer.HtmlToken"><code class="xref py py-class docutils literal notranslate"><span class="pre">HtmlToken</span></code></a> |
| 238 |
instances and a list of tags encoded using <a class="reference external" href="http://en.wikipedia.org/wiki/Inside_Outside_Beginning">IOB2</a> encoding |
| 239 |
(also known as BIO encoding). So in our example <code class="docutils literal notranslate"><span class="pre">X</span></code> is a list of |
| 240 |
lists of <a class="reference internal" href="ref/features.html#webstruct.html_tokenizer.HtmlToken" title="webstruct.html_tokenizer.HtmlToken"><code class="xref py py-class docutils literal notranslate"><span class="pre">HtmlToken</span></code></a> instances, and <code class="docutils literal notranslate"><span class="pre">y</span></code> is a list of lists |
| 241 |
of strings.</p> |
| 242 |
</div> |
| 243 |
<div class="section" id="feature-extraction"> |
| 244 |
<h2>Feature Extraction<a class="headerlink" href="#feature-extraction" title="Permalink to this headline">¶</a></h2> |
| 245 |
<p>For supervised machine learning algorithms to work we need to extract |
| 246 |
<a class="reference external" href="http://en.wikipedia.org/wiki/Features_%28pattern_recognition%29">features</a>.</p> |
| 247 |
<p>In WebStruct feature vectors are Python dicts |
| 248 |
<code class="docutils literal notranslate"><span class="pre">{"feature_name":</span> <span class="pre">"feature_value"}</span></code>; a dict is computed for |
| 249 |
each HTML token. How to convert these dicts into representation required |
| 250 |
by a sequence labelling toolkit depends on a toolkit used; we will cover |
| 251 |
that later.</p> |
| 252 |
<p>To compute feature dicts we’ll use <a class="reference internal" href="ref/features.html#webstruct.feature_extraction.HtmlFeatureExtractor" title="webstruct.feature_extraction.HtmlFeatureExtractor"><code class="xref py py-class docutils literal notranslate"><span class="pre">HtmlFeatureExtractor</span></code></a>.</p> |
| 253 |
<p>First, define your feature functions. A feature function should take |
| 254 |
an <a class="reference internal" href="ref/features.html#webstruct.html_tokenizer.HtmlToken" title="webstruct.html_tokenizer.HtmlToken"><code class="xref py py-class docutils literal notranslate"><span class="pre">HtmlToken</span></code></a> instance and return a feature dict; |
| 255 |
feature dicts from individual feature functions will be merged |
| 256 |
into the final feature dict for a token. Feature functions can ask questions |
| 257 |
about token itself, its neighbours (in the same HTML element), |
| 258 |
its position in HTML.</p> |
| 259 |
<div class="admonition note"> |
| 260 |
<p class="first admonition-title">Note</p> |
| 261 |
<p class="last">WebStruct supports other kind of feature functions that work on multiple |
| 262 |
tokens; we don’t cover them in this tutorial.</p> |
| 263 |
</div> |
| 264 |
<p>There are predefined feature functions in <a class="reference internal" href="ref/features.html#module-webstruct.features" title="webstruct.features"><code class="xref py py-mod docutils literal notranslate"><span class="pre">webstruct.features</span></code></a>, |
| 265 |
but for this tutorial let’s create some functions ourselves:</p> |
| 266 |
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">token_identity</span><span class="p">(</span><span class="n">html_token</span><span class="p">):</span> |
| 267 |
<span class="k">return</span> <span class="p">{</span><span class="s1">'token'</span><span class="p">:</span> <span class="n">html_token</span><span class="o">.</span><span class="n">token</span><span class="p">}</span> |
| 268 |
|
| 269 |
<span class="k">def</span> <span class="nf">token_isupper</span><span class="p">(</span><span class="n">html_token</span><span class="p">):</span> |
| 270 |
<span class="k">return</span> <span class="p">{</span><span class="s1">'isupper'</span><span class="p">:</span> <span class="n">html_token</span><span class="o">.</span><span class="n">token</span><span class="o">.</span><span class="n">isupper</span><span class="p">()}</span> |
| 271 |
|
| 272 |
<span class="k">def</span> <span class="nf">parent_tag</span><span class="p">(</span><span class="n">html_token</span><span class="p">):</span> |
| 273 |
<span class="k">return</span> <span class="p">{</span><span class="s1">'parent_tag'</span><span class="p">:</span> <span class="n">html_token</span><span class="o">.</span><span class="n">parent</span><span class="o">.</span><span class="n">tag</span><span class="p">}</span> |
| 274 |
|
| 275 |
<span class="k">def</span> <span class="nf">border_at_left</span><span class="p">(</span><span class="n">html_token</span><span class="p">):</span> |
| 276 |
<span class="k">return</span> <span class="p">{</span><span class="s1">'border_at_left'</span><span class="p">:</span> <span class="n">html_token</span><span class="o">.</span><span class="n">index</span> <span class="o">==</span> <span class="mi">0</span><span class="p">}</span> |
| 277 |
</pre></div> |
| 278 |
</div> |
| 279 |
<p>Next, create <a class="reference internal" href="ref/features.html#webstruct.feature_extraction.HtmlFeatureExtractor" title="webstruct.feature_extraction.HtmlFeatureExtractor"><code class="xref py py-class docutils literal notranslate"><span class="pre">HtmlFeatureExtractor</span></code></a>:</p> |
| 280 |
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">feature_extractor</span> <span class="o">=</span> <span class="n">HtmlFeatureExtractor</span><span class="p">(</span> |
| 281 |
<span class="n">token_features</span> <span class="o">=</span> <span class="p">[</span> |
| 282 |
<span class="n">token_identity</span><span class="p">,</span> |
| 283 |
<span class="n">token_isupper</span><span class="p">,</span> |
| 284 |
<span class="n">parent_tag</span><span class="p">,</span> |
| 285 |
<span class="n">border_at_left</span> |
| 286 |
<span class="p">]</span> |
| 287 |
<span class="p">)</span> |
| 288 |
</pre></div> |
| 289 |
</div> |
| 290 |
<p>and use it to extract feature dicts:</p> |
| 291 |
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">features</span> <span class="o">=</span> <span class="n">feature_extractor</span><span class="o">.</span><span class="n">fit_transform</span><span class="p">(</span><span class="n">X</span><span class="p">)</span> |
| 292 |
</pre></div> |
| 293 |
</div> |
| 294 |
<p>See <a class="reference internal" href="ref/features.html#feature-extraction"><span class="std std-ref">Feature Extraction</span></a> for more info about HTML tokenization and |
| 295 |
feature extraction.</p> |
| 296 |
</div> |
| 297 |
<div class="section" id="using-a-sequence-labelling-toolkit"> |
| 298 |
<h2>Using a Sequence Labelling Toolkit<a class="headerlink" href="#using-a-sequence-labelling-toolkit" title="Permalink to this headline">¶</a></h2> |
| 299 |
<p>WebStruct doesn’t provide a CRF or Structured Perceptron implementation; |
| 300 |
learning and prediction is supposed to be handled by an external |
| 301 |
sequence labelling toolkit like <a class="reference external" href="http://www.chokkan.org/software/crfsuite/">CRFSuite</a>, <a class="reference external" href="http://wapiti.limsi.fr">Wapiti</a> or <a class="reference external" href="https://github.com/larsmans/seqlearn">seqlearn</a>.</p> |
| 302 |
<p>Once feature dicts are extracted from HTML you should convert them to |
| 303 |
a format required by your sequence labelling tooklit and use this toolkit |
| 304 |
to train a model and do the prediction. For example, you may use |
| 305 |
DictVectorizer from scikit-learn to convert feature dicts |
| 306 |
into <a class="reference external" href="https://github.com/larsmans/seqlearn">seqlearn</a> input format.</p> |
| 307 |
<p>We’ll use <a class="reference external" href="http://www.chokkan.org/software/crfsuite/">CRFSuite</a> in this tutorial.</p> |
| 308 |
<p>WebStruct provides some helpers for CRFSuite sequence labelling toolkit. |
| 309 |
To use CRFSuite with WebStruct, you need</p> |
| 310 |
<ul class="simple"> |
| 311 |
<li>sklearn-crfsuite package (which depends on python-crfsuite and sklearn)</li> |
| 312 |
</ul> |
| 313 |
<div class="section" id="defining-a-model"> |
| 314 |
<h3>Defining a Model<a class="headerlink" href="#defining-a-model" title="Permalink to this headline">¶</a></h3> |
| 315 |
<p>Basic way to define CRF model is the following:</p> |
| 316 |
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">model</span> <span class="o">=</span> <span class="n">webstruct</span><span class="o">.</span><span class="n">create_crfsuite_pipeline</span><span class="p">(</span> |
| 317 |
<span class="n">token_features</span><span class="o">=</span><span class="p">[</span><span class="n">token_identity</span><span class="p">,</span> <span class="n">token_isupper</span><span class="p">,</span> <span class="n">parent_tag</span><span class="p">,</span> <span class="n">border_at_left</span><span class="p">],</span> |
| 318 |
<span class="n">verbose</span><span class="o">=</span><span class="kc">True</span> |
| 319 |
<span class="p">)</span> |
| 320 |
</pre></div> |
| 321 |
</div> |
| 322 |
<p>First <a class="reference internal" href="ref/crfsuite.html#webstruct.crfsuite.create_crfsuite_pipeline" title="webstruct.crfsuite.create_crfsuite_pipeline"><code class="xref py py-func docutils literal notranslate"><span class="pre">create_crfsuite_pipeline()</span></code></a> argument is a list of feature functions which will be used for training. |
| 323 |
<code class="docutils literal notranslate"><span class="pre">verbose</span></code> is a boolean parameter enabling verbose output of various training information; |
| 324 |
check sklearn-crfsuite <a class="reference external" href="https://sklearn-crfsuite.readthedocs.io/en/latest/api.html#sklearn_crfsuite.CRF">API reference</a> |
| 325 |
for available options.</p> |
| 326 |
<p>Under the hood <a class="reference internal" href="ref/crfsuite.html#webstruct.crfsuite.create_crfsuite_pipeline" title="webstruct.crfsuite.create_crfsuite_pipeline"><code class="xref py py-func docutils literal notranslate"><span class="pre">create_crfsuite_pipeline()</span></code></a> creates a |
| 327 |
<code class="docutils literal notranslate"><span class="pre">sklearn.pipeline.Pipeline</span></code> with an <a class="reference internal" href="ref/features.html#webstruct.feature_extraction.HtmlFeatureExtractor" title="webstruct.feature_extraction.HtmlFeatureExtractor"><code class="xref py py-class docutils literal notranslate"><span class="pre">HtmlFeatureExtractor</span></code></a> instance |
| 328 |
followed by <code class="xref py py-class docutils literal notranslate"><span class="pre">sklearn_crfsuite.CRF</span></code> instance. The example above is just a shortcut |
| 329 |
for the following:</p> |
| 330 |
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">model</span> <span class="o">=</span> <span class="n">Pipeline</span><span class="p">([</span> |
| 331 |
<span class="p">(</span><span class="s1">'fe'</span><span class="p">,</span> <span class="n">HtmlFeatureExtractor</span><span class="p">(</span> |
| 332 |
<span class="n">token_features</span> <span class="o">=</span> <span class="p">[</span> |
| 333 |
<span class="n">token_identity</span><span class="p">,</span> |
| 334 |
<span class="n">token_isupper</span><span class="p">,</span> |
| 335 |
<span class="n">parent_tag</span><span class="p">,</span> |
| 336 |
<span class="n">border_at_left</span><span class="p">,</span> |
| 337 |
<span class="p">]</span> |
| 338 |
<span class="p">)),</span> |
| 339 |
<span class="p">(</span><span class="s1">'crf'</span><span class="p">,</span> <span class="n">sklearn_crfsuite</span><span class="o">.</span><span class="n">CRF</span><span class="p">(</span> |
| 340 |
<span class="n">verbose</span><span class="o">=</span><span class="kc">True</span> |
| 341 |
<span class="p">)),</span> |
| 342 |
<span class="p">])</span> |
| 343 |
</pre></div> |
| 344 |
</div> |
| 345 |
</div> |
| 346 |
<div class="section" id="training"> |
| 347 |
<h3>Training<a class="headerlink" href="#training" title="Permalink to this headline">¶</a></h3> |
| 348 |
<p>To train a model use its <code class="docutils literal notranslate"><span class="pre">fit</span></code> method:</p> |
| 349 |
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">model</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">X</span><span class="p">,</span> <span class="n">y</span><span class="p">)</span> |
| 350 |
</pre></div> |
| 351 |
</div> |
| 352 |
<p><code class="docutils literal notranslate"><span class="pre">X</span></code> and <code class="docutils literal notranslate"><span class="pre">y</span></code> are return values of <a class="reference internal" href="ref/features.html#webstruct.html_tokenizer.HtmlTokenizer.tokenize" title="webstruct.html_tokenizer.HtmlTokenizer.tokenize"><code class="xref py py-meth docutils literal notranslate"><span class="pre">HtmlTokenizer.tokenize()</span></code></a> |
| 353 |
(a list of lists of <a class="reference internal" href="ref/features.html#webstruct.html_tokenizer.HtmlToken" title="webstruct.html_tokenizer.HtmlToken"><code class="xref py py-class docutils literal notranslate"><span class="pre">HtmlToken</span></code></a> instances and a list of |
| 354 |
lists of string IOB labels).</p> |
| 355 |
<p>If you use <code class="xref py py-class docutils literal notranslate"><span class="pre">sklearn_crfsuite.CRF</span></code> directly then train it using |
| 356 |
<code class="xref py py-meth docutils literal notranslate"><span class="pre">CRF.fit()</span></code> method. It accepts 2 lists: a list of lists of |
| 357 |
feature dicts, and a list of lists of tags:</p> |
| 358 |
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">model</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">features</span><span class="p">,</span> <span class="n">y</span><span class="p">)</span> |
| 359 |
</pre></div> |
| 360 |
</div> |
| 361 |
</div> |
| 362 |
</div> |
| 363 |
<div class="section" id="named-entity-recognition"> |
| 364 |
<h2>Named Entity Recognition<a class="headerlink" href="#named-entity-recognition" title="Permalink to this headline">¶</a></h2> |
| 365 |
<p>Once you got a trained model you can use it to extract entities |
| 366 |
from unseen (unannotated) webpages. First, get some binary HTML data:</p> |
| 367 |
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="kn">import</span> <span class="nn">urllib2</span> |
| 368 |
<span class="gp">>>> </span><span class="n">html</span> <span class="o">=</span> <span class="n">urllib2</span><span class="o">.</span><span class="n">urlopen</span><span class="p">(</span><span class="s2">"http://scrapinghub.com/contact"</span><span class="p">)</span><span class="o">.</span><span class="n">read</span><span class="p">()</span> |
| 369 |
</pre></div> |
| 370 |
</div> |
| 371 |
<p>Then create a <a class="reference internal" href="ref/model.html#webstruct.model.NER" title="webstruct.model.NER"><code class="xref py py-class docutils literal notranslate"><span class="pre">NER</span></code></a> instance initialized with a trained model:</p> |
| 372 |
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ner</span> <span class="o">=</span> <span class="n">webstruct</span><span class="o">.</span><span class="n">NER</span><span class="p">(</span><span class="n">model</span><span class="p">)</span> |
| 373 |
</pre></div> |
| 374 |
</div> |
| 375 |
<p>The <code class="docutils literal notranslate"><span class="pre">model</span></code> must provide a <code class="docutils literal notranslate"><span class="pre">predict</span></code> method that extracts features |
| 376 |
from HTML tokens and predicts labels for these tokens. A pipeline created with |
| 377 |
<a class="reference internal" href="ref/crfsuite.html#webstruct.crfsuite.create_crfsuite_pipeline" title="webstruct.crfsuite.create_crfsuite_pipeline"><code class="xref py py-func docutils literal notranslate"><span class="pre">create_crfsuite_pipeline()</span></code></a> function fits this definition.</p> |
| 378 |
<p>Finally, use <a class="reference internal" href="ref/model.html#webstruct.model.NER.extract" title="webstruct.model.NER.extract"><code class="xref py py-meth docutils literal notranslate"><span class="pre">NER.extract()</span></code></a> method to extract entities:</p> |
| 379 |
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ner</span><span class="o">.</span><span class="n">extract</span><span class="p">(</span><span class="n">html</span><span class="p">)</span> |
| 380 |
<span class="go">[('Scrapinghub', 'ORG'), ..., ('Iturriaga 3429 ap. 1', 'STREET'), ...]</span> |
| 381 |
</pre></div> |
| 382 |
</div> |
| 383 |
<p>Generally, the steps are:</p> |
| 384 |
<ol class="arabic simple"> |
| 385 |
<li>Load data using <a class="reference internal" href="ref/loaders.html#webstruct.loaders.HtmlLoader" title="webstruct.loaders.HtmlLoader"><code class="xref py py-class docutils literal notranslate"><span class="pre">HtmlLoader</span></code></a> loader. If a custom HTML cleaner |
| 386 |
was used for loading training data make sure to apply it here as well.</li> |
| 387 |
<li>Use the same <code class="docutils literal notranslate"><span class="pre">html_tokenizer</span></code> as used for training to extract HTML tokens |
| 388 |
from loaded trees. All labels would be “O” when using <a class="reference internal" href="ref/loaders.html#webstruct.loaders.HtmlLoader" title="webstruct.loaders.HtmlLoader"><code class="xref py py-class docutils literal notranslate"><span class="pre">HtmlLoader</span></code></a> |
| 389 |
loader - <code class="docutils literal notranslate"><span class="pre">y</span></code> can be discarded.</li> |
| 390 |
<li>Use the same <code class="docutils literal notranslate"><span class="pre">feature_extractor</span></code> as used for training to extract |
| 391 |
features.</li> |
| 392 |
<li>Run <code class="docutils literal notranslate"><span class="pre">your_crf.predict()</span></code> method (e.g. <code class="xref py py-meth docutils literal notranslate"><span class="pre">CRF.predict()</span></code>) |
| 393 |
on features extracted in (3) to get the prediction - a list of IOB2-encoded |
| 394 |
tags for each input document.</li> |
| 395 |
<li>Build entities from input tokens based on predicted tags |
| 396 |
(check <a class="reference internal" href="ref/misc.html#webstruct.sequence_encoding.IobEncoder.group" title="webstruct.sequence_encoding.IobEncoder.group"><code class="xref py py-meth docutils literal notranslate"><span class="pre">IobEncoder.group()</span></code></a> and <a class="reference internal" href="ref/misc.html#webstruct.utils.smart_join" title="webstruct.utils.smart_join"><code class="xref py py-func docutils literal notranslate"><span class="pre">smart_join()</span></code></a>).</li> |
| 397 |
<li>Split entities into groups (optional). One way to do it is to use |
| 398 |
<a class="reference internal" href="ref/grouping.html#module-webstruct.grouping" title="webstruct.grouping"><code class="xref py py-mod docutils literal notranslate"><span class="pre">webstruct.grouping</span></code></a>.</li> |
| 399 |
</ol> |
| 400 |
<p><a class="reference internal" href="ref/model.html#webstruct.model.NER" title="webstruct.model.NER"><code class="xref py py-class docutils literal notranslate"><span class="pre">NER</span></code></a> helper class combines HTML loading, HTML tokenization, |
| 401 |
feature extraction, CRF model, entity building and grouping.</p> |
| 402 |
</div> |
| 403 |
<div class="section" id="entity-grouping"> |
| 404 |
<h2>Entity Grouping<a class="headerlink" href="#entity-grouping" title="Permalink to this headline">¶</a></h2> |
| 405 |
<p>Detecting entities on their own is not always enough; in many cases |
| 406 |
what is wanted is to find the relationship between them. For example, |
| 407 |
“<strong>street_name/STREET city_name/CITY zipcode_number/ZIPCODE</strong> |
| 408 |
form an address”, or “<strong>phone/TEL</strong> is a phone of <strong>person/PER</strong>”.</p> |
| 409 |
<p>The first approximation is to say that all entities from a single webpage |
| 410 |
are related. For example, if we have extracted some <strong>organizaion/ORG</strong> and some |
| 411 |
<strong>phone/TEL</strong> from a single webpage we may assume that the phone |
| 412 |
is a contact phone of the organization.</p> |
| 413 |
<p>Sometimes there are several “entity groups” on a webpage. If a page |
| 414 |
contains contact phones of several persons or several business locations |
| 415 |
it is better to split all entities into groups of related |
| 416 |
entities - “person name + his/her phone(s)” or “address”.</p> |
| 417 |
<p>WebStruct provides an <a class="reference internal" href="ref/grouping.html#grouping-algorithm"><span class="std std-ref">unsupervised algorithm</span></a> |
| 418 |
for extracting such entity groups. Algorithm prefers to build |
| 419 |
large groups without entities of duplicate types; if a split is needed |
| 420 |
algorithm tries to split at points where distance between entities is larger.</p> |
| 421 |
<p>Use <a class="reference internal" href="ref/model.html#webstruct.model.NER.extract_groups" title="webstruct.model.NER.extract_groups"><code class="xref py py-meth docutils literal notranslate"><span class="pre">NER.extract_groups()</span></code></a> to extract groups of entities:</p> |
| 422 |
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">ner</span><span class="o">.</span><span class="n">extract_groups</span><span class="p">(</span><span class="n">html</span><span class="p">)</span> |
| 423 |
<span class="go">[[...], ... [('Iturriaga 3429 ap. 1', 'STREET'), ('Montevideo', 'CITY'), ...]]</span> |
| 424 |
</pre></div> |
| 425 |
</div> |
| 426 |
<p>Sometimes it is better to allow some entity types to appear |
| 427 |
multuple times in a group. For example, a person (PER entity) may have |
| 428 |
several contact phones and faxes (TEL and FAX entities) - we should penalize |
| 429 |
groups with multiple PERs, but multiple TELs and FAXes are fine. |
| 430 |
Use <code class="docutils literal notranslate"><span class="pre">dont_penalize</span></code> argument if you want to allow some entity types |
| 431 |
to appear multiple times in a group:</p> |
| 432 |
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">ner</span><span class="o">.</span><span class="n">extract_groups</span><span class="p">(</span><span class="n">html</span><span class="p">,</span> <span class="n">dont_penalize</span><span class="o">=</span><span class="p">{</span><span class="s1">'TEL'</span><span class="p">,</span> <span class="s1">'FAX'</span><span class="p">})</span> |
| 433 |
</pre></div> |
| 434 |
</div> |
| 435 |
<p>The simple algorithm WebStruct provides is by no means a general solution |
| 436 |
to relation detection, but give it a try - maybe it is enough for your task.</p> |
| 437 |
</div> |
| 438 |
<div class="section" id="model-development"> |
| 439 |
<h2>Model Development<a class="headerlink" href="#model-development" title="Permalink to this headline">¶</a></h2> |
| 440 |
<p>To develop the model you need to choose the learning algorithm, |
| 441 |
features, hyperparameters, etc. To do that you need scoring metrics, |
| 442 |
cross-validation utilities and tools for debugging what classifier learned. |
| 443 |
WebStruct helps in the following way:</p> |
| 444 |
<ol class="arabic"> |
| 445 |
<li><p class="first">Pipeline created by <a class="reference internal" href="ref/crfsuite.html#webstruct.crfsuite.create_crfsuite_pipeline" title="webstruct.crfsuite.create_crfsuite_pipeline"><code class="xref py py-func docutils literal notranslate"><span class="pre">create_crfsuite_pipeline()</span></code></a> is compatible with |
| 446 |
<a class="reference external" href="http://scikit-learn.org/stable/modules/cross_validation.html">cross-validation</a> and <a class="reference external" href="http://scikit-learn.org/stable/modules/grid_search.html">grid search</a> utilities from scikit-learn; |
| 447 |
use them to select model parameters and check the quality.</p> |
| 448 |
<p>One limitation of <a class="reference internal" href="ref/crfsuite.html#webstruct.crfsuite.create_crfsuite_pipeline" title="webstruct.crfsuite.create_crfsuite_pipeline"><code class="xref py py-func docutils literal notranslate"><span class="pre">create_crfsuite_pipeline()</span></code></a> is that <code class="docutils literal notranslate"><span class="pre">n_jobs</span></code> |
| 449 |
in scikit-learn functions and classes should be 1, but other than that |
| 450 |
WebStruct objects should work fine with scikit-learn. Just keep in mind |
| 451 |
that for WebStruct an “observation” is a document, not an individual token, |
| 452 |
and a “label” is a sequence of labels for a document, not an individual |
| 453 |
IOB tag.</p> |
| 454 |
</li> |
| 455 |
<li><p class="first">There is <a class="reference internal" href="ref/metrics.html#module-webstruct.metrics" title="webstruct.metrics"><code class="xref py py-mod docutils literal notranslate"><span class="pre">webstruct.metrics</span></code></a> module with a couple of metrics useful |
| 456 |
for sequence classification.</p> |
| 457 |
</li> |
| 458 |
</ol> |
| 459 |
<p>To debug what CRFSuite learned you could use <a class="reference external" href="https://github.com/TeamHG-Memex/eli5">eli5</a> library. With eli5 it would be two calls to |
| 460 |
<code class="xref py py-func docutils literal notranslate"><span class="pre">eli5.explain_weights()</span></code> and <code class="xref py py-func docutils literal notranslate"><span class="pre">eli5.format_as_html()</span></code> with <code class="xref py py-class docutils literal notranslate"><span class="pre">sklearn_crfsuite.CRF</span></code> instance as argument. |
| 461 |
As a result you will get transitions and feature weights.</p> |
| 462 |
</div> |
| 463 |
</div> |
| 464 |
|
| 465 |
|
| 466 |
</div> |
| 467 |
|
| 468 |
</div> |
| 469 |
<footer> |
| 470 |
|
| 471 |
<div class="rst-footer-buttons" role="navigation" aria-label="footer navigation"> |
| 472 |
|
| 473 |
<a href="ref/index.html" class="btn btn-neutral float-right" title="Reference" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right"></span></a> |
| 474 |
|
| 475 |
|
| 476 |
<a href="intro.html" class="btn btn-neutral" title="Webstruct" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left"></span> Previous</a> |
| 477 |
|
| 478 |
</div> |
| 479 |
|
| 480 |
|
| 481 |
<hr/> |
| 482 |
|
| 483 |
<div role="contentinfo"> |
| 484 |
<p> |
| 485 |
© Copyright 2014-2017, Scrapinghub Inc.. |
| 486 |
|
| 487 |
<span class="commit"> |
| 488 |
Revision <code>9e461566</code>. |
| 489 |
</span> |
| 490 |
|
| 491 |
|
| 492 |
</p> |
| 493 |
</div> |
| 494 |
Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. |
| 495 |
|
| 496 |
</footer> |
| 497 |
|
| 498 |
</div> |
| 499 |
</div> |
| 500 |
|
| 501 |
</section> |
| 502 |
|
| 503 |
</div> |
| 504 |
|
| 505 |
|
| 506 |
<div class="rst-versions" data-toggle="rst-versions" role="note" aria-label="versions"> |
| 507 |
<span class="rst-current-version" data-toggle="rst-current-version"> |
| 508 |
<span class="fa fa-book"> Read the Docs</span> |
| 509 |
v: latest |
| 510 |
<span class="fa fa-caret-down"></span> |
| 511 |
</span> |
| 512 |
<div class="rst-other-versions"> |
| 513 |
<dl> |
| 514 |
<dt>Versions</dt> |
| 515 |
|
| 516 |
<dd><a href="/en/latest/">latest</a></dd> |
| 517 |
|
| 518 |
<dd><a href="/en/stable/">stable</a></dd> |
| 519 |
|
| 520 |
<dd><a href="/en/0.6/">0.6</a></dd> |
| 521 |
|
| 522 |
<dd><a href="/en/0.5/">0.5</a></dd> |
| 523 |
|
| 524 |
<dd><a href="/en/0.4.1/">0.4.1</a></dd> |
| 525 |
|
| 526 |
<dd><a href="/en/0.4/">0.4</a></dd> |
| 527 |
|
| 528 |
<dd><a href="/en/0.3/">0.3</a></dd> |
| 529 |
|
| 530 |
<dd><a href="/en/0.2/">0.2</a></dd> |
| 531 |
|
| 532 |
</dl> |
| 533 |
<dl> |
| 534 |
<dt>Downloads</dt> |
| 535 |
|
| 536 |
<dd><a href="//readthedocs.org/projects/webstruct/downloads/pdf/latest/">pdf</a></dd> |
| 537 |
|
| 538 |
<dd><a href="//readthedocs.org/projects/webstruct/downloads/htmlzip/latest/">htmlzip</a></dd> |
| 539 |
|
| 540 |
<dd><a href="//readthedocs.org/projects/webstruct/downloads/epub/latest/">epub</a></dd> |
| 541 |
|
| 542 |
</dl> |
| 543 |
<dl> |
| 544 |
<dt>On Read the Docs</dt> |
| 545 |
<dd> |
| 546 |
<a href="//readthedocs.org/projects/webstruct/?fromdocs=webstruct">Project Home</a> |
| 547 |
</dd> |
| 548 |
<dd> |
| 549 |
<a href="//readthedocs.org/builds/webstruct/?fromdocs=webstruct">Builds</a> |
| 550 |
</dd> |
| 551 |
</dl> |
| 552 |
<hr/> |
| 553 |
Free document hosting provided by <a href="http://www.readthedocs.org">Read the Docs</a>. |
| 554 |
|
| 555 |
</div> |
| 556 |
</div> |
| 557 |
|
| 558 |
|
| 559 |
|
| 560 |
|
| 561 |
|
| 562 |
<script type="text/javascript"> |
| 563 |
var DOCUMENTATION_OPTIONS = { |
| 564 |
URL_ROOT:'./', |
| 565 |
VERSION:'0.6', |
| 566 |
LANGUAGE:'en', |
| 567 |
COLLAPSE_INDEX:false, |
| 568 |
FILE_SUFFIX:'.html', |
| 569 |
HAS_SOURCE: true, |
| 570 |
SOURCELINK_SUFFIX: '.txt' |
| 571 |
}; |
| 572 |
</script> |
| 573 |
<script type="text/javascript" src="https://media.readthedocs.org/javascript/jquery/jquery-2.0.3.min.js"></script> |
| 574 |
<script type="text/javascript" src="https://media.readthedocs.org/javascript/jquery/jquery-migrate-1.2.1.min.js"></script> |
| 575 |
<script type="text/javascript" src="https://media.readthedocs.org/javascript/underscore.js"></script> |
| 576 |
<script type="text/javascript" src="https://media.readthedocs.org/javascript/doctools.js"></script> |
| 577 |
<script type="text/javascript" src="https://media.readthedocs.org/javascript/readthedocs-doc-embed.js"></script> |
| 578 |
|
| 579 |
|
| 580 |
|
| 581 |
<script type="text/javascript" src="_static/js/theme.js"></script> |
| 582 |
|
| 583 |
<script type="text/javascript"> |
| 584 |
jQuery(function () { |
| 585 |
SphinxRtdTheme.Navigation.enable(true); |
| 586 |
}); |
| 587 |
</script> |
| 588 |
|
| 589 |
</body> |
| 590 |
</html> |