Debian Codegraph @

django/contrib/gis/utils/layermapping.py

raw
1 # LayerMapping -- A Django Model/OGR Layer Mapping Utility
2 """
3 The LayerMapping class provides a way to map the contents of OGR
4 vector files (e.g. SHP files) to Geographic-enabled Django models.
5
6 For more information, please consult the GeoDjango documentation:
7 https://docs.djangoproject.com/en/dev/ref/contrib/gis/layermapping/
8 """
9
10 import sys
11 from decimal import Decimal
12 from decimal import InvalidOperation as DecimalInvalidOperation
13 from pathlib import Path
14
15 from django.contrib.gis.db.models import GeometryField
16 from django.contrib.gis.gdal import (
17 CoordTransform,
18 DataSource,
19 GDALException,
20 OGRGeometry,
21 OGRGeomType,
22 SpatialReference,
23 )
24 from django.contrib.gis.gdal.field import (
25 OFTDate,
26 OFTDateTime,
27 OFTInteger,
28 OFTInteger64,
29 OFTReal,
30 OFTString,
31 OFTTime,
32 )
33 from django.core.exceptions import FieldDoesNotExist, ObjectDoesNotExist
34 from django.db import connections, models, router, transaction
35 from django.utils.encoding import force_str
36
37
38 # LayerMapping exceptions.
39 class LayerMapError(Exception):
40 pass
41
42
43 class InvalidString(LayerMapError):
44 pass
45
46
47 class InvalidDecimal(LayerMapError):
48 pass
49
50
51 class InvalidInteger(LayerMapError):
52 pass
53
54
55 class MissingForeignKey(LayerMapError):
56 pass
57
58
59 class LayerMapping:
60 "A class that maps OGR Layers to GeoDjango Models."
61
62 # Acceptable 'base' types for a multi-geometry type.
63 MULTI_TYPES = {
64 1: OGRGeomType("MultiPoint"),
65 2: OGRGeomType("MultiLineString"),
66 3: OGRGeomType("MultiPolygon"),
67 OGRGeomType("Point25D").num: OGRGeomType("MultiPoint25D"),
68 OGRGeomType("LineString25D").num: OGRGeomType("MultiLineString25D"),
69 OGRGeomType("Polygon25D").num: OGRGeomType("MultiPolygon25D"),
70 }
71 # Acceptable Django field types and corresponding acceptable OGR
72 # counterparts.
73 FIELD_TYPES = {
74 models.AutoField: OFTInteger,
75 models.BigAutoField: OFTInteger64,
76 models.SmallAutoField: OFTInteger,
77 models.BooleanField: (OFTInteger, OFTReal, OFTString),
78 models.IntegerField: (OFTInteger, OFTReal, OFTString),
79 models.FloatField: (OFTInteger, OFTReal),
80 models.DateField: OFTDate,
81 models.DateTimeField: OFTDateTime,
82 models.EmailField: OFTString,
83 models.TimeField: OFTTime,
84 models.DecimalField: (OFTInteger, OFTReal),
85 models.CharField: OFTString,
86 models.SlugField: OFTString,
87 models.TextField: OFTString,
88 models.URLField: OFTString,
89 models.UUIDField: OFTString,
90 models.BigIntegerField: (OFTInteger, OFTReal, OFTString),
91 models.SmallIntegerField: (OFTInteger, OFTReal, OFTString),
92 models.PositiveBigIntegerField: (OFTInteger, OFTReal, OFTString),
93 models.PositiveIntegerField: (OFTInteger, OFTReal, OFTString),
94 models.PositiveSmallIntegerField: (OFTInteger, OFTReal, OFTString),
95 }
96
97 def __init__(
98 self,
99 model,
100 data,
101 mapping,
102 layer=0,
103 source_srs=None,
104 encoding="utf-8",
105 transaction_mode="commit_on_success",
106 transform=True,
107 unique=None,
108 using=None,
109 ):
110 """
111 A LayerMapping object is initialized using the given Model (not an instance),
112 a DataSource (or string path to an OGR-supported data file), and a mapping
113 dictionary. See the module level docstring for more details and keyword
114 argument usage.
115 """
116 # Getting the DataSource and the associated Layer.
117 if isinstance(data, (str, Path)):
118 self.ds = DataSource(data, encoding=encoding)
119 else:
120 self.ds = data
121 self.layer = self.ds[layer]
122
123 self.using = using if using is not None else router.db_for_write(model)
124 connection = connections[self.using]
125 self.spatial_backend = connection.ops
126
127 # Setting the mapping & model attributes.
128 self.mapping = mapping
129 self.model = model
130
131 # Checking the layer -- initialization of the object will fail if
132 # things don't check out before hand.
133 self.check_layer()
134
135 # Getting the geometry column associated with the model (an
136 # exception will be raised if there is no geometry column).
137 if connection.features.supports_transform:
138 self.geo_field = self.geometry_field()
139 else:
140 transform = False
141
142 # Checking the source spatial reference system, and getting
143 # the coordinate transformation object (unless the `transform`
144 # keyword is set to False)
145 if transform:
146 self.source_srs = self.check_srs(source_srs)
147 self.transform = self.coord_transform()
148 else:
149 self.transform = transform
150
151 # Setting the encoding for OFTString fields, if specified.
152 if encoding:
153 # Making sure the encoding exists, if not a LookupError
154 # exception will be thrown.
155 from codecs import lookup
156
157 lookup(encoding)
158 self.encoding = encoding
159 else:
160 self.encoding = None
161
162 if unique:
163 self.check_unique(unique)
164 transaction_mode = "autocommit" # Has to be set to autocommit.
165 self.unique = unique
166 else:
167 self.unique = None
168
169 # Setting the transaction decorator with the function in the
170 # transaction modes dictionary.
171 self.transaction_mode = transaction_mode
172 if transaction_mode == "autocommit":
173 self.transaction_decorator = None
174 elif transaction_mode == "commit_on_success":
175 self.transaction_decorator = transaction.atomic
176 else:
177 raise LayerMapError("Unrecognized transaction mode: %s" % transaction_mode)
178
179 # #### Checking routines used during initialization ####
180 def check_fid_range(self, fid_range):
181 "Check the `fid_range` keyword."
182 if fid_range:
183 if isinstance(fid_range, (tuple, list)):
184 return slice(*fid_range)
185 elif isinstance(fid_range, slice):
186 return fid_range
187 else:
188 raise TypeError
189 else:
190 return None
191
192 def check_layer(self):
193 """
194 Check the Layer metadata and ensure that it's compatible with the
195 mapping information and model. Unlike previous revisions, there is no
196 need to increment through each feature in the Layer.
197 """
198 # The geometry field of the model is set here.
199 # TODO: Support more than one geometry field / model. However, this
200 # depends on the GDAL Driver in use.
201 self.geom_field = False
202 self.fields = {}
203
204 # Getting lists of the field names and the field types available in
205 # the OGR Layer.
206 ogr_fields = self.layer.fields
207 ogr_field_types = self.layer.field_types
208
209 # Function for determining if the OGR mapping field is in the Layer.
210 def check_ogr_fld(ogr_map_fld):
211 try:
212 idx = ogr_fields.index(ogr_map_fld)
213 except ValueError:
214 raise LayerMapError(
215 'Given mapping OGR field "%s" not found in OGR Layer.' % ogr_map_fld
216 )
217 return idx
218
219 # No need to increment through each feature in the model, simply check
220 # the Layer metadata against what was given in the mapping dictionary.
221 for field_name, ogr_name in self.mapping.items():
222 # Ensuring that a corresponding field exists in the model
223 # for the given field name in the mapping.
224 try:
225 model_field = self.model._meta.get_field(field_name)
226 except FieldDoesNotExist:
227 raise LayerMapError(
228 'Given mapping field "%s" not in given Model fields.' % field_name
229 )
230
231 # Getting the string name for the Django field class (e.g., 'PointField').
232 fld_name = model_field.__class__.__name__
233
234 if isinstance(model_field, GeometryField):
235 if self.geom_field:
236 raise LayerMapError(
237 "LayerMapping does not support more than one GeometryField per "
238 "model."
239 )
240
241 # Getting the coordinate dimension of the geometry field.
242 coord_dim = model_field.dim
243
244 try:
245 if coord_dim == 3:
246 gtype = OGRGeomType(ogr_name + "25D")
247 else:
248 gtype = OGRGeomType(ogr_name)
249 except GDALException:
250 raise LayerMapError(
251 'Invalid mapping for GeometryField "%s".' % field_name
252 )
253
254 # Making sure that the OGR Layer's Geometry is compatible.
255 ltype = self.layer.geom_type
256 if not (
257 ltype.name.startswith(gtype.name)
258 or self.make_multi(ltype, model_field)
259 ):
260 raise LayerMapError(
261 "Invalid mapping geometry; model has %s%s, "
262 "layer geometry type is %s."
263 % (fld_name, "(dim=3)" if coord_dim == 3 else "", ltype)
264 )
265
266 # Setting the `geom_field` attribute w/the name of the model field
267 # that is a Geometry. Also setting the coordinate dimension
268 # attribute.
269 self.geom_field = field_name
270 self.coord_dim = coord_dim
271 fields_val = model_field
272 elif isinstance(model_field, models.ForeignKey):
273 if isinstance(ogr_name, dict):
274 # Is every given related model mapping field in the Layer?
275 rel_model = model_field.remote_field.model
276 for rel_name, ogr_field in ogr_name.items():
277 idx = check_ogr_fld(ogr_field)
278 try:
279 rel_model._meta.get_field(rel_name)
280 except FieldDoesNotExist:
281 raise LayerMapError(
282 'ForeignKey mapping field "%s" not in %s fields.'
283 % (rel_name, rel_model.__class__.__name__)
284 )
285 fields_val = rel_model
286 else:
287 raise TypeError("ForeignKey mapping must be of dictionary type.")
288 else:
289 # Is the model field type supported by LayerMapping?
290 if model_field.__class__ not in self.FIELD_TYPES:
291 raise LayerMapError(
292 'Django field type "%s" has no OGR mapping (yet).' % fld_name
293 )
294
295 # Is the OGR field in the Layer?
296 idx = check_ogr_fld(ogr_name)
297 ogr_field = ogr_field_types[idx]
298
299 # Can the OGR field type be mapped to the Django field type?
300 if not issubclass(ogr_field, self.FIELD_TYPES[model_field.__class__]):
301 raise LayerMapError(
302 'OGR field "%s" (of type %s) cannot be mapped to Django %s.'
303 % (ogr_field, ogr_field.__name__, fld_name)
304 )
305 fields_val = model_field
306
307 self.fields[field_name] = fields_val
308
309 def check_srs(self, source_srs):
310 "Check the compatibility of the given spatial reference object."
311
312 if isinstance(source_srs, SpatialReference):
313 sr = source_srs
314 elif isinstance(source_srs, self.spatial_backend.spatial_ref_sys()):
315 sr = source_srs.srs
316 elif isinstance(source_srs, (int, str)):
317 sr = SpatialReference(source_srs)
318 else:
319 # Otherwise just pulling the SpatialReference from the layer
320 sr = self.layer.srs
321
322 if not sr:
323 raise LayerMapError("No source reference system defined.")
324 else:
325 return sr
326
327 def check_unique(self, unique):
328 "Check the `unique` keyword parameter -- may be a sequence or string."
329 if isinstance(unique, (list, tuple)):
330 # List of fields to determine uniqueness with
331 for attr in unique:
332 if attr not in self.mapping:
333 raise ValueError
334 elif isinstance(unique, str):
335 # Only a single field passed in.
336 if unique not in self.mapping:
337 raise ValueError
338 else:
339 raise TypeError(
340 "Unique keyword argument must be set with a tuple, list, or string."
341 )
342
343 # Keyword argument retrieval routines ####
344 def feature_kwargs(self, feat):
345 """
346 Given an OGR Feature, return a dictionary of keyword arguments for
347 constructing the mapped model.
348 """
349 # The keyword arguments for model construction.
350 kwargs = {}
351
352 # Incrementing through each model field and OGR field in the
353 # dictionary mapping.
354 for field_name, ogr_name in self.mapping.items():
355 model_field = self.fields[field_name]
356
357 if isinstance(model_field, GeometryField):
358 # Verify OGR geometry.
359 try:
360 val = self.verify_geom(feat.geom, model_field)
361 except GDALException:
362 raise LayerMapError("Could not retrieve geometry from feature.")
363 elif isinstance(model_field, models.base.ModelBase):
364 # The related _model_, not a field was passed in -- indicating
365 # another mapping for the related Model.
366 val = self.verify_fk(feat, model_field, ogr_name)
367 else:
368 # Otherwise, verify OGR Field type.
369 val = self.verify_ogr_field(feat[ogr_name], model_field)
370
371 # Setting the keyword arguments for the field name with the
372 # value obtained above.
373 kwargs[field_name] = val
374
375 return kwargs
376
377 def unique_kwargs(self, kwargs):
378 """
379 Given the feature keyword arguments (from `feature_kwargs`), construct
380 and return the uniqueness keyword arguments -- a subset of the feature
381 kwargs.
382 """
383 if isinstance(self.unique, str):
384 return {self.unique: kwargs[self.unique]}
385 else:
386 return {fld: kwargs[fld] for fld in self.unique}
387
388 # #### Verification routines used in constructing model keyword arguments. ####
389 def verify_ogr_field(self, ogr_field, model_field):
390 """
391 Verify if the OGR Field contents are acceptable to the model field. If
392 they are, return the verified value, otherwise raise an exception.
393 """
394 if isinstance(ogr_field, OFTString) and isinstance(
395 model_field, (models.CharField, models.TextField)
396 ):
397 if self.encoding and ogr_field.value is not None:
398 # The encoding for OGR data sources may be specified here
399 # (e.g., 'cp437' for Census Bureau boundary files).
400 val = force_str(ogr_field.value, self.encoding)
401 else:
402 val = ogr_field.value
403 if (
404 model_field.max_length
405 and val is not None
406 and len(val) > model_field.max_length
407 ):
408 raise InvalidString(
409 "%s model field maximum string length is %s, given %s characters."
410 % (model_field.name, model_field.max_length, len(val))
411 )
412 elif isinstance(ogr_field, OFTReal) and isinstance(
413 model_field, models.DecimalField
414 ):
415 try:
416 # Creating an instance of the Decimal value to use.
417 d = Decimal(str(ogr_field.value))
418 except DecimalInvalidOperation:
419 raise InvalidDecimal(
420 "Could not construct decimal from: %s" % ogr_field.value
421 )
422
423 # Getting the decimal value as a tuple.
424 dtup = d.as_tuple()
425 digits = dtup[1]
426 d_idx = dtup[2] # index where the decimal is
427
428 # Maximum amount of precision, or digits to the left of the decimal.
429 max_prec = model_field.max_digits - model_field.decimal_places
430
431 # Getting the digits to the left of the decimal place for the
432 # given decimal.
433 if d_idx < 0:
434 n_prec = len(digits[:d_idx])
435 else:
436 n_prec = len(digits) + d_idx
437
438 # If we have more than the maximum digits allowed, then throw an
439 # InvalidDecimal exception.
440 if n_prec > max_prec:
441 raise InvalidDecimal(
442 "A DecimalField with max_digits %d, decimal_places %d must "
443 "round to an absolute value less than 10^%d."
444 % (model_field.max_digits, model_field.decimal_places, max_prec)
445 )
446 val = d
447 elif isinstance(ogr_field, (OFTReal, OFTString)) and isinstance(
448 model_field, models.IntegerField
449 ):
450 # Attempt to convert any OFTReal and OFTString value to an OFTInteger.
451 try:
452 val = int(ogr_field.value)
453 except ValueError:
454 raise InvalidInteger(
455 "Could not construct integer from: %s" % ogr_field.value
456 )
457 else:
458 val = ogr_field.value
459 return val
460
461 def verify_fk(self, feat, rel_model, rel_mapping):
462 """
463 Given an OGR Feature, the related model and its dictionary mapping,
464 retrieve the related model for the ForeignKey mapping.
465 """
466 # TODO: It is expensive to retrieve a model for every record --
467 # explore if an efficient mechanism exists for caching related
468 # ForeignKey models.
469
470 # Constructing and verifying the related model keyword arguments.
471 fk_kwargs = {}
472 for field_name, ogr_name in rel_mapping.items():
473 fk_kwargs[field_name] = self.verify_ogr_field(
474 feat[ogr_name], rel_model._meta.get_field(field_name)
475 )
476
477 # Attempting to retrieve and return the related model.
478 try:
479 return rel_model.objects.using(self.using).get(**fk_kwargs)
480 except ObjectDoesNotExist:
481 raise MissingForeignKey(
482 "No ForeignKey %s model found with keyword arguments: %s"
483 % (rel_model.__name__, fk_kwargs)
484 )
485
486 def verify_geom(self, geom, model_field):
487 """
488 Verify the geometry -- construct and return a GeometryCollection
489 if necessary (for example if the model field is MultiPolygonField while
490 the mapped shapefile only contains Polygons).
491 """
492 # Measured geometries are not yet supported by GeoDjango models.
493 if geom.is_measured:
494 geom.set_measured(False)
495
496 # Downgrade a 3D geom to a 2D one, if necessary.
497 if self.coord_dim == 2 and geom.is_3d:
498 geom.set_3d(False)
499
500 if self.make_multi(geom.geom_type, model_field):
501 # Constructing a multi-geometry type to contain the single geometry
502 multi_type = self.MULTI_TYPES[geom.geom_type.num]
503 g = OGRGeometry(multi_type)
504 g.add(geom)
505 else:
506 g = geom
507
508 # Transforming the geometry with our Coordinate Transformation object,
509 # but only if the class variable `transform` is set w/a CoordTransform
510 # object.
511 if self.transform:
512 g.transform(self.transform)
513
514 # Returning the WKT of the geometry.
515 return g.wkt
516
517 # #### Other model methods ####
518 def coord_transform(self):
519 "Return the coordinate transformation object."
520 SpatialRefSys = self.spatial_backend.spatial_ref_sys()
521 try:
522 # Getting the target spatial reference system
523 target_srs = (
524 SpatialRefSys.objects.using(self.using)
525 .get(srid=self.geo_field.srid)
526 .srs
527 )
528
529 # Creating the CoordTransform object
530 return CoordTransform(self.source_srs, target_srs)
531 except Exception as exc:
532 raise LayerMapError(
533 "Could not translate between the data source and model geometry."
534 ) from exc
535
536 def geometry_field(self):
537 "Return the GeometryField instance associated with the geographic column."
538 # Use `get_field()` on the model's options so that we
539 # get the correct field instance if there's model inheritance.
540 opts = self.model._meta
541 return opts.get_field(self.geom_field)
542
543 def make_multi(self, geom_type, model_field):
544 """
545 Given the OGRGeomType for a geometry and its associated GeometryField,
546 determine whether the geometry should be turned into a GeometryCollection.
547 """
548 return (
549 geom_type.num in self.MULTI_TYPES
550 and model_field.__class__.__name__ == "Multi%s" % geom_type.django
551 )
552
553 def save(
554 self,
555 verbose=False,
556 fid_range=False,
557 step=False,
558 progress=False,
559 silent=False,
560 stream=sys.stdout,
561 strict=False,
562 ):
563 """
564 Save the contents from the OGR DataSource Layer into the database
565 according to the mapping dictionary given at initialization.
566
567 Keyword Parameters:
568 verbose:
569 If set, information will be printed subsequent to each model save
570 executed on the database.
571
572 fid_range:
573 May be set with a slice or tuple of (begin, end) feature ID's to map
574 from the data source. In other words, this keyword enables the user
575 to selectively import a subset range of features in the geographic
576 data source.
577
578 step:
579 If set with an integer, transactions will occur at every step
580 interval. For example, if step=1000, a commit would occur after
581 the 1,000th feature, the 2,000th feature etc.
582
583 progress:
584 When this keyword is set, status information will be printed giving
585 the number of features processed and successfully saved. By default,
586 progress information will pe printed every 1000 features processed,
587 however, this default may be overridden by setting this keyword with an
588 integer for the desired interval.
589
590 stream:
591 Status information will be written to this file handle. Defaults to
592 using `sys.stdout`, but any object with a `write` method is supported.
593
594 silent:
595 By default, non-fatal error notifications are printed to stdout, but
596 this keyword may be set to disable these notifications.
597
598 strict:
599 Execution of the model mapping will cease upon the first error
600 encountered. The default behavior is to attempt to continue.
601 """
602 # Getting the default Feature ID range.
603 default_range = self.check_fid_range(fid_range)
604
605 # Setting the progress interval, if requested.
606 if progress:
607 if progress is True or not isinstance(progress, int):
608 progress_interval = 1000
609 else:
610 progress_interval = progress
611
612 def _save(feat_range=default_range, num_feat=0, num_saved=0):
613 if feat_range:
614 layer_iter = self.layer[feat_range]
615 else:
616 layer_iter = self.layer
617
618 for feat in layer_iter:
619 num_feat += 1
620 # Getting the keyword arguments
621 try:
622 kwargs = self.feature_kwargs(feat)
623 except LayerMapError as msg:
624 # Something borked the validation
625 if strict:
626 raise
627 elif not silent:
628 stream.write(
629 "Ignoring Feature ID %s because: %s\n" % (feat.fid, msg)
630 )
631 else:
632 # Constructing the model using the keyword args
633 is_update = False
634 if self.unique:
635 # If we want unique models on a particular field, handle the
636 # geometry appropriately.
637 try:
638 # Getting the keyword arguments and retrieving
639 # the unique model.
640 u_kwargs = self.unique_kwargs(kwargs)
641 m = self.model.objects.using(self.using).get(**u_kwargs)
642 is_update = True
643
644 # Getting the geometry (in OGR form), creating
645 # one from the kwargs WKT, adding in additional
646 # geometries, and update the attribute with the
647 # just-updated geometry WKT.
648 geom_value = getattr(m, self.geom_field)
649 if geom_value is None:
650 geom = OGRGeometry(kwargs[self.geom_field])
651 else:
652 geom = geom_value.ogr
653 new = OGRGeometry(kwargs[self.geom_field])
654 for g in new:
655 geom.add(g)
656 setattr(m, self.geom_field, geom.wkt)
657 except ObjectDoesNotExist:
658 # No unique model exists yet, create.
659 m = self.model(**kwargs)
660 else:
661 m = self.model(**kwargs)
662
663 try:
664 # Attempting to save.
665 m.save(using=self.using)
666 num_saved += 1
667 if verbose:
668 stream.write(
669 "%s: %s\n" % ("Updated" if is_update else "Saved", m)
670 )
671 except Exception as msg:
672 if strict:
673 # Bailing out if the `strict` keyword is set.
674 if not silent:
675 stream.write(
676 "Failed to save the feature (id: %s) into the "
677 "model with the keyword arguments:\n" % feat.fid
678 )
679 stream.write("%s\n" % kwargs)
680 raise
681 elif not silent:
682 stream.write(
683 "Failed to save %s:\n %s\nContinuing\n" % (kwargs, msg)
684 )
685
686 # Printing progress information, if requested.
687 if progress and num_feat % progress_interval == 0:
688 stream.write(
689 "Processed %d features, saved %d ...\n" % (num_feat, num_saved)
690 )
691
692 # Only used for status output purposes -- incremental saving uses the
693 # values returned here.
694 return num_saved, num_feat
695
696 if self.transaction_decorator is not None:
697 _save = self.transaction_decorator(_save)
698
699 nfeat = self.layer.num_feat
700 if step and isinstance(step, int) and step < nfeat:
701 # Incremental saving is requested at the given interval (step)
702 if default_range:
703 raise LayerMapError(
704 "The `step` keyword may not be used in conjunction with the "
705 "`fid_range` keyword."
706 )
707 beg, num_feat, num_saved = (0, 0, 0)
708 indices = range(step, nfeat, step)
709 n_i = len(indices)
710
711 for i, end in enumerate(indices):
712 # Constructing the slice to use for this step; the last slice is
713 # special (e.g, [100:] instead of [90:100]).
714 if i + 1 == n_i:
715 step_slice = slice(beg, None)
716 else:
717 step_slice = slice(beg, end)
718
719 try:
720 num_feat, num_saved = _save(step_slice, num_feat, num_saved)
721 beg = end
722 except Exception: # Deliberately catch everything
723 stream.write(
724 "%s\nFailed to save slice: %s\n" % ("=-" * 20, step_slice)
725 )
726 raise
727 else:
728 # Otherwise, just calling the previously defined _save() function.
729 _save()