| 1 |
# LayerMapping -- A Django Model/OGR Layer Mapping Utility |
| 2 |
""" |
| 3 |
The LayerMapping class provides a way to map the contents of OGR |
| 4 |
vector files (e.g. SHP files) to Geographic-enabled Django models. |
| 5 |
|
| 6 |
For more information, please consult the GeoDjango documentation: |
| 7 |
https://docs.djangoproject.com/en/dev/ref/contrib/gis/layermapping/ |
| 8 |
""" |
| 9 |
|
| 10 |
import sys |
| 11 |
from decimal import Decimal |
| 12 |
from decimal import InvalidOperation as DecimalInvalidOperation |
| 13 |
from pathlib import Path |
| 14 |
|
| 15 |
from django.contrib.gis.db.models import GeometryField |
| 16 |
from django.contrib.gis.gdal import ( |
| 17 |
CoordTransform, |
| 18 |
DataSource, |
| 19 |
GDALException, |
| 20 |
OGRGeometry, |
| 21 |
OGRGeomType, |
| 22 |
SpatialReference, |
| 23 |
) |
| 24 |
from django.contrib.gis.gdal.field import ( |
| 25 |
OFTDate, |
| 26 |
OFTDateTime, |
| 27 |
OFTInteger, |
| 28 |
OFTInteger64, |
| 29 |
OFTReal, |
| 30 |
OFTString, |
| 31 |
OFTTime, |
| 32 |
) |
| 33 |
from django.core.exceptions import FieldDoesNotExist, ObjectDoesNotExist |
| 34 |
from django.db import connections, models, router, transaction |
| 35 |
from django.utils.encoding import force_str |
| 36 |
|
| 37 |
|
| 38 |
# LayerMapping exceptions. |
| 39 |
class LayerMapError(Exception): |
| 40 |
pass |
| 41 |
|
| 42 |
|
| 43 |
class InvalidString(LayerMapError): |
| 44 |
pass |
| 45 |
|
| 46 |
|
| 47 |
class InvalidDecimal(LayerMapError): |
| 48 |
pass |
| 49 |
|
| 50 |
|
| 51 |
class InvalidInteger(LayerMapError): |
| 52 |
pass |
| 53 |
|
| 54 |
|
| 55 |
class MissingForeignKey(LayerMapError): |
| 56 |
pass |
| 57 |
|
| 58 |
|
| 59 |
class LayerMapping: |
| 60 |
"A class that maps OGR Layers to GeoDjango Models." |
| 61 |
|
| 62 |
# Acceptable 'base' types for a multi-geometry type. |
| 63 |
MULTI_TYPES = { |
| 64 |
1: OGRGeomType("MultiPoint"), |
| 65 |
2: OGRGeomType("MultiLineString"), |
| 66 |
3: OGRGeomType("MultiPolygon"), |
| 67 |
OGRGeomType("Point25D").num: OGRGeomType("MultiPoint25D"), |
| 68 |
OGRGeomType("LineString25D").num: OGRGeomType("MultiLineString25D"), |
| 69 |
OGRGeomType("Polygon25D").num: OGRGeomType("MultiPolygon25D"), |
| 70 |
} |
| 71 |
# Acceptable Django field types and corresponding acceptable OGR |
| 72 |
# counterparts. |
| 73 |
FIELD_TYPES = { |
| 74 |
models.AutoField: OFTInteger, |
| 75 |
models.BigAutoField: OFTInteger64, |
| 76 |
models.SmallAutoField: OFTInteger, |
| 77 |
models.BooleanField: (OFTInteger, OFTReal, OFTString), |
| 78 |
models.IntegerField: (OFTInteger, OFTReal, OFTString), |
| 79 |
models.FloatField: (OFTInteger, OFTReal), |
| 80 |
models.DateField: OFTDate, |
| 81 |
models.DateTimeField: OFTDateTime, |
| 82 |
models.EmailField: OFTString, |
| 83 |
models.TimeField: OFTTime, |
| 84 |
models.DecimalField: (OFTInteger, OFTReal), |
| 85 |
models.CharField: OFTString, |
| 86 |
models.SlugField: OFTString, |
| 87 |
models.TextField: OFTString, |
| 88 |
models.URLField: OFTString, |
| 89 |
models.UUIDField: OFTString, |
| 90 |
models.BigIntegerField: (OFTInteger, OFTReal, OFTString), |
| 91 |
models.SmallIntegerField: (OFTInteger, OFTReal, OFTString), |
| 92 |
models.PositiveBigIntegerField: (OFTInteger, OFTReal, OFTString), |
| 93 |
models.PositiveIntegerField: (OFTInteger, OFTReal, OFTString), |
| 94 |
models.PositiveSmallIntegerField: (OFTInteger, OFTReal, OFTString), |
| 95 |
} |
| 96 |
|
| 97 |
def __init__( |
| 98 |
self, |
| 99 |
model, |
| 100 |
data, |
| 101 |
mapping, |
| 102 |
layer=0, |
| 103 |
source_srs=None, |
| 104 |
encoding="utf-8", |
| 105 |
transaction_mode="commit_on_success", |
| 106 |
transform=True, |
| 107 |
unique=None, |
| 108 |
using=None, |
| 109 |
): |
| 110 |
""" |
| 111 |
A LayerMapping object is initialized using the given Model (not an instance), |
| 112 |
a DataSource (or string path to an OGR-supported data file), and a mapping |
| 113 |
dictionary. See the module level docstring for more details and keyword |
| 114 |
argument usage. |
| 115 |
""" |
| 116 |
# Getting the DataSource and the associated Layer. |
| 117 |
if isinstance(data, (str, Path)): |
| 118 |
self.ds = DataSource(data, encoding=encoding) |
| 119 |
else: |
| 120 |
self.ds = data |
| 121 |
self.layer = self.ds[layer] |
| 122 |
|
| 123 |
self.using = using if using is not None else router.db_for_write(model) |
| 124 |
connection = connections[self.using] |
| 125 |
self.spatial_backend = connection.ops |
| 126 |
|
| 127 |
# Setting the mapping & model attributes. |
| 128 |
self.mapping = mapping |
| 129 |
self.model = model |
| 130 |
|
| 131 |
# Checking the layer -- initialization of the object will fail if |
| 132 |
# things don't check out before hand. |
| 133 |
self.check_layer() |
| 134 |
|
| 135 |
# Getting the geometry column associated with the model (an |
| 136 |
# exception will be raised if there is no geometry column). |
| 137 |
if connection.features.supports_transform: |
| 138 |
self.geo_field = self.geometry_field() |
| 139 |
else: |
| 140 |
transform = False |
| 141 |
|
| 142 |
# Checking the source spatial reference system, and getting |
| 143 |
# the coordinate transformation object (unless the `transform` |
| 144 |
# keyword is set to False) |
| 145 |
if transform: |
| 146 |
self.source_srs = self.check_srs(source_srs) |
| 147 |
self.transform = self.coord_transform() |
| 148 |
else: |
| 149 |
self.transform = transform |
| 150 |
|
| 151 |
# Setting the encoding for OFTString fields, if specified. |
| 152 |
if encoding: |
| 153 |
# Making sure the encoding exists, if not a LookupError |
| 154 |
# exception will be thrown. |
| 155 |
from codecs import lookup |
| 156 |
|
| 157 |
lookup(encoding) |
| 158 |
self.encoding = encoding |
| 159 |
else: |
| 160 |
self.encoding = None |
| 161 |
|
| 162 |
if unique: |
| 163 |
self.check_unique(unique) |
| 164 |
transaction_mode = "autocommit" # Has to be set to autocommit. |
| 165 |
self.unique = unique |
| 166 |
else: |
| 167 |
self.unique = None |
| 168 |
|
| 169 |
# Setting the transaction decorator with the function in the |
| 170 |
# transaction modes dictionary. |
| 171 |
self.transaction_mode = transaction_mode |
| 172 |
if transaction_mode == "autocommit": |
| 173 |
self.transaction_decorator = None |
| 174 |
elif transaction_mode == "commit_on_success": |
| 175 |
self.transaction_decorator = transaction.atomic |
| 176 |
else: |
| 177 |
raise LayerMapError("Unrecognized transaction mode: %s" % transaction_mode) |
| 178 |
|
| 179 |
# #### Checking routines used during initialization #### |
| 180 |
def check_fid_range(self, fid_range): |
| 181 |
"Check the `fid_range` keyword." |
| 182 |
if fid_range: |
| 183 |
if isinstance(fid_range, (tuple, list)): |
| 184 |
return slice(*fid_range) |
| 185 |
elif isinstance(fid_range, slice): |
| 186 |
return fid_range |
| 187 |
else: |
| 188 |
raise TypeError |
| 189 |
else: |
| 190 |
return None |
| 191 |
|
| 192 |
def check_layer(self): |
| 193 |
""" |
| 194 |
Check the Layer metadata and ensure that it's compatible with the |
| 195 |
mapping information and model. Unlike previous revisions, there is no |
| 196 |
need to increment through each feature in the Layer. |
| 197 |
""" |
| 198 |
# The geometry field of the model is set here. |
| 199 |
# TODO: Support more than one geometry field / model. However, this |
| 200 |
# depends on the GDAL Driver in use. |
| 201 |
self.geom_field = False |
| 202 |
self.fields = {} |
| 203 |
|
| 204 |
# Getting lists of the field names and the field types available in |
| 205 |
# the OGR Layer. |
| 206 |
ogr_fields = self.layer.fields |
| 207 |
ogr_field_types = self.layer.field_types |
| 208 |
|
| 209 |
# Function for determining if the OGR mapping field is in the Layer. |
| 210 |
def check_ogr_fld(ogr_map_fld): |
| 211 |
try: |
| 212 |
idx = ogr_fields.index(ogr_map_fld) |
| 213 |
except ValueError: |
| 214 |
raise LayerMapError( |
| 215 |
'Given mapping OGR field "%s" not found in OGR Layer.' % ogr_map_fld |
| 216 |
) |
| 217 |
return idx |
| 218 |
|
| 219 |
# No need to increment through each feature in the model, simply check |
| 220 |
# the Layer metadata against what was given in the mapping dictionary. |
| 221 |
for field_name, ogr_name in self.mapping.items(): |
| 222 |
# Ensuring that a corresponding field exists in the model |
| 223 |
# for the given field name in the mapping. |
| 224 |
try: |
| 225 |
model_field = self.model._meta.get_field(field_name) |
| 226 |
except FieldDoesNotExist: |
| 227 |
raise LayerMapError( |
| 228 |
'Given mapping field "%s" not in given Model fields.' % field_name |
| 229 |
) |
| 230 |
|
| 231 |
# Getting the string name for the Django field class (e.g., 'PointField'). |
| 232 |
fld_name = model_field.__class__.__name__ |
| 233 |
|
| 234 |
if isinstance(model_field, GeometryField): |
| 235 |
if self.geom_field: |
| 236 |
raise LayerMapError( |
| 237 |
"LayerMapping does not support more than one GeometryField per " |
| 238 |
"model." |
| 239 |
) |
| 240 |
|
| 241 |
# Getting the coordinate dimension of the geometry field. |
| 242 |
coord_dim = model_field.dim |
| 243 |
|
| 244 |
try: |
| 245 |
if coord_dim == 3: |
| 246 |
gtype = OGRGeomType(ogr_name + "25D") |
| 247 |
else: |
| 248 |
gtype = OGRGeomType(ogr_name) |
| 249 |
except GDALException: |
| 250 |
raise LayerMapError( |
| 251 |
'Invalid mapping for GeometryField "%s".' % field_name |
| 252 |
) |
| 253 |
|
| 254 |
# Making sure that the OGR Layer's Geometry is compatible. |
| 255 |
ltype = self.layer.geom_type |
| 256 |
if not ( |
| 257 |
ltype.name.startswith(gtype.name) |
| 258 |
or self.make_multi(ltype, model_field) |
| 259 |
): |
| 260 |
raise LayerMapError( |
| 261 |
"Invalid mapping geometry; model has %s%s, " |
| 262 |
"layer geometry type is %s." |
| 263 |
% (fld_name, "(dim=3)" if coord_dim == 3 else "", ltype) |
| 264 |
) |
| 265 |
|
| 266 |
# Setting the `geom_field` attribute w/the name of the model field |
| 267 |
# that is a Geometry. Also setting the coordinate dimension |
| 268 |
# attribute. |
| 269 |
self.geom_field = field_name |
| 270 |
self.coord_dim = coord_dim |
| 271 |
fields_val = model_field |
| 272 |
elif isinstance(model_field, models.ForeignKey): |
| 273 |
if isinstance(ogr_name, dict): |
| 274 |
# Is every given related model mapping field in the Layer? |
| 275 |
rel_model = model_field.remote_field.model |
| 276 |
for rel_name, ogr_field in ogr_name.items(): |
| 277 |
idx = check_ogr_fld(ogr_field) |
| 278 |
try: |
| 279 |
rel_model._meta.get_field(rel_name) |
| 280 |
except FieldDoesNotExist: |
| 281 |
raise LayerMapError( |
| 282 |
'ForeignKey mapping field "%s" not in %s fields.' |
| 283 |
% (rel_name, rel_model.__class__.__name__) |
| 284 |
) |
| 285 |
fields_val = rel_model |
| 286 |
else: |
| 287 |
raise TypeError("ForeignKey mapping must be of dictionary type.") |
| 288 |
else: |
| 289 |
# Is the model field type supported by LayerMapping? |
| 290 |
if model_field.__class__ not in self.FIELD_TYPES: |
| 291 |
raise LayerMapError( |
| 292 |
'Django field type "%s" has no OGR mapping (yet).' % fld_name |
| 293 |
) |
| 294 |
|
| 295 |
# Is the OGR field in the Layer? |
| 296 |
idx = check_ogr_fld(ogr_name) |
| 297 |
ogr_field = ogr_field_types[idx] |
| 298 |
|
| 299 |
# Can the OGR field type be mapped to the Django field type? |
| 300 |
if not issubclass(ogr_field, self.FIELD_TYPES[model_field.__class__]): |
| 301 |
raise LayerMapError( |
| 302 |
'OGR field "%s" (of type %s) cannot be mapped to Django %s.' |
| 303 |
% (ogr_field, ogr_field.__name__, fld_name) |
| 304 |
) |
| 305 |
fields_val = model_field |
| 306 |
|
| 307 |
self.fields[field_name] = fields_val |
| 308 |
|
| 309 |
def check_srs(self, source_srs): |
| 310 |
"Check the compatibility of the given spatial reference object." |
| 311 |
|
| 312 |
if isinstance(source_srs, SpatialReference): |
| 313 |
sr = source_srs |
| 314 |
elif isinstance(source_srs, self.spatial_backend.spatial_ref_sys()): |
| 315 |
sr = source_srs.srs |
| 316 |
elif isinstance(source_srs, (int, str)): |
| 317 |
sr = SpatialReference(source_srs) |
| 318 |
else: |
| 319 |
# Otherwise just pulling the SpatialReference from the layer |
| 320 |
sr = self.layer.srs |
| 321 |
|
| 322 |
if not sr: |
| 323 |
raise LayerMapError("No source reference system defined.") |
| 324 |
else: |
| 325 |
return sr |
| 326 |
|
| 327 |
def check_unique(self, unique): |
| 328 |
"Check the `unique` keyword parameter -- may be a sequence or string." |
| 329 |
if isinstance(unique, (list, tuple)): |
| 330 |
# List of fields to determine uniqueness with |
| 331 |
for attr in unique: |
| 332 |
if attr not in self.mapping: |
| 333 |
raise ValueError |
| 334 |
elif isinstance(unique, str): |
| 335 |
# Only a single field passed in. |
| 336 |
if unique not in self.mapping: |
| 337 |
raise ValueError |
| 338 |
else: |
| 339 |
raise TypeError( |
| 340 |
"Unique keyword argument must be set with a tuple, list, or string." |
| 341 |
) |
| 342 |
|
| 343 |
# Keyword argument retrieval routines #### |
| 344 |
def feature_kwargs(self, feat): |
| 345 |
""" |
| 346 |
Given an OGR Feature, return a dictionary of keyword arguments for |
| 347 |
constructing the mapped model. |
| 348 |
""" |
| 349 |
# The keyword arguments for model construction. |
| 350 |
kwargs = {} |
| 351 |
|
| 352 |
# Incrementing through each model field and OGR field in the |
| 353 |
# dictionary mapping. |
| 354 |
for field_name, ogr_name in self.mapping.items(): |
| 355 |
model_field = self.fields[field_name] |
| 356 |
|
| 357 |
if isinstance(model_field, GeometryField): |
| 358 |
# Verify OGR geometry. |
| 359 |
try: |
| 360 |
val = self.verify_geom(feat.geom, model_field) |
| 361 |
except GDALException: |
| 362 |
raise LayerMapError("Could not retrieve geometry from feature.") |
| 363 |
elif isinstance(model_field, models.base.ModelBase): |
| 364 |
# The related _model_, not a field was passed in -- indicating |
| 365 |
# another mapping for the related Model. |
| 366 |
val = self.verify_fk(feat, model_field, ogr_name) |
| 367 |
else: |
| 368 |
# Otherwise, verify OGR Field type. |
| 369 |
val = self.verify_ogr_field(feat[ogr_name], model_field) |
| 370 |
|
| 371 |
# Setting the keyword arguments for the field name with the |
| 372 |
# value obtained above. |
| 373 |
kwargs[field_name] = val |
| 374 |
|
| 375 |
return kwargs |
| 376 |
|
| 377 |
def unique_kwargs(self, kwargs): |
| 378 |
""" |
| 379 |
Given the feature keyword arguments (from `feature_kwargs`), construct |
| 380 |
and return the uniqueness keyword arguments -- a subset of the feature |
| 381 |
kwargs. |
| 382 |
""" |
| 383 |
if isinstance(self.unique, str): |
| 384 |
return {self.unique: kwargs[self.unique]} |
| 385 |
else: |
| 386 |
return {fld: kwargs[fld] for fld in self.unique} |
| 387 |
|
| 388 |
# #### Verification routines used in constructing model keyword arguments. #### |
| 389 |
def verify_ogr_field(self, ogr_field, model_field): |
| 390 |
""" |
| 391 |
Verify if the OGR Field contents are acceptable to the model field. If |
| 392 |
they are, return the verified value, otherwise raise an exception. |
| 393 |
""" |
| 394 |
if isinstance(ogr_field, OFTString) and isinstance( |
| 395 |
model_field, (models.CharField, models.TextField) |
| 396 |
): |
| 397 |
if self.encoding and ogr_field.value is not None: |
| 398 |
# The encoding for OGR data sources may be specified here |
| 399 |
# (e.g., 'cp437' for Census Bureau boundary files). |
| 400 |
val = force_str(ogr_field.value, self.encoding) |
| 401 |
else: |
| 402 |
val = ogr_field.value |
| 403 |
if ( |
| 404 |
model_field.max_length |
| 405 |
and val is not None |
| 406 |
and len(val) > model_field.max_length |
| 407 |
): |
| 408 |
raise InvalidString( |
| 409 |
"%s model field maximum string length is %s, given %s characters." |
| 410 |
% (model_field.name, model_field.max_length, len(val)) |
| 411 |
) |
| 412 |
elif isinstance(ogr_field, OFTReal) and isinstance( |
| 413 |
model_field, models.DecimalField |
| 414 |
): |
| 415 |
try: |
| 416 |
# Creating an instance of the Decimal value to use. |
| 417 |
d = Decimal(str(ogr_field.value)) |
| 418 |
except DecimalInvalidOperation: |
| 419 |
raise InvalidDecimal( |
| 420 |
"Could not construct decimal from: %s" % ogr_field.value |
| 421 |
) |
| 422 |
|
| 423 |
# Getting the decimal value as a tuple. |
| 424 |
dtup = d.as_tuple() |
| 425 |
digits = dtup[1] |
| 426 |
d_idx = dtup[2] # index where the decimal is |
| 427 |
|
| 428 |
# Maximum amount of precision, or digits to the left of the decimal. |
| 429 |
max_prec = model_field.max_digits - model_field.decimal_places |
| 430 |
|
| 431 |
# Getting the digits to the left of the decimal place for the |
| 432 |
# given decimal. |
| 433 |
if d_idx < 0: |
| 434 |
n_prec = len(digits[:d_idx]) |
| 435 |
else: |
| 436 |
n_prec = len(digits) + d_idx |
| 437 |
|
| 438 |
# If we have more than the maximum digits allowed, then throw an |
| 439 |
# InvalidDecimal exception. |
| 440 |
if n_prec > max_prec: |
| 441 |
raise InvalidDecimal( |
| 442 |
"A DecimalField with max_digits %d, decimal_places %d must " |
| 443 |
"round to an absolute value less than 10^%d." |
| 444 |
% (model_field.max_digits, model_field.decimal_places, max_prec) |
| 445 |
) |
| 446 |
val = d |
| 447 |
elif isinstance(ogr_field, (OFTReal, OFTString)) and isinstance( |
| 448 |
model_field, models.IntegerField |
| 449 |
): |
| 450 |
# Attempt to convert any OFTReal and OFTString value to an OFTInteger. |
| 451 |
try: |
| 452 |
val = int(ogr_field.value) |
| 453 |
except ValueError: |
| 454 |
raise InvalidInteger( |
| 455 |
"Could not construct integer from: %s" % ogr_field.value |
| 456 |
) |
| 457 |
else: |
| 458 |
val = ogr_field.value |
| 459 |
return val |
| 460 |
|
| 461 |
def verify_fk(self, feat, rel_model, rel_mapping): |
| 462 |
""" |
| 463 |
Given an OGR Feature, the related model and its dictionary mapping, |
| 464 |
retrieve the related model for the ForeignKey mapping. |
| 465 |
""" |
| 466 |
# TODO: It is expensive to retrieve a model for every record -- |
| 467 |
# explore if an efficient mechanism exists for caching related |
| 468 |
# ForeignKey models. |
| 469 |
|
| 470 |
# Constructing and verifying the related model keyword arguments. |
| 471 |
fk_kwargs = {} |
| 472 |
for field_name, ogr_name in rel_mapping.items(): |
| 473 |
fk_kwargs[field_name] = self.verify_ogr_field( |
| 474 |
feat[ogr_name], rel_model._meta.get_field(field_name) |
| 475 |
) |
| 476 |
|
| 477 |
# Attempting to retrieve and return the related model. |
| 478 |
try: |
| 479 |
return rel_model.objects.using(self.using).get(**fk_kwargs) |
| 480 |
except ObjectDoesNotExist: |
| 481 |
raise MissingForeignKey( |
| 482 |
"No ForeignKey %s model found with keyword arguments: %s" |
| 483 |
% (rel_model.__name__, fk_kwargs) |
| 484 |
) |
| 485 |
|
| 486 |
def verify_geom(self, geom, model_field): |
| 487 |
""" |
| 488 |
Verify the geometry -- construct and return a GeometryCollection |
| 489 |
if necessary (for example if the model field is MultiPolygonField while |
| 490 |
the mapped shapefile only contains Polygons). |
| 491 |
""" |
| 492 |
# Measured geometries are not yet supported by GeoDjango models. |
| 493 |
if geom.is_measured: |
| 494 |
geom.set_measured(False) |
| 495 |
|
| 496 |
# Downgrade a 3D geom to a 2D one, if necessary. |
| 497 |
if self.coord_dim == 2 and geom.is_3d: |
| 498 |
geom.set_3d(False) |
| 499 |
|
| 500 |
if self.make_multi(geom.geom_type, model_field): |
| 501 |
# Constructing a multi-geometry type to contain the single geometry |
| 502 |
multi_type = self.MULTI_TYPES[geom.geom_type.num] |
| 503 |
g = OGRGeometry(multi_type) |
| 504 |
g.add(geom) |
| 505 |
else: |
| 506 |
g = geom |
| 507 |
|
| 508 |
# Transforming the geometry with our Coordinate Transformation object, |
| 509 |
# but only if the class variable `transform` is set w/a CoordTransform |
| 510 |
# object. |
| 511 |
if self.transform: |
| 512 |
g.transform(self.transform) |
| 513 |
|
| 514 |
# Returning the WKT of the geometry. |
| 515 |
return g.wkt |
| 516 |
|
| 517 |
# #### Other model methods #### |
| 518 |
def coord_transform(self): |
| 519 |
"Return the coordinate transformation object." |
| 520 |
SpatialRefSys = self.spatial_backend.spatial_ref_sys() |
| 521 |
try: |
| 522 |
# Getting the target spatial reference system |
| 523 |
target_srs = ( |
| 524 |
SpatialRefSys.objects.using(self.using) |
| 525 |
.get(srid=self.geo_field.srid) |
| 526 |
.srs |
| 527 |
) |
| 528 |
|
| 529 |
# Creating the CoordTransform object |
| 530 |
return CoordTransform(self.source_srs, target_srs) |
| 531 |
except Exception as exc: |
| 532 |
raise LayerMapError( |
| 533 |
"Could not translate between the data source and model geometry." |
| 534 |
) from exc |
| 535 |
|
| 536 |
def geometry_field(self): |
| 537 |
"Return the GeometryField instance associated with the geographic column." |
| 538 |
# Use `get_field()` on the model's options so that we |
| 539 |
# get the correct field instance if there's model inheritance. |
| 540 |
opts = self.model._meta |
| 541 |
return opts.get_field(self.geom_field) |
| 542 |
|
| 543 |
def make_multi(self, geom_type, model_field): |
| 544 |
""" |
| 545 |
Given the OGRGeomType for a geometry and its associated GeometryField, |
| 546 |
determine whether the geometry should be turned into a GeometryCollection. |
| 547 |
""" |
| 548 |
return ( |
| 549 |
geom_type.num in self.MULTI_TYPES |
| 550 |
and model_field.__class__.__name__ == "Multi%s" % geom_type.django |
| 551 |
) |
| 552 |
|
| 553 |
def save( |
| 554 |
self, |
| 555 |
verbose=False, |
| 556 |
fid_range=False, |
| 557 |
step=False, |
| 558 |
progress=False, |
| 559 |
silent=False, |
| 560 |
stream=sys.stdout, |
| 561 |
strict=False, |
| 562 |
): |
| 563 |
""" |
| 564 |
Save the contents from the OGR DataSource Layer into the database |
| 565 |
according to the mapping dictionary given at initialization. |
| 566 |
|
| 567 |
Keyword Parameters: |
| 568 |
verbose: |
| 569 |
If set, information will be printed subsequent to each model save |
| 570 |
executed on the database. |
| 571 |
|
| 572 |
fid_range: |
| 573 |
May be set with a slice or tuple of (begin, end) feature ID's to map |
| 574 |
from the data source. In other words, this keyword enables the user |
| 575 |
to selectively import a subset range of features in the geographic |
| 576 |
data source. |
| 577 |
|
| 578 |
step: |
| 579 |
If set with an integer, transactions will occur at every step |
| 580 |
interval. For example, if step=1000, a commit would occur after |
| 581 |
the 1,000th feature, the 2,000th feature etc. |
| 582 |
|
| 583 |
progress: |
| 584 |
When this keyword is set, status information will be printed giving |
| 585 |
the number of features processed and successfully saved. By default, |
| 586 |
progress information will pe printed every 1000 features processed, |
| 587 |
however, this default may be overridden by setting this keyword with an |
| 588 |
integer for the desired interval. |
| 589 |
|
| 590 |
stream: |
| 591 |
Status information will be written to this file handle. Defaults to |
| 592 |
using `sys.stdout`, but any object with a `write` method is supported. |
| 593 |
|
| 594 |
silent: |
| 595 |
By default, non-fatal error notifications are printed to stdout, but |
| 596 |
this keyword may be set to disable these notifications. |
| 597 |
|
| 598 |
strict: |
| 599 |
Execution of the model mapping will cease upon the first error |
| 600 |
encountered. The default behavior is to attempt to continue. |
| 601 |
""" |
| 602 |
# Getting the default Feature ID range. |
| 603 |
default_range = self.check_fid_range(fid_range) |
| 604 |
|
| 605 |
# Setting the progress interval, if requested. |
| 606 |
if progress: |
| 607 |
if progress is True or not isinstance(progress, int): |
| 608 |
progress_interval = 1000 |
| 609 |
else: |
| 610 |
progress_interval = progress |
| 611 |
|
| 612 |
def _save(feat_range=default_range, num_feat=0, num_saved=0): |
| 613 |
if feat_range: |
| 614 |
layer_iter = self.layer[feat_range] |
| 615 |
else: |
| 616 |
layer_iter = self.layer |
| 617 |
|
| 618 |
for feat in layer_iter: |
| 619 |
num_feat += 1 |
| 620 |
# Getting the keyword arguments |
| 621 |
try: |
| 622 |
kwargs = self.feature_kwargs(feat) |
| 623 |
except LayerMapError as msg: |
| 624 |
# Something borked the validation |
| 625 |
if strict: |
| 626 |
raise |
| 627 |
elif not silent: |
| 628 |
stream.write( |
| 629 |
"Ignoring Feature ID %s because: %s\n" % (feat.fid, msg) |
| 630 |
) |
| 631 |
else: |
| 632 |
# Constructing the model using the keyword args |
| 633 |
is_update = False |
| 634 |
if self.unique: |
| 635 |
# If we want unique models on a particular field, handle the |
| 636 |
# geometry appropriately. |
| 637 |
try: |
| 638 |
# Getting the keyword arguments and retrieving |
| 639 |
# the unique model. |
| 640 |
u_kwargs = self.unique_kwargs(kwargs) |
| 641 |
m = self.model.objects.using(self.using).get(**u_kwargs) |
| 642 |
is_update = True |
| 643 |
|
| 644 |
# Getting the geometry (in OGR form), creating |
| 645 |
# one from the kwargs WKT, adding in additional |
| 646 |
# geometries, and update the attribute with the |
| 647 |
# just-updated geometry WKT. |
| 648 |
geom_value = getattr(m, self.geom_field) |
| 649 |
if geom_value is None: |
| 650 |
geom = OGRGeometry(kwargs[self.geom_field]) |
| 651 |
else: |
| 652 |
geom = geom_value.ogr |
| 653 |
new = OGRGeometry(kwargs[self.geom_field]) |
| 654 |
for g in new: |
| 655 |
geom.add(g) |
| 656 |
setattr(m, self.geom_field, geom.wkt) |
| 657 |
except ObjectDoesNotExist: |
| 658 |
# No unique model exists yet, create. |
| 659 |
m = self.model(**kwargs) |
| 660 |
else: |
| 661 |
m = self.model(**kwargs) |
| 662 |
|
| 663 |
try: |
| 664 |
# Attempting to save. |
| 665 |
m.save(using=self.using) |
| 666 |
num_saved += 1 |
| 667 |
if verbose: |
| 668 |
stream.write( |
| 669 |
"%s: %s\n" % ("Updated" if is_update else "Saved", m) |
| 670 |
) |
| 671 |
except Exception as msg: |
| 672 |
if strict: |
| 673 |
# Bailing out if the `strict` keyword is set. |
| 674 |
if not silent: |
| 675 |
stream.write( |
| 676 |
"Failed to save the feature (id: %s) into the " |
| 677 |
"model with the keyword arguments:\n" % feat.fid |
| 678 |
) |
| 679 |
stream.write("%s\n" % kwargs) |
| 680 |
raise |
| 681 |
elif not silent: |
| 682 |
stream.write( |
| 683 |
"Failed to save %s:\n %s\nContinuing\n" % (kwargs, msg) |
| 684 |
) |
| 685 |
|
| 686 |
# Printing progress information, if requested. |
| 687 |
if progress and num_feat % progress_interval == 0: |
| 688 |
stream.write( |
| 689 |
"Processed %d features, saved %d ...\n" % (num_feat, num_saved) |
| 690 |
) |
| 691 |
|
| 692 |
# Only used for status output purposes -- incremental saving uses the |
| 693 |
# values returned here. |
| 694 |
return num_saved, num_feat |
| 695 |
|
| 696 |
if self.transaction_decorator is not None: |
| 697 |
_save = self.transaction_decorator(_save) |
| 698 |
|
| 699 |
nfeat = self.layer.num_feat |
| 700 |
if step and isinstance(step, int) and step < nfeat: |
| 701 |
# Incremental saving is requested at the given interval (step) |
| 702 |
if default_range: |
| 703 |
raise LayerMapError( |
| 704 |
"The `step` keyword may not be used in conjunction with the " |
| 705 |
"`fid_range` keyword." |
| 706 |
) |
| 707 |
beg, num_feat, num_saved = (0, 0, 0) |
| 708 |
indices = range(step, nfeat, step) |
| 709 |
n_i = len(indices) |
| 710 |
|
| 711 |
for i, end in enumerate(indices): |
| 712 |
# Constructing the slice to use for this step; the last slice is |
| 713 |
# special (e.g, [100:] instead of [90:100]). |
| 714 |
if i + 1 == n_i: |
| 715 |
step_slice = slice(beg, None) |
| 716 |
else: |
| 717 |
step_slice = slice(beg, end) |
| 718 |
|
| 719 |
try: |
| 720 |
num_feat, num_saved = _save(step_slice, num_feat, num_saved) |
| 721 |
beg = end |
| 722 |
except Exception: # Deliberately catch everything |
| 723 |
stream.write( |
| 724 |
"%s\nFailed to save slice: %s\n" % ("=-" * 20, step_slice) |
| 725 |
) |
| 726 |
raise |
| 727 |
else: |
| 728 |
# Otherwise, just calling the previously defined _save() function. |
| 729 |
_save() |