| 1 |
import datetime |
| 2 |
import unittest |
| 3 |
from copy import copy |
| 4 |
from decimal import Decimal |
| 5 |
from pathlib import Path |
| 6 |
|
| 7 |
from django.conf import settings |
| 8 |
from django.contrib.gis.gdal import DataSource |
| 9 |
from django.contrib.gis.utils.layermapping import ( |
| 10 |
InvalidDecimal, |
| 11 |
InvalidString, |
| 12 |
LayerMapError, |
| 13 |
LayerMapping, |
| 14 |
MissingForeignKey, |
| 15 |
) |
| 16 |
from django.db import connection |
| 17 |
from django.test import TestCase, override_settings |
| 18 |
|
| 19 |
from .models import ( |
| 20 |
City, |
| 21 |
County, |
| 22 |
CountyFeat, |
| 23 |
DoesNotAllowNulls, |
| 24 |
HasNulls, |
| 25 |
ICity1, |
| 26 |
ICity2, |
| 27 |
Interstate, |
| 28 |
Invalid, |
| 29 |
State, |
| 30 |
city_mapping, |
| 31 |
co_mapping, |
| 32 |
cofeat_mapping, |
| 33 |
has_nulls_mapping, |
| 34 |
inter_mapping, |
| 35 |
) |
| 36 |
|
| 37 |
shp_path = Path(__file__).resolve().parent.parent / "data" |
| 38 |
city_shp = shp_path / "cities" / "cities.shp" |
| 39 |
co_shp = shp_path / "counties" / "counties.shp" |
| 40 |
inter_shp = shp_path / "interstates" / "interstates.shp" |
| 41 |
invalid_shp = shp_path / "invalid" / "emptypoints.shp" |
| 42 |
has_nulls_geojson = shp_path / "has_nulls" / "has_nulls.geojson" |
| 43 |
|
| 44 |
# Dictionaries to hold what's expected in the county shapefile. |
| 45 |
NAMES = ["Bexar", "Galveston", "Harris", "Honolulu", "Pueblo"] |
| 46 |
NUMS = [1, 2, 1, 19, 1] # Number of polygons for each. |
| 47 |
STATES = ["Texas", "Texas", "Texas", "Hawaii", "Colorado"] |
| 48 |
|
| 49 |
|
| 50 |
class LayerMapTest(TestCase): |
| 51 |
def test_init(self): |
| 52 |
"Testing LayerMapping initialization." |
| 53 |
|
| 54 |
# Model field that does not exist. |
| 55 |
bad1 = copy(city_mapping) |
| 56 |
bad1["foobar"] = "FooField" |
| 57 |
|
| 58 |
# Shapefile field that does not exist. |
| 59 |
bad2 = copy(city_mapping) |
| 60 |
bad2["name"] = "Nombre" |
| 61 |
|
| 62 |
# Nonexistent geographic field type. |
| 63 |
bad3 = copy(city_mapping) |
| 64 |
bad3["point"] = "CURVE" |
| 65 |
|
| 66 |
# Incrementing through the bad mapping dictionaries and |
| 67 |
# ensuring that a LayerMapError is raised. |
| 68 |
for bad_map in (bad1, bad2, bad3): |
| 69 |
with self.assertRaises(LayerMapError): |
| 70 |
LayerMapping(City, city_shp, bad_map) |
| 71 |
|
| 72 |
# A LookupError should be thrown for bogus encodings. |
| 73 |
with self.assertRaises(LookupError): |
| 74 |
LayerMapping(City, city_shp, city_mapping, encoding="foobar") |
| 75 |
|
| 76 |
def test_simple_layermap(self): |
| 77 |
"Test LayerMapping import of a simple point shapefile." |
| 78 |
# Setting up for the LayerMapping. |
| 79 |
lm = LayerMapping(City, city_shp, city_mapping) |
| 80 |
lm.save() |
| 81 |
|
| 82 |
# There should be three cities in the shape file. |
| 83 |
self.assertEqual(3, City.objects.count()) |
| 84 |
|
| 85 |
# Opening up the shapefile, and verifying the values in each |
| 86 |
# of the features made it to the model. |
| 87 |
ds = DataSource(city_shp) |
| 88 |
layer = ds[0] |
| 89 |
for feat in layer: |
| 90 |
city = City.objects.get(name=feat["Name"].value) |
| 91 |
self.assertEqual(feat["Population"].value, city.population) |
| 92 |
self.assertEqual(Decimal(str(feat["Density"])), city.density) |
| 93 |
self.assertEqual(feat["Created"].value, city.dt) |
| 94 |
|
| 95 |
# Comparing the geometries. |
| 96 |
pnt1, pnt2 = feat.geom, city.point |
| 97 |
self.assertAlmostEqual(pnt1.x, pnt2.x, 5) |
| 98 |
self.assertAlmostEqual(pnt1.y, pnt2.y, 5) |
| 99 |
|
| 100 |
def test_data_source_str(self): |
| 101 |
lm = LayerMapping(City, str(city_shp), city_mapping) |
| 102 |
lm.save() |
| 103 |
self.assertEqual(City.objects.count(), 3) |
| 104 |
|
| 105 |
def test_layermap_strict(self): |
| 106 |
"Testing the `strict` keyword, and import of a LineString shapefile." |
| 107 |
# When the `strict` keyword is set an error encountered will force |
| 108 |
# the importation to stop. |
| 109 |
with self.assertRaises(InvalidDecimal): |
| 110 |
lm = LayerMapping(Interstate, inter_shp, inter_mapping) |
| 111 |
lm.save(silent=True, strict=True) |
| 112 |
Interstate.objects.all().delete() |
| 113 |
|
| 114 |
# This LayerMapping should work b/c `strict` is not set. |
| 115 |
lm = LayerMapping(Interstate, inter_shp, inter_mapping) |
| 116 |
lm.save(silent=True) |
| 117 |
|
| 118 |
# Two interstate should have imported correctly. |
| 119 |
self.assertEqual(2, Interstate.objects.count()) |
| 120 |
|
| 121 |
# Verifying the values in the layer w/the model. |
| 122 |
ds = DataSource(inter_shp) |
| 123 |
|
| 124 |
# Only the first two features of this shapefile are valid. |
| 125 |
valid_feats = ds[0][:2] |
| 126 |
for feat in valid_feats: |
| 127 |
istate = Interstate.objects.get(name=feat["Name"].value) |
| 128 |
|
| 129 |
if feat.fid == 0: |
| 130 |
self.assertEqual(Decimal(str(feat["Length"])), istate.length) |
| 131 |
elif feat.fid == 1: |
| 132 |
# Everything but the first two decimal digits were truncated, |
| 133 |
# because the Interstate model's `length` field has decimal_places=2. |
| 134 |
self.assertAlmostEqual(feat.get("Length"), float(istate.length), 2) |
| 135 |
|
| 136 |
for p1, p2 in zip(feat.geom, istate.path): |
| 137 |
self.assertAlmostEqual(p1[0], p2[0], 6) |
| 138 |
self.assertAlmostEqual(p1[1], p2[1], 6) |
| 139 |
|
| 140 |
def county_helper(self, county_feat=True): |
| 141 |
"Helper function for ensuring the integrity of the mapped County models." |
| 142 |
for name, n, st in zip(NAMES, NUMS, STATES): |
| 143 |
# Should only be one record b/c of `unique` keyword. |
| 144 |
c = County.objects.get(name=name) |
| 145 |
self.assertEqual(n, len(c.mpoly)) |
| 146 |
self.assertEqual(st, c.state.name) # Checking ForeignKey mapping. |
| 147 |
|
| 148 |
# Multiple records because `unique` was not set. |
| 149 |
if county_feat: |
| 150 |
qs = CountyFeat.objects.filter(name=name) |
| 151 |
self.assertEqual(n, qs.count()) |
| 152 |
|
| 153 |
def test_layermap_unique_multigeometry_fk(self): |
| 154 |
""" |
| 155 |
The `unique`, and `transform`, geometry collection conversion, and |
| 156 |
ForeignKey mappings. |
| 157 |
""" |
| 158 |
# All the following should work. |
| 159 |
|
| 160 |
# Telling LayerMapping that we want no transformations performed on the data. |
| 161 |
lm = LayerMapping(County, co_shp, co_mapping, transform=False) |
| 162 |
|
| 163 |
# Specifying the source spatial reference system via the `source_srs` keyword. |
| 164 |
lm = LayerMapping(County, co_shp, co_mapping, source_srs=4269) |
| 165 |
lm = LayerMapping(County, co_shp, co_mapping, source_srs="NAD83") |
| 166 |
|
| 167 |
# Unique may take tuple or string parameters. |
| 168 |
for arg in ("name", ("name", "mpoly")): |
| 169 |
lm = LayerMapping(County, co_shp, co_mapping, transform=False, unique=arg) |
| 170 |
|
| 171 |
# Now test for failures |
| 172 |
|
| 173 |
# Testing invalid params for the `unique` keyword. |
| 174 |
for e, arg in ( |
| 175 |
(TypeError, 5.0), |
| 176 |
(ValueError, "foobar"), |
| 177 |
(ValueError, ("name", "mpolygon")), |
| 178 |
): |
| 179 |
with self.assertRaises(e): |
| 180 |
LayerMapping(County, co_shp, co_mapping, transform=False, unique=arg) |
| 181 |
|
| 182 |
# No source reference system defined in the shapefile, should raise an error. |
| 183 |
if connection.features.supports_transform: |
| 184 |
with self.assertRaises(LayerMapError): |
| 185 |
LayerMapping(County, co_shp, co_mapping) |
| 186 |
|
| 187 |
# Passing in invalid ForeignKey mapping parameters -- must be a dictionary |
| 188 |
# mapping for the model the ForeignKey points to. |
| 189 |
bad_fk_map1 = copy(co_mapping) |
| 190 |
bad_fk_map1["state"] = "name" |
| 191 |
bad_fk_map2 = copy(co_mapping) |
| 192 |
bad_fk_map2["state"] = {"nombre": "State"} |
| 193 |
with self.assertRaises(TypeError): |
| 194 |
LayerMapping(County, co_shp, bad_fk_map1, transform=False) |
| 195 |
with self.assertRaises(LayerMapError): |
| 196 |
LayerMapping(County, co_shp, bad_fk_map2, transform=False) |
| 197 |
|
| 198 |
# There exist no State models for the ForeignKey mapping to work -- should raise |
| 199 |
# a MissingForeignKey exception (this error would be ignored if the `strict` |
| 200 |
# keyword is not set). |
| 201 |
lm = LayerMapping(County, co_shp, co_mapping, transform=False, unique="name") |
| 202 |
with self.assertRaises(MissingForeignKey): |
| 203 |
lm.save(silent=True, strict=True) |
| 204 |
|
| 205 |
# Now creating the state models so the ForeignKey mapping may work. |
| 206 |
State.objects.bulk_create( |
| 207 |
[State(name="Colorado"), State(name="Hawaii"), State(name="Texas")] |
| 208 |
) |
| 209 |
|
| 210 |
# If a mapping is specified as a collection, all OGR fields that |
| 211 |
# are not collections will be converted into them. For example, |
| 212 |
# a Point column would be converted to MultiPoint. Other things being done |
| 213 |
# w/the keyword args: |
| 214 |
# `transform=False`: Specifies that no transform is to be done; this |
| 215 |
# has the effect of ignoring the spatial reference check (because the |
| 216 |
# county shapefile does not have implicit spatial reference info). |
| 217 |
# |
| 218 |
# `unique='name'`: Creates models on the condition that they have |
| 219 |
# unique county names; geometries from each feature however will be |
| 220 |
# appended to the geometry collection of the unique model. Thus, |
| 221 |
# all of the various islands in Honolulu county will be in one |
| 222 |
# database record with a MULTIPOLYGON type. |
| 223 |
lm = LayerMapping(County, co_shp, co_mapping, transform=False, unique="name") |
| 224 |
lm.save(silent=True, strict=True) |
| 225 |
|
| 226 |
# A reference that doesn't use the unique keyword; a new database record will |
| 227 |
# created for each polygon. |
| 228 |
lm = LayerMapping(CountyFeat, co_shp, cofeat_mapping, transform=False) |
| 229 |
lm.save(silent=True, strict=True) |
| 230 |
|
| 231 |
# The county helper is called to ensure integrity of County models. |
| 232 |
self.county_helper() |
| 233 |
|
| 234 |
def test_test_fid_range_step(self): |
| 235 |
"Tests the `fid_range` keyword and the `step` keyword of .save()." |
| 236 |
|
| 237 |
# Function for clearing out all the counties before testing. |
| 238 |
def clear_counties(): |
| 239 |
County.objects.all().delete() |
| 240 |
|
| 241 |
State.objects.bulk_create( |
| 242 |
[State(name="Colorado"), State(name="Hawaii"), State(name="Texas")] |
| 243 |
) |
| 244 |
|
| 245 |
# Initializing the LayerMapping object to use in these tests. |
| 246 |
lm = LayerMapping(County, co_shp, co_mapping, transform=False, unique="name") |
| 247 |
|
| 248 |
# Bad feature id ranges should raise a type error. |
| 249 |
bad_ranges = (5.0, "foo", co_shp) |
| 250 |
for bad in bad_ranges: |
| 251 |
with self.assertRaises(TypeError): |
| 252 |
lm.save(fid_range=bad) |
| 253 |
|
| 254 |
# Step keyword should not be allowed w/`fid_range`. |
| 255 |
fr = (3, 5) # layer[3:5] |
| 256 |
with self.assertRaises(LayerMapError): |
| 257 |
lm.save(fid_range=fr, step=10) |
| 258 |
lm.save(fid_range=fr) |
| 259 |
|
| 260 |
# Features IDs 3 & 4 are for Galveston County, Texas -- only |
| 261 |
# one model is returned because the `unique` keyword was set. |
| 262 |
qs = County.objects.all() |
| 263 |
self.assertEqual(1, qs.count()) |
| 264 |
self.assertEqual("Galveston", qs[0].name) |
| 265 |
|
| 266 |
# Features IDs 5 and beyond for Honolulu County, Hawaii, and |
| 267 |
# FID 0 is for Pueblo County, Colorado. |
| 268 |
clear_counties() |
| 269 |
lm.save(fid_range=slice(5, None), silent=True, strict=True) # layer[5:] |
| 270 |
lm.save(fid_range=slice(None, 1), silent=True, strict=True) # layer[:1] |
| 271 |
|
| 272 |
# Only Pueblo & Honolulu counties should be present because of |
| 273 |
# the `unique` keyword. Have to set `order_by` on this QuerySet |
| 274 |
# or else MySQL will return a different ordering than the other dbs. |
| 275 |
qs = County.objects.order_by("name") |
| 276 |
self.assertEqual(2, qs.count()) |
| 277 |
hi, co = tuple(qs) |
| 278 |
hi_idx, co_idx = tuple(map(NAMES.index, ("Honolulu", "Pueblo"))) |
| 279 |
self.assertEqual("Pueblo", co.name) |
| 280 |
self.assertEqual(NUMS[co_idx], len(co.mpoly)) |
| 281 |
self.assertEqual("Honolulu", hi.name) |
| 282 |
self.assertEqual(NUMS[hi_idx], len(hi.mpoly)) |
| 283 |
|
| 284 |
# Testing the `step` keyword -- should get the same counties |
| 285 |
# regardless of we use a step that divides equally, that is odd, |
| 286 |
# or that is larger than the dataset. |
| 287 |
for st in (4, 7, 1000): |
| 288 |
clear_counties() |
| 289 |
lm.save(step=st, strict=True) |
| 290 |
self.county_helper(county_feat=False) |
| 291 |
|
| 292 |
def test_model_inheritance(self): |
| 293 |
"Tests LayerMapping on inherited models. See #12093." |
| 294 |
icity_mapping = { |
| 295 |
"name": "Name", |
| 296 |
"population": "Population", |
| 297 |
"density": "Density", |
| 298 |
"point": "POINT", |
| 299 |
"dt": "Created", |
| 300 |
} |
| 301 |
# Parent model has geometry field. |
| 302 |
lm1 = LayerMapping(ICity1, city_shp, icity_mapping) |
| 303 |
lm1.save() |
| 304 |
|
| 305 |
# Grandparent has geometry field. |
| 306 |
lm2 = LayerMapping(ICity2, city_shp, icity_mapping) |
| 307 |
lm2.save() |
| 308 |
|
| 309 |
self.assertEqual(6, ICity1.objects.count()) |
| 310 |
self.assertEqual(3, ICity2.objects.count()) |
| 311 |
|
| 312 |
def test_invalid_layer(self): |
| 313 |
"Tests LayerMapping on invalid geometries. See #15378." |
| 314 |
invalid_mapping = {"point": "POINT"} |
| 315 |
lm = LayerMapping(Invalid, invalid_shp, invalid_mapping, source_srs=4326) |
| 316 |
lm.save(silent=True) |
| 317 |
|
| 318 |
def test_charfield_too_short(self): |
| 319 |
mapping = copy(city_mapping) |
| 320 |
mapping["name_short"] = "Name" |
| 321 |
lm = LayerMapping(City, city_shp, mapping) |
| 322 |
with self.assertRaises(InvalidString): |
| 323 |
lm.save(silent=True, strict=True) |
| 324 |
|
| 325 |
def test_textfield(self): |
| 326 |
"String content fits also in a TextField" |
| 327 |
mapping = copy(city_mapping) |
| 328 |
mapping["name_txt"] = "Name" |
| 329 |
lm = LayerMapping(City, city_shp, mapping) |
| 330 |
lm.save(silent=True, strict=True) |
| 331 |
self.assertEqual(City.objects.count(), 3) |
| 332 |
self.assertEqual(City.objects.get(name="Houston").name_txt, "Houston") |
| 333 |
|
| 334 |
def test_encoded_name(self): |
| 335 |
"""Test a layer containing utf-8-encoded name""" |
| 336 |
city_shp = shp_path / "ch-city" / "ch-city.shp" |
| 337 |
lm = LayerMapping(City, city_shp, city_mapping) |
| 338 |
lm.save(silent=True, strict=True) |
| 339 |
self.assertEqual(City.objects.count(), 1) |
| 340 |
self.assertEqual(City.objects.all()[0].name, "Zürich") |
| 341 |
|
| 342 |
def test_null_geom_with_unique(self): |
| 343 |
"""LayerMapping may be created with a unique and a null geometry.""" |
| 344 |
State.objects.bulk_create( |
| 345 |
[State(name="Colorado"), State(name="Hawaii"), State(name="Texas")] |
| 346 |
) |
| 347 |
hw = State.objects.get(name="Hawaii") |
| 348 |
hu = County.objects.create(name="Honolulu", state=hw, mpoly=None) |
| 349 |
lm = LayerMapping(County, co_shp, co_mapping, transform=False, unique="name") |
| 350 |
lm.save(silent=True, strict=True) |
| 351 |
hu.refresh_from_db() |
| 352 |
self.assertIsNotNone(hu.mpoly) |
| 353 |
self.assertEqual(hu.mpoly.ogr.num_coords, 449) |
| 354 |
|
| 355 |
def test_null_number_imported(self): |
| 356 |
"""LayerMapping import of GeoJSON with a null numeric value.""" |
| 357 |
lm = LayerMapping(HasNulls, has_nulls_geojson, has_nulls_mapping) |
| 358 |
lm.save() |
| 359 |
self.assertEqual(HasNulls.objects.count(), 3) |
| 360 |
self.assertEqual(HasNulls.objects.filter(num=0).count(), 1) |
| 361 |
self.assertEqual(HasNulls.objects.filter(num__isnull=True).count(), 1) |
| 362 |
|
| 363 |
def test_null_string_imported(self): |
| 364 |
"Test LayerMapping import of GeoJSON with a null string value." |
| 365 |
lm = LayerMapping(HasNulls, has_nulls_geojson, has_nulls_mapping) |
| 366 |
lm.save() |
| 367 |
self.assertEqual(HasNulls.objects.filter(name="None").count(), 0) |
| 368 |
num_empty = 1 if connection.features.interprets_empty_strings_as_nulls else 0 |
| 369 |
self.assertEqual(HasNulls.objects.filter(name="").count(), num_empty) |
| 370 |
self.assertEqual(HasNulls.objects.filter(name__isnull=True).count(), 1) |
| 371 |
|
| 372 |
def test_nullable_boolean_imported(self): |
| 373 |
"""LayerMapping import of GeoJSON with a nullable boolean value.""" |
| 374 |
lm = LayerMapping(HasNulls, has_nulls_geojson, has_nulls_mapping) |
| 375 |
lm.save() |
| 376 |
self.assertEqual(HasNulls.objects.filter(boolean=True).count(), 1) |
| 377 |
self.assertEqual(HasNulls.objects.filter(boolean=False).count(), 1) |
| 378 |
self.assertEqual(HasNulls.objects.filter(boolean__isnull=True).count(), 1) |
| 379 |
|
| 380 |
def test_nullable_datetime_imported(self): |
| 381 |
"""LayerMapping import of GeoJSON with a nullable date/time value.""" |
| 382 |
lm = LayerMapping(HasNulls, has_nulls_geojson, has_nulls_mapping) |
| 383 |
lm.save() |
| 384 |
self.assertEqual( |
| 385 |
HasNulls.objects.filter(datetime__lt=datetime.date(1994, 8, 15)).count(), 1 |
| 386 |
) |
| 387 |
self.assertEqual( |
| 388 |
HasNulls.objects.filter(datetime="2018-11-29T03:02:52").count(), 1 |
| 389 |
) |
| 390 |
self.assertEqual(HasNulls.objects.filter(datetime__isnull=True).count(), 1) |
| 391 |
|
| 392 |
def test_uuids_imported(self): |
| 393 |
"""LayerMapping import of GeoJSON with UUIDs.""" |
| 394 |
lm = LayerMapping(HasNulls, has_nulls_geojson, has_nulls_mapping) |
| 395 |
lm.save() |
| 396 |
self.assertEqual( |
| 397 |
HasNulls.objects.filter( |
| 398 |
uuid="1378c26f-cbe6-44b0-929f-eb330d4991f5" |
| 399 |
).count(), |
| 400 |
1, |
| 401 |
) |
| 402 |
|
| 403 |
def test_null_number_imported_not_allowed(self): |
| 404 |
""" |
| 405 |
LayerMapping import of GeoJSON with nulls to fields that don't permit |
| 406 |
them. |
| 407 |
""" |
| 408 |
lm = LayerMapping(DoesNotAllowNulls, has_nulls_geojson, has_nulls_mapping) |
| 409 |
lm.save(silent=True) |
| 410 |
# When a model fails to save due to IntegrityError (null in non-null |
| 411 |
# column), subsequent saves fail with "An error occurred in the current |
| 412 |
# transaction. You can't execute queries until the end of the 'atomic' |
| 413 |
# block." On Oracle and MySQL, the one object that did load appears in |
| 414 |
# this count. On other databases, no records appear. |
| 415 |
self.assertLessEqual(DoesNotAllowNulls.objects.count(), 1) |
| 416 |
|
| 417 |
|
| 418 |
class OtherRouter: |
| 419 |
def db_for_read(self, model, **hints): |
| 420 |
return "other" |
| 421 |
|
| 422 |
def db_for_write(self, model, **hints): |
| 423 |
return self.db_for_read(model, **hints) |
| 424 |
|
| 425 |
def allow_relation(self, obj1, obj2, **hints): |
| 426 |
# ContentType objects are created during a post-migrate signal while |
| 427 |
# performing fixture teardown using the default database alias and |
| 428 |
# don't abide by the database specified by this router. |
| 429 |
return True |
| 430 |
|
| 431 |
def allow_migrate(self, db, app_label, **hints): |
| 432 |
return True |
| 433 |
|
| 434 |
|
| 435 |
@override_settings(DATABASE_ROUTERS=[OtherRouter()]) |
| 436 |
class LayerMapRouterTest(TestCase): |
| 437 |
databases = {"default", "other"} |
| 438 |
|
| 439 |
@unittest.skipUnless(len(settings.DATABASES) > 1, "multiple databases required") |
| 440 |
def test_layermapping_default_db(self): |
| 441 |
lm = LayerMapping(City, city_shp, city_mapping) |
| 442 |
self.assertEqual(lm.using, "other") |