Debian Codegraph @

tests/gis_tests/layermap/tests.py

raw
1 import datetime
2 import unittest
3 from copy import copy
4 from decimal import Decimal
5 from pathlib import Path
6
7 from django.conf import settings
8 from django.contrib.gis.gdal import DataSource
9 from django.contrib.gis.utils.layermapping import (
10 InvalidDecimal,
11 InvalidString,
12 LayerMapError,
13 LayerMapping,
14 MissingForeignKey,
15 )
16 from django.db import connection
17 from django.test import TestCase, override_settings
18
19 from .models import (
20 City,
21 County,
22 CountyFeat,
23 DoesNotAllowNulls,
24 HasNulls,
25 ICity1,
26 ICity2,
27 Interstate,
28 Invalid,
29 State,
30 city_mapping,
31 co_mapping,
32 cofeat_mapping,
33 has_nulls_mapping,
34 inter_mapping,
35 )
36
37 shp_path = Path(__file__).resolve().parent.parent / "data"
38 city_shp = shp_path / "cities" / "cities.shp"
39 co_shp = shp_path / "counties" / "counties.shp"
40 inter_shp = shp_path / "interstates" / "interstates.shp"
41 invalid_shp = shp_path / "invalid" / "emptypoints.shp"
42 has_nulls_geojson = shp_path / "has_nulls" / "has_nulls.geojson"
43
44 # Dictionaries to hold what's expected in the county shapefile.
45 NAMES = ["Bexar", "Galveston", "Harris", "Honolulu", "Pueblo"]
46 NUMS = [1, 2, 1, 19, 1] # Number of polygons for each.
47 STATES = ["Texas", "Texas", "Texas", "Hawaii", "Colorado"]
48
49
50 class LayerMapTest(TestCase):
51 def test_init(self):
52 "Testing LayerMapping initialization."
53
54 # Model field that does not exist.
55 bad1 = copy(city_mapping)
56 bad1["foobar"] = "FooField"
57
58 # Shapefile field that does not exist.
59 bad2 = copy(city_mapping)
60 bad2["name"] = "Nombre"
61
62 # Nonexistent geographic field type.
63 bad3 = copy(city_mapping)
64 bad3["point"] = "CURVE"
65
66 # Incrementing through the bad mapping dictionaries and
67 # ensuring that a LayerMapError is raised.
68 for bad_map in (bad1, bad2, bad3):
69 with self.assertRaises(LayerMapError):
70 LayerMapping(City, city_shp, bad_map)
71
72 # A LookupError should be thrown for bogus encodings.
73 with self.assertRaises(LookupError):
74 LayerMapping(City, city_shp, city_mapping, encoding="foobar")
75
76 def test_simple_layermap(self):
77 "Test LayerMapping import of a simple point shapefile."
78 # Setting up for the LayerMapping.
79 lm = LayerMapping(City, city_shp, city_mapping)
80 lm.save()
81
82 # There should be three cities in the shape file.
83 self.assertEqual(3, City.objects.count())
84
85 # Opening up the shapefile, and verifying the values in each
86 # of the features made it to the model.
87 ds = DataSource(city_shp)
88 layer = ds[0]
89 for feat in layer:
90 city = City.objects.get(name=feat["Name"].value)
91 self.assertEqual(feat["Population"].value, city.population)
92 self.assertEqual(Decimal(str(feat["Density"])), city.density)
93 self.assertEqual(feat["Created"].value, city.dt)
94
95 # Comparing the geometries.
96 pnt1, pnt2 = feat.geom, city.point
97 self.assertAlmostEqual(pnt1.x, pnt2.x, 5)
98 self.assertAlmostEqual(pnt1.y, pnt2.y, 5)
99
100 def test_data_source_str(self):
101 lm = LayerMapping(City, str(city_shp), city_mapping)
102 lm.save()
103 self.assertEqual(City.objects.count(), 3)
104
105 def test_layermap_strict(self):
106 "Testing the `strict` keyword, and import of a LineString shapefile."
107 # When the `strict` keyword is set an error encountered will force
108 # the importation to stop.
109 with self.assertRaises(InvalidDecimal):
110 lm = LayerMapping(Interstate, inter_shp, inter_mapping)
111 lm.save(silent=True, strict=True)
112 Interstate.objects.all().delete()
113
114 # This LayerMapping should work b/c `strict` is not set.
115 lm = LayerMapping(Interstate, inter_shp, inter_mapping)
116 lm.save(silent=True)
117
118 # Two interstate should have imported correctly.
119 self.assertEqual(2, Interstate.objects.count())
120
121 # Verifying the values in the layer w/the model.
122 ds = DataSource(inter_shp)
123
124 # Only the first two features of this shapefile are valid.
125 valid_feats = ds[0][:2]
126 for feat in valid_feats:
127 istate = Interstate.objects.get(name=feat["Name"].value)
128
129 if feat.fid == 0:
130 self.assertEqual(Decimal(str(feat["Length"])), istate.length)
131 elif feat.fid == 1:
132 # Everything but the first two decimal digits were truncated,
133 # because the Interstate model's `length` field has decimal_places=2.
134 self.assertAlmostEqual(feat.get("Length"), float(istate.length), 2)
135
136 for p1, p2 in zip(feat.geom, istate.path):
137 self.assertAlmostEqual(p1[0], p2[0], 6)
138 self.assertAlmostEqual(p1[1], p2[1], 6)
139
140 def county_helper(self, county_feat=True):
141 "Helper function for ensuring the integrity of the mapped County models."
142 for name, n, st in zip(NAMES, NUMS, STATES):
143 # Should only be one record b/c of `unique` keyword.
144 c = County.objects.get(name=name)
145 self.assertEqual(n, len(c.mpoly))
146 self.assertEqual(st, c.state.name) # Checking ForeignKey mapping.
147
148 # Multiple records because `unique` was not set.
149 if county_feat:
150 qs = CountyFeat.objects.filter(name=name)
151 self.assertEqual(n, qs.count())
152
153 def test_layermap_unique_multigeometry_fk(self):
154 """
155 The `unique`, and `transform`, geometry collection conversion, and
156 ForeignKey mappings.
157 """
158 # All the following should work.
159
160 # Telling LayerMapping that we want no transformations performed on the data.
161 lm = LayerMapping(County, co_shp, co_mapping, transform=False)
162
163 # Specifying the source spatial reference system via the `source_srs` keyword.
164 lm = LayerMapping(County, co_shp, co_mapping, source_srs=4269)
165 lm = LayerMapping(County, co_shp, co_mapping, source_srs="NAD83")
166
167 # Unique may take tuple or string parameters.
168 for arg in ("name", ("name", "mpoly")):
169 lm = LayerMapping(County, co_shp, co_mapping, transform=False, unique=arg)
170
171 # Now test for failures
172
173 # Testing invalid params for the `unique` keyword.
174 for e, arg in (
175 (TypeError, 5.0),
176 (ValueError, "foobar"),
177 (ValueError, ("name", "mpolygon")),
178 ):
179 with self.assertRaises(e):
180 LayerMapping(County, co_shp, co_mapping, transform=False, unique=arg)
181
182 # No source reference system defined in the shapefile, should raise an error.
183 if connection.features.supports_transform:
184 with self.assertRaises(LayerMapError):
185 LayerMapping(County, co_shp, co_mapping)
186
187 # Passing in invalid ForeignKey mapping parameters -- must be a dictionary
188 # mapping for the model the ForeignKey points to.
189 bad_fk_map1 = copy(co_mapping)
190 bad_fk_map1["state"] = "name"
191 bad_fk_map2 = copy(co_mapping)
192 bad_fk_map2["state"] = {"nombre": "State"}
193 with self.assertRaises(TypeError):
194 LayerMapping(County, co_shp, bad_fk_map1, transform=False)
195 with self.assertRaises(LayerMapError):
196 LayerMapping(County, co_shp, bad_fk_map2, transform=False)
197
198 # There exist no State models for the ForeignKey mapping to work -- should raise
199 # a MissingForeignKey exception (this error would be ignored if the `strict`
200 # keyword is not set).
201 lm = LayerMapping(County, co_shp, co_mapping, transform=False, unique="name")
202 with self.assertRaises(MissingForeignKey):
203 lm.save(silent=True, strict=True)
204
205 # Now creating the state models so the ForeignKey mapping may work.
206 State.objects.bulk_create(
207 [State(name="Colorado"), State(name="Hawaii"), State(name="Texas")]
208 )
209
210 # If a mapping is specified as a collection, all OGR fields that
211 # are not collections will be converted into them. For example,
212 # a Point column would be converted to MultiPoint. Other things being done
213 # w/the keyword args:
214 # `transform=False`: Specifies that no transform is to be done; this
215 # has the effect of ignoring the spatial reference check (because the
216 # county shapefile does not have implicit spatial reference info).
217 #
218 # `unique='name'`: Creates models on the condition that they have
219 # unique county names; geometries from each feature however will be
220 # appended to the geometry collection of the unique model. Thus,
221 # all of the various islands in Honolulu county will be in one
222 # database record with a MULTIPOLYGON type.
223 lm = LayerMapping(County, co_shp, co_mapping, transform=False, unique="name")
224 lm.save(silent=True, strict=True)
225
226 # A reference that doesn't use the unique keyword; a new database record will
227 # created for each polygon.
228 lm = LayerMapping(CountyFeat, co_shp, cofeat_mapping, transform=False)
229 lm.save(silent=True, strict=True)
230
231 # The county helper is called to ensure integrity of County models.
232 self.county_helper()
233
234 def test_test_fid_range_step(self):
235 "Tests the `fid_range` keyword and the `step` keyword of .save()."
236
237 # Function for clearing out all the counties before testing.
238 def clear_counties():
239 County.objects.all().delete()
240
241 State.objects.bulk_create(
242 [State(name="Colorado"), State(name="Hawaii"), State(name="Texas")]
243 )
244
245 # Initializing the LayerMapping object to use in these tests.
246 lm = LayerMapping(County, co_shp, co_mapping, transform=False, unique="name")
247
248 # Bad feature id ranges should raise a type error.
249 bad_ranges = (5.0, "foo", co_shp)
250 for bad in bad_ranges:
251 with self.assertRaises(TypeError):
252 lm.save(fid_range=bad)
253
254 # Step keyword should not be allowed w/`fid_range`.
255 fr = (3, 5) # layer[3:5]
256 with self.assertRaises(LayerMapError):
257 lm.save(fid_range=fr, step=10)
258 lm.save(fid_range=fr)
259
260 # Features IDs 3 & 4 are for Galveston County, Texas -- only
261 # one model is returned because the `unique` keyword was set.
262 qs = County.objects.all()
263 self.assertEqual(1, qs.count())
264 self.assertEqual("Galveston", qs[0].name)
265
266 # Features IDs 5 and beyond for Honolulu County, Hawaii, and
267 # FID 0 is for Pueblo County, Colorado.
268 clear_counties()
269 lm.save(fid_range=slice(5, None), silent=True, strict=True) # layer[5:]
270 lm.save(fid_range=slice(None, 1), silent=True, strict=True) # layer[:1]
271
272 # Only Pueblo & Honolulu counties should be present because of
273 # the `unique` keyword. Have to set `order_by` on this QuerySet
274 # or else MySQL will return a different ordering than the other dbs.
275 qs = County.objects.order_by("name")
276 self.assertEqual(2, qs.count())
277 hi, co = tuple(qs)
278 hi_idx, co_idx = tuple(map(NAMES.index, ("Honolulu", "Pueblo")))
279 self.assertEqual("Pueblo", co.name)
280 self.assertEqual(NUMS[co_idx], len(co.mpoly))
281 self.assertEqual("Honolulu", hi.name)
282 self.assertEqual(NUMS[hi_idx], len(hi.mpoly))
283
284 # Testing the `step` keyword -- should get the same counties
285 # regardless of we use a step that divides equally, that is odd,
286 # or that is larger than the dataset.
287 for st in (4, 7, 1000):
288 clear_counties()
289 lm.save(step=st, strict=True)
290 self.county_helper(county_feat=False)
291
292 def test_model_inheritance(self):
293 "Tests LayerMapping on inherited models. See #12093."
294 icity_mapping = {
295 "name": "Name",
296 "population": "Population",
297 "density": "Density",
298 "point": "POINT",
299 "dt": "Created",
300 }
301 # Parent model has geometry field.
302 lm1 = LayerMapping(ICity1, city_shp, icity_mapping)
303 lm1.save()
304
305 # Grandparent has geometry field.
306 lm2 = LayerMapping(ICity2, city_shp, icity_mapping)
307 lm2.save()
308
309 self.assertEqual(6, ICity1.objects.count())
310 self.assertEqual(3, ICity2.objects.count())
311
312 def test_invalid_layer(self):
313 "Tests LayerMapping on invalid geometries. See #15378."
314 invalid_mapping = {"point": "POINT"}
315 lm = LayerMapping(Invalid, invalid_shp, invalid_mapping, source_srs=4326)
316 lm.save(silent=True)
317
318 def test_charfield_too_short(self):
319 mapping = copy(city_mapping)
320 mapping["name_short"] = "Name"
321 lm = LayerMapping(City, city_shp, mapping)
322 with self.assertRaises(InvalidString):
323 lm.save(silent=True, strict=True)
324
325 def test_textfield(self):
326 "String content fits also in a TextField"
327 mapping = copy(city_mapping)
328 mapping["name_txt"] = "Name"
329 lm = LayerMapping(City, city_shp, mapping)
330 lm.save(silent=True, strict=True)
331 self.assertEqual(City.objects.count(), 3)
332 self.assertEqual(City.objects.get(name="Houston").name_txt, "Houston")
333
334 def test_encoded_name(self):
335 """Test a layer containing utf-8-encoded name"""
336 city_shp = shp_path / "ch-city" / "ch-city.shp"
337 lm = LayerMapping(City, city_shp, city_mapping)
338 lm.save(silent=True, strict=True)
339 self.assertEqual(City.objects.count(), 1)
340 self.assertEqual(City.objects.all()[0].name, "Zürich")
341
342 def test_null_geom_with_unique(self):
343 """LayerMapping may be created with a unique and a null geometry."""
344 State.objects.bulk_create(
345 [State(name="Colorado"), State(name="Hawaii"), State(name="Texas")]
346 )
347 hw = State.objects.get(name="Hawaii")
348 hu = County.objects.create(name="Honolulu", state=hw, mpoly=None)
349 lm = LayerMapping(County, co_shp, co_mapping, transform=False, unique="name")
350 lm.save(silent=True, strict=True)
351 hu.refresh_from_db()
352 self.assertIsNotNone(hu.mpoly)
353 self.assertEqual(hu.mpoly.ogr.num_coords, 449)
354
355 def test_null_number_imported(self):
356 """LayerMapping import of GeoJSON with a null numeric value."""
357 lm = LayerMapping(HasNulls, has_nulls_geojson, has_nulls_mapping)
358 lm.save()
359 self.assertEqual(HasNulls.objects.count(), 3)
360 self.assertEqual(HasNulls.objects.filter(num=0).count(), 1)
361 self.assertEqual(HasNulls.objects.filter(num__isnull=True).count(), 1)
362
363 def test_null_string_imported(self):
364 "Test LayerMapping import of GeoJSON with a null string value."
365 lm = LayerMapping(HasNulls, has_nulls_geojson, has_nulls_mapping)
366 lm.save()
367 self.assertEqual(HasNulls.objects.filter(name="None").count(), 0)
368 num_empty = 1 if connection.features.interprets_empty_strings_as_nulls else 0
369 self.assertEqual(HasNulls.objects.filter(name="").count(), num_empty)
370 self.assertEqual(HasNulls.objects.filter(name__isnull=True).count(), 1)
371
372 def test_nullable_boolean_imported(self):
373 """LayerMapping import of GeoJSON with a nullable boolean value."""
374 lm = LayerMapping(HasNulls, has_nulls_geojson, has_nulls_mapping)
375 lm.save()
376 self.assertEqual(HasNulls.objects.filter(boolean=True).count(), 1)
377 self.assertEqual(HasNulls.objects.filter(boolean=False).count(), 1)
378 self.assertEqual(HasNulls.objects.filter(boolean__isnull=True).count(), 1)
379
380 def test_nullable_datetime_imported(self):
381 """LayerMapping import of GeoJSON with a nullable date/time value."""
382 lm = LayerMapping(HasNulls, has_nulls_geojson, has_nulls_mapping)
383 lm.save()
384 self.assertEqual(
385 HasNulls.objects.filter(datetime__lt=datetime.date(1994, 8, 15)).count(), 1
386 )
387 self.assertEqual(
388 HasNulls.objects.filter(datetime="2018-11-29T03:02:52").count(), 1
389 )
390 self.assertEqual(HasNulls.objects.filter(datetime__isnull=True).count(), 1)
391
392 def test_uuids_imported(self):
393 """LayerMapping import of GeoJSON with UUIDs."""
394 lm = LayerMapping(HasNulls, has_nulls_geojson, has_nulls_mapping)
395 lm.save()
396 self.assertEqual(
397 HasNulls.objects.filter(
398 uuid="1378c26f-cbe6-44b0-929f-eb330d4991f5"
399 ).count(),
400 1,
401 )
402
403 def test_null_number_imported_not_allowed(self):
404 """
405 LayerMapping import of GeoJSON with nulls to fields that don't permit
406 them.
407 """
408 lm = LayerMapping(DoesNotAllowNulls, has_nulls_geojson, has_nulls_mapping)
409 lm.save(silent=True)
410 # When a model fails to save due to IntegrityError (null in non-null
411 # column), subsequent saves fail with "An error occurred in the current
412 # transaction. You can't execute queries until the end of the 'atomic'
413 # block." On Oracle and MySQL, the one object that did load appears in
414 # this count. On other databases, no records appear.
415 self.assertLessEqual(DoesNotAllowNulls.objects.count(), 1)
416
417
418 class OtherRouter:
419 def db_for_read(self, model, **hints):
420 return "other"
421
422 def db_for_write(self, model, **hints):
423 return self.db_for_read(model, **hints)
424
425 def allow_relation(self, obj1, obj2, **hints):
426 # ContentType objects are created during a post-migrate signal while
427 # performing fixture teardown using the default database alias and
428 # don't abide by the database specified by this router.
429 return True
430
431 def allow_migrate(self, db, app_label, **hints):
432 return True
433
434
435 @override_settings(DATABASE_ROUTERS=[OtherRouter()])
436 class LayerMapRouterTest(TestCase):
437 databases = {"default", "other"}
438
439 @unittest.skipUnless(len(settings.DATABASES) > 1, "multiple databases required")
440 def test_layermapping_default_db(self):
441 lm = LayerMapping(City, city_shp, city_mapping)
442 self.assertEqual(lm.using, "other")