Debian Codegraph @

ogr/ogrsf_frmts/parquet/ogrparquetlayer.cpp

raw
1 /******************************************************************************
2 *
3 * Project: Parquet Translator
4 * Purpose: Implements OGRParquetDriver.
5 * Author: Even Rouault, <even.rouault at spatialys.com>
6 *
7 ******************************************************************************
8 * Copyright (c) 2022, Planet Labs
9 *
10 * SPDX-License-Identifier: MIT
11 ****************************************************************************/
12
13 #include "cpl_json.h"
14 #include "cpl_time.h"
15 #include "cpl_multiproc.h"
16 #include "gdal_pam.h"
17 #include "ogrsf_frmts.h"
18 #include "ogr_p.h"
19 #include "gdal_thread_pool.h"
20
21 #include <algorithm>
22 #include <cinttypes>
23 #include <cmath>
24 #include <limits>
25 #include <map>
26 #include <set>
27 #include <utility>
28
29 #include "ogr_parquet.h"
30
31 #include "../arrow_common/ograrrowlayer.hpp"
32 #include "../arrow_common/ograrrowdataset.hpp"
33
34 /************************************************************************/
35 /* OGRParquetLayerBase() */
36 /************************************************************************/
37
38 OGRParquetLayerBase::OGRParquetLayerBase(OGRParquetDataset *poDS,
39 const char *pszLayerName,
40 CSLConstList papszOpenOptions)
41 : OGRArrowLayer(poDS, pszLayerName,
42 CPLTestBool(CSLFetchNameValueDef(
43 papszOpenOptions, "LISTS_AS_STRING_JSON", "NO"))),
44 m_poDS(poDS),
45 m_aosGeomPossibleNames(CSLTokenizeString2(
46 CSLFetchNameValueDef(papszOpenOptions, "GEOM_POSSIBLE_NAMES",
47 "geometry,wkb_geometry,wkt_geometry"),
48 ",", 0)),
49 m_osCRS(CSLFetchNameValueDef(papszOpenOptions, "CRS", ""))
50 {
51 }
52
53 /************************************************************************/
54 /* GetDataset() */
55 /************************************************************************/
56
57 GDALDataset *OGRParquetLayerBase::GetDataset()
58 {
59 return m_poDS;
60 }
61
62 /************************************************************************/
63 /* ResetReading() */
64 /************************************************************************/
65
66 void OGRParquetLayerBase::ResetReading()
67 {
68 if (m_iRecordBatch != 0)
69 {
70 m_poRecordBatchReader.reset();
71 }
72 OGRArrowLayer::ResetReading();
73 }
74
75 /************************************************************************/
76 /* InvalidateCachedBatches() */
77 /************************************************************************/
78
79 void OGRParquetLayerBase::InvalidateCachedBatches()
80 {
81 m_iRecordBatch = -1;
82 ResetReading();
83 }
84
85 /************************************************************************/
86 /* LoadGeoMetadata() */
87 /************************************************************************/
88
89 void OGRParquetLayerBase::LoadGeoMetadata(
90 const std::shared_ptr<const arrow::KeyValueMetadata> &kv_metadata)
91 {
92 if (kv_metadata && kv_metadata->Contains("geo"))
93 {
94 auto geo = kv_metadata->Get("geo");
95 if (geo.ok())
96 {
97 CPLDebug("PARQUET", "geo = %s", geo->c_str());
98 CPLJSONDocument oDoc;
99 if (oDoc.LoadMemory(*geo))
100 {
101 auto oRoot = oDoc.GetRoot();
102 const auto osVersion = oRoot.GetString("version");
103 if (osVersion != "0.1.0" && osVersion != "0.2.0" &&
104 osVersion != "0.3.0" && osVersion != "0.4.0" &&
105 osVersion != "1.0.0-beta.1" && osVersion != "1.0.0-rc.1" &&
106 osVersion != "1.0.0" && osVersion != "1.1.0")
107 {
108 CPLDebug(
109 "PARQUET",
110 "version = %s not explicitly handled by the driver",
111 osVersion.c_str());
112 }
113
114 auto oColumns = oRoot.GetObj("columns");
115 if (oColumns.IsValid())
116 {
117 for (const auto &oColumn : oColumns.GetChildren())
118 {
119 m_oMapGeometryColumns[oColumn.GetName()] = oColumn;
120 }
121 }
122 }
123 else
124 {
125 CPLError(CE_Warning, CPLE_AppDefined,
126 "Cannot parse 'geo' metadata");
127 }
128 }
129 }
130 }
131
132 /************************************************************************/
133 /* ParseGeometryColumnCovering() */
134 /************************************************************************/
135
136 //! Parse bounding box column definition
137 /*static */
138 bool OGRParquetLayerBase::ParseGeometryColumnCovering(
139 const CPLJSONObject &oJSONDef, std::string &osBBOXColumn,
140 std::string &osXMin, std::string &osYMin, std::string &osXMax,
141 std::string &osYMax)
142 {
143 const auto oCovering = oJSONDef["covering"];
144 if (oCovering.IsValid() &&
145 oCovering.GetType() == CPLJSONObject::Type::Object)
146 {
147 const auto oBBOX = oCovering["bbox"];
148 if (oBBOX.IsValid() && oBBOX.GetType() == CPLJSONObject::Type::Object)
149 {
150 const auto oXMin = oBBOX["xmin"];
151 const auto oYMin = oBBOX["ymin"];
152 const auto oXMax = oBBOX["xmax"];
153 const auto oYMax = oBBOX["ymax"];
154 if (oXMin.IsValid() && oYMin.IsValid() && oXMax.IsValid() &&
155 oYMax.IsValid() &&
156 oXMin.GetType() == CPLJSONObject::Type::Array &&
157 oYMin.GetType() == CPLJSONObject::Type::Array &&
158 oXMax.GetType() == CPLJSONObject::Type::Array &&
159 oYMax.GetType() == CPLJSONObject::Type::Array)
160 {
161 const auto osXMinArray = oXMin.ToArray();
162 const auto osYMinArray = oYMin.ToArray();
163 const auto osXMaxArray = oXMax.ToArray();
164 const auto osYMaxArray = oYMax.ToArray();
165 if (osXMinArray.Size() == 2 && osYMinArray.Size() == 2 &&
166 osXMaxArray.Size() == 2 && osYMaxArray.Size() == 2 &&
167 osXMinArray[0].GetType() == CPLJSONObject::Type::String &&
168 osXMinArray[1].GetType() == CPLJSONObject::Type::String &&
169 osYMinArray[0].GetType() == CPLJSONObject::Type::String &&
170 osYMinArray[1].GetType() == CPLJSONObject::Type::String &&
171 osXMaxArray[0].GetType() == CPLJSONObject::Type::String &&
172 osXMaxArray[1].GetType() == CPLJSONObject::Type::String &&
173 osYMaxArray[0].GetType() == CPLJSONObject::Type::String &&
174 osYMaxArray[1].GetType() == CPLJSONObject::Type::String &&
175 osXMinArray[0].ToString() == osYMinArray[0].ToString() &&
176 osXMinArray[0].ToString() == osXMaxArray[0].ToString() &&
177 osXMinArray[0].ToString() == osYMaxArray[0].ToString())
178 {
179 osBBOXColumn = osXMinArray[0].ToString();
180 osXMin = osXMinArray[1].ToString();
181 osYMin = osYMinArray[1].ToString();
182 osXMax = osXMaxArray[1].ToString();
183 osYMax = osYMaxArray[1].ToString();
184 return true;
185 }
186 }
187 }
188 }
189 return false;
190 }
191
192 /************************************************************************/
193 /* DealWithArrow21GeometryGeographyNativeTypes() */
194 /************************************************************************/
195
196 #if PARQUET_VERSION_MAJOR >= 21
197
198 /** Returns true if the passed field is detected to be a native Geometry/Geography
199 * column as introduced in libarrow >= 21.
200 *
201 * In that case, m_aeGeomEncoding, m_poFeatureDefn and m_anMapGeomFieldIndexToArrowColumn
202 * are updated with the new geometry column.
203 */
204 bool OGRParquetLayerBase::DealWithArrow21GeometryGeographyNativeTypes(
205 int iFieldIdx, const std::shared_ptr<arrow::Field> &field,
206 const parquet::ColumnDescriptor *parquetColumn,
207 const parquet::FileMetaData *fileMetadata, int iColumn)
208 {
209 std::shared_ptr<arrow::DataType> fieldType = field->type();
210 auto fieldTypeId = fieldType->id();
211
212 // Arrow >= 21 GEOMETRY/GEOGRAPHY logical type are seen as an extension
213 // because OGRParquetDriverOpen registers OGRGeoArrowWkbExtensionType
214 // as dealing with geoarrow.wkb
215 if (fieldTypeId != arrow::Type::EXTENSION)
216 return false;
217
218 auto extensionType =
219 cpl::down_cast<arrow::ExtensionType *>(fieldType.get());
220 if (extensionType->extension_name() != EXTENSION_NAME_GEOARROW_WKB)
221 return false;
222
223 fieldTypeId = extensionType->storage_type()->id();
224 if (fieldTypeId != arrow::Type::BINARY &&
225 fieldTypeId != arrow::Type::LARGE_BINARY)
226 {
227 return false;
228 }
229
230 const auto arrowWkb =
231 dynamic_cast<const OGRGeoArrowWkbExtensionType *>(extensionType);
232 #ifdef DEBUG
233 if (arrowWkb)
234 {
235 CPLDebug("PARQUET", "arrowWkb = '%s'", arrowWkb->Serialize().c_str());
236 }
237 #endif
238
239 OGRwkbGeometryType eGeomType = wkbUnknown;
240 bool bSkipRowGroups = false;
241
242 std::string crs(m_osCRS);
243 if (parquetColumn && crs.empty())
244 {
245 const auto &logicalType = parquetColumn->logical_type();
246 if (logicalType->is_geometry())
247 {
248 crs = static_cast<const parquet::GeometryLogicalType *>(
249 logicalType.get())
250 ->crs();
251 if (crs.empty())
252 crs = "EPSG:4326";
253 CPLDebugOnly("PARQUET", "GeometryLogicalType crs=%s", crs.c_str());
254 }
255 else if (logicalType->is_geography())
256 {
257 const auto *geographyType =
258 static_cast<const parquet::GeographyLogicalType *>(
259 logicalType.get());
260 crs = geographyType->crs();
261 if (crs.empty())
262 crs = "EPSG:4326";
263
264 SetMetadataItem(
265 "EDGES", CPLString(std::string(geographyType->algorithm_name()))
266 .toupper());
267 CPLDebugOnly("PARQUET", "GeographyLogicalType crs=%s", crs.c_str());
268 }
269 else
270 {
271 CPLDebug("PARQUET", "geoarrow.wkb column is neither a "
272 "geometry or geography one");
273
274 return false;
275 }
276
277 // Cf https://github.com/apache/parquet-format/blob/master/Geospatial.md#crs-customization
278 // "projjson: PROJJSON, identifier is the name of a table property or a file property where the projjson string is stored."
279 // Here the property is interpreted as the key of a file metadata (as done in libarrow)
280 constexpr const char *PROJJSON_PREFIX = "projjson:";
281 if (cpl::starts_with(crs, PROJJSON_PREFIX) && fileMetadata)
282 {
283 auto projjson_value = fileMetadata->key_value_metadata()->Get(
284 crs.substr(strlen(PROJJSON_PREFIX)));
285 if (projjson_value.ok())
286 {
287 crs = *projjson_value;
288 }
289 else
290 {
291 CPLDebug("PARQUET", "Cannot find file metadata for %s",
292 crs.c_str());
293 }
294 }
295 }
296 else if (!parquetColumn && arrowWkb)
297 {
298 // For a OGRParquetDatasetLayer for example
299 const std::string arrowWkbMetadata = arrowWkb->Serialize();
300 if (arrowWkbMetadata.empty() || arrowWkbMetadata == "{}")
301 {
302 crs = "EPSG:4326";
303 }
304 else if (arrowWkbMetadata[0] == '{')
305 {
306 CPLJSONDocument oDoc;
307 if (oDoc.LoadMemory(arrowWkbMetadata))
308 {
309 auto jCrs = oDoc.GetRoot()["crs"];
310 if (jCrs.GetType() == CPLJSONObject::Type::Object)
311 {
312 crs = jCrs.Format(CPLJSONObject::PrettyFormat::Plain);
313 }
314 else if (jCrs.GetType() == CPLJSONObject::Type::String)
315 {
316 crs = jCrs.ToString();
317 }
318 if (oDoc.GetRoot()["edges"].ToString() == "spherical")
319 {
320 SetMetadataItem("EDGES", "SPHERICAL");
321 }
322 }
323 }
324 }
325
326 bool bGeomTypeInvalid = false;
327 bool bHasMulti = false;
328 bool bHasZ = false;
329 bool bHasM = false;
330 bool bFirst = true;
331 OGRwkbGeometryType eFirstType = wkbUnknown;
332 OGRwkbGeometryType eFirstTypeCollection = wkbUnknown;
333 const auto numRowGroups = fileMetadata ? fileMetadata->num_row_groups() : 0;
334 bool bEnvelopeValid = true;
335 OGREnvelope sEnvelope;
336 bool bEnvelope3DValid = true;
337 OGREnvelope3D sEnvelope3D;
338 for (int iRowGroup = 0; !bSkipRowGroups && iRowGroup < numRowGroups;
339 ++iRowGroup)
340 {
341 const auto columnChunk =
342 fileMetadata->RowGroup(iRowGroup)->ColumnChunk(iColumn);
343 if (auto geostats = columnChunk->geo_statistics())
344 {
345 double dfMinX = std::numeric_limits<double>::quiet_NaN();
346 double dfMinY = std::numeric_limits<double>::quiet_NaN();
347 double dfMinZ = std::numeric_limits<double>::quiet_NaN();
348 double dfMaxX = std::numeric_limits<double>::quiet_NaN();
349 double dfMaxY = std::numeric_limits<double>::quiet_NaN();
350 double dfMaxZ = std::numeric_limits<double>::quiet_NaN();
351 if (bEnvelopeValid && geostats->dimension_valid()[0] &&
352 geostats->dimension_valid()[1])
353 {
354 dfMinX = geostats->lower_bound()[0];
355 dfMaxX = geostats->upper_bound()[0];
356 dfMinY = geostats->lower_bound()[1];
357 dfMaxY = geostats->upper_bound()[1];
358
359 // Deal as best as we can with wrap around bounding box
360 if (dfMinX > dfMaxX && std::fabs(dfMinX) <= 180 &&
361 std::fabs(dfMaxX) <= 180)
362 {
363 dfMinX = -180;
364 dfMaxX = 180;
365 }
366
367 if (std::isfinite(dfMinX) && std::isfinite(dfMaxX) &&
368 std::isfinite(dfMinY) && std::isfinite(dfMaxY))
369 {
370 sEnvelope.Merge(dfMinX, dfMinY);
371 sEnvelope.Merge(dfMaxX, dfMaxY);
372 if (bEnvelope3DValid && geostats->dimension_valid()[2])
373 {
374 dfMinZ = geostats->lower_bound()[2];
375 dfMaxZ = geostats->upper_bound()[2];
376 if (std::isfinite(dfMinZ) && std::isfinite(dfMaxZ))
377 {
378 sEnvelope3D.Merge(dfMinX, dfMinY, dfMinZ);
379 sEnvelope3D.Merge(dfMaxX, dfMaxY, dfMaxZ);
380 }
381 }
382 }
383 }
384
385 bEnvelopeValid = bEnvelopeValid && std::isfinite(dfMinX) &&
386 std::isfinite(dfMaxX) && std::isfinite(dfMinY) &&
387 std::isfinite(dfMaxY);
388
389 bEnvelope3DValid = bEnvelope3DValid && std::isfinite(dfMinZ) &&
390 std::isfinite(dfMaxZ);
391
392 if (auto geometry_types = geostats->geometry_types())
393 {
394 const auto PromoteToCollection = [](OGRwkbGeometryType eType)
395 {
396 if (eType == wkbPoint)
397 return wkbMultiPoint;
398 if (eType == wkbLineString)
399 return wkbMultiLineString;
400 if (eType == wkbPolygon)
401 return wkbMultiPolygon;
402 return eType;
403 };
404
405 for (int nGeomType : *geometry_types)
406 {
407 OGRwkbGeometryType eThisGeom = wkbUnknown;
408 if ((nGeomType > 0 && nGeomType <= 17) ||
409 (nGeomType > 2000 && nGeomType <= 2017) ||
410 (nGeomType > 3000 && nGeomType <= 3017))
411 {
412 eThisGeom = static_cast<OGRwkbGeometryType>(nGeomType);
413 }
414 else if (nGeomType > 1000 && nGeomType <= 1017)
415 {
416 eThisGeom = OGR_GT_SetZ(
417 static_cast<OGRwkbGeometryType>(nGeomType - 1000));
418 ;
419 }
420 else
421 {
422 CPLDebug("PARQUET", "Unknown geometry type: %d",
423 nGeomType);
424 bGeomTypeInvalid = true;
425 break;
426 }
427 if (bFirst)
428 {
429 bFirst = false;
430 eFirstType = eThisGeom;
431 eFirstTypeCollection = PromoteToCollection(eFirstType);
432 }
433 else if (PromoteToCollection(OGR_GT_Flatten(eThisGeom)) !=
434 eFirstTypeCollection)
435 {
436 bGeomTypeInvalid = true;
437 break;
438 }
439 bHasZ |= OGR_GT_HasZ(eThisGeom) != FALSE;
440 bHasM |= OGR_GT_HasM(eThisGeom) != FALSE;
441 bHasMulti |=
442 (PromoteToCollection(OGR_GT_Flatten(eThisGeom)) ==
443 OGR_GT_Flatten(eThisGeom));
444 }
445 }
446 }
447 else
448 {
449 bEnvelopeValid = false;
450 bEnvelope3DValid = false;
451 bGeomTypeInvalid = true;
452 }
453 }
454
455 if (bEnvelopeValid && sEnvelope.IsInit())
456 {
457 CPLDebug("PARQUET", "Got bounding box from geo_statistics");
458 m_geoStatsWithBBOXAvailable.insert(
459 m_poFeatureDefn->GetGeomFieldCount());
460 m_oMapExtents[m_poFeatureDefn->GetGeomFieldCount()] =
461 std::move(sEnvelope);
462
463 if (bEnvelope3DValid && sEnvelope3D.IsInit())
464 {
465 CPLDebug("PARQUET", "Got bounding box 3D from geo_statistics");
466 m_oMapExtents3D[m_poFeatureDefn->GetGeomFieldCount()] =
467 std::move(sEnvelope3D);
468 }
469 }
470
471 if (!bSkipRowGroups && !bGeomTypeInvalid)
472 {
473 if (eFirstTypeCollection == wkbMultiPoint ||
474 eFirstTypeCollection == wkbMultiPolygon ||
475 eFirstTypeCollection == wkbMultiLineString)
476 {
477 if (bHasMulti)
478 eGeomType =
479 OGR_GT_SetModifier(eFirstTypeCollection, bHasZ, bHasM);
480 else
481 eGeomType = OGR_GT_SetModifier(eFirstType, bHasZ, bHasM);
482 }
483 }
484
485 OGRGeomFieldDefn oField(field->name().c_str(), eGeomType);
486 oField.SetNullable(field->nullable());
487
488 if (!crs.empty())
489 {
490 // Cf https://github.com/apache/parquet-format/blob/master/Geospatial.md#crs-customization
491 // "srid: Spatial reference identifier, identifier is the SRID itself.."
492 constexpr const char *SRID_PREFIX = "srid:";
493 if (cpl::starts_with(crs, SRID_PREFIX))
494 {
495 // When getting the value from the GeometryLogicalType::crs() method
496 crs = crs.substr(strlen(SRID_PREFIX));
497 }
498 if (CPLGetValueType(crs.c_str()) == CPL_VALUE_INTEGER)
499 {
500 // Getting here from above if, or if reading the ArrowWkb
501 // metadata directly (typically from a OGRParquetDatasetLayer)
502
503 // Assumes a SRID code is an EPSG code...
504 crs = std::string("EPSG:") + crs;
505 }
506
507 auto poSRS = OGRSpatialReferenceRefCountedPtr::makeInstance();
508 poSRS->SetAxisMappingStrategy(OAMS_TRADITIONAL_GIS_ORDER);
509 if (poSRS->SetFromUserInput(
510 crs.c_str(),
511 OGRSpatialReference::SET_FROM_USER_INPUT_LIMITATIONS_get()) ==
512 OGRERR_NONE)
513 {
514 const char *pszAuthName = poSRS->GetAuthorityName();
515 const char *pszAuthCode = poSRS->GetAuthorityCode();
516 if (pszAuthName && pszAuthCode && EQUAL(pszAuthName, "OGC") &&
517 EQUAL(pszAuthCode, "CRS84"))
518 poSRS->importFromEPSG(4326);
519 oField.SetSpatialRef(poSRS.get());
520 }
521 }
522
523 m_aeGeomEncoding.push_back(OGRArrowGeomEncoding::WKB);
524 m_poFeatureDefn->AddGeomFieldDefn(&oField);
525 m_anMapGeomFieldIndexToArrowColumn.push_back(iFieldIdx);
526
527 return true;
528 }
529
530 #endif
531
532 /************************************************************************/
533 /* DealWithGeometryColumn() */
534 /************************************************************************/
535
536 bool OGRParquetLayerBase::DealWithGeometryColumn(
537 int iFieldIdx, const std::shared_ptr<arrow::Field> &field,
538 std::function<OGRwkbGeometryType(void)> computeGeometryTypeFun,
539 [[maybe_unused]] const parquet::ColumnDescriptor *parquetColumn,
540 [[maybe_unused]] const parquet::FileMetaData *fileMetadata,
541 [[maybe_unused]] int iColumn)
542 {
543 #if PARQUET_VERSION_MAJOR >= 21
544 if (DealWithArrow21GeometryGeographyNativeTypes(
545 iFieldIdx, field, parquetColumn, fileMetadata, iColumn))
546 return true;
547 #endif
548
549 const auto &field_kv_metadata = field->metadata();
550 std::string osExtensionName;
551 if (field_kv_metadata)
552 {
553 #ifdef DEBUG
554 const auto keyValueSortedPairs = field_kv_metadata->sorted_pairs();
555 if (!keyValueSortedPairs.empty())
556 {
557 CPLDebug("PARQUET",
558 "Metadata for field '%s':", field->name().c_str());
559 for (const auto &keyValue : keyValueSortedPairs)
560 {
561 CPLDebug("PARQUET", " '%s' = '%s'", keyValue.first.c_str(),
562 keyValue.second.c_str());
563 }
564 }
565 #endif
566 auto extension_name = field_kv_metadata->Get(ARROW_EXTENSION_NAME_KEY);
567 if (extension_name.ok())
568 {
569 // This code is needed for geo'ish-parquet file that aren't GeoParquet 1.1,
570 // nor use Parquet new native geometry/geography field columns, but
571 // have a ARROW:extension:name == ogc.wkb field metadata,
572 // and when libarrow extension handling geoarrow.wkb is NOT loaded
573 // Such extension is automatically registered in OGRParquetDriverOpen()
574 // for libarrow >= 21. It may also be registered by Python geoarrow.pyarrow
575 osExtensionName = *extension_name;
576 }
577 }
578
579 std::shared_ptr<arrow::DataType> fieldType = field->type();
580 const auto fieldTypeId = fieldType->id();
581 if (osExtensionName.empty() && fieldTypeId == arrow::Type::EXTENSION)
582 {
583 // This code is needed for geo'ish-parquet file that aren't GeoParquet 1.1,
584 // nor use Parquet new native geometry/geography field columns, but
585 // have a ARROW:extension:name == ogc.wkb field metadata,
586 // and when libarrow extension handling geoarrow.wkb is loaded
587 // Such extension is automatically registered in OGRParquetDriverOpen()
588 // for libarrow >= 21. It may also be registered by Python geoarrow.pyarrow
589 auto extensionType =
590 cpl::down_cast<arrow::ExtensionType *>(fieldType.get());
591 osExtensionName = extensionType->extension_name();
592 }
593
594 bool bRegularField = true;
595
596 auto oIter = m_oMapGeometryColumns.find(field->name());
597 // cppcheck-suppress knownConditionTrueFalse
598 if (bRegularField && (oIter != m_oMapGeometryColumns.end() ||
599 STARTS_WITH(osExtensionName.c_str(), "ogc.") ||
600 STARTS_WITH(osExtensionName.c_str(), "geoarrow.")))
601 {
602 CPLJSONObject oJSONDef;
603 if (oIter != m_oMapGeometryColumns.end())
604 oJSONDef = oIter->second;
605 auto osEncoding = oJSONDef.GetString("encoding");
606 if (osEncoding.empty() && !osExtensionName.empty())
607 osEncoding = osExtensionName;
608
609 OGRwkbGeometryType eGeomType = wkbUnknown;
610 auto eGeomEncoding = OGRArrowGeomEncoding::WKB;
611 if (IsValidGeometryEncoding(field, osEncoding,
612 oIter != m_oMapGeometryColumns.end(),
613 eGeomType, eGeomEncoding))
614 {
615 bRegularField = false;
616 OGRGeomFieldDefn oField(field->name().c_str(), wkbUnknown);
617
618 auto oCRS = oJSONDef["crs"];
619 OGRSpatialReference *poSRS = nullptr;
620 if (!oCRS.IsValid())
621 {
622 if (!m_oMapGeometryColumns.empty())
623 {
624 // WGS 84 is implied if no crs member is found.
625 poSRS = new OGRSpatialReference();
626 poSRS->SetAxisMappingStrategy(OAMS_TRADITIONAL_GIS_ORDER);
627 poSRS->importFromEPSG(4326);
628 }
629 }
630 else if (oCRS.GetType() == CPLJSONObject::Type::String)
631 {
632 const auto osWKT = oCRS.ToString();
633 poSRS = new OGRSpatialReference();
634 poSRS->SetAxisMappingStrategy(OAMS_TRADITIONAL_GIS_ORDER);
635
636 if (poSRS->importFromWkt(osWKT.c_str()) != OGRERR_NONE)
637 {
638 poSRS->Release();
639 poSRS = nullptr;
640 }
641 }
642 else if (oCRS.GetType() == CPLJSONObject::Type::Object)
643 {
644 // CRS encoded as PROJJSON (extension)
645 const auto oType = oCRS["type"];
646 if (oType.IsValid() &&
647 oType.GetType() == CPLJSONObject::Type::String)
648 {
649 const auto osType = oType.ToString();
650 if (osType.find("CRS") != std::string::npos)
651 {
652 poSRS = new OGRSpatialReference();
653 poSRS->SetAxisMappingStrategy(
654 OAMS_TRADITIONAL_GIS_ORDER);
655
656 if (poSRS->SetFromUserInput(
657 oCRS.ToString().c_str(),
658 OGRSpatialReference::
659 SET_FROM_USER_INPUT_LIMITATIONS_get()) !=
660 OGRERR_NONE)
661 {
662 poSRS->Release();
663 poSRS = nullptr;
664 }
665 }
666 }
667 }
668
669 if (poSRS)
670 {
671 const double dfCoordEpoch = oJSONDef.GetDouble("epoch");
672 if (dfCoordEpoch > 0)
673 poSRS->SetCoordinateEpoch(dfCoordEpoch);
674
675 oField.SetSpatialRef(poSRS);
676
677 poSRS->Release();
678 }
679
680 if (!m_osCRS.empty())
681 {
682 poSRS = new OGRSpatialReference();
683 poSRS->SetAxisMappingStrategy(OAMS_TRADITIONAL_GIS_ORDER);
684 if (poSRS->SetFromUserInput(
685 m_osCRS.c_str(),
686 OGRSpatialReference::
687 SET_FROM_USER_INPUT_LIMITATIONS_get()) ==
688 OGRERR_NONE)
689 {
690 oField.SetSpatialRef(poSRS);
691 }
692 poSRS->Release();
693 }
694
695 if (oJSONDef.GetString("edges") == "spherical")
696 {
697 SetMetadataItem("EDGES", "SPHERICAL");
698 }
699
700 // m_aeGeomEncoding be filled before calling
701 // ComputeGeometryColumnType()
702 m_aeGeomEncoding.push_back(eGeomEncoding);
703 if (eGeomType == wkbUnknown)
704 {
705 // geometry_types since 1.0.0-beta1. Was geometry_type
706 // before
707 auto oType = oJSONDef.GetObj("geometry_types");
708 if (!oType.IsValid())
709 oType = oJSONDef.GetObj("geometry_type");
710 if (oType.GetType() == CPLJSONObject::Type::String)
711 {
712 // string is no longer valid since 1.0.0-beta1
713 const auto osType = oType.ToString();
714 if (osType != "Unknown")
715 eGeomType = GetGeometryTypeFromString(osType);
716 }
717 else if (oType.GetType() == CPLJSONObject::Type::Array)
718 {
719 const auto oTypeArray = oType.ToArray();
720 if (oTypeArray.Size() == 1)
721 {
722 eGeomType =
723 GetGeometryTypeFromString(oTypeArray[0].ToString());
724 }
725 else if (oTypeArray.Size() > 1)
726 {
727 const auto PromoteToCollection =
728 [](OGRwkbGeometryType eType)
729 {
730 if (eType == wkbPoint)
731 return wkbMultiPoint;
732 if (eType == wkbLineString)
733 return wkbMultiLineString;
734 if (eType == wkbPolygon)
735 return wkbMultiPolygon;
736 return eType;
737 };
738 bool bMixed = false;
739 bool bHasMulti = false;
740 bool bHasZ = false;
741 bool bHasM = false;
742 const auto eFirstType =
743 OGR_GT_Flatten(GetGeometryTypeFromString(
744 oTypeArray[0].ToString()));
745 const auto eFirstTypeCollection =
746 PromoteToCollection(eFirstType);
747 for (int i = 0; i < oTypeArray.Size(); ++i)
748 {
749 const auto eThisGeom = GetGeometryTypeFromString(
750 oTypeArray[i].ToString());
751 if (PromoteToCollection(OGR_GT_Flatten(
752 eThisGeom)) != eFirstTypeCollection)
753 {
754 bMixed = true;
755 break;
756 }
757 bHasZ |= OGR_GT_HasZ(eThisGeom) != FALSE;
758 bHasM |= OGR_GT_HasM(eThisGeom) != FALSE;
759 bHasMulti |=
760 (PromoteToCollection(OGR_GT_Flatten(
761 eThisGeom)) == OGR_GT_Flatten(eThisGeom));
762 }
763 if (!bMixed)
764 {
765 if (eFirstTypeCollection == wkbMultiPolygon ||
766 eFirstTypeCollection == wkbMultiLineString)
767 {
768 if (bHasMulti)
769 eGeomType = OGR_GT_SetModifier(
770 eFirstTypeCollection, bHasZ, bHasM);
771 else
772 eGeomType = OGR_GT_SetModifier(
773 eFirstType, bHasZ, bHasM);
774 }
775 }
776 }
777 }
778 else if (CPLTestBool(CPLGetConfigOption(
779 "OGR_PARQUET_COMPUTE_GEOMETRY_TYPE", "YES")))
780 {
781 eGeomType = computeGeometryTypeFun();
782 }
783 }
784
785 oField.SetType(eGeomType);
786 oField.SetNullable(field->nullable());
787 m_poFeatureDefn->AddGeomFieldDefn(&oField);
788 m_anMapGeomFieldIndexToArrowColumn.push_back(iFieldIdx);
789 }
790 }
791
792 // Try to autodetect a (WKB) geometry column from the GEOM_POSSIBLE_NAMES
793 // open option
794 if (bRegularField && osExtensionName.empty() &&
795 m_oMapGeometryColumns.empty() &&
796 m_aosGeomPossibleNames.FindString(field->name().c_str()) >= 0)
797 {
798 if (fieldTypeId == arrow::Type::BINARY ||
799 fieldTypeId == arrow::Type::LARGE_BINARY)
800 {
801 CPLDebug("PARQUET",
802 "Field %s detected as likely WKB geometry field",
803 field->name().c_str());
804 bRegularField = false;
805 m_aeGeomEncoding.push_back(OGRArrowGeomEncoding::WKB);
806 }
807 else if ((fieldTypeId == arrow::Type::STRING ||
808 fieldTypeId == arrow::Type::LARGE_STRING) &&
809 (field->name().find("wkt") != std::string::npos ||
810 field->name().find("WKT") != std::string::npos))
811 {
812 CPLDebug("PARQUET",
813 "Field %s detected as likely WKT geometry field",
814 field->name().c_str());
815 bRegularField = false;
816 m_aeGeomEncoding.push_back(OGRArrowGeomEncoding::WKT);
817 }
818 if (!bRegularField)
819 {
820 OGRGeomFieldDefn oField(field->name().c_str(), wkbUnknown);
821 oField.SetNullable(field->nullable());
822
823 if (!m_osCRS.empty())
824 {
825 auto poSRS = new OGRSpatialReference();
826 poSRS->SetAxisMappingStrategy(OAMS_TRADITIONAL_GIS_ORDER);
827 if (poSRS->SetFromUserInput(
828 m_osCRS.c_str(),
829 OGRSpatialReference::
830 SET_FROM_USER_INPUT_LIMITATIONS_get()) ==
831 OGRERR_NONE)
832 {
833 oField.SetSpatialRef(poSRS);
834 }
835 poSRS->Release();
836 }
837
838 m_poFeatureDefn->AddGeomFieldDefn(&oField);
839 m_anMapGeomFieldIndexToArrowColumn.push_back(iFieldIdx);
840 }
841 }
842
843 return !bRegularField;
844 }
845
846 /************************************************************************/
847 /* TestCapability() */
848 /************************************************************************/
849
850 int OGRParquetLayerBase::TestCapability(const char *pszCap) const
851 {
852 if (EQUAL(pszCap, OLCMeasuredGeometries))
853 return true;
854
855 if (EQUAL(pszCap, OLCFastSetNextByIndex))
856 return true;
857
858 if (EQUAL(pszCap, OLCFastSpatialFilter))
859 {
860 if (m_oMapGeomFieldIndexToGeomColBBOX.find(m_iGeomFieldFilter) !=
861 m_oMapGeomFieldIndexToGeomColBBOX.end())
862 {
863 return true;
864 }
865 return false;
866 }
867
868 return OGRArrowLayer::TestCapability(pszCap);
869 }
870
871 /************************************************************************/
872 /* GetNumCPUs() */
873 /************************************************************************/
874
875 /* static */
876 int OGRParquetLayerBase::GetNumCPUs()
877 {
878 const char *pszNumThreads = nullptr;
879 int nNumThreads =
880 GDALGetNumThreads(pszNumThreads,
881 /* nMaxVal = */ -1,
882 /* bDefaultToAllCPUs = */ false, &pszNumThreads);
883 if (pszNumThreads == nullptr)
884 nNumThreads = std::min(4, CPLGetNumCPUs());
885 if (nNumThreads > 1)
886 {
887 CPL_IGNORE_RET_VAL(arrow::SetCpuThreadPoolCapacity(nNumThreads));
888 }
889 return nNumThreads;
890 }
891
892 /************************************************************************/
893 /* OGRParquetLayer() */
894 /************************************************************************/
895
896 OGRParquetLayer::OGRParquetLayer(
897 OGRParquetDataset *poDS, const char *pszLayerName,
898 std::unique_ptr<parquet::arrow::FileReader> &&arrow_reader,
899 CSLConstList papszOpenOptions)
900 : OGRParquetLayerBase(poDS, pszLayerName, papszOpenOptions),
901 m_poArrowReader(std::move(arrow_reader))
902 {
903 EstablishFeatureDefn();
904 CPLAssert(static_cast<int>(m_aeGeomEncoding.size()) ==
905 m_poFeatureDefn->GetGeomFieldCount());
906
907 m_oFeatureIdxRemappingIter = m_asFeatureIdxRemapping.begin();
908 }
909
910 /************************************************************************/
911 /* EstablishFeatureDefn() */
912 /************************************************************************/
913
914 void OGRParquetLayer::EstablishFeatureDefn()
915 {
916 const auto metadata = m_poArrowReader->parquet_reader()->metadata();
917 const auto &kv_metadata = metadata->key_value_metadata();
918
919 LoadGeoMetadata(kv_metadata);
920 const auto oMapFieldNameToGDALSchemaFieldDefn =
921 LoadGDALSchema(kv_metadata.get());
922
923 LoadGDALMetadata(kv_metadata.get());
924
925 if (kv_metadata && kv_metadata->Contains("gdal:creation-options"))
926 {
927 auto co = kv_metadata->Get("gdal:creation-options");
928 if (co.ok())
929 {
930 CPLDebugOnly("PARQUET", "gdal:creation-options = %s", co->c_str());
931 CPLJSONDocument oDoc;
932 if (oDoc.LoadMemory(*co))
933 {
934 auto oRoot = oDoc.GetRoot();
935 if (oRoot.GetType() == CPLJSONObject::Type::Object)
936 {
937 for (const auto &oChild : oRoot.GetChildren())
938 {
939 if (oChild.GetType() == CPLJSONObject::Type::String)
940 {
941 m_aosCreationOptions.SetNameValue(
942 oChild.GetName().c_str(),
943 oChild.ToString().c_str());
944 }
945 }
946 }
947 }
948 }
949 }
950
951 if (!m_poArrowReader->GetSchema(&m_poSchema).ok())
952 {
953 return;
954 }
955
956 const bool bUseBBOX =
957 CPLTestBool(CPLGetConfigOption("OGR_PARQUET_USE_BBOX", "YES"));
958
959 // Keep track of declared bounding box columns in GeoParquet JSON metadata,
960 // in order not to expose them as regular fields.
961 std::set<std::string> oSetBBOXColumns;
962 if (bUseBBOX)
963 {
964 for (const auto &iter : m_oMapGeometryColumns)
965 {
966 std::string osBBOXColumn;
967 std::string osXMin, osYMin, osXMax, osYMax;
968 if (ParseGeometryColumnCovering(iter.second, osBBOXColumn, osXMin,
969 osYMin, osXMax, osYMax))
970 {
971 oSetBBOXColumns.insert(std::move(osBBOXColumn));
972 }
973 }
974 }
975
976 const auto &fields = m_poSchema->fields();
977 const auto poParquetSchema = metadata->schema();
978
979 // Map from Parquet column name (with dot separator) to Parquet index
980 std::map<std::string, int> oMapParquetColumnNameToIdx;
981 const int nParquetColumns = poParquetSchema->num_columns();
982 for (int iParquetCol = 0; iParquetCol < nParquetColumns; ++iParquetCol)
983 {
984 const auto parquetColumn = poParquetSchema->Column(iParquetCol);
985 const auto parquetColumnName = parquetColumn->path()->ToDotString();
986 oMapParquetColumnNameToIdx[parquetColumnName] = iParquetCol;
987 }
988
989 // Synthetize a GeoParquet bounding box column definition when detecting
990 // a Overture Map dataset < 2024-04-16-beta.0
991 if ((m_oMapGeometryColumns.empty() ||
992 // Below is for release 2024-01-17-alpha.0
993 (m_oMapGeometryColumns.find("geometry") !=
994 m_oMapGeometryColumns.end() &&
995 !m_oMapGeometryColumns["geometry"].GetObj("covering").IsValid() &&
996 m_oMapGeometryColumns["geometry"].GetString("encoding") == "WKB")) &&
997 bUseBBOX &&
998 oMapParquetColumnNameToIdx.find("geometry") !=
999 oMapParquetColumnNameToIdx.end() &&
1000 oMapParquetColumnNameToIdx.find("bbox.minx") !=
1001 oMapParquetColumnNameToIdx.end() &&
1002 oMapParquetColumnNameToIdx.find("bbox.miny") !=
1003 oMapParquetColumnNameToIdx.end() &&
1004 oMapParquetColumnNameToIdx.find("bbox.maxx") !=
1005 oMapParquetColumnNameToIdx.end() &&
1006 oMapParquetColumnNameToIdx.find("bbox.maxy") !=
1007 oMapParquetColumnNameToIdx.end())
1008 {
1009 CPLJSONObject oDef;
1010 if (m_oMapGeometryColumns.find("geometry") !=
1011 m_oMapGeometryColumns.end())
1012 {
1013 oDef = m_oMapGeometryColumns["geometry"];
1014 }
1015 CPLJSONObject oCovering;
1016 oDef.Add("covering", oCovering);
1017 CPLJSONObject oBBOX;
1018 oCovering.Add("bbox", oBBOX);
1019 {
1020 CPLJSONArray oArray;
1021 oArray.Add("bbox");
1022 oArray.Add("minx");
1023 oBBOX.Add("xmin", oArray);
1024 }
1025 {
1026 CPLJSONArray oArray;
1027 oArray.Add("bbox");
1028 oArray.Add("miny");
1029 oBBOX.Add("ymin", oArray);
1030 }
1031 {
1032 CPLJSONArray oArray;
1033 oArray.Add("bbox");
1034 oArray.Add("maxx");
1035 oBBOX.Add("xmax", oArray);
1036 }
1037 {
1038 CPLJSONArray oArray;
1039 oArray.Add("bbox");
1040 oArray.Add("maxy");
1041 oBBOX.Add("ymax", oArray);
1042 }
1043 oSetBBOXColumns.insert("bbox");
1044 oDef.Add("encoding", "WKB");
1045 m_oMapGeometryColumns["geometry"] = std::move(oDef);
1046 }
1047 // Overture Maps 2024-04-16-beta.0 almost follows GeoParquet 1.1, except
1048 // they don't declare the "covering" element in the GeoParquet JSON metadata
1049 else if (m_oMapGeometryColumns.find("geometry") !=
1050 m_oMapGeometryColumns.end() &&
1051 bUseBBOX &&
1052 !m_oMapGeometryColumns["geometry"].GetObj("covering").IsValid() &&
1053 m_oMapGeometryColumns["geometry"].GetString("encoding") == "WKB" &&
1054 oMapParquetColumnNameToIdx.find("geometry") !=
1055 oMapParquetColumnNameToIdx.end() &&
1056 oMapParquetColumnNameToIdx.find("bbox.xmin") !=
1057 oMapParquetColumnNameToIdx.end() &&
1058 oMapParquetColumnNameToIdx.find("bbox.ymin") !=
1059 oMapParquetColumnNameToIdx.end() &&
1060 oMapParquetColumnNameToIdx.find("bbox.xmax") !=
1061 oMapParquetColumnNameToIdx.end() &&
1062 oMapParquetColumnNameToIdx.find("bbox.ymax") !=
1063 oMapParquetColumnNameToIdx.end())
1064 {
1065 CPLJSONObject oDef = m_oMapGeometryColumns["geometry"];
1066 CPLJSONObject oCovering;
1067 oDef.Add("covering", oCovering);
1068 CPLJSONObject oBBOX;
1069 oCovering.Add("bbox", oBBOX);
1070 {
1071 CPLJSONArray oArray;
1072 oArray.Add("bbox");
1073 oArray.Add("xmin");
1074 oBBOX.Add("xmin", oArray);
1075 }
1076 {
1077 CPLJSONArray oArray;
1078 oArray.Add("bbox");
1079 oArray.Add("ymin");
1080 oBBOX.Add("ymin", oArray);
1081 }
1082 {
1083 CPLJSONArray oArray;
1084 oArray.Add("bbox");
1085 oArray.Add("xmax");
1086 oBBOX.Add("xmax", oArray);
1087 }
1088 {
1089 CPLJSONArray oArray;
1090 oArray.Add("bbox");
1091 oArray.Add("ymax");
1092 oBBOX.Add("ymax", oArray);
1093 }
1094 oSetBBOXColumns.insert("bbox");
1095 m_oMapGeometryColumns["geometry"] = std::move(oDef);
1096 }
1097
1098 int iParquetCol = 0;
1099 for (int i = 0; i < m_poSchema->num_fields(); ++i)
1100 {
1101 const auto &field = fields[i];
1102
1103 bool bParquetColValid =
1104 CheckMatchArrowParquetColumnNames(iParquetCol, field);
1105 if (!bParquetColValid)
1106 m_bHasMissingMappingToParquet = true;
1107
1108 if (!m_osFIDColumn.empty() && field->name() == m_osFIDColumn &&
1109 (field->type()->id() == arrow::Type::INT32 ||
1110 field->type()->id() == arrow::Type::INT64))
1111 {
1112 m_poFIDType = field->type();
1113 m_iFIDArrowColumn = i;
1114 if (bParquetColValid)
1115 {
1116 m_iFIDParquetColumn = iParquetCol;
1117 iParquetCol++;
1118 }
1119 continue;
1120 }
1121
1122 if (oSetBBOXColumns.find(field->name()) != oSetBBOXColumns.end())
1123 {
1124 m_oSetBBoxArrowColumns.insert(i);
1125 if (bParquetColValid)
1126 iParquetCol++;
1127 continue;
1128 }
1129
1130 const auto ComputeGeometryColumnTypeLambda =
1131 [this, bParquetColValid, iParquetCol, &poParquetSchema]()
1132 {
1133 // only with GeoParquet < 0.2.0
1134 if (bParquetColValid &&
1135 poParquetSchema->Column(iParquetCol)->physical_type() ==
1136 parquet::Type::BYTE_ARRAY)
1137 {
1138 return ComputeGeometryColumnType(
1139 m_poFeatureDefn->GetGeomFieldCount(), iParquetCol);
1140 }
1141 return wkbUnknown;
1142 };
1143
1144 const bool bGeometryField = DealWithGeometryColumn(
1145 i, field, ComputeGeometryColumnTypeLambda,
1146 bParquetColValid ? poParquetSchema->Column(iParquetCol) : nullptr,
1147 metadata.get(), bParquetColValid ? iParquetCol : -1);
1148 if (bGeometryField)
1149 {
1150 const auto oIter = m_oMapGeometryColumns.find(field->name());
1151 if (bUseBBOX && oIter != m_oMapGeometryColumns.end())
1152 {
1153 ProcessGeometryColumnCovering(field, oIter->second,
1154 oMapParquetColumnNameToIdx);
1155 }
1156
1157 if (bParquetColValid &&
1158 (field->type()->id() == arrow::Type::STRUCT ||
1159 field->type()->id() == arrow::Type::LIST))
1160 {
1161 // GeoArrow types
1162 std::vector<int> anParquetCols;
1163 for (const auto &iterParquetCols : oMapParquetColumnNameToIdx)
1164 {
1165 if (STARTS_WITH(
1166 iterParquetCols.first.c_str(),
1167 std::string(field->name()).append(".").c_str()))
1168 {
1169 iParquetCol =
1170 std::max(iParquetCol, iterParquetCols.second);
1171 anParquetCols.push_back(iterParquetCols.second);
1172 }
1173 }
1174 m_anMapGeomFieldIndexToParquetColumns.push_back(
1175 std::move(anParquetCols));
1176 ++iParquetCol;
1177 }
1178 else
1179 {
1180 m_anMapGeomFieldIndexToParquetColumns.push_back(
1181 {bParquetColValid ? iParquetCol : -1});
1182 if (bParquetColValid)
1183 iParquetCol++;
1184 }
1185 }
1186 else
1187 {
1188 CreateFieldFromSchema(field, bParquetColValid, iParquetCol, {i},
1189 oMapFieldNameToGDALSchemaFieldDefn);
1190 }
1191 }
1192
1193 CPLAssert(static_cast<int>(m_anMapFieldIndexToArrowColumn.size()) ==
1194 m_poFeatureDefn->GetFieldCount());
1195 CPLAssert(static_cast<int>(m_anMapGeomFieldIndexToArrowColumn.size()) ==
1196 m_poFeatureDefn->GetGeomFieldCount());
1197 CPLAssert(static_cast<int>(m_anMapGeomFieldIndexToParquetColumns.size()) ==
1198 m_poFeatureDefn->GetGeomFieldCount());
1199
1200 if (!fields.empty())
1201 {
1202 try
1203 {
1204 auto poRowGroup = m_poArrowReader->parquet_reader()->RowGroup(0);
1205 if (poRowGroup)
1206 {
1207 auto poColumn = poRowGroup->metadata()->ColumnChunk(0);
1208 CPLDebug("PARQUET", "Compression (of first column): %s",
1209 arrow::util::Codec::GetCodecAsString(
1210 poColumn->compression())
1211 .c_str());
1212 }
1213 }
1214 catch (const std::exception &)
1215 {
1216 }
1217 }
1218 }
1219
1220 /************************************************************************/
1221 /* ProcessGeometryColumnCovering() */
1222 /************************************************************************/
1223
1224 /** Process GeoParquet JSON geometry field object to extract information about
1225 * its bounding box column, and appropriately fill m_oMapGeomFieldIndexToGeomColBBOX
1226 * and m_oMapGeomFieldIndexToGeomColBBOXParquet members with information on that
1227 * bounding box column.
1228 */
1229 void OGRParquetLayer::ProcessGeometryColumnCovering(
1230 const std::shared_ptr<arrow::Field> &field,
1231 const CPLJSONObject &oJSONGeometryColumn,
1232 const std::map<std::string, int> &oMapParquetColumnNameToIdx)
1233 {
1234 std::string osBBOXColumn;
1235 std::string osXMin, osYMin, osXMax, osYMax;
1236 if (ParseGeometryColumnCovering(oJSONGeometryColumn, osBBOXColumn, osXMin,
1237 osYMin, osXMax, osYMax))
1238 {
1239 OGRArrowLayer::GeomColBBOX sDesc;
1240 sDesc.iArrowCol = m_poSchema->GetFieldIndex(osBBOXColumn);
1241 const auto fieldBBOX = m_poSchema->GetFieldByName(osBBOXColumn);
1242 if (sDesc.iArrowCol >= 0 && fieldBBOX &&
1243 fieldBBOX->type()->id() == arrow::Type::STRUCT)
1244 {
1245 const auto fieldBBOXStruct =
1246 std::static_pointer_cast<arrow::StructType>(fieldBBOX->type());
1247 const auto fieldXMin = fieldBBOXStruct->GetFieldByName(osXMin);
1248 const auto fieldYMin = fieldBBOXStruct->GetFieldByName(osYMin);
1249 const auto fieldXMax = fieldBBOXStruct->GetFieldByName(osXMax);
1250 const auto fieldYMax = fieldBBOXStruct->GetFieldByName(osYMax);
1251 const int nXMinIdx = fieldBBOXStruct->GetFieldIndex(osXMin);
1252 const int nYMinIdx = fieldBBOXStruct->GetFieldIndex(osYMin);
1253 const int nXMaxIdx = fieldBBOXStruct->GetFieldIndex(osXMax);
1254 const int nYMaxIdx = fieldBBOXStruct->GetFieldIndex(osYMax);
1255 const auto oIterParquetIdxXMin = oMapParquetColumnNameToIdx.find(
1256 std::string(osBBOXColumn).append(".").append(osXMin));
1257 const auto oIterParquetIdxYMin = oMapParquetColumnNameToIdx.find(
1258 std::string(osBBOXColumn).append(".").append(osYMin));
1259 const auto oIterParquetIdxXMax = oMapParquetColumnNameToIdx.find(
1260 std::string(osBBOXColumn).append(".").append(osXMax));
1261 const auto oIterParquetIdxYMax = oMapParquetColumnNameToIdx.find(
1262 std::string(osBBOXColumn).append(".").append(osYMax));
1263 if (nXMinIdx >= 0 && nYMinIdx >= 0 && nXMaxIdx >= 0 &&
1264 nYMaxIdx >= 0 && fieldXMin && fieldYMin && fieldXMax &&
1265 fieldYMax &&
1266 oIterParquetIdxXMin != oMapParquetColumnNameToIdx.end() &&
1267 oIterParquetIdxYMin != oMapParquetColumnNameToIdx.end() &&
1268 oIterParquetIdxXMax != oMapParquetColumnNameToIdx.end() &&
1269 oIterParquetIdxYMax != oMapParquetColumnNameToIdx.end() &&
1270 (fieldXMin->type()->id() == arrow::Type::FLOAT ||
1271 fieldXMin->type()->id() == arrow::Type::DOUBLE) &&
1272 fieldXMin->type()->id() == fieldYMin->type()->id() &&
1273 fieldXMin->type()->id() == fieldXMax->type()->id() &&
1274 fieldXMin->type()->id() == fieldYMax->type()->id())
1275 {
1276 CPLDebug("PARQUET",
1277 "Bounding box column '%s' detected for "
1278 "geometry column '%s'",
1279 osBBOXColumn.c_str(), field->name().c_str());
1280 sDesc.iArrowSubfieldXMin = nXMinIdx;
1281 sDesc.iArrowSubfieldYMin = nYMinIdx;
1282 sDesc.iArrowSubfieldXMax = nXMaxIdx;
1283 sDesc.iArrowSubfieldYMax = nYMaxIdx;
1284 sDesc.bIsFloat =
1285 (fieldXMin->type()->id() == arrow::Type::FLOAT);
1286
1287 m_oMapGeomFieldIndexToGeomColBBOX
1288 [m_poFeatureDefn->GetGeomFieldCount() - 1] =
1289 std::move(sDesc);
1290
1291 GeomColBBOXParquet sDescParquet;
1292 sDescParquet.iParquetXMin = oIterParquetIdxXMin->second;
1293 sDescParquet.iParquetYMin = oIterParquetIdxYMin->second;
1294 sDescParquet.iParquetXMax = oIterParquetIdxXMax->second;
1295 sDescParquet.iParquetYMax = oIterParquetIdxYMax->second;
1296 for (const auto &iterParquetCols : oMapParquetColumnNameToIdx)
1297 {
1298 if (STARTS_WITH(
1299 iterParquetCols.first.c_str(),
1300 std::string(osBBOXColumn).append(".").c_str()))
1301 {
1302 sDescParquet.anParquetCols.push_back(
1303 iterParquetCols.second);
1304 }
1305 }
1306 m_oMapGeomFieldIndexToGeomColBBOXParquet
1307 [m_poFeatureDefn->GetGeomFieldCount() - 1] =
1308 std::move(sDescParquet);
1309 }
1310 }
1311 }
1312 }
1313
1314 /************************************************************************/
1315 /* CollectLeaveNodes() */
1316 /************************************************************************/
1317
1318 static void CollectLeaveNodes(
1319 const parquet::schema::Node *node,
1320 const std::map<const parquet::schema::Node *, int> &oMapNodeToColIdx,
1321 std::vector<int> &anParquetCols)
1322 {
1323 CPLAssert(node);
1324 if (node->is_primitive())
1325 {
1326 const auto it = oMapNodeToColIdx.find(node);
1327 if (it != oMapNodeToColIdx.end())
1328 anParquetCols.push_back(it->second);
1329 }
1330 else if (node->is_group())
1331 {
1332 const auto groupNode =
1333 cpl::down_cast<const parquet::schema::GroupNode *>(node);
1334 for (int i = 0; i < groupNode->field_count(); ++i)
1335 {
1336 CollectLeaveNodes(groupNode->field(i).get(), oMapNodeToColIdx,
1337 anParquetCols);
1338 }
1339 }
1340 }
1341
1342 /************************************************************************/
1343 /* GetParquetColumnIndicesForArrowField() */
1344 /************************************************************************/
1345
1346 std::vector<int> OGRParquetLayer::GetParquetColumnIndicesForArrowField(
1347 const std::string &arrowFieldName) const
1348 {
1349 const auto metadata = m_poArrowReader->parquet_reader()->metadata();
1350 const auto schema = metadata->schema();
1351
1352 std::vector<int> anParquetCols;
1353 const auto *rootNode = schema->schema_root().get();
1354 const parquet::schema::Node *fieldNode = nullptr;
1355 if (rootNode->is_group())
1356 {
1357 const auto groupNode =
1358 cpl::down_cast<const parquet::schema::GroupNode *>(rootNode);
1359 for (int i = 0; i < groupNode->field_count(); ++i)
1360 {
1361 if (groupNode->field(i).get()->name() == arrowFieldName)
1362 {
1363 fieldNode = groupNode->field(i).get();
1364 break;
1365 }
1366 }
1367 }
1368 if (!fieldNode)
1369 {
1370 CPLDebug("Parquet",
1371 "Cannot find Parquet node corresponding to Arrow field %s",
1372 arrowFieldName.c_str());
1373 return anParquetCols;
1374 }
1375
1376 /// Build mapping from schema node to column index
1377 std::map<const parquet::schema::Node *, int> oMapNodeToColIdx;
1378 const int num_cols = schema->num_columns();
1379 for (int i = 0; i < num_cols; ++i)
1380 {
1381 const auto *node = schema->Column(i)->schema_node().get();
1382 oMapNodeToColIdx[node] = i;
1383 }
1384
1385 CollectLeaveNodes(fieldNode, oMapNodeToColIdx, anParquetCols);
1386
1387 return anParquetCols;
1388 }
1389
1390 /************************************************************************/
1391 /* CheckMatchArrowParquetColumnNames() */
1392 /************************************************************************/
1393
1394 bool OGRParquetLayer::CheckMatchArrowParquetColumnNames(
1395 int &iParquetCol, const std::shared_ptr<arrow::Field> &field) const
1396 {
1397 const auto metadata = m_poArrowReader->parquet_reader()->metadata();
1398 const auto poParquetSchema = metadata->schema();
1399 const int nParquetColumns = poParquetSchema->num_columns();
1400 const auto &fieldName = field->name();
1401 const int iParquetColBefore = iParquetCol;
1402
1403 while (iParquetCol < nParquetColumns)
1404 {
1405 const auto parquetColumn = poParquetSchema->Column(iParquetCol);
1406 const auto parquetColumnName = parquetColumn->path()->ToDotString();
1407 if (fieldName == parquetColumnName ||
1408 (parquetColumnName.size() > fieldName.size() &&
1409 STARTS_WITH(parquetColumnName.c_str(), fieldName.c_str()) &&
1410 parquetColumnName[fieldName.size()] == '.'))
1411 {
1412 return true;
1413 }
1414 else
1415 {
1416 iParquetCol++;
1417 }
1418 }
1419
1420 CPLError(CE_Warning, CPLE_AppDefined,
1421 "Cannot match Arrow column name %s with a Parquet one",
1422 fieldName.c_str());
1423 iParquetCol = iParquetColBefore;
1424 return false;
1425 }
1426
1427 /************************************************************************/
1428 /* CreateFieldFromSchema() */
1429 /************************************************************************/
1430
1431 void OGRParquetLayer::CreateFieldFromSchema(
1432 const std::shared_ptr<arrow::Field> &field, bool bParquetColValid,
1433 int &iParquetCol, const std::vector<int> &path,
1434 const std::map<std::string, std::unique_ptr<OGRFieldDefn>>
1435 &oMapFieldNameToGDALSchemaFieldDefn)
1436 {
1437 OGRFieldDefn oField(field->name().c_str(), OFTString);
1438 OGRFieldType eType = OFTString;
1439 OGRFieldSubType eSubType = OFSTNone;
1440 bool bTypeOK = true;
1441
1442 auto type = field->type();
1443 if (type->id() == arrow::Type::DICTIONARY && path.size() == 1)
1444 {
1445 const auto dictionaryType =
1446 std::static_pointer_cast<arrow::DictionaryType>(field->type());
1447 auto indexType = dictionaryType->index_type();
1448 if (dictionaryType->value_type()->id() == arrow::Type::STRING &&
1449 IsIntegerArrowType(indexType->id()))
1450 {
1451 if (bParquetColValid)
1452 {
1453 std::string osDomainName(field->name() + "Domain");
1454 m_poDS->RegisterDomainName(osDomainName,
1455 m_poFeatureDefn->GetFieldCount());
1456 oField.SetDomainName(osDomainName);
1457 }
1458 type = std::move(indexType);
1459 }
1460 else
1461 {
1462 bTypeOK = false;
1463 }
1464 }
1465
1466 int nParquetColIncrement = 1;
1467 switch (type->id())
1468 {
1469 case arrow::Type::STRUCT:
1470 {
1471 const auto subfields = field->Flatten();
1472 const std::string osExtensionName =
1473 GetFieldExtensionName(field, type, GetDriverUCName().c_str());
1474 if (osExtensionName == EXTENSION_NAME_ARROW_TIMESTAMP_WITH_OFFSET &&
1475 subfields.size() == 2 &&
1476 subfields[0]->name() ==
1477 field->name() + "." + ATSWO_TIMESTAMP_FIELD_NAME &&
1478 subfields[0]->type()->id() == arrow::Type::TIMESTAMP &&
1479 subfields[1]->name() ==
1480 field->name() + "." + ATSWO_OFFSET_MINUTES_FIELD_NAME &&
1481 subfields[1]->type()->id() == arrow::Type::INT16)
1482 {
1483 oField.SetType(OFTDateTime);
1484 oField.SetTZFlag(OGR_TZFLAG_MIXED_TZ);
1485 oField.SetNullable(field->nullable());
1486 m_poFeatureDefn->AddFieldDefn(&oField);
1487 m_anMapFieldIndexToArrowColumn.push_back(path);
1488 m_apoArrowDataTypes.push_back(std::move(type));
1489 }
1490 else
1491 {
1492 auto newpath = path;
1493 newpath.push_back(0);
1494 for (int j = 0; j < static_cast<int>(subfields.size()); j++)
1495 {
1496 const auto &subfield = subfields[j];
1497 bParquetColValid = CheckMatchArrowParquetColumnNames(
1498 iParquetCol, subfield);
1499 if (!bParquetColValid)
1500 m_bHasMissingMappingToParquet = true;
1501 newpath.back() = j;
1502 CreateFieldFromSchema(subfield, bParquetColValid,
1503 iParquetCol, newpath,
1504 oMapFieldNameToGDALSchemaFieldDefn);
1505 }
1506 }
1507 return; // return intended, not break
1508 }
1509
1510 case arrow::Type::MAP:
1511 {
1512 // A arrow map maps to 2 Parquet columns
1513 nParquetColIncrement = 2;
1514 break;
1515 }
1516
1517 default:
1518 break;
1519 }
1520
1521 if (bTypeOK)
1522 {
1523 bTypeOK = MapArrowTypeToOGR(type, field, oField, eType, eSubType, path,
1524 oMapFieldNameToGDALSchemaFieldDefn);
1525 if (bTypeOK)
1526 {
1527 m_apoArrowDataTypes.push_back(std::move(type));
1528 }
1529 }
1530
1531 if (bParquetColValid)
1532 iParquetCol += nParquetColIncrement;
1533 }
1534
1535 /************************************************************************/
1536 /* BuildDomain() */
1537 /************************************************************************/
1538
1539 std::unique_ptr<OGRFieldDomain>
1540 OGRParquetLayer::BuildDomain(const std::string &osDomainName,
1541 int iFieldIndex) const
1542 {
1543 const int iArrowCol = m_anMapFieldIndexToArrowColumn[iFieldIndex][0];
1544 const std::string osArrowColName = m_poSchema->fields()[iArrowCol]->name();
1545 CPLAssert(m_poSchema->fields()[iArrowCol]->type()->id() ==
1546 arrow::Type::DICTIONARY);
1547 const auto anParquetColsForField =
1548 GetParquetColumnIndicesForArrowField(osArrowColName.c_str());
1549 CPLAssert(!anParquetColsForField.empty());
1550 const auto oldBatchSize = m_poArrowReader->properties().batch_size();
1551 m_poArrowReader->set_batch_size(1);
1552 #if PARQUET_VERSION_MAJOR >= 21
1553 std::unique_ptr<arrow::RecordBatchReader> poRecordBatchReader;
1554 auto result =
1555 m_poArrowReader->GetRecordBatchReader({0}, anParquetColsForField);
1556 if (result.ok())
1557 poRecordBatchReader = std::move(*result);
1558 #else
1559 std::shared_ptr<arrow::RecordBatchReader> poRecordBatchReader;
1560 CPL_IGNORE_RET_VAL(m_poArrowReader->GetRecordBatchReader(
1561 {0}, anParquetColsForField, &poRecordBatchReader));
1562 #endif
1563 if (poRecordBatchReader != nullptr)
1564 {
1565 std::shared_ptr<arrow::RecordBatch> poBatch;
1566 auto status = poRecordBatchReader->ReadNext(&poBatch);
1567 if (!status.ok())
1568 {
1569 CPLError(CE_Failure, CPLE_AppDefined, "ReadNext() failed: %s",
1570 status.message().c_str());
1571 }
1572 else if (poBatch)
1573 {
1574 m_poArrowReader->set_batch_size(oldBatchSize);
1575 return BuildDomainFromBatch(osDomainName, poBatch, 0);
1576 }
1577 }
1578 m_poArrowReader->set_batch_size(oldBatchSize);
1579 return nullptr;
1580 }
1581
1582 /************************************************************************/
1583 /* ComputeGeometryColumnType() */
1584 /************************************************************************/
1585
1586 OGRwkbGeometryType
1587 OGRParquetLayer::ComputeGeometryColumnType(int iGeomCol, int iParquetCol) const
1588 {
1589 // Compute type of geometry column by iterating over each geometry, and
1590 // looking at the WKB geometry type in the first 5 bytes of each geometry.
1591
1592 OGRwkbGeometryType eGeomType = wkbNone;
1593
1594 std::vector<int> anRowGroups;
1595 const int nNumGroups = m_poArrowReader->num_row_groups();
1596 anRowGroups.reserve(nNumGroups);
1597 for (int i = 0; i < nNumGroups; ++i)
1598 anRowGroups.push_back(i);
1599 #if PARQUET_VERSION_MAJOR >= 21
1600 std::unique_ptr<arrow::RecordBatchReader> poRecordBatchReader;
1601 auto result =
1602 m_poArrowReader->GetRecordBatchReader(anRowGroups, {iParquetCol});
1603 if (result.ok())
1604 poRecordBatchReader = std::move(*result);
1605 #else
1606 std::shared_ptr<arrow::RecordBatchReader> poRecordBatchReader;
1607 CPL_IGNORE_RET_VAL(m_poArrowReader->GetRecordBatchReader(
1608 anRowGroups, {iParquetCol}, &poRecordBatchReader));
1609 #endif
1610 if (poRecordBatchReader != nullptr)
1611 {
1612 std::shared_ptr<arrow::RecordBatch> poBatch;
1613 while (true)
1614 {
1615 auto status = poRecordBatchReader->ReadNext(&poBatch);
1616 if (!status.ok())
1617 {
1618 CPLError(CE_Failure, CPLE_AppDefined, "ReadNext() failed: %s",
1619 status.message().c_str());
1620 break;
1621 }
1622 else if (!poBatch)
1623 break;
1624
1625 eGeomType = ComputeGeometryColumnTypeProcessBatch(poBatch, iGeomCol,
1626 0, eGeomType);
1627 if (eGeomType == wkbUnknown)
1628 break;
1629 }
1630 }
1631
1632 return eGeomType == wkbNone ? wkbUnknown : eGeomType;
1633 }
1634
1635 /************************************************************************/
1636 /* GetFeatureExplicitFID() */
1637 /************************************************************************/
1638
1639 OGRFeature *OGRParquetLayer::GetFeatureExplicitFID(GIntBig nFID)
1640 {
1641 std::vector<int> anRowGroups;
1642 const int nNumGroups = m_poArrowReader->num_row_groups();
1643 anRowGroups.reserve(nNumGroups);
1644 for (int i = 0; i < nNumGroups; ++i)
1645 anRowGroups.push_back(i);
1646 #if PARQUET_VERSION_MAJOR >= 21
1647 std::unique_ptr<arrow::RecordBatchReader> poRecordBatchReader;
1648 auto result = m_bIgnoredFields
1649 ? m_poArrowReader->GetRecordBatchReader(
1650 anRowGroups, m_anRequestedParquetColumns)
1651 : m_poArrowReader->GetRecordBatchReader(anRowGroups);
1652 if (result.ok())
1653 {
1654 poRecordBatchReader = std::move(*result);
1655 }
1656 #else
1657 std::shared_ptr<arrow::RecordBatchReader> poRecordBatchReader;
1658 if (m_bIgnoredFields)
1659 {
1660 CPL_IGNORE_RET_VAL(m_poArrowReader->GetRecordBatchReader(
1661 anRowGroups, m_anRequestedParquetColumns, &poRecordBatchReader));
1662 }
1663 else
1664 {
1665 CPL_IGNORE_RET_VAL(m_poArrowReader->GetRecordBatchReader(
1666 anRowGroups, &poRecordBatchReader));
1667 }
1668 #endif
1669 if (poRecordBatchReader != nullptr)
1670 {
1671 std::shared_ptr<arrow::RecordBatch> poBatch;
1672 while (true)
1673 {
1674 auto status = poRecordBatchReader->ReadNext(&poBatch);
1675 if (!status.ok())
1676 {
1677 CPLError(CE_Failure, CPLE_AppDefined, "ReadNext() failed: %s",
1678 status.message().c_str());
1679 break;
1680 }
1681 else if (!poBatch)
1682 break;
1683
1684 const auto array = poBatch->column(
1685 m_bIgnoredFields ? m_nRequestedFIDColumn : m_iFIDArrowColumn);
1686 const auto arrayPtr = array.get();
1687 const auto arrayTypeId = array->type_id();
1688 for (int64_t nIdxInBatch = 0; nIdxInBatch < poBatch->num_rows();
1689 nIdxInBatch++)
1690 {
1691 if (!array->IsNull(nIdxInBatch))
1692 {
1693 if (arrayTypeId == arrow::Type::INT64)
1694 {
1695 const auto castArray =
1696 static_cast<const arrow::Int64Array *>(arrayPtr);
1697 if (castArray->Value(nIdxInBatch) == nFID)
1698 {
1699 return ReadFeature(nIdxInBatch, poBatch->columns());
1700 }
1701 }
1702 else if (arrayTypeId == arrow::Type::INT32)
1703 {
1704 const auto castArray =
1705 static_cast<const arrow::Int32Array *>(arrayPtr);
1706 if (castArray->Value(nIdxInBatch) == nFID)
1707 {
1708 return ReadFeature(nIdxInBatch, poBatch->columns());
1709 }
1710 }
1711 }
1712 }
1713 }
1714 }
1715 return nullptr;
1716 }
1717
1718 /************************************************************************/
1719 /* GetFeatureByIndex() */
1720 /************************************************************************/
1721
1722 OGRFeature *OGRParquetLayer::GetFeatureByIndex(GIntBig nFID)
1723 {
1724
1725 if (nFID < 0)
1726 return nullptr;
1727
1728 const auto metadata = m_poArrowReader->parquet_reader()->metadata();
1729 const int nNumGroups = m_poArrowReader->num_row_groups();
1730 int64_t nAccRows = 0;
1731 for (int iGroup = 0; iGroup < nNumGroups; ++iGroup)
1732 {
1733 const int64_t nNextAccRows =
1734 nAccRows + metadata->RowGroup(iGroup)->num_rows();
1735 if (nFID < nNextAccRows)
1736 {
1737 #if PARQUET_VERSION_MAJOR >= 21
1738 std::unique_ptr<arrow::RecordBatchReader> poRecordBatchReader;
1739 auto result = m_bIgnoredFields
1740 ? m_poArrowReader->GetRecordBatchReader(
1741 {iGroup}, m_anRequestedParquetColumns)
1742 : m_poArrowReader->GetRecordBatchReader({iGroup});
1743 if (result.ok())
1744 {
1745 poRecordBatchReader = std::move(*result);
1746 }
1747 else
1748 {
1749 CPLError(CE_Failure, CPLE_AppDefined,
1750 "GetRecordBatchReader() failed: %s",
1751 result.status().message().c_str());
1752 return nullptr;
1753 }
1754 #else
1755 std::shared_ptr<arrow::RecordBatchReader> poRecordBatchReader;
1756 {
1757 arrow::Status status;
1758 if (m_bIgnoredFields)
1759 {
1760 status = m_poArrowReader->GetRecordBatchReader(
1761 {iGroup}, m_anRequestedParquetColumns,
1762 &poRecordBatchReader);
1763 }
1764 else
1765 {
1766 status = m_poArrowReader->GetRecordBatchReader(
1767 {iGroup}, &poRecordBatchReader);
1768 }
1769 if (poRecordBatchReader == nullptr)
1770 {
1771 CPLError(CE_Failure, CPLE_AppDefined,
1772 "GetRecordBatchReader() failed: %s",
1773 status.message().c_str());
1774 return nullptr;
1775 }
1776 }
1777 #endif
1778
1779 const int64_t nExpectedIdxInGroup = nFID - nAccRows;
1780 int64_t nIdxInGroup = 0;
1781 while (true)
1782 {
1783 std::shared_ptr<arrow::RecordBatch> poBatch;
1784 arrow::Status status = poRecordBatchReader->ReadNext(&poBatch);
1785 if (!status.ok())
1786 {
1787 CPLError(CE_Failure, CPLE_AppDefined,
1788 "ReadNext() failed: %s", status.message().c_str());
1789 return nullptr;
1790 }
1791 if (poBatch == nullptr)
1792 {
1793 return nullptr;
1794 }
1795 if (nExpectedIdxInGroup < nIdxInGroup + poBatch->num_rows())
1796 {
1797 const auto nIdxInBatch = nExpectedIdxInGroup - nIdxInGroup;
1798 auto poFeature =
1799 ReadFeature(nIdxInBatch, poBatch->columns());
1800 poFeature->SetFID(nFID);
1801 return poFeature;
1802 }
1803 nIdxInGroup += poBatch->num_rows();
1804 }
1805 }
1806 nAccRows = nNextAccRows;
1807 }
1808 return nullptr;
1809 }
1810
1811 /************************************************************************/
1812 /* GetFeature() */
1813 /************************************************************************/
1814
1815 OGRFeature *OGRParquetLayer::GetFeature(GIntBig nFID)
1816 {
1817 if (!m_osFIDColumn.empty())
1818 {
1819 return GetFeatureExplicitFID(nFID);
1820 }
1821 else
1822 {
1823 return GetFeatureByIndex(nFID);
1824 }
1825 }
1826
1827 /************************************************************************/
1828 /* ResetReading() */
1829 /************************************************************************/
1830
1831 void OGRParquetLayer::ResetReading()
1832 {
1833 OGRParquetLayerBase::ResetReading();
1834 m_oFeatureIdxRemappingIter = m_asFeatureIdxRemapping.begin();
1835 m_nFeatureIdxSelected = 0;
1836 if (!m_asFeatureIdxRemapping.empty())
1837 {
1838 m_nFeatureIdx = m_oFeatureIdxRemappingIter->second;
1839 ++m_oFeatureIdxRemappingIter;
1840 }
1841 }
1842
1843 /************************************************************************/
1844 /* CreateRecordBatchReader() */
1845 /************************************************************************/
1846
1847 bool OGRParquetLayer::CreateRecordBatchReader(int iStartingRowGroup)
1848 {
1849 std::vector<int> anRowGroups;
1850 const int nNumGroups = m_poArrowReader->num_row_groups();
1851 anRowGroups.reserve(nNumGroups - iStartingRowGroup);
1852 for (int i = iStartingRowGroup; i < nNumGroups; ++i)
1853 anRowGroups.push_back(i);
1854 return CreateRecordBatchReader(anRowGroups);
1855 }
1856
1857 bool OGRParquetLayer::CreateRecordBatchReader(
1858 const std::vector<int> &anRowGroups)
1859 {
1860 #if PARQUET_VERSION_MAJOR >= 21
1861 auto result = m_bIgnoredFields
1862 ? m_poArrowReader->GetRecordBatchReader(
1863 anRowGroups, m_anRequestedParquetColumns)
1864 : m_poArrowReader->GetRecordBatchReader(anRowGroups);
1865 if (result.ok())
1866 {
1867 m_poRecordBatchReader = std::move(*result);
1868 return true;
1869 }
1870 else
1871 {
1872 CPLError(CE_Failure, CPLE_AppDefined,
1873 "GetRecordBatchReader() failed: %s",
1874 result.status().message().c_str());
1875 return false;
1876 }
1877 #else
1878 arrow::Status status;
1879 if (m_bIgnoredFields)
1880 {
1881 status = m_poArrowReader->GetRecordBatchReader(
1882 anRowGroups, m_anRequestedParquetColumns, &m_poRecordBatchReader);
1883 }
1884 else
1885 {
1886 status = m_poArrowReader->GetRecordBatchReader(anRowGroups,
1887 &m_poRecordBatchReader);
1888 }
1889 if (m_poRecordBatchReader == nullptr)
1890 {
1891 CPLError(CE_Failure, CPLE_AppDefined,
1892 "GetRecordBatchReader() failed: %s", status.message().c_str());
1893 return false;
1894 }
1895 return true;
1896 #endif
1897 }
1898
1899 /************************************************************************/
1900 /* IsConstraintPossible() */
1901 /************************************************************************/
1902
1903 enum class IsConstraintPossibleRes
1904 {
1905 YES,
1906 NO,
1907 UNKNOWN
1908 };
1909
1910 template <class T>
1911 static IsConstraintPossibleRes IsConstraintPossible(int nOperation, T v, T min,
1912 T max)
1913 {
1914 if (nOperation == SWQ_EQ)
1915 {
1916 if (v < min || v > max)
1917 {
1918 return IsConstraintPossibleRes::NO;
1919 }
1920 }
1921 else if (nOperation == SWQ_NE)
1922 {
1923 if (v == min && v == max)
1924 {
1925 return IsConstraintPossibleRes::NO;
1926 }
1927 }
1928 else if (nOperation == SWQ_LE)
1929 {
1930 if (v < min)
1931 {
1932 return IsConstraintPossibleRes::NO;
1933 }
1934 }
1935 else if (nOperation == SWQ_LT)
1936 {
1937 if (v <= min)
1938 {
1939 return IsConstraintPossibleRes::NO;
1940 }
1941 }
1942 else if (nOperation == SWQ_GE)
1943 {
1944 if (v > max)
1945 {
1946 return IsConstraintPossibleRes::NO;
1947 }
1948 }
1949 else if (nOperation == SWQ_GT)
1950 {
1951 if (v >= max)
1952 {
1953 return IsConstraintPossibleRes::NO;
1954 }
1955 }
1956 else
1957 {
1958 CPLDebug("PARQUET",
1959 "IsConstraintPossible: Unhandled operation type: %d",
1960 nOperation);
1961 return IsConstraintPossibleRes::UNKNOWN;
1962 }
1963 return IsConstraintPossibleRes::YES;
1964 }
1965
1966 /************************************************************************/
1967 /* IncrFeatureIdx() */
1968 /************************************************************************/
1969
1970 void OGRParquetLayer::IncrFeatureIdx()
1971 {
1972 ++m_nFeatureIdxSelected;
1973 ++m_nFeatureIdx;
1974 if (m_iFIDArrowColumn < 0 && !m_asFeatureIdxRemapping.empty() &&
1975 m_oFeatureIdxRemappingIter != m_asFeatureIdxRemapping.end())
1976 {
1977 if (m_nFeatureIdxSelected == m_oFeatureIdxRemappingIter->first)
1978 {
1979 m_nFeatureIdx = m_oFeatureIdxRemappingIter->second;
1980 ++m_oFeatureIdxRemappingIter;
1981 }
1982 }
1983 }
1984
1985 /************************************************************************/
1986 /* ReadNextBatch() */
1987 /************************************************************************/
1988
1989 bool OGRParquetLayer::ReadNextBatch()
1990 {
1991 m_nIdxInBatch = 0;
1992
1993 const int nNumGroups = m_poArrowReader->num_row_groups();
1994 if (nNumGroups == 0)
1995 return false;
1996
1997 if (m_bSingleBatch)
1998 {
1999 CPLAssert(m_iRecordBatch == 0);
2000 CPLAssert(m_poBatch != nullptr);
2001 return false;
2002 }
2003
2004 CPLAssert((m_iRecordBatch == -1 && m_poRecordBatchReader == nullptr) ||
2005 (m_iRecordBatch >= 0 && m_poRecordBatchReader != nullptr));
2006
2007 if (m_poRecordBatchReader == nullptr)
2008 {
2009 m_asFeatureIdxRemapping.clear();
2010
2011 bool bIterateEverything = false;
2012 std::vector<int> anSelectedGroups;
2013 const auto oIterToGeomColBBOX =
2014 m_oMapGeomFieldIndexToGeomColBBOXParquet.find(m_iGeomFieldFilter);
2015 const bool bUSEBBOXFields =
2016 (m_poFilterGeom &&
2017 oIterToGeomColBBOX !=
2018 m_oMapGeomFieldIndexToGeomColBBOXParquet.end() &&
2019 CPLTestBool(CPLGetConfigOption(
2020 ("OGR_" + GetDriverUCName() + "_USE_BBOX").c_str(), "YES")));
2021 const bool bIsGeoArrowStruct =
2022 (m_iGeomFieldFilter >= 0 &&
2023 m_iGeomFieldFilter < static_cast<int>(m_aeGeomEncoding.size()) &&
2024 m_iGeomFieldFilter <
2025 static_cast<int>(
2026 m_anMapGeomFieldIndexToParquetColumns.size()) &&
2027 m_anMapGeomFieldIndexToParquetColumns[m_iGeomFieldFilter].size() >=
2028 2 &&
2029 OGRArrowIsGeoArrowStruct(m_aeGeomEncoding[m_iGeomFieldFilter]));
2030 #if PARQUET_VERSION_MAJOR >= 21
2031 const bool bUseParquetGeoStat =
2032 (m_poFilterGeom && m_iGeomFieldFilter >= 0 &&
2033 m_geoStatsWithBBOXAvailable.find(m_iGeomFieldFilter) !=
2034 m_geoStatsWithBBOXAvailable.end() &&
2035 m_iGeomFieldFilter <
2036 static_cast<int>(
2037 m_anMapGeomFieldIndexToParquetColumns.size()) &&
2038 m_anMapGeomFieldIndexToParquetColumns[m_iGeomFieldFilter].size() ==
2039 1 &&
2040 m_anMapGeomFieldIndexToParquetColumns[m_iGeomFieldFilter][0] >= 0);
2041 #endif
2042 if (m_asAttributeFilterConstraints.empty() && !bUSEBBOXFields &&
2043 !(bIsGeoArrowStruct && m_poFilterGeom)
2044 #if PARQUET_VERSION_MAJOR >= 21
2045 && !bUseParquetGeoStat
2046 #endif
2047 )
2048 {
2049 bIterateEverything = true;
2050 }
2051 else
2052 {
2053 OGRField sMin;
2054 OGRField sMax;
2055 OGR_RawField_SetNull(&sMin);
2056 OGR_RawField_SetNull(&sMax);
2057 bool bFoundMin = false;
2058 bool bFoundMax = false;
2059 OGRFieldType eType = OFTMaxType;
2060 OGRFieldSubType eSubType = OFSTNone;
2061 std::string osMinTmp, osMaxTmp;
2062 int64_t nFeatureIdxSelected = 0;
2063 int64_t nFeatureIdxTotal = 0;
2064
2065 int iXMinField = -1;
2066 int iYMinField = -1;
2067 int iXMaxField = -1;
2068 int iYMaxField = -1;
2069
2070 if (bIsGeoArrowStruct)
2071 {
2072 const auto metadata =
2073 m_poArrowReader->parquet_reader()->metadata();
2074 const auto poParquetSchema = metadata->schema();
2075 for (int iParquetCol :
2076 m_anMapGeomFieldIndexToParquetColumns[m_iGeomFieldFilter])
2077 {
2078 const auto parquetColumn =
2079 poParquetSchema->Column(iParquetCol);
2080 const auto parquetColumnName =
2081 parquetColumn->path()->ToDotString();
2082 if (parquetColumnName.size() > 2 &&
2083 parquetColumnName.find(".x") ==
2084 parquetColumnName.size() - 2)
2085 {
2086 iXMinField = iParquetCol;
2087 iXMaxField = iParquetCol;
2088 }
2089 else if (parquetColumnName.size() > 2 &&
2090 parquetColumnName.find(".y") ==
2091 parquetColumnName.size() - 2)
2092 {
2093 iYMinField = iParquetCol;
2094 iYMaxField = iParquetCol;
2095 }
2096 }
2097 }
2098 else if (bUSEBBOXFields)
2099 {
2100 iXMinField = oIterToGeomColBBOX->second.iParquetXMin;
2101 iYMinField = oIterToGeomColBBOX->second.iParquetYMin;
2102 iXMaxField = oIterToGeomColBBOX->second.iParquetXMax;
2103 iYMaxField = oIterToGeomColBBOX->second.iParquetYMax;
2104 }
2105
2106 for (int iRowGroup = 0;
2107 iRowGroup < nNumGroups && !bIterateEverything; ++iRowGroup)
2108 {
2109 bool bSelectGroup = true;
2110 auto poRowGroup =
2111 GetReader()->parquet_reader()->RowGroup(iRowGroup);
2112
2113 if (iXMinField >= 0 && iYMinField >= 0 && iXMaxField >= 0 &&
2114 iYMaxField >= 0)
2115 {
2116 if (GetMinMaxForParquetCol(iRowGroup, iXMinField, nullptr,
2117 true, sMin, bFoundMin, false,
2118 sMax, bFoundMax, eType, eSubType,
2119 osMinTmp, osMaxTmp) &&
2120 bFoundMin && eType == OFTReal)
2121 {
2122 const double dfGroupMinX = sMin.Real;
2123 if (dfGroupMinX > m_sFilterEnvelope.MaxX)
2124 {
2125 bSelectGroup = false;
2126 }
2127 else if (GetMinMaxForParquetCol(
2128 iRowGroup, iYMinField, nullptr, true, sMin,
2129 bFoundMin, false, sMax, bFoundMax, eType,
2130 eSubType, osMinTmp, osMaxTmp) &&
2131 bFoundMin && eType == OFTReal)
2132 {
2133 const double dfGroupMinY = sMin.Real;
2134 if (dfGroupMinY > m_sFilterEnvelope.MaxY)
2135 {
2136 bSelectGroup = false;
2137 }
2138 else if (GetMinMaxForParquetCol(
2139 iRowGroup, iXMaxField, nullptr, false,
2140 sMin, bFoundMin, true, sMax, bFoundMax,
2141 eType, eSubType, osMinTmp, osMaxTmp) &&
2142 bFoundMax && eType == OFTReal)
2143 {
2144 const double dfGroupMaxX = sMax.Real;
2145 if (dfGroupMaxX < m_sFilterEnvelope.MinX)
2146 {
2147 bSelectGroup = false;
2148 }
2149 else if (GetMinMaxForParquetCol(
2150 iRowGroup, iYMaxField, nullptr,
2151 false, sMin, bFoundMin, true, sMax,
2152 bFoundMax, eType, eSubType,
2153 osMinTmp, osMaxTmp) &&
2154 bFoundMax && eType == OFTReal)
2155 {
2156 const double dfGroupMaxY = sMax.Real;
2157 if (dfGroupMaxY < m_sFilterEnvelope.MinY)
2158 {
2159 bSelectGroup = false;
2160 }
2161 }
2162 }
2163 }
2164 }
2165 }
2166 #if PARQUET_VERSION_MAJOR >= 21
2167 else if (bUseParquetGeoStat)
2168 {
2169 const int iParquetCol =
2170 m_anMapGeomFieldIndexToParquetColumns
2171 [m_iGeomFieldFilter][0];
2172 CPLAssert(iParquetCol >= 0);
2173
2174 const auto metadata =
2175 m_poArrowReader->parquet_reader()->metadata();
2176 const auto columnChunk =
2177 metadata->RowGroup(iRowGroup)->ColumnChunk(iParquetCol);
2178 if (auto geostats = columnChunk->geo_statistics())
2179 {
2180 if (geostats->dimension_valid()[0] &&
2181 geostats->dimension_valid()[1])
2182 {
2183 double dfMinX = geostats->lower_bound()[0];
2184 double dfMaxX = geostats->upper_bound()[0];
2185 double dfMinY = geostats->lower_bound()[1];
2186 double dfMaxY = geostats->upper_bound()[1];
2187
2188 // Deal as best as we can with wrap around bounding box
2189 if (dfMinX > dfMaxX && std::fabs(dfMinX) <= 180 &&
2190 std::fabs(dfMaxX) <= 180)
2191 {
2192 dfMinX = -180;
2193 dfMaxX = 180;
2194 }
2195
2196 // Check if there is an intersection between
2197 // the geostatistics for this rowgroup and
2198 // the bbox of interest
2199 if (dfMinX > m_sFilterEnvelope.MaxX ||
2200 dfMaxX < m_sFilterEnvelope.MinX ||
2201 dfMinY > m_sFilterEnvelope.MaxY ||
2202 dfMaxY < m_sFilterEnvelope.MinY)
2203 {
2204 bSelectGroup = false;
2205 }
2206 }
2207 }
2208 }
2209 #endif
2210
2211 if (bSelectGroup)
2212 {
2213 for (auto &constraint : m_asAttributeFilterConstraints)
2214 {
2215 int iOGRField = constraint.iField;
2216 if (constraint.iField ==
2217 m_poFeatureDefn->GetFieldCount() + SPF_FID)
2218 {
2219 iOGRField = OGR_FID_INDEX;
2220 }
2221 if (constraint.nOperation != SWQ_ISNULL &&
2222 constraint.nOperation != SWQ_ISNOTNULL)
2223 {
2224 if (iOGRField == OGR_FID_INDEX &&
2225 m_iFIDParquetColumn < 0)
2226 {
2227 sMin.Integer64 = nFeatureIdxTotal;
2228 sMax.Integer64 =
2229 nFeatureIdxTotal +
2230 poRowGroup->metadata()->num_rows() - 1;
2231 eType = OFTInteger64;
2232 }
2233 else if (!GetMinMaxForOGRField(
2234 iRowGroup, iOGRField, true, sMin,
2235 bFoundMin, true, sMax, bFoundMax,
2236 eType, eSubType, osMinTmp, osMaxTmp) ||
2237 !bFoundMin || !bFoundMax)
2238 {
2239 bIterateEverything = true;
2240 break;
2241 }
2242 }
2243
2244 IsConstraintPossibleRes res =
2245 IsConstraintPossibleRes::UNKNOWN;
2246 if (constraint.eType ==
2247 OGRArrowLayer::Constraint::Type::Integer &&
2248 eType == OFTInteger)
2249 {
2250 #if 0
2251 CPLDebug("PARQUET",
2252 "Group %d, field %s, min = %d, max = %d",
2253 iRowGroup,
2254 iOGRField == OGR_FID_INDEX
2255 ? m_osFIDColumn.c_str()
2256 : m_poFeatureDefn->GetFieldDefn(iOGRField)
2257 ->GetNameRef(),
2258 sMin.Integer, sMax.Integer);
2259 #endif
2260 res = IsConstraintPossible(
2261 constraint.nOperation,
2262 constraint.sValue.Integer, sMin.Integer,
2263 sMax.Integer);
2264 }
2265 else if (constraint.eType == OGRArrowLayer::Constraint::
2266 Type::Integer64 &&
2267 eType == OFTInteger64)
2268 {
2269 #if 0
2270 CPLDebug("PARQUET",
2271 "Group %d, field %s, min = " CPL_FRMT_GIB
2272 ", max = " CPL_FRMT_GIB,
2273 iRowGroup,
2274 iOGRField == OGR_FID_INDEX
2275 ? m_osFIDColumn.c_str()
2276 : m_poFeatureDefn->GetFieldDefn(iOGRField)
2277 ->GetNameRef(),
2278 static_cast<GIntBig>(sMin.Integer64),
2279 static_cast<GIntBig>(sMax.Integer64));
2280 #endif
2281 res = IsConstraintPossible(
2282 constraint.nOperation,
2283 constraint.sValue.Integer64, sMin.Integer64,
2284 sMax.Integer64);
2285 }
2286 else if (constraint.eType ==
2287 OGRArrowLayer::Constraint::Type::Real &&
2288 eType == OFTReal)
2289 {
2290 #if 0
2291 CPLDebug("PARQUET",
2292 "Group %d, field %s, min = %g, max = %g",
2293 iRowGroup,
2294 iOGRField == OGR_FID_INDEX
2295 ? m_osFIDColumn.c_str()
2296 : m_poFeatureDefn->GetFieldDefn(iOGRField)
2297 ->GetNameRef(),
2298 sMin.Real, sMax.Real);
2299 #endif
2300 res = IsConstraintPossible(constraint.nOperation,
2301 constraint.sValue.Real,
2302 sMin.Real, sMax.Real);
2303 }
2304 else if (constraint.eType ==
2305 OGRArrowLayer::Constraint::Type::String &&
2306 eType == OFTString)
2307 {
2308 #if 0
2309 CPLDebug("PARQUET",
2310 "Group %d, field %s, min = %s, max = %s",
2311 iRowGroup,
2312 iOGRField == OGR_FID_INDEX
2313 ? m_osFIDColumn.c_str()
2314 : m_poFeatureDefn->GetFieldDefn(iOGRField)
2315 ->GetNameRef(),
2316 sMin.String, sMax.String);
2317 #endif
2318 res = IsConstraintPossible(
2319 constraint.nOperation,
2320 std::string(constraint.sValue.String),
2321 std::string(sMin.String),
2322 std::string(sMax.String));
2323 }
2324 else if (constraint.nOperation == SWQ_ISNULL ||
2325 constraint.nOperation == SWQ_ISNOTNULL)
2326 {
2327 const std::vector<int> anCols =
2328 iOGRField == OGR_FID_INDEX
2329 ? std::vector<int>{m_iFIDParquetColumn}
2330 : GetParquetColumnIndicesForArrowField(
2331 GetLayerDefn()
2332 ->GetFieldDefn(iOGRField)
2333 ->GetNameRef());
2334 if (anCols.size() == 1 && anCols[0] >= 0)
2335 {
2336 const auto metadata =
2337 m_poArrowReader->parquet_reader()
2338 ->metadata();
2339 const auto rowGroupColumnChunk =
2340 metadata->RowGroup(iRowGroup)->ColumnChunk(
2341 anCols[0]);
2342 const auto rowGroupStats =
2343 rowGroupColumnChunk->statistics();
2344 if (rowGroupColumnChunk->is_stats_set() &&
2345 rowGroupStats)
2346 {
2347 res = IsConstraintPossibleRes::YES;
2348 if (constraint.nOperation == SWQ_ISNULL &&
2349 rowGroupStats->num_values() ==
2350 poRowGroup->metadata()->num_rows())
2351 {
2352 res = IsConstraintPossibleRes::NO;
2353 }
2354 else if (constraint.nOperation ==
2355 SWQ_ISNOTNULL &&
2356 rowGroupStats->num_values() == 0)
2357 {
2358 res = IsConstraintPossibleRes::NO;
2359 }
2360 }
2361 }
2362 }
2363 else
2364 {
2365 CPLDebug(
2366 "PARQUET",
2367 "Unhandled combination of constraint.eType "
2368 "(%d) and eType (%d)",
2369 static_cast<int>(constraint.eType), eType);
2370 }
2371
2372 if (res == IsConstraintPossibleRes::NO)
2373 {
2374 bSelectGroup = false;
2375 break;
2376 }
2377 else if (res == IsConstraintPossibleRes::UNKNOWN)
2378 {
2379 bIterateEverything = true;
2380 break;
2381 }
2382 }
2383 }
2384
2385 if (bSelectGroup)
2386 {
2387 // CPLDebug("PARQUET", "Selecting row group %d", iRowGroup);
2388 m_asFeatureIdxRemapping.emplace_back(
2389 std::make_pair(nFeatureIdxSelected, nFeatureIdxTotal));
2390 anSelectedGroups.push_back(iRowGroup);
2391 nFeatureIdxSelected += poRowGroup->metadata()->num_rows();
2392 }
2393
2394 nFeatureIdxTotal += poRowGroup->metadata()->num_rows();
2395 }
2396 }
2397
2398 if (bIterateEverything)
2399 {
2400 m_asFeatureIdxRemapping.clear();
2401 m_oFeatureIdxRemappingIter = m_asFeatureIdxRemapping.begin();
2402 if (!CreateRecordBatchReader(0))
2403 return false;
2404 }
2405 else
2406 {
2407 m_oFeatureIdxRemappingIter = m_asFeatureIdxRemapping.begin();
2408 if (anSelectedGroups.empty())
2409 {
2410 return false;
2411 }
2412 CPLDebug("PARQUET", "%d/%d row groups selected",
2413 int(anSelectedGroups.size()),
2414 m_poArrowReader->num_row_groups());
2415 m_nFeatureIdx = m_oFeatureIdxRemappingIter->second;
2416 ++m_oFeatureIdxRemappingIter;
2417 if (!CreateRecordBatchReader(anSelectedGroups))
2418 {
2419 return false;
2420 }
2421 }
2422 }
2423
2424 std::shared_ptr<arrow::RecordBatch> poNextBatch;
2425
2426 do
2427 {
2428 ++m_iRecordBatch;
2429 poNextBatch.reset();
2430 auto status = m_poRecordBatchReader->ReadNext(&poNextBatch);
2431 if (!status.ok())
2432 {
2433 CPLError(CE_Failure, CPLE_AppDefined, "ReadNext() failed: %s",
2434 status.message().c_str());
2435 poNextBatch.reset();
2436 }
2437 if (poNextBatch == nullptr)
2438 {
2439 if (m_iRecordBatch == 1 && m_poBatch && m_poAttrQuery == nullptr &&
2440 m_poFilterGeom == nullptr)
2441 {
2442 m_iRecordBatch = 0;
2443 m_bSingleBatch = true;
2444 }
2445 else
2446 m_poBatch.reset();
2447 return false;
2448 }
2449 } while (poNextBatch->num_rows() == 0);
2450
2451 SetBatch(poNextBatch);
2452
2453 return true;
2454 }
2455
2456 /************************************************************************/
2457 /* InvalidateCachedBatches() */
2458 /************************************************************************/
2459
2460 void OGRParquetLayer::InvalidateCachedBatches()
2461 {
2462 m_bSingleBatch = false;
2463 OGRParquetLayerBase::InvalidateCachedBatches();
2464 }
2465
2466 /************************************************************************/
2467 /* SetIgnoredFields() */
2468 /************************************************************************/
2469
2470 OGRErr OGRParquetLayer::SetIgnoredFields(CSLConstList papszFields)
2471 {
2472 m_bIgnoredFields = false;
2473 m_anRequestedParquetColumns.clear();
2474 m_anMapFieldIndexToArrayIndex.clear();
2475 m_anMapGeomFieldIndexToArrayIndex.clear();
2476 m_nRequestedFIDColumn = -1;
2477 OGRErr eErr = OGRLayer::SetIgnoredFields(papszFields);
2478 int nBatchColumns = 0;
2479 if (!m_bHasMissingMappingToParquet && eErr == OGRERR_NONE)
2480 {
2481 m_bIgnoredFields = papszFields != nullptr && papszFields[0] != nullptr;
2482 if (m_bIgnoredFields)
2483 {
2484 if (m_iFIDParquetColumn >= 0)
2485 {
2486 m_nRequestedFIDColumn = nBatchColumns;
2487 nBatchColumns++;
2488 m_anRequestedParquetColumns.push_back(m_iFIDParquetColumn);
2489 }
2490
2491 for (int i = 0; i < m_poFeatureDefn->GetFieldCount(); ++i)
2492 {
2493 const auto eArrowType =
2494 m_poSchema->fields()[m_anMapFieldIndexToArrowColumn[i][0]]
2495 ->type()
2496 ->id();
2497 if (eArrowType == arrow::Type::STRUCT)
2498 {
2499 // For a struct, for the sake of simplicity in
2500 // GetNextRawFeature(), as soon as one of the member if
2501 // requested, request all Parquet columns, so that the Arrow
2502 // type doesn't change
2503 bool bFoundNotIgnored = false;
2504 for (int j = i; j < m_poFeatureDefn->GetFieldCount() &&
2505 m_anMapFieldIndexToArrowColumn[i][0] ==
2506 m_anMapFieldIndexToArrowColumn[j][0];
2507 ++j)
2508 {
2509 if (!m_poFeatureDefn->GetFieldDefn(j)->IsIgnored())
2510 {
2511 bFoundNotIgnored = true;
2512 break;
2513 }
2514 }
2515 if (bFoundNotIgnored)
2516 {
2517 int j;
2518 for (j = i; j < m_poFeatureDefn->GetFieldCount() &&
2519 m_anMapFieldIndexToArrowColumn[i][0] ==
2520 m_anMapFieldIndexToArrowColumn[j][0];
2521 ++j)
2522 {
2523 if (!m_poFeatureDefn->GetFieldDefn(j)->IsIgnored())
2524 {
2525 m_anMapFieldIndexToArrayIndex.push_back(
2526 nBatchColumns);
2527 }
2528 else
2529 {
2530 m_anMapFieldIndexToArrayIndex.push_back(-1);
2531 }
2532
2533 const int iArrowCol =
2534 m_anMapFieldIndexToArrowColumn[i][0];
2535 const std::string osArrowColName =
2536 m_poSchema->fields()[iArrowCol]->name();
2537 const auto anParquetColsForField =
2538 GetParquetColumnIndicesForArrowField(
2539 osArrowColName.c_str());
2540 m_anRequestedParquetColumns.insert(
2541 m_anRequestedParquetColumns.end(),
2542 anParquetColsForField.begin(),
2543 anParquetColsForField.end());
2544 }
2545 i = j - 1;
2546 nBatchColumns++;
2547 }
2548 else
2549 {
2550 int j;
2551 for (j = i; j < m_poFeatureDefn->GetFieldCount() &&
2552 m_anMapFieldIndexToArrowColumn[i][0] ==
2553 m_anMapFieldIndexToArrowColumn[j][0];
2554 ++j)
2555 {
2556 m_anMapFieldIndexToArrayIndex.push_back(-1);
2557 }
2558 i = j - 1;
2559 }
2560 }
2561 else if (!m_poFeatureDefn->GetFieldDefn(i)->IsIgnored())
2562 {
2563 m_anMapFieldIndexToArrayIndex.push_back(nBatchColumns);
2564 nBatchColumns++;
2565 const int iArrowCol = m_anMapFieldIndexToArrowColumn[i][0];
2566 const std::string osArrowColName =
2567 m_poSchema->fields()[iArrowCol]->name();
2568 const auto anParquetColsForField =
2569 GetParquetColumnIndicesForArrowField(osArrowColName);
2570 m_anRequestedParquetColumns.insert(
2571 m_anRequestedParquetColumns.end(),
2572 anParquetColsForField.begin(),
2573 anParquetColsForField.end());
2574 }
2575 else
2576 {
2577 m_anMapFieldIndexToArrayIndex.push_back(-1);
2578 }
2579 }
2580
2581 CPLAssert(static_cast<int>(m_anMapFieldIndexToArrayIndex.size()) ==
2582 m_poFeatureDefn->GetFieldCount());
2583
2584 for (int i = 0; i < m_poFeatureDefn->GetGeomFieldCount(); ++i)
2585 {
2586 if (!m_poFeatureDefn->GetGeomFieldDefn(i)->IsIgnored())
2587 {
2588 const auto &anVals =
2589 m_anMapGeomFieldIndexToParquetColumns[i];
2590 CPLAssert(!anVals.empty() && anVals[0] >= 0);
2591 m_anRequestedParquetColumns.insert(
2592 m_anRequestedParquetColumns.end(), anVals.begin(),
2593 anVals.end());
2594 m_anMapGeomFieldIndexToArrayIndex.push_back(nBatchColumns);
2595 nBatchColumns++;
2596
2597 auto oIter = m_oMapGeomFieldIndexToGeomColBBOX.find(i);
2598 const auto oIterParquet =
2599 m_oMapGeomFieldIndexToGeomColBBOXParquet.find(i);
2600 if (oIter != m_oMapGeomFieldIndexToGeomColBBOX.end() &&
2601 oIterParquet !=
2602 m_oMapGeomFieldIndexToGeomColBBOXParquet.end())
2603 {
2604 oIter->second.iArrayIdx = nBatchColumns++;
2605 m_anRequestedParquetColumns.insert(
2606 m_anRequestedParquetColumns.end(),
2607 oIterParquet->second.anParquetCols.begin(),
2608 oIterParquet->second.anParquetCols.end());
2609 }
2610 }
2611 else
2612 {
2613 m_anMapGeomFieldIndexToArrayIndex.push_back(-1);
2614 }
2615 }
2616
2617 CPLAssert(
2618 static_cast<int>(m_anMapGeomFieldIndexToArrayIndex.size()) ==
2619 m_poFeatureDefn->GetGeomFieldCount());
2620 }
2621 }
2622
2623 m_nExpectedBatchColumns = m_bIgnoredFields ? nBatchColumns : -1;
2624
2625 ComputeConstraintsArrayIdx();
2626
2627 // Full invalidation
2628 InvalidateCachedBatches();
2629
2630 return eErr;
2631 }
2632
2633 /************************************************************************/
2634 /* GetFeatureCount() */
2635 /************************************************************************/
2636
2637 GIntBig OGRParquetLayer::GetFeatureCount(int bForce)
2638 {
2639 if (m_poAttrQuery == nullptr && m_poFilterGeom == nullptr)
2640 {
2641 auto metadata = m_poArrowReader->parquet_reader()->metadata();
2642 if (metadata)
2643 return metadata->num_rows();
2644 }
2645 return OGRLayer::GetFeatureCount(bForce);
2646 }
2647
2648 /************************************************************************/
2649 /* FastGetExtent() */
2650 /************************************************************************/
2651
2652 bool OGRParquetLayer::FastGetExtent(int iGeomField, OGREnvelope *psExtent) const
2653 {
2654 if (OGRParquetLayerBase::FastGetExtent(iGeomField, psExtent))
2655 return true;
2656
2657 const auto oIterToGeomColBBOX =
2658 m_oMapGeomFieldIndexToGeomColBBOXParquet.find(iGeomField);
2659 if (oIterToGeomColBBOX != m_oMapGeomFieldIndexToGeomColBBOXParquet.end() &&
2660 CPLTestBool(CPLGetConfigOption("OGR_PARQUET_USE_BBOX", "YES")))
2661 {
2662 OGREnvelope sExtent;
2663 OGRField sMin, sMax;
2664 OGR_RawField_SetNull(&sMin);
2665 OGR_RawField_SetNull(&sMax);
2666 bool bFoundMin, bFoundMax;
2667 OGRFieldType eType = OFTMaxType;
2668 OGRFieldSubType eSubType = OFSTNone;
2669 std::string osMinTmp, osMaxTmp;
2670 if (GetMinMaxForParquetCol(-1, oIterToGeomColBBOX->second.iParquetXMin,
2671 nullptr, true, sMin, bFoundMin, false, sMax,
2672 bFoundMax, eType, eSubType, osMinTmp,
2673 osMaxTmp) &&
2674 eType == OFTReal)
2675 {
2676 sExtent.MinX = sMin.Real;
2677
2678 if (GetMinMaxForParquetCol(
2679 -1, oIterToGeomColBBOX->second.iParquetYMin, nullptr, true,
2680 sMin, bFoundMin, false, sMax, bFoundMax, eType, eSubType,
2681 osMinTmp, osMaxTmp) &&
2682 eType == OFTReal)
2683 {
2684 sExtent.MinY = sMin.Real;
2685
2686 if (GetMinMaxForParquetCol(
2687 -1, oIterToGeomColBBOX->second.iParquetXMax, nullptr,
2688 false, sMin, bFoundMin, true, sMax, bFoundMax, eType,
2689 eSubType, osMinTmp, osMaxTmp) &&
2690 eType == OFTReal)
2691 {
2692 sExtent.MaxX = sMax.Real;
2693
2694 if (GetMinMaxForParquetCol(
2695 -1, oIterToGeomColBBOX->second.iParquetYMax,
2696 nullptr, false, sMin, bFoundMin, true, sMax,
2697 bFoundMax, eType, eSubType, osMinTmp, osMaxTmp) &&
2698 eType == OFTReal)
2699 {
2700 sExtent.MaxY = sMax.Real;
2701
2702 CPLDebug("PARQUET",
2703 "Using statistics of bbox.minx, bbox.miny, "
2704 "bbox.maxx, bbox.maxy columns to get extent");
2705 m_oMapExtents[iGeomField] = sExtent;
2706 *psExtent = sExtent;
2707 return true;
2708 }
2709 }
2710 }
2711 }
2712 }
2713
2714 return false;
2715 }
2716
2717 /************************************************************************/
2718 /* TestCapability() */
2719 /************************************************************************/
2720
2721 int OGRParquetLayer::TestCapability(const char *pszCap) const
2722 {
2723 if (EQUAL(pszCap, OLCFastFeatureCount))
2724 return m_poAttrQuery == nullptr && m_poFilterGeom == nullptr;
2725
2726 if (EQUAL(pszCap, OLCIgnoreFields))
2727 return !m_bHasMissingMappingToParquet;
2728
2729 if (EQUAL(pszCap, OLCFastSpatialFilter))
2730 {
2731 if (m_iGeomFieldFilter >= 0 &&
2732 m_iGeomFieldFilter < static_cast<int>(m_aeGeomEncoding.size()) &&
2733 OGRArrowIsGeoArrowStruct(m_aeGeomEncoding[m_iGeomFieldFilter]))
2734 {
2735 return true;
2736 }
2737
2738 #if PARQUET_VERSION_MAJOR >= 21
2739 if (m_iGeomFieldFilter >= 0 &&
2740 m_iGeomFieldFilter < static_cast<int>(m_aeGeomEncoding.size()) &&
2741 m_aeGeomEncoding[m_iGeomFieldFilter] == OGRArrowGeomEncoding::WKB &&
2742 m_iGeomFieldFilter <
2743 static_cast<int>(
2744 m_anMapGeomFieldIndexToParquetColumns.size()) &&
2745 m_anMapGeomFieldIndexToParquetColumns[m_iGeomFieldFilter].size() ==
2746 1)
2747 {
2748 const int iParquetCol =
2749 m_anMapGeomFieldIndexToParquetColumns[m_iGeomFieldFilter][0];
2750 if (iParquetCol >= 0)
2751 {
2752 const auto metadata =
2753 m_poArrowReader->parquet_reader()->metadata();
2754
2755 int nCountRowGroupsStatsValid = 0;
2756 const int nNumGroups = m_poArrowReader->num_row_groups();
2757 for (int iRowGroup = 0; iRowGroup < nNumGroups &&
2758 nCountRowGroupsStatsValid == iRowGroup;
2759 ++iRowGroup)
2760 {
2761 const auto columnChunk =
2762 metadata->RowGroup(iRowGroup)->ColumnChunk(iParquetCol);
2763 if (auto geostats = columnChunk->geo_statistics())
2764 {
2765 if (geostats->dimension_valid()[0] &&
2766 geostats->dimension_valid()[1])
2767 {
2768 const double dfMinX = geostats->lower_bound()[0];
2769 const double dfMaxX = geostats->upper_bound()[0];
2770 const double dfMinY = geostats->lower_bound()[1];
2771 const double dfMaxY = geostats->upper_bound()[1];
2772 if (std::isfinite(dfMinX) &&
2773 std::isfinite(dfMaxX) &&
2774 std::isfinite(dfMinY) && std::isfinite(dfMaxY))
2775 {
2776 nCountRowGroupsStatsValid++;
2777 }
2778 }
2779 }
2780 }
2781 if (nCountRowGroupsStatsValid == nNumGroups)
2782 {
2783 return true;
2784 }
2785 }
2786 }
2787 #endif
2788
2789 // fallback to base method
2790 }
2791
2792 return OGRParquetLayerBase::TestCapability(pszCap);
2793 }
2794
2795 /************************************************************************/
2796 /* GetMetadataItem() */
2797 /************************************************************************/
2798
2799 const char *OGRParquetLayer::GetMetadataItem(const char *pszName,
2800 const char *pszDomain)
2801 {
2802 // Mostly for unit test purposes
2803 if (pszDomain != nullptr && EQUAL(pszDomain, "_PARQUET_"))
2804 {
2805 int nRowGroupIdx = -1;
2806 int nColumn = -1;
2807 if (EQUAL(pszName, "NUM_ROW_GROUPS"))
2808 {
2809 return CPLSPrintf("%d", m_poArrowReader->num_row_groups());
2810 }
2811 if (EQUAL(pszName, "CREATOR"))
2812 {
2813 return CPLSPrintf("%s", m_poArrowReader->parquet_reader()
2814 ->metadata()
2815 ->created_by()
2816 .c_str());
2817 }
2818 else if (sscanf(pszName, "ROW_GROUPS[%d]", &nRowGroupIdx) == 1 &&
2819 strstr(pszName, ".NUM_ROWS"))
2820 {
2821 try
2822 {
2823 auto poRowGroup =
2824 m_poArrowReader->parquet_reader()->RowGroup(nRowGroupIdx);
2825 if (poRowGroup == nullptr)
2826 return nullptr;
2827 return CPLSPrintf("%" PRId64,
2828 poRowGroup->metadata()->num_rows());
2829 }
2830 catch (const std::exception &)
2831 {
2832 }
2833 }
2834 else if (sscanf(pszName, "ROW_GROUPS[%d].COLUMNS[%d]", &nRowGroupIdx,
2835 &nColumn) == 2 &&
2836 strstr(pszName, ".COMPRESSION"))
2837 {
2838 try
2839 {
2840 auto poRowGroup =
2841 m_poArrowReader->parquet_reader()->RowGroup(nRowGroupIdx);
2842 if (poRowGroup == nullptr)
2843 return nullptr;
2844 auto poColumn = poRowGroup->metadata()->ColumnChunk(nColumn);
2845 return CPLSPrintf("%s", arrow::util::Codec::GetCodecAsString(
2846 poColumn->compression())
2847 .c_str());
2848 }
2849 catch (const std::exception &)
2850 {
2851 }
2852 }
2853 return nullptr;
2854 }
2855 if (pszDomain != nullptr && EQUAL(pszDomain, "_PARQUET_METADATA_"))
2856 {
2857 const auto metadata = m_poArrowReader->parquet_reader()->metadata();
2858 const auto &kv_metadata = metadata->key_value_metadata();
2859 if (kv_metadata && kv_metadata->Contains(pszName))
2860 {
2861 auto metadataItem = kv_metadata->Get(pszName);
2862 if (metadataItem.ok())
2863 {
2864 return CPLSPrintf("%s", metadataItem->c_str());
2865 }
2866 }
2867 return nullptr;
2868 }
2869 return OGRLayer::GetMetadataItem(pszName, pszDomain);
2870 }
2871
2872 /************************************************************************/
2873 /* GetMetadata() */
2874 /************************************************************************/
2875
2876 CSLConstList OGRParquetLayer::GetMetadata(const char *pszDomain)
2877 {
2878 // Mostly for unit test purposes
2879 if (pszDomain != nullptr && EQUAL(pszDomain, "_PARQUET_METADATA_"))
2880 {
2881 m_aosFeatherMetadata.Clear();
2882 const auto metadata = m_poArrowReader->parquet_reader()->metadata();
2883 const auto &kv_metadata = metadata->key_value_metadata();
2884 if (kv_metadata)
2885 {
2886 for (const auto &kv : kv_metadata->sorted_pairs())
2887 {
2888 m_aosFeatherMetadata.SetNameValue(kv.first.c_str(),
2889 kv.second.c_str());
2890 }
2891 }
2892 return m_aosFeatherMetadata.List();
2893 }
2894
2895 // Mostly for unit test purposes
2896 if (pszDomain != nullptr && EQUAL(pszDomain, "_GDAL_CREATION_OPTIONS_"))
2897 {
2898 return m_aosCreationOptions.List();
2899 }
2900
2901 return OGRLayer::GetMetadata(pszDomain);
2902 }
2903
2904 /************************************************************************/
2905 /* GetArrowStream() */
2906 /************************************************************************/
2907
2908 bool OGRParquetLayer::GetArrowStream(struct ArrowArrayStream *out_stream,
2909 CSLConstList papszOptions)
2910 {
2911 const char *pszMaxFeaturesInBatch =
2912 CSLFetchNameValue(papszOptions, "MAX_FEATURES_IN_BATCH");
2913 if (pszMaxFeaturesInBatch)
2914 {
2915 int nMaxBatchSize = atoi(pszMaxFeaturesInBatch);
2916 if (nMaxBatchSize <= 0)
2917 nMaxBatchSize = 1;
2918 if (nMaxBatchSize > INT_MAX - 1)
2919 nMaxBatchSize = INT_MAX - 1;
2920 m_poArrowReader->set_batch_size(nMaxBatchSize);
2921 }
2922 return OGRArrowLayer::GetArrowStream(out_stream, papszOptions);
2923 }
2924
2925 /************************************************************************/
2926 /* SetNextByIndex() */
2927 /************************************************************************/
2928
2929 OGRErr OGRParquetLayer::SetNextByIndex(GIntBig nIndex)
2930 {
2931 if (nIndex < 0)
2932 {
2933 m_bEOF = true;
2934 return OGRERR_NON_EXISTING_FEATURE;
2935 }
2936
2937 const auto metadata = m_poArrowReader->parquet_reader()->metadata();
2938 if (nIndex >= metadata->num_rows())
2939 {
2940 m_bEOF = true;
2941 return OGRERR_NON_EXISTING_FEATURE;
2942 }
2943
2944 m_bEOF = false;
2945
2946 if (m_bSingleBatch)
2947 {
2948 ResetReading();
2949 m_nIdxInBatch = nIndex;
2950 m_nFeatureIdx = nIndex;
2951 return OGRERR_NONE;
2952 }
2953
2954 const int nNumGroups = m_poArrowReader->num_row_groups();
2955 int64_t nAccRows = 0;
2956 const auto nBatchSize = m_poArrowReader->properties().batch_size();
2957 m_iRecordBatch = -1;
2958 ResetReading();
2959 m_iRecordBatch = 0;
2960 for (int iGroup = 0; iGroup < nNumGroups; ++iGroup)
2961 {
2962 const auto nRowsInRowGroup = metadata->RowGroup(iGroup)->num_rows();
2963 const int64_t nNextAccRows = nAccRows + nRowsInRowGroup;
2964 if (nIndex < nNextAccRows)
2965 {
2966 if (!CreateRecordBatchReader(iGroup))
2967 return OGRERR_FAILURE;
2968
2969 std::shared_ptr<arrow::RecordBatch> poBatch;
2970 while (true)
2971 {
2972 auto status = m_poRecordBatchReader->ReadNext(&poBatch);
2973 if (!status.ok())
2974 {
2975 CPLError(CE_Failure, CPLE_AppDefined,
2976 "ReadNext() failed: %s", status.message().c_str());
2977 m_iRecordBatch = -1;
2978 ResetReading();
2979 return OGRERR_FAILURE;
2980 }
2981 if (poBatch == nullptr)
2982 {
2983 m_iRecordBatch = -1;
2984 ResetReading();
2985 return OGRERR_FAILURE;
2986 }
2987 if (nIndex < nAccRows + poBatch->num_rows())
2988 {
2989 break;
2990 }
2991 nAccRows += poBatch->num_rows();
2992 m_iRecordBatch++;
2993 }
2994 m_nIdxInBatch = nIndex - nAccRows;
2995 m_nFeatureIdx = nIndex;
2996 SetBatch(poBatch);
2997 return OGRERR_NONE;
2998 }
2999 nAccRows = nNextAccRows;
3000 m_iRecordBatch +=
3001 static_cast<int>(cpl::div_round_up(nRowsInRowGroup, nBatchSize));
3002 }
3003
3004 m_iRecordBatch = -1;
3005 ResetReading();
3006 return OGRERR_FAILURE;
3007 }
3008
3009 /************************************************************************/
3010 /* GetStats() */
3011 /************************************************************************/
3012
3013 template <class STAT_TYPE> struct GetStats
3014 {
3015 using T = typename STAT_TYPE::T;
3016
3017 static T min(const std::shared_ptr<parquet::FileMetaData> &metadata,
3018 const int iRowGroup, const int numRowGroups, const int iCol,
3019 bool &bFound)
3020 {
3021 T v{};
3022 bFound = false;
3023 for (int i = 0; i < (iRowGroup < 0 ? numRowGroups : 1); i++)
3024 {
3025 const auto columnChunk =
3026 metadata->RowGroup(iRowGroup < 0 ? i : iRowGroup)
3027 ->ColumnChunk(iCol);
3028 const auto colStats = columnChunk->statistics();
3029 if (columnChunk->is_stats_set() && colStats &&
3030 colStats->HasMinMax())
3031 {
3032 auto castStats = static_cast<STAT_TYPE *>(colStats.get());
3033 const auto rowGroupVal = castStats->min();
3034 if (i == 0 || rowGroupVal < v)
3035 {
3036 bFound = true;
3037 v = rowGroupVal;
3038 }
3039 }
3040 else if (columnChunk->num_values() > 0)
3041 {
3042 bFound = false;
3043 break;
3044 }
3045 }
3046 return v;
3047 }
3048
3049 static T max(const std::shared_ptr<parquet::FileMetaData> &metadata,
3050 const int iRowGroup, const int numRowGroups, const int iCol,
3051 bool &bFound)
3052 {
3053 T v{};
3054 bFound = false;
3055 for (int i = 0; i < (iRowGroup < 0 ? numRowGroups : 1); i++)
3056 {
3057 const auto columnChunk =
3058 metadata->RowGroup(iRowGroup < 0 ? i : iRowGroup)
3059 ->ColumnChunk(iCol);
3060 const auto colStats = columnChunk->statistics();
3061 if (columnChunk->is_stats_set() && colStats &&
3062 colStats->HasMinMax())
3063 {
3064 auto castStats = static_cast<STAT_TYPE *>(colStats.get());
3065 const auto rowGroupVal = castStats->max();
3066 if (i == 0 || rowGroupVal > v)
3067 {
3068 bFound = true;
3069 v = rowGroupVal;
3070 }
3071 }
3072 else if (columnChunk->num_values() > 0)
3073 {
3074 bFound = false;
3075 break;
3076 }
3077 }
3078 return v;
3079 }
3080 };
3081
3082 template <> struct GetStats<parquet::ByteArrayStatistics>
3083 {
3084 static std::string
3085 min(const std::shared_ptr<parquet::FileMetaData> &metadata,
3086 const int iRowGroup, const int numRowGroups, const int iCol,
3087 bool &bFound)
3088 {
3089 std::string v{};
3090 bFound = false;
3091 for (int i = 0; i < (iRowGroup < 0 ? numRowGroups : 1); i++)
3092 {
3093 const auto columnChunk =
3094 metadata->RowGroup(iRowGroup < 0 ? i : iRowGroup)
3095 ->ColumnChunk(iCol);
3096 const auto colStats = columnChunk->statistics();
3097 if (columnChunk->is_stats_set() && colStats &&
3098 colStats->HasMinMax())
3099 {
3100 auto castStats =
3101 static_cast<parquet::ByteArrayStatistics *>(colStats.get());
3102 const auto rowGroupValRaw = castStats->min();
3103 std::string rowGroupVal(
3104 reinterpret_cast<const char *>(rowGroupValRaw.ptr),
3105 rowGroupValRaw.len);
3106 if (i == 0 || rowGroupVal < v)
3107 {
3108 bFound = true;
3109 v = std::move(rowGroupVal);
3110 }
3111 }
3112 }
3113 return v;
3114 }
3115
3116 static std::string
3117 max(const std::shared_ptr<parquet::FileMetaData> &metadata,
3118 const int iRowGroup, const int numRowGroups, const int iCol,
3119 bool &bFound)
3120 {
3121 std::string v{};
3122 bFound = false;
3123 for (int i = 0; i < (iRowGroup < 0 ? numRowGroups : 1); i++)
3124 {
3125 const auto columnChunk =
3126 metadata->RowGroup(iRowGroup < 0 ? i : iRowGroup)
3127 ->ColumnChunk(iCol);
3128 const auto colStats = columnChunk->statistics();
3129 if (columnChunk->is_stats_set() && colStats &&
3130 colStats->HasMinMax())
3131 {
3132 auto castStats =
3133 static_cast<parquet::ByteArrayStatistics *>(colStats.get());
3134 const auto rowGroupValRaw = castStats->max();
3135 std::string rowGroupVal(
3136 reinterpret_cast<const char *>(rowGroupValRaw.ptr),
3137 rowGroupValRaw.len);
3138 if (i == 0 || rowGroupVal > v)
3139 {
3140 bFound = true;
3141 v = std::move(rowGroupVal);
3142 }
3143 }
3144 else
3145 {
3146 bFound = false;
3147 break;
3148 }
3149 }
3150 return v;
3151 }
3152 };
3153
3154 /************************************************************************/
3155 /* GetMinMaxForOGRField() */
3156 /************************************************************************/
3157
3158 bool OGRParquetLayer::GetMinMaxForOGRField(int iRowGroup, // -1 for all
3159 int iOGRField, bool bComputeMin,
3160 OGRField &sMin, bool &bFoundMin,
3161 bool bComputeMax, OGRField &sMax,
3162 bool &bFoundMax, OGRFieldType &eType,
3163 OGRFieldSubType &eSubType,
3164 std::string &osMinTmp,
3165 std::string &osMaxTmp) const
3166 {
3167 OGR_RawField_SetNull(&sMin);
3168 OGR_RawField_SetNull(&sMax);
3169 eType = OFTReal;
3170 eSubType = OFSTNone;
3171 bFoundMin = false;
3172 bFoundMax = false;
3173
3174 const std::vector<int> anCols =
3175 iOGRField == OGR_FID_INDEX
3176 ? std::vector<int>{m_iFIDParquetColumn}
3177 : GetParquetColumnIndicesForArrowField(
3178 GetLayerDefn()->GetFieldDefn(iOGRField)->GetNameRef());
3179 if (anCols.empty() || anCols[0] < 0)
3180 return false;
3181 const int iCol = anCols[0];
3182 const auto &arrowType = iOGRField == OGR_FID_INDEX
3183 ? m_poFIDType
3184 : GetArrowFieldTypes()[iOGRField];
3185
3186 const bool bRet = GetMinMaxForParquetCol(
3187 iRowGroup, iCol, arrowType, bComputeMin, sMin, bFoundMin, bComputeMax,
3188 sMax, bFoundMax, eType, eSubType, osMinTmp, osMaxTmp);
3189
3190 if (eType == OFTInteger64 && arrowType->id() == arrow::Type::TIMESTAMP)
3191 {
3192 const OGRFieldDefn oDummyFIDFieldDefn(m_osFIDColumn.c_str(),
3193 OFTInteger64);
3194 const OGRFieldDefn *poFieldDefn =
3195 iOGRField == OGR_FID_INDEX ? &oDummyFIDFieldDefn
3196 : const_cast<OGRParquetLayer *>(this)
3197 ->GetLayerDefn()
3198 ->GetFieldDefn(iOGRField);
3199 if (poFieldDefn->GetType() == OFTDateTime)
3200 {
3201 const auto timestampType =
3202 static_cast<arrow::TimestampType *>(arrowType.get());
3203 if (bFoundMin)
3204 {
3205 const int64_t timestamp = sMin.Integer64;
3206 OGRArrowLayer::TimestampToOGR(timestamp, timestampType,
3207 poFieldDefn->GetTZFlag(), &sMin);
3208 }
3209 if (bFoundMax)
3210 {
3211 const int64_t timestamp = sMax.Integer64;
3212 OGRArrowLayer::TimestampToOGR(timestamp, timestampType,
3213 poFieldDefn->GetTZFlag(), &sMax);
3214 }
3215 eType = OFTDateTime;
3216 }
3217 }
3218
3219 return bRet;
3220 }
3221
3222 /************************************************************************/
3223 /* GetMinMaxForParquetCol() */
3224 /************************************************************************/
3225
3226 bool OGRParquetLayer::GetMinMaxForParquetCol(
3227 int iRowGroup, // -1 for all
3228 int iCol,
3229 const std::shared_ptr<arrow::DataType> &arrowType, // potentially nullptr
3230 bool bComputeMin, OGRField &sMin, bool &bFoundMin, bool bComputeMax,
3231 OGRField &sMax, bool &bFoundMax, OGRFieldType &eType,
3232 OGRFieldSubType &eSubType, std::string &osMinTmp,
3233 std::string &osMaxTmp) const
3234 {
3235 OGR_RawField_SetNull(&sMin);
3236 OGR_RawField_SetNull(&sMax);
3237 eType = OFTReal;
3238 eSubType = OFSTNone;
3239 bFoundMin = false;
3240 bFoundMax = false;
3241
3242 const auto metadata = GetReader()->parquet_reader()->metadata();
3243 const auto numRowGroups = metadata->num_row_groups();
3244
3245 if (numRowGroups == 0)
3246 return false;
3247
3248 const auto rowGroup0 = metadata->RowGroup(0);
3249 if (iCol < 0 || iCol >= rowGroup0->num_columns())
3250 {
3251 CPLError(CE_Failure, CPLE_AppDefined,
3252 "GetMinMaxForParquetCol(): invalid iCol=%d", iCol);
3253 return false;
3254 }
3255 const auto rowGroup0columnChunk = rowGroup0->ColumnChunk(iCol);
3256 const auto rowGroup0Stats = rowGroup0columnChunk->statistics();
3257 if (!(rowGroup0columnChunk->is_stats_set() && rowGroup0Stats))
3258 {
3259 CPLDebug("PARQUET", "Statistics not available for field %s",
3260 rowGroup0columnChunk->path_in_schema()->ToDotString().c_str());
3261 return false;
3262 }
3263
3264 const auto physicalType = rowGroup0Stats->physical_type();
3265
3266 if (bComputeMin)
3267 {
3268 if (physicalType == parquet::Type::BOOLEAN)
3269 {
3270 eType = OFTInteger;
3271 eSubType = OFSTBoolean;
3272 sMin.Integer = GetStats<parquet::BoolStatistics>::min(
3273 metadata, iRowGroup, numRowGroups, iCol, bFoundMin);
3274 }
3275 else if (physicalType == parquet::Type::INT32)
3276 {
3277 if (arrowType && arrowType->id() == arrow::Type::UINT32)
3278 {
3279 // With parquet file version 2.0,
3280 // statistics of uint32 fields are
3281 // stored as signed int32 values...
3282 eType = OFTInteger64;
3283 int nVal = GetStats<parquet::Int32Statistics>::min(
3284 metadata, iRowGroup, numRowGroups, iCol, bFoundMin);
3285 if (bFoundMin)
3286 {
3287 sMin.Integer64 = static_cast<uint32_t>(nVal);
3288 }
3289 }
3290 else
3291 {
3292 eType = OFTInteger;
3293 if (arrowType && arrowType->id() == arrow::Type::INT16)
3294 eSubType = OFSTInt16;
3295 sMin.Integer = GetStats<parquet::Int32Statistics>::min(
3296 metadata, iRowGroup, numRowGroups, iCol, bFoundMin);
3297 }
3298 }
3299 else if (physicalType == parquet::Type::INT64)
3300 {
3301 eType = OFTInteger64;
3302 sMin.Integer64 = GetStats<parquet::Int64Statistics>::min(
3303 metadata, iRowGroup, numRowGroups, iCol, bFoundMin);
3304 }
3305 else if (physicalType == parquet::Type::FLOAT)
3306 {
3307 eType = OFTReal;
3308 eSubType = OFSTFloat32;
3309 sMin.Real = GetStats<parquet::FloatStatistics>::min(
3310 metadata, iRowGroup, numRowGroups, iCol, bFoundMin);
3311 }
3312 else if (physicalType == parquet::Type::DOUBLE)
3313 {
3314 eType = OFTReal;
3315 sMin.Real = GetStats<parquet::DoubleStatistics>::min(
3316 metadata, iRowGroup, numRowGroups, iCol, bFoundMin);
3317 }
3318 else if (arrowType &&
3319 (arrowType->id() == arrow::Type::STRING ||
3320 arrowType->id() == arrow::Type::LARGE_STRING) &&
3321 physicalType == parquet::Type::BYTE_ARRAY)
3322 {
3323 osMinTmp = GetStats<parquet::ByteArrayStatistics>::min(
3324 metadata, iRowGroup, numRowGroups, iCol, bFoundMin);
3325 if (bFoundMin)
3326 {
3327 eType = OFTString;
3328 sMin.String = &osMinTmp[0];
3329 }
3330 }
3331 }
3332
3333 if (bComputeMax)
3334 {
3335 if (physicalType == parquet::Type::BOOLEAN)
3336 {
3337 eType = OFTInteger;
3338 eSubType = OFSTBoolean;
3339 sMax.Integer = GetStats<parquet::BoolStatistics>::max(
3340 metadata, iRowGroup, numRowGroups, iCol, bFoundMax);
3341 }
3342 else if (physicalType == parquet::Type::INT32)
3343 {
3344 if (arrowType && arrowType->id() == arrow::Type::UINT32)
3345 {
3346 // With parquet file version 2.0,
3347 // statistics of uint32 fields are
3348 // stored as signed int32 values...
3349 eType = OFTInteger64;
3350 int nVal = GetStats<parquet::Int32Statistics>::max(
3351 metadata, iRowGroup, numRowGroups, iCol, bFoundMax);
3352 if (bFoundMax)
3353 {
3354 sMax.Integer64 = static_cast<uint32_t>(nVal);
3355 }
3356 }
3357 else
3358 {
3359 eType = OFTInteger;
3360 if (arrowType && arrowType->id() == arrow::Type::INT16)
3361 eSubType = OFSTInt16;
3362 sMax.Integer = GetStats<parquet::Int32Statistics>::max(
3363 metadata, iRowGroup, numRowGroups, iCol, bFoundMax);
3364 }
3365 }
3366 else if (physicalType == parquet::Type::INT64)
3367 {
3368 eType = OFTInteger64;
3369 sMax.Integer64 = GetStats<parquet::Int64Statistics>::max(
3370 metadata, iRowGroup, numRowGroups, iCol, bFoundMax);
3371 }
3372 else if (physicalType == parquet::Type::FLOAT)
3373 {
3374 eType = OFTReal;
3375 eSubType = OFSTFloat32;
3376 sMax.Real = GetStats<parquet::FloatStatistics>::max(
3377 metadata, iRowGroup, numRowGroups, iCol, bFoundMax);
3378 }
3379 else if (physicalType == parquet::Type::DOUBLE)
3380 {
3381 eType = OFTReal;
3382 sMax.Real = GetStats<parquet::DoubleStatistics>::max(
3383 metadata, iRowGroup, numRowGroups, iCol, bFoundMax);
3384 }
3385 else if (arrowType &&
3386 (arrowType->id() == arrow::Type::STRING ||
3387 arrowType->id() == arrow::Type::LARGE_STRING) &&
3388 physicalType == parquet::Type::BYTE_ARRAY)
3389 {
3390 osMaxTmp = GetStats<parquet::ByteArrayStatistics>::max(
3391 metadata, iRowGroup, numRowGroups, iCol, bFoundMax);
3392 if (bFoundMax)
3393 {
3394 eType = OFTString;
3395 sMax.String = &osMaxTmp[0];
3396 }
3397 }
3398 }
3399
3400 return bFoundMin || bFoundMax;
3401 }
3402
3403 /************************************************************************/
3404 /* GeomColsBBOXParquet() */
3405 /************************************************************************/
3406
3407 /** Return for a given geometry column (iGeom: in [0, GetGeomFieldCount()-1] range),
3408 * the Parquet column number of the corresponding xmin,ymin,xmax,ymax bounding
3409 * box columns, if existing.
3410 */
3411 bool OGRParquetLayer::GeomColsBBOXParquet(int iGeom, int &iParquetXMin,
3412 int &iParquetYMin, int &iParquetXMax,
3413 int &iParquetYMax) const
3414 {
3415 const auto oIter = m_oMapGeomFieldIndexToGeomColBBOXParquet.find(iGeom);
3416 const bool bFound =
3417 (oIter != m_oMapGeomFieldIndexToGeomColBBOXParquet.end());
3418 if (bFound)
3419 {
3420 iParquetXMin = oIter->second.iParquetXMin;
3421 iParquetYMin = oIter->second.iParquetYMin;
3422 iParquetXMax = oIter->second.iParquetXMax;
3423 iParquetYMax = oIter->second.iParquetYMax;
3424 }
3425 return bFound;
3426 }