Debian Codegraph @

ogr/ogrsf_frmts/parquet/ogrparquetdatasetlayer.cpp

raw
1 /******************************************************************************
2 *
3 * Project: Parquet Translator
4 * Purpose: Implements OGRParquetDriver.
5 * Author: Even Rouault, <even.rouault at spatialys.com>
6 *
7 ******************************************************************************
8 * Copyright (c) 2022-2024, Planet Labs
9 *
10 * SPDX-License-Identifier: MIT
11 ****************************************************************************/
12
13 #include "ogrsf_frmts.h"
14
15 #include <algorithm>
16 #include <cassert>
17 #include <map>
18 #include <set>
19 #include <utility>
20
21 #include "cpl_time.h"
22 #include "ogr_api.h"
23
24 #include "ogr_parquet.h"
25
26 #include "../arrow_common/ograrrowlayer.hpp"
27 #include "../arrow_common/ograrrowdataset.hpp"
28
29 #if PARQUET_VERSION_MAJOR >= 13
30 // Using field indices for FieldRef is only supported since
31 // https://github.com/apache/arrow/commit/10eedbe63c71f4cf8f0621f3a2304ab3168a2ae5
32 #define SUPPORTS_INDICES_IN_FIELD_REF
33 #endif
34
35 namespace cp = ::arrow::compute;
36
37 /************************************************************************/
38 /* OGRParquetLayer() */
39 /************************************************************************/
40
41 OGRParquetDatasetLayer::OGRParquetDatasetLayer(
42 OGRParquetDataset *poDS, const char *pszLayerName, bool bIsVSI,
43 const std::shared_ptr<arrow::dataset::Dataset> &dataset,
44 CSLConstList papszOpenOptions)
45 : OGRParquetLayerBase(poDS, pszLayerName, papszOpenOptions),
46 m_bIsVSI(bIsVSI), m_poDataset(dataset)
47 {
48 m_poSchema = m_poDataset->schema();
49 EstablishFeatureDefn();
50 CPLAssert(static_cast<int>(m_aeGeomEncoding.size()) ==
51 m_poFeatureDefn->GetGeomFieldCount());
52 }
53
54 /************************************************************************/
55 /* ProcessGeometryColumnCovering() */
56 /************************************************************************/
57
58 /** Process GeoParquet JSON geometry field object to extract information about
59 * its bounding box column, and appropriately fill m_oMapGeomFieldIndexToGeomColBBOX
60 * member with information on that bounding box column.
61 */
62 void OGRParquetDatasetLayer::ProcessGeometryColumnCovering(
63 const std::shared_ptr<arrow::Field> &field,
64 const CPLJSONObject &oJSONGeometryColumn)
65 {
66 std::string osBBOXColumn;
67 std::string osXMin, osYMin, osXMax, osYMax;
68 if (ParseGeometryColumnCovering(oJSONGeometryColumn, osBBOXColumn, osXMin,
69 osYMin, osXMax, osYMax))
70 {
71 OGRArrowLayer::GeomColBBOX sDesc;
72 sDesc.iArrowCol = m_poSchema->GetFieldIndex(osBBOXColumn);
73 const auto fieldBBOX = m_poSchema->GetFieldByName(osBBOXColumn);
74 if (sDesc.iArrowCol >= 0 && fieldBBOX &&
75 fieldBBOX->type()->id() == arrow::Type::STRUCT)
76 {
77 const auto fieldBBOXStruct =
78 std::static_pointer_cast<arrow::StructType>(fieldBBOX->type());
79 const auto fieldXMin = fieldBBOXStruct->GetFieldByName(osXMin);
80 const auto fieldYMin = fieldBBOXStruct->GetFieldByName(osYMin);
81 const auto fieldXMax = fieldBBOXStruct->GetFieldByName(osXMax);
82 const auto fieldYMax = fieldBBOXStruct->GetFieldByName(osYMax);
83 const int nXMinIdx = fieldBBOXStruct->GetFieldIndex(osXMin);
84 const int nYMinIdx = fieldBBOXStruct->GetFieldIndex(osYMin);
85 const int nXMaxIdx = fieldBBOXStruct->GetFieldIndex(osXMax);
86 const int nYMaxIdx = fieldBBOXStruct->GetFieldIndex(osYMax);
87 if (nXMinIdx >= 0 && nYMinIdx >= 0 && nXMaxIdx >= 0 &&
88 nYMaxIdx >= 0 && fieldXMin && fieldYMin && fieldXMax &&
89 fieldYMax &&
90 (fieldXMin->type()->id() == arrow::Type::FLOAT ||
91 fieldXMin->type()->id() == arrow::Type::DOUBLE) &&
92 fieldXMin->type()->id() == fieldYMin->type()->id() &&
93 fieldXMin->type()->id() == fieldXMax->type()->id() &&
94 fieldXMin->type()->id() == fieldYMax->type()->id())
95 {
96 CPLDebug("PARQUET",
97 "Bounding box column '%s' detected for "
98 "geometry column '%s'",
99 osBBOXColumn.c_str(), field->name().c_str());
100 sDesc.iArrowSubfieldXMin = nXMinIdx;
101 sDesc.iArrowSubfieldYMin = nYMinIdx;
102 sDesc.iArrowSubfieldXMax = nXMaxIdx;
103 sDesc.iArrowSubfieldYMax = nYMaxIdx;
104 sDesc.bIsFloat =
105 (fieldXMin->type()->id() == arrow::Type::FLOAT);
106
107 m_oMapGeomFieldIndexToGeomColBBOX
108 [m_poFeatureDefn->GetGeomFieldCount() - 1] =
109 std::move(sDesc);
110 }
111 }
112 }
113 }
114
115 /************************************************************************/
116 /* EstablishFeatureDefn() */
117 /************************************************************************/
118
119 void OGRParquetDatasetLayer::EstablishFeatureDefn()
120 {
121 const auto &kv_metadata = m_poSchema->metadata();
122
123 LoadGeoMetadata(kv_metadata);
124 const auto oMapFieldNameToGDALSchemaFieldDefn =
125 LoadGDALSchema(kv_metadata.get());
126
127 LoadGDALMetadata(kv_metadata.get());
128
129 const bool bUseBBOX =
130 CPLTestBool(CPLGetConfigOption("OGR_PARQUET_USE_BBOX", "YES"));
131
132 // Keep track of declared bounding box columns in GeoParquet JSON metadata,
133 // in order not to expose them as regular fields.
134 std::set<std::string> oSetBBOXColumns;
135 if (bUseBBOX)
136 {
137 for (const auto &iter : m_oMapGeometryColumns)
138 {
139 std::string osBBOXColumn;
140 std::string osXMin, osYMin, osXMax, osYMax;
141 if (ParseGeometryColumnCovering(iter.second, osBBOXColumn, osXMin,
142 osYMin, osXMax, osYMax))
143 {
144 oSetBBOXColumns.insert(std::move(osBBOXColumn));
145 }
146 }
147 }
148
149 const auto &fields = m_poSchema->fields();
150
151 // Overture Maps 2024-04-16-beta.0 almost follows GeoParquet 1.1, except
152 // they don't declare the "covering" element in the GeoParquet JSON metadata
153 if (m_oMapGeometryColumns.find("geometry") != m_oMapGeometryColumns.end() &&
154 bUseBBOX &&
155 !m_oMapGeometryColumns["geometry"].GetObj("covering").IsValid() &&
156 m_oMapGeometryColumns["geometry"].GetString("encoding") == "WKB")
157 {
158 for (int i = 0; i < m_poSchema->num_fields(); ++i)
159 {
160 const auto &field = fields[i];
161 if (field->name() == "bbox" &&
162 field->type()->id() == arrow::Type::STRUCT)
163 {
164 bool bXMin = false;
165 bool bXMax = false;
166 bool bYMin = false;
167 bool bYMax = false;
168 const auto subfields = field->Flatten();
169 if (subfields.size() == 4)
170 {
171 for (int j = 0; j < static_cast<int>(subfields.size()); j++)
172 {
173 const auto &subfield = subfields[j];
174 if (subfield->name() == "bbox.xmin")
175 bXMin = true;
176 else if (subfield->name() == "bbox.xmax")
177 bXMax = true;
178 else if (subfield->name() == "bbox.ymin")
179 bYMin = true;
180 else if (subfield->name() == "bbox.ymax")
181 bYMax = true;
182 }
183 }
184 if (bXMin && bXMax && bYMin && bYMax)
185 {
186 CPLJSONObject oDef = m_oMapGeometryColumns["geometry"];
187 CPLJSONObject oCovering;
188 oDef.Add("covering", oCovering);
189 CPLJSONObject oBBOX;
190 oCovering.Add("bbox", oBBOX);
191 {
192 CPLJSONArray oArray;
193 oArray.Add("bbox");
194 oArray.Add("xmin");
195 oBBOX.Add("xmin", oArray);
196 }
197 {
198 CPLJSONArray oArray;
199 oArray.Add("bbox");
200 oArray.Add("ymin");
201 oBBOX.Add("ymin", oArray);
202 }
203 {
204 CPLJSONArray oArray;
205 oArray.Add("bbox");
206 oArray.Add("xmax");
207 oBBOX.Add("xmax", oArray);
208 }
209 {
210 CPLJSONArray oArray;
211 oArray.Add("bbox");
212 oArray.Add("ymax");
213 oBBOX.Add("ymax", oArray);
214 }
215 oSetBBOXColumns.insert("bbox");
216 m_oMapGeometryColumns["geometry"] = std::move(oDef);
217 }
218 break;
219 }
220 }
221 }
222
223 for (int i = 0; i < m_poSchema->num_fields(); ++i)
224 {
225 const auto &field = fields[i];
226
227 if (!m_osFIDColumn.empty() && field->name() == m_osFIDColumn)
228 {
229 m_iFIDArrowColumn = i;
230 continue;
231 }
232
233 if (oSetBBOXColumns.find(field->name()) != oSetBBOXColumns.end())
234 {
235 m_oSetBBoxArrowColumns.insert(i);
236 continue;
237 }
238
239 const bool bGeometryField = DealWithGeometryColumn(
240 i, field, []() { return wkbUnknown; }, nullptr, nullptr, -1);
241 if (bGeometryField)
242 {
243 const auto oIter = m_oMapGeometryColumns.find(field->name());
244 if (bUseBBOX && oIter != m_oMapGeometryColumns.end())
245 {
246 ProcessGeometryColumnCovering(field, oIter->second);
247 }
248 }
249 else
250 {
251 CreateFieldFromSchema(field, {i},
252 oMapFieldNameToGDALSchemaFieldDefn);
253 }
254 }
255
256 CPLAssert(static_cast<int>(m_anMapFieldIndexToArrowColumn.size()) ==
257 m_poFeatureDefn->GetFieldCount());
258 CPLAssert(static_cast<int>(m_anMapGeomFieldIndexToArrowColumn.size()) ==
259 m_poFeatureDefn->GetGeomFieldCount());
260 }
261
262 namespace
263 {
264
265 /************************************************************************/
266 /* WKBGeometryOptionsType */
267 /************************************************************************/
268
269 class WKBGeometryOptions;
270
271 class WKBGeometryOptionsType : public cp::FunctionOptionsType
272 {
273 WKBGeometryOptionsType() = default;
274
275 static const WKBGeometryOptions &Cast(const cp::FunctionOptions &opts);
276
277 public:
278 const char *type_name() const override
279 {
280 return "WKBGeometryOptionsType";
281 }
282
283 std::string Stringify(const cp::FunctionOptions &) const override;
284 bool Compare(const cp::FunctionOptions &,
285 const cp::FunctionOptions &) const override;
286 std::unique_ptr<cp::FunctionOptions>
287 Copy(const cp::FunctionOptions &) const override;
288
289 static WKBGeometryOptionsType *GetSingleton()
290 {
291 static WKBGeometryOptionsType instance;
292 return &instance;
293 }
294 };
295
296 /************************************************************************/
297 /* WKBGeometryOptions */
298 /************************************************************************/
299
300 class WKBGeometryOptions : public cp::FunctionOptions
301 {
302
303 public:
304 explicit WKBGeometryOptions(
305 const std::vector<GByte> &abyFilterGeomWkbIn = std::vector<GByte>())
306 : cp::FunctionOptions(WKBGeometryOptionsType::GetSingleton()),
307 abyFilterGeomWkb(abyFilterGeomWkbIn)
308 {
309 }
310
311 bool operator==(const WKBGeometryOptions &other) const
312 {
313 return abyFilterGeomWkb == other.abyFilterGeomWkb;
314 }
315
316 std::vector<GByte> abyFilterGeomWkb;
317 };
318
319 const WKBGeometryOptions &
320 WKBGeometryOptionsType::Cast(const cp::FunctionOptions &opts)
321 {
322 return *cpl::down_cast<const WKBGeometryOptions *>(&opts);
323 }
324
325 bool WKBGeometryOptionsType::Compare(const cp::FunctionOptions &optsA,
326 const cp::FunctionOptions &optsB) const
327 {
328 return Cast(optsA) == Cast(optsB);
329 }
330
331 std::string
332 WKBGeometryOptionsType::Stringify(const cp::FunctionOptions &opts) const
333 {
334 const auto &bboxOptions = Cast(opts);
335 std::string osRet(type_name());
336 osRet += '-';
337 for (GByte byVal : bboxOptions.abyFilterGeomWkb)
338 osRet += CPLSPrintf("%02X", byVal);
339 return osRet;
340 }
341
342 std::unique_ptr<cp::FunctionOptions>
343 WKBGeometryOptionsType::Copy(const cp::FunctionOptions &opts) const
344 {
345 return std::make_unique<WKBGeometryOptions>(Cast(opts));
346 }
347
348 /************************************************************************/
349 /* OptionsWrapper */
350 /************************************************************************/
351
352 /// KernelState adapter for the common case of kernels whose only
353 /// state is an instance of a subclass of FunctionOptions.
354 template <typename OptionsType> struct OptionsWrapper : public cp::KernelState
355 {
356 explicit OptionsWrapper(OptionsType optionsIn)
357 : options(std::move(optionsIn))
358 {
359 }
360
361 static arrow::Result<std::unique_ptr<cp::KernelState>>
362 Init(cp::KernelContext *, const cp::KernelInitArgs &args)
363 {
364 auto options = cpl::down_cast<const OptionsType *>(args.options);
365 CPLAssert(options);
366 return std::make_unique<OptionsWrapper>(*options);
367 }
368
369 static const OptionsType &Get(cp::KernelContext *ctx)
370 {
371 return cpl::down_cast<const OptionsWrapper *>(ctx->state())->options;
372 }
373
374 OptionsType options;
375 };
376 } // namespace
377
378 /************************************************************************/
379 /* ExecOGRWKBIntersects() */
380 /************************************************************************/
381
382 static arrow::Status ExecOGRWKBIntersects(cp::KernelContext *ctx,
383 const cp::ExecSpan &batch,
384 cp::ExecResult *out)
385 {
386 // Get filter geometry
387 const auto &opts = OptionsWrapper<WKBGeometryOptions>::Get(ctx);
388 OGRGeometry *poGeomTmp = nullptr;
389 OGRErr eErr = OGRGeometryFactory::createFromWkb(
390 opts.abyFilterGeomWkb.data(), nullptr, &poGeomTmp,
391 opts.abyFilterGeomWkb.size());
392 CPL_IGNORE_RET_VAL(eErr);
393 CPLAssert(eErr == OGRERR_NONE);
394 CPLAssert(poGeomTmp != nullptr);
395 std::unique_ptr<OGRGeometry> poFilterGeom(poGeomTmp);
396 OGREnvelope sFilterEnvelope;
397 poFilterGeom->getEnvelope(&sFilterEnvelope);
398 const bool bFilterIsEnvelope = poFilterGeom->IsRectangle();
399
400 // Deal with input array
401 CPLAssert(batch.num_values() == 1);
402 const arrow::ArraySpan &input = batch[0].array;
403 CPLAssert(input.type->id() == arrow::Type::BINARY);
404 // Packed array of bits
405 const auto pabyInputValidity = input.buffers[0].data;
406 const auto nInputOffsets = input.offset;
407 const auto panWkbOffsets = input.GetValues<int32_t>(1);
408 const auto pabyWkbArray = input.buffers[2].data;
409
410 // Deal with output array
411 CPLAssert(out->type()->id() == arrow::Type::BOOL);
412 auto out_span = out->array_span();
413 // Below array holds 8 bits per uint8_t
414 uint8_t *pabitsOutValues = out_span->buffers[1].data;
415 const auto nOutOffset = out_span->offset;
416
417 // Iterate over WKB geometries
418 OGRPreparedGeometry *pPreparedFilterGeom = nullptr;
419 OGREnvelope sEnvelope;
420 for (int64_t i = 0; i < batch.length; ++i)
421 {
422 const bool bInputIsNull =
423 (pabyInputValidity &&
424 arrow::bit_util::GetBit(pabyInputValidity, i + nInputOffsets) ==
425 0);
426 bool bOutputVal = false;
427 if (!bInputIsNull)
428 {
429 const GByte *pabyWkb = pabyWkbArray + panWkbOffsets[i];
430 const size_t nWkbSize = panWkbOffsets[i + 1] - panWkbOffsets[i];
431 bOutputVal = OGRLayer::FilterWKBGeometry(
432 pabyWkb, nWkbSize,
433 /* bEnvelopeAlreadySet = */ false, sEnvelope,
434 poFilterGeom.get(), bFilterIsEnvelope, sFilterEnvelope,
435 pPreparedFilterGeom);
436 }
437 if (bOutputVal)
438 arrow::bit_util::SetBit(pabitsOutValues, i + nOutOffset);
439 else
440 arrow::bit_util::ClearBit(pabitsOutValues, i + nOutOffset);
441 }
442
443 // Cleanup
444 if (pPreparedFilterGeom)
445 OGRDestroyPreparedGeometry(pPreparedFilterGeom);
446
447 return arrow::Status::OK();
448 }
449
450 /************************************************************************/
451 /* RegisterOGRWKBIntersectsIfNeeded() */
452 /************************************************************************/
453
454 static bool RegisterOGRWKBIntersectsIfNeeded()
455 {
456 auto registry = cp::GetFunctionRegistry();
457 bool bRet =
458 registry->GetFunction("OGRWKBIntersects").ValueOr(nullptr) != nullptr;
459 if (!bRet)
460 {
461 static const WKBGeometryOptions defaultOpts;
462
463 // Below assert is completely useless but helps improve test coverage
464 CPLAssert(WKBGeometryOptionsType::GetSingleton()->Compare(
465 defaultOpts, *(WKBGeometryOptionsType::GetSingleton()
466 ->Copy(defaultOpts)
467 .get())));
468
469 auto func = std::make_shared<cp::ScalarFunction>(
470 "OGRWKBIntersects", cp::Arity::Unary(), cp::FunctionDoc(),
471 &defaultOpts);
472 cp::ScalarKernel kernel({arrow::binary()}, arrow::boolean(),
473 ExecOGRWKBIntersects,
474 OptionsWrapper<WKBGeometryOptions>::Init);
475 kernel.null_handling = cp::NullHandling::OUTPUT_NOT_NULL;
476 bRet = func->AddKernel(std::move(kernel)).ok() &&
477 registry->AddFunction(std::move(func)).ok();
478 }
479 return bRet;
480 }
481
482 /************************************************************************/
483 /* BuildScanner() */
484 /************************************************************************/
485
486 void OGRParquetDatasetLayer::BuildScanner()
487 {
488 m_bRebuildScanner = false;
489 m_bSkipFilterGeometry = false;
490 m_bBaseArrowIgnoreSpatialFilterRect = false;
491 m_bBaseArrowIgnoreSpatialFilter = false;
492 m_bBaseArrowIgnoreAttributeFilter = false;
493
494 try
495 {
496 std::shared_ptr<arrow::dataset::ScannerBuilder> scannerBuilder;
497 PARQUET_ASSIGN_OR_THROW(scannerBuilder, m_poDataset->NewScan());
498 assert(scannerBuilder);
499
500 // We cannot use the shared memory pool. Otherwise we get random
501 // crashes in multi-threaded arrow code (apparently some cleanup code),
502 // that may used the memory pool after it has been destroyed.
503 // At least this was true with some older libarrow version
504 // PARQUET_THROW_NOT_OK(scannerBuilder->Pool(m_poMemoryPool));
505
506 if (m_bIsVSI)
507 {
508 const int nFragmentReadAhead = atoi(
509 CPLGetConfigOption("OGR_PARQUET_FRAGMENT_READ_AHEAD", "2"));
510 PARQUET_THROW_NOT_OK(
511 scannerBuilder->FragmentReadahead(nFragmentReadAhead));
512 }
513
514 const char *pszBatchSize =
515 CPLGetConfigOption("OGR_PARQUET_BATCH_SIZE", nullptr);
516 if (pszBatchSize)
517 {
518 PARQUET_THROW_NOT_OK(
519 scannerBuilder->BatchSize(CPLAtoGIntBig(pszBatchSize)));
520 }
521
522 const int nNumCPUs = GetNumCPUs();
523 const char *pszUseThreads =
524 CPLGetConfigOption("OGR_PARQUET_USE_THREADS", nullptr);
525 if (!pszUseThreads && nNumCPUs > 1)
526 {
527 pszUseThreads = "YES";
528 }
529 if (pszUseThreads && CPLTestBool(pszUseThreads))
530 {
531 PARQUET_THROW_NOT_OK(scannerBuilder->UseThreads(true));
532 }
533
534 #if PARQUET_VERSION_MAJOR >= 10
535 const char *pszBatchReadAhead =
536 CPLGetConfigOption("OGR_PARQUET_BATCH_READ_AHEAD", nullptr);
537 if (pszBatchReadAhead)
538 {
539 PARQUET_THROW_NOT_OK(
540 scannerBuilder->BatchReadahead(atoi(pszBatchReadAhead)));
541 }
542 #endif
543
544 cp::Expression expression;
545 if (m_poFilterGeom && !m_poFilterGeom->IsEmpty() &&
546 CPLTestBool(CPLGetConfigOption(
547 "OGR_PARQUET_OPTIMIZED_SPATIAL_FILTER", "YES")))
548 {
549 const auto oIter =
550 m_oMapGeomFieldIndexToGeomColBBOX.find(m_iGeomFieldFilter);
551 if (oIter != m_oMapGeomFieldIndexToGeomColBBOX.end())
552 {
553 // This actually requires Arrow >= 15 (https://github.com/apache/arrow/issues/39064)
554 // to be more efficient.
555 #ifdef SUPPORTS_INDICES_IN_FIELD_REF
556 const auto &oBBOXDef = oIter->second;
557 expression = cp::and_(
558 {cp::less_equal(
559 cp::field_ref(arrow::FieldRef(
560 oBBOXDef.iArrowCol, oBBOXDef.iArrowSubfieldXMin)),
561 cp::literal(m_sFilterEnvelope.MaxX)),
562 cp::less_equal(
563 cp::field_ref(arrow::FieldRef(
564 oBBOXDef.iArrowCol, oBBOXDef.iArrowSubfieldYMin)),
565 cp::literal(m_sFilterEnvelope.MaxY)),
566 cp::greater_equal(
567 cp::field_ref(arrow::FieldRef(
568 oBBOXDef.iArrowCol, oBBOXDef.iArrowSubfieldXMax)),
569 cp::literal(m_sFilterEnvelope.MinX)),
570 cp::greater_equal(
571 cp::field_ref(arrow::FieldRef(
572 oBBOXDef.iArrowCol, oBBOXDef.iArrowSubfieldYMax)),
573 cp::literal(m_sFilterEnvelope.MinY))});
574 #else
575 const auto oIter2 = m_oMapGeometryColumns.find(
576 m_poFeatureDefn->GetGeomFieldDefn(m_iGeomFieldFilter)
577 ->GetNameRef());
578 std::string osBBOXColumn;
579 std::string osXMin, osYMin, osXMax, osYMax;
580 if (ParseGeometryColumnCovering(oIter2->second, osBBOXColumn,
581 osXMin, osYMin, osXMax, osYMax))
582 {
583 expression = cp::and_(
584 {cp::less_equal(cp::field_ref(arrow::FieldRef(
585 osBBOXColumn, osXMin)),
586 cp::literal(m_sFilterEnvelope.MaxX)),
587 cp::less_equal(cp::field_ref(arrow::FieldRef(
588 osBBOXColumn, osYMin)),
589 cp::literal(m_sFilterEnvelope.MaxY)),
590 cp::greater_equal(cp::field_ref(arrow::FieldRef(
591 osBBOXColumn, osXMax)),
592 cp::literal(m_sFilterEnvelope.MinX)),
593 cp::greater_equal(
594 cp::field_ref(
595 arrow::FieldRef(osBBOXColumn, osYMax)),
596 cp::literal(m_sFilterEnvelope.MinY))});
597 }
598 #endif
599 }
600 else if (m_iGeomFieldFilter >= 0 &&
601 m_iGeomFieldFilter <
602 static_cast<int>(m_aeGeomEncoding.size()) &&
603 m_aeGeomEncoding[m_iGeomFieldFilter] ==
604 OGRArrowGeomEncoding::GEOARROW_STRUCT_POINT)
605 {
606 const int iCol =
607 m_anMapGeomFieldIndexToArrowColumn[m_iGeomFieldFilter];
608 const auto &field = m_poSchema->fields()[iCol];
609 auto type = field->type();
610 std::vector<arrow::FieldRef> fieldRefs;
611 #ifdef SUPPORTS_INDICES_IN_FIELD_REF
612 fieldRefs.emplace_back(iCol);
613 #else
614 fieldRefs.emplace_back(field->name());
615 #endif
616 if (type->id() == arrow::Type::STRUCT)
617 {
618 const auto fieldStruct =
619 std::static_pointer_cast<arrow::StructType>(type);
620 const auto fieldX = fieldStruct->GetFieldByName("x");
621 const auto fieldY = fieldStruct->GetFieldByName("y");
622 if (fieldX && fieldY)
623 {
624 auto fieldRefX(fieldRefs);
625 fieldRefX.emplace_back("x");
626 auto fieldRefY(std::move(fieldRefs));
627 fieldRefY.emplace_back("y");
628 expression = cp::and_(
629 {cp::less_equal(
630 cp::field_ref(arrow::FieldRef(fieldRefX)),
631 cp::literal(m_sFilterEnvelope.MaxX)),
632 cp::less_equal(
633 cp::field_ref(arrow::FieldRef(fieldRefY)),
634 cp::literal(m_sFilterEnvelope.MaxY)),
635 cp::greater_equal(
636 cp::field_ref(arrow::FieldRef(fieldRefX)),
637 cp::literal(m_sFilterEnvelope.MinX)),
638 cp::greater_equal(
639 cp::field_ref(arrow::FieldRef(fieldRefY)),
640 cp::literal(m_sFilterEnvelope.MinY))});
641 }
642 }
643 }
644 else if (m_iGeomFieldFilter >= 0 &&
645 m_iGeomFieldFilter <
646 static_cast<int>(m_aeGeomEncoding.size()) &&
647 m_aeGeomEncoding[m_iGeomFieldFilter] ==
648 OGRArrowGeomEncoding::WKB)
649 {
650 const int iCol =
651 m_anMapGeomFieldIndexToArrowColumn[m_iGeomFieldFilter];
652 const auto &field = m_poSchema->fields()[iCol];
653 if (field->type()->id() == arrow::Type::BINARY &&
654 RegisterOGRWKBIntersectsIfNeeded())
655 {
656 #ifdef SUPPORTS_INDICES_IN_FIELD_REF
657 auto oFieldRef = arrow::FieldRef(iCol);
658 #else
659 auto oFieldRef = arrow::FieldRef(field->name());
660 #endif
661 std::vector<GByte> abyFilterGeomWkb;
662 abyFilterGeomWkb.resize(m_poFilterGeom->WkbSize());
663 m_poFilterGeom->exportToWkb(wkbNDR, abyFilterGeomWkb.data(),
664 wkbVariantIso);
665 // Silence 'Using uninitialized value oFieldRef. Field oFieldRef.impl_._M_u is uninitialized when calling FieldRef.'
666 // coverity[uninit_use_in_call]
667 expression = cp::call("OGRWKBIntersects",
668 {cp::field_ref(std::move(oFieldRef))},
669 WKBGeometryOptions(abyFilterGeomWkb));
670
671 if (expression.is_valid())
672 {
673 m_bBaseArrowIgnoreSpatialFilterRect = true;
674 m_bBaseArrowIgnoreSpatialFilter = true;
675 m_bSkipFilterGeometry = true;
676 }
677 }
678 }
679
680 if (expression.is_valid() && !m_bSkipFilterGeometry)
681 {
682 m_bBaseArrowIgnoreSpatialFilterRect = true;
683
684 const bool bIsPoint =
685 wkbFlatten(
686 m_poFeatureDefn->GetGeomFieldDefn(m_iGeomFieldFilter)
687 ->GetType()) == wkbPoint;
688 m_bBaseArrowIgnoreSpatialFilter =
689 m_bFilterIsEnvelope && bIsPoint;
690
691 m_bSkipFilterGeometry =
692 m_bFilterIsEnvelope &&
693 (bIsPoint ||
694 m_poFeatureDefn->GetGeomFieldDefn(m_iGeomFieldFilter)
695 ->IsIgnored());
696 }
697 }
698
699 if (m_poAttrQuery &&
700 CPLTestBool(CPLGetConfigOption(
701 "OGR_PARQUET_OPTIMIZED_ATTRIBUTE_FILTER", "YES")))
702 {
703 const swq_expr_node *poNode =
704 static_cast<swq_expr_node *>(m_poAttrQuery->GetSWQExpr());
705 bool bFullyTranslated = true;
706 auto expressionFilter = BuildArrowFilter(poNode, bFullyTranslated);
707 if (expressionFilter.is_valid())
708 {
709 if (bFullyTranslated)
710 {
711 CPLDebugOnly("PARQUET",
712 "Attribute filter fully translated to Arrow");
713 m_asAttributeFilterConstraints.clear();
714 m_bBaseArrowIgnoreAttributeFilter = true;
715 }
716
717 if (expression.is_valid())
718 expression =
719 cp::and_(expression, std::move(expressionFilter));
720 else
721 expression = std::move(expressionFilter);
722 }
723 }
724
725 if (expression.is_valid())
726 {
727 PARQUET_ASSIGN_OR_THROW(expression,
728 expression.Bind(*m_poDataset->schema()));
729
730 #ifdef DEBUG
731 arrow::dataset::FragmentIterator fragment_it;
732 PARQUET_ASSIGN_OR_THROW(fragment_it,
733 m_poDataset->GetFragments(expression));
734 for (const auto &maybe_fragment : fragment_it)
735 {
736 auto fragment = maybe_fragment.ValueOrDie();
737 CPLDebug("PARQUET", "Scanner planning to read '%s'",
738 fragment->ToString().c_str());
739 }
740 #endif
741
742 PARQUET_THROW_NOT_OK(scannerBuilder->Filter(expression));
743 }
744
745 if (m_bIgnoredFields)
746 {
747 #ifdef DEBUG
748 std::string osFields;
749 for (const std::string &osField : m_aosProjectedFields)
750 {
751 if (!osFields.empty())
752 osFields += ',';
753 osFields += osField;
754 }
755 CPLDebug("PARQUET", "Projected fields: %s", osFields.c_str());
756 #endif
757 PARQUET_THROW_NOT_OK(scannerBuilder->Project(m_aosProjectedFields));
758 }
759
760 PARQUET_ASSIGN_OR_THROW(m_poScanner, scannerBuilder->Finish());
761 }
762 catch (const std::exception &e)
763 {
764 CPLError(CE_Failure, CPLE_AppDefined, "Arrow/Parquet exception: %s",
765 e.what());
766 }
767 }
768
769 /************************************************************************/
770 /* BuildArrowFilter() */
771 /************************************************************************/
772
773 cp::Expression
774 OGRParquetDatasetLayer::BuildArrowFilter(const swq_expr_node *poNode,
775 bool &bFullyTranslated)
776 {
777 if (poNode->eNodeType == SNT_OPERATION && poNode->nOperation == SWQ_AND &&
778 poNode->nSubExprCount == 2)
779 {
780 auto sLeft = BuildArrowFilter(poNode->papoSubExpr[0], bFullyTranslated);
781 auto sRight =
782 BuildArrowFilter(poNode->papoSubExpr[1], bFullyTranslated);
783 if (sLeft.is_valid() && sRight.is_valid())
784 return cp::and_(std::move(sLeft), std::move(sRight));
785 else if (sLeft.is_valid())
786 return sLeft;
787 else if (sRight.is_valid())
788 return sRight;
789 }
790
791 else if (poNode->eNodeType == SNT_OPERATION &&
792 poNode->nOperation == SWQ_OR && poNode->nSubExprCount == 2)
793 {
794 auto sLeft = BuildArrowFilter(poNode->papoSubExpr[0], bFullyTranslated);
795 auto sRight =
796 BuildArrowFilter(poNode->papoSubExpr[1], bFullyTranslated);
797 if (sLeft.is_valid() && sRight.is_valid())
798 return cp::or_(std::move(sLeft), std::move(sRight));
799 }
800
801 else if (poNode->eNodeType == SNT_OPERATION &&
802 poNode->nOperation == SWQ_NOT && poNode->nSubExprCount == 1)
803 {
804 auto expr = BuildArrowFilter(poNode->papoSubExpr[0], bFullyTranslated);
805 if (expr.is_valid())
806 return cp::not_(std::move(expr));
807 }
808
809 else if (poNode->eNodeType == SNT_COLUMN)
810 {
811 if (poNode->field_index >= 0 &&
812 poNode->field_index < m_poFeatureDefn->GetFieldCount())
813 {
814 std::vector<arrow::FieldRef> fieldRefs;
815 std::shared_ptr<arrow::Field> field;
816 for (int idx : m_anMapFieldIndexToArrowColumn[poNode->field_index])
817 {
818 if (!field)
819 {
820 field = m_poSchema->fields()[idx];
821 }
822 else
823 {
824 CPLAssert(field->type()->id() == arrow::Type::STRUCT);
825 const auto fieldStruct =
826 std::static_pointer_cast<arrow::StructType>(
827 field->type());
828 field = fieldStruct->fields()[idx];
829 }
830 fieldRefs.emplace_back(field->name());
831 }
832
833 auto expr = cp::field_ref(arrow::FieldRef(std::move(fieldRefs)));
834
835 // Comparing a boolean column to 0 or 1 fails without explicit cast
836 if (m_poFeatureDefn->GetFieldDefn(poNode->field_index)
837 ->GetSubType() == OFSTBoolean)
838 {
839 expr = cp::call("cast", {expr},
840 cp::CastOptions::Safe(arrow::uint8()));
841 }
842 return expr;
843 }
844 else if (poNode->field_index ==
845 m_poFeatureDefn->GetFieldCount() + SPF_FID &&
846 m_iFIDArrowColumn >= 0)
847 {
848 #ifdef SUPPORTS_INDICES_IN_FIELD_REF
849 return cp::field_ref(arrow::FieldRef(m_iFIDArrowColumn));
850 #else
851 return cp::field_ref(arrow::FieldRef(
852 m_poSchema->fields()[m_iFIDArrowColumn]->name()));
853 #endif
854 }
855 }
856
857 else if (poNode->eNodeType == SNT_CONSTANT)
858 {
859 switch (poNode->field_type)
860 {
861 case SWQ_INTEGER:
862 return cp::literal(static_cast<int32_t>(poNode->int_value));
863
864 case SWQ_INTEGER64:
865 return cp::literal(static_cast<int64_t>(poNode->int_value));
866
867 case SWQ_FLOAT:
868 return cp::literal(poNode->float_value);
869
870 case SWQ_STRING:
871 return cp::literal(poNode->string_value);
872
873 case SWQ_TIMESTAMP:
874 {
875 OGRField sField;
876 if (OGRParseDate(poNode->string_value, &sField, 0))
877 {
878 struct tm brokenDown;
879 brokenDown.tm_year = sField.Date.Year - 1900;
880 brokenDown.tm_mon = sField.Date.Month - 1;
881 brokenDown.tm_mday = sField.Date.Day;
882 brokenDown.tm_hour = sField.Date.Hour;
883 brokenDown.tm_min = sField.Date.Minute;
884 brokenDown.tm_sec = static_cast<int>(sField.Date.Second);
885 int64_t nVal =
886 CPLYMDHMSToUnixTime(&brokenDown) * 1000 +
887 (static_cast<int>(sField.Date.Second * 1000 + 0.5) %
888 1000);
889 if (sField.Date.TZFlag > OGR_TZFLAG_MIXED_TZ)
890 {
891 // Convert for sField.Date.TZFlag to UTC
892 const int TZOffset =
893 (sField.Date.TZFlag - OGR_TZFLAG_UTC) * 15;
894 const int TZOffsetMS = TZOffset * 60 * 1000;
895 nVal -= TZOffsetMS;
896 return cp::literal(arrow::TimestampScalar(
897 nVal, arrow::TimeUnit::MILLI, "UTC"));
898 }
899 else
900 {
901 return cp::literal(arrow::TimestampScalar(
902 nVal, arrow::TimeUnit::MILLI));
903 }
904 }
905 break;
906 }
907
908 default:
909 break;
910 }
911 }
912
913 else if (poNode->eNodeType == SNT_OPERATION && poNode->nSubExprCount == 2 &&
914 IsComparisonOp(poNode->nOperation))
915 {
916 auto sLeft = BuildArrowFilter(poNode->papoSubExpr[0], bFullyTranslated);
917 auto sRight =
918 BuildArrowFilter(poNode->papoSubExpr[1], bFullyTranslated);
919 if (sLeft.is_valid() && sRight.is_valid())
920 {
921 if (poNode->nOperation == SWQ_EQ)
922 return cp::equal(std::move(sLeft), std::move(sRight));
923 if (poNode->nOperation == SWQ_LT)
924 return cp::less(std::move(sLeft), std::move(sRight));
925 if (poNode->nOperation == SWQ_LE)
926 return cp::less_equal(std::move(sLeft), std::move(sRight));
927 if (poNode->nOperation == SWQ_GT)
928 return cp::greater(std::move(sLeft), std::move(sRight));
929 if (poNode->nOperation == SWQ_GE)
930 return cp::greater_equal(std::move(sLeft), std::move(sRight));
931 if (poNode->nOperation == SWQ_NE)
932 return cp::not_equal(std::move(sLeft), std::move(sRight));
933 }
934 }
935
936 else if (poNode->eNodeType == SNT_OPERATION && poNode->nSubExprCount == 2 &&
937 (poNode->nOperation == SWQ_LIKE ||
938 poNode->nOperation == SWQ_ILIKE) &&
939 poNode->papoSubExpr[1]->eNodeType == SNT_CONSTANT &&
940 poNode->papoSubExpr[1]->field_type == SWQ_STRING)
941 {
942 auto sLeft = BuildArrowFilter(poNode->papoSubExpr[0], bFullyTranslated);
943 if (sLeft.is_valid())
944 {
945 if (cp::GetFunctionRegistry()
946 ->GetFunction("match_like")
947 .ValueOr(nullptr))
948 {
949 // match_like is only available is Arrow built against RE2.
950 return cp::call(
951 "match_like", {std::move(sLeft)},
952 cp::MatchSubstringOptions(
953 poNode->papoSubExpr[1]->string_value,
954 /* ignore_case=*/poNode->nOperation == SWQ_ILIKE));
955 }
956 }
957 }
958
959 else if (poNode->eNodeType == SNT_OPERATION &&
960 poNode->nOperation == SWQ_ISNULL && poNode->nSubExprCount == 1)
961 {
962 auto expr = BuildArrowFilter(poNode->papoSubExpr[0], bFullyTranslated);
963 if (expr.is_valid())
964 return cp::is_null(std::move(expr));
965 }
966
967 bFullyTranslated = false;
968 return {};
969 }
970
971 /************************************************************************/
972 /* ReadNextBatch() */
973 /************************************************************************/
974
975 bool OGRParquetDatasetLayer::ReadNextBatch()
976 {
977 if (m_bRebuildScanner)
978 BuildScanner();
979
980 m_nIdxInBatch = 0;
981
982 if (m_poRecordBatchReader == nullptr)
983 {
984 if (!m_poScanner)
985 return false;
986 auto result = m_poScanner->ToRecordBatchReader();
987 if (!result.ok())
988 {
989 CPLError(CE_Failure, CPLE_AppDefined,
990 "ToRecordBatchReader() failed: %s",
991 result.status().message().c_str());
992 return false;
993 }
994 m_poRecordBatchReader = *result;
995 if (m_poRecordBatchReader == nullptr)
996 return false;
997 }
998
999 std::shared_ptr<arrow::RecordBatch> poNextBatch;
1000 do
1001 {
1002 ++m_iRecordBatch;
1003
1004 poNextBatch.reset();
1005 auto status = m_poRecordBatchReader->ReadNext(&poNextBatch);
1006 if (!status.ok())
1007 {
1008 CPLError(CE_Failure, CPLE_AppDefined, "ReadNext() failed: %s",
1009 status.message().c_str());
1010 poNextBatch.reset();
1011 }
1012 if (poNextBatch == nullptr)
1013 {
1014 m_poBatch.reset();
1015 return false;
1016 }
1017 } while (poNextBatch->num_rows() == 0);
1018
1019 // CPLDebug("PARQUET", "Current batch has %d rows", int(poNextBatch->num_rows()));
1020
1021 SetBatch(poNextBatch);
1022
1023 return true;
1024 }
1025
1026 /************************************************************************/
1027 /* GetNextFeature() */
1028 /************************************************************************/
1029
1030 OGRFeature *OGRParquetDatasetLayer::GetNextFeature()
1031 {
1032 while (true)
1033 {
1034 OGRFeature *poFeature = GetNextRawFeature();
1035 if (poFeature == nullptr)
1036 return nullptr;
1037
1038 if ((m_poFilterGeom == nullptr || m_bSkipFilterGeometry ||
1039 FilterGeometry(poFeature->GetGeometryRef())) &&
1040 (m_poAttrQuery == nullptr || m_bBaseArrowIgnoreAttributeFilter ||
1041 m_poAttrQuery->Evaluate(poFeature)))
1042 {
1043 return poFeature;
1044 }
1045 else
1046 delete poFeature;
1047 }
1048 }
1049
1050 /************************************************************************/
1051 /* GetFeatureCount() */
1052 /************************************************************************/
1053
1054 GIntBig OGRParquetDatasetLayer::GetFeatureCount(int bForce)
1055 {
1056 if (m_poAttrQuery == nullptr && m_poFilterGeom == nullptr)
1057 {
1058 if (m_bRebuildScanner)
1059 BuildScanner();
1060 if (!m_poScanner)
1061 return -1;
1062 auto status = m_poScanner->CountRows();
1063 if (status.ok())
1064 return *status;
1065 }
1066 return OGRLayer::GetFeatureCount(bForce);
1067 }
1068
1069 /************************************************************************/
1070 /* FastGetExtent() */
1071 /************************************************************************/
1072
1073 bool OGRParquetDatasetLayer::FastGetExtent(int iGeomField,
1074 OGREnvelope *psExtent) const
1075 {
1076 const auto oIter = m_oMapExtents.find(iGeomField);
1077 if (oIter != m_oMapExtents.end())
1078 {
1079 *psExtent = oIter->second;
1080 return true;
1081 }
1082
1083 return false;
1084 }
1085
1086 /************************************************************************/
1087 /* IGetExtent() */
1088 /************************************************************************/
1089
1090 OGRErr OGRParquetDatasetLayer::IGetExtent(int iGeomField, OGREnvelope *psExtent,
1091 bool bForce)
1092 {
1093 if (FastGetExtent(iGeomField, psExtent))
1094 {
1095 return OGRERR_NONE;
1096 }
1097
1098 // bbox in general m_oMapGeometryColumns can not be trusted (at least at
1099 // time of writing), so we have to iterate over each fragment.
1100 const char *pszGeomFieldName =
1101 m_poFeatureDefn->GetGeomFieldDefn(iGeomField)->GetNameRef();
1102 auto oIter = m_oMapGeometryColumns.find(pszGeomFieldName);
1103 if (oIter != m_oMapGeometryColumns.end())
1104 {
1105 auto statusFragments = m_poDataset->GetFragments();
1106 if (statusFragments.ok())
1107 {
1108 *psExtent = OGREnvelope();
1109 int nFragmentCount = 0;
1110 int nBBoxFragmentCount = 0;
1111 for (const auto &oFragmentStatus : *statusFragments)
1112 {
1113 if (oFragmentStatus.ok())
1114 {
1115 auto statusSchema =
1116 (*oFragmentStatus)->ReadPhysicalSchema();
1117 if (statusSchema.ok())
1118 {
1119 nFragmentCount++;
1120 const auto &kv_metadata = (*statusSchema)->metadata();
1121 if (kv_metadata && kv_metadata->Contains("geo"))
1122 {
1123 auto geo = kv_metadata->Get("geo");
1124 CPLJSONDocument oDoc;
1125 if (geo.ok() && oDoc.LoadMemory(*geo))
1126 {
1127 auto oRoot = oDoc.GetRoot();
1128 auto oColumns = oRoot.GetObj("columns");
1129 auto oCol = oColumns.GetObj(pszGeomFieldName);
1130 OGREnvelope3D sFragmentExtent;
1131 if (oCol.IsValid() &&
1132 GetExtentFromMetadata(
1133 oCol, &sFragmentExtent) == OGRERR_NONE)
1134 {
1135 nBBoxFragmentCount++;
1136 psExtent->Merge(sFragmentExtent);
1137 }
1138 }
1139 }
1140 if (nFragmentCount != nBBoxFragmentCount)
1141 break;
1142 }
1143 }
1144 }
1145 if (nFragmentCount == nBBoxFragmentCount)
1146 {
1147 m_oMapExtents[iGeomField] = *psExtent;
1148 return OGRERR_NONE;
1149 }
1150 }
1151 }
1152
1153 return OGRParquetLayerBase::IGetExtent(iGeomField, psExtent, bForce);
1154 }
1155
1156 /************************************************************************/
1157 /* ISetSpatialFilter() */
1158 /************************************************************************/
1159
1160 OGRErr OGRParquetDatasetLayer::ISetSpatialFilter(int iGeomField,
1161 const OGRGeometry *poGeomIn)
1162
1163 {
1164 const OGRErr eErr =
1165 OGRParquetLayerBase::ISetSpatialFilter(iGeomField, poGeomIn);
1166 m_bRebuildScanner = true;
1167
1168 // Full invalidation
1169 InvalidateCachedBatches();
1170 return eErr;
1171 }
1172
1173 /************************************************************************/
1174 /* SetIgnoredFields() */
1175 /************************************************************************/
1176
1177 OGRErr OGRParquetDatasetLayer::SetIgnoredFields(CSLConstList papszFields)
1178 {
1179 m_bRebuildScanner = true;
1180 m_aosProjectedFields.clear();
1181 m_bIgnoredFields = false;
1182 m_anMapFieldIndexToArrayIndex.clear();
1183 m_anMapGeomFieldIndexToArrayIndex.clear();
1184 m_nRequestedFIDColumn = -1;
1185 OGRErr eErr = OGRParquetLayerBase::SetIgnoredFields(papszFields);
1186 if (eErr == OGRERR_NONE)
1187 {
1188 m_bIgnoredFields = papszFields != nullptr && papszFields[0] != nullptr;
1189 if (m_bIgnoredFields)
1190 {
1191 if (m_iFIDArrowColumn >= 0)
1192 {
1193 m_nRequestedFIDColumn =
1194 static_cast<int>(m_aosProjectedFields.size());
1195 m_aosProjectedFields.emplace_back(GetFIDColumn());
1196 }
1197
1198 const auto &fields = m_poSchema->fields();
1199 for (int i = 0; i < m_poFeatureDefn->GetFieldCount(); ++i)
1200 {
1201 const auto &field =
1202 fields[m_anMapFieldIndexToArrowColumn[i][0]];
1203 const auto eArrowType = field->type()->id();
1204 if (eArrowType == arrow::Type::STRUCT)
1205 {
1206 // For a struct, for the sake of simplicity in
1207 // GetNextRawFeature(), as soon as one of the member if
1208 // requested, request the struct field, so that the Arrow
1209 // type doesn't change
1210 bool bFoundNotIgnored = false;
1211 for (int j = i; j < m_poFeatureDefn->GetFieldCount() &&
1212 m_anMapFieldIndexToArrowColumn[i][0] ==
1213 m_anMapFieldIndexToArrowColumn[j][0];
1214 ++j)
1215 {
1216 if (!m_poFeatureDefn->GetFieldDefn(j)->IsIgnored())
1217 {
1218 bFoundNotIgnored = true;
1219 break;
1220 }
1221 }
1222 if (bFoundNotIgnored)
1223 {
1224 int j;
1225 for (j = i; j < m_poFeatureDefn->GetFieldCount() &&
1226 m_anMapFieldIndexToArrowColumn[i][0] ==
1227 m_anMapFieldIndexToArrowColumn[j][0];
1228 ++j)
1229 {
1230 if (!m_poFeatureDefn->GetFieldDefn(j)->IsIgnored())
1231 {
1232 m_anMapFieldIndexToArrayIndex.push_back(
1233 static_cast<int>(
1234 m_aosProjectedFields.size()));
1235 }
1236 else
1237 {
1238 m_anMapFieldIndexToArrayIndex.push_back(-1);
1239 }
1240 }
1241 i = j - 1;
1242
1243 m_aosProjectedFields.emplace_back(field->name());
1244 }
1245 else
1246 {
1247 int j;
1248 for (j = i; j < m_poFeatureDefn->GetFieldCount() &&
1249 m_anMapFieldIndexToArrowColumn[i][0] ==
1250 m_anMapFieldIndexToArrowColumn[j][0];
1251 ++j)
1252 {
1253 m_anMapFieldIndexToArrayIndex.push_back(-1);
1254 }
1255 i = j - 1;
1256 }
1257 }
1258 else if (!m_poFeatureDefn->GetFieldDefn(i)->IsIgnored())
1259 {
1260 m_anMapFieldIndexToArrayIndex.push_back(
1261 static_cast<int>(m_aosProjectedFields.size()));
1262 m_aosProjectedFields.emplace_back(field->name());
1263 }
1264 else
1265 {
1266 m_anMapFieldIndexToArrayIndex.push_back(-1);
1267 }
1268 }
1269
1270 for (int i = 0; i < m_poFeatureDefn->GetGeomFieldCount(); ++i)
1271 {
1272 const auto &field =
1273 fields[m_anMapGeomFieldIndexToArrowColumn[i]];
1274 if (!m_poFeatureDefn->GetGeomFieldDefn(i)->IsIgnored())
1275 {
1276 m_anMapGeomFieldIndexToArrayIndex.push_back(
1277 static_cast<int>(m_aosProjectedFields.size()));
1278 m_aosProjectedFields.emplace_back(field->name());
1279 }
1280 else
1281 {
1282 m_anMapGeomFieldIndexToArrayIndex.push_back(-1);
1283 }
1284 }
1285 }
1286 }
1287
1288 m_nExpectedBatchColumns =
1289 m_bIgnoredFields ? static_cast<int>(m_aosProjectedFields.size()) : -1;
1290
1291 // Full invalidation
1292 InvalidateCachedBatches();
1293
1294 return eErr;
1295 }
1296
1297 /************************************************************************/
1298 /* TestCapability() */
1299 /************************************************************************/
1300
1301 int OGRParquetDatasetLayer::TestCapability(const char *pszCap) const
1302 {
1303 if (EQUAL(pszCap, OLCIgnoreFields))
1304 return true;
1305
1306 if (EQUAL(pszCap, OLCFastSpatialFilter))
1307 {
1308 if (m_iGeomFieldFilter >= 0 &&
1309 m_iGeomFieldFilter < static_cast<int>(m_aeGeomEncoding.size()) &&
1310 m_aeGeomEncoding[m_iGeomFieldFilter] ==
1311 OGRArrowGeomEncoding::GEOARROW_STRUCT_POINT)
1312 {
1313 return true;
1314 }
1315 // fallback to base method
1316 }
1317
1318 return OGRParquetLayerBase::TestCapability(pszCap);
1319 }
1320
1321 /************************************************************************/
1322 /* SetAttributeFilter() */
1323 /************************************************************************/
1324
1325 OGRErr OGRParquetDatasetLayer::SetAttributeFilter(const char *pszFilter)
1326 {
1327 m_bRebuildScanner = true;
1328 return OGRParquetLayerBase::SetAttributeFilter(pszFilter);
1329 }