Debian Codegraph @

ogr/ogrsf_frmts/arrow_common/ograrrowlayer.hpp

raw
1 /******************************************************************************
2 *
3 * Project: Arrow generic code
4 * Purpose: Arrow generic code
5 * Author: Even Rouault, <even.rouault at spatialys.com>
6 *
7 ******************************************************************************
8 * Copyright (c) 2022, Planet Labs
9 *
10 * SPDX-License-Identifier: MIT
11 ****************************************************************************/
12
13 #ifndef OGARROWLAYER_HPP_INCLUDED
14 #define OGARROWLAYER_HPP_INCLUDED
15
16 #include "ogr_arrow.h"
17
18 #include "cpl_float.h"
19 #include "cpl_json.h"
20 #include "cpl_time.h"
21 #include "ogrlayerarrow.h"
22 #include "ogr_p.h"
23 #include "ogr_swq.h"
24 #include "ogr_wkb.h"
25
26 #include <algorithm>
27 #include <cassert>
28 #include <cinttypes>
29 #include <limits>
30 #include <string_view>
31
32 #define SWQ_ISNOTNULL (-SWQ_ISNULL)
33
34 #if defined(__clang__)
35 #pragma clang diagnostic push
36 #pragma clang diagnostic ignored "-Wweak-vtables"
37 #endif
38
39 inline IOGRArrowLayer::~IOGRArrowLayer() = default;
40
41 /************************************************************************/
42 /* OGRArrowLayer() */
43 /************************************************************************/
44
45 inline OGRArrowLayer::OGRArrowLayer(OGRArrowDataset *poDS,
46 const char *pszLayerName,
47 bool bListsAsStringJson)
48 : m_poArrowDS(poDS), m_bListsAsStringJson(bListsAsStringJson),
49 m_poMemoryPool(poDS->GetMemoryPool())
50 {
51 m_poFeatureDefn = new OGRFeatureDefn(pszLayerName);
52 m_poFeatureDefn->SetGeomType(wkbNone);
53 m_poFeatureDefn->Reference();
54 SetDescription(pszLayerName);
55 }
56
57 /************************************************************************/
58 /* ~OGRFeatherLayer() */
59 /************************************************************************/
60
61 inline OGRArrowLayer::~OGRArrowLayer()
62 {
63 if (m_sCachedSchema.release)
64 m_sCachedSchema.release(&m_sCachedSchema);
65
66 CPLDebug("ARROW", "Memory pool: bytes_allocated = %" PRId64,
67 m_poMemoryPool->bytes_allocated());
68 CPLDebug("ARROW", "Memory pool: max_memory = %" PRId64,
69 m_poMemoryPool->max_memory());
70 m_poFeatureDefn->Release();
71 }
72
73 /************************************************************************/
74 /* LoadGDALSchema() */
75 /************************************************************************/
76
77 inline std::map<std::string, std::unique_ptr<OGRFieldDefn>>
78 OGRArrowLayer::LoadGDALSchema(const arrow::KeyValueMetadata *kv_metadata)
79 {
80 std::map<std::string, std::unique_ptr<OGRFieldDefn>>
81 oMapFieldNameToGDALSchemaFieldDefn;
82 if (kv_metadata && kv_metadata->Contains("gdal:schema") &&
83 CPLTestBool(CPLGetConfigOption(
84 ("OGR_" + GetDriverUCName() + "_READ_GDAL_SCHEMA").c_str(), "YES")))
85 {
86 auto gdalSchema = kv_metadata->Get("gdal:schema");
87 if (gdalSchema.ok())
88 {
89 CPLDebug(GetDriverUCName().c_str(), "gdal:schema = %s",
90 gdalSchema->c_str());
91 CPLJSONDocument oDoc;
92 if (oDoc.LoadMemory(*gdalSchema))
93 {
94 auto oRoot = oDoc.GetRoot();
95
96 m_osFIDColumn = oRoot.GetString("fid");
97
98 auto oColumns = oRoot.GetObj("columns");
99 if (oColumns.IsValid())
100 {
101 for (const auto &oColumn : oColumns.GetChildren())
102 {
103 const auto osName = oColumn.GetName();
104 const auto osType = oColumn.GetString("type");
105 const auto osSubType = oColumn.GetString("subtype");
106 auto poFieldDefn = std::make_unique<OGRFieldDefn>(
107 osName.c_str(), OFTString);
108 for (int iType = 0;
109 iType <= static_cast<int>(OFTMaxType); iType++)
110 {
111 if (EQUAL(osType.c_str(),
112 OGRFieldDefn::GetFieldTypeName(
113 static_cast<OGRFieldType>(iType))))
114 {
115 poFieldDefn->SetType(
116 static_cast<OGRFieldType>(iType));
117 break;
118 }
119 }
120 if (!osSubType.empty())
121 {
122 for (int iSubType = 0;
123 iSubType <= static_cast<int>(OFSTMaxSubType);
124 iSubType++)
125 {
126 if (EQUAL(osSubType.c_str(),
127 OGRFieldDefn::GetFieldSubTypeName(
128 static_cast<OGRFieldSubType>(
129 iSubType))))
130 {
131 poFieldDefn->SetSubType(
132 static_cast<OGRFieldSubType>(iSubType));
133 break;
134 }
135 }
136 }
137 poFieldDefn->SetWidth(oColumn.GetInteger("width"));
138 poFieldDefn->SetPrecision(
139 oColumn.GetInteger("precision"));
140
141 const auto osAlternativeName =
142 oColumn.GetString("alternative_name");
143 if (!osAlternativeName.empty())
144 poFieldDefn->SetAlternativeName(
145 osAlternativeName.c_str());
146
147 const auto osComment = oColumn.GetString("comment");
148 if (!osComment.empty())
149 poFieldDefn->SetComment(osComment);
150
151 oMapFieldNameToGDALSchemaFieldDefn[osName] =
152 std::move(poFieldDefn);
153 }
154 }
155 }
156 }
157 }
158 return oMapFieldNameToGDALSchemaFieldDefn;
159 }
160
161 /************************************************************************/
162 /* LoadGDALMetadata() */
163 /************************************************************************/
164
165 inline void
166 OGRArrowLayer::LoadGDALMetadata(const arrow::KeyValueMetadata *kv_metadata)
167 {
168 if (kv_metadata && kv_metadata->Contains("gdal:metadata"))
169 {
170 auto gdalMetadata = kv_metadata->Get("gdal:metadata");
171 if (gdalMetadata.ok())
172 {
173 CPLJSONDocument oDoc;
174 if (oDoc.LoadMemory(*gdalMetadata))
175 {
176 auto oRoot = oDoc.GetRoot();
177 for (const auto &oDomain : oRoot.GetChildren())
178 {
179 if (STARTS_WITH(oDomain.GetName().c_str(), "json:") &&
180 oDomain.GetType() == CPLJSONObject::Type::Object)
181 {
182 char **papszMD = nullptr;
183 papszMD = CSLAddString(
184 papszMD,
185 oDomain.Format(CPLJSONObject::PrettyFormat::Plain)
186 .c_str());
187 SetMetadata(papszMD, oDomain.GetName().c_str());
188 CSLDestroy(papszMD);
189 }
190 else if (STARTS_WITH(oDomain.GetName().c_str(), "xml:") &&
191 oDomain.GetType() == CPLJSONObject::Type::String)
192 {
193 char **papszMD = nullptr;
194 papszMD =
195 CSLAddString(papszMD, oDomain.ToString().c_str());
196 SetMetadata(papszMD, oDomain.GetName().c_str());
197 CSLDestroy(papszMD);
198 }
199 else
200 {
201 for (const auto &oItem : oDomain.GetChildren())
202 {
203 if (oItem.GetType() == CPLJSONObject::Type::String)
204 {
205 SetMetadataItem(oItem.GetName().c_str(),
206 oItem.ToString().c_str(),
207 oDomain.GetName().c_str());
208 }
209 }
210 }
211 }
212 }
213 }
214 }
215 }
216
217 /************************************************************************/
218 /* IsIntegerArrowType() */
219 /************************************************************************/
220
221 inline bool OGRArrowLayer::IsIntegerArrowType(arrow::Type::type typeId)
222 {
223 return typeId == arrow::Type::INT8 || typeId == arrow::Type::UINT8 ||
224 typeId == arrow::Type::INT16 || typeId == arrow::Type::UINT16 ||
225 typeId == arrow::Type::INT32 || typeId == arrow::Type::UINT32 ||
226 typeId == arrow::Type::INT64 || typeId == arrow::Type::UINT64;
227 }
228
229 /************************************************************************/
230 /* IsHandledListOrMapType() */
231 /************************************************************************/
232
233 inline bool OGRArrowLayer::IsHandledListOrMapType(
234 const std::shared_ptr<arrow::DataType> &valueType)
235 {
236 const auto itemTypeId = valueType->id();
237 return itemTypeId == arrow::Type::BOOL || IsIntegerArrowType(itemTypeId) ||
238 itemTypeId == arrow::Type::HALF_FLOAT ||
239 itemTypeId == arrow::Type::FLOAT ||
240 itemTypeId == arrow::Type::DOUBLE ||
241 #if ARROW_VERSION_MAJOR >= 18
242 itemTypeId == arrow::Type::DECIMAL32 ||
243 itemTypeId == arrow::Type::DECIMAL64 ||
244 #endif
245 itemTypeId == arrow::Type::DECIMAL128 ||
246 itemTypeId == arrow::Type::DECIMAL256 ||
247 itemTypeId == arrow::Type::STRING ||
248 itemTypeId == arrow::Type::LARGE_STRING ||
249 #if ARROW_VERSION_MAJOR >= 15
250 itemTypeId == arrow::Type::STRING_VIEW ||
251 #endif
252 itemTypeId == arrow::Type::BINARY ||
253 #if ARROW_VERSION_MAJOR >= 15
254 itemTypeId == arrow::Type::BINARY_VIEW ||
255 #endif
256 itemTypeId == arrow::Type::STRUCT ||
257 (itemTypeId == arrow::Type::MAP &&
258 IsHandledMapType(
259 std::static_pointer_cast<arrow::MapType>(valueType))) ||
260 ((itemTypeId == arrow::Type::LIST ||
261 itemTypeId == arrow::Type::LARGE_LIST ||
262 itemTypeId == arrow::Type::FIXED_SIZE_LIST) &&
263 IsHandledListType(
264 std::static_pointer_cast<arrow::BaseListType>(valueType)));
265 }
266
267 /************************************************************************/
268 /* IsHandledListType() */
269 /************************************************************************/
270
271 inline bool OGRArrowLayer::IsHandledListType(
272 const std::shared_ptr<arrow::BaseListType> &listType)
273 {
274 return IsHandledListOrMapType(listType->value_type());
275 }
276
277 /************************************************************************/
278 /* IsHandledMapType() */
279 /************************************************************************/
280
281 inline bool
282 OGRArrowLayer::IsHandledMapType(const std::shared_ptr<arrow::MapType> &mapType)
283 {
284 const auto typeId = mapType->key_type()->id();
285 return (typeId == arrow::Type::STRING
286 #if ARROW_VERSION_MAJOR >= 15
287 || typeId == arrow::Type::STRING_VIEW
288 #endif
289 ) &&
290 IsHandledListOrMapType(mapType->item_type());
291 }
292
293 /************************************************************************/
294 /* GetFieldExtensionName() */
295 /************************************************************************/
296
297 inline static std::string
298 GetFieldExtensionName(const std::shared_ptr<arrow::Field> &field,
299 std::shared_ptr<arrow::DataType> &type,
300 const char *pszDebugKey)
301 {
302 std::string osExtensionName;
303 if (type->id() == arrow::Type::EXTENSION)
304 {
305 auto extensionType = cpl::down_cast<arrow::ExtensionType *>(type.get());
306 osExtensionName = extensionType->extension_name();
307 type = extensionType->storage_type();
308 }
309 else if (const auto &field_kv_metadata = field->metadata())
310 {
311 auto extension_name = field_kv_metadata->Get(ARROW_EXTENSION_NAME_KEY);
312 if (extension_name.ok())
313 {
314 osExtensionName = *extension_name;
315 }
316 }
317
318 if (!osExtensionName.empty() &&
319 osExtensionName != EXTENSION_NAME_ARROW_JSON &&
320 osExtensionName != EXTENSION_NAME_ARROW_TIMESTAMP_WITH_OFFSET)
321 {
322 CPLDebug(pszDebugKey,
323 "Dealing with field %s of extension type %s as %s",
324 field->name().c_str(), osExtensionName.c_str(),
325 type->ToString().c_str());
326 }
327 return osExtensionName;
328 }
329
330 /************************************************************************/
331 /* MapArrowTypeToOGR() */
332 /************************************************************************/
333
334 inline bool OGRArrowLayer::MapArrowTypeToOGR(
335 const std::shared_ptr<arrow::DataType> &typeIn,
336 const std::shared_ptr<arrow::Field> &field, OGRFieldDefn &oField,
337 OGRFieldType &eType, OGRFieldSubType &eSubType,
338 const std::vector<int> &path,
339 const std::map<std::string, std::unique_ptr<OGRFieldDefn>>
340 &oMapFieldNameToGDALSchemaFieldDefn)
341 {
342 bool bTypeOK = false;
343
344 std::shared_ptr<arrow::DataType> type(typeIn);
345 const std::string osExtensionName =
346 GetFieldExtensionName(field, type, GetDriverUCName().c_str());
347 switch (type->id())
348 {
349 case arrow::Type::NA:
350 break;
351
352 case arrow::Type::BOOL:
353 bTypeOK = true;
354 eType = OFTInteger;
355 eSubType = OFSTBoolean;
356 break;
357 case arrow::Type::UINT8:
358 case arrow::Type::INT8:
359 case arrow::Type::UINT16:
360 bTypeOK = true;
361 eType = OFTInteger;
362 break;
363 case arrow::Type::INT16:
364 bTypeOK = true;
365 eType = OFTInteger;
366 eSubType = OFSTInt16;
367 break;
368 case arrow::Type::UINT32:
369 bTypeOK = true;
370 eType = OFTInteger64;
371 break;
372 case arrow::Type::INT32:
373 bTypeOK = true;
374 eType = OFTInteger;
375 break;
376 case arrow::Type::UINT64:
377 bTypeOK = true;
378 eType = OFTReal; // potential loss
379 break;
380 case arrow::Type::INT64:
381 bTypeOK = true;
382 eType = OFTInteger64;
383 break;
384 case arrow::Type::HALF_FLOAT: // should use OFSTFloat16 if we had it
385 case arrow::Type::FLOAT:
386 bTypeOK = true;
387 eType = OFTReal;
388 eSubType = OFSTFloat32;
389 break;
390 case arrow::Type::DOUBLE:
391 bTypeOK = true;
392 eType = OFTReal;
393 break;
394 case arrow::Type::STRING:
395 case arrow::Type::LARGE_STRING:
396 #if ARROW_VERSION_MAJOR >= 15
397 case arrow::Type::STRING_VIEW:
398 #endif
399 bTypeOK = true;
400 eType = OFTString;
401 if (osExtensionName == EXTENSION_NAME_ARROW_JSON)
402 eSubType = OFSTJSON;
403 break;
404 case arrow::Type::BINARY:
405 case arrow::Type::LARGE_BINARY:
406 #if ARROW_VERSION_MAJOR >= 15
407 case arrow::Type::BINARY_VIEW:
408 #endif
409 bTypeOK = true;
410 eType = OFTBinary;
411 break;
412 case arrow::Type::FIXED_SIZE_BINARY:
413 bTypeOK = true;
414 eType = OFTBinary;
415 oField.SetWidth(
416 std::static_pointer_cast<arrow::FixedSizeBinaryType>(type)
417 ->byte_width());
418 break;
419
420 case arrow::Type::DATE32:
421 case arrow::Type::DATE64:
422 bTypeOK = true;
423 eType = OFTDate;
424 break;
425
426 case arrow::Type::TIMESTAMP:
427 {
428 bTypeOK = true;
429 const auto timestampType =
430 static_cast<arrow::TimestampType *>(type.get());
431 eType = OFTDateTime;
432 const auto &osTZ = timestampType->timezone();
433 int nTZFlag = OGRTimezoneToTZFlag(osTZ.c_str(), false);
434 if (nTZFlag == OGR_TZFLAG_UNKNOWN && !osTZ.empty())
435 {
436 CPLDebug(GetDriverUCName().c_str(),
437 "Field %s has unrecognized timezone %s. "
438 "UTC datetime will be used instead.",
439 field->name().c_str(), osTZ.c_str());
440 nTZFlag = OGR_TZFLAG_UTC;
441 }
442 oField.SetTZFlag(nTZFlag);
443 break;
444 }
445
446 case arrow::Type::TIME32:
447 bTypeOK = true;
448 eType = OFTTime;
449 break;
450
451 case arrow::Type::TIME64:
452 bTypeOK = true;
453 eType = OFTInteger64; // our OFTTime doesn't have micro or
454 // nanosecond accuracy
455 break;
456
457 #if ARROW_VERSION_MAJOR >= 18
458 case arrow::Type::DECIMAL32:
459 case arrow::Type::DECIMAL64:
460 #endif
461 case arrow::Type::DECIMAL128:
462 case arrow::Type::DECIMAL256:
463 {
464 bTypeOK = true;
465 const auto decimalType =
466 std::static_pointer_cast<arrow::DecimalType>(type);
467 eType = OFTReal;
468 oField.SetWidth(decimalType->precision());
469 oField.SetPrecision(decimalType->scale());
470 break;
471 }
472
473 case arrow::Type::LIST:
474 case arrow::Type::LARGE_LIST:
475 case arrow::Type::FIXED_SIZE_LIST:
476 {
477 bTypeOK = true;
478 auto listType = std::static_pointer_cast<arrow::BaseListType>(type);
479 switch (listType->value_type()->id())
480 {
481 case arrow::Type::BOOL:
482 eType = OFTIntegerList;
483 eSubType = OFSTBoolean;
484 break;
485 case arrow::Type::UINT8:
486 case arrow::Type::INT8:
487 case arrow::Type::UINT16:
488 case arrow::Type::INT16:
489 case arrow::Type::INT32:
490 eType = OFTIntegerList;
491 break;
492 case arrow::Type::UINT32:
493 eType = OFTInteger64List;
494 break;
495 case arrow::Type::UINT64:
496 eType = OFTRealList; // potential loss
497 break;
498 case arrow::Type::INT64:
499 eType = OFTInteger64List;
500 break;
501 case arrow::Type::HALF_FLOAT: // should use OFSTFloat16 if we
502 // had it
503 case arrow::Type::FLOAT:
504 eType = OFTRealList;
505 eSubType = OFSTFloat32;
506 break;
507 case arrow::Type::DOUBLE:
508 #if ARROW_VERSION_MAJOR >= 18
509 case arrow::Type::DECIMAL32:
510 case arrow::Type::DECIMAL64:
511 #endif
512 case arrow::Type::DECIMAL128:
513 case arrow::Type::DECIMAL256:
514 eType = OFTRealList;
515 break;
516 case arrow::Type::STRING:
517 case arrow::Type::LARGE_STRING:
518 case arrow::Type::BINARY:
519 #if ARROW_VERSION_MAJOR >= 15
520 case arrow::Type::STRING_VIEW:
521 #endif
522 eType = OFTStringList;
523 break;
524 default:
525 {
526 if (IsHandledListType(listType))
527 {
528 eType = OFTString;
529 eSubType = OFSTJSON;
530 }
531 else
532 {
533 bTypeOK = false;
534 CPLError(CE_Warning, CPLE_AppDefined,
535 "Field %s of unhandled type %s ignored",
536 field->name().c_str(),
537 type->ToString().c_str());
538 }
539 break;
540 }
541 }
542
543 if (bTypeOK && m_bListsAsStringJson)
544 {
545 eType = OFTString;
546 eSubType = OFSTJSON;
547 }
548
549 break;
550 }
551
552 case arrow::Type::MAP:
553 {
554 bTypeOK = true;
555 auto mapType = std::static_pointer_cast<arrow::MapType>(type);
556 if (IsHandledMapType(mapType))
557 {
558 eType = OFTString;
559 eSubType = OFSTJSON;
560 }
561 else
562 {
563 bTypeOK = false;
564 CPLError(CE_Warning, CPLE_AppDefined,
565 "Field %s of unhandled type %s ignored",
566 field->name().c_str(), type->ToString().c_str());
567 }
568 break;
569 }
570
571 case arrow::Type::STRUCT:
572 // should be handled by specialized code
573 CPLAssert(false);
574 break;
575
576 // unhandled types
577
578 case arrow::Type::INTERVAL_MONTHS:
579 case arrow::Type::INTERVAL_DAY_TIME:
580 case arrow::Type::SPARSE_UNION:
581 case arrow::Type::DENSE_UNION:
582 case arrow::Type::DICTIONARY:
583 case arrow::Type::EXTENSION:
584 case arrow::Type::DURATION:
585 case arrow::Type::INTERVAL_MONTH_DAY_NANO:
586 #if ARROW_VERSION_MAJOR >= 12
587 case arrow::Type::RUN_END_ENCODED:
588 #endif
589 #if ARROW_VERSION_MAJOR >= 15
590 case arrow::Type::LIST_VIEW:
591 case arrow::Type::LARGE_LIST_VIEW:
592 #endif
593 case arrow::Type::MAX_ID:
594 {
595 CPLError(CE_Warning, CPLE_AppDefined,
596 "Field %s of unhandled type %s ignored",
597 field->name().c_str(), type->ToString().c_str());
598 break;
599 }
600 }
601
602 if (bTypeOK)
603 {
604 const auto oIter =
605 oMapFieldNameToGDALSchemaFieldDefn.find(field->name());
606 oField.SetType(eType);
607 if (oIter != oMapFieldNameToGDALSchemaFieldDefn.end())
608 {
609 const auto &poGDALFieldDefn = oIter->second;
610 if (poGDALFieldDefn->GetType() == eType)
611 {
612 if (eSubType == OFSTNone)
613 {
614 eSubType = poGDALFieldDefn->GetSubType();
615 }
616 else if (eSubType != poGDALFieldDefn->GetSubType())
617 {
618 CPLDebug(
619 GetDriverUCName().c_str(),
620 "Field subtype inferred from Parquet/Arrow schema is "
621 "%s, "
622 "whereas the one in gdal:schema is %s. "
623 "Using the former one.",
624 OGR_GetFieldSubTypeName(eSubType),
625 OGR_GetFieldSubTypeName(poGDALFieldDefn->GetSubType()));
626 }
627 }
628 else
629 {
630 CPLDebug(GetDriverUCName().c_str(),
631 "Field type inferred from Parquet/Arrow schema is %s, "
632 "whereas the one in gdal:schema is %s. "
633 "Using the former one.",
634 OGR_GetFieldTypeName(eType),
635 OGR_GetFieldTypeName(poGDALFieldDefn->GetType()));
636 }
637 if (poGDALFieldDefn->GetWidth() > 0)
638 oField.SetWidth(poGDALFieldDefn->GetWidth());
639 if (poGDALFieldDefn->GetPrecision() > 0)
640 oField.SetPrecision(poGDALFieldDefn->GetPrecision());
641 if (poGDALFieldDefn->GetAlternativeNameRef()[0])
642 oField.SetAlternativeName(
643 poGDALFieldDefn->GetAlternativeNameRef());
644 if (!poGDALFieldDefn->GetComment().empty())
645 oField.SetComment(poGDALFieldDefn->GetComment());
646 }
647 oField.SetSubType(eSubType);
648 oField.SetNullable(field->nullable());
649 m_poFeatureDefn->AddFieldDefn(&oField);
650 m_anMapFieldIndexToArrowColumn.push_back(path);
651 }
652
653 return bTypeOK;
654 }
655
656 /************************************************************************/
657 /* CreateFieldFromSchema() */
658 /************************************************************************/
659
660 inline void OGRArrowLayer::CreateFieldFromSchema(
661 const std::shared_ptr<arrow::Field> &field, const std::vector<int> &path,
662 const std::map<std::string, std::unique_ptr<OGRFieldDefn>>
663 &oMapFieldNameToGDALSchemaFieldDefn)
664 {
665 OGRFieldDefn oField(field->name().c_str(), OFTString);
666 OGRFieldType eType = OFTString;
667 OGRFieldSubType eSubType = OFSTNone;
668 bool bTypeOK = true;
669
670 auto type = field->type();
671 if (type->id() == arrow::Type::DICTIONARY && path.size() == 1)
672 {
673 const auto dictionaryType =
674 std::static_pointer_cast<arrow::DictionaryType>(field->type());
675 auto indexType = dictionaryType->index_type();
676 if (dictionaryType->value_type()->id() == arrow::Type::STRING &&
677 IsIntegerArrowType(indexType->id()))
678 {
679 std::string osDomainName(field->name() + "Domain");
680 m_poArrowDS->RegisterDomainName(osDomainName,
681 m_poFeatureDefn->GetFieldCount());
682 oField.SetDomainName(osDomainName);
683 type = std::move(indexType);
684 }
685 else
686 {
687 bTypeOK = false;
688 }
689 }
690
691 if (type->id() == arrow::Type::STRUCT)
692 {
693 const auto subfields = field->Flatten();
694 const std::string osExtensionName =
695 GetFieldExtensionName(field, type, GetDriverUCName().c_str());
696 if (osExtensionName == EXTENSION_NAME_ARROW_TIMESTAMP_WITH_OFFSET &&
697 subfields.size() == 2 &&
698 subfields[0]->name() ==
699 field->name() + "." + ATSWO_TIMESTAMP_FIELD_NAME &&
700 subfields[0]->type()->id() == arrow::Type::TIMESTAMP &&
701 subfields[1]->name() ==
702 field->name() + "." + ATSWO_OFFSET_MINUTES_FIELD_NAME &&
703 subfields[1]->type()->id() == arrow::Type::INT16)
704 {
705 oField.SetType(OFTDateTime);
706 oField.SetTZFlag(OGR_TZFLAG_MIXED_TZ);
707 oField.SetNullable(field->nullable());
708 m_poFeatureDefn->AddFieldDefn(&oField);
709 m_anMapFieldIndexToArrowColumn.push_back(path);
710 }
711 else
712 {
713 auto newpath = path;
714 newpath.push_back(0);
715 for (int j = 0; j < static_cast<int>(subfields.size()); j++)
716 {
717 const auto &subfield = subfields[j];
718 newpath.back() = j;
719 CreateFieldFromSchema(subfield, newpath,
720 oMapFieldNameToGDALSchemaFieldDefn);
721 }
722 }
723 }
724 else if (bTypeOK)
725 {
726 MapArrowTypeToOGR(type, field, oField, eType, eSubType, path,
727 oMapFieldNameToGDALSchemaFieldDefn);
728 }
729 }
730
731 /************************************************************************/
732 /* BuildDomainFromBatch() */
733 /************************************************************************/
734
735 inline std::unique_ptr<OGRFieldDomain> OGRArrowLayer::BuildDomainFromBatch(
736 const std::string &osDomainName,
737 const std::shared_ptr<arrow::RecordBatch> &poBatch, int iCol) const
738 {
739 const auto array = poBatch->column(iCol);
740 auto castArray = std::static_pointer_cast<arrow::DictionaryArray>(array);
741 auto dict = castArray->dictionary();
742 CPLAssert(dict->type_id() == arrow::Type::STRING);
743 OGRFieldType eType = OFTInteger;
744 const auto indexTypeId = castArray->dict_type()->index_type()->id();
745 if (indexTypeId == arrow::Type::UINT32 ||
746 indexTypeId == arrow::Type::UINT64 || indexTypeId == arrow::Type::INT64)
747 eType = OFTInteger64;
748 auto values = std::static_pointer_cast<arrow::StringArray>(dict);
749 std::vector<OGRCodedValue> asValues;
750 if (values->length() > INT_MAX)
751 {
752 CPLError(CE_Failure, CPLE_OutOfMemory,
753 "BuildDomainFromBatch(): too many values");
754 return nullptr;
755 }
756 try
757 {
758 asValues.reserve(static_cast<size_t>(values->length()));
759 }
760 catch (const std::bad_alloc &)
761 {
762 CPLError(CE_Failure, CPLE_OutOfMemory,
763 "BuildDomainFromBatch(): out of memory");
764 return nullptr;
765 }
766 for (int i = 0; i < static_cast<int>(values->length()); ++i)
767 {
768 if (!values->IsNull(i))
769 {
770 OGRCodedValue val;
771 val.pszCode = CPLStrdup(CPLSPrintf("%d", i));
772 val.pszValue = CPLStrdup(values->GetString(i).c_str());
773 asValues.emplace_back(val);
774 }
775 }
776 return std::make_unique<OGRCodedFieldDomain>(
777 osDomainName, std::string(), eType, OFSTNone, std::move(asValues));
778 }
779
780 /************************************************************************/
781 /* GetStorageArray() */
782 /************************************************************************/
783
784 static const arrow::Array *GetStorageArray(const arrow::Array *array)
785 {
786 if (array->type_id() == arrow::Type::EXTENSION)
787 {
788 auto extensionArray =
789 cpl::down_cast<const arrow::ExtensionArray *>(array);
790 array = extensionArray->storage().get();
791 }
792 return array;
793 }
794
795 /************************************************************************/
796 /* ComputeGeometryColumnTypeProcessBatch() */
797 /************************************************************************/
798
799 inline OGRwkbGeometryType OGRArrowLayer::ComputeGeometryColumnTypeProcessBatch(
800 const std::shared_ptr<arrow::RecordBatch> &poBatch, int iGeomCol,
801 int iBatchCol, OGRwkbGeometryType eGeomType) const
802 {
803 const auto array = poBatch->column(iBatchCol);
804 const auto storageArray = GetStorageArray(array.get());
805 const auto castBinaryArray =
806 (m_aeGeomEncoding[iGeomCol] == OGRArrowGeomEncoding::WKB)
807 ? dynamic_cast<const arrow::BinaryArray *>(storageArray)
808 : nullptr;
809 const auto castLargeBinaryArray =
810 (m_aeGeomEncoding[iGeomCol] == OGRArrowGeomEncoding::WKB)
811 ? dynamic_cast<const arrow::LargeBinaryArray *>(storageArray)
812 : nullptr;
813 const auto castStringArray =
814 (m_aeGeomEncoding[iGeomCol] == OGRArrowGeomEncoding::WKT)
815 ? dynamic_cast<const arrow::StringArray *>(storageArray)
816 : nullptr;
817 const auto castLargeStringArray =
818 (m_aeGeomEncoding[iGeomCol] == OGRArrowGeomEncoding::WKT)
819 ? dynamic_cast<const arrow::LargeStringArray *>(storageArray)
820 : nullptr;
821 for (int64_t i = 0; i < poBatch->num_rows(); i++)
822 {
823 if (!storageArray->IsNull(i))
824 {
825 OGRwkbGeometryType eThisGeomType = wkbNone;
826 if (m_aeGeomEncoding[iGeomCol] == OGRArrowGeomEncoding::WKB &&
827 castBinaryArray)
828 {
829 arrow::BinaryArray::offset_type out_length = 0;
830 const uint8_t *data = castBinaryArray->GetValue(i, &out_length);
831 if (out_length >= 5)
832 {
833 OGRReadWKBGeometryType(data, wkbVariantIso, &eThisGeomType);
834 }
835 }
836 else if (m_aeGeomEncoding[iGeomCol] == OGRArrowGeomEncoding::WKB &&
837 castLargeBinaryArray)
838 {
839 arrow::LargeBinaryArray::offset_type out_length = 0;
840 const uint8_t *data =
841 castLargeBinaryArray->GetValue(i, &out_length);
842 if (out_length >= 5)
843 {
844 OGRReadWKBGeometryType(data, wkbVariantIso, &eThisGeomType);
845 }
846 }
847 else if (m_aeGeomEncoding[iGeomCol] == OGRArrowGeomEncoding::WKT &&
848 castStringArray)
849 {
850 const auto osWKT = castStringArray->GetString(i);
851 if (!osWKT.empty())
852 {
853 OGRReadWKTGeometryType(osWKT.c_str(), &eThisGeomType);
854 }
855 }
856 else if (m_aeGeomEncoding[iGeomCol] == OGRArrowGeomEncoding::WKT &&
857 castLargeStringArray)
858 {
859 const auto osWKT = castLargeStringArray->GetString(i);
860 if (!osWKT.empty())
861 {
862 OGRReadWKTGeometryType(osWKT.c_str(), &eThisGeomType);
863 }
864 }
865
866 if (eThisGeomType != wkbNone)
867 {
868 if (eGeomType == wkbNone)
869 eGeomType = eThisGeomType;
870 else if (wkbFlatten(eThisGeomType) == wkbFlatten(eGeomType))
871 ;
872 else if (wkbFlatten(eThisGeomType) == wkbMultiLineString &&
873 wkbFlatten(eGeomType) == wkbLineString)
874 {
875 eGeomType = OGR_GT_SetModifier(
876 wkbMultiLineString,
877 OGR_GT_HasZ(eThisGeomType) || OGR_GT_HasZ(eGeomType),
878 OGR_GT_HasM(eThisGeomType) || OGR_GT_HasM(eGeomType));
879 }
880 else if (wkbFlatten(eThisGeomType) == wkbLineString &&
881 wkbFlatten(eGeomType) == wkbMultiLineString)
882 ;
883 else if (wkbFlatten(eThisGeomType) == wkbMultiPolygon &&
884 wkbFlatten(eGeomType) == wkbPolygon)
885 {
886 eGeomType = OGR_GT_SetModifier(
887 wkbMultiPolygon,
888 OGR_GT_HasZ(eThisGeomType) || OGR_GT_HasZ(eGeomType),
889 OGR_GT_HasM(eThisGeomType) || OGR_GT_HasM(eGeomType));
890 }
891 else if (wkbFlatten(eThisGeomType) == wkbPolygon &&
892 wkbFlatten(eGeomType) == wkbMultiPolygon)
893 ;
894 else
895 return wkbUnknown;
896
897 eGeomType = OGR_GT_SetModifier(
898 eGeomType,
899 OGR_GT_HasZ(eThisGeomType) || OGR_GT_HasZ(eGeomType),
900 OGR_GT_HasM(eThisGeomType) || OGR_GT_HasM(eGeomType));
901 }
902 }
903 }
904 return eGeomType;
905 }
906
907 /************************************************************************/
908 /* IsPointType() */
909 /************************************************************************/
910
911 static bool IsPointType(const std::shared_ptr<arrow::DataType> &type,
912 bool &bHasZOut, bool &bHasMOut)
913 {
914 if (type->id() != arrow::Type::FIXED_SIZE_LIST)
915 return false;
916 auto poListType = std::static_pointer_cast<arrow::FixedSizeListType>(type);
917 const int nOutDimensionality = poListType->list_size();
918 const std::string osValueFieldName(poListType->value_field()->name());
919 if (nOutDimensionality == 2)
920 {
921 bHasZOut = false;
922 bHasMOut = false;
923 }
924 else if (nOutDimensionality == 3)
925 {
926 if (osValueFieldName == "xym")
927 {
928 bHasZOut = false;
929 bHasMOut = true;
930 }
931 else /* if (osValueFieldName == "xyz" || osValueFieldName == "element") */
932 {
933 bHasMOut = false;
934 bHasZOut = true;
935 }
936 }
937 else if (nOutDimensionality == 4)
938 {
939 bHasMOut = true;
940 bHasZOut = true;
941 }
942 else
943 {
944 return false;
945 }
946 return poListType->value_type()->id() == arrow::Type::DOUBLE;
947 }
948
949 /************************************************************************/
950 /* IsListOfPointType() */
951 /************************************************************************/
952
953 static bool IsListOfPointType(const std::shared_ptr<arrow::DataType> &type,
954 int nDepth, bool &bHasZOut, bool &bHasMOut)
955 {
956 if (type->id() != arrow::Type::LIST)
957 return false;
958 auto poListType = std::static_pointer_cast<arrow::ListType>(type);
959 return nDepth == 1
960 ? IsPointType(poListType->value_type(), bHasZOut, bHasMOut)
961 : IsListOfPointType(poListType->value_type(), nDepth - 1,
962 bHasZOut, bHasMOut);
963 }
964
965 /************************************************************************/
966 /* IsPointStructType() */
967 /************************************************************************/
968
969 static bool IsPointStructType(const std::shared_ptr<arrow::DataType> &type,
970 bool &bHasZOut, bool &bHasMOut)
971 {
972 if (type->id() != arrow::Type::STRUCT)
973 return false;
974 auto poStructType = std::static_pointer_cast<arrow::StructType>(type);
975 const int nNumFields = poStructType->num_fields();
976 if (nNumFields < 2 || nNumFields > 4)
977 return false;
978 bHasZOut = false;
979 bHasMOut = false;
980 const auto poFieldX = poStructType->field(0);
981 if (poFieldX->name() != "x" ||
982 poFieldX->type()->id() != arrow::Type::DOUBLE)
983 return false;
984 const auto poFieldY = poStructType->field(1);
985 if (poFieldY->name() != "y" ||
986 poFieldY->type()->id() != arrow::Type::DOUBLE)
987 return false;
988 if (nNumFields == 2)
989 return true;
990 const auto poField2 = poStructType->field(2);
991 if (poField2->type()->id() != arrow::Type::DOUBLE)
992 return false;
993 if (poField2->name() == "z")
994 {
995 bHasZOut = true;
996 if (nNumFields == 4)
997 {
998 const auto poField3 = poStructType->field(3);
999 if (poField3->name() != "m" ||
1000 poField3->type()->id() != arrow::Type::DOUBLE)
1001 return false;
1002 bHasMOut = true;
1003 }
1004 }
1005 else if (poField2->name() == "m")
1006 {
1007 bHasMOut = true;
1008 }
1009 else
1010 {
1011 return false;
1012 }
1013 return true;
1014 }
1015
1016 /************************************************************************/
1017 /* IsListOfPointStructType() */
1018 /************************************************************************/
1019
1020 static bool
1021 IsListOfPointStructType(const std::shared_ptr<arrow::DataType> &type,
1022 int nDepth, bool &bHasZOut, bool &bHasMOut)
1023 {
1024 if (type->id() != arrow::Type::LIST)
1025 return false;
1026 auto poListType = std::static_pointer_cast<arrow::ListType>(type);
1027 return nDepth == 1
1028 ? IsPointStructType(poListType->value_type(), bHasZOut, bHasMOut)
1029 : IsListOfPointStructType(poListType->value_type(), nDepth - 1,
1030 bHasZOut, bHasMOut);
1031 }
1032
1033 /************************************************************************/
1034 /* IsValidGeometryEncoding() */
1035 /************************************************************************/
1036
1037 inline bool OGRArrowLayer::IsValidGeometryEncoding(
1038 const std::shared_ptr<arrow::Field> &field, const std::string &osEncoding,
1039 bool bWarnIfUnknownEncoding, OGRwkbGeometryType &eGeomTypeOut,
1040 OGRArrowGeomEncoding &eOGRArrowGeomEncodingOut)
1041 {
1042 const auto &fieldName = field->name();
1043 std::shared_ptr<arrow::DataType> fieldType = field->type();
1044 auto fieldTypeId = fieldType->id();
1045
1046 if (fieldTypeId == arrow::Type::EXTENSION)
1047 {
1048 auto extensionType =
1049 cpl::down_cast<arrow::ExtensionType *>(fieldType.get());
1050 fieldType = extensionType->storage_type();
1051 fieldTypeId = fieldType->id();
1052 }
1053
1054 eGeomTypeOut = wkbUnknown;
1055
1056 if (osEncoding == "WKT" || // As used in Parquet geo metadata
1057 osEncoding ==
1058 "ogc.wkt" || // As used in ARROW:extension:name field metadata
1059 osEncoding ==
1060 "geoarrow.wkt" // As used in ARROW:extension:name field metadata
1061 )
1062 {
1063 if (fieldTypeId != arrow::Type::LARGE_STRING &&
1064 fieldTypeId != arrow::Type::STRING)
1065 {
1066 CPLError(CE_Warning, CPLE_AppDefined,
1067 "Geometry column %s has a non String type: %s. "
1068 "Handling it as a regular field",
1069 fieldName.c_str(), fieldType->ToString().c_str());
1070 return false;
1071 }
1072 eOGRArrowGeomEncodingOut = OGRArrowGeomEncoding::WKT;
1073 return true;
1074 }
1075
1076 if (osEncoding == "WKB" || // As used in Parquet geo metadata
1077 osEncoding ==
1078 "ogc.wkb" || // As used in ARROW:extension:name field metadata
1079 osEncoding ==
1080 "geoarrow.wkb" // As used in ARROW:extension:name field metadata
1081 )
1082 {
1083 if (fieldTypeId != arrow::Type::LARGE_BINARY &&
1084 fieldTypeId != arrow::Type::BINARY)
1085 {
1086 CPLError(CE_Warning, CPLE_AppDefined,
1087 "Geometry column %s has a non Binary type: %s. "
1088 "Handling it as a regular field",
1089 fieldName.c_str(), fieldType->ToString().c_str());
1090 return false;
1091 }
1092 eOGRArrowGeomEncodingOut = OGRArrowGeomEncoding::WKB;
1093 return true;
1094 }
1095
1096 bool bHasZ = false;
1097 bool bHasM = false;
1098 if (osEncoding == "geoarrow.point" || osEncoding == "point")
1099 {
1100 if (IsPointType(fieldType, bHasZ, bHasM))
1101 {
1102 eOGRArrowGeomEncodingOut = OGRArrowGeomEncoding::GEOARROW_FSL_POINT;
1103 }
1104 else if (IsPointStructType(fieldType, bHasZ, bHasM))
1105 {
1106 eOGRArrowGeomEncodingOut =
1107 OGRArrowGeomEncoding::GEOARROW_STRUCT_POINT;
1108 }
1109 else
1110 {
1111 CPLError(CE_Warning, CPLE_AppDefined,
1112 "Geometry column %s has a type != fixed_size_list<xy: "
1113 "double>[2]> and != struct<x: double, y: double>: %s. "
1114 "Handling it as a regular field",
1115 fieldName.c_str(), fieldType->name().c_str());
1116 return false;
1117 }
1118 eGeomTypeOut = OGR_GT_SetModifier(wkbPoint, static_cast<int>(bHasZ),
1119 static_cast<int>(bHasM));
1120 return true;
1121 }
1122
1123 else if (osEncoding == "geoarrow.linestring" || osEncoding == "linestring")
1124 {
1125 if (IsListOfPointType(fieldType, 1, bHasZ, bHasM))
1126 {
1127 eOGRArrowGeomEncodingOut =
1128 OGRArrowGeomEncoding::GEOARROW_FSL_LINESTRING;
1129 }
1130 else if (IsListOfPointStructType(fieldType, 1, bHasZ, bHasM))
1131 {
1132 eOGRArrowGeomEncodingOut =
1133 OGRArrowGeomEncoding::GEOARROW_STRUCT_LINESTRING;
1134 }
1135 else
1136 {
1137 CPLError(CE_Warning, CPLE_AppDefined,
1138 "Geometry column %s has a type != fixed_size_list<xy: "
1139 "double>[2]> and != list<element: struct<x: double, y: "
1140 "double>>: %s. "
1141 "Handling it as a regular field",
1142 fieldName.c_str(), fieldType->ToString().c_str());
1143 return false;
1144 }
1145 eGeomTypeOut = OGR_GT_SetModifier(
1146 wkbLineString, static_cast<int>(bHasZ), static_cast<int>(bHasM));
1147 return true;
1148 }
1149
1150 else if (osEncoding == "geoarrow.polygon" || osEncoding == "polygon")
1151 {
1152 if (IsListOfPointType(fieldType, 2, bHasZ, bHasM))
1153 {
1154 eOGRArrowGeomEncodingOut =
1155 OGRArrowGeomEncoding::GEOARROW_FSL_POLYGON;
1156 }
1157 else if (IsListOfPointStructType(fieldType, 2, bHasZ, bHasM))
1158 {
1159 eOGRArrowGeomEncodingOut =
1160 OGRArrowGeomEncoding::GEOARROW_STRUCT_POLYGON;
1161 }
1162 else
1163 {
1164 CPLError(CE_Warning, CPLE_AppDefined,
1165 "Geometry column %s has a type != list<vertices: "
1166 "fixed_size_list<xy: double>[2]>> and != list<element: "
1167 "list<element: struct<x: double, y: double>>>: %s. "
1168 "Handling it as a regular field",
1169 fieldName.c_str(), fieldType->ToString().c_str());
1170 return false;
1171 }
1172 eGeomTypeOut = OGR_GT_SetModifier(wkbPolygon, static_cast<int>(bHasZ),
1173 static_cast<int>(bHasM));
1174 return true;
1175 }
1176
1177 else if (osEncoding == "geoarrow.multipoint" || osEncoding == "multipoint")
1178 {
1179 if (IsListOfPointType(fieldType, 1, bHasZ, bHasM))
1180 {
1181 eOGRArrowGeomEncodingOut =
1182 OGRArrowGeomEncoding::GEOARROW_FSL_MULTIPOINT;
1183 }
1184 else if (IsListOfPointStructType(fieldType, 1, bHasZ, bHasM))
1185 {
1186 eOGRArrowGeomEncodingOut =
1187 OGRArrowGeomEncoding::GEOARROW_STRUCT_MULTIPOINT;
1188 }
1189 else
1190 {
1191 CPLError(CE_Warning, CPLE_AppDefined,
1192 "Geometry column %s has a type != fixed_size_list<xy: "
1193 "double>[2]> and != list<element: struct<x: double, y: "
1194 "double>>: %s. "
1195 "Handling it as a regular field",
1196 fieldName.c_str(), fieldType->ToString().c_str());
1197 return false;
1198 }
1199 eGeomTypeOut = OGR_GT_SetModifier(
1200 wkbMultiPoint, static_cast<int>(bHasZ), static_cast<int>(bHasM));
1201 return true;
1202 }
1203
1204 else if (osEncoding == "geoarrow.multilinestring" ||
1205 osEncoding == "multilinestring")
1206 {
1207 if (IsListOfPointType(fieldType, 2, bHasZ, bHasM))
1208 {
1209 eOGRArrowGeomEncodingOut =
1210 OGRArrowGeomEncoding::GEOARROW_FSL_MULTILINESTRING;
1211 }
1212 else if (IsListOfPointStructType(fieldType, 2, bHasZ, bHasM))
1213 {
1214 eOGRArrowGeomEncodingOut =
1215 OGRArrowGeomEncoding::GEOARROW_STRUCT_MULTILINESTRING;
1216 }
1217 else
1218 {
1219 CPLError(CE_Warning, CPLE_AppDefined,
1220 "Geometry column %s has a type != list<vertices: "
1221 "fixed_size_list<xy: double>[2]>> and != list<element: "
1222 "list<element: struct<x: double, y: double>>>: %s. "
1223 "Handling it as a regular field",
1224 fieldName.c_str(), fieldType->ToString().c_str());
1225 return false;
1226 }
1227 eGeomTypeOut =
1228 OGR_GT_SetModifier(wkbMultiLineString, static_cast<int>(bHasZ),
1229 static_cast<int>(bHasM));
1230 return true;
1231 }
1232
1233 else if (osEncoding == "geoarrow.multipolygon" ||
1234 osEncoding == "multipolygon")
1235 {
1236 if (IsListOfPointType(fieldType, 3, bHasZ, bHasM))
1237 {
1238 eOGRArrowGeomEncodingOut =
1239 OGRArrowGeomEncoding::GEOARROW_FSL_MULTIPOLYGON;
1240 }
1241 else if (IsListOfPointStructType(fieldType, 3, bHasZ, bHasM))
1242 {
1243 eOGRArrowGeomEncodingOut =
1244 OGRArrowGeomEncoding::GEOARROW_STRUCT_MULTIPOLYGON;
1245 }
1246 else
1247 {
1248 CPLError(
1249 CE_Warning, CPLE_AppDefined,
1250 "Geometry column %s has a type != list<polygons: list<rings: "
1251 "list<vertices: fixed_size_list<xy: double>[2]>>> and != "
1252 "list<element: list<element: list<element: struct<x: double, "
1253 "y: double>>>>: %s. "
1254 "Handling it as a regular field",
1255 fieldName.c_str(), fieldType->ToString().c_str());
1256 return false;
1257 }
1258 eGeomTypeOut = OGR_GT_SetModifier(
1259 wkbMultiPolygon, static_cast<int>(bHasZ), static_cast<int>(bHasM));
1260 return true;
1261 }
1262
1263 if (bWarnIfUnknownEncoding)
1264 {
1265 CPLError(CE_Warning, CPLE_AppDefined,
1266 "Geometry column %s uses a unhandled encoding: %s. "
1267 "Handling it as a regular field",
1268 fieldName.c_str(), osEncoding.c_str());
1269 }
1270 return false;
1271 }
1272
1273 /************************************************************************/
1274 /* GetGeometryTypeFromString() */
1275 /************************************************************************/
1276
1277 inline OGRwkbGeometryType
1278 OGRArrowLayer::GetGeometryTypeFromString(const std::string &osType)
1279 {
1280 OGRwkbGeometryType eGeomType = wkbUnknown;
1281 OGRReadWKTGeometryType(osType.c_str(), &eGeomType);
1282 if (eGeomType == wkbUnknown && !osType.empty())
1283 {
1284 CPLDebug("ARROW", "Unknown geometry type: %s", osType.c_str());
1285 }
1286 return eGeomType;
1287 }
1288
1289 static CPLJSONObject GetObjectAsJSON(const arrow::Array *array,
1290 const size_t nIdx);
1291
1292 /************************************************************************/
1293 /* AddToArray() */
1294 /************************************************************************/
1295
1296 template <class Container>
1297 void AddToContainer(Container &oContainer, const arrow::Array *array,
1298 const size_t nIdx)
1299 {
1300 switch (array->type()->id())
1301 {
1302 case arrow::Type::BOOL:
1303 {
1304 oContainer.Add(
1305 static_cast<const arrow::BooleanArray *>(array)->Value(nIdx));
1306 break;
1307 }
1308 case arrow::Type::UINT8:
1309 {
1310 oContainer.Add(
1311 static_cast<const arrow::UInt8Array *>(array)->Value(nIdx));
1312 break;
1313 }
1314 case arrow::Type::INT8:
1315 {
1316 oContainer.Add(
1317 static_cast<const arrow::Int8Array *>(array)->Value(nIdx));
1318 break;
1319 }
1320 case arrow::Type::UINT16:
1321 {
1322 oContainer.Add(
1323 static_cast<const arrow::UInt16Array *>(array)->Value(nIdx));
1324 break;
1325 }
1326 case arrow::Type::INT16:
1327 {
1328 oContainer.Add(
1329 static_cast<const arrow::Int16Array *>(array)->Value(nIdx));
1330 break;
1331 }
1332 case arrow::Type::INT32:
1333 {
1334 oContainer.Add(
1335 static_cast<const arrow::Int32Array *>(array)->Value(nIdx));
1336 break;
1337 }
1338 case arrow::Type::UINT32:
1339 {
1340 oContainer.Add(static_cast<GInt64>(
1341 static_cast<const arrow::UInt32Array *>(array)->Value(nIdx)));
1342 break;
1343 }
1344 case arrow::Type::INT64:
1345 {
1346 oContainer.Add(static_cast<GInt64>(
1347 static_cast<const arrow::Int64Array *>(array)->Value(nIdx)));
1348 break;
1349 }
1350 case arrow::Type::UINT64:
1351 {
1352 oContainer.Add(static_cast<uint64_t>(
1353 static_cast<const arrow::UInt64Array *>(array)->Value(nIdx)));
1354 break;
1355 }
1356 case arrow::Type::HALF_FLOAT:
1357 {
1358 const uint16_t nFloat16 =
1359 static_cast<const arrow::HalfFloatArray *>(array)->Value(nIdx);
1360 uint32_t nFloat32 = CPLHalfToFloat(nFloat16);
1361 float f;
1362 memcpy(&f, &nFloat32, sizeof(nFloat32));
1363 oContainer.Add(f);
1364 break;
1365 }
1366 case arrow::Type::FLOAT:
1367 {
1368 oContainer.Add(
1369 static_cast<const arrow::FloatArray *>(array)->Value(nIdx));
1370 break;
1371 }
1372 case arrow::Type::DOUBLE:
1373 {
1374 oContainer.Add(
1375 static_cast<const arrow::DoubleArray *>(array)->Value(nIdx));
1376 break;
1377 }
1378
1379 #if ARROW_VERSION_MAJOR >= 18
1380 case arrow::Type::DECIMAL32:
1381 {
1382 oContainer.Add(
1383 CPLAtof(static_cast<const arrow::Decimal32Array *>(array)
1384 ->FormatValue(nIdx)
1385 .c_str()));
1386 break;
1387 }
1388 case arrow::Type::DECIMAL64:
1389 {
1390 oContainer.Add(
1391 CPLAtof(static_cast<const arrow::Decimal64Array *>(array)
1392 ->FormatValue(nIdx)
1393 .c_str()));
1394 break;
1395 }
1396 #endif
1397 case arrow::Type::DECIMAL128:
1398 {
1399 oContainer.Add(
1400 CPLAtof(static_cast<const arrow::Decimal128Array *>(array)
1401 ->FormatValue(nIdx)
1402 .c_str()));
1403 break;
1404 }
1405 case arrow::Type::DECIMAL256:
1406 {
1407 oContainer.Add(
1408 CPLAtof(static_cast<const arrow::Decimal256Array *>(array)
1409 ->FormatValue(nIdx)
1410 .c_str()));
1411 break;
1412 }
1413 case arrow::Type::STRING:
1414 {
1415 oContainer.Add(
1416 static_cast<const arrow::StringArray *>(array)->GetString(
1417 nIdx));
1418 break;
1419 }
1420 case arrow::Type::LARGE_STRING:
1421 {
1422 oContainer.Add(
1423 static_cast<const arrow::LargeStringArray *>(array)->GetString(
1424 nIdx));
1425 break;
1426 }
1427 #if ARROW_VERSION_MAJOR >= 15
1428 case arrow::Type::STRING_VIEW:
1429 {
1430 oContainer.Add(
1431 static_cast<const arrow::StringViewArray *>(array)->GetString(
1432 nIdx));
1433 break;
1434 }
1435 #endif
1436 case arrow::Type::LIST:
1437 case arrow::Type::LARGE_LIST:
1438 case arrow::Type::FIXED_SIZE_LIST:
1439 case arrow::Type::MAP:
1440 case arrow::Type::STRUCT:
1441 {
1442 oContainer.Add(GetObjectAsJSON(array, nIdx));
1443 break;
1444 }
1445
1446 case arrow::Type::BINARY:
1447 {
1448 const auto castArray =
1449 static_cast<const arrow::BinaryArray *>(array);
1450 int length = 0;
1451 const uint8_t *data = castArray->GetValue(nIdx, &length);
1452 bool bIsASCII = true;
1453 for (int i = 0; i < length && bIsASCII; ++i)
1454 bIsASCII = data[i] >= 32 && data[i] <= 127;
1455 if (bIsASCII)
1456 {
1457 oContainer.Add(
1458 std::string(reinterpret_cast<const char *>(data), length));
1459 }
1460 else
1461 {
1462 char *pszBase64 = CPLBase64Encode(length, data);
1463 oContainer.Add(std::string("base64:").append(pszBase64));
1464 CPLFree(pszBase64);
1465 }
1466 break;
1467 }
1468
1469 #if ARROW_VERSION_MAJOR >= 15
1470 case arrow::Type::BINARY_VIEW:
1471 {
1472 const auto castArray =
1473 static_cast<const arrow::BinaryViewArray *>(array);
1474 const auto view = castArray->GetView(nIdx);
1475 if (view.size() <= INT_MAX)
1476 {
1477 const int length = static_cast<int>(view.size());
1478 const char *data = view.data();
1479 bool bIsASCII = true;
1480 for (int i = 0; i < length && bIsASCII; ++i)
1481 bIsASCII =
1482 data[i] >= 32 && static_cast<unsigned>(data[i]) <= 127;
1483 if (bIsASCII)
1484 {
1485 oContainer.Add(std::string(view));
1486 }
1487 else
1488 {
1489 char *pszBase64 = CPLBase64Encode(
1490 length, reinterpret_cast<const GByte *>(data));
1491 oContainer.Add(std::string("base64:").append(pszBase64));
1492 CPLFree(pszBase64);
1493 }
1494 }
1495 else
1496 {
1497 CPLError(CE_Failure, CPLE_AppDefined, "Too large binary view");
1498 }
1499 break;
1500 }
1501 #endif
1502
1503 default:
1504 {
1505 CPLDebug("ARROW", "AddToContainer(): unexpected data type %s",
1506 array->type()->ToString().c_str());
1507 break;
1508 }
1509 }
1510 }
1511
1512 /************************************************************************/
1513 /* AddToArray() */
1514 /************************************************************************/
1515
1516 static void AddToArray(CPLJSONArray &oArray, const arrow::Array *array,
1517 const size_t nIdx)
1518 {
1519 AddToContainer(oArray, array, nIdx);
1520 }
1521
1522 /************************************************************************/
1523 /* GetListAsJSON() */
1524 /************************************************************************/
1525
1526 template <class ArrowType>
1527 static CPLJSONArray GetListAsJSON(const ArrowType *array,
1528 const size_t nIdxInArray)
1529 {
1530 const auto values = array->values();
1531 const auto nIdxStart = array->value_offset(nIdxInArray);
1532 const auto nCount = array->value_length(nIdxInArray);
1533 CPLJSONArray oArray;
1534 for (auto k = decltype(nCount){0}; k < nCount; k++)
1535 {
1536 if (values->IsNull(nIdxStart + k))
1537 oArray.AddNull();
1538 else
1539 AddToArray(oArray, values.get(),
1540 static_cast<size_t>(nIdxStart + k));
1541 }
1542 return oArray;
1543 }
1544
1545 /************************************************************************/
1546 /* AddToDict() */
1547 /************************************************************************/
1548
1549 namespace
1550 {
1551 struct ContainerAdapter
1552 {
1553 CPLJSONObject &m_oDict;
1554 const std::string m_osKey;
1555
1556 ContainerAdapter(CPLJSONObject &oDictIn, const std::string &osKeyIn)
1557 : m_oDict(oDictIn), m_osKey(osKeyIn)
1558 {
1559 }
1560
1561 template <class T> void Add(const T &v)
1562 {
1563 m_oDict.Add(m_osKey, v);
1564 }
1565 };
1566 } // namespace
1567
1568 static void AddToDict(CPLJSONObject &oDict, const std::string &osKey,
1569 const arrow::Array *array, const size_t nIdx)
1570 {
1571 ContainerAdapter dictAdapter(oDict, osKey);
1572 AddToContainer(dictAdapter, array, nIdx);
1573 }
1574
1575 /************************************************************************/
1576 /* GetMapAsJSON() */
1577 /************************************************************************/
1578
1579 template <class KeyArrayType>
1580 static CPLJSONObject GetMapAsJSON(const arrow::Array *array,
1581 const size_t nIdxInArray)
1582 {
1583 const auto mapArray = static_cast<const arrow::MapArray *>(array);
1584 const auto keys = std::static_pointer_cast<KeyArrayType>(mapArray->keys());
1585 const auto values = mapArray->items();
1586 const auto nIdxStart = mapArray->value_offset(nIdxInArray);
1587 const int nCount = mapArray->value_length(nIdxInArray);
1588 CPLJSONObject oRoot;
1589 for (int k = 0; k < nCount; k++)
1590 {
1591 if (!keys->IsNull(nIdxStart + k))
1592 {
1593 const auto osKey = keys->GetString(nIdxStart + k);
1594 if (!values->IsNull(nIdxStart + k))
1595 AddToDict(oRoot, osKey, values.get(), nIdxStart + k);
1596 else
1597 oRoot.AddNull(osKey);
1598 }
1599 }
1600 return oRoot;
1601 }
1602
1603 static CPLJSONObject GetMapAsJSON(const arrow::Array *array,
1604 const size_t nIdxInArray)
1605 {
1606 const auto mapArray = static_cast<const arrow::MapArray *>(array);
1607 const auto eKeyType = mapArray->keys()->type()->id();
1608 if (eKeyType == arrow::Type::STRING)
1609 return GetMapAsJSON<arrow::StringArray>(array, nIdxInArray);
1610 #if ARROW_VERSION_MAJOR >= 15
1611 else if (eKeyType == arrow::Type::STRING_VIEW)
1612 return GetMapAsJSON<arrow::StringViewArray>(array, nIdxInArray);
1613 #endif
1614 else
1615 {
1616 CPLAssert(false);
1617 return CPLJSONObject();
1618 }
1619 }
1620
1621 /************************************************************************/
1622 /* GetStructureAsJSON() */
1623 /************************************************************************/
1624
1625 static CPLJSONObject GetStructureAsJSON(const arrow::Array *array,
1626 const size_t nIdxInArray)
1627 {
1628 CPLJSONObject oRoot;
1629 const auto structArray = static_cast<const arrow::StructArray *>(array);
1630 const auto structArrayType = structArray->type();
1631 for (int i = 0; i < structArrayType->num_fields(); ++i)
1632 {
1633 const auto field = structArray->field(i);
1634 if (!field->IsNull(nIdxInArray))
1635 {
1636 AddToDict(oRoot, structArrayType->field(i)->name(), field.get(),
1637 nIdxInArray);
1638 }
1639 else
1640 oRoot.AddNull(structArrayType->field(i)->name());
1641 }
1642
1643 return oRoot;
1644 }
1645
1646 /************************************************************************/
1647 /* GetObjectAsJSON() */
1648 /************************************************************************/
1649
1650 static CPLJSONObject GetObjectAsJSON(const arrow::Array *array,
1651 const size_t nIdxInArray)
1652 {
1653 switch (array->type()->id())
1654 {
1655 case arrow::Type::MAP:
1656 return GetMapAsJSON(array, nIdxInArray);
1657 case arrow::Type::LIST:
1658 return GetListAsJSON(static_cast<const arrow::ListArray *>(array),
1659 nIdxInArray);
1660 case arrow::Type::LARGE_LIST:
1661 return GetListAsJSON(
1662 static_cast<const arrow::LargeListArray *>(array), nIdxInArray);
1663 case arrow::Type::FIXED_SIZE_LIST:
1664 return GetListAsJSON(
1665 static_cast<const arrow::FixedSizeListArray *>(array),
1666 nIdxInArray);
1667 case arrow::Type::STRUCT:
1668 return GetStructureAsJSON(array, nIdxInArray);
1669 default:
1670 {
1671 CPLError(CE_Failure, CPLE_AppDefined,
1672 "GetObjectAsJSON(): unhandled value format: %s",
1673 array->type()->ToString().c_str());
1674 return CPLJSONObject();
1675 }
1676 }
1677 }
1678
1679 template <class ArrowType>
1680 static int GetListLength(const ArrowType *array, int64_t nIdxInArray)
1681 {
1682 const auto nCount = array->value_length(nIdxInArray);
1683 if constexpr (!std::is_same_v<decltype(nCount), int>)
1684 {
1685 // Should be >, but >= makes coverity scan happy
1686 if (nCount >= INT_MAX)
1687 {
1688 CPLError(CE_Failure, CPLE_NotSupported,
1689 "More than %d values in list. Clamping to it", INT_MAX);
1690 return INT_MAX;
1691 }
1692 }
1693 return static_cast<int>(nCount);
1694 }
1695
1696 template <class OGRType, class ArrowType, class ArrayType>
1697 static void ReadList(OGRFeature *poFeature, int i, int64_t nIdxInArray,
1698 const ArrayType *array)
1699 {
1700 const auto values = std::static_pointer_cast<ArrowType>(array->values());
1701 const auto nIdxStart = array->value_offset(nIdxInArray);
1702 const int nCount = GetListLength(array, nIdxInArray);
1703 std::vector<OGRType> aValues;
1704 aValues.reserve(nCount);
1705 for (int k = 0; k < nCount; k++)
1706 {
1707 aValues.push_back(static_cast<OGRType>(values->Value(nIdxStart + k)));
1708 }
1709 poFeature->SetField(i, nCount, aValues.data());
1710 }
1711
1712 template <class ArrowType, class ArrayType>
1713 static void ReadListDouble(OGRFeature *poFeature, int i, int64_t nIdxInArray,
1714 const ArrayType *array)
1715 {
1716 const auto values = std::static_pointer_cast<ArrowType>(array->values());
1717 const auto rawValues = values->raw_values();
1718 const auto nIdxStart = array->value_offset(nIdxInArray);
1719 const int nCount = GetListLength(array, nIdxInArray);
1720 std::vector<double> aValues;
1721 aValues.reserve(nCount);
1722 for (int k = 0; k < nCount; k++)
1723 {
1724 if (values->IsNull(nIdxStart + k))
1725 aValues.push_back(std::numeric_limits<double>::quiet_NaN());
1726 else
1727 aValues.push_back(rawValues[nIdxStart + k]);
1728 }
1729 poFeature->SetField(i, nCount, aValues.data());
1730 }
1731
1732 template <class ArrayType>
1733 static void ReadList(OGRFeature *poFeature, int i, int64_t nIdxInArray,
1734 const ArrayType *array, arrow::Type::type valueTypeId)
1735 {
1736 switch (valueTypeId)
1737 {
1738 case arrow::Type::BOOL:
1739 {
1740 ReadList<int, arrow::BooleanArray>(poFeature, i, nIdxInArray,
1741 array);
1742 break;
1743 }
1744 case arrow::Type::UINT8:
1745 {
1746 ReadList<int, arrow::UInt8Array>(poFeature, i, nIdxInArray, array);
1747 break;
1748 }
1749 case arrow::Type::INT8:
1750 {
1751 ReadList<int, arrow::Int8Array>(poFeature, i, nIdxInArray, array);
1752 break;
1753 }
1754 case arrow::Type::UINT16:
1755 {
1756 ReadList<int, arrow::UInt16Array>(poFeature, i, nIdxInArray, array);
1757 break;
1758 }
1759 case arrow::Type::INT16:
1760 {
1761 ReadList<int, arrow::Int16Array>(poFeature, i, nIdxInArray, array);
1762 break;
1763 }
1764 case arrow::Type::INT32:
1765 {
1766 ReadList<int, arrow::Int32Array>(poFeature, i, nIdxInArray, array);
1767 break;
1768 }
1769 case arrow::Type::UINT32:
1770 {
1771 ReadList<GIntBig, arrow::UInt32Array>(poFeature, i, nIdxInArray,
1772 array);
1773 break;
1774 }
1775 case arrow::Type::INT64:
1776 {
1777 ReadList<GIntBig, arrow::Int64Array>(poFeature, i, nIdxInArray,
1778 array);
1779 break;
1780 }
1781 case arrow::Type::UINT64:
1782 {
1783 ReadList<double, arrow::UInt64Array>(poFeature, i, nIdxInArray,
1784 array);
1785 break;
1786 }
1787 case arrow::Type::HALF_FLOAT:
1788 {
1789 const auto values = std::static_pointer_cast<arrow::HalfFloatArray>(
1790 array->values());
1791 const auto nIdxStart = array->value_offset(nIdxInArray);
1792 const int nCount = GetListLength(array, nIdxInArray);
1793 std::vector<double> aValues;
1794 aValues.reserve(nCount);
1795 for (int k = 0; k < nCount; k++)
1796 {
1797 if (values->IsNull(nIdxStart + k))
1798 aValues.push_back(std::numeric_limits<double>::quiet_NaN());
1799 else
1800 {
1801 const uint16_t nFloat16 = values->Value(nIdxStart + k);
1802 uint32_t nFloat32 = CPLHalfToFloat(nFloat16);
1803 float f;
1804 memcpy(&f, &nFloat32, sizeof(nFloat32));
1805 aValues.push_back(f);
1806 }
1807 }
1808 poFeature->SetField(i, nCount, aValues.data());
1809 break;
1810 }
1811 case arrow::Type::FLOAT:
1812 {
1813 ReadListDouble<arrow::FloatArray>(poFeature, i, nIdxInArray, array);
1814 break;
1815 }
1816 case arrow::Type::DOUBLE:
1817 {
1818 ReadListDouble<arrow::DoubleArray>(poFeature, i, nIdxInArray,
1819 array);
1820 break;
1821 }
1822
1823 #if ARROW_VERSION_MAJOR >= 18
1824 case arrow::Type::DECIMAL32:
1825 {
1826 const auto values = std::static_pointer_cast<arrow::Decimal32Array>(
1827 array->values());
1828 const auto nIdxStart = array->value_offset(nIdxInArray);
1829 const int nCount = GetListLength(array, nIdxInArray);
1830 std::vector<double> aValues;
1831 aValues.reserve(nCount);
1832 for (int k = 0; k < nCount; k++)
1833 {
1834 if (values->IsNull(nIdxStart + k))
1835 aValues.push_back(std::numeric_limits<double>::quiet_NaN());
1836 else
1837 aValues.push_back(
1838 CPLAtof(values->FormatValue(nIdxStart + k).c_str()));
1839 }
1840 poFeature->SetField(i, nCount, aValues.data());
1841 break;
1842 }
1843
1844 case arrow::Type::DECIMAL64:
1845 {
1846 const auto values = std::static_pointer_cast<arrow::Decimal64Array>(
1847 array->values());
1848 const auto nIdxStart = array->value_offset(nIdxInArray);
1849 const int nCount = GetListLength(array, nIdxInArray);
1850 std::vector<double> aValues;
1851 aValues.reserve(nCount);
1852 for (int k = 0; k < nCount; k++)
1853 {
1854 if (values->IsNull(nIdxStart + k))
1855 aValues.push_back(std::numeric_limits<double>::quiet_NaN());
1856 else
1857 aValues.push_back(
1858 CPLAtof(values->FormatValue(nIdxStart + k).c_str()));
1859 }
1860 poFeature->SetField(i, nCount, aValues.data());
1861 break;
1862 }
1863 #endif
1864
1865 case arrow::Type::DECIMAL128:
1866 {
1867 const auto values =
1868 std::static_pointer_cast<arrow::Decimal128Array>(
1869 array->values());
1870 const auto nIdxStart = array->value_offset(nIdxInArray);
1871 const int nCount = GetListLength(array, nIdxInArray);
1872 std::vector<double> aValues;
1873 aValues.reserve(nCount);
1874 for (int k = 0; k < nCount; k++)
1875 {
1876 if (values->IsNull(nIdxStart + k))
1877 aValues.push_back(std::numeric_limits<double>::quiet_NaN());
1878 else
1879 aValues.push_back(
1880 CPLAtof(values->FormatValue(nIdxStart + k).c_str()));
1881 }
1882 poFeature->SetField(i, nCount, aValues.data());
1883 break;
1884 }
1885
1886 case arrow::Type::DECIMAL256:
1887 {
1888 const auto values =
1889 std::static_pointer_cast<arrow::Decimal256Array>(
1890 array->values());
1891 const auto nIdxStart = array->value_offset(nIdxInArray);
1892 const int nCount = GetListLength(array, nIdxInArray);
1893 std::vector<double> aValues;
1894 aValues.reserve(nCount);
1895 for (int k = 0; k < nCount; k++)
1896 {
1897 if (values->IsNull(nIdxStart + k))
1898 aValues.push_back(std::numeric_limits<double>::quiet_NaN());
1899 else
1900 aValues.push_back(
1901 CPLAtof(values->FormatValue(nIdxStart + k).c_str()));
1902 }
1903 poFeature->SetField(i, nCount, aValues.data());
1904 break;
1905 }
1906
1907 case arrow::Type::STRING:
1908 {
1909 const auto values =
1910 std::static_pointer_cast<arrow::StringArray>(array->values());
1911 const auto nIdxStart = array->value_offset(nIdxInArray);
1912 const int nCount = GetListLength(array, nIdxInArray);
1913 CPLStringList aosList;
1914 for (int k = 0; k < nCount; k++)
1915 {
1916 if (values->IsNull(nIdxStart + k))
1917 aosList.AddString(
1918 ""); // we cannot have null strings in a list
1919 else
1920 aosList.AddString(values->GetString(nIdxStart + k).c_str());
1921 }
1922 poFeature->SetField(i, aosList.List());
1923 break;
1924 }
1925
1926 case arrow::Type::LARGE_STRING:
1927 {
1928 const auto values =
1929 std::static_pointer_cast<arrow::LargeStringArray>(
1930 array->values());
1931 const auto nIdxStart = array->value_offset(nIdxInArray);
1932 const auto nCount = GetListLength(array, nIdxInArray);
1933 CPLStringList aosList;
1934 for (auto k = decltype(nCount){0}; k < nCount; k++)
1935 {
1936 if (values->IsNull(nIdxStart + k))
1937 aosList.AddString(
1938 ""); // we cannot have null strings in a list
1939 else
1940 aosList.AddString(values->GetString(nIdxStart + k).c_str());
1941 }
1942 poFeature->SetField(i, aosList.List());
1943 break;
1944 }
1945
1946 #if ARROW_VERSION_MAJOR >= 15
1947 case arrow::Type::STRING_VIEW:
1948 {
1949 const auto values =
1950 std::static_pointer_cast<arrow::StringViewArray>(
1951 array->values());
1952 const auto nIdxStart = array->value_offset(nIdxInArray);
1953 const int nCount = GetListLength(array, nIdxInArray);
1954 CPLStringList aosList;
1955 for (int k = 0; k < nCount; k++)
1956 {
1957 if (values->IsNull(nIdxStart + k))
1958 aosList.AddString(
1959 ""); // we cannot have null strings in a list
1960 else
1961 aosList.AddString(values->GetString(nIdxStart + k).c_str());
1962 }
1963 poFeature->SetField(i, aosList.List());
1964 break;
1965 }
1966 #endif
1967
1968 case arrow::Type::BINARY:
1969 {
1970 const auto values =
1971 std::static_pointer_cast<arrow::BinaryArray>(array->values());
1972 const auto nIdxStart = array->value_offset(nIdxInArray);
1973 const auto nCount = GetListLength(array, nIdxInArray);
1974 CPLStringList aosList;
1975 for (auto k = decltype(nCount){0}; k < nCount; k++)
1976 {
1977 if (values->IsNull(nIdxStart + k))
1978 {
1979 aosList.AddString(
1980 ""); // we cannot have null strings in a list
1981 }
1982 else
1983 {
1984 int length = 0;
1985 const uint8_t *data =
1986 values->GetValue(nIdxStart + k, &length);
1987 bool bIsASCII = true;
1988 for (int j = 0; j < length && bIsASCII; ++j)
1989 bIsASCII = data[j] >= 32 && data[j] <= 127;
1990 if (bIsASCII)
1991 {
1992 aosList.AddString(
1993 std::string(reinterpret_cast<const char *>(data),
1994 length)
1995 .c_str());
1996 }
1997 else
1998 {
1999 char *pszBase64 = CPLBase64Encode(length, data);
2000 aosList.AddString(
2001 std::string("base64:").append(pszBase64).c_str());
2002 CPLFree(pszBase64);
2003 }
2004 }
2005 }
2006 poFeature->SetField(i, aosList.List());
2007 break;
2008 }
2009
2010 case arrow::Type::LIST:
2011 case arrow::Type::LARGE_LIST:
2012 case arrow::Type::FIXED_SIZE_LIST:
2013 case arrow::Type::MAP:
2014 case arrow::Type::STRUCT:
2015 {
2016 poFeature->SetField(
2017 i, GetListAsJSON(array, static_cast<size_t>(nIdxInArray))
2018 .Format(CPLJSONObject::PrettyFormat::Plain)
2019 .c_str());
2020 break;
2021 }
2022
2023 default:
2024 {
2025 CPLDebug("ARROW", "ReadList(): unexpected data type %s",
2026 array->values()->type()->ToString().c_str());
2027 break;
2028 }
2029 }
2030 }
2031
2032 /************************************************************************/
2033 /* SetPointsOfLine() */
2034 /************************************************************************/
2035
2036 template <bool bHasZ, bool bHasM, int nDim>
2037 void SetPointsOfLine(OGRLineString *poLS, const arrow::DoubleArray *pointValues,
2038 size_t pointOffset, int numPoints)
2039 {
2040 if (!bHasZ && !bHasM)
2041 {
2042 static_assert(sizeof(OGRRawPoint) == 2 * sizeof(double),
2043 "sizeof(OGRRawPoint) == 2 * sizeof(double)");
2044 poLS->setPoints(numPoints,
2045 reinterpret_cast<const OGRRawPoint *>(
2046 pointValues->raw_values() + pointOffset));
2047 return;
2048 }
2049
2050 poLS->setNumPoints(numPoints, FALSE);
2051 for (int k = 0; k < numPoints; k++)
2052 {
2053 if constexpr (bHasZ)
2054 {
2055 if constexpr (bHasM)
2056 {
2057 poLS->setPoint(k, pointValues->Value(pointOffset + nDim * k),
2058 pointValues->Value(pointOffset + nDim * k + 1),
2059 pointValues->Value(pointOffset + nDim * k + 2),
2060 pointValues->Value(pointOffset + nDim * k + 3));
2061 }
2062 else
2063 {
2064 poLS->setPoint(k, pointValues->Value(pointOffset + nDim * k),
2065 pointValues->Value(pointOffset + nDim * k + 1),
2066 pointValues->Value(pointOffset + nDim * k + 2));
2067 }
2068 }
2069 else /* if( bHasM ) */
2070 {
2071 poLS->setPointM(k, pointValues->Value(pointOffset + nDim * k),
2072 pointValues->Value(pointOffset + nDim * k + 1),
2073 pointValues->Value(pointOffset + nDim * k + 2));
2074 }
2075 }
2076 }
2077
2078 typedef void (*SetPointsOfLineType)(OGRLineString *, const arrow::DoubleArray *,
2079 size_t, int);
2080
2081 static SetPointsOfLineType GetSetPointsOfLine(bool bHasZ, bool bHasM)
2082 {
2083 if (bHasZ && bHasM)
2084 return SetPointsOfLine<true, true, 4>;
2085 if (bHasZ)
2086 return SetPointsOfLine<true, false, 3>;
2087 if (bHasM)
2088 return SetPointsOfLine<false, true, 3>;
2089 return SetPointsOfLine<false, false, 2>;
2090 }
2091
2092 /************************************************************************/
2093 /* SetPointsOfLineStruct() */
2094 /************************************************************************/
2095
2096 template <bool bHasZ, bool bHasM, int nDim>
2097 void SetPointsOfLineStruct(OGRLineString *poLS,
2098 const arrow::StructArray *structArray,
2099 size_t pointOffset, int numPoints)
2100 {
2101 CPLAssert(structArray->num_fields() == nDim);
2102 const auto &fields = structArray->fields();
2103 const auto &fieldX = fields[0];
2104 CPLAssert(fieldX->type_id() == arrow::Type::DOUBLE);
2105 const auto fieldXDouble = static_cast<arrow::DoubleArray *>(fieldX.get());
2106 const auto &fieldY = fields[1];
2107 CPLAssert(fieldY->type_id() == arrow::Type::DOUBLE);
2108 const auto fieldYDouble = static_cast<arrow::DoubleArray *>(fieldY.get());
2109 const arrow::DoubleArray *fieldZDouble = nullptr;
2110 const arrow::DoubleArray *fieldMDouble = nullptr;
2111 int iField = 2;
2112 if constexpr (bHasZ)
2113 {
2114 const auto &field = fields[iField];
2115 ++iField;
2116 CPLAssert(field->type_id() == arrow::Type::DOUBLE);
2117 fieldZDouble = static_cast<arrow::DoubleArray *>(field.get());
2118 }
2119 if constexpr (bHasM)
2120 {
2121 const auto &field = fields[iField];
2122 CPLAssert(field->type_id() == arrow::Type::DOUBLE);
2123 fieldMDouble = static_cast<arrow::DoubleArray *>(field.get());
2124 }
2125
2126 poLS->setNumPoints(numPoints, FALSE);
2127 for (int k = 0; k < numPoints; k++)
2128 {
2129 if constexpr (bHasZ)
2130 {
2131 if constexpr (bHasM)
2132 {
2133 poLS->setPoint(k, fieldXDouble->Value(pointOffset + k),
2134 fieldYDouble->Value(pointOffset + k),
2135 fieldZDouble->Value(pointOffset + k),
2136 fieldMDouble->Value(pointOffset + k));
2137 }
2138 else
2139 {
2140 poLS->setPoint(k, fieldXDouble->Value(pointOffset + k),
2141 fieldYDouble->Value(pointOffset + k),
2142 fieldZDouble->Value(pointOffset + k));
2143 }
2144 }
2145 else if constexpr (bHasM)
2146 {
2147 poLS->setPointM(k, fieldXDouble->Value(pointOffset + k),
2148 fieldYDouble->Value(pointOffset + k),
2149 fieldMDouble->Value(pointOffset + k));
2150 }
2151 else
2152 {
2153 poLS->setPoint(k, fieldXDouble->Value(pointOffset + k),
2154 fieldYDouble->Value(pointOffset + k));
2155 }
2156 }
2157 }
2158
2159 typedef void (*SetPointsOfLineStructType)(OGRLineString *,
2160 const arrow::StructArray *, size_t,
2161 int);
2162
2163 static SetPointsOfLineStructType GetSetPointsOfLineStruct(bool bHasZ,
2164 bool bHasM)
2165 {
2166 if (bHasZ && bHasM)
2167 return SetPointsOfLineStruct<true, true, 4>;
2168 if (bHasZ)
2169 return SetPointsOfLineStruct<true, false, 3>;
2170 if (bHasM)
2171 return SetPointsOfLineStruct<false, true, 3>;
2172 return SetPointsOfLineStruct<false, false, 2>;
2173 }
2174
2175 /************************************************************************/
2176 /* TimestampToOGR() */
2177 /************************************************************************/
2178
2179 inline void
2180 OGRArrowLayer::TimestampToOGR(int64_t timestamp,
2181 const arrow::TimestampType *timestampType,
2182 int nTZFlag, OGRField *psField)
2183 {
2184 const auto unit = timestampType->unit();
2185 double floatingPart = 0;
2186 if (unit == arrow::TimeUnit::MILLI)
2187 {
2188 floatingPart = (timestamp % 1000) / 1e3;
2189 timestamp /= 1000;
2190 }
2191 else if (unit == arrow::TimeUnit::MICRO)
2192 {
2193 floatingPart = (timestamp % (1000 * 1000)) / 1e6;
2194 timestamp /= 1000 * 1000;
2195 }
2196 else if (unit == arrow::TimeUnit::NANO)
2197 {
2198 floatingPart = (timestamp % (1000 * 1000 * 1000)) / 1e9;
2199 timestamp /= 1000 * 1000 * 1000;
2200 }
2201 if (nTZFlag > OGR_TZFLAG_MIXED_TZ)
2202 {
2203 const int TZOffset = (nTZFlag - OGR_TZFLAG_UTC) * 15;
2204 timestamp += TZOffset * 60;
2205 }
2206 struct tm dt;
2207 CPLUnixTimeToYMDHMS(timestamp, &dt);
2208 psField->Date.Year = static_cast<GInt16>(dt.tm_year + 1900);
2209 psField->Date.Month = static_cast<GByte>(dt.tm_mon + 1);
2210 psField->Date.Day = static_cast<GByte>(dt.tm_mday);
2211 psField->Date.Hour = static_cast<GByte>(dt.tm_hour);
2212 psField->Date.Minute = static_cast<GByte>(dt.tm_min);
2213 psField->Date.TZFlag = static_cast<GByte>(nTZFlag);
2214 psField->Date.Second = static_cast<float>(dt.tm_sec + floatingPart);
2215 }
2216
2217 /************************************************************************/
2218 /* ReadFeature() */
2219 /************************************************************************/
2220
2221 inline OGRFeature *OGRArrowLayer::ReadFeature(
2222 int64_t nIdxInBatch,
2223 const std::vector<std::shared_ptr<arrow::Array>> &poColumnArrays) const
2224 {
2225 OGRFeature *poFeature = new OGRFeature(m_poFeatureDefn);
2226
2227 if (m_iFIDArrowColumn >= 0)
2228 {
2229 const int iCol =
2230 m_bIgnoredFields ? m_nRequestedFIDColumn : m_iFIDArrowColumn;
2231 const arrow::Array *array = poColumnArrays[iCol].get();
2232 if (!array->IsNull(nIdxInBatch))
2233 {
2234 if (array->type_id() == arrow::Type::INT64)
2235 {
2236 const auto castArray =
2237 static_cast<const arrow::Int64Array *>(array);
2238 poFeature->SetFID(
2239 static_cast<GIntBig>(castArray->Value(nIdxInBatch)));
2240 }
2241 else if (array->type_id() == arrow::Type::INT32)
2242 {
2243 const auto castArray =
2244 static_cast<const arrow::Int32Array *>(array);
2245 poFeature->SetFID(castArray->Value(nIdxInBatch));
2246 }
2247 }
2248 }
2249
2250 const auto ReadList =
2251 [this, poFeature, nIdxInBatch](const auto *array, int iField)
2252 {
2253 const auto listType =
2254 static_cast<const arrow::ListType *>(array->data()->type.get());
2255
2256 if (m_bListsAsStringJson)
2257 {
2258 poFeature->SetField(
2259 iField, GetListAsJSON(array, static_cast<size_t>(nIdxInBatch))
2260 .Format(CPLJSONObject::PrettyFormat::Plain)
2261 .c_str());
2262 }
2263 else
2264 {
2265 ::ReadList(poFeature, iField, nIdxInBatch, array,
2266 listType->value_field()->type()->id());
2267 }
2268 };
2269
2270 const int nFieldCount = m_poFeatureDefn->GetFieldCount();
2271 for (int i = 0; i < nFieldCount; ++i)
2272 {
2273 int iCol;
2274 if (m_bIgnoredFields)
2275 {
2276 iCol = m_anMapFieldIndexToArrayIndex[i];
2277 if (iCol < 0)
2278 continue;
2279 }
2280 else
2281 {
2282 iCol = m_anMapFieldIndexToArrowColumn[i][0];
2283 }
2284
2285 const arrow::Array *array = GetStorageArray(poColumnArrays[iCol].get());
2286 if (array->IsNull(nIdxInBatch))
2287 {
2288 poFeature->SetFieldNull(i);
2289 continue;
2290 }
2291
2292 int j = 1;
2293 bool bSkipToNextField = false;
2294 if (array->type_id() == arrow::Type::STRUCT &&
2295 m_poFeatureDefn->GetFieldDefn(i)->GetType() == OFTDateTime &&
2296 m_poFeatureDefn->GetFieldDefn(i)->GetTZFlag() ==
2297 OGR_TZFLAG_MIXED_TZ)
2298 {
2299 const auto structArray =
2300 static_cast<const arrow::StructArray *>(array);
2301 const auto &subArrays = structArray->fields();
2302 CPLAssert(subArrays.size() == 2);
2303 const auto timestampType = static_cast<arrow::TimestampType *>(
2304 subArrays[0]->data()->type.get());
2305 const auto timestampArray =
2306 static_cast<const arrow::TimestampArray *>(subArrays[0].get());
2307 const int64_t timestamp = timestampArray->Value(nIdxInBatch);
2308 const auto offsetMinutesArray =
2309 static_cast<const arrow::Int16Array *>(subArrays[1].get());
2310 const int nOffsetMinutes = offsetMinutesArray->Value(nIdxInBatch);
2311 const int MAX_TIME_ZONE_HOUR = 14;
2312 const int nTZFlag =
2313 nOffsetMinutes >= -MAX_TIME_ZONE_HOUR * 60 &&
2314 nOffsetMinutes <= MAX_TIME_ZONE_HOUR * 60
2315 ? OGR_TZFLAG_UTC + nOffsetMinutes / 15
2316 : OGR_TZFLAG_UTC;
2317 OGRField sField;
2318 sField.Set.nMarker1 = OGRUnsetMarker;
2319 sField.Set.nMarker2 = OGRUnsetMarker;
2320 sField.Set.nMarker3 = OGRUnsetMarker;
2321 TimestampToOGR(timestamp, timestampType, nTZFlag, &sField);
2322 poFeature->SetField(i, &sField);
2323 continue;
2324 }
2325 else
2326 {
2327 while (array->type_id() == arrow::Type::STRUCT)
2328 {
2329 const auto castArray =
2330 static_cast<const arrow::StructArray *>(array);
2331 const auto &subArrays = castArray->fields();
2332 CPLAssert(j < static_cast<int>(
2333 m_anMapFieldIndexToArrowColumn[i].size()));
2334 const int iArrowSubcol = m_anMapFieldIndexToArrowColumn[i][j];
2335 j++;
2336 CPLAssert(iArrowSubcol < static_cast<int>(subArrays.size()));
2337 array = GetStorageArray(subArrays[iArrowSubcol].get());
2338 if (array->IsNull(nIdxInBatch))
2339 {
2340 poFeature->SetFieldNull(i);
2341 bSkipToNextField = true;
2342 break;
2343 }
2344 }
2345 }
2346 if (bSkipToNextField)
2347 continue;
2348
2349 if (array->type_id() == arrow::Type::DICTIONARY)
2350 {
2351 const auto castArray =
2352 static_cast<const arrow::DictionaryArray *>(array);
2353 m_poReadFeatureTmpArray =
2354 castArray->indices(); // does not return a const reference
2355 array = GetStorageArray(m_poReadFeatureTmpArray.get());
2356 if (array->IsNull(nIdxInBatch))
2357 {
2358 poFeature->SetFieldNull(i);
2359 continue;
2360 }
2361 }
2362
2363 switch (array->type_id())
2364 {
2365 case arrow::Type::NA:
2366 break;
2367
2368 case arrow::Type::BOOL:
2369 {
2370 const auto castArray =
2371 static_cast<const arrow::BooleanArray *>(array);
2372 poFeature->SetFieldSameTypeUnsafe(
2373 i, castArray->Value(nIdxInBatch));
2374 break;
2375 }
2376 case arrow::Type::UINT8:
2377 {
2378 const auto castArray =
2379 static_cast<const arrow::UInt8Array *>(array);
2380 poFeature->SetFieldSameTypeUnsafe(
2381 i, castArray->Value(nIdxInBatch));
2382 break;
2383 }
2384 case arrow::Type::INT8:
2385 {
2386 const auto castArray =
2387 static_cast<const arrow::Int8Array *>(array);
2388 poFeature->SetFieldSameTypeUnsafe(
2389 i, castArray->Value(nIdxInBatch));
2390 break;
2391 }
2392 case arrow::Type::UINT16:
2393 {
2394 const auto castArray =
2395 static_cast<const arrow::UInt16Array *>(array);
2396 poFeature->SetFieldSameTypeUnsafe(
2397 i, castArray->Value(nIdxInBatch));
2398 break;
2399 }
2400 case arrow::Type::INT16:
2401 {
2402 const auto castArray =
2403 static_cast<const arrow::Int16Array *>(array);
2404 poFeature->SetFieldSameTypeUnsafe(
2405 i, castArray->Value(nIdxInBatch));
2406 break;
2407 }
2408 case arrow::Type::UINT32:
2409 {
2410 const auto castArray =
2411 static_cast<const arrow::UInt32Array *>(array);
2412 poFeature->SetFieldSameTypeUnsafe(
2413 i, static_cast<GIntBig>(castArray->Value(nIdxInBatch)));
2414 break;
2415 }
2416 case arrow::Type::INT32:
2417 {
2418 const auto castArray =
2419 static_cast<const arrow::Int32Array *>(array);
2420 poFeature->SetFieldSameTypeUnsafe(
2421 i, castArray->Value(nIdxInBatch));
2422 break;
2423 }
2424 case arrow::Type::UINT64:
2425 {
2426 const auto castArray =
2427 static_cast<const arrow::UInt64Array *>(array);
2428 poFeature->SetFieldSameTypeUnsafe(
2429 i, static_cast<double>(castArray->Value(nIdxInBatch)));
2430 break;
2431 }
2432 case arrow::Type::INT64:
2433 {
2434 const auto castArray =
2435 static_cast<const arrow::Int64Array *>(array);
2436 poFeature->SetFieldSameTypeUnsafe(
2437 i, static_cast<GIntBig>(castArray->Value(nIdxInBatch)));
2438 break;
2439 }
2440 case arrow::Type::HALF_FLOAT:
2441 {
2442 const auto castArray =
2443 static_cast<const arrow::HalfFloatArray *>(array);
2444 const uint16_t nFloat16 = castArray->Value(nIdxInBatch);
2445 uint32_t nFloat32 = CPLHalfToFloat(nFloat16);
2446 float f;
2447 memcpy(&f, &nFloat32, sizeof(nFloat32));
2448 poFeature->SetFieldSameTypeUnsafe(i, f);
2449 break;
2450 }
2451 case arrow::Type::FLOAT:
2452 {
2453 const auto castArray =
2454 static_cast<const arrow::FloatArray *>(array);
2455 poFeature->SetFieldSameTypeUnsafe(
2456 i, castArray->Value(nIdxInBatch));
2457 break;
2458 }
2459 case arrow::Type::DOUBLE:
2460 {
2461 const auto castArray =
2462 static_cast<const arrow::DoubleArray *>(array);
2463 poFeature->SetFieldSameTypeUnsafe(
2464 i, castArray->Value(nIdxInBatch));
2465 break;
2466 }
2467 case arrow::Type::STRING:
2468 {
2469 const auto castArray =
2470 static_cast<const arrow::StringArray *>(array);
2471 int out_length = 0;
2472 const uint8_t *data =
2473 castArray->GetValue(nIdxInBatch, &out_length);
2474 char *pszString =
2475 static_cast<char *>(CPLMalloc(out_length + 1));
2476 memcpy(pszString, data, out_length);
2477 pszString[out_length] = 0;
2478 poFeature->SetFieldSameTypeUnsafe(i, pszString);
2479 break;
2480 }
2481 case arrow::Type::BINARY:
2482 {
2483 const auto castArray =
2484 static_cast<const arrow::BinaryArray *>(array);
2485 int out_length = 0;
2486 const uint8_t *data =
2487 castArray->GetValue(nIdxInBatch, &out_length);
2488 poFeature->SetField(i, out_length, data);
2489 break;
2490 }
2491 #if ARROW_VERSION_MAJOR >= 15
2492 case arrow::Type::BINARY_VIEW:
2493 {
2494 const auto castArray =
2495 static_cast<const arrow::BinaryViewArray *>(array);
2496 const auto view = castArray->GetView(nIdxInBatch);
2497 poFeature->SetField(i, static_cast<int>(view.size()),
2498 view.data());
2499 break;
2500 }
2501 #endif
2502 #if ARROW_VERSION_MAJOR >= 15
2503 case arrow::Type::STRING_VIEW:
2504 {
2505 const auto castArray =
2506 static_cast<const arrow::StringViewArray *>(array);
2507 const auto strView = castArray->GetView(nIdxInBatch);
2508 char *pszString =
2509 static_cast<char *>(CPLMalloc(strView.length() + 1));
2510 memcpy(pszString, strView.data(), strView.length());
2511 pszString[strView.length()] = 0;
2512 poFeature->SetFieldSameTypeUnsafe(i, pszString);
2513 break;
2514 }
2515 #endif
2516 case arrow::Type::FIXED_SIZE_BINARY:
2517 {
2518 const auto castArray =
2519 static_cast<const arrow::FixedSizeBinaryArray *>(array);
2520 const uint8_t *data = castArray->GetValue(nIdxInBatch);
2521 poFeature->SetField(i, castArray->byte_width(), data);
2522 break;
2523 }
2524 case arrow::Type::DATE32:
2525 {
2526 // number of days since Epoch
2527 const auto castArray =
2528 static_cast<const arrow::Date32Array *>(array);
2529 int64_t timestamp =
2530 static_cast<int64_t>(castArray->Value(nIdxInBatch)) * 3600 *
2531 24;
2532 struct tm dt;
2533 CPLUnixTimeToYMDHMS(timestamp, &dt);
2534 poFeature->SetField(i, dt.tm_year + 1900, dt.tm_mon + 1,
2535 dt.tm_mday, 0, 0, 0);
2536 break;
2537 }
2538 case arrow::Type::DATE64:
2539 {
2540 // number of milliseconds since Epoch
2541 const auto castArray =
2542 static_cast<const arrow::Date64Array *>(array);
2543 int64_t timestamp =
2544 static_cast<int64_t>(castArray->Value(nIdxInBatch)) / 1000;
2545 struct tm dt;
2546 CPLUnixTimeToYMDHMS(timestamp, &dt);
2547 poFeature->SetField(i, dt.tm_year + 1900, dt.tm_mon + 1,
2548 dt.tm_mday, 0, 0, 0);
2549 break;
2550 }
2551 case arrow::Type::TIMESTAMP:
2552 {
2553 const auto timestampType = static_cast<arrow::TimestampType *>(
2554 array->data()->type.get());
2555 const auto castArray =
2556 static_cast<const arrow::TimestampArray *>(array);
2557 const int64_t timestamp = castArray->Value(nIdxInBatch);
2558 OGRField sField;
2559 sField.Set.nMarker1 = OGRUnsetMarker;
2560 sField.Set.nMarker2 = OGRUnsetMarker;
2561 sField.Set.nMarker3 = OGRUnsetMarker;
2562 TimestampToOGR(timestamp, timestampType,
2563 m_poFeatureDefn->GetFieldDefn(i)->GetTZFlag(),
2564 &sField);
2565 poFeature->SetField(i, &sField);
2566 break;
2567 }
2568 case arrow::Type::TIME32:
2569 {
2570 const auto timestampType =
2571 static_cast<arrow::Time32Type *>(array->data()->type.get());
2572 const auto castArray =
2573 static_cast<const arrow::Time32Array *>(array);
2574 const auto unit = timestampType->unit();
2575 int value = castArray->Value(nIdxInBatch);
2576 double floatingPart = 0;
2577 if (unit == arrow::TimeUnit::MILLI)
2578 {
2579 floatingPart = (value % 1000) / 1e3;
2580 value /= 1000;
2581 }
2582 const int nHour = value / 3600;
2583 const int nMinute = (value / 60) % 60;
2584 const int nSecond = value % 60;
2585 poFeature->SetField(i, 0, 0, 0, nHour, nMinute,
2586 static_cast<float>(nSecond + floatingPart));
2587 break;
2588 }
2589 case arrow::Type::TIME64:
2590 {
2591 const auto castArray =
2592 static_cast<const arrow::Time64Array *>(array);
2593 poFeature->SetField(
2594 i, static_cast<GIntBig>(castArray->Value(nIdxInBatch)));
2595 break;
2596 }
2597
2598 #if ARROW_VERSION_MAJOR >= 18
2599 case arrow::Type::DECIMAL32:
2600 {
2601 const auto castArray =
2602 static_cast<const arrow::Decimal32Array *>(array);
2603 poFeature->SetField(
2604 i, CPLAtof(castArray->FormatValue(nIdxInBatch).c_str()));
2605 break;
2606 }
2607
2608 case arrow::Type::DECIMAL64:
2609 {
2610 const auto castArray =
2611 static_cast<const arrow::Decimal64Array *>(array);
2612 poFeature->SetField(
2613 i, CPLAtof(castArray->FormatValue(nIdxInBatch).c_str()));
2614 break;
2615 }
2616 #endif
2617
2618 case arrow::Type::DECIMAL128:
2619 {
2620 const auto castArray =
2621 static_cast<const arrow::Decimal128Array *>(array);
2622 poFeature->SetField(
2623 i, CPLAtof(castArray->FormatValue(nIdxInBatch).c_str()));
2624 break;
2625 }
2626
2627 case arrow::Type::DECIMAL256:
2628 {
2629 const auto castArray =
2630 static_cast<const arrow::Decimal256Array *>(array);
2631 poFeature->SetField(
2632 i, CPLAtof(castArray->FormatValue(nIdxInBatch).c_str()));
2633 break;
2634 }
2635
2636 case arrow::Type::LIST:
2637 {
2638 ReadList(static_cast<const arrow::ListArray *>(array), i);
2639 break;
2640 }
2641
2642 case arrow::Type::LARGE_LIST:
2643 {
2644 ReadList(static_cast<const arrow::LargeListArray *>(array), i);
2645 break;
2646 }
2647
2648 case arrow::Type::FIXED_SIZE_LIST:
2649 {
2650 ReadList(static_cast<const arrow::FixedSizeListArray *>(array),
2651 i);
2652 break;
2653 }
2654
2655 case arrow::Type::LARGE_STRING:
2656 {
2657 const auto castArray =
2658 static_cast<const arrow::LargeStringArray *>(array);
2659 poFeature->SetField(i,
2660 castArray->GetString(nIdxInBatch).c_str());
2661 break;
2662 }
2663 case arrow::Type::LARGE_BINARY:
2664 {
2665 const auto castArray =
2666 static_cast<const arrow::LargeBinaryArray *>(array);
2667 arrow::LargeBinaryArray::offset_type out_length = 0;
2668 const uint8_t *data =
2669 castArray->GetValue(nIdxInBatch, &out_length);
2670 if (out_length >= 0 && out_length <= INT_MAX - 1)
2671 {
2672 // coverity[overflow_sink]
2673 poFeature->SetField(i, static_cast<int>(out_length), data);
2674 }
2675 else
2676 {
2677 // this is probably the most likely code path if people use
2678 // LargeBinary...
2679 CPLError(CE_Warning, CPLE_AppDefined,
2680 "Too large binary: " CPL_FRMT_GUIB " bytes",
2681 static_cast<GUIntBig>(out_length));
2682 }
2683 break;
2684 }
2685
2686 case arrow::Type::MAP:
2687 {
2688 const auto castArray =
2689 static_cast<const arrow::MapArray *>(array);
2690 poFeature->SetField(
2691 i, GetMapAsJSON(castArray, static_cast<size_t>(nIdxInBatch))
2692 .Format(CPLJSONObject::PrettyFormat::Plain)
2693 .c_str());
2694 break;
2695 }
2696
2697 // unhandled types
2698 case arrow::Type::STRUCT: // should not happen
2699 case arrow::Type::INTERVAL_MONTHS:
2700 case arrow::Type::INTERVAL_DAY_TIME:
2701 case arrow::Type::SPARSE_UNION:
2702 case arrow::Type::DENSE_UNION:
2703 case arrow::Type::DICTIONARY:
2704 case arrow::Type::EXTENSION:
2705 case arrow::Type::DURATION:
2706 case arrow::Type::INTERVAL_MONTH_DAY_NANO:
2707 #if ARROW_VERSION_MAJOR >= 12
2708 case arrow::Type::RUN_END_ENCODED:
2709 #endif
2710 #if ARROW_VERSION_MAJOR >= 15
2711 case arrow::Type::LIST_VIEW:
2712 case arrow::Type::LARGE_LIST_VIEW:
2713 #endif
2714 case arrow::Type::MAX_ID:
2715 {
2716 // Shouldn't happen normally as we should have discarded those
2717 // fields when creating OGR field definitions
2718 CPLError(CE_Warning, CPLE_AppDefined,
2719 "Cannot read content for field %s",
2720 m_poFeatureDefn->GetFieldDefn(i)->GetNameRef());
2721 break;
2722 }
2723 }
2724 }
2725
2726 const int nGeomFieldCount = m_poFeatureDefn->GetGeomFieldCount();
2727 for (int i = 0; i < nGeomFieldCount; ++i)
2728 {
2729 int iCol;
2730 if (m_bIgnoredFields)
2731 {
2732 iCol = m_anMapGeomFieldIndexToArrayIndex[i];
2733 if (iCol < 0)
2734 continue;
2735 }
2736 else
2737 {
2738 iCol = m_anMapGeomFieldIndexToArrowColumn[i];
2739 }
2740
2741 const auto array = GetStorageArray(poColumnArrays[iCol].get());
2742 auto poGeometry = ReadGeometry(i, array, nIdxInBatch);
2743 if (poGeometry)
2744 {
2745 const auto poGeomFieldDefn = m_poFeatureDefn->GetGeomFieldDefn(i);
2746 if (wkbFlatten(poGeometry->getGeometryType()) == wkbLineString &&
2747 wkbFlatten(poGeomFieldDefn->GetType()) == wkbMultiLineString)
2748 {
2749 poGeometry =
2750 OGRGeometryFactory::forceToMultiLineString(poGeometry);
2751 }
2752 else if (wkbFlatten(poGeometry->getGeometryType()) == wkbPolygon &&
2753 wkbFlatten(poGeomFieldDefn->GetType()) == wkbMultiPolygon)
2754 {
2755 poGeometry =
2756 OGRGeometryFactory::forceToMultiPolygon(poGeometry);
2757 }
2758 if (OGR_GT_HasZ(poGeomFieldDefn->GetType()) && !poGeometry->Is3D())
2759 {
2760 poGeometry->set3D(true);
2761 }
2762 poFeature->SetGeomFieldDirectly(i, poGeometry);
2763 }
2764 }
2765
2766 return poFeature;
2767 }
2768
2769 /************************************************************************/
2770 /* ReadGeometry() */
2771 /************************************************************************/
2772
2773 inline OGRGeometry *OGRArrowLayer::ReadGeometry(int iGeomField,
2774 const arrow::Array *array,
2775 int64_t nIdxInBatch) const
2776 {
2777 if (array->IsNull(nIdxInBatch))
2778 {
2779 return nullptr;
2780 }
2781 OGRGeometry *poGeometry = nullptr;
2782 const auto poGeomFieldDefn = m_poFeatureDefn->GetGeomFieldDefn(iGeomField);
2783 const auto eGeomType = poGeomFieldDefn->GetType();
2784 const bool bHasZ = CPL_TO_BOOL(OGR_GT_HasZ(eGeomType));
2785 const bool bHasM = CPL_TO_BOOL(OGR_GT_HasM(eGeomType));
2786 const int nDim = 2 + (bHasZ ? 1 : 0) + (bHasM ? 1 : 0);
2787
2788 const auto CreatePoint =
2789 [bHasZ, bHasM](const arrow::DoubleArray *pointValues, int pointOffset)
2790 {
2791 if (bHasZ)
2792 {
2793 if (bHasM)
2794 {
2795 return new OGRPoint(pointValues->Value(pointOffset),
2796 pointValues->Value(pointOffset + 1),
2797 pointValues->Value(pointOffset + 2),
2798 pointValues->Value(pointOffset + 3));
2799 }
2800 else
2801 {
2802 return new OGRPoint(pointValues->Value(pointOffset),
2803 pointValues->Value(pointOffset + 1),
2804 pointValues->Value(pointOffset + 2));
2805 }
2806 }
2807 else if (bHasM)
2808 {
2809 return OGRPoint::createXYM(pointValues->Value(pointOffset),
2810 pointValues->Value(pointOffset + 1),
2811 pointValues->Value(pointOffset + 2));
2812 }
2813 else
2814 {
2815 return new OGRPoint(pointValues->Value(pointOffset),
2816 pointValues->Value(pointOffset + 1));
2817 }
2818 };
2819
2820 const auto CreateStructPoint =
2821 [nDim, bHasZ, bHasM](const arrow::StructArray *structArray,
2822 int64_t pointOffset)
2823 {
2824 CPL_IGNORE_RET_VAL(nDim);
2825 CPLAssert(structArray->num_fields() == nDim);
2826 const auto &fieldX = structArray->field(0);
2827 CPLAssert(fieldX->type_id() == arrow::Type::DOUBLE);
2828 const auto fieldXDouble =
2829 static_cast<arrow::DoubleArray *>(fieldX.get());
2830 const auto &fieldY = structArray->field(1);
2831 CPLAssert(fieldY->type_id() == arrow::Type::DOUBLE);
2832 const auto fieldYDouble =
2833 static_cast<arrow::DoubleArray *>(fieldY.get());
2834 if (bHasZ)
2835 {
2836 const auto &fieldZ = structArray->field(2);
2837 CPLAssert(fieldZ->type_id() == arrow::Type::DOUBLE);
2838 const auto fieldZDouble =
2839 static_cast<arrow::DoubleArray *>(fieldZ.get());
2840 if (bHasM)
2841 {
2842 const auto &fieldM = structArray->field(3);
2843 CPLAssert(fieldM->type_id() == arrow::Type::DOUBLE);
2844 const auto fieldMDouble =
2845 static_cast<arrow::DoubleArray *>(fieldM.get());
2846 return new OGRPoint(fieldXDouble->Value(pointOffset),
2847 fieldYDouble->Value(pointOffset),
2848 fieldZDouble->Value(pointOffset),
2849 fieldMDouble->Value(pointOffset));
2850 }
2851 else
2852 {
2853 return new OGRPoint(fieldXDouble->Value(pointOffset),
2854 fieldYDouble->Value(pointOffset),
2855 fieldZDouble->Value(pointOffset));
2856 }
2857 }
2858 else if (bHasM)
2859 {
2860 const auto &fieldM = structArray->field(2);
2861 CPLAssert(fieldM->type_id() == arrow::Type::DOUBLE);
2862 const auto fieldMDouble =
2863 static_cast<arrow::DoubleArray *>(fieldM.get());
2864 return OGRPoint::createXYM(fieldXDouble->Value(pointOffset),
2865 fieldYDouble->Value(pointOffset),
2866 fieldMDouble->Value(pointOffset));
2867 }
2868 else
2869 {
2870 return new OGRPoint(fieldXDouble->Value(pointOffset),
2871 fieldYDouble->Value(pointOffset));
2872 }
2873 };
2874
2875 // Arrow 14 since https://github.com/apache/arrow/commit/95a8bfb319b2729c8f6daa069433caba3b4ddddd
2876 // returns reference to shared pointers, so we can safely take the raw pointer
2877 // and cast it.
2878 // Earlier versions returned a non-reference shared pointer, so formally it
2879 // is safer to use static_pointer_cast (although in practice given that
2880 // "values" is a member variable), the Arrow >= 14 path might work...
2881 #if ARROW_VERSION_MAJOR >= 14
2882 #define GET_PTR_FROM_VALUES(var, type, values) \
2883 const auto var = static_cast<const type *>((values).get())
2884 #else
2885 #define GET_PTR_FROM_VALUES(var, type, values) \
2886 const auto var##tmp = std::static_pointer_cast<type>(values); \
2887 const auto var = var##tmp.get()
2888 #endif
2889
2890 switch (m_aeGeomEncoding[iGeomField])
2891 {
2892 case OGRArrowGeomEncoding::WKB:
2893 {
2894 int out_length = 0;
2895 const uint8_t *data;
2896 if (array->type_id() == arrow::Type::BINARY)
2897 {
2898 const auto castArray =
2899 static_cast<const arrow::BinaryArray *>(array);
2900 data = castArray->GetValue(nIdxInBatch, &out_length);
2901 }
2902 else
2903 {
2904 CPLAssert(array->type_id() == arrow::Type::LARGE_BINARY);
2905 const auto castArray =
2906 static_cast<const arrow::LargeBinaryArray *>(array);
2907 int64_t out_length64 = 0;
2908 data = castArray->GetValue(nIdxInBatch, &out_length64);
2909 if (out_length64 > INT_MAX)
2910 {
2911 CPLError(CE_Failure, CPLE_AppDefined, "Too large geometry");
2912 return nullptr;
2913 }
2914 out_length = static_cast<int>(out_length64);
2915 }
2916 if (OGRGeometryFactory::createFromWkb(
2917 data, poGeomFieldDefn->GetSpatialRef(), &poGeometry,
2918 out_length) == OGRERR_NONE)
2919 {
2920 #ifdef DEBUG_ReadWKBBoundingBox
2921 OGREnvelope sEnvelopeFromWKB;
2922 bool bRet =
2923 OGRWKBGetBoundingBox(data, out_length, sEnvelopeFromWKB);
2924 CPLAssert(bRet);
2925 OGREnvelope sEnvelopeFromGeom;
2926 poGeometry->getEnvelope(&sEnvelopeFromGeom);
2927 CPLAssert(sEnvelopeFromWKB == sEnvelopeFromGeom);
2928 #endif
2929 }
2930 break;
2931 }
2932
2933 case OGRArrowGeomEncoding::WKT:
2934 {
2935 if (array->type_id() == arrow::Type::STRING)
2936 {
2937 const auto castArray =
2938 static_cast<const arrow::StringArray *>(array);
2939 const auto osWKT = castArray->GetString(nIdxInBatch);
2940 OGRGeometryFactory::createFromWkt(
2941 osWKT.c_str(), poGeomFieldDefn->GetSpatialRef(),
2942 &poGeometry);
2943 }
2944 else
2945 {
2946 CPLAssert(array->type_id() == arrow::Type::LARGE_STRING);
2947 const auto castArray =
2948 static_cast<const arrow::LargeStringArray *>(array);
2949 const auto osWKT = castArray->GetString(nIdxInBatch);
2950 OGRGeometryFactory::createFromWkt(
2951 osWKT.c_str(), poGeomFieldDefn->GetSpatialRef(),
2952 &poGeometry);
2953 }
2954 break;
2955 }
2956
2957 case OGRArrowGeomEncoding::GEOARROW_FSL_GENERIC:
2958 case OGRArrowGeomEncoding::GEOARROW_STRUCT_GENERIC:
2959 {
2960 CPLAssert(false);
2961 break;
2962 }
2963
2964 case OGRArrowGeomEncoding::GEOARROW_FSL_POINT:
2965 {
2966 CPLAssert(array->type_id() == arrow::Type::FIXED_SIZE_LIST);
2967 const auto listArray =
2968 static_cast<const arrow::FixedSizeListArray *>(array);
2969 CPLAssert(listArray->values()->type_id() == arrow::Type::DOUBLE);
2970 GET_PTR_FROM_VALUES(pointValues, arrow::DoubleArray,
2971 listArray->values());
2972 if (!pointValues->IsNull(nDim * nIdxInBatch))
2973 {
2974 poGeometry = CreatePoint(pointValues,
2975 static_cast<int>(nDim * nIdxInBatch));
2976 poGeometry->assignSpatialReference(
2977 poGeomFieldDefn->GetSpatialRef());
2978 }
2979 break;
2980 }
2981
2982 case OGRArrowGeomEncoding::GEOARROW_FSL_LINESTRING:
2983 {
2984 CPLAssert(array->type_id() == arrow::Type::LIST);
2985 const auto listArray = static_cast<const arrow::ListArray *>(array);
2986 CPLAssert(listArray->values()->type_id() ==
2987 arrow::Type::FIXED_SIZE_LIST);
2988 GET_PTR_FROM_VALUES(listOfPointsValues, arrow::FixedSizeListArray,
2989 listArray->values());
2990 CPLAssert(listOfPointsValues->values()->type_id() ==
2991 arrow::Type::DOUBLE);
2992 GET_PTR_FROM_VALUES(pointValues, arrow::DoubleArray,
2993 listOfPointsValues->values());
2994 const auto nPoints = listArray->value_length(nIdxInBatch);
2995 const auto nPointOffset =
2996 listArray->value_offset(nIdxInBatch) * nDim;
2997 auto poLineString = new OGRLineString();
2998 poGeometry = poLineString;
2999 poGeometry->assignSpatialReference(
3000 poGeomFieldDefn->GetSpatialRef());
3001 if (nPoints)
3002 {
3003 GetSetPointsOfLine(bHasZ, bHasM)(poLineString, pointValues,
3004 nPointOffset, nPoints);
3005 }
3006 else
3007 {
3008 poGeometry->set3D(bHasZ);
3009 poGeometry->setMeasured(bHasM);
3010 }
3011 break;
3012 }
3013
3014 case OGRArrowGeomEncoding::GEOARROW_FSL_POLYGON:
3015 {
3016 CPLAssert(array->type_id() == arrow::Type::LIST);
3017 const auto listOfRingsArray =
3018 static_cast<const arrow::ListArray *>(array);
3019 CPLAssert(listOfRingsArray->values()->type_id() ==
3020 arrow::Type::LIST);
3021 GET_PTR_FROM_VALUES(listOfRingsValues, arrow::ListArray,
3022 listOfRingsArray->values());
3023 CPLAssert(listOfRingsValues->values()->type_id() ==
3024 arrow::Type::FIXED_SIZE_LIST);
3025 GET_PTR_FROM_VALUES(listOfPointsValues, arrow::FixedSizeListArray,
3026 listOfRingsValues->values());
3027 CPLAssert(listOfPointsValues->values()->type_id() ==
3028 arrow::Type::DOUBLE);
3029 GET_PTR_FROM_VALUES(pointValues, arrow::DoubleArray,
3030 listOfPointsValues->values());
3031 const auto setPointsFun = GetSetPointsOfLine(bHasZ, bHasM);
3032 const auto nRings = listOfRingsArray->value_length(nIdxInBatch);
3033 const auto nRingOffset =
3034 listOfRingsArray->value_offset(nIdxInBatch);
3035 auto poPoly = new OGRPolygon();
3036 poGeometry = poPoly;
3037 poGeometry->assignSpatialReference(
3038 poGeomFieldDefn->GetSpatialRef());
3039 for (auto k = decltype(nRings){0}; k < nRings; k++)
3040 {
3041 const auto nPoints =
3042 listOfRingsValues->value_length(nRingOffset + k);
3043 const auto nPointOffset =
3044 listOfRingsValues->value_offset(nRingOffset + k) * nDim;
3045 auto poRing = new OGRLinearRing();
3046 if (nPoints)
3047 {
3048 setPointsFun(poRing, pointValues, nPointOffset, nPoints);
3049 }
3050 poPoly->addRingDirectly(poRing);
3051 }
3052 if (poGeometry->IsEmpty())
3053 {
3054 poGeometry->set3D(bHasZ);
3055 poGeometry->setMeasured(bHasM);
3056 }
3057 break;
3058 }
3059
3060 case OGRArrowGeomEncoding::GEOARROW_FSL_MULTIPOINT:
3061 {
3062 CPLAssert(array->type_id() == arrow::Type::LIST);
3063 const auto listArray = static_cast<const arrow::ListArray *>(array);
3064 CPLAssert(listArray->values()->type_id() ==
3065 arrow::Type::FIXED_SIZE_LIST);
3066 GET_PTR_FROM_VALUES(listOfPointsValues, arrow::FixedSizeListArray,
3067 listArray->values());
3068 CPLAssert(listOfPointsValues->values()->type_id() ==
3069 arrow::Type::DOUBLE);
3070 GET_PTR_FROM_VALUES(pointValues, arrow::DoubleArray,
3071 listOfPointsValues->values());
3072 const auto nPoints = listArray->value_length(nIdxInBatch);
3073 const auto nPointOffset =
3074 listArray->value_offset(nIdxInBatch) * nDim;
3075 auto poMultiPoint = new OGRMultiPoint();
3076 poGeometry = poMultiPoint;
3077 poGeometry->assignSpatialReference(
3078 poGeomFieldDefn->GetSpatialRef());
3079 for (auto k = decltype(nPoints){0}; k < nPoints; k++)
3080 {
3081 poMultiPoint->addGeometryDirectly(
3082 CreatePoint(pointValues, nPointOffset + k * nDim));
3083 }
3084 if (poGeometry->IsEmpty())
3085 {
3086 poGeometry->set3D(bHasZ);
3087 poGeometry->setMeasured(bHasM);
3088 }
3089 break;
3090 }
3091
3092 case OGRArrowGeomEncoding::GEOARROW_FSL_MULTILINESTRING:
3093 {
3094 CPLAssert(array->type_id() == arrow::Type::LIST);
3095 const auto listOfStringsArray =
3096 static_cast<const arrow::ListArray *>(array);
3097 CPLAssert(listOfStringsArray->values()->type_id() ==
3098 arrow::Type::LIST);
3099 GET_PTR_FROM_VALUES(listOfStringsValues, arrow::ListArray,
3100 listOfStringsArray->values());
3101 CPLAssert(listOfStringsValues->values()->type_id() ==
3102 arrow::Type::FIXED_SIZE_LIST);
3103 GET_PTR_FROM_VALUES(listOfPointsValues, arrow::FixedSizeListArray,
3104 listOfStringsValues->values());
3105 CPLAssert(listOfPointsValues->values()->type_id() ==
3106 arrow::Type::DOUBLE);
3107 GET_PTR_FROM_VALUES(pointValues, arrow::DoubleArray,
3108 listOfPointsValues->values());
3109 const auto setPointsFun = GetSetPointsOfLine(bHasZ, bHasM);
3110 const auto nStrings = listOfStringsArray->value_length(nIdxInBatch);
3111 const auto nRingOffset =
3112 listOfStringsArray->value_offset(nIdxInBatch);
3113 auto poMLS = new OGRMultiLineString();
3114 poGeometry = poMLS;
3115 poGeometry->assignSpatialReference(
3116 poGeomFieldDefn->GetSpatialRef());
3117 for (auto k = decltype(nStrings){0}; k < nStrings; k++)
3118 {
3119 const auto nPoints =
3120 listOfStringsValues->value_length(nRingOffset + k);
3121 const auto nPointOffset =
3122 listOfStringsValues->value_offset(nRingOffset + k) * nDim;
3123 auto poLS = new OGRLineString();
3124 if (nPoints)
3125 {
3126 setPointsFun(poLS, pointValues, nPointOffset, nPoints);
3127 }
3128 poMLS->addGeometryDirectly(poLS);
3129 }
3130 if (poGeometry->IsEmpty())
3131 {
3132 poGeometry->set3D(bHasZ);
3133 poGeometry->setMeasured(bHasM);
3134 }
3135 break;
3136 }
3137
3138 case OGRArrowGeomEncoding::GEOARROW_FSL_MULTIPOLYGON:
3139 {
3140 CPLAssert(array->type_id() == arrow::Type::LIST);
3141 const auto listOfPartsArray =
3142 static_cast<const arrow::ListArray *>(array);
3143 CPLAssert(listOfPartsArray->values()->type_id() ==
3144 arrow::Type::LIST);
3145 GET_PTR_FROM_VALUES(listOfPartsValues, arrow::ListArray,
3146 listOfPartsArray->values());
3147 CPLAssert(listOfPartsValues->values()->type_id() ==
3148 arrow::Type::LIST);
3149 GET_PTR_FROM_VALUES(listOfRingsValues, arrow::ListArray,
3150 listOfPartsValues->values());
3151 CPLAssert(listOfRingsValues->values()->type_id() ==
3152 arrow::Type::FIXED_SIZE_LIST);
3153 GET_PTR_FROM_VALUES(listOfPointsValues, arrow::FixedSizeListArray,
3154 listOfRingsValues->values());
3155 CPLAssert(listOfPointsValues->values()->type_id() ==
3156 arrow::Type::DOUBLE);
3157 GET_PTR_FROM_VALUES(pointValues, arrow::DoubleArray,
3158 listOfPointsValues->values());
3159 auto poMP = new OGRMultiPolygon();
3160 poGeometry = poMP;
3161 poGeometry->assignSpatialReference(
3162 poGeomFieldDefn->GetSpatialRef());
3163 const auto setPointsFun = GetSetPointsOfLine(bHasZ, bHasM);
3164 const auto nParts = listOfPartsArray->value_length(nIdxInBatch);
3165 const auto nPartOffset =
3166 listOfPartsArray->value_offset(nIdxInBatch);
3167 for (auto j = decltype(nParts){0}; j < nParts; j++)
3168 {
3169 const auto nRings =
3170 listOfPartsValues->value_length(nPartOffset + j);
3171 const auto nRingOffset =
3172 listOfPartsValues->value_offset(nPartOffset + j);
3173 auto poPoly = new OGRPolygon();
3174 for (auto k = decltype(nRings){0}; k < nRings; k++)
3175 {
3176 const auto nPoints =
3177 listOfRingsValues->value_length(nRingOffset + k);
3178 const auto nPointOffset =
3179 listOfRingsValues->value_offset(nRingOffset + k) * nDim;
3180 auto poRing = new OGRLinearRing();
3181 if (nPoints)
3182 {
3183 setPointsFun(poRing, pointValues, nPointOffset,
3184 nPoints);
3185 }
3186 poPoly->addRingDirectly(poRing);
3187 }
3188 poMP->addGeometryDirectly(poPoly);
3189 }
3190 if (poGeometry->IsEmpty())
3191 {
3192 poGeometry->set3D(bHasZ);
3193 poGeometry->setMeasured(bHasM);
3194 }
3195 break;
3196 }
3197
3198 case OGRArrowGeomEncoding::GEOARROW_STRUCT_POINT:
3199 {
3200 CPLAssert(array->type_id() == arrow::Type::STRUCT);
3201 const auto structArray =
3202 static_cast<const arrow::StructArray *>(array);
3203 if (!structArray->IsNull(nIdxInBatch))
3204 {
3205 poGeometry = CreateStructPoint(structArray, nIdxInBatch);
3206 poGeometry->assignSpatialReference(
3207 poGeomFieldDefn->GetSpatialRef());
3208 }
3209 break;
3210 }
3211
3212 case OGRArrowGeomEncoding::GEOARROW_STRUCT_LINESTRING:
3213 {
3214 CPLAssert(array->type_id() == arrow::Type::LIST);
3215 const auto listArray = static_cast<const arrow::ListArray *>(array);
3216 CPLAssert(listArray->values()->type_id() == arrow::Type::STRUCT);
3217 GET_PTR_FROM_VALUES(pointValues, arrow::StructArray,
3218 listArray->values());
3219 const auto nPoints = listArray->value_length(nIdxInBatch);
3220 const auto nPointOffset = listArray->value_offset(nIdxInBatch);
3221 auto poLineString = new OGRLineString();
3222 poGeometry = poLineString;
3223 poGeometry->assignSpatialReference(
3224 poGeomFieldDefn->GetSpatialRef());
3225 if (nPoints)
3226 {
3227 GetSetPointsOfLineStruct(bHasZ, bHasM)(
3228 poLineString, pointValues, nPointOffset, nPoints);
3229 }
3230 else
3231 {
3232 poGeometry->set3D(bHasZ);
3233 poGeometry->setMeasured(bHasM);
3234 }
3235 break;
3236 }
3237
3238 case OGRArrowGeomEncoding::GEOARROW_STRUCT_POLYGON:
3239 {
3240 CPLAssert(array->type_id() == arrow::Type::LIST);
3241 const auto listOfRingsArray =
3242 static_cast<const arrow::ListArray *>(array);
3243 CPLAssert(listOfRingsArray->values()->type_id() ==
3244 arrow::Type::LIST);
3245 GET_PTR_FROM_VALUES(listOfRingsValues, arrow::ListArray,
3246 listOfRingsArray->values());
3247 CPLAssert(listOfRingsValues->values()->type_id() ==
3248 arrow::Type::STRUCT);
3249 GET_PTR_FROM_VALUES(pointValues, arrow::StructArray,
3250 listOfRingsValues->values());
3251 const auto setPointsFun = GetSetPointsOfLineStruct(bHasZ, bHasM);
3252 const auto nRings = listOfRingsArray->value_length(nIdxInBatch);
3253 const auto nRingOffset =
3254 listOfRingsArray->value_offset(nIdxInBatch);
3255 auto poPoly = new OGRPolygon();
3256 poGeometry = poPoly;
3257 poGeometry->assignSpatialReference(
3258 poGeomFieldDefn->GetSpatialRef());
3259 for (auto k = decltype(nRings){0}; k < nRings; k++)
3260 {
3261 const auto nPoints =
3262 listOfRingsValues->value_length(nRingOffset + k);
3263 const auto nPointOffset =
3264 listOfRingsValues->value_offset(nRingOffset + k);
3265 auto poRing = new OGRLinearRing();
3266 if (nPoints)
3267 {
3268 setPointsFun(poRing, pointValues, nPointOffset, nPoints);
3269 }
3270 poPoly->addRingDirectly(poRing);
3271 }
3272 if (poGeometry->IsEmpty())
3273 {
3274 poGeometry->set3D(bHasZ);
3275 poGeometry->setMeasured(bHasM);
3276 }
3277 break;
3278 }
3279
3280 case OGRArrowGeomEncoding::GEOARROW_STRUCT_MULTIPOINT:
3281 {
3282 CPLAssert(array->type_id() == arrow::Type::LIST);
3283 const auto listArray = static_cast<const arrow::ListArray *>(array);
3284 CPLAssert(listArray->values()->type_id() == arrow::Type::STRUCT);
3285 GET_PTR_FROM_VALUES(pointValues, arrow::StructArray,
3286 listArray->values());
3287 const auto nPoints = listArray->value_length(nIdxInBatch);
3288 const auto nPointOffset = listArray->value_offset(nIdxInBatch);
3289 auto poMultiPoint = new OGRMultiPoint();
3290 poGeometry = poMultiPoint;
3291 poGeometry->assignSpatialReference(
3292 poGeomFieldDefn->GetSpatialRef());
3293 for (auto k = decltype(nPoints){0}; k < nPoints; k++)
3294 {
3295 poMultiPoint->addGeometryDirectly(
3296 CreateStructPoint(pointValues, nPointOffset + k));
3297 }
3298 if (poGeometry->IsEmpty())
3299 {
3300 poGeometry->set3D(bHasZ);
3301 poGeometry->setMeasured(bHasM);
3302 }
3303 break;
3304 }
3305
3306 case OGRArrowGeomEncoding::GEOARROW_STRUCT_MULTILINESTRING:
3307 {
3308 CPLAssert(array->type_id() == arrow::Type::LIST);
3309 const auto listOfStringsArray =
3310 static_cast<const arrow::ListArray *>(array);
3311 CPLAssert(listOfStringsArray->values()->type_id() ==
3312 arrow::Type::LIST);
3313 GET_PTR_FROM_VALUES(listOfStringsValues, arrow::ListArray,
3314 listOfStringsArray->values());
3315 CPLAssert(listOfStringsValues->values()->type_id() ==
3316 arrow::Type::STRUCT);
3317 GET_PTR_FROM_VALUES(pointValues, arrow::StructArray,
3318 listOfStringsValues->values());
3319 const auto setPointsFun = GetSetPointsOfLineStruct(bHasZ, bHasM);
3320 const auto nStrings = listOfStringsArray->value_length(nIdxInBatch);
3321 const auto nRingOffset =
3322 listOfStringsArray->value_offset(nIdxInBatch);
3323 auto poMLS = new OGRMultiLineString();
3324 poGeometry = poMLS;
3325 poGeometry->assignSpatialReference(
3326 poGeomFieldDefn->GetSpatialRef());
3327 for (auto k = decltype(nStrings){0}; k < nStrings; k++)
3328 {
3329 const auto nPoints =
3330 listOfStringsValues->value_length(nRingOffset + k);
3331 const auto nPointOffset =
3332 listOfStringsValues->value_offset(nRingOffset + k);
3333 auto poLS = new OGRLineString();
3334 if (nPoints)
3335 {
3336 setPointsFun(poLS, pointValues, nPointOffset, nPoints);
3337 }
3338 poMLS->addGeometryDirectly(poLS);
3339 }
3340 if (poGeometry->IsEmpty())
3341 {
3342 poGeometry->set3D(bHasZ);
3343 poGeometry->setMeasured(bHasM);
3344 }
3345 break;
3346 }
3347
3348 case OGRArrowGeomEncoding::GEOARROW_STRUCT_MULTIPOLYGON:
3349 {
3350 CPLAssert(array->type_id() == arrow::Type::LIST);
3351 const auto listOfPartsArray =
3352 static_cast<const arrow::ListArray *>(array);
3353 CPLAssert(listOfPartsArray->values()->type_id() ==
3354 arrow::Type::LIST);
3355 GET_PTR_FROM_VALUES(listOfPartsValues, arrow::ListArray,
3356 listOfPartsArray->values());
3357 CPLAssert(listOfPartsValues->values()->type_id() ==
3358 arrow::Type::LIST);
3359 GET_PTR_FROM_VALUES(listOfRingsValues, arrow::ListArray,
3360 listOfPartsValues->values());
3361 CPLAssert(listOfRingsValues->values()->type_id() ==
3362 arrow::Type::STRUCT);
3363 GET_PTR_FROM_VALUES(pointValues, arrow::StructArray,
3364 listOfRingsValues->values());
3365 auto poMP = new OGRMultiPolygon();
3366 poGeometry = poMP;
3367 poGeometry->assignSpatialReference(
3368 poGeomFieldDefn->GetSpatialRef());
3369 const auto setPointsFun = GetSetPointsOfLineStruct(bHasZ, bHasM);
3370 const auto nParts = listOfPartsArray->value_length(nIdxInBatch);
3371 const auto nPartOffset =
3372 listOfPartsArray->value_offset(nIdxInBatch);
3373 for (auto j = decltype(nParts){0}; j < nParts; j++)
3374 {
3375 const auto nRings =
3376 listOfPartsValues->value_length(nPartOffset + j);
3377 const auto nRingOffset =
3378 listOfPartsValues->value_offset(nPartOffset + j);
3379 auto poPoly = new OGRPolygon();
3380 for (auto k = decltype(nRings){0}; k < nRings; k++)
3381 {
3382 const auto nPoints =
3383 listOfRingsValues->value_length(nRingOffset + k);
3384 const auto nPointOffset =
3385 listOfRingsValues->value_offset(nRingOffset + k);
3386 auto poRing = new OGRLinearRing();
3387 if (nPoints)
3388 {
3389 setPointsFun(poRing, pointValues, nPointOffset,
3390 nPoints);
3391 }
3392 poPoly->addRingDirectly(poRing);
3393 }
3394 poMP->addGeometryDirectly(poPoly);
3395 }
3396 if (poGeometry->IsEmpty())
3397 {
3398 poGeometry->set3D(bHasZ);
3399 poGeometry->setMeasured(bHasM);
3400 }
3401 break;
3402 }
3403 }
3404 return poGeometry;
3405 }
3406
3407 /************************************************************************/
3408 /* ResetReading() */
3409 /************************************************************************/
3410
3411 inline void OGRArrowLayer::ResetReading()
3412 {
3413 m_bEOF = false;
3414 m_nFeatureIdx = 0;
3415 m_nIdxInBatch = 0;
3416 m_poReadFeatureTmpArray.reset();
3417 if (m_iRecordBatch != 0)
3418 {
3419 m_iRecordBatch = -1;
3420 m_poBatch.reset();
3421 m_poBatchColumns.clear();
3422 }
3423 }
3424
3425 /************************************************************************/
3426 /* GetColumnSubNode() */
3427 /************************************************************************/
3428
3429 /* static*/
3430 inline const swq_expr_node *
3431 OGRArrowLayer::GetColumnSubNode(const swq_expr_node *poNode)
3432 {
3433 if (poNode->eNodeType == SNT_OPERATION && poNode->nSubExprCount == 2)
3434 {
3435 if (poNode->papoSubExpr[0]->eNodeType == SNT_COLUMN)
3436 return poNode->papoSubExpr[0];
3437 if (poNode->papoSubExpr[1]->eNodeType == SNT_COLUMN)
3438 return poNode->papoSubExpr[1];
3439 }
3440 return nullptr;
3441 }
3442
3443 /************************************************************************/
3444 /* GetConstantSubNode() */
3445 /************************************************************************/
3446
3447 /* static */
3448 inline const swq_expr_node *
3449 OGRArrowLayer::GetConstantSubNode(const swq_expr_node *poNode)
3450 {
3451 if (poNode->eNodeType == SNT_OPERATION && poNode->nSubExprCount == 2)
3452 {
3453 if (poNode->papoSubExpr[1]->eNodeType == SNT_CONSTANT)
3454 return poNode->papoSubExpr[1];
3455 if (poNode->papoSubExpr[0]->eNodeType == SNT_CONSTANT)
3456 return poNode->papoSubExpr[0];
3457 }
3458 return nullptr;
3459 }
3460
3461 /************************************************************************/
3462 /* IsComparisonOp() */
3463 /************************************************************************/
3464
3465 /* static*/
3466 inline bool OGRArrowLayer::IsComparisonOp(int op)
3467 {
3468 return (op == SWQ_EQ || op == SWQ_NE || op == SWQ_LT || op == SWQ_LE ||
3469 op == SWQ_GT || op == SWQ_GE);
3470 }
3471
3472 /************************************************************************/
3473 /* FillTargetValueFromSrcExpr() */
3474 /************************************************************************/
3475
3476 static bool FillTargetValueFromSrcExpr(const OGRFieldDefn *poFieldDefn,
3477 OGRArrowLayer::Constraint *psConstraint,
3478 const swq_expr_node *poSrcValue)
3479 {
3480 switch (poFieldDefn->GetType())
3481 {
3482 case OFTInteger:
3483 psConstraint->eType = OGRArrowLayer::Constraint::Type::Integer;
3484 if (poSrcValue->field_type == SWQ_FLOAT)
3485 psConstraint->sValue.Integer =
3486 static_cast<int>(poSrcValue->float_value);
3487 else
3488 psConstraint->sValue.Integer =
3489 static_cast<int>(poSrcValue->int_value);
3490 psConstraint->osValue =
3491 std::to_string(psConstraint->sValue.Integer);
3492 break;
3493
3494 case OFTInteger64:
3495 psConstraint->eType = OGRArrowLayer::Constraint::Type::Integer64;
3496 if (poSrcValue->field_type == SWQ_FLOAT)
3497 psConstraint->sValue.Integer64 =
3498 static_cast<GIntBig>(poSrcValue->float_value);
3499 else
3500 psConstraint->sValue.Integer64 = poSrcValue->int_value;
3501 psConstraint->osValue =
3502 std::to_string(psConstraint->sValue.Integer64);
3503 break;
3504
3505 case OFTReal:
3506 psConstraint->eType = OGRArrowLayer::Constraint::Type::Real;
3507 psConstraint->sValue.Real = poSrcValue->float_value;
3508 psConstraint->osValue = std::to_string(psConstraint->sValue.Real);
3509 break;
3510
3511 case OFTString:
3512 psConstraint->eType = OGRArrowLayer::Constraint::Type::String;
3513 psConstraint->sValue.String = poSrcValue->string_value;
3514 psConstraint->osValue = psConstraint->sValue.String;
3515 break;
3516 #ifdef not_yet_handled
3517 case OFTDate:
3518 case OFTTime:
3519 case OFTDateTime:
3520 if (poSrcValue->field_type == SWQ_TIMESTAMP ||
3521 poSrcValue->field_type == SWQ_DATE ||
3522 poSrcValue->field_type == SWQ_TIME)
3523 {
3524 int nYear = 0, nMonth = 0, nDay = 0, nHour = 0, nMin = 0,
3525 nSec = 0;
3526 if (sscanf(poSrcValue->string_value,
3527 "%04d/%02d/%02d %02d:%02d:%02d", &nYear, &nMonth,
3528 &nDay, &nHour, &nMin, &nSec) == 6 ||
3529 sscanf(poSrcValue->string_value, "%04d/%02d/%02d", &nYear,
3530 &nMonth, &nDay) == 3 ||
3531 sscanf(poSrcValue->string_value, "%02d:%02d:%02d", &nHour,
3532 &nMin, &nSec) == 3)
3533 {
3534 psConstraint->eType =
3535 OGRArrowLayer::Constraint::Type::DateTime;
3536 psConstraint->sValue.Date.Year = (GInt16)nYear;
3537 psConstraint->sValue.Date.Month = (GByte)nMonth;
3538 psConstraint->sValue.Date.Day = (GByte)nDay;
3539 psConstraint->sValue.Date.Hour = (GByte)nHour;
3540 psConstraint->sValue.Date.Minute = (GByte)nMin;
3541 psConstraint->sValue.Date.Second = (GByte)nSec;
3542 psConstraint->sValue.Date.TZFlag = 0;
3543 psConstraint->sValue.Date.Reserved = 0;
3544 }
3545 else
3546 return false;
3547 }
3548 else
3549 return false;
3550 break;
3551 #endif
3552 default:
3553 return false;
3554 }
3555 return true;
3556 }
3557
3558 /************************************************************************/
3559 /* ComputeConstraintsArrayIdx() */
3560 /************************************************************************/
3561
3562 inline void OGRArrowLayer::ComputeConstraintsArrayIdx()
3563 {
3564 for (auto &constraint : m_asAttributeFilterConstraints)
3565 {
3566 if (m_bIgnoredFields)
3567 {
3568 if (constraint.iField == m_poFeatureDefn->GetFieldCount() + SPF_FID)
3569 {
3570 constraint.iArrayIdx = m_nRequestedFIDColumn;
3571 if (constraint.iArrayIdx < 0 && m_osFIDColumn.empty())
3572 return;
3573 }
3574 else
3575 {
3576 constraint.iArrayIdx =
3577 m_anMapFieldIndexToArrayIndex[constraint.iField];
3578 }
3579 if (constraint.iArrayIdx < 0)
3580 {
3581 CPLError(CE_Failure, CPLE_AppDefined,
3582 "Constraint on field %s cannot be applied due to "
3583 "it being ignored",
3584 constraint.iField ==
3585 m_poFeatureDefn->GetFieldCount() + SPF_FID
3586 ? m_osFIDColumn.c_str()
3587 : m_poFeatureDefn->GetFieldDefn(constraint.iField)
3588 ->GetNameRef());
3589 }
3590 }
3591 else
3592 {
3593 if (constraint.iField == m_poFeatureDefn->GetFieldCount() + SPF_FID)
3594 {
3595 constraint.iArrayIdx = m_iFIDArrowColumn;
3596 if (constraint.iArrayIdx < 0 && !m_osFIDColumn.empty())
3597 {
3598 CPLDebug(GetDriverUCName().c_str(),
3599 "Constraint on field %s cannot be applied",
3600 m_osFIDColumn.c_str());
3601 }
3602 }
3603 else
3604 {
3605 constraint.iArrayIdx =
3606 m_anMapFieldIndexToArrowColumn[constraint.iField][0];
3607 }
3608 }
3609 }
3610 }
3611
3612 /************************************************************************/
3613 /* ExploreExprNode() */
3614 /************************************************************************/
3615
3616 inline void OGRArrowLayer::ExploreExprNode(const swq_expr_node *poNode)
3617 {
3618 const auto AddConstraint = [this](Constraint &constraint)
3619 { m_asAttributeFilterConstraints.emplace_back(constraint); };
3620
3621 if (poNode->eNodeType == SNT_OPERATION && poNode->nOperation == SWQ_AND &&
3622 poNode->nSubExprCount == 2)
3623 {
3624 ExploreExprNode(poNode->papoSubExpr[0]);
3625 ExploreExprNode(poNode->papoSubExpr[1]);
3626 }
3627
3628 else if (poNode->eNodeType == SNT_OPERATION &&
3629 IsComparisonOp(poNode->nOperation) && poNode->nSubExprCount == 2)
3630 {
3631 const swq_expr_node *poColumn = GetColumnSubNode(poNode);
3632 const swq_expr_node *poValue = GetConstantSubNode(poNode);
3633 if (poColumn != nullptr && poValue != nullptr &&
3634 (poColumn->field_index < m_poFeatureDefn->GetFieldCount() ||
3635 poColumn->field_index ==
3636 m_poFeatureDefn->GetFieldCount() + SPF_FID))
3637 {
3638 const OGRFieldDefn oDummyFIDFieldDefn(m_osFIDColumn.c_str(),
3639 OFTInteger64);
3640 const OGRFieldDefn *poFieldDefn =
3641 (poColumn->field_index ==
3642 m_poFeatureDefn->GetFieldCount() + SPF_FID)
3643 ? &oDummyFIDFieldDefn
3644 : m_poFeatureDefn->GetFieldDefn(poColumn->field_index);
3645
3646 Constraint constraint;
3647 constraint.iField = poColumn->field_index;
3648 constraint.nOperation = poNode->nOperation;
3649
3650 if (FillTargetValueFromSrcExpr(poFieldDefn, &constraint, poValue))
3651 {
3652 if (poColumn == poNode->papoSubExpr[0])
3653 {
3654 // nothing to do
3655 }
3656 else
3657 {
3658 /* If "constant op column", then we must reverse */
3659 /* the operator for LE, LT, GE, GT */
3660 switch (poNode->nOperation)
3661 {
3662 case SWQ_LE:
3663 constraint.nOperation = SWQ_GE;
3664 break;
3665 case SWQ_LT:
3666 constraint.nOperation = SWQ_GT;
3667 break;
3668 case SWQ_NE: /* do nothing */;
3669 break;
3670 case SWQ_EQ: /* do nothing */;
3671 break;
3672 case SWQ_GE:
3673 constraint.nOperation = SWQ_LE;
3674 break;
3675 case SWQ_GT:
3676 constraint.nOperation = SWQ_LT;
3677 break;
3678 default:
3679 CPLAssert(false);
3680 break;
3681 }
3682 }
3683
3684 AddConstraint(constraint);
3685 }
3686 }
3687 }
3688
3689 else if (poNode->eNodeType == SNT_OPERATION &&
3690 poNode->nOperation == SWQ_ISNULL && poNode->nSubExprCount == 1)
3691 {
3692 const swq_expr_node *poColumn = poNode->papoSubExpr[0];
3693 if (poColumn->eNodeType == SNT_COLUMN &&
3694 poColumn->field_index < m_poFeatureDefn->GetFieldCount())
3695 {
3696 Constraint constraint;
3697 constraint.iField = poColumn->field_index;
3698 constraint.nOperation = poNode->nOperation;
3699 AddConstraint(constraint);
3700 }
3701 }
3702
3703 else if (poNode->eNodeType == SNT_OPERATION &&
3704 poNode->nOperation == SWQ_NOT && poNode->nSubExprCount == 1 &&
3705 poNode->papoSubExpr[0]->eNodeType == SNT_OPERATION &&
3706 poNode->papoSubExpr[0]->nOperation == SWQ_ISNULL &&
3707 poNode->papoSubExpr[0]->nSubExprCount == 1)
3708 {
3709 const swq_expr_node *poColumn = poNode->papoSubExpr[0]->papoSubExpr[0];
3710 if (poColumn->eNodeType == SNT_COLUMN &&
3711 poColumn->field_index < m_poFeatureDefn->GetFieldCount())
3712 {
3713 Constraint constraint;
3714 constraint.iField = poColumn->field_index;
3715 constraint.nOperation = SWQ_ISNOTNULL;
3716 AddConstraint(constraint);
3717 }
3718 }
3719 }
3720
3721 /************************************************************************/
3722 /* SetAttributeFilter() */
3723 /************************************************************************/
3724
3725 inline OGRErr OGRArrowLayer::SetAttributeFilter(const char *pszFilter)
3726 {
3727 m_asAttributeFilterConstraints.clear();
3728
3729 // When changing filters, we need to invalidate cached batches, as
3730 // PostFilterArrowArray() has potentially modified array contents
3731 if (m_poAttrQuery)
3732 InvalidateCachedBatches();
3733
3734 OGRErr eErr = OGRLayer::SetAttributeFilter(pszFilter);
3735 if (eErr != OGRERR_NONE)
3736 return eErr;
3737
3738 if (m_poAttrQuery != nullptr)
3739 {
3740 if (m_nUseOptimizedAttributeFilter < 0)
3741 {
3742 m_nUseOptimizedAttributeFilter = CPLTestBool(CPLGetConfigOption(
3743 ("OGR_" + GetDriverUCName() + "_OPTIMIZED_ATTRIBUTE_FILTER")
3744 .c_str(),
3745 "YES"));
3746 }
3747 if (m_nUseOptimizedAttributeFilter)
3748 {
3749 swq_expr_node *poNode =
3750 static_cast<swq_expr_node *>(m_poAttrQuery->GetSWQExpr());
3751 poNode->ReplaceBetweenByGEAndLERecurse();
3752 ExploreExprNode(poNode);
3753 ComputeConstraintsArrayIdx();
3754 }
3755 }
3756
3757 return OGRERR_NONE;
3758 }
3759
3760 /************************************************************************/
3761 /* ConstraintEvaluator() */
3762 /************************************************************************/
3763
3764 namespace
3765 {
3766 template <class T, class U> struct CompareGeneric
3767 {
3768 static inline bool get(int op, const T &val1, const U &val2)
3769 {
3770 switch (op)
3771 {
3772 case SWQ_LE:
3773 return val1 <= val2;
3774 case SWQ_LT:
3775 return val1 < val2;
3776 case SWQ_NE:
3777 return val1 != val2;
3778 case SWQ_EQ:
3779 return val1 == val2;
3780 case SWQ_GE:
3781 return val1 >= val2;
3782 case SWQ_GT:
3783 return val1 > val2;
3784 break;
3785 default:
3786 CPLAssert(false);
3787 }
3788 return true;
3789 }
3790 };
3791
3792 template <class T, class U> struct Compare
3793 {
3794 };
3795
3796 template <class T> struct Compare<T, T> : public CompareGeneric<T, T>
3797 {
3798 };
3799
3800 template <> struct Compare<int, GIntBig> : public CompareGeneric<int, GIntBig>
3801 {
3802 };
3803
3804 template <> struct Compare<double, GIntBig>
3805 {
3806 static inline bool get(int op, double val1, GIntBig val2)
3807 {
3808 return CompareGeneric<double, double>::get(op, val1,
3809 static_cast<double>(val2));
3810 }
3811 };
3812
3813 template <> struct Compare<GIntBig, int> : public CompareGeneric<GIntBig, int>
3814 {
3815 };
3816
3817 template <> struct Compare<double, int> : public CompareGeneric<double, int>
3818 {
3819 };
3820
3821 template <class T>
3822 static bool ConstraintEvaluator(const OGRArrowLayer::Constraint &constraint,
3823 const T value)
3824 {
3825 bool b = false;
3826 switch (constraint.eType)
3827 {
3828 case OGRArrowLayer::Constraint::Type::Integer:
3829 b = Compare<T, int>::get(constraint.nOperation, value,
3830 constraint.sValue.Integer);
3831 break;
3832 case OGRArrowLayer::Constraint::Type::Integer64:
3833 b = Compare<T, GIntBig>::get(constraint.nOperation, value,
3834 constraint.sValue.Integer64);
3835 break;
3836 case OGRArrowLayer::Constraint::Type::Real:
3837 b = Compare<double, double>::get(constraint.nOperation,
3838 static_cast<double>(value),
3839 constraint.sValue.Real);
3840 break;
3841 case OGRArrowLayer::Constraint::Type::String:
3842 b = Compare<std::string, std::string>::get(constraint.nOperation,
3843 std::to_string(value),
3844 constraint.osValue);
3845 break;
3846 }
3847 return b;
3848 }
3849
3850 inline bool CompareStr(int op, const std::string_view &val1,
3851 const std::string &val2)
3852 {
3853 if (op == SWQ_EQ)
3854 {
3855 return val1 == val2;
3856 }
3857 const int cmpRes = val2.compare(val1);
3858 switch (op)
3859 {
3860 case SWQ_LE:
3861 return cmpRes >= 0;
3862 case SWQ_LT:
3863 return cmpRes > 0;
3864 case SWQ_NE:
3865 return cmpRes != 0;
3866 // case SWQ_EQ: return cmpRes == 0;
3867 case SWQ_GE:
3868 return cmpRes <= 0;
3869 case SWQ_GT:
3870 return cmpRes < 0;
3871 break;
3872 default:
3873 CPLAssert(false);
3874 }
3875 return true;
3876 }
3877
3878 inline bool ConstraintEvaluator(const OGRArrowLayer::Constraint &constraint,
3879 const std::string_view &value)
3880 {
3881 return CompareStr(constraint.nOperation, value, constraint.osValue);
3882 }
3883
3884 } // namespace
3885
3886 /************************************************************************/
3887 /* SkipToNextFeatureDueToAttributeFilter() */
3888 /************************************************************************/
3889
3890 inline bool OGRArrowLayer::SkipToNextFeatureDueToAttributeFilter() const
3891 {
3892 for (const auto &constraint : m_asAttributeFilterConstraints)
3893 {
3894 if (constraint.iArrayIdx < 0)
3895 {
3896 if (constraint.iField ==
3897 m_poFeatureDefn->GetFieldCount() + SPF_FID &&
3898 m_osFIDColumn.empty())
3899 {
3900 if (!ConstraintEvaluator(constraint,
3901 static_cast<GIntBig>(m_nFeatureIdx)))
3902 {
3903 return true;
3904 }
3905 continue;
3906 }
3907 else
3908 {
3909 // can happen if ignoring a field that is needed by the
3910 // attribute filter. ComputeConstraintsArrayIdx() will have
3911 // warned about that
3912 continue;
3913 }
3914 }
3915
3916 const arrow::Array *array =
3917 m_poBatchColumns[constraint.iArrayIdx].get();
3918
3919 const bool bIsNull = array->IsNull(m_nIdxInBatch);
3920 if (constraint.nOperation == SWQ_ISNULL)
3921 {
3922 if (bIsNull)
3923 {
3924 continue;
3925 }
3926 return true;
3927 }
3928 else if (constraint.nOperation == SWQ_ISNOTNULL)
3929 {
3930 if (!bIsNull)
3931 {
3932 continue;
3933 }
3934 return true;
3935 }
3936 else if (bIsNull)
3937 {
3938 return true;
3939 }
3940
3941 switch (array->type_id())
3942 {
3943 case arrow::Type::NA:
3944 break;
3945
3946 case arrow::Type::BOOL:
3947 {
3948 const auto castArray =
3949 static_cast<const arrow::BooleanArray *>(array);
3950 if (!ConstraintEvaluator(
3951 constraint,
3952 static_cast<int>(castArray->Value(m_nIdxInBatch))))
3953 {
3954 return true;
3955 }
3956 break;
3957 }
3958 case arrow::Type::UINT8:
3959 {
3960 const auto castArray =
3961 static_cast<const arrow::UInt8Array *>(array);
3962 if (!ConstraintEvaluator(
3963 constraint,
3964 static_cast<int>(castArray->Value(m_nIdxInBatch))))
3965 {
3966 return true;
3967 }
3968 break;
3969 }
3970 case arrow::Type::INT8:
3971 {
3972 const auto castArray =
3973 static_cast<const arrow::Int8Array *>(array);
3974 if (!ConstraintEvaluator(
3975 constraint,
3976 static_cast<int>(castArray->Value(m_nIdxInBatch))))
3977 {
3978 return true;
3979 }
3980 break;
3981 }
3982 case arrow::Type::UINT16:
3983 {
3984 const auto castArray =
3985 static_cast<const arrow::UInt16Array *>(array);
3986 if (!ConstraintEvaluator(
3987 constraint,
3988 static_cast<int>(castArray->Value(m_nIdxInBatch))))
3989 {
3990 return true;
3991 }
3992 break;
3993 }
3994 case arrow::Type::INT16:
3995 {
3996 const auto castArray =
3997 static_cast<const arrow::Int16Array *>(array);
3998 if (!ConstraintEvaluator(
3999 constraint,
4000 static_cast<int>(castArray->Value(m_nIdxInBatch))))
4001 {
4002 return true;
4003 }
4004 break;
4005 }
4006 case arrow::Type::UINT32:
4007 {
4008 const auto castArray =
4009 static_cast<const arrow::UInt32Array *>(array);
4010 if (!ConstraintEvaluator(
4011 constraint,
4012 static_cast<GIntBig>(castArray->Value(m_nIdxInBatch))))
4013 {
4014 return true;
4015 }
4016 break;
4017 }
4018 case arrow::Type::INT32:
4019 {
4020 const auto castArray =
4021 static_cast<const arrow::Int32Array *>(array);
4022 if (!ConstraintEvaluator(constraint,
4023 castArray->Value(m_nIdxInBatch)))
4024 {
4025 return true;
4026 }
4027 break;
4028 }
4029 case arrow::Type::UINT64:
4030 {
4031 const auto castArray =
4032 static_cast<const arrow::UInt64Array *>(array);
4033 if (!ConstraintEvaluator(
4034 constraint,
4035 static_cast<double>(castArray->Value(m_nIdxInBatch))))
4036 {
4037 return true;
4038 }
4039 break;
4040 }
4041 case arrow::Type::INT64:
4042 {
4043 const auto castArray =
4044 static_cast<const arrow::Int64Array *>(array);
4045 if (!ConstraintEvaluator(
4046 constraint,
4047 static_cast<GIntBig>(castArray->Value(m_nIdxInBatch))))
4048 {
4049 return true;
4050 }
4051 break;
4052 }
4053 case arrow::Type::HALF_FLOAT:
4054 {
4055 const auto castArray =
4056 static_cast<const arrow::HalfFloatArray *>(array);
4057 const uint16_t nFloat16 = castArray->Value(m_nIdxInBatch);
4058 uint32_t nFloat32 = CPLHalfToFloat(nFloat16);
4059 float f;
4060 memcpy(&f, &nFloat32, sizeof(nFloat32));
4061 if (!ConstraintEvaluator(constraint, static_cast<double>(f)))
4062 {
4063 return true;
4064 }
4065 break;
4066 }
4067 case arrow::Type::FLOAT:
4068 {
4069 const auto castArray =
4070 static_cast<const arrow::FloatArray *>(array);
4071 if (!ConstraintEvaluator(
4072 constraint,
4073 static_cast<double>(castArray->Value(m_nIdxInBatch))))
4074 {
4075 return true;
4076 }
4077 break;
4078 }
4079 case arrow::Type::DOUBLE:
4080 {
4081 const auto castArray =
4082 static_cast<const arrow::DoubleArray *>(array);
4083 if (!ConstraintEvaluator(constraint,
4084 castArray->Value(m_nIdxInBatch)))
4085 {
4086 return true;
4087 }
4088 break;
4089 }
4090 case arrow::Type::STRING:
4091 {
4092 const auto castArray =
4093 static_cast<const arrow::StringArray *>(array);
4094 int out_length = 0;
4095 const uint8_t *data =
4096 castArray->GetValue(m_nIdxInBatch, &out_length);
4097 if (!ConstraintEvaluator(
4098 constraint,
4099 std::string_view(reinterpret_cast<const char *>(data),
4100 out_length)))
4101 {
4102 return true;
4103 }
4104 break;
4105 }
4106
4107 #if ARROW_VERSION_MAJOR >= 18
4108 case arrow::Type::DECIMAL32:
4109 {
4110 const auto castArray =
4111 static_cast<const arrow::Decimal32Array *>(array);
4112 if (!ConstraintEvaluator(
4113 constraint,
4114 CPLAtof(castArray->FormatValue(m_nIdxInBatch).c_str())))
4115 {
4116 return true;
4117 }
4118 break;
4119 }
4120
4121 case arrow::Type::DECIMAL64:
4122 {
4123 const auto castArray =
4124 static_cast<const arrow::Decimal64Array *>(array);
4125 if (!ConstraintEvaluator(
4126 constraint,
4127 CPLAtof(castArray->FormatValue(m_nIdxInBatch).c_str())))
4128 {
4129 return true;
4130 }
4131 break;
4132 }
4133 #endif
4134
4135 case arrow::Type::DECIMAL128:
4136 {
4137 const auto castArray =
4138 static_cast<const arrow::Decimal128Array *>(array);
4139 if (!ConstraintEvaluator(
4140 constraint,
4141 CPLAtof(castArray->FormatValue(m_nIdxInBatch).c_str())))
4142 {
4143 return true;
4144 }
4145 break;
4146 }
4147
4148 case arrow::Type::DECIMAL256:
4149 {
4150 const auto castArray =
4151 static_cast<const arrow::Decimal256Array *>(array);
4152 if (!ConstraintEvaluator(
4153 constraint,
4154 CPLAtof(castArray->FormatValue(m_nIdxInBatch).c_str())))
4155 {
4156 return true;
4157 }
4158 break;
4159 }
4160
4161 default:
4162 break;
4163 }
4164 }
4165 return false;
4166 }
4167
4168 /************************************************************************/
4169 /* SetBatch() */
4170 /************************************************************************/
4171
4172 inline void
4173 OGRArrowLayer::SetBatch(const std::shared_ptr<arrow::RecordBatch> &poBatch)
4174 {
4175 m_poBatch = poBatch;
4176 m_poBatchColumns.clear();
4177 m_poArrayWKB = nullptr;
4178 m_poArrayWKBLarge = nullptr;
4179 m_poArrayBBOX = nullptr;
4180 m_poArrayXMinDouble = nullptr;
4181 m_poArrayYMinDouble = nullptr;
4182 m_poArrayXMaxDouble = nullptr;
4183 m_poArrayYMaxDouble = nullptr;
4184 m_poArrayXMinFloat = nullptr;
4185 m_poArrayYMinFloat = nullptr;
4186 m_poArrayXMaxFloat = nullptr;
4187 m_poArrayYMaxFloat = nullptr;
4188
4189 if (m_poBatch)
4190 {
4191 m_poBatchColumns = m_poBatch->columns();
4192 if (!SanityCheckOfSetBatch())
4193 {
4194 m_poBatch.reset();
4195 m_poBatchColumns.clear();
4196 }
4197 }
4198
4199 if (m_poBatch && m_poFilterGeom && !m_bBaseArrowIgnoreSpatialFilterRect)
4200 {
4201 int iCol;
4202 if (m_bIgnoredFields)
4203 {
4204 iCol = m_anMapGeomFieldIndexToArrayIndex[m_iGeomFieldFilter];
4205 }
4206 else
4207 {
4208 iCol = m_anMapGeomFieldIndexToArrowColumn[m_iGeomFieldFilter];
4209 }
4210 if (iCol >= 0 &&
4211 m_aeGeomEncoding[m_iGeomFieldFilter] == OGRArrowGeomEncoding::WKB)
4212 {
4213 const arrow::Array *poArrayWKB =
4214 GetStorageArray(m_poBatchColumns[iCol].get());
4215 if (poArrayWKB->type_id() == arrow::Type::BINARY)
4216 m_poArrayWKB =
4217 static_cast<const arrow::BinaryArray *>(poArrayWKB);
4218 else
4219 {
4220 CPLAssert(poArrayWKB->type_id() == arrow::Type::LARGE_BINARY);
4221 m_poArrayWKBLarge =
4222 static_cast<const arrow::LargeBinaryArray *>(poArrayWKB);
4223 }
4224 }
4225
4226 if (iCol >= 0 &&
4227 CPLTestBool(CPLGetConfigOption(
4228 ("OGR_" + GetDriverUCName() + "_USE_BBOX").c_str(), "YES")))
4229 {
4230 const auto oIter =
4231 m_oMapGeomFieldIndexToGeomColBBOX.find(m_iGeomFieldFilter);
4232 if (oIter != m_oMapGeomFieldIndexToGeomColBBOX.end())
4233 {
4234 const int idx = m_bIgnoredFields ? oIter->second.iArrayIdx
4235 : oIter->second.iArrowCol;
4236 if (idx >= 0)
4237 {
4238 CPLAssert(static_cast<size_t>(idx) <
4239 m_poBatchColumns.size());
4240 m_poArrayBBOX = m_poBatchColumns[idx].get();
4241 CPLAssert(m_poArrayBBOX->type_id() == arrow::Type::STRUCT);
4242 const auto castArray =
4243 static_cast<const arrow::StructArray *>(m_poArrayBBOX);
4244 const auto &subArrays = castArray->fields();
4245 CPLAssert(
4246 static_cast<size_t>(oIter->second.iArrowSubfieldXMin) <
4247 subArrays.size());
4248 const auto xminArray =
4249 subArrays[oIter->second.iArrowSubfieldXMin].get();
4250 CPLAssert(
4251 static_cast<size_t>(oIter->second.iArrowSubfieldYMin) <
4252 subArrays.size());
4253 const auto yminArray =
4254 subArrays[oIter->second.iArrowSubfieldYMin].get();
4255 CPLAssert(
4256 static_cast<size_t>(oIter->second.iArrowSubfieldXMax) <
4257 subArrays.size());
4258 const auto xmaxArray =
4259 subArrays[oIter->second.iArrowSubfieldXMax].get();
4260 CPLAssert(
4261 static_cast<size_t>(oIter->second.iArrowSubfieldYMax) <
4262 subArrays.size());
4263 const auto ymaxArray =
4264 subArrays[oIter->second.iArrowSubfieldYMax].get();
4265 if (oIter->second.bIsFloat)
4266 {
4267 CPLAssert(xminArray->type_id() == arrow::Type::FLOAT);
4268 m_poArrayXMinFloat =
4269 static_cast<const arrow::FloatArray *>(xminArray);
4270 CPLAssert(yminArray->type_id() == arrow::Type::FLOAT);
4271 m_poArrayYMinFloat =
4272 static_cast<const arrow::FloatArray *>(yminArray);
4273 CPLAssert(xmaxArray->type_id() == arrow::Type::FLOAT);
4274 m_poArrayXMaxFloat =
4275 static_cast<const arrow::FloatArray *>(xmaxArray);
4276 CPLAssert(ymaxArray->type_id() == arrow::Type::FLOAT);
4277 m_poArrayYMaxFloat =
4278 static_cast<const arrow::FloatArray *>(ymaxArray);
4279 }
4280 else
4281 {
4282 CPLAssert(xminArray->type_id() == arrow::Type::DOUBLE);
4283 m_poArrayXMinDouble =
4284 static_cast<const arrow::DoubleArray *>(xminArray);
4285 CPLAssert(yminArray->type_id() == arrow::Type::DOUBLE);
4286 m_poArrayYMinDouble =
4287 static_cast<const arrow::DoubleArray *>(yminArray);
4288 CPLAssert(xmaxArray->type_id() == arrow::Type::DOUBLE);
4289 m_poArrayXMaxDouble =
4290 static_cast<const arrow::DoubleArray *>(xmaxArray);
4291 CPLAssert(ymaxArray->type_id() == arrow::Type::DOUBLE);
4292 m_poArrayYMaxDouble =
4293 static_cast<const arrow::DoubleArray *>(ymaxArray);
4294 }
4295 }
4296 }
4297 }
4298 }
4299 }
4300
4301 /************************************************************************/
4302 /* SanityCheckOfSetBatch() */
4303 /************************************************************************/
4304
4305 inline bool OGRArrowLayer::SanityCheckOfSetBatch() const
4306 {
4307 CPLAssert(m_poBatch);
4308
4309 // Sanity checks
4310 const auto nExpectedBatchColumns =
4311 (m_bIgnoredFields ? m_nExpectedBatchColumns : m_poSchema->num_fields());
4312 if (m_poBatch->num_columns() != nExpectedBatchColumns)
4313 {
4314 CPLError(
4315 CE_Failure, CPLE_AppDefined,
4316 "BUG! m_poBatch->num_columns() (=%d) != nExpectedBatchColumns(=%d)",
4317 static_cast<int>(m_poBatch->num_columns()),
4318 static_cast<int>(nExpectedBatchColumns));
4319 return false;
4320 }
4321
4322 #ifdef DEBUG
4323
4324 const auto &poColumns = m_poBatch->columns();
4325
4326 const auto &fields = m_poSchema->fields();
4327
4328 for (int i = 0; i < m_poFeatureDefn->GetFieldCount(); ++i)
4329 {
4330 int iCol;
4331 if (m_bIgnoredFields)
4332 {
4333 iCol = m_anMapFieldIndexToArrayIndex[i];
4334 if (iCol < 0)
4335 continue;
4336 }
4337 else
4338 {
4339 iCol = m_anMapFieldIndexToArrowColumn[i][0];
4340 }
4341 CPL_IGNORE_RET_VAL(iCol); // to make cppcheck happy
4342
4343 CPLAssert(iCol < static_cast<int>(poColumns.size()));
4344 CPLAssert(fields[m_anMapFieldIndexToArrowColumn[i][0]]->type()->id() ==
4345 poColumns[iCol]->type_id());
4346 }
4347
4348 for (int i = 0; i < m_poFeatureDefn->GetGeomFieldCount(); ++i)
4349 {
4350 int iCol;
4351 if (m_bIgnoredFields)
4352 {
4353 iCol = m_anMapGeomFieldIndexToArrayIndex[i];
4354 if (iCol < 0)
4355 continue;
4356 }
4357 else
4358 {
4359 iCol = m_anMapGeomFieldIndexToArrowColumn[i];
4360 }
4361 CPL_IGNORE_RET_VAL(iCol); // to make cppcheck happy
4362
4363 CPLAssert(iCol < static_cast<int>(poColumns.size()));
4364 CPLAssert(fields[m_anMapGeomFieldIndexToArrowColumn[i]]->type()->id() ==
4365 poColumns[iCol]->type_id());
4366 }
4367 #endif
4368
4369 return true;
4370 }
4371
4372 /************************************************************************/
4373 /* GetNextRawFeature() */
4374 /************************************************************************/
4375
4376 inline OGRFeature *OGRArrowLayer::GetNextRawFeature()
4377 {
4378 if (m_bEOF || !m_bSpatialFilterIntersectsLayerExtent)
4379 return nullptr;
4380
4381 if (m_poBatch == nullptr || m_nIdxInBatch == m_poBatch->num_rows())
4382 {
4383 m_bEOF = !ReadNextBatch();
4384 if (m_bEOF)
4385 return nullptr;
4386 }
4387
4388 // Evaluate spatial filter by computing the bounding box of each geometry
4389 // but without creating a OGRGeometry
4390 if (m_poFilterGeom && !m_bBaseArrowIgnoreSpatialFilterRect)
4391 {
4392 int iCol;
4393 if (m_bIgnoredFields)
4394 {
4395 iCol = m_anMapGeomFieldIndexToArrayIndex[m_iGeomFieldFilter];
4396 }
4397 else
4398 {
4399 iCol = m_anMapGeomFieldIndexToArrowColumn[m_iGeomFieldFilter];
4400 }
4401
4402 if (iCol >= 0 && (m_poArrayXMinFloat || m_poArrayXMinDouble))
4403 {
4404 OGREnvelope sEnvelopeSkipToNextFeatureDueToBBOX;
4405 const auto IntersectsBBOX =
4406 [this, &sEnvelopeSkipToNextFeatureDueToBBOX]()
4407 {
4408 if (m_poArrayXMinFloat &&
4409 !m_poArrayXMinFloat->IsNull(m_nIdxInBatch))
4410 {
4411 sEnvelopeSkipToNextFeatureDueToBBOX.MinX =
4412 m_poArrayXMinFloat->Value(m_nIdxInBatch);
4413 sEnvelopeSkipToNextFeatureDueToBBOX.MinY =
4414 m_poArrayYMinFloat->Value(m_nIdxInBatch);
4415 sEnvelopeSkipToNextFeatureDueToBBOX.MaxX =
4416 m_poArrayXMaxFloat->Value(m_nIdxInBatch);
4417 sEnvelopeSkipToNextFeatureDueToBBOX.MaxY =
4418 m_poArrayYMaxFloat->Value(m_nIdxInBatch);
4419 if (m_sFilterEnvelope.Intersects(
4420 sEnvelopeSkipToNextFeatureDueToBBOX))
4421 {
4422 return true;
4423 }
4424 }
4425 else if (m_poArrayXMinDouble &&
4426 !m_poArrayXMinDouble->IsNull(m_nIdxInBatch))
4427 {
4428 sEnvelopeSkipToNextFeatureDueToBBOX.MinX =
4429 m_poArrayXMinDouble->Value(m_nIdxInBatch);
4430 sEnvelopeSkipToNextFeatureDueToBBOX.MinY =
4431 m_poArrayYMinDouble->Value(m_nIdxInBatch);
4432 sEnvelopeSkipToNextFeatureDueToBBOX.MaxX =
4433 m_poArrayXMaxDouble->Value(m_nIdxInBatch);
4434 sEnvelopeSkipToNextFeatureDueToBBOX.MaxY =
4435 m_poArrayYMaxDouble->Value(m_nIdxInBatch);
4436 if (m_sFilterEnvelope.Intersects(
4437 sEnvelopeSkipToNextFeatureDueToBBOX))
4438 {
4439 return true;
4440 }
4441 }
4442 return false;
4443 };
4444
4445 while (true)
4446 {
4447 if (!m_poArrayBBOX->IsNull(m_nIdxInBatch) && IntersectsBBOX() &&
4448 (m_asAttributeFilterConstraints.empty() ||
4449 !SkipToNextFeatureDueToAttributeFilter()))
4450 {
4451 break;
4452 }
4453
4454 IncrFeatureIdx();
4455 m_nIdxInBatch++;
4456 if (m_nIdxInBatch == m_poBatch->num_rows())
4457 {
4458 m_bEOF = !ReadNextBatch();
4459 if (m_bEOF)
4460 return nullptr;
4461 }
4462 }
4463 }
4464 else if (iCol >= 0 && m_aeGeomEncoding[m_iGeomFieldFilter] ==
4465 OGRArrowGeomEncoding::WKB)
4466 {
4467 CPLAssert(m_poArrayWKB || m_poArrayWKBLarge);
4468 OGREnvelope sEnvelope;
4469
4470 while (true)
4471 {
4472 bool bMatchBBOX = false;
4473 if ((m_poArrayWKB && m_poArrayWKB->IsNull(m_nIdxInBatch)) ||
4474 (m_poArrayWKBLarge &&
4475 m_poArrayWKBLarge->IsNull(m_nIdxInBatch)))
4476 {
4477 // nothing to do
4478 }
4479 else
4480 {
4481 if (m_poArrayWKB)
4482 {
4483 int out_length = 0;
4484 const uint8_t *data =
4485 m_poArrayWKB->GetValue(m_nIdxInBatch, &out_length);
4486 if (OGRWKBGetBoundingBox(data, out_length, sEnvelope) &&
4487 m_sFilterEnvelope.Intersects(sEnvelope))
4488 {
4489 bMatchBBOX = true;
4490 }
4491 }
4492 else
4493 {
4494 CPLAssert(m_poArrayWKBLarge);
4495 int64_t out_length64 = 0;
4496 const uint8_t *data = m_poArrayWKBLarge->GetValue(
4497 m_nIdxInBatch, &out_length64);
4498 if (out_length64 < INT_MAX &&
4499 OGRWKBGetBoundingBox(data,
4500 static_cast<int>(out_length64),
4501 sEnvelope) &&
4502 m_sFilterEnvelope.Intersects(sEnvelope))
4503 {
4504 bMatchBBOX = true;
4505 }
4506 }
4507 }
4508 if (bMatchBBOX && (m_asAttributeFilterConstraints.empty() ||
4509 !SkipToNextFeatureDueToAttributeFilter()))
4510 {
4511 break;
4512 }
4513
4514 IncrFeatureIdx();
4515 m_nIdxInBatch++;
4516 if (m_nIdxInBatch == m_poBatch->num_rows())
4517 {
4518 m_bEOF = !ReadNextBatch();
4519 if (m_bEOF)
4520 return nullptr;
4521 }
4522 }
4523 }
4524 else if (iCol >= 0 &&
4525 m_aeGeomEncoding[m_iGeomFieldFilter] ==
4526 OGRArrowGeomEncoding::GEOARROW_FSL_MULTIPOLYGON)
4527 {
4528 const auto poGeomFieldDefn =
4529 m_poFeatureDefn->GetGeomFieldDefn(m_iGeomFieldFilter);
4530 const auto eGeomType = poGeomFieldDefn->GetType();
4531 const bool bHasZ = CPL_TO_BOOL(OGR_GT_HasZ(eGeomType));
4532 const bool bHasM = CPL_TO_BOOL(OGR_GT_HasM(eGeomType));
4533 const int nDim = 2 + (bHasZ ? 1 : 0) + (bHasM ? 1 : 0);
4534
4535 bool bReturnFeature;
4536 do
4537 {
4538 bReturnFeature = false;
4539 auto array = GetStorageArray(m_poBatchColumns[iCol].get());
4540 CPLAssert(array->type_id() == arrow::Type::LIST);
4541 auto listOfPartsArray =
4542 static_cast<const arrow::ListArray *>(array);
4543 CPLAssert(listOfPartsArray->values()->type_id() ==
4544 arrow::Type::LIST);
4545 auto listOfPartsValues =
4546 std::static_pointer_cast<arrow::ListArray>(
4547 listOfPartsArray->values());
4548 CPLAssert(listOfPartsValues->values()->type_id() ==
4549 arrow::Type::LIST);
4550 auto listOfRingsValues =
4551 std::static_pointer_cast<arrow::ListArray>(
4552 listOfPartsValues->values());
4553 CPLAssert(listOfRingsValues->values()->type_id() ==
4554 arrow::Type::FIXED_SIZE_LIST);
4555 auto listOfPointsValues =
4556 std::static_pointer_cast<arrow::FixedSizeListArray>(
4557 listOfRingsValues->values());
4558 CPLAssert(listOfPointsValues->values()->type_id() ==
4559 arrow::Type::DOUBLE);
4560 auto pointValues = std::static_pointer_cast<arrow::DoubleArray>(
4561 listOfPointsValues->values());
4562
4563 while (true)
4564 {
4565 bool bMatchBBOX = false;
4566 if (!listOfPartsArray->IsNull(m_nIdxInBatch))
4567 {
4568 OGREnvelope sEnvelope;
4569 const auto nParts =
4570 listOfPartsArray->value_length(m_nIdxInBatch);
4571 const auto nPartOffset =
4572 listOfPartsArray->value_offset(m_nIdxInBatch);
4573 for (auto j = decltype(nParts){0}; j < nParts; j++)
4574 {
4575 const auto nRings = listOfPartsValues->value_length(
4576 nPartOffset + j);
4577 const auto nRingOffset =
4578 listOfPartsValues->value_offset(nPartOffset +
4579 j);
4580 if (nRings >= 1)
4581 {
4582 const auto nPoints =
4583 listOfRingsValues->value_length(
4584 nRingOffset);
4585 const auto nPointOffset =
4586 listOfRingsValues->value_offset(
4587 nRingOffset) *
4588 nDim;
4589 const double *padfRawValue =
4590 pointValues->raw_values() + nPointOffset;
4591 for (auto l = decltype(nPoints){0}; l < nPoints;
4592 ++l)
4593 {
4594 sEnvelope.Merge(padfRawValue[nDim * l],
4595 padfRawValue[nDim * l + 1]);
4596 }
4597 // for bounding box, only the first ring matters
4598 }
4599 }
4600
4601 if (nParts != 0 &&
4602 m_sFilterEnvelope.Intersects(sEnvelope))
4603 {
4604 bMatchBBOX = true;
4605 }
4606 }
4607 if (bMatchBBOX &&
4608 (m_asAttributeFilterConstraints.empty() ||
4609 !SkipToNextFeatureDueToAttributeFilter()))
4610 {
4611 bReturnFeature = true;
4612 break;
4613 }
4614
4615 IncrFeatureIdx();
4616 m_nIdxInBatch++;
4617 if (m_nIdxInBatch == m_poBatch->num_rows())
4618 {
4619 m_bEOF = !ReadNextBatch();
4620 if (m_bEOF)
4621 return nullptr;
4622 break;
4623 }
4624 }
4625 } while (!bReturnFeature);
4626 }
4627 else if (iCol >= 0 && m_aeGeomEncoding[m_iGeomFieldFilter] ==
4628 OGRArrowGeomEncoding::GEOARROW_STRUCT_POINT)
4629 {
4630 bool bReturnFeature;
4631 do
4632 {
4633 bReturnFeature = false;
4634 auto array = GetStorageArray(m_poBatchColumns[iCol].get());
4635 CPLAssert(array->type_id() == arrow::Type::STRUCT);
4636 auto pointValues =
4637 static_cast<const arrow::StructArray *>(array);
4638 const auto &fields = pointValues->fields();
4639 const auto &fieldX = fields[0];
4640 CPLAssert(fieldX->type_id() == arrow::Type::DOUBLE);
4641 const auto fieldXDouble =
4642 static_cast<arrow::DoubleArray *>(fieldX.get());
4643 const auto &fieldY = fields[1];
4644 CPLAssert(fieldY->type_id() == arrow::Type::DOUBLE);
4645 const auto fieldYDouble =
4646 static_cast<arrow::DoubleArray *>(fieldY.get());
4647
4648 while (true)
4649 {
4650 bool bMatchBBOX = false;
4651 if (!array->IsNull(m_nIdxInBatch))
4652 {
4653 const double dfX = fieldXDouble->Value(m_nIdxInBatch);
4654 const double dfY = fieldYDouble->Value(m_nIdxInBatch);
4655 if (dfX >= m_sFilterEnvelope.MinX &&
4656 dfY >= m_sFilterEnvelope.MinY &&
4657 dfX <= m_sFilterEnvelope.MaxX &&
4658 dfY <= m_sFilterEnvelope.MaxY)
4659 {
4660 bMatchBBOX = true;
4661 }
4662 }
4663 if (bMatchBBOX &&
4664 (m_asAttributeFilterConstraints.empty() ||
4665 !SkipToNextFeatureDueToAttributeFilter()))
4666 {
4667 bReturnFeature = true;
4668 break;
4669 }
4670
4671 IncrFeatureIdx();
4672 m_nIdxInBatch++;
4673 if (m_nIdxInBatch == m_poBatch->num_rows())
4674 {
4675 m_bEOF = !ReadNextBatch();
4676 if (m_bEOF)
4677 return nullptr;
4678 break;
4679 }
4680 }
4681 } while (!bReturnFeature);
4682 }
4683 else if (iCol >= 0 &&
4684 m_aeGeomEncoding[m_iGeomFieldFilter] ==
4685 OGRArrowGeomEncoding::GEOARROW_STRUCT_LINESTRING)
4686 {
4687 bool bReturnFeature;
4688 do
4689 {
4690 bReturnFeature = false;
4691 auto array = GetStorageArray(m_poBatchColumns[iCol].get());
4692 CPLAssert(array->type_id() == arrow::Type::LIST);
4693 const auto listArray =
4694 static_cast<const arrow::ListArray *>(array);
4695 CPLAssert(listArray->values()->type_id() ==
4696 arrow::Type::STRUCT);
4697 auto pointValues = std::static_pointer_cast<arrow::StructArray>(
4698 listArray->values());
4699 const auto &fields = pointValues->fields();
4700 const auto &fieldX = fields[0];
4701 CPLAssert(fieldX->type_id() == arrow::Type::DOUBLE);
4702 const auto fieldXDouble =
4703 static_cast<arrow::DoubleArray *>(fieldX.get());
4704 const auto &fieldY = fields[1];
4705 CPLAssert(fieldY->type_id() == arrow::Type::DOUBLE);
4706 const auto fieldYDouble =
4707 static_cast<arrow::DoubleArray *>(fieldY.get());
4708
4709 while (true)
4710 {
4711 bool bMatchBBOX = false;
4712 if (!listArray->IsNull(m_nIdxInBatch))
4713 {
4714 OGREnvelope sEnvelope;
4715 const auto nPoints =
4716 listArray->value_length(m_nIdxInBatch);
4717 const auto nPointOffset =
4718 listArray->value_offset(m_nIdxInBatch);
4719 if (nPoints > 0)
4720 {
4721 const double *padfRawXValue =
4722 fieldXDouble->raw_values() + nPointOffset;
4723 const double *padfRawYValue =
4724 fieldYDouble->raw_values() + nPointOffset;
4725 for (auto l = decltype(nPoints){0}; l < nPoints;
4726 ++l)
4727 {
4728 sEnvelope.Merge(padfRawXValue[l],
4729 padfRawYValue[l]);
4730 }
4731 if (m_sFilterEnvelope.Intersects(sEnvelope))
4732 {
4733 bMatchBBOX = true;
4734 }
4735 }
4736 }
4737 if (bMatchBBOX &&
4738 (m_asAttributeFilterConstraints.empty() ||
4739 !SkipToNextFeatureDueToAttributeFilter()))
4740 {
4741 bReturnFeature = true;
4742 break;
4743 }
4744
4745 IncrFeatureIdx();
4746 m_nIdxInBatch++;
4747 if (m_nIdxInBatch == m_poBatch->num_rows())
4748 {
4749 m_bEOF = !ReadNextBatch();
4750 if (m_bEOF)
4751 return nullptr;
4752 break;
4753 }
4754 }
4755 } while (!bReturnFeature);
4756 }
4757 else if (iCol >= 0 && m_aeGeomEncoding[m_iGeomFieldFilter] ==
4758 OGRArrowGeomEncoding::GEOARROW_STRUCT_POLYGON)
4759 {
4760 bool bReturnFeature;
4761 do
4762 {
4763 bReturnFeature = false;
4764 auto array = GetStorageArray(m_poBatchColumns[iCol].get());
4765 CPLAssert(array->type_id() == arrow::Type::LIST);
4766 const auto listOfRingsArray =
4767 static_cast<const arrow::ListArray *>(array);
4768 CPLAssert(listOfRingsArray->values()->type_id() ==
4769 arrow::Type::LIST);
4770 const auto listOfRingsValues =
4771 std::static_pointer_cast<arrow::ListArray>(
4772 listOfRingsArray->values());
4773 CPLAssert(listOfRingsValues->values()->type_id() ==
4774 arrow::Type::STRUCT);
4775 auto pointValues = std::static_pointer_cast<arrow::StructArray>(
4776 listOfRingsValues->values());
4777 const auto &fields = pointValues->fields();
4778 const auto &fieldX = fields[0];
4779 CPLAssert(fieldX->type_id() == arrow::Type::DOUBLE);
4780 const auto fieldXDouble =
4781 static_cast<arrow::DoubleArray *>(fieldX.get());
4782 const auto &fieldY = fields[1];
4783 CPLAssert(fieldY->type_id() == arrow::Type::DOUBLE);
4784 const auto fieldYDouble =
4785 static_cast<arrow::DoubleArray *>(fieldY.get());
4786
4787 while (true)
4788 {
4789 bool bMatchBBOX = false;
4790 if (!listOfRingsArray->IsNull(m_nIdxInBatch))
4791 {
4792 OGREnvelope sEnvelope;
4793 const auto nRings =
4794 listOfRingsArray->value_length(m_nIdxInBatch);
4795 const auto nRingOffset =
4796 listOfRingsArray->value_offset(m_nIdxInBatch);
4797 if (nRings >= 1)
4798 {
4799 const auto nPoints =
4800 listOfRingsValues->value_length(nRingOffset);
4801 const auto nPointOffset =
4802 listOfRingsValues->value_offset(nRingOffset);
4803 const double *padfRawXValue =
4804 fieldXDouble->raw_values() + nPointOffset;
4805 const double *padfRawYValue =
4806 fieldYDouble->raw_values() + nPointOffset;
4807 for (auto l = decltype(nPoints){0}; l < nPoints;
4808 ++l)
4809 {
4810 sEnvelope.Merge(padfRawXValue[l],
4811 padfRawYValue[l]);
4812 }
4813 // for bounding box, only the first ring matters
4814
4815 if (m_sFilterEnvelope.Intersects(sEnvelope))
4816 {
4817 bMatchBBOX = true;
4818 }
4819 }
4820 }
4821 if (bMatchBBOX &&
4822 (m_asAttributeFilterConstraints.empty() ||
4823 !SkipToNextFeatureDueToAttributeFilter()))
4824 {
4825 bReturnFeature = true;
4826 break;
4827 }
4828
4829 IncrFeatureIdx();
4830 m_nIdxInBatch++;
4831 if (m_nIdxInBatch == m_poBatch->num_rows())
4832 {
4833 m_bEOF = !ReadNextBatch();
4834 if (m_bEOF)
4835 return nullptr;
4836 break;
4837 }
4838 }
4839 } while (!bReturnFeature);
4840 }
4841 else if (iCol >= 0 &&
4842 m_aeGeomEncoding[m_iGeomFieldFilter] ==
4843 OGRArrowGeomEncoding::GEOARROW_STRUCT_MULTIPOINT)
4844 {
4845 bool bReturnFeature;
4846 do
4847 {
4848 bReturnFeature = false;
4849 auto array = GetStorageArray(m_poBatchColumns[iCol].get());
4850 CPLAssert(array->type_id() == arrow::Type::LIST);
4851 const auto listArray =
4852 static_cast<const arrow::ListArray *>(array);
4853 CPLAssert(listArray->values()->type_id() ==
4854 arrow::Type::STRUCT);
4855 auto pointValues = std::static_pointer_cast<arrow::StructArray>(
4856 listArray->values());
4857 const auto &fields = pointValues->fields();
4858 const auto &fieldX = fields[0];
4859 CPLAssert(fieldX->type_id() == arrow::Type::DOUBLE);
4860 const auto fieldXDouble =
4861 static_cast<arrow::DoubleArray *>(fieldX.get());
4862 const auto &fieldY = fields[1];
4863 CPLAssert(fieldY->type_id() == arrow::Type::DOUBLE);
4864 const auto fieldYDouble =
4865 static_cast<arrow::DoubleArray *>(fieldY.get());
4866
4867 while (true)
4868 {
4869 bool bMatchBBOX = false;
4870 if (!listArray->IsNull(m_nIdxInBatch))
4871 {
4872 const auto nPoints =
4873 listArray->value_length(m_nIdxInBatch);
4874 const auto nPointOffset =
4875 listArray->value_offset(m_nIdxInBatch);
4876 if (nPoints > 0)
4877 {
4878 const double *padfRawXValue =
4879 fieldXDouble->raw_values() + nPointOffset;
4880 const double *padfRawYValue =
4881 fieldYDouble->raw_values() + nPointOffset;
4882 for (auto l = decltype(nPoints){0}; l < nPoints;
4883 ++l)
4884 {
4885 if (padfRawXValue[l] >=
4886 m_sFilterEnvelope.MinX &&
4887 padfRawYValue[l] >=
4888 m_sFilterEnvelope.MinY &&
4889 padfRawXValue[l] <=
4890 m_sFilterEnvelope.MaxX &&
4891 padfRawYValue[l] <= m_sFilterEnvelope.MaxY)
4892 {
4893 bMatchBBOX = true;
4894 break;
4895 }
4896 }
4897 }
4898 }
4899 if (bMatchBBOX &&
4900 (m_asAttributeFilterConstraints.empty() ||
4901 !SkipToNextFeatureDueToAttributeFilter()))
4902 {
4903 bReturnFeature = true;
4904 break;
4905 }
4906
4907 IncrFeatureIdx();
4908 m_nIdxInBatch++;
4909 if (m_nIdxInBatch == m_poBatch->num_rows())
4910 {
4911 m_bEOF = !ReadNextBatch();
4912 if (m_bEOF)
4913 return nullptr;
4914 break;
4915 }
4916 }
4917 } while (!bReturnFeature);
4918 }
4919 else if (iCol >= 0 &&
4920 m_aeGeomEncoding[m_iGeomFieldFilter] ==
4921 OGRArrowGeomEncoding::GEOARROW_STRUCT_MULTILINESTRING)
4922 {
4923 bool bReturnFeature;
4924 do
4925 {
4926 bReturnFeature = false;
4927 auto array = GetStorageArray(m_poBatchColumns[iCol].get());
4928 CPLAssert(array->type_id() == arrow::Type::LIST);
4929 auto listOfPartsArray =
4930 static_cast<const arrow::ListArray *>(array);
4931 CPLAssert(listOfPartsArray->values()->type_id() ==
4932 arrow::Type::LIST);
4933 auto listOfPartsValues =
4934 std::static_pointer_cast<arrow::ListArray>(
4935 listOfPartsArray->values());
4936 CPLAssert(listOfPartsValues->values()->type_id() ==
4937 arrow::Type::STRUCT);
4938 auto pointValues = std::static_pointer_cast<arrow::StructArray>(
4939 listOfPartsValues->values());
4940 const auto &fields = pointValues->fields();
4941 const auto &fieldX = fields[0];
4942 CPLAssert(fieldX->type_id() == arrow::Type::DOUBLE);
4943 const auto fieldXDouble =
4944 static_cast<arrow::DoubleArray *>(fieldX.get());
4945 const auto &fieldY = fields[1];
4946 CPLAssert(fieldY->type_id() == arrow::Type::DOUBLE);
4947 const auto fieldYDouble =
4948 static_cast<arrow::DoubleArray *>(fieldY.get());
4949
4950 while (true)
4951 {
4952 bool bMatchBBOX = false;
4953 if (!listOfPartsArray->IsNull(m_nIdxInBatch))
4954 {
4955 const auto nParts =
4956 listOfPartsArray->value_length(m_nIdxInBatch);
4957 const auto nPartOffset =
4958 listOfPartsArray->value_offset(m_nIdxInBatch);
4959 for (auto j = decltype(nParts){0};
4960 j < nParts && !bMatchBBOX; j++)
4961 {
4962 OGREnvelope sEnvelope;
4963 const auto nPoints =
4964 listOfPartsValues->value_length(nPartOffset +
4965 j);
4966 const auto nPointOffset =
4967 listOfPartsValues->value_offset(nPartOffset +
4968 j);
4969 const double *padfRawXValue =
4970 fieldXDouble->raw_values() + nPointOffset;
4971 const double *padfRawYValue =
4972 fieldYDouble->raw_values() + nPointOffset;
4973 for (auto l = decltype(nPoints){0}; l < nPoints;
4974 ++l)
4975 {
4976 sEnvelope.Merge(padfRawXValue[l],
4977 padfRawYValue[l]);
4978 }
4979
4980 if (m_sFilterEnvelope.Intersects(sEnvelope))
4981 {
4982 bMatchBBOX = true;
4983 }
4984 }
4985 }
4986 if (bMatchBBOX &&
4987 (m_asAttributeFilterConstraints.empty() ||
4988 !SkipToNextFeatureDueToAttributeFilter()))
4989 {
4990 bReturnFeature = true;
4991 break;
4992 }
4993
4994 IncrFeatureIdx();
4995 m_nIdxInBatch++;
4996 if (m_nIdxInBatch == m_poBatch->num_rows())
4997 {
4998 m_bEOF = !ReadNextBatch();
4999 if (m_bEOF)
5000 return nullptr;
5001 break;
5002 }
5003 }
5004 } while (!bReturnFeature);
5005 }
5006 else if (iCol >= 0 &&
5007 m_aeGeomEncoding[m_iGeomFieldFilter] ==
5008 OGRArrowGeomEncoding::GEOARROW_STRUCT_MULTIPOLYGON)
5009 {
5010 bool bReturnFeature;
5011 do
5012 {
5013 bReturnFeature = false;
5014 auto array = GetStorageArray(m_poBatchColumns[iCol].get());
5015 CPLAssert(array->type_id() == arrow::Type::LIST);
5016 auto listOfPartsArray =
5017 static_cast<const arrow::ListArray *>(array);
5018 CPLAssert(listOfPartsArray->values()->type_id() ==
5019 arrow::Type::LIST);
5020 auto listOfPartsValues =
5021 std::static_pointer_cast<arrow::ListArray>(
5022 listOfPartsArray->values());
5023 CPLAssert(listOfPartsValues->values()->type_id() ==
5024 arrow::Type::LIST);
5025 auto listOfRingsValues =
5026 std::static_pointer_cast<arrow::ListArray>(
5027 listOfPartsValues->values());
5028 CPLAssert(listOfRingsValues->values()->type_id() ==
5029 arrow::Type::STRUCT);
5030 auto pointValues = std::static_pointer_cast<arrow::StructArray>(
5031 listOfRingsValues->values());
5032 const auto &fields = pointValues->fields();
5033 const auto &fieldX = fields[0];
5034 CPLAssert(fieldX->type_id() == arrow::Type::DOUBLE);
5035 const auto fieldXDouble =
5036 static_cast<arrow::DoubleArray *>(fieldX.get());
5037 const auto &fieldY = fields[1];
5038 CPLAssert(fieldY->type_id() == arrow::Type::DOUBLE);
5039 const auto fieldYDouble =
5040 static_cast<arrow::DoubleArray *>(fieldY.get());
5041
5042 while (true)
5043 {
5044 bool bMatchBBOX = false;
5045 if (!listOfPartsArray->IsNull(m_nIdxInBatch))
5046 {
5047 const auto nParts =
5048 listOfPartsArray->value_length(m_nIdxInBatch);
5049 const auto nPartOffset =
5050 listOfPartsArray->value_offset(m_nIdxInBatch);
5051 for (auto j = decltype(nParts){0};
5052 j < nParts && !bMatchBBOX; j++)
5053 {
5054 OGREnvelope sEnvelope;
5055 const auto nRings = listOfPartsValues->value_length(
5056 nPartOffset + j);
5057 const auto nRingOffset =
5058 listOfPartsValues->value_offset(nPartOffset +
5059 j);
5060 if (nRings >= 1)
5061 {
5062 const auto nPoints =
5063 listOfRingsValues->value_length(
5064 nRingOffset);
5065 const auto nPointOffset =
5066 listOfRingsValues->value_offset(
5067 nRingOffset);
5068 const double *padfRawXValue =
5069 fieldXDouble->raw_values() + nPointOffset;
5070 const double *padfRawYValue =
5071 fieldYDouble->raw_values() + nPointOffset;
5072 for (auto l = decltype(nPoints){0}; l < nPoints;
5073 ++l)
5074 {
5075 sEnvelope.Merge(padfRawXValue[l],
5076 padfRawYValue[l]);
5077 }
5078
5079 if (m_sFilterEnvelope.Intersects(sEnvelope))
5080 {
5081 bMatchBBOX = true;
5082 }
5083 // for bounding box, only the first ring matters
5084 }
5085 }
5086 }
5087 if (bMatchBBOX &&
5088 (m_asAttributeFilterConstraints.empty() ||
5089 !SkipToNextFeatureDueToAttributeFilter()))
5090 {
5091 bReturnFeature = true;
5092 break;
5093 }
5094
5095 IncrFeatureIdx();
5096 m_nIdxInBatch++;
5097 if (m_nIdxInBatch == m_poBatch->num_rows())
5098 {
5099 m_bEOF = !ReadNextBatch();
5100 if (m_bEOF)
5101 return nullptr;
5102 break;
5103 }
5104 }
5105 } while (!bReturnFeature);
5106 }
5107 else if (iCol >= 0)
5108 {
5109 auto array = GetStorageArray(m_poBatchColumns[iCol].get());
5110 while (true)
5111 {
5112 bool bMatchBBOX = false;
5113
5114 auto poGeometry = std::unique_ptr<OGRGeometry>(
5115 ReadGeometry(m_iGeomFieldFilter, array, m_nIdxInBatch));
5116 if (poGeometry && !poGeometry->IsEmpty())
5117 {
5118 OGREnvelope sEnvelope;
5119 poGeometry->getEnvelope(&sEnvelope);
5120 if (m_sFilterEnvelope.Intersects(sEnvelope))
5121 {
5122 bMatchBBOX = true;
5123 }
5124 }
5125 if (bMatchBBOX && (m_asAttributeFilterConstraints.empty() ||
5126 !SkipToNextFeatureDueToAttributeFilter()))
5127 {
5128 break;
5129 }
5130
5131 IncrFeatureIdx();
5132 m_nIdxInBatch++;
5133 if (m_nIdxInBatch == m_poBatch->num_rows())
5134 {
5135 m_bEOF = !ReadNextBatch();
5136 if (m_bEOF)
5137 return nullptr;
5138 array = GetStorageArray(m_poBatchColumns[iCol].get());
5139 }
5140 }
5141 }
5142 }
5143
5144 else if (!m_asAttributeFilterConstraints.empty())
5145 {
5146 while (true)
5147 {
5148 if (!SkipToNextFeatureDueToAttributeFilter())
5149 {
5150 break;
5151 }
5152
5153 IncrFeatureIdx();
5154 m_nIdxInBatch++;
5155 if (m_nIdxInBatch == m_poBatch->num_rows())
5156 {
5157 m_bEOF = !ReadNextBatch();
5158 if (m_bEOF)
5159 return nullptr;
5160 }
5161 }
5162 }
5163
5164 auto poFeature = ReadFeature(m_nIdxInBatch, m_poBatchColumns);
5165
5166 if (m_iFIDArrowColumn < 0)
5167 poFeature->SetFID(m_nFeatureIdx);
5168
5169 IncrFeatureIdx();
5170 m_nIdxInBatch++;
5171
5172 return poFeature;
5173 }
5174
5175 /************************************************************************/
5176 /* GetExtentFromMetadata() */
5177 /************************************************************************/
5178
5179 inline OGRErr
5180 OGRArrowLayer::GetExtentFromMetadata(const CPLJSONObject &oJSONDef,
5181 OGREnvelope3D *psExtent)
5182 {
5183 const auto oBBox = oJSONDef.GetArray("bbox");
5184 if (oBBox.IsValid() && oBBox.Size() == 4)
5185 {
5186 psExtent->MinX = oBBox[0].ToDouble();
5187 psExtent->MinY = oBBox[1].ToDouble();
5188 psExtent->MinZ = std::numeric_limits<double>::infinity();
5189 psExtent->MaxX = oBBox[2].ToDouble();
5190 psExtent->MaxY = oBBox[3].ToDouble();
5191 psExtent->MaxZ = -std::numeric_limits<double>::infinity();
5192 if (psExtent->MinX <= psExtent->MaxX)
5193 return OGRERR_NONE;
5194 }
5195 else if (oBBox.IsValid() && oBBox.Size() == 6)
5196 {
5197 psExtent->MinX = oBBox[0].ToDouble();
5198 psExtent->MinY = oBBox[1].ToDouble();
5199 psExtent->MinZ = oBBox[2].ToDouble();
5200 psExtent->MaxX = oBBox[3].ToDouble();
5201 psExtent->MaxY = oBBox[4].ToDouble();
5202 psExtent->MaxZ = oBBox[5].ToDouble();
5203 if (psExtent->MinX <= psExtent->MaxX)
5204 return OGRERR_NONE;
5205 }
5206 return OGRERR_FAILURE;
5207 }
5208
5209 /************************************************************************/
5210 /* ISetSpatialFilter() */
5211 /************************************************************************/
5212
5213 inline OGRErr OGRArrowLayer::ISetSpatialFilter(int iGeomField,
5214 const OGRGeometry *poGeomIn)
5215
5216 {
5217 // When changing filters, we need to invalidate cached batches, as
5218 // PostFilterArrowArray() has potentially modified array contents
5219 if (m_poFilterGeom)
5220 InvalidateCachedBatches();
5221
5222 m_bSpatialFilterIntersectsLayerExtent = true;
5223 if (iGeomField < GetLayerDefn()->GetGeomFieldCount())
5224 {
5225 m_iGeomFieldFilter = iGeomField;
5226 if (InstallFilter(poGeomIn))
5227 ResetReading();
5228 if (m_poFilterGeom != nullptr)
5229 {
5230 OGREnvelope sLayerExtent;
5231 if (FastGetExtent(iGeomField, &sLayerExtent))
5232 {
5233 m_bSpatialFilterIntersectsLayerExtent =
5234 CPL_TO_BOOL(m_sFilterEnvelope.Intersects(sLayerExtent));
5235 }
5236 }
5237 }
5238
5239 SetBatch(m_poBatch);
5240 return OGRERR_NONE;
5241 }
5242
5243 /************************************************************************/
5244 /* FastGetExtent() */
5245 /************************************************************************/
5246
5247 inline bool OGRArrowLayer::FastGetExtent(int iGeomField,
5248 OGREnvelope *psExtent) const
5249 {
5250 {
5251 const auto oIter = m_oMapExtents.find(iGeomField);
5252 if (oIter != m_oMapExtents.end())
5253 {
5254 *psExtent = oIter->second;
5255 return true;
5256 }
5257 }
5258
5259 const char *pszGeomFieldName =
5260 m_poFeatureDefn->GetGeomFieldDefn(iGeomField)->GetNameRef();
5261 const auto oIter = m_oMapGeometryColumns.find(pszGeomFieldName);
5262 if (oIter != m_oMapGeometryColumns.end() &&
5263 CPLTestBool(CPLGetConfigOption(
5264 ("OGR_" + GetDriverUCName() + "_USE_BBOX").c_str(), "YES")))
5265 {
5266 const auto &oJSONDef = oIter->second;
5267 OGREnvelope3D sEnvelope3D;
5268 if (GetExtentFromMetadata(oJSONDef, &sEnvelope3D) == OGRERR_NONE)
5269 {
5270 *psExtent = sEnvelope3D;
5271 return true;
5272 }
5273 }
5274 return false;
5275 }
5276
5277 /************************************************************************/
5278 /* IGetExtent() */
5279 /************************************************************************/
5280
5281 inline OGRErr OGRArrowLayer::IGetExtent(int iGeomField, OGREnvelope *psExtent,
5282 bool bForce)
5283 {
5284 if (FastGetExtent(iGeomField, psExtent))
5285 {
5286 return OGRERR_NONE;
5287 }
5288
5289 if (!bForce && !CanRunNonForcedGetExtent())
5290 {
5291 return OGRERR_FAILURE;
5292 }
5293
5294 int iCol;
5295 if (m_bIgnoredFields)
5296 {
5297 iCol = m_anMapGeomFieldIndexToArrayIndex[iGeomField];
5298 }
5299 else
5300 {
5301 iCol = m_anMapGeomFieldIndexToArrowColumn[iGeomField];
5302 }
5303 if (iCol < 0)
5304 {
5305 return OGRERR_FAILURE;
5306 }
5307
5308 if (m_aeGeomEncoding[iGeomField] == OGRArrowGeomEncoding::WKB)
5309 {
5310 ResetReading();
5311 if (m_poBatch == nullptr)
5312 {
5313 m_bEOF = !ReadNextBatch();
5314 if (m_bEOF)
5315 return OGRERR_FAILURE;
5316 }
5317 *psExtent = OGREnvelope();
5318
5319 auto array = GetStorageArray(m_poBatchColumns[iCol].get());
5320 const arrow::BinaryArray *smallArray = nullptr;
5321 const arrow::LargeBinaryArray *largeArray = nullptr;
5322 if (array->type_id() == arrow::Type::BINARY)
5323 smallArray = static_cast<const arrow::BinaryArray *>(array);
5324 else
5325 {
5326 CPLAssert(array->type_id() == arrow::Type::LARGE_BINARY);
5327 largeArray = static_cast<const arrow::LargeBinaryArray *>(array);
5328 }
5329 OGREnvelope sEnvelope;
5330 while (true)
5331 {
5332 if (!array->IsNull(m_nIdxInBatch))
5333 {
5334 if (smallArray)
5335 {
5336 int out_length = 0;
5337 const uint8_t *data =
5338 smallArray->GetValue(m_nIdxInBatch, &out_length);
5339 if (OGRWKBGetBoundingBox(data, out_length, sEnvelope))
5340 {
5341 psExtent->Merge(sEnvelope);
5342 }
5343 }
5344 else
5345 {
5346 assert(largeArray);
5347 int64_t out_length = 0;
5348 const uint8_t *data =
5349 largeArray->GetValue(m_nIdxInBatch, &out_length);
5350 if (out_length < INT_MAX &&
5351 OGRWKBGetBoundingBox(data, static_cast<int>(out_length),
5352 sEnvelope))
5353 {
5354 psExtent->Merge(sEnvelope);
5355 }
5356 }
5357 }
5358
5359 m_nIdxInBatch++;
5360 if (m_nIdxInBatch == m_poBatch->num_rows())
5361 {
5362 m_bEOF = !ReadNextBatch();
5363 if (m_bEOF)
5364 {
5365 ResetReading();
5366 if (psExtent->IsInit())
5367 {
5368 m_oMapExtents[iGeomField] = *psExtent;
5369 return OGRERR_NONE;
5370 }
5371 return OGRERR_FAILURE;
5372 }
5373 array = GetStorageArray(m_poBatchColumns[iCol].get());
5374 if (array->type_id() == arrow::Type::BINARY)
5375 smallArray = static_cast<const arrow::BinaryArray *>(array);
5376 else
5377 {
5378 CPLAssert(array->type_id() == arrow::Type::LARGE_BINARY);
5379 largeArray =
5380 static_cast<const arrow::LargeBinaryArray *>(array);
5381 }
5382 }
5383 }
5384 }
5385 else if (m_aeGeomEncoding[iGeomField] ==
5386 OGRArrowGeomEncoding::GEOARROW_FSL_MULTIPOLYGON)
5387 {
5388 ResetReading();
5389 if (m_poBatch == nullptr)
5390 {
5391 m_bEOF = !ReadNextBatch();
5392 if (m_bEOF)
5393 return OGRERR_FAILURE;
5394 }
5395 *psExtent = OGREnvelope();
5396
5397 const auto poGeomFieldDefn =
5398 m_poFeatureDefn->GetGeomFieldDefn(iGeomField);
5399 const auto eGeomType = poGeomFieldDefn->GetType();
5400 const bool bHasZ = CPL_TO_BOOL(OGR_GT_HasZ(eGeomType));
5401 const bool bHasM = CPL_TO_BOOL(OGR_GT_HasM(eGeomType));
5402 const int nDim = 2 + (bHasZ ? 1 : 0) + (bHasM ? 1 : 0);
5403
5404 begin_multipolygon:
5405 auto array = m_poBatchColumns[iCol].get();
5406 CPLAssert(array->type_id() == arrow::Type::LIST);
5407 auto listOfPartsArray = static_cast<const arrow::ListArray *>(array);
5408 CPLAssert(listOfPartsArray->values()->type_id() == arrow::Type::LIST);
5409 auto listOfPartsValues = std::static_pointer_cast<arrow::ListArray>(
5410 listOfPartsArray->values());
5411 CPLAssert(listOfPartsValues->values()->type_id() == arrow::Type::LIST);
5412 auto listOfRingsValues = std::static_pointer_cast<arrow::ListArray>(
5413 listOfPartsValues->values());
5414 CPLAssert(listOfRingsValues->values()->type_id() ==
5415 arrow::Type::FIXED_SIZE_LIST);
5416 auto listOfPointsValues =
5417 std::static_pointer_cast<arrow::FixedSizeListArray>(
5418 listOfRingsValues->values());
5419 CPLAssert(listOfPointsValues->values()->type_id() ==
5420 arrow::Type::DOUBLE);
5421 auto pointValues = std::static_pointer_cast<arrow::DoubleArray>(
5422 listOfPointsValues->values());
5423
5424 while (true)
5425 {
5426 if (!listOfPartsArray->IsNull(m_nIdxInBatch))
5427 {
5428 const auto nParts =
5429 listOfPartsArray->value_length(m_nIdxInBatch);
5430 const auto nPartOffset =
5431 listOfPartsArray->value_offset(m_nIdxInBatch);
5432 for (auto j = decltype(nParts){0}; j < nParts; j++)
5433 {
5434 const auto nRings =
5435 listOfPartsValues->value_length(nPartOffset + j);
5436 const auto nRingOffset =
5437 listOfPartsValues->value_offset(nPartOffset + j);
5438 if (nRings >= 1)
5439 {
5440 const auto nPoints =
5441 listOfRingsValues->value_length(nRingOffset);
5442 const auto nPointOffset =
5443 listOfRingsValues->value_offset(nRingOffset) * nDim;
5444 const double *padfRawValue =
5445 pointValues->raw_values() + nPointOffset;
5446 for (auto l = decltype(nPoints){0}; l < nPoints; ++l)
5447 {
5448 psExtent->Merge(padfRawValue[nDim * l],
5449 padfRawValue[nDim * l + 1]);
5450 }
5451 // for bounding box, only the first ring matters
5452 }
5453 }
5454 }
5455
5456 m_nIdxInBatch++;
5457 if (m_nIdxInBatch == m_poBatch->num_rows())
5458 {
5459 m_bEOF = !ReadNextBatch();
5460 if (m_bEOF)
5461 {
5462 ResetReading();
5463 if (psExtent->IsInit())
5464 {
5465 m_oMapExtents[iGeomField] = *psExtent;
5466 return OGRERR_NONE;
5467 }
5468 return OGRERR_FAILURE;
5469 }
5470 goto begin_multipolygon;
5471 }
5472 }
5473 }
5474
5475 return OGRLayer::IGetExtent(iGeomField, psExtent, bForce);
5476 }
5477
5478 /************************************************************************/
5479 /* FastGetExtent3D() */
5480 /************************************************************************/
5481
5482 inline bool OGRArrowLayer::FastGetExtent3D(int iGeomField,
5483 OGREnvelope3D *psExtent) const
5484 {
5485 {
5486 const auto oIter = m_oMapExtents3D.find(iGeomField);
5487 if (oIter != m_oMapExtents3D.end())
5488 {
5489 *psExtent = oIter->second;
5490 return true;
5491 }
5492 }
5493
5494 const char *pszGeomFieldName =
5495 m_poFeatureDefn->GetGeomFieldDefn(iGeomField)->GetNameRef();
5496 const auto oIter = m_oMapGeometryColumns.find(pszGeomFieldName);
5497 if (oIter != m_oMapGeometryColumns.end() &&
5498 CPLTestBool(CPLGetConfigOption(
5499 ("OGR_" + GetDriverUCName() + "_USE_BBOX").c_str(), "YES")))
5500 {
5501 const auto &oJSONDef = oIter->second;
5502 if (GetExtentFromMetadata(oJSONDef, psExtent) == OGRERR_NONE &&
5503 psExtent->Is3D())
5504 {
5505 return true;
5506 }
5507 }
5508 return false;
5509 }
5510
5511 /************************************************************************/
5512 /* IGetExtent3D() */
5513 /************************************************************************/
5514
5515 inline OGRErr OGRArrowLayer::IGetExtent3D(int iGeomField,
5516 OGREnvelope3D *psExtent, bool bForce)
5517 {
5518 if (FastGetExtent3D(iGeomField, psExtent))
5519 {
5520 return OGRERR_NONE;
5521 }
5522
5523 return OGRLayer::IGetExtent3D(iGeomField, psExtent, bForce);
5524 }
5525
5526 /************************************************************************/
5527 /* OverrideArrowSchemaRelease() */
5528 /************************************************************************/
5529
5530 template <class T>
5531 static void OverrideArrowRelease(OGRArrowDataset *poDS, T *obj)
5532 {
5533 // We override the release callback, since it can use the memory pool,
5534 // and we need to make sure it is still alive when the object (ArrowArray
5535 // or ArrowSchema) is deleted
5536 struct OverriddenPrivate
5537 {
5538 OverriddenPrivate() = default;
5539 OverriddenPrivate(const OverriddenPrivate &) = delete;
5540 OverriddenPrivate &operator=(const OverriddenPrivate &) = delete;
5541
5542 std::shared_ptr<arrow::MemoryPool> poMemoryPool{};
5543 void (*pfnPreviousRelease)(T *) = nullptr;
5544 void *pPreviousPrivateData = nullptr;
5545
5546 static void release(T *l_obj)
5547 {
5548 OverriddenPrivate *myPrivate =
5549 static_cast<OverriddenPrivate *>(l_obj->private_data);
5550 l_obj->private_data = myPrivate->pPreviousPrivateData;
5551 l_obj->release = myPrivate->pfnPreviousRelease;
5552 l_obj->release(l_obj);
5553 delete myPrivate;
5554 }
5555 };
5556
5557 auto overriddenPrivate = new OverriddenPrivate();
5558 overriddenPrivate->poMemoryPool = poDS->GetSharedMemoryPool();
5559 overriddenPrivate->pPreviousPrivateData = obj->private_data;
5560 overriddenPrivate->pfnPreviousRelease = obj->release;
5561
5562 obj->release = OverriddenPrivate::release;
5563 obj->private_data = overriddenPrivate;
5564 }
5565
5566 /************************************************************************/
5567 /* UseRecordBatchBaseImplementation() */
5568 /************************************************************************/
5569
5570 inline bool OGRArrowLayer::UseRecordBatchBaseImplementation() const
5571 {
5572 if (CPLTestBool(CPLGetConfigOption("OGR_ARROW_STREAM_BASE_IMPL", "NO")))
5573 {
5574 return true;
5575 }
5576
5577 if (m_aosArrowArrayStreamOptions.FetchBool(GAS_OPT_DATETIME_AS_STRING,
5578 false))
5579 {
5580 const int nFieldCount = m_poFeatureDefn->GetFieldCount();
5581 for (int i = 0; i < nFieldCount; ++i)
5582 {
5583 const auto poFieldDefn = m_poFeatureDefn->GetFieldDefn(i);
5584 if (!poFieldDefn->IsIgnored() &&
5585 poFieldDefn->GetType() == OFTDateTime)
5586 {
5587 CPLDebug("ARROW",
5588 "DATETIME_AS_STRING=YES not compatible of fast "
5589 "Arrow implementation");
5590 return true;
5591 }
5592 }
5593 }
5594
5595 if (EQUAL(m_aosArrowArrayStreamOptions.FetchNameValueDef(
5596 "GEOMETRY_ENCODING", ""),
5597 "WKB"))
5598 {
5599 const int nGeomFieldCount = m_poFeatureDefn->GetGeomFieldCount();
5600 for (int i = 0; i < nGeomFieldCount; i++)
5601 {
5602 if (!m_poFeatureDefn->GetGeomFieldDefn(i)->IsIgnored() &&
5603 m_aeGeomEncoding[i] != OGRArrowGeomEncoding::WKB &&
5604 m_aeGeomEncoding[i] != OGRArrowGeomEncoding::WKT)
5605 {
5606 CPLDebug("ARROW", "Geometry encoding not compatible of fast "
5607 "Arrow implementation");
5608 return true;
5609 }
5610 }
5611 }
5612
5613 if (m_bIgnoredFields)
5614 {
5615 std::vector<int> ignoredState(m_anMapFieldIndexToArrowColumn.size(),
5616 -1);
5617 for (size_t i = 0; i < m_anMapFieldIndexToArrowColumn.size(); i++)
5618 {
5619 const int nArrowCol = m_anMapFieldIndexToArrowColumn[i][0];
5620 if (nArrowCol >= static_cast<int>(ignoredState.size()))
5621 ignoredState.resize(nArrowCol + 1, -1);
5622 const auto bIsIgnored =
5623 m_poFeatureDefn->GetFieldDefn(static_cast<int>(i))->IsIgnored();
5624 if (ignoredState[nArrowCol] < 0)
5625 {
5626 ignoredState[nArrowCol] = static_cast<int>(bIsIgnored);
5627 }
5628 else
5629 {
5630 // struct fields will point to the same arrow column
5631 if (ignoredState[nArrowCol] != static_cast<int>(bIsIgnored))
5632 {
5633 CPLDebug("ARROW",
5634 "Inconsistent ignore state for Arrow Columns");
5635 return true;
5636 }
5637 }
5638 }
5639 }
5640
5641 if (m_poAttrQuery || m_poFilterGeom)
5642 {
5643 struct ArrowSchema *psSchema = &m_sCachedSchema;
5644 if (psSchema->release)
5645 psSchema->release(psSchema);
5646 memset(psSchema, 0, sizeof(*psSchema));
5647
5648 const bool bCanPostFilter = GetArrowSchemaInternal(psSchema) == 0 &&
5649 CanPostFilterArrowArray(psSchema);
5650 if (!bCanPostFilter)
5651 return true;
5652 }
5653
5654 return false;
5655 }
5656
5657 /************************************************************************/
5658 /* GetArrowStream() */
5659 /************************************************************************/
5660
5661 inline bool OGRArrowLayer::GetArrowStream(struct ArrowArrayStream *out_stream,
5662 CSLConstList papszOptions)
5663 {
5664 if (!OGRLayer::GetArrowStream(out_stream, papszOptions))
5665 return false;
5666
5667 m_bUseRecordBatchBaseImplementation = UseRecordBatchBaseImplementation();
5668 return true;
5669 }
5670
5671 /************************************************************************/
5672 /* GetArrowSchema() */
5673 /************************************************************************/
5674
5675 inline int OGRArrowLayer::GetArrowSchema(struct ArrowArrayStream *stream,
5676 struct ArrowSchema *out_schema)
5677 {
5678 if (m_bUseRecordBatchBaseImplementation)
5679 return OGRLayer::GetArrowSchema(stream, out_schema);
5680
5681 return GetArrowSchemaInternal(out_schema);
5682 }
5683
5684 /************************************************************************/
5685 /* GetArrowSchemaInternal() */
5686 /************************************************************************/
5687
5688 static bool IsSilentlyIgnoredFormatForGetArrowSchemaArray(const char *format)
5689 {
5690 // n: null
5691 return strcmp(format, "n") == 0;
5692 }
5693
5694 inline int
5695 OGRArrowLayer::GetArrowSchemaInternal(struct ArrowSchema *out_schema) const
5696 {
5697 auto status = arrow::ExportSchema(*m_poSchema, out_schema);
5698 if (!status.ok())
5699 {
5700 CPLError(CE_Failure, CPLE_AppDefined, "ExportSchema() failed with %s",
5701 status.message().c_str());
5702 return EIO;
5703 }
5704
5705 CPLAssert(out_schema->n_children == m_poSchema->num_fields());
5706
5707 // Remove ignored fields from the ArrowSchema.
5708
5709 struct FieldDesc
5710 {
5711 bool bIsRegularField =
5712 false; // true = attribute field, false = geometry field
5713 int nIdx = -1;
5714 };
5715
5716 // cppcheck-suppress unreadVariable
5717 std::vector<FieldDesc> fieldDesc(
5718 static_cast<size_t>(out_schema->n_children));
5719 for (size_t i = 0; i < m_anMapFieldIndexToArrowColumn.size(); i++)
5720 {
5721 const int nArrowCol = m_anMapFieldIndexToArrowColumn[i][0];
5722 if (fieldDesc[nArrowCol].nIdx < 0)
5723 {
5724 fieldDesc[nArrowCol].bIsRegularField = true;
5725 fieldDesc[nArrowCol].nIdx = static_cast<int>(i);
5726 }
5727 }
5728 for (size_t i = 0; i < m_anMapGeomFieldIndexToArrowColumn.size(); i++)
5729 {
5730 const int nArrowCol = m_anMapGeomFieldIndexToArrowColumn[i];
5731 CPLAssert(fieldDesc[nArrowCol].nIdx < 0);
5732 fieldDesc[nArrowCol].bIsRegularField = false;
5733 fieldDesc[nArrowCol].nIdx = static_cast<int>(i);
5734 }
5735
5736 int j = 0;
5737 const char *pszReqGeomEncoding =
5738 m_aosArrowArrayStreamOptions.FetchNameValueDef("GEOMETRY_ENCODING", "");
5739
5740 const char *pszExtensionName = EXTENSION_NAME_OGC_WKB;
5741 if (EQUAL(pszReqGeomEncoding, "WKB") || EQUAL(pszReqGeomEncoding, ""))
5742 {
5743 const char *const pszGeometryMetadataEncoding =
5744 m_aosArrowArrayStreamOptions.FetchNameValue(
5745 "GEOMETRY_METADATA_ENCODING");
5746 if (pszGeometryMetadataEncoding)
5747 {
5748 if (EQUAL(pszGeometryMetadataEncoding, "OGC"))
5749 pszExtensionName = EXTENSION_NAME_OGC_WKB;
5750 else if (EQUAL(pszGeometryMetadataEncoding, "GEOARROW"))
5751 pszExtensionName = EXTENSION_NAME_GEOARROW_WKB;
5752 else
5753 CPLError(CE_Warning, CPLE_NotSupported,
5754 "Unsupported GEOMETRY_METADATA_ENCODING value: %s",
5755 pszGeometryMetadataEncoding);
5756 }
5757 }
5758
5759 for (int i = 0; i < out_schema->n_children; ++i)
5760 {
5761 if (fieldDesc[i].nIdx < 0)
5762 {
5763 if (m_iFIDArrowColumn == i)
5764 {
5765 out_schema->children[j] = out_schema->children[i];
5766 ++j;
5767 }
5768 else if (cpl::contains(m_oSetBBoxArrowColumns, i))
5769 {
5770 // Remove bounding box columns from exported schema
5771 out_schema->children[i]->release(out_schema->children[i]);
5772 out_schema->children[i] = nullptr;
5773 }
5774 else if (IsSilentlyIgnoredFormatForGetArrowSchemaArray(
5775 out_schema->children[i]->format))
5776 {
5777 // Silently ignore columns with null data type...
5778 out_schema->children[i]->release(out_schema->children[i]);
5779 }
5780 else
5781 {
5782 // can happen with data types we don't support
5783 if (m_aosArrowArrayStreamOptions.FetchBool(
5784 "SILENCE_GET_SCHEMA_ERROR", false))
5785 {
5786 CPLDebug(GetDriverUCName().c_str(),
5787 "GetArrowSchema() error: fieldDesc[%d].nIdx < 0 "
5788 "not expected: name=%s, format=%s",
5789 i, out_schema->children[i]->name,
5790 out_schema->children[i]->format);
5791 }
5792 else
5793 {
5794 CPLError(CE_Failure, CPLE_NotSupported,
5795 "GetArrowSchema() error: fieldDesc[%d].nIdx < 0 "
5796 "not expected: name=%s, format=%s",
5797 i, out_schema->children[i]->name,
5798 out_schema->children[i]->format);
5799 }
5800 for (; i < out_schema->n_children; ++i, ++j)
5801 out_schema->children[j] = out_schema->children[i];
5802 out_schema->n_children = j;
5803
5804 out_schema->release(out_schema);
5805
5806 return EIO;
5807 }
5808 continue;
5809 }
5810
5811 const auto bIsIgnored =
5812 fieldDesc[i].bIsRegularField
5813 ? m_poFeatureDefn->GetFieldDefn(fieldDesc[i].nIdx)->IsIgnored()
5814 : m_poFeatureDefn->GetGeomFieldDefn(fieldDesc[i].nIdx)
5815 ->IsIgnored();
5816 if (bIsIgnored)
5817 {
5818 out_schema->children[i]->release(out_schema->children[i]);
5819 }
5820 else
5821 {
5822 if (!fieldDesc[i].bIsRegularField &&
5823 EQUAL(pszReqGeomEncoding, "WKB"))
5824 {
5825 const int iGeomField = fieldDesc[i].nIdx;
5826 if (m_aeGeomEncoding[iGeomField] == OGRArrowGeomEncoding::WKT)
5827 {
5828 const auto poGeomFieldDefn =
5829 m_poFeatureDefn->GetGeomFieldDefn(iGeomField);
5830 CPLAssert(strcmp(out_schema->children[i]->name,
5831 poGeomFieldDefn->GetNameRef()) == 0);
5832 auto poSchema = CreateSchemaForWKBGeometryColumn(
5833 poGeomFieldDefn, "z", pszExtensionName);
5834 out_schema->children[i]->release(out_schema->children[i]);
5835 *(out_schema->children[j]) = *poSchema;
5836 CPLFree(poSchema);
5837 }
5838 else if (m_aeGeomEncoding[iGeomField] !=
5839 OGRArrowGeomEncoding::WKB)
5840 {
5841 // Shouldn't happen if UseRecordBatchBaseImplementation()
5842 // is up to date
5843 CPLAssert(false);
5844 }
5845 else
5846 {
5847 out_schema->children[j] = out_schema->children[i];
5848 }
5849 }
5850 else
5851 {
5852 out_schema->children[j] = out_schema->children[i];
5853 }
5854
5855 if (!fieldDesc[i].bIsRegularField &&
5856 (EQUAL(pszReqGeomEncoding, "WKB") ||
5857 EQUAL(pszReqGeomEncoding, "")))
5858 {
5859 const int iGeomField = fieldDesc[i].nIdx;
5860 const char *pszFormat = out_schema->children[j]->format;
5861 if (m_aeGeomEncoding[iGeomField] == OGRArrowGeomEncoding::WKB &&
5862 !out_schema->children[j]->metadata &&
5863 (strcmp(pszFormat, "z") == 0 ||
5864 strcmp(pszFormat, "Z") == 0))
5865 {
5866 const auto poGeomFieldDefn =
5867 m_poFeatureDefn->GetGeomFieldDefn(iGeomField);
5868 // Set ARROW:extension:name = ogc:wkb
5869 auto poSchema = CreateSchemaForWKBGeometryColumn(
5870 poGeomFieldDefn, pszFormat, pszExtensionName);
5871 out_schema->children[i]->release(out_schema->children[i]);
5872 *(out_schema->children[j]) = *poSchema;
5873 CPLFree(poSchema);
5874 }
5875 }
5876
5877 ++j;
5878 }
5879 }
5880
5881 out_schema->n_children = j;
5882
5883 OverrideArrowRelease(m_poArrowDS, out_schema);
5884
5885 return 0;
5886 }
5887
5888 /************************************************************************/
5889 /* GetNextArrowArray() */
5890 /************************************************************************/
5891
5892 inline int OGRArrowLayer::GetNextArrowArray(struct ArrowArrayStream *stream,
5893 struct ArrowArray *out_array)
5894 {
5895 if (m_bUseRecordBatchBaseImplementation)
5896 return OGRLayer::GetNextArrowArray(stream, out_array);
5897
5898 while (true)
5899 {
5900 if (m_bEOF)
5901 {
5902 memset(out_array, 0, sizeof(*out_array));
5903 return 0;
5904 }
5905
5906 if (m_poBatch == nullptr || m_nIdxInBatch == m_poBatch->num_rows())
5907 {
5908 if (!ReadNextBatch())
5909 {
5910 if (m_poAttrQuery || m_poFilterGeom)
5911 {
5912 InvalidateCachedBatches();
5913 }
5914 m_bEOF = true;
5915 memset(out_array, 0, sizeof(*out_array));
5916 return 0;
5917 }
5918 }
5919
5920 const bool bNeedsPostFilter =
5921 (m_poAttrQuery && !m_bBaseArrowIgnoreAttributeFilter) ||
5922 (m_poFilterGeom && !m_bBaseArrowIgnoreSpatialFilter);
5923
5924 struct ArrowSchema schema;
5925 memset(&schema, 0, sizeof(schema));
5926 auto status = arrow::ExportRecordBatch(*m_poBatch, out_array, &schema);
5927 m_nIdxInBatch = m_poBatch->num_rows();
5928 if (!status.ok())
5929 {
5930 CPLError(CE_Failure, CPLE_AppDefined,
5931 "ExportRecordBatch() failed with %s",
5932 status.message().c_str());
5933 return EIO;
5934 }
5935
5936 // Remove bounding box columns from exported array, or columns
5937 // of unsupported data types that we voluntarily strip off.
5938 const auto RemoveBBoxOrUnsupportedColumns =
5939 [out_array, &schema](const std::set<int> &oSetBBoxArrayIndex)
5940 {
5941 int j = 0;
5942 for (int i = 0; i < static_cast<int>(schema.n_children); ++i)
5943 {
5944 if (cpl::contains(oSetBBoxArrayIndex, i) ||
5945 IsSilentlyIgnoredFormatForGetArrowSchemaArray(
5946 schema.children[i]->format))
5947 {
5948 out_array->children[i]->release(out_array->children[i]);
5949 out_array->children[i] = nullptr;
5950
5951 schema.children[i]->release(schema.children[i]);
5952 schema.children[i] = nullptr;
5953 }
5954 else
5955 {
5956 out_array->children[j] = out_array->children[i];
5957 schema.children[j] = schema.children[i];
5958 ++j;
5959 }
5960 }
5961 out_array->n_children = j;
5962 schema.n_children = j;
5963 };
5964
5965 if (m_bIgnoredFields)
5966 {
5967 std::set<int> oSetBBoxArrayIndex;
5968 for (const auto &iter : m_oMapGeomFieldIndexToGeomColBBOX)
5969 {
5970 if (iter.second.iArrayIdx >= 0)
5971 oSetBBoxArrayIndex.insert(iter.second.iArrayIdx);
5972 }
5973 RemoveBBoxOrUnsupportedColumns(oSetBBoxArrayIndex);
5974 }
5975 else
5976 {
5977 RemoveBBoxOrUnsupportedColumns(m_oSetBBoxArrowColumns);
5978 }
5979
5980 if (EQUAL(m_aosArrowArrayStreamOptions.FetchNameValueDef(
5981 "GEOMETRY_ENCODING", ""),
5982 "WKB"))
5983 {
5984 const int nGeomFieldCount = m_poFeatureDefn->GetGeomFieldCount();
5985 for (int i = 0; i < nGeomFieldCount; i++)
5986 {
5987 const auto poGeomFieldDefn =
5988 m_poFeatureDefn->GetGeomFieldDefn(i);
5989 if (!poGeomFieldDefn->IsIgnored())
5990 {
5991 if (m_aeGeomEncoding[i] == OGRArrowGeomEncoding::WKT)
5992 {
5993 const int nArrayIdx =
5994 m_bIgnoredFields
5995 ? m_anMapGeomFieldIndexToArrayIndex[i]
5996 : m_anMapGeomFieldIndexToArrowColumn[i];
5997 auto sourceArray = out_array->children[nArrayIdx];
5998 auto targetArray =
5999 strcmp(schema.children[nArrayIdx]->format, "u") == 0
6000 ? CreateWKBArrayFromWKTArray<uint32_t>(
6001 sourceArray)
6002 : CreateWKBArrayFromWKTArray<uint64_t>(
6003 sourceArray);
6004 if (targetArray)
6005 {
6006 sourceArray->release(sourceArray);
6007 *(out_array->children[nArrayIdx]) = *targetArray;
6008 CPLFree(targetArray);
6009 }
6010 else
6011 {
6012 out_array->release(out_array);
6013 memset(out_array, 0, sizeof(*out_array));
6014 if (schema.release)
6015 schema.release(&schema);
6016 return ENOMEM;
6017 }
6018 }
6019 else if (m_aeGeomEncoding[i] != OGRArrowGeomEncoding::WKB)
6020 {
6021 // Shouldn't happen if UseRecordBatchBaseImplementation()
6022 // is up to date
6023 CPLAssert(false);
6024 }
6025 }
6026 }
6027 }
6028
6029 if (schema.release)
6030 schema.release(&schema);
6031
6032 OverrideArrowRelease(m_poArrowDS, out_array);
6033
6034 const auto nFeatureIdxCur = m_nFeatureIdx;
6035 // TODO: We likely have an issue regarding FIDs based on m_nFeatureIdx
6036 // when m_iFIDArrowColumn < 0, only a subset of row groups is
6037 // selected, and this batch goes across non consecutive row groups.
6038 for (int64_t i = 0; i < m_nIdxInBatch; ++i)
6039 IncrFeatureIdx();
6040
6041 if (bNeedsPostFilter)
6042 {
6043 CPLStringList aosOptions;
6044 if (m_iFIDArrowColumn < 0)
6045 aosOptions.SetNameValue(
6046 "BASE_SEQUENTIAL_FID",
6047 CPLSPrintf(CPL_FRMT_GIB,
6048 static_cast<GIntBig>(nFeatureIdxCur)));
6049
6050 // If there might be more than one record batch, it is more
6051 // prudent to clone the array before modifying it.
6052 if (nFeatureIdxCur > 0 || !TestCapability(OLCFastFeatureCount) ||
6053 out_array->length < GetFeatureCount(false))
6054 {
6055 struct ArrowArray new_array;
6056 if (!OGRCloneArrowArray(&m_sCachedSchema, out_array,
6057 &new_array))
6058 {
6059 if (out_array->release)
6060 out_array->release(out_array);
6061 memset(out_array, 0, sizeof(*out_array));
6062 return ENOMEM;
6063 }
6064 if (out_array->release)
6065 out_array->release(out_array);
6066 memcpy(out_array, &new_array, sizeof(new_array));
6067 }
6068
6069 PostFilterArrowArray(&m_sCachedSchema, out_array,
6070 aosOptions.List());
6071 if (out_array->length == 0)
6072 {
6073 if (out_array->release)
6074 out_array->release(out_array);
6075 memset(out_array, 0, sizeof(*out_array));
6076 // If there are no records after filtering, start again
6077 // with a new batch
6078 continue;
6079 }
6080 }
6081
6082 break;
6083 }
6084
6085 return 0;
6086 }
6087
6088 /************************************************************************/
6089 /* OGRArrowLayerAppendBuffer */
6090 /************************************************************************/
6091
6092 class OGRArrowLayerAppendBuffer : public OGRAppendBuffer
6093 {
6094 public:
6095 OGRArrowLayerAppendBuffer(struct ArrowArray *targetArrayIn,
6096 size_t nInitialCapacityIn)
6097 : m_psTargetArray(targetArrayIn)
6098 {
6099 m_nCapacity = nInitialCapacityIn;
6100 m_pRawBuffer = const_cast<void *>(m_psTargetArray->buffers[2]);
6101 }
6102
6103 protected:
6104 bool Grow(size_t nItemSize) override
6105 {
6106 constexpr uint32_t MAX_SIZE_SINT32 =
6107 static_cast<uint32_t>(std::numeric_limits<int32_t>::max());
6108 if (nItemSize > MAX_SIZE_SINT32 - m_nSize)
6109 {
6110 CPLError(CE_Failure, CPLE_AppDefined, "Too large WKT content");
6111 return false;
6112 }
6113 size_t nNewCapacity = m_nSize + nItemSize;
6114 CPLAssert(m_nCapacity <= MAX_SIZE_SINT32);
6115 const size_t nDoubleCapacity =
6116 std::min<size_t>(MAX_SIZE_SINT32, 2 * m_nCapacity);
6117 if (nNewCapacity < nDoubleCapacity)
6118 nNewCapacity = nDoubleCapacity;
6119 CPLAssert(nNewCapacity <= MAX_SIZE_SINT32);
6120 void *newBuffer = VSI_MALLOC_ALIGNED_AUTO_VERBOSE(nNewCapacity);
6121 if (newBuffer == nullptr)
6122 {
6123 return false;
6124 }
6125 m_nCapacity = nNewCapacity;
6126 memcpy(newBuffer, m_pRawBuffer, m_nSize);
6127 VSIFreeAligned(m_pRawBuffer);
6128 m_pRawBuffer = newBuffer;
6129 m_psTargetArray->buffers[2] = m_pRawBuffer;
6130 return true;
6131 }
6132
6133 private:
6134 struct ArrowArray *m_psTargetArray;
6135
6136 OGRArrowLayerAppendBuffer(const OGRArrowLayerAppendBuffer &) = delete;
6137 OGRArrowLayerAppendBuffer &
6138 operator=(const OGRArrowLayerAppendBuffer &) = delete;
6139 };
6140
6141 /************************************************************************/
6142 /* CreateWKBArrayFromWKTArray() */
6143 /************************************************************************/
6144
6145 template <typename SourceOffset>
6146 inline struct ArrowArray *
6147 OGRArrowLayer::CreateWKBArrayFromWKTArray(const struct ArrowArray *sourceArray)
6148 {
6149 CPLAssert(sourceArray->n_buffers == 3);
6150 CPLAssert(sourceArray->buffers[1] != nullptr);
6151 CPLAssert(sourceArray->buffers[2] != nullptr);
6152
6153 const size_t nLength = static_cast<size_t>(sourceArray->length);
6154 auto targetArray = static_cast<struct ArrowArray *>(
6155 CPLCalloc(1, sizeof(struct ArrowArray)));
6156 targetArray->release = OGRLayer::ReleaseArray;
6157 targetArray->length = nLength;
6158
6159 targetArray->n_buffers = 3;
6160 targetArray->buffers =
6161 static_cast<const void **>(CPLCalloc(3, sizeof(void *)));
6162
6163 // Allocate validity map buffer if needed
6164 const auto sourceNull =
6165 static_cast<const uint8_t *>(sourceArray->buffers[0]);
6166 const size_t nOffset = static_cast<size_t>(sourceArray->offset);
6167 uint8_t *targetNull = nullptr;
6168 if (sourceArray->null_count && sourceNull)
6169 {
6170 targetArray->buffers[0] =
6171 VSI_MALLOC_ALIGNED_AUTO_VERBOSE((nLength + 7) / 8);
6172 if (targetArray->buffers[0])
6173 {
6174 targetArray->null_count = sourceArray->null_count;
6175 targetNull = static_cast<uint8_t *>(
6176 const_cast<void *>(targetArray->buffers[0]));
6177 if (nOffset == 0)
6178 {
6179 memcpy(targetNull, sourceNull, (nLength + 7) / 8);
6180 }
6181 else
6182 {
6183 memset(targetNull, 0, (nLength + 7) / 8);
6184 for (size_t i = 0; i < nLength; ++i)
6185 {
6186 if ((sourceNull[(i + nOffset) / 8] >> ((i + nOffset) % 8)) &
6187 1)
6188 targetNull[i / 8] |= (1 << (i % 8));
6189 }
6190 }
6191 }
6192 }
6193
6194 // Allocate offset buffer
6195 targetArray->buffers[1] =
6196 VSI_MALLOC_ALIGNED_AUTO_VERBOSE(sizeof(uint32_t) * (1 + nLength));
6197
6198 // Allocate data (WKB) buffer
6199 constexpr size_t DEFAULT_WKB_SIZE = 100;
6200 uint32_t nInitialCapacity = static_cast<uint32_t>(std::min<size_t>(
6201 std::numeric_limits<int32_t>::max(), DEFAULT_WKB_SIZE * nLength));
6202 targetArray->buffers[2] = VSI_MALLOC_ALIGNED_AUTO_VERBOSE(nInitialCapacity);
6203
6204 // Check buffers have been allocated
6205 if ((sourceArray->null_count && sourceNull && !targetNull) ||
6206 targetArray->buffers[1] == nullptr ||
6207 targetArray->buffers[2] == nullptr)
6208 {
6209 targetArray->release(targetArray);
6210 return nullptr;
6211 }
6212
6213 OGRArrowLayerAppendBuffer oOGRAppendBuffer(targetArray, nInitialCapacity);
6214 OGRWKTToWKBTranslator oTranslator(oOGRAppendBuffer);
6215
6216 const auto sourceOffsets =
6217 static_cast<const SourceOffset *>(sourceArray->buffers[1]) + nOffset;
6218 auto sourceBytes =
6219 static_cast<char *>(const_cast<void *>(sourceArray->buffers[2]));
6220 auto targetOffsets =
6221 static_cast<uint32_t *>(const_cast<void *>(targetArray->buffers[1]));
6222 for (size_t i = 0; i < nLength; ++i)
6223 {
6224 targetOffsets[i] = static_cast<uint32_t>(oOGRAppendBuffer.GetSize());
6225
6226 if (targetNull && ((targetNull[i / 8] >> (i % 8)) & 1) == 0)
6227 {
6228 continue;
6229 }
6230
6231 const size_t nWKBSize = oTranslator.TranslateWKT(
6232 sourceBytes + sourceOffsets[i],
6233 static_cast<size_t>(sourceOffsets[i + 1] - sourceOffsets[i]),
6234 sourceOffsets[i + 1] < sourceOffsets[nLength]);
6235 if (nWKBSize == static_cast<size_t>(-1))
6236 {
6237 targetArray->release(targetArray);
6238 return nullptr;
6239 }
6240 }
6241 targetOffsets[nLength] = static_cast<uint32_t>(oOGRAppendBuffer.GetSize());
6242
6243 return targetArray;
6244 }
6245
6246 /************************************************************************/
6247 /* TestCapability() */
6248 /************************************************************************/
6249
6250 inline int OGRArrowLayer::TestCapability(const char *pszCap) const
6251 {
6252
6253 if (EQUAL(pszCap, OLCStringsAsUTF8))
6254 return true;
6255
6256 else if (EQUAL(pszCap, OLCFastGetArrowStream) &&
6257 !UseRecordBatchBaseImplementation())
6258 {
6259 return true;
6260 }
6261
6262 if (EQUAL(pszCap, OLCFastGetExtent))
6263 {
6264 OGREnvelope sEnvelope;
6265 for (int i = 0; i < m_poFeatureDefn->GetGeomFieldCount(); i++)
6266 {
6267 if (!FastGetExtent(i, &sEnvelope))
6268 return false;
6269 }
6270 return true;
6271 }
6272
6273 if (EQUAL(pszCap, OLCFastGetExtent3D))
6274 {
6275 OGREnvelope3D sEnvelope;
6276 for (int i = 0; i < m_poFeatureDefn->GetGeomFieldCount(); i++)
6277 {
6278 if (!FastGetExtent3D(i, &sEnvelope))
6279 return false;
6280 }
6281 return true;
6282 }
6283
6284 return false;
6285 }
6286
6287 #if defined(__clang__)
6288 #pragma clang diagnostic pop
6289 #endif
6290
6291 #endif /* OGARROWLAYER_HPP_INCLUDED */