Debian Codegraph @

ogr/ogrsf_frmts/arrow_common/ograrrowwriterlayer.hpp

raw
1 /******************************************************************************
2 *
3 * Project: Arrow generic code
4 * Purpose: Arrow generic code
5 * Author: Even Rouault, <even.rouault at spatialys.com>
6 *
7 ******************************************************************************
8 * Copyright (c) 2022, Planet Labs
9 *
10 * SPDX-License-Identifier: MIT
11 ****************************************************************************/
12
13 #ifndef OGARROWWRITERLAYER_HPP_INCLUDED
14 #define OGARROWWRITERLAYER_HPP_INCLUDED
15
16 #include "ogr_arrow.h"
17
18 #include "cpl_json.h"
19 #include "cpl_time.h"
20
21 #include "ogrlayerarrow.h"
22 #include "ogr_wkb.h"
23
24 #include <array>
25 #include <cinttypes>
26 #include <limits>
27
28 static constexpr int TZFLAG_UNINITIALIZED = -1;
29
30 #define OGR_ARROW_RETURN_NOT_OK(status, ret_value) \
31 do \
32 { \
33 if (!(status).ok()) \
34 { \
35 CPLError(CE_Failure, CPLE_AppDefined, "%s failed", \
36 (status).message().c_str()); \
37 return (ret_value); \
38 } \
39 } while (false)
40
41 #define OGR_ARROW_RETURN_FALSE_NOT_OK(status) \
42 OGR_ARROW_RETURN_NOT_OK(status, false)
43
44 #define OGR_ARROW_RETURN_OGRERR_NOT_OK(status) \
45 OGR_ARROW_RETURN_NOT_OK(status, OGRERR_FAILURE)
46
47 #define OGR_ARROW_PROPAGATE_OGRERR(ret_value) \
48 do \
49 { \
50 if ((ret_value) != OGRERR_NONE) \
51 return OGRERR_FAILURE; \
52 } while (0)
53
54 /************************************************************************/
55 /* OGRArrowWriterLayer() */
56 /************************************************************************/
57
58 inline OGRArrowWriterLayer::OGRArrowWriterLayer(
59 arrow::MemoryPool *poMemoryPool,
60 const std::shared_ptr<arrow::io::OutputStream> &poOutputStream,
61 const char *pszLayerName)
62 : m_poMemoryPool(poMemoryPool), m_poOutputStream(poOutputStream)
63 {
64 m_poFeatureDefn = new OGRFeatureDefn(pszLayerName);
65 m_poFeatureDefn->SetGeomType(wkbNone);
66 m_poFeatureDefn->Reference();
67 SetDescription(pszLayerName);
68 }
69
70 /************************************************************************/
71 /* ~OGRArrowWriterLayer() */
72 /************************************************************************/
73
74 inline OGRArrowWriterLayer::~OGRArrowWriterLayer()
75 {
76 CPLDebug("ARROW", "Memory pool (writer layer): bytes_allocated = %" PRId64,
77 m_poMemoryPool->bytes_allocated());
78 CPLDebug("ARROW", "Memory pool (writer layer): max_memory = %" PRId64,
79 m_poMemoryPool->max_memory());
80
81 m_poFeatureDefn->Release();
82 }
83
84 /************************************************************************/
85 /* FinalizeWriting() */
86 /************************************************************************/
87
88 inline bool OGRArrowWriterLayer::FinalizeWriting()
89 {
90 bool ret = true;
91
92 if (!IsFileWriterCreated())
93 {
94 CreateWriter();
95 }
96 if (IsFileWriterCreated())
97 {
98 PerformStepsBeforeFinalFlushGroup();
99
100 if (!m_apoBuilders.empty() && m_apoFieldsFromArrowSchema.empty())
101 ret = FlushGroup();
102
103 if (!CloseFileWriter())
104 ret = false;
105 }
106
107 return ret;
108 }
109
110 /************************************************************************/
111 /* RemoveIDFromMemberOfEnsembles() */
112 /************************************************************************/
113
114 /* static */
115 inline void
116 OGRArrowWriterLayer::RemoveIDFromMemberOfEnsembles(CPLJSONObject &obj)
117 {
118 // Remove "id" from members of datum ensembles for compatibility with
119 // older PROJ versions
120 // Cf https://github.com/opengeospatial/geoparquet/discussions/110
121 // and https://github.com/OSGeo/PROJ/pull/3221
122 if (obj.GetType() == CPLJSONObject::Type::Object)
123 {
124 for (auto &subObj : obj.GetChildren())
125 {
126 RemoveIDFromMemberOfEnsembles(subObj);
127 }
128 }
129 else if (obj.GetType() == CPLJSONObject::Type::Array &&
130 obj.GetName() == "members")
131 {
132 for (auto &subObj : obj.ToArray())
133 {
134 subObj.Delete("id");
135 }
136 }
137 }
138
139 /************************************************************************/
140 /* IdentifyCRS() */
141 /************************************************************************/
142
143 /* static */
144 inline OGRSpatialReference
145 OGRArrowWriterLayer::IdentifyCRS(const OGRSpatialReference *poSRS)
146 {
147 OGRSpatialReference oSRSIdentified(*poSRS);
148
149 if (poSRS->GetAuthorityName(nullptr) == nullptr)
150 {
151 // Try to find a registered CRS that matches the input one
152 int nEntries = 0;
153 int *panConfidence = nullptr;
154 OGRSpatialReferenceH *pahSRS =
155 poSRS->FindMatches(nullptr, &nEntries, &panConfidence);
156
157 // If there are several matches >= 90%, take the only one
158 // that is EPSG
159 int iOtherAuthority = -1;
160 int iEPSG = -1;
161 const char *const apszOptions[] = {
162 "IGNORE_DATA_AXIS_TO_SRS_AXIS_MAPPING=YES", nullptr};
163 int iConfidenceBestMatch = -1;
164 for (int iSRS = 0; iSRS < nEntries; iSRS++)
165 {
166 auto poCandidateCRS = OGRSpatialReference::FromHandle(pahSRS[iSRS]);
167 if (panConfidence[iSRS] < iConfidenceBestMatch ||
168 panConfidence[iSRS] < 70)
169 {
170 break;
171 }
172 if (poSRS->IsSame(poCandidateCRS, apszOptions))
173 {
174 const char *pszAuthName =
175 poCandidateCRS->GetAuthorityName(nullptr);
176 if (pszAuthName != nullptr && EQUAL(pszAuthName, "EPSG"))
177 {
178 iOtherAuthority = -2;
179 if (iEPSG < 0)
180 {
181 iConfidenceBestMatch = panConfidence[iSRS];
182 iEPSG = iSRS;
183 }
184 else
185 {
186 iEPSG = -1;
187 break;
188 }
189 }
190 else if (iEPSG < 0 && pszAuthName != nullptr)
191 {
192 if (EQUAL(pszAuthName, "OGC"))
193 {
194 const char *pszAuthCode =
195 poCandidateCRS->GetAuthorityCode(nullptr);
196 if (pszAuthCode && EQUAL(pszAuthCode, "CRS84"))
197 {
198 iOtherAuthority = iSRS;
199 break;
200 }
201 }
202 else if (iOtherAuthority == -1)
203 {
204 iConfidenceBestMatch = panConfidence[iSRS];
205 iOtherAuthority = iSRS;
206 }
207 else
208 iOtherAuthority = -2;
209 }
210 }
211 }
212 if (iEPSG >= 0)
213 {
214 oSRSIdentified = *OGRSpatialReference::FromHandle(pahSRS[iEPSG]);
215 }
216 else if (iOtherAuthority >= 0)
217 {
218 oSRSIdentified =
219 *OGRSpatialReference::FromHandle(pahSRS[iOtherAuthority]);
220 }
221 OSRFreeSRSArray(pahSRS);
222 CPLFree(panConfidence);
223 }
224
225 return oSRSIdentified;
226 }
227
228 /************************************************************************/
229 /* CreateSchemaCommon() */
230 /************************************************************************/
231
232 inline void OGRArrowWriterLayer::CreateSchemaCommon()
233 {
234 CPLAssert(static_cast<int>(m_aeGeomEncoding.size()) ==
235 m_poFeatureDefn->GetGeomFieldCount());
236
237 std::vector<std::shared_ptr<arrow::Field>> fields;
238 bool bNeedGDALSchema = false;
239
240 m_anTZFlag.resize(m_poFeatureDefn->GetFieldCount(), TZFLAG_UNINITIALIZED);
241
242 if (!m_osFIDColumn.empty())
243 {
244 bNeedGDALSchema = true;
245 fields.emplace_back(arrow::field(m_osFIDColumn, arrow::int64(), false));
246 }
247
248 if (!m_apoFieldsFromArrowSchema.empty())
249 {
250 fields.insert(fields.end(), m_apoFieldsFromArrowSchema.begin(),
251 m_apoFieldsFromArrowSchema.end());
252 }
253
254 for (int i = 0; i < m_poFeatureDefn->GetFieldCount(); ++i)
255 {
256 const auto poFieldDefn = m_poFeatureDefn->GetFieldDefn(i);
257 std::shared_ptr<arrow::DataType> dt;
258 const auto eDT = poFieldDefn->GetType();
259 const auto eSubDT = poFieldDefn->GetSubType();
260 const auto &osDomainName = poFieldDefn->GetDomainName();
261 const OGRFieldDomain *poFieldDomain = nullptr;
262 const int nWidth = poFieldDefn->GetWidth();
263 if (!osDomainName.empty())
264 {
265 const auto oIter = m_oMapFieldDomains.find(osDomainName);
266 if (oIter == m_oMapFieldDomains.end())
267 {
268 CPLError(CE_Warning, CPLE_AppDefined,
269 "Field %s references domain %s, but the later one "
270 "has not been created",
271 poFieldDefn->GetNameRef(), osDomainName.c_str());
272 }
273 else
274 {
275 poFieldDomain = oIter->second.get();
276 }
277 }
278 const char *pszFieldMetadata = nullptr;
279 switch (eDT)
280 {
281 case OFTInteger:
282 if (eSubDT == OFSTBoolean)
283 dt = arrow::boolean();
284 else if (eSubDT == OFSTInt16)
285 dt = arrow::int16();
286 else
287 dt = arrow::int32();
288 if (poFieldDomain != nullptr)
289 {
290 dt = arrow::dictionary(dt, arrow::utf8());
291 }
292 break;
293
294 case OFTInteger64:
295 dt = arrow::int64();
296 if (poFieldDomain != nullptr)
297 {
298 dt = arrow::dictionary(dt, arrow::utf8());
299 }
300 break;
301
302 case OFTReal:
303 {
304 const int nPrecision = poFieldDefn->GetPrecision();
305 if (nWidth != 0 && nPrecision != 0)
306 {
307 // Since arrow 18.0, we could use arrow::smallest_decimal()
308 // to return the smallest representation (i.e. possibly
309 // decimal32 and decimal64). But for now keep decimal128
310 // as the minimum for backwards compatibility.
311 // GetValueDecimal() and other functions in
312 // ogrlayerarrow.cpp would have to be adapted for decimal32
313 // and decimal64 compatibility.
314 if (nWidth > 38)
315 dt = arrow::decimal256(nWidth, nPrecision);
316 else
317 dt = arrow::decimal128(nWidth, nPrecision);
318 }
319 else if (eSubDT == OFSTFloat32)
320 dt = arrow::float32();
321 else
322 dt = arrow::float64();
323 break;
324 }
325
326 case OFTString:
327 case OFTWideString:
328 if ((eSubDT != OFSTNone && eSubDT != OFSTJSON) || nWidth > 0)
329 bNeedGDALSchema = true;
330 if (eSubDT == OFSTJSON)
331 pszFieldMetadata = EXTENSION_NAME_ARROW_JSON;
332 dt = arrow::utf8();
333 break;
334
335 case OFTBinary:
336 if (nWidth != 0)
337 dt = arrow::fixed_size_binary(nWidth);
338 else
339 dt = arrow::binary();
340 break;
341
342 case OFTIntegerList:
343 if (eSubDT == OFSTBoolean)
344 dt = arrow::list(arrow::boolean());
345 else if (eSubDT == OFSTInt16)
346 dt = arrow::list(arrow::int16());
347 else
348 dt = arrow::list(arrow::int32());
349 break;
350
351 case OFTInteger64List:
352 dt = arrow::list(arrow::int64());
353 break;
354
355 case OFTRealList:
356 if (eSubDT == OFSTFloat32)
357 dt = arrow::list(arrow::float32());
358 else
359 dt = arrow::list(arrow::float64());
360 break;
361
362 case OFTStringList:
363 case OFTWideStringList:
364 dt = arrow::list(arrow::utf8());
365 break;
366
367 case OFTDate:
368 dt = arrow::date32();
369 break;
370
371 case OFTTime:
372 dt = arrow::time32(arrow::TimeUnit::MILLI);
373 break;
374
375 case OFTDateTime:
376 {
377 const int nTZFlag = poFieldDefn->GetTZFlag();
378 const char *pszTIMESTAMP_WITH_OFFSET =
379 m_aosCreationOptions.FetchNameValueDef(
380 "TIMESTAMP_WITH_OFFSET", "AUTO");
381 if ((nTZFlag == OGR_TZFLAG_MIXED_TZ &&
382 !EQUAL(pszTIMESTAMP_WITH_OFFSET, "NO")) ||
383 EQUAL(pszTIMESTAMP_WITH_OFFSET, "YES"))
384 {
385 m_anTZFlag[i] = nTZFlag;
386 std::vector<std::shared_ptr<arrow::Field>>
387 tsWithOffsetFields{
388 arrow::field(
389 ATSWO_TIMESTAMP_FIELD_NAME,
390 arrow::timestamp(arrow::TimeUnit::MILLI, "UTC"),
391 false),
392 arrow::field(ATSWO_OFFSET_MINUTES_FIELD_NAME,
393 arrow::int16(), false)};
394 dt = arrow::struct_(std::move(tsWithOffsetFields));
395 pszFieldMetadata =
396 EXTENSION_NAME_ARROW_TIMESTAMP_WITH_OFFSET;
397 }
398 else
399 {
400 if (nTZFlag >= OGR_TZFLAG_MIXED_TZ)
401 {
402 m_anTZFlag[i] = nTZFlag;
403 }
404 dt = arrow::timestamp(arrow::TimeUnit::MILLI);
405 }
406 break;
407 }
408 }
409
410 auto field = arrow::field(poFieldDefn->GetNameRef(), std::move(dt),
411 CPL_TO_BOOL(poFieldDefn->IsNullable()));
412 if (pszFieldMetadata)
413 {
414 auto kvMetadata = std::make_shared<arrow::KeyValueMetadata>();
415 kvMetadata->Append(ARROW_EXTENSION_NAME_KEY, pszFieldMetadata);
416 field = field->WithMetadata(kvMetadata);
417 }
418
419 fields.emplace_back(std::move(field));
420 if (poFieldDefn->GetAlternativeNameRef()[0])
421 bNeedGDALSchema = true;
422 if (!poFieldDefn->GetComment().empty())
423 bNeedGDALSchema = true;
424 }
425
426 for (int i = 0; i < m_poFeatureDefn->GetGeomFieldCount(); ++i)
427 {
428 const auto poGeomFieldDefn = m_poFeatureDefn->GetGeomFieldDefn(i);
429 const auto eGType = poGeomFieldDefn->GetType();
430 const int nDim =
431 2 + (OGR_GT_HasZ(eGType) ? 1 : 0) + (OGR_GT_HasM(eGType) ? 1 : 0);
432
433 const bool pointFieldNullable = GetDriverUCName() == "PARQUET";
434
435 // Fixed Size List GeoArrow encoding
436 const auto getFixedSizeListOfPoint =
437 [nDim, eGType, pointFieldNullable]()
438 {
439 return arrow::fixed_size_list(
440 arrow::field(nDim == 2 ? "xy"
441 : nDim == 3 ? (OGR_GT_HasZ(eGType) ? "xyz" : "xym")
442 : "xyzm",
443 arrow::float64(), pointFieldNullable),
444 nDim);
445 };
446
447 // Struct GeoArrow encoding
448 auto xField(arrow::field("x", arrow::float64(), false));
449 auto yField(arrow::field("y", arrow::float64(), false));
450 std::vector<std::shared_ptr<arrow::Field>> pointFields{
451 arrow::field("x", arrow::float64(), false),
452 arrow::field("y", arrow::float64(), false)};
453 if (OGR_GT_HasZ(eGType))
454 pointFields.emplace_back(
455 arrow::field("z", arrow::float64(), false));
456 if (OGR_GT_HasM(eGType))
457 pointFields.emplace_back(
458 arrow::field("m", arrow::float64(), false));
459 auto pointStructType(arrow::struct_(std::move(pointFields)));
460
461 const auto getListOfVertices = [&getFixedSizeListOfPoint]()
462 {
463 return arrow::list(std::make_shared<arrow::Field>(
464 "vertices", getFixedSizeListOfPoint()));
465 };
466
467 const auto getListOfRings = [&getListOfVertices]()
468 {
469 return arrow::list(
470 std::make_shared<arrow::Field>("rings", getListOfVertices()));
471 };
472
473 const auto getListOfVerticesStruct = [&pointStructType]()
474 {
475 return arrow::list(
476 std::make_shared<arrow::Field>("vertices", pointStructType));
477 };
478
479 const auto getListOfRingsStruct = [&getListOfVerticesStruct]()
480 {
481 return arrow::list(std::make_shared<arrow::Field>(
482 "rings", getListOfVerticesStruct()));
483 };
484
485 std::shared_ptr<arrow::DataType> dt;
486 switch (m_aeGeomEncoding[i])
487 {
488 case OGRArrowGeomEncoding::WKB:
489 #if ARROW_VERSION_MAJOR >= 21
490 if (m_bUseArrowWKBExtension)
491 {
492 CPLJSONDocument oMetadataDoc;
493
494 const auto poSRS = poGeomFieldDefn->GetSpatialRef();
495 if (poSRS)
496 {
497 OGRSpatialReference oSRSIdentified(IdentifyCRS(poSRS));
498
499 // CRS encoded as PROJJSON
500 char *pszPROJJSON = nullptr;
501 oSRSIdentified.exportToPROJJSON(&pszPROJJSON, nullptr);
502 CPLJSONDocument oCRSDoc;
503 CPL_IGNORE_RET_VAL(oCRSDoc.LoadMemory(pszPROJJSON));
504 CPLFree(pszPROJJSON);
505 CPLJSONObject oCRSRoot = oCRSDoc.GetRoot();
506 RemoveIDFromMemberOfEnsembles(oCRSRoot);
507
508 oMetadataDoc.GetRoot().Add("crs", oCRSRoot);
509 }
510
511 if (m_bEdgesSpherical)
512 {
513 oMetadataDoc.GetRoot().Add("edges", "spherical");
514 }
515
516 const std::string metadata = oMetadataDoc.GetRoot().Format(
517 CPLJSONObject::PrettyFormat::Plain);
518 dt = std::make_shared<OGRGeoArrowWkbExtensionType>(
519 arrow::binary(), metadata);
520 }
521 else
522 #endif
523 {
524 dt = arrow::binary();
525 }
526 break;
527
528 case OGRArrowGeomEncoding::WKT:
529 dt = arrow::utf8();
530 break;
531
532 case OGRArrowGeomEncoding::GEOARROW_FSL_GENERIC:
533 case OGRArrowGeomEncoding::GEOARROW_STRUCT_GENERIC:
534 CPLAssert(false);
535 break;
536
537 case OGRArrowGeomEncoding::GEOARROW_FSL_POINT:
538 dt = getFixedSizeListOfPoint();
539 break;
540
541 case OGRArrowGeomEncoding::GEOARROW_FSL_LINESTRING:
542 dt = getListOfVertices();
543 break;
544
545 case OGRArrowGeomEncoding::GEOARROW_FSL_POLYGON:
546 dt = getListOfRings();
547 break;
548
549 case OGRArrowGeomEncoding::GEOARROW_FSL_MULTIPOINT:
550 dt = arrow::list(std::make_shared<arrow::Field>(
551 "points", getFixedSizeListOfPoint()));
552 break;
553
554 case OGRArrowGeomEncoding::GEOARROW_FSL_MULTILINESTRING:
555 dt = arrow::list(std::make_shared<arrow::Field>(
556 "linestrings", getListOfVertices()));
557 break;
558
559 case OGRArrowGeomEncoding::GEOARROW_FSL_MULTIPOLYGON:
560 dt = arrow::list(std::make_shared<arrow::Field>(
561 "polygons", getListOfRings()));
562 break;
563
564 case OGRArrowGeomEncoding::GEOARROW_STRUCT_POINT:
565 dt = pointStructType;
566 break;
567
568 case OGRArrowGeomEncoding::GEOARROW_STRUCT_LINESTRING:
569 dt = getListOfVerticesStruct();
570 break;
571
572 case OGRArrowGeomEncoding::GEOARROW_STRUCT_POLYGON:
573 dt = getListOfRingsStruct();
574 break;
575
576 case OGRArrowGeomEncoding::GEOARROW_STRUCT_MULTIPOINT:
577 dt = arrow::list(
578 std::make_shared<arrow::Field>("points", pointStructType));
579 break;
580
581 case OGRArrowGeomEncoding::GEOARROW_STRUCT_MULTILINESTRING:
582 dt = arrow::list(std::make_shared<arrow::Field>(
583 "linestrings", getListOfVerticesStruct()));
584 break;
585
586 case OGRArrowGeomEncoding::GEOARROW_STRUCT_MULTIPOLYGON:
587 dt = arrow::list(std::make_shared<arrow::Field>(
588 "polygons", getListOfRingsStruct()));
589 break;
590 }
591
592 std::shared_ptr<arrow::Field> field(
593 arrow::field(poGeomFieldDefn->GetNameRef(), std::move(dt),
594 CPL_TO_BOOL(poGeomFieldDefn->IsNullable())));
595 if (m_bWriteFieldArrowExtensionName)
596 {
597 auto kvMetadata = field->metadata()
598 ? field->metadata()->Copy()
599 : std::make_shared<arrow::KeyValueMetadata>();
600 kvMetadata->Append(
601 ARROW_EXTENSION_NAME_KEY,
602 GetGeomEncodingAsString(m_aeGeomEncoding[i], false));
603 field = field->WithMetadata(kvMetadata);
604 }
605
606 m_apoBaseStructGeomType.emplace_back(std::move(pointStructType));
607
608 fields.emplace_back(std::move(field));
609 }
610
611 if (m_bWriteBBoxStruct)
612 {
613 for (int i = 0; i < m_poFeatureDefn->GetGeomFieldCount(); ++i)
614 {
615 const auto poGeomFieldDefn = m_poFeatureDefn->GetGeomFieldDefn(i);
616 auto bbox_field_xmin(arrow::field("xmin", arrow::float32(), false));
617 auto bbox_field_ymin(arrow::field("ymin", arrow::float32(), false));
618 auto bbox_field_xmax(arrow::field("xmax", arrow::float32(), false));
619 auto bbox_field_ymax(arrow::field("ymax", arrow::float32(), false));
620 auto bbox_field(arrow::field(
621 m_oBBoxStructFieldName.empty()
622 ? std::string(poGeomFieldDefn->GetNameRef()).append("_bbox")
623 : m_oBBoxStructFieldName,
624 arrow::struct_(
625 {std::move(bbox_field_xmin), std::move(bbox_field_ymin),
626 std::move(bbox_field_xmax), std::move(bbox_field_ymax)}),
627 CPL_TO_BOOL(poGeomFieldDefn->IsNullable())));
628 fields.emplace_back(bbox_field);
629 m_apoFieldsBBOX.emplace_back(bbox_field);
630 }
631 }
632
633 m_aoEnvelopes.resize(m_poFeatureDefn->GetGeomFieldCount());
634 m_oSetWrittenGeometryTypes.resize(m_poFeatureDefn->GetGeomFieldCount());
635
636 m_poSchema = arrow::schema(std::move(fields));
637 CPLAssert(m_poSchema);
638 if (bNeedGDALSchema &&
639 CPLTestBool(CPLGetConfigOption(
640 ("OGR_" + GetDriverUCName() + "_WRITE_GDAL_SCHEMA").c_str(),
641 "YES")))
642 {
643 CPLJSONObject oRoot;
644 CPLJSONObject oColumns;
645
646 if (!m_osFIDColumn.empty())
647 oRoot.Add("fid", m_osFIDColumn);
648
649 oRoot.Add("columns", oColumns);
650 for (int i = 0; i < m_poFeatureDefn->GetFieldCount(); ++i)
651 {
652 const auto poFieldDefn = m_poFeatureDefn->GetFieldDefn(i);
653 CPLJSONObject oColumn;
654 oColumns.Add(poFieldDefn->GetNameRef(), oColumn);
655 oColumn.Add("type", OGR_GetFieldTypeName(poFieldDefn->GetType()));
656 const auto eSubDT = poFieldDefn->GetSubType();
657 if (eSubDT != OFSTNone)
658 oColumn.Add("subtype", OGR_GetFieldSubTypeName(eSubDT));
659 const int nWidth = poFieldDefn->GetWidth();
660 if (nWidth > 0)
661 oColumn.Add("width", nWidth);
662 const int nPrecision = poFieldDefn->GetPrecision();
663 if (nPrecision > 0)
664 oColumn.Add("precision", nPrecision);
665 if (poFieldDefn->GetAlternativeNameRef()[0])
666 oColumn.Add("alternative_name",
667 poFieldDefn->GetAlternativeNameRef());
668 if (!poFieldDefn->GetComment().empty())
669 oColumn.Add("comment", poFieldDefn->GetComment());
670 }
671
672 auto kvMetadata = m_poSchema->metadata()
673 ? m_poSchema->metadata()->Copy()
674 : std::make_shared<arrow::KeyValueMetadata>();
675 kvMetadata->Append("gdal:schema",
676 oRoot.Format(CPLJSONObject::PrettyFormat::Plain));
677 m_poSchema = m_poSchema->WithMetadata(kvMetadata);
678 CPLAssert(m_poSchema);
679 }
680 }
681
682 /************************************************************************/
683 /* FinalizeSchema() */
684 /************************************************************************/
685
686 inline void OGRArrowWriterLayer::FinalizeSchema()
687 {
688 // Final tuning of schema taking into actual timezone values
689 // from features
690 int nArrowIdxFirstField = !m_osFIDColumn.empty() ? 1 : 0;
691 for (int i = 0; i < m_poFeatureDefn->GetFieldCount(); ++i)
692 {
693 const auto poFieldDefn = m_poFeatureDefn->GetFieldDefn(i);
694 if (m_anTZFlag[i] >= OGR_TZFLAG_MIXED_TZ &&
695 poFieldDefn->GetTZFlag() != OGR_TZFLAG_MIXED_TZ &&
696 m_poSchema->field(nArrowIdxFirstField + i)->type()->id() !=
697 arrow::Type::STRUCT)
698 {
699 const int nOffset = m_anTZFlag[i] == OGR_TZFLAG_MIXED_TZ
700 ? 0
701 : (m_anTZFlag[i] - OGR_TZFLAG_UTC) * 15;
702 int nHours = static_cast<int>(nOffset / 60); // Round towards zero.
703 const int nMinutes = std::abs(nOffset - nHours * 60);
704
705 const std::string osTZ =
706 CPLSPrintf("%c%02d:%02d", nOffset >= 0 ? '+' : '-',
707 std::abs(nHours), nMinutes);
708 auto dt = arrow::timestamp(arrow::TimeUnit::MILLI, osTZ);
709 auto field = arrow::field(poFieldDefn->GetNameRef(), std::move(dt),
710 CPL_TO_BOOL(poFieldDefn->IsNullable()));
711 auto result = m_poSchema->SetField(nArrowIdxFirstField + i, field);
712 if (!result.ok())
713 {
714 CPLError(CE_Warning, CPLE_AppDefined,
715 "Schema::SetField() failed with %s",
716 result.status().message().c_str());
717 }
718 else
719 {
720 m_poSchema = *result;
721 }
722 }
723 }
724 }
725
726 /************************************************************************/
727 /* AddFieldDomain() */
728 /************************************************************************/
729
730 inline bool
731 OGRArrowWriterLayer::AddFieldDomain(std::unique_ptr<OGRFieldDomain> &&domain,
732 std::string &failureReason)
733 {
734 if (domain->GetDomainType() != OFDT_CODED)
735 {
736 failureReason = "Only coded field domains are supported by Arrow";
737 return false;
738 }
739
740 const OGRCodedFieldDomain *poDomain =
741 static_cast<const OGRCodedFieldDomain *>(domain.get());
742 const OGRCodedValue *psIter = poDomain->GetEnumeration();
743
744 auto poStringBuilder =
745 std::make_shared<arrow::StringBuilder>(m_poMemoryPool);
746
747 int nLastCode = -1;
748 for (; psIter->pszCode; ++psIter)
749 {
750 if (CPLGetValueType(psIter->pszCode) != CPL_VALUE_INTEGER)
751 {
752 failureReason = "Non integer code in domain ";
753 failureReason += domain->GetName();
754 return false;
755 }
756 int nCode = atoi(psIter->pszCode);
757 if (nCode <= nLastCode || nCode - nLastCode > 100)
758 {
759 failureReason = "Too sparse codes in domain ";
760 failureReason += domain->GetName();
761 return false;
762 }
763 for (int i = nLastCode + 1; i < nCode; ++i)
764 {
765 OGR_ARROW_RETURN_FALSE_NOT_OK(poStringBuilder->AppendNull());
766 }
767 if (psIter->pszValue)
768 OGR_ARROW_RETURN_FALSE_NOT_OK(
769 poStringBuilder->Append(psIter->pszValue));
770 else
771 OGR_ARROW_RETURN_FALSE_NOT_OK(poStringBuilder->AppendNull());
772 nLastCode = nCode;
773 }
774
775 std::shared_ptr<arrow::Array> stringArray;
776 auto status = poStringBuilder->Finish(&stringArray);
777 if (!status.ok())
778 {
779 CPLError(CE_Failure, CPLE_AppDefined,
780 "StringArray::Finish() failed with %s",
781 status.message().c_str());
782 return false;
783 }
784
785 m_oMapFieldDomainToStringArray[domain->GetName()] = std::move(stringArray);
786 m_oMapFieldDomains[domain->GetName()] = std::move(domain);
787 return true;
788 }
789
790 /************************************************************************/
791 /* GetFieldDomainNames() */
792 /************************************************************************/
793
794 inline std::vector<std::string> OGRArrowWriterLayer::GetFieldDomainNames() const
795 {
796 std::vector<std::string> names;
797 names.reserve(m_oMapFieldDomains.size());
798 for (const auto &it : m_oMapFieldDomains)
799 {
800 names.emplace_back(it.first);
801 }
802 return names;
803 }
804
805 /************************************************************************/
806 /* GetFieldDomain() */
807 /************************************************************************/
808
809 inline const OGRFieldDomain *
810 OGRArrowWriterLayer::GetFieldDomain(const std::string &name) const
811 {
812 const auto iter = m_oMapFieldDomains.find(name);
813 if (iter == m_oMapFieldDomains.end())
814 return nullptr;
815 return iter->second.get();
816 }
817
818 /************************************************************************/
819 /* CreateField() */
820 /************************************************************************/
821
822 inline OGRErr OGRArrowWriterLayer::CreateField(const OGRFieldDefn *poField,
823 int /* bApproxOK */)
824 {
825 if (m_poSchema)
826 {
827 CPLError(CE_Failure, CPLE_NotSupported,
828 "Cannot add field after a first feature has been written");
829 return OGRERR_FAILURE;
830 }
831 if (!m_apoFieldsFromArrowSchema.empty())
832 {
833 CPLError(CE_Failure, CPLE_NotSupported,
834 "Cannot mix calls to CreateField() and "
835 "CreateFieldFromArrowSchema()");
836 return OGRERR_FAILURE;
837 }
838 m_poFeatureDefn->AddFieldDefn(poField);
839 return OGRERR_NONE;
840 }
841
842 /************************************************************************/
843 /* OGRLayer::CreateFieldFromArrowSchema() */
844 /************************************************************************/
845
846 inline bool OGRArrowWriterLayer::CreateFieldFromArrowSchema(
847 const struct ArrowSchema *schema, CSLConstList /*papszOptions*/)
848 {
849 if (m_poSchema)
850 {
851 CPLError(CE_Failure, CPLE_NotSupported,
852 "Cannot add field after a first feature has been written");
853 return false;
854 }
855
856 if (m_poFeatureDefn->GetFieldCount())
857 {
858 CPLError(CE_Failure, CPLE_NotSupported,
859 "Cannot mix calls to CreateField() and "
860 "CreateFieldFromArrowSchema()");
861 return false;
862 }
863
864 if (m_osFIDColumn == schema->name)
865 {
866 CPLError(CE_Failure, CPLE_AppDefined,
867 "FID column has the same name as this field: %s",
868 schema->name);
869 return false;
870 }
871
872 for (auto &apoField : m_apoFieldsFromArrowSchema)
873 {
874 if (apoField->name() == schema->name)
875 {
876 CPLError(CE_Failure, CPLE_AppDefined,
877 "Field of name %s already exists", schema->name);
878 return false;
879 }
880 }
881
882 if (m_poFeatureDefn->GetGeomFieldIndex(schema->name) >= 0)
883 {
884 CPLError(CE_Failure, CPLE_AppDefined,
885 "Geometry field of name %s already exists", schema->name);
886 return false;
887 }
888
889 // ImportField() would release the schema, but we don't want that
890 // So copy the structure content into a local variable, and override its
891 // release callback to a no-op. This may be a bit fragile, but it doesn't
892 // look like ImportField implementation tries to access the C ArrowSchema
893 // after it has been called.
894 struct ArrowSchema lSchema = *schema;
895 const auto DummyFreeSchema = [](struct ArrowSchema *ptrSchema)
896 { ptrSchema->release = nullptr; };
897 lSchema.release = DummyFreeSchema;
898 auto result = arrow::ImportField(&lSchema);
899 CPLAssert(lSchema.release == nullptr);
900 if (!result.ok())
901 {
902 CPLError(CE_Failure, CPLE_AppDefined,
903 "CreateFieldFromArrowSchema() failed: %s",
904 result.status().message().c_str());
905 return false;
906 }
907 m_apoFieldsFromArrowSchema.emplace_back(std::move(*result));
908 return true;
909 }
910
911 /************************************************************************/
912 /* GetPreciseArrowGeomEncoding() */
913 /************************************************************************/
914
915 inline OGRArrowGeomEncoding OGRArrowWriterLayer::GetPreciseArrowGeomEncoding(
916 OGRArrowGeomEncoding eEncodingType, OGRwkbGeometryType eGType)
917 {
918 CPLAssert(eEncodingType == OGRArrowGeomEncoding::GEOARROW_FSL_GENERIC ||
919 eEncodingType == OGRArrowGeomEncoding::GEOARROW_STRUCT_GENERIC);
920 const auto eFlatType = wkbFlatten(eGType);
921 if (eFlatType == wkbPoint)
922 {
923 return eEncodingType == OGRArrowGeomEncoding::GEOARROW_FSL_GENERIC
924 ? OGRArrowGeomEncoding::GEOARROW_FSL_POINT
925 : OGRArrowGeomEncoding::GEOARROW_STRUCT_POINT;
926 }
927 else if (eFlatType == wkbLineString)
928 {
929 return eEncodingType == OGRArrowGeomEncoding::GEOARROW_FSL_GENERIC
930 ? OGRArrowGeomEncoding::GEOARROW_FSL_LINESTRING
931 : OGRArrowGeomEncoding::GEOARROW_STRUCT_LINESTRING;
932 }
933 else if (eFlatType == wkbPolygon)
934 {
935 return eEncodingType == OGRArrowGeomEncoding::GEOARROW_FSL_GENERIC
936 ? OGRArrowGeomEncoding::GEOARROW_FSL_POLYGON
937 : OGRArrowGeomEncoding::GEOARROW_STRUCT_POLYGON;
938 }
939 else if (eFlatType == wkbMultiPoint)
940 {
941 return eEncodingType == OGRArrowGeomEncoding::GEOARROW_FSL_GENERIC
942 ? OGRArrowGeomEncoding::GEOARROW_FSL_MULTIPOINT
943 : OGRArrowGeomEncoding::GEOARROW_STRUCT_MULTIPOINT;
944 }
945 else if (eFlatType == wkbMultiLineString)
946 {
947 return eEncodingType == OGRArrowGeomEncoding::GEOARROW_FSL_GENERIC
948 ? OGRArrowGeomEncoding::GEOARROW_FSL_MULTILINESTRING
949 : OGRArrowGeomEncoding::GEOARROW_STRUCT_MULTILINESTRING;
950 }
951 else if (eFlatType == wkbMultiPolygon)
952 {
953 return eEncodingType == OGRArrowGeomEncoding::GEOARROW_FSL_GENERIC
954 ? OGRArrowGeomEncoding::GEOARROW_FSL_MULTIPOLYGON
955 : OGRArrowGeomEncoding::GEOARROW_STRUCT_MULTIPOLYGON;
956 }
957 else
958 {
959 CPLError(CE_Failure, CPLE_NotSupported,
960 "GeoArrow encoding is currently not supported for %s",
961 OGRGeometryTypeToName(eGType));
962 return eEncodingType;
963 }
964 }
965
966 /************************************************************************/
967 /* GetGeomEncodingAsString() */
968 /************************************************************************/
969
970 inline const char *
971 OGRArrowWriterLayer::GetGeomEncodingAsString(OGRArrowGeomEncoding eGeomEncoding,
972 bool bForParquetGeo)
973 {
974 switch (eGeomEncoding)
975 {
976 case OGRArrowGeomEncoding::WKB:
977 return bForParquetGeo ? "WKB" : "geoarrow.wkb";
978 case OGRArrowGeomEncoding::WKT:
979 return bForParquetGeo ? "WKT" : "geoarrow.wkt";
980 case OGRArrowGeomEncoding::GEOARROW_FSL_GENERIC:
981 case OGRArrowGeomEncoding::GEOARROW_STRUCT_GENERIC:
982 CPLAssert(false);
983 break;
984 case OGRArrowGeomEncoding::GEOARROW_FSL_POINT:
985 return "geoarrow.point";
986 case OGRArrowGeomEncoding::GEOARROW_FSL_LINESTRING:
987 return "geoarrow.linestring";
988 case OGRArrowGeomEncoding::GEOARROW_FSL_POLYGON:
989 return "geoarrow.polygon";
990 case OGRArrowGeomEncoding::GEOARROW_FSL_MULTIPOINT:
991 return "geoarrow.multipoint";
992 case OGRArrowGeomEncoding::GEOARROW_FSL_MULTILINESTRING:
993 return "geoarrow.multilinestring";
994 case OGRArrowGeomEncoding::GEOARROW_FSL_MULTIPOLYGON:
995 return "geoarrow.multipolygon";
996 case OGRArrowGeomEncoding::GEOARROW_STRUCT_POINT:
997 return bForParquetGeo ? "point" : "geoarrow.point";
998 case OGRArrowGeomEncoding::GEOARROW_STRUCT_LINESTRING:
999 return bForParquetGeo ? "linestring" : "geoarrow.linestring";
1000 case OGRArrowGeomEncoding::GEOARROW_STRUCT_POLYGON:
1001 return bForParquetGeo ? "polygon" : "geoarrow.polygon";
1002 case OGRArrowGeomEncoding::GEOARROW_STRUCT_MULTIPOINT:
1003 return bForParquetGeo ? "multipoint" : "geoarrow.multipoint";
1004 case OGRArrowGeomEncoding::GEOARROW_STRUCT_MULTILINESTRING:
1005 return bForParquetGeo ? "multilinestring"
1006 : "geoarrow.multilinestring";
1007 case OGRArrowGeomEncoding::GEOARROW_STRUCT_MULTIPOLYGON:
1008 return bForParquetGeo ? "multipolygon" : "geoarrow.multipolygon";
1009 }
1010 return nullptr;
1011 }
1012
1013 /************************************************************************/
1014 /* CreateGeomField() */
1015 /************************************************************************/
1016
1017 inline OGRErr
1018 OGRArrowWriterLayer::CreateGeomField(const OGRGeomFieldDefn *poField,
1019 int /* bApproxOK */)
1020 {
1021 if (m_poSchema)
1022 {
1023 CPLError(CE_Failure, CPLE_NotSupported,
1024 "Cannot add field after a first feature has been written");
1025 return OGRERR_FAILURE;
1026 }
1027 const auto eGType = poField->GetType();
1028 if (!IsSupportedGeometryType(eGType))
1029 {
1030 return OGRERR_FAILURE;
1031 }
1032
1033 if (IsSRSRequired() && poField->GetSpatialRef() == nullptr)
1034 {
1035 CPLError(CE_Warning, CPLE_AppDefined,
1036 "Geometry column should have an associated CRS");
1037 }
1038 auto eGeomEncoding = m_eGeomEncoding;
1039 if (eGeomEncoding == OGRArrowGeomEncoding::GEOARROW_FSL_GENERIC ||
1040 eGeomEncoding == OGRArrowGeomEncoding::GEOARROW_STRUCT_GENERIC)
1041 {
1042 const auto eEncodingType = eGeomEncoding;
1043 eGeomEncoding = GetPreciseArrowGeomEncoding(eEncodingType, eGType);
1044 if (eGeomEncoding == eEncodingType)
1045 return OGRERR_FAILURE;
1046 }
1047 m_aeGeomEncoding.push_back(eGeomEncoding);
1048 m_poFeatureDefn->AddGeomFieldDefn(poField);
1049 return OGRERR_NONE;
1050 }
1051
1052 /************************************************************************/
1053 /* MakeGeoArrowBuilder() */
1054 /************************************************************************/
1055
1056 static std::shared_ptr<arrow::ArrayBuilder>
1057 MakeGeoArrowBuilder(arrow::MemoryPool *poMemoryPool, int nDim, int nDepth)
1058 {
1059 if (nDepth == 0)
1060 return std::make_shared<arrow::FixedSizeListBuilder>(
1061 poMemoryPool, std::make_shared<arrow::DoubleBuilder>(poMemoryPool),
1062 nDim);
1063 else
1064 return std::make_shared<arrow::ListBuilder>(
1065 poMemoryPool, MakeGeoArrowBuilder(poMemoryPool, nDim, nDepth - 1));
1066 }
1067
1068 /************************************************************************/
1069 /* MakeGeoArrowStructBuilder() */
1070 /************************************************************************/
1071
1072 static std::shared_ptr<arrow::ArrayBuilder>
1073 MakeGeoArrowStructBuilder(arrow::MemoryPool *poMemoryPool, int nDim, int nDepth,
1074 const std::shared_ptr<arrow::DataType> &eBaseType)
1075 {
1076 if (nDepth == 0)
1077 {
1078 std::vector<std::shared_ptr<arrow::ArrayBuilder>> builders;
1079 for (int i = 0; i < nDim; ++i)
1080 builders.emplace_back(
1081 std::make_shared<arrow::DoubleBuilder>(poMemoryPool));
1082 return std::make_shared<arrow::StructBuilder>(eBaseType, poMemoryPool,
1083 std::move(builders));
1084 }
1085 else
1086 return std::make_shared<arrow::ListBuilder>(
1087 poMemoryPool, MakeGeoArrowStructBuilder(poMemoryPool, nDim,
1088 nDepth - 1, eBaseType));
1089 }
1090
1091 /************************************************************************/
1092 /* ClearArrayBuilers() */
1093 /************************************************************************/
1094
1095 inline void OGRArrowWriterLayer::ClearArrayBuilers()
1096 {
1097 m_apoBuilders.clear();
1098 m_apoBuildersBBOXStruct.clear();
1099 m_apoBuildersBBOXXMin.clear();
1100 m_apoBuildersBBOXYMin.clear();
1101 m_apoBuildersBBOXXMax.clear();
1102 m_apoBuildersBBOXYMax.clear();
1103 }
1104
1105 /************************************************************************/
1106 /* CreateArrayBuilders() */
1107 /************************************************************************/
1108
1109 inline void OGRArrowWriterLayer::CreateArrayBuilders()
1110 {
1111 m_apoBuilders.reserve(1 + m_poFeatureDefn->GetFieldCount() +
1112 m_poFeatureDefn->GetGeomFieldCount());
1113
1114 int nArrowIdx = 0;
1115 if (!m_osFIDColumn.empty())
1116 {
1117 m_apoBuilders.emplace_back(std::make_shared<arrow::Int64Builder>());
1118 nArrowIdx++;
1119 }
1120
1121 for (int i = 0; i < m_poFeatureDefn->GetFieldCount(); ++i, ++nArrowIdx)
1122 {
1123 const auto poFieldDefn = m_poFeatureDefn->GetFieldDefn(i);
1124 std::shared_ptr<arrow::ArrayBuilder> builder;
1125 const auto eSubDT = poFieldDefn->GetSubType();
1126 switch (poFieldDefn->GetType())
1127 {
1128 case OFTInteger:
1129 if (eSubDT == OFSTBoolean)
1130 builder =
1131 std::make_shared<arrow::BooleanBuilder>(m_poMemoryPool);
1132 else if (eSubDT == OFSTInt16)
1133 builder =
1134 std::make_shared<arrow::Int16Builder>(m_poMemoryPool);
1135 else
1136 builder =
1137 std::make_shared<arrow::Int32Builder>(m_poMemoryPool);
1138 break;
1139
1140 case OFTInteger64:
1141 builder = std::make_shared<arrow::Int64Builder>(m_poMemoryPool);
1142 break;
1143
1144 case OFTReal:
1145 {
1146 const auto arrowType = m_poSchema->fields()[nArrowIdx]->type();
1147 if (arrowType->id() == arrow::Type::DECIMAL128)
1148 builder = std::make_shared<arrow::Decimal128Builder>(
1149 arrowType, m_poMemoryPool);
1150 else if (arrowType->id() == arrow::Type::DECIMAL256)
1151 builder = std::make_shared<arrow::Decimal256Builder>(
1152 arrowType, m_poMemoryPool);
1153 else if (eSubDT == OFSTFloat32)
1154 builder =
1155 std::make_shared<arrow::FloatBuilder>(m_poMemoryPool);
1156 else
1157 builder =
1158 std::make_shared<arrow::DoubleBuilder>(m_poMemoryPool);
1159 break;
1160 }
1161
1162 case OFTString:
1163 case OFTWideString:
1164 builder =
1165 std::make_shared<arrow::StringBuilder>(m_poMemoryPool);
1166 break;
1167
1168 case OFTBinary:
1169 if (poFieldDefn->GetWidth() != 0)
1170 builder = std::make_shared<arrow::FixedSizeBinaryBuilder>(
1171 arrow::fixed_size_binary(poFieldDefn->GetWidth()),
1172 m_poMemoryPool);
1173 else
1174 builder =
1175 std::make_shared<arrow::BinaryBuilder>(m_poMemoryPool);
1176 break;
1177
1178 case OFTIntegerList:
1179 {
1180 std::shared_ptr<arrow::ArrayBuilder> poBaseBuilder;
1181 if (eSubDT == OFSTBoolean)
1182 poBaseBuilder =
1183 std::make_shared<arrow::BooleanBuilder>(m_poMemoryPool);
1184 else if (eSubDT == OFSTInt16)
1185 poBaseBuilder =
1186 std::make_shared<arrow::Int16Builder>(m_poMemoryPool);
1187 else
1188 poBaseBuilder =
1189 std::make_shared<arrow::Int32Builder>(m_poMemoryPool);
1190 builder = std::make_shared<arrow::ListBuilder>(m_poMemoryPool,
1191 poBaseBuilder);
1192 break;
1193 }
1194
1195 case OFTInteger64List:
1196 builder = std::make_shared<arrow::ListBuilder>(
1197 m_poMemoryPool,
1198 std::make_shared<arrow::Int64Builder>(m_poMemoryPool));
1199
1200 break;
1201
1202 case OFTRealList:
1203 if (eSubDT == OFSTFloat32)
1204 builder = std::make_shared<arrow::ListBuilder>(
1205 m_poMemoryPool,
1206 std::make_shared<arrow::FloatBuilder>(m_poMemoryPool));
1207 else
1208 builder = std::make_shared<arrow::ListBuilder>(
1209 m_poMemoryPool,
1210 std::make_shared<arrow::DoubleBuilder>(m_poMemoryPool));
1211 break;
1212
1213 case OFTStringList:
1214 case OFTWideStringList:
1215 builder = std::make_shared<arrow::ListBuilder>(
1216 m_poMemoryPool,
1217 std::make_shared<arrow::StringBuilder>(m_poMemoryPool));
1218
1219 break;
1220
1221 case OFTDate:
1222 builder =
1223 std::make_shared<arrow::Date32Builder>(m_poMemoryPool);
1224 break;
1225
1226 case OFTTime:
1227 builder = std::make_shared<arrow::Time32Builder>(
1228 arrow::time32(arrow::TimeUnit::MILLI), m_poMemoryPool);
1229 break;
1230
1231 case OFTDateTime:
1232 {
1233 const auto arrowType = m_poSchema->fields()[nArrowIdx]->type();
1234 if (arrowType->id() == arrow::Type::STRUCT)
1235 {
1236 builder = std::make_shared<arrow::StructBuilder>(
1237 arrowType, m_poMemoryPool,
1238 std::vector<std::shared_ptr<arrow::ArrayBuilder>>{
1239 std::make_shared<arrow::TimestampBuilder>(
1240 arrow::timestamp(arrow::TimeUnit::MILLI),
1241 m_poMemoryPool),
1242 std::make_shared<arrow::Int16Builder>(
1243 m_poMemoryPool)});
1244 }
1245 else
1246 {
1247 builder = std::make_shared<arrow::TimestampBuilder>(
1248 arrow::timestamp(arrow::TimeUnit::MILLI),
1249 m_poMemoryPool);
1250 }
1251 break;
1252 }
1253 }
1254 m_apoBuilders.emplace_back(builder);
1255 }
1256
1257 for (int i = 0; i < m_poFeatureDefn->GetGeomFieldCount(); ++i, ++nArrowIdx)
1258 {
1259 std::shared_ptr<arrow::ArrayBuilder> builder;
1260 const auto poGeomFieldDefn = m_poFeatureDefn->GetGeomFieldDefn(i);
1261 const auto eGType = poGeomFieldDefn->GetType();
1262 const int nDim =
1263 2 + (OGR_GT_HasZ(eGType) ? 1 : 0) + (OGR_GT_HasM(eGType) ? 1 : 0);
1264
1265 switch (m_aeGeomEncoding[i])
1266 {
1267 case OGRArrowGeomEncoding::WKB:
1268 builder =
1269 std::make_shared<arrow::BinaryBuilder>(m_poMemoryPool);
1270 break;
1271
1272 case OGRArrowGeomEncoding::WKT:
1273 builder =
1274 std::make_shared<arrow::StringBuilder>(m_poMemoryPool);
1275 break;
1276
1277 case OGRArrowGeomEncoding::GEOARROW_FSL_POINT:
1278 builder = MakeGeoArrowBuilder(m_poMemoryPool, nDim, 0);
1279 break;
1280
1281 case OGRArrowGeomEncoding::GEOARROW_FSL_LINESTRING:
1282 builder = MakeGeoArrowBuilder(m_poMemoryPool, nDim, 1);
1283 break;
1284
1285 case OGRArrowGeomEncoding::GEOARROW_FSL_POLYGON:
1286 builder = MakeGeoArrowBuilder(m_poMemoryPool, nDim, 2);
1287 break;
1288
1289 case OGRArrowGeomEncoding::GEOARROW_FSL_MULTIPOINT:
1290 builder = MakeGeoArrowBuilder(m_poMemoryPool, nDim, 1);
1291 break;
1292
1293 case OGRArrowGeomEncoding::GEOARROW_FSL_MULTILINESTRING:
1294 builder = MakeGeoArrowBuilder(m_poMemoryPool, nDim, 2);
1295 break;
1296
1297 case OGRArrowGeomEncoding::GEOARROW_FSL_MULTIPOLYGON:
1298 builder = MakeGeoArrowBuilder(m_poMemoryPool, nDim, 3);
1299 break;
1300
1301 case OGRArrowGeomEncoding::GEOARROW_STRUCT_POINT:
1302 builder = MakeGeoArrowStructBuilder(m_poMemoryPool, nDim, 0,
1303 m_apoBaseStructGeomType[i]);
1304 break;
1305
1306 case OGRArrowGeomEncoding::GEOARROW_STRUCT_LINESTRING:
1307 builder = MakeGeoArrowStructBuilder(m_poMemoryPool, nDim, 1,
1308 m_apoBaseStructGeomType[i]);
1309 break;
1310
1311 case OGRArrowGeomEncoding::GEOARROW_STRUCT_POLYGON:
1312 builder = MakeGeoArrowStructBuilder(m_poMemoryPool, nDim, 2,
1313 m_apoBaseStructGeomType[i]);
1314 break;
1315
1316 case OGRArrowGeomEncoding::GEOARROW_STRUCT_MULTIPOINT:
1317 builder = MakeGeoArrowStructBuilder(m_poMemoryPool, nDim, 1,
1318 m_apoBaseStructGeomType[i]);
1319 break;
1320
1321 case OGRArrowGeomEncoding::GEOARROW_STRUCT_MULTILINESTRING:
1322 builder = MakeGeoArrowStructBuilder(m_poMemoryPool, nDim, 2,
1323 m_apoBaseStructGeomType[i]);
1324 break;
1325
1326 case OGRArrowGeomEncoding::GEOARROW_STRUCT_MULTIPOLYGON:
1327 builder = MakeGeoArrowStructBuilder(m_poMemoryPool, nDim, 3,
1328 m_apoBaseStructGeomType[i]);
1329 break;
1330
1331 case OGRArrowGeomEncoding::GEOARROW_FSL_GENERIC:
1332 case OGRArrowGeomEncoding::GEOARROW_STRUCT_GENERIC:
1333 CPLAssert(false);
1334 break;
1335 }
1336
1337 m_apoBuilders.emplace_back(builder);
1338
1339 if (m_bWriteBBoxStruct)
1340 {
1341 m_apoBuildersBBOXXMin.emplace_back(
1342 std::make_shared<arrow::FloatBuilder>(m_poMemoryPool));
1343 m_apoBuildersBBOXYMin.emplace_back(
1344 std::make_shared<arrow::FloatBuilder>(m_poMemoryPool));
1345 m_apoBuildersBBOXXMax.emplace_back(
1346 std::make_shared<arrow::FloatBuilder>(m_poMemoryPool));
1347 m_apoBuildersBBOXYMax.emplace_back(
1348 std::make_shared<arrow::FloatBuilder>(m_poMemoryPool));
1349 m_apoBuildersBBOXStruct.emplace_back(
1350 std::make_shared<arrow::StructBuilder>(
1351 m_apoFieldsBBOX[i]->type(), m_poMemoryPool,
1352 std::vector<std::shared_ptr<arrow::ArrayBuilder>>{
1353 m_apoBuildersBBOXXMin.back(),
1354 m_apoBuildersBBOXYMin.back(),
1355 m_apoBuildersBBOXXMax.back(),
1356 m_apoBuildersBBOXYMax.back()}));
1357 }
1358 }
1359 }
1360
1361 /************************************************************************/
1362 /* castToFloatDown() */
1363 /************************************************************************/
1364
1365 // Cf https://github.com/sqlite/sqlite/blob/90e4a3b7fcdf63035d6f35eb44d11ff58ff4b068/ext/rtree/rtree.c#L2993C1-L2995C3
1366 /*
1367 ** Rounding constants for float->double conversion.
1368 */
1369 #define RNDTOWARDS (1.0 - 1.0 / 8388608.0) /* Round towards zero */
1370 #define RNDAWAY (1.0 + 1.0 / 8388608.0) /* Round away from zero */
1371
1372 /*
1373 ** Convert an sqlite3_value into an RtreeValue (presumably a float)
1374 ** while taking care to round toward negative or positive, respectively.
1375 */
1376 static float castToFloatDown(double d)
1377 {
1378 float f = static_cast<float>(d);
1379 if (f > d)
1380 {
1381 f = static_cast<float>(d * (d < 0 ? RNDAWAY : RNDTOWARDS));
1382 }
1383 return f;
1384 }
1385
1386 static float castToFloatUp(double d)
1387 {
1388 float f = static_cast<float>(d);
1389 if (f < d)
1390 {
1391 f = static_cast<float>(d * (d < 0 ? RNDTOWARDS : RNDAWAY));
1392 }
1393 return f;
1394 }
1395
1396 /************************************************************************/
1397 /* GeoArrowLineBuilder() */
1398 /************************************************************************/
1399
1400 template <class PointBuilderType>
1401 static OGRErr GeoArrowLineBuilder(const OGRLineString *poLS,
1402 PointBuilderType *poPointBuilder,
1403 arrow::DoubleBuilder *poXBuilder,
1404 arrow::DoubleBuilder *poYBuilder,
1405 arrow::DoubleBuilder *poZBuilder,
1406 arrow::DoubleBuilder *poMBuilder)
1407 {
1408 for (int j = 0; j < poLS->getNumPoints(); ++j)
1409 {
1410 OGR_ARROW_RETURN_OGRERR_NOT_OK(poPointBuilder->Append());
1411 OGR_ARROW_RETURN_OGRERR_NOT_OK(poXBuilder->Append(poLS->getX(j)));
1412 OGR_ARROW_RETURN_OGRERR_NOT_OK(poYBuilder->Append(poLS->getY(j)));
1413 if (poZBuilder)
1414 OGR_ARROW_RETURN_OGRERR_NOT_OK(poZBuilder->Append(poLS->getZ(j)));
1415 if (poMBuilder)
1416 OGR_ARROW_RETURN_OGRERR_NOT_OK(poMBuilder->Append(poLS->getM(j)));
1417 }
1418 return OGRERR_NONE;
1419 }
1420
1421 /************************************************************************/
1422 /* BuildGeometry() */
1423 /************************************************************************/
1424
1425 inline OGRErr OGRArrowWriterLayer::BuildGeometry(OGRGeometry *poGeom,
1426 int iGeomField,
1427 arrow::ArrayBuilder *poBuilder)
1428 {
1429 const auto eGType = poGeom ? poGeom->getGeometryType() : wkbNone;
1430 const auto eColumnGType =
1431 m_poFeatureDefn->GetGeomFieldDefn(iGeomField)->GetType();
1432 const bool bHasZ = CPL_TO_BOOL(OGR_GT_HasZ(eColumnGType));
1433 const bool bHasM = CPL_TO_BOOL(OGR_GT_HasM(eColumnGType));
1434 const bool bIsEmpty = poGeom != nullptr && poGeom->IsEmpty();
1435 OGREnvelope3D oEnvelope;
1436 if (poGeom != nullptr && !bIsEmpty)
1437 {
1438 if (poGeom->Is3D())
1439 {
1440 poGeom->getEnvelope(&oEnvelope);
1441 m_aoEnvelopes[iGeomField].Merge(oEnvelope);
1442 }
1443 else
1444 {
1445 poGeom->getEnvelope(static_cast<OGREnvelope *>(&oEnvelope));
1446 m_aoEnvelopes[iGeomField].Merge(oEnvelope);
1447 }
1448 m_oSetWrittenGeometryTypes[iGeomField].insert(eGType);
1449 }
1450
1451 if (m_bWriteBBoxStruct)
1452 {
1453 if (poGeom && !bIsEmpty)
1454 {
1455 OGR_ARROW_RETURN_OGRERR_NOT_OK(
1456 m_apoBuildersBBOXXMin[iGeomField]->Append(
1457 castToFloatDown(oEnvelope.MinX)));
1458 OGR_ARROW_RETURN_OGRERR_NOT_OK(
1459 m_apoBuildersBBOXYMin[iGeomField]->Append(
1460 castToFloatDown(oEnvelope.MinY)));
1461 OGR_ARROW_RETURN_OGRERR_NOT_OK(
1462 m_apoBuildersBBOXXMax[iGeomField]->Append(
1463 castToFloatUp(oEnvelope.MaxX)));
1464 OGR_ARROW_RETURN_OGRERR_NOT_OK(
1465 m_apoBuildersBBOXYMax[iGeomField]->Append(
1466 castToFloatUp(oEnvelope.MaxY)));
1467 OGR_ARROW_RETURN_OGRERR_NOT_OK(
1468 m_apoBuildersBBOXStruct[iGeomField]->Append());
1469 }
1470 else
1471 {
1472 OGR_ARROW_RETURN_OGRERR_NOT_OK(
1473 m_apoBuildersBBOXStruct[iGeomField]->AppendNull());
1474 }
1475 }
1476
1477 if (poGeom == nullptr)
1478 {
1479 if (m_aeGeomEncoding[iGeomField] ==
1480 OGRArrowGeomEncoding::GEOARROW_FSL_POINT &&
1481 GetDriverUCName() == "PARQUET")
1482 {
1483 // For some reason, Parquet doesn't support a NULL FixedSizeList
1484 // on reading
1485 auto poPointBuilder =
1486 static_cast<arrow::FixedSizeListBuilder *>(poBuilder);
1487 OGR_ARROW_RETURN_OGRERR_NOT_OK(poPointBuilder->Append());
1488 auto poValueBuilder = static_cast<arrow::DoubleBuilder *>(
1489 poPointBuilder->value_builder());
1490 OGR_ARROW_RETURN_OGRERR_NOT_OK(poValueBuilder->Append(
1491 std::numeric_limits<double>::quiet_NaN()));
1492 OGR_ARROW_RETURN_OGRERR_NOT_OK(poValueBuilder->Append(
1493 std::numeric_limits<double>::quiet_NaN()));
1494 if (bHasZ)
1495 OGR_ARROW_RETURN_OGRERR_NOT_OK(poValueBuilder->Append(
1496 std::numeric_limits<double>::quiet_NaN()));
1497 if (bHasM)
1498 OGR_ARROW_RETURN_OGRERR_NOT_OK(poValueBuilder->Append(
1499 std::numeric_limits<double>::quiet_NaN()));
1500 }
1501 else
1502 {
1503 OGR_ARROW_RETURN_OGRERR_NOT_OK(poBuilder->AppendNull());
1504 }
1505
1506 return OGRERR_NONE;
1507 }
1508
1509 // The following checks are only valid for GeoArrow encoding
1510 if (m_aeGeomEncoding[iGeomField] != OGRArrowGeomEncoding::WKB &&
1511 m_aeGeomEncoding[iGeomField] != OGRArrowGeomEncoding::WKT)
1512 {
1513 if ((!bIsEmpty && eGType != eColumnGType) ||
1514 (bIsEmpty && wkbFlatten(eGType) != wkbFlatten(eColumnGType)))
1515 {
1516 CPLError(CE_Warning, CPLE_AppDefined,
1517 "Geometry of type %s found, whereas %s is expected. "
1518 "Writing null geometry",
1519 OGRGeometryTypeToName(eGType),
1520 OGRGeometryTypeToName(eColumnGType));
1521 OGR_ARROW_RETURN_OGRERR_NOT_OK(poBuilder->AppendNull());
1522
1523 return OGRERR_NONE;
1524 }
1525 }
1526
1527 switch (m_aeGeomEncoding[iGeomField])
1528 {
1529 case OGRArrowGeomEncoding::WKB:
1530 {
1531 std::unique_ptr<OGRGeometry> poGeomModified;
1532 if (OGR_GT_HasM(eGType) && !OGR_GT_HasM(eColumnGType))
1533 {
1534 static bool bHasWarned = false;
1535 if (!bHasWarned)
1536 {
1537 CPLError(CE_Warning, CPLE_AppDefined,
1538 "Removing M component from geometry");
1539 bHasWarned = true;
1540 }
1541 poGeomModified.reset(poGeom->clone());
1542 poGeomModified->setMeasured(false);
1543 poGeom = poGeomModified.get();
1544 }
1545 FixupGeometryBeforeWriting(poGeom);
1546 const auto nSize = poGeom->WkbSize();
1547 if (nSize < INT_MAX)
1548 {
1549 m_abyBuffer.resize(nSize);
1550 poGeom->exportToWkb(wkbNDR, &m_abyBuffer[0], wkbVariantIso);
1551 OGR_ARROW_RETURN_OGRERR_NOT_OK(
1552 static_cast<arrow::BinaryBuilder *>(poBuilder)->Append(
1553 m_abyBuffer.data(),
1554 static_cast<int>(m_abyBuffer.size())));
1555 }
1556 else
1557 {
1558 CPLError(CE_Warning, CPLE_AppDefined,
1559 "Too big geometry. "
1560 "Writing null geometry");
1561 OGR_ARROW_RETURN_OGRERR_NOT_OK(poBuilder->AppendNull());
1562 }
1563 break;
1564 }
1565
1566 case OGRArrowGeomEncoding::WKT:
1567 {
1568 OGRWktOptions options;
1569 options.variant = wkbVariantIso;
1570 if (m_nWKTCoordinatePrecision >= 0)
1571 {
1572 options.format = OGRWktFormat::F;
1573 options.xyPrecision = m_nWKTCoordinatePrecision;
1574 options.zPrecision = m_nWKTCoordinatePrecision;
1575 options.mPrecision = m_nWKTCoordinatePrecision;
1576 }
1577 OGR_ARROW_RETURN_OGRERR_NOT_OK(
1578 static_cast<arrow::StringBuilder *>(poBuilder)->Append(
1579 poGeom->exportToWkt(options)));
1580 break;
1581 }
1582
1583 case OGRArrowGeomEncoding::GEOARROW_FSL_POINT:
1584 {
1585 const auto poPoint = poGeom->toPoint();
1586 auto poPointBuilder =
1587 static_cast<arrow::FixedSizeListBuilder *>(poBuilder);
1588 OGR_ARROW_RETURN_OGRERR_NOT_OK(poPointBuilder->Append());
1589 auto poValueBuilder = static_cast<arrow::DoubleBuilder *>(
1590 poPointBuilder->value_builder());
1591 if (bIsEmpty)
1592 {
1593 OGR_ARROW_RETURN_OGRERR_NOT_OK(poValueBuilder->Append(
1594 std::numeric_limits<double>::quiet_NaN()));
1595 OGR_ARROW_RETURN_OGRERR_NOT_OK(poValueBuilder->Append(
1596 std::numeric_limits<double>::quiet_NaN()));
1597 if (bHasZ)
1598 OGR_ARROW_RETURN_OGRERR_NOT_OK(poValueBuilder->Append(
1599 std::numeric_limits<double>::quiet_NaN()));
1600 if (bHasM)
1601 OGR_ARROW_RETURN_OGRERR_NOT_OK(poValueBuilder->Append(
1602 std::numeric_limits<double>::quiet_NaN()));
1603 }
1604 else
1605 {
1606 OGR_ARROW_RETURN_OGRERR_NOT_OK(
1607 poValueBuilder->Append(poPoint->getX()));
1608 OGR_ARROW_RETURN_OGRERR_NOT_OK(
1609 poValueBuilder->Append(poPoint->getY()));
1610 if (bHasZ)
1611 OGR_ARROW_RETURN_OGRERR_NOT_OK(
1612 poValueBuilder->Append(poPoint->getZ()));
1613 if (bHasM)
1614 OGR_ARROW_RETURN_OGRERR_NOT_OK(
1615 poValueBuilder->Append(poPoint->getM()));
1616 }
1617 break;
1618 }
1619
1620 #define GET_XYZM_STRUCT_FIELD_BUILDERS_FROM(poPointBuilder) \
1621 auto poXBuilder = \
1622 static_cast<arrow::DoubleBuilder *>(poPointBuilder->field_builder(0)); \
1623 auto poYBuilder = \
1624 static_cast<arrow::DoubleBuilder *>(poPointBuilder->field_builder(1)); \
1625 int iSubField = 2; \
1626 arrow::DoubleBuilder *poZBuilder = nullptr; \
1627 if (bHasZ) \
1628 { \
1629 poZBuilder = static_cast<arrow::DoubleBuilder *>( \
1630 poPointBuilder->field_builder(iSubField)); \
1631 ++iSubField; \
1632 } \
1633 arrow::DoubleBuilder *poMBuilder = nullptr; \
1634 if (bHasM) \
1635 { \
1636 poMBuilder = static_cast<arrow::DoubleBuilder *>( \
1637 poPointBuilder->field_builder(iSubField)); \
1638 } \
1639 do \
1640 { \
1641 } while (0)
1642
1643 case OGRArrowGeomEncoding::GEOARROW_STRUCT_POINT:
1644 {
1645 const auto poPoint = poGeom->toPoint();
1646 auto poPointBuilder =
1647 static_cast<arrow::StructBuilder *>(poBuilder);
1648 GET_XYZM_STRUCT_FIELD_BUILDERS_FROM(poPointBuilder);
1649 OGR_ARROW_RETURN_OGRERR_NOT_OK(poPointBuilder->Append());
1650
1651 if (bIsEmpty)
1652 {
1653 OGR_ARROW_RETURN_OGRERR_NOT_OK(poXBuilder->Append(
1654 std::numeric_limits<double>::quiet_NaN()));
1655 OGR_ARROW_RETURN_OGRERR_NOT_OK(poYBuilder->Append(
1656 std::numeric_limits<double>::quiet_NaN()));
1657 }
1658 else
1659 {
1660 OGR_ARROW_RETURN_OGRERR_NOT_OK(
1661 poXBuilder->Append(poPoint->getX()));
1662 OGR_ARROW_RETURN_OGRERR_NOT_OK(
1663 poYBuilder->Append(poPoint->getY()));
1664 }
1665 if (poZBuilder)
1666 {
1667 OGR_ARROW_RETURN_OGRERR_NOT_OK(poZBuilder->Append(
1668 bIsEmpty ? std::numeric_limits<double>::quiet_NaN()
1669 : poPoint->getZ()));
1670 }
1671 if (poMBuilder)
1672 {
1673 OGR_ARROW_RETURN_OGRERR_NOT_OK(poMBuilder->Append(
1674 bIsEmpty ? std::numeric_limits<double>::quiet_NaN()
1675 : poPoint->getM()));
1676 }
1677 break;
1678 }
1679
1680 case OGRArrowGeomEncoding::GEOARROW_FSL_LINESTRING:
1681 {
1682 const auto poLS = poGeom->toLineString();
1683 auto poListBuilder = static_cast<arrow::ListBuilder *>(poBuilder);
1684 auto poPointBuilder = static_cast<arrow::FixedSizeListBuilder *>(
1685 poListBuilder->value_builder());
1686 auto poValueBuilder = static_cast<arrow::DoubleBuilder *>(
1687 poPointBuilder->value_builder());
1688
1689 OGR_ARROW_RETURN_OGRERR_NOT_OK(poListBuilder->Append());
1690 OGR_ARROW_PROPAGATE_OGRERR(GeoArrowLineBuilder(
1691 poLS, poPointBuilder, poValueBuilder, poValueBuilder,
1692 bHasZ ? poValueBuilder : nullptr,
1693 bHasM ? poValueBuilder : nullptr));
1694 break;
1695 }
1696
1697 case OGRArrowGeomEncoding::GEOARROW_STRUCT_LINESTRING:
1698 {
1699 const auto poLS = poGeom->toLineString();
1700 auto poListBuilder = static_cast<arrow::ListBuilder *>(poBuilder);
1701 auto poPointBuilder = static_cast<arrow::StructBuilder *>(
1702 poListBuilder->value_builder());
1703 GET_XYZM_STRUCT_FIELD_BUILDERS_FROM(poPointBuilder);
1704
1705 OGR_ARROW_RETURN_OGRERR_NOT_OK(poListBuilder->Append());
1706 OGR_ARROW_PROPAGATE_OGRERR(
1707 GeoArrowLineBuilder(poLS, poPointBuilder, poXBuilder,
1708 poYBuilder, poZBuilder, poMBuilder));
1709 break;
1710 }
1711
1712 case OGRArrowGeomEncoding::GEOARROW_FSL_POLYGON:
1713 {
1714 const auto poPolygon = poGeom->toPolygon();
1715 auto poPolygonBuilder =
1716 static_cast<arrow::ListBuilder *>(poBuilder);
1717 auto poRingBuilder = static_cast<arrow::ListBuilder *>(
1718 poPolygonBuilder->value_builder());
1719 auto poPointBuilder = static_cast<arrow::FixedSizeListBuilder *>(
1720 poRingBuilder->value_builder());
1721 auto poValueBuilder = static_cast<arrow::DoubleBuilder *>(
1722 poPointBuilder->value_builder());
1723 OGR_ARROW_RETURN_OGRERR_NOT_OK(poPolygonBuilder->Append());
1724 for (const auto *poRing : *poPolygon)
1725 {
1726 OGR_ARROW_RETURN_OGRERR_NOT_OK(poRingBuilder->Append());
1727 OGR_ARROW_PROPAGATE_OGRERR(GeoArrowLineBuilder(
1728 poRing, poPointBuilder, poValueBuilder, poValueBuilder,
1729 bHasZ ? poValueBuilder : nullptr,
1730 bHasM ? poValueBuilder : nullptr));
1731 }
1732 break;
1733 }
1734
1735 case OGRArrowGeomEncoding::GEOARROW_STRUCT_POLYGON:
1736 {
1737 const auto poPolygon = poGeom->toPolygon();
1738 auto poPolygonBuilder =
1739 static_cast<arrow::ListBuilder *>(poBuilder);
1740 auto poRingBuilder = static_cast<arrow::ListBuilder *>(
1741 poPolygonBuilder->value_builder());
1742 auto poPointBuilder = static_cast<arrow::StructBuilder *>(
1743 poRingBuilder->value_builder());
1744 GET_XYZM_STRUCT_FIELD_BUILDERS_FROM(poPointBuilder);
1745
1746 OGR_ARROW_RETURN_OGRERR_NOT_OK(poPolygonBuilder->Append());
1747 for (const auto *poRing : *poPolygon)
1748 {
1749 OGR_ARROW_RETURN_OGRERR_NOT_OK(poRingBuilder->Append());
1750 OGR_ARROW_PROPAGATE_OGRERR(
1751 GeoArrowLineBuilder(poRing, poPointBuilder, poXBuilder,
1752 poYBuilder, poZBuilder, poMBuilder));
1753 }
1754 break;
1755 }
1756
1757 case OGRArrowGeomEncoding::GEOARROW_FSL_MULTIPOINT:
1758 {
1759 const auto poMultiPoint = poGeom->toMultiPoint();
1760 auto poListBuilder = static_cast<arrow::ListBuilder *>(poBuilder);
1761 auto poPointBuilder = static_cast<arrow::FixedSizeListBuilder *>(
1762 poListBuilder->value_builder());
1763 auto poValueBuilder = static_cast<arrow::DoubleBuilder *>(
1764 poPointBuilder->value_builder());
1765 OGR_ARROW_RETURN_OGRERR_NOT_OK(poListBuilder->Append());
1766 for (const auto *poPoint : *poMultiPoint)
1767 {
1768 OGR_ARROW_RETURN_OGRERR_NOT_OK(poPointBuilder->Append());
1769 OGR_ARROW_RETURN_OGRERR_NOT_OK(
1770 poValueBuilder->Append(poPoint->getX()));
1771 OGR_ARROW_RETURN_OGRERR_NOT_OK(
1772 poValueBuilder->Append(poPoint->getY()));
1773 if (bHasZ)
1774 OGR_ARROW_RETURN_OGRERR_NOT_OK(
1775 poValueBuilder->Append(poPoint->getZ()));
1776 if (bHasM)
1777 OGR_ARROW_RETURN_OGRERR_NOT_OK(
1778 poValueBuilder->Append(poPoint->getM()));
1779 }
1780 break;
1781 }
1782
1783 case OGRArrowGeomEncoding::GEOARROW_STRUCT_MULTIPOINT:
1784 {
1785 const auto poMultiPoint = poGeom->toMultiPoint();
1786 auto poListBuilder = static_cast<arrow::ListBuilder *>(poBuilder);
1787 auto poPointBuilder = static_cast<arrow::StructBuilder *>(
1788 poListBuilder->value_builder());
1789 GET_XYZM_STRUCT_FIELD_BUILDERS_FROM(poPointBuilder);
1790
1791 OGR_ARROW_RETURN_OGRERR_NOT_OK(poListBuilder->Append());
1792 for (const auto *poPoint : *poMultiPoint)
1793 {
1794 OGR_ARROW_RETURN_OGRERR_NOT_OK(poPointBuilder->Append());
1795 OGR_ARROW_RETURN_OGRERR_NOT_OK(
1796 poXBuilder->Append(poPoint->getX()));
1797 OGR_ARROW_RETURN_OGRERR_NOT_OK(
1798 poYBuilder->Append(poPoint->getY()));
1799 if (poZBuilder)
1800 OGR_ARROW_RETURN_OGRERR_NOT_OK(
1801 poZBuilder->Append(poPoint->getZ()));
1802 if (poMBuilder)
1803 OGR_ARROW_RETURN_OGRERR_NOT_OK(
1804 poMBuilder->Append(poPoint->getM()));
1805 }
1806 break;
1807 }
1808
1809 case OGRArrowGeomEncoding::GEOARROW_FSL_MULTILINESTRING:
1810 {
1811 const auto poMLS = poGeom->toMultiLineString();
1812 auto poMLSBuilder = static_cast<arrow::ListBuilder *>(poBuilder);
1813 auto poLSBuilder = static_cast<arrow::ListBuilder *>(
1814 poMLSBuilder->value_builder());
1815 auto poPointBuilder = static_cast<arrow::FixedSizeListBuilder *>(
1816 poLSBuilder->value_builder());
1817 auto poValueBuilder = static_cast<arrow::DoubleBuilder *>(
1818 poPointBuilder->value_builder());
1819 OGR_ARROW_RETURN_OGRERR_NOT_OK(poMLSBuilder->Append());
1820 for (const auto *poLS : *poMLS)
1821 {
1822 OGR_ARROW_RETURN_OGRERR_NOT_OK(poLSBuilder->Append());
1823 OGR_ARROW_PROPAGATE_OGRERR(GeoArrowLineBuilder(
1824 poLS, poPointBuilder, poValueBuilder, poValueBuilder,
1825 bHasZ ? poValueBuilder : nullptr,
1826 bHasM ? poValueBuilder : nullptr));
1827 }
1828 break;
1829 }
1830
1831 case OGRArrowGeomEncoding::GEOARROW_STRUCT_MULTILINESTRING:
1832 {
1833 const auto poMLS = poGeom->toMultiLineString();
1834 auto poMLSBuilder = static_cast<arrow::ListBuilder *>(poBuilder);
1835 auto poLSBuilder = static_cast<arrow::ListBuilder *>(
1836 poMLSBuilder->value_builder());
1837 auto poPointBuilder = static_cast<arrow::StructBuilder *>(
1838 poLSBuilder->value_builder());
1839 GET_XYZM_STRUCT_FIELD_BUILDERS_FROM(poPointBuilder);
1840
1841 OGR_ARROW_RETURN_OGRERR_NOT_OK(poMLSBuilder->Append());
1842 for (const auto *poLS : *poMLS)
1843 {
1844 OGR_ARROW_RETURN_OGRERR_NOT_OK(poLSBuilder->Append());
1845 OGR_ARROW_PROPAGATE_OGRERR(
1846 GeoArrowLineBuilder(poLS, poPointBuilder, poXBuilder,
1847 poYBuilder, poZBuilder, poMBuilder));
1848 }
1849 break;
1850 }
1851
1852 case OGRArrowGeomEncoding::GEOARROW_FSL_MULTIPOLYGON:
1853 {
1854 const auto poMPoly = poGeom->toMultiPolygon();
1855 auto poMPolyBuilder = static_cast<arrow::ListBuilder *>(poBuilder);
1856 auto poPolyBuilder = static_cast<arrow::ListBuilder *>(
1857 poMPolyBuilder->value_builder());
1858 auto poRingBuilder = static_cast<arrow::ListBuilder *>(
1859 poPolyBuilder->value_builder());
1860 auto poPointBuilder = static_cast<arrow::FixedSizeListBuilder *>(
1861 poRingBuilder->value_builder());
1862 auto poValueBuilder = static_cast<arrow::DoubleBuilder *>(
1863 poPointBuilder->value_builder());
1864 OGR_ARROW_RETURN_OGRERR_NOT_OK(poMPolyBuilder->Append());
1865 for (const auto *poPolygon : *poMPoly)
1866 {
1867 OGR_ARROW_RETURN_OGRERR_NOT_OK(poPolyBuilder->Append());
1868 for (const auto *poRing : *poPolygon)
1869 {
1870 OGR_ARROW_RETURN_OGRERR_NOT_OK(poRingBuilder->Append());
1871 OGR_ARROW_PROPAGATE_OGRERR(GeoArrowLineBuilder(
1872 poRing, poPointBuilder, poValueBuilder, poValueBuilder,
1873 bHasZ ? poValueBuilder : nullptr,
1874 bHasM ? poValueBuilder : nullptr));
1875 }
1876 }
1877 break;
1878 }
1879
1880 case OGRArrowGeomEncoding::GEOARROW_STRUCT_MULTIPOLYGON:
1881 {
1882 const auto poMPoly = poGeom->toMultiPolygon();
1883 auto poMPolyBuilder = static_cast<arrow::ListBuilder *>(poBuilder);
1884 auto poPolyBuilder = static_cast<arrow::ListBuilder *>(
1885 poMPolyBuilder->value_builder());
1886 auto poRingBuilder = static_cast<arrow::ListBuilder *>(
1887 poPolyBuilder->value_builder());
1888 auto poPointBuilder = static_cast<arrow::StructBuilder *>(
1889 poRingBuilder->value_builder());
1890 GET_XYZM_STRUCT_FIELD_BUILDERS_FROM(poPointBuilder);
1891
1892 OGR_ARROW_RETURN_OGRERR_NOT_OK(poMPolyBuilder->Append());
1893 for (const auto *poPolygon : *poMPoly)
1894 {
1895 OGR_ARROW_RETURN_OGRERR_NOT_OK(poPolyBuilder->Append());
1896 for (const auto *poRing : *poPolygon)
1897 {
1898 OGR_ARROW_RETURN_OGRERR_NOT_OK(poRingBuilder->Append());
1899 OGR_ARROW_PROPAGATE_OGRERR(GeoArrowLineBuilder(
1900 poRing, poPointBuilder, poXBuilder, poYBuilder,
1901 poZBuilder, poMBuilder));
1902 }
1903 }
1904 break;
1905 }
1906
1907 case OGRArrowGeomEncoding::GEOARROW_FSL_GENERIC:
1908 case OGRArrowGeomEncoding::GEOARROW_STRUCT_GENERIC:
1909 {
1910 CPLAssert(false);
1911 break;
1912 }
1913 }
1914
1915 return OGRERR_NONE;
1916 }
1917
1918 /************************************************************************/
1919 /* ICreateFeature() */
1920 /************************************************************************/
1921
1922 inline OGRErr OGRArrowWriterLayer::ICreateFeature(OGRFeature *poFeature)
1923 {
1924 if (m_poSchema == nullptr)
1925 {
1926 CreateSchema();
1927 }
1928
1929 if (m_apoBuilders.empty())
1930 {
1931 if (!m_apoFieldsFromArrowSchema.empty())
1932 {
1933 CPLError(CE_Failure, CPLE_NotSupported,
1934 "ICreateFeature() cannot be used after "
1935 "CreateFieldFromArrowSchema()");
1936 return OGRERR_FAILURE;
1937 }
1938 CreateArrayBuilders();
1939 }
1940
1941 // First pass to check not-null constraints as Arrow doesn't seem
1942 // to do that on the writing side. But such files can't be read.
1943 const int nFieldCount = m_poFeatureDefn->GetFieldCount();
1944 for (int i = 0; i < nFieldCount; ++i)
1945 {
1946 const auto poFieldDefn = m_poFeatureDefn->GetFieldDefn(i);
1947 if (!poFieldDefn->IsNullable() &&
1948 !poFeature->IsFieldSetAndNotNullUnsafe(i))
1949 {
1950 CPLError(CE_Failure, CPLE_AppDefined,
1951 "Null value found in non-nullable field %s",
1952 poFieldDefn->GetNameRef());
1953 return OGRERR_FAILURE;
1954 }
1955 }
1956
1957 const int nGeomFieldCount = m_poFeatureDefn->GetGeomFieldCount();
1958 for (int i = 0; i < nGeomFieldCount; ++i)
1959 {
1960 const auto poGeomFieldDefn = m_poFeatureDefn->GetGeomFieldDefn(i);
1961 if (!poGeomFieldDefn->IsNullable() &&
1962 poFeature->GetGeomFieldRef(i) == nullptr)
1963 {
1964 CPLError(CE_Failure, CPLE_AppDefined,
1965 "Null value found in non-nullable geometry field %s",
1966 poGeomFieldDefn->GetNameRef());
1967 return OGRERR_FAILURE;
1968 }
1969 }
1970
1971 // Write FID, if FID column present
1972 int nArrowIdx = 0;
1973 if (!m_osFIDColumn.empty())
1974 {
1975 int64_t nFID = poFeature->GetFID();
1976 if (nFID == OGRNullFID)
1977 {
1978 nFID = m_nFeatureCount;
1979 poFeature->SetFID(nFID);
1980 }
1981 auto poBuilder =
1982 static_cast<arrow::Int64Builder *>(m_apoBuilders[0].get());
1983 OGR_ARROW_RETURN_OGRERR_NOT_OK(poBuilder->Append(nFID));
1984 nArrowIdx++;
1985 }
1986
1987 // Write attributes
1988 for (int i = 0; i < nFieldCount; ++i, ++nArrowIdx)
1989 {
1990 auto poBuilder = m_apoBuilders[nArrowIdx].get();
1991 if (!poFeature->IsFieldSetAndNotNullUnsafe(i))
1992 {
1993 OGR_ARROW_RETURN_OGRERR_NOT_OK(poBuilder->AppendNull());
1994 continue;
1995 }
1996
1997 const auto poFieldDefn = m_poFeatureDefn->GetFieldDefn(i);
1998 const auto eSubDT = poFieldDefn->GetSubType();
1999 switch (poFieldDefn->GetType())
2000 {
2001 case OFTInteger:
2002 if (eSubDT == OFSTBoolean)
2003 OGR_ARROW_RETURN_OGRERR_NOT_OK(
2004 static_cast<arrow::BooleanBuilder *>(poBuilder)->Append(
2005 poFeature->GetFieldAsIntegerUnsafe(i) != 0));
2006 else if (eSubDT == OFSTInt16)
2007 OGR_ARROW_RETURN_OGRERR_NOT_OK(
2008 static_cast<arrow::Int16Builder *>(poBuilder)->Append(
2009 static_cast<int16_t>(
2010 poFeature->GetFieldAsIntegerUnsafe(i))));
2011 else
2012 OGR_ARROW_RETURN_OGRERR_NOT_OK(
2013 static_cast<arrow::Int32Builder *>(poBuilder)->Append(
2014 poFeature->GetFieldAsIntegerUnsafe(i)));
2015 break;
2016
2017 case OFTInteger64:
2018 OGR_ARROW_RETURN_OGRERR_NOT_OK(
2019 static_cast<arrow::Int64Builder *>(poBuilder)->Append(
2020 static_cast<int64_t>(
2021 poFeature->GetFieldAsInteger64Unsafe(i))));
2022 break;
2023
2024 case OFTReal:
2025 {
2026 const auto arrowType = m_poSchema->fields()[nArrowIdx]->type();
2027 const double dfVal = poFeature->GetFieldAsDoubleUnsafe(i);
2028 if (arrowType->id() == arrow::Type::DECIMAL128)
2029 {
2030 auto res = arrow::Decimal128::FromReal(
2031 dfVal, poFieldDefn->GetWidth(),
2032 poFieldDefn->GetPrecision());
2033 if (res.ok())
2034 {
2035 OGR_ARROW_RETURN_OGRERR_NOT_OK(
2036 static_cast<arrow::Decimal128Builder *>(poBuilder)
2037 ->Append(*res));
2038 }
2039 else
2040 {
2041 CPLError(CE_Warning, CPLE_AppDefined,
2042 "Cannot parse %.18g as a %d.%d decimal", dfVal,
2043 poFieldDefn->GetWidth(),
2044 poFieldDefn->GetPrecision());
2045 OGR_ARROW_RETURN_OGRERR_NOT_OK(poBuilder->AppendNull());
2046 }
2047 }
2048 else if (arrowType->id() == arrow::Type::DECIMAL256)
2049 {
2050 auto res = arrow::Decimal256::FromReal(
2051 dfVal, poFieldDefn->GetWidth(),
2052 poFieldDefn->GetPrecision());
2053 if (res.ok())
2054 {
2055 OGR_ARROW_RETURN_OGRERR_NOT_OK(
2056 static_cast<arrow::Decimal256Builder *>(poBuilder)
2057 ->Append(*res));
2058 }
2059 else
2060 {
2061 CPLError(CE_Warning, CPLE_AppDefined,
2062 "Cannot parse %.18g as a %d.%d decimal", dfVal,
2063 poFieldDefn->GetWidth(),
2064 poFieldDefn->GetPrecision());
2065 OGR_ARROW_RETURN_OGRERR_NOT_OK(poBuilder->AppendNull());
2066 }
2067 }
2068 else if (eSubDT == OFSTFloat32)
2069 {
2070 OGR_ARROW_RETURN_OGRERR_NOT_OK(
2071 static_cast<arrow::FloatBuilder *>(poBuilder)->Append(
2072 static_cast<float>(dfVal)));
2073 }
2074 else
2075 {
2076 OGR_ARROW_RETURN_OGRERR_NOT_OK(
2077 static_cast<arrow::DoubleBuilder *>(poBuilder)->Append(
2078 dfVal));
2079 }
2080 break;
2081 }
2082
2083 case OFTString:
2084 case OFTWideString:
2085 OGR_ARROW_RETURN_OGRERR_NOT_OK(
2086 static_cast<arrow::StringBuilder *>(poBuilder)->Append(
2087 poFeature->GetFieldAsStringUnsafe(i)));
2088 break;
2089
2090 case OFTBinary:
2091 {
2092 int nSize = 0;
2093 const auto pData = poFeature->GetFieldAsBinary(i, &nSize);
2094 if (poFieldDefn->GetWidth() != 0)
2095 {
2096 if (poFieldDefn->GetWidth() != nSize)
2097 {
2098 CPLError(
2099 CE_Warning, CPLE_AppDefined,
2100 "Cannot write field %s. Got %d bytes, expected %d",
2101 poFieldDefn->GetNameRef(), nSize,
2102 poFieldDefn->GetWidth());
2103 OGR_ARROW_RETURN_OGRERR_NOT_OK(poBuilder->AppendNull());
2104 }
2105 else
2106 {
2107 OGR_ARROW_RETURN_OGRERR_NOT_OK(
2108 static_cast<arrow::FixedSizeBinaryBuilder *>(
2109 poBuilder)
2110 ->Append(pData));
2111 }
2112 }
2113 else
2114 OGR_ARROW_RETURN_OGRERR_NOT_OK(
2115 static_cast<arrow::BinaryBuilder *>(poBuilder)->Append(
2116 pData, nSize));
2117 break;
2118 }
2119
2120 case OFTIntegerList:
2121 {
2122 auto poListBuilder =
2123 static_cast<arrow::ListBuilder *>(poBuilder);
2124 if (eSubDT == OFSTBoolean)
2125 {
2126 OGR_ARROW_RETURN_OGRERR_NOT_OK(poListBuilder->Append());
2127 auto poValueBuilder = static_cast<arrow::BooleanBuilder *>(
2128 poListBuilder->value_builder());
2129 int nValues = 0;
2130 const auto panValues =
2131 poFeature->GetFieldAsIntegerList(i, &nValues);
2132 for (int j = 0; j < nValues; ++j)
2133 OGR_ARROW_RETURN_OGRERR_NOT_OK(
2134 poValueBuilder->Append(panValues[j] != 0));
2135 }
2136 else if (eSubDT == OFSTInt16)
2137 {
2138 OGR_ARROW_RETURN_OGRERR_NOT_OK(poListBuilder->Append());
2139 auto poValueBuilder = static_cast<arrow::Int16Builder *>(
2140 poListBuilder->value_builder());
2141 int nValues = 0;
2142 const auto panValues =
2143 poFeature->GetFieldAsIntegerList(i, &nValues);
2144 for (int j = 0; j < nValues; ++j)
2145 OGR_ARROW_RETURN_OGRERR_NOT_OK(poValueBuilder->Append(
2146 static_cast<int16_t>(panValues[j])));
2147 }
2148 else
2149 {
2150 OGR_ARROW_RETURN_OGRERR_NOT_OK(poListBuilder->Append());
2151 auto poValueBuilder = static_cast<arrow::Int32Builder *>(
2152 poListBuilder->value_builder());
2153 int nValues = 0;
2154 const auto panValues =
2155 poFeature->GetFieldAsIntegerList(i, &nValues);
2156 for (int j = 0; j < nValues; ++j)
2157 OGR_ARROW_RETURN_OGRERR_NOT_OK(
2158 poValueBuilder->Append(panValues[j]));
2159 }
2160 break;
2161 }
2162
2163 case OFTInteger64List:
2164 {
2165 auto poListBuilder =
2166 static_cast<arrow::ListBuilder *>(poBuilder);
2167 OGR_ARROW_RETURN_OGRERR_NOT_OK(poListBuilder->Append());
2168 auto poValueBuilder = static_cast<arrow::Int64Builder *>(
2169 poListBuilder->value_builder());
2170 int nValues = 0;
2171 const auto panValues =
2172 poFeature->GetFieldAsInteger64List(i, &nValues);
2173 for (int j = 0; j < nValues; ++j)
2174 OGR_ARROW_RETURN_OGRERR_NOT_OK(poValueBuilder->Append(
2175 static_cast<int64_t>(panValues[j])));
2176 break;
2177 }
2178
2179 case OFTRealList:
2180 {
2181 auto poListBuilder =
2182 static_cast<arrow::ListBuilder *>(poBuilder);
2183 if (eSubDT == OFSTFloat32)
2184 {
2185 OGR_ARROW_RETURN_OGRERR_NOT_OK(poListBuilder->Append());
2186 auto poValueBuilder = static_cast<arrow::FloatBuilder *>(
2187 poListBuilder->value_builder());
2188 int nValues = 0;
2189 const auto padfValues =
2190 poFeature->GetFieldAsDoubleList(i, &nValues);
2191 for (int j = 0; j < nValues; ++j)
2192 OGR_ARROW_RETURN_OGRERR_NOT_OK(poValueBuilder->Append(
2193 static_cast<float>(padfValues[j])));
2194 }
2195 else
2196 {
2197 OGR_ARROW_RETURN_OGRERR_NOT_OK(poListBuilder->Append());
2198 auto poValueBuilder = static_cast<arrow::DoubleBuilder *>(
2199 poListBuilder->value_builder());
2200 int nValues = 0;
2201 const auto padfValues =
2202 poFeature->GetFieldAsDoubleList(i, &nValues);
2203 for (int j = 0; j < nValues; ++j)
2204 OGR_ARROW_RETURN_OGRERR_NOT_OK(
2205 poValueBuilder->Append(padfValues[j]));
2206 }
2207 break;
2208 }
2209
2210 case OFTStringList:
2211 case OFTWideStringList:
2212 {
2213 auto poListBuilder =
2214 static_cast<arrow::ListBuilder *>(poBuilder);
2215 OGR_ARROW_RETURN_OGRERR_NOT_OK(poListBuilder->Append());
2216 auto poValueBuilder = static_cast<arrow::StringBuilder *>(
2217 poListBuilder->value_builder());
2218 const auto papszValues = poFeature->GetFieldAsStringList(i);
2219 for (int j = 0; papszValues && papszValues[j]; ++j)
2220 OGR_ARROW_RETURN_OGRERR_NOT_OK(
2221 poValueBuilder->Append(papszValues[j]));
2222 break;
2223 }
2224
2225 case OFTDate:
2226 {
2227 int nYear, nMonth, nDay, nHour, nMinute;
2228 float fSec;
2229 int nTZFlag;
2230 poFeature->GetFieldAsDateTime(i, &nYear, &nMonth, &nDay, &nHour,
2231 &nMinute, &fSec, &nTZFlag);
2232 struct tm brokenDown;
2233 memset(&brokenDown, 0, sizeof(brokenDown));
2234 brokenDown.tm_year = nYear - 1900;
2235 brokenDown.tm_mon = nMonth - 1;
2236 brokenDown.tm_mday = nDay;
2237 GIntBig nVal = CPLYMDHMSToUnixTime(&brokenDown);
2238 OGR_ARROW_RETURN_OGRERR_NOT_OK(
2239 static_cast<arrow::Date32Builder *>(poBuilder)->Append(
2240 static_cast<int>(nVal / 86400)));
2241 break;
2242 }
2243
2244 case OFTTime:
2245 {
2246 int nYear, nMonth, nDay, nHour, nMinute;
2247 float fSec;
2248 int nTZFlag;
2249 poFeature->GetFieldAsDateTime(i, &nYear, &nMonth, &nDay, &nHour,
2250 &nMinute, &fSec, &nTZFlag);
2251 int nVal = nHour * 3600 + nMinute * 60;
2252 OGR_ARROW_RETURN_OGRERR_NOT_OK(
2253 static_cast<arrow::Time32Builder *>(poBuilder)->Append(
2254 static_cast<int>(
2255 (static_cast<double>(nVal) + fSec) * 1000 + 0.5)));
2256 break;
2257 }
2258
2259 case OFTDateTime:
2260 {
2261 int nYear, nMonth, nDay, nHour, nMinute;
2262 float fSec;
2263 int nTZFlag;
2264 poFeature->GetFieldAsDateTime(i, &nYear, &nMonth, &nDay, &nHour,
2265 &nMinute, &fSec, &nTZFlag);
2266 struct tm brokenDown;
2267 memset(&brokenDown, 0, sizeof(brokenDown));
2268 brokenDown.tm_year = nYear - 1900;
2269 brokenDown.tm_mon = nMonth - 1;
2270 brokenDown.tm_mday = nDay;
2271 brokenDown.tm_hour = nHour;
2272 brokenDown.tm_min = nMinute;
2273 brokenDown.tm_sec = 0;
2274 GIntBig nVal = CPLYMDHMSToUnixTime(&brokenDown);
2275 if (!IsFileWriterCreated() &&
2276 m_anTZFlag[i] != OGR_TZFLAG_UNKNOWN)
2277 {
2278 if (m_anTZFlag[i] == TZFLAG_UNINITIALIZED)
2279 m_anTZFlag[i] = nTZFlag;
2280 else if (m_anTZFlag[i] != nTZFlag)
2281 {
2282 if (m_anTZFlag[i] >= OGR_TZFLAG_MIXED_TZ &&
2283 nTZFlag >= OGR_TZFLAG_MIXED_TZ)
2284 {
2285 m_anTZFlag[i] =
2286 OGR_TZFLAG_MIXED_TZ; // harmonize on UTC ultimately
2287 }
2288 else
2289 {
2290 CPLError(CE_Warning, CPLE_AppDefined,
2291 "Field %s contains a mix of "
2292 "timezone-aware and local/without "
2293 "timezone values.",
2294 poFieldDefn->GetNameRef());
2295 m_anTZFlag[i] = OGR_TZFLAG_UNKNOWN;
2296 }
2297 }
2298 }
2299 if (nTZFlag > OGR_TZFLAG_MIXED_TZ)
2300 {
2301 const int nOffsetSec = (nTZFlag - OGR_TZFLAG_UTC) * 15 * 60;
2302 nVal -= nOffsetSec;
2303 }
2304 const int64_t nTimestamp = static_cast<int64_t>(
2305 (static_cast<double>(nVal) + fSec) * 1000 + 0.5);
2306 auto structBuilder =
2307 dynamic_cast<arrow::StructBuilder *>(poBuilder);
2308 if (structBuilder)
2309 {
2310 OGR_ARROW_RETURN_OGRERR_NOT_OK(
2311 static_cast<arrow::TimestampBuilder *>(
2312 structBuilder->field_builder(0))
2313 ->Append(nTimestamp));
2314 const int16_t nUTCOffsetMin =
2315 static_cast<int16_t>((nTZFlag - OGR_TZFLAG_UTC) * 15);
2316 OGR_ARROW_RETURN_OGRERR_NOT_OK(
2317 static_cast<arrow::Int16Builder *>(
2318 structBuilder->field_builder(1))
2319 ->Append(nUTCOffsetMin));
2320 OGR_ARROW_RETURN_OGRERR_NOT_OK(structBuilder->Append());
2321 }
2322 else
2323 {
2324 OGR_ARROW_RETURN_OGRERR_NOT_OK(
2325 static_cast<arrow::TimestampBuilder *>(poBuilder)
2326 ->Append(nTimestamp));
2327 }
2328 break;
2329 }
2330 }
2331 }
2332
2333 // Write geometries
2334 for (int i = 0; i < nGeomFieldCount; ++i, ++nArrowIdx)
2335 {
2336 auto poBuilder = m_apoBuilders[nArrowIdx].get();
2337 OGRGeometry *poGeom = poFeature->GetGeomFieldRef(i);
2338 if (BuildGeometry(poGeom, i, poBuilder) != OGRERR_NONE)
2339 return OGRERR_FAILURE;
2340 }
2341
2342 m_nFeatureCount++;
2343
2344 // Flush the current row group if reaching the limit of rows per group.
2345 if (!m_apoBuilders.empty() && m_apoBuilders[0]->length() == m_nRowGroupSize)
2346 {
2347 if (!FlushFeatures())
2348 return OGRERR_FAILURE;
2349 }
2350
2351 return OGRERR_NONE;
2352 }
2353
2354 /************************************************************************/
2355 /* FlushFeatures() */
2356 /************************************************************************/
2357
2358 inline bool OGRArrowWriterLayer::FlushFeatures()
2359 {
2360 if (m_apoBuilders.empty() || m_apoBuilders[0]->length() == 0)
2361 return true;
2362
2363 if (!IsFileWriterCreated())
2364 {
2365 CreateWriter();
2366 if (!IsFileWriterCreated())
2367 return false;
2368 }
2369
2370 return FlushGroup();
2371 }
2372
2373 /************************************************************************/
2374 /* GetFeatureCount() */
2375 /************************************************************************/
2376
2377 inline GIntBig OGRArrowWriterLayer::GetFeatureCount(int bForce)
2378 {
2379 if (m_poAttrQuery == nullptr && m_poFilterGeom == nullptr)
2380 {
2381 return m_nFeatureCount;
2382 }
2383 return OGRLayer::GetFeatureCount(bForce);
2384 }
2385
2386 /************************************************************************/
2387 /* TestCapability() */
2388 /************************************************************************/
2389
2390 inline int OGRArrowWriterLayer::TestCapability(const char *pszCap) const
2391 {
2392 if (EQUAL(pszCap, OLCCreateField) || EQUAL(pszCap, OLCCreateGeomField))
2393 return m_poSchema == nullptr;
2394
2395 if (EQUAL(pszCap, OLCSequentialWrite))
2396 return true;
2397
2398 if (EQUAL(pszCap, OLCFastWriteArrowBatch))
2399 return true;
2400
2401 if (EQUAL(pszCap, OLCStringsAsUTF8))
2402 return true;
2403
2404 if (EQUAL(pszCap, OLCMeasuredGeometries))
2405 return true;
2406
2407 return false;
2408 }
2409
2410 /************************************************************************/
2411 /* WriteArrays() */
2412 /************************************************************************/
2413
2414 inline bool OGRArrowWriterLayer::WriteArrays(
2415 std::function<bool(const std::shared_ptr<arrow::Field> &,
2416 const std::shared_ptr<arrow::Array> &)>
2417 postProcessArray)
2418 {
2419 int nArrowIdx = 0;
2420 int nArrowIdxFirstField = !m_osFIDColumn.empty() ? 1 : 0;
2421 for (const auto &poBuilder : m_apoBuilders)
2422 {
2423 const auto &field = m_poSchema->fields()[nArrowIdx];
2424
2425 std::shared_ptr<arrow::Array> array;
2426 auto status = poBuilder->Finish(&array);
2427 if (!status.ok())
2428 {
2429 CPLError(CE_Failure, CPLE_AppDefined,
2430 "builder::Finish() for field %s failed with %s",
2431 field->name().c_str(), status.message().c_str());
2432 return false;
2433 }
2434
2435 // CPLDebug("ARROW", "%s", array->ToString().c_str());
2436
2437 const int iCol = nArrowIdx - nArrowIdxFirstField;
2438 if (iCol >= 0 && iCol < m_poFeatureDefn->GetFieldCount())
2439 {
2440 const auto poFieldDefn = m_poFeatureDefn->GetFieldDefn(iCol);
2441 const auto eFieldType = poFieldDefn->GetType();
2442 if (eFieldType == OFTInteger || eFieldType == OFTInteger64)
2443 {
2444 const auto &osDomainName = poFieldDefn->GetDomainName();
2445 const auto oIter =
2446 m_oMapFieldDomainToStringArray.find(osDomainName);
2447 if (oIter != m_oMapFieldDomainToStringArray.end())
2448 {
2449 auto result = arrow::DictionaryArray::FromArrays(
2450 field->type(), array, oIter->second);
2451 if (!result.ok())
2452 {
2453 CPLError(CE_Failure, CPLE_AppDefined,
2454 "DictionaryArray::FromArrays() for field %s "
2455 "failed with %s",
2456 field->name().c_str(),
2457 result.status().message().c_str());
2458 return false;
2459 }
2460 array = *result;
2461 }
2462 }
2463 }
2464
2465 if (!postProcessArray(field, array))
2466 {
2467 return false;
2468 }
2469
2470 nArrowIdx++;
2471 }
2472
2473 if (m_bWriteBBoxStruct)
2474 {
2475 const int nGeomFieldCount = m_poFeatureDefn->GetGeomFieldCount();
2476 for (int i = 0; i < nGeomFieldCount; ++i)
2477 {
2478 const auto &field = m_apoFieldsBBOX[i];
2479 std::shared_ptr<arrow::Array> array;
2480 auto status = m_apoBuildersBBOXStruct[i]->Finish(&array);
2481 if (!status.ok())
2482 {
2483 CPLError(CE_Failure, CPLE_AppDefined,
2484 "builder::Finish() for field %s failed with %s",
2485 field->name().c_str(), status.message().c_str());
2486 return false;
2487 }
2488
2489 if (!postProcessArray(field, array))
2490 {
2491 return false;
2492 }
2493 }
2494 }
2495
2496 return true;
2497 }
2498
2499 /************************************************************************/
2500 /* TestBit() */
2501 /************************************************************************/
2502
2503 static inline bool TestBit(const uint8_t *pabyData, size_t nIdx)
2504 {
2505 return (pabyData[nIdx / 8] & (1 << (nIdx % 8))) != 0;
2506 }
2507
2508 /************************************************************************/
2509 /* WriteArrowBatchInternal() */
2510 /************************************************************************/
2511
2512 inline bool OGRArrowWriterLayer::WriteArrowBatchInternal(
2513 const struct ArrowSchema *schema, struct ArrowArray *array,
2514 CSLConstList papszOptions,
2515 std::function<bool(const std::shared_ptr<arrow::RecordBatch> &)> writeBatch)
2516 {
2517 #ifdef __COVERITY__
2518 (void)schema;
2519 (void)array;
2520 (void)papszOptions;
2521 (void)writeBatch;
2522 CPLError(CE_Failure, CPLE_AppDefined, "Not implemented");
2523 return false;
2524 #else
2525 if (m_poSchema == nullptr)
2526 {
2527 CreateSchema();
2528 }
2529
2530 if (!IsFileWriterCreated())
2531 {
2532 CreateWriter();
2533 if (!IsFileWriterCreated())
2534 return false;
2535 }
2536
2537 if (m_apoBuilders.empty())
2538 {
2539 CreateArrayBuilders();
2540 }
2541
2542 const int nGeomFieldCount = m_poFeatureDefn->GetGeomFieldCount();
2543 const int nGeomFieldCountBBoxFields =
2544 m_bWriteBBoxStruct ? nGeomFieldCount : 0;
2545
2546 const char *pszFIDName = CSLFetchNameValueDef(
2547 papszOptions, "FID", OGRLayer::DEFAULT_ARROW_FID_NAME);
2548 const char *pszSingleGeomFieldName =
2549 CSLFetchNameValue(papszOptions, "GEOMETRY_NAME");
2550
2551 // Sort schema and array children in the same order as m_poSchema.
2552 // This is needed for non-WKB geometry encoding
2553 std::map<std::string, int> oMapSchemaChildrenNameToIdx;
2554 for (int i = 0; i < static_cast<int>(schema->n_children); ++i)
2555 {
2556 if (cpl::contains(oMapSchemaChildrenNameToIdx,
2557 schema->children[i]->name))
2558 {
2559 CPLError(CE_Failure, CPLE_AppDefined,
2560 "Several fields with same name '%s' found",
2561 schema->children[i]->name);
2562 return false;
2563 }
2564 oMapSchemaChildrenNameToIdx[schema->children[i]->name] = i;
2565
2566 if (!pszSingleGeomFieldName && schema->children[i]->metadata)
2567 {
2568 const auto oMetadata =
2569 OGRParseArrowMetadata(schema->children[i]->metadata);
2570 const auto oIter = oMetadata.find(ARROW_EXTENSION_NAME_KEY);
2571 if (oIter != oMetadata.end() &&
2572 (oIter->second == EXTENSION_NAME_OGC_WKB ||
2573 oIter->second == EXTENSION_NAME_GEOARROW_WKB))
2574 {
2575 pszSingleGeomFieldName = schema->children[i]->name;
2576 }
2577 }
2578 }
2579 if (!pszSingleGeomFieldName)
2580 pszSingleGeomFieldName = OGRLayer::DEFAULT_ARROW_GEOMETRY_NAME;
2581
2582 std::vector<int> anMapLayerSchemaToArraySchema(m_poSchema->num_fields(),
2583 -1);
2584 struct ArrowArray fidArray;
2585 struct ArrowSchema fidSchema;
2586 memset(&fidArray, 0, sizeof(fidArray));
2587 memset(&fidSchema, 0, sizeof(fidSchema));
2588 std::vector<void *> apBuffersFid;
2589 std::vector<int64_t> fids;
2590
2591 std::set<int> oSetReferencedFieldsInArraySchema;
2592 const auto DummyFreeArray = [](struct ArrowArray *ptrArray)
2593 { ptrArray->release = nullptr; };
2594 const auto DummyFreeSchema = [](struct ArrowSchema *ptrSchema)
2595 { ptrSchema->release = nullptr; };
2596 bool bRebuildBatch = false;
2597 for (int i = 0; i < m_poSchema->num_fields() - nGeomFieldCountBBoxFields;
2598 ++i)
2599 {
2600 auto oIter =
2601 oMapSchemaChildrenNameToIdx.find(m_poSchema->field(i)->name());
2602 if (oIter == oMapSchemaChildrenNameToIdx.end())
2603 {
2604 if (m_poSchema->field(i)->name() == m_osFIDColumn)
2605 {
2606 oIter = oMapSchemaChildrenNameToIdx.find(pszFIDName);
2607 if (oIter == oMapSchemaChildrenNameToIdx.end())
2608 {
2609 // If the input data does not contain a FID column, but
2610 // the output file requires it, creates a default FID column
2611 fidArray.release = DummyFreeArray;
2612 fidArray.n_buffers = 2;
2613 apBuffersFid.resize(2);
2614 fidArray.buffers =
2615 const_cast<const void **>(apBuffersFid.data());
2616 fids.reserve(static_cast<size_t>(array->length));
2617 for (size_t iRow = 0;
2618 iRow < static_cast<size_t>(array->length); ++iRow)
2619 fids.push_back(m_nFeatureCount + iRow);
2620 fidArray.buffers[1] = fids.data();
2621 fidArray.length = array->length;
2622 fidSchema.release = DummyFreeSchema;
2623 fidSchema.name = m_osFIDColumn.c_str();
2624 fidSchema.format = "l"; // int64
2625 continue;
2626 }
2627 }
2628 else if (nGeomFieldCount == 1 &&
2629 m_poFeatureDefn->GetGeomFieldIndex(
2630 m_poSchema->field(i)->name().c_str()) == 0)
2631 {
2632 oIter =
2633 oMapSchemaChildrenNameToIdx.find(pszSingleGeomFieldName);
2634 if (oIter != oMapSchemaChildrenNameToIdx.end())
2635 bRebuildBatch = true;
2636 }
2637
2638 if (oIter == oMapSchemaChildrenNameToIdx.end())
2639 {
2640 CPLError(CE_Failure, CPLE_AppDefined,
2641 "Cannot find field '%s' in schema",
2642 m_poSchema->field(i)->name().c_str());
2643 return false;
2644 }
2645 }
2646 anMapLayerSchemaToArraySchema[i] = oIter->second;
2647 oSetReferencedFieldsInArraySchema.insert(oIter->second);
2648 }
2649
2650 // Note: we cheat a bit by declaring a single instance of the minx/miny/
2651 // maxx/maxy sub-field ArrowSchema*, and make all struct ArrowSchema point
2652 // to them. That's OK because we use DummyFreeSchema to release, which does
2653 // nothing.
2654 struct ArrowSchema bboxStructSchemaXMin;
2655 struct ArrowSchema bboxStructSchemaYMin;
2656 struct ArrowSchema bboxStructSchemaXMax;
2657 struct ArrowSchema bboxStructSchemaYMax;
2658 constexpr int BBOX_SUBFIELD_COUNT = 4;
2659 std::array<struct ArrowSchema *, BBOX_SUBFIELD_COUNT>
2660 bboxStructSchemaChildren;
2661 constexpr int BBOX_STRUCT_BUFFER_COUNT = 1; // validity bitmap array
2662 // cppcheck-suppress constStatement
2663 std::vector<std::array<const void *, BBOX_STRUCT_BUFFER_COUNT>>
2664 bboxStructBuffersPtr;
2665 std::vector<std::vector<GByte>> aabyBboxStructValidity;
2666 std::vector<std::vector<float>> aadfMinX, aadfMinY, aadfMaxX, aadfMaxY;
2667 // cppcheck-suppress constStatement
2668 std::vector<std::array<struct ArrowArray, BBOX_SUBFIELD_COUNT>> bboxArrays;
2669 // cppcheck-suppress constStatement
2670 std::vector<std::array<struct ArrowArray *, BBOX_SUBFIELD_COUNT>>
2671 bboxArraysPtr;
2672 constexpr int BBOX_SUBFIELD_BUFFER_COUNT =
2673 2; // validity bitmap array and float array
2674 std::vector<std::array<std::array<const void *, BBOX_SUBFIELD_BUFFER_COUNT>,
2675 BBOX_SUBFIELD_COUNT>>
2676 bboxBuffersPtr;
2677
2678 // Temporary arrays to hold the geometry bounding boxes.
2679 std::vector<struct ArrowArray> bboxStructArray;
2680 std::vector<struct ArrowSchema> bboxStructSchema;
2681
2682 std::vector<struct ArrowSchema *> newSchemaChildren;
2683 std::vector<struct ArrowArray *> newArrayChildren;
2684 newSchemaChildren.reserve(m_poSchema->num_fields());
2685 newArrayChildren.reserve(m_poSchema->num_fields());
2686 for (int i = 0; i < m_poSchema->num_fields() - nGeomFieldCountBBoxFields;
2687 ++i)
2688 {
2689 if (anMapLayerSchemaToArraySchema[i] < 0)
2690 {
2691 CPLAssert(m_poSchema->field(i)->name() == m_osFIDColumn);
2692 newSchemaChildren.emplace_back(&fidSchema);
2693 newArrayChildren.emplace_back(&fidArray);
2694 }
2695 else
2696 {
2697 newSchemaChildren.emplace_back(
2698 schema->children[anMapLayerSchemaToArraySchema[i]]);
2699 newArrayChildren.emplace_back(
2700 array->children[anMapLayerSchemaToArraySchema[i]]);
2701 }
2702 }
2703
2704 if (m_bWriteBBoxStruct)
2705 {
2706 memset(&bboxStructSchemaXMin, 0, sizeof(bboxStructSchemaXMin));
2707 memset(&bboxStructSchemaYMin, 0, sizeof(bboxStructSchemaYMin));
2708 memset(&bboxStructSchemaXMax, 0, sizeof(bboxStructSchemaXMax));
2709 memset(&bboxStructSchemaYMax, 0, sizeof(bboxStructSchemaYMax));
2710
2711 bboxStructSchemaXMin.release = DummyFreeSchema;
2712 bboxStructSchemaXMin.name = "xmin";
2713 bboxStructSchemaXMin.format = "f"; // float32
2714
2715 bboxStructSchemaYMin.release = DummyFreeSchema;
2716 bboxStructSchemaYMin.name = "ymin";
2717 bboxStructSchemaYMin.format = "f"; // float32
2718
2719 bboxStructSchemaXMax.release = DummyFreeSchema;
2720 bboxStructSchemaXMax.name = "xmax";
2721 bboxStructSchemaXMax.format = "f"; // float32
2722
2723 bboxStructSchemaYMax.release = DummyFreeSchema;
2724 bboxStructSchemaYMax.name = "ymax";
2725 bboxStructSchemaYMax.format = "f"; // float32
2726
2727 try
2728 {
2729 constexpr int XMIN_IDX = 0;
2730 constexpr int YMIN_IDX = 1;
2731 constexpr int XMAX_IDX = 2;
2732 constexpr int YMAX_IDX = 3;
2733 bboxStructSchemaChildren[XMIN_IDX] = &bboxStructSchemaXMin;
2734 // cppcheck-suppress objectIndex
2735 bboxStructSchemaChildren[YMIN_IDX] = &bboxStructSchemaYMin;
2736 // cppcheck-suppress objectIndex
2737 bboxStructSchemaChildren[XMAX_IDX] = &bboxStructSchemaXMax;
2738 // cppcheck-suppress objectIndex
2739 bboxStructSchemaChildren[YMAX_IDX] = &bboxStructSchemaYMax;
2740
2741 bboxStructArray.resize(nGeomFieldCount);
2742 bboxStructSchema.resize(nGeomFieldCount);
2743 bboxArrays.resize(nGeomFieldCount);
2744 bboxArraysPtr.resize(nGeomFieldCount);
2745 bboxBuffersPtr.resize(nGeomFieldCount);
2746 bboxStructBuffersPtr.resize(nGeomFieldCount);
2747 aabyBboxStructValidity.resize(nGeomFieldCount);
2748 memset(bboxStructArray.data(), 0,
2749 nGeomFieldCount * sizeof(bboxStructArray[0]));
2750 memset(bboxStructSchema.data(), 0,
2751 nGeomFieldCount * sizeof(bboxStructSchema[0]));
2752 memset(bboxArrays.data(), 0,
2753 nGeomFieldCount * sizeof(bboxArrays[0]));
2754 aadfMinX.resize(nGeomFieldCount);
2755 aadfMinY.resize(nGeomFieldCount);
2756 aadfMaxX.resize(nGeomFieldCount);
2757 aadfMaxY.resize(nGeomFieldCount);
2758 for (int i = 0; i < nGeomFieldCount; ++i)
2759 {
2760 const bool bIsNullable = CPL_TO_BOOL(
2761 m_poFeatureDefn->GetGeomFieldDefn(i)->IsNullable());
2762 aadfMinX[i].reserve(static_cast<size_t>(array->length));
2763 aadfMinY[i].reserve(static_cast<size_t>(array->length));
2764 aadfMaxX[i].reserve(static_cast<size_t>(array->length));
2765 aadfMaxY[i].reserve(static_cast<size_t>(array->length));
2766 aabyBboxStructValidity[i].resize(
2767 static_cast<size_t>(array->length + 7) / 8, 0xFF);
2768
2769 bboxStructSchema[i].release = DummyFreeSchema;
2770 bboxStructSchema[i].name = m_apoFieldsBBOX[i]->name().c_str();
2771 bboxStructSchema[i].format = "+s"; // structure
2772 bboxStructSchema[i].flags =
2773 bIsNullable ? ARROW_FLAG_NULLABLE : 0;
2774 bboxStructSchema[i].n_children = BBOX_SUBFIELD_COUNT;
2775 bboxStructSchema[i].children = bboxStructSchemaChildren.data();
2776
2777 constexpr int VALIDITY_ARRAY_IDX = 0;
2778 constexpr int BBOX_SUBFIELD_FLOAT_VALUE_IDX = 1;
2779 bboxBuffersPtr[i][XMIN_IDX][BBOX_SUBFIELD_FLOAT_VALUE_IDX] =
2780 aadfMinX[i].data();
2781 bboxBuffersPtr[i][YMIN_IDX][BBOX_SUBFIELD_FLOAT_VALUE_IDX] =
2782 aadfMinY[i].data();
2783 bboxBuffersPtr[i][XMAX_IDX][BBOX_SUBFIELD_FLOAT_VALUE_IDX] =
2784 aadfMaxX[i].data();
2785 bboxBuffersPtr[i][YMAX_IDX][BBOX_SUBFIELD_FLOAT_VALUE_IDX] =
2786 aadfMaxY[i].data();
2787
2788 for (int j = 0; j < BBOX_SUBFIELD_COUNT; ++j)
2789 {
2790 bboxBuffersPtr[i][j][VALIDITY_ARRAY_IDX] = nullptr;
2791
2792 bboxArrays[i][j].release = DummyFreeArray;
2793 bboxArrays[i][j].length = array->length;
2794 bboxArrays[i][j].n_buffers = BBOX_SUBFIELD_BUFFER_COUNT;
2795 bboxArrays[i][j].buffers = bboxBuffersPtr[i][j].data();
2796
2797 bboxArraysPtr[i][j] = &bboxArrays[i][j];
2798 }
2799
2800 bboxStructArray[i].release = DummyFreeArray;
2801 bboxStructArray[i].n_children = BBOX_SUBFIELD_COUNT;
2802 // coverity[escape]
2803 bboxStructArray[i].children = bboxArraysPtr[i].data();
2804 bboxStructArray[i].length = array->length;
2805 bboxStructArray[i].n_buffers = BBOX_STRUCT_BUFFER_COUNT;
2806 bboxStructBuffersPtr[i][VALIDITY_ARRAY_IDX] =
2807 bIsNullable ? aabyBboxStructValidity[i].data() : nullptr;
2808 // coverity[escape]
2809 bboxStructArray[i].buffers = bboxStructBuffersPtr[i].data();
2810
2811 newSchemaChildren.emplace_back(&bboxStructSchema[i]);
2812 newArrayChildren.emplace_back(&bboxStructArray[i]);
2813 }
2814 }
2815 catch (const std::bad_alloc &)
2816 {
2817 CPLError(CE_Failure, CPLE_OutOfMemory,
2818 "Out of memory in "
2819 "OGRArrowWriterLayer::WriteArrowBatchInternal()");
2820 return false;
2821 }
2822 }
2823
2824 for (int i = 0; i < static_cast<int>(schema->n_children); ++i)
2825 {
2826 if (!cpl::contains(oSetReferencedFieldsInArraySchema, i))
2827 {
2828 if (m_osFIDColumn.empty() &&
2829 strcmp(schema->children[i]->name, pszFIDName) == 0)
2830 {
2831 // If the input data contains a FID column, but the output data
2832 // does not, then ignore it.
2833 }
2834 else
2835 {
2836 CPLError(CE_Failure, CPLE_AppDefined,
2837 "Found field '%s' in array schema that does not exist "
2838 "in layer schema",
2839 schema->children[i]->name);
2840 return false;
2841 }
2842 }
2843 }
2844
2845 // ImportSchema() would release the schema, but we don't want that
2846 // So copy the structure content into a local variable, and override its
2847 // release callback to a no-op. This may be a bit fragile, but it doesn't
2848 // look like ImportSchema implementation tries to access the C ArrowSchema
2849 // after it has been called.
2850 struct ArrowSchema lSchema = *schema;
2851 schema = &lSchema;
2852 CPL_IGNORE_RET_VAL(schema);
2853
2854 lSchema.n_children = newSchemaChildren.size();
2855 lSchema.children = newSchemaChildren.data();
2856
2857 lSchema.release = DummyFreeSchema;
2858 auto poSchemaResult = arrow::ImportSchema(&lSchema);
2859 CPLAssert(lSchema.release == nullptr);
2860 if (!poSchemaResult.ok())
2861 {
2862 CPLError(CE_Failure, CPLE_AppDefined, "ImportSchema() failed with %s",
2863 poSchemaResult.status().message().c_str());
2864 return false;
2865 }
2866 auto poSchema = *poSchemaResult;
2867
2868 // Hack the array to use the new children we've computed above
2869 // but make sure the original release() callback sees the original children
2870 struct ArrayReleaser
2871 {
2872 struct ArrowArray ori_array{};
2873
2874 explicit ArrayReleaser(struct ArrowArray *array)
2875 {
2876 memcpy(&ori_array, array, sizeof(*array));
2877 array->release = ArrayReleaser::release;
2878 array->private_data = this;
2879 }
2880
2881 static void release(struct ArrowArray *array)
2882 {
2883 struct ArrayReleaser *releaser =
2884 static_cast<struct ArrayReleaser *>(array->private_data);
2885 memcpy(array, &(releaser->ori_array), sizeof(*array));
2886 CPLAssert(array->release != nullptr);
2887 array->release(array);
2888 CPLAssert(array->release == nullptr);
2889 delete releaser;
2890 }
2891 };
2892
2893 // Must be allocated on the heap, since ArrayReleaser::release() will be
2894 // called after this method has ended.
2895 ArrayReleaser *releaser = new ArrayReleaser(array);
2896 array->private_data = releaser;
2897 array->n_children = newArrayChildren.size();
2898 // cppcheck-suppress autoVariables
2899 array->children = newArrayChildren.data();
2900
2901 // Process geometry columns:
2902 // - if the output encoding is WKB, then just note the geometry type and
2903 // envelope.
2904 // - otherwise convert to the output encoding.
2905 int nBuilderIdx = 0;
2906 if (!m_osFIDColumn.empty())
2907 {
2908 nBuilderIdx++;
2909 }
2910 std::map<std::string, std::shared_ptr<arrow::Array>>
2911 oMapGeomFieldNameToArray;
2912 for (int i = 0; i < nGeomFieldCount; ++i, ++nBuilderIdx)
2913 {
2914 const char *pszThisGeomFieldName =
2915 m_poFeatureDefn->GetGeomFieldDefn(i)->GetNameRef();
2916 int nIdx = poSchema->GetFieldIndex(pszThisGeomFieldName);
2917 if (nIdx < 0)
2918 {
2919 if (nGeomFieldCount == 1)
2920 nIdx = poSchema->GetFieldIndex(pszSingleGeomFieldName);
2921 if (nIdx < 0)
2922 {
2923 CPLError(CE_Failure, CPLE_AppDefined,
2924 "Cannot find geometry field '%s' in schema",
2925 pszThisGeomFieldName);
2926 return false;
2927 }
2928 }
2929
2930 if (strcmp(lSchema.children[nIdx]->format, "z") != 0 &&
2931 strcmp(lSchema.children[nIdx]->format, "Z") != 0)
2932 {
2933 CPLError(CE_Failure, CPLE_AppDefined,
2934 "Type of geometry field '%s' is not binary, but '%s'",
2935 pszThisGeomFieldName, lSchema.children[nIdx]->format);
2936 return false;
2937 }
2938
2939 const auto psGeomArray = array->children[nIdx];
2940 const uint8_t *pabyValidity =
2941 psGeomArray->null_count != 0
2942 ? static_cast<const uint8_t *>(psGeomArray->buffers[0])
2943 : nullptr;
2944 const bool bUseOffsets32 =
2945 (strcmp(lSchema.children[nIdx]->format, "z") == 0);
2946 const uint32_t *panOffsets32 =
2947 static_cast<const uint32_t *>(psGeomArray->buffers[1]) +
2948 psGeomArray->offset;
2949 const uint64_t *panOffsets64 =
2950 static_cast<const uint64_t *>(psGeomArray->buffers[1]) +
2951 psGeomArray->offset;
2952 GByte *pabyData =
2953 static_cast<GByte *>(const_cast<void *>(psGeomArray->buffers[2]));
2954 OGREnvelope sEnvelope;
2955 auto poBuilder = m_apoBuilders[nBuilderIdx].get();
2956
2957 for (size_t iRow = 0; iRow < static_cast<size_t>(psGeomArray->length);
2958 ++iRow)
2959 {
2960 bool bValidGeom = false;
2961
2962 if (!pabyValidity ||
2963 TestBit(pabyValidity,
2964 static_cast<size_t>(iRow + psGeomArray->offset)))
2965 {
2966 const auto nLen =
2967 bUseOffsets32 ? static_cast<size_t>(panOffsets32[iRow + 1] -
2968 panOffsets32[iRow])
2969 : static_cast<size_t>(panOffsets64[iRow + 1] -
2970 panOffsets64[iRow]);
2971 GByte *pabyWkb =
2972 pabyData + (bUseOffsets32
2973 ? panOffsets32[iRow]
2974 : static_cast<size_t>(panOffsets64[iRow]));
2975 if (m_aeGeomEncoding[i] == OGRArrowGeomEncoding::WKB)
2976 {
2977 FixupWKBGeometryBeforeWriting(pabyWkb, nLen);
2978
2979 uint32_t nType = 0;
2980 bool bNeedSwap = false;
2981 if (OGRWKBGetGeomType(pabyWkb, nLen, bNeedSwap, nType))
2982 {
2983 m_oSetWrittenGeometryTypes[i].insert(
2984 static_cast<OGRwkbGeometryType>(nType));
2985 if (OGRWKBGetBoundingBox(pabyWkb, nLen, sEnvelope))
2986 {
2987 bValidGeom = true;
2988 m_aoEnvelopes[i].Merge(sEnvelope);
2989
2990 if (m_bWriteBBoxStruct)
2991 {
2992 aadfMinX[i].push_back(
2993 castToFloatDown(sEnvelope.MinX));
2994 aadfMinY[i].push_back(
2995 castToFloatDown(sEnvelope.MinY));
2996 aadfMaxX[i].push_back(
2997 castToFloatUp(sEnvelope.MaxX));
2998 aadfMaxY[i].push_back(
2999 castToFloatUp(sEnvelope.MaxY));
3000 }
3001 }
3002 }
3003 }
3004 else
3005 {
3006 size_t nBytesConsumedOut = 0;
3007 OGRGeometry *poGeometry = nullptr;
3008 OGRGeometryFactory::createFromWkb(
3009 pabyWkb, nullptr, &poGeometry, nLen, wkbVariantIso,
3010 nBytesConsumedOut);
3011 if (BuildGeometry(poGeometry, i, poBuilder) != OGRERR_NONE)
3012 {
3013 delete poGeometry;
3014 return false;
3015 }
3016 bValidGeom = true;
3017 if (m_bWriteBBoxStruct)
3018 {
3019 poGeometry->getEnvelope(&sEnvelope);
3020 aadfMinX[i].push_back(castToFloatDown(sEnvelope.MinX));
3021 aadfMinY[i].push_back(castToFloatDown(sEnvelope.MinY));
3022 aadfMaxX[i].push_back(castToFloatUp(sEnvelope.MaxX));
3023 aadfMaxY[i].push_back(castToFloatUp(sEnvelope.MaxY));
3024 }
3025 delete poGeometry;
3026 }
3027 }
3028 else
3029 {
3030 if (m_aeGeomEncoding[i] != OGRArrowGeomEncoding::WKB)
3031 {
3032 if (BuildGeometry(nullptr, i, poBuilder) != OGRERR_NONE)
3033 return false;
3034 }
3035 }
3036
3037 if (!bValidGeom && m_bWriteBBoxStruct)
3038 {
3039 if ((bboxStructSchema[i].flags & ARROW_FLAG_NULLABLE))
3040 {
3041 bboxStructArray[i].null_count++;
3042 aabyBboxStructValidity[i][iRow / 8] &=
3043 ~(1 << static_cast<int>(iRow % 8));
3044 }
3045 aadfMinX[i].push_back(0.0f);
3046 aadfMinY[i].push_back(0.0f);
3047 aadfMaxX[i].push_back(0.0f);
3048 aadfMaxY[i].push_back(0.0f);
3049 }
3050 }
3051
3052 if (m_aeGeomEncoding[i] != OGRArrowGeomEncoding::WKB)
3053 {
3054 std::shared_ptr<arrow::Array> geomArray;
3055 auto status = poBuilder->Finish(&geomArray);
3056 if (!status.ok())
3057 {
3058 CPLError(CE_Failure, CPLE_AppDefined,
3059 "builder::Finish() for field %s failed with %s",
3060 pszThisGeomFieldName, status.message().c_str());
3061 return false;
3062 }
3063 oMapGeomFieldNameToArray[pszThisGeomFieldName] =
3064 std::move(geomArray);
3065 }
3066 }
3067
3068 auto poRecordBatchResult =
3069 arrow::ImportRecordBatch(array, std::move(poSchema));
3070 if (!poRecordBatchResult.ok())
3071 {
3072 CPLError(CE_Failure, CPLE_AppDefined,
3073 "ImportRecordBatch() failed with %s",
3074 poRecordBatchResult.status().message().c_str());
3075 return false;
3076 }
3077 auto poRecordBatch = *poRecordBatchResult;
3078
3079 if (!(bRebuildBatch || !oMapGeomFieldNameToArray.empty()))
3080 {
3081 for (int i = 0; i < m_poSchema->num_fields(); ++i)
3082 {
3083 const auto oIter =
3084 oMapGeomFieldNameToArray.find(m_poSchema->field(i)->name());
3085 auto l_array = (oIter != oMapGeomFieldNameToArray.end())
3086 ? oIter->second
3087 : poRecordBatch->column(i);
3088 const auto schemaType = m_poSchema->field(i)->type();
3089 const auto arrayType = l_array->type();
3090 if (schemaType->id() != arrow::Type::EXTENSION &&
3091 arrayType->id() == arrow::Type::EXTENSION)
3092 {
3093 bRebuildBatch = true;
3094 }
3095 else if (schemaType->id() != arrayType->id())
3096 {
3097 CPLDebug(
3098 "Arrow",
3099 "Field idx=%d name='%s', schema type=%s, array type=%s", i,
3100 m_poSchema->field(i)->name().c_str(),
3101 schemaType->ToString().c_str(),
3102 arrayType->ToString().c_str());
3103 }
3104 }
3105 }
3106
3107 // below assertion commented out since it is not strictly necessary, but
3108 // reflects what ImportRecordBatch() does.
3109 // CPLAssert(array->release == nullptr);
3110
3111 // We may need to reconstruct a final record batch that perfectly matches
3112 // the expected schema.
3113 if (bRebuildBatch || !oMapGeomFieldNameToArray.empty())
3114 {
3115 std::vector<std::shared_ptr<arrow::Array>> apoArrays;
3116 for (int i = 0; i < m_poSchema->num_fields(); ++i)
3117 {
3118 const auto oIter =
3119 oMapGeomFieldNameToArray.find(m_poSchema->field(i)->name());
3120 if (oIter != oMapGeomFieldNameToArray.end())
3121 apoArrays.emplace_back(oIter->second);
3122 else
3123 apoArrays.emplace_back(poRecordBatch->column(i));
3124
3125 auto expectedFieldType = m_poSchema->field(i)->type();
3126 if (expectedFieldType->id() == arrow::Type::EXTENSION)
3127 {
3128 auto extensionType = cpl::down_cast<arrow::ExtensionType *>(
3129 expectedFieldType.get());
3130 expectedFieldType = extensionType->storage_type();
3131 }
3132
3133 if (apoArrays.back()->type()->id() == arrow::Type::EXTENSION)
3134 {
3135 apoArrays.back() =
3136 std::static_pointer_cast<arrow::ExtensionArray>(
3137 apoArrays.back())
3138 ->storage();
3139 }
3140
3141 if (apoArrays.back()->type()->id() != expectedFieldType->id())
3142 {
3143 CPLError(
3144 CE_Failure, CPLE_AppDefined,
3145 "Field '%s' of unexpected type. Got '%s', expected '%s'",
3146 m_poSchema->field(i)->name().c_str(),
3147 apoArrays.back()->type()->name().c_str(),
3148 expectedFieldType->name().c_str());
3149 return false;
3150 }
3151 }
3152 poRecordBatchResult = arrow::RecordBatch::Make(
3153 m_poSchema, poRecordBatch->num_rows(), std::move(apoArrays));
3154 if (!poRecordBatchResult.ok())
3155 {
3156 CPLError(CE_Failure, CPLE_AppDefined,
3157 "RecordBatch::Make() failed with %s",
3158 poRecordBatchResult.status().message().c_str());
3159 return false;
3160 }
3161 poRecordBatch = *poRecordBatchResult;
3162 }
3163
3164 if (writeBatch(poRecordBatch))
3165 {
3166 m_nFeatureCount += poRecordBatch->num_rows();
3167 return true;
3168 }
3169 return false;
3170 #endif
3171 }
3172
3173 #endif /* OGARROWWRITERLAYER_HPP_INCLUDED */