Debian Codegraph @

ogr/ogrsf_frmts/arrow/ogrfeatherwriterlayer.cpp

raw
1 /******************************************************************************
2 *
3 * Project: Feather Translator
4 * Purpose: Implements OGRFeatherDriver.
5 * Author: Even Rouault, <even.rouault at spatialys.com>
6 *
7 ******************************************************************************
8 * Copyright (c) 2022, Planet Labs
9 *
10 * SPDX-License-Identifier: MIT
11 ****************************************************************************/
12
13 #include "ogr_feather.h"
14
15 #include "../arrow_common/ograrrowwriterlayer.hpp"
16
17 /************************************************************************/
18 /* OGRFeatherWriterLayer() */
19 /************************************************************************/
20
21 OGRFeatherWriterLayer::OGRFeatherWriterLayer(
22 GDALDataset *poDS, arrow::MemoryPool *poMemoryPool,
23 const std::shared_ptr<arrow::io::OutputStream> &poOutputStream,
24 const char *pszLayerName)
25 : OGRArrowWriterLayer(poMemoryPool, poOutputStream, pszLayerName),
26 m_poDS(poDS)
27 {
28 m_bWriteFieldArrowExtensionName = true;
29 }
30
31 /************************************************************************/
32 /* Close() */
33 /************************************************************************/
34
35 bool OGRFeatherWriterLayer::Close()
36 {
37 if (m_bInitializationOK)
38 {
39 if (!FinalizeWriting())
40 return false;
41 }
42
43 return true;
44 }
45
46 /************************************************************************/
47 /* IsSupportedGeometryType() */
48 /************************************************************************/
49
50 bool OGRFeatherWriterLayer::IsSupportedGeometryType(
51 OGRwkbGeometryType eGType) const
52 {
53 if (eGType != wkbFlatten(eGType))
54 {
55 const auto osConfigOptionName =
56 "OGR_" + GetDriverUCName() + "_ALLOW_ALL_DIMS";
57 if (!CPLTestBool(CPLGetConfigOption(osConfigOptionName.c_str(), "NO")))
58 {
59 CPLError(CE_Failure, CPLE_NotSupported,
60 "Only 2D geometry types are supported (unless the "
61 "%s configuration option is set to YES)",
62 osConfigOptionName.c_str());
63 return false;
64 }
65 }
66 return true;
67 }
68
69 /************************************************************************/
70 /* SetOptions() */
71 /************************************************************************/
72
73 bool OGRFeatherWriterLayer::SetOptions(const std::string &osFilename,
74 CSLConstList papszOptions,
75 const OGRSpatialReference *poSpatialRef,
76 OGRwkbGeometryType eGType)
77 {
78 m_aosCreationOptions = papszOptions;
79
80 const char *pszDefaultFormat =
81 (EQUAL(CPLGetExtensionSafe(osFilename.c_str()).c_str(), "arrows") ||
82 STARTS_WITH_CI(osFilename.c_str(), "/vsistdout"))
83 ? "STREAM"
84 : "FILE";
85 m_bStreamFormat =
86 EQUAL(CSLFetchNameValueDef(papszOptions, "FORMAT", pszDefaultFormat),
87 "STREAM");
88
89 const char *pszGeomEncoding =
90 CSLFetchNameValue(papszOptions, "GEOMETRY_ENCODING");
91 m_eGeomEncoding = OGRArrowGeomEncoding::GEOARROW_STRUCT_GENERIC;
92 if (pszGeomEncoding)
93 {
94 if (EQUAL(pszGeomEncoding, "WKB"))
95 m_eGeomEncoding = OGRArrowGeomEncoding::WKB;
96 else if (EQUAL(pszGeomEncoding, "WKT"))
97 m_eGeomEncoding = OGRArrowGeomEncoding::WKT;
98 else if (EQUAL(pszGeomEncoding, "GEOARROW_INTERLEAVED"))
99 m_eGeomEncoding = OGRArrowGeomEncoding::GEOARROW_FSL_GENERIC;
100 else if (EQUAL(pszGeomEncoding, "GEOARROW") ||
101 EQUAL(pszGeomEncoding, "GEOARROW_STRUCT"))
102 m_eGeomEncoding = OGRArrowGeomEncoding::GEOARROW_STRUCT_GENERIC;
103 else
104 {
105 CPLError(CE_Failure, CPLE_NotSupported,
106 "Unsupported GEOMETRY_ENCODING = %s", pszGeomEncoding);
107 return false;
108 }
109 }
110
111 if (eGType != wkbNone)
112 {
113 if (!IsSupportedGeometryType(eGType))
114 {
115 return false;
116 }
117
118 if (poSpatialRef == nullptr)
119 {
120 CPLError(CE_Warning, CPLE_AppDefined,
121 "Geometry column should have an associated CRS");
122 }
123
124 m_poFeatureDefn->SetGeomType(eGType);
125 auto eGeomEncoding = m_eGeomEncoding;
126 if (eGeomEncoding == OGRArrowGeomEncoding::GEOARROW_FSL_GENERIC ||
127 eGeomEncoding == OGRArrowGeomEncoding::GEOARROW_STRUCT_GENERIC)
128 {
129 const auto eEncodingType = eGeomEncoding;
130 eGeomEncoding = GetPreciseArrowGeomEncoding(eEncodingType, eGType);
131 if (eGeomEncoding == eEncodingType)
132 return false;
133 }
134 m_aeGeomEncoding.push_back(eGeomEncoding);
135 m_poFeatureDefn->GetGeomFieldDefn(0)->SetName(
136 CSLFetchNameValueDef(papszOptions, "GEOMETRY_NAME", "geometry"));
137 if (poSpatialRef)
138 {
139 auto poSRS = poSpatialRef->Clone();
140 m_poFeatureDefn->GetGeomFieldDefn(0)->SetSpatialRef(poSRS);
141 poSRS->Release();
142 }
143 }
144
145 m_osFIDColumn = CSLFetchNameValueDef(papszOptions, "FID", "");
146
147 const char *pszCompression = CSLFetchNameValue(papszOptions, "COMPRESSION");
148 if (pszCompression == nullptr)
149 {
150 auto oResult = arrow::util::Codec::GetCompressionType("lz4");
151 if (oResult.ok() && arrow::util::Codec::IsAvailable(*oResult))
152 {
153 pszCompression = "LZ4";
154 }
155 else
156 {
157 pszCompression = "NONE";
158 }
159 }
160
161 if (EQUAL(pszCompression, "NONE"))
162 pszCompression = "UNCOMPRESSED";
163 auto oResult = arrow::util::Codec::GetCompressionType(
164 CPLString(pszCompression).tolower());
165 if (!oResult.ok())
166 {
167 CPLError(CE_Failure, CPLE_NotSupported,
168 "Unrecognized compression method: %s", pszCompression);
169 return false;
170 }
171 m_eCompression = *oResult;
172 if (!arrow::util::Codec::IsAvailable(m_eCompression))
173 {
174 CPLError(CE_Failure, CPLE_NotSupported,
175 "Compression method %s is known, but libarrow has not "
176 "been built with support for it",
177 pszCompression);
178 return false;
179 }
180
181 const char *pszRowGroupSize = CSLFetchNameValue(papszOptions, "BATCH_SIZE");
182 if (pszRowGroupSize)
183 {
184 auto nRowGroupSize = static_cast<int64_t>(atoll(pszRowGroupSize));
185 if (nRowGroupSize > 0)
186 {
187 if (nRowGroupSize > INT_MAX)
188 nRowGroupSize = INT_MAX;
189 m_nRowGroupSize = nRowGroupSize;
190 }
191 }
192
193 m_bInitializationOK = true;
194 return true;
195 }
196
197 /************************************************************************/
198 /* CloseFileWriter() */
199 /************************************************************************/
200
201 bool OGRFeatherWriterLayer::CloseFileWriter()
202 {
203 auto status = m_poFileWriter->Close();
204 if (!status.ok())
205 {
206 CPLError(CE_Failure, CPLE_AppDefined,
207 "FileWriter::Close() failed with %s",
208 status.message().c_str());
209 }
210 return status.ok();
211 }
212
213 /************************************************************************/
214 /* CreateSchema() */
215 /************************************************************************/
216
217 void OGRFeatherWriterLayer::CreateSchema()
218 {
219 CreateSchemaCommon();
220
221 if (m_poFeatureDefn->GetGeomFieldCount() != 0 &&
222 CPLTestBool(CPLGetConfigOption("OGR_ARROW_WRITE_GEO", "YES")))
223 {
224 CPLJSONObject oRoot;
225 oRoot.Add("schema_version", "0.1.0");
226 oRoot.Add("primary_column",
227 m_poFeatureDefn->GetGeomFieldDefn(0)->GetNameRef());
228 CPLJSONObject oColumns;
229 oRoot.Add("columns", oColumns);
230 for (int i = 0; i < m_poFeatureDefn->GetGeomFieldCount(); ++i)
231 {
232 const auto poGeomFieldDefn = m_poFeatureDefn->GetGeomFieldDefn(i);
233 CPLJSONObject oColumn;
234 oColumns.Add(poGeomFieldDefn->GetNameRef(), oColumn);
235 oColumn.Add("encoding",
236 GetGeomEncodingAsString(m_aeGeomEncoding[i], false));
237
238 const auto poSRS = poGeomFieldDefn->GetSpatialRef();
239 if (poSRS)
240 {
241 const char *const apszOptions[] = {"FORMAT=WKT2_2019",
242 "MULTILINE=NO", nullptr};
243 char *pszWKT = nullptr;
244 poSRS->exportToWkt(&pszWKT, apszOptions);
245 if (pszWKT)
246 oColumn.Add("crs", pszWKT);
247 CPLFree(pszWKT);
248
249 const double dfCoordEpoch = poSRS->GetCoordinateEpoch();
250 if (dfCoordEpoch > 0)
251 oColumn.Add("epoch", dfCoordEpoch);
252 }
253
254 #if 0
255 if( m_aoEnvelopes[i].IsInit() &&
256 CPLTestBool(CPLGetConfigOption(
257 "OGR_ARROW_WRITE_BBOX", "YES")) )
258 {
259 CPLJSONArray oBBOX;
260 oBBOX.Add(m_aoEnvelopes[i].MinX);
261 oBBOX.Add(m_aoEnvelopes[i].MinY);
262 oBBOX.Add(m_aoEnvelopes[i].MaxX);
263 oBBOX.Add(m_aoEnvelopes[i].MaxY);
264 oColumn.Add("bbox", oBBOX);
265 }
266 #endif
267 const auto eType = poGeomFieldDefn->GetType();
268 if (CPLTestBool(CPLGetConfigOption(
269 "OGR_ARROW_WRITE_GDAL_GEOMETRY_TYPE", "YES")) &&
270 eType == wkbFlatten(eType))
271 {
272 // Geometry type, place under a temporary "gdal:geometry_type"
273 // property pending acceptance of proposal at
274 // https://github.com/opengeospatial/geoparquet/issues/41
275 const char *pszType = "mixed";
276 if (wkbPoint == eType)
277 pszType = "Point";
278 else if (wkbLineString == eType)
279 pszType = "LineString";
280 else if (wkbPolygon == eType)
281 pszType = "Polygon";
282 else if (wkbMultiPoint == eType)
283 pszType = "MultiPoint";
284 else if (wkbMultiLineString == eType)
285 pszType = "MultiLineString";
286 else if (wkbMultiPolygon == eType)
287 pszType = "MultiPolygon";
288 else if (wkbGeometryCollection == eType)
289 pszType = "GeometryCollection";
290 oColumn.Add("gdal:geometry_type", pszType);
291 }
292 }
293
294 auto kvMetadata = m_poSchema->metadata()
295 ? m_poSchema->metadata()->Copy()
296 : std::make_shared<arrow::KeyValueMetadata>();
297 kvMetadata->Append("geo",
298 oRoot.Format(CPLJSONObject::PrettyFormat::Plain));
299 m_poSchema = m_poSchema->WithMetadata(kvMetadata);
300 CPLAssert(m_poSchema);
301 }
302 }
303
304 /************************************************************************/
305 /* CreateWriter() */
306 /************************************************************************/
307
308 void OGRFeatherWriterLayer::CreateWriter()
309 {
310 CPLAssert(m_poFileWriter == nullptr);
311
312 if (m_poSchema == nullptr)
313 {
314 CreateSchema();
315 }
316 else
317 {
318 FinalizeSchema();
319 }
320
321 auto options = arrow::ipc::IpcWriteOptions::Defaults();
322 options.memory_pool = m_poMemoryPool;
323
324 {
325 auto result = arrow::util::Codec::Create(m_eCompression);
326 if (!result.ok())
327 {
328 CPLError(CE_Failure, CPLE_AppDefined,
329 "Codec::Create() failed with %s",
330 result.status().message().c_str());
331 }
332 else
333 {
334 options.codec.reset(result->release());
335 }
336 }
337
338 if (m_bStreamFormat)
339 {
340 auto result =
341 arrow::ipc::MakeStreamWriter(m_poOutputStream, m_poSchema, options);
342 if (!result.ok())
343 {
344 CPLError(CE_Failure, CPLE_AppDefined,
345 "arrow::ipc::MakeStreamWriter() failed with %s",
346 result.status().message().c_str());
347 }
348 else
349 {
350 m_poFileWriter = *result;
351 }
352 }
353 else
354 {
355 m_poFooterKeyValueMetadata =
356 std::make_shared<arrow::KeyValueMetadata>();
357 auto result = arrow::ipc::MakeFileWriter(
358 m_poOutputStream, m_poSchema, options, m_poFooterKeyValueMetadata);
359 if (!result.ok())
360 {
361 CPLError(CE_Failure, CPLE_AppDefined,
362 "arrow::ipc::MakeFileWriter() failed with %s",
363 result.status().message().c_str());
364 }
365 else
366 {
367 m_poFileWriter = *result;
368 }
369 }
370 }
371
372 /************************************************************************/
373 /* PerformStepsBeforeFinalFlushGroup() */
374 /************************************************************************/
375
376 // Add a gdal:geo extension metadata for now, which embeds a bbox
377 void OGRFeatherWriterLayer::PerformStepsBeforeFinalFlushGroup()
378 {
379 if (m_poFooterKeyValueMetadata &&
380 m_poFeatureDefn->GetGeomFieldCount() != 0 &&
381 CPLTestBool(CPLGetConfigOption("OGR_ARROW_WRITE_GDAL_FOOTER", "YES")))
382 {
383 CPLJSONObject oRoot;
384 oRoot.Add("primary_column",
385 m_poFeatureDefn->GetGeomFieldDefn(0)->GetNameRef());
386 CPLJSONObject oColumns;
387 oRoot.Add("columns", oColumns);
388 for (int i = 0; i < m_poFeatureDefn->GetGeomFieldCount(); ++i)
389 {
390 const auto poGeomFieldDefn = m_poFeatureDefn->GetGeomFieldDefn(i);
391 CPLJSONObject oColumn;
392 oColumns.Add(poGeomFieldDefn->GetNameRef(), oColumn);
393 oColumn.Add("encoding",
394 GetGeomEncodingAsString(m_aeGeomEncoding[i], true));
395
396 const auto poSRS = poGeomFieldDefn->GetSpatialRef();
397 if (poSRS)
398 {
399 const char *const apszOptions[] = {"FORMAT=WKT2_2019",
400 "MULTILINE=NO", nullptr};
401 char *pszWKT = nullptr;
402 poSRS->exportToWkt(&pszWKT, apszOptions);
403 if (pszWKT)
404 oColumn.Add("crs", pszWKT);
405 CPLFree(pszWKT);
406
407 const double dfCoordEpoch = poSRS->GetCoordinateEpoch();
408 if (dfCoordEpoch > 0)
409 oColumn.Add("epoch", dfCoordEpoch);
410 }
411
412 if (m_aoEnvelopes[i].IsInit())
413 {
414 CPLJSONArray oBBOX;
415 oBBOX.Add(m_aoEnvelopes[i].MinX);
416 oBBOX.Add(m_aoEnvelopes[i].MinY);
417 oBBOX.Add(m_aoEnvelopes[i].MaxX);
418 oBBOX.Add(m_aoEnvelopes[i].MaxY);
419 oColumn.Add("bbox", oBBOX);
420 }
421 }
422
423 m_poFooterKeyValueMetadata->Append(
424 GDAL_GEO_FOOTER_KEY,
425 oRoot.Format(CPLJSONObject::PrettyFormat::Plain));
426 }
427 }
428
429 /************************************************************************/
430 /* FlushGroup() */
431 /************************************************************************/
432
433 bool OGRFeatherWriterLayer::FlushGroup()
434 {
435 std::vector<std::shared_ptr<arrow::Array>> columns;
436 auto ret = WriteArrays(
437 [&columns](const std::shared_ptr<arrow::Field> &,
438 const std::shared_ptr<arrow::Array> &array)
439 {
440 columns.emplace_back(array);
441 return true;
442 });
443
444 if (ret)
445 {
446 auto poRecordBatch = arrow::RecordBatch::Make(
447 m_poSchema, !columns.empty() ? columns[0]->length() : 0, columns);
448 auto status = m_poFileWriter->WriteRecordBatch(*poRecordBatch);
449 if (!status.ok())
450 {
451 CPLError(CE_Failure, CPLE_AppDefined,
452 "WriteRecordBatch() failed with %s",
453 status.message().c_str());
454 ret = false;
455 }
456 }
457
458 ClearArrayBuilers();
459 return ret;
460 }
461
462 /************************************************************************/
463 /* WriteArrowBatch() */
464 /************************************************************************/
465
466 inline bool
467 OGRFeatherWriterLayer::WriteArrowBatch(const struct ArrowSchema *schema,
468 struct ArrowArray *array,
469 CSLConstList papszOptions)
470 {
471 return WriteArrowBatchInternal(
472 schema, array, papszOptions,
473 [this](const std::shared_ptr<arrow::RecordBatch> &poBatch)
474 {
475 auto status = m_poFileWriter->WriteRecordBatch(*poBatch);
476 if (!status.ok())
477 {
478 CPLError(CE_Failure, CPLE_AppDefined,
479 "WriteRecordBatch() failed: %s",
480 status.message().c_str());
481 return false;
482 }
483
484 return true;
485 });
486 }