Debian Codegraph @

ogr/ogrsf_frmts/arrow/ogrfeatherdriver.cpp

raw
1 /******************************************************************************
2 *
3 * Project: Feather Translator
4 * Purpose: Implements OGRFeatherDriver.
5 * Author: Even Rouault, <even.rouault at spatialys.com>
6 *
7 ******************************************************************************
8 * Copyright (c) 2022, Planet Labs
9 *
10 * SPDX-License-Identifier: MIT
11 ****************************************************************************/
12
13 #include "gdal_pam.h"
14 #include "ogrsf_frmts.h"
15
16 #include <map>
17 #include <mutex>
18
19 #include "ogr_feather.h"
20 #include "../arrow_common/ograrrowrandomaccessfile.h"
21 #include "../arrow_common/ograrrowwritablefile.h"
22 #include "../arrow_common/ograrrowdataset.hpp"
23
24 #include "ogrfeatherdrivercore.h"
25
26 /************************************************************************/
27 /* IsArrowIPCStream() */
28 /************************************************************************/
29
30 static bool IsArrowIPCStream(GDALOpenInfo *poOpenInfo)
31 {
32 // WARNING: if making changes in that method, reflect them in
33 // OGRFeatherDriverIsArrowIPCStreamBasic() in ogrfeatherdrivercore.cpp
34
35 if (STARTS_WITH_CI(poOpenInfo->pszFilename, "ARROW_IPC_STREAM:"))
36 return true;
37
38 constexpr int CONTINUATION_SIZE = 4; // 0xFFFFFFFF
39 constexpr int METADATA_SIZE_SIZE = 4;
40
41 // See
42 // https://arrow.apache.org/docs/format/Columnar.html#encapsulated-message-format
43 if (poOpenInfo->fpL != nullptr &&
44 poOpenInfo->nHeaderBytes >= CONTINUATION_SIZE + METADATA_SIZE_SIZE &&
45 memcmp(poOpenInfo->pabyHeader, "\xFF\xFF\xFF\xFF", CONTINUATION_SIZE) ==
46 0)
47 {
48 const char *pszExt = poOpenInfo->osExtension.c_str();
49 if (EQUAL(pszExt, "arrows") || EQUAL(pszExt, "ipc"))
50 return true;
51
52 const uint32_t nMetadataSize =
53 CPL_LSBUINT32PTR(poOpenInfo->pabyHeader + CONTINUATION_SIZE);
54 if (strcmp(poOpenInfo->pszFilename, "/vsistdin/") == 0)
55 {
56 if (poOpenInfo->IsSingleAllowedDriver("ARROW"))
57 return true;
58
59 // Padding after metadata and before body is not necessarily present
60 // but the body must be at least 4 bytes
61 constexpr int PADDING_MAX_SIZE = 4;
62
63 // /vsistdin/ cannot seek back beyond first MB
64 if (nMetadataSize >
65 1024 * 1024 -
66 (CONTINUATION_SIZE + METADATA_SIZE_SIZE + PADDING_MAX_SIZE))
67 {
68 return false;
69 }
70 const int nSizeToRead = CONTINUATION_SIZE + METADATA_SIZE_SIZE +
71 nMetadataSize + PADDING_MAX_SIZE;
72 if (!poOpenInfo->TryToIngest(nSizeToRead))
73 {
74 return false;
75 }
76
77 const std::string osTmpFilename(
78 VSIMemGenerateHiddenFilename("arrow"));
79 auto fp = VSIVirtualHandleUniquePtr(VSIFileFromMemBuffer(
80 osTmpFilename.c_str(), poOpenInfo->pabyHeader, nSizeToRead,
81 false));
82 auto infile = std::make_shared<OGRArrowRandomAccessFile>(
83 osTmpFilename.c_str(), std::move(fp));
84 auto options = arrow::ipc::IpcReadOptions::Defaults();
85 auto result =
86 arrow::ipc::RecordBatchStreamReader::Open(infile, options);
87 CPLDebug("ARROW", "RecordBatchStreamReader::Open(): %s",
88 result.status().message().c_str());
89 VSIUnlink(osTmpFilename.c_str());
90 return result.ok();
91 }
92
93 VSIFSeekL(poOpenInfo->fpL, 0, SEEK_END);
94 const auto nFileSize = VSIFTellL(poOpenInfo->fpL);
95 VSIFSeekL(poOpenInfo->fpL, 0, SEEK_SET);
96 if (nMetadataSize >
97 nFileSize - (CONTINUATION_SIZE + METADATA_SIZE_SIZE))
98 return false;
99
100 // Do not give ownership of poOpenInfo->fpL to infile
101 auto infile = std::make_shared<OGRArrowRandomAccessFile>(
102 poOpenInfo->pszFilename, poOpenInfo->fpL, false);
103 auto options = arrow::ipc::IpcReadOptions::Defaults();
104 auto result =
105 arrow::ipc::RecordBatchStreamReader::Open(infile, options);
106 VSIFSeekL(poOpenInfo->fpL, 0, SEEK_SET);
107 return result.ok();
108 }
109 return false;
110 }
111
112 /************************************************************************/
113 /* Open() */
114 /************************************************************************/
115
116 static GDALDataset *OGRFeatherDriverOpen(GDALOpenInfo *poOpenInfo)
117 {
118 if (poOpenInfo->eAccess == GA_Update)
119 {
120 return nullptr;
121 }
122
123 #if ARROW_VERSION_MAJOR >= 21
124 // Register geoarrow.wkb extension only if requested for Arrow driver
125 if (CPLTestBool(CPLGetConfigOption(
126 "OGR_ARROW_REGISTER_GEOARROW_WKB_EXTENSION", "NO")) &&
127 arrow::GetExtensionType(EXTENSION_NAME_GEOARROW_WKB))
128 {
129 CPL_IGNORE_RET_VAL(arrow::RegisterExtensionType(
130 std::make_shared<OGRGeoArrowWkbExtensionType>(
131 std::move(arrow::binary()), std::string())));
132 }
133 #endif
134
135 GDALOpenInfo *poOpenInfoForIdentify = poOpenInfo;
136 std::unique_ptr<GDALOpenInfo> poOpenInfoTmp;
137 if (STARTS_WITH(poOpenInfo->pszFilename, "gdalvsi://"))
138 {
139 poOpenInfoTmp = std::make_unique<GDALOpenInfo>(poOpenInfo->pszFilename +
140 strlen("gdalvsi://"),
141 poOpenInfo->nOpenFlags);
142 poOpenInfoForIdentify = poOpenInfoTmp.get();
143 }
144
145 const bool bIsStreamingFormat = IsArrowIPCStream(poOpenInfoForIdentify);
146 if (!bIsStreamingFormat &&
147 !OGRFeatherDriverIsArrowFileFormat(poOpenInfoForIdentify))
148 {
149 return nullptr;
150 }
151
152 std::shared_ptr<arrow::io::RandomAccessFile> infile;
153 if (STARTS_WITH_CI(poOpenInfo->pszFilename, "ARROW_IPC_STREAM:"))
154 {
155 const std::string osFilename(poOpenInfo->pszFilename +
156 strlen("ARROW_IPC_STREAM:"));
157 auto fp =
158 VSIVirtualHandleUniquePtr(VSIFOpenL(osFilename.c_str(), "rb"));
159 if (fp == nullptr)
160 {
161 CPLError(CE_Failure, CPLE_FileIO, "Cannot open %s",
162 osFilename.c_str());
163 return nullptr;
164 }
165 infile = std::make_shared<OGRArrowRandomAccessFile>(osFilename.c_str(),
166 std::move(fp));
167 }
168 else if (STARTS_WITH(poOpenInfo->pszFilename, "/vsi") ||
169 CPLTestBool(CPLGetConfigOption("OGR_ARROW_USE_VSI", "NO")))
170 {
171 VSIVirtualHandleUniquePtr fp(poOpenInfo->fpL);
172 poOpenInfo->fpL = nullptr;
173 infile = std::make_shared<OGRArrowRandomAccessFile>(
174 poOpenInfo->pszFilename, std::move(fp));
175 }
176 else
177 {
178 // FileSystemFromUriOrPath() doesn't like relative paths
179 // so transform them to absolute.
180 std::string osPath(poOpenInfo->pszFilename);
181 if (CPLIsFilenameRelative(osPath.c_str()))
182 {
183 char *pszCurDir = CPLGetCurrentDir();
184 if (pszCurDir == nullptr)
185 return nullptr;
186 osPath = CPLFormFilenameSafe(pszCurDir, osPath.c_str(), nullptr);
187 CPLFree(pszCurDir);
188 }
189
190 std::string osFSPath;
191 auto poFS =
192 arrow::fs::FileSystemFromUriOrPath(osPath.c_str(), &osFSPath);
193 if (!poFS.ok())
194 {
195 CPLError(CE_Failure, CPLE_AppDefined,
196 "arrow::fs::FileSystemFromUriOrPath failed with %s",
197 poFS.status().message().c_str());
198 return nullptr;
199 }
200 auto result = (*poFS)->OpenInputFile(osFSPath);
201 if (!result.ok())
202 {
203 CPLError(CE_Failure, CPLE_AppDefined,
204 "OpenInputFile() failed with %s",
205 result.status().message().c_str());
206 return nullptr;
207 }
208 infile = *result;
209 }
210
211 auto poMemoryPool = std::shared_ptr<arrow::MemoryPool>(
212 arrow::MemoryPool::CreateDefault().release());
213 auto options = arrow::ipc::IpcReadOptions::Defaults();
214 options.memory_pool = poMemoryPool.get();
215
216 auto poDS = std::make_unique<OGRFeatherDataset>(poMemoryPool);
217 if (bIsStreamingFormat)
218 {
219 auto result =
220 arrow::ipc::RecordBatchStreamReader::Open(infile, options);
221 if (!result.ok())
222 {
223 CPLError(CE_Failure, CPLE_AppDefined,
224 "RecordBatchStreamReader::Open() failed with %s",
225 result.status().message().c_str());
226 return nullptr;
227 }
228 auto poRecordBatchStreamReader = *result;
229 const bool bSeekable =
230 !STARTS_WITH_CI(poOpenInfo->pszFilename, "ARROW_IPC_STREAM:") &&
231 strcmp(poOpenInfo->pszFilename, "/vsistdin/") != 0;
232 std::string osLayername = CPLGetBasenameSafe(poOpenInfo->pszFilename);
233 if (osLayername.empty())
234 osLayername = "layer";
235 auto poLayer = std::make_unique<OGRFeatherLayer>(
236 poDS.get(), osLayername.c_str(), infile, bSeekable, options,
237 poRecordBatchStreamReader, poOpenInfo->papszOpenOptions);
238 poDS->SetLayer(std::move(poLayer));
239
240 // Pre-load field domains, as this depends on the first record batch
241 auto poLayerPtr = poDS->GetLayer(0);
242 const auto poFeatureDefn = poLayerPtr->GetLayerDefn();
243 bool bHasReadBatch = false;
244 for (int i = 0; i < poFeatureDefn->GetFieldCount(); ++i)
245 {
246 const auto poFieldDefn = poFeatureDefn->GetFieldDefn(i);
247 const auto &osDomainName = poFieldDefn->GetDomainName();
248 if (!osDomainName.empty())
249 {
250 if (!bHasReadBatch)
251 {
252 bHasReadBatch = true;
253 delete poLayerPtr->GetNextFeature();
254 poLayerPtr->ResetReading();
255 }
256 poDS->GetFieldDomain(osDomainName);
257 }
258 }
259 }
260 else
261 {
262 auto result = arrow::ipc::RecordBatchFileReader::Open(infile, options);
263 if (!result.ok())
264 {
265 CPLError(CE_Failure, CPLE_AppDefined,
266 "RecordBatchFileReader::Open() failed with %s",
267 result.status().message().c_str());
268 return nullptr;
269 }
270 auto poRecordBatchReader = *result;
271 auto poLayer = std::make_unique<OGRFeatherLayer>(
272 poDS.get(), CPLGetBasenameSafe(poOpenInfo->pszFilename).c_str(),
273 poRecordBatchReader, poOpenInfo->papszOpenOptions);
274 poDS->SetLayer(std::move(poLayer));
275 }
276 return poDS.release();
277 }
278
279 /************************************************************************/
280 /* Create() */
281 /************************************************************************/
282
283 static GDALDataset *OGRFeatherDriverCreate(const char *pszName, int nXSize,
284 int nYSize, int nBands,
285 GDALDataType eType,
286 CSLConstList /* papszOptions */)
287 {
288 if (!(nXSize == 0 && nYSize == 0 && nBands == 0 && eType == GDT_Unknown))
289 return nullptr;
290
291 std::shared_ptr<arrow::io::OutputStream> out_file;
292 if (STARTS_WITH(pszName, "/vsi") ||
293 CPLTestBool(CPLGetConfigOption("OGR_ARROW_USE_VSI", "YES")))
294 {
295 VSIVirtualHandleUniquePtr fp =
296 VSIFilesystemHandler::OpenStatic(pszName, "wb");
297 if (fp == nullptr)
298 {
299 CPLError(CE_Failure, CPLE_FileIO, "Cannot create %s", pszName);
300 return nullptr;
301 }
302 out_file = std::make_shared<OGRArrowWritableFile>(std::move(fp));
303 }
304 else
305 {
306 auto result = arrow::io::FileOutputStream::Open(pszName);
307 if (!result.ok())
308 {
309 CPLError(CE_Failure, CPLE_FileIO, "Cannot create %s: %s", pszName,
310 result.status().message().c_str());
311 return nullptr;
312 }
313 out_file = *result;
314 }
315
316 return new OGRFeatherWriterDataset(pszName, out_file);
317 }
318
319 /************************************************************************/
320 /* OGRFeatherDriver() */
321 /************************************************************************/
322
323 class OGRFeatherDriver final : public GDALDriver
324 {
325 std::recursive_mutex m_oMutex{};
326 bool m_bMetadataInitialized = false;
327 void InitMetadata();
328
329 public:
330 const char *GetMetadataItem(const char *pszName,
331 const char *pszDomain) override;
332
333 CSLConstList GetMetadata(const char *pszDomain) override
334 {
335 std::lock_guard oLock(m_oMutex);
336 InitMetadata();
337 return GDALDriver::GetMetadata(pszDomain);
338 }
339 };
340
341 const char *OGRFeatherDriver::GetMetadataItem(const char *pszName,
342 const char *pszDomain)
343 {
344 std::lock_guard oLock(m_oMutex);
345 if (EQUAL(pszName, GDAL_DS_LAYER_CREATIONOPTIONLIST))
346 {
347 InitMetadata();
348 }
349 return GDALDriver::GetMetadataItem(pszName, pszDomain);
350 }
351
352 void OGRFeatherDriver::InitMetadata()
353 {
354 if (m_bMetadataInitialized)
355 return;
356 m_bMetadataInitialized = true;
357
358 CPLXMLTreeCloser oTree(
359 CPLCreateXMLNode(nullptr, CXT_Element, "LayerCreationOptionList"));
360
361 std::vector<const char *> apszCompressionMethods;
362 bool bHasLZ4 = false;
363 for (const char *pszMethod : {"ZSTD", "LZ4"})
364 {
365 auto oResult = arrow::util::Codec::GetCompressionType(
366 CPLString(pszMethod).tolower());
367 if (oResult.ok() && arrow::util::Codec::IsAvailable(*oResult))
368 {
369 if (EQUAL(pszMethod, "LZ4"))
370 bHasLZ4 = true;
371 apszCompressionMethods.emplace_back(pszMethod);
372 }
373 }
374
375 {
376 auto psOption = CPLCreateXMLNode(oTree.get(), CXT_Element, "Option");
377 CPLAddXMLAttributeAndValue(psOption, "name", "FORMAT");
378 CPLAddXMLAttributeAndValue(psOption, "type", "string-select");
379 CPLAddXMLAttributeAndValue(psOption, "description",
380 "File format variant");
381 for (const char *pszEncoding : {"FILE", "STREAM"})
382 {
383 auto poValueNode = CPLCreateXMLNode(psOption, CXT_Element, "Value");
384 CPLCreateXMLNode(poValueNode, CXT_Text, pszEncoding);
385 }
386 }
387
388 {
389 auto psOption = CPLCreateXMLNode(oTree.get(), CXT_Element, "Option");
390 CPLAddXMLAttributeAndValue(psOption, "name", "COMPRESSION");
391 CPLAddXMLAttributeAndValue(psOption, "type", "string-select");
392 CPLAddXMLAttributeAndValue(psOption, "description",
393 "Compression method");
394 CPLAddXMLAttributeAndValue(psOption, "default",
395 bHasLZ4 ? "LZ4" : "NONE");
396 {
397 auto poValueNode = CPLCreateXMLNode(psOption, CXT_Element, "Value");
398 CPLAddXMLAttributeAndValue(poValueNode, "alias", "UNCOMPRESSED");
399 CPLCreateXMLNode(poValueNode, CXT_Text, "NONE");
400 }
401 for (const char *pszMethod : apszCompressionMethods)
402 {
403 auto poValueNode = CPLCreateXMLNode(psOption, CXT_Element, "Value");
404 CPLCreateXMLNode(poValueNode, CXT_Text, pszMethod);
405 }
406 }
407
408 {
409 auto psOption = CPLCreateXMLNode(oTree.get(), CXT_Element, "Option");
410 CPLAddXMLAttributeAndValue(psOption, "name", "GEOMETRY_ENCODING");
411 CPLAddXMLAttributeAndValue(psOption, "type", "string-select");
412 CPLAddXMLAttributeAndValue(psOption, "description",
413 "Encoding of geometry columns");
414 CPLAddXMLAttributeAndValue(psOption, "default", "GEOARROW");
415 for (const char *pszEncoding :
416 {"GEOARROW", "GEOARROW_INTERLEAVED", "WKB", "WKT"})
417 {
418 auto poValueNode = CPLCreateXMLNode(psOption, CXT_Element, "Value");
419 CPLCreateXMLNode(poValueNode, CXT_Text, pszEncoding);
420 if (EQUAL(pszEncoding, "GEOARROW"))
421 CPLAddXMLAttributeAndValue(poValueNode, "alias",
422 "GEOARROW_STRUCT");
423 }
424 }
425
426 {
427 auto psOption = CPLCreateXMLNode(oTree.get(), CXT_Element, "Option");
428 CPLAddXMLAttributeAndValue(psOption, "name", "BATCH_SIZE");
429 CPLAddXMLAttributeAndValue(psOption, "type", "integer");
430 CPLAddXMLAttributeAndValue(psOption, "description",
431 "Maximum number of rows per batch");
432 CPLAddXMLAttributeAndValue(psOption, "default", "65536");
433 }
434
435 {
436 auto psOption = CPLCreateXMLNode(oTree.get(), CXT_Element, "Option");
437 CPLAddXMLAttributeAndValue(psOption, "name", "GEOMETRY_NAME");
438 CPLAddXMLAttributeAndValue(psOption, "type", "string");
439 CPLAddXMLAttributeAndValue(psOption, "description",
440 "Name of geometry column");
441 CPLAddXMLAttributeAndValue(psOption, "default", "geometry");
442 }
443
444 {
445 auto psOption = CPLCreateXMLNode(oTree.get(), CXT_Element, "Option");
446 CPLAddXMLAttributeAndValue(psOption, "name", "FID");
447 CPLAddXMLAttributeAndValue(psOption, "type", "string");
448 CPLAddXMLAttributeAndValue(psOption, "description",
449 "Name of the FID column to create");
450 }
451
452 {
453 auto psOption = CPLCreateXMLNode(oTree.get(), CXT_Element, "Option");
454 CPLAddXMLAttributeAndValue(psOption, "name", "TIMESTAMP_WITH_OFFSET");
455 CPLAddXMLAttributeAndValue(psOption, "type", "string-select");
456 CPLAddXMLAttributeAndValue(psOption, "default", "AUTO");
457 CPLAddXMLAttributeAndValue(
458 psOption, "description",
459 "Whether timestamp with offset fields should be used");
460 CPLCreateXMLElementAndValue(psOption, "Value", "AUTO");
461 CPLCreateXMLElementAndValue(psOption, "Value", "YES");
462 CPLCreateXMLElementAndValue(psOption, "Value", "NO");
463 }
464
465 char *pszXML = CPLSerializeXMLTree(oTree.get());
466 GDALDriver::SetMetadataItem(GDAL_DS_LAYER_CREATIONOPTIONLIST, pszXML);
467 CPLFree(pszXML);
468 }
469
470 /************************************************************************/
471 /* RegisterOGRArrow() */
472 /************************************************************************/
473
474 void RegisterOGRArrow()
475 {
476 if (GDALGetDriverByName(DRIVER_NAME) != nullptr)
477 return;
478
479 auto poDriver = std::make_unique<OGRFeatherDriver>();
480
481 OGRFeatherDriverSetCommonMetadata(poDriver.get());
482
483 poDriver->pfnOpen = OGRFeatherDriverOpen;
484 poDriver->pfnCreate = OGRFeatherDriverCreate;
485
486 poDriver->SetMetadataItem("ARROW_VERSION", ARROW_VERSION_STRING);
487
488 GetGDALDriverManager()->RegisterDriver(poDriver.release());
489
490 #if ARROW_VERSION_MAJOR >= 16
491 // Mostly for tests
492 const char *pszPath =
493 CPLGetConfigOption("OGR_ARROW_LOAD_FILE_SYSTEM_FACTORIES", nullptr);
494 if (pszPath)
495 {
496 auto result = arrow::fs::LoadFileSystemFactories(pszPath);
497 if (!result.ok())
498 {
499 CPLError(CE_Warning, CPLE_AppDefined,
500 "arrow::fs::LoadFileSystemFactories() failed with %s",
501 result.message().c_str());
502 }
503 }
504 #endif
505 }