Debian Codegraph @

ogr/ogrsf_frmts/parquet/ogrparquetdataset.cpp

raw
1 /******************************************************************************
2 *
3 * Project: Parquet Translator
4 * Purpose: Implements OGRParquetDriver.
5 * Author: Even Rouault, <even.rouault at spatialys.com>
6 *
7 ******************************************************************************
8 * Copyright (c) 2022, Planet Labs
9 *
10 * SPDX-License-Identifier: MIT
11 ****************************************************************************/
12
13 #include "ogr_parquet.h"
14 #include "memdataset.h"
15 #include "ogr_swq.h"
16
17 #include "../arrow_common/ograrrowdataset.hpp"
18 #include "../arrow_common/ograrrowlayer.hpp"
19 #include "../arrow_common/vsiarrowfilesystem.hpp"
20
21 /************************************************************************/
22 /* OGRParquetDataset() */
23 /************************************************************************/
24
25 OGRParquetDataset::OGRParquetDataset()
26 : OGRArrowDataset(arrow::MemoryPool::CreateDefault())
27 {
28 }
29
30 /************************************************************************/
31 /* ~OGRParquetDataset() */
32 /************************************************************************/
33
34 OGRParquetDataset::~OGRParquetDataset()
35 {
36 OGRParquetDataset::Close();
37 }
38
39 /************************************************************************/
40 /* Close() */
41 /************************************************************************/
42
43 CPLErr OGRParquetDataset::Close(GDALProgressFunc, void *)
44 {
45 CPLErr eErr = CE_None;
46 if (nOpenFlags != OPEN_FLAGS_CLOSED)
47 {
48 // libarrow might continue to do I/O in auxiliary threads on the underlying
49 // files when using the arrow::dataset API even after we closed the dataset.
50 // This is annoying as it can cause crashes when closing GDAL, in particular
51 // the virtual file manager, as this could result in VSI files being
52 // accessed after their VSIVirtualFileSystem has been destroyed, resulting
53 // in crashes. The workaround is to make sure that VSIArrowFileSystem
54 // waits for all file handles it is aware of to have been destroyed.
55 eErr = OGRArrowDataset::Close();
56
57 auto poFS = std::dynamic_pointer_cast<VSIArrowFileSystem>(m_poFS);
58 if (poFS)
59 poFS->AskToClose();
60 }
61
62 return eErr;
63 }
64
65 /************************************************************************/
66 /* CreateReaderLayer() */
67 /************************************************************************/
68
69 std::unique_ptr<OGRParquetLayer>
70 OGRParquetDataset::CreateReaderLayer(const std::string &osFilename,
71 VSILFILE *&fpIn,
72 CSLConstList papszOpenOptionsIn)
73 {
74 try
75 {
76 std::shared_ptr<arrow::io::RandomAccessFile> infile;
77 if (STARTS_WITH(osFilename.c_str(), "/vsi") ||
78 CPLTestBool(CPLGetConfigOption("OGR_PARQUET_USE_VSI", "NO")))
79 {
80 VSIVirtualHandleUniquePtr fp(fpIn);
81 fpIn = nullptr;
82 if (fp == nullptr)
83 {
84 fp.reset(VSIFOpenL(osFilename.c_str(), "rb"));
85 if (fp == nullptr)
86 return nullptr;
87 }
88 infile = std::make_shared<OGRArrowRandomAccessFile>(osFilename,
89 std::move(fp));
90 }
91 else
92 {
93 PARQUET_ASSIGN_OR_THROW(infile,
94 arrow::io::ReadableFile::Open(osFilename));
95 }
96
97 // Open Parquet file reader
98 std::unique_ptr<parquet::arrow::FileReader> arrow_reader;
99
100 const int nNumCPUs = OGRParquetLayerBase::GetNumCPUs();
101 const char *pszUseThreads =
102 CPLGetConfigOption("OGR_PARQUET_USE_THREADS", nullptr);
103 if (!pszUseThreads && nNumCPUs > 1)
104 {
105 pszUseThreads = "YES";
106 }
107 const bool bUseThreads = pszUseThreads && CPLTestBool(pszUseThreads);
108
109 const char *pszParquetBatchSize =
110 CPLGetConfigOption("OGR_PARQUET_BATCH_SIZE", nullptr);
111
112 auto poMemoryPool = GetMemoryPool();
113 #if ARROW_VERSION_MAJOR >= 21
114 parquet::arrow::FileReaderBuilder fileReaderBuilder;
115 {
116 auto st = fileReaderBuilder.Open(std::move(infile));
117 if (!st.ok())
118 {
119 CPLError(CE_Failure, CPLE_AppDefined,
120 "parquet::arrow::FileReaderBuilder::Open() failed: %s",
121 st.message().c_str());
122 return nullptr;
123 }
124 }
125 fileReaderBuilder.memory_pool(poMemoryPool);
126 parquet::ArrowReaderProperties fileReaderProperties;
127 fileReaderProperties.set_arrow_extensions_enabled(CPLTestBool(
128 CPLGetConfigOption("OGR_PARQUET_ENABLE_ARROW_EXTENSIONS", "YES")));
129 if (pszParquetBatchSize)
130 {
131 fileReaderProperties.set_batch_size(
132 CPLAtoGIntBig(pszParquetBatchSize));
133 }
134 if (bUseThreads)
135 {
136 fileReaderProperties.set_use_threads(true);
137 }
138 fileReaderBuilder.properties(fileReaderProperties);
139 {
140 auto res = fileReaderBuilder.Build();
141 if (!res.ok())
142 {
143 CPLError(
144 CE_Failure, CPLE_AppDefined,
145 "parquet::arrow::FileReaderBuilder::Build() failed: %s",
146 res.status().message().c_str());
147 return nullptr;
148 }
149 arrow_reader = std::move(*res);
150 }
151 #elif ARROW_VERSION_MAJOR >= 19
152 PARQUET_ASSIGN_OR_THROW(
153 arrow_reader,
154 parquet::arrow::OpenFile(std::move(infile), poMemoryPool));
155 #else
156 auto st = parquet::arrow::OpenFile(std::move(infile), poMemoryPool,
157 &arrow_reader);
158 if (!st.ok())
159 {
160 CPLError(CE_Failure, CPLE_AppDefined,
161 "parquet::arrow::OpenFile() failed: %s",
162 st.message().c_str());
163 return nullptr;
164 }
165 #endif
166
167 #if ARROW_VERSION_MAJOR < 21
168 if (pszParquetBatchSize)
169 {
170 arrow_reader->set_batch_size(CPLAtoGIntBig(pszParquetBatchSize));
171 }
172
173 if (bUseThreads)
174 {
175 arrow_reader->set_use_threads(true);
176 }
177 #endif
178
179 return std::make_unique<OGRParquetLayer>(
180 this, CPLGetBasenameSafe(osFilename.c_str()).c_str(),
181 std::move(arrow_reader), papszOpenOptionsIn);
182 }
183 catch (const std::exception &e)
184 {
185 CPLError(CE_Failure, CPLE_AppDefined, "Parquet exception: %s",
186 e.what());
187 return nullptr;
188 }
189 }
190
191 /************************************************************************/
192 /* ExecuteSQL() */
193 /************************************************************************/
194
195 OGRLayer *OGRParquetDataset::ExecuteSQL(const char *pszSQLCommand,
196 OGRGeometry *poSpatialFilter,
197 const char *pszDialect)
198 {
199 /* -------------------------------------------------------------------- */
200 /* Special cases for SQL optimizations */
201 /* -------------------------------------------------------------------- */
202 if (STARTS_WITH_CI(pszSQLCommand, "SELECT ") &&
203 (pszDialect == nullptr || EQUAL(pszDialect, "") ||
204 EQUAL(pszDialect, "OGRSQL")))
205 {
206 swq_select oSelect;
207 if (oSelect.preparse(pszSQLCommand) != CE_None)
208 return nullptr;
209
210 /* --------------------------------------------------------------------
211 */
212 /* MIN/MAX/COUNT optimization */
213 /* --------------------------------------------------------------------
214 */
215 if (oSelect.join_count == 0 && oSelect.poOtherSelect == nullptr &&
216 oSelect.table_count == 1 && oSelect.order_specs == 0 &&
217 oSelect.query_mode != SWQM_DISTINCT_LIST &&
218 oSelect.where_expr == nullptr &&
219 CPLTestBool(
220 CPLGetConfigOption("OGR_PARQUET_USE_STATISTICS", "YES")))
221 {
222 auto poLayer = dynamic_cast<OGRParquetLayer *>(
223 GetLayerByName(oSelect.table_defs[0].table_name));
224 if (poLayer)
225 {
226 OGRMemLayer *poMemLayer = nullptr;
227 const auto poLayerDefn = poLayer->GetLayerDefn();
228
229 int i = 0; // Used after for.
230 for (; i < oSelect.result_columns(); i++)
231 {
232 swq_col_func col_func = oSelect.column_defs[i].col_func;
233 if (!(col_func == SWQCF_MIN || col_func == SWQCF_MAX ||
234 col_func == SWQCF_COUNT))
235 break;
236
237 const char *pszFieldName =
238 oSelect.column_defs[i].field_name;
239 if (pszFieldName == nullptr)
240 break;
241 if (oSelect.column_defs[i].target_type != SWQ_OTHER)
242 break;
243
244 const int iOGRField =
245 (EQUAL(pszFieldName, poLayer->GetFIDColumn()) &&
246 pszFieldName[0])
247 ? OGRParquetLayer::OGR_FID_INDEX
248 : poLayerDefn->GetFieldIndex(pszFieldName);
249 if (iOGRField < 0 &&
250 iOGRField != OGRParquetLayer::OGR_FID_INDEX)
251 break;
252
253 OGRField sField;
254 OGR_RawField_SetNull(&sField);
255 OGRFieldType eType = OFTReal;
256 OGRFieldSubType eSubType = OFSTNone;
257 const std::vector<int> anCols =
258 iOGRField == OGRParquetLayer::OGR_FID_INDEX
259 ? std::vector<int>{poLayer->GetFIDParquetColumn()}
260 : poLayer->GetParquetColumnIndicesForArrowField(
261 pszFieldName);
262 if (anCols.size() != 1 || anCols[0] < 0)
263 break;
264 const int iCol = anCols[0];
265 const auto metadata =
266 poLayer->GetReader()->parquet_reader()->metadata();
267 const auto numRowGroups = metadata->num_row_groups();
268 bool bFound = false;
269 std::string sVal;
270
271 if (numRowGroups > 0)
272 {
273 const auto rowGroup0columnChunk =
274 metadata->RowGroup(0)->ColumnChunk(iCol);
275 const auto rowGroup0Stats =
276 rowGroup0columnChunk->statistics();
277 if (rowGroup0columnChunk->is_stats_set() &&
278 rowGroup0Stats)
279 {
280 OGRField sFieldDummy;
281 bool bFoundDummy;
282 std::string sValDummy;
283
284 if (col_func == SWQCF_MIN)
285 {
286 CPL_IGNORE_RET_VAL(
287 poLayer->GetMinMaxForOGRField(
288 /* iRowGroup=*/-1, // -1 for all
289 iOGRField, true, sField, bFound, false,
290 sFieldDummy, bFoundDummy, eType,
291 eSubType, sVal, sValDummy));
292 }
293 else if (col_func == SWQCF_MAX)
294 {
295 CPL_IGNORE_RET_VAL(
296 poLayer->GetMinMaxForOGRField(
297 /* iRowGroup=*/-1, // -1 for all
298 iOGRField, false, sFieldDummy,
299 bFoundDummy, true, sField, bFound,
300 eType, eSubType, sValDummy, sVal));
301 }
302 else if (col_func == SWQCF_COUNT)
303 {
304 if (oSelect.column_defs[i].distinct_flag)
305 {
306 eType = OFTInteger64;
307 sField.Integer64 = 0;
308 for (int iGroup = 0; iGroup < numRowGroups;
309 iGroup++)
310 {
311 const auto columnChunk =
312 metadata->RowGroup(iGroup)
313 ->ColumnChunk(iCol);
314 const auto colStats =
315 columnChunk->statistics();
316 if (columnChunk->is_stats_set() &&
317 colStats &&
318 colStats->HasDistinctCount())
319 {
320 // Statistics generated by arrow-cpp
321 // Parquet writer seem to be buggy,
322 // as distinct_count() is always
323 // zero. We can detect this: if
324 // there are non-null values, then
325 // distinct_count() should be > 0.
326 if (colStats->distinct_count() ==
327 0 &&
328 colStats->num_values() > 0)
329 {
330 bFound = false;
331 break;
332 }
333 sField.Integer64 +=
334 colStats->distinct_count();
335 bFound = true;
336 }
337 else
338 {
339 bFound = false;
340 break;
341 }
342 }
343 }
344 else
345 {
346 eType = OFTInteger64;
347 sField.Integer64 = 0;
348 bFound = true;
349 for (int iGroup = 0; iGroup < numRowGroups;
350 iGroup++)
351 {
352 const auto columnChunk =
353 metadata->RowGroup(iGroup)
354 ->ColumnChunk(iCol);
355 const auto colStats =
356 columnChunk->statistics();
357 if (columnChunk->is_stats_set() &&
358 colStats)
359 {
360 sField.Integer64 +=
361 colStats->num_values();
362 }
363 else
364 {
365 bFound = false;
366 }
367 }
368 }
369 }
370 }
371 else
372 {
373 CPLDebug("PARQUET",
374 "Statistics not available for field %s",
375 pszFieldName);
376 }
377 }
378 if (!bFound)
379 {
380 break;
381 }
382
383 if (poMemLayer == nullptr)
384 {
385 poMemLayer =
386 new OGRMemLayer("SELECT", nullptr, wkbNone);
387 auto poFeature = std::make_unique<OGRFeature>(
388 poMemLayer->GetLayerDefn());
389 CPL_IGNORE_RET_VAL(
390 poMemLayer->CreateFeature(std::move(poFeature)));
391 }
392
393 const char *pszMinMaxFieldName =
394 oSelect.column_defs[i].field_alias
395 ? oSelect.column_defs[i].field_alias
396 : CPLSPrintf("%s_%s",
397 (col_func == SWQCF_MIN) ? "MIN"
398 : (col_func == SWQCF_MAX) ? "MAX"
399 : "COUNT",
400 oSelect.column_defs[i].field_name);
401 OGRFieldDefn oFieldDefn(pszMinMaxFieldName, eType);
402 oFieldDefn.SetSubType(eSubType);
403 poMemLayer->CreateField(&oFieldDefn);
404
405 auto poFeature =
406 std::unique_ptr<OGRFeature>(poMemLayer->GetFeature(0));
407 poFeature->SetField(oFieldDefn.GetNameRef(), &sField);
408 CPL_IGNORE_RET_VAL(
409 poMemLayer->SetFeature(std::move(poFeature)));
410 }
411 if (i != oSelect.result_columns())
412 {
413 delete poMemLayer;
414 }
415 else
416 {
417 CPLDebug("PARQUET",
418 "Using optimized MIN/MAX/COUNT implementation");
419 return poMemLayer;
420 }
421 }
422 }
423 }
424
425 else if (EQUAL(pszSQLCommand, "GET_SET_FILES_ASKED_TO_BE_OPEN") &&
426 pszDialect && EQUAL(pszDialect, "_DEBUG_"))
427 {
428 if (auto poFS = dynamic_cast<VSIArrowFileSystem *>(m_poFS.get()))
429 {
430 auto poMemLayer = std::make_unique<OGRMemLayer>(
431 "SET_FILES_ASKED_TO_BE_OPEN", nullptr, wkbNone);
432 OGRFieldDefn oFieldDefn("path", OFTString);
433 CPL_IGNORE_RET_VAL(poMemLayer->CreateField(&oFieldDefn));
434 for (const std::string &path : poFS->GetSetFilesAskedToOpen())
435 {
436 auto poFeature =
437 std::make_unique<OGRFeature>(poMemLayer->GetLayerDefn());
438 poFeature->SetField(0, path.c_str());
439 CPL_IGNORE_RET_VAL(
440 poMemLayer->CreateFeature(std::move(poFeature)));
441 }
442 poFS->ResetSetFilesAskedToOpen();
443 return poMemLayer.release();
444 }
445 return nullptr;
446 }
447
448 return GDALDataset::ExecuteSQL(pszSQLCommand, poSpatialFilter, pszDialect);
449 }
450
451 /************************************************************************/
452 /* ReleaseResultSet() */
453 /************************************************************************/
454
455 void OGRParquetDataset::ReleaseResultSet(OGRLayer *poResultsSet)
456 {
457 delete poResultsSet;
458 }
459
460 /************************************************************************/
461 /* TestCapability() */
462 /************************************************************************/
463
464 int OGRParquetDataset::TestCapability(const char *pszCap) const
465
466 {
467 if (EQUAL(pszCap, ODsCZGeometries))
468 return true;
469 else if (EQUAL(pszCap, ODsCMeasuredGeometries))
470 return true;
471
472 return false;
473 }