| 1 |
/****************************************************************************** |
| 2 |
* |
| 3 |
* Project: Parquet Translator |
| 4 |
* Purpose: Implements OGRParquetDriver. |
| 5 |
* Author: Even Rouault, <even.rouault at spatialys.com> |
| 6 |
* |
| 7 |
****************************************************************************** |
| 8 |
* Copyright (c) 2022, Planet Labs |
| 9 |
* |
| 10 |
* SPDX-License-Identifier: MIT |
| 11 |
****************************************************************************/ |
| 12 |
|
| 13 |
#include "ogr_parquet.h" |
| 14 |
#include "memdataset.h" |
| 15 |
#include "ogr_swq.h" |
| 16 |
|
| 17 |
#include "../arrow_common/ograrrowdataset.hpp" |
| 18 |
#include "../arrow_common/ograrrowlayer.hpp" |
| 19 |
#include "../arrow_common/vsiarrowfilesystem.hpp" |
| 20 |
|
| 21 |
/************************************************************************/ |
| 22 |
/* OGRParquetDataset() */ |
| 23 |
/************************************************************************/ |
| 24 |
|
| 25 |
OGRParquetDataset::OGRParquetDataset() |
| 26 |
: OGRArrowDataset(arrow::MemoryPool::CreateDefault()) |
| 27 |
{ |
| 28 |
} |
| 29 |
|
| 30 |
/************************************************************************/ |
| 31 |
/* ~OGRParquetDataset() */ |
| 32 |
/************************************************************************/ |
| 33 |
|
| 34 |
OGRParquetDataset::~OGRParquetDataset() |
| 35 |
{ |
| 36 |
OGRParquetDataset::Close(); |
| 37 |
} |
| 38 |
|
| 39 |
/************************************************************************/ |
| 40 |
/* Close() */ |
| 41 |
/************************************************************************/ |
| 42 |
|
| 43 |
CPLErr OGRParquetDataset::Close(GDALProgressFunc, void *) |
| 44 |
{ |
| 45 |
CPLErr eErr = CE_None; |
| 46 |
if (nOpenFlags != OPEN_FLAGS_CLOSED) |
| 47 |
{ |
| 48 |
// libarrow might continue to do I/O in auxiliary threads on the underlying |
| 49 |
// files when using the arrow::dataset API even after we closed the dataset. |
| 50 |
// This is annoying as it can cause crashes when closing GDAL, in particular |
| 51 |
// the virtual file manager, as this could result in VSI files being |
| 52 |
// accessed after their VSIVirtualFileSystem has been destroyed, resulting |
| 53 |
// in crashes. The workaround is to make sure that VSIArrowFileSystem |
| 54 |
// waits for all file handles it is aware of to have been destroyed. |
| 55 |
eErr = OGRArrowDataset::Close(); |
| 56 |
|
| 57 |
auto poFS = std::dynamic_pointer_cast<VSIArrowFileSystem>(m_poFS); |
| 58 |
if (poFS) |
| 59 |
poFS->AskToClose(); |
| 60 |
} |
| 61 |
|
| 62 |
return eErr; |
| 63 |
} |
| 64 |
|
| 65 |
/************************************************************************/ |
| 66 |
/* CreateReaderLayer() */ |
| 67 |
/************************************************************************/ |
| 68 |
|
| 69 |
std::unique_ptr<OGRParquetLayer> |
| 70 |
OGRParquetDataset::CreateReaderLayer(const std::string &osFilename, |
| 71 |
VSILFILE *&fpIn, |
| 72 |
CSLConstList papszOpenOptionsIn) |
| 73 |
{ |
| 74 |
try |
| 75 |
{ |
| 76 |
std::shared_ptr<arrow::io::RandomAccessFile> infile; |
| 77 |
if (STARTS_WITH(osFilename.c_str(), "/vsi") || |
| 78 |
CPLTestBool(CPLGetConfigOption("OGR_PARQUET_USE_VSI", "NO"))) |
| 79 |
{ |
| 80 |
VSIVirtualHandleUniquePtr fp(fpIn); |
| 81 |
fpIn = nullptr; |
| 82 |
if (fp == nullptr) |
| 83 |
{ |
| 84 |
fp.reset(VSIFOpenL(osFilename.c_str(), "rb")); |
| 85 |
if (fp == nullptr) |
| 86 |
return nullptr; |
| 87 |
} |
| 88 |
infile = std::make_shared<OGRArrowRandomAccessFile>(osFilename, |
| 89 |
std::move(fp)); |
| 90 |
} |
| 91 |
else |
| 92 |
{ |
| 93 |
PARQUET_ASSIGN_OR_THROW(infile, |
| 94 |
arrow::io::ReadableFile::Open(osFilename)); |
| 95 |
} |
| 96 |
|
| 97 |
// Open Parquet file reader |
| 98 |
std::unique_ptr<parquet::arrow::FileReader> arrow_reader; |
| 99 |
|
| 100 |
const int nNumCPUs = OGRParquetLayerBase::GetNumCPUs(); |
| 101 |
const char *pszUseThreads = |
| 102 |
CPLGetConfigOption("OGR_PARQUET_USE_THREADS", nullptr); |
| 103 |
if (!pszUseThreads && nNumCPUs > 1) |
| 104 |
{ |
| 105 |
pszUseThreads = "YES"; |
| 106 |
} |
| 107 |
const bool bUseThreads = pszUseThreads && CPLTestBool(pszUseThreads); |
| 108 |
|
| 109 |
const char *pszParquetBatchSize = |
| 110 |
CPLGetConfigOption("OGR_PARQUET_BATCH_SIZE", nullptr); |
| 111 |
|
| 112 |
auto poMemoryPool = GetMemoryPool(); |
| 113 |
#if ARROW_VERSION_MAJOR >= 21 |
| 114 |
parquet::arrow::FileReaderBuilder fileReaderBuilder; |
| 115 |
{ |
| 116 |
auto st = fileReaderBuilder.Open(std::move(infile)); |
| 117 |
if (!st.ok()) |
| 118 |
{ |
| 119 |
CPLError(CE_Failure, CPLE_AppDefined, |
| 120 |
"parquet::arrow::FileReaderBuilder::Open() failed: %s", |
| 121 |
st.message().c_str()); |
| 122 |
return nullptr; |
| 123 |
} |
| 124 |
} |
| 125 |
fileReaderBuilder.memory_pool(poMemoryPool); |
| 126 |
parquet::ArrowReaderProperties fileReaderProperties; |
| 127 |
fileReaderProperties.set_arrow_extensions_enabled(CPLTestBool( |
| 128 |
CPLGetConfigOption("OGR_PARQUET_ENABLE_ARROW_EXTENSIONS", "YES"))); |
| 129 |
if (pszParquetBatchSize) |
| 130 |
{ |
| 131 |
fileReaderProperties.set_batch_size( |
| 132 |
CPLAtoGIntBig(pszParquetBatchSize)); |
| 133 |
} |
| 134 |
if (bUseThreads) |
| 135 |
{ |
| 136 |
fileReaderProperties.set_use_threads(true); |
| 137 |
} |
| 138 |
fileReaderBuilder.properties(fileReaderProperties); |
| 139 |
{ |
| 140 |
auto res = fileReaderBuilder.Build(); |
| 141 |
if (!res.ok()) |
| 142 |
{ |
| 143 |
CPLError( |
| 144 |
CE_Failure, CPLE_AppDefined, |
| 145 |
"parquet::arrow::FileReaderBuilder::Build() failed: %s", |
| 146 |
res.status().message().c_str()); |
| 147 |
return nullptr; |
| 148 |
} |
| 149 |
arrow_reader = std::move(*res); |
| 150 |
} |
| 151 |
#elif ARROW_VERSION_MAJOR >= 19 |
| 152 |
PARQUET_ASSIGN_OR_THROW( |
| 153 |
arrow_reader, |
| 154 |
parquet::arrow::OpenFile(std::move(infile), poMemoryPool)); |
| 155 |
#else |
| 156 |
auto st = parquet::arrow::OpenFile(std::move(infile), poMemoryPool, |
| 157 |
&arrow_reader); |
| 158 |
if (!st.ok()) |
| 159 |
{ |
| 160 |
CPLError(CE_Failure, CPLE_AppDefined, |
| 161 |
"parquet::arrow::OpenFile() failed: %s", |
| 162 |
st.message().c_str()); |
| 163 |
return nullptr; |
| 164 |
} |
| 165 |
#endif |
| 166 |
|
| 167 |
#if ARROW_VERSION_MAJOR < 21 |
| 168 |
if (pszParquetBatchSize) |
| 169 |
{ |
| 170 |
arrow_reader->set_batch_size(CPLAtoGIntBig(pszParquetBatchSize)); |
| 171 |
} |
| 172 |
|
| 173 |
if (bUseThreads) |
| 174 |
{ |
| 175 |
arrow_reader->set_use_threads(true); |
| 176 |
} |
| 177 |
#endif |
| 178 |
|
| 179 |
return std::make_unique<OGRParquetLayer>( |
| 180 |
this, CPLGetBasenameSafe(osFilename.c_str()).c_str(), |
| 181 |
std::move(arrow_reader), papszOpenOptionsIn); |
| 182 |
} |
| 183 |
catch (const std::exception &e) |
| 184 |
{ |
| 185 |
CPLError(CE_Failure, CPLE_AppDefined, "Parquet exception: %s", |
| 186 |
e.what()); |
| 187 |
return nullptr; |
| 188 |
} |
| 189 |
} |
| 190 |
|
| 191 |
/************************************************************************/ |
| 192 |
/* ExecuteSQL() */ |
| 193 |
/************************************************************************/ |
| 194 |
|
| 195 |
OGRLayer *OGRParquetDataset::ExecuteSQL(const char *pszSQLCommand, |
| 196 |
OGRGeometry *poSpatialFilter, |
| 197 |
const char *pszDialect) |
| 198 |
{ |
| 199 |
/* -------------------------------------------------------------------- */ |
| 200 |
/* Special cases for SQL optimizations */ |
| 201 |
/* -------------------------------------------------------------------- */ |
| 202 |
if (STARTS_WITH_CI(pszSQLCommand, "SELECT ") && |
| 203 |
(pszDialect == nullptr || EQUAL(pszDialect, "") || |
| 204 |
EQUAL(pszDialect, "OGRSQL"))) |
| 205 |
{ |
| 206 |
swq_select oSelect; |
| 207 |
if (oSelect.preparse(pszSQLCommand) != CE_None) |
| 208 |
return nullptr; |
| 209 |
|
| 210 |
/* -------------------------------------------------------------------- |
| 211 |
*/ |
| 212 |
/* MIN/MAX/COUNT optimization */ |
| 213 |
/* -------------------------------------------------------------------- |
| 214 |
*/ |
| 215 |
if (oSelect.join_count == 0 && oSelect.poOtherSelect == nullptr && |
| 216 |
oSelect.table_count == 1 && oSelect.order_specs == 0 && |
| 217 |
oSelect.query_mode != SWQM_DISTINCT_LIST && |
| 218 |
oSelect.where_expr == nullptr && |
| 219 |
CPLTestBool( |
| 220 |
CPLGetConfigOption("OGR_PARQUET_USE_STATISTICS", "YES"))) |
| 221 |
{ |
| 222 |
auto poLayer = dynamic_cast<OGRParquetLayer *>( |
| 223 |
GetLayerByName(oSelect.table_defs[0].table_name)); |
| 224 |
if (poLayer) |
| 225 |
{ |
| 226 |
OGRMemLayer *poMemLayer = nullptr; |
| 227 |
const auto poLayerDefn = poLayer->GetLayerDefn(); |
| 228 |
|
| 229 |
int i = 0; // Used after for. |
| 230 |
for (; i < oSelect.result_columns(); i++) |
| 231 |
{ |
| 232 |
swq_col_func col_func = oSelect.column_defs[i].col_func; |
| 233 |
if (!(col_func == SWQCF_MIN || col_func == SWQCF_MAX || |
| 234 |
col_func == SWQCF_COUNT)) |
| 235 |
break; |
| 236 |
|
| 237 |
const char *pszFieldName = |
| 238 |
oSelect.column_defs[i].field_name; |
| 239 |
if (pszFieldName == nullptr) |
| 240 |
break; |
| 241 |
if (oSelect.column_defs[i].target_type != SWQ_OTHER) |
| 242 |
break; |
| 243 |
|
| 244 |
const int iOGRField = |
| 245 |
(EQUAL(pszFieldName, poLayer->GetFIDColumn()) && |
| 246 |
pszFieldName[0]) |
| 247 |
? OGRParquetLayer::OGR_FID_INDEX |
| 248 |
: poLayerDefn->GetFieldIndex(pszFieldName); |
| 249 |
if (iOGRField < 0 && |
| 250 |
iOGRField != OGRParquetLayer::OGR_FID_INDEX) |
| 251 |
break; |
| 252 |
|
| 253 |
OGRField sField; |
| 254 |
OGR_RawField_SetNull(&sField); |
| 255 |
OGRFieldType eType = OFTReal; |
| 256 |
OGRFieldSubType eSubType = OFSTNone; |
| 257 |
const std::vector<int> anCols = |
| 258 |
iOGRField == OGRParquetLayer::OGR_FID_INDEX |
| 259 |
? std::vector<int>{poLayer->GetFIDParquetColumn()} |
| 260 |
: poLayer->GetParquetColumnIndicesForArrowField( |
| 261 |
pszFieldName); |
| 262 |
if (anCols.size() != 1 || anCols[0] < 0) |
| 263 |
break; |
| 264 |
const int iCol = anCols[0]; |
| 265 |
const auto metadata = |
| 266 |
poLayer->GetReader()->parquet_reader()->metadata(); |
| 267 |
const auto numRowGroups = metadata->num_row_groups(); |
| 268 |
bool bFound = false; |
| 269 |
std::string sVal; |
| 270 |
|
| 271 |
if (numRowGroups > 0) |
| 272 |
{ |
| 273 |
const auto rowGroup0columnChunk = |
| 274 |
metadata->RowGroup(0)->ColumnChunk(iCol); |
| 275 |
const auto rowGroup0Stats = |
| 276 |
rowGroup0columnChunk->statistics(); |
| 277 |
if (rowGroup0columnChunk->is_stats_set() && |
| 278 |
rowGroup0Stats) |
| 279 |
{ |
| 280 |
OGRField sFieldDummy; |
| 281 |
bool bFoundDummy; |
| 282 |
std::string sValDummy; |
| 283 |
|
| 284 |
if (col_func == SWQCF_MIN) |
| 285 |
{ |
| 286 |
CPL_IGNORE_RET_VAL( |
| 287 |
poLayer->GetMinMaxForOGRField( |
| 288 |
/* iRowGroup=*/-1, // -1 for all |
| 289 |
iOGRField, true, sField, bFound, false, |
| 290 |
sFieldDummy, bFoundDummy, eType, |
| 291 |
eSubType, sVal, sValDummy)); |
| 292 |
} |
| 293 |
else if (col_func == SWQCF_MAX) |
| 294 |
{ |
| 295 |
CPL_IGNORE_RET_VAL( |
| 296 |
poLayer->GetMinMaxForOGRField( |
| 297 |
/* iRowGroup=*/-1, // -1 for all |
| 298 |
iOGRField, false, sFieldDummy, |
| 299 |
bFoundDummy, true, sField, bFound, |
| 300 |
eType, eSubType, sValDummy, sVal)); |
| 301 |
} |
| 302 |
else if (col_func == SWQCF_COUNT) |
| 303 |
{ |
| 304 |
if (oSelect.column_defs[i].distinct_flag) |
| 305 |
{ |
| 306 |
eType = OFTInteger64; |
| 307 |
sField.Integer64 = 0; |
| 308 |
for (int iGroup = 0; iGroup < numRowGroups; |
| 309 |
iGroup++) |
| 310 |
{ |
| 311 |
const auto columnChunk = |
| 312 |
metadata->RowGroup(iGroup) |
| 313 |
->ColumnChunk(iCol); |
| 314 |
const auto colStats = |
| 315 |
columnChunk->statistics(); |
| 316 |
if (columnChunk->is_stats_set() && |
| 317 |
colStats && |
| 318 |
colStats->HasDistinctCount()) |
| 319 |
{ |
| 320 |
// Statistics generated by arrow-cpp |
| 321 |
// Parquet writer seem to be buggy, |
| 322 |
// as distinct_count() is always |
| 323 |
// zero. We can detect this: if |
| 324 |
// there are non-null values, then |
| 325 |
// distinct_count() should be > 0. |
| 326 |
if (colStats->distinct_count() == |
| 327 |
0 && |
| 328 |
colStats->num_values() > 0) |
| 329 |
{ |
| 330 |
bFound = false; |
| 331 |
break; |
| 332 |
} |
| 333 |
sField.Integer64 += |
| 334 |
colStats->distinct_count(); |
| 335 |
bFound = true; |
| 336 |
} |
| 337 |
else |
| 338 |
{ |
| 339 |
bFound = false; |
| 340 |
break; |
| 341 |
} |
| 342 |
} |
| 343 |
} |
| 344 |
else |
| 345 |
{ |
| 346 |
eType = OFTInteger64; |
| 347 |
sField.Integer64 = 0; |
| 348 |
bFound = true; |
| 349 |
for (int iGroup = 0; iGroup < numRowGroups; |
| 350 |
iGroup++) |
| 351 |
{ |
| 352 |
const auto columnChunk = |
| 353 |
metadata->RowGroup(iGroup) |
| 354 |
->ColumnChunk(iCol); |
| 355 |
const auto colStats = |
| 356 |
columnChunk->statistics(); |
| 357 |
if (columnChunk->is_stats_set() && |
| 358 |
colStats) |
| 359 |
{ |
| 360 |
sField.Integer64 += |
| 361 |
colStats->num_values(); |
| 362 |
} |
| 363 |
else |
| 364 |
{ |
| 365 |
bFound = false; |
| 366 |
} |
| 367 |
} |
| 368 |
} |
| 369 |
} |
| 370 |
} |
| 371 |
else |
| 372 |
{ |
| 373 |
CPLDebug("PARQUET", |
| 374 |
"Statistics not available for field %s", |
| 375 |
pszFieldName); |
| 376 |
} |
| 377 |
} |
| 378 |
if (!bFound) |
| 379 |
{ |
| 380 |
break; |
| 381 |
} |
| 382 |
|
| 383 |
if (poMemLayer == nullptr) |
| 384 |
{ |
| 385 |
poMemLayer = |
| 386 |
new OGRMemLayer("SELECT", nullptr, wkbNone); |
| 387 |
auto poFeature = std::make_unique<OGRFeature>( |
| 388 |
poMemLayer->GetLayerDefn()); |
| 389 |
CPL_IGNORE_RET_VAL( |
| 390 |
poMemLayer->CreateFeature(std::move(poFeature))); |
| 391 |
} |
| 392 |
|
| 393 |
const char *pszMinMaxFieldName = |
| 394 |
oSelect.column_defs[i].field_alias |
| 395 |
? oSelect.column_defs[i].field_alias |
| 396 |
: CPLSPrintf("%s_%s", |
| 397 |
(col_func == SWQCF_MIN) ? "MIN" |
| 398 |
: (col_func == SWQCF_MAX) ? "MAX" |
| 399 |
: "COUNT", |
| 400 |
oSelect.column_defs[i].field_name); |
| 401 |
OGRFieldDefn oFieldDefn(pszMinMaxFieldName, eType); |
| 402 |
oFieldDefn.SetSubType(eSubType); |
| 403 |
poMemLayer->CreateField(&oFieldDefn); |
| 404 |
|
| 405 |
auto poFeature = |
| 406 |
std::unique_ptr<OGRFeature>(poMemLayer->GetFeature(0)); |
| 407 |
poFeature->SetField(oFieldDefn.GetNameRef(), &sField); |
| 408 |
CPL_IGNORE_RET_VAL( |
| 409 |
poMemLayer->SetFeature(std::move(poFeature))); |
| 410 |
} |
| 411 |
if (i != oSelect.result_columns()) |
| 412 |
{ |
| 413 |
delete poMemLayer; |
| 414 |
} |
| 415 |
else |
| 416 |
{ |
| 417 |
CPLDebug("PARQUET", |
| 418 |
"Using optimized MIN/MAX/COUNT implementation"); |
| 419 |
return poMemLayer; |
| 420 |
} |
| 421 |
} |
| 422 |
} |
| 423 |
} |
| 424 |
|
| 425 |
else if (EQUAL(pszSQLCommand, "GET_SET_FILES_ASKED_TO_BE_OPEN") && |
| 426 |
pszDialect && EQUAL(pszDialect, "_DEBUG_")) |
| 427 |
{ |
| 428 |
if (auto poFS = dynamic_cast<VSIArrowFileSystem *>(m_poFS.get())) |
| 429 |
{ |
| 430 |
auto poMemLayer = std::make_unique<OGRMemLayer>( |
| 431 |
"SET_FILES_ASKED_TO_BE_OPEN", nullptr, wkbNone); |
| 432 |
OGRFieldDefn oFieldDefn("path", OFTString); |
| 433 |
CPL_IGNORE_RET_VAL(poMemLayer->CreateField(&oFieldDefn)); |
| 434 |
for (const std::string &path : poFS->GetSetFilesAskedToOpen()) |
| 435 |
{ |
| 436 |
auto poFeature = |
| 437 |
std::make_unique<OGRFeature>(poMemLayer->GetLayerDefn()); |
| 438 |
poFeature->SetField(0, path.c_str()); |
| 439 |
CPL_IGNORE_RET_VAL( |
| 440 |
poMemLayer->CreateFeature(std::move(poFeature))); |
| 441 |
} |
| 442 |
poFS->ResetSetFilesAskedToOpen(); |
| 443 |
return poMemLayer.release(); |
| 444 |
} |
| 445 |
return nullptr; |
| 446 |
} |
| 447 |
|
| 448 |
return GDALDataset::ExecuteSQL(pszSQLCommand, poSpatialFilter, pszDialect); |
| 449 |
} |
| 450 |
|
| 451 |
/************************************************************************/ |
| 452 |
/* ReleaseResultSet() */ |
| 453 |
/************************************************************************/ |
| 454 |
|
| 455 |
void OGRParquetDataset::ReleaseResultSet(OGRLayer *poResultsSet) |
| 456 |
{ |
| 457 |
delete poResultsSet; |
| 458 |
} |
| 459 |
|
| 460 |
/************************************************************************/ |
| 461 |
/* TestCapability() */ |
| 462 |
/************************************************************************/ |
| 463 |
|
| 464 |
int OGRParquetDataset::TestCapability(const char *pszCap) const |
| 465 |
|
| 466 |
{ |
| 467 |
if (EQUAL(pszCap, ODsCZGeometries)) |
| 468 |
return true; |
| 469 |
else if (EQUAL(pszCap, ODsCMeasuredGeometries)) |
| 470 |
return true; |
| 471 |
|
| 472 |
return false; |
| 473 |
} |