Debian Codegraph @

ogr/ogrsf_frmts/arrow/ogrfeatherdrivercore.cpp

raw
1 /******************************************************************************
2 *
3 * Project: Feather Translator
4 * Purpose: Implements OGRFeatherDriver.
5 * Author: Even Rouault, <even.rouault at spatialys.com>
6 *
7 ******************************************************************************
8 * Copyright (c) 2022, Planet Labs
9 *
10 * SPDX-License-Identifier: MIT
11 ****************************************************************************/
12
13 #include "gdal_frmts.h"
14
15 #ifdef PLUGIN_FILENAME
16 #include "gdalplugindriverproxy.h"
17 #endif
18
19 #include "ogrsf_frmts.h"
20 #include "gdal_priv.h"
21
22 #include "ogrfeatherdrivercore.h"
23
24 /************************************************************************/
25 /* OGRFeatherDriverIsArrowIPCStreamBasic() */
26 /************************************************************************/
27
28 static int OGRFeatherDriverIsArrowIPCStreamBasic(GDALOpenInfo *poOpenInfo)
29 {
30 // WARNING: if making changes in that method, reflect them in
31 // IsArrowIPCStream() in ogrfeatherdriver.cpp
32
33 if (STARTS_WITH_CI(poOpenInfo->pszFilename, "ARROW_IPC_STREAM:"))
34 return true;
35
36 constexpr int CONTINUATION_SIZE = 4; // 0xFFFFFFFF
37 constexpr int METADATA_SIZE_SIZE = 4;
38
39 // See
40 // https://arrow.apache.org/docs/format/Columnar.html#encapsulated-message-format
41 if (poOpenInfo->fpL != nullptr &&
42 poOpenInfo->nHeaderBytes >= CONTINUATION_SIZE + METADATA_SIZE_SIZE &&
43 memcmp(poOpenInfo->pabyHeader, "\xFF\xFF\xFF\xFF", CONTINUATION_SIZE) ==
44 0)
45 {
46 const char *pszExt = poOpenInfo->osExtension.c_str();
47 if (EQUAL(pszExt, "arrows") || EQUAL(pszExt, "ipc"))
48 return true;
49
50 const uint32_t nMetadataSize =
51 CPL_LSBUINT32PTR(poOpenInfo->pabyHeader + CONTINUATION_SIZE);
52 if (strcmp(poOpenInfo->pszFilename, "/vsistdin/") == 0)
53 {
54 // Padding after metadata and before body is not necessarily present
55 // but the body must be at least 4 bytes
56 constexpr int PADDING_MAX_SIZE = 4;
57
58 // /vsistdin/ cannot seek back beyond first MB
59 if (nMetadataSize >
60 1024 * 1024 -
61 (CONTINUATION_SIZE + METADATA_SIZE_SIZE + PADDING_MAX_SIZE))
62 {
63 if (poOpenInfo->IsSingleAllowedDriver("ARROW"))
64 return true;
65 return GDAL_IDENTIFY_UNKNOWN;
66 }
67 const int nSizeToRead = CONTINUATION_SIZE + METADATA_SIZE_SIZE +
68 nMetadataSize + PADDING_MAX_SIZE;
69 if (!poOpenInfo->TryToIngest(nSizeToRead))
70 {
71 return false;
72 }
73
74 if (poOpenInfo->IsSingleAllowedDriver("ARROW"))
75 return true;
76 return GDAL_IDENTIFY_UNKNOWN;
77 }
78
79 VSIFSeekL(poOpenInfo->fpL, 0, SEEK_END);
80 const auto nFileSize = VSIFTellL(poOpenInfo->fpL);
81 VSIFSeekL(poOpenInfo->fpL, 0, SEEK_SET);
82 if (nMetadataSize >
83 nFileSize - (CONTINUATION_SIZE + METADATA_SIZE_SIZE))
84 return false;
85
86 if (poOpenInfo->IsSingleAllowedDriver("ARROW"))
87 return true;
88 return GDAL_IDENTIFY_UNKNOWN;
89 }
90 return false;
91 }
92
93 /************************************************************************/
94 /* OGRFeatherDriverIsArrowFileFormat() */
95 /************************************************************************/
96
97 template <size_t N> constexpr int constexpr_length(const char (&)[N])
98 {
99 return static_cast<int>(N - 1);
100 }
101
102 bool OGRFeatherDriverIsArrowFileFormat(GDALOpenInfo *poOpenInfo)
103 {
104 // See https://arrow.apache.org/docs/format/Columnar.html#ipc-file-format
105 bool bRet = false;
106 constexpr const char SIGNATURE[] = "ARROW1";
107 constexpr int SIGNATURE_SIZE = constexpr_length(SIGNATURE);
108 static_assert(SIGNATURE_SIZE == 6, "SIGNATURE_SIZE == 6");
109 constexpr int SIGNATURE_PLUS_PADDING = SIGNATURE_SIZE + 2;
110 constexpr int FOOTERSIZE_SIZE = 4;
111 if (poOpenInfo->fpL != nullptr &&
112 poOpenInfo->nHeaderBytes >=
113 SIGNATURE_PLUS_PADDING + FOOTERSIZE_SIZE + SIGNATURE_SIZE &&
114 memcmp(poOpenInfo->pabyHeader, SIGNATURE, SIGNATURE_SIZE) == 0)
115 {
116 VSIFSeekL(poOpenInfo->fpL, 0, SEEK_END);
117 const auto nFileSize = VSIFTellL(poOpenInfo->fpL);
118 VSIFSeekL(poOpenInfo->fpL,
119 nFileSize - (FOOTERSIZE_SIZE + SIGNATURE_SIZE), SEEK_SET);
120 uint32_t nFooterSize = 0;
121 static_assert(sizeof(nFooterSize) == FOOTERSIZE_SIZE,
122 "sizeof(nFooterSize) == FOOTERSIZE_SIZE");
123 VSIFReadL(&nFooterSize, 1, sizeof(nFooterSize), poOpenInfo->fpL);
124 CPL_LSBPTR32(&nFooterSize);
125 unsigned char abyTrailingBytes[SIGNATURE_SIZE] = {0};
126 VSIFReadL(&abyTrailingBytes[0], 1, SIGNATURE_SIZE, poOpenInfo->fpL);
127 bRet = memcmp(abyTrailingBytes, SIGNATURE, SIGNATURE_SIZE) == 0 &&
128 nFooterSize < nFileSize;
129 VSIFSeekL(poOpenInfo->fpL, 0, SEEK_SET);
130 }
131 return bRet;
132 }
133
134 /************************************************************************/
135 /* Identify() */
136 /************************************************************************/
137
138 int OGRFeatherDriverIdentify(GDALOpenInfo *poOpenInfo)
139 {
140 if (STARTS_WITH(poOpenInfo->pszFilename, "gdalvsi://"))
141 {
142 GDALOpenInfo oOpenInfo(poOpenInfo->pszFilename + strlen("gdalvsi://"),
143 poOpenInfo->nOpenFlags);
144 return OGRFeatherDriverIdentify(&oOpenInfo);
145 }
146
147 int ret = OGRFeatherDriverIsArrowIPCStreamBasic(poOpenInfo);
148 if (ret == GDAL_IDENTIFY_TRUE || ret == GDAL_IDENTIFY_UNKNOWN)
149 return ret;
150 return OGRFeatherDriverIsArrowFileFormat(poOpenInfo);
151 }
152
153 /************************************************************************/
154 /* OGRFeatherDriverSetCommonMetadata() */
155 /************************************************************************/
156
157 void OGRFeatherDriverSetCommonMetadata(GDALDriver *poDriver)
158 {
159 poDriver->SetDescription(DRIVER_NAME);
160 poDriver->SetMetadataItem(GDAL_DCAP_VECTOR, "YES");
161 poDriver->SetMetadataItem(GDAL_DCAP_CREATE_LAYER, "YES");
162 poDriver->SetMetadataItem(GDAL_DCAP_CREATE_FIELD, "YES");
163 poDriver->SetMetadataItem(GDAL_DMD_LONGNAME,
164 "(Geo)Arrow IPC File Format / Stream");
165 poDriver->SetMetadataItem(GDAL_DMD_EXTENSIONS, "arrow feather arrows ipc");
166 poDriver->SetMetadataItem(GDAL_DMD_HELPTOPIC,
167 "drivers/vector/feather.html");
168 poDriver->SetMetadataItem(GDAL_DCAP_VIRTUALIO, "YES");
169 poDriver->SetMetadataItem(GDAL_DCAP_MEASURED_GEOMETRIES, "YES");
170 poDriver->SetMetadataItem(GDAL_DCAP_Z_GEOMETRIES, "YES");
171 poDriver->SetMetadataItem(GDAL_DMD_SUPPORTED_SQL_DIALECTS, "OGRSQL SQLITE");
172 poDriver->SetMetadataItem(GDAL_DCAP_REOPEN_AFTER_WRITE_REQUIRED, "YES");
173 poDriver->SetMetadataItem(GDAL_DCAP_CAN_READ_AFTER_DELETE, "YES");
174
175 poDriver->SetMetadataItem(
176 GDAL_DMD_CREATIONFIELDDATATYPES,
177 "Integer Integer64 Real String Date Time DateTime "
178 "Binary IntegerList Integer64List RealList StringList");
179 poDriver->SetMetadataItem(GDAL_DMD_CREATIONFIELDDATASUBTYPES,
180 "Boolean Int16 Float32 JSON UUID");
181 poDriver->SetMetadataItem(GDAL_DMD_CREATION_FIELD_DEFN_FLAGS,
182 "WidthPrecision Nullable "
183 "Comment AlternativeName Domain");
184
185 poDriver->SetMetadataItem(
186 GDAL_DMD_OPENOPTIONLIST,
187 "<OpenOptionList>"
188 " <Option name='LISTS_AS_STRING_JSON' type='boolean' description='"
189 "Whether lists of strings/integers/reals should be reported as "
190 "String(JSON) fields rather than String/Integer[64]/RealList. Useful "
191 "when null values in such lists must be exactly mapped as such.' "
192 "default='NO'/>"
193 "</OpenOptionList>");
194
195 poDriver->pfnIdentify = OGRFeatherDriverIdentify;
196 poDriver->SetMetadataItem(GDAL_DCAP_OPEN, "YES");
197 poDriver->SetMetadataItem(GDAL_DCAP_CREATE, "YES");
198 }
199
200 /************************************************************************/
201 /* DeclareDeferredOGRArrowPlugin() */
202 /************************************************************************/
203
204 #ifdef PLUGIN_FILENAME
205 void DeclareDeferredOGRArrowPlugin()
206 {
207 if (GDALGetDriverByName(DRIVER_NAME) != nullptr)
208 {
209 return;
210 }
211 auto poDriver = new GDALPluginDriverProxy(PLUGIN_FILENAME);
212 #ifdef PLUGIN_INSTALLATION_MESSAGE
213 poDriver->SetMetadataItem(GDAL_DMD_PLUGIN_INSTALLATION_MESSAGE,
214 PLUGIN_INSTALLATION_MESSAGE);
215 #endif
216 OGRFeatherDriverSetCommonMetadata(poDriver);
217 GetGDALDriverManager()->DeclareDeferredPluginDriver(poDriver);
218 }
219 #endif