Debian Codegraph @

ogr/ogrsf_frmts/arrow_common/vsiarrowfilesystem.hpp

raw
1 /******************************************************************************
2 *
3 * Project: Parquet Translator
4 * Purpose: Implements OGRParquetDriver.
5 * Author: Even Rouault, <even.rouault at spatialys.com>
6 *
7 ******************************************************************************
8 * Copyright (c) 2022, Planet Labs
9 *
10 * SPDX-License-Identifier: MIT
11 ****************************************************************************/
12
13 #ifndef VSIARROWFILESYSTEM_HPP_INCLUDED
14 #define VSIARROWFILESYSTEM_HPP_INCLUDED
15
16 #include "cpl_multiproc.h"
17
18 #include "arrow/util/config.h"
19
20 #include "ograrrowrandomaccessfile.h"
21
22 #include <atomic>
23 #include <memory>
24 #include <mutex>
25 #include <set>
26 #include <vector>
27 #include <utility>
28
29 #if defined(__clang__)
30 #pragma clang diagnostic push
31 #pragma clang diagnostic ignored "-Wweak-vtables"
32 #endif
33
34 /************************************************************************/
35 /* VSIArrowFileSystem */
36 /************************************************************************/
37
38 class VSIArrowFileSystem final : public arrow::fs::FileSystem
39 {
40 const std::string m_osEnvVarPrefix;
41 const std::string m_osQueryParameters;
42
43 std::atomic<bool> m_bAskedToClosed = false;
44 std::mutex m_oMutex{};
45 std::vector<std::pair<std::string, std::weak_ptr<OGRArrowRandomAccessFile>>>
46 m_oSetFiles{};
47 std::set<std::string> m_oSetFilesAskedToOpen{};
48
49 public:
50 VSIArrowFileSystem(const std::string &osEnvVarPrefix,
51 const std::string &osQueryParameters)
52 : m_osEnvVarPrefix(osEnvVarPrefix),
53 m_osQueryParameters(osQueryParameters)
54 {
55 }
56
57 // Cf comment in OGRParquetDataset::~OGRParquetDataset() for rationale
58 // for this method
59 void AskToClose()
60 {
61 m_bAskedToClosed = true;
62 std::vector<
63 std::pair<std::string, std::weak_ptr<OGRArrowRandomAccessFile>>>
64 oSetFiles;
65 {
66 std::lock_guard oLock(m_oMutex);
67 oSetFiles = m_oSetFiles;
68 }
69 for (auto &[osName, poFile] : oSetFiles)
70 {
71 bool bWarned = false;
72 while (!poFile.expired())
73 {
74 if (!bWarned)
75 {
76 bWarned = true;
77 auto poFileLocked = poFile.lock();
78 if (poFileLocked)
79 {
80 CPLDebug("PARQUET",
81 "Still on-going reads on %s. Waiting for it "
82 "to be closed.",
83 osName.c_str());
84 poFileLocked->AskToClose();
85 }
86 }
87 CPLSleep(0.01);
88 }
89 }
90 }
91
92 std::string type_name() const override
93 {
94 return "vsi" + m_osEnvVarPrefix;
95 }
96
97 using arrow::fs::FileSystem::Equals;
98
99 bool Equals(const arrow::fs::FileSystem &other) const override
100 {
101 const auto poOther = dynamic_cast<const VSIArrowFileSystem *>(&other);
102 return poOther != nullptr &&
103 poOther->m_osEnvVarPrefix == m_osEnvVarPrefix &&
104 poOther->m_osQueryParameters == m_osQueryParameters;
105 }
106
107 using arrow::fs::FileSystem::GetFileInfo;
108
109 arrow::Result<arrow::fs::FileInfo>
110 GetFileInfo(const std::string &path) override
111 {
112 auto fileType = arrow::fs::FileType::Unknown;
113 VSIStatBufL sStat;
114 if (VSIStatL(path.c_str(), &sStat) == 0)
115 {
116 if (VSI_ISREG(sStat.st_mode))
117 fileType = arrow::fs::FileType::File;
118 else if (VSI_ISDIR(sStat.st_mode))
119 fileType = arrow::fs::FileType::Directory;
120 }
121 else
122 {
123 fileType = arrow::fs::FileType::NotFound;
124 }
125 arrow::fs::FileInfo info(path, fileType);
126 if (fileType == arrow::fs::FileType::File)
127 info.set_size(sStat.st_size);
128 return info;
129 }
130
131 arrow::Result<arrow::fs::FileInfoVector>
132 GetFileInfo(const arrow::fs::FileSelector &select) override
133 {
134 arrow::fs::FileInfoVector res;
135 VSIDIR *psDir = VSIOpenDir(select.base_dir.c_str(),
136 select.recursive ? -1 : 0, nullptr);
137 if (psDir == nullptr)
138 return res;
139
140 bool bParquetFound = false;
141 const int nMaxNonParquetFiles = atoi(
142 CPLGetConfigOption("OGR_PARQUET_MAX_NON_PARQUET_FILES", "100"));
143 const int nMaxListedFiles =
144 atoi(CPLGetConfigOption("OGR_PARQUET_MAX_LISTED_FILES", "1000000"));
145 while (const auto psEntry = VSIGetNextDirEntry(psDir))
146 {
147 if (!bParquetFound)
148 bParquetFound = EQUAL(
149 CPLGetExtensionSafe(psEntry->pszName).c_str(), "parquet");
150
151 std::string osFilename = select.base_dir + '/' + psEntry->pszName;
152 int nMode = psEntry->nMode;
153 if (!psEntry->bModeKnown)
154 {
155 VSIStatBufL sStat;
156 if (VSIStatL(osFilename.c_str(), &sStat) == 0)
157 nMode = sStat.st_mode;
158 }
159
160 auto fileType = arrow::fs::FileType::Unknown;
161 if (VSI_ISREG(nMode))
162 fileType = arrow::fs::FileType::File;
163 else if (VSI_ISDIR(nMode))
164 fileType = arrow::fs::FileType::Directory;
165
166 arrow::fs::FileInfo info(std::move(osFilename), fileType);
167 if (fileType == arrow::fs::FileType::File && psEntry->bSizeKnown)
168 {
169 info.set_size(psEntry->nSize);
170 }
171 res.push_back(std::move(info));
172
173 if (m_osEnvVarPrefix == "PARQUET")
174 {
175 // Avoid iterating over too many files if there's no likely parquet
176 // files.
177 if (static_cast<int>(res.size()) == nMaxNonParquetFiles &&
178 !bParquetFound)
179 break;
180 if (static_cast<int>(res.size()) == nMaxListedFiles)
181 break;
182 }
183 }
184 VSICloseDir(psDir);
185 return res;
186 }
187
188 arrow::Status CreateDir(const std::string & /*path*/,
189 bool /*recursive*/ = true) override
190 {
191 return arrow::Status::IOError("CreateDir() unimplemented");
192 }
193
194 arrow::Status DeleteDir(const std::string & /*path*/) override
195 {
196 return arrow::Status::IOError("DeleteDir() unimplemented");
197 }
198
199 arrow::Status DeleteDirContents(const std::string & /*path*/
200 #if ARROW_VERSION_MAJOR >= 8
201 ,
202 bool /*missing_dir_ok*/ = false
203 #endif
204 ) override
205 {
206 return arrow::Status::IOError("DeleteDirContents() unimplemented");
207 }
208
209 arrow::Status DeleteRootDirContents() override
210 {
211 return arrow::Status::IOError("DeleteRootDirContents() unimplemented");
212 }
213
214 arrow::Status DeleteFile(const std::string & /*path*/) override
215 {
216 return arrow::Status::IOError("DeleteFile() unimplemented");
217 }
218
219 arrow::Status Move(const std::string & /*src*/,
220 const std::string & /*dest*/) override
221 {
222 return arrow::Status::IOError("Move() unimplemented");
223 }
224
225 arrow::Status CopyFile(const std::string & /*src*/,
226 const std::string & /*dest*/) override
227 {
228 return arrow::Status::IOError("CopyFile() unimplemented");
229 }
230
231 using arrow::fs::FileSystem::OpenInputStream;
232
233 arrow::Result<std::shared_ptr<arrow::io::InputStream>>
234 OpenInputStream(const std::string &path) override
235 {
236 return OpenInputFile(path);
237 }
238
239 using arrow::fs::FileSystem::OpenInputFile;
240
241 arrow::Result<std::shared_ptr<arrow::io::RandomAccessFile>>
242 OpenInputFile(const std::string &path) override
243 {
244 if (m_bAskedToClosed)
245 return arrow::Status::IOError(
246 "OpenInputFile(): file system in shutdown");
247
248 std::string osPath(path);
249 osPath += m_osQueryParameters;
250 m_oSetFilesAskedToOpen.insert(osPath);
251 CPLDebugOnly(m_osEnvVarPrefix.c_str(), "Opening %s", osPath.c_str());
252 auto fp = VSIVirtualHandleUniquePtr(VSIFOpenL(osPath.c_str(), "rb"));
253 if (fp == nullptr)
254 return arrow::Status::IOError("OpenInputFile() failed for " +
255 osPath);
256 auto poFile =
257 std::make_shared<OGRArrowRandomAccessFile>(osPath, std::move(fp));
258 {
259 std::lock_guard oLock(m_oMutex);
260 m_oSetFiles.emplace_back(path, poFile);
261 }
262 return poFile;
263 }
264
265 using arrow::fs::FileSystem::OpenOutputStream;
266
267 arrow::Result<std::shared_ptr<arrow::io::OutputStream>>
268 OpenOutputStream(const std::string & /*path*/,
269 const std::shared_ptr<const arrow::KeyValueMetadata>
270 & /* metadata */) override
271 {
272 return arrow::Status::IOError("OpenOutputStream() unimplemented");
273 }
274
275 arrow::Result<std::shared_ptr<arrow::io::OutputStream>>
276 OpenAppendStream(const std::string & /*path*/,
277 const std::shared_ptr<const arrow::KeyValueMetadata>
278 & /* metadata */) override
279 {
280 return arrow::Status::IOError("OpenAppendStream() unimplemented");
281 }
282
283 // For debugging
284 const std::set<std::string> &GetSetFilesAskedToOpen() const
285 {
286 return m_oSetFilesAskedToOpen;
287 }
288
289 // For debugging
290 void ResetSetFilesAskedToOpen()
291 {
292 m_oSetFilesAskedToOpen.clear();
293 }
294 };
295
296 #if defined(__clang__)
297 #pragma clang diagnostic pop
298 #endif
299
300 #endif // VSIARROWFILESYSTEM_HPP_INCLUDED